Skip to content
This repository was archived by the owner on Sep 23, 2025. It is now read-only.

Commit fb04df8

Browse files
mattmoormattmoor
authored
Refactor cosigned to take advantage of duck typing. (#637)
* Refactor cosigned to take advantage of duck typing. With this change, the webhook can take advantage of duck typing to parse all of the "Pod Specable" types currently supported. This also takes advantage of the `knative.dev/pkg` webhook infrastructure to reduce boilerplate and eliminate the need for `cert-manager`. Lastly, this starts to sketch out some cosigned e2e tests to verify that things work. Signed-off-by: Matt Moore <mattomata@gmail.com> * Make port configurable, pull tests out into a script. Signed-off-by: Matt Moore <mattomata@gmail.com> * Drop GO111MODULE, drop v1beta1 admission review, improve flag desc, hoist and comment webhook name as constant Signed-off-by: Matt Moore <mattomata@gmail.com>
1 parent 739947d commit fb04df8

26 files changed

Lines changed: 1091 additions & 450 deletions

‎.github/workflows/kind-e2e-cosigned.yaml‎

Lines changed: 183 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,183 @@
1+
name: Cosigned KinD E2E
2+
3+
on:
4+
pull_request:
5+
branches: [ 'main', 'release-*' ]
6+
7+
defaults:
8+
run:
9+
shell: bash
10+
working-directory: ./src/github.com/sigstore/cosign
11+
12+
jobs:
13+
14+
e2e-tests:
15+
name: e2e tests
16+
runs-on: ubuntu-latest
17+
strategy:
18+
fail-fast: false # Keep running if one leg fails.
19+
matrix:
20+
k8s-version:
21+
- v1.19.11
22+
- v1.20.7
23+
- v1.21.1
24+
25+
include:
26+
# Map between K8s and KinD versions.
27+
# This is attempting to make it a bit clearer what's being tested.
28+
# See: https://github.com/kubernetes-sigs/kind/releases
29+
- k8s-version: v1.19.11
30+
kind-version: v0.11.1
31+
kind-image-sha: sha256:07db187ae84b4b7de440a73886f008cf903fcf5764ba8106a9fd5243d6f32729
32+
cluster-suffix: c${{ github.run_id }}.local
33+
- k8s-version: v1.20.7
34+
kind-version: v0.11.1
35+
kind-image-sha: sha256:cbeaf907fc78ac97ce7b625e4bf0de16e3ea725daf6b04f930bd14c67c671ff9
36+
cluster-suffix: c${{ github.run_id }}.local
37+
- k8s-version: v1.21.1
38+
kind-version: v0.11.1
39+
kind-image-sha: sha256:69860bda5563ac81e3c0057d654b5253219618a22ec3a346306239bba8cfa1a6
40+
cluster-suffix: c${{ github.run_id }}.local
41+
42+
env:
43+
GOPATH: ${{ github.workspace }}
44+
# https://github.com/google/go-containerregistry/pull/125 allows insecure registry for
45+
# '*.local' hostnames.
46+
REGISTRY_NAME: registry.local
47+
REGISTRY_PORT: 5000
48+
KO_DOCKER_REPO: registry.local:5000/cosigned
49+
50+
steps:
51+
- name: Set up Go 1.16.x
52+
uses: actions/setup-go@v2
53+
with:
54+
go-version: 1.16.x
55+
56+
- name: Install Dependencies
57+
working-directory: ./
58+
run: |
59+
echo '::group:: install ko'
60+
curl -L https://github.com/google/ko/releases/download/v0.8.3/ko_0.8.3_Linux_x86_64.tar.gz | tar xzf - ko
61+
chmod +x ./ko
62+
sudo mv ko /usr/local/bin
63+
echo '::endgroup::'
64+
65+
- name: Check out code onto GOPATH
66+
uses: actions/checkout@v2
67+
with:
68+
path: ./src/github.com/sigstore/cosign
69+
70+
- name: Install Cosign
71+
run: |
72+
go install ./cmd/cosign
73+
74+
# This KinD setup is based on what we use for knative/serving on GHA, and it includes several "fun"
75+
# monkey wrenches (e.g. randomizing cluster suffix: `.svc.cluster.local`) to make sure we don't bake
76+
# in any invalid assumptions about a particular Kubernetes configuration.
77+
- name: Install KinD
78+
run: |
79+
set -x
80+
# Disable swap otherwise memory enforcement doesn't work
81+
# See: https://kubernetes.slack.com/archives/CEKK1KTN2/p1600009955324200
82+
sudo swapoff -a
83+
sudo rm -f /swapfile
84+
# Use in-memory storage to avoid etcd server timeouts.
85+
# https://kubernetes.slack.com/archives/CEKK1KTN2/p1615134111016300
86+
# https://github.com/kubernetes-sigs/kind/issues/845
87+
sudo mkdir -p /tmp/etcd
88+
sudo mount -t tmpfs tmpfs /tmp/etcd
89+
curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${{ matrix.kind-version }}/kind-$(uname)-amd64
90+
chmod +x ./kind
91+
sudo mv kind /usr/local/bin
92+
93+
- name: Configure KinD Cluster
94+
run: |
95+
set -x
96+
# KinD configuration.
97+
cat > kind.yaml <<EOF
98+
apiVersion: kind.x-k8s.io/v1alpha4
99+
kind: Cluster
100+
# Configure registry for KinD.
101+
containerdConfigPatches:
102+
- |-
103+
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."$REGISTRY_NAME:$REGISTRY_PORT"]
104+
endpoint = ["http://$REGISTRY_NAME:$REGISTRY_PORT"]
105+
# This is needed in order to support projected volumes with service account tokens.
106+
# See: https://kubernetes.slack.com/archives/CEKK1KTN2/p1600268272383600
107+
kubeadmConfigPatches:
108+
- |
109+
apiVersion: kubeadm.k8s.io/v1beta2
110+
kind: ClusterConfiguration
111+
metadata:
112+
name: config
113+
apiServer:
114+
extraArgs:
115+
"service-account-issuer": "kubernetes.default.svc"
116+
"service-account-signing-key-file": "/etc/kubernetes/pki/sa.key"
117+
networking:
118+
dnsDomain: "${{ matrix.cluster-suffix }}"
119+
nodes:
120+
- role: control-plane
121+
image: kindest/node:${{ matrix.k8s-version }}@${{ matrix.kind-image-sha }}
122+
extraMounts:
123+
- containerPath: /var/lib/etcd
124+
hostPath: /tmp/etcd
125+
- role: worker
126+
image: kindest/node:${{ matrix.k8s-version }}@${{ matrix.kind-image-sha }}
127+
EOF
128+
129+
- name: Create KinD Cluster
130+
run: |
131+
set -x
132+
kind create cluster --config kind.yaml
133+
134+
- name: Setup local registry
135+
run: |
136+
# Run a registry.
137+
docker run -d --restart=always \
138+
-p $REGISTRY_PORT:$REGISTRY_PORT --name $REGISTRY_NAME registry:2
139+
140+
# Connect the registry to the KinD network.
141+
docker network connect "kind" $REGISTRY_NAME
142+
143+
# Make the $REGISTRY_NAME -> 127.0.0.1, to tell `ko` to publish to
144+
# local reigstry, even when pushing $REGISTRY_NAME:$REGISTRY_PORT/some/image
145+
sudo echo "127.0.0.1 $REGISTRY_NAME" | sudo tee -a /etc/hosts
146+
147+
- name: Install cosigned
148+
run: |
149+
ko apply -Bf config/
150+
151+
# Update the cosign verification-key secret with a proper key pair.
152+
cosign generate-key-pair k8s://cosign-system/verification-key
153+
154+
# Wait for the webhook to come up and become Ready
155+
kubectl rollout status --timeout 5m --namespace cosign-system deployments/webhook
156+
157+
- name: Run Tests
158+
run: |
159+
./test/e2e_test_cosigned.sh
160+
161+
- name: Collect diagnostics
162+
if: ${{ failure() }}
163+
run: |
164+
# Add more namespaces to dump here.
165+
for ns in cosign-system; do
166+
kubectl get pods -n${ns}
167+
168+
echo '::group:: describe'
169+
kubectl describe pods -n${ns}
170+
echo '::endgroup::'
171+
172+
for x in $(kubectl get pods -n${ns} -oname); do
173+
174+
echo "::group:: describe $x"
175+
kubectl describe -n${ns} $x
176+
echo '::endgroup::'
177+
178+
echo "::group:: $x logs"
179+
kubectl logs -n${ns} $x --all-containers
180+
echo '::endgroup::'
181+
182+
done
183+
done

‎cmd/cosign/webhook/depcheck_test.go‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
//
2+
// Copyright 2021 The Sigstore Authors.
3+
//
4+
// Licensed under the Apache License, Version 2.0 (the "License");
5+
// you may not use this file except in compliance with the License.
6+
// You may obtain a copy of the License at
7+
//
8+
// http://www.apache.org/licenses/LICENSE-2.0
9+
//
10+
// Unless required by applicable law or agreed to in writing, software
11+
// distributed under the License is distributed on an "AS IS" BASIS,
12+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
// See the License for the specific language governing permissions and
14+
// limitations under the License.
15+
16+
package main_test
17+
18+
import (
19+
"testing"
20+
21+
"knative.dev/pkg/depcheck"
22+
)
23+
24+
func TestNoDeps(t *testing.T) {
25+
depcheck.AssertNoDependency(t, map[string][]string{
26+
"github.com/sigstore/cosign/cmd/cosign/webhook": {
27+
// This conflicts with klog, we error on startup about
28+
// `-log_dir` being defined multiple times.
29+
"github.com/golang/glog",
30+
},
31+
})
32+
}

‎cmd/cosign/webhook/kodata/HEAD‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../../../../.git/HEAD

‎cmd/cosign/webhook/kodata/LICENSE‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../../../../LICENSE

‎cmd/cosign/webhook/kodata/refs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../../../../.git/refs

‎cmd/cosign/webhook/main.go‎

Lines changed: 60 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -17,99 +17,83 @@ package main
1717

1818
import (
1919
"context"
20-
goflag "flag"
21-
"net"
22-
"net/http"
23-
"os"
24-
"strconv"
20+
"flag"
2521

26-
flag "github.com/spf13/pflag"
2722
appsv1 "k8s.io/api/apps/v1"
2823
batchv1 "k8s.io/api/batch/v1"
2924
corev1 "k8s.io/api/core/v1"
3025
"k8s.io/apimachinery/pkg/runtime/schema"
31-
"k8s.io/klog/v2"
32-
ctrl "sigs.k8s.io/controller-runtime"
33-
"sigs.k8s.io/controller-runtime/pkg/healthz"
34-
"sigs.k8s.io/controller-runtime/pkg/log/zap"
35-
36-
"github.com/sigstore/cosign/pkg/cosign/kubernetes/webhook"
26+
duckv1 "knative.dev/pkg/apis/duck/v1"
27+
"knative.dev/pkg/configmap"
28+
"knative.dev/pkg/controller"
29+
"knative.dev/pkg/injection/sharedmain"
30+
"knative.dev/pkg/signals"
31+
"knative.dev/pkg/webhook"
32+
"knative.dev/pkg/webhook/certificates"
33+
"knative.dev/pkg/webhook/resourcesemantics"
34+
"knative.dev/pkg/webhook/resourcesemantics/validation"
35+
36+
cwebhook "github.com/sigstore/cosign/pkg/cosign/kubernetes/webhook"
3737
)
3838

39-
func main() {
40-
ctrl.SetLogger(zap.New(func(o *zap.Options) {
41-
o.Development = true
42-
}))
43-
44-
var (
45-
metricsAddr = net.ParseIP("127.0.0.1")
46-
metricsPort uint16 = 8080
39+
var secretName = flag.String("secret-name", "", "The name of the secret in the webhook's namespace that holds the public key for verification.")
4740

48-
bindAddr = net.ParseIP("0.0.0.0")
49-
bindPort uint16 = 8443
41+
// webhookName holds the name of the validating webhook to set up with the
42+
// types we are watching. If this changes, you must also change:
43+
// ./config/500-webhook-configuration.yaml
44+
const webhookName = "cosigned.sigstore.dev"
5045

51-
tlsCertDirectory string
52-
secretKeyRef string
53-
)
54-
55-
klog.InitFlags(goflag.CommandLine)
56-
flags := flag.NewFlagSet("main", flag.ExitOnError)
57-
flags.AddGoFlagSet(goflag.CommandLine)
58-
flags.StringVar(&secretKeyRef, "secret-key-ref", "", "The secret that includes pub/private key pair")
59-
flags.IPVar(&metricsAddr, "metrics-address", metricsAddr, "The address the metric endpoint binds to.")
60-
flags.Uint16Var(&metricsPort, "metrics-port", metricsPort, "The port the metric endpoint binds to.")
61-
flags.IPVar(&bindAddr, "bind-address", bindAddr, ""+
62-
"The IP address on which to listen for the --secure-port port.")
63-
flags.Uint16Var(&bindPort, "secure-port", bindPort, "The port on which to serve HTTPS.")
64-
flags.StringVar(&tlsCertDirectory, "tls-cert-dir", tlsCertDirectory, "The directory where the TLS certs are located.")
65-
66-
err := flags.Parse(os.Args[1:])
67-
if err != nil {
68-
klog.Error(err)
69-
os.Exit(1)
46+
func main() {
47+
opts := webhook.Options{
48+
ServiceName: "webhook",
49+
Port: 8443,
50+
SecretName: "webhook-certs",
7051
}
52+
ctx := webhook.WithOptions(signals.NewContext(), opts)
7153

72-
cosignedValidationFuncs := map[schema.GroupVersionKind]webhook.ValidationFunc{
73-
corev1.SchemeGroupVersion.WithKind("Pod"): webhook.ValidateSignedResources,
74-
batchv1.SchemeGroupVersion.WithKind("Job"): webhook.ValidateSignedResources,
75-
appsv1.SchemeGroupVersion.WithKind("Deployment"): webhook.ValidateSignedResources,
76-
appsv1.SchemeGroupVersion.WithKind("StatefulSet"): webhook.ValidateSignedResources,
77-
appsv1.SchemeGroupVersion.WithKind("ReplicateSet"): webhook.ValidateSignedResources,
78-
appsv1.SchemeGroupVersion.WithKind("DaemonSet"): webhook.ValidateSignedResources,
79-
}
54+
// Allow folks to configure the port the webhook serves on.
55+
flag.IntVar(&opts.Port, "secure-port", opts.Port, "The port on which to serve HTTPS.")
8056

81-
cosignedValidationHook := webhook.NewFuncAdmissionValidator(webhook.Scheme, cosignedValidationFuncs, secretKeyRef)
57+
// This calls flag.Parse()
58+
sharedmain.MainWithContext(ctx, "cosigned",
59+
certificates.NewController,
60+
NewValidatingAdmissionController,
61+
)
62+
}
8263

83-
opts := ctrl.Options{
84-
Scheme: webhook.Scheme,
85-
MetricsBindAddress: net.JoinHostPort(metricsAddr.String(), strconv.Itoa(int(metricsPort))),
86-
Host: bindAddr.String(),
87-
Port: int(bindPort),
88-
CertDir: tlsCertDirectory,
89-
}
64+
func NewValidatingAdmissionController(ctx context.Context, cmw configmap.Watcher) *controller.Impl {
65+
validator := cwebhook.NewValidator(ctx, *secretName)
9066

91-
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), opts)
92-
if err != nil {
93-
klog.Error(err, "Failed to create manager")
94-
os.Exit(1)
95-
}
67+
return validation.NewAdmissionController(ctx,
68+
// Name of the resource webhook.
69+
webhookName,
9670

97-
// Get the controller manager webhook server.
98-
webhookServer := mgr.GetWebhookServer()
71+
// The path on which to serve the webhook.
72+
"/validations",
9973

100-
// Register the webhooks in the server.
101-
webhookServer.Register("/validations", cosignedValidationHook)
74+
// The resources to validate.
75+
map[schema.GroupVersionKind]resourcesemantics.GenericCRD{
76+
corev1.SchemeGroupVersion.WithKind("Pod"): &duckv1.Pod{},
10277

103-
// Add healthz and readyz handlers to webhook server. The controller-runtime AddHealthzCheck/AddReadyzCheck methods
104-
// are served via separate http server - better to serve these from the same webhook http server.
105-
webhookServer.WebhookMux.Handle("/readyz/", http.StripPrefix("/readyz/", &healthz.Handler{}))
106-
webhookServer.WebhookMux.Handle("/healthz/", http.StripPrefix("/healthz/", &healthz.Handler{}))
78+
appsv1.SchemeGroupVersion.WithKind("ReplicaSet"): &duckv1.WithPod{},
79+
appsv1.SchemeGroupVersion.WithKind("Deployment"): &duckv1.WithPod{},
80+
appsv1.SchemeGroupVersion.WithKind("StatefulSet"): &duckv1.WithPod{},
81+
appsv1.SchemeGroupVersion.WithKind("DaemonSet"): &duckv1.WithPod{},
82+
batchv1.SchemeGroupVersion.WithKind("Job"): &duckv1.WithPod{},
83+
},
10784

108-
klog.Info("Starting the webhook...")
85+
// A function that infuses the context passed to Validate/SetDefaults with custom metadata.
86+
func(ctx context.Context) context.Context {
87+
ctx = duckv1.WithPodValidator(ctx, validator.ValidatePod)
88+
ctx = duckv1.WithPodSpecValidator(ctx, validator.ValidatePodSpecable)
89+
return ctx
90+
},
10991

110-
// Start the server by starting a previously-set-up manager
111-
if err := mgr.Start(context.Background()); err != nil {
112-
klog.Error(err)
113-
os.Exit(1)
114-
}
92+
// Whether to disallow unknown fields.
93+
// We pass false because we're using partial schemas.
94+
false,
95+
96+
// Extra validating callbacks to be applied to resources.
97+
nil,
98+
)
11599
}

0 commit comments

Comments
 (0)