From 48bcec2c11a14ca2c4336db3fe798825065ae6fa Mon Sep 17 00:00:00 2001 From: Lingxian Kong Date: Mon, 7 Sep 2020 22:02:55 +1200 Subject: [PATCH 01/90] Added gh-pages branch --- .github/ISSUE_TEMPLATE.md | 39 - .github/PULL_REQUEST_TEMPLATE.md | 33 - .github/workflows/pr.yaml | 50 - .github/workflows/release.yaml | 26 - .zuul.yaml | 208 -- CONTRIBUTING.md | 25 - LICENSE | 202 -- Makefile | 329 -- OWNERS | 18 - README.md | 45 - SECURITY.md | 22 - SECURITY_CONTACTS | 15 - bindep.txt | 6 - charts/cinder-csi-plugin/.helmignore | 22 - charts/cinder-csi-plugin/Chart.yaml | 10 - charts/cinder-csi-plugin/README.md | 21 - charts/cinder-csi-plugin/templates/NOTES.txt | 1 - .../cinder-csi-plugin/templates/_helpers.tpl | 90 - .../templates/cinder-csi-driver.yaml | 10 - .../templates/controllerplugin-rbac.yaml | 197 -- .../controllerplugin-statefulset.yaml | 107 - .../templates/nodeplugin-daemonset.yaml | 103 - .../templates/nodeplugin-rbac.yaml | 28 - .../templates/storageclass.yaml | 25 - charts/cinder-csi-plugin/values.yaml | 43 - charts/manila-csi-plugin/Chart.yaml | 10 - charts/manila-csi-plugin/README.md | 14 - .../manila-csi-plugin/templates/_helpers.tpl | 83 - .../controllerplugin-clusterrole.yaml | 15 - .../controllerplugin-clusterrolebinding.yaml | 18 - .../templates/controllerplugin-role.yaml | 17 - .../controllerplugin-rolebinding.yaml | 18 - .../controllerplugin-rules-clusterrole.yaml | 48 - .../templates/controllerplugin-service.yaml | 18 - .../controllerplugin-serviceaccount.yaml | 10 - .../controllerplugin-statefulset.yaml | 149 - .../templates/csidriver.yaml | 10 - .../templates/nodeplugin-clusterrole.yaml | 15 - .../nodeplugin-clusterrolebinding.yaml | 18 - .../templates/nodeplugin-daemonset.yaml | 139 - .../nodeplugin-rules-clusterrole.yaml | 24 - .../templates/nodeplugin-serviceaccount.yaml | 10 - .../templates/runtimeconfig-cm.yaml | 9 - charts/manila-csi-plugin/values.yaml | 84 - ...loud-controller-manager-role-bindings.yaml | 40 - .../rbac/cloud-controller-manager-roles.yaml | 129 - .../storage-class/openstack/default.yaml | 10 - cluster/images/barbican-kms-plugin/Dockerfile | 32 - .../barbican-kms-plugin/Dockerfile.build | 7 - cluster/images/cinder-csi-plugin/Dockerfile | 32 - .../images/cinder-csi-plugin/Dockerfile.build | 9 - cluster/images/k8s-keystone-auth/Dockerfile | 34 - .../images/k8s-keystone-auth/Dockerfile.build | 20 - cluster/images/magnum-auto-healer/Dockerfile | 32 - .../magnum-auto-healer/Dockerfile.build | 20 - cluster/images/manila-csi-plugin/Dockerfile | 33 - .../images/manila-csi-plugin/Dockerfile.build | 8 - .../octavia-ingress-controller/Dockerfile | 32 - .../Dockerfile.build | 18 - .../Dockerfile | 32 - .../Dockerfile.build | 19 - cmd/barbican-kms-plugin/main.go | 86 - cmd/cinder-csi-plugin/main.go | 119 - cmd/client-keystone-auth/main.go | 232 -- cmd/k8s-keystone-auth/main.go | 66 - cmd/magnum-auto-healer/main.go | 25 - cmd/manila-csi-plugin/main.go | 190 -- cmd/octavia-ingress-controller/main.go | 25 - .../main.go | 259 -- cmd/tests/cinder_csi_e2e_suite_test.go | 45 - code-of-conduct.md | 3 - contrib/gate_hook.sh | 18 - contrib/post_test_hook.sh | 176 -- contrib/pre_test_hook.sh | 19 - devstack/plugin.sh | 216 -- devstack/settings | 6 - docs/developers-csi-manila.md | 107 - ...cations-using-loadbalancer-type-service.md | 269 -- docs/getting-started-provider-dev.md | 321 -- docs/migrate-to-ccm-with-csimigration.md | 212 -- docs/using-auth-data-synchronization.md | 143 - docs/using-barbican-kms-plugin.md | 109 - docs/using-cinder-csi-plugin.md | 621 ---- docs/using-client-keystone-auth.md | 169 - ...ne-webhook-authenticator-and-authorizer.md | 777 ----- docs/using-magnum-auto-healer.md | 242 -- docs/using-manila-csi-plugin.md | 236 -- docs/using-octavia-ingress-controller.md | 394 --- ...sing-openstack-cloud-controller-manager.md | 223 -- examples/cinder-csi-plugin/block/block.yaml | 41 - .../clone/example-clone.yaml | 41 - examples/cinder-csi-plugin/clone/sc.yaml | 5 - .../cinder-csi-plugin/clone/source-pvc.yaml | 11 - .../inline/inline-example.yaml | 20 - examples/cinder-csi-plugin/nginx.yaml | 43 - .../cinder-csi-plugin/resize/example.yaml | 44 - .../cinder-csi-plugin/snapshot/example.yaml | 30 - .../snapshot/snapshotcreate.yaml | 8 - .../snapshot/snapshotrestore.yaml | 15 - .../cinder-csi-plugin/topology/example.yaml | 46 - examples/loadbalancers/README.MD | 145 - .../loadbalancers/external-http-nginx.yaml | 36 - .../loadbalancers/internal-http-nginx.yaml | 35 - examples/manila-csi-plugin/README.md | 40 - examples/manila-csi-plugin/exec-bash.sh | 15 - examples/manila-csi-plugin/logs.sh | 15 - .../nfs/dynamic-provisioning/pod.yaml | 17 - .../nfs/dynamic-provisioning/pvc.yaml | 12 - .../dynamic-provisioning/storageclass.yaml | 15 - examples/manila-csi-plugin/nfs/secrets.yaml | 20 - .../manila-csi-plugin/nfs/snapshot/pod.yaml | 17 - .../nfs/snapshot/snapshotclass.yaml | 8 - .../nfs/snapshot/snapshotcreate.yaml | 9 - .../nfs/snapshot/snapshotrestore.yaml | 15 - .../nfs/static-provisioning/pod.yaml | 17 - .../preprovisioned-pvc.yaml | 39 - .../nfs/topology-aware/pod.yaml | 17 - .../nfs/topology-aware/pvc.yaml | 12 - .../nfs/topology-aware/storageclass.yaml | 37 - .../manila-csi-plugin/runtimeconfig-cm.yaml | 17 - examples/mysql-cinder-pd/README.md | 1 - examples/mysql-cinder-pd/mysql-service.yaml | 13 - examples/mysql-cinder-pd/mysql.yaml | 33 - .../cinder/cinder-full.yaml | 43 - .../cinder/cinder-in-tree-full.yaml | 38 - .../cinder/cinder-storage-class.yaml | 7 - .../cinder/example-pod.yaml | 23 - examples/volumes/cinder/README.md | 1 - examples/volumes/cinder/cinder-web.yaml | 21 - .../webhook/keystone-apiserver-webhook.yaml | 16 - examples/webhook/keystone-deployment.yaml | 41 - .../webhook/keystone-policy-configmap.yaml | 62 - examples/webhook/keystone-rbac.yaml | 32 - examples/webhook/keystone-service.yaml | 12 - examples/webhook/policy.json | 32 - go.mod | 81 - go.sum | 915 ------ hack/make.sh | 23 - hack/update-gofmt.sh | 21 - hack/verify-gofmt.sh | 35 - hack/verify-golint.sh | 132 - hack/verify-import-boss.sh | 28 - hack/.golint_failures => index.yaml | 0 manifests/barbican-kms/encryption-config.yaml | 11 - manifests/barbican-kms/pod.yaml | 24 - .../cinder-csi-controllerplugin-rbac.yaml | 197 -- .../cinder-csi-controllerplugin.yaml | 114 - .../cinder-csi-nodeplugin-rbac.yaml | 30 - .../cinder-csi-nodeplugin.yaml | 98 - .../cinder-csi-plugin/csi-cinder-driver.yaml | 10 - .../csi-secret-cinderplugin.yaml | 10 - manifests/controller-manager/cloud-config | 19 - manifests/controller-manager/kubeadm.conf | 19 - ...openstack-cloud-controller-manager-ds.yaml | 75 - ...penstack-cloud-controller-manager-pod.yaml | 52 - .../magnum-auto-healer.yaml | 104 - .../csi-controllerplugin-rbac.yaml | 113 - .../csi-controllerplugin.yaml | 119 - .../csi-nodeplugin-rbac.yaml | 58 - .../manila-csi-plugin/csi-nodeplugin.yaml | 100 - manifests/manila-csi-plugin/csidriver.yaml | 8 - pkg/autohealing/OWNERS | 6 - .../cloudprovider/cloudprovider.go | 70 - .../cloudprovider/openstack/provider.go | 490 --- .../cloudprovider/register/register.go | 81 - pkg/autohealing/cmd/root.go | 138 - pkg/autohealing/config/config.go | 94 - pkg/autohealing/controller/controller.go | 409 --- pkg/autohealing/healthcheck/healthcheck.go | 91 - .../healthcheck/plugin_endpoint.go | 216 -- .../healthcheck/plugin_nodecondition.go | 116 - pkg/autohealing/utils/utils.go | 37 - pkg/cloudprovider/.import-restrictions | 23 - pkg/cloudprovider/providers/openstack/OWNERS | 8 - pkg/cloudprovider/providers/openstack/doc.go | 17 - .../providers/openstack/openstack.go | 1038 ------ .../providers/openstack/openstack_client.go | 76 - .../openstack/openstack_instances.go | 285 -- .../openstack/openstack_loadbalancer.go | 2808 ----------------- .../providers/openstack/openstack_metrics.go | 57 - .../providers/openstack/openstack_routes.go | 348 -- .../openstack/openstack_routes_test.go | 123 - .../providers/openstack/openstack_test.go | 1198 ------- pkg/csi/cinder/controllerserver.go | 623 ---- pkg/csi/cinder/controllerserver_test.go | 523 --- pkg/csi/cinder/driver.go | 157 - pkg/csi/cinder/driver_test.go | 66 - pkg/csi/cinder/etc/cloud.conf | 6 - pkg/csi/cinder/fake.go | 120 - pkg/csi/cinder/identityserver.go | 97 - pkg/csi/cinder/identityserver_test.go | 40 - pkg/csi/cinder/nodeserver.go | 551 ---- pkg/csi/cinder/nodeserver_test.go | 448 --- pkg/csi/cinder/openstack/openstack.go | 185 -- .../cinder/openstack/openstack_instances.go | 14 - pkg/csi/cinder/openstack/openstack_mock.go | 385 --- .../cinder/openstack/openstack_snapshots.go | 175 - pkg/csi/cinder/openstack/openstack_test.go | 143 - pkg/csi/cinder/openstack/openstack_volumes.go | 343 -- pkg/csi/cinder/server.go | 112 - pkg/csi/cinder/utils.go | 89 - pkg/csi/cinder/utils_test.go | 76 - pkg/csi/manila/OWNERS | 10 - pkg/csi/manila/adapters.go | 37 - .../manila/capabilities/manilacapabilities.go | 70 - pkg/csi/manila/compatibility/compatibility.go | 44 - pkg/csi/manila/controllerserver.go | 420 --- pkg/csi/manila/csiclient.go | 48 - pkg/csi/manila/csiclient/builder.go | 109 - pkg/csi/manila/csiclient/client.go | 70 - pkg/csi/manila/csiclient/interface.go | 48 - pkg/csi/manila/driver.go | 331 -- pkg/csi/manila/identityserver.go | 74 - pkg/csi/manila/manilaclient/builder.go | 124 - pkg/csi/manila/manilaclient/client.go | 116 - pkg/csi/manila/manilaclient/interface.go | 54 - pkg/csi/manila/nodeserver.go | 338 -- .../manila/options/compatibilityoptions.go | 31 - pkg/csi/manila/options/openstackoptions.go | 35 - pkg/csi/manila/options/shareoptions.go | 66 - pkg/csi/manila/runtimeconfig/nfsconfig.go | 26 - pkg/csi/manila/runtimeconfig/runtimeconfig.go | 52 - pkg/csi/manila/share.go | 152 - pkg/csi/manila/shareadapters/cephfs.go | 132 - pkg/csi/manila/shareadapters/nfs.go | 165 - pkg/csi/manila/shareadapters/shareadapter.go | 57 - pkg/csi/manila/shareadapters/util.go | 35 - pkg/csi/manila/snapshot.go | 150 - pkg/csi/manila/util.go | 372 --- pkg/csi/manila/util/exportlocation.go | 68 - pkg/csi/manila/util/exportlocation_test.go | 127 - pkg/csi/manila/validator/dependsexpr.go | 110 - pkg/csi/manila/validator/identity.go | 55 - pkg/csi/manila/validator/matchexpr.go | 51 - pkg/csi/manila/validator/precludesexpr.go | 79 - pkg/csi/manila/validator/util.go | 28 - pkg/csi/manila/validator/validator.go | 147 - pkg/csi/manila/validator/validator_test.go | 195 -- pkg/csi/manila/validator/valueexpr.go | 187 -- pkg/csi/manila/volumesource.go | 126 - pkg/identity/keystone/authenticator.go | 144 - pkg/identity/keystone/authenticator_test.go | 65 - pkg/identity/keystone/authorizer.go | 388 --- pkg/identity/keystone/authorizer_test.go | 380 --- .../keystone/authorizer_test_policy.json | 259 -- .../authorizer_test_policy_version2.json | 33 - pkg/identity/keystone/config.go | 94 - pkg/identity/keystone/keystone.go | 613 ---- pkg/identity/keystone/keystone_test.go | 61 - pkg/identity/keystone/mock_IKeystone.go | 56 - pkg/identity/keystone/policy.go | 101 - pkg/identity/keystone/sync.go | 329 -- pkg/identity/keystone/sync_test.go | 156 - pkg/identity/keystone/sync_test.yaml | 16 - pkg/identity/keystone/token_getter.go | 126 - pkg/identity/keystone/token_getter_test.go | 110 - pkg/ingress/cmd/root.go | 118 - pkg/ingress/config/config.go | 56 - pkg/ingress/controller/controller.go | 921 ------ pkg/ingress/controller/openstack/client.go | 94 - pkg/ingress/controller/openstack/neutron.go | 294 -- pkg/ingress/controller/openstack/octavia.go | 472 --- pkg/ingress/utils/utils.go | 95 - pkg/kms/barbican/barbican.go | 53 - pkg/kms/barbican/fake_barbican.go | 10 - pkg/kms/client/client.go | 54 - pkg/kms/encryption/aescbc/aescbc.go | 72 - pkg/kms/encryption/aescbc/aescbc_test.go | 40 - pkg/kms/server/server.go | 137 - pkg/kms/server/server_test.go | 40 - pkg/util/blockdevice/blockdevice_linux.go | 119 - .../blockdevice/blockdevice_unsupported.go | 35 - pkg/util/errors/errors.go | 55 - pkg/util/file/file.go | 57 - pkg/util/io/consistentread.go | 45 - pkg/util/metadata/metadata.go | 294 -- pkg/util/metadata/metadata_mock.go | 50 - pkg/util/metadata/metadata_test.go | 107 - pkg/util/mount/mount.go | 281 -- pkg/util/mount/mount_mock.go | 190 -- pkg/util/net/sets/ipnet.go | 121 - pkg/util/openstack/keymanager.go | 123 - pkg/util/openstack/loadbalancer.go | 341 -- pkg/util/openstack/network.go | 55 - pkg/util/util.go | 76 - pkg/version/version.go | 30 - release-procedure.md | 53 - snapcraft.yaml | 22 - tests/e2e/cloudprovider/test-lb-service.sh | 207 -- tests/e2e/csi/cinder/cinder-testdriver.go | 101 - tests/e2e/csi/cinder/csi-volumes.go | 33 - tests/e2e/k8s-keystone-auth/test-authz.sh | 104 - tests/sanity/cinder/fakecloud.go | 273 -- tests/sanity/cinder/fakemetadata.go | 16 - tests/sanity/cinder/fakemount.go | 69 - tests/sanity/cinder/sanity_test.go | 44 - tests/sanity/manila/fake-secrets.yaml | 49 - tests/sanity/manila/fakecsiclient.go | 95 - tests/sanity/manila/fakemanilaclient.go | 249 -- tests/sanity/manila/sanity_test.go | 67 - tools/install-distro-packages.sh | 44 - tools/test-setup.sh | 57 - 302 files changed, 36882 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE.md delete mode 100644 .github/PULL_REQUEST_TEMPLATE.md delete mode 100644 .github/workflows/pr.yaml delete mode 100644 .github/workflows/release.yaml delete mode 100644 .zuul.yaml delete mode 100644 CONTRIBUTING.md delete mode 100644 LICENSE delete mode 100644 Makefile delete mode 100644 OWNERS delete mode 100644 README.md delete mode 100644 SECURITY.md delete mode 100644 SECURITY_CONTACTS delete mode 100644 bindep.txt delete mode 100644 charts/cinder-csi-plugin/.helmignore delete mode 100644 charts/cinder-csi-plugin/Chart.yaml delete mode 100644 charts/cinder-csi-plugin/README.md delete mode 100644 charts/cinder-csi-plugin/templates/NOTES.txt delete mode 100644 charts/cinder-csi-plugin/templates/_helpers.tpl delete mode 100644 charts/cinder-csi-plugin/templates/cinder-csi-driver.yaml delete mode 100644 charts/cinder-csi-plugin/templates/controllerplugin-rbac.yaml delete mode 100644 charts/cinder-csi-plugin/templates/controllerplugin-statefulset.yaml delete mode 100644 charts/cinder-csi-plugin/templates/nodeplugin-daemonset.yaml delete mode 100644 charts/cinder-csi-plugin/templates/nodeplugin-rbac.yaml delete mode 100644 charts/cinder-csi-plugin/templates/storageclass.yaml delete mode 100644 charts/cinder-csi-plugin/values.yaml delete mode 100644 charts/manila-csi-plugin/Chart.yaml delete mode 100644 charts/manila-csi-plugin/README.md delete mode 100644 charts/manila-csi-plugin/templates/_helpers.tpl delete mode 100644 charts/manila-csi-plugin/templates/controllerplugin-clusterrole.yaml delete mode 100644 charts/manila-csi-plugin/templates/controllerplugin-clusterrolebinding.yaml delete mode 100644 charts/manila-csi-plugin/templates/controllerplugin-role.yaml delete mode 100644 charts/manila-csi-plugin/templates/controllerplugin-rolebinding.yaml delete mode 100644 charts/manila-csi-plugin/templates/controllerplugin-rules-clusterrole.yaml delete mode 100644 charts/manila-csi-plugin/templates/controllerplugin-service.yaml delete mode 100644 charts/manila-csi-plugin/templates/controllerplugin-serviceaccount.yaml delete mode 100644 charts/manila-csi-plugin/templates/controllerplugin-statefulset.yaml delete mode 100644 charts/manila-csi-plugin/templates/csidriver.yaml delete mode 100644 charts/manila-csi-plugin/templates/nodeplugin-clusterrole.yaml delete mode 100644 charts/manila-csi-plugin/templates/nodeplugin-clusterrolebinding.yaml delete mode 100644 charts/manila-csi-plugin/templates/nodeplugin-daemonset.yaml delete mode 100644 charts/manila-csi-plugin/templates/nodeplugin-rules-clusterrole.yaml delete mode 100644 charts/manila-csi-plugin/templates/nodeplugin-serviceaccount.yaml delete mode 100644 charts/manila-csi-plugin/templates/runtimeconfig-cm.yaml delete mode 100644 charts/manila-csi-plugin/values.yaml delete mode 100644 cluster/addons/rbac/cloud-controller-manager-role-bindings.yaml delete mode 100644 cluster/addons/rbac/cloud-controller-manager-roles.yaml delete mode 100644 cluster/addons/storage-class/openstack/default.yaml delete mode 100644 cluster/images/barbican-kms-plugin/Dockerfile delete mode 100644 cluster/images/barbican-kms-plugin/Dockerfile.build delete mode 100644 cluster/images/cinder-csi-plugin/Dockerfile delete mode 100644 cluster/images/cinder-csi-plugin/Dockerfile.build delete mode 100644 cluster/images/k8s-keystone-auth/Dockerfile delete mode 100644 cluster/images/k8s-keystone-auth/Dockerfile.build delete mode 100644 cluster/images/magnum-auto-healer/Dockerfile delete mode 100644 cluster/images/magnum-auto-healer/Dockerfile.build delete mode 100644 cluster/images/manila-csi-plugin/Dockerfile delete mode 100644 cluster/images/manila-csi-plugin/Dockerfile.build delete mode 100644 cluster/images/octavia-ingress-controller/Dockerfile delete mode 100644 cluster/images/octavia-ingress-controller/Dockerfile.build delete mode 100644 cluster/images/openstack-cloud-controller-manager/Dockerfile delete mode 100644 cluster/images/openstack-cloud-controller-manager/Dockerfile.build delete mode 100644 cmd/barbican-kms-plugin/main.go delete mode 100644 cmd/cinder-csi-plugin/main.go delete mode 100644 cmd/client-keystone-auth/main.go delete mode 100644 cmd/k8s-keystone-auth/main.go delete mode 100644 cmd/magnum-auto-healer/main.go delete mode 100644 cmd/manila-csi-plugin/main.go delete mode 100644 cmd/octavia-ingress-controller/main.go delete mode 100644 cmd/openstack-cloud-controller-manager/main.go delete mode 100644 cmd/tests/cinder_csi_e2e_suite_test.go delete mode 100644 code-of-conduct.md delete mode 100755 contrib/gate_hook.sh delete mode 100755 contrib/post_test_hook.sh delete mode 100755 contrib/pre_test_hook.sh delete mode 100755 devstack/plugin.sh delete mode 100644 devstack/settings delete mode 100644 docs/developers-csi-manila.md delete mode 100644 docs/expose-applications-using-loadbalancer-type-service.md delete mode 100644 docs/getting-started-provider-dev.md delete mode 100644 docs/migrate-to-ccm-with-csimigration.md delete mode 100644 docs/using-auth-data-synchronization.md delete mode 100644 docs/using-barbican-kms-plugin.md delete mode 100644 docs/using-cinder-csi-plugin.md delete mode 100644 docs/using-client-keystone-auth.md delete mode 100644 docs/using-keystone-webhook-authenticator-and-authorizer.md delete mode 100644 docs/using-magnum-auto-healer.md delete mode 100644 docs/using-manila-csi-plugin.md delete mode 100644 docs/using-octavia-ingress-controller.md delete mode 100644 docs/using-openstack-cloud-controller-manager.md delete mode 100644 examples/cinder-csi-plugin/block/block.yaml delete mode 100644 examples/cinder-csi-plugin/clone/example-clone.yaml delete mode 100644 examples/cinder-csi-plugin/clone/sc.yaml delete mode 100644 examples/cinder-csi-plugin/clone/source-pvc.yaml delete mode 100644 examples/cinder-csi-plugin/inline/inline-example.yaml delete mode 100644 examples/cinder-csi-plugin/nginx.yaml delete mode 100644 examples/cinder-csi-plugin/resize/example.yaml delete mode 100644 examples/cinder-csi-plugin/snapshot/example.yaml delete mode 100644 examples/cinder-csi-plugin/snapshot/snapshotcreate.yaml delete mode 100644 examples/cinder-csi-plugin/snapshot/snapshotrestore.yaml delete mode 100644 examples/cinder-csi-plugin/topology/example.yaml delete mode 100644 examples/loadbalancers/README.MD delete mode 100644 examples/loadbalancers/external-http-nginx.yaml delete mode 100644 examples/loadbalancers/internal-http-nginx.yaml delete mode 100644 examples/manila-csi-plugin/README.md delete mode 100755 examples/manila-csi-plugin/exec-bash.sh delete mode 100755 examples/manila-csi-plugin/logs.sh delete mode 100644 examples/manila-csi-plugin/nfs/dynamic-provisioning/pod.yaml delete mode 100644 examples/manila-csi-plugin/nfs/dynamic-provisioning/pvc.yaml delete mode 100644 examples/manila-csi-plugin/nfs/dynamic-provisioning/storageclass.yaml delete mode 100644 examples/manila-csi-plugin/nfs/secrets.yaml delete mode 100644 examples/manila-csi-plugin/nfs/snapshot/pod.yaml delete mode 100644 examples/manila-csi-plugin/nfs/snapshot/snapshotclass.yaml delete mode 100644 examples/manila-csi-plugin/nfs/snapshot/snapshotcreate.yaml delete mode 100644 examples/manila-csi-plugin/nfs/snapshot/snapshotrestore.yaml delete mode 100644 examples/manila-csi-plugin/nfs/static-provisioning/pod.yaml delete mode 100644 examples/manila-csi-plugin/nfs/static-provisioning/preprovisioned-pvc.yaml delete mode 100644 examples/manila-csi-plugin/nfs/topology-aware/pod.yaml delete mode 100644 examples/manila-csi-plugin/nfs/topology-aware/pvc.yaml delete mode 100644 examples/manila-csi-plugin/nfs/topology-aware/storageclass.yaml delete mode 100644 examples/manila-csi-plugin/runtimeconfig-cm.yaml delete mode 100644 examples/mysql-cinder-pd/README.md delete mode 100644 examples/mysql-cinder-pd/mysql-service.yaml delete mode 100644 examples/mysql-cinder-pd/mysql.yaml delete mode 100644 examples/persistent-volume-provisioning/cinder/cinder-full.yaml delete mode 100644 examples/persistent-volume-provisioning/cinder/cinder-in-tree-full.yaml delete mode 100644 examples/persistent-volume-provisioning/cinder/cinder-storage-class.yaml delete mode 100644 examples/persistent-volume-provisioning/cinder/example-pod.yaml delete mode 100644 examples/volumes/cinder/README.md delete mode 100644 examples/volumes/cinder/cinder-web.yaml delete mode 100644 examples/webhook/keystone-apiserver-webhook.yaml delete mode 100644 examples/webhook/keystone-deployment.yaml delete mode 100644 examples/webhook/keystone-policy-configmap.yaml delete mode 100644 examples/webhook/keystone-rbac.yaml delete mode 100644 examples/webhook/keystone-service.yaml delete mode 100644 examples/webhook/policy.json delete mode 100644 go.mod delete mode 100644 go.sum delete mode 100755 hack/make.sh delete mode 100755 hack/update-gofmt.sh delete mode 100755 hack/verify-gofmt.sh delete mode 100755 hack/verify-golint.sh delete mode 100755 hack/verify-import-boss.sh rename hack/.golint_failures => index.yaml (100%) delete mode 100755 manifests/barbican-kms/encryption-config.yaml delete mode 100644 manifests/barbican-kms/pod.yaml delete mode 100644 manifests/cinder-csi-plugin/cinder-csi-controllerplugin-rbac.yaml delete mode 100644 manifests/cinder-csi-plugin/cinder-csi-controllerplugin.yaml delete mode 100644 manifests/cinder-csi-plugin/cinder-csi-nodeplugin-rbac.yaml delete mode 100644 manifests/cinder-csi-plugin/cinder-csi-nodeplugin.yaml delete mode 100644 manifests/cinder-csi-plugin/csi-cinder-driver.yaml delete mode 100644 manifests/cinder-csi-plugin/csi-secret-cinderplugin.yaml delete mode 100644 manifests/controller-manager/cloud-config delete mode 100644 manifests/controller-manager/kubeadm.conf delete mode 100644 manifests/controller-manager/openstack-cloud-controller-manager-ds.yaml delete mode 100644 manifests/controller-manager/openstack-cloud-controller-manager-pod.yaml delete mode 100644 manifests/magnum-auto-healer/magnum-auto-healer.yaml delete mode 100644 manifests/manila-csi-plugin/csi-controllerplugin-rbac.yaml delete mode 100644 manifests/manila-csi-plugin/csi-controllerplugin.yaml delete mode 100644 manifests/manila-csi-plugin/csi-nodeplugin-rbac.yaml delete mode 100644 manifests/manila-csi-plugin/csi-nodeplugin.yaml delete mode 100644 manifests/manila-csi-plugin/csidriver.yaml delete mode 100644 pkg/autohealing/OWNERS delete mode 100644 pkg/autohealing/cloudprovider/cloudprovider.go delete mode 100644 pkg/autohealing/cloudprovider/openstack/provider.go delete mode 100644 pkg/autohealing/cloudprovider/register/register.go delete mode 100644 pkg/autohealing/cmd/root.go delete mode 100644 pkg/autohealing/config/config.go delete mode 100644 pkg/autohealing/controller/controller.go delete mode 100644 pkg/autohealing/healthcheck/healthcheck.go delete mode 100644 pkg/autohealing/healthcheck/plugin_endpoint.go delete mode 100644 pkg/autohealing/healthcheck/plugin_nodecondition.go delete mode 100644 pkg/autohealing/utils/utils.go delete mode 100644 pkg/cloudprovider/.import-restrictions delete mode 100644 pkg/cloudprovider/providers/openstack/OWNERS delete mode 100644 pkg/cloudprovider/providers/openstack/doc.go delete mode 100644 pkg/cloudprovider/providers/openstack/openstack.go delete mode 100644 pkg/cloudprovider/providers/openstack/openstack_client.go delete mode 100644 pkg/cloudprovider/providers/openstack/openstack_instances.go delete mode 100644 pkg/cloudprovider/providers/openstack/openstack_loadbalancer.go delete mode 100644 pkg/cloudprovider/providers/openstack/openstack_metrics.go delete mode 100644 pkg/cloudprovider/providers/openstack/openstack_routes.go delete mode 100644 pkg/cloudprovider/providers/openstack/openstack_routes_test.go delete mode 100644 pkg/cloudprovider/providers/openstack/openstack_test.go delete mode 100644 pkg/csi/cinder/controllerserver.go delete mode 100644 pkg/csi/cinder/controllerserver_test.go delete mode 100644 pkg/csi/cinder/driver.go delete mode 100644 pkg/csi/cinder/driver_test.go delete mode 100644 pkg/csi/cinder/etc/cloud.conf delete mode 100644 pkg/csi/cinder/fake.go delete mode 100644 pkg/csi/cinder/identityserver.go delete mode 100644 pkg/csi/cinder/identityserver_test.go delete mode 100644 pkg/csi/cinder/nodeserver.go delete mode 100644 pkg/csi/cinder/nodeserver_test.go delete mode 100644 pkg/csi/cinder/openstack/openstack.go delete mode 100644 pkg/csi/cinder/openstack/openstack_instances.go delete mode 100644 pkg/csi/cinder/openstack/openstack_mock.go delete mode 100644 pkg/csi/cinder/openstack/openstack_snapshots.go delete mode 100644 pkg/csi/cinder/openstack/openstack_test.go delete mode 100644 pkg/csi/cinder/openstack/openstack_volumes.go delete mode 100644 pkg/csi/cinder/server.go delete mode 100644 pkg/csi/cinder/utils.go delete mode 100644 pkg/csi/cinder/utils_test.go delete mode 100644 pkg/csi/manila/OWNERS delete mode 100644 pkg/csi/manila/adapters.go delete mode 100644 pkg/csi/manila/capabilities/manilacapabilities.go delete mode 100644 pkg/csi/manila/compatibility/compatibility.go delete mode 100644 pkg/csi/manila/controllerserver.go delete mode 100644 pkg/csi/manila/csiclient.go delete mode 100644 pkg/csi/manila/csiclient/builder.go delete mode 100644 pkg/csi/manila/csiclient/client.go delete mode 100644 pkg/csi/manila/csiclient/interface.go delete mode 100644 pkg/csi/manila/driver.go delete mode 100644 pkg/csi/manila/identityserver.go delete mode 100644 pkg/csi/manila/manilaclient/builder.go delete mode 100644 pkg/csi/manila/manilaclient/client.go delete mode 100644 pkg/csi/manila/manilaclient/interface.go delete mode 100644 pkg/csi/manila/nodeserver.go delete mode 100644 pkg/csi/manila/options/compatibilityoptions.go delete mode 100644 pkg/csi/manila/options/openstackoptions.go delete mode 100644 pkg/csi/manila/options/shareoptions.go delete mode 100644 pkg/csi/manila/runtimeconfig/nfsconfig.go delete mode 100644 pkg/csi/manila/runtimeconfig/runtimeconfig.go delete mode 100644 pkg/csi/manila/share.go delete mode 100644 pkg/csi/manila/shareadapters/cephfs.go delete mode 100644 pkg/csi/manila/shareadapters/nfs.go delete mode 100644 pkg/csi/manila/shareadapters/shareadapter.go delete mode 100644 pkg/csi/manila/shareadapters/util.go delete mode 100644 pkg/csi/manila/snapshot.go delete mode 100644 pkg/csi/manila/util.go delete mode 100644 pkg/csi/manila/util/exportlocation.go delete mode 100644 pkg/csi/manila/util/exportlocation_test.go delete mode 100644 pkg/csi/manila/validator/dependsexpr.go delete mode 100644 pkg/csi/manila/validator/identity.go delete mode 100644 pkg/csi/manila/validator/matchexpr.go delete mode 100644 pkg/csi/manila/validator/precludesexpr.go delete mode 100644 pkg/csi/manila/validator/util.go delete mode 100644 pkg/csi/manila/validator/validator.go delete mode 100644 pkg/csi/manila/validator/validator_test.go delete mode 100644 pkg/csi/manila/validator/valueexpr.go delete mode 100644 pkg/csi/manila/volumesource.go delete mode 100644 pkg/identity/keystone/authenticator.go delete mode 100644 pkg/identity/keystone/authenticator_test.go delete mode 100644 pkg/identity/keystone/authorizer.go delete mode 100644 pkg/identity/keystone/authorizer_test.go delete mode 100644 pkg/identity/keystone/authorizer_test_policy.json delete mode 100644 pkg/identity/keystone/authorizer_test_policy_version2.json delete mode 100644 pkg/identity/keystone/config.go delete mode 100644 pkg/identity/keystone/keystone.go delete mode 100644 pkg/identity/keystone/keystone_test.go delete mode 100644 pkg/identity/keystone/mock_IKeystone.go delete mode 100644 pkg/identity/keystone/policy.go delete mode 100644 pkg/identity/keystone/sync.go delete mode 100644 pkg/identity/keystone/sync_test.go delete mode 100644 pkg/identity/keystone/sync_test.yaml delete mode 100644 pkg/identity/keystone/token_getter.go delete mode 100644 pkg/identity/keystone/token_getter_test.go delete mode 100644 pkg/ingress/cmd/root.go delete mode 100644 pkg/ingress/config/config.go delete mode 100644 pkg/ingress/controller/controller.go delete mode 100644 pkg/ingress/controller/openstack/client.go delete mode 100644 pkg/ingress/controller/openstack/neutron.go delete mode 100644 pkg/ingress/controller/openstack/octavia.go delete mode 100644 pkg/ingress/utils/utils.go delete mode 100644 pkg/kms/barbican/barbican.go delete mode 100644 pkg/kms/barbican/fake_barbican.go delete mode 100644 pkg/kms/client/client.go delete mode 100644 pkg/kms/encryption/aescbc/aescbc.go delete mode 100644 pkg/kms/encryption/aescbc/aescbc_test.go delete mode 100644 pkg/kms/server/server.go delete mode 100644 pkg/kms/server/server_test.go delete mode 100644 pkg/util/blockdevice/blockdevice_linux.go delete mode 100644 pkg/util/blockdevice/blockdevice_unsupported.go delete mode 100644 pkg/util/errors/errors.go delete mode 100644 pkg/util/file/file.go delete mode 100644 pkg/util/io/consistentread.go delete mode 100644 pkg/util/metadata/metadata.go delete mode 100644 pkg/util/metadata/metadata_mock.go delete mode 100644 pkg/util/metadata/metadata_test.go delete mode 100644 pkg/util/mount/mount.go delete mode 100644 pkg/util/mount/mount_mock.go delete mode 100644 pkg/util/net/sets/ipnet.go delete mode 100644 pkg/util/openstack/keymanager.go delete mode 100644 pkg/util/openstack/loadbalancer.go delete mode 100644 pkg/util/openstack/network.go delete mode 100644 pkg/util/util.go delete mode 100644 pkg/version/version.go delete mode 100644 release-procedure.md delete mode 100644 snapcraft.yaml delete mode 100755 tests/e2e/cloudprovider/test-lb-service.sh delete mode 100644 tests/e2e/csi/cinder/cinder-testdriver.go delete mode 100644 tests/e2e/csi/cinder/csi-volumes.go delete mode 100755 tests/e2e/k8s-keystone-auth/test-authz.sh delete mode 100644 tests/sanity/cinder/fakecloud.go delete mode 100644 tests/sanity/cinder/fakemetadata.go delete mode 100644 tests/sanity/cinder/fakemount.go delete mode 100644 tests/sanity/cinder/sanity_test.go delete mode 100644 tests/sanity/manila/fake-secrets.yaml delete mode 100644 tests/sanity/manila/fakecsiclient.go delete mode 100644 tests/sanity/manila/fakemanilaclient.go delete mode 100644 tests/sanity/manila/sanity_test.go delete mode 100755 tools/install-distro-packages.sh delete mode 100755 tools/test-setup.sh diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md deleted file mode 100644 index 2380ad460d..0000000000 --- a/.github/ISSUE_TEMPLATE.md +++ /dev/null @@ -1,39 +0,0 @@ - - -**Is this a BUG REPORT or FEATURE REQUEST?**: - -> Uncomment only one, leave it on its own line: -> -> /kind bug -> /kind feature - -**What happened**: - - -**What you expected to happen**: - - -**How to reproduce it**: - - -**Anything else we need to know?**: - - -**Environment**: -- openstack-cloud-controller-manager(or other related binary) version: -- OpenStack version: -- Others: diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md deleted file mode 100644 index bd2247841b..0000000000 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ /dev/null @@ -1,33 +0,0 @@ - - -**What this PR does / why we need it**: - -**Which issue this PR fixes(if applicable)**: -fixes # - -**Special notes for reviewers**: - - -**Release note**: - -```release-note -NONE -``` diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml deleted file mode 100644 index a0a4f365d5..0000000000 --- a/.github/workflows/pr.yaml +++ /dev/null @@ -1,50 +0,0 @@ -name: Lint and Test Charts - -on: pull_request - -jobs: - changes: - outputs: - charts: ${{ steps.filter.outputs.charts }} - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - - id: filter - uses: dorny/paths-filter@v2.2.0 - with: - filters: | - charts: - - 'charts/**/Chart.yaml' - - 'charts/**/*' - token: ${{ secrets.GITHUB_TOKEN }} - - helm-lint-test: - if: ${{ needs.changes.outputs.charts == 'true' }} - name: Helm chart - needs: - - changes - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Fetch history - run: git fetch --prune --unshallow - - - id: lint - name: Run chart-testing (lint) - uses: helm/chart-testing-action@v1.0.0 - with: - command: lint - - # Only build a kind cluster if there are chart changes to test. - - if: steps.lint.outputs.changed == 'true' - name: Create kind cluster - uses: helm/kind-action@v1.0.0 - - - name: Run chart-testing (install) - uses: helm/chart-testing-action@v1.0.0 - with: - command: install diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml deleted file mode 100644 index 7b5746ab57..0000000000 --- a/.github/workflows/release.yaml +++ /dev/null @@ -1,26 +0,0 @@ -name: Release Charts - -on: - push: - branches: - - master - -jobs: - release: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Fetch history - run: git fetch --prune --unshallow - - - name: Configure Git - run: | - git config user.name "$GITHUB_ACTOR" - git config user.email "$GITHUB_ACTOR@users.noreply.github.com" - - - name: Run chart-releaser - uses: helm/chart-releaser-action@v1.0.0 - env: - CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" diff --git a/.zuul.yaml b/.zuul.yaml deleted file mode 100644 index 446a791362..0000000000 --- a/.zuul.yaml +++ /dev/null @@ -1,208 +0,0 @@ -- project: - cloud-provider-openstack-unittest: - jobs: - - cloud-provider-openstack-unittest: - irrelevant-files: - - ^docs/.*$ - - ^.*\.md$ - - ^OWNERS$ - - ^SECURITY_CONTACTS$ - - ^.gitignore$ - - ^.*\.sh$ - cloud-provider-openstack-acceptance-test-lb-octavia: - jobs: - - cloud-provider-openstack-acceptance-test-lb-octavia: - files: - - cmd/openstack-cloud-controller-manager/.* - - pkg/cloudprovider/.* - - pkg/util/.* - - tests/e2e/cloudprovider/.* - - go.mod$ - - go.sum$ - - Makefile$ - irrelevant-files: - - docs/.*$ - - ^.*\.md$ - - ^OWNERS$ - - ^SECURITY_CONTACTS$ - - ^.gitignore$ - cloud-provider-openstack-format: - jobs: - - cloud-provider-openstack-format: - irrelevant-files: - - ^docs/.*$ - - ^.*\.md$ - - ^OWNERS$ - - ^SECURITY_CONTACTS$ - - ^.gitignore$ - - ^.*\.sh$ - cloud-provider-openstack-acceptance-test-keystone-authentication-authorization: - jobs: - - cloud-provider-openstack-acceptance-test-keystone-authentication-authorization: - files: - - cmd/k8s-keystone-auth/.* - - pkg/identity/.* - - go.mod$ - - go.sum$ - - Makefile$ - - tests/e2e/k8s-keystone-auth/.* - irrelevant-files: - - ^docs/.*$ - - ^.*\.md$ - - ^OWNERS$ - - ^SECURITY_CONTACTS$ - - ^.gitignore$ - cloud-provider-openstack-acceptance-test-csi-cinder: - jobs: - - cloud-provider-openstack-acceptance-test-csi-cinder: - files: - - cluster/images/cinder-csi-plugin/.* - - cmd/cinder-csi-plugin/.* - - manifests/cinder-csi-plugin/.* - - pkg/csi/cinder/.* - - pkg/util/.* - - pkg/cloudprovider/providers/openstack/.* - - test/.* - - cmd/tests/.* - - go.mod$ - - go.sum$ - - Makefile$ - irrelevant-files: - - ^docs/.*$ - - ^.*\.md$ - - ^OWNERS$ - - ^SECURITY_CONTACTS$ - - ^.gitignore$ - cloud-provider-openstack-e2e-test-csi-cinder: - jobs: - - cloud-provider-openstack-e2e-test-csi-cinder: - files: - - cmd/cinder-csi-plugin/.* - - manifests/cinder-csi-plugin/.* - - pkg/csi/cinder/.* - - pkg/util/.* - - tests/e2e/csi/cinder/.* - - cmd/tests/.* - - go.mod$ - - go.sum$ - - Makefile$ - irrelevant-files: - - docs/.*$ - - ^.*\.md$ - - ^OWNERS$ - - ^SECURITY_CONTACTS$ - cloud-provider-openstack-sanity-test-csi-cinder: - jobs: - - cloud-provider-openstack-sanity-test-csi-cinder: - files: - - cmd/cinder-csi-plugin/.* - - pkg/csi/cinder/.* - - pkg/util/.* - - tests/sanity/cinder/.* - - go.mod$ - - go.sum$ - - Makefile$ - irrelevant-files: - - docs/.*$ - - ^.*\.md$ - - ^OWNERS$ - - ^SECURITY_CONTACTS$ - cloud-provider-openstack-multinode-csi-migration-test: - jobs: - - cloud-provider-openstack-multinode-csi-migration-test: - files: - - cmd/cinder-csi-plugin/.* - - manifests/cinder-csi-plugin/.* - - pkg/csi/cinder/.* - - pkg/util/.* - - go.mod$ - - go.sum$ - - Makefile$ - - .zuul.yaml$ - cloud-provider-openstack-acceptance-test-csi-manila: - jobs: - - cloud-provider-openstack-acceptance-test-csi-manila: - files: - - cmd/manila-csi-plugin/.* - - examples/manila-csi-plugin/.* - - pkg/csi/manila/.* - - pkg/util/errors/.* - - pkg/cloudprovider/providers/openstack/.* - - pkg/share/manila/shareoptions/validator/.* - - go.mod$ - - go.sum$ - - Makefile$ - irrelevant-files: - - ^docs/.*$ - - ^.*\.md$ - - ^OWNERS$ - - ^SECURITY_CONTACTS$ - - ^.gitignore$ - cloud-provider-openstack-acceptance-test-e2e-conformance: - jobs: - - cloud-provider-openstack-acceptance-test-e2e-conformance: - files: - - cmd/openstack-cloud-controller-manager/.* - - pkg/cloudprovider/.* - - pkg/util/.* - - tests/e2e/cloudprovider/.* - - go.mod$ - - go.sum$ - - Makefile$ - irrelevant-files: - - ^docs/.*$ - - ^.*\.md$ - - ^OWNERS$ - - ^SECURITY_CONTACTS$ - - ^.gitignore$ - cloud-provider-openstack-acceptance-test-e2e-conformance-stable-branch-v1.16: - jobs: - - cloud-provider-openstack-acceptance-test-e2e-conformance-stable-branch-v1.16: - files: - - cmd/openstack-cloud-controller-manager/.* - - pkg/cloudprovider/.* - - pkg/util/.* - - tests/e2e/cloudprovider/.* - - go.mod$ - - go.sum$ - - Makefile$ - irrelevant-files: - - ^docs/.*$ - - ^.*\.md$ - - ^OWNERS$ - - ^SECURITY_CONTACTS$ - - ^.gitignore$ - cloud-provider-openstack-acceptance-test-e2e-conformance-stable-branch-v1.17: - jobs: - - cloud-provider-openstack-acceptance-test-e2e-conformance-stable-branch-v1.17: - files: - - cmd/openstack-cloud-controller-manager/.* - - pkg/cloudprovider/.* - - pkg/util/.* - - tests/e2e/cloudprovider/.* - - go.mod$ - - go.sum$ - - Makefile$ - irrelevant-files: - - ^docs/.*$ - - ^.*\.md$ - - ^OWNERS$ - - ^SECURITY_CONTACTS$ - - ^.gitignore$ - cloud-provider-openstack-acceptance-test-e2e-conformance-stable-branch-v1.18: - jobs: - - cloud-provider-openstack-acceptance-test-e2e-conformance-stable-branch-v1.18: - files: - - cmd/openstack-cloud-controller-manager/.* - - pkg/cloudprovider/.* - - pkg/util/.* - - tests/e2e/cloudprovider/.* - - go.mod$ - - go.sum$ - - Makefile$ - irrelevant-files: - - ^docs/.*$ - - ^.*\.md$ - - ^OWNERS$ - - ^SECURITY_CONTACTS$ - - ^.gitignore$ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index ec3d25c92f..0000000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,25 +0,0 @@ -# Contributing Guidelines - -The **kubernetes/cloud-provider-openstack** project accepts contribution via github [pull request](https://help.github.com/articles/about-pull-requests/). This document outlines the process to help get your contribution accepted. Please also read the [Kubernetes contributor guide](https://github.com/kubernetes/community/blob/master/contributors/guide/README.md) - -### Sign the Contributor License Agreement - -We'd love to accept your patches! Before we can accept them you need to sign Cloud Native Computing Foundation (CNCF) [CLA](https://github.com/kubernetes/community/blob/master/CLA.md). - -### Setup the Development Environment -Refer [getting started](https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/getting-started-provider-dev.md) document to setup development environment - -### Reporting an issue -If you find a bug or a feature request related to cloud-provider-openstack you can create a new github issue in this repo @[kubernetes/cloud-provider-openstack](https://github.com/kubernetes/cloud-provider-openstack/issues). - -### Contributing a Patch -1. Submit an issue describing your proposed change to the repo. -2. Fork the cloud-provider-openstack repo, develop and test your code changes. -3. Submit a pull request. -4. The bot will automatically assigns someone to review your PR. Check the full list of bot commands [here](https://prow.k8s.io/command-help) - -### Contact and Meeting Time - -* [Slack](https://kubernetes.slack.com/messages/sig-openstack). -* [Mailing list](https://groups.google.com/forum/#!forum/kubernetes-sig-openstack). -* Please check the [sig-openstack community page](https://github.com/kubernetes/community/tree/master/sig-cloud-provider#provider-openstack) for meeting time. diff --git a/LICENSE b/LICENSE deleted file mode 100644 index d645695673..0000000000 --- a/LICENSE +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/Makefile b/Makefile deleted file mode 100644 index b52b26188b..0000000000 --- a/Makefile +++ /dev/null @@ -1,329 +0,0 @@ -# golang-client Makefile -# Follows the interface defined in the Golang CTI proposed -# in https://review.openstack.org/410355 - -#REPO_VERSION?=$(shell git describe --tags) - -GIT_HOST = k8s.io - -PWD := $(shell pwd) -BASE_DIR := $(shell basename $(PWD)) -# Keep an existing GOPATH, make a private one if it is undefined -GOPATH_DEFAULT := $(PWD)/.go -export GOPATH ?= $(GOPATH_DEFAULT) -GOBIN_DEFAULT := $(GOPATH)/bin -export GOBIN ?= $(GOBIN_DEFAULT) -export GO111MODULE := on -TESTARGS_DEFAULT := "-v" -export TESTARGS ?= $(TESTARGS_DEFAULT) -PKG := $(shell awk '/^module/ { print $$2 }' go.mod) -DEST := $(GOPATH)/src/$(GIT_HOST)/$(BASE_DIR) -SOURCES := $(shell find $(DEST) -name '*.go' 2>/dev/null) -HAS_LINT := $(shell command -v golint;) -HAS_GOX := $(shell command -v gox;) -HAS_IMPORT_BOSS := $(shell command -v import-boss;) -GOX_PARALLEL ?= 3 - -TARGETS ?= darwin/amd64 linux/amd64 linux/386 linux/arm linux/arm64 linux/ppc64le linux/s390x -DIST_DIRS = find * -type d -exec - -TEMP_DIR :=$(shell mktemp -d) -TAR_FILE ?= rootfs.tar - -GOOS ?= $(shell go env GOOS) -VERSION ?= $(shell git describe --exact-match 2> /dev/null || \ - git describe --match=$(git rev-parse --short=8 HEAD) --always --dirty --abbrev=8) -ALPINE_ARCH := -DEBIAN_ARCH := -QEMUARCH := -QEMUVERSION := "v4.2.0-4" -GOARCH := -GOFLAGS := -TAGS := -LDFLAGS := "-w -s -X 'k8s.io/cloud-provider-openstack/pkg/version.Version=${VERSION}'" -REGISTRY ?= k8scloudprovider -IMAGE_OS ?= linux -IMAGE_NAMES ?= openstack-cloud-controller-manager \ - cinder-csi-plugin \ - k8s-keystone-auth \ - octavia-ingress-controller \ - manila-csi-plugin \ - barbican-kms-plugin \ - magnum-auto-healer -ARCH ?= amd64 -ARCHS ?= amd64 arm arm64 ppc64le s390x -BUILD_CMDS ?= openstack-cloud-controller-manager \ - cinder-csi-plugin \ - k8s-keystone-auth \ - octavia-ingress-controller \ - manila-csi-plugin \ - barbican-kms-plugin \ - magnum-auto-healer \ - client-keystone-auth - -# This option is for running docker manifest command -export DOCKER_CLI_EXPERIMENTAL := enabled - -# CTI targets - -$(GOBIN): - echo "create gobin" - mkdir -p $(GOBIN) - -work: $(GOBIN) - -ifeq ($(ARCH),arm) - DEBIAN_ARCH=$(ARCH) - GOARCH=$(ARCH) - QEMUARCH=$(ARCH) - ALPINE_ARCH=arm32v7 -else ifeq ($(ARCH),arm64) - DEBIAN_ARCH=$(ARCH) - GOARCH=$(ARCH) - QEMUARCH=aarch64 - ALPINE_ARCH=arm64v8 -else - DEBIAN_ARCH=$(ARCH) - GOARCH=$(ARCH) - QEMUARCH=$(ARCH) - ALPINE_ARCH=$(ARCH) -endif - -build-all-archs: - @for arch in $(ARCHS); do $(MAKE) ARCH=$${arch} build ; done - -build: $(addprefix build-cmd-,$(BUILD_CMDS)) - -# Remove individual go build targets, once we migrate openlab-zuul-jobs -# to use new build-cmd-% targets. -cinder-csi-plugin: work $(SOURCES) - CGO_ENABLED=0 GOOS=$(GOOS) go build \ - -ldflags $(LDFLAGS) \ - -o cinder-csi-plugin \ - cmd/cinder-csi-plugin/main.go - -# This target is for supporting CI jobs of release-1.17 branch. We should delete this target once 1.17 support is dropped and change the cinder-csi-plugin related CI jobs to use target image-cinder-csi-plugin -image-csi-plugin: - $(MAKE) image-cinder-csi-plugin - -manila-csi-plugin: work $(SOURCES) - CGO_ENABLED=0 GOOS=$(GOOS) go build \ - -ldflags $(LDFLAGS) \ - -o manila-csi-plugin \ - cmd/manila-csi-plugin/main.go - -# Remove this individual go build target, once we remove -# image-controller-manager below. -openstack-cloud-controller-manager: work $(SOURCES) - CGO_ENABLED=0 GOOS=$(GOOS) go build \ - -ldflags $(LDFLAGS) \ - -o openstack-cloud-controller-manager-$(ARCH) \ - cmd/openstack-cloud-controller-manager/main.go - -# Remove individual image builder once we migrate openlab-zuul-jobs -# to use new image-openstack-cloud-controller-manager target. -image-controller-manager: work openstack-cloud-controller-manager -ifeq ($(GOOS),linux) - cp -r cluster/images/openstack-cloud-controller-manager $(TEMP_DIR) - cp openstack-cloud-controller-manager-$(ARCH) $(TEMP_DIR)/openstack-cloud-controller-manager - cp $(TEMP_DIR)/openstack-cloud-controller-manager/Dockerfile.build $(TEMP_DIR)/openstack-cloud-controller-manager/Dockerfile - docker build -t $(REGISTRY)/openstack-cloud-controller-manager:$(VERSION) $(TEMP_DIR)/openstack-cloud-controller-manager - rm -rf $(TEMP_DIR)/openstack-cloud-controller-manager -else - $(error Please set GOOS=linux for building the image) -endif - -build-cmd-%: work $(SOURCES) - @# Keep binary with no arch mark. We should remove this once we correct - @# openlab-zuul-jobs. - CGO_ENABLED=0 GOOS=$(GOOS) go build \ - -ldflags $(LDFLAGS) \ - -o $* \ - cmd/$*/main.go - CGO_ENABLED=0 GOOS=$(GOOS) GOARCH=$(GOARCH) go build \ - -ldflags $(LDFLAGS) \ - -o $*-$(ARCH) \ - cmd/$*/main.go - -test: unit functional - -check: work fmt vet lint import-boss - -unit: work - go test -tags=unit $(shell go list ./... | sed -e '/sanity/ { N; d; }' | sed -e '/tests/ {N; d;}') $(TESTARGS) - -functional: - @echo "$@ not yet implemented" - -test-cinder-csi-sanity: work - go test $(GIT_HOST)/$(BASE_DIR)/tests/sanity/cinder - -test-manila-csi-sanity: work - go test $(GIT_HOST)/$(BASE_DIR)/tests/sanity/manila - -fmt: - hack/verify-gofmt.sh - -lint: -ifndef HAS_LINT - echo "installing lint" - GO111MODULE=off go get -u golang.org/x/lint/golint -endif - hack/verify-golint.sh - -import-boss: -ifndef HAS_IMPORT_BOSS - echo "installing import-boss" - go get -u k8s.io/code-generator/cmd/import-boss -endif - hack/verify-import-boss.sh - -vet: - go vet ./... - -cover: work - go test -tags=unit $(shell go list ./...) -cover - -docs: - @echo "$@ not yet implemented" - -godoc: - @echo "$@ not yet implemented" - -releasenotes: - @echo "Reno not yet implemented for this repo" - -translation: - @echo "$@ not yet implemented" - -# Do the work here - -# Set up the development environment -env: - @echo "PWD: $(PWD)" - @echo "BASE_DIR: $(BASE_DIR)" - @echo "GOPATH: $(GOPATH)" - @echo "GOROOT: $(GOROOT)" - @echo "DEST: $(DEST)" - @echo "PKG: $(PKG)" - go version - go env - -# Get our dev/test dependencies in place -bootstrap: - tools/test-setup.sh - -.bindep: - virtualenv .bindep - .bindep/bin/pip install -i https://pypi.python.org/simple bindep - -bindep: .bindep - @.bindep/bin/bindep -b -f bindep.txt || true - -install-distro-packages: - tools/install-distro-packages.sh - -clean: - rm -rf _dist .bindep - @echo "clean builds binary" - @for binary in $(BUILD_CMDS); do rm -rf $${binary}*; done - -realclean: clean - rm -rf vendor - if [ "$(GOPATH)" = "$(GOPATH_DEFAULT)" ]; then \ - rm -rf $(GOPATH); \ - fi - -shell: - $(SHELL) -i - -push-manifest-%: - docker manifest create --amend $(REGISTRY)/$*:$(VERSION) $(shell echo $(ARCHS) | sed -e "s~[^ ]*~$(REGISTRY)/$*\-&:$(VERSION)~g") - @for arch in $(ARCHS); do docker manifest annotate --os $(IMAGE_OS) --arch $${arch} $(REGISTRY)/$*:${VERSION} $(REGISTRY)/$*-$${arch}:${VERSION}; done - docker manifest push --purge $(REGISTRY)/$*:${VERSION} - -push-all-manifest: $(addprefix push-manifest-,$(IMAGE_NAMES)) - -build-images: $(addprefix image-,$(IMAGE_NAMES)) - -push-images: $(addprefix push-image-,$(IMAGE_NAMES)) - -image-%: work - $(MAKE) $(addprefix build-cmd-,$*) -ifeq ($(GOOS),linux) - cp -r cluster/images/$* $(TEMP_DIR) - -ifneq ($(ARCH),amd64) - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - curl -sSL https://github.com/multiarch/qemu-user-static/releases/download/$(QEMUVERSION)/x86_64_qemu-$(QEMUARCH)-static.tar.gz | tar -xz -C $(TEMP_DIR)/$* - @# Ensure we don't get surprised by umask settings - chmod 0755 $(TEMP_DIR)/$*/qemu-$(QEMUARCH)-static - sed "/^FROM .*/a COPY qemu-$(QEMUARCH)-static /usr/bin/" $(TEMP_DIR)/$*/Dockerfile.build > $(TEMP_DIR)/$*/Dockerfile.build.tmp - mv $(TEMP_DIR)/$*/Dockerfile.build.tmp $(TEMP_DIR)/$*/Dockerfile.build -endif - - cp $*-$(ARCH) $(TEMP_DIR)/$* - docker build --build-arg ALPINE_ARCH=$(ALPINE_ARCH) --build-arg ARCH=$(ARCH) --build-arg DEBIAN_ARCH=$(DEBIAN_ARCH) --pull -t build-$*-$(ARCH) -f $(TEMP_DIR)/$*/Dockerfile.build $(TEMP_DIR)/$* - docker create --name build-$*-$(ARCH) build-$*-$(ARCH) - docker export build-$*-$(ARCH) > $(TEMP_DIR)/$*/$(TAR_FILE) - - @echo "build image $(REGISTRY)/$*-$(ARCH)" - docker build --build-arg ALPINE_ARCH=$(ALPINE_ARCH) --build-arg ARCH=$(ARCH) --build-arg DEBIAN_ARCH=$(DEBIAN_ARCH) --pull -t $(REGISTRY)/$*-$(ARCH):$(VERSION) $(TEMP_DIR)/$* - - rm -rf $(TEMP_DIR)/$* - docker rm build-$*-$(ARCH) - docker rmi build-$*-$(ARCH) -else - $(error Please set GOOS=linux for building the image) -endif - -push-image-%: - @echo "push image $*-$(ARCH) to $(REGISTRY)" -ifneq ($(and $(DOCKER_USERNAME),$(DOCKER_PASSWORD)),) - @docker login -u="$(DOCKER_USERNAME)" -p="$(DOCKER_PASSWORD)" -endif - docker push $(REGISTRY)/$*-$(ARCH):$(VERSION) - -images: $(addprefix build-arch-image-,$(ARCH)) - -images-all-archs: $(addprefix build-arch-image-,$(ARCHS)) - -build-arch-image-%: - @echo "Building images for ARCH=$*" - $(MAKE) ARCH=$* build-images - -upload-image-%: - $(MAKE) ARCH=$* build-images push-images - -upload-images: $(addprefix upload-image-,$(ARCHS)) push-all-manifest - -version: - @echo ${VERSION} - -.PHONY: build-cross -build-cross: LDFLAGS += -extldflags "-static" -build-cross: work -ifndef HAS_GOX - echo "installing gox" - GO111MODULE=off go get -u github.com/mitchellh/gox -endif - CGO_ENABLED=0 gox -parallel=$(GOX_PARALLEL) -output="_dist/{{.OS}}-{{.Arch}}/{{.Dir}}" -osarch='$(TARGETS)' $(GOFLAGS) $(if $(TAGS),-tags '$(TAGS)',) -ldflags '$(LDFLAGS)' $(GIT_HOST)/$(BASE_DIR)/cmd/openstack-cloud-controller-manager/ - CGO_ENABLED=0 gox -parallel=$(GOX_PARALLEL) -output="_dist/{{.OS}}-{{.Arch}}/{{.Dir}}" -osarch='$(TARGETS)' $(GOFLAGS) $(if $(TAGS),-tags '$(TAGS)',) -ldflags '$(LDFLAGS)' $(GIT_HOST)/$(BASE_DIR)/cmd/cinder-csi-plugin/ - CGO_ENABLED=0 gox -parallel=$(GOX_PARALLEL) -output="_dist/{{.OS}}-{{.Arch}}/{{.Dir}}" -osarch='$(TARGETS)' $(GOFLAGS) $(if $(TAGS),-tags '$(TAGS)',) -ldflags '$(LDFLAGS)' $(GIT_HOST)/$(BASE_DIR)/cmd/k8s-keystone-auth/ - CGO_ENABLED=0 gox -parallel=$(GOX_PARALLEL) -output="_dist/{{.OS}}-{{.Arch}}/{{.Dir}}" -osarch='$(TARGETS)' $(GOFLAGS) $(if $(TAGS),-tags '$(TAGS)',) -ldflags '$(LDFLAGS)' $(GIT_HOST)/$(BASE_DIR)/cmd/client-keystone-auth/ - CGO_ENABLED=0 gox -parallel=$(GOX_PARALLEL) -output="_dist/{{.OS}}-{{.Arch}}/{{.Dir}}" -osarch='$(TARGETS)' $(GOFLAGS) $(if $(TAGS),-tags '$(TAGS)',) -ldflags '$(LDFLAGS)' $(GIT_HOST)/$(BASE_DIR)/cmd/octavia-ingress-controller/ - CGO_ENABLED=0 gox -parallel=$(GOX_PARALLEL) -output="_dist/{{.OS}}-{{.Arch}}/{{.Dir}}" -osarch='$(TARGETS)' $(GOFLAGS) $(if $(TAGS),-tags '$(TAGS)',) -ldflags '$(LDFLAGS)' $(GIT_HOST)/$(BASE_DIR)/cmd/manila-csi-plugin/ - CGO_ENABLED=0 gox -parallel=$(GOX_PARALLEL) -output="_dist/{{.OS}}-{{.Arch}}/{{.Dir}}" -osarch='$(TARGETS)' $(GOFLAGS) $(if $(TAGS),-tags '$(TAGS)',) -ldflags '$(LDFLAGS)' $(GIT_HOST)/$(BASE_DIR)/cmd/magnum-auto-healer/ - -.PHONY: dist -dist: build-cross - ( \ - cd _dist && \ - $(DIST_DIRS) cp ../LICENSE {} \; && \ - $(DIST_DIRS) cp ../README.md {} \; && \ - $(DIST_DIRS) tar -zcf cloud-provider-openstack-$(VERSION)-{}.tar.gz {} \; && \ - $(DIST_DIRS) zip -r cloud-provider-openstack-$(VERSION)-{}.zip {} \; \ - ) - -.PHONY: bindep build clean cover work docs fmt functional lint import-boss realclean \ - relnotes test translation version build-cross dist diff --git a/OWNERS b/OWNERS deleted file mode 100644 index fe940d9208..0000000000 --- a/OWNERS +++ /dev/null @@ -1,18 +0,0 @@ -approvers: -- adisky -- chrigl -- dims -- lingxiankong -- ramineni -- zetaab -- jichenjc -reviewers: -- adisky -- chrigl -- dims -- Fedosin -- jichenjc -- lingxiankong -- ramineni -- ricolin -- zetaab diff --git a/README.md b/README.md deleted file mode 100644 index 8c056fa13f..0000000000 --- a/README.md +++ /dev/null @@ -1,45 +0,0 @@ -# Cloud Provider OpenStack - -Thank you for visiting the `Cloud Provider OpenStack` repository! - -This Repository hosts various plugins relevant to OpenStack and Kubernetes Integration - -* [OpenStack Cloud Controller Manager](/docs/using-openstack-cloud-controller-manager.md/) -* [Octavia Ingress Controller](/docs/using-octavia-ingress-controller.md/) -* [Cinder CSI Plugin](/docs/using-cinder-csi-plugin.md/) -* [Keystone Webhook Authentication Authorization](/docs/using-keystone-webhook-authenticator-and-authorizer.md/) -* [Client Keystone](/docs/using-client-keystone-auth.md/) -* [Manila CSI Plugin](/docs/using-manila-csi-plugin.md/) -* [Barbican KMS Plugin](/docs/using-barbican-kms-plugin.md/) -* [Magnum Auto Healer](/docs/using-magnum-auto-healer.md/) - -> NOTE: Cinder Standalone Provisioner, Manila Provisioner and Cinder FlexVolume Driver were removed since release v1.18.0. - -> Version 1.17 was the last release of Manila Provisioner, which is unmaintained from now on. Due to dependency issues, we removed the code from master but it is still accessible in the [release-1.17](https://github.com/kubernetes/cloud-provider-openstack/tree/release-1.17) branch. Please consider migrating to Manila CSI Plugin. - -## Developing - -Please Refer [Getting Started Guide](/docs/getting-started-provider-dev.md/) for setting up development environment. - -## Contact - -Please join us on [Kubernetes provider-openstack slack channel](https://kubernetes.slack.com/messages/provider-openstack) - -Project Co-Leads: -* @lxkong - Lingxian Kong -* @ramineni - Anusha Ramineni -* @chrigl - Christoph Glaubitz - -## License - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -[http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0) - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. diff --git a/SECURITY.md b/SECURITY.md deleted file mode 100644 index 2083d44cdf..0000000000 --- a/SECURITY.md +++ /dev/null @@ -1,22 +0,0 @@ -# Security Policy - -## Security Announcements - -Join the [kubernetes-security-announce] group for security and vulnerability announcements. - -You can also subscribe to an RSS feed of the above using [this link][kubernetes-security-announce-rss]. - -## Reporting a Vulnerability - -Instructions for reporting a vulnerability can be found on the -[Kubernetes Security and Disclosure Information] page. - -## Supported Versions - -Information about supported Kubernetes versions can be found on the -[Kubernetes version and version skew support policy] page on the Kubernetes website. - -[kubernetes-security-announce]: https://groups.google.com/forum/#!forum/kubernetes-security-announce -[kubernetes-security-announce-rss]: https://groups.google.com/forum/feed/kubernetes-security-announce/msgs/rss_v2_0.xml?num=50 -[Kubernetes version and version skew support policy]: https://kubernetes.io/docs/setup/release/version-skew-policy/#supported-versions -[Kubernetes Security and Disclosure Information]: https://kubernetes.io/docs/reference/issues-security/security/#report-a-vulnerability diff --git a/SECURITY_CONTACTS b/SECURITY_CONTACTS deleted file mode 100644 index d8868d2a1d..0000000000 --- a/SECURITY_CONTACTS +++ /dev/null @@ -1,15 +0,0 @@ -# Defined below are the security contacts for this repo. -# -# They are the contact point for the Product Security Committee to reach out -# to for triaging and handling of incoming issues. -# -# The below names agree to abide by the -# [Embargo Policy](https://git.k8s.io/security/private-distributors-list.md#embargo-policy) -# and will be removed and replaced if they violate that agreement. -# -# DO NOT REPORT SECURITY VULNERABILITIES DIRECTLY TO THESE NAMES, FOLLOW THE -# INSTRUCTIONS AT https://kubernetes.io/security/ - -dims -hogepodge -dklyle diff --git a/bindep.txt b/bindep.txt deleted file mode 100644 index 656f43506a..0000000000 --- a/bindep.txt +++ /dev/null @@ -1,6 +0,0 @@ -pkg-config -build-essential -mercurial -golang-go -golint -make diff --git a/charts/cinder-csi-plugin/.helmignore b/charts/cinder-csi-plugin/.helmignore deleted file mode 100644 index 50af031725..0000000000 --- a/charts/cinder-csi-plugin/.helmignore +++ /dev/null @@ -1,22 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/charts/cinder-csi-plugin/Chart.yaml b/charts/cinder-csi-plugin/Chart.yaml deleted file mode 100644 index e07bc162e6..0000000000 --- a/charts/cinder-csi-plugin/Chart.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -appVersion: latest -description: Cinder CSI Plugin for OpenStack -name: openstack-cinder-csi -version: 1.1.2 -home: https://github.com/kubernetes/cloud-provider-openstack -icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png -maintainers: - - name: brtknr - email: brtknr@bath.edu diff --git a/charts/cinder-csi-plugin/README.md b/charts/cinder-csi-plugin/README.md deleted file mode 100644 index c90dfcf1ca..0000000000 --- a/charts/cinder-csi-plugin/README.md +++ /dev/null @@ -1,21 +0,0 @@ -# Cinder CSI volume provisioner - -Deploys a Cinder csi provisioner to your cluster, with the appropriate storageClass. - -## How To install -- Enable deployment of storageclasses using `storageClass.enabled` -- Tag the retain or delete class as default class using `storageClass.delete.isDefault` in your value yaml -- Set `storageClass..allowVolumeExpansion` to `true` or `false` - -First add the repo: - - helm repo add cpo https://kubernetes.github.io/cloud-provider-openstack - helm repo update - -If you are using Helm v3: - - helm install cinder-csi cpo/openstack-cinder-csi - -If you are using Helm v2: - - helm install --name cinder-csi cpo/openstack-cinder-csi diff --git a/charts/cinder-csi-plugin/templates/NOTES.txt b/charts/cinder-csi-plugin/templates/NOTES.txt deleted file mode 100644 index 2ed02ae645..0000000000 --- a/charts/cinder-csi-plugin/templates/NOTES.txt +++ /dev/null @@ -1 +0,0 @@ -Use the following storageClass csi-cinder-sc-retain and csi-cinder-sc-delete only for RWO volumes. \ No newline at end of file diff --git a/charts/cinder-csi-plugin/templates/_helpers.tpl b/charts/cinder-csi-plugin/templates/_helpers.tpl deleted file mode 100644 index 75fc64064d..0000000000 --- a/charts/cinder-csi-plugin/templates/_helpers.tpl +++ /dev/null @@ -1,90 +0,0 @@ -{{/* vim: set filetype=mustache: */}} -{{/* -Expand the name of the chart. -*/}} -{{- define "cinder-csi.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "cinder-csi.fullname" -}} -{{- if .Values.fullnameOverride -}} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- .Release.Name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "cinder-csi.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Common labels -*/}} -{{- define "cinder-csi.labels" -}} -app.kubernetes.io/name: {{ include "cinder-csi.name" . }} -helm.sh/chart: {{ include "cinder-csi.chart" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end -}} - - -{{/* -Create the name of the service account to use -*/}} -{{- define "cinder-csi.serviceAccountName" -}} -{{- if .Values.serviceAccount.create -}} - {{ default (include "cinder-csi.fullname" .) .Values.serviceAccount.name }} -{{- else -}} - {{ default "default" .Values.serviceAccount.name }} -{{- end -}} -{{- end -}} - -{{/* -Create unified labels for cinder-csi components -*/}} -{{- define "cinder-csi.common.matchLabels" -}} -app: {{ template "cinder-csi.name" . }} -release: {{ .Release.Name }} -{{- end -}} - -{{- define "cinder-csi.common.metaLabels" -}} -chart: {{ template "cinder-csi.chart" . }} -heritage: {{ .Release.Service }} -{{- end -}} - -{{- define "cinder-csi.controllerplugin.matchLabels" -}} -component: controllerplugin -{{ include "cinder-csi.common.matchLabels" . }} -{{- end -}} - -{{- define "cinder-csi.controllerplugin.labels" -}} -{{ include "cinder-csi.controllerplugin.matchLabels" . }} -{{ include "cinder-csi.common.metaLabels" . }} -{{- end -}} - -{{- define "cinder-csi.nodeplugin.matchLabels" -}} -component: nodeplugin -{{ include "cinder-csi.common.matchLabels" . }} -{{- end -}} - -{{- define "cinder-csi.nodeplugin.labels" -}} -{{ include "cinder-csi.nodeplugin.matchLabels" . }} -{{ include "cinder-csi.common.metaLabels" . }} -{{- end -}} diff --git a/charts/cinder-csi-plugin/templates/cinder-csi-driver.yaml b/charts/cinder-csi-plugin/templates/cinder-csi-driver.yaml deleted file mode 100644 index 5b681e4c4d..0000000000 --- a/charts/cinder-csi-plugin/templates/cinder-csi-driver.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: storage.k8s.io/v1 -kind: CSIDriver -metadata: - name: cinder.csi.openstack.org -spec: - attachRequired: true - podInfoOnMount: true - volumeLifecycleModes: - - Persistent - - Ephemeral diff --git a/charts/cinder-csi-plugin/templates/controllerplugin-rbac.yaml b/charts/cinder-csi-plugin/templates/controllerplugin-rbac.yaml deleted file mode 100644 index 1415fcc840..0000000000 --- a/charts/cinder-csi-plugin/templates/controllerplugin-rbac.yaml +++ /dev/null @@ -1,197 +0,0 @@ -# This YAML file contains RBAC API objects, -# which are necessary to run csi controller plugin - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: csi-cinder-controller-sa - namespace: {{ .Release.Namespace }} ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-attacher-role -rules: - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["csinodes"] - verbs: ["get", "list", "watch"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-attacher-binding -subjects: - - kind: ServiceAccount - name: csi-cinder-controller-sa - namespace: {{ .Release.Namespace }} -roleRef: - kind: ClusterRole - name: csi-attacher-role - apiGroup: rbac.authorization.k8s.io ---- -# external Provisioner -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-provisioner-role -rules: - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["csinodes"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["get", "list"] - ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-provisioner-binding -subjects: - - kind: ServiceAccount - name: csi-cinder-controller-sa - namespace: {{ .Release.Namespace }} -roleRef: - kind: ClusterRole - name: csi-provisioner-role - apiGroup: rbac.authorization.k8s.io - ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-snapshotter-role -rules: - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["create", "get", "list", "watch", "update", "delete"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots/status"] - verbs: ["update"] - - apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: ["create", "list", "watch", "delete"] - ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-snapshotter-binding -subjects: - - kind: ServiceAccount - name: csi-cinder-controller-sa - namespace: {{ .Release.Namespace }} -roleRef: - kind: ClusterRole - name: csi-snapshotter-role - apiGroup: rbac.authorization.k8s.io ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-resizer-role -rules: - # The following rule should be uncommented for plugins that require secrets - # for provisioning. - # - apiGroups: [""] - # resources: ["secrets"] - # verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims/status"] - verbs: ["update", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-resizer-binding -subjects: - - kind: ServiceAccount - name: csi-cinder-controller-sa - namespace: {{ .Release.Namespace }} -roleRef: - kind: ClusterRole - name: csi-resizer-role - apiGroup: rbac.authorization.k8s.io ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: external-resizer-cfg - namespace: {{ .Release.Namespace }} -rules: -- apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "watch", "list", "delete", "update", "create"] ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-resizer-role-cfg -subjects: - - kind: ServiceAccount - name: csi-cinder-controller-sa - namespace: {{ .Release.Namespace }} -roleRef: - kind: Role - name: external-resizer-cfg - apiGroup: rbac.authorization.k8s.io diff --git a/charts/cinder-csi-plugin/templates/controllerplugin-statefulset.yaml b/charts/cinder-csi-plugin/templates/controllerplugin-statefulset.yaml deleted file mode 100644 index 4da22eb090..0000000000 --- a/charts/cinder-csi-plugin/templates/controllerplugin-statefulset.yaml +++ /dev/null @@ -1,107 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ include "cinder-csi.name" . }}-controllerplugin - labels: - {{- include "cinder-csi.controllerplugin.labels" . | nindent 4 }} -spec: - clusterIP: None ---- -kind: StatefulSet -apiVersion: apps/v1 -metadata: - name: {{ include "cinder-csi.name" . }}-controllerplugin - labels: - {{- include "cinder-csi.controllerplugin.labels" . | nindent 4 }} -spec: - serviceName: {{ include "cinder-csi.name" . }}-controllerplugin - replicas: 1 - selector: - matchLabels: - {{- include "cinder-csi.controllerplugin.matchLabels" . | nindent 6 }} - template: - metadata: - labels: - {{- include "cinder-csi.controllerplugin.labels" . | nindent 8 }} - spec: - serviceAccount: csi-cinder-controller-sa - containers: - - name: csi-attacher - image: "{{ .Values.csi.attacher.image.repository }}:{{ .Values.csi.attacher.image.tag }}" - imagePullPolicy: {{ .Values.csi.attacher.image.pullPolicy }} - args: - - "--csi-address=$(ADDRESS)" - - "--timeout=3m" - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - - name: csi-provisioner - image: "{{ .Values.csi.provisioner.image.repository }}:{{ .Values.csi.provisioner.image.tag }}" - imagePullPolicy: {{ .Values.csi.provisioner.image.pullPolicy }} - args: - - "--csi-address=$(ADDRESS)" - - "--timeout=3m" - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - - name: csi-snapshotter - image: "{{ .Values.csi.snapshotter.image.repository }}:{{ .Values.csi.snapshotter.image.tag }}" - imagePullPolicy: {{ .Values.csi.snapshotter.image.pullPolicy }} - args: - - "--csi-address=$(ADDRESS)" - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - volumeMounts: - - mountPath: /var/lib/csi/sockets/pluginproxy/ - name: socket-dir - - name: csi-resizer - image: "{{ .Values.csi.resizer.image.repository }}:{{ .Values.csi.resizer.image.tag }}" - imagePullPolicy: {{ .Values.csi.resizer.image.pullPolicy }} - args: - - "--csi-address=$(ADDRESS)" - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - - name: cinder-csi-plugin - image: "{{ .Values.csi.plugin.image.repository }}:{{ .Values.csi.plugin.image.tag }}" - imagePullPolicy: {{ .Values.csi.plugin.image.pullPolicy }} - args: - - /bin/cinder-csi-plugin - - "--nodeid=$(NODE_ID)" - - "--endpoint=$(CSI_ENDPOINT)" - - "--cloud-config=$(CLOUD_CONFIG)" - - "--cluster=$(CLUSTER_NAME)" - env: - - name: NODE_ID - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: CSI_ENDPOINT - value: unix://csi/csi.sock - - name: CLOUD_CONFIG - value: /etc/kubernetes/cloud-config - - name: CLUSTER_NAME - value: kubernetes - volumeMounts: - - name: socket-dir - mountPath: /csi - - name: cacert - mountPath: /etc/cacert - readOnly: true - volumes: - - name: cacert - hostPath: - path: /etc/cacert - type: Directory - - name: socket-dir - emptyDir: diff --git a/charts/cinder-csi-plugin/templates/nodeplugin-daemonset.yaml b/charts/cinder-csi-plugin/templates/nodeplugin-daemonset.yaml deleted file mode 100644 index bbb87c271f..0000000000 --- a/charts/cinder-csi-plugin/templates/nodeplugin-daemonset.yaml +++ /dev/null @@ -1,103 +0,0 @@ -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: {{ include "cinder-csi.name" . }}-nodeplugin - labels: - {{- include "cinder-csi.nodeplugin.labels" . | nindent 4 }} -spec: - selector: - matchLabels: - {{- include "cinder-csi.nodeplugin.matchLabels" . | nindent 6 }} - template: - metadata: - labels: - {{- include "cinder-csi.nodeplugin.labels" . | nindent 8 }} - spec: - serviceAccount: csi-cinder-node-sa - hostNetwork: true - containers: - - name: node-driver-registrar - image: "{{ .Values.csi.nodeDriverRegistrar.image.repository }}:{{ .Values.csi.nodeDriverRegistrar.image.tag }}" - imagePullPolicy: {{ .Values.csi.nodeDriverRegistrar.image.pullPolicy }} - args: - - "--csi-address=$(ADDRESS)" - - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)" - lifecycle: - preStop: - exec: - command: ["/bin/sh", "-c", "rm -rf /registration/cinder.csi.openstack.org /registration/cinder.csi.openstack.org-reg.sock"] - env: - - name: ADDRESS - value: /csi/csi.sock - - name: DRIVER_REG_SOCK_PATH - value: /var/lib/kubelet/plugins/cinder.csi.openstack.org/csi.sock - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - imagePullPolicy: "IfNotPresent" - volumeMounts: - - name: socket-dir - mountPath: /csi - - name: registration-dir - mountPath: /registration - - name: cinder-csi-plugin - securityContext: - privileged: true - capabilities: - add: ["SYS_ADMIN"] - allowPrivilegeEscalation: true - image: "{{ .Values.csi.plugin.image.repository }}:{{ .Values.csi.plugin.image.tag }}" - imagePullPolicy: {{ .Values.csi.plugin.image.pullPolicy }} - args: - - /bin/cinder-csi-plugin - - "--nodeid=$(NODE_ID)" - - "--endpoint=$(CSI_ENDPOINT)" - - "--cloud-config=$(CLOUD_CONFIG)" - env: - - name: NODE_ID - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: CSI_ENDPOINT - value: unix://csi/csi.sock - - name: CLOUD_CONFIG - value: /etc/kubernetes/cloud-config - imagePullPolicy: "IfNotPresent" - volumeMounts: - - name: socket-dir - mountPath: /csi - - name: kubelet-dir - mountPath: /var/lib/kubelet - mountPropagation: "Bidirectional" - - name: pods-probe-dir - mountPath: /dev - mountPropagation: "HostToContainer" - - name: cacert - mountPath: /etc/cacert - readOnly: true - volumes: - - name: cacert - hostPath: - path: /etc/cacert - type: Directory - - name: socket-dir - hostPath: - path: /var/lib/kubelet/plugins/cinder.csi.openstack.org - type: DirectoryOrCreate - - name: registration-dir - hostPath: - path: /var/lib/kubelet/plugins_registry/ - type: Directory - - name: kubelet-dir - hostPath: - path: /var/lib/kubelet - type: Directory - # - name: pods-cloud-data - # hostPath: - # path: /var/lib/cloud/data - # type: Directory - - name: pods-probe-dir - hostPath: - path: /dev - type: Directory diff --git a/charts/cinder-csi-plugin/templates/nodeplugin-rbac.yaml b/charts/cinder-csi-plugin/templates/nodeplugin-rbac.yaml deleted file mode 100644 index 894a8df219..0000000000 --- a/charts/cinder-csi-plugin/templates/nodeplugin-rbac.yaml +++ /dev/null @@ -1,28 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: csi-cinder-node-sa - namespace: {{ .Release.Namespace }} ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-nodeplugin-role -rules: - - apiGroups: [""] - resources: ["events"] - verbs: ["get", "list", "watch", "create", "update", "patch"] - ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-nodeplugin-binding -subjects: - - kind: ServiceAccount - name: csi-cinder-node-sa - namespace: {{ .Release.Namespace }} -roleRef: - kind: ClusterRole - name: csi-nodeplugin-role - apiGroup: rbac.authorization.k8s.io diff --git a/charts/cinder-csi-plugin/templates/storageclass.yaml b/charts/cinder-csi-plugin/templates/storageclass.yaml deleted file mode 100644 index 704c30038b..0000000000 --- a/charts/cinder-csi-plugin/templates/storageclass.yaml +++ /dev/null @@ -1,25 +0,0 @@ -{{- if .Values.storageClass.enabled }} -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: csi-cinder-sc-delete - {{- if .Values.storageClass.delete.isDefault }} - annotations: - storageclass.kubernetes.io/is-default-class: "true" - {{- end }} -provisioner: cinder.csi.openstack.org -reclaimPolicy: Delete -allowVolumeExpansion: {{ .Values.storageClass.delete.allowVolumeExpansion }} ---- -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: csi-cinder-sc-retain - {{- if .Values.storageClass.retain.isDefault }} - annotations: - storageclass.kubernetes.io/is-default-class: "true" - {{- end }} -provisioner: cinder.csi.openstack.org -reclaimPolicy: Retain -allowVolumeExpansion: {{ .Values.storageClass.retain.allowVolumeExpansion }} -{{- end }} \ No newline at end of file diff --git a/charts/cinder-csi-plugin/values.yaml b/charts/cinder-csi-plugin/values.yaml deleted file mode 100644 index 0becf4b15c..0000000000 --- a/charts/cinder-csi-plugin/values.yaml +++ /dev/null @@ -1,43 +0,0 @@ -nameOverride: "" -fullnameOverride: "" - -csi: - attacher: - image: - repository: quay.io/k8scsi/csi-attacher - tag: v2.2.0 - pullPolicy: IfNotPresent - provisioner: - image: - repository: quay.io/k8scsi/csi-provisioner - tag: v1.6.0 - pullPolicy: IfNotPresent - snapshotter: - image: - repository: quay.io/k8scsi/csi-snapshotter - tag: v1.2.2 - pullPolicy: IfNotPresent - resizer: - image: - repository: quay.io/k8scsi/csi-resizer - tag: v0.4.0 - pullPolicy: IfNotPresent - nodeDriverRegistrar: - image: - repository: quay.io/k8scsi/csi-node-driver-registrar - tag: v1.2.0 - pullPolicy: IfNotPresent - plugin: - image: - repository: docker.io/k8scloudprovider/cinder-csi-plugin - tag: latest - pullPolicy: IfNotPresent - -storageClass: - enabled: true - delete: - isDefault: false - allowVolumeExpansion: true - retain: - isDefault: false - allowVolumeExpansion: true diff --git a/charts/manila-csi-plugin/Chart.yaml b/charts/manila-csi-plugin/Chart.yaml deleted file mode 100644 index b9835de688..0000000000 --- a/charts/manila-csi-plugin/Chart.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -appVersion: latest -description: Manila CSI Plugin for OpenStack -name: openstack-manila-csi -version: 0.1.2 -home: http://github.com/kubernetes/cloud-provider-openstack -icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png -maintainers: - - name: brtknr - email: brtknr@bath.edu diff --git a/charts/manila-csi-plugin/README.md b/charts/manila-csi-plugin/README.md deleted file mode 100644 index c2f4742c49..0000000000 --- a/charts/manila-csi-plugin/README.md +++ /dev/null @@ -1,14 +0,0 @@ -# Manila CSI provisioner - -First add the repo: - - helm repo add cpo https://kubernetes.github.io/cloud-provider-openstack - helm repo update - -If you are using Helm v3: - - helm install manila-csi cpo/openstack-manila-csi - -If you are using Helm v2: - - helm install --name manila-csi cpo/openstack-manila-csi diff --git a/charts/manila-csi-plugin/templates/_helpers.tpl b/charts/manila-csi-plugin/templates/_helpers.tpl deleted file mode 100644 index 73708d55b8..0000000000 --- a/charts/manila-csi-plugin/templates/_helpers.tpl +++ /dev/null @@ -1,83 +0,0 @@ -{{/* vim: set filetype=mustache: */}} -{{/* -Expand the name of the chart. -*/}} -{{- define "openstack-manila-csi.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "openstack-manila-csi.fullname" -}} -{{- if .Values.fullnameOverride -}} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- .Release.Name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Create fully qualified app name for the node plugin. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "openstack-manila-csi.nodeplugin.fullname" -}} -{{- if .Values.nodeplugin.fullnameOverride -}} -{{- .Values.nodeplugin.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- printf "%s-%s" .Release.Name .Values.nodeplugin.name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s-%s" .Release.Name $name .Values.nodeplugin.name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Create the name of the service account to use for the node plugin. -*/}} -{{- define "openstack-manila-csi.serviceAccountName.nodeplugin" -}} -{{ include "openstack-manila-csi.nodeplugin.fullname" . }} -{{- end -}} - -{{/* -Create fully qualified app name for the controller plugin. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "openstack-manila-csi.controllerplugin.fullname" -}} -{{- if .Values.controllerplugin.fullnameOverride -}} -{{- .Values.controllerplugin.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- printf "%s-%s" .Release.Name .Values.controllerplugin.name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s-%s" .Release.Name $name .Values.controllerplugin.name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Create the name of the service account to use for the controller plugin. -*/}} -{{- define "openstack-manila-csi.serviceAccountName.controllerplugin" -}} -{{ include "openstack-manila-csi.controllerplugin.fullname" . }} -{{- end -}} - - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "openstack-manila-csi.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} -{{- end -}} diff --git a/charts/manila-csi-plugin/templates/controllerplugin-clusterrole.yaml b/charts/manila-csi-plugin/templates/controllerplugin-clusterrole.yaml deleted file mode 100644 index 71e5ecbc64..0000000000 --- a/charts/manila-csi-plugin/templates/controllerplugin-clusterrole.yaml +++ /dev/null @@ -1,15 +0,0 @@ -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: {{ include "openstack-manila-csi.controllerplugin.fullname" . }} - labels: - app: {{ include "openstack-manila-csi.name" . }} - chart: {{ include "openstack-manila-csi.chart" . }} - component: {{ .Values.controllerplugin.name }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -aggregationRule: - clusterRoleSelectors: - - matchLabels: - rbac.manila.csi.openstack.org/aggregate-to-{{ include "openstack-manila-csi.controllerplugin.fullname" . }}: "true" -rules: [] diff --git a/charts/manila-csi-plugin/templates/controllerplugin-clusterrolebinding.yaml b/charts/manila-csi-plugin/templates/controllerplugin-clusterrolebinding.yaml deleted file mode 100644 index 6c92491cf7..0000000000 --- a/charts/manila-csi-plugin/templates/controllerplugin-clusterrolebinding.yaml +++ /dev/null @@ -1,18 +0,0 @@ -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: {{ include "openstack-manila-csi.controllerplugin.fullname" . }} - labels: - app: {{ include "openstack-manila-csi.name" . }} - chart: {{ include "openstack-manila-csi.chart" . }} - component: {{ .Values.controllerplugin.name }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -subjects: - - kind: ServiceAccount - name: {{ include "openstack-manila-csi.serviceAccountName.controllerplugin" . }} - namespace: {{ .Release.Namespace }} -roleRef: - kind: ClusterRole - name: {{ include "openstack-manila-csi.controllerplugin.fullname" . }} - apiGroup: rbac.authorization.k8s.io diff --git a/charts/manila-csi-plugin/templates/controllerplugin-role.yaml b/charts/manila-csi-plugin/templates/controllerplugin-role.yaml deleted file mode 100644 index 5502f0c2bf..0000000000 --- a/charts/manila-csi-plugin/templates/controllerplugin-role.yaml +++ /dev/null @@ -1,17 +0,0 @@ -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: {{ include "openstack-manila-csi.controllerplugin.fullname" . }} - labels: - app: {{ include "openstack-manila-csi.name" . }} - chart: {{ include "openstack-manila-csi.chart" . }} - component: {{ .Values.controllerplugin.name }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -rules: - - apiGroups: [""] - resources: ["endpoints"] - verbs: ["get", "watch", "list", "delete", "update", "create"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "watch", "create", "delete"] diff --git a/charts/manila-csi-plugin/templates/controllerplugin-rolebinding.yaml b/charts/manila-csi-plugin/templates/controllerplugin-rolebinding.yaml deleted file mode 100644 index acb8fa6966..0000000000 --- a/charts/manila-csi-plugin/templates/controllerplugin-rolebinding.yaml +++ /dev/null @@ -1,18 +0,0 @@ -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: {{ include "openstack-manila-csi.controllerplugin.fullname" . }} - labels: - app: {{ include "openstack-manila-csi.name" . }} - chart: {{ include "openstack-manila-csi.chart" . }} - component: {{ .Values.controllerplugin.name }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -subjects: - - kind: ServiceAccount - name: {{ include "openstack-manila-csi.serviceAccountName.controllerplugin" . }} - namespace: {{ .Release.Namespace }} -roleRef: - kind: Role - name: {{ include "openstack-manila-csi.controllerplugin.fullname" . }} - apiGroup: rbac.authorization.k8s.io diff --git a/charts/manila-csi-plugin/templates/controllerplugin-rules-clusterrole.yaml b/charts/manila-csi-plugin/templates/controllerplugin-rules-clusterrole.yaml deleted file mode 100644 index e17e6e9366..0000000000 --- a/charts/manila-csi-plugin/templates/controllerplugin-rules-clusterrole.yaml +++ /dev/null @@ -1,48 +0,0 @@ -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: {{ include "openstack-manila-csi.controllerplugin.fullname" . }}-rules - labels: - app: {{ include "openstack-manila-csi.name" . }} - chart: {{ include "openstack-manila-csi.chart" . }} - component: {{ .Values.controllerplugin.name }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} - rbac.manila.csi.openstack.org/aggregate-to-{{ include "openstack-manila-csi.controllerplugin.fullname" . }}: "true" -rules: - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["csinodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["create", "get", "list", "watch", "update", "delete"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots/status"] - verbs: ["update"] - - apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: ["create", "list", "watch", "delete", "get", "update"] diff --git a/charts/manila-csi-plugin/templates/controllerplugin-service.yaml b/charts/manila-csi-plugin/templates/controllerplugin-service.yaml deleted file mode 100644 index 374f782947..0000000000 --- a/charts/manila-csi-plugin/templates/controllerplugin-service.yaml +++ /dev/null @@ -1,18 +0,0 @@ -kind: Service -apiVersion: v1 -metadata: - name: {{ include "openstack-manila-csi.controllerplugin.fullname" . }} - labels: - app: {{ include "openstack-manila-csi.name" . }} - chart: {{ include "openstack-manila-csi.chart" . }} - component: {{ .Values.controllerplugin.name }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -spec: - selector: - app: {{ include "openstack-manila-csi.name" . }} - component: {{ .Values.controllerplugin.name }} - release: {{ .Release.Name }} - ports: - - name: dummy - port: 12345 diff --git a/charts/manila-csi-plugin/templates/controllerplugin-serviceaccount.yaml b/charts/manila-csi-plugin/templates/controllerplugin-serviceaccount.yaml deleted file mode 100644 index f724916b0e..0000000000 --- a/charts/manila-csi-plugin/templates/controllerplugin-serviceaccount.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "openstack-manila-csi.serviceAccountName.controllerplugin" . }} - labels: - app: {{ include "openstack-manila-csi.name" . }} - chart: {{ include "openstack-manila-csi.chart" . }} - component: {{ .Values.controllerplugin.name }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} diff --git a/charts/manila-csi-plugin/templates/controllerplugin-statefulset.yaml b/charts/manila-csi-plugin/templates/controllerplugin-statefulset.yaml deleted file mode 100644 index 1ecf2c709b..0000000000 --- a/charts/manila-csi-plugin/templates/controllerplugin-statefulset.yaml +++ /dev/null @@ -1,149 +0,0 @@ -kind: StatefulSet -apiVersion: apps/v1 -metadata: - name: {{ include "openstack-manila-csi.controllerplugin.fullname" . }} - labels: - app: {{ include "openstack-manila-csi.name" . }} - chart: {{ include "openstack-manila-csi.chart" . }} - component: {{ .Values.controllerplugin.name }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -spec: - serviceName: {{ include "openstack-manila-csi.controllerplugin.fullname" . }} - replicas: 1 - selector: - matchLabels: - app: {{ include "openstack-manila-csi.name" . }} - component: {{ .Values.controllerplugin.name }} - release: {{ .Release.Name }} - template: - metadata: - labels: - app: {{ include "openstack-manila-csi.name" . }} - chart: {{ include "openstack-manila-csi.chart" . }} - component: {{ .Values.controllerplugin.name }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} - spec: - serviceAccountName: {{ include "openstack-manila-csi.serviceAccountName.controllerplugin" . }} - containers: - {{- range .Values.shareProtocols }} - - name: {{ .protocolSelector | lower }}-provisioner - image: "{{ $.Values.controllerplugin.provisioner.image.repository }}:{{ $.Values.controllerplugin.provisioner.image.tag }}" - args: - - "--v=5" - - "--csi-address=$(ADDRESS)" - {{- if $.Values.csimanila.topologyAwarenessEnabled }} - - "--feature-gates=Topology=true" - {{- end }} - env: - - name: ADDRESS - value: "unix:///var/lib/kubelet/plugins/{{ printf "%s.%s" .protocolSelector $.Values.driverName | lower }}/csi-controllerplugin.sock" - imagePullPolicy: {{ $.Values.controllerplugin.provisioner.image.pullPolicy }} - volumeMounts: - - name: {{ .protocolSelector | lower }}-plugin-dir - mountPath: /var/lib/kubelet/plugins/{{ printf "%s.%s" .protocolSelector $.Values.driverName | lower }} - resources: -{{ toYaml $.Values.controllerplugin.provisioner.resources | indent 12 }} - - name: {{ .protocolSelector | lower }}-snapshotter - image: "{{ $.Values.controllerplugin.snapshotter.image.repository }}:{{ $.Values.controllerplugin.snapshotter.image.tag }}" - args: - - "--v=5" - - "--csi-address=$(ADDRESS)" - env: - - name: ADDRESS - value: "unix:///var/lib/kubelet/plugins/{{ printf "%s.%s" .protocolSelector $.Values.driverName | lower }}/csi-controllerplugin.sock" - imagePullPolicy: {{ $.Values.controllerplugin.snapshotter.image.pullPolicy }} - volumeMounts: - - name: {{ .protocolSelector | lower }}-plugin-dir - mountPath: /var/lib/kubelet/plugins/{{ printf "%s.%s" .protocolSelector $.Values.driverName | lower }} - resources: -{{ toYaml $.Values.controllerplugin.snapshotter.resources | indent 12 }} - - name: {{ .protocolSelector | lower }}-nodeplugin - securityContext: - privileged: true - capabilities: - add: ["SYS_ADMIN"] - allowPrivilegeEscalation: true - image: "{{ $.Values.csimanila.image.repository }}:{{ $.Values.csimanila.image.tag }}" - command: ["/bin/sh", "-c", - '/bin/manila-csi-plugin - --v=5 - --nodeid=$(NODE_ID) - {{- if $.Values.csimanila.topologyAwarenessEnabled }} - --with-topology - --nodeaz={{ $.Values.csimanila.nodeAZ }} - {{- end }} - {{- if $.Values.csimanila.runtimeConfig.enabled }} - --runtime-config-file=/runtimeconfig/runtimeconfig.json - {{- end }} - --endpoint=$(CSI_ENDPOINT) - --drivername=$(DRIVER_NAME) - --share-protocol-selector=$(MANILA_SHARE_PROTO) - --fwdendpoint=$(FWD_CSI_ENDPOINT) - {{- if .compatibilitySettings }} - --compatibility-settings={{ .compatibilitySettings }} - {{- end }}' - ] - env: - - name: DRIVER_NAME - value: {{ printf "%s.%s" .protocolSelector $.Values.driverName | lower }} - - name: NODE_ID - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: CSI_ENDPOINT - value: "unix:///var/lib/kubelet/plugins/{{ printf "%s.%s" .protocolSelector $.Values.driverName | lower }}/csi-controllerplugin.sock" - - name: FWD_CSI_ENDPOINT - value: "unix://{{ .fwdNodePluginEndpoint.dir }}/{{ .fwdNodePluginEndpoint.sockFile }}" - - name: MANILA_SHARE_PROTO - value: "{{ .protocolSelector }}" - imagePullPolicy: {{ $.Values.csimanila.image.pullPolicy }} - volumeMounts: - - name: {{ .protocolSelector | lower }}-plugin-dir - mountPath: /var/lib/kubelet/plugins/{{ printf "%s.%s" .protocolSelector $.Values.driverName | lower }} - - name: {{ .protocolSelector | lower }}-fwd-plugin-dir - mountPath: {{ .fwdNodePluginEndpoint.dir }} - - name: pod-mounts - mountPath: /var/lib/kubelet/pods - mountPropagation: Bidirectional - {{- if $.Values.csimanila.runtimeConfig.enabled }} - - name: {{ .protocolSelector | lower }}-runtime-config-dir - mountPath: /runtimeconfig - readOnly: true - {{- end }} - resources: -{{ toYaml $.Values.controllerplugin.nodeplugin.resources | indent 12 }} - {{- end }} - volumes: - {{- range .Values.shareProtocols }} - - name: {{ .protocolSelector | lower }}-plugin-dir - hostPath: - path: /var/lib/kubelet/plugins/{{ printf "%s.%s" .protocolSelector $.Values.driverName | lower }} - type: DirectoryOrCreate - - name: {{ .protocolSelector | lower }}-fwd-plugin-dir - hostPath: - path: {{ .fwdNodePluginEndpoint.dir }} - type: Directory - {{- if $.Values.csimanila.runtimeConfig.enabled }} - - name: {{ .protocolSelector | lower }}-runtime-config-dir - configMap: - name: manila-csi-runtimeconf-cm - {{- end }} - {{- end }} - - name: pod-mounts - hostPath: - path: /var/lib/kubelet/pods - type: Directory - {{- if .Values.controllerplugin.affinity -}} - affinity: -{{ toYaml .Values.controllerplugin.affinity . | indent 8 }} - {{- end -}} - {{- if .Values.controllerplugin.nodeSelector -}} - nodeSelector: -{{ toYaml .Values.controllerplugin.nodeSelector | indent 8 }} - {{- end -}} - {{- if .Values.controllerplugin.tolerations -}} - tolerations: -{{ toYaml .Values.controllerplugin.tolerations | indent 8 }} - {{- end -}} diff --git a/charts/manila-csi-plugin/templates/csidriver.yaml b/charts/manila-csi-plugin/templates/csidriver.yaml deleted file mode 100644 index 6107af7c44..0000000000 --- a/charts/manila-csi-plugin/templates/csidriver.yaml +++ /dev/null @@ -1,10 +0,0 @@ -{{- range .Values.shareProtocols }} -apiVersion: storage.k8s.io/v1beta1 -kind: CSIDriver -metadata: - name: {{ printf "%s.%s" .protocolSelector $.Values.driverName | lower }} -spec: - attachRequired: false - podInfoOnMount: false ---- -{{- end }} diff --git a/charts/manila-csi-plugin/templates/nodeplugin-clusterrole.yaml b/charts/manila-csi-plugin/templates/nodeplugin-clusterrole.yaml deleted file mode 100644 index cbf2115372..0000000000 --- a/charts/manila-csi-plugin/templates/nodeplugin-clusterrole.yaml +++ /dev/null @@ -1,15 +0,0 @@ -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: {{ include "openstack-manila-csi.nodeplugin.fullname" . }} - labels: - app: {{ include "openstack-manila-csi.name" . }} - chart: {{ include "openstack-manila-csi.chart" . }} - component: {{ .Values.nodeplugin.name }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -aggregationRule: - clusterRoleSelectors: - - matchLabels: - rbac.manila.csi.openstack.org/aggregate-to-{{ include "openstack-manila-csi.nodeplugin.fullname" . }}: "true" -rules: [] diff --git a/charts/manila-csi-plugin/templates/nodeplugin-clusterrolebinding.yaml b/charts/manila-csi-plugin/templates/nodeplugin-clusterrolebinding.yaml deleted file mode 100644 index 863b14e2d3..0000000000 --- a/charts/manila-csi-plugin/templates/nodeplugin-clusterrolebinding.yaml +++ /dev/null @@ -1,18 +0,0 @@ -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: {{ include "openstack-manila-csi.nodeplugin.fullname" . }} - labels: - app: {{ include "openstack-manila-csi.name" . }} - chart: {{ include "openstack-manila-csi.chart" . }} - component: {{ .Values.nodeplugin.name }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -subjects: - - kind: ServiceAccount - name: {{ include "openstack-manila-csi.serviceAccountName.nodeplugin" . }} - namespace: {{ .Release.Namespace }} -roleRef: - kind: ClusterRole - name: {{ include "openstack-manila-csi.nodeplugin.fullname" . }} - apiGroup: rbac.authorization.k8s.io diff --git a/charts/manila-csi-plugin/templates/nodeplugin-daemonset.yaml b/charts/manila-csi-plugin/templates/nodeplugin-daemonset.yaml deleted file mode 100644 index 33b24e6b7a..0000000000 --- a/charts/manila-csi-plugin/templates/nodeplugin-daemonset.yaml +++ /dev/null @@ -1,139 +0,0 @@ -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: {{ include "openstack-manila-csi.nodeplugin.fullname" . }} - labels: - app: {{ include "openstack-manila-csi.name" . }} - chart: {{ include "openstack-manila-csi.chart" . }} - component: {{ .Values.nodeplugin.name }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -spec: - selector: - matchLabels: - app: {{ include "openstack-manila-csi.name" . }} - component: {{ .Values.nodeplugin.name }} - release: {{ .Release.Name }} - template: - metadata: - labels: - app: {{ include "openstack-manila-csi.name" . }} - chart: {{ include "openstack-manila-csi.chart" . }} - component: {{ .Values.nodeplugin.name }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} - spec: - serviceAccountName: {{ include "openstack-manila-csi.serviceAccountName.nodeplugin" . }} - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet - containers: - {{- range .Values.shareProtocols }} - - name: {{ .protocolSelector | lower }}-registrar - image: "{{ $.Values.nodeplugin.registrar.image.repository }}:{{ $.Values.nodeplugin.registrar.image.tag }}" - args: - - "--v=5" - - "--csi-address=/csi/csi.sock" - - "--kubelet-registration-path=/var/lib/kubelet/plugins/{{ printf "%s.%s" .protocolSelector $.Values.driverName | lower }}/csi.sock" - lifecycle: - preStop: - exec: - command: [ - "/bin/sh", "-c", - 'rm -rf /registration/{{ printf "%s.%s" .protocolSelector $.Values.driverName | lower }} - /registration/{{ printf "%s.%s" .protocolSelector $.Values.driverName | lower }}-reg.sock' - ] - env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - imagePullPolicy: {{ $.Values.nodeplugin.registrar.image.pullPolicy }} - volumeMounts: - - name: {{ .protocolSelector | lower }}-plugin-dir - mountPath: /csi - - name: registration-dir - mountPath: /registration - resources: -{{ toYaml $.Values.nodeplugin.registrar.resources | indent 12 }} - - name: {{ .protocolSelector | lower }}-nodeplugin - securityContext: - privileged: true - capabilities: - add: ["SYS_ADMIN"] - allowPrivilegeEscalation: true - image: "{{ $.Values.csimanila.image.repository }}:{{ $.Values.csimanila.image.tag }}" - command: ["/bin/sh", "-c", - '/bin/manila-csi-plugin - --v=5 - --nodeid=$(NODE_ID) - {{- if $.Values.csimanila.runtimeConfig.enabled }} - --runtime-config-file=/runtimeconfig/runtimeconfig.json - {{- end }} - {{- if $.Values.csimanila.topologyAwarenessEnabled }} - --with-topology - --nodeaz={{ $.Values.csimanila.nodeAZ }} - {{- end }} - --endpoint=$(CSI_ENDPOINT) - --drivername=$(DRIVER_NAME) - --share-protocol-selector=$(MANILA_SHARE_PROTO) - --fwdendpoint=$(FWD_CSI_ENDPOINT)' - ] - env: - - name: DRIVER_NAME - value: {{ printf "%s.%s" .protocolSelector $.Values.driverName | lower }} - - name: NODE_ID - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: CSI_ENDPOINT - value: "unix:///var/lib/kubelet/plugins/{{ printf "%s.%s" .protocolSelector $.Values.driverName | lower }}/csi.sock" - - name: FWD_CSI_ENDPOINT - value: "unix://{{ .fwdNodePluginEndpoint.dir }}/{{ .fwdNodePluginEndpoint.sockFile }}" - - name: MANILA_SHARE_PROTO - value: "{{ .protocolSelector }}" - imagePullPolicy: {{ $.Values.csimanila.image.pullPolicy }} - volumeMounts: - - name: {{ .protocolSelector | lower }}-plugin-dir - mountPath: /var/lib/kubelet/plugins/{{ printf "%s.%s" .protocolSelector $.Values.driverName | lower }} - - name: {{ .protocolSelector | lower }}-fwd-plugin-dir - mountPath: {{ .fwdNodePluginEndpoint.dir }} - {{- if $.Values.csimanila.runtimeConfig.enabled }} - - name: {{ .protocolSelector | lower }}-runtime-config-dir - mountPath: /runtimeconfig - readOnly: true - {{- end }} - resources: -{{ toYaml $.Values.nodeplugin.nodeplugin.resources | indent 12 }} - {{- end }} - volumes: - - name: registration-dir - hostPath: - path: /var/lib/kubelet/plugins_registry - type: Directory - {{- range .Values.shareProtocols }} - - name: {{ .protocolSelector | lower }}-plugin-dir - hostPath: - path: /var/lib/kubelet/plugins/{{ printf "%s.%s" .protocolSelector $.Values.driverName | lower }} - type: DirectoryOrCreate - - name: {{ .protocolSelector | lower }}-fwd-plugin-dir - hostPath: - path: {{ .fwdNodePluginEndpoint.dir }} - type: DirectoryOrCreate - {{- if $.Values.csimanila.runtimeConfig.enabled }} - - name: {{ .protocolSelector | lower }}-runtime-config-dir - configMap: - name: manila-csi-runtimeconf-cm - {{- end }} - {{- end }} - {{- if .Values.nodeplugin.affinity -}} - affinity: -{{ toYaml .Values.nodeplugin.affinity . | indent 8 }} - {{- end -}} - {{- if .Values.nodeplugin.nodeSelector -}} - nodeSelector: -{{ toYaml .Values.nodeplugin.nodeSelector | indent 8 }} - {{- end -}} - {{- if .Values.nodeplugin.tolerations -}} - tolerations: -{{ toYaml .Values.nodeplugin.tolerations | indent 8 }} - {{- end -}} diff --git a/charts/manila-csi-plugin/templates/nodeplugin-rules-clusterrole.yaml b/charts/manila-csi-plugin/templates/nodeplugin-rules-clusterrole.yaml deleted file mode 100644 index 54abeda0a8..0000000000 --- a/charts/manila-csi-plugin/templates/nodeplugin-rules-clusterrole.yaml +++ /dev/null @@ -1,24 +0,0 @@ -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: {{ include "openstack-manila-csi.nodeplugin.fullname" . }}-rules - labels: - app: {{ include "openstack-manila-csi.name" . }} - chart: {{ include "openstack-manila-csi.chart" . }} - component: {{ .Values.nodeplugin.name }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} - rbac.manila.csi.openstack.org/aggregate-to-{{ include "openstack-manila-csi.nodeplugin.fullname" . }}: "true" -rules: - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "update"] - - apiGroups: [""] - resources: ["namespaces"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update"] diff --git a/charts/manila-csi-plugin/templates/nodeplugin-serviceaccount.yaml b/charts/manila-csi-plugin/templates/nodeplugin-serviceaccount.yaml deleted file mode 100644 index 9abbf463f5..0000000000 --- a/charts/manila-csi-plugin/templates/nodeplugin-serviceaccount.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "openstack-manila-csi.serviceAccountName.nodeplugin" . }} - labels: - app: {{ include "openstack-manila-csi.name" . }} - chart: {{ include "openstack-manila-csi.chart" . }} - component: {{ .Values.nodeplugin.name }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} diff --git a/charts/manila-csi-plugin/templates/runtimeconfig-cm.yaml b/charts/manila-csi-plugin/templates/runtimeconfig-cm.yaml deleted file mode 100644 index d739ce1640..0000000000 --- a/charts/manila-csi-plugin/templates/runtimeconfig-cm.yaml +++ /dev/null @@ -1,9 +0,0 @@ -{{- if .Values.csimanila.runtimeConfig.enabled }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: manila-csi-runtimeconf-cm -data: - runtimeconfig.json: |- -{{ .Values.csimanila.runtimeConfig.jsonData | indent 4 }} -{{- end }} diff --git a/charts/manila-csi-plugin/values.yaml b/charts/manila-csi-plugin/values.yaml deleted file mode 100644 index 7a8cf30db0..0000000000 --- a/charts/manila-csi-plugin/values.yaml +++ /dev/null @@ -1,84 +0,0 @@ -# Base name of the CSI Manila driver -driverName: manila.csi.openstack.org - -# Enabled Manila share protocols -shareProtocols: - - protocolSelector: NFS - fwdNodePluginEndpoint: - dir: /var/lib/kubelet/plugins/csi-nfsplugin - sockFile: csi.sock -# - protocolSelector: CEPHFS -# fwdNodePluginEndpoint: -# dir: /var/lib/kubelet/plugins/cephfs.csi.ceph.com -# sockFile: csi.sock - -# CSI Manila spec -csimanila: - # Set topologyAwarenessEnabled to true to enable topology awareness - topologyAwarenessEnabled: false - # Runtime configuration - runtimeConfig: - enabled: false - jsonData: |- - { - "nfs": { - "matchExportLocationAddress": "172.168.122.0/24" - } - } - # Availability zone for each node. topologyAwarenessEnabled must be set to true for this option to have any effect. - # If your Kubernetes cluster runs atop of Nova and want to use Nova AZs as AZs for the nodes of the cluster, uncomment the line below: - # nodeAZ: "$(curl http://169.254.169.254/openstack/latest/meta_data.json | jq -r .availability_zone)" - # Image spec - image: - repository: k8scloudprovider/manila-csi-plugin - tag: latest - pullPolicy: IfNotPresent - -# DeamonSet deployment -nodeplugin: - # Component name - name: nodeplugin - # CSI Manila container compute resources constraints - nodeplugin: - resources: {} - # csi-node-driver-registrar - registrar: - image: - repository: quay.io/k8scsi/csi-node-driver-registrar - tag: v1.1.0 - pullPolicy: IfNotPresent - resources: {} - nodeSelector: {} - tolerations: [] - affinity: {} - # Use fullnameOverride to fully override the name of this component - # fullnameOverride: some-other-name - -# StatefulSet deployment -controllerplugin: - name: controllerplugin - # CSI Manila container compute resources constraints - nodeplugin: - resources: {} - # CSI external-provisioner container spec - provisioner: - image: - repository: quay.io/k8scsi/csi-provisioner - tag: v1.4.0 - pullPolicy: IfNotPresent - resources: {} - # CSI external-snapshotter container spec - snapshotter: - image: - repository: quay.io/k8scsi/csi-snapshotter - tag: v1.2.2 - pullPolicy: IfNotPresent - resources: {} - nodeSelector: {} - tolerations: [] - affinity: {} - # Use fullnameOverride to fully override the name of this component - # fullnameOverride: some-other-name - -# Override the default app name using nameOverride -# nameOverride: some-other-name diff --git a/cluster/addons/rbac/cloud-controller-manager-role-bindings.yaml b/cluster/addons/rbac/cloud-controller-manager-role-bindings.yaml deleted file mode 100644 index 4f102516fe..0000000000 --- a/cluster/addons/rbac/cloud-controller-manager-role-bindings.yaml +++ /dev/null @@ -1,40 +0,0 @@ -apiVersion: v1 -items: -- apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRoleBinding - metadata: - name: system:cloud-node-controller - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:cloud-node-controller - subjects: - - kind: ServiceAccount - name: cloud-node-controller - namespace: kube-system -- apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRoleBinding - metadata: - name: system:pvl-controller - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:pvl-controller - subjects: - - kind: ServiceAccount - name: pvl-controller - namespace: kube-system -- apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRoleBinding - metadata: - name: system:cloud-controller-manager - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:cloud-controller-manager - subjects: - - kind: ServiceAccount - name: cloud-controller-manager - namespace: kube-system -kind: List -metadata: {} \ No newline at end of file diff --git a/cluster/addons/rbac/cloud-controller-manager-roles.yaml b/cluster/addons/rbac/cloud-controller-manager-roles.yaml deleted file mode 100644 index 1e8816e29e..0000000000 --- a/cluster/addons/rbac/cloud-controller-manager-roles.yaml +++ /dev/null @@ -1,129 +0,0 @@ -apiVersion: v1 -items: -- apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRole - metadata: - name: system:cloud-controller-manager - rules: - - apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - create - - update - - apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - update - - apiGroups: - - "" - resources: - - nodes - verbs: - - '*' - - apiGroups: - - "" - resources: - - nodes/status - verbs: - - patch - - apiGroups: - - "" - resources: - - services - verbs: - - list - - patch - - update - - watch - - apiGroups: - - "" - resources: - - serviceaccounts - verbs: - - create - - get - - apiGroups: - - "" - resources: - - persistentvolumes - verbs: - - '*' - - apiGroups: - - "" - resources: - - endpoints - verbs: - - create - - get - - list - - watch - - update - - apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch - - apiGroups: - - "" - resources: - - secrets - verbs: - - list - - get - - watch -- apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRole - metadata: - name: system:cloud-node-controller - rules: - - apiGroups: - - "" - resources: - - nodes - verbs: - - '*' - - apiGroups: - - "" - resources: - - nodes/status - verbs: - - patch - - apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - update -- apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRole - metadata: - name: system:pvl-controller - rules: - - apiGroups: - - "" - resources: - - persistentvolumes - verbs: - - '*' - - apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - update -kind: List -metadata: {} diff --git a/cluster/addons/storage-class/openstack/default.yaml b/cluster/addons/storage-class/openstack/default.yaml deleted file mode 100644 index 6df7cb76b9..0000000000 --- a/cluster/addons/storage-class/openstack/default.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: standard - annotations: - storageclass.beta.kubernetes.io/is-default-class: "true" - labels: - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: EnsureExists -provisioner: openstack.org/standalone-cinder diff --git a/cluster/images/barbican-kms-plugin/Dockerfile b/cluster/images/barbican-kms-plugin/Dockerfile deleted file mode 100644 index b9ffb73cd8..0000000000 --- a/cluster/images/barbican-kms-plugin/Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -ARG ALPINE_ARCH=amd64 -# We not using scratch because we need to keep the basic image information -# from parent image -FROM ${ALPINE_ARCH}/alpine:3.11 - -ARG ARCH=amd64 - -# Fill out the labels -LABEL name="barbican-kms-plugin" \ - license="Apache Version 2.0" \ - maintainers="Kubernetes Authors" \ - description="Barbican kms plugin" \ - architecture=$ARCH \ - distribution-scope="public" \ - summary="Barbican kms plugin" \ - help="none" - -ADD rootfs.tar / - -CMD ["sh", "-c", "/bin/barbican-kms-plugin --socketpath ${socketpath} --cloud-config ${cloudconfig}"] diff --git a/cluster/images/barbican-kms-plugin/Dockerfile.build b/cluster/images/barbican-kms-plugin/Dockerfile.build deleted file mode 100644 index 4dbd13232b..0000000000 --- a/cluster/images/barbican-kms-plugin/Dockerfile.build +++ /dev/null @@ -1,7 +0,0 @@ -ARG ALPINE_ARCH=amd64 -FROM ${ALPINE_ARCH}/alpine:3.11 -LABEL maintainers="Kubernetes Authors" -LABEL description="Barbican KMS Plugin" - -ARG ARCH=amd64 -ADD barbican-kms-plugin-${ARCH} /bin/barbican-kms-plugin diff --git a/cluster/images/cinder-csi-plugin/Dockerfile b/cluster/images/cinder-csi-plugin/Dockerfile deleted file mode 100644 index c5d0c6684c..0000000000 --- a/cluster/images/cinder-csi-plugin/Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -ARG DEBIAN_ARCH=amd64 -# We not using scratch because we need to keep the basic image information -# from parent image -FROM k8s.gcr.io/build-image/debian-base-${DEBIAN_ARCH}:v2.1.3 - -ARG ARCH=amd64 - -# Fill out the labels -LABEL name="cinder-csi-plugin" \ - license="Apache Version 2.0" \ - maintainers="Kubernetes Authors" \ - description="Cinder CSI Plugin" \ - architecture=$ARCH \ - distribution-scope="public" \ - summary="Cinder CSI Plugin" \ - help="none" - -ADD rootfs.tar / - -CMD ["/bin/cinder-csi-plugin"] diff --git a/cluster/images/cinder-csi-plugin/Dockerfile.build b/cluster/images/cinder-csi-plugin/Dockerfile.build deleted file mode 100644 index 4919e31d85..0000000000 --- a/cluster/images/cinder-csi-plugin/Dockerfile.build +++ /dev/null @@ -1,9 +0,0 @@ -ARG DEBIAN_ARCH=amd64 -FROM k8s.gcr.io/build-image/debian-base-${DEBIAN_ARCH}:v2.1.3 - -ARG ARCH=amd64 - -# Install e4fsprogs for format -RUN clean-install ca-certificates e2fsprogs mount xfsprogs udev - -ADD cinder-csi-plugin-${ARCH} /bin/cinder-csi-plugin diff --git a/cluster/images/k8s-keystone-auth/Dockerfile b/cluster/images/k8s-keystone-auth/Dockerfile deleted file mode 100644 index b6d2251a41..0000000000 --- a/cluster/images/k8s-keystone-auth/Dockerfile +++ /dev/null @@ -1,34 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -ARG ALPINE_ARCH=amd64 -# We not using scratch because we need to keep the basic image information -# from parent image -FROM ${ALPINE_ARCH}/alpine:3.11 - -ARG ARCH=amd64 - -# Fill out the labels -LABEL name="k8s-keystone-auth" \ - license="Apache Version 2.0" \ - maintainers="Kubernetes Authors" \ - description="K8s Keystone Auth" \ - architecture=$ARCH \ - distribution-scope="public" \ - summary="K8s Keystone Auth" \ - help="none" - -ADD rootfs.tar / - -EXPOSE 8443 - -CMD ["/bin/k8s-keystone-auth"] diff --git a/cluster/images/k8s-keystone-auth/Dockerfile.build b/cluster/images/k8s-keystone-auth/Dockerfile.build deleted file mode 100644 index e8ef5a716b..0000000000 --- a/cluster/images/k8s-keystone-auth/Dockerfile.build +++ /dev/null @@ -1,20 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -ARG ALPINE_ARCH=amd64 -FROM ${ALPINE_ARCH}/alpine:3.11 - -ARG ARCH=amd64 - -RUN apk add --no-cache ca-certificates - -ADD k8s-keystone-auth-${ARCH} /bin/k8s-keystone-auth diff --git a/cluster/images/magnum-auto-healer/Dockerfile b/cluster/images/magnum-auto-healer/Dockerfile deleted file mode 100644 index 4204f62f67..0000000000 --- a/cluster/images/magnum-auto-healer/Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -ARG ALPINE_ARCH=amd64 -# We not using scratch because we need to keep the basic image information -# from parent image -FROM ${ALPINE_ARCH}/alpine:3.11 - -ARG ARCH=amd64 - -# Fill out the labels -LABEL name="magnum-auto-healer" \ - license="Apache Version 2.0" \ - maintainers="Kubernetes Authors" \ - description="Magnum auto healer" \ - architecture=$ARCH \ - distribution-scope="public" \ - summary="Magnum auto healer" \ - help="none" - -ADD rootfs.tar / - -CMD ["/bin/magnum-auto-healer"] diff --git a/cluster/images/magnum-auto-healer/Dockerfile.build b/cluster/images/magnum-auto-healer/Dockerfile.build deleted file mode 100644 index 53519b2fc3..0000000000 --- a/cluster/images/magnum-auto-healer/Dockerfile.build +++ /dev/null @@ -1,20 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -ARG ALPINE_ARCH=amd64 -FROM ${ALPINE_ARCH}/alpine:3.11 - -ARG ARCH=amd64 - -RUN apk add --no-cache ca-certificates - -ADD magnum-auto-healer-${ARCH} /bin/magnum-auto-healer diff --git a/cluster/images/manila-csi-plugin/Dockerfile b/cluster/images/manila-csi-plugin/Dockerfile deleted file mode 100644 index b776289503..0000000000 --- a/cluster/images/manila-csi-plugin/Dockerfile +++ /dev/null @@ -1,33 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -ARG ALPINE_ARCH=amd64 -# We not using scratch because we need to keep the basic image information -# from parent image -FROM ${ALPINE_ARCH}/alpine:3.11 - -ARG ARCH=amd64 - -# Fill out the labels -LABEL name="manila-csi-plugin" \ - license="Apache Version 2.0" \ - maintainers="Kubernetes Authors" \ - description="Manila CSI Plugin" \ - architecture=$ARCH \ - distribution-scope="public" \ - summary="Manila CSI Plugin" \ - help="none" - - -ADD rootfs.tar / - -ENTRYPOINT ["/bin/manila-csi-plugin"] diff --git a/cluster/images/manila-csi-plugin/Dockerfile.build b/cluster/images/manila-csi-plugin/Dockerfile.build deleted file mode 100644 index da914df1ad..0000000000 --- a/cluster/images/manila-csi-plugin/Dockerfile.build +++ /dev/null @@ -1,8 +0,0 @@ -ARG ALPINE_ARCH=amd64 -FROM ${ALPINE_ARCH}/alpine:3.11 - -ARG ARCH=amd64 - -RUN apk add --no-cache ca-certificates jq curl - -ADD manila-csi-plugin-${ARCH} /bin/manila-csi-plugin diff --git a/cluster/images/octavia-ingress-controller/Dockerfile b/cluster/images/octavia-ingress-controller/Dockerfile deleted file mode 100644 index 8e2ff9cb84..0000000000 --- a/cluster/images/octavia-ingress-controller/Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -ARG ALPINE_ARCH=amd64 -# We not using scratch because we need to keep the basic image information -# from parent image -FROM ${ALPINE_ARCH}/alpine:3.11 - -ARG ARCH=amd64 - -# Fill out the labels -LABEL name="octavia-ingress-controller" \ - license="Apache Version 2.0" \ - maintainers="Kubernetes Authors" \ - description="Octavia ingress controller" \ - architecture=$ARCH \ - distribution-scope="public" \ - summary="Octavia ingress controller" \ - help="none" - -ADD rootfs.tar / - -CMD ["/bin/octavia-ingress-controller"] diff --git a/cluster/images/octavia-ingress-controller/Dockerfile.build b/cluster/images/octavia-ingress-controller/Dockerfile.build deleted file mode 100644 index f7c5024c87..0000000000 --- a/cluster/images/octavia-ingress-controller/Dockerfile.build +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -ARG ALPINE_ARCH=amd64 -FROM ${ALPINE_ARCH}/alpine:3.11 - -ARG ARCH=amd64 - -RUN apk add --no-cache ca-certificates -ADD octavia-ingress-controller-${ARCH} /bin/octavia-ingress-controller diff --git a/cluster/images/openstack-cloud-controller-manager/Dockerfile b/cluster/images/openstack-cloud-controller-manager/Dockerfile deleted file mode 100644 index d30b5a4e5d..0000000000 --- a/cluster/images/openstack-cloud-controller-manager/Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -ARG ALPINE_ARCH=amd64 -# We not using scratch because we need to keep the basic image information -# from parent image -FROM ${ALPINE_ARCH}/alpine:3.11 - -ARG ARCH=amd64 - -# Fill out the labels -LABEL name="openstack-cloud-controller-manager" \ - license="Apache Version 2.0" \ - maintainers="Kubernetes Authors" \ - description="OpenStack cloud controller manager" \ - architecture=$ARCH \ - distribution-scope="public" \ - summary="OpenStack cloud controller manager" \ - help="none" - -ADD rootfs.tar / - -CMD ["/bin/openstack-cloud-controller-manager"] diff --git a/cluster/images/openstack-cloud-controller-manager/Dockerfile.build b/cluster/images/openstack-cloud-controller-manager/Dockerfile.build deleted file mode 100644 index d660238afb..0000000000 --- a/cluster/images/openstack-cloud-controller-manager/Dockerfile.build +++ /dev/null @@ -1,19 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -ARG ALPINE_ARCH=amd64 -FROM ${ALPINE_ARCH}/alpine:3.11 - -ARG ARCH=amd64 -RUN apk add --no-cache ca-certificates - -ADD openstack-cloud-controller-manager-${ARCH} /bin/openstack-cloud-controller-manager diff --git a/cmd/barbican-kms-plugin/main.go b/cmd/barbican-kms-plugin/main.go deleted file mode 100644 index 431f1d6d93..0000000000 --- a/cmd/barbican-kms-plugin/main.go +++ /dev/null @@ -1,86 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "flag" - "fmt" - "os" - "os/signal" - - "github.com/spf13/cobra" - "golang.org/x/sys/unix" - "k8s.io/cloud-provider-openstack/pkg/kms/server" - "k8s.io/component-base/logs" - "k8s.io/klog/v2" -) - -var ( - socketpath string - cloudconfig string -) - -func init() { - flag.Set("logtostderr", "true") -} - -func main() { - // Glog requires this otherwise it complains. - flag.CommandLine.Parse(nil) - // This is a temporary hack to enable proper logging until upstream dependencies - // are migrated to fully utilize klog instead of glog. - klogFlags := flag.NewFlagSet("klog", flag.ExitOnError) - klog.InitFlags(klogFlags) - - // Sync the glog and klog flags. - flag.CommandLine.VisitAll(func(f1 *flag.Flag) { - f2 := klogFlags.Lookup(f1.Name) - if f2 != nil { - value := f1.Value.String() - f2.Value.Set(value) - } - }) - - logs.InitLogs() - defer logs.FlushLogs() - - cmd := &cobra.Command{ - Use: "barbican-kms-plugin", - Short: "Barbican KMS plugin for kubernetes", - RunE: func(cmd *cobra.Command, args []string) error { - sigchan := make(chan os.Signal, 1) - signal.Notify(sigchan, unix.SIGTERM, unix.SIGINT) - err := server.Run(cloudconfig, socketpath, sigchan) - return err - }, - } - - cmd.Flags().AddGoFlagSet(flag.CommandLine) - - cmd.PersistentFlags().StringVar(&socketpath, "socketpath", "", "Barbican KMS Plugin unix socket endpoint") - cmd.MarkPersistentFlagRequired("socketpath") - - cmd.PersistentFlags().StringVar(&cloudconfig, "cloud-config", "", "Barbican KMS Plugin cloud config") - cmd.MarkPersistentFlagRequired("cloud-config") - - if err := cmd.Execute(); err != nil { - fmt.Fprintf(os.Stderr, "%s", err.Error()) - os.Exit(1) - } - - os.Exit(0) -} diff --git a/cmd/cinder-csi-plugin/main.go b/cmd/cinder-csi-plugin/main.go deleted file mode 100644 index 6ecd7fb040..0000000000 --- a/cmd/cinder-csi-plugin/main.go +++ /dev/null @@ -1,119 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "flag" - "fmt" - "os" - - "github.com/spf13/cobra" - "github.com/spf13/pflag" - "k8s.io/cloud-provider-openstack/pkg/csi/cinder" - "k8s.io/cloud-provider-openstack/pkg/csi/cinder/openstack" - "k8s.io/cloud-provider-openstack/pkg/util/metadata" - "k8s.io/cloud-provider-openstack/pkg/util/mount" - "k8s.io/component-base/logs" - "k8s.io/klog/v2" -) - -var ( - endpoint string - nodeID string - cloudconfig string - cluster string -) - -func init() { - flag.Set("logtostderr", "true") -} - -func main() { - - flag.CommandLine.Parse([]string{}) - - cmd := &cobra.Command{ - Use: "Cinder", - Short: "CSI based Cinder driver", - PersistentPreRun: func(cmd *cobra.Command, args []string) { - // Glog requires this otherwise it complains. - flag.CommandLine.Parse(nil) - - // This is a temporary hack to enable proper logging until upstream dependencies - // are migrated to fully utilize klog instead of glog. - klogFlags := flag.NewFlagSet("klog", flag.ExitOnError) - klog.InitFlags(klogFlags) - - // Sync the glog and klog flags. - cmd.Flags().VisitAll(func(f1 *pflag.Flag) { - f2 := klogFlags.Lookup(f1.Name) - if f2 != nil { - value := f1.Value.String() - f2.Value.Set(value) - } - }) - }, - Run: func(cmd *cobra.Command, args []string) { - handle() - }, - } - - cmd.Flags().AddGoFlagSet(flag.CommandLine) - - cmd.PersistentFlags().StringVar(&nodeID, "nodeid", "", "node id") - cmd.MarkPersistentFlagRequired("nodeid") - - cmd.PersistentFlags().StringVar(&endpoint, "endpoint", "", "CSI endpoint") - cmd.MarkPersistentFlagRequired("endpoint") - - cmd.PersistentFlags().StringVar(&cloudconfig, "cloud-config", "", "CSI driver cloud config") - cmd.MarkPersistentFlagRequired("cloud-config") - - cmd.PersistentFlags().StringVar(&cluster, "cluster", "", "The identifier of the cluster that the plugin is running in.") - - openstack.AddExtraFlags(pflag.CommandLine) - - logs.InitLogs() - defer logs.FlushLogs() - - if err := cmd.Execute(); err != nil { - fmt.Fprintf(os.Stderr, "%s", err.Error()) - os.Exit(1) - } - - os.Exit(0) -} - -func handle() { - - d := cinder.NewDriver(nodeID, endpoint, cluster) - // Initiliaze cloud - openstack.InitOpenStackProvider(cloudconfig) - cloud, err := openstack.GetOpenStackProvider() - if err != nil { - klog.Warningf("Failed to GetOpenStackProvider: %v", err) - return - } - //Intiliaze mount - mount := mount.GetMountProvider() - - //Intiliaze Metadatda - metadatda := metadata.GetMetadataProvider(cloud.GetMetadataOpts().SearchOrder) - - d.SetupDriver(cloud, mount, metadatda) - d.Run() -} diff --git a/cmd/client-keystone-auth/main.go b/cmd/client-keystone-auth/main.go deleted file mode 100644 index d378bfcc61..0000000000 --- a/cmd/client-keystone-auth/main.go +++ /dev/null @@ -1,232 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "flag" - "fmt" - "io" - "os" - "time" - - "github.com/gophercloud/gophercloud" - "github.com/gophercloud/utils/openstack/clientconfig" - "github.com/spf13/pflag" - - "golang.org/x/crypto/ssh/terminal" - - "k8s.io/cloud-provider-openstack/pkg/identity/keystone" - kflag "k8s.io/component-base/cli/flag" - "k8s.io/klog/v2" -) - -const errRespTemplate string = `{ - "apiVersion": "client.authentication.k8s.io/v1beta1", - "kind": "ExecCredential", - "status": {} -}` - -const respTemplate string = `{ - "apiVersion": "client.authentication.k8s.io/v1beta1", - "kind": "ExecCredential", - "status": { - "token": "%v", - "expirationTimestamp": "%v" - } -}` - -func promptForString(field string, r io.Reader, show bool) (result string, err error) { - // We have to print output to Stderr, because Stdout is redirected and not shown to the user. - fmt.Fprintf(os.Stderr, "Please enter %s: ", field) - - if show { - _, err = fmt.Fscan(r, &result) - } else { - var data []byte - data, err = terminal.ReadPassword(int(os.Stdin.Fd())) - result = string(data) - fmt.Fprintf(os.Stderr, "\n") - } - return result, err -} - -// prompt pulls keystone auth url, domain, project, username and password from stdin, -// if they are not specified initially (i.e. equal ""). -func prompt(url string, domain string, user string, project string, password string, applicationCredentialID string, applicationCredentialName string, applicationCredentialSecret string) (gophercloud.AuthOptions, error) { - var err error - var options gophercloud.AuthOptions - - if url == "" { - url, err = promptForString("Keystone Auth URL", os.Stdin, true) - if err != nil { - return options, err - } - } - - if domain == "" { - domain, err = promptForString("domain name", os.Stdin, true) - if err != nil { - return options, err - } - } - - if user == "" { - user, err = promptForString("user name", os.Stdin, true) - if err != nil { - return options, err - } - } - - if project == "" && applicationCredentialID == "" && applicationCredentialName == "" { - project, err = promptForString("project name", os.Stdin, true) - if err != nil { - return options, err - } - } - - if password == "" && applicationCredentialID == "" && applicationCredentialName == "" { - password, err = promptForString("password", nil, false) - if err != nil { - return options, err - } - } - - options = gophercloud.AuthOptions{ - IdentityEndpoint: url, - Username: user, - TenantName: project, - Password: password, - DomainName: domain, - ApplicationCredentialID: applicationCredentialID, - ApplicationCredentialName: applicationCredentialName, - ApplicationCredentialSecret: applicationCredentialSecret, - } - - return options, nil -} - -func argumentsAreSet(url, user, project, password, domain, applicationCredentialID, applicationCredentialName, applicationCredentialSecret string) bool { - if url == "" { - return false - } - - if user != "" && project != "" && domain != "" && password != "" { - return true - } - - if applicationCredentialID != "" && applicationCredentialName != "" && applicationCredentialSecret != "" { - return true - } - - return false -} - -func main() { - // Glog requires this otherwise it complains. - flag.CommandLine.Parse(nil) - // This is a temporary hack to enable proper logging until upstream dependencies - // are migrated to fully utilize klog instead of glog. - klogFlags := flag.NewFlagSet("klog", flag.ExitOnError) - klog.InitFlags(klogFlags) - - // Sync the glog and klog flags. - flag.CommandLine.VisitAll(func(f1 *flag.Flag) { - f2 := klogFlags.Lookup(f1.Name) - if f2 != nil { - value := f1.Value.String() - f2.Value.Set(value) - } - }) - - var url string - var domain string - var user string - var project string - var password string - var clientCertPath string - var clientKeyPath string - var clientCAPath string - var options keystone.Options - var err error - var applicationCredentialID string - var applicationCredentialName string - var applicationCredentialSecret string - - pflag.StringVar(&url, "keystone-url", os.Getenv("OS_AUTH_URL"), "URL for the OpenStack Keystone API") - pflag.StringVar(&domain, "domain-name", os.Getenv("OS_DOMAIN_NAME"), "Keystone domain name") - pflag.StringVar(&user, "user-name", os.Getenv("OS_USERNAME"), "User name") - pflag.StringVar(&project, "project-name", os.Getenv("OS_PROJECT_NAME"), "Keystone project name") - pflag.StringVar(&password, "password", os.Getenv("OS_PASSWORD"), "Password") - pflag.StringVar(&clientCertPath, "cert", os.Getenv("OS_CERT"), "Client certificate bundle file") - pflag.StringVar(&clientKeyPath, "key", os.Getenv("OS_KEY"), "Client certificate key file") - pflag.StringVar(&clientCAPath, "cacert", os.Getenv("OS_CACERT"), "Certificate authority file") - pflag.StringVar(&applicationCredentialID, "application-credential-id", os.Getenv("OS_APPLICATION_CREDENTIAL_ID"), "Application Credential ID") - pflag.StringVar(&applicationCredentialName, "application-credential-name", os.Getenv("OS_APPLICATION_CREDENTIAL_NAME"), "Application Credential Name") - pflag.StringVar(&applicationCredentialSecret, "application-credential-secret", os.Getenv("OS_APPLICATION_CREDENTIAL_SECRET"), "Application Credential Secret") - pflag.CommandLine.AddGoFlagSet(klogFlags) - kflag.InitFlags() - - // Generate Gophercloud Auth Options based on input data from stdin - // if IsTerminal returns "true", or from env variables otherwise. - if !terminal.IsTerminal(int(os.Stdin.Fd())) { - // If all requiered arguments are set use them - if argumentsAreSet(url, user, project, password, domain, applicationCredentialID, applicationCredentialName, applicationCredentialSecret) { - options.AuthOptions = gophercloud.AuthOptions{ - IdentityEndpoint: url, - Username: user, - TenantName: project, - Password: password, - DomainName: domain, - ApplicationCredentialID: applicationCredentialID, - ApplicationCredentialName: applicationCredentialName, - ApplicationCredentialSecret: applicationCredentialSecret, - } - } else { - // Use environment variables if arguments are missing - authOpts, err := clientconfig.AuthOptions(nil) - if err != nil { - fmt.Fprintf(os.Stderr, "Failed to read openstack env vars: %s\n", err) - os.Exit(1) - } - options.AuthOptions = *authOpts - } - } else { - options.AuthOptions, err = prompt(url, domain, user, project, password, applicationCredentialID, applicationCredentialName, applicationCredentialSecret) - if err != nil { - fmt.Fprintf(os.Stderr, "Failed to read data from console: %s\n", err) - os.Exit(1) - } - } - - options.ClientCertPath = clientCertPath - options.ClientKeyPath = clientKeyPath - options.ClientCAPath = clientCAPath - - token, err := keystone.GetToken(options) - if err != nil { - if _, ok := err.(gophercloud.ErrDefault401); ok { - fmt.Println(errRespTemplate) - os.Stderr.WriteString("Invalid user credentials were provided\n") - os.Exit(0) - } - fmt.Fprintf(os.Stderr, "An error occurred: %v\n", err) - os.Exit(1) - } - - out := fmt.Sprintf(respTemplate, token.ID, token.ExpiresAt.Format(time.RFC3339Nano)) - fmt.Println(out) -} diff --git a/cmd/k8s-keystone-auth/main.go b/cmd/k8s-keystone-auth/main.go deleted file mode 100644 index 4ef602273b..0000000000 --- a/cmd/k8s-keystone-auth/main.go +++ /dev/null @@ -1,66 +0,0 @@ -/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "flag" - "os" - - "github.com/spf13/pflag" - "k8s.io/klog/v2" - - "k8s.io/cloud-provider-openstack/pkg/identity/keystone" - kflag "k8s.io/component-base/cli/flag" - "k8s.io/component-base/logs" -) - -func main() { - // Glog requires this otherwise it complains. - flag.CommandLine.Parse(nil) - // This is a temporary hack to enable proper logging until upstream dependencies - // are migrated to fully utilize klog instead of glog. - klogFlags := flag.NewFlagSet("klog", flag.ExitOnError) - klog.InitFlags(klogFlags) - - keystone.AddExtraFlags(pflag.CommandLine) - - // Sync the glog and klog flags. - flag.CommandLine.VisitAll(func(f1 *flag.Flag) { - f2 := klogFlags.Lookup(f1.Name) - if f2 != nil { - value := f1.Value.String() - f2.Value.Set(value) - } - }) - - logs.InitLogs() - defer logs.FlushLogs() - - config := keystone.NewConfig() - config.AddFlags(pflag.CommandLine) - kflag.InitFlags() - - if err := config.ValidateFlags(); err != nil { - klog.Errorf("%v", err) - os.Exit(1) - } - - keystoneAuth, err := keystone.NewKeystoneAuth(config) - if err != nil { - klog.Errorf("%v", err) - os.Exit(1) - } - keystoneAuth.Run() -} diff --git a/cmd/magnum-auto-healer/main.go b/cmd/magnum-auto-healer/main.go deleted file mode 100644 index 9e54de6f3a..0000000000 --- a/cmd/magnum-auto-healer/main.go +++ /dev/null @@ -1,25 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "k8s.io/cloud-provider-openstack/pkg/autohealing/cmd" -) - -func main() { - cmd.Execute() -} diff --git a/cmd/manila-csi-plugin/main.go b/cmd/manila-csi-plugin/main.go deleted file mode 100644 index 5e4459c922..0000000000 --- a/cmd/manila-csi-plugin/main.go +++ /dev/null @@ -1,190 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "flag" - "fmt" - "os" - "strings" - - "github.com/spf13/cobra" - "github.com/spf13/pflag" - "k8s.io/cloud-provider-openstack/pkg/csi/manila" - "k8s.io/cloud-provider-openstack/pkg/csi/manila/csiclient" - "k8s.io/cloud-provider-openstack/pkg/csi/manila/manilaclient" - "k8s.io/cloud-provider-openstack/pkg/csi/manila/options" - "k8s.io/cloud-provider-openstack/pkg/csi/manila/runtimeconfig" - "k8s.io/component-base/logs" - "k8s.io/klog/v2" -) - -var ( - endpoint string - driverName string - nodeID string - nodeAZ string - runtimeConfigFile string - withTopology bool - protoSelector string - fwdEndpoint string - userAgentData []string - compatibilitySettings string -) - -func validateShareProtocolSelector(v string) error { - supportedShareProtocols := []string{"NFS", "CEPHFS"} - - v = strings.ToUpper(v) - for _, proto := range supportedShareProtocols { - if v == proto { - return nil - } - } - - return fmt.Errorf("share protocol %q not supported; supported protocols are %v", v, supportedShareProtocols) -} - -func parseCompatOpts() (*options.CompatibilityOptions, error) { - data := make(map[string]string) - - if compatibilitySettings == "" { - return options.NewCompatibilityOptions(data) - } - - knownCompatSettings := map[string]interface{}{} - - isKnown := func(v string) bool { - _, ok := knownCompatSettings[v] - return ok - } - - settings := strings.Split(compatibilitySettings, ",") - for _, elem := range settings { - setting := strings.SplitN(elem, "=", 2) - - if len(setting) != 2 || setting[0] == "" || setting[1] == "" { - return nil, fmt.Errorf("invalid format in option %v, expected KEY=VALUE", setting) - } - - if !isKnown(setting[0]) { - return nil, fmt.Errorf("unrecognized option '%s'", setting[0]) - } - - data[setting[0]] = setting[1] - } - - return options.NewCompatibilityOptions(data) -} - -func main() { - flag.CommandLine.Parse([]string{}) - - cmd := &cobra.Command{ - Use: os.Args[0], - Short: "CSI Manila driver", - PersistentPreRun: func(cmd *cobra.Command, args []string) { - // Glog requires this otherwise it complains. - flag.CommandLine.Parse(nil) - - // This is a temporary hack to enable proper logging until upstream dependencies - // are migrated to fully utilize klog instead of glog. - klogFlags := flag.NewFlagSet("klog", flag.ExitOnError) - klog.InitFlags(klogFlags) - - // Sync the glog and klog flags. - cmd.Flags().VisitAll(func(f1 *pflag.Flag) { - f2 := klogFlags.Lookup(f1.Name) - if f2 != nil { - value := f1.Value.String() - f2.Value.Set(value) - } - }) - }, - Run: func(cmd *cobra.Command, args []string) { - if err := validateShareProtocolSelector(protoSelector); err != nil { - klog.Fatalf(err.Error()) - } - - compatOpts, err := parseCompatOpts() - if err != nil { - klog.Fatalf("failed to parse compatibility settings: %v", err) - } - - manilaClientBuilder := &manilaclient.ClientBuilder{UserAgent: "manila-csi-plugin", ExtraUserAgentData: userAgentData} - csiClientBuilder := &csiclient.ClientBuilder{} - - d, err := manila.NewDriver( - &manila.DriverOpts{ - DriverName: driverName, - NodeID: nodeID, - NodeAZ: nodeAZ, - WithTopology: withTopology, - ShareProto: protoSelector, - ServerCSIEndpoint: endpoint, - FwdCSIEndpoint: fwdEndpoint, - ManilaClientBuilder: manilaClientBuilder, - CSIClientBuilder: csiClientBuilder, - CompatOpts: compatOpts, - }, - ) - - if err != nil { - klog.Fatalf("driver initialization failed: %v", err) - } - - runtimeconfig.RuntimeConfigFilename = runtimeConfigFile - - d.Run() - }, - } - - cmd.Flags().AddGoFlagSet(flag.CommandLine) - - cmd.PersistentFlags().StringVar(&endpoint, "endpoint", "unix://tmp/csi.sock", "CSI endpoint") - - cmd.PersistentFlags().StringVar(&driverName, "drivername", "manila.csi.openstack.org", "name of the driver") - - cmd.PersistentFlags().StringVar(&nodeID, "nodeid", "", "this node's ID") - cmd.MarkPersistentFlagRequired("nodeid") - - cmd.PersistentFlags().StringVar(&nodeAZ, "nodeaz", "", "this node's availability zone") - - cmd.PersistentFlags().StringVar(&runtimeConfigFile, "runtime-config-file", "", "path to the runtime configuration file") - - cmd.PersistentFlags().BoolVar(&withTopology, "with-topology", false, "cluster is topology-aware") - - cmd.PersistentFlags().StringVar(&protoSelector, "share-protocol-selector", "", "specifies which Manila share protocol to use. Valid values are NFS and CEPHFS") - cmd.MarkPersistentFlagRequired("share-protocol-selector") - - cmd.PersistentFlags().StringVar(&fwdEndpoint, "fwdendpoint", "", "CSI Node Plugin endpoint to which all Node Service RPCs are forwarded. Must be able to handle the file-system specified in share-protocol-selector") - cmd.MarkPersistentFlagRequired("fwdendpoint") - - cmd.PersistentFlags().StringVar(&compatibilitySettings, "compatibility-settings", "", "settings for the compatibility layer") - - cmd.PersistentFlags().StringArrayVar(&userAgentData, "user-agent", nil, "extra data to add to gophercloud user-agent. Use multiple times to add more than one component.") - - logs.InitLogs() - defer logs.FlushLogs() - - if err := cmd.Execute(); err != nil { - fmt.Fprintf(os.Stderr, "%s\n", err.Error()) - os.Exit(1) - } - - os.Exit(0) -} diff --git a/cmd/octavia-ingress-controller/main.go b/cmd/octavia-ingress-controller/main.go deleted file mode 100644 index 0725877d88..0000000000 --- a/cmd/octavia-ingress-controller/main.go +++ /dev/null @@ -1,25 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "k8s.io/cloud-provider-openstack/pkg/ingress/cmd" -) - -func main() { - cmd.Execute() -} diff --git a/cmd/openstack-cloud-controller-manager/main.go b/cmd/openstack-cloud-controller-manager/main.go deleted file mode 100644 index 4bc23f111f..0000000000 --- a/cmd/openstack-cloud-controller-manager/main.go +++ /dev/null @@ -1,259 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// The external controller manager is responsible for running controller loops that -// are cloud provider dependent. It uses the API to listen to new events on resources. - -package main - -import ( - goflag "flag" - "fmt" - "math/rand" - "net" - "net/http" - "os" - "strings" - "time" - - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/apiserver/pkg/server/healthz" - cloudprovider "k8s.io/cloud-provider" - "k8s.io/cloud-provider-openstack/pkg/cloudprovider/providers/openstack" - "k8s.io/cloud-provider-openstack/pkg/version" - "k8s.io/component-base/cli/flag" - "k8s.io/component-base/logs" - _ "k8s.io/component-base/metrics/prometheus/restclient" // for client metric registration - _ "k8s.io/component-base/metrics/prometheus/version" // for version metric registration - "k8s.io/klog/v2" - "k8s.io/kubernetes/cmd/cloud-controller-manager/app" - cloudcontrollerconfig "k8s.io/kubernetes/cmd/cloud-controller-manager/app/config" - "k8s.io/kubernetes/cmd/cloud-controller-manager/app/options" - cloudcontrollers "k8s.io/kubernetes/pkg/controller/cloud" - routecontroller "k8s.io/kubernetes/pkg/controller/route" - servicecontroller "k8s.io/kubernetes/pkg/controller/service" - _ "k8s.io/kubernetes/pkg/features" // add the kubernetes feature gates - utilflag "k8s.io/kubernetes/pkg/util/flag" - - "github.com/spf13/cobra" - "github.com/spf13/pflag" -) - -func init() { - mux := http.NewServeMux() - healthz.InstallHandler(mux) -} - -var ( - versionFlag bool -) - -func main() { - rand.Seed(time.Now().UTC().UnixNano()) - - goflag.CommandLine.Parse([]string{}) - s, err := options.NewCloudControllerManagerOptions() - if err != nil { - klog.Fatalf("unable to initialize command options: %v", err) - } - - command := &cobra.Command{ - Use: "openstack-cloud-controller-manager", - Long: `The Cloud controller manager is a daemon that embeds -the cloud specific control loops shipped with Kubernetes.`, - PersistentPreRun: func(cmd *cobra.Command, args []string) { - // Glog requires this otherwise it complains. - goflag.CommandLine.Parse(nil) - - // This is a temporary hack to enable proper logging until upstream dependencies - // are migrated to fully utilize klog instead of glog. - klogFlags := goflag.NewFlagSet("klog", goflag.ExitOnError) - klog.InitFlags(klogFlags) - - // Sync the glog and klog flags. - cmd.Flags().VisitAll(func(f1 *pflag.Flag) { - f2 := klogFlags.Lookup(f1.Name) - if f2 != nil { - value := f1.Value.String() - f2.Value.Set(value) - } - }) - }, - Run: func(cmd *cobra.Command, args []string) { - if versionFlag { - version.PrintVersionAndExit() - } - - utilflag.PrintFlags(cmd.Flags()) - - c, err := s.Config(KnownControllers(), ControllersDisabledByDefault.List()) - if err != nil { - fmt.Fprintf(os.Stderr, "%v\n", err) - os.Exit(1) - } - - if err := app.Run(c.Complete(), wait.NeverStop); err != nil { - fmt.Fprintf(os.Stderr, "%v\n", err) - os.Exit(1) - } - }, - } - - fs := command.Flags() - namedFlagSets := s.Flags(KnownControllers(), ControllersDisabledByDefault.List()) - for _, f := range namedFlagSets.FlagSets { - fs.AddFlagSet(f) - } - - fs.BoolVar(&versionFlag, "version", false, "Print version and exit") - - openstack.AddExtraFlags(pflag.CommandLine) - - // TODO: once we switch everything over to Cobra commands, we can go back to calling - // utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the - // normalize func and add the go flag set by hand. - pflag.CommandLine.SetNormalizeFunc(flag.WordSepNormalizeFunc) - pflag.CommandLine.AddGoFlagSet(goflag.CommandLine) - // utilflag.InitFlags() - logs.InitLogs() - defer logs.FlushLogs() - - klog.V(1).Infof("openstack-cloud-controller-manager version: %s", version.Version) - - s.KubeCloudShared.CloudProvider.Name = openstack.ProviderName - if err := command.Execute(); err != nil { - fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } -} - -// initFunc is used to launch a particular controller. It may run additional "should I activate checks". -// Any error returned will cause the controller process to `Fatal` -// The bool indicates whether the controller was enabled. -type initFunc func(ctx *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface, stop <-chan struct{}) (debuggingHandler http.Handler, enabled bool, err error) - -// KnownControllers indicate the default controller we are known. -func KnownControllers() []string { - ret := sets.StringKeySet(newControllerInitializers()) - return ret.List() -} - -// ControllersDisabledByDefault is the controller disabled default when starting cloud-controller managers. -var ControllersDisabledByDefault = sets.NewString() - -// newControllerInitializers is a private map of named controller groups (you can start more than one in an init func) -// paired to their initFunc. This allows for structured downstream composition and subdivision. -func newControllerInitializers() map[string]initFunc { - controllers := map[string]initFunc{} - controllers["cloud-node"] = startCloudNodeController - controllers["cloud-node-lifecycle"] = startCloudNodeLifecycleController - controllers["service"] = startServiceController - controllers["route"] = startRouteController - return controllers -} - -func startCloudNodeController(ctx *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface, stopCh <-chan struct{}) (http.Handler, bool, error) { - // Start the CloudNodeController - nodeController, err := cloudcontrollers.NewCloudNodeController( - ctx.SharedInformers.Core().V1().Nodes(), - // cloud node controller uses existing cluster role from node-controller - ctx.ClientBuilder.ClientOrDie("node-controller"), - cloud, - ctx.ComponentConfig.NodeStatusUpdateFrequency.Duration) - - if err != nil { - klog.Warningf("failed to start cloud node controller: %s", err) - return nil, false, nil - } - - go nodeController.Run(stopCh) - - return nil, true, nil -} - -func startCloudNodeLifecycleController(ctx *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface, stopCh <-chan struct{}) (http.Handler, bool, error) { - // Start the cloudNodeLifecycleController - cloudNodeLifecycleController, err := cloudcontrollers.NewCloudNodeLifecycleController( - ctx.SharedInformers.Core().V1().Nodes(), - // cloud node lifecycle controller uses existing cluster role from node-controller - ctx.ClientBuilder.ClientOrDie("node-controller"), - cloud, - ctx.ComponentConfig.KubeCloudShared.NodeMonitorPeriod.Duration, - ) - if err != nil { - klog.Warningf("failed to start cloud node lifecycle controller: %s", err) - return nil, false, nil - } - - go cloudNodeLifecycleController.Run(stopCh) - - return nil, true, nil -} - -func startServiceController(ctx *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface, stopCh <-chan struct{}) (http.Handler, bool, error) { - // Start the service controller - serviceController, err := servicecontroller.New( - cloud, - ctx.ClientBuilder.ClientOrDie("service-controller"), - ctx.SharedInformers.Core().V1().Services(), - ctx.SharedInformers.Core().V1().Nodes(), - ctx.ComponentConfig.KubeCloudShared.ClusterName, - ) - if err != nil { - // This error shouldn't fail. It lives like this as a legacy. - klog.Errorf("Failed to start service controller: %v", err) - return nil, false, nil - } - - go serviceController.Run(stopCh, int(ctx.ComponentConfig.ServiceController.ConcurrentServiceSyncs)) - - return nil, true, nil -} - -func startRouteController(ctx *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface, stopCh <-chan struct{}) (http.Handler, bool, error) { - if !ctx.ComponentConfig.KubeCloudShared.AllocateNodeCIDRs || !ctx.ComponentConfig.KubeCloudShared.ConfigureCloudRoutes { - klog.Infof("Will not configure cloud provider routes for allocate-node-cidrs: %v, configure-cloud-routes: %v.", ctx.ComponentConfig.KubeCloudShared.AllocateNodeCIDRs, ctx.ComponentConfig.KubeCloudShared.ConfigureCloudRoutes) - return nil, false, nil - } - - // If CIDRs should be allocated for pods and set on the CloudProvider, then start the route controller - routes, ok := cloud.Routes() - if !ok { - klog.Warning("configure-cloud-routes is set, but cloud provider does not support routes. Will not configure cloud provider routes.") - return nil, false, nil - } - var clusterCIDRs []*net.IPNet - if len(strings.TrimSpace(ctx.ComponentConfig.KubeCloudShared.ClusterCIDR)) != 0 { - _, clusterCIDR, err := net.ParseCIDR(ctx.ComponentConfig.KubeCloudShared.ClusterCIDR) - if err != nil { - klog.Warningf("Unsuccessful parsing of cluster CIDR %v: %v", ctx.ComponentConfig.KubeCloudShared.ClusterCIDR, err) - } - clusterCIDRs = append(clusterCIDRs, clusterCIDR) - - } - - routeController := routecontroller.New( - routes, - ctx.ClientBuilder.ClientOrDie("route-controller"), - ctx.SharedInformers.Core().V1().Nodes(), - ctx.ComponentConfig.KubeCloudShared.ClusterName, - clusterCIDRs, - ) - go routeController.Run(stopCh, ctx.ComponentConfig.KubeCloudShared.RouteReconciliationPeriod.Duration) - - return nil, true, nil -} diff --git a/cmd/tests/cinder_csi_e2e_suite_test.go b/cmd/tests/cinder_csi_e2e_suite_test.go deleted file mode 100644 index 9837ca4b6f..0000000000 --- a/cmd/tests/cinder_csi_e2e_suite_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package main - -import ( - "flag" - "fmt" - "log" - "os" - "path" - "testing" - - "github.com/onsi/ginkgo" - "github.com/onsi/ginkgo/config" - "github.com/onsi/ginkgo/reporters" - "github.com/onsi/gomega" - _ "k8s.io/cloud-provider-openstack/tests/e2e/csi/cinder" - "k8s.io/kubernetes/test/e2e/framework" - frameworkconfig "k8s.io/kubernetes/test/e2e/framework/config" -) - -func init() { - frameworkconfig.CopyFlags(frameworkconfig.Flags, flag.CommandLine) - framework.RegisterCommonFlags(flag.CommandLine) - framework.RegisterClusterFlags(flag.CommandLine) - framework.AfterReadingAllFlags(&framework.TestContext) -} - -func Test(t *testing.T) { - gomega.RegisterFailHandler(ginkgo.Fail) - var r []ginkgo.Reporter - if framework.TestContext.ReportDir != "" { - if err := os.MkdirAll(framework.TestContext.ReportDir, 0755); err != nil { - log.Fatalf("Failed creating report directory: %v", err) - } else { - r = append(r, reporters.NewJUnitReporter(path.Join(framework.TestContext.ReportDir, fmt.Sprintf("junit_%v%02d.xml", framework.TestContext.ReportPrefix, config.GinkgoConfig.ParallelNode)))) - } - } - log.Printf("Starting e2e run %q on Ginkgo node %d", framework.RunID, config.GinkgoConfig.ParallelNode) - - ginkgo.RunSpecsWithDefaultAndCustomReporters(t, "Cinder CSI Suite", r) -} - -func main() { - flag.Parse() - Test(&testing.T{}) -} diff --git a/code-of-conduct.md b/code-of-conduct.md deleted file mode 100644 index 0d15c00cf3..0000000000 --- a/code-of-conduct.md +++ /dev/null @@ -1,3 +0,0 @@ -# Kubernetes Community Code of Conduct - -Please refer to our [Kubernetes Community Code of Conduct](https://git.k8s.io/community/code-of-conduct.md) diff --git a/contrib/gate_hook.sh b/contrib/gate_hook.sh deleted file mode 100755 index 37d1d7dcda..0000000000 --- a/contrib/gate_hook.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -set -ex - -echo "In gate_test_hook" -$BASE/new/devstack-gate/devstack-vm-gate.sh diff --git a/contrib/post_test_hook.sh b/contrib/post_test_hook.sh deleted file mode 100755 index 8ccdaa2f3e..0000000000 --- a/contrib/post_test_hook.sh +++ /dev/null @@ -1,176 +0,0 @@ -#!/bin/bash -xe - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This script is executed inside post_test_hook function in devstack gate. - -BASE_DIR=$(cd $(dirname $BASH_SOURCE)/.. && pwd) - - -TESTS_LIST_REGEX=( - '\[Slow\]' - '\[Serial\]' - '\[Disruptive\]' - '\[Flaky\]' - '\[Feature:.+\]' - '\[HPA\]' -) - -FLAKY_TESTS_LIST=( - # https://github.com/kubernetes/kubernetes/issues/44226 - 'Downward API volume [It] should update labels on modification [Conformance] [Volume]' - 'Downward API volume [It] should update annotations on modification [Conformance] [Volume]' - 'Projected [It] should update labels on modification [Conformance] [Volume]' - 'Secrets [It] should be consumable from pods in volume with mappings [Conformance] [Volume]' -) - -FAILING_TESTS_LIST=( - 'Services [It] should be able to create a functioning NodePort service' - 'Services [It] should serve multiport endpoints from pods [Conformance]' -) - -function escape_test_name() { - sed 's/\[[^]]*\]//g' <<< "$1" | sed "s/[^[:alnum:]]/ /g" | tr -s " " | sed "s/^\s\+//" | sed "s/\s/.*/g" -} - -function test_names () { - local first=y - for name in "${TESTS_LIST_REGEX[@]}"; do - if [ -z "${first}" ]; then - echo -n "|" - else - first= - fi - echo -n "${name}" - done - for name in "${FLAKY_TESTS_LIST[@]}"; do - if [ -z "${first}" ]; then - echo -n "|" - else - first= - fi - echo -n "$(escape_test_name "${name}")" - done - for name in "${FAILING_TESTS_LIST[@]}"; do - if [ -z "${first}" ]; then - echo -n "|" - else - first= - fi - echo -n "$(escape_test_name "${name}")" - done -} - -cd $BASE/new/devstack -source openrc admin admin - -echo "In post_test_hook" - -# Get the latest stable version of kubernetes -export K8S_VERSION=$(curl -sS https://storage.googleapis.com/kubernetes-release/release/stable.txt) -echo "K8S_VERSION : ${K8S_VERSION}" - -echo "Download Kubernetes CLI" -sudo wget -O kubectl "http://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl" -sudo chmod 755 kubectl - -export KUBECONFIG=/var/run/kubernetes/admin.kubeconfig - -echo "Waiting for kubernetes service to start..." -for i in {1..600} -do - if [[ -f $KUBECONFIG ]]; then - running_count=$(./kubectl get nodes --no-headers 2>/dev/null | grep "Ready" | wc -l) - if [ "$running_count" -ge 1 ]; then - break - fi - fi - echo -n "." - sleep 1 -done - -echo "Cluster created!" -echo "" - -sudo journalctl -u devstack@kubernetes.service - -echo "Dump Kubernetes Objects..." -./kubectl get componentstatuses -./kubectl get configmaps -./kubectl get daemonsets -./kubectl get deployments -./kubectl get events -./kubectl get endpoints -./kubectl get horizontalpodautoscalers -./kubectl get ingress -./kubectl get jobs -./kubectl get limitranges -./kubectl get nodes -./kubectl get namespaces -./kubectl get pods -./kubectl get persistentvolumes -./kubectl get persistentvolumeclaims -./kubectl get quota -./kubectl get resourcequotas -./kubectl get replicasets -./kubectl get replicationcontrollers -./kubectl get secrets -./kubectl get serviceaccounts -./kubectl get services - -echo "Clear the taint to make sure we can schedule jobs to the master node" -./kubectl taint nodes --all node.cloudprovider.kubernetes.io/uninitialized- - -./kubectl get node -o json - -echo "Create a default StorageClass since we do not have a cloud provider" -./kubectl create -f - </dev/null -sudo -E PATH=$GOPATH/bin:$PATH make all WHAT=cmd/kubectl -sudo -E PATH=$GOPATH/bin:$PATH make all WHAT=vendor/github.com/onsi/ginkgo/ginkgo - -# open up access for containers -sudo ifconfig -a -export HOST_INTERFACE=$(ip -f inet route | awk '/default/ {print $5}') -sudo iptables -t nat -A POSTROUTING -o $HOST_INTERFACE -s 10.0.0.0/24 -j MASQUERADE -sudo iptables -t nat -A POSTROUTING -o $HOST_INTERFACE -s 172.17.0.0/24 -j MASQUERADE - - -sudo -E PATH=$GOPATH/bin:$PATH make all WHAT=test/e2e/e2e.test -sudo -E PATH=$GOPATH/bin:$PATH go run hack/e2e.go -- -v --test --test_args="--ginkgo.trace=true --ginkgo.seed=1378936983 --logtostderr --v 4 --provider=local --report-dir=/opt/stack/logs/ --ginkgo.v --ginkgo.skip=$(test_names)" -popd >/dev/null diff --git a/contrib/pre_test_hook.sh b/contrib/pre_test_hook.sh deleted file mode 100755 index ce572a9055..0000000000 --- a/contrib/pre_test_hook.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -xe -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This script is executed inside pre_test_hook function in devstack gate. - -export localconf=$BASE/new/devstack/local.conf - -echo "In pre_test_hook" diff --git a/devstack/plugin.sh b/devstack/plugin.sh deleted file mode 100755 index e2ff9310ec..0000000000 --- a/devstack/plugin.sh +++ /dev/null @@ -1,216 +0,0 @@ -#!/bin/bash -# -# lib/dlm -# -# Functions to control the installation and configuration of kubernetes with the -# external OpenStack cloud provider enabled. - -# Save trace setting -_XTRACE_K8S_PROVIDER=$(set +o | grep xtrace) -set -o xtrace - - -BASE_DIR=$(cd $(dirname $BASH_SOURCE)/.. && pwd) - -# Defaults -# -------- - -export GOPATH=${BASE_DIR}/go - -CONFORMANCE_REPO=${CONFORMANCE_REPO:-github.com/kubernetes/kubernetes} -K8S_SRC=${GOPATH}/src/k8s.io/kubernetes - -function install_prereqs { - # Install pre-reqs - $BASE_DIR/tools/install-distro-packages.sh - $BASE_DIR/tools/test-setup.sh - - install_package nfs-common -} - - -function install_docker { - # Install docker if needed - if ! is_package_installed docker-engine; then - sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D || true - sudo apt-key adv --keyserver hkp://pgp.mit.edu:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D || true - sudo apt-add-repository 'deb http://apt.dockerproject.org/repo ubuntu-xenial main' - sudo apt-get update - sudo apt-cache policy docker-engine - sudo apt-get install -y docker-engine=1.12.6-0~ubuntu-xenial - sudo cat /lib/systemd/system/docker.service - sudo sed -r -i "s|(ExecStart)=(.+)|\1=\2 --iptables=false|" /lib/systemd/system/docker.service - sudo cat /lib/systemd/system/docker.service - sudo systemctl daemon-reload - sudo systemctl restart docker - sudo ifconfig -a - fi - docker --version - - # Get the latest stable version of kubernetes - export K8S_VERSION=$(curl -sS https://storage.googleapis.com/kubernetes-release/release/stable.txt) - echo "K8S_VERSION : ${K8S_VERSION}" - - echo "Starting docker service" - sudo systemctl enable docker.service - sudo systemctl start docker.service --ignore-dependencies - echo "Checking docker service" - sudo docker ps -} - -function install_k8s_cloud_provider { - echo_summary "Installing Devstack Plugin for k8s-cloud-provider" - - # golang env details - go env - go version - - GO111MODULE=off go get -u github.com/jteeuwen/go-bindata/go-bindata || true - GO111MODULE=off go get -u github.com/cloudflare/cfssl/cmd/... || true - - # Get Kubernetes from source - mkdir -p ${GOPATH}/src/k8s.io/ - if [ ! -d "${K8S_SRC}" ]; then - git clone https://${CONFORMANCE_REPO} ${K8S_SRC} - pushd ${K8S_SRC} >/dev/null - git remote update - git fetch --all --tags --prune - - popd >/dev/null - fi - - # Run the script that builds kubernetes from source and starts the processes - pushd ${K8S_SRC} >/dev/null - - # local-up-cluster needs the etcd path and the GOPATH bin as cf-ssl - # and go-bindata are located there - export PATH=$DEST/bin:$GOPATH/bin:${PATH} - - # Seed the log files so devstack-gate can capture the logs - export LOG_DIR=${SCREEN_LOGDIR:-/opt/stack/logs} - sudo mkdir -p $LOG_DIR - sudo touch $LOG_DIR/kube-apiserver.log;sudo ln -sf $LOG_DIR/kube-apiserver.log $LOG_DIR/screen-kube-apiserver.log - sudo touch $LOG_DIR/kube-controller-manager.log;sudo ln -sf $LOG_DIR/kube-controller-manager.log $LOG_DIR/screen-kube-controller-manager.log - sudo touch $LOG_DIR/kube-proxy.log;sudo ln -sf $LOG_DIR/kube-proxy.log $LOG_DIR/screen-kube-proxy.log - sudo touch $LOG_DIR/kube-scheduler.log;sudo ln -sf $LOG_DIR/kube-scheduler.log $LOG_DIR/screen-kube-scheduler.log - sudo touch $LOG_DIR/kubelet.log;sudo ln -sf $LOG_DIR/kubelet.log $LOG_DIR/screen-kubelet.log - - echo "Stopping firewall and allow all traffic..." - sudo iptables -F - sudo iptables -X - sudo iptables -t nat -F - sudo iptables -t nat -X - sudo iptables -t mangle -F - sudo iptables -t mangle -X - sudo iptables -P INPUT ACCEPT - sudo iptables -P FORWARD ACCEPT - sudo iptables -P OUTPUT ACCEPT - - # Turn on/off a few things in local-up-cluster.sh - export ALLOW_PRIVILEGED=true - export ALLOW_SECURITY_CONTEXT=true - export CLOUD_PROVIDER=external - export ENABLE_CRI=false - export ENABLE_DAEMON=true - export ENABLE_HOSTPATH_PROVISIONER=true - export ENABLE_SINGLE_CA_SIGNER=true - export HOSTNAME_OVERRIDE=$(ip route get 1.1.1.1 | awk '{print $7}') - export KUBE_ENABLE_CLUSTER_DASHBOARD=true - export KUBE_ENABLE_CLUSTER_DNS=true - export LOG_LEVEL=10 - - pushd ${BASE_DIR} - make build - popd - - export EXTERNAL_CLOUD_PROVIDER_BINARY=${BASE_DIR}/openstack-cloud-controller-manager - export EXTERNAL_CLOUD_PROVIDER=true - export CLOUD_PROVIDER=openstack - export CLOUD_CONFIG=${CLOUD_CONFIG_FILE} - - sudo pwd - if [[ ! -d "${CLOUD_CONFIG_DIR}" ]]; then - sudo mkdir -p ${CLOUD_CONFIG_DIR} - fi - sudo chown $STACK_USER /etc/kubernetes/ - iniset ${CLOUD_CONFIG} Global region RegionOne - iniset ${CLOUD_CONFIG} Global username admin - iniset ${CLOUD_CONFIG} Global password secretadmin - iniset ${CLOUD_CONFIG} Global auth-url "http://localhost/identity" - iniset ${CLOUD_CONFIG} Global tenant-id $(openstack project show admin -f value -c id) - iniset ${CLOUD_CONFIG} Global domain-id default - iniset ${CLOUD_CONFIG} LoadBalancer subnet-id $(openstack subnet show lb-mgmt-subnet -f value -c id) - iniset ${CLOUD_CONFIG} LoadBalancer floating-network-id $(openstack network show public -f value -c id) - iniset ${CLOUD_CONFIG} BlockStorage bs-version v2 - - # Listen on all interfaces - export KUBELET_HOST="0.0.0.0" - - # Use the docker0's ip address for kubedns to work - export API_HOST_IP="172.17.0.1" - - # kill etcd and let local-up-cluster start it up - $SYSTEMCTL stop $ETCD_SYSTEMD_SERVICE - - # local-up-cluster.sh compiles everything from source and starts the services. - sudo -E PATH=$PATH SHELLOPTS=$SHELLOPTS ./hack/local-up-cluster.sh - - popd >/dev/null -} - -# cleanup_k8s_cloud_provider() - Remove residual data files, anything left over from previous -# runs that a clean run would need to clean up -function cleanup_k8s_cloud_provider { - echo_summary "Cleaning up Devstack Plugin for k8s-cloud-provider" - # Kill the k8s processes - ps -ef | grep -e hyperkube | grep -v grep | awk '{print $2}' | xargs sudo kill -9 - - # Cleanup docker images and containers - sudo docker rm -f $(docker ps -a -q) || true - sudo docker rmi -f $(docker images -q -a) || true - - # Stop docker - sudo systemctl stop docker.service - sudo rm -rf "$K8S_SRC" -} - -function stop_k8s_cloud_provider { - echo_summary "Stop Devstack Plugin for k8s-cloud-provider" - stop_process kubernetes -} - -# check for service enabled -if is_service_enabled k8s-cloud-provider; then - - if [[ "$1" == "stack" && "$2" == "pre-install" ]]; then - # no-op - : - - elif [[ "$1" == "stack" && "$2" == "install" ]]; then - install_docker - install_prereqs - - elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then - # no-op - : - elif [[ "$1" == "stack" && "$2" == "extra" ]]; then - install_k8s_cloud_provider - : - fi - - if [[ "$1" == "unstack" ]]; then - stop_k8s_cloud_provider - fi - - if [[ "$1" == "clean" ]]; then - cleanup_k8s_cloud_provider - fi -fi - -# Restore xtrace -$_XTRACE_K8S_PROVIDER - -# Tell emacs to use shell-script-mode -## Local variables: -## mode: shell-script -## End: diff --git a/devstack/settings b/devstack/settings deleted file mode 100644 index 9104ba9f25..0000000000 --- a/devstack/settings +++ /dev/null @@ -1,6 +0,0 @@ -# settings file for devstack plugin - -enable_service k8s-cloud-provider -enable_service kubernetes -CLOUD_CONFIG_DIR=${CLOUD_CONFIG:-/etc/kubernetes} -CLOUD_CONFIG_FILE=${CLOUD_CONFIG_DIR}/cloud-config diff --git a/docs/developers-csi-manila.md b/docs/developers-csi-manila.md deleted file mode 100644 index 4092ac63e7..0000000000 --- a/docs/developers-csi-manila.md +++ /dev/null @@ -1,107 +0,0 @@ - - -**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* - -- [CSI Manila developer's guide](#csi-manila-developers-guide) - - [Running CSI Sanity tests](#running-csi-sanity-tests) - - [Share adapters](#share-adapters) - - [Adding support for more share protocols](#adding-support-for-more-share-protocols) - - [Passing volume options to share adapters](#passing-volume-options-to-share-adapters) - - [Service capabilities](#service-capabilities) - - [Notes on design...](#notes-on-design) - - - -# CSI Manila developer's guide - -## Running CSI Sanity tests - -Sanity tests create a real instance of driver with fake Manila client and CSI forwarding node plugin. -See [Sanity check](https://github.com/kubernetes-csi/csi-test/tree/master/pkg/sanity) for more info. - -Run the test suite with `make test-manila-csi-sanity`. - -## Share adapters - -A share adapter is an interface found here `pkg/csi/manila/shareadapters/shareadapter.go` that forms an adapter between a Manila share and a CSI plugin. - -### Adding support for more share protocols - -As of writing this document, CSI Manila supports only NFS and CephFS shares. If you'd like to expand on this set and contribute with a new adapter for a share protocol, keep reading! - -1. Create a new file `some-protocol.go` under `pkg/csi/manila/shareadapters` -2. Create a new struct that implements the `ShareAdapter` interface -3. Add a case block in `getShareAdapter()` function in `pkg/csi/manila/adapters.go`. The condition string must match one of Manila's supported share protocols. -4. Add the protocol name to the `matches` expression (regexp syntax) inside `ControllerVolumeContext.Protocol` field tags in `pkg/csi/manila/options/shareoptions.go`. Again, it must match one of Manila's supported share protocols. -5. Update the docs in `docs/using-manila-csi-plugin.md`, namely any parameters that the protocol or node plugin may use. There's also a dedicated section "Share protocols support matrix" at the bottom of the document which needs to be updated: name of the share protocol, link to the proxy'd CSI driver and its supported version(s). - -### Passing volume options to share adapters - -Usually, shares / share adapters offer a set of options which users may want to configure. Those options can be specified in `pkg/csi/manila/options/shareoptions.go`, in either `ControllerVolumeContext` or `NodeVolumeContext` structs, or both. Each struct field must contain `name` field tag which is then used for parsing input values. It's highly recommended that, if necessary, you use validator tags instead of hard-coding validation checks in share adapters. See `pkg/share/manila/shareoptions/validator/validator.go` for info on supported validator tags. - -## Service capabilities - -**Controller Service:** -* `CREATE_DELETE_VOLUME` -* `CREATE_DELETE_SNAPSHOT` (snapshotting CephFS shares is not supported yet - planned as a part of GSoC 2019) - -Availability Zones are not supported yet. - -**Node Service:** - -Node Service capabilities of the proxy'd Node Plugin - -## Notes on design... - -**Problem 1:** - -OpenStack Manila supports NFS, CIFS, GLUSTERFS, HDFS, CEPHFS and MAPRFS share protocols. Implementing support for all of those backends within a single CSI driver doesn't scale very well because: -* a dedicated CSI driver for each of those file-systems already exists - or eventually will -* not reusing those existing drivers means more maintenance for devs, possible fragmented/missing features between drivers -* mounting the file-systems mentioned above requires tools that are not usually present on the host system and they'd need to be built into the container image - -**Solution:** - -CSI Manila's Controller Plugin deals with Manila and nothing else. All node-related operations (attachments, mounts) are carried out by *another CSI driver* dedicated for that particular filesystem, effectively offloading all FS-specific code out of CSI Manila. This is achieved by Node Plugin acting only as a proxy and forwarding CSI RPCs to the other CSI driver. - -For example, creating and mounting a CephFS Manila share would go like this: - -(1) CO requests a volume provisioning with `CreateVolume`: - -``` - +--------------------+ - | MASTER | -+------+ CreateVolume | +--------------+ | -| CO +------------------>+ csi-manila | | -+------+ | +--------------+ | - +--------------------+ - -1. Authenticate with Manila v2 client -2. Create a CephFS share -3. Create a cephx access rule -``` - -(2) And when CO publishes the volume onto a node with `NodePublishVolume`: -``` - +--------------------------------------+ - | NODE | -+------+ NodePublishVolume | +--------------+ | -| CO +----------------------->+ csi-manila | | -+------+ | +------+-------+ | - | | | - | | FORWARD NodePublishVolume | - | V | - | +------+-------+ | - | | csi-cephfs | | - | +--------------+ | - +--------------------------------------+ - -1. Authenticate with Manila v2 client -2. Retrieve the CephFS share -3. Retrieve the cephx access rule -4. Connect to csi-cephfs socket -5. Call csi-cephfs's NodePublishVolume, return its response -``` - -The initial idea was to encompass all Manila share protocols within a single instance of csi-manila. Due to limitations of CSI, this cannot be achieved and there have to be multiple instances of csi-manila running in order to serve multiple share protocols, one for each. There are couple of RPCs in Node Service that don't bring enough context to decide which proxy'd driver that particular RPC should be forwarded to (e.g. `NodeGetCapabilities` would be ambiguous and we can't know which proxy'd driver should answer) => therefore with current design, there may be only a single proxy'd driver per csi-manila instance, i.e. one share protocol per csi-manila instance. - diff --git a/docs/expose-applications-using-loadbalancer-type-service.md b/docs/expose-applications-using-loadbalancer-type-service.md deleted file mode 100644 index 9e1eb29abd..0000000000 --- a/docs/expose-applications-using-loadbalancer-type-service.md +++ /dev/null @@ -1,269 +0,0 @@ - - -**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* - -- [Exposing applications using services of LoadBalancer type](#exposing-applications-using-services-of-loadbalancer-type) - - [Creating a Service of LoadBalancer type](#creating-a-service-of-loadbalancer-type) - - [Supported Features](#supported-features) - - [Service annotations](#service-annotations) - - [Switching between Floating Subnets by using preconfigured Classes](#switching-between-floating-subnets-by-using-preconfigured-classes) - - [Creating Service by specifying a floating IP](#creating-service-by-specifying-a-floating-ip) - - [Restrict Access For LoadBalancer Service](#restrict-access-for-loadbalancer-service) - - [Issues](#issues) - - - -# Exposing applications using services of LoadBalancer type - -This page shows how to create Services of LoadBalancer type in Kubernetes cluster which is running inside OpenStack. For an explanation of the Service concept and a discussion of the various types of Services, see [Services](https://kubernetes.io/docs/concepts/services-networking/service/). - -A LoadBalancer type Service is a typical way to expose an application to the internet. It relies on the cloud provider to create an external load balancer with an IP address in the relevant network space. Any traffic that is then directed to this IP address is forwarded on to the application’s service. - -> Note: Different cloud providers may support different Service annotations and features. - -## Creating a Service of LoadBalancer type - -Create an application of Deployment as the Service backend: - -```shell -kubectl run echoserver --image=gcr.io/google-containers/echoserver:1.10 --port=8080 -``` - -To provide the echoserver application with an internet-facing loadbalancer we can simply run the following: - -```shell -cat < NOTE: If 122.112.219.229 is not available, a new floating IP will be created automatically from the configured public network. If 122.112.219.229 is already associated with another port, the Service creation will fail. - -```yaml -apiVersion: v1 -kind: Service -metadata: - name: nginx-internet -spec: - type: LoadBalancer - selector: - app: nginx - ports: - - port: 80 - targetPort: 80 - loadBalancerIP: 122.112.219.229 -``` - -### Restrict Access For LoadBalancer Service - -When using a Service with `spec.type: LoadBalancer`, you can specify the IP ranges that are allowed to access the load balancer by using `spec.loadBalancerSourceRanges`. This field takes a list of IP CIDR ranges, which Kubernetes will use to configure firewall exceptions. - -This feature is only supported in the OpenStack Cloud with Octavia(API version >= v2.12) service deployed, otherwise `loadBalancerSourceRanges` is ignored. - -In the following example, a load balancer will be created that is only accessible to clients with IP addresses in 192.168.32.1/24. - -```yaml -apiVersion: v1 -kind: Service -metadata: - name: test - namespace: default -spec: - type: LoadBalancer - loadBalancerSourceRanges: - - 192.168.32.1/24 - selector: - run: echoserver - ports: - - protocol: TCP - port: 80 - targetPort: 8080 -``` - -`loadBalancerSourceRanges` field supports to be updated. - -## Issues - -- `spec.externalTrafficPolicy` is not supported. diff --git a/docs/getting-started-provider-dev.md b/docs/getting-started-provider-dev.md deleted file mode 100644 index 360b15d1a2..0000000000 --- a/docs/getting-started-provider-dev.md +++ /dev/null @@ -1,321 +0,0 @@ - - -**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* - -- [Getting Started with Cloud Provider OpenStack Development](#getting-started-with-cloud-provider-openstack-development) - - [Prerequisites](#prerequisites) - - [OpenStack Cloud](#openstack-cloud) - - [Docker](#docker) - - [Development tools](#development-tools) - - [Development](#development) - - [Getting and Building Cloud Provider OpenStack](#getting-and-building-cloud-provider-openstack) - - [Building inside container](#building-inside-container) - - [Getting and Building Kubernetes](#getting-and-building-kubernetes) - - [Running the Cloud Provider](#running-the-cloud-provider) - - [Publish Container images](#publish-container-images) - - [Build and push images](#build-and-push-images) - - [Only Build Image](#only-build-image) - - [Troubleshooting](#troubleshooting) - - - -# Getting Started with Cloud Provider OpenStack Development - -This guide will help you get started with building a development environment for you -to build and run a single node Kubernetes cluster with the OpenStack Cloud Provider -enabled. - -## Prerequisites - -To get started, you will need to set up your development environment. - -### OpenStack Cloud -You will need access to an OpenStack cloud, either public or private. You can sign up -for a public OpenStack Cloud through the [OpenStack Passport](https://www.openstack.org/passport) -program, or you can install a small private development cloud with -[DevStack](https://docs.openstack.org/devstack/latest/). - -Once you have obtained access to an OpenStack cloud, you will need to start a development -VM. The rest of this guide assumes a CentOS 7 cloud image, but should be easily transferrable -to whatever development environment you prefer. You will need to have your cloud credentials -loaded into your environment. For example, I use this `openrc` file: - -``` -export OS_PROJECT_DOMAIN_NAME=Default -export OS_USER_DOMAIN_NAME=Default -export OS_DOMAIN_ID= -export OS_PROJECT_NAME= -export OS_TENANT_NAME= -export OS_TENANT_ID= -export OS_USERNAME= -export OS_PASSWORD= -export OS_AUTH_URL=http:///v3 -export OS_INTERFACE=public -export OS_IDENTITY_API_VERSION=3 -export OS_REGION_NAME= -``` - -The specific values you use will vary based on your particular environment. You may -notice that several values are aliases of one another. This is in part because the -values expected by the OpenStack client and -[Gopher Cloud](https://github.com/gophercloud/gophercloud) are slightly different, -especially with respect to the change from using `tenant` to `project`. One of our -development goals is to make this setup easier and more consistent. - -### Docker - -Your cloud instance will need to have Docker installed. To install, please refer to [Docker Container runtime](https://kubernetes.io/docs/setup/production-environment/container-runtimes/#docker). - -### Development tools - -You're going to need a few basic development tools and applications to get, build, and run -the source code. With your package manager you can install `git`, `gcc` and `etcd`. - -``` -sudo yum install -y -q git gcc etcd - -``` - -You will also need a recent version of Go and set your environment variables. - -``` -GO_VERSION=1.13.4 -GO_ARCH=linux-amd64 -curl -o go.tgz https://dl.google.com/go/go${GO_VERSION}.${GO_ARCH}.tar.gz -sudo tar -C /usr/local/ -xvzf go.tgz -export GOROOT=/usr/local/go -export GOPATH=$HOME/go -``` - -Finally, set up your Git identity and GitHub integrations. - -More comprehensive setup instructions are available in the -[Development Guide](https://github.com/kubernetes/community/blob/master/contributors/devel/development.md#building-kubernetes-on-a-local-osshell-environment) -in the Kubernetes repository. When in doubt, check there for additional setup -and versioning information. - -## Development - -### Getting and Building Cloud Provider OpenStack - -Following the [GitHub Workflow](https://github.com/kubernetes/community/blob/master/contributors/guide/github-workflow.md) guidelines for Kubernetes development, set up your environment and get the latest development repository. Begin by forking both the Kubernetes and Cloud-Provider-OpenStack projects into your GitHub into your local workspace (or bringing your current fork up to date with the current state of both repositories). - -`make` will build, test, and package this project. This project uses [go modules](https://github.com/golang/go/wiki/Modules) for dependency management since v1.17. - -Set up some environment variables to help download the repositories - -``` -export GOPATH=$HOME/go -export GOROOT=/usr/local/go -export PATH=$PATH:$GOROOT/bin -export user={your github profile name} -export working_dir=$GOPATH/src/k8s.io -``` - -With your environment variables set up, clone the forks into your go environment. - -``` -mkdir -p $working_dir -cd $working_dir -git clone https://github.com/{user}/cloud-provider-openstack -cd cloud-provider-openstack -``` - -If you want to build the plugins for single architecture: - -``` -# ARCH default to amd64 -export ARCH=arm64 -make build -``` - -And if you want to build the plugins for multiple architectures: - -``` -# ARCHS default to 'amd64 arm arm64 ppc64le s390x' -export ARCHS='amd64 arm arm64 ppc64le s390x' -make build-all-archs -``` - -If you want to run unit tests: - -``` -make test -``` - -#### Building inside container - -If you don't have a Go Environment setup, we also offer the ability to run make -in a Docker Container. The only requirement for this is that you have Docker -installed and configured (of course). You don't need to have a Golang -environment setup, and you don't need to follow rules in terms of directory -structure for the code checkout. - -To use this method, just call the `hack/make.sh` script with the desired argument: - `hack/make.sh build` for example will run `make build` in a container. - -> NOTE: You MUST run the script from the root source directory as shown above, -attempting to do something like `cd hack && make.sh build` will not work -because we won't bind mount the source files into the container. - -### Getting and Building Kubernetes - -To get and build Kubernetes - -``` -cd $working_dir -export KUBE_FASTBUILD=true -git clone https://github.com/{user}/kubernetes -cd kubernetes -make cross -``` - -### Running the Cloud Provider - -To run the OpenStack provider, integrated with your cloud, be sure to have sourced the -environment variables. You will also need to create an `/etc/kubernetes/cloud-config` file -with the minimum options: - -``` -[Global] -username= -password= -auth-url=http:///v3 -tenant-id= -domain-id= -``` - -Start your cluster with the `hack/local-up-cluster.sh` with the proper environment variable set to -enable the external cloud provider: - -``` -export EXTERNAL_CLOUD_PROVIDER_BINARY=$GOPATH/src/k8s.io/cloud-provider-openstack/openstack-cloud-controller-manager -export EXTERNAL_CLOUD_PROVIDER=true -export CLOUD_PROVIDER=openstack -export CLOUD_CONFIG=/etc/kubernetes/cloud-config -./hack/local-up-cluster.sh -``` - -After giving the cluster time to build and start, you can access it through the directions -provided by the script: - -``` -export KUBECONFIG=/var/run/kubernetes/admin.kubeconfig -./cluster/kubectl.sh -``` - -The cluster/addons/rbac has a set of yaml files, currently it has -cloud-controller-manager-role-bindings.yaml -cloud-controller-manager-roles.yaml - -you need use following command to create ClusterRole and ClusterRoleBinding -otherwise the cloud-controller-manager is not able to access k8s API. -``` -./cluster/kubectl.sh create -f $working_dir/cloud-provider-openstack/cluster/addons/rbac/ -``` - -Have a good time with OpenStack and Kubernetes! - -## Publish Container images - -We allow build and publish container images for multiple architectures. - -### Build and push images - -To build and push all supported images for all architecture. - -``` -# default value of VERSION is "{part of latest commit id}-dirty" -export VERSION=latest -# default registry is "k8scloudprovider" -export REGISTRY=k8scloudprovider -# No need to provide "DOCKER_PASSWORD" and "DOCKER_USERNAME" if environment already logged in. -export DOCKER_PASSWORD=password -export DOCKER_USERNAME=username - -# default value for IMAGE_NAMES includes all supported images. -export IMAGE_NAMES='openstack-cloud-controller-manager cinder-flex-volume-driver cinder-provisioner cinder-csi-plugin k8s-keystone-auth octavia-ingress-controller manila-provisioner manila-csi-plugin barbican-kms-plugin magnum-auto-healer' -# default value for ARCHS includes all supported architectures. -export ARCHS='amd64 arm arm64 ppc64le s390x' - -make upload-images -``` - -Here's example to explain `make upload-images` -`upload-images` will first build images with architectures. -For example with `REGISTRY=k8scloudprovider`, `VERSION=latest`, `IMAGE_NAMES=openstack-cloud-controller-manager`, -and `ARCHS=arm64 amd64` will build following two images -- `k8scloudprovider/openstack-cloud-controller-manager-arm64:latest` -- `k8scloudprovider/openstack-cloud-controller-manager-amd64:latest` - -Second, it will than upload above two images. -And finally, it will build image manifest to upload image `k8scloudprovider/openstack-cloud-controller-manager:latest` (with no architecture in image name). - -Image `k8scloudprovider/openstack-cloud-controller-manager:latest` will support two different `OS/ARCH` tag: `linux/arm64` and `linux/amd64`. - -From this point all images are ready in image repositories. -To start using image with specific architecture. You can use `--platform` or without if you plan to use default platform: - -``` -docker pull --platform=linux/amd64 k8scloudprovider/openstack-cloud-controller-manager:latest -``` - -Or you can use image with architecture tagged in name: - -``` -docker pull k8scloudprovider/openstack-cloud-controller-manager-amd64:latest -``` - -Note that since image `k8scloudprovider/openstack-cloud-controller-manager:latest` is build by manifest, -the image digest for image pull from `k8scloudprovider/openstack-cloud-controller-manager:latest` will be different to -image you pull from `k8scloudprovider/openstack-cloud-controller-manager-amd64:latest`. - -### Only Build Image - -It's generally more recommended to use `make upload-images` to build and push image. -But you might need to build image and run test on it if you're running a test job to verify that image. - -To prepare default environment variables: - -``` -# default value of VERSION is "{part of latest commit id}-dirty" -export VERSION=latest - -# default value for IMAGE_NAMES includes all supported images. -export IMAGE_NAMES='openstack-cloud-controller-manager magnum-auto-healer' -# default value for ARCHS includes all supported architectures. -``` - -If you not plan to build image, you can directly run `make images-all-archs` to support build for all specified images with all specified architectures. - -``` -export ARCHS='amd64 arm arm64 ppc64le s390x' -make images-all-archs -``` - -If you plan to build image for single architecture, you can either specify `ARCHS` as `amd64` or run below commands - -``` -export ARCH=amd64 -make build-images -``` - -Or if you wish to ony build for specific architecture with specific image - -``` -export ARCH=amd64 -make image-openstack-cloud-controller-manager -``` -In case of building images, by default, images built are appended with this -$ARCH tag at the end. -For example, for the above case, openstack-cloud-controller-manager-amd64 is built. -In case needed without it, tag it separately as required. - -``` -docker image tag openstack-cloud-controller-manager-amd64:latest openstack-cloud-controller-manager:latest -``` - -This will create image with tag `openstack-cloud-controller-manager:latest` in your local environment. - -## Troubleshooting - -You can increase a log level verbosity (`-v` parameter) to know better what is happening during the OpenStack Cloud Controller Manager runtime. Setting the log level to **6** allows you to see OpenStack API JSON requests and responses. It is recommended to set a `--concurrent-service-syncs` parameter to **1** for a better output tracking. diff --git a/docs/migrate-to-ccm-with-csimigration.md b/docs/migrate-to-ccm-with-csimigration.md deleted file mode 100644 index e3108ec66f..0000000000 --- a/docs/migrate-to-ccm-with-csimigration.md +++ /dev/null @@ -1,212 +0,0 @@ - - -**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* - -- [Migrate from in-tree cloud provider to openstack-cloud-controller-manager and enable CSIMigration](#migrate-from-in-tree-cloud-provider-to-openstack-cloud-controller-manager-and-enable-csimigration) - - [Before you begin](#before-you-begin) - - [Migrate to openstack-cloud-controller-manager](#migrate-to-openstack-cloud-controller-manager) - - [Enable CSIMigration](#enable-csimigration) - - [Finalize the migration](#finalize-the-migration) - - [Caveats](#caveats) - - - -# Migrate from in-tree cloud provider to openstack-cloud-controller-manager and enable CSIMigration - -This guide walks you through the process of migrating from using the Kubernetes in-tree cloud provider (specified by using `--cloud-provider=openstack` on `kube-controller-manager`) to use the `cloud-controller-manager` (CCM) for OpenStack. This document tries to give an example for using multiple steps in order to get the migration fully done. We expect users to want to migrate to `cloud-provider-openstack` but stay with the in-tree `cinder-volume-provisioner` up until `CSIMigration` has become GA. - - -A little bit of background on CSI and `CSIMigration`. These days, storage providers should implement the [Container Storage Interface](https://github.com/container-storage-interface/spec/blob/master/spec.md) to provide storage for Kubernetes clusters (but not limited to Kubernetes). `cloud-provider-openstack` also provides [Cinder CSI Plugin](https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/using-cinder-csi-plugin.md) to serve exact this purpose. This plugin can be installed and used alongside in-tree `cloud-provider`, because it requires its own `StorageClass`. What this means is: All volumes, created by the in-tree volume APIs will be handled by the old in-tree-provisioner one. Everything provisioned by Cinder CSI Plugin, will be handled by `cinder-csi-plugin`. - -Sometimes it can be hard to migrate an entire cluster to use the new `StorageClass`. This is where `CSIMigration` comes into play (see [CSIMigration design proposal](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/storage/csi-migration.md)). With this enabled, calls to the in-tree volume APIs will call out to the CSI plugins. See [In-tree Storage Plugin to CSI Migration Design Doc](https://github.com/kubernetes/enhancements/blob/master/keps/sig-storage/20190129-csi-migration.md) for more detail. - -## Before you begin - -Note: This guide works on a cluster created by `kubeadm`. - -Example `kubeadm.yaml`: - -```yaml -apiVersion: kubeadm.k8s.io/v1beta2 -kind: ClusterConfiguration -kubernetesVersion: 1.17.0-beta.1 -networking: - podSubnet: 10.96.0.0/16 - serviceSubnet: 10.97.0.0/16 -controllerManager: - extraArgs: - cloud-provider: openstack - cloud-config: /etc/kubernetes/cloud.conf - extraVolumes: - - name: cloud - hostPath: /etc/kubernetes/cloud.conf - mountPath: /etc/kubernetes/cloud.conf - readOnly: true ---- -apiVersion: kubeadm.k8s.io/v1beta2 -kind: InitConfiguration -localAPIEndpoint: - bindPort: 6443 -nodeRegistration: - kubeletExtraArgs: - cloud-provider: openstack - cloud-config: /etc/kubernetes/cloud.conf -``` - -## Migrate to openstack-cloud-controller-manager - -Remember... In the first step, we want to keep the in-tree volume API, but for all of the rest, we want to use `openstack-cloud-controller-manager`. So we can't just remove the `--cloud-provider` flag from `kube-controller-manager`. In fact, we must disable all cloud related controllers. -Create the file `kubeadm-disable-controllers.yaml`: - -```yaml -apiVersion: kubeadm.k8s.io/v1beta2 -kind: ClusterConfiguration -kubernetesVersion: 1.17.0-beta.1 -networking: - podSubnet: 10.96.0.0/16 - serviceSubnet: 10.97.0.0/16 -controllerManager: - extraArgs: - cloud-provider: openstack - cloud-config: /etc/kubernetes/cloud.conf - controllers: '*,bootstrapsigner,tokencleaner,-cloud-node-lifecycle,-route,-service' - extraVolumes: - - name: cloud - hostPath: /etc/kubernetes/cloud.conf - mountPath: /etc/kubernetes/cloud.conf - readOnly: true -``` - -And create new manifests for `kube-controller-manager`: - -```bash -kubeadm init --config kubeadm-disable-controllers.yaml phase control-plane controller-manager -``` - -To verify, check the logs of `kube-controller-manager` for the following lines. - -``` -W1117 19:59:34.094182 1 controllermanager.go:513] "service" is disabled -W1117 19:59:34.094187 1 controllermanager.go:513] "route" is disabled -W1117 19:59:44.988353 1 controllermanager.go:513] "cloud-node-lifecycle" is disabled -``` - -You can now deploy `openstack-cloud-controller-manager` using [RBAC](https://github.com/kubernetes/cloud-provider-openstack/tree/master/cluster/addons/rbac) and the [CCM manifest](https://github.com/kubernetes/cloud-provider-openstack/blob/master/manifests/controller-manager/openstack-cloud-controller-manager-ds.yaml). But disable the controller `cloud-node` by adding the argument `--controllers=*,-cloud-node`. This will be done by the `kubelets` up until the entire migration is done. Note: You must create a secret `cloud-config` with a valid `cloud.conf` for the CCM. E.g. - -```bash -kubectl -n kube-system create secret generic cloud-config --from-file=cloud.conf=/etc/kubernetes/cloud.conf -``` - -Note: Because you still want to use the in-tree volume API, you **must** keep the arguments on the `kubelets` untouched (meaning, stick with `--cloud-provider=openstack` and `--cloud-config=/etc/kubernetes/cloud.conf`) for now. - -At this point, the cluster should work as before. LoadBalancers can be created and `StatefulSets` create and use persistent volumes. Note: The logs of `openstack-cloud-controller-manager` in namespace `kube-system` reveal, that this now takes care of creating LoadBalancers. All further management of existing LoadBalancers is taken over by `openstack-cloud-controller-manager` as well. There is no need to recreate any LoadBalancer. - -At this point, you should also deploy [Cinder CSI Plugin](./using-cinder-csi-plugin.md). As mentioned above, this can be deployed alogside the in-tree provider. We would recommend to create a separate `StorageClass` for Cinder CSI Plugin, and make it the default. (see https://github.com/kubernetes/cloud-provider-openstack/tree/master/examples/cinder-csi-plugin for examples) - -## Enable CSIMigration - -This walks you through enabling needed `feature-gates` and explains necessary steps. - -The first step is to ebale the `feature-gates` on the control plane. We again use `kubeadm.yaml` and `kubeadm` to perform necessary tasks. - -```yaml -apiVersion: kubeadm.k8s.io/v1beta2 -kind: ClusterConfiguration -kubernetesVersion: "1.17.0-beta.1" -networking: - podSubnet: 10.96.0.0/16 - serviceSubnet: 10.97.0.0/16 -apiServer: - extraArgs: - feature-gates: "CSIMigration=true,CSIMigrationOpenStack=true,ExpandCSIVolumes=true" -controllerManager: - extraArgs: - feature-gates: "CSIMigration=true,CSIMigrationOpenStack=true,ExpandCSIVolumes=true" - controllers: "*,bootstrapsigner,tokencleaner,-cloud-node-lifecycle,-route,-service" - cloud-config: /etc/kubernetes/cloud.conf - cloud-provider: openstack - extraVolumes: - - name: cloud - hostPath: /etc/kubernetes/cloud.conf - mountPath: /etc/kubernetes/cloud.conf - readOnly: true -``` - -In this config, we still leave the `cloud-provider` and `cloud-config` arguments untouched, but enable the needed feature-gates. - -``` -# enable feature-gates for CSIMigration on apiserver and controller-manager -kubeadm init --config ~ubuntu/kubeadm-migrate-1.yaml phase control-plane apiserver -kubeadm init --config ~ubuntu/kubeadm-migrate-1.yaml phase control-plane controller-manager -``` - -You must now drain an existing node and enable the `feature-gates` on `kubelet`. (At this point, you could also change to `--cloud-provider=external` and remove the `--cloud-config` argument) - -```bash -cat >> /var/lib/kubelet/config.yaml< - -**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* - -- [Authentication synchronization between Keystone and Kubernetes](#authentication-synchronization-between-keystone-and-kubernetes) - - [Overview](#overview) - - [Configuration](#configuration) - - [Example of sync config file](#example-of-sync-config-file) - - [Full example using Keystone for Authentication and Kubernetes RBAC for Authorization](#full-example-using-keystone-for-authentication-and-kubernetes-rbac-for-authorization) - - - -# Authentication synchronization between Keystone and Kubernetes - -## Overview - -Authentication synchronization is an experimental feature allowing to create Kubernetes auth related entities during user authentication. - -As an example, you can take the synchronization of Keystone projects with Kubernetes namespaces. If the user belongs to the project in Keystone, then, when attempting to authenticate in Kubernetes using [k8s-keystone-auth](./using-keystone-webhook-authenticator-and-authorizer.md) webhook for the first time, a Kubernetes namespace will automatically be created for the user. - -Another typical use case is, the Kubernetes cluster admin wants to config RBAC based on Keystone roles, e.g. all users with *member* role should have access to resources of *default* namespace, see the configuration example below. - -## Configuration - -To enable the feature two new arguments were added to the k8s-keystone-auth binary: - -- `--sync-config-file` - points to a local config file. -- `--sync-configmap-name` - defines a ConfigMap object containing the configuration. The ConfigMap must have `syncConfig` key containing config data. - -Sync config file takes precedence over the ConfigMap, but the sync config definition will be refreshed based on the ConfigMap change on-the-fly. It is possible that both are not provided, in this case, the keystone webhook will not synchronize data. - -The config options: - -* **role-mappings** - - A list of role mappings that apply to the user identity after authentication, works with Keystone authentication webhook. This option could be used alone without all others. This allows the cluster admin to config RBAC based on Keystone roles, which is more Kubernetes-native than using the policy definition in the Keystone authorization webhook. The supported keys are: keystone-role, username, groups. See a full example below. - -* **data-types-to-sync** - - Defines a list of available data types, that the webhook will synchronize. Default: [] - - Available options are *projects* and *role_assignments*. In the first case the webhook will convert Keystone projects into Kubernetes namespaces. In the second, the webhook creates Kubernetes RBAC *rolebindings* based on the Keystone role names. - - To correctly create the *rolebindings*, the cluster admin should create the *clusterrole* (same with Keystone role name) first. For example, if the user *alice* has a role assignment *member* in Keystone, when *alice* accesses the cluster, before Kubernetes actually does authorization, the webhook should create a new *rolebinding* in the new namespace with the *clusterrole* name *member* and the user *alice*. As a result, the user *alice* should have some pre-defined resource permissions even it's the first time to access the cluster. - -* **projects-blacklist** - - Contains a list of Keystone project ids, that should be excluded from synchronization. Default: [] - -* **projects-name-blacklist** - - Contains a list of Keystone project names, that should be excluded from synchronization. Default: [] - -* **namespace-format** - - Defines a string with a format of namespace name after synchronization allowing to create more mnemonic names. The string may contain wildcards ``%d``, ``%n`` and ``%i`` representing keystone domain id, project name and project id respectively. Default: "%i" - - For example if the format string was set to ``prefix-%d-%n-%i-suffix``, then syncer will create a namespace as ``prefix-default-my_proj-2f240589c9e44a59836892bfa5abd698-suffix``. - - By convention the namespace name must consist of alphanumeric characters, '-', '_' or '.', and must start and end with an alphanumeric character. - - Due to the restriction of Kubernetes to the length of the name in 63 characters, if the generated string is longer than the specified limit, then only the keystone id will be used as the namespace name. - - The string must contain ``%i`` wildcard. If this is absent the webhook won't start. - -## Example of sync config file - -Here is an example of sync configuration *configmap*: - -```yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: keystone-sync-policy - namespace: kube-system -data: - syncConfig: | - data_types_to_sync: - - projects - - role_assignments - namespace_format: "%i" - role-mappings: - - keystone-role: member - username: myuser - groups: ["mytest"] -``` - -## Full example using Keystone for Authentication and Kubernetes RBAC for Authorization - -* Make sure you have deployed k8s-keystone-auth webhook server by following [k8s-keystone-auth installation guide](./using-keystone-webhook-authenticator-and-authorizer.md). However, we are going to use Kubernetes RBAC for authorization, so remove the `--authorization-webhook-config-file` option for *kube-apiserver* service and make sure `--authorization-mode=Node,RBAC`. Restart *kube-apiserver* as needed. - -* Assuming your k8s-keystone-auth webhook server is running in *kube-system* namespace. As the cluster admin, create a *configmap* that assign the group *project-1-group* to the user who has been assigned *member* role in Keystone: - - ``` - cat < NOTE: After --sync-configmap-name configured, the configmap keystone-sync-policy could be changed on-the-fly, k8s-keystone-auth is able to pick up the change automatically. - -* As the cluster admin, create RBAC *roles* and *rolebindings*, e.g. we want to allow users in the group *project-1-group* to access *pods* in the namespace *project-1*. - - ``` - kubectl create ns project-1 - kubectl -n project-1 create role pod-reader --verb=get,list --resource=pods - kubectl -n project-1 create rolebinding pod-reader --role=pod-reader --group=project-1-group - ``` - -* Let's summarize what we have configured so far, according to the configuration, when the Keystone user *alice* with role *member* trying to access the Kubernetes cluster, she could list the *pods* in the *project-1* namespace, but nothing else. - - ```shell - $ # Switch to OpenStack user. - $ kubectl get po - Error from server (Forbidden): pods is forbidden: User "alice" cannot list resource "pods" in API group "" in the namespace "default" - $ kubectl get ns - Error from server (Forbidden): namespaces is forbidden: User "alice" cannot list resource "namespaces" in API group "" at the cluster scope - $ kubectl -n project-1 get pod - No resources found. - $ kubectl -n project-1 get deployment - Error from server (Forbidden): deployments.extensions is forbidden: User "alice" cannot list resource "deployments" in API group "extensions" in the namespace "project-1" - ``` diff --git a/docs/using-barbican-kms-plugin.md b/docs/using-barbican-kms-plugin.md deleted file mode 100644 index 83c87c502b..0000000000 --- a/docs/using-barbican-kms-plugin.md +++ /dev/null @@ -1,109 +0,0 @@ - - -**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* - -- [OpenStack Barbican KMS Plugin](#openstack-barbican-kms-plugin) - - [Installation Steps:](#installation-steps) - - [Verify](#verify) - - - -# OpenStack Barbican KMS Plugin -Kubernetes supports to encrypt etcd data with various providers listed [here](https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/#providers), one of which is *kms*. The Kubernetes *kms provider* uses envelope encryption scheme. The data is encrypted using *DEK's* by kubernetes *kms provider*, *DEK's* are encrypted by *kms plugin* (e.g. barbican) using *KEK*. *Barbican-kms-plugin* uses *key* from barbican to encrypt/decrypt the *DEK's* as requested by kubernetes api server. -The *KMS provider* uses gRPC to communicate with a specific *KMS plugin*. - -It is recommended to read following kubernetes documents - -* [Encrypting Secret Data at Rest](https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/#verifying-that-data-is-encrypted) -* [Using a KMS provider for data encryption](https://kubernetes.io/docs/tasks/administer-cluster/kms-provider/) - -## Installation Steps: -The following installation steps assumes that you have a Kubernetes cluster(v1.10+) running on OpenStack Cloud. - -1. Create 256bit(32 byte) cbc key and store in barbican -``` -$ openstack secret order create --name k8s_key --algorithm aes --mode cbc --bit-length 256 --payload-content-type=application/octet-stream key -+----------------+-----------------------------------------------------------------------+ -| Field | Value | -+----------------+-----------------------------------------------------------------------+ -| Order href | http://hostname:9311/v1/orders/e477a578-4a46-4c3f-b071-79e220207b0e | -| Type | Key | -| Container href | N/A | -| Secret href | None | -| Created | None | -| Status | None | -| Error code | None | -| Error message | None | -+----------------+----------------------------------------------------------------------+ -``` - -2. Get the Key Id, It is the uuid in *Secret href* -``` -$ openstack secret order get http://hostname:9311/v1/orders/e477a578-4a46-4c3f-b071-79e220207b0e - +----------------+-----------------------------------------------------------------------+ -| Field | Value | -+----------------+-----------------------------------------------------------------------+ -| Order href | http://hostname:9311/v1/orders/e477a578-4a46-4c3f-b071-79e220207b0e | -| Type | Key | -| Container href | N/A | -| Secret href | http://hostname:9311/v1/secrets/b5309dfb-b326-4148-b0ad-e9cd1ec223a8 | -| Created | 2018-10-10T06:29:56+00:00 | -| Status | ACTIVE | -| Error code | None | -| Error message | None | -+----------------+-----------------------------------------------------------------------+ -``` - -3. Add the key-id in your cloud-config file -``` -[Global] -username = -password = -domain-name = -auth-url = -tenant-id = -region = - -[KeyManager] -key-id = -``` - -4. Clone the cloud-provider-openstack repo and build the docker image for barbican-kms-plugin in architecture amd64 -``` -$ git clone https://github.com/kubernetes/cloud-provider-openstack.git $GOPATH/k8s.io/src/ -$ cd $GOPATH/k8s.io/src/cloud-provider-openstack/ -$ export ARCH=amd64 -$ export VERSION=latest -$ make image-barbican-kms-plugin -``` - -5. Run the KMS Plugin in docker container -``` -$ docker run -d --volume=/var/lib/kms:/var/lib/kms \ ---volume=/etc/kubernetes:/etc/kubernetes \ --e socketpath=/var/lib/kms/kms.sock \ --e cloudconfig=/etc/kubernetes/cloud-config \ -docker.io/k8scloudprovider/barbican-kms-plugin-amd64:latest -``` -6. Create /etc/kubernetes/encryption-config.yaml -``` -kind: EncryptionConfig -apiVersion: v1 -resources: - - resources: - - secrets - providers: - - kms: - name : barbican - endpoint: unix:///var/lib/kms/kms.sock - cachesize: 100 - - identity: {} - ``` -7. Enable --experimental-encryption-provider-config flag in kube-apiserver and restart it. -``` ---experimental-encryption-provider-config=/etc/kubernetes/encryption-config.yaml -``` - -### Verify -[Verify the secret data is encrypted](https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/#verifying-that-data-is-encrypted -) diff --git a/docs/using-cinder-csi-plugin.md b/docs/using-cinder-csi-plugin.md deleted file mode 100644 index 1190f5ab58..0000000000 --- a/docs/using-cinder-csi-plugin.md +++ /dev/null @@ -1,621 +0,0 @@ - - -**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* - -- [CSI Cinder driver](#csi-cinder-driver) - - [Kubernetes](#kubernetes) - - [Compatibility](#compatibility) - - [Requirements](#requirements) - - [Example local-up-cluster.sh](#example-local-up-clustersh) - - [Deploy](#deploy) - - [Using the Helm chart](#using-the-helm-chart) - - [Using the manifests](#using-the-manifests) - - [Example Nginx application usage](#example-nginx-application-usage) - - [Enable Topology-aware dynamic provisioning for Cinder Volumes](#enable-topology-aware-dynamic-provisioning-for-cinder-volumes) - - [Example Snapshot Create and Restore](#example-snapshot-create-and-restore) - - [Example: Raw Block Volume](#example-raw-block-volume) - - [Volume Expansion](#volume-expansion) - - [Rescan block device geometry on in-use volume resize](#rescan-block-device-geometry-on-in-use-volume-resize) - - [Inline Volumes](#inline-volumes) - - [Volume Cloning](#volume-cloning) - - [Running Sanity Tests](#running-sanity-tests) - - [Using CSC tool](#using-csc-tool) - - [Test using csc](#test-using-csc) - - [Start Cinder driver](#start-cinder-driver) - - [Get plugin info](#get-plugin-info) - - [Get supported capabilities](#get-supported-capabilities) - - [Get controller implemented capabilities](#get-controller-implemented-capabilities) - - [Create a volume](#create-a-volume) - - [List volumes](#list-volumes) - - [Delete a volume](#delete-a-volume) - - [Create a snapshot from volume](#create-a-snapshot-from-volume) - - [List snapshots](#list-snapshots) - - [Delete a snapshot](#delete-a-snapshot) - - [ControllerPublish a volume](#controllerpublish-a-volume) - - [ControllerUnpublish a volume](#controllerunpublish-a-volume) - - [NodePublish a volume](#nodepublish-a-volume) - - [NodeUnpublish a volume](#nodeunpublish-a-volume) - - [Get NodeID](#get-nodeid) - - - -# CSI Cinder driver - -## Kubernetes - -### Compatibility - -CSI version | Cinder CSI Plugin Version | Kubernetes Version -:------ | :------------ | :----------- -v1.1.0 | v1.0.0, v1.1.0 docker image: k8scloudprovider/cinder-csi-plugin:latest | v1.15+ -v1.0.0 | v1.0.0 docker image: k8scloudprovider/cinder-csi-plugin:v1.14 | v1.13, v1.14 -v0.3.0 | v0.3.0 docker image: k8scloudprovider/cinder-csi-plugin:1.13.x| v1.11, v1.12, v1.13 -v0.2.0 | v0.2.0 docker image: k8scloudprovider/cinder-csi-plugin:v0.2.0 | v1.10, v1.9 -v0.1.0 | v0.1.0 docker image: k8scloudprovider/cinder-csi-plugin:v0.1.0| v1.9 - -For sidecar version compatibility , please refer compatibility matrix for each sidecar here - -https://kubernetes-csi.github.io/docs/sidecar-containers.html - -### Requirements - -``` -RUNTIME_CONFIG="storage.k8s.io/v1=true" -``` -MountPropagation requires support for privileged containers. So, make sure privileged containers are enabled in the cluster. - -Check [kubernetes CSI Docs](https://kubernetes-csi.github.io/docs/) for flag details and latest update. - -> NOTE: All following examples need to be used inside instance(s) provisoned by openstack, otherwise the attach action will fail due to fail to find instance ID from given openstack cloud. - -### Example local-up-cluster.sh - -```ALLOW_PRIVILEGED=true RUNTIME_CONFIG="storage.k8s.io/v1=true" LOG_LEVEL=5 hack/local-up-cluster.sh``` - -### Deploy - -You can either use the manifests under `manifests/cinder-csi-plugin` or the Helm chart `charts/cinder-csi-plugin`. - -#### Using the Helm chart - -> NOTE: This chart assumes that the `cloud-config` is found on the host under `/etc/kubernetes/` and that your OpenStack cloud has cert under `/etc/cacert`. - -To install the chart use the following command: -``` -helm install --namespace kube-system --name cinder-csi ./charts/cinder-csi-plugin -``` - -#### Using the manifests - -If you already created the `cloud-config` secret used by the [cloud-controller-manager](https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/using-openstack-cloud-controller-manager.md#steps), remove the file ```manifests/cinder-csi-plugin/csi-secret-cinderplugin.yaml``` from [manifests](https://github.com/kubernetes/cloud-provider-openstack/tree/master/manifests/cinder-csi-plugin) and then jump directly to the `kubectl apply ...` command. - -Encode your ```$CLOUD_CONFIG``` file content using base64. - -```base64 -w 0 $CLOUD_CONFIG``` - -Update ```cloud.conf``` configuration in ```manifests/cinder-csi-plugin/csi-secret-cinderplugin.yaml``` file -by using the result of the above command. - -> NOTE: In OpenStack, the compute instance uses either config drive or metadata service to retrieve instance-specific data. As the cluster administrator, you are able to config the order in the cloud config file for cinder-csi-plugin, the default configuration is as follows: -``` -[Metadata] -search-order = configDrive,metadataService -``` - -> NOTE: if your openstack cloud has cert (which means you already has [ca-file](provider-configuration.md#global-optional-parameters) definition in cloud-config), please make sure that you also updated the volumes list of `cinder-csi-controllerplugin.yaml` and `cinder-csi-nodeplugin.yaml` to include the cacert. e.g following sample then mount the volume to the pod as well. - -``` -volumes: -... -- name: cacert - hostPath: - path: /etc/cacert - type: Directory -``` - -Then call following command by using existing [manifests](https://github.com/kubernetes/cloud-provider-openstack/tree/master/manifests/cinder-csi-plugin): - -```kubectl -f manifests/cinder-csi-plugin apply``` - -This creates a set of cluster roles, cluster role bindings, and statefulsets etc to communicate with openstack(cinder). -For detailed list of created objects, explore the yaml files in the directory. -You should make sure following similar pods are ready before proceed: - -``` -NAME READY STATUS RESTARTS AGE -csi-cinder-controllerplugin 5/5 Running 0 29h -csi-cinder-nodeplugin 2/2 Running 0 46h -``` - -you can get information about CSI Drivers running in a cluster, using **CSIDriver** object - -``` -$ kubectl get csidrivers.storage.k8s.io -NAME CREATED AT -cinder.csi.openstack.org 2019-07-29T09:02:40Z - -$ kubectl describe csidrivers.storage.k8s.io -Name: cinder.csi.openstack.org -Namespace: -Labels: -Annotations: -API Version: storage.k8s.io/v1beta1 -Kind: CSIDriver -Metadata: - Creation Timestamp: 2019-07-29T09:02:40Z - Resource Version: 1891 - Self Link: /apis/storage.k8s.io/v1beta1/csidrivers/cinder.csi.openstack.org - UID: 2bd1f3bf-3c41-46a8-b99b-5773cb5eacd3 -Spec: - Attach Required: true - Pod Info On Mount: false -Events: -``` - -### Example Nginx application usage - -After performing above steps, you can try to create StorageClass, PersistentVolumeClaim and pod to consume it. -Try following command by using [examples](https://github.com/kubernetes/cloud-provider-openstack/blob/master/examples/cinder-csi-plugin/nginx.yaml): - -```kubectl -f examples/cinder-csi-plugin/nginx.yaml create``` - -You will get pvc which claims one volume from cinder - -``` -$ kubectl get pvc -NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE -csi-pvc-cinderplugin Bound pvc-72a8f9c9-f462-11e8-b6b6-fa163e18b7b5 1Gi RWO csi-sc-cinderplugin 58m - -$ openstack volume list -+--------------------------------------+--------+------------------------------------------+------+-------------+----------+--------------------------------------+ -| ID | Status | Name | Size | Volume Type | Bootable | Attached to | -+--------------------------------------+--------+------------------------------------------+------+-------------+----------+--------------------------------------+ -| b2e7be02-cdd7-487f-8eb9-6f10f3df790b | in-use | pvc-72a8f9c9-f462-11e8-b6b6-fa163e18b7b5 | 1 | lvmdriver-1 | false | 39859899-2985-43cf-8cdf-21de2548dfd9 | -+--------------------------------------+--------+------------------------------------------+------+-------------+----------+--------------------------------------+ -``` - -Check the volume created and attached to the pod -``` -$ ls /dev/vdb -/dev/vdb - -$ mount | grep vdb -/dev/vdb on /var/lib/kubelet/pods/8196212e-f462-11e8-b6b6-fa163e18b7b5/volumes/kubernetes.io~csi/pvc-72a8f9c9-f462-11e8-b6b6-fa163e18b7b5/mount type ext4 (rw,relatime,data=ordered) - -$ fdisk -l /dev/vdb | grep Disk -Disk /dev/vdb: 1 GiB, 1073741824 bytes, 2097152 sectors -``` - -Then try to add a file in the pod's mounted position (in our case, /var/lib/www/html) -``` -$ kubectl exec -it nginx bash - -root@nginx:/# ls /var/lib/www/html -lost+found - -root@nginx:/# touch /var/lib/www/html/index.html - -root@nginx:/# exit -exit -``` - -Next, make sure the pod is deleted so that the persistent volume will be freed -``` -kubectl delete pod nginx -``` - -Then the volume is back to available state: -``` -$ ls /dev/vdb -ls: cannot access '/dev/vdb': No such file or directory - -$ openstack volume list -+--------------------------------------+-----------+------------------------------------------+------+-------------+----------+-------------+ -| ID | Status | Name | Size | Volume Type | Bootable | Attached to | -+--------------------------------------+-----------+------------------------------------------+------+-------------+----------+-------------+ -| b2e7be02-cdd7-487f-8eb9-6f10f3df790b | available | pvc-72a8f9c9-f462-11e8-b6b6-fa163e18b7b5 | 1 | lvmdriver-1 | false | | -+--------------------------------------+-----------+------------------------------------------+------+-------------+----------+-------------+ -``` - -Optionally you can verify the volume does contain the info we created in pod by attaching to a VM in openstack: -``` -$ nova volume-attach ji1 b2e7be02-cdd7-487f-8eb9-6f10f3df790b -+----------+--------------------------------------+ -| Property | Value | -+----------+--------------------------------------+ -| device | /dev/vdb | -| id | b2e7be02-cdd7-487f-8eb9-6f10f3df790b | -| serverId | 39859899-2985-43cf-8cdf-21de2548dfd9 | -| volumeId | b2e7be02-cdd7-487f-8eb9-6f10f3df790b | -+----------+--------------------------------------+ - -$ ls /dev/vdb -/dev/vdb -$ mount /dev/vdb /mnt; ls /mnt -index.html lost+found -``` - -### Enable Topology-aware dynamic provisioning for Cinder Volumes - -Following feature gates needs to be enabled as below: -1. `--feature-gates=CSINodeInfo=true,CSIDriverRegistry=true` in the manifest entries of kubelet and kube-apiserver. (Enabled by default in kubernetes v1.14) -2. `--feature-gates=Topology=true` needs to be enabled in external-provisioner. - -Currently, driver supports only one topology key: `topology.cinder.csi.openstack.org/zone` that represents availability by zone. - -Note: `allowedTopologies` can be specified in storage class to restrict the topology of provisioned volumes to specific zones and should be used as replacement of `availability` parameter. - -### Example Snapshot Create and Restore - -Following prerequisite needed for volume snapshot feature to work. - -1. Download all yaml files from [snapshot crd](https://github.com/kubernetes-csi/external-snapshotter/tree/v2.1.1/config/crd) -2. Apply all yaml files downloaded at step 1) by using `kubectl apply -f ` command, as currently there is no released yaml file from `external-snapshotter` community. -3. Download all yaml files from [snapshot-controller](https://github.com/kubernetes-csi/external-snapshotter/tree/master/deploy/kubernetes/snapshot-controller) and optionally update all namespaces from `default` to `kube-system` in the yaml files. -4. Apply all yaml files downloaded at step 3) by using `kubectl apply -f ` command. - -Supported parameters for VolumeSnapshotClass: -* `force-create`: Support creating snapshot for a volume in in-use status. - -For Snapshot Creation and Volume Restore, please follow below steps: -* Create Storage Class, Snapshot Class and PVC -``` -$ kubectl -f examples/cinder-csi-plugin/snapshot/example.yaml create -storageclass.storage.k8s.io/csi-sc-cinderplugin created -volumesnapshotclass.snapshot.storage.k8s.io/csi-cinder-snapclass created -persistentvolumeclaim/pvc-snapshot-demo created -``` - -* Verify that pvc is bounded -``` -$ kubectl get pvc --all-namespaces -NAMESPACE NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE -default pvc-snapshot-demo Bound pvc-4699fa78-4149-4772-b900-9536891fe200 1Gi RWO -``` -* Create Snapshot of the PVC -``` -$ kubectl -f examples/cinder-csi-plugin/snapshot/snapshotcreate.yaml create -volumesnapshot.snapshot.storage.k8s.io/new-snapshot-demo created -``` -* Verify that snapshot is created -``` -$ kubectl get volumesnapshot -NAME AGE -new-snapshot-demo 2m54s -$ openstack snapshot list -+--------------------------------------+-----------------------------------------------+-------------+-----------+------+ -| ID | Name | Description | Status | Size | -+--------------------------------------+-----------------------------------------------+-------------+-----------+------+ -| 1b673af2-3a69-4cc6-8dd0-9ac62e29df9e | snapshot-332a8a7e-c5f2-4df9-b6a0-cf52e18e72b1 | None | available | 1 | -+--------------------------------------+-----------------------------------------------+-------------+-----------+------+ -``` -* Restore volume from snapshot -``` -$ kubectl -f examples/cinder-csi-plugin/snapshot/snapshotrestore.yaml create -persistentvolumeclaim/snapshot-demo-restore created -``` -* Verify that volume from snapshot is created -``` -$ kubectl get pvc -NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE -pvc-snapshot-demo Bound pvc-4699fa78-4149-4772-b900-9536891fe200 1Gi RWO csi-sc-cinderplugin 4m5s -snapshot-demo-restore Bound pvc-400b1ca8-8786-435f-a6cc-f684afddbbea 1Gi RWO csi-sc-cinderplugin 8s - -$ openstack volume list -+--------------------------------------+------------------------------------------+-----------+------+-------------------------------------------------+ -| ID | Display Name | Status | Size | Attached to | -+--------------------------------------+------------------------------------------+-----------+------+-------------------------------------------------+ -| 07522a3b-95db-4bfa-847c-ffa179d08c39 | pvc-400b1ca8-8786-435f-a6cc-f684afddbbea | available | 1 | | -| bf8f9ae9-87b4-42bb-b74c-ba4645634be6 | pvc-4699fa78-4149-4772-b900-9536891fe200 | available | 1 | | -``` -### Example: Raw Block Volume - -For consuming a cinder volume as a raw block device - -1. Make sure the volumeMode is `Block` in Persistence Volume Claim Spec -2. Make sure the pod is consuming the PVC with the defined name and `volumeDevices` is used instead of `volumeMounts` -3. Deploy the Application - -Example : - -``` -$ kubectl create -f examples/cinder-csi-plugin/block/block.yaml -``` - -Make sure Pod is running -``` -$ kubectl get pods -``` - -Verify the device node is mounted inside the container - -``` -$ kubectl exec -ti test-block -- ls -al /dev/xvda -brw-rw---- 1 root disk 202, 23296 Mar 12 04:23 /dev/xvda -``` - -### Volume Expansion - -1. As of kubernetes v1.16, Volume Expansion is a beta feature and enabled by default. -2. Make sure to set `allowVolumeExpansion` to `true` in Storage class spec. -3. Deploy the application. - -Example: - -``` -$ kubectl create -f examples/cinder-csi-plugin/resize/example.yaml -``` - -Verify PV is created and bound to PVC - -``` -$ kubectl get pvc -NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE -csi-pvc-cinderplugin Bound pvc-e36abf50-84f3-11e8-8538-42010a800002 1Gi RWO csi-sc-cinderplugin 9s -``` -Make sure Pod is running -``` -$ kubectl get pods -NAME READY STATUS RESTARTS AGE -nginx 1/1 Running 0 1m -``` - -Check current filesystem size on the running pod -``` -$ kubectl exec nginx -- df -h /var/lib/www/html -Filesystem Size Used Avail Use% Mounted on -/dev/vdb 1.0G 24M 0.98G 1% /var/lib/www/html -``` -Resize volume by modifying the field `spec -> resources -> requests -> storage` -``` -$ kubectl edit pvc csi-pvc-cinderplugin -apiVersion: v1 -kind: PersistentVolumeClaim -... -spec: - resources: - requests: - storage: 2Gi - ... -... -``` - -Verify filesystem resized on the running pod -``` -$ kubectl exec nginx -- df -h /var/lib/www/html -Filesystem Size Used Avail Use% Mounted on -/dev/vdb 2.0G 27M 1.97G 1% /var/lib/www/html -``` - -#### Rescan block device geometry on in-use volume resize - -Some hypervizors (like VMware) don't automatically send a new volume size to a Linux kernel, when a volume is in-use. Sending a "1" to `/sys/class/block/XXX/device/rescan` is telling the SCSI block device to refresh it's information about where it's ending boundary is (among other things) to give the kernel information about it's updated size. When a `rescan-on-resize` flag is set in a CSI node driver cloud-config `[BlockStorage]` section, a CSI node driver will rescan block device and verify its size before expanding the filesystem. CSI driver will raise an error, when expected volume size cannot be detected. - -Not all hypervizors have a `/sys/class/block/XXX/device/rescan` location, therefore if you enable this option and your hypervizor doesn't support this, you'll get a warning log on resize event. It is recommended to disable this option in this case. - -### Inline Volumes - -This feature allows CSI volumes to be directly embedded in the Pod specification instead of a PersistentVolume. Volumes specified in this way are ephemeral and do not persist across Pod restarts. As of Kubernetes v1.16 this feature is beta so enabled by default. To enable this feature for CSI Driver, `volumeLifecycleModes` needs to be specified in [CSIDriver](https://github.com/kubernetes/cloud-provider-openstack/blob/master/manifests/cinder-csi-plugin/csi-cinder-driver.yaml) object. The driver can run in `Persistent` mode, `Ephemeral` or in both modes. `podInfoOnMount` must be `true` to use this feature. - -Example: -1. Deploy CSI Driver, in default yamls both modes are enabled. -``` -$ kubectl create -f manifests/cinder-csi-plugin/ -``` -2. Create a pod with inline volume -``` -$ kubectl create -f examples/cinder-csi-plugin/inline/inline-example.yaml -``` -3. Get the pod description, verify created volume in Volumes section. -``` -$ kubectl describe pod - -Volumes: - my-csi-volume: - Type: CSI (a Container Storage Interface (CSI) volume source) - Driver: cinder.csi.openstack.org - FSType: ext4 - ReadOnly: false - VolumeAttributes: capacity=1Gi - default-token-dn78p: - Type: Secret (a volume populated by a Secret) - SecretName: default-token-dn78p - Optional: false - -``` - -### Volume Cloning - -As of Kubernetes v1.16, volume cloning is beta feature and enabled by default. -This feature enables support of cloning a volume from existing PVCs. - -Following prerequisites needed for volume cloning to work : -1. The source PVC must be bound and available (not in use). -2. source and destination PVCs must be in the same namespace. -3. Cloning is only supported within the same Storage Class. Destination volume must - be the same storage class as the source - -Sample yamls can be found [here](https://github.com/kubernetes/cloud-provider-openstack/tree/master/examples/cinder-csi-plugin/clone) - -## Running Sanity Tests - -Sanity tests create a real instance of driver and fake cloud provider. -see [Sanity check](https://github.com/kubernetes-csi/csi-test/tree/master/pkg/sanity) for more info. -``` -$ make test-cinder-csi-sanity -``` - -## Using CSC tool - -### Test using csc -Get ```csc``` tool from https://github.com/thecodeteam/gocsi/tree/master/csc - -### Start Cinder driver - -First, you need to start the plugin as daemon to accept request from csc. -Following example is starting listening at localhost port 10000 with cloud configuration -given in /etc/cloud.conf and the node id is CSINodeID. ClusterID is the identifier of -the cluster in which the plugin is running. - -``` -$ sudo cinder-csi-plugin --endpoint tcp://127.0.0.1:10000 --cloud-config /etc/cloud.conf --nodeid CSINodeID --cluster ClusterID -``` - -### Get plugin info -``` -$ csc identity plugin-info --endpoint tcp://127.0.0.1:10000 -"cinder.csi.openstack.org" "1.0.0" -``` - -### Get supported capabilities -``` -$ csc identity plugin-capabilities --endpoint tcp://127.0.0.1:10000 -CONTROLLER_SERVICE -VOLUME_ACCESSIBILITY_CONSTRAINTS -``` - -### Get controller implemented capabilities -``` -$ csc controller get-capabilities --endpoint tcp://127.0.0.1:10000 -&{type:LIST_VOLUMES } -&{type:CREATE_DELETE_VOLUME } -&{type:PUBLISH_UNPUBLISH_VOLUME } -&{type:CREATE_DELETE_SNAPSHOT } -&{type:LIST_SNAPSHOTS } -``` - -### Create a volume -Following sample creates a volume named ``CSIVolumeName`` and the -volume id returned is ``8a55f98f-e987-43ab-a9f5-973352bee19c`` with size ``1073741824`` bytes (1Gb) -``` -$ csc controller create-volume --endpoint tcp://127.0.0.1:10000 CSIVolumeName -"8a55f98f-e987-43ab-a9f5-973352bee19c" 1073741824 "availability"="nova" -``` - -### List volumes -Following sample list all volumes: -``` -$ csc controller list-volumes --endpoint tcp://127.0.0.1:10000 -"8a55f98f-e987-43ab-a9f5-973352bee19c" 1073741824 -``` - -### Delete a volume -Following sample deletes a volume ``01217e93-bd1b-4760-b5d8-18b8b3d47f91`` -``` -$ csc controller delete-volume --endpoint tcp://127.0.0.1:10000 01217e93-bd1b-4760-b5d8-18b8b3d47f91 -01217e93-bd1b-4760-b5d8-18b8b3d47f91 -``` - -### Create a snapshot from volume -Following sample creates a snapshot from volume `40615da4-3fda-4e78-bf58-820692536e68`. -After execution, snapshot `e2df8c2a-58eb-40fb-8ec9-45aee5b8f39f` will be created. -``` -$ csc controller create-snapshot --source-volume 40615da4-3fda-4e78-bf58-820692536e68 --endpoint tcp://127.0.0.1:10000 s1 -"e2df8c2a-58eb-40fb-8ec9-45aee5b8f39f" 1073741824 40615da4-3fda-4e78-bf58-820692536e68 seconds:1561530261 true -``` - -Use openstack command to verify: -``` -openstack volume snapshot list -+--------------------------------------+------+-------------+-----------+------+ -| ID | Name | Description | Status | Size | -+--------------------------------------+------+-------------+-----------+------+ -| e2df8c2a-58eb-40fb-8ec9-45aee5b8f39f | s1 | None | available | 1 | -+--------------------------------------+------+-------------+-----------+------+ -``` - -### List snapshots - -Following sample lists all snapshots: -``` -$ csc controller list-snapshots --endpoint tcp://127.0.0.1:10000 -"e2df8c2a-58eb-40fb-8ec9-45aee5b8f39f" 1073741824 40615da4-3fda-4e78-bf58-820692536e68 seconds:1561532425 true -``` - -### Delete a snapshot - -Following sample deletes the snapshot `e2df8c2a-58eb-40fb-8ec9-45aee5b8f39f`. -``` -$ csc controller delete-snapshot e2df8c2a-58eb-40fb-8ec9-45aee5b8f39f --endpoint tcp://127.0.0.1:10000 -e2df8c2a-58eb-40fb-8ec9-45aee5b8f39f -``` - -Use openstack command to verify: -``` -$ openstack volume snapshot list - -$ -``` - -### ControllerPublish a volume -The action has similar result to ``nova volume-attach`` command: - -Assume we have following result in openstack now: -``` -$ openstack server list -+--------------------------------------+-------+--------+--------------------------------+--------+---------+ -| ID | Name | Status | Networks | Image | Flavor | -+--------------------------------------+-------+--------+--------------------------------+--------+---------+ -| 17e540e6-8d08-4a5a-8835-668bc8fe913c | demo1 | ACTIVE | demo-net=10.0.0.13 | cirros | m1.tiny | -+--------------------------------------+-------+--------+--------------------------------+--------+---------+ - -$ openstack volume list -+--------------------------------------+-----------------------------------+-----------+------+-------------+ -| ID | Name | Status | Size | Attached to | -+--------------------------------------+-----------------------------------+-----------+------+-------------+ -| ed893ce1-807d-4c6e-a558-88c61b439659 | v1 | available | 1 | | -+--------------------------------------+-----------------------------------+-----------+------+-------------+ -``` - -Then execute: - -``` -# csc controller publish --endpoint tcp://127.0.0.1:10000 --node-id=17e540e6-8d08-4a5a-8835-668bc8fe913c ed893ce1-807d-4c6e-a558-88c61b439659 -"ed893ce1-807d-4c6e-a558-88c61b439659" "DevicePath"="/dev/vdb" -``` - -From openstack side you will see following result: - -``` -$ openstack server list -+--------------------------------------+-------+--------+--------------------------------+--------+---------+ -| ID | Name | Status | Networks | Image | Flavor | -+--------------------------------------+-------+--------+--------------------------------+--------+---------+ -| 17e540e6-8d08-4a5a-8835-668bc8fe913c | demo1 | ACTIVE | demo-net=10.0.0.13 | cirros | m1.tiny | -+--------------------------------------+-------+--------+--------------------------------+--------+---------+ -$ openstack volume list -+--------------------------------------+-----------------------------------+-----------+------+--------------------------------+ -| ID | Name | Status | Size | Attached to | -+--------------------------------------+-----------------------------------+-----------+------+--------------------------------+ -| ed893ce1-807d-4c6e-a558-88c61b439659 | v1 | in-use | 1 | Attached to demo1 on /dev/vdb | -+--------------------------------------+-----------------------------------+-----------+------+--------------------------------+ - -Note: -volume "Status" field will change to "in-use" afterwards. -"Attached to" field will change to volume mount point. -``` - -### ControllerUnpublish a volume -ControllerUnpublish is reserver action of ControllerPublish, which is similar to ``nova volume-detach`` -``` -[root@kvm-017212 docs]# csc controller unpublish --endpoint tcp://127.0.0.1:10000 --node-id=17e540e6-8d08-4a5a-8835-668bc8fe913c ed893ce1-807d-4c6e-a558-88c61b439659 - -ed893ce1-807d-4c6e-a558-88c61b439659 -``` - -### NodePublish a volume -``` -$ csc node publish --endpoint tcp://127.0.0.1:10000 --target-path /mnt/cinder --pub-info DevicePath="/dev/xxx" CSIVolumeID -CSIVolumeID -``` - -### NodeUnpublish a volume -``` -$ csc node unpublish --endpoint tcp://127.0.0.1:10000 --target-path /mnt/cinder CSIVolumeID -CSIVolumeID -``` - -### Get NodeID -``` -$ csc node get-id --endpoint tcp://127.0.0.1:10000 -CSINodeID -``` diff --git a/docs/using-client-keystone-auth.md b/docs/using-client-keystone-auth.md deleted file mode 100644 index ce0367fc6d..0000000000 --- a/docs/using-client-keystone-auth.md +++ /dev/null @@ -1,169 +0,0 @@ - - -**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* - -- [Keystone client-go credential plugin usage](#keystone-client-go-credential-plugin-usage) - - [Overview](#overview) - - [Example use case](#example-use-case) - - [Configuration](#configuration) - - [Input and output formats](#input-and-output-formats) - - [References](#references) - - - -# Keystone client-go credential plugin usage - -## Overview - -`k8s.io/client-go` and tools using it such as `kubectl` and `kubelet` are able to execute an -external command to receive user credentials. - -This feature allows client side integrations with authentication using Keystone API, that is -not natively supported by `k8s.io/client-go`. The plugin implements the protocol specific logic, -then returns opaque credentials to use. This credential plugin use cases require a server side -component with support for the [webhook token authenticator](https://kubernetes.io/docs/admin/authentication/#webhook-token-authentication) -to interpret the credential format produced by the client plugin. The webhook authenticator is -provided by [k8s-keystone-auth binary](https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/using-keystone-webhook-authenticator-and-authorizer.md) - -## Example use case - -In a hypothetical use case, an organization would run an external service that exchanges Openstack Keystone -credentials for user specific, signed tokens. The service would also be capable of responding to [webhook token -authenticator](https://kubernetes.io/docs/admin/authentication/#webhook-token-authentication) requests to -validate the tokens. Users would be required to install a credential plugin on their workstation. - -To authenticate against the API: - -* The user issues a `kubectl` command. -* Credential plugin prompts the user for Keystone credentials, exchanges credentials with external service for a token. -* Credential plugin returns token to client-go, which uses it as a bearer token against the API server. -* API server uses the [webhook token authenticator](https://kubernetes.io/docs/admin/authentication/#webhook-token-authentication) - to submit the token to the Keystone service. -* Keystone service verifies the token and returns the user's username and groups. - -## Configuration - -The credential plugin is configured through [`kubectl` config files](https://kubernetes.io/docs/tasks/access-application-cluster/configure-access-multiple-clusters/) -as part of the user fields. - -```yaml -apiVersion: v1 -kind: Config -users: -- name: my-user - user: - exec: - # Command to execute. Required. - command: "client-keystone-auth" - - # API version to use when encoding and decoding the ExecCredentials - # resource. Required. - # - # The API version returned by the plugin MUST match the version encoded. - apiVersion: "client.authentication.k8s.io/v1beta1" - - # Environment variables to set when executing the plugin. Optional. - env: - - name: "OS_USERNAME" - value: "admin" - - name: "OS_PASSWORD" - value: "passw0rd" - - name: "OS_PROJECT_NAME" - value: "myproject" - - # Arguments to pass when executing the plugin. Optional. - args: - - "--domain-name=default" - - "--keystone-url=https://127.0.0.1/identity" -clusters: -- name: my-cluster - cluster: - server: "https://172.17.4.100:6443" - certificate-authority: "/etc/kubernetes/ca.pem" -contexts: -- name: my-cluster - context: - cluster: my-cluster - user: my-user -current-context: my-cluster -``` - -Relative command paths are interpreted as relative to the directory of the config file. If -KUBECONFIG is set to `/home/jane/kubeconfig` and the exec command is `./bin/client-keystone-auth`, -the binary `/home/jane/bin/client-keystone-auth` is executed. - -```yaml -- name: my-user - user: - exec: - # Path relative to the directory of the kubeconfig - command: "./bin/client-keystone-auth" - apiVersion: "client.authentication.k8s.io/v1beta1" -``` - -## Input and output formats - -The executed command prints an `ExecCredential` object to `stdout`. `k8s.io/client-go` -authenticates against the Kubernetes API using the returned credentials in the `status`. - -When run from an interactive session, `stdin` is exposed directly to the plugin. The plugin uses a -[TTY check](https://godoc.org/golang.org/x/crypto/ssh/terminal#IsTerminal) to determine if it's -appropriate to prompt a user interactively. - -When the plugin is executed from an interactive session, `stdin` and `stderr` are directly -exposed to the plugin so it can prompt the user for input for interactive logins. - -To authenticate in Keystone from an interactive session, the user needs to provide the address -domain name, user name, password, and, optionally, a project name. These values can be specified -using environment variables -(`OS_AUTH_URL`, `OS_DOMAIN_NAME`, `OS_USERNAME`, `OS_PASSWORD` and `OS_PROJECT_NAME`), or through command -arguments -(`--keystone-url`, `--domain-name`, `--user-name`, `--password` and `--project-name`), respectively. -If they are not specified, the user will be prompted to enter them at the time of the interactive -session. -It is also possible to authenticate with [Keystone Application Credentials](https://docs.openstack.org/keystone/latest/user/application_credentials.html). -In this case, the user needs to provide the id or the name of the Application Credential, and the Secret. -The environment variables are `OS_APPLICATION_CREDENTIAL_ID`,`OS_APPLICATION_CREDENTIAL_NAME` and -`OS_APPLICATION_CREDENTIAL_SECRET` and the command line arguments are `--application-credential-name`, -`--application-credential-id` and `--application-credential-secret`. - -When responding to a 401 HTTP status code (indicating invalid credentials), this object will -include metadata about the response. - -```json -{ - "apiVersion": "client.authentication.k8s.io/v1beta1", - "kind": "ExecCredential", - "spec": { - "response": { - "code": 401, - "header": {}, - }, - "interactive": true - } -} -``` - -The executed command prints an `ExecCredential` to `stdout`. This objects contains a bearer -token as `token` and the expiry of the token formatted as a RFC3339 timestamp as `expirationTimestamp`. -`k8s.io/client-go` will then use the returned bearer token in the `status` when authenticating against the -Kubernetes API. - -```json -{ - "apiVersion": "client.authentication.k8s.io/v1beta1", - "kind": "ExecCredential", - "status": { - "token": "my-bearer-token", - "expirationTimestamp": "2018-03-05T17:30:20-08:00" - } -} -``` - -## References - -More details about Kubernetes Authentication Webhook using Bearer Tokens is at : -https://kubernetes.io/docs/admin/authentication/#webhook-token-authentication - -Client-go credential plugins documentation is at: -https://kubernetes.io/docs/admin/authentication/#client-go-credential-plugins diff --git a/docs/using-keystone-webhook-authenticator-and-authorizer.md b/docs/using-keystone-webhook-authenticator-and-authorizer.md deleted file mode 100644 index ec3938b3b4..0000000000 --- a/docs/using-keystone-webhook-authenticator-and-authorizer.md +++ /dev/null @@ -1,777 +0,0 @@ - - -**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* - -- [k8s-keystone-auth](#k8s-keystone-auth) - - [Prerequisites](#prerequisites) - - [Deploy k8s-keystone-auth webhook server](#deploy-k8s-keystone-auth-webhook-server) - - [Prepare the authorization policy (optional)](#prepare-the-authorization-policy-optional) - - [Non-resource permission](#non-resource-permission) - - [Sub-resource permission](#sub-resource-permission) - - [Prepare the service certificates](#prepare-the-service-certificates) - - [Create service account for k8s-keystone-auth](#create-service-account-for-k8s-keystone-auth) - - [Deploy k8s-keystone-auth](#deploy-k8s-keystone-auth) - - [Test k8s-keystone-auth service](#test-k8s-keystone-auth-service) - - [Configuration on K8S master for authentication and/or authorization](#configuration-on-k8s-master-for-authentication-andor-authorization) - - [Authorization policy definition(version 2)](#authorization-policy-definitionversion-2) - - [Client(kubectl) configuration](#clientkubectl-configuration) - - [Old kubectl clients](#old-kubectl-clients) - - [kubectl clients from v1.8.0 to v1.10.x](#kubectl-clients-from-v180-to-v110x) - - [New kubectl clients from v1.11.0 and later](#new-kubectl-clients-from-v1110-and-later) - - - -# k8s-keystone-auth - -[Kubernetes webhook authentication and authorization](https://kubernetes.io/docs/reference/access-authn-authz/webhook/) -for OpenStack Keystone. With k8s-keystone-auth, the Kubernetes cluster -administrator only need to know the OpenStack project names or roles, -it's up to the OpenStack project admin for user management, as a result, -the OpenStack users could have access to the Kubernetes cluster. - -The k8s-keystone-auth can be running either as a static pod(controlled -by kubelet) or a normal kubernetes service. - -## Prerequisites - -- You already have a Kubernetes cluster(version >= 1.9.3) up and running and you - have the admin permission for the cluster. -- You have an OpenStack environment and admin credentials. - -> If you run k8s-keystone-auth service as a static pod, the pod creation could - be a part of kubernetes cluster initialization process. - -## Deploy k8s-keystone-auth webhook server - -### Prepare the authorization policy (optional) - -> The authorization feature is optional, you can choose to deploy k8s-keystone-auth webhook server for authentication only and rely on Kubernetes RBAC for authorization. See more details [here](./using-auth-data-synchronization.md). However, k8s-keystone-auth authorization provides more flexible configurations than Kubernetes native RBAC. - -The authorization policy can be specified using an existing ConfigMap name in -the cluster, by doing this, the policy could be changed dynamically without the -k8s-keystone-auth service restart. The ConfigMap needs to be created before -running the k8s-keystone-auth service. - -k8s-keystone-auth service supports two versions of policy definition. -Version 2 is recommended because of its better flexibility. However, -both versions are described in this guide. You can see more information -of version 2 in [Authorization policy definition(version -2)](#authorization-policy-definitionversion-2). - -For testing purpose, in the following ConfigMap, we only allow the users in -project `demo` with `member` role in OpenStack to query the Pods information -from all the namespaces. We create the ConfigMap in `kube-system` namespace -because we will also run k8s-keystone-auth service there. - -Version 1: - -```shell -$ cat < /etc/kubernetes/keystone-auth/policy-config.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: k8s-auth-policy - namespace: kube-system -data: - policies: | - [ - { - "resource": { - "verbs": ["get", "list", "watch"], - "resources": ["pods"], - "version": "*", - "namespace": "default" - }, - "match": [ - { - "type": "role", - "values": ["member"] - }, - { - "type": "project", - "values": ["demo"] - } - ] - } - ] -EOF -$ kubectl apply -f /etc/kubernetes/keystone-auth/policy-config.yaml -``` -Version 2: - -please refer to [configmap](../examples/webhook/keystone-policy-configmap.yaml) for the definition of -policy config map. -```shell -$ kubectl apply -f examples/webhook/keystone-policy-configmap.yaml -``` - -As you can see, the version 2 policy definition is much simpler and -more succinct. - -#### Non-resource permission - -For many scenarios clients require access to `nonresourse` paths. -`nonresource` paths include: `/api`, `/apis`, `/metrics`, `/resetMetrics`, -`/logs`, `/debug`, `/healthz`, `/swagger-ui/`, `/swaggerapi/`, `/ui`, and -`/version`. Clients require access to `/api`, `/api/*`, `/apis`, `/apis/*`, -and `/version` to discover what resources and versions are present on the -server. Access to other `nonresource` paths can be disallowed without -restricting access to the REST api. - -#### Sub-resource permission - -In order to describe subresource (e.g `logs` or `exec`) of a certain resource -(e.g. `pod`)it is possible to use `/` in order to combine resource and -subresource. This is similar to the way resources described in `rules` list -of k8s `Role` object. - -For an example of using of subresources as well as `nonresourse` paths please -see policy below. With this policy we only want to allow client to be able to -`kubectl exec` into pod and only in `utility` namespace. For this purpose we -define resource as `"resources": ["pods/exec"]`. But in order for client to be -able to discover pods and versions as mentioned above we also need to allow -read access to `nonresource` paths `/api`, `/api/*`, `/apis`, `/apis/*`. -At this moment only one path (type string) is supported per `nonresource` json -object, this is why we have entry for each of them. - -```shell -cat <" - }, - "status": { - "authenticated": true, - "user": { - "extra": { - "alpha.kubernetes.io/identity/project/id": [ - "423d41d3a02f4b77b4a9bbfbc3a1b3c6" - ], - "alpha.kubernetes.io/identity/project/name": [ - "demo" - ], - "alpha.kubernetes.io/identity/roles": [ - "member", - "load-balancer_member" - ], - "alpha.kubernetes.io/identity/user/domain/id": [ - "default" - ], - "alpha.kubernetes.io/identity/user/domain/name": [ - "Default" - ] - }, - "groups": [ - "mygroup", - "423d41d3a02f4b77b4a9bbfbc3a1b3c6" - ], - "uid": "ff369be2cbb14ee9bb775c0bcf2a1061", - "username": "demo" - } - } - } - ``` - -- Authorization (optional) - - > Please skip this validation if you are using Kubernetes RBAC for authorization. - - From the above response, we know the `demo` user in the `demo` project - does have `member` role associated: - - ```shell - $ kubectl --namespace kube-system run --rm --restart=Never --attach=true \ - --image curlimages/curl curl -- \ - -ks -XPOST https://k8s-keystone-auth.kube-system:8443/webhook -d ' - { - "apiVersion": "authorization.k8s.io/v1beta1", - "kind": "SubjectAccessReview", - "spec": { - "resourceAttributes": { - "namespace": "default", - "verb": "get", - "group": "", - "resource": "pods", - "name": "pod1" - }, - "user": "demo", - "group": ["423d41d3a02f4b77b4a9bbfbc3a1b3c6"], - "extra": { - "alpha.kubernetes.io/identity/project/id": ["423d41d3a02f4b77b4a9bbfbc3a1b3c6"], - "alpha.kubernetes.io/identity/project/name": ["demo"], - "alpha.kubernetes.io/identity/roles": ["load-balancer_member","member"] - } - } - }' - ``` - - Response: - - ```shell - { - "apiVersion": "authorization.k8s.io/v1beta1", - "kind": "SubjectAccessReview", - "status": { - "allowed": true - } - } - ``` - - According to the policy definition, pod creation should fail: - - ```shell - $ kubectl --namespace kube-system run --rm --restart=Never --attach=true \ - --image curlimages/curl curl -- \ - -ks -XPOST https://k8s-keystone-auth.kube-system:8443/webhook -d ' - { - "apiVersion": "authorization.k8s.io/v1beta1", - "kind": "SubjectAccessReview", - "spec": { - "resourceAttributes": { - "namespace": "default", - "verb": "create", - "group": "", - "resource": "pods", - "name": "pod1" - }, - "user": "demo", - "group": ["423d41d3a02f4b77b4a9bbfbc3a1b3c6"], - "extra": { - "alpha.kubernetes.io/identity/project/id": ["423d41d3a02f4b77b4a9bbfbc3a1b3c6"], - "alpha.kubernetes.io/identity/project/name": ["demo"], - "alpha.kubernetes.io/identity/roles": ["load-balancer_member","member"] - } - } - }' - ``` - - Response: - - ```shell - { - "apiVersion": "authorization.k8s.io/v1beta1", - "kind": "SubjectAccessReview", - "status": { - "allowed": false - } - } - ``` - -Now the k8s-keystone-auth service works as expected, we could go ahead to -config kubernetes API server to use the k8s-keystone-auth service as a webhook -service for both authentication and authorization. In fact, the -k8s-keystone-auth service can be used for authentication or authorization only, -and both as well, depending on your requirement. In this example, -`10.109.16.219` is the cluster IP of k8s-keystone-auth service. - -### Configuration on K8S master for authentication and/or authorization - -- Create the webhook config file. - - ```shell - keystone_auth_service_addr=10.109.16.219 - mkdir /etc/kubernetes/webhooks - cat < /etc/kubernetes/webhooks/webhookconfig.yaml - --- - apiVersion: v1 - kind: Config - preferences: {} - clusters: - - cluster: - insecure-skip-tls-verify: true - server: https://${keystone_auth_service_addr}:8443/webhook - name: webhook - users: - - name: webhook - contexts: - - context: - cluster: webhook - user: webhook - name: webhook - current-context: webhook - EOF - ``` - -- Modify kube-apiserver config file to use the webhook service for - authentication and/or authorization. - - Authentication: - - ``` - --authentication-token-webhook-config-file=/etc/kubernetes/webhooks/webhookconfig.yaml - ``` - - Authorization (optional): - - ``` - --authorization-webhook-config-file=/etc/kubernetes/webhooks/webhookconfig.yaml - --authorization-mode=Node,Webhook,RBAC - ``` - - Also mount the new webhooks directory: - - ``` - containers: - ... - volumeMounts: - ... - - mountPath: /etc/kubernetes/webhooks - name: webhooks - readOnly: true - volumes: - ... - - hostPath: - path: /etc/kubernetes/webhooks - type: DirectoryOrCreate - name: webhooks - ``` - -- Wait for the API server to restart successfully until you can see all the - pods are running in `kube-system` namespace. - -## Authorization policy definition(version 2) - -The version 2 definition could be used together with version 1 but will -take precedence over version 1 if both are defined. The version 1 -definition is still supported but may be considered deprecated in the future. - -The authorization policy definition is based on whitelist, which means -the operation is allowed if *ANY* rule defined in the permissions is -satisfied. - -- "users" defines which projects the OpenStack users belong to and what - roles they have. You could define multiple projects or roles, if the project - of the target user is included in the projects, the permission is going to be - checked. -- "resource_permissions" is a map with the key defines namespaces and - resources, the value defines the allowed operations. `/` is used as separator - for namespace and resource. `!` and `*` are supported both for namespaces and - resources, see examples below. -- "nonresource_permissions" is a map with the key defines the - non-resource endpoint such as `/healthz`, the value defines the - allowed operations. - -Some examples: - -- Any operation is allowed on any resource in any namespace. - - ```json - "resource_permissions": { - "*/*": ["*"] - } - ``` - -- Only "get" and "list" are allowed for Pods in the "default" namespace. - - ```json - "resource_permissions": { - "default/pods": ["get", "list"] - } - ``` - -- "create" is allowed for any resource except Secrets and ClusterRoles - in the "default" namespace. - - ```json - "resource_permissions": { - "default/!['secrets', 'clusterroles']": ["create"] - } - ``` - -- Any operation is allowed for any resource in any namespace except - "kube-system". - - ```json - "resource_permissions": { - "!kube-system/*": ["*"] - } - ``` - -- Any operation is allowed for any resource except Secrets and - ClusterRoles in any namespace except "kube-system". - - ```json - "resource_permissions": { - "!kube-system/!['secrets', 'clusterroles']": ["*"] - } - ``` - -## Client(kubectl) configuration - -If the k8s-keystone-auth service is configured for both authentication and -authorization, make sure your OpenStack user in the following steps has the -`member` role in Keystone as defined above, otherwise listing pod operation -will fail. - -### Old kubectl clients - -- Run `openstack token issue` to generate a token -- Run `kubectl --token $TOKEN get po` or `curl -k -v -XGET -H "Accept: application/json" -H "Authorization: Bearer $TOKEN" https://localhost:6443/api/v1/namespaces/default/pods` - -### kubectl clients from v1.8.0 to v1.10.x - -The client is able to read the `OS_` env variables used also by the -openstackclient. You don't have to pass a token with `--token`, but the client -will contact Keystone directly, will get a token and will use it. To configure -the client do the following: - -- Run `kubectl config set-credentials openstackuser --auth-provider=openstack` - -This command creates the following entry in your ~/.kube/config -``` -- name: openstackuser - user: - as-user-extra: {} - auth-provider: - name: openstack -``` -- Run `kubectl config set-context --cluster=mycluster --user=openstackuser openstackuser@kubernetes` -- Run `kubectl config use-context openstackuser@kubernetes` to activate the context - -After running above commands, your kubeconfig file should be like below: - -``` -apiVersion: v1 -clusters: -- cluster: - certificate-authority: /tmp/certs/ca.pem - server: https://172.24.4.6:6443 - name: mycluster -contexts: -- context: - cluster: mycluster - user: admin - name: default -- context: - cluster: mycluster - user: openstackuser - name: openstackuser@kubernetes -current-context: openstackuser@kubernetes -kind: Config -preferences: {} -users: -- name: admin - user: - client-certificate: /tmp/certs/cert.pem - client-key: /tmp/certs/key.pem -- name: openstackuser - user: - auth-provider: - config: - ttl: 10m0s - name: openstack - -``` - -In above kubeconfig, the cluster name is `mycluster`, the kube API address is -`https://172.24.4.6:6443`. And in this kubeconfig file, there are two contexts. -One for normal certs auth, and one for Keystone auth. Please note, the current -context is `openstackuser@kubernetes`. - -Source your env vars. Make sure you include `OS_DOMAIN_NAME` or the client will -fallback to Keystone V2 that is not supported by the webhook.This env should be -ok: - -``` -OS_AUTH_URL="https://keystone.example.com:5000/v3" -OS_DOMAIN_NAME="default" -OS_IDENTITY_API_VERSION="3" -OS_PASSWORD="mysecret" -OS_PROJECT_NAME="myproject" -OS_REGION_NAME="myRegion" -OS_USERNAME="username" -``` -- Try: `kubectl get pods` - -### New kubectl clients from v1.11.0 and later - -Client auth providers are deprecated in v1.11.0 and to be removed in the next -version. The recommended way of client authentication is to use ``exec`` mode -with the ``client-keystone-auth`` binary. - -To configure the client do the following: - -- Download the client-keystone-auth binary(maintained by Lingxian Kong), you can also build the binary by yourself. - - ``` - curl https://github.com/kubernetes/cloud-provider-openstack/releases/latest/download/client-keystone-auth -o ~/client-keystone-auth - sudo chmod u+x ~/client-keystone-auth - ``` - -- Run `kubectl config set-credentials openstackuser`, this command creates the - following entry in the `~/.kube/config` file. - - ``` - - name: openstackuser - user: {} - ``` - -- Config kubectl to use client-keystone-auth binary for the user - `openstackuser`. We assume `mycluster` is the cluster name defined in - `~/.kube/config`. - - ``` - sed -i '/user: {}/ d' ~/.kube/config - cat <> ~/.kube/config - user: - exec: - command: "/home/ubuntu/client-keystone-auth" - apiVersion: "client.authentication.k8s.io/v1beta1" - EOF - kubectl config set-context --cluster=mycluster --user=openstackuser openstackuser@mycluster - ``` - -After running above commands, your kubeconfig file should be like below: - -``` -apiVersion: v1 -clusters: -- cluster: - certificate-authority: /tmp/certs/ca.pem - server: https://172.24.4.6:6443 - name: mycluster -contexts: -- context: - cluster: mycluster - user: admin - name: default -- context: - cluster: mycluster - user: openstackuser - name: openstackuser@mycluster -current-context: openstackuser@mycluster -kind: Config -preferences: {} -users: -- name: admin - user: - client-certificate: /tmp/certs/cert.pem - client-key: /tmp/certs/key.pem -- name: openstackuser - user: - exec: - command: "/home/ubuntu/client-keystone-auth" - apiVersion: "client.authentication.k8s.io/v1alpha1" -``` - -In above kubeconfig, the cluster name is `mycluster`, the kube API address is -`https://172.24.4.6:6443`. And in this kubeconfig file, there are two contexts. -One for normal certs auth, and one for Keystone auth. - -Next you have several ways to specify additional auth parameters: - -1. Source your env vars(recommended). Make sure you include `OS_DOMAIN_NAME` - otherwise the client will fallback to Keystone V2 that is not supported by - the webhook. - - ``` - export OS_AUTH_URL="https://keystone.example.com:5000/v3" - export OS_DOMAIN_NAME="default" - export OS_PASSWORD="mysecret" - export OS_USERNAME="username" - export OS_PROJECT_NAME="demo" - ``` - -2. Specify auth parameters in the `~/.kube/config` file. For more information - read - [client keystone auth configuaration doc](./using-client-keystone-auth.md) - and - [credential plugins documentation](https://kubernetes.io/docs/admin/authentication/#client-go-credential-plugins) -3. Use the interactive mode. If auth parameters are not specified initially, - neither as env variables, nor the `~/.kube/config` file, the user will be - prompted to enter them from keyboard at the time of the interactive session. - -To test that everything works as expected try: -``` -kubectl get pods --context openstackuser@mycluster -``` - -In case you are using this Webhook just for the authentication, you should get -an authorization error: - -``` -Error from server (Forbidden): pods is forbidden: User "username" cannot list pods in the namespace "default" -``` - -You need to configure the RBAC with roles to be authorized to do something, for -example: - -``` -kubectl create rolebinding username-view --clusterrole view --user username --namespace default -``` - -Try now again and you should see the pods. diff --git a/docs/using-magnum-auto-healer.md b/docs/using-magnum-auto-healer.md deleted file mode 100644 index 757eb20c7e..0000000000 --- a/docs/using-magnum-auto-healer.md +++ /dev/null @@ -1,242 +0,0 @@ - - -**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* - -- [Using magnum-auto-healer](#using-magnum-auto-healer) - - [What is magnum-auto-healer](#what-is-magnum-auto-healer) - - [magnum-auto-healer Design](#magnum-auto-healer-design) - - [Deploying and testing magnum-auto-healer](#deploying-and-testing-magnum-auto-healer) - - [Prerequisites](#prerequisites) - - [Deploy magnum-auto-healer](#deploy-magnum-auto-healer) - - [Testing magnum-auto-healer](#testing-magnum-auto-healer) - - [magnum-auto-healer video demo](#magnum-auto-healer-video-demo) - - - -# Using magnum-auto-healer - -## What is magnum-auto-healer - -Kubernetes is self-healing container orchestration platform that will detect failures from your pods and redeploy that workload, however, magnum-auto-healer is a self-healing cluster management service that will automatically recover a failed master or worker node within your Magnum cluster. Basically, magnum-auto-healer ensures the running Kubernetes nodes are healthy by monitoring the nodes' status periodically, searching for unhealthy instances and triggering replacements when needed, maximizing your cluster's high availability and reliability, and protecting your application from downtime when the node it's running on fails. - -The other side of concerns for Kubernetes cluster is scalability. Kubernetes [cluster-autoscaler](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler) can scale the worker pools in your cluster automatically to increase or decrease the number of worker nodes based on the sizing needs of the scheduled workloads. cluster-autoscaler periodically scans the cluster to adjust the number of worker nodes in response to your workload resource requests and any custom settings that you configure, such as scanning intervals. The main purpose of cluster-autoscaler is autoscaling, not autohealing. cluster-autoscaler can be deployed together with magnum-auto-healer. - -Like cluster-autoscaler, magnum-auto-healer is implemented to use together with cloud providers as well, [OpenStack Magnum](https://docs.openstack.org/magnum/latest/user/) is supported by default. - -## magnum-auto-healer Design - -There are some considerations when we were designing the magnum-auto-healer service: - -- We want to have a single component for the cluster autohealing purpose. There are already some other components out there in the community to deal with some specific tasks separately, combining them together with some customization may work, but will lead to much complexity and maintenance overhead. -- Support both master nodes and worker nodes. -- Cluster administrator is able to disable the autohealing feature on the fly, which is very important for the cluster operations like upgrade or scheduled maintenance. -- The Kubernetes cluster is not necessary to be exposed to either the public or the OpenStack control plane. For example, In Magnum, the end user may create a private cluster which is not accessible even from Magnum control services. -- The health check should be pluggable. Deployers should be able to write their own health check plugin with customized health check parameters. -- Support different cloud providers. - -## Deploying and testing magnum-auto-healer - -### Prerequisites - -1. A multi-node cluster(3 masters and 3 workers) is created in Magnum. - - ``` - $ openstack coe cluster list - +--------------------------------------+-----------------------------+-----------------+------------+--------------+-----------------+ - | uuid | name | keypair | node_count | master_count | status | - +--------------------------------------+-----------------------------+-----------------+------------+--------------+-----------------+ - | c418c335-0e52-42fc-bd68-baa8d264e072 | lingxian_por_test_1.12.7_ha | lingxian_laptop | 3 | 3 | CREATE_COMPLETE | - +--------------------------------------+-----------------------------+-----------------+------------+--------------+-----------------+ - $ openstack server list --name lingxian-por-test-1-12-7-ha - +--------------------------------------+---------------------------------------------------+--------+-----------------------------------------+-------------------------+---------+ - | ID | Name | Status | Networks | Image | Flavor | - +--------------------------------------+---------------------------------------------------+--------+-----------------------------------------+-------------------------+---------+ - | 908957c2-ac88-4b54-a1fc-91f9cc8f98f1 | lingxian-por-test-1-12-7-ha-bbgjts5g4xhb-minion-2 | ACTIVE | lingxian_net=10.0.10.33, 150.242.42.234 | fedora-atomic-27-x86_64 | c1.c4r8 | - | 8f0c3ad9-caf5-45b6-bf3a-97b3bb6de623 | lingxian-por-test-1-12-7-ha-bbgjts5g4xhb-minion-0 | ACTIVE | lingxian_net=10.0.10.32, 150.242.42.233 | fedora-atomic-27-x86_64 | c1.c4r8 | - | a6ae4cee-7cf2-4b25-89bc-a5c6cb2c364d | lingxian-por-test-1-12-7-ha-bbgjts5g4xhb-minion-1 | ACTIVE | lingxian_net=10.0.10.34, 150.242.42.245 | fedora-atomic-27-x86_64 | c1.c4r8 | - | 2af96203-cc6f-4b55-8fb2-062340207ebb | lingxian-por-test-1-12-7-ha-bbgjts5g4xhb-master-2 | ACTIVE | lingxian_net=10.0.10.31, 150.242.42.226 | fedora-atomic-27-x86_64 | c1.c2r4 | - | 10bef366-b5a8-4400-b2c3-82188ec06b13 | lingxian-por-test-1-12-7-ha-bbgjts5g4xhb-master-1 | ACTIVE | lingxian_net=10.0.10.30, 150.242.42.22 | fedora-atomic-27-x86_64 | c1.c2r4 | - | 9c17f034-6825-4e49-b3cb-0ecddd1a8dd8 | lingxian-por-test-1-12-7-ha-bbgjts5g4xhb-master-0 | ACTIVE | lingxian_net=10.0.10.29, 150.242.42.213 | fedora-atomic-27-x86_64 | c1.c2r4 | - +--------------------------------------+---------------------------------------------------+--------+-----------------------------------------+-------------------------+---------+ - ``` - -2. The kubeconfig file of the cluster is in place. - -### Deploy magnum-auto-healer - -It's recommended to run magnum-auto-healer service as a DaemonSet on the master nodes, the service is running in active-passive mode using leader election mechanism. There is a sample manifest file in `manifests/magnum-auto-healer/magnum-auto-healer.yaml`, you need to change some variables as needed before actually running `kubectl apply` command. The following commands are just examples: - -```shell -magnum_cluster_uuid=c418c335-0e52-42fc-bd68-baa8d264e072 -keystone_auth_url=https://api.nz-por-1.catalystcloud.io:5000/v3 -user_id=ceb61464a3d341ebabdf97d1d4b97099 -user_project_id=b23a5e41d1af4c20974bf58b4dff8e5a -password=password -region=RegionOne -image=k8scloudprovider/magnum-auto-healer:latest - -cat < - -**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* - -- [CSI Manila driver](#csi-manila-driver) - - [Table of contents](#table-of-contents) - - [Configuration](#configuration) - - [Command line arguments](#command-line-arguments) - - [Controller Service volume parameters](#controller-service-volume-parameters) - - [Node Service volume context](#node-service-volume-context) - - [Secrets, authentication](#secrets-authentication) - - [Topology-aware dynamic provisioning](#topology-aware-dynamic-provisioning) - - [Runtime configuration file](#runtime-configuration-file) - - [Deployment](#deployment) - - [Kubernetes 1.15+](#kubernetes-115) - - [Verifying the deployment](#verifying-the-deployment) - - [Enabling topology awareness](#enabling-topology-awareness) - - [Share protocol support matrix](#share-protocol-support-matrix) - - [For developers](#for-developers) - - - -# CSI Manila driver - -The CSI Manila driver is able to create and mount OpenStack Manila shares. Snapshots and recovering shares from snapshots is supported as well (support for CephFS snapshots will be added soon). - -## Configuration - -### Command line arguments - -Option | Default value | Description --------|---------------|------------ -`--endpoint` | `unix:///tmp/csi.sock` | CSI Manila's CSI endpoint -`--drivername` | `manila.csi.openstack.org` | Name of this driver -`--nodeid` | _none_ | ID of this node -`--nodeaz` | _none_ | Availability zone of this node -`--runtime-config-file` | _none_ | Path to the [runtime configuration file](#runtime-configuration-file) -`--with-topology` | _none_ | CSI Manila is topology-aware. See [Topology-aware dynamic provisioning](#topology-aware-dynamic-provisioning) for more info -`--share-protocol-selector` | _none_ | Specifies which Manila share protocol to use for this instance of the driver. See [supported protocols](#share-protocol-support-matrix) for valid values. -`--fwdendpoint` | _none_ | [CSI Node Plugin](https://github.com/container-storage-interface/spec/blob/master/spec.md#rpc-interface) endpoint to which all Node Service RPCs are forwarded. Must be able to handle the file-system specified in `share-protocol-selector`. Check out the [Deployment](#deployment) section to see why this is necessary. - -### Controller Service volume parameters - -_Kubernetes storage class parameters for dynamically provisioned volumes_ - -Parameter | Required | Description -----------|----------|------------ -`type` | _yes_ | Manila [share type](https://wiki.openstack.org/wiki/Manila/Concepts#share_type) -`shareNetworkID` | _no_ | Manila [share network ID](https://wiki.openstack.org/wiki/Manila/Concepts#share_network) -`availability` | _no_ | Manila availability zone of the provisioned share. If none is provided, the default Manila zone will be used. Note that this parameter is opaque to the CO and does not influence placement of workloads that will consume this share, meaning they may be scheduled onto any node of the cluster. If the specified Manila AZ is not equally accessible from all compute nodes of the cluster, use [Topology-aware dynamic provisioning](#topology-aware-dynamic-provisioning). -`cephfs-mounter` | _no_ | Relevant for CephFS Manila shares. Specifies which mounting method to use with the CSI CephFS driver. Available options are `kernel` and `fuse`, defaults to `fuse`. See [CSI CephFS docs](https://github.com/ceph/ceph-csi/blob/csi-v1.0/docs/deploy-cephfs.md#configuration) for further information. -`nfs-shareClient` | _no_ | Relevant for NFS Manila shares. Specifies what address has access to the NFS share. Defaults to `0.0.0.0/0`, i.e. anyone. - -### Node Service volume context - -_Kubernetes PV CSI volume attributes for pre-provisioned volumes_ - -Parameter | Required | Description -----------|----------|------------ -`shareID` | if `shareName` is not given | The UUID of the share -`shareName` | if `shareID` is not given | The name of the share -`shareAccessID` | _yes_ | The UUID of the access rule for the share -`cephfs-mounter` | _no_ | Relevant for CephFS Manila shares. Specifies which mounting method to use with the CSI CephFS driver. Available options are `kernel` and `fuse`, defaults to `fuse`. See [CSI CephFS docs](https://github.com/ceph/ceph-csi/blob/csi-v1.0/docs/deploy-cephfs.md#configuration) for further information. - -_Note that the Node Plugin of CSI Manila doesn't care about the origin of a share. As long as the share protocol is supported, CSI Manila is able to consume dynamically provisioned as well as pre-provisioned shares (e.g. shares created manually)._ - -### Secrets, authentication - -Authentication with OpenStack may be done using either user or trustee credentials. - -Mandatory secrets: `os-authURL`, `os-region`. - -Mandatory secrets for _user authentication:_ `os-password`, `os-userID` or `os-userName`, `os-domainID` or `os-domainName`, `os-projectID` or `os-projectName`. - -Optional secrets for _user authentication:_ `os-projectDomainID` or `os-projectDomainName`, `os-userDomainID` or `os-userDomainName` - -Mandatory secrets for _application credential authentication:_ `os-applicationCredentialID` or `os-applicationCredentialName` (when `os-userID` or `os-userName` and `os-domainName` are set), `os-applicationCredentialSecret`. - -Mandatory secrets for _trustee authentication:_ `os-trustID`, `os-trusteeID`, `os-trusteePassword`. - -Optionally, a custom certificate may be sourced via `os-certAuthorityPath` (path to a PEM file inside the plugin container). By default, the usual TLS verification is performed. To override this behavior and accept insecure certificates, set `os-TLSInsecure` to `true` (defaults to `false`). - -### Topology-aware dynamic provisioning - -Topology-aware dynamic provisioning makes it possible to reliably provision and use shares that are _not_ equally accessible from all compute nodes due to storage topology constraints. -With topology awareness enabled, administrators can specify the mapping between compute and Manila availability zones. -Doing so will instruct the CO scheduler to place the workloads+shares only on nodes that are able to reach the underlying storage. - -CSI Manila uses `topology.manila.csi.openstack.org/zone` _topology key_ to identify node's affinity to a certain compute availability zone. -Each node of the cluster then gets labeled with a key/value pair of `topology.manila.csi.openstack.org/zone` / value of [`--nodeaz`](#command-line-arguments) cmd arg. - -This label may be used as a node selector when defining topology constraints for dynamic provisioning. -Administrators are also free to pass arbitrary labels, and as long as they are valid node selectors, they will be honored by the scheduler. - -``` - Topology-aware storage class example: - - -Storage AZ does not influence - the placement of workloads. Compute AZs do. - - +-----------+ +---------------+ - | Manila AZ | | Compute AZs | - | zone-a | apiVersion: storage.k8s.io/v1 | nova-1 | - +-----------+ kind: StorageClass | nova-2 | - | metadata: +---------------+ - | name: cephfs-gold | - | provisioner: cephfs.manila.csi.openstack.org | - | parameters: | - +---------+ availability: zone-a | - ... | - allowedTopologies: +------------------------+ - - matchLabelExpressions: - - key: topology.manila.csi.openstack.org/zone - values: - - nova-1 - - nova-2 - - - Shares in zone-a are accessible only from nodes in nova-1 and nova-2. -``` - -[Enabling topology awareness in Kubernetes](#enabling-topology-awareness) - -### Runtime configuration file - -CSI Manila's runtime configuration file is a JSON document for modifying behavior of the driver at runtime. - -Schema: - -* Root object: - Attribute | Type | Description - ----------|------|------------ - `nfs` | `NfsConfig` | Configuration for NFS shares. Optional. -* `NfsConfig`: - Attribute | Type | Description - ----------|------|------------ - `matchExportLocationAddress` | `string` | When mounting an NFS share, select an export location with matching IP address. No match between this address and at least a single export location for this share will result in an error. Expects a CIDR-formatted address. If prefix is not provided, /32 or /128 prefix is assumed for IPv4 and IPv6 respectively. Optional. - -In Kubernetes, you may store this configuration in a [ConfigMap](https://kubernetes.io/docs/concepts/configuration/configmap/) and expose it to CSI Manila pods as a [volume](https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#add-configmap-data-to-a-volume). Then enter the path to the file populated by the ConfigMap into `--runtime-config-file`. Demo ConfigMap is located in `examples/manila-csi-plugin/runtimeconfig-cm.yaml`. If you're deploying CSI Manila with Helm, setting `csimanila.runtimeConfig.enabled` to `true` will take care of the setup. - -## Deployment - -The CSI Manila driver deals with the Manila service only. All node-related operations (attachments, mounts) are performed by a dedicated CSI Node Plugin, to which all Node Service RPCs are forwarded. This means that the operator is expected to already have a working deployment of that dedicated CSI Node Plugin. - -A single instance of the driver may serve only a single Manila share protocol. To serve multiple share protocols, multiple deployments of the driver need to be made. In order to avoid deployment collisions, each instance of the driver should be named differently, e.g. `csi-manila-cephfs`, `csi-manila-nfs`. - -### Kubernetes 1.15+ - -Snapshots require `VolumeSnapshotDataSource=true` feature gate. - -The deployment consists of two main components: Controller and Node plugins along with their respective RBACs. Controller plugin is deployed as a StatefulSet and runs CSI Manila, [external-provisioner](https://github.com/kubernetes-csi/external-provisioner) and [external-snapshotter](https://github.com/kubernetes-csi/external-snapshotter). Node plugin is deployed as a DaemonSet and runs CSI Manila and [csi-node-driver-registrar](https://github.com/kubernetes-csi/node-driver-registrar). - -**Deploying with Helm** - -This is the preferred way of deployment because it greatly simplifies the difficulties with managing multiple share protocols. - -CSI Manila Helm chart is located in `charts/manila-csi-plugin`. - -First, modify `values.yaml` to suite your environment, and then simply install the Helm chart with `$ helm install ./charts/manila-csi-plugin`. - -Note that the release name generated by `helm install` may not be suitable due to their length. The chart generates object names with the release name included in them, which may cause the names to exceed 63 characters and result in chart installation failure. You may use `--name` flag to set the release name manually. See [helm installation docs](https://helm.sh/docs/helm/#helm-install) for more info. Alternatively, you may also use `nameOverride` or `fullnameOverride` variables in `values.yaml` to override the respective names. - -**Manual deployment** - -All Kubernetes YAML manifests are located in `manifests/manila-csi-plugin`. - -First, deploy the RBACs: -``` -kubectl create -f csi-controllerplugin-rbac.yaml -kubectl create -f csi-nodeplugin-rbac.yaml -``` - -Deploy the CSIDriver CRD: -``` -kubectl create -f csidriver.yaml -``` - -Before continuing, `csi-controllerplugin.yaml` and `csi-nodeplugin.yaml` manifests need to be modified: `fwd-plugin-dir` needs to point to the correct path containing the `.sock` file of the other CSI driver. -Next, `MANILA_SHARE_PROTO` and `FWD_CSI_ENDPOINT` environment variables must be set. Consult `--share-protocol-selector` and `--fwdendpoint` [command line flags](#command-line-arguments) for valid values. - -Finally, deploy Controller and Node plugins: -``` -kubectl create -f csi-controllerplugin.yaml -kubectl create -f csi-nodeplugin.yaml -``` - -#### Verifying the deployment - -Successful deployment should look similar to this: - -``` -$ kubectl get all -NAME READY STATUS RESTARTS AGE -pod/openstack-manila-csi-controllerplugin-0 3/3 Running 0 2m8s -pod/openstack-manila-csi-nodeplugin-ln2xk 2/2 Running 0 2m2s - -NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE -service/openstack-manila-csi-controllerplugin ClusterIP 10.102.188.171 12345/TCP 2m8s - -NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE -daemonset.apps/openstack-manila-csi-nodeplugin 1 1 1 1 1 2m2s - -NAME READY AGE -statefulset.apps/openstack-manila-csi-controllerplugin 1/1 2m8s - -... -``` - -To test the deployment further, see `examples/csi-manila-plugin`. - -#### Enabling topology awareness - -If you're deploying CSI Manila with Helm: -1. Set `csimanila.topologyAwarenessEnabled` to `true` -2. Set `csimanila.nodeAZ`. This value will be sourced into the [`--nodeaz`](#command-line-arguments) cmd flag. Bash expressions are also allowed. - -If you're deploying CSI Manila manually: -1. Run the [external-provisioner](https://github.com/kubernetes-csi/external-provisioner) with `--feature-gates=Topology=true` cmd flag. -2. Run CSI Manila with [`--with-topology`](#command-line-arguments) and set [`--nodeaz`](#command-line-arguments) to node's availability zone. For Nova, the zone may be retrieved via the Metadata service like so: `--nodeaz=$(curl http://169.254.169.254/openstack/latest/meta_data.json | jq -r .availability_zone)` - -See `examples/csi-manila-plugin/nfs/topology-aware` for examples on defining topology constraints. - -## Share protocol support matrix - -The table below shows Manila share protocols currently supported by CSI Manila and their corresponding CSI Node Plugins which must be deployed alongside CSI Manila. - -Manila share protocol | CSI Node Plugin -----------------------|---------------- -`CEPHFS` | [CSI CephFS](https://github.com/ceph/ceph-csi) : v1.0.0 -`NFS` | [CSI NFS](https://github.com/kubernetes-csi/csi-driver-nfs) : v1.0.0 - -## For developers - -If you'd like to contribute to CSI Manila, check out `docs/developers-csi-manila.md` to get you started. - diff --git a/docs/using-octavia-ingress-controller.md b/docs/using-octavia-ingress-controller.md deleted file mode 100644 index 0a85953607..0000000000 --- a/docs/using-octavia-ingress-controller.md +++ /dev/null @@ -1,394 +0,0 @@ - - -**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* - -- [Get started with octavia-ingress-controller for Kubernetes](#get-started-with-octavia-ingress-controller-for-kubernetes) - - [What is an Ingress Controller?](#what-is-an-ingress-controller) - - [Why octavia-ingress-controller](#why-octavia-ingress-controller) - - [Requirements](#requirements) - - [Deploy octavia-ingress-controller in the Kubernetes cluster](#deploy-octavia-ingress-controller-in-the-kubernetes-cluster) - - [Create service account and grant permissions](#create-service-account-and-grant-permissions) - - [Prepare octavia-ingress-controller configuration](#prepare-octavia-ingress-controller-configuration) - - [Deploy octavia-ingress-controller](#deploy-octavia-ingress-controller) - - [Setting up HTTP Load Balancing with Ingress](#setting-up-http-load-balancing-with-ingress) - - [Create a backend service](#create-a-backend-service) - - [Create an Ingress resource](#create-an-ingress-resource) - - [Enable TLS encryption](#enable-tls-encryption) - - - -# Get started with octavia-ingress-controller for Kubernetes - -This guide explains how to deploy and config the octavia-ingress-controller in Kubernetes cluster on top of OpenStack cloud. - -> NOTE: octavia-ingress-controller is still in Beta, support for the overall feature will not be dropped, though details may change. - -## What is an Ingress Controller? -In Kubernetes, Ingress allows external users and client applications access to HTTP services. Ingress consists of two components. - -- [Ingress Resource](https://kubernetes.io/docs/concepts/services-networking/ingress/#the-ingress-resource) is a collection of rules for the inbound traffic to reach Services. These are Layer 7 (L7) rules that allow hostnames (and optionally paths) to be directed to specific Services in Kubernetes. -- [Ingress Controller](https://kubernetes.io/docs/concepts/services-networking/ingress/#ingress-controllers) which acts upon the rules set by the Ingress Resource, typically via an HTTP or L7 load balancer. - -It is vital that both pieces are properly configured to route traffic from an outside client to a Kubernetes Service. - -## Why octavia-ingress-controller -After creating a Kubernetes cluster, the most common way to expose the application to the outside world is to use [LoadBalancer](https://kubernetes.io/docs/concepts/services-networking/service/#type-loadbalancer) type service. In the OpenStack cloud, Octavia(LBaaS v2) is the default implementation of LoadBalancer type service, as a result, for each LoadBalancer type service, there is a load balancer created in the cloud tenant account. We could see some drawbacks of this way: - -- The cost of Kubernetes Service is a little bit high if it's one-to-one mapping from the service to Octavia load balancer, the customers have to pay for a load balancer per exposed service, which can get expensive. -- There is no filtering, no routing, etc. for the service. This means you can send almost any kind of traffic to it, like HTTP, TCP, UDP, Websockets, gRPC, or whatever. -- The traditional ingress controllers(such as NGINX ingress controller, HAProxy ingress controller, Traefik ingress controller, etc.) don't make much sense in the cloud environment because they still rely on the cloud load balancing service to expose themselves behind a Service of LoadBalancer type, not to mention the overhead of managing the extra softwares. - -The octavia-ingress-controller could solve all the above problems in the OpenStack environment by creating a single load balancer for multiple [NodePort](https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport) type services in an Ingress. In order to use the octavia-ingress-controller in Kubernetes cluster, set the annotation `kubernetes.io/ingress.class` in the `metadata` section of the Ingress resource as shown below: - -```yaml -annotations: - kubernetes.io/ingress.class: "openstack" -``` - -## Requirements - -octavia-ingress-controller implementation relies on load balancer management by OpenStack Octavia service, so: - -- Communication between octavia-ingress-controller and Octavia is needed. -- Octavia stable/queens or higher version is required because of some needed features such as bulk pool members operation. -- OpenStack Key Manager(Barbican) service is required for TLS Ingress, otherwise Ingress creation will fail. - -## Deploy octavia-ingress-controller in the Kubernetes cluster - -In the guide, we will deploy octavia-ingress-controller as a StatefulSet(with only one pod) in the kube-system namespace in the cluster. Alternatively, you can also deploy the controller as a static pod by providing a manifest file in the `/etc/kubernetes/manifests` folder in a typical Kubernetes cluster installed by kubeadm. All the manifest files in this guide are saved in `/etc/kubernetes/octavia-ingress-controller` folder, so create the folder first. - -```shell -mkdir -p /etc/kubernetes/octavia-ingress-controller -``` - -### Create service account and grant permissions - -For testing purpose, we grant the cluster admin role to the serviceaccount created. - -```shell -cat < /etc/kubernetes/octavia-ingress-controller/serviceaccount.yaml ---- -kind: ServiceAccount -apiVersion: v1 -metadata: - name: octavia-ingress-controller - namespace: kube-system ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: octavia-ingress-controller -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cluster-admin -subjects: - - kind: ServiceAccount - name: octavia-ingress-controller - namespace: kube-system -EOF -kubectl apply -f /etc/kubernetes/octavia-ingress-controller/serviceaccount.yaml -``` - -### Prepare octavia-ingress-controller configuration - -The octavia-ingress-controller needs to communicate with OpenStack cloud to create resources corresponding to the Kubernetes Ingress resource, so the credentials of an OpenStack user(doesn't need to be the admin user) need to be provided in `openstack` section. Additionally, in order to differentiate the Ingresses between kubernetes clusters, `cluster-name` needs to be unique. - -```shell -cat < /etc/kubernetes/octavia-ingress-controller/config.yaml ---- -kind: ConfigMap -apiVersion: v1 -metadata: - name: octavia-ingress-controller-config - namespace: kube-system -data: - config: | - cluster-name: ${cluster_name} - openstack: - auth-url: ${auth_url} - domain-name: ${domain-name} - username: ${user_id} - password: ${password} - project-id: ${project_id} - region: ${region} - octavia: - subnet-id: ${subnet_id} - floating-network-id: ${public_net_id} -EOF -kubectl apply -f /etc/kubernetes/octavia-ingress-controller/config.yaml -``` - -Here are several other config options are not included in the example configuration above: - -- Options for connecting to the kubernetes cluster. The configuration above will leverage the service account credential which is going to be injected into the pod automatically(see more details [here](https://kubernetes.io/docs/tasks/access-application-cluster/access-cluster/#accessing-the-api-from-a-pod)). However, there may be some reasons to specify the configuration explicitly. - - ```yaml - kubernetes: - api-host: https://127.0.0.1:6443 - kubeconfig: /home/ubuntu/.kube/config - ``` - -- Options for security group management. The octavia-ingress-controller creates an Octavia load balancer per Ingress and adds the worker nodes as members of the load balancer. In order for the Octavia amphorae talking to the Service NodePort, either the kubernetes cluster administrator manually manages the security group for the worker nodes or leave it to octavia-ingress-controller. For the latter case, you should config: - - ```yaml - octavia: - manage-security-groups: true - ``` - - Notes for the security group: - - - The security group name is in the format: `k8s_ing___` - - The security group description is in the format: `Security group created for Ingress / from cluster ` - - The security group has tags: `["octavia.ingress.kubernetes.io", "_"]` - - The security group is associated with all the Neutron ports of the Kubernetes worker nodes. - -### Deploy octavia-ingress-controller - -```shell -image="docker.io/k8scloudprovider/octavia-ingress-controller:latest" - -cat < /etc/kubernetes/octavia-ingress-controller/deployment.yaml ---- -kind: StatefulSet -apiVersion: apps/v1 -metadata: - name: octavia-ingress-controller - namespace: kube-system - labels: - k8s-app: octavia-ingress-controller -spec: - replicas: 1 - selector: - matchLabels: - k8s-app: octavia-ingress-controller - serviceName: octavia-ingress-controller - template: - metadata: - labels: - k8s-app: octavia-ingress-controller - spec: - serviceAccountName: octavia-ingress-controller - tolerations: - - effect: NoSchedule # Make sure the pod can be scheduled on master kubelet. - operator: Exists - - key: CriticalAddonsOnly # Mark the pod as a critical add-on for rescheduling. - operator: Exists - - effect: NoExecute - operator: Exists - containers: - - name: octavia-ingress-controller - image: ${image} - imagePullPolicy: IfNotPresent - args: - - /bin/octavia-ingress-controller - - --config=/etc/config/octavia-ingress-controller-config.yaml - volumeMounts: - - mountPath: /etc/kubernetes - name: kubernetes-config - readOnly: true - - name: ingress-config - mountPath: /etc/config - hostNetwork: true - volumes: - - name: kubernetes-config - hostPath: - path: /etc/kubernetes - type: Directory - - name: ingress-config - configMap: - name: octavia-ingress-controller-config - items: - - key: config - path: octavia-ingress-controller-config.yaml -EOF -kubectl apply -f /etc/kubernetes/octavia-ingress-controller/deployment.yaml -``` - -Wait until the StatefulSet is up and running. - -## Setting up HTTP Load Balancing with Ingress - -### Create a backend service -Create a simple web service that is listening on a HTTP server on port 8080. - -```bash -$ cat < 8080:32461/TCP 9h -``` - -When you create a Service of type NodePort, Kubernetes makes your Service available on a randomly selected high port number (e.g. 32461) on all the nodes in your cluster. Generally the Kubernetes nodes are not externally accessible by default, creating this Service does not make your application accessible from the Internet. However, we could verify the service using its `CLUSTER-IP` on Kubernetes master node: - -```bash -$ curl http://10.105.129.150:8080 -webserver-58fcfb75fb-dz5kn -``` - -Next, we create an Ingress resource to make your HTTP web server application publicly accessible. - -### Create an Ingress resource - -The following command defines an Ingress resource that forwards traffic that requests `http://foo.bar.com/ping` to the webserver: - -```bash -cat < If you don't want your Ingress to be accessible from the public internet, you could set the annotation `octavia.ingress.kubernetes.io/internal` to true. - -Verify that Ingress Resource has been created. Please note that the IP address for the Ingress Resource will not be defined right away (wait for the ADDRESS field to get populated): - -```bash -$ kubectl get ing -NAME CLASS HOSTS ADDRESS PORTS AGE -test-octavia-ingress foo.bar.com 80 12s -$ # Wait until the ingress gets an IP address -$ kubectl get ing -NAME CLASS HOSTS ADDRESS PORTS AGE -test-octavia-ingress foo.bar.com 103.197.62.239 80 25s -``` - -For testing purpose, you can log into a host that could connect to the floating IP, you should be able to access the backend service by sending HTTP request to the domain name specified in the Ingress resource: - -```shell -$ IPADDRESS=103.197.62.239 -$ curl -H "Host: foo.bar.com" http://$IPADDRESS/ping -webserver-58fcfb75fb-dz5kn -``` - -## Enable TLS encryption - -In the example below, we are going generate TLS certificates and keys for the -Ingress and enable the more secure HTTPS protocol. - -1. Generate server TLS certificate and key for foo.bar.com using a - self-signed CA. When generating certificates using the script, just use - simple password e.g. 1234, the passphrase is removed from the private key - in the end. In production, it's recommended to use [Let's - Encrypt](https://letsencrypt.org/) or other certificate authorities to - generate real server certificates and keys. - - ```shell - $ ll - total 0 - $ curl -SLO https://gist.github.com/lingxiankong/47aa743de380a1f122a900d39cff02b3/raw/f7886bedeb615bef2964775b7ca67a38552180c3/gen_certs.sh - % Total % Received % Xferd Average Speed Time Time Time Current - Dload Upload Total Spent Left Speed - 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 - 100 966 100 966 0 0 1588 0 --:--:-- --:--:-- --:--:-- 1588 - $ bash gen_certs.sh - Enter your server domain [foo.bar.com]: - Create CA cert(self-signed) and key... - Create server key... - Enter pass phrase for .key: - Verifying - Enter pass phrase for .key: - Remove password... - Enter pass phrase for .key: - Create server certificate signing request... - Sign SSL certificate... - Succeed! - $ ll - total 24 - -rw-rw-r-- 1 stack stack 1346 Feb 14 15:38 ca.crt - -rw------- 1 stack stack 1704 Feb 14 15:38 ca.key - -rw-rw-r-- 1 stack stack 966 Feb 14 15:38 gen_certs.sh - -rw-rw-r-- 1 stack stack 1038 Feb 14 15:38 foo.bar.com.crt - -rw-rw-r-- 1 stack stack 672 Feb 14 15:38 foo.bar.com.csr - -rw------- 1 stack stack 887 Feb 14 15:38 foo.bar.com.key - ``` - -1. Create Kubernetes secret using the certificates created. - - ```shell script - kubectl create secret tls tls-secret --cert foo.bar.com.crt --key foo.bar.com.key - ``` - -1. Create a TLS Ingress and wait for it's allocated the IP address. - - ```shell script - cat < NOTE: octavia-ingress-controller currently doesn't support to integrate with -`cert-manager` to create the non-existing secret dynamically. Could be improved -in the future. diff --git a/docs/using-openstack-cloud-controller-manager.md b/docs/using-openstack-cloud-controller-manager.md deleted file mode 100644 index c00708f243..0000000000 --- a/docs/using-openstack-cloud-controller-manager.md +++ /dev/null @@ -1,223 +0,0 @@ - - -**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* - -- [Get started with external openstack-cloud-controller-manager in Kubernetes](#get-started-with-external-openstack-cloud-controller-manager-in-kubernetes) - - [Deploy a Kubernetes cluster with openstack-cloud-controller-manager using kubeadm](#deploy-a-kubernetes-cluster-with-openstack-cloud-controller-manager-using-kubeadm) - - [Prerequisites](#prerequisites) - - [Steps](#steps) - - [Migrating from in-tree openstack cloud provider to external openstack-cloud-controller-manager](#migrating-from-in-tree-openstack-cloud-provider-to-external-openstack-cloud-controller-manager) - - [Config openstack-cloud-controller-manager](#config-openstack-cloud-controller-manager) - - [Global](#global) - - [Networking](#networking) - - [Load Balancer](#load-balancer) - - [Metadata](#metadata) - - [Exposing applications using services of LoadBalancer type](#exposing-applications-using-services-of-loadbalancer-type) - - - -# Get started with external openstack-cloud-controller-manager in Kubernetes - -External cloud providers were introduced as an Alpha feature in Kubernetes release 1.6. openstack-cloud-controller-manager is the implementation of external cloud provider for OpenStack clusters. An external cloud provider is a kubernetes controller that runs cloud provider-specific loops required for the functioning of kubernetes. These loops were originally a part of the `kube-controller-manager`, but they were tightly coupling the `kube-controller-manager` to cloud-provider specific code. In order to free the kubernetes project of this dependency, the `cloud-controller-manager` was introduced. - -`cloud-controller-manager` allows cloud vendors and kubernetes core to evolve independent of each other. In prior releases, the core Kubernetes code was dependent upon cloud provider-specific code for functionality. In future releases, code specific to cloud vendors should be maintained by the cloud vendor themselves, and linked to `cloud-controller-manager` while running Kubernetes. - -For more information about cloud-controller-manager, please see: - -- -- -- - -**NOTE: Now, the openstack-cloud-controller-manager implementation is based on OpenStack Octavia, Neutron-LBaaS has been deprecated in OpenStack since Queens release and no longer maintained in openstack-cloud-controller-manager. So make sure to use Octavia if upgrade to the latest openstack-cloud-controller-manager docker image.** - -## Deploy a Kubernetes cluster with openstack-cloud-controller-manager using kubeadm - -The following guide has been tested to install Kubernetes v1.17 on Ubuntu 18.04. - -### Prerequisites - -- docker, kubeadm, kubelet and kubectl has been installed. - -### Steps - -- Create the kubeadm config file according to [`manifests/controller-manager/kubeadm.conf`](https://raw.githubusercontent.com/kubernetes/cloud-provider-openstack/master/manifests/controller-manager/kubeadm.conf) - -- Bootstrap the cluster, make sure to install the [CNI network plugin](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/create-cluster-kubeadm/#pod-network) as well. - - ``` - kubeadm init --config kubeadm.conf - ``` - -- Bootstrap worker nodes. You need to set `--cloud-provider=external` for kubelet service before running `kubeadm join`. - -- Create a secret containing the cloud configuration. You can find an example config file in [`manifests/controller-manager/cloud-config`](https://raw.githubusercontent.com/kubernetes/cloud-provider-openstack/master/manifests/controller-manager/cloud-config). If you have certs you need put the cert file into folder `/etc/ssl/certs/` and update `ca-file` in the configuration file, refer to `ca-file` option [here](https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/using-openstack-cloud-controller-manager.md#global) for further information. After that, Save the configuration to a file named *cloud.conf*, then: - - ```shell - kubectl create secret -n kube-system generic cloud-config --from-file=cloud.conf - ``` - -- Create RBAC resources and openstack-cloud-controller-manager deamonset. - - ```shell - kubectl apply -f https://raw.githubusercontent.com/kubernetes/cloud-provider-openstack/master/cluster/addons/rbac/cloud-controller-manager-roles.yaml - kubectl apply -f https://raw.githubusercontent.com/kubernetes/cloud-provider-openstack/master/cluster/addons/rbac/cloud-controller-manager-role-bindings.yaml - kubectl apply -f https://raw.githubusercontent.com/kubernetes/cloud-provider-openstack/master/manifests/controller-manager/openstack-cloud-controller-manager-ds.yaml - ``` - -- Waiting for all the pods in kube-system namespace up and running. - -## Migrating from in-tree openstack cloud provider to external openstack-cloud-controller-manager - -If you are already running a Kubernetes cluster (installed by kubeadm) but using in-tree openstack cloud provider, switching to openstack-cloud-controller-manager is easy by following the steps in the demo below. - -[![asciicast](https://asciinema.org/a/303399.svg)](https://asciinema.org/a/303399?speed=2) - -## Config openstack-cloud-controller-manager - -Implementation of openstack-cloud-controller-manager relies on several OpenStack services. - -| Service | API Version(s) | Deprecated | Required | -|--------------------------------|----------------|------------|----------| -| Identity (Keystone) | v2 | Yes | No | -| Identity (Keystone) | v3 | No | Yes | -| Compute (Nova) | v2 | No | Yes | -| Load Balancing (Neutron-LBaaS) | v1, v2 | Yes | No | -| Load Balancing (Octavia) | v2 | No | Yes | - -> NOTE: Block Storage is not needed for openstack-cloud-controller-manager in favor of [cinder-csi-plugin](./using-cinder-csi-plugin.md). - -### Global - -The options in `Global` section are used for openstack-cloud-controller-manager authentication with OpenStack Keystone, they are similar to the global options when using `openstack` CLI, see more information in [openstack man page](https://docs.openstack.org/python-openstackclient/latest/cli/man/openstack.html). - -* `auth-url` - Required. Keystone service URL, e.g. http://128.110.154.166/identity -* `ca-file` - Optional. CA certificate bundle file for communication with Keystone service, this is required when using the https protocol in the Keystone service URL. -* `username` - Keystone user name. If you are using [Keystone application credential](https://docs.openstack.org/keystone/latest/user/application_credentials.html), this option is not required. -* `password` - Keystone user password. If you are using [Keystone application credential](https://docs.openstack.org/keystone/latest/user/application_credentials.html), this option is not required. -* `region` - Required. Keystone region name. -* `domain-id` - Keystone user domain ID. If you are using [Keystone application credential](https://docs.openstack.org/keystone/latest/user/application_credentials.html), this option is not required. -* `domain-name` - Keystone user domain name, not required if `domain-id` is set. -* `tenant-id` - Keystone project ID. When using Keystone V3 - which changed the identifier `tenant` to `project` - the `tenant-id` value is automatically mapped to the project construct in the API. - - `tenant-id` is not needed when using `trust-id` or [Keystone application credential](https://docs.openstack.org/keystone/latest/user/application_credentials.html) -* `tenant-name` - Keystone project name, not required if `tenant-id` is set. -* `tenant-domain-id` - Keystone project domain ID. -* `tenant-domain-name` - Keystone project domain name. -* `user-domain-id` - Keystone user domain ID. -* `user-domain-name` - Keystone user domain name. -* `trust-id` - Keystone trust ID. A trust represents a user's (the trustor) authorization to delegate roles to another user (the trustee), and optionally allow the trustee to impersonate the trustor. Available trusts are found under the `/v3/OS-TRUST/trusts` endpoint of the Keystone API. -* `use-clouds` - Set this option to `true` to get authorization credentials from a clouds.yaml file. Options explicitly set in this section are prioritized over values read from clouds.yaml, the file path can be set in `clouds-file` option. Otherwise, the following order is applied: - 1. A file path stored in the environment variable `OS_CLIENT_CONFIG_FILE` - 2. The directory `pkg/cloudprovider/providers/openstack/` - 3. The directory `~/.config/openstack` - 4. The directory `/etc/openstack` -* `clouds-file` - File path of a clouds.yaml file, used together with `use-clouds=true`. -* `cloud` - Used to specify which named cloud in the clouds.yaml file that you want to use, used together with `use-clouds=true`. -* `application-credential-id` - The ID of an application credential to authenticate with. An `application-credential-secret` has to be set along with this parameter. -* `application-credential-name` - The name of an application credential to authenticate with. If `application-credential-id` is not set, the user name and domain need to be set. -* `application-credential-secret` - The secret of an application credential to authenticate with. - -### Networking - -* `ipv6-support-disabled` - Indicates whether or not IPv6 is supported. Default: false -* `public-network-name` - The name of Neutron external network. openstack-cloud-controller-manager uses this option when getting the external IP of the Kubernetes node. Can be specified multiple times. Specified network names will be ORed. Default: "" -* `internal-network-name` - The name of Neutron internal network. openstack-cloud-controller-manager uses this option when getting the internal IP of the Kubernetes node, this is useful if the node has multiple interfaces. Can be specified multiple times. Specified network names will be ORed. Default: "" - -### Load Balancer - -Although the openstack-cloud-controller-manager was initially implemented with Neutron-LBaaS support, Octavia is recommended now because Neutron-LBaaS has been deprecated since Queens OpenStack release cycle and no longer accepted new feature enhancements. As a result, lots of advanced features in openstack-cloud-controller-manager rely on Octavia, even the CI is running based on Octavia enabled OpenStack environment. Functionalities are not guaranteed if using Neutron-LBaaS. - -* `use-octavia` - Whether or not to use Octavia for LoadBalancer type of Service implementation instead of using Neutron-LBaaS. Default: true - -* `floating-network-id` - Optional. The external network used to create floating IP for the load balancer VIP. If there are multiple external networks in the cloud, either this option must be set or user must specify `loadbalancer.openstack.org/floating-network-id` in the Service annotation. - -* `floating-subnet-id` - Optional. The external network subnet used to create floating IP for the load balancer VIP. Can be overridden by the Service annotation `loadbalancer.openstack.org/floating-subnet-id`. - -* `lb-method` - The load balancing algorithm used to create the load balancer pool. The value can be `ROUND_ROBIN`, `LEAST_CONNECTIONS`, or `SOURCE_IP`. Default: `ROUND_ROBIN` - -* `lb-provider` - Optional. Used to specify the provider of the load balancer, e.g. "amphora" or "octavia". - -* `lb-version` - Optional. If specified, only "v2" is supported. - -* `subnet-id` - ID of the Neutron subnet on which to create load balancer VIP, this ID is also used to create pool members. - -* `network-id` - ID of the Neutron network on which to create load balancer VIP, not needed if `subnet-id` is set. - -* `manage-security-groups` - If the Neutron security groups should be managed separately. Default: false - - This option is not supported for Octavia. The worker nodes and the Octavia amphorae are usually in the same subnet, so it's sufficient to config the port security group rules manually for worker nodes, to allow the traffic coming from the the subnet IP range to the node port range(i.e. 30000-32767). - -* `create-monitor` - Indicates whether or not to create a health monitor for the service load balancer. Default: false - -* `monitor-delay` - The time, in seconds, between sending probes to members of the load balancer. Default: 5 - -* `monitor-max-retries` - The number of successful checks before changing the operating status of the load balancer member to ONLINE. A valid value is from 1 to 10. Default: 1 - -* `monitor-timeout` - The maximum time, in seconds, that a monitor waits to connect backend before it times out. Default: 3 - -* `internal-lb` - Determines whether or not to create an internal load balancer (no floating IP) by default. Default: false. - -* `cascade-delete` - Determines whether or not to perform cascade deletion of load balancers. Default: true. - -* `flavor-id` - The id of the loadbalancer flavor to use. Uses octavia default if not set. - -* `LoadBalancerClass "ClassName"` - This is a config section including a set of config options. User can choose the `ClassName` by specifying the Service annotation `loadbalancer.openstack.org/class`. The following options are supported: - - * floating-network-id. The same with `floating-network-id` option above. - * floating-subnet-id. The same with `floating-subnet-id` option above. - * network-id. The same with `network-id` option above. - * subnet-id. The same with `subnet-id` option above. - -### Metadata - -* `search-order` - This configuration key influences the way that the provider retrieves metadata relating to the instance(s) in which it runs. The default value of `configDrive,metadataService` results in the provider retrieving metadata relating to the instance from the config drive first if available and then the metadata service. Alternative values are: - * `configDrive` - Only retrieve instance metadata from the configuration drive. - * `metadataService` - Only retrieve instance metadata from the metadata service. - * `metadataService,configDrive` - Retrieve instance metadata from the metadata service first if available, then the configuration drive. - - Not all OpenStack clouds provide both configuration drive and metadata service though and only one or the other may be available which is why the default is to check both. Especially, the metadata on the config drive may grow stale over time, whereas the metadata service always provides the most up to date data. - -## Exposing applications using services of LoadBalancer type - -Refer to [Exposing applications using services of LoadBalancer type](./expose-applications-using-loadbalancer-type-service.md) diff --git a/examples/cinder-csi-plugin/block/block.yaml b/examples/cinder-csi-plugin/block/block.yaml deleted file mode 100644 index 90ffd556ae..0000000000 --- a/examples/cinder-csi-plugin/block/block.yaml +++ /dev/null @@ -1,41 +0,0 @@ -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: csi-sc-cinderplugin -provisioner: cinder.csi.openstack.org - ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: csi-pvc-cinderplugin -spec: - accessModes: - - ReadWriteOnce - volumeMode: Block - resources: - requests: - storage: 1Gi - storageClassName: csi-sc-cinderplugin - ---- -apiVersion: v1 -kind: Pod -metadata: - name: test-block -spec: - containers: - - image: nginx - imagePullPolicy: IfNotPresent - name: nginx - ports: - - containerPort: 80 - protocol: TCP - volumeDevices: - - devicePath: /dev/xvda - name: csi-data-cinderplugin - volumes: - - name: csi-data-cinderplugin - persistentVolumeClaim: - claimName: csi-pvc-cinderplugin - readOnly: false diff --git a/examples/cinder-csi-plugin/clone/example-clone.yaml b/examples/cinder-csi-plugin/clone/example-clone.yaml deleted file mode 100644 index 1b4f1b7218..0000000000 --- a/examples/cinder-csi-plugin/clone/example-clone.yaml +++ /dev/null @@ -1,41 +0,0 @@ -# This YAML file contains nginx & csi cinder driver objects, -# which are necessary to run nginx with csi cinder driver. - ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: csi-pvc-clone -spec: - dataSource: - name: source-pvc - kind: PersistentVolumeClaim - apiGroup: "" - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi - storageClassName: csi-cinderplugin-sc - ---- -apiVersion: v1 -kind: Pod -metadata: - name: nginx -spec: - containers: - - image: nginx - imagePullPolicy: IfNotPresent - name: nginx - ports: - - containerPort: 80 - protocol: TCP - volumeMounts: - - mountPath: /var/lib/www/html - name: csi-data-cinderplugin - volumes: - - name: csi-data-cinderplugin - persistentVolumeClaim: - claimName: csi-pvc-clone - readOnly: false diff --git a/examples/cinder-csi-plugin/clone/sc.yaml b/examples/cinder-csi-plugin/clone/sc.yaml deleted file mode 100644 index 86122f8f86..0000000000 --- a/examples/cinder-csi-plugin/clone/sc.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: csi-cinderplugin-sc -provisioner: cinder.csi.openstack.org diff --git a/examples/cinder-csi-plugin/clone/source-pvc.yaml b/examples/cinder-csi-plugin/clone/source-pvc.yaml deleted file mode 100644 index abd2b73442..0000000000 --- a/examples/cinder-csi-plugin/clone/source-pvc.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: source-pvc -spec: - storageClassName: csi-cinderplugin-sc - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi diff --git a/examples/cinder-csi-plugin/inline/inline-example.yaml b/examples/cinder-csi-plugin/inline/inline-example.yaml deleted file mode 100644 index f7d3530918..0000000000 --- a/examples/cinder-csi-plugin/inline/inline-example.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: inline-pod -spec: - containers: - - image: nginx - imagePullPolicy: IfNotPresent - name: nginx-inline - volumeMounts: - - name: my-csi-volume - mountPath: /var/lib/www/html - volumes: - - name: my-csi-volume - csi: - driver: cinder.csi.openstack.org - volumeAttributes: - capacity: 1Gi # default is 1Gi - readOnly: false # default is false - fsType: ext4 # default is ext4 diff --git a/examples/cinder-csi-plugin/nginx.yaml b/examples/cinder-csi-plugin/nginx.yaml deleted file mode 100644 index c030889657..0000000000 --- a/examples/cinder-csi-plugin/nginx.yaml +++ /dev/null @@ -1,43 +0,0 @@ -# This YAML file contains nginx & csi cinder driver objects, -# which are necessary to run nginx with csi cinder driver. - -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: csi-sc-cinderplugin -provisioner: cinder.csi.openstack.org - ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: csi-pvc-cinderplugin -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi - storageClassName: csi-sc-cinderplugin - ---- -apiVersion: v1 -kind: Pod -metadata: - name: nginx -spec: - containers: - - image: nginx - imagePullPolicy: IfNotPresent - name: nginx - ports: - - containerPort: 80 - protocol: TCP - volumeMounts: - - mountPath: /var/lib/www/html - name: csi-data-cinderplugin - volumes: - - name: csi-data-cinderplugin - persistentVolumeClaim: - claimName: csi-pvc-cinderplugin - readOnly: false diff --git a/examples/cinder-csi-plugin/resize/example.yaml b/examples/cinder-csi-plugin/resize/example.yaml deleted file mode 100644 index 702dd57362..0000000000 --- a/examples/cinder-csi-plugin/resize/example.yaml +++ /dev/null @@ -1,44 +0,0 @@ -# This YAML file contains nginx & csi cinder driver objects, -# which are necessary to run nginx with csi cinder driver. - -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: csi-sc-cinderplugin -provisioner: cinder.csi.openstack.org -allowVolumeExpansion: true - ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: csi-pvc-cinderplugin -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi - storageClassName: csi-sc-cinderplugin - ---- -apiVersion: v1 -kind: Pod -metadata: - name: nginx -spec: - containers: - - image: nginx - imagePullPolicy: IfNotPresent - name: nginx - ports: - - containerPort: 80 - protocol: TCP - volumeMounts: - - mountPath: /var/lib/www/html - name: csi-data-cinderplugin - volumes: - - name: csi-data-cinderplugin - persistentVolumeClaim: - claimName: csi-pvc-cinderplugin - readOnly: false diff --git a/examples/cinder-csi-plugin/snapshot/example.yaml b/examples/cinder-csi-plugin/snapshot/example.yaml deleted file mode 100644 index 2538a78357..0000000000 --- a/examples/cinder-csi-plugin/snapshot/example.yaml +++ /dev/null @@ -1,30 +0,0 @@ -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: csi-sc-cinderplugin -provisioner: cinder.csi.openstack.org - ---- - -apiVersion: snapshot.storage.k8s.io/v1beta1 -kind: VolumeSnapshotClass -metadata: - name: csi-cinder-snapclass -driver: cinder.csi.openstack.org -deletionPolicy: Delete -parameters: - force-create: "false" - ---- - -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: pvc-snapshot-demo -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi - storageClassName: csi-sc-cinderplugin diff --git a/examples/cinder-csi-plugin/snapshot/snapshotcreate.yaml b/examples/cinder-csi-plugin/snapshot/snapshotcreate.yaml deleted file mode 100644 index 4c98ab9625..0000000000 --- a/examples/cinder-csi-plugin/snapshot/snapshotcreate.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: snapshot.storage.k8s.io/v1beta1 -kind: VolumeSnapshot -metadata: - name: new-snapshot-demo -spec: - volumeSnapshotClassName: csi-cinder-snapclass - source: - persistentVolumeClaimName: pvc-snapshot-demo diff --git a/examples/cinder-csi-plugin/snapshot/snapshotrestore.yaml b/examples/cinder-csi-plugin/snapshot/snapshotrestore.yaml deleted file mode 100644 index 523b0a900f..0000000000 --- a/examples/cinder-csi-plugin/snapshot/snapshotrestore.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: snapshot-demo-restore -spec: - storageClassName: csi-sc-cinderplugin - dataSource: - name: new-snapshot-demo - kind: VolumeSnapshot - apiGroup: snapshot.storage.k8s.io - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi diff --git a/examples/cinder-csi-plugin/topology/example.yaml b/examples/cinder-csi-plugin/topology/example.yaml deleted file mode 100644 index ce667bf840..0000000000 --- a/examples/cinder-csi-plugin/topology/example.yaml +++ /dev/null @@ -1,46 +0,0 @@ -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: topology-aware-standard -provisioner: cinder.csi.openstack.org -volumeBindingMode: WaitForFirstConsumer -allowedTopologies: -- matchLabelExpressions: - - key: topology.cinder.csi.openstack.org/zone - values: - - nova1 - ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: pvc1 -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi - storageClassName: topology-aware-standard - ---- -apiVersion: v1 -kind: Pod -metadata: - name: app -spec: - containers: - - image: nginx - imagePullPolicy: IfNotPresent - name: nginx - ports: - - containerPort: 80 - protocol: TCP - volumeMounts: - - mountPath: /var/lib/www/data - name: mydata - volumes: - - name: mydata - persistentVolumeClaim: - claimName: pvc1 - readOnly: false diff --git a/examples/loadbalancers/README.MD b/examples/loadbalancers/README.MD deleted file mode 100644 index e08ca1e98e..0000000000 --- a/examples/loadbalancers/README.MD +++ /dev/null @@ -1,145 +0,0 @@ -# Loadbalancers - -OpenStack Cloud Controller Manager runs service controller, -which is responsible for watching services of type ```LoadBalancer``` -and creating OpenStack loadbalancers to satisfy its requirements. -Here are some examples of how it's used. - -## External HTTP loadbalancer - -When you create a service with ```type: LoadBalancer```, an OpenStack load balancer will be created. -The example below will create a nginx deployment and expose it via an OpenStack External load balancer. - -```yaml ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: external-http-nginx-deployment -spec: - replicas: 2 - selector: - matchLabels: - app: nginx - template: - metadata: - labels: - app: nginx - spec: - containers: - - name: nginx - image: nginx - ports: - - containerPort: 80 ---- -kind: Service -apiVersion: v1 -metadata: - name: external-http-nginx-service - annotations: - service.beta.kubernetes.io/openstack-internal-load-balancer: "false" - loadbalancer.openstack.org/floating-network-id: "9be23551-38e2-4d27-b5ea-ea2ea1321bd6" -spec: - selector: - app: nginx - type: LoadBalancer - ports: - - name: http - port: 80 - targetPort: 80 -``` - -The ```service.beta.kubernetes.io/openstack-internal-load-balancer``` annotation -is used on the service to indicate that we want an internal loadbalancer service. -If the value of ```service.beta.kubernetes.io/openstack-internal-load-balancer``` is false, -it indicates that we want an external loadbalancer service. Default to false. - -The ```loadbalancer.openstack.org/floating-network-id``` annotation -indicates that it will create a floating IP for the external loadbalancer service -on the specified floating network id. This annotation works when the value of -```service.beta.kubernetes.io/openstack-internal-load-balancer``` is false. -If this annotation is not specified, it will use the default floating network id. - - -```bash -$ kubectl create -f examples/loadbalancers/external-http-nginx.yaml -``` - -Watch the service and await an ```EXTERNAL-IP``` by the following command. -This will be the load balancer IP which you can use to connect to your service. - -```bash -$ watch kubectl get service -NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE -http-nginx-service 10.0.0.10 122.112.219.229 80:30000/TCP 5m -``` - -You can now access your service via the provisioned load balancer. - -```bash -$ curl http://122.112.219.229 -``` - -## Internal HTTP loadbalancer - -The example below will create a nginx deployment and expose it via an OpenStack Internal load balancer. - -```yaml ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: internal-http-nginx-deployment -spec: - replicas: 2 - selector: - matchLabels: - app: nginx - template: - metadata: - labels: - app: nginx - spec: - containers: - - name: nginx - image: nginx - ports: - - containerPort: 80 ---- -kind: Service -apiVersion: v1 -metadata: - name: internal-http-nginx-service - annotations: - service.beta.kubernetes.io/openstack-internal-load-balancer: "true" -spec: - selector: - app: nginx - type: LoadBalancer - ports: - - name: http - port: 80 - targetPort: 80 -``` - -The value of ```service.beta.kubernetes.io/openstack-internal-load-balancer``` is true, -it indicates that we want an internal loadbalancer service. - -```bash -$ kubectl create -f examples/loadbalancers/internal-http-nginx.yaml -``` - -Watch the service and await an ```EXTERNAL-IP``` by the following command. -This will be the load balancer IP which you can use to connect to your service. - -```bash -$ watch kubectl get service -NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE -http-nginx-service 10.0.0.10 192.168.0.181 80:30000/TCP 5m -``` - -You can now access your service via the provisioned load balancer. - -```bash -$ curl http://192.168.0.181 -``` diff --git a/examples/loadbalancers/external-http-nginx.yaml b/examples/loadbalancers/external-http-nginx.yaml deleted file mode 100644 index a5f963a513..0000000000 --- a/examples/loadbalancers/external-http-nginx.yaml +++ /dev/null @@ -1,36 +0,0 @@ ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: external-http-nginx-deployment -spec: - replicas: 2 - selector: - matchLabels: - app: nginx - template: - metadata: - labels: - app: nginx - spec: - containers: - - name: nginx - image: nginx - ports: - - containerPort: 80 ---- -kind: Service -apiVersion: v1 -metadata: - name: external-http-nginx-service - annotations: - service.beta.kubernetes.io/openstack-internal-load-balancer: "false" - loadbalancer.openstack.org/floating-network-id: "9be23551-38e2-4d27-b5ea-ea2ea1321bd6" -spec: - selector: - app: nginx - type: LoadBalancer - ports: - - name: http - port: 80 - targetPort: 80 diff --git a/examples/loadbalancers/internal-http-nginx.yaml b/examples/loadbalancers/internal-http-nginx.yaml deleted file mode 100644 index 7b7864a850..0000000000 --- a/examples/loadbalancers/internal-http-nginx.yaml +++ /dev/null @@ -1,35 +0,0 @@ ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: internal-http-nginx-deployment -spec: - replicas: 2 - selector: - matchLabels: - app: nginx - template: - metadata: - labels: - app: nginx - spec: - containers: - - name: nginx - image: nginx - ports: - - containerPort: 80 ---- -kind: Service -apiVersion: v1 -metadata: - name: internal-http-nginx-service - annotations: - service.beta.kubernetes.io/openstack-internal-load-balancer: "true" -spec: - selector: - app: nginx - type: LoadBalancer - ports: - - name: http - port: 80 - targetPort: 80 diff --git a/examples/manila-csi-plugin/README.md b/examples/manila-csi-plugin/README.md deleted file mode 100644 index 298db2bc98..0000000000 --- a/examples/manila-csi-plugin/README.md +++ /dev/null @@ -1,40 +0,0 @@ -## General notes before continuing - -1. Make sure you've completed all the steps in `docs/using-manila-csi-plugin.md`: e.g. you've deployed CSI NFS and CSI Manila plugins and CSI Manila is running with `--share-protocol-selector=NFS` and `--fwdendpoint=unix:///csi/csi-nfsplugin/csi.sock` (or similar, based on your environment) -2. Modify `secrets.yaml` to suite your OpenStack cloud environment. Refer to the _"Secrets, authentication"_ section of CSI Manila docs. You may also use helper scripts from `examples/manila-provisioner` to generate the Secrets manifest. -3. The same steps apply to all supported Manila share protocols -4. `exec-bash.sh`, `logs.sh` are convenience scripts for debugging CSI Manila - -## Example CSI Manila usage with NFS shares - -``` -nfs/ -├── dynamic-provisioning/ -│   ├── pod.yaml -│   ├── pvc.yaml -│   └── --> storageclass.yaml <-- -├── snapshot/ -│   ├── pod.yaml -│   ├── --> snapshotclass.yaml <-- -│   ├── snapshotcreate.yaml -│   └── snapshotrestore.yaml -├── static-provisioning/ -│ ├── pod.yaml -│ └── --> preprovisioned-pvc.yaml <-- -├── topology-aware/ -│   ├── pod.yaml -│   ├── pvc.yaml -│   └── --> storageclass.yaml <-- -└── --> secrets.yaml <-- -``` - -Files marked with `--> ... <--` may need to be customized. - -* `dynamic-provisioning/` : creates a new Manila NFS share and mounts it in a Pod. -* `static-provisioning/` : fetches an existing Manila NFS share and mounts it in a Pod -* `snapshot/` : takes a snapshot from a PVC source, restores it into a new share and mounts it in a Pod. Deploy manifests in `dynamic-provisioning/` first -* `topology-aware/` : topology-aware dynamic provisioning - -Make sure the `provisioner` field in `storageclass.yaml` and `snapshotclass.yaml` matches the driver name in your deployment! - -After deploying each example you should see the corresponding Pod with status _Running_ soon. diff --git a/examples/manila-csi-plugin/exec-bash.sh b/examples/manila-csi-plugin/exec-bash.sh deleted file mode 100755 index b984b18a3f..0000000000 --- a/examples/manila-csi-plugin/exec-bash.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -CONTAINER_NAME=csi-nodeplugin-manilaplugin -POD_NAME=$(kubectl get pods -l app=$CONTAINER_NAME -o=name | head -n 1) - -function get_pod_status() { - echo -n "$(kubectl get "$POD_NAME" -o jsonpath="{.status.phase}")" -} - -while [[ "$(get_pod_status)" != "Running" ]]; do - sleep 1 - echo "Waiting for $POD_NAME (status $(get_pod_status))" -done - -kubectl exec -it "${POD_NAME#*/}" -c "$CONTAINER_NAME" bash diff --git a/examples/manila-csi-plugin/logs.sh b/examples/manila-csi-plugin/logs.sh deleted file mode 100755 index b087307d8a..0000000000 --- a/examples/manila-csi-plugin/logs.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -POD_NAME=$(kubectl get pods -l app=csi-nodeplugin-manilaplugin -o=name | head -n 1) - -function get_pod_status() { - echo -n $(kubectl get $POD_NAME -o jsonpath="{.status.phase}") -} - -while [[ $(get_pod_status) != "Running" ]]; do - sleep 1 - echo "Waiting for $POD_NAME (status $(get_pod_status))" -done - - -kubectl logs -f $POD_NAME -c csi-nodeplugin-manilaplugin diff --git a/examples/manila-csi-plugin/nfs/dynamic-provisioning/pod.yaml b/examples/manila-csi-plugin/nfs/dynamic-provisioning/pod.yaml deleted file mode 100644 index 483489aed6..0000000000 --- a/examples/manila-csi-plugin/nfs/dynamic-provisioning/pod.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: new-nfs-share-pod -spec: - containers: - - name: web-server - image: nginx - imagePullPolicy: IfNotPresent - volumeMounts: - - name: mypvc - mountPath: /var/lib/www - volumes: - - name: mypvc - persistentVolumeClaim: - claimName: new-nfs-share-pvc - readOnly: false diff --git a/examples/manila-csi-plugin/nfs/dynamic-provisioning/pvc.yaml b/examples/manila-csi-plugin/nfs/dynamic-provisioning/pvc.yaml deleted file mode 100644 index 5b95d3482c..0000000000 --- a/examples/manila-csi-plugin/nfs/dynamic-provisioning/pvc.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: new-nfs-share-pvc -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 1Gi - storageClassName: csi-manila-nfs - diff --git a/examples/manila-csi-plugin/nfs/dynamic-provisioning/storageclass.yaml b/examples/manila-csi-plugin/nfs/dynamic-provisioning/storageclass.yaml deleted file mode 100644 index ad690767ce..0000000000 --- a/examples/manila-csi-plugin/nfs/dynamic-provisioning/storageclass.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: csi-manila-nfs -provisioner: nfs.manila.csi.openstack.org -parameters: - # Manila share type - type: default - - csi.storage.k8s.io/provisioner-secret-name: csi-manila-secrets - csi.storage.k8s.io/provisioner-secret-namespace: default - csi.storage.k8s.io/node-stage-secret-name: csi-manila-secrets - csi.storage.k8s.io/node-stage-secret-namespace: default - csi.storage.k8s.io/node-publish-secret-name: csi-manila-secrets - csi.storage.k8s.io/node-publish-secret-namespace: default diff --git a/examples/manila-csi-plugin/nfs/secrets.yaml b/examples/manila-csi-plugin/nfs/secrets.yaml deleted file mode 100644 index bc863e075d..0000000000 --- a/examples/manila-csi-plugin/nfs/secrets.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: csi-manila-secrets - namespace: default -stringData: - # Mandatory - os-authURL: "some-auth-url" - os-region: "some-region" - - # Authentication using user credentials - os-userName: "some-username" - os-password: "some-password" - os-projectName: "some-project-name" - os-domainID: "some-domain-id" - - # Authentication using trustee credentials - os-trustID: "some-trust-id" - os-trusteeID: "some-trustee-id" - os-trusteePassword: "some-trustee-password" diff --git a/examples/manila-csi-plugin/nfs/snapshot/pod.yaml b/examples/manila-csi-plugin/nfs/snapshot/pod.yaml deleted file mode 100644 index baec55b053..0000000000 --- a/examples/manila-csi-plugin/nfs/snapshot/pod.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: new-nfs-share-snap-restore-pod -spec: - containers: - - name: web-server - image: nginx - imagePullPolicy: IfNotPresent - volumeMounts: - - name: mypvc - mountPath: /var/lib/www - volumes: - - name: mypvc - persistentVolumeClaim: - claimName: new-nfs-share-snap-restore - readOnly: false diff --git a/examples/manila-csi-plugin/nfs/snapshot/snapshotclass.yaml b/examples/manila-csi-plugin/nfs/snapshot/snapshotclass.yaml deleted file mode 100644 index fb5b37fc9c..0000000000 --- a/examples/manila-csi-plugin/nfs/snapshot/snapshotclass.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: snapshot.storage.k8s.io/v1alpha1 -kind: VolumeSnapshotClass -metadata: - name: csi-manila-nfs -snapshotter: nfs.manila.csi.openstack.org -parameters: - csi.storage.k8s.io/snapshotter-secret-name: csi-manila-secrets - csi.storage.k8s.io/snapshotter-secret-namespace: default diff --git a/examples/manila-csi-plugin/nfs/snapshot/snapshotcreate.yaml b/examples/manila-csi-plugin/nfs/snapshot/snapshotcreate.yaml deleted file mode 100644 index cce0ace345..0000000000 --- a/examples/manila-csi-plugin/nfs/snapshot/snapshotcreate.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: snapshot.storage.k8s.io/v1alpha1 -kind: VolumeSnapshot -metadata: - name: new-nfs-share-snap -spec: - snapshotClassName: csi-manila-nfs - source: - name: new-nfs-share-pvc - kind: PersistentVolumeClaim diff --git a/examples/manila-csi-plugin/nfs/snapshot/snapshotrestore.yaml b/examples/manila-csi-plugin/nfs/snapshot/snapshotrestore.yaml deleted file mode 100644 index d613152f2e..0000000000 --- a/examples/manila-csi-plugin/nfs/snapshot/snapshotrestore.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: new-nfs-share-snap-restore -spec: - storageClassName: csi-manila-nfs - dataSource: - name: new-nfs-share-snap - kind: VolumeSnapshot - apiGroup: snapshot.storage.k8s.io - accessModes: - - ReadWriteMany - resources: - requests: - storage: 1Gi diff --git a/examples/manila-csi-plugin/nfs/static-provisioning/pod.yaml b/examples/manila-csi-plugin/nfs/static-provisioning/pod.yaml deleted file mode 100644 index 97d27978e4..0000000000 --- a/examples/manila-csi-plugin/nfs/static-provisioning/pod.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: existing-nfs-share-pod -spec: - containers: - - name: web-server - image: nginx - imagePullPolicy: IfNotPresent - volumeMounts: - - name: mypvc - mountPath: /var/lib/www - volumes: - - name: mypvc - persistentVolumeClaim: - claimName: existing-nfs-share-pvc - readOnly: false diff --git a/examples/manila-csi-plugin/nfs/static-provisioning/preprovisioned-pvc.yaml b/examples/manila-csi-plugin/nfs/static-provisioning/preprovisioned-pvc.yaml deleted file mode 100644 index 3b572d1ebf..0000000000 --- a/examples/manila-csi-plugin/nfs/static-provisioning/preprovisioned-pvc.yaml +++ /dev/null @@ -1,39 +0,0 @@ -apiVersion: v1 -kind: PersistentVolume -metadata: - name: preprovisioned-nfs-share - labels: - name: preprovisioned-nfs-share -spec: - accessModes: - - ReadWriteMany - capacity: - storage: 1Gi - csi: - driver: nfs.manila.csi.openstack.org - volumeHandle: preprovisioned-nfs-share - nodeStageSecretRef: - name: csi-manila-secrets - namespace: default - nodePublishSecretRef: - name: csi-manila-secrets - namespace: default - volumeAttributes: - shareID: SHARE-UUID-GOES-HERE - shareAccessID: ACCESS-UUID-OF-THE-SHARE ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: existing-nfs-share-pvc -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 1Gi - selector: - matchExpressions: - - key: name - operator: In - values: ["preprovisioned-nfs-share"] diff --git a/examples/manila-csi-plugin/nfs/topology-aware/pod.yaml b/examples/manila-csi-plugin/nfs/topology-aware/pod.yaml deleted file mode 100644 index 483489aed6..0000000000 --- a/examples/manila-csi-plugin/nfs/topology-aware/pod.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: new-nfs-share-pod -spec: - containers: - - name: web-server - image: nginx - imagePullPolicy: IfNotPresent - volumeMounts: - - name: mypvc - mountPath: /var/lib/www - volumes: - - name: mypvc - persistentVolumeClaim: - claimName: new-nfs-share-pvc - readOnly: false diff --git a/examples/manila-csi-plugin/nfs/topology-aware/pvc.yaml b/examples/manila-csi-plugin/nfs/topology-aware/pvc.yaml deleted file mode 100644 index 9ae1423036..0000000000 --- a/examples/manila-csi-plugin/nfs/topology-aware/pvc.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: new-nfs-share-pvc -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 1Gi - storageClassName: csi-manila-nfs-az - diff --git a/examples/manila-csi-plugin/nfs/topology-aware/storageclass.yaml b/examples/manila-csi-plugin/nfs/topology-aware/storageclass.yaml deleted file mode 100644 index 5c9fd89808..0000000000 --- a/examples/manila-csi-plugin/nfs/topology-aware/storageclass.yaml +++ /dev/null @@ -1,37 +0,0 @@ -# Topology constraints example: -# -# Let's have two Manila AZs: manila-zone-1, manila-zone-2 -# Let's have six Nova AZs: nova-{1..6} -# -# manila-zone-1 is accessible from nodes in nova-{1,2,3} only -# manila-zone-2 is accessible from nodes in nova-{4,5,6} only -# -# We're provisioning into manila-zone-1 -# allowedTopologies reflects the topology constraints - -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: csi-manila-nfs-az -provisioner: nfs.manila.csi.openstack.org -parameters: - type: default - availability: manila-zone-1 - - csi.storage.k8s.io/provisioner-secret-name: csi-manila-secrets - csi.storage.k8s.io/provisioner-secret-namespace: default - csi.storage.k8s.io/node-stage-secret-name: csi-manila-secrets - csi.storage.k8s.io/node-stage-secret-namespace: default - csi.storage.k8s.io/node-publish-secret-name: csi-manila-secrets - csi.storage.k8s.io/node-publish-secret-namespace: default -allowedTopologies: - - matchLabelExpressions: - - key: topology.manila.csi.openstack.org/zone - values: - - nova-1 - - nova-2 - - nova-3 - # ...or you may use other node labels: - # - key: my-zone-label - # values: - # - nodes-that-can-reach-manila-az-1 diff --git a/examples/manila-csi-plugin/runtimeconfig-cm.yaml b/examples/manila-csi-plugin/runtimeconfig-cm.yaml deleted file mode 100644 index d255940070..0000000000 --- a/examples/manila-csi-plugin/runtimeconfig-cm.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: manila-csi-runtimeconf-cm -data: - runtimeconfig.json: | - { - "nfs": { - # When mounting an NFS share, select an export location with - # this IP address. No match between this address and - # at least a single export location for this share will - # result in an error. - # Expects a CIDR-formatted address. If prefix is not provided, - # /32 or /128 prefix is assumed for IPv4 and IPv6 respectively. - "matchExportLocationAddress": "172.168.122.0/24" - } - } diff --git a/examples/mysql-cinder-pd/README.md b/examples/mysql-cinder-pd/README.md deleted file mode 100644 index 40d805bef1..0000000000 --- a/examples/mysql-cinder-pd/README.md +++ /dev/null @@ -1 +0,0 @@ -This file has moved to [https://github.com/kubernetes/examples/blob/master/mysql-cinder-pd/README.md](https://github.com/kubernetes/examples/blob/master/mysql-cinder-pd/README.md) diff --git a/examples/mysql-cinder-pd/mysql-service.yaml b/examples/mysql-cinder-pd/mysql-service.yaml deleted file mode 100644 index 6e2c019ac3..0000000000 --- a/examples/mysql-cinder-pd/mysql-service.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - labels: - name: mysql - name: mysql -spec: - ports: - # the port that this service should serve on - - port: 3306 - # label keys and values that must match in order to receive traffic for this service - selector: - name: mysql \ No newline at end of file diff --git a/examples/mysql-cinder-pd/mysql.yaml b/examples/mysql-cinder-pd/mysql.yaml deleted file mode 100644 index 282f06246e..0000000000 --- a/examples/mysql-cinder-pd/mysql.yaml +++ /dev/null @@ -1,33 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: mysql - labels: - name: mysql -spec: - containers: - - resources: - limits: - cpu: 0.5 - image: mysql - name: mysql - args: - - "--ignore-db-dir" - - "lost+found" - env: - - name: MYSQL_ROOT_PASSWORD - # change this - value: yourpassword - ports: - - containerPort: 3306 - name: mysql - volumeMounts: - # name must match the volume name below - - name: mysql-persistent-storage - # mount path within the container - mountPath: /var/lib/mysql - volumes: - - name: mysql-persistent-storage - cinder: - volumeID: bd82f7e2-wece-4c01-a505-4acf60b07f4a - fsType: ext4 diff --git a/examples/persistent-volume-provisioning/cinder/cinder-full.yaml b/examples/persistent-volume-provisioning/cinder/cinder-full.yaml deleted file mode 100644 index 549253c6ce..0000000000 --- a/examples/persistent-volume-provisioning/cinder/cinder-full.yaml +++ /dev/null @@ -1,43 +0,0 @@ -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: standard - annotations: - storageclass.beta.kubernetes.io/is-default-class: "true" - labels: - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: EnsureExists -provisioner: openstack.org/standalone-cinder ---- -kind: PersistentVolumeClaim -apiVersion: v1 -metadata: - name: myclaim -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi # pass here the size of the volume - storageClassName: standard ---- -apiVersion: v1 -kind: Pod -metadata: - name: web -spec: - containers: - - name: web - image: nginx - ports: - - name: web - containerPort: 80 - hostPort: 8081 - protocol: TCP - volumeMounts: - - mountPath: "/usr/share/nginx/html" - name: mypd - volumes: - - name: mypd - persistentVolumeClaim: - claimName: myclaim \ No newline at end of file diff --git a/examples/persistent-volume-provisioning/cinder/cinder-in-tree-full.yaml b/examples/persistent-volume-provisioning/cinder/cinder-in-tree-full.yaml deleted file mode 100644 index 49f80afd32..0000000000 --- a/examples/persistent-volume-provisioning/cinder/cinder-in-tree-full.yaml +++ /dev/null @@ -1,38 +0,0 @@ -kind: StorageClass -apiVersion: storage.k8s.io/v1 -metadata: - name: gold -provisioner: kubernetes.io/cinder ---- -kind: PersistentVolumeClaim -apiVersion: v1 -metadata: - name: myclaim-gold -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi # pass here the size of the volume - storageClassName: gold ---- -apiVersion: v1 -kind: Pod -metadata: - name: web -spec: - containers: - - name: web - image: nginx - ports: - - name: web - containerPort: 80 - hostPort: 8081 - protocol: TCP - volumeMounts: - - mountPath: "/usr/share/nginx/html" - name: mypd - volumes: - - name: mypd - persistentVolumeClaim: - claimName: myclaim-gold diff --git a/examples/persistent-volume-provisioning/cinder/cinder-storage-class.yaml b/examples/persistent-volume-provisioning/cinder/cinder-storage-class.yaml deleted file mode 100644 index fa442b0b07..0000000000 --- a/examples/persistent-volume-provisioning/cinder/cinder-storage-class.yaml +++ /dev/null @@ -1,7 +0,0 @@ -kind: StorageClass -apiVersion: storage.k8s.io/v1 -metadata: - name: gold -provisioner: openstack.org/standalone-cinder -parameters: - availability: nova diff --git a/examples/persistent-volume-provisioning/cinder/example-pod.yaml b/examples/persistent-volume-provisioning/cinder/example-pod.yaml deleted file mode 100644 index bc14b12054..0000000000 --- a/examples/persistent-volume-provisioning/cinder/example-pod.yaml +++ /dev/null @@ -1,23 +0,0 @@ -apiVersion: v1 -kind: ReplicationController -metadata: - name: server -spec: - replicas: 1 - selector: - role: server - template: - metadata: - labels: - role: server - spec: - containers: - - name: server - image: nginx - volumeMounts: - - mountPath: /var/lib/www/html - name: cinderpvc - volumes: - - name: cinderpvc - persistentVolumeClaim: - claimName: claim1 diff --git a/examples/volumes/cinder/README.md b/examples/volumes/cinder/README.md deleted file mode 100644 index 4d008c58e1..0000000000 --- a/examples/volumes/cinder/README.md +++ /dev/null @@ -1 +0,0 @@ -This file has moved to [https://github.com/kubernetes/examples/blob/master/staging/volumes/cinder/README.md](https://github.com/kubernetes/examples/blob/master/staging/volumes/cinder/README.md) diff --git a/examples/volumes/cinder/cinder-web.yaml b/examples/volumes/cinder/cinder-web.yaml deleted file mode 100644 index 4692ac868a..0000000000 --- a/examples/volumes/cinder/cinder-web.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: cinder-web -spec: - containers: - - name: web - image: nginx - ports: - - name: web - containerPort: 80 - protocol: tcp - volumeMounts: - - name: html-volume - mountPath: "/usr/share/nginx/html" - volumes: - - name: html-volume - cinder: - # Enter the volume ID below - volumeID: volume_ID - fsType: ext4 diff --git a/examples/webhook/keystone-apiserver-webhook.yaml b/examples/webhook/keystone-apiserver-webhook.yaml deleted file mode 100644 index 359c69e77a..0000000000 --- a/examples/webhook/keystone-apiserver-webhook.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: v1 -kind: Config -preferences: {} -clusters: - - cluster: - insecure-skip-tls-verify: true - server: https://{{ keystone_auth_service_ip }}:8443/webhook - name: webhook -users: - - name: webhook -contexts: - - context: - cluster: webhook - user: webhook - name: webhook -current-context: webhook diff --git a/examples/webhook/keystone-deployment.yaml b/examples/webhook/keystone-deployment.yaml deleted file mode 100644 index 902318bcb9..0000000000 --- a/examples/webhook/keystone-deployment.yaml +++ /dev/null @@ -1,41 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: k8s-keystone-auth - namespace: kube-system - labels: - app: k8s-keystone-auth -spec: - replicas: 2 - selector: - matchLabels: - app: k8s-keystone-auth - template: - metadata: - labels: - app: k8s-keystone-auth - spec: - serviceAccountName: k8s-keystone - containers: - - name: k8s-keystone-auth - image: k8scloudprovider/k8s-keystone-auth:latest - args: - - ./bin/k8s-keystone-auth - - --tls-cert-file - - /etc/pki/tls.crt - - --tls-private-key-file - - /etc/pki/tls.key - - --policy-configmap-name - - k8s-auth-policy - - --keystone-url - - {{ keystone_server_url }} - volumeMounts: - - mountPath: /etc/pki - name: certs - readOnly: true - ports: - - containerPort: 8443 - volumes: - - name: certs - secret: - secretName: keystone-auth-certs diff --git a/examples/webhook/keystone-policy-configmap.yaml b/examples/webhook/keystone-policy-configmap.yaml deleted file mode 100644 index 45290868a1..0000000000 --- a/examples/webhook/keystone-policy-configmap.yaml +++ /dev/null @@ -1,62 +0,0 @@ -# This file lays out the policies of the cluster. It is only used -# if using Keystone for authorization. Note the difference between -# authentication and authorization. - -# This configmap allows the users in the project 'demo' that have -# the 'k8s-viewer' role in Keystone to query the pod information -# from all the namespaces. The configmap is in the kube-system -# namespace due to the k8s-keystone-auth service existing in -# the kube-system namespace. - -# This is version 1 policy definition - -# apiVersion: v1 -# kind: ConfigMap -# metadata: -# name: k8s-auth-policy -# namespace: kube-system -# labels: -# k8s-app: k8s-keystone-auth -# data: -# policies: | -# [ -# { -# "resource": { -# "verbs": ["get", "list", "watch"], -# "resources": ["pods"], -# "version": "*", -# "namespace": "default" -# }, -# "match": [ -# { -# "type": "role", -# "values": ["k8s-viewer"] -# }, -# { -# "type": "project", -# "values": ["demo"] -# } -# ] -# } -# ] - -# This is version 2 policy definition - -apiVersion: v1 -kind: ConfigMap -metadata: - name: k8s-auth-policy - namespace: kube-system -data: - policies: | - [ - { - "users": { - "projects": ["demo"], - "roles": ["member"] - }, - "resource_permissions": { - "*/pods": ["get", "list", "watch"] - } - } - ] diff --git a/examples/webhook/keystone-rbac.yaml b/examples/webhook/keystone-rbac.yaml deleted file mode 100644 index cde6aee47d..0000000000 --- a/examples/webhook/keystone-rbac.yaml +++ /dev/null @@ -1,32 +0,0 @@ -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - labels: - k8s-app: k8s-keystone-auth - name: k8s-keystone-auth -rules: - # Allow k8s-keystone-auth to get k8s-auth-policy configmap -- apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "watch", "list"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: k8s-keystone-auth - labels: - k8s-app: k8s-keystone-auth -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: k8s-keystone-auth -subjects: -- kind: ServiceAccount - name: k8s-keystone - namespace: kube-system ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: k8s-keystone - namespace: kube-system diff --git a/examples/webhook/keystone-service.yaml b/examples/webhook/keystone-service.yaml deleted file mode 100644 index bf5e98f271..0000000000 --- a/examples/webhook/keystone-service.yaml +++ /dev/null @@ -1,12 +0,0 @@ -kind: Service -apiVersion: v1 -metadata: - name: k8s-keystone-auth-service - namespace: kube-system -spec: - selector: - app: k8s-keystone-auth - ports: - - protocol: TCP - port: 8443 - targetPort: 8443 diff --git a/examples/webhook/policy.json b/examples/webhook/policy.json deleted file mode 100644 index 53a868aab8..0000000000 --- a/examples/webhook/policy.json +++ /dev/null @@ -1,32 +0,0 @@ -[ - { - "resource": { - "verbs": [ - "get", - "list", - "watch" - ], - "resources": [ - "pods" - ], - "version": "*", - "namespace": "default" - }, - "match": [ - { - "type": "role", - "values": [ - "k8s-admin", - "k8s-viewer", - "k8s-editor" - ] - }, - { - "type": "project", - "values": [ - "c1f7910086964990847dc6c8b128f63c" - ] - } - ] - } -] \ No newline at end of file diff --git a/go.mod b/go.mod deleted file mode 100644 index 80165de35a..0000000000 --- a/go.mod +++ /dev/null @@ -1,81 +0,0 @@ -module k8s.io/cloud-provider-openstack - -go 1.13 - -require ( - github.com/NYTimes/gziphandler v1.1.1 // indirect - github.com/container-storage-interface/spec v1.2.0 - github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a // indirect - github.com/emicklei/go-restful v2.9.6+incompatible // indirect - github.com/evanphx/json-patch v4.5.0+incompatible // indirect - github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect - github.com/golang/protobuf v1.3.2 - github.com/googleapis/gnostic v0.3.1 // indirect - github.com/gophercloud/gophercloud v0.11.1-0.20200518183226-7aec46f32c19 - github.com/gophercloud/utils v0.0.0-20200423144003-7c72efc7435d - github.com/gorilla/mux v1.7.3 - github.com/hashicorp/go-version v1.2.0 - github.com/hashicorp/golang-lru v0.5.3 // indirect - github.com/imdario/mergo v0.3.7 // indirect - github.com/kubernetes-csi/csi-lib-utils v0.6.1 - github.com/kubernetes-csi/csi-test v2.2.0+incompatible - github.com/kubernetes-csi/csi-test/v3 v3.1.0 - github.com/mitchellh/go-homedir v1.1.0 - github.com/mitchellh/mapstructure v1.1.2 - github.com/onsi/ginkgo v1.12.0 - github.com/onsi/gomega v1.9.0 - github.com/pborman/uuid v1.2.0 - github.com/pelletier/go-toml v1.4.0 // indirect - github.com/prometheus/client_golang v1.4.1 - github.com/sirupsen/logrus v1.4.2 - github.com/spf13/cobra v0.0.5 - github.com/spf13/pflag v1.0.5 - github.com/spf13/viper v1.4.0 - github.com/stretchr/testify v1.5.1 - golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975 - golang.org/x/net v0.0.0-20200226121028-0de0cce0169b - golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect - golang.org/x/sys v0.0.0-20200122134326-e047566fdf82 - golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect - google.golang.org/appengine v1.6.2 // indirect - google.golang.org/grpc v1.26.0 - gopkg.in/gcfg.v1 v1.2.3 - gopkg.in/square/go-jose.v2 v2.3.1 // indirect - gopkg.in/warnings.v0 v0.1.2 // indirect - gopkg.in/yaml.v2 v2.2.8 - k8s.io/api v0.18.0 - k8s.io/apimachinery v0.18.0 - k8s.io/apiserver v0.18.0 - k8s.io/client-go v0.18.0 - k8s.io/cloud-provider v0.18.0 - k8s.io/component-base v0.18.0 - k8s.io/klog/v2 v2.0.0 - k8s.io/kubernetes v1.18.0 - k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89 - software.sslmate.com/src/go-pkcs12 v0.0.0-20190209200317-47dd539968c4 -) - -replace ( - github.com/opencontainers/runc => github.com/opencontainers/runc v1.0.0-rc9 - k8s.io/api => k8s.io/api v0.18.0 - k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.18.0 - k8s.io/apimachinery => k8s.io/apimachinery v0.18.0 - k8s.io/apiserver => k8s.io/apiserver v0.18.0 - k8s.io/cli-runtime => k8s.io/cli-runtime v0.18.0 - k8s.io/client-go => k8s.io/client-go v0.18.0 - k8s.io/cloud-provider => k8s.io/cloud-provider v0.18.0 - k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.18.0 - k8s.io/code-generator => k8s.io/code-generator v0.18.0 - k8s.io/component-base => k8s.io/component-base v0.18.0 - k8s.io/cri-api => k8s.io/cri-api v0.18.0 - k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.18.0 - k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.18.0 - k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.18.0 - k8s.io/kube-proxy => k8s.io/kube-proxy v0.18.0 - k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.18.0 - k8s.io/kubectl => k8s.io/kubectl v0.18.0 - k8s.io/kubelet => k8s.io/kubelet v0.18.0 - k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.18.0 - k8s.io/metrics => k8s.io/metrics v0.18.0 - k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.18.0 -) diff --git a/go.sum b/go.sum deleted file mode 100644 index 33e87322d6..0000000000 --- a/go.sum +++ /dev/null @@ -1,915 +0,0 @@ -bitbucket.org/bertimus9/systemstat v0.0.0-20180207000608-0eeff89b0690/go.mod h1:Ulb78X89vxKYgdL24HMTiXYHlyHEvruOj1ZPlqeNEZM= -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -github.com/Azure/azure-sdk-for-go v35.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= -github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= -github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= -github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= -github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= -github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= -github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= -github.com/Azure/go-autorest/autorest/to v0.2.0/go.mod h1:GunWKJp1AEqgMaGLV+iocmRAJWqST1wQYhyyjXJ3SJc= -github.com/Azure/go-autorest/autorest/validation v0.1.0/go.mod h1:Ha3z/SqBeaalWQvokg3NZAlQTalVMtOIAs1aGK7G6u8= -github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= -github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/GoogleCloudPlatform/k8s-cloud-provider v0.0.0-20190822182118-27a4ced34534/go.mod h1:iroGtC8B3tQiqtds1l+mgk/BBOrxbqjH+eUfFQYRc14= -github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab/go.mod h1:3VYc5hodBMJ5+l/7J4xAyMeuM2PNuepvHlGs8yilUCA= -github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E= -github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= -github.com/Microsoft/hcsshim v0.0.0-20190417211021-672e52e9209d/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= -github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= -github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= -github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/OpenPeeDeeP/depguard v1.0.0/go.mod h1:7/4sitnI9YlQgTLLk734QlzXT8DuHVnAyztLplQjk+o= -github.com/OpenPeeDeeP/depguard v1.0.1/go.mod h1:xsIw86fROiiwelg+jB2uM9PiKihMMmUx/1V+TNhjQvM= -github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= -github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/Rican7/retry v0.1.0/go.mod h1:FgOROf8P5bebcC1DS0PdOQiqGUridaZvikzUmkFW6gg= -github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= -github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= -github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= -github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/auth0/go-jwt-middleware v0.0.0-20170425171159-5493cabe49f7/go.mod h1:LWMyo4iOLWXHGdBki7NIht1kHru/0wM179h+d3g8ATM= -github.com/aws/aws-sdk-go v1.28.2/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/bazelbuild/bazel-gazelle v0.18.2/go.mod h1:D0ehMSbS+vesFsLGiD6JXu3mVEzOlfUl8wNnq+x/9p0= -github.com/bazelbuild/bazel-gazelle v0.19.1-0.20191105222053-70208cbdc798/go.mod h1:rPwzNHUqEzngx1iVBfO/2X2npKaT3tqPqqHW6rVsn/A= -github.com/bazelbuild/buildtools v0.0.0-20190731111112-f720930ceb60/go.mod h1:5JP0TXzWDHXv8qvxRC4InIazwdyDseBDbzESUMKk1yU= -github.com/bazelbuild/buildtools v0.0.0-20190917191645-69366ca98f89/go.mod h1:5JP0TXzWDHXv8qvxRC4InIazwdyDseBDbzESUMKk1yU= -github.com/bazelbuild/rules_go v0.0.0-20190719190356-6dae44dc5cab/go.mod h1:MC23Dc/wkXEyk3Wpq6lCqz0ZAYOZDw2DR5y3N1q2i7M= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bifurcation/mint v0.0.0-20180715133206-93c51c6ce115/go.mod h1:zVt7zX3K/aDCk9Tj+VM7YymsX66ERvzCJzw8rFCX2JU= -github.com/blang/semver v3.5.0+incompatible h1:CGxCgetQ64DKk7rdZ++Vfnb1+ogGNnB17OJKJXD2Cfs= -github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= -github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= -github.com/caddyserver/caddy v1.0.3/go.mod h1:G+ouvOY32gENkJC+jhgl62TyhvqEsFaDiZ4uw0RzP1E= -github.com/cenkalti/backoff v2.1.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/prettybench v0.0.0-20150116022406-03b8cfe5406c/go.mod h1:Xe6ZsFhtM8HrDku0pxJ3/Lr51rwykrzgFwpmTzleatY= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw= -github.com/checkpoint-restore/go-criu v0.0.0-20181120144056-17b0214f6c48/go.mod h1:TrMrLQfeENAPYPRsJuq3jsqdlRh3lvi6trTZJG8+tho= -github.com/cheekybits/genny v0.0.0-20170328200008-9127e812e1e9/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= -github.com/cilium/ebpf v0.0.0-20191025125908-95b36a581eed/go.mod h1:MA5e5Lr8slmEg9bt0VpxxWqJlO4iwu3FBdHUzV7wQVg= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/clusterhq/flocker-go v0.0.0-20160920122132-2b8b7259d313/go.mod h1:P1wt9Z3DP8O6W3rvwCt0REIlshg1InHImaLW0t3ObY0= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= -github.com/codegangsta/negroni v1.0.0/go.mod h1:v0y3T5G7Y1UlFfyxFn/QLRU4a2EuNau2iZY63YTKWo0= -github.com/container-storage-interface/spec v1.2.0 h1:bD9KIVgaVKKkQ/UbVUY9kCaH/CJbhNxe0eeB4JeJV2s= -github.com/container-storage-interface/spec v1.2.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4= -github.com/containerd/console v0.0.0-20170925154832-84eeaae905fa/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= -github.com/containerd/containerd v1.0.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/typeurl v0.0.0-20190228175220-2a93cfde8c20/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= -github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= -github.com/coredns/corefile-migration v1.0.6/go.mod h1:OFwBp/Wc9dJt5cAZzHWMNhK1r5L0p0jDwIBc6j8NC8E= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= -github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a h1:W8b4lQ4tFF21aspRGoBuCNV6V2fFJBF+pm1J6OY8Lys= -github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= -github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= -github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/daviddengcn/go-colortext v0.0.0-20160507010035-511bcaf42ccd/go.mod h1:dv4zxwHi5C/8AeI+4gX4dCWOIvNi7I6JCSX0HvlKPgE= -github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= -github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= -github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0 h1:w3NnFcKR5241cfmQU5ZZAsf0xcpId6mWOupTvJlUX2U= -github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/go-connections v0.3.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= -github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/libnetwork v0.8.0-dev.2.0.20190925143933-c8a5fca4a652/go.mod h1:93m0aTqz6z+g32wla4l4WxTrdtvBRmVzYRkYvasA5Z8= -github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96 h1:cenwrSVm+Z7QLSV/BsnenAOcDXdX4cMv4wP0B/5QbPg= -github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= -github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= -github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= -github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful v2.9.6+incompatible h1:tfrHha8zJ01ywiOEC1miGY8st1/igzWB8OmvPgoYX7w= -github.com/emicklei/go-restful v2.9.6+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/euank/go-kmsg-parser v2.0.0+incompatible/go.mod h1:MhmAMZ8V4CYH4ybgdRwPr2TU5ThnS43puaKEMpja1uw= -github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M= -github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= -github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= -github.com/fatih/color v1.6.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= -github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= -github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= -github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= -github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= -github.com/go-acme/lego v2.5.0+incompatible/go.mod h1:yzMNe9CasVUhkquNvti5nAtPmG94USbYxYrZfTkIn0M= -github.com/go-bindata/go-bindata v3.1.1+incompatible/go.mod h1:xK8Dsgwmeed+BBsSy2XTopBn/8uK2HWuGSnA11C3Joo= -github.com/go-critic/go-critic v0.3.5-0.20190526074819-1df300866540/go.mod h1:+sE8vrLDS2M0pZkBk0wy6+nLdKexVDrl/jBqQOTDThA= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logr/logr v0.1.0 h1:M1Tv3VzNlEHg6uyACnRdtrploV2P7wZqH8BoQMtz0cg= -github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= -github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= -github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= -github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= -github.com/go-openapi/analysis v0.19.2/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk= -github.com/go-openapi/analysis v0.19.5/go.mod h1:hkEAkxagaIvIP7VTn8ygJNkd4kAYON2rCu0v0ObL0AU= -github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= -github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= -github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= -github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= -github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= -github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= -github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= -github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w= -github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= -github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= -github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= -github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= -github.com/go-openapi/jsonreference v0.19.3 h1:5cxNfTy0UVC3X8JL5ymxzyoUZmo8iZb+jeTWn7tUa8o= -github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= -github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= -github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= -github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= -github.com/go-openapi/loads v0.19.2/go.mod h1:QAskZPMX5V0C2gvfkGZzJlINuP7Hx/4+ix5jWFxsNPs= -github.com/go-openapi/loads v0.19.4/go.mod h1:zZVHonKd8DXyxyw4yfnVjPzBjIQcLt0CCsn0N0ZrQsk= -github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA= -github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64= -github.com/go-openapi/runtime v0.19.4/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29gLDlFGtJ8NL4= -github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= -github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= -github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= -github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY= -github.com/go-openapi/spec v0.19.3 h1:0XRyw8kguri6Yw4SxhsQA/atC88yqrk0+G4YhI2wabc= -github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= -github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= -github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= -github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY= -github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= -github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= -github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= -github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= -github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= -github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= -github.com/go-openapi/validate v0.19.5/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4= -github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4= -github.com/go-toolsmith/astcopy v1.0.0/go.mod h1:vrgyG+5Bxrnz4MZWPF+pI4R8h3qKRjjyvV/DSez4WVQ= -github.com/go-toolsmith/astequal v0.0.0-20180903214952-dcb477bfacd6/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY= -github.com/go-toolsmith/astequal v1.0.0/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY= -github.com/go-toolsmith/astfmt v0.0.0-20180903215011-8f8ee99c3086/go.mod h1:mP93XdblcopXwlyN4X4uodxXQhldPGZbcEJIimQHrkg= -github.com/go-toolsmith/astfmt v1.0.0/go.mod h1:cnWmsOAuq4jJY6Ct5YWlVLmcmLMn1JUPuQIHCY7CJDw= -github.com/go-toolsmith/astinfo v0.0.0-20180906194353-9809ff7efb21/go.mod h1:dDStQCHtmZpYOmjRP/8gHHnCCch3Zz3oEgCdZVdtweU= -github.com/go-toolsmith/astp v0.0.0-20180903215135-0af7e3c24f30/go.mod h1:SV2ur98SGypH1UjcPpCatrV5hPazG6+IfNHbkDXBRrk= -github.com/go-toolsmith/astp v1.0.0/go.mod h1:RSyrtpVlfTFGDYRbrjyWP1pYu//tSFcvdYrA8meBmLI= -github.com/go-toolsmith/pkgload v0.0.0-20181119091011-e9e65178eee8/go.mod h1:WoMrjiy4zvdS+Bg6z9jZH82QXwkcgCBX6nOfnmdaHks= -github.com/go-toolsmith/pkgload v1.0.0/go.mod h1:5eFArkbO80v7Z0kdngIxsRXRMTaX4Ilcwuh3clNrQJc= -github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8= -github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU= -github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= -github.com/godbus/dbus v0.0.0-20181101234600-2ff6f7ffd60f/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.0.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1 h1:qGJ6qTW+x6xX/my+8YUVl4WNpX9B7+/l2tRsHGZ7f2s= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4= -github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a/go.mod h1:ryS0uhF+x9jgbj/N71xsEqODy9BN81/GonCZiOzirOk= -github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6/go.mod h1:DbHgvLiFKX1Sh2T1w8Q/h4NAI8MHIpzCdnBUDTXU3I0= -github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2NxV+sN8upjjeVYr5W7tyxaT1JVtvhKhOn2ii8= -github.com/golangci/go-tools v0.0.0-20190318055746-e32c54105b7c/go.mod h1:unzUULGw35sjyOYjUt0jMTXqHlZPpPc6e+xfO4cd6mM= -github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o= -github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU= -github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU= -github.com/golangci/golangci-lint v1.18.0/go.mod h1:kaqo8l0OZKYPtjNmG4z4HrWLgcYNIJ9B9q3LWri9uLg= -github.com/golangci/gosec v0.0.0-20190211064107-66fb7fc33547/go.mod h1:0qUabqiIQgfmlAmulqxyiGkkyF6/tOGSnY2cnPVwrzU= -github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU= -github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg= -github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o= -github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA= -github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21/go.mod h1:tf5+bzsHdTM0bsB7+8mt0GUMvjCgwLpTapNZHU8AajI= -github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4= -github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ= -github.com/golangplus/bytes v0.0.0-20160111154220-45c989fe5450/go.mod h1:Bk6SMAONeMXrxql8uvOKuAZSu8aM5RUGv+1C6IJaEho= -github.com/golangplus/fmt v0.0.0-20150411045040-2a5d6d7d2995/go.mod h1:lJgMEyOkYFkPcDKwRXegd+iM6E7matEszMG5HhwytU8= -github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/cadvisor v0.35.0/go.mod h1:1nql6U13uTHaLYB8rLS5x9IJc2qT6Xd/Tr1sTX6NE48= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= -github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= -github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= -github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= -github.com/googleapis/gnostic v0.3.1 h1:WeAefnSUHlBb0iJKwxFDZdbfGwkd7xRNuV+IpXMJhYk= -github.com/googleapis/gnostic v0.3.1/go.mod h1:on+2t9HRStVgn95RSsFWFz+6Q0Snyqv1awfrALZdbtU= -github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= -github.com/gophercloud/gophercloud v0.6.1-0.20191122030953-d8ac278c1c9d/go.mod h1:ozGNgr9KYOVATV5jsgHl/ceCDXGuguqOZAzoQ/2vcNM= -github.com/gophercloud/gophercloud v0.11.1-0.20200518183226-7aec46f32c19 h1:Amaxs7PsvtzbahUHadno+OZI0IrMqwbPhoGUVLdM1NA= -github.com/gophercloud/gophercloud v0.11.1-0.20200518183226-7aec46f32c19/go.mod h1:gmC5oQqMDOMO1t1gq5DquX/yAU808e/4mzjjDA76+Ss= -github.com/gophercloud/utils v0.0.0-20200423144003-7c72efc7435d h1:fduaPzWwIfvOMLuHk2Al3GZH0XbUqG8MbElPop+Igzs= -github.com/gophercloud/utils v0.0.0-20200423144003-7c72efc7435d/go.mod h1:ehWUbLQJPqS0Ep+CxeD559hsm9pthPXadJNKwZkp43w= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= -github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= -github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE= -github.com/gostaticanalysis/analysisutil v0.0.3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE= -github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.9.5 h1:UImYN5qQ8tuGpGE16ZmjvcTtTw24zw1QAp/SlnNrZhI= -github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-version v1.2.0 h1:3vNe/fWF5CBgRIguda1meWhsZHy3m8gCJ5wx+dIzX/E= -github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk= -github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/heketi/heketi v9.0.1-0.20190917153846-c2e2a4ab7ab9+incompatible/go.mod h1:bB9ly3RchcQqsQ9CpyaQwvva7RS5ytVoSoholZQON6o= -github.com/heketi/tests v0.0.0-20151005000721-f3775cbcefd6/go.mod h1:xGMAM8JLi7UkZt1i4FQeQy0R2T8GLUwQhOP5M1gBhy4= -github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.7 h1:Y+UAYTZ7gDEuOfhxKWy+dvb5dRQ6rJjFSdX2HZY1/gI= -github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= -github.com/jimstudt/http-authentication v0.0.0-20140401203705-3eca13d6893a/go.mod h1:wK6yTYYcgjHE1Z1QtXACPDjcFJyBskHEdagmnq3vsP8= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= -github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= -github.com/karrick/godirwalk v1.7.5/go.mod h1:2c9FRhkDxdIbgkOnCEvnSWs71Bhugbl46shStcFDJ34= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= -github.com/kisielk/gotool v0.0.0-20161130080628-0de1eaf82fa3/go.mod h1:jxZFDH7ILpTPQTk+E2s+z4CUas9lVNjIuKR4c5/zKgM= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= -github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kubernetes-csi/csi-lib-utils v0.6.1 h1:+AZ58SRSRWh2vmMoWAAGcv7x6fIyBMpyCXAgIc9kT28= -github.com/kubernetes-csi/csi-lib-utils v0.6.1/go.mod h1:GVmlUmxZ+SUjVLXicRFjqWUUvWez0g0Y78zNV9t7KfQ= -github.com/kubernetes-csi/csi-test v2.2.0+incompatible h1:ksIV60Q+4mY0Fg8LKvBssjEcvbyxo7nz0eAD6ZLMux0= -github.com/kubernetes-csi/csi-test v2.2.0+incompatible/go.mod h1:YxJ4UiuPWIhMBkxUKY5c267DyA0uDZ/MtAimhx/2TA0= -github.com/kubernetes-csi/csi-test/v3 v3.1.0 h1:XPiXQgEhDV9y+6vHAklH273PKA/ocqQdC8R+WcIKZKw= -github.com/kubernetes-csi/csi-test/v3 v3.1.0/go.mod h1:UWxYP5cDlD6iSNVKEiLFqfJnJinuhtI7MLt61rQQOfI= -github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= -github.com/libopenstorage/openstorage v1.0.0/go.mod h1:Sp1sIObHjat1BeXhfMqLZ14wnOzEhNx2YQedreMcUyc= -github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= -github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= -github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= -github.com/lpabon/godbc v0.1.1/go.mod h1:Jo9QV0cf3U6jZABgiJ2skINAXb9j8m51r07g4KI92ZA= -github.com/lucas-clemente/aes12 v0.0.0-20171027163421-cd47fb39b79f/go.mod h1:JpH9J1c9oX6otFSgdUHwUBUizmKlrMjxWnIAjff4m04= -github.com/lucas-clemente/quic-clients v0.1.0/go.mod h1:y5xVIEoObKqULIKivu+gD/LU90pL73bTdtQjPBvtCBk= -github.com/lucas-clemente/quic-go v0.10.2/go.mod h1:hvaRS9IHjFLMq76puFJeWNfmn+H70QZ/CXoxqw9bzao= -github.com/lucas-clemente/quic-go-certificates v0.0.0-20160823095156-d2f86524cced/go.mod h1:NCcRLrOTZbzhZvixZLlERbJtDtYsmMw8Jc4vS8Z0g58= -github.com/magiconair/properties v1.7.6/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= -github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM= -github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= -github.com/marten-seemann/qtls v0.2.3/go.mod h1:xzjG7avBwGGbdZ8dTGxlBnLArsVKLvwmjgmPuiQEcYk= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= -github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-shellwords v1.0.5/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= -github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= -github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/mesos/mesos-go v0.0.9/go.mod h1:kPYCMQ9gsOXVAle1OsoY4I1+9kPu8GHkf88aV59fDr4= -github.com/mholt/certmagic v0.6.2-0.20190624175158-6a42ef9fe8c2/go.mod h1:g4cOPxcjV0oFq3qwpjSA30LReKD8AoIfwAY9VvG35NY= -github.com/miekg/dns v1.1.3/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/miekg/dns v1.1.4/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/mindprince/gonvml v0.0.0-20190828220739-9ebdce4bb989/go.mod h1:2eu9pRWp8mo84xCg6KswZ+USQHjwgRhNp06sozOdsTY= -github.com/mistifyio/go-zfs v2.1.1+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk= -github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= -github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/mohae/deepcopy v0.0.0-20170603005431-491d3605edfb/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= -github.com/mozilla/tls-observatory v0.0.0-20180409132520-8791a200eb40/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk= -github.com/mrunalp/fileutils v0.0.0-20171103030105-7d4729fb3618/go.mod h1:x8F1gnqOkIEiO4rqoeEEEqQbo7HjGMTvyoq3gej4iT0= -github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mvdan/xurls v1.1.0/go.mod h1:tQlNn3BED8bE/15hnSL2HLkDeLWpNPAwtw7wkEq44oU= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0= -github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= -github.com/nbutton23/zxcvbn-go v0.0.0-20160627004424-a22cb81b2ecd/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU= -github.com/nbutton23/zxcvbn-go v0.0.0-20171102151520-eafdab6b0663/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.0 h1:Iw5WCbBcaAAd0fpRb1c9r5YCylv4XDoCSigm1zLevwU= -github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= -github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.9.0 h1:R1uwffexN6Pr340GtYRIdZmAiN4J+iw6WG4wog1DUXg= -github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= -github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= -github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= -github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runtime-spec v1.0.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/selinux v1.3.1-0.20190929122143-5215b1806f52/go.mod h1:+BLncwf63G4dgOzykXAxcmnFlUaOlkDdmw/CqsW6pjs= -github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g= -github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/pelletier/go-toml v1.1.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.4.0 h1:u3Z1r+oOXJIkxqw34zVhyPgjBsm6X2wn21NWs/HfSeg= -github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= -github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= -github.com/pquerna/ffjson v0.0.0-20180717144149-af8b230fcd20/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.4.1 h1:FFSuS004yOQEtDdTq+TAOLP5xUq63KqAFYyOi8zA+Y8= -github.com/prometheus/client_golang v1.4.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U= -github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8= -github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI= -github.com/quobyte/api v0.1.2/go.mod h1:jL7lIHrmqQ7yh05OJ+eEEdHr0u/kmT1Ff9iHd+4H6VI= -github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= -github.com/robertkrimen/otto v0.0.0-20191219234010-c382bd3c16ff/go.mod h1:xvqspoSXJTIpemEonrMDFq6XzwHYYgToXWj5eRX1OtY= -github.com/robfig/cron v1.1.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= -github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rubiojr/go-vhd v0.0.0-20160810183302-0bfd3b39853c/go.mod h1:DM5xW0nvfNNm2uytzsvhI3OnX8uzaRAg8UX/CnDqbto= -github.com/russross/blackfriday v0.0.0-20170610170232-067529f716f4/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= -github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= -github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/shirou/gopsutil v0.0.0-20180427012116-c95755e4bcd7/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= -github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= -github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ= -github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= -github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= -github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/sourcegraph/go-diff v0.5.1/go.mod h1:j2dHj3m8aZgQO8lMTcTnBcXkRRRqi34cd2MNlA9u1mE= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.0/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= -github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg= -github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.2/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= -github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= -github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= -github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM= -github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU= -github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= -github.com/storageos/go-api v0.0.0-20180912212459-343b3eff91fc/go.mod h1:ZrLn+e0ZuF3Y65PNF6dIwbJPZqfmtCXxFm9ckv0agOY= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= -github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= -github.com/thecodeteam/goscaleio v0.1.0/go.mod h1:68sdkZAsK8bvEwBlbQnlLS+xU+hvLYM/iQ8KXej1AwM= -github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/timakin/bodyclose v0.0.0-20190721030226-87058b9bfcec/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk= -github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= -github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/ultraware/funlen v0.0.1/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA= -github.com/ultraware/funlen v0.0.2/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA= -github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= -github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s= -github.com/valyala/quicktemplate v1.1.1/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4= -github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= -github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= -github.com/vishvananda/netlink v1.0.0/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= -github.com/vishvananda/netns v0.0.0-20171111001504-be1fbeda1936/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= -github.com/vmware/govmomi v0.20.3/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1/go.mod h1:QcJo0QPSfTONNIgpN5RA8prR7fF8nkF6cTWTcNerRO8= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= -go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738 h1:VcrIfasaLFkyjk6KNlXQSzO+B0fZcnECiDrKJsfxka0= -go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= -go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= -go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= -go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= -golang.org/x/build v0.0.0-20190927031335-2835ba2e683f/go.mod h1:fYw7AShPAhGMdXqA9gRadk/CcMsvLlClpE5oBwnS3dM= -golang.org/x/crypto v0.0.0-20180426230345-b49d69b5da94/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190123085648-057139ce5d2b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190228161510-8dd112bcdc25/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190424203555-c05e17bb3b2d/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191202143827-86a70503ff7e/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975 h1:/Tl7pH94bvbAAHBdZJT947M/+gp0+CqQXDtMRC0fseo= -golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20170915142106-8351a756f30f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181102091132-c10e9556a7bc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190328230028-74de082e2cca/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190502183928-7f726cade0ab/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191126235420-ef20fe5d7933/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20171026204733-164713f0dfce/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190122071731-054c452bb702/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190124100055-b90733256f2e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502175342-a43fa875dd82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191113165036-4c7a9d0fe056/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191128015809-6d18c012aee9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82 h1:ywK/j/KkyTHcdyYSZNXGjMwgmDSfjglYZ3vStQ/gSCU= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.0.0-20170915090833-1cbadb444a80/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20170915040203-e531a2a1c15f/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190121143147-24cd39ecf745/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190122202912-9c309ee22fab/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190909030654-5b82db07426d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191203134012-c197fd4bf371/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= -gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= -gonum.org/v1/gonum v0.6.2/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= -gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= -gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ= -gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.6.1-0.20190607001116-5213b8090861/go.mod h1:btoxGiFvQNVUZQ8W08zLtrVS08CNpINPEfxXxgJL1Q4= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.2 h1:j8RI1yW0SkI+paT6uGwMlrMI/6zwYA6/CFil8rxOzGI= -google.golang.org/appengine v1.6.2/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20191114150713-6bbd007550de h1:dFEMUWudT9iV1JMk6i6NwbfIw2V/2VDFyDYCZFypRxE= -google.golang.org/genproto v0.0.0-20191114150713-6bbd007550de/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/gcfg.v1 v1.2.0/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= -gopkg.in/gcfg.v1 v1.2.3 h1:m8OOJ4ccYHnx2f4gQwpno8nAX5OGOh7RLaaz0pj3Ogs= -gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= -gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= -gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= -gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/mcuadros/go-syslog.v2 v2.2.1/go.mod h1:l5LPIyOOyIdQquNg+oU6Z3524YwrcqEm0aKH+5zpt2U= -gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= -gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb78= -gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= -gopkg.in/square/go-jose.v2 v2.3.1 h1:SK5KegNXmKmqE342YYN2qPHEnUYeoMiXXl1poUlI+o4= -gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/warnings.v0 v0.1.1/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= -gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= -gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gotest.tools v2.1.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= -gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= -gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= -gotest.tools/gotestsum v0.3.5/go.mod h1:Mnf3e5FUzXbkCfynWBGOwLssY7gTQgCHObK9tMpAriY= -grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.2/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -k8s.io/api v0.18.0 h1:lwYk8Vt7rsVTwjRU6pzEsa9YNhThbmbocQlKvNBB4EQ= -k8s.io/api v0.18.0/go.mod h1:q2HRQkfDzHMBZL9l/y9rH63PkQl4vae0xRT+8prbrK8= -k8s.io/apiextensions-apiserver v0.18.0/go.mod h1:18Cwn1Xws4xnWQNC00FLq1E350b9lUF+aOdIWDOZxgo= -k8s.io/apimachinery v0.18.0 h1:fuPfYpk3cs1Okp/515pAf0dNhL66+8zk8RLbSX+EgAE= -k8s.io/apimachinery v0.18.0/go.mod h1:9SnR/e11v5IbyPCGbvJViimtJ0SwHG4nfZFjU77ftcA= -k8s.io/apiserver v0.18.0 h1:ELAWpGWC6XdbRLi5lwAbEbvksD7hkXxPdxaJsdpist4= -k8s.io/apiserver v0.18.0/go.mod h1:3S2O6FeBBd6XTo0njUrLxiqk8GNy6wWOftjhJcXYnjw= -k8s.io/cli-runtime v0.18.0/go.mod h1:1eXfmBsIJosjn9LjEBUd2WVPoPAY9XGTqTFcPMIBsUQ= -k8s.io/client-go v0.18.0 h1:yqKw4cTUQraZK3fcVCMeSa+lqKwcjZ5wtcOIPnxQno4= -k8s.io/client-go v0.18.0/go.mod h1:uQSYDYs4WhVZ9i6AIoEZuwUggLVEF64HOD37boKAtF8= -k8s.io/cloud-provider v0.18.0 h1:Ri82BTdMutzIoiBWz/IS5o7I/+fCgRbO09iPWNfrZEM= -k8s.io/cloud-provider v0.18.0/go.mod h1:ZBq1FhoJ+XoQ8JYBYoyx81LS3JV0RAW/UmHf/6w9E6k= -k8s.io/cluster-bootstrap v0.18.0/go.mod h1:xSe+bOZ3asS/ciT91ESQYGhjOql43aBETfvbCzNvad8= -k8s.io/code-generator v0.18.0/go.mod h1:+UHX5rSbxmR8kzS+FAv7um6dtYrZokQvjHpDSYRVkTc= -k8s.io/component-base v0.18.0 h1:I+lP0fNfsEdTDpHaL61bCAqTZLoiWjEEP304Mo5ZQgE= -k8s.io/component-base v0.18.0/go.mod h1:u3BCg0z1uskkzrnAKFzulmYaEpZF7XC9Pf/uFyb1v2c= -k8s.io/cri-api v0.18.0 h1:ryVwCnvQ2OxtQHe56o85xZ9MkIoW4j0FrjgJG7x/Amw= -k8s.io/cri-api v0.18.0/go.mod h1:OJtpjDvfsKoLGhvcc0qfygved0S0dGX56IJzPbqTG1s= -k8s.io/csi-translation-lib v0.18.0 h1:tmQWO6UIXUFboUqlsxT27yCJS//RvC7BZvCDdTRyPyU= -k8s.io/csi-translation-lib v0.18.0/go.mod h1:iF8TE4ACSaPqN1qxmiIjvcU1A8VgkOrpcFGD7Z0hVu0= -k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20200114144118-36b2048a9120/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/heapster v1.2.0-beta.1/go.mod h1:h1uhptVXMwC8xtZBYsPXKVi8fpdlYkTs6k949KozGrM= -k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= -k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= -k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= -k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= -k8s.io/klog/v2 v2.0.0 h1:Foj74zO6RbjjP4hBEKjnYtjjAhGg4jNynUdYF6fJrok= -k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= -k8s.io/kube-aggregator v0.18.0/go.mod h1:ateewQ5QbjMZF/dihEFXwaEwoA4v/mayRvzfmvb6eqI= -k8s.io/kube-controller-manager v0.18.0 h1:Xo8WFWqm3538TbXwtdmUNk2WBQMiEt9HgMohyVb+8Hk= -k8s.io/kube-controller-manager v0.18.0/go.mod h1:pIRGUrSo+skWzwr5pgWNbgiFWEGSotbamGQpR/gKd5U= -k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c h1:/KUFqjjqAcY4Us6luF5RDNZ16KJtb49HfR3ZHB9qYXM= -k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= -k8s.io/kube-proxy v0.18.0/go.mod h1:st3Gcg9wYAd1sn6UMeAs5AHN3R0NOItfB5P6qObKrr8= -k8s.io/kube-scheduler v0.18.0 h1:dwVfXf1AWfGgNUHJdNvxRpk8qGnVQqv+KshALC3aWkg= -k8s.io/kube-scheduler v0.18.0/go.mod h1:GFaNT5Z5/zPZsjXmkGihac2qsT+0u2KIHDgXdFfPHPc= -k8s.io/kubectl v0.18.0 h1:hu52Ndq/d099YW+3sS3VARxFz61Wheiq8K9S7oa82Dk= -k8s.io/kubectl v0.18.0/go.mod h1:LOkWx9Z5DXMEg5KtOjHhRiC1fqJPLyCr3KtQgEolCkU= -k8s.io/kubelet v0.18.0/go.mod h1:1VULM2m7c/ePlIeNOVVK+kkprayDr1RPf1T8oaNaHuQ= -k8s.io/kubernetes v1.18.0 h1:rVe+edi5GwutPQJ4KIZq1Nk506nmnfyz/KOZVCLv7Yo= -k8s.io/kubernetes v1.18.0/go.mod h1:z8xjOOO1Ljz+TaHpOxVGC7cxtF32TesIamoQ+BZrVS0= -k8s.io/legacy-cloud-providers v0.18.0/go.mod h1:4Bc9CdZg8wl0mskyhnaXa8DdqLpTUfPEMkw3FZok+H8= -k8s.io/metrics v0.18.0/go.mod h1:8aYTW18koXqjLVKL7Ds05RPMX9ipJZI3mywYvBOxXd4= -k8s.io/repo-infra v0.0.1-alpha.1/go.mod h1:wO1t9WaB99V80ljbeENTnayuEEwNZt7gECYh/CEyOJ8= -k8s.io/sample-apiserver v0.18.0/go.mod h1:1RKw7QEixom4PIw/vjUvDgl2QQbuTXbeCUHLlNCzOjg= -k8s.io/system-validators v1.0.4/go.mod h1:HgSgTg4NAGNoYYjKsUyk52gdNi2PVDswQ9Iyn66R7NI= -k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89 h1:d4vVOjXm687F1iLSP2q3lyPPuyvTUt3aVoBpi2DqRsU= -k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= -modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= -modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= -modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= -modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= -modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I= -mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc= -mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4= -mvdan.cc/unparam v0.0.0-20190209190245-fbb59629db34/go.mod h1:H6SUd1XjIs+qQCyskXg5OFSrilMRUkD8ePJpHKDPaeY= -rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.7 h1:uuHDyjllyzRyCIvvn0OBjiRB0SgBZGqHNYAmjR7fO50= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.7/go.mod h1:PHgbrJT7lCHcxMU+mDHEm+nx46H4zuuHZkDP6icnhu0= -sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU= -sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= -sigs.k8s.io/structured-merge-diff/v3 v3.0.0 h1:dOmIZBMfhcHS09XZkMyUgkq5trg3/jRyJYFZUiaOp8E= -sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= -sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= -sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= -sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= -software.sslmate.com/src/go-pkcs12 v0.0.0-20190209200317-47dd539968c4 h1:bgAHvGcJ9+ho1l9ItDc5F14pNQ4iZnd65UHEYwJLbrI= -software.sslmate.com/src/go-pkcs12 v0.0.0-20190209200317-47dd539968c4/go.mod h1:/xvNRWUqm0+/ZMiF4EX00vrSCMsE4/NHb+Pt3freEeQ= -sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= -vbom.ml/util v0.0.0-20160121211510-db5cfe13f5cc/go.mod h1:so/NYdZXCz+E3ZpW0uAoCj6uzU2+8OWDFv/HxUSs7kI= diff --git a/hack/make.sh b/hack/make.sh deleted file mode 100755 index a89488ea35..0000000000 --- a/hack/make.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -# Copyright 2018 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Simple script to offer the ability to run the make directives in a Docker Container. -# In other words you don't need a Go env on your system to run Make (build, test etc) -# This script will just bind-mount the source directory into a container under the correct -# GOPATH and handle all of the Go ENV stuff for you. All you need is Docker -docker run -it -v "$PWD":/go/src/k8s.io/cloud-provider-openstack:z \ - -w /go/src/k8s.io/cloud-provider-openstack \ - golang:1.13 make $1 diff --git a/hack/update-gofmt.sh b/hack/update-gofmt.sh deleted file mode 100755 index a14aecbbd2..0000000000 --- a/hack/update-gofmt.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -# Copyright 2017 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o nounset -set -o pipefail - -find . -name "*.go" | grep -v "\/vendor\/" | xargs gofmt -s -w diff --git a/hack/verify-gofmt.sh b/hack/verify-gofmt.sh deleted file mode 100755 index 4072becfad..0000000000 --- a/hack/verify-gofmt.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2017 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o nounset -set -o pipefail - -if ! which gofmt > /dev/null; then - echo "Can not find gofmt" - exit 1 -fi - -# gofmt exits with non-zero exit code if it finds a problem unrelated to -# formatting (e.g., a file does not parse correctly). Without "|| true" this -# would have led to no useful error message from gofmt, because the script would -# have failed before getting to the "echo" in the block below. -diff=$(find . -name "*.go" | grep -v "\/vendor\/" | xargs gofmt -s -d 2>&1) || true -if [[ -n "${diff}" ]]; then - echo "${diff}" - echo - echo "Please run hack/update-gofmt.sh" - exit 1 -fi diff --git a/hack/verify-golint.sh b/hack/verify-golint.sh deleted file mode 100755 index f3ae2136da..0000000000 --- a/hack/verify-golint.sh +++ /dev/null @@ -1,132 +0,0 @@ -#!/bin/bash - -# Copyright 2014 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o pipefail - -KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. -#source "${KUBE_ROOT}/hack/lib/init.sh" - -#kube::golang::verify_go_version - -if ! which golint > /dev/null; then - echo 'Can not find golint, install with:' - echo 'go get -u github.com/golang/lint/golint' - exit 1 -fi - -cd "${KUBE_ROOT}" - -array_contains () { - local seeking=$1; shift # shift will iterate through the array - local in=1 # in holds the exit status for the function - for element; do - if [[ "$element" == "$seeking" ]]; then - in=0 # set in to 0 since we found it - break - fi - done - return $in -} - -# Check that the file is in alphabetical order -failure_file="${KUBE_ROOT}/hack/.golint_failures" -if ! diff -u "${failure_file}" <(LC_ALL=C sort "${failure_file}"); then - { - echo - echo "hack/.golint_failures is not in alphabetical order. Please sort it:" - echo - echo " LC_ALL=C sort -o hack/.golint_failures hack/.golint_failures" - echo - } >&2 - false -fi - -export IFS=$'\n' -# NOTE: when "go list -e ./..." is run within GOPATH, it turns the k8s.io/kubernetes -# as the prefix, however if we run it outside it returns the full path of the file -# with a leading underscore. We'll need to support both scenarios for all_packages. -all_packages=( - $(go list -e ./... | egrep -v "/(third_party|vendor|staging/src/k8s.io/client-go/pkg|generated|clientset_generated)" | sed -e 's|^k8s.io/kubernetes/||' -e "s|^_${KUBE_ROOT}/\?||") -) -failing_packages=( - $(cat $failure_file) -) -unset IFS -errors=() -not_failing=() -for p in "${all_packages[@]}"; do - # Run golint on package/*.go file explicitly to validate all go files - # and not just the ones for the current platform. - # Packages with a corresponding foo_test package will make golint fail - # with a useless error. Just ignore that, see golang/lint#68. - failedLint=$(golint "$p"/*.go 2>/dev/null) - array_contains "$p" "${failing_packages[@]}" && in_failing=$? || in_failing=$? - if [[ -n "${failedLint}" ]] && [[ "${in_failing}" -ne "0" ]]; then - errors+=( "${failedLint}" ) - fi - if [[ -z "${failedLint}" ]] && [[ "${in_failing}" -eq "0" ]]; then - not_failing+=( $p ) - fi -done - -# Check that all failing_packages actually still exist -gone=() -for p in "${failing_packages[@]}"; do - array_contains "$p" "${all_packages[@]}" || gone+=( "$p" ) -done - -# Check to be sure all the packages that should pass lint are. -if [ ${#errors[@]} -eq 0 ]; then - echo 'Congratulations! All Go source files have been linted.' -else - { - echo "Errors from golint:" - for err in "${errors[@]}"; do - echo "$err" - done - echo - echo 'Please review the above warnings. You can test via "golint" and commit the result.' - echo 'If the above warnings do not make sense, you can exempt this package from golint' - echo 'checking by adding it to hack/.golint_failures (if your reviewer is okay with it).' - echo - } >&2 - false -fi - -if [[ ${#not_failing[@]} -gt 0 ]]; then - { - echo "Some packages in hack/.golint_failures are passing golint. Please remove them." - echo - for p in "${not_failing[@]}"; do - echo " $p" - done - echo - } >&2 - false -fi - -if [[ ${#gone[@]} -gt 0 ]]; then - { - echo "Some packages in hack/.golint_failures do not exist anymore. Please remove them." - echo - for p in "${gone[@]}"; do - echo " $p" - done - echo - } >&2 - false -fi diff --git a/hack/verify-import-boss.sh b/hack/verify-import-boss.sh deleted file mode 100755 index fe032de16f..0000000000 --- a/hack/verify-import-boss.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2018 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o nounset -set -o pipefail - -if ! which import-boss > /dev/null; then - echo "Can not find import-boss" - exit 1 -fi - -# prepare the vendor dir -go mod vendor - -GO111MODULE=off import-boss -i k8s.io/cloud-provider-openstack/... --verify-only diff --git a/hack/.golint_failures b/index.yaml similarity index 100% rename from hack/.golint_failures rename to index.yaml diff --git a/manifests/barbican-kms/encryption-config.yaml b/manifests/barbican-kms/encryption-config.yaml deleted file mode 100755 index 6ffeb5b3da..0000000000 --- a/manifests/barbican-kms/encryption-config.yaml +++ /dev/null @@ -1,11 +0,0 @@ -kind: EncryptionConfig -apiVersion: v1 -resources: - - resources: - - secrets - providers: - - kms: - name: barbican - endpoint: unix:///var/lib/kms/kms.sock - cachesize: 20 - - identity: {} diff --git a/manifests/barbican-kms/pod.yaml b/manifests/barbican-kms/pod.yaml deleted file mode 100644 index c0e8a20c9a..0000000000 --- a/manifests/barbican-kms/pod.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: barbican-kms -spec: - containers: - - name: barbican-kms - image: docker.io/k8scloudprovider/barbican-kms-plugin:latest - args: - - "--socketpath=/kms/kms.sock" - - "--cloud-config=/etc/kubernetes/cloud-config" - volumeMounts: - - name: cloud-config - mountPath: /etc/kubernetes/ - - name: socket-dir - mountPath: /kms/ - volumes: - - name: config - hostPath: - path: /etc/kubernetes - - name: socket-dir - hostPath: - path: /var/lib/kms/ - type: DirectoryOrCreate diff --git a/manifests/cinder-csi-plugin/cinder-csi-controllerplugin-rbac.yaml b/manifests/cinder-csi-plugin/cinder-csi-controllerplugin-rbac.yaml deleted file mode 100644 index 3de44b0136..0000000000 --- a/manifests/cinder-csi-plugin/cinder-csi-controllerplugin-rbac.yaml +++ /dev/null @@ -1,197 +0,0 @@ -# This YAML file contains RBAC API objects, -# which are necessary to run csi controller plugin - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: csi-cinder-controller-sa - namespace: kube-system - ---- -# external attacher -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-attacher-role -rules: - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["csinodes"] - verbs: ["get", "list", "watch"] - - ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-attacher-binding -subjects: - - kind: ServiceAccount - name: csi-cinder-controller-sa - namespace: kube-system -roleRef: - kind: ClusterRole - name: csi-attacher-role - apiGroup: rbac.authorization.k8s.io - ---- -# external Provisioner -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-provisioner-role -rules: - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["csinodes"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["get", "list"] - ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-provisioner-binding -subjects: - - kind: ServiceAccount - name: csi-cinder-controller-sa - namespace: kube-system -roleRef: - kind: ClusterRole - name: csi-provisioner-role - apiGroup: rbac.authorization.k8s.io - ---- -# external snapshotter -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-snapshotter-role -rules: - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - # Secret permission is optional. - # Enable it if your driver needs secret. - # For example, `csi.storage.k8s.io/snapshotter-secret-name` is set in VolumeSnapshotClass. - # See https://kubernetes-csi.github.io/docs/secrets-and-credentials.html for more details. - # - apiGroups: [""] - # resources: ["secrets"] - # verbs: ["get", "list"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["create", "get", "list", "watch", "update", "delete"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents/status"] - verbs: ["update"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-snapshotter-binding -subjects: - - kind: ServiceAccount - name: csi-cinder-controller-sa - namespace: kube-system -roleRef: - kind: ClusterRole - name: csi-snapshotter-role - apiGroup: rbac.authorization.k8s.io ---- - -# External Resizer -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-resizer-role -rules: - # The following rule should be uncommented for plugins that require secrets - # for provisioning. - # - apiGroups: [""] - # resources: ["secrets"] - # verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims/status"] - verbs: ["update", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-resizer-binding -subjects: - - kind: ServiceAccount - name: csi-cinder-controller-sa - namespace: kube-system -roleRef: - kind: ClusterRole - name: csi-resizer-role - apiGroup: rbac.authorization.k8s.io - ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - namespace: kube-system - name: external-resizer-cfg -rules: -- apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "watch", "list", "delete", "update", "create"] - ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-resizer-role-cfg - namespace: kube-system -subjects: - - kind: ServiceAccount - name: csi-cinder-controller-sa - namespace: kube-system -roleRef: - kind: Role - name: external-resizer-cfg - apiGroup: rbac.authorization.k8s.io diff --git a/manifests/cinder-csi-plugin/cinder-csi-controllerplugin.yaml b/manifests/cinder-csi-plugin/cinder-csi-controllerplugin.yaml deleted file mode 100644 index b1b20f3864..0000000000 --- a/manifests/cinder-csi-plugin/cinder-csi-controllerplugin.yaml +++ /dev/null @@ -1,114 +0,0 @@ -# This YAML file contains CSI Controller Plugin Sidecars -# external-attacher, external-provisioner, external-snapshotter - -kind: Service -apiVersion: v1 -metadata: - name: csi-cinder-controller-service - namespace: kube-system - labels: - app: csi-cinder-controllerplugin -spec: - selector: - app: csi-cinder-controllerplugin - ports: - - name: dummy - port: 12345 - ---- -kind: StatefulSet -apiVersion: apps/v1 -metadata: - name: csi-cinder-controllerplugin - namespace: kube-system -spec: - serviceName: "csi-cinder-controller-service" - replicas: 1 - selector: - matchLabels: - app: csi-cinder-controllerplugin - template: - metadata: - labels: - app: csi-cinder-controllerplugin - spec: - serviceAccount: csi-cinder-controller-sa - containers: - - name: csi-attacher - image: quay.io/k8scsi/csi-attacher:v2.2.0 - args: - - "--csi-address=$(ADDRESS)" - - "--timeout=3m" - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - imagePullPolicy: "IfNotPresent" - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - - name: csi-provisioner - image: quay.io/k8scsi/csi-provisioner:v1.6.0 - args: - - "--csi-address=$(ADDRESS)" - - "--timeout=3m" - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - imagePullPolicy: "IfNotPresent" - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - - name: csi-snapshotter - image: quay.io/k8scsi/csi-snapshotter:v2.1.1 - args: - - "--csi-address=$(ADDRESS)" - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - imagePullPolicy: Always - volumeMounts: - - mountPath: /var/lib/csi/sockets/pluginproxy/ - name: socket-dir - - name: csi-resizer - image: quay.io/k8scsi/csi-resizer:v0.4.0 - args: - - "--csi-address=$(ADDRESS)" - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - imagePullPolicy: "IfNotPresent" - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - - name: cinder-csi-plugin - image: docker.io/k8scloudprovider/cinder-csi-plugin:latest - args: - - /bin/cinder-csi-plugin - - "--nodeid=$(NODE_ID)" - - "--endpoint=$(CSI_ENDPOINT)" - - "--cloud-config=$(CLOUD_CONFIG)" - - "--cluster=$(CLUSTER_NAME)" - env: - - name: NODE_ID - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: CSI_ENDPOINT - value: unix://csi/csi.sock - - name: CLOUD_CONFIG - value: /etc/config/cloud.conf - - name: CLUSTER_NAME - value: kubernetes - imagePullPolicy: "IfNotPresent" - volumeMounts: - - name: socket-dir - mountPath: /csi - - name: secret-cinderplugin - mountPath: /etc/config - readOnly: true - volumes: - - name: socket-dir - emptyDir: - - name: secret-cinderplugin - secret: - secretName: cloud-config diff --git a/manifests/cinder-csi-plugin/cinder-csi-nodeplugin-rbac.yaml b/manifests/cinder-csi-plugin/cinder-csi-nodeplugin-rbac.yaml deleted file mode 100644 index 912923fcad..0000000000 --- a/manifests/cinder-csi-plugin/cinder-csi-nodeplugin-rbac.yaml +++ /dev/null @@ -1,30 +0,0 @@ -# This YAML defines all API objects to create RBAC roles for csi node plugin. - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: csi-cinder-node-sa - namespace: kube-system ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-nodeplugin-role -rules: - - apiGroups: [""] - resources: ["events"] - verbs: ["get", "list", "watch", "create", "update", "patch"] - ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-nodeplugin-binding -subjects: - - kind: ServiceAccount - name: csi-cinder-node-sa - namespace: kube-system -roleRef: - kind: ClusterRole - name: csi-nodeplugin-role - apiGroup: rbac.authorization.k8s.io diff --git a/manifests/cinder-csi-plugin/cinder-csi-nodeplugin.yaml b/manifests/cinder-csi-plugin/cinder-csi-nodeplugin.yaml deleted file mode 100644 index 8b2f60c89b..0000000000 --- a/manifests/cinder-csi-plugin/cinder-csi-nodeplugin.yaml +++ /dev/null @@ -1,98 +0,0 @@ -# This YAML file contains driver-registrar & csi driver nodeplugin API objects, -# which are necessary to run csi nodeplugin for cinder. - -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: csi-cinder-nodeplugin - namespace: kube-system -spec: - selector: - matchLabels: - app: csi-cinder-nodeplugin - template: - metadata: - labels: - app: csi-cinder-nodeplugin - spec: - serviceAccount: csi-cinder-node-sa - hostNetwork: true - containers: - - name: node-driver-registrar - image: quay.io/k8scsi/csi-node-driver-registrar:v1.2.0 - args: - - "--csi-address=$(ADDRESS)" - - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)" - lifecycle: - preStop: - exec: - command: ["/bin/sh", "-c", "rm -rf /registration/cinder.csi.openstack.org /registration/cinder.csi.openstack.org-reg.sock"] - env: - - name: ADDRESS - value: /csi/csi.sock - - name: DRIVER_REG_SOCK_PATH - value: /var/lib/kubelet/plugins/cinder.csi.openstack.org/csi.sock - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - imagePullPolicy: "IfNotPresent" - volumeMounts: - - name: socket-dir - mountPath: /csi - - name: registration-dir - mountPath: /registration - - name: cinder-csi-plugin - securityContext: - privileged: true - capabilities: - add: ["SYS_ADMIN"] - allowPrivilegeEscalation: true - image: docker.io/k8scloudprovider/cinder-csi-plugin:latest - args: - - /bin/cinder-csi-plugin - - "--nodeid=$(NODE_ID)" - - "--endpoint=$(CSI_ENDPOINT)" - - "--cloud-config=$(CLOUD_CONFIG)" - env: - - name: NODE_ID - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: CSI_ENDPOINT - value: unix://csi/csi.sock - - name: CLOUD_CONFIG - value: /etc/config/cloud.conf - imagePullPolicy: "IfNotPresent" - volumeMounts: - - name: socket-dir - mountPath: /csi - - name: kubelet-dir - mountPath: /var/lib/kubelet - mountPropagation: "Bidirectional" - - name: pods-probe-dir - mountPath: /dev - mountPropagation: "HostToContainer" - - name: secret-cinderplugin - mountPath: /etc/config - readOnly: true - volumes: - - name: socket-dir - hostPath: - path: /var/lib/kubelet/plugins/cinder.csi.openstack.org - type: DirectoryOrCreate - - name: registration-dir - hostPath: - path: /var/lib/kubelet/plugins_registry/ - type: Directory - - name: kubelet-dir - hostPath: - path: /var/lib/kubelet - type: Directory - - name: pods-probe-dir - hostPath: - path: /dev - type: Directory - - name: secret-cinderplugin - secret: - secretName: cloud-config diff --git a/manifests/cinder-csi-plugin/csi-cinder-driver.yaml b/manifests/cinder-csi-plugin/csi-cinder-driver.yaml deleted file mode 100644 index 5b681e4c4d..0000000000 --- a/manifests/cinder-csi-plugin/csi-cinder-driver.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: storage.k8s.io/v1 -kind: CSIDriver -metadata: - name: cinder.csi.openstack.org -spec: - attachRequired: true - podInfoOnMount: true - volumeLifecycleModes: - - Persistent - - Ephemeral diff --git a/manifests/cinder-csi-plugin/csi-secret-cinderplugin.yaml b/manifests/cinder-csi-plugin/csi-secret-cinderplugin.yaml deleted file mode 100644 index 5813c5c02c..0000000000 --- a/manifests/cinder-csi-plugin/csi-secret-cinderplugin.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# This YAML file contains secret objects, -# which are necessary to run csi cinder plugin. - -kind: Secret -apiVersion: v1 -metadata: - name: cloud-config - namespace: kube-system -data: - cloud.conf: W0dsb2JhbF0KdXNlcm5hbWUgPSBhZG1pbgpwYXNzd29yZCA9IG5vbW9yZXNlY3JldApkb21haW4tbmFtZSA9IGRlZmF1bHQKYXV0aC11cmwgPSBodHRwOi8vMTkyLjE2OC4yMDAuOS9pZGVudGl0eQp0ZW5hbnQtaWQgPSBjYzM0YjExZmY5NWQ0MjMwOTA4MWQwYmQ0NmMwZmY4OQpyZWdpb24gPSBSZWdpb25PbmUK diff --git a/manifests/controller-manager/cloud-config b/manifests/controller-manager/cloud-config deleted file mode 100644 index 7d670a25c3..0000000000 --- a/manifests/controller-manager/cloud-config +++ /dev/null @@ -1,19 +0,0 @@ -[Global] -auth-url=http://192.128.1.15:5000/v3 -#Tip: You can also use Application Credential ID and Secret in place of username, password, tenant-id, and domain-id. -#application-credential-id= -#application-credential-secret= -username=admin -# user-id= -password=passw0rd -region=RegionOne -tenant-id= -domain-id= - -[LoadBalancer] -use-octavia=true -subnet-id=9c018814-f2a8-45dd-9679-5400d49b333a -floating-network-id=80c9fc78-a2be-45d5-a85b-f17a6c98df92 - -[BlockStorage] -bs-version=v2 diff --git a/manifests/controller-manager/kubeadm.conf b/manifests/controller-manager/kubeadm.conf deleted file mode 100644 index 1222ec65be..0000000000 --- a/manifests/controller-manager/kubeadm.conf +++ /dev/null @@ -1,19 +0,0 @@ ---- -apiVersion: kubeadm.k8s.io/v1beta2 -kind: InitConfiguration -nodeRegistration: - kubeletExtraArgs: - cloud-provider: "external" - ---- -apiVersion: kubeadm.k8s.io/v1beta2 -kind: ClusterConfiguration -kubernetesVersion: v1.17.0 -networking: - podSubnet: 192.168.0.0/16 -apiServer: - extraArgs: - cloud-provider: "external" -controllerManager: - extraArgs: - cloud-provider: "external" diff --git a/manifests/controller-manager/openstack-cloud-controller-manager-ds.yaml b/manifests/controller-manager/openstack-cloud-controller-manager-ds.yaml deleted file mode 100644 index 615ac9c84d..0000000000 --- a/manifests/controller-manager/openstack-cloud-controller-manager-ds.yaml +++ /dev/null @@ -1,75 +0,0 @@ ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: cloud-controller-manager - namespace: kube-system ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: openstack-cloud-controller-manager - namespace: kube-system - labels: - k8s-app: openstack-cloud-controller-manager -spec: - selector: - matchLabels: - k8s-app: openstack-cloud-controller-manager - updateStrategy: - type: RollingUpdate - template: - metadata: - labels: - k8s-app: openstack-cloud-controller-manager - spec: - nodeSelector: - node-role.kubernetes.io/master: "" - securityContext: - runAsUser: 1001 - tolerations: - - key: node.cloudprovider.kubernetes.io/uninitialized - value: "true" - effect: NoSchedule - - key: node-role.kubernetes.io/master - effect: NoSchedule - serviceAccountName: cloud-controller-manager - containers: - - name: openstack-cloud-controller-manager - image: docker.io/k8scloudprovider/openstack-cloud-controller-manager:latest - args: - - /bin/openstack-cloud-controller-manager - - --v=1 - - --cloud-config=$(CLOUD_CONFIG) - - --cloud-provider=openstack - - --use-service-account-credentials=true - - --address=127.0.0.1 - volumeMounts: - - mountPath: /etc/kubernetes/pki - name: k8s-certs - readOnly: true - - mountPath: /etc/ssl/certs - name: ca-certs - readOnly: true - - mountPath: /etc/config - name: cloud-config-volume - readOnly: true - resources: - requests: - cpu: 200m - env: - - name: CLOUD_CONFIG - value: /etc/config/cloud.conf - hostNetwork: true - volumes: - - hostPath: - path: /etc/kubernetes/pki - type: DirectoryOrCreate - name: k8s-certs - - hostPath: - path: /etc/ssl/certs - type: DirectoryOrCreate - name: ca-certs - - name: cloud-config-volume - secret: - secretName: cloud-config diff --git a/manifests/controller-manager/openstack-cloud-controller-manager-pod.yaml b/manifests/controller-manager/openstack-cloud-controller-manager-pod.yaml deleted file mode 100644 index 0777ec92c7..0000000000 --- a/manifests/controller-manager/openstack-cloud-controller-manager-pod.yaml +++ /dev/null @@ -1,52 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - annotations: - scheduler.alpha.kubernetes.io/critical-pod: "" - labels: - component: cloud-controller-manager - tier: control-plane - name: openstack-cloud-controller-manager - namespace: kube-system -spec: - containers: - - name: openstack-cloud-controller-manager - image: docker.io/k8scloudprovider/openstack-cloud-controller-manager:latest - args: - - /bin/openstack-cloud-controller-manager - - --v=1 - - --cloud-config=$(CLOUD_CONFIG) - - --cloud-provider=openstack - - --use-service-account-credentials=true - - --address=127.0.0.1 - volumeMounts: - - mountPath: /etc/kubernetes/pki - name: k8s-certs - readOnly: true - - mountPath: /etc/ssl/certs - name: ca-certs - readOnly: true - - mountPath: /etc/config - name: cloud-config-volume - readOnly: true - resources: - requests: - cpu: 200m - env: - - name: CLOUD_CONFIG - value: /etc/config/cloud.conf - hostNetwork: true - securityContext: - runAsUser: 1001 - volumes: - - hostPath: - path: /etc/kubernetes/pki - type: DirectoryOrCreate - name: k8s-certs - - hostPath: - path: /etc/ssl/certs - type: DirectoryOrCreate - name: ca-certs - - name: cloud-config-volume - secret: - secretName: cloud-config diff --git a/manifests/magnum-auto-healer/magnum-auto-healer.yaml b/manifests/magnum-auto-healer/magnum-auto-healer.yaml deleted file mode 100644 index b750498b87..0000000000 --- a/manifests/magnum-auto-healer/magnum-auto-healer.yaml +++ /dev/null @@ -1,104 +0,0 @@ ---- -kind: ServiceAccount -apiVersion: v1 -metadata: - name: magnum-auto-healer - namespace: kube-system - ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: magnum-auto-healer -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cluster-admin -subjects: - - kind: ServiceAccount - name: magnum-auto-healer - namespace: kube-system - ---- -kind: ConfigMap -apiVersion: v1 -metadata: - name: magnum-auto-healer-config - namespace: kube-system -data: - config.yaml: | - cluster-name: ${magnum_cluster_uuid} - dry-run: false - monitor-interval: 15s - check-delay-after-add: 20m - leader-elect: true - healthcheck: - master: - - type: Endpoint - params: - unhealthy-duration: 30s - protocol: HTTPS - port: 6443 - endpoints: ["/healthz"] - ok-codes: [200] - - type: NodeCondition - params: - unhealthy-duration: 1m - types: ["Ready"] - ok-values: ["True"] - worker: - - type: NodeCondition - params: - unhealthy-duration: 1m - types: ["Ready"] - ok-values: ["True"] - openstack: - auth-url: ${keystone_auth_url} - user-id: ${user_id} - project-id: ${user_project_id} - password: ${password} - region: ${region} - ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: magnum-auto-healer - namespace: kube-system - labels: - k8s-app: magnum-auto-healer -spec: - selector: - matchLabels: - k8s-app: magnum-auto-healer - template: - metadata: - labels: - k8s-app: magnum-auto-healer - spec: - serviceAccountName: magnum-auto-healer - tolerations: - - effect: NoSchedule - operator: Exists - - key: CriticalAddonsOnly - operator: Exists - - effect: NoExecute - operator: Exists - nodeSelector: - node-role.kubernetes.io/master: "" - containers: - - name: magnum-auto-healer - image: docker.io/k8scloudprovider/magnum-auto-healer:latest - imagePullPolicy: Always - args: - - /bin/magnum-auto-healer - - --config=/etc/magnum-auto-healer/config.yaml - - --v - - "2" - volumeMounts: - - name: config - mountPath: /etc/magnum-auto-healer - volumes: - - name: config - configMap: - name: magnum-auto-healer-config diff --git a/manifests/manila-csi-plugin/csi-controllerplugin-rbac.yaml b/manifests/manila-csi-plugin/csi-controllerplugin-rbac.yaml deleted file mode 100644 index 9da05d017e..0000000000 --- a/manifests/manila-csi-plugin/csi-controllerplugin-rbac.yaml +++ /dev/null @@ -1,113 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: openstack-manila-csi-controllerplugin - labels: - app: openstack-manila-csi - component: controllerplugin ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: openstack-manila-csi-controllerplugin - labels: - app: openstack-manila-csi - component: controllerplugin -aggregationRule: - clusterRoleSelectors: - - matchLabels: - rbac.manila.csi.openstack.org/aggregate-to-openstack-manila-csi-controllerplugin: "true" -rules: [] ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: openstack-manila-csi-controllerplugin-rules - labels: - app: openstack-manila-csi - component: controllerplugin - rbac.manila.csi.openstack.org/aggregate-to-openstack-manila-csi-controllerplugin: "true" -rules: - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["csinodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["create", "get", "list", "watch", "update", "delete"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots/status"] - verbs: ["update"] - - apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: ["create", "list", "watch", "delete", "get", "update"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: openstack-manila-csi-controllerplugin - labels: - app: openstack-manila-csi - component: controllerplugin -subjects: - - kind: ServiceAccount - name: openstack-manila-csi-controllerplugin - namespace: default -roleRef: - kind: ClusterRole - name: openstack-manila-csi-controllerplugin - apiGroup: rbac.authorization.k8s.io ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: openstack-manila-csi-controllerplugin - labels: - app: openstack-manila-csi - component: controllerplugin -rules: - - apiGroups: [""] - resources: ["endpoints"] - verbs: ["get", "watch", "list", "delete", "update", "create"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "watch", "create", "delete"] ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: openstack-manila-csi-controllerplugin - labels: - app: openstack-manila-csi - component: controllerplugin -subjects: - - kind: ServiceAccount - name: openstack-manila-csi-controllerplugin - namespace: default -roleRef: - kind: Role - name: openstack-manila-csi-controllerplugin - apiGroup: rbac.authorization.k8s.io diff --git a/manifests/manila-csi-plugin/csi-controllerplugin.yaml b/manifests/manila-csi-plugin/csi-controllerplugin.yaml deleted file mode 100644 index 6fe83c389a..0000000000 --- a/manifests/manila-csi-plugin/csi-controllerplugin.yaml +++ /dev/null @@ -1,119 +0,0 @@ -kind: Service -apiVersion: v1 -metadata: - name: openstack-manila-csi-controllerplugin - labels: - app: openstack-manila-csi - component: controllerplugin -spec: - selector: - app: openstack-manila-csi - component: controllerplugin - ports: - - name: dummy - port: 12345 ---- -kind: StatefulSet -apiVersion: apps/v1 -metadata: - name: openstack-manila-csi-controllerplugin - labels: - app: openstack-manila-csi - component: controllerplugin -spec: - serviceName: openstack-manila-csi-controllerplugin - replicas: 1 - selector: - matchLabels: - app: openstack-manila-csi - component: controllerplugin - template: - metadata: - labels: - app: openstack-manila-csi - component: controllerplugin - spec: - serviceAccountName: openstack-manila-csi-controllerplugin - containers: - - name: provisioner - image: "quay.io/k8scsi/csi-provisioner:v1.4.0" - args: - - "--v=5" - - "--csi-address=$(ADDRESS)" - # To enable topology awareness in csi-provisioner, uncomment the following line: - # - "--feature-gates=Topology=true" - env: - - name: ADDRESS - value: "unix:///var/lib/kubelet/plugins/manila.csi.openstack.org/csi-controllerplugin.sock" - imagePullPolicy: IfNotPresent - volumeMounts: - - name: plugin-dir - mountPath: /var/lib/kubelet/plugins/manila.csi.openstack.org - - name: snapshotter - image: "quay.io/k8scsi/csi-snapshotter:v1.2.2" - args: - - "--v=5" - - "--csi-address=$(ADDRESS)" - env: - - name: ADDRESS - value: "unix:///var/lib/kubelet/plugins/manila.csi.openstack.org/csi-controllerplugin.sock" - imagePullPolicy: IfNotPresent - volumeMounts: - - name: plugin-dir - mountPath: /var/lib/kubelet/plugins/manila.csi.openstack.org - - name: nodeplugin - securityContext: - privileged: true - capabilities: - add: ["SYS_ADMIN"] - allowPrivilegeEscalation: true - image: "k8scloudprovider/manila-csi-plugin:latest" - command: ["/bin/sh", "-c", - '/bin/manila-csi-plugin - --v=5 - --nodeid=$(NODE_ID) - --endpoint=$(CSI_ENDPOINT) - --drivername=$(DRIVER_NAME) - --share-protocol-selector=$(MANILA_SHARE_PROTO) - --fwdendpoint=$(FWD_CSI_ENDPOINT)' - # To enable topology awareness and retrieve compute node AZs from the OpenStack Metadata Service, add the following flags: - # --with-topology - # --nodeaz=$(curl http://169.254.169.254/openstack/latest/meta_data.json | jq -r .availability_zone) - # Those flags need to be added to csi-nodeplugin.yaml as well. - ] - env: - - name: DRIVER_NAME - value: manila.csi.openstack.org - - name: NODE_ID - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: CSI_ENDPOINT - value: "unix:///var/lib/kubelet/plugins/manila.csi.openstack.org/csi-controllerplugin.sock" - - name: FWD_CSI_ENDPOINT - value: "unix:///var/lib/kubelet/plugins/FWD-NODEPLUGIN/csi.sock" - - name: MANILA_SHARE_PROTO - value: "SHARE-PROTOCOL" - imagePullPolicy: IfNotPresent - volumeMounts: - - name: plugin-dir - mountPath: /var/lib/kubelet/plugins/manila.csi.openstack.org - - name: fwd-plugin-dir - mountPath: /var/lib/kubelet/plugins/FWD-NODEPLUGIN - - name: pod-mounts - mountPath: /var/lib/kubelet/pods - mountPropagation: Bidirectional - volumes: - - name: plugin-dir - hostPath: - path: /var/lib/kubelet/plugins/manila.csi.openstack.org - type: DirectoryOrCreate - - name: fwd-plugin-dir - hostPath: - path: /var/lib/kubelet/plugins/FWD-NODEPLUGIN - type: Directory - - name: pod-mounts - hostPath: - path: /var/lib/kubelet/pods - type: Directory - diff --git a/manifests/manila-csi-plugin/csi-nodeplugin-rbac.yaml b/manifests/manila-csi-plugin/csi-nodeplugin-rbac.yaml deleted file mode 100644 index 6d6228cdfd..0000000000 --- a/manifests/manila-csi-plugin/csi-nodeplugin-rbac.yaml +++ /dev/null @@ -1,58 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: openstack-manila-csi-nodeplugin - labels: - app: openstack-manila-csi - component: nodeplugin ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: openstack-manila-csi-nodeplugin - labels: - app: openstack-manila-csi - component: nodeplugin -aggregationRule: - clusterRoleSelectors: - - matchLabels: - rbac.manila.csi.openstack.org/aggregate-to-openstack-manila-csi-nodeplugin: "true" -rules: [] ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: openstack-manila-csi-nodeplugin-rules - labels: - app: openstack-manila-csi - component: nodeplugin - rbac.manila.csi.openstack.org/aggregate-to-openstack-manila-csi-nodeplugin: "true" -rules: - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "update"] - - apiGroups: [""] - resources: ["namespaces"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: openstack-manila-csi-nodeplugin - labels: - app: openstack-manila-csi - component: nodeplugin -subjects: - - kind: ServiceAccount - name: openstack-manila-csi-nodeplugin - namespace: default -roleRef: - kind: ClusterRole - name: openstack-manila-csi-nodeplugin - apiGroup: rbac.authorization.k8s.io diff --git a/manifests/manila-csi-plugin/csi-nodeplugin.yaml b/manifests/manila-csi-plugin/csi-nodeplugin.yaml deleted file mode 100644 index 46f33887f7..0000000000 --- a/manifests/manila-csi-plugin/csi-nodeplugin.yaml +++ /dev/null @@ -1,100 +0,0 @@ -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: openstack-manila-csi-nodeplugin - labels: - app: openstack-manila-csi - component: nodeplugin -spec: - selector: - matchLabels: - app: openstack-manila-csi - component: nodeplugin - template: - metadata: - labels: - app: openstack-manila-csi - component: nodeplugin - spec: - serviceAccountName: openstack-manila-csi-nodeplugin - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet - containers: - - name: registrar - image: "quay.io/k8scsi/csi-node-driver-registrar:v1.1.0" - args: - - "--v=5" - - "--csi-address=/csi/csi.sock" - - "--kubelet-registration-path=/var/lib/kubelet/plugins/manila.csi.openstack.org/csi.sock" - lifecycle: - preStop: - exec: - command: [ - "/bin/sh", "-c", - 'rm -rf /registration/manila.csi.openstack.org - /registration/manila.csi.openstack.org-reg.sock' - ] - env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - imagePullPolicy: IfNotPresent - volumeMounts: - - name: plugin-dir - mountPath: /csi - - name: registration-dir - mountPath: /registration - - name: nodeplugin - securityContext: - privileged: true - capabilities: - add: ["SYS_ADMIN"] - allowPrivilegeEscalation: true - image: "k8scloudprovider/manila-csi-plugin:latest" - command: ["/bin/sh", "-c", - '/bin/manila-csi-plugin - --v=5 - --nodeid=$(NODE_ID) - --endpoint=$(CSI_ENDPOINT) - --drivername=$(DRIVER_NAME) - --share-protocol-selector=$(MANILA_SHARE_PROTO) - --fwdendpoint=$(FWD_CSI_ENDPOINT)' - # To enable topology awareness and retrieve compute node AZs from the OpenStack Metadata Service, add the following flags: - # --with-topology - # --nodeaz=$(curl http://169.254.169.254/openstack/latest/meta_data.json | jq -r .availability_zone) - # Those flags need to be added to csi-controllerplugin.yaml as well. - ] - env: - - name: DRIVER_NAME - value: manila.csi.openstack.org - - name: NODE_ID - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: CSI_ENDPOINT - value: "unix:///var/lib/kubelet/plugins/manila.csi.openstack.org/csi.sock" - - name: FWD_CSI_ENDPOINT - value: "unix:///var/lib/kubelet/plugins/FWD-NODEPLUGIN/csi.sock" - - name: MANILA_SHARE_PROTO - value: "SHARE-PROTOCOL" - imagePullPolicy: IfNotPresent - volumeMounts: - - name: plugin-dir - mountPath: /var/lib/kubelet/plugins/manila.csi.openstack.org - - name: fwd-plugin-dir - mountPath: /var/lib/kubelet/plugins/FWD-NODEPLUGIN - volumes: - - name: registration-dir - hostPath: - path: /var/lib/kubelet/plugins_registry - type: Directory - - name: plugin-dir - hostPath: - path: /var/lib/kubelet/plugins/manila.csi.openstack.org - type: DirectoryOrCreate - - name: fwd-plugin-dir - hostPath: - path: /var/lib/kubelet/plugins/FWD-NODEPLUGIN - type: DirectoryOrCreate - diff --git a/manifests/manila-csi-plugin/csidriver.yaml b/manifests/manila-csi-plugin/csidriver.yaml deleted file mode 100644 index d0140122f2..0000000000 --- a/manifests/manila-csi-plugin/csidriver.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: storage.k8s.io/v1beta1 -kind: CSIDriver -metadata: - name: manila.csi.openstack.org -spec: - attachRequired: false - podInfoOnMount: false - diff --git a/pkg/autohealing/OWNERS b/pkg/autohealing/OWNERS deleted file mode 100644 index c014c763ef..0000000000 --- a/pkg/autohealing/OWNERS +++ /dev/null @@ -1,6 +0,0 @@ -approvers: -- lingxiankong -- openstacker -reviewers: -- lingxiankong -- openstacker diff --git a/pkg/autohealing/cloudprovider/cloudprovider.go b/pkg/autohealing/cloudprovider/cloudprovider.go deleted file mode 100644 index ef45479bcf..0000000000 --- a/pkg/autohealing/cloudprovider/cloudprovider.go +++ /dev/null @@ -1,70 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cloudprovider - -import ( - "k8s.io/client-go/kubernetes" - log "k8s.io/klog/v2" - - "k8s.io/cloud-provider-openstack/pkg/autohealing/config" - "k8s.io/cloud-provider-openstack/pkg/autohealing/healthcheck" -) - -var ( - providers = make(map[string]RegisterFunc) -) - -// CloudProvider is an abstract, pluggable interface for cloud providers. -type CloudProvider interface { - // GetName returns the cloud provider name. - GetName() string - - // Update cluster health status. - UpdateHealthStatus([]healthcheck.NodeInfo, []healthcheck.NodeInfo) error - - // Repair triggers the node repair process in the cloud. - Repair([]healthcheck.NodeInfo) error - - // Enabled decides if the repair should be triggered. - // It's recommended that the `Enabled()` function of the cloud provider doesn't allow to re-trigger when the repair - // is in place, e.g. before the repair process is finished, `Enabled()` should return false so that we won't - // re-trigger the repair process in the subsequent checks. - // This function also provides the cluster admin the capability to disable the cluster auto healing on the fly. - Enabled() bool -} - -type RegisterFunc func(config config.Config, client kubernetes.Interface) (CloudProvider, error) - -// RegisterCloudProvider registers a cloudprovider.Factory by name. This -// is expected to happen during app startup. -func RegisterCloudProvider(name string, register RegisterFunc) { - if _, found := providers[name]; found { - log.Fatalf("Cloud provider %s is already registered.", name) - } - - log.Infof("Registered cloud provider %s.", name) - providers[name] = register -} - -// GetCloudProvider creates an instance of the named cloud provider -func GetCloudProvider(name string, config config.Config, client kubernetes.Interface) (CloudProvider, error) { - f, found := providers[name] - if !found { - return nil, nil - } - return f(config, client) -} diff --git a/pkg/autohealing/cloudprovider/openstack/provider.go b/pkg/autohealing/cloudprovider/openstack/provider.go deleted file mode 100644 index abc8edad3b..0000000000 --- a/pkg/autohealing/cloudprovider/openstack/provider.go +++ /dev/null @@ -1,490 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package openstack - -import ( - "context" - "encoding/json" - "fmt" - "strconv" - "strings" - "time" - - "github.com/gophercloud/gophercloud" - "github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/startstop" - "github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/volumeattach" - "github.com/gophercloud/gophercloud/openstack/compute/v2/servers" - "github.com/gophercloud/gophercloud/openstack/containerinfra/v1/clusters" - "github.com/gophercloud/gophercloud/openstack/containerinfra/v1/nodegroups" - "github.com/gophercloud/gophercloud/openstack/orchestration/v1/stackresources" - "github.com/gophercloud/gophercloud/openstack/orchestration/v1/stacks" - "github.com/gophercloud/gophercloud/pagination" - uuid "github.com/pborman/uuid" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/kubernetes" - log "k8s.io/klog/v2" - - "k8s.io/cloud-provider-openstack/pkg/autohealing/config" - "k8s.io/cloud-provider-openstack/pkg/autohealing/healthcheck" -) - -const ( - ProviderName = "openstack" - ClusterAutoHealingLabel = "auto_healing_enabled" - clusterStatusUpdateInProgress = "UPDATE_IN_PROGRESS" - clusterStatusUpdateComplete = "UPDATE_COMPLETE" - clusterStatusUpdateFailed = "UPDATE_FAILED" - stackStatusUpdateFailed = "UPDATE_FAILED" - stackStatusUpdateInProgress = "UPDATE_IN_PROGRESS" - stackStatusUpdateComplete = "UPDATE_COMPLETE" -) - -var statusesPreventingRepair = sets.NewString( - stackStatusUpdateInProgress, - stackStatusUpdateFailed, -) - -// OpenStack is an implementation of cloud provider Interface for OpenStack. -type OpenStackCloudProvider struct { - KubeClient kubernetes.Interface - Nova *gophercloud.ServiceClient - Heat *gophercloud.ServiceClient - Magnum *gophercloud.ServiceClient - Config config.Config - ResourceStackMapping map[string]ResourceStackRelationship -} - -type ResourceStackRelationship struct { - ResourceID string - ResourceName string - StackID string - StackName string -} - -func (provider OpenStackCloudProvider) GetName() string { - return ProviderName -} - -// getStackName finds the name of a stack matching a given ID. -func (provider *OpenStackCloudProvider) getStackName(stackID string) (string, error) { - stack, err := stacks.Find(provider.Heat, stackID).Extract() - if err != nil { - return "", err - } - - return stack.Name, nil -} - -// getAllStackResourceMapping returns all mapping relationships including -// masters and minions(workers). The key in the map is the server/instance ID -// in Nova and the value is the resource ID and name of the server, and the -// parent stack ID and name. -func (provider *OpenStackCloudProvider) getAllStackResourceMapping(stackName, stackID string) (m map[string]ResourceStackRelationship, err error) { - if provider.ResourceStackMapping != nil { - return provider.ResourceStackMapping, nil - } - - mapping := make(map[string]ResourceStackRelationship) - - serverPages, err := stackresources.List(provider.Heat, stackName, stackID, stackresources.ListOpts{Depth: 2}).AllPages() - if err != nil { - return m, err - } - serverResources, err := stackresources.ExtractResources(serverPages) - if err != nil { - return m, err - } - - for _, sr := range serverResources { - if sr.Type != "OS::Nova::Server" { - continue - } - - for _, link := range sr.Links { - if link.Rel == "self" { - continue - } - - paths := strings.Split(link.Href, "/") - if len(paths) > 2 { - m := ResourceStackRelationship{ - ResourceID: sr.PhysicalID, - ResourceName: sr.Name, - StackID: paths[len(paths)-1], - StackName: paths[len(paths)-2], - } - - log.V(4).Infof("resource ID: %s, resource name: %s, parent stack ID: %s, parent stack name: %s", sr.PhysicalID, sr.Name, paths[len(paths)-1], paths[len(paths)-2]) - - mapping[sr.PhysicalID] = m - } - } - } - provider.ResourceStackMapping = mapping - - return provider.ResourceStackMapping, nil -} - -func (provider OpenStackCloudProvider) waitForServerPoweredOff(serverID string, timeout time.Duration) error { - err := startstop.Stop(provider.Nova, serverID).ExtractErr() - if err != nil { - return err - } - - err = wait.Poll(3*time.Second, timeout, - func() (bool, error) { - server, err := servers.Get(provider.Nova, serverID).Extract() - if err != nil { - return false, err - } - - if server.Status == "SHUTOFF" { - return true, nil - } - - return false, nil - }) - - return err -} - -// waitForClusterStatus checks periodically to see if the cluster has entered a given status. -// Returns when the status is observed or the timeout is reached. -func (provider OpenStackCloudProvider) waitForClusterComplete(clusterID string, timeout time.Duration) error { - log.V(2).Infof("Waiting for cluster %s in complete status", clusterID) - - err := wait.Poll(3*time.Second, timeout, - func() (bool, error) { - cluster, err := clusters.Get(provider.Magnum, clusterID).Extract() - log.V(5).Infof("Cluster %s in status %s", clusterID, cluster.Status) - if err != nil { - return false, err - } - if strings.HasSuffix(cluster.Status, "_COMPLETE") { - return true, nil - } - return false, nil - }) - return err -} - -func (provider OpenStackCloudProvider) waitForServerDetachVolumes(serverID string, timeout time.Duration) error { - err := volumeattach.List(provider.Nova, serverID).EachPage(func(page pagination.Page) (bool, error) { - attachments, err := volumeattach.ExtractVolumeAttachments(page) - if err != nil { - return false, err - } - for _, attachment := range attachments { - log.Infof("detaching volume %s for instance %s", attachment.VolumeID, serverID) - err := volumeattach.Delete(provider.Nova, serverID, attachment.ID).ExtractErr() - if err != nil { - return false, fmt.Errorf("failed to detach volume %s from instance %s", attachment.VolumeID, serverID) - } - } - return true, err - }) - if err != nil { - return err - } - err = wait.Poll(3*time.Second, timeout, - func() (bool, error) { - server, err := servers.Get(provider.Nova, serverID).Extract() - if err != nil { - return false, err - } - - if len(server.AttachedVolumes) == 0 { - return true, nil - } - - return false, nil - }) - - return err -} - -// For master nodes: Soft deletes the VMs, marks the heat resource "unhealthy" then trigger Heat stack update in order to rebuild -// the VMs. The information this function needs: -// - Nova VM IDs -// - Heat stack ID and resource ID. -// For worker nodes: Call Magnum resize API directly. -func (provider OpenStackCloudProvider) Repair(nodes []healthcheck.NodeInfo) error { - if len(nodes) == 0 { - return nil - } - - masters := []healthcheck.NodeInfo{} - workers := []healthcheck.NodeInfo{} - - clusterName := provider.Config.ClusterName - isWorkerNode := nodes[0].IsWorker - if isWorkerNode { - workers = nodes - } else { - masters = nodes - } - - err := provider.UpdateHealthStatus(masters, workers) - if err != nil { - return fmt.Errorf("Failed to update the helath status of cluster %s, error: %v", clusterName, err) - } - - cluster, err := clusters.Get(provider.Magnum, clusterName).Extract() - if err != nil { - return fmt.Errorf("failed to get the cluster %s, error: %v", clusterName, err) - } - - if isWorkerNode { - for _, n := range nodes { - nodesToReplace := sets.NewString() - machineID := uuid.Parse(n.KubeNode.Status.NodeInfo.MachineID) - if machineID == nil { - log.Warningf("Failed to get the correct server ID for server %s", n.KubeNode.Name) - continue - } - serverID := machineID.String() - - if err := provider.waitForServerDetachVolumes(serverID, 30*time.Second); err != nil { - log.Warningf("Failed to detach volumes from server %s, error: %v", serverID, err) - } - - if err := provider.waitForServerPoweredOff(serverID, 30*time.Second); err != nil { - log.Warningf("Failed to shutdown the server %s, error: %v", serverID, err) - } - - nodesToReplace.Insert(serverID) - ng, err := provider.getNodeGroup(clusterName, n) - ngName := "default-worker" - ngNodeCount := &cluster.NodeCount - if err == nil { - ngName = ng.Name - ngNodeCount = &ng.NodeCount - } - - opts := clusters.ResizeOpts{ - NodeGroup: ngName, - NodeCount: ngNodeCount, - NodesToRemove: nodesToReplace.List(), - } - - clusters.Resize(provider.Magnum, clusterName, opts) - // Wait 10 seconds to make sure Magnum has already got the request - // to avoid sending all of the resize API calls at the same time. - time.Sleep(10 * time.Second) - // TODO: Ignore the result value until https://github.com/gophercloud/gophercloud/pull/1649 is merged. - //if ret.Err != nil { - // return fmt.Errorf("failed to resize cluster %s, error: %v", clusterName, ret.Err) - //} - - log.Infof("Cluster %s resized", clusterName) - } - } else { - clusterStackName, err := provider.getStackName(cluster.StackID) - if err != nil { - return fmt.Errorf("failed to get the Heat stack for cluster %s, error: %v", clusterName, err) - } - - // In order to rebuild the nodes by Heat stack update, we need to know the parent stack ID of the resources and - // mark them unhealthy first. - allMapping, err := provider.getAllStackResourceMapping(clusterStackName, cluster.StackID) - if err != nil { - return fmt.Errorf("failed to get the resource stack mapping for cluster %s, error: %v", clusterName, err) - } - - for _, n := range nodes { - id := uuid.Parse(n.KubeNode.Status.NodeInfo.MachineID) - if id == nil { - log.Warningf("Failed to get the correct server ID for server %s", n.KubeNode.Name) - continue - } - serverID := id.String() - - if err := provider.waitForServerPoweredOff(serverID, 30*time.Second); err != nil { - log.Warningf("Failed to shutdown the server %s, error: %v", serverID, err) - } - - log.Infof("Marking Nova VM %s(Heat resource %s) unhealthy for Heat stack %s", serverID, allMapping[serverID].ResourceID, cluster.StackID) - - opts := stackresources.MarkUnhealthyOpts{ - MarkUnhealthy: true, - ResourceStatusReason: "Mark resource unhealthy by autohealing service", - } - err = stackresources.MarkUnhealthy(provider.Heat, allMapping[serverID].StackName, allMapping[serverID].StackID, allMapping[serverID].ResourceID, opts).ExtractErr() - if err != nil { - log.Errorf("failed to mark resource %s unhealthy, error: %v", serverID, err) - } - } - - if err := stacks.UpdatePatch(provider.Heat, clusterStackName, cluster.StackID, stacks.UpdateOpts{}).ExtractErr(); err != nil { - return fmt.Errorf("failed to update Heat stack to rebuild resources, error: %v", err) - } - - log.Infof("Started Heat stack update to rebuild resources, cluster: %s, stack: %s", clusterName, cluster.StackID) - } - - // Remove the broken nodes from the cluster - for _, n := range nodes { - if err := provider.KubeClient.CoreV1().Nodes().Delete(context.TODO(), n.KubeNode.Name, metav1.DeleteOptions{}); err != nil { - log.Errorf("Failed to remove the node %s from cluster, error: %v", n.KubeNode.Name, err) - } - } - - return nil -} - -func (provider OpenStackCloudProvider) getNodeGroup(clusterName string, node healthcheck.NodeInfo) (nodegroups.NodeGroup, error) { - var ng nodegroups.NodeGroup - - ngPages, err := nodegroups.List(provider.Magnum, clusterName, nodegroups.ListOpts{}).AllPages() - if err == nil { - ngs, err := nodegroups.ExtractNodeGroups(ngPages) - if err != nil { - log.Warningf("Failed to get node group for cluster %s, error: %v", clusterName, err) - return ng, err - } - for _, ng := range ngs { - ngInfo, err := nodegroups.Get(provider.Magnum, clusterName, ng.UUID).Extract() - if err != nil { - log.Warningf("Failed to get node group for cluster %s, error: %v", clusterName, err) - return ng, err - } - log.Infof("Got node addresses %v, node group's node addresses %v ", node.KubeNode.Status.Addresses, ngInfo.NodeAddresses) - for _, na := range node.KubeNode.Status.Addresses { - for _, nodeAddress := range ngInfo.NodeAddresses { - if na.Address == nodeAddress { - log.Infof("Got matched node group %s", ngInfo.Name) - return *ngInfo, nil - } - } - } - } - } - - return ng, fmt.Errorf("failed to find node group") -} - -// UpdateHealthStatus can update the cluster health status to reflect the -// real-time health status of the k8s cluster. -func (provider OpenStackCloudProvider) UpdateHealthStatus(masters []healthcheck.NodeInfo, workers []healthcheck.NodeInfo) error { - log.Infof("start to update cluster health status.") - clusterName := provider.Config.ClusterName - - healthStatus := "UNHEALTHY" - healthStatusReasonMap := make(map[string]string) - healthStatusReasonMap["updated_at"] = time.Now().String() - - if len(masters) == 0 && len(workers) == 0 { - // No unhealthy node passed in means the cluster is healthy - healthStatus = "HEALTHY" - healthStatusReasonMap["api"] = "ok" - healthStatusReasonMap["nodes"] = "ok" - } else { - if len(workers) > 0 { - for _, n := range workers { - // TODO: Need to figure out a way to reflect the detailed error information - healthStatusReasonMap[n.KubeNode.Name+"."+n.FailedCheck] = "error" - } - } else { - // TODO: Need to figure out a way to reflect detailed error information - healthStatusReasonMap["api"] = "error" - } - } - - jsonDumps, err := json.Marshal(healthStatusReasonMap) - if err != nil { - return fmt.Errorf("failed to build health status reason for cluster %s, error: %v", clusterName, err) - } - healthStatusReason := strings.Replace(string(jsonDumps), "\"", "'", -1) - - updateOpts := []clusters.UpdateOptsBuilder{ - clusters.UpdateOpts{ - Op: clusters.ReplaceOp, - Path: "/health_status", - Value: healthStatus, - }, - clusters.UpdateOpts{ - Op: clusters.ReplaceOp, - Path: "/health_status_reason", - Value: healthStatusReason, - }, - } - - log.Infof("updating cluster health status as %s for reason %s.", healthStatus, healthStatusReason) - res := clusters.Update(provider.Magnum, clusterName, updateOpts) - - if res.Err != nil { - return fmt.Errorf("failed to update the health status of cluster %s error: %v", clusterName, res.Err) - } - - if err := provider.waitForClusterComplete(clusterName, 30*time.Second); err != nil { - log.Warningf("failed to wait the cluster %s in complete status, error: %v", clusterName, err) - } - - return nil -} - -// Enabled decides if the repair should be triggered. -// There are two conditions that we disable the repair: -// - The cluster admin disables the auto healing via OpenStack API. -// - The Magnum cluster is not in stable status. -func (provider OpenStackCloudProvider) Enabled() bool { - clusterName := provider.Config.ClusterName - - cluster, err := clusters.Get(provider.Magnum, clusterName).Extract() - if err != nil { - log.Warningf("failed to get the cluster %s, error: %v", clusterName, err) - return false - } - - if !strings.HasSuffix(cluster.Status, "_COMPLETE") { - return false - } - - if _, isPresent := cluster.Labels[ClusterAutoHealingLabel]; !isPresent { - log.Infof("Autohealing is disalbed for cluster %s", clusterName) - return false - } - autoHealingEnabled, err := strconv.ParseBool(cluster.Labels[ClusterAutoHealingLabel]) - if err != nil { - log.Warningf("Unexpected value for %s label in cluster %s", ClusterAutoHealingLabel, clusterName) - return false - } - if !autoHealingEnabled { - log.Infof("Autohealing is disalbed for cluster %s", clusterName) - return false - } - - clusterStackName, err := provider.getStackName(cluster.StackID) - if err != nil { - log.Warningf("Failed to get the Heat stack ID for cluster %s, error: %v", clusterName, err) - return false - } - stack, err := stacks.Get(provider.Heat, clusterStackName, cluster.StackID).Extract() - if err != nil { - log.Warningf("Failed to get Heat stack %s for cluster %s, error: %v", cluster.StackID, clusterName, err) - return false - } - - if statusesPreventingRepair.Has(stack.Status) { - log.Infof("Current cluster stack is in status %s, skip the repair", stack.Status) - return false - } - - return true -} diff --git a/pkg/autohealing/cloudprovider/register/register.go b/pkg/autohealing/cloudprovider/register/register.go deleted file mode 100644 index 0c4eb5668c..0000000000 --- a/pkg/autohealing/cloudprovider/register/register.go +++ /dev/null @@ -1,81 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// register package is introduced in order to avoid circle imports between openstack and cloudprovider packages. -package register - -import ( - "fmt" - - "github.com/gophercloud/gophercloud" - gopenstack "github.com/gophercloud/gophercloud/openstack" - "k8s.io/client-go/kubernetes" - - "k8s.io/cloud-provider-openstack/pkg/autohealing/cloudprovider" - "k8s.io/cloud-provider-openstack/pkg/autohealing/cloudprovider/openstack" - "k8s.io/cloud-provider-openstack/pkg/autohealing/config" - openstack_provider "k8s.io/cloud-provider-openstack/pkg/cloudprovider/providers/openstack" -) - -func registerOpenStack(cfg config.Config, kubeClient kubernetes.Interface) (cloudprovider.CloudProvider, error) { - client, err := openstack_provider.NewOpenStackClient(&cfg.OpenStack, "magnum-auto-healer") - if err != nil { - return nil, err - } - - // get nova service client - var novaClient *gophercloud.ServiceClient - novaClient, err = gopenstack.NewComputeV2(client, gophercloud.EndpointOpts{ - Region: cfg.OpenStack.Region, - }) - if err != nil { - return nil, fmt.Errorf("failed to find Nova service endpoint in the region %s: %v", cfg.OpenStack.Region, err) - } - - // get heat service client - var heatClient *gophercloud.ServiceClient - heatClient, err = gopenstack.NewOrchestrationV1(client, gophercloud.EndpointOpts{ - Region: cfg.OpenStack.Region, - }) - if err != nil { - return nil, fmt.Errorf("failed to find Heat service endpoint in the region %s: %v", cfg.OpenStack.Region, err) - } - - // get magnum service client - var magnumClient *gophercloud.ServiceClient - magnumClient, err = gopenstack.NewContainerInfraV1(client, gophercloud.EndpointOpts{ - Region: cfg.OpenStack.Region, - }) - if err != nil { - return nil, fmt.Errorf("failed to find Magnum service endpoint in the region %s: %v", cfg.OpenStack.Region, err) - } - magnumClient.Microversion = "latest" - - var p cloudprovider.CloudProvider - p = openstack.OpenStackCloudProvider{ - KubeClient: kubeClient, - Nova: novaClient, - Heat: heatClient, - Magnum: magnumClient, - Config: cfg, - } - - return p, nil -} - -func init() { - cloudprovider.RegisterCloudProvider(openstack.ProviderName, registerOpenStack) -} diff --git a/pkg/autohealing/cmd/root.go b/pkg/autohealing/cmd/root.go deleted file mode 100644 index ce79e67eca..0000000000 --- a/pkg/autohealing/cmd/root.go +++ /dev/null @@ -1,138 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cmd - -import ( - "context" - goflag "flag" - "fmt" - "os" - "os/signal" - "syscall" - "time" - - "github.com/mitchellh/go-homedir" - "github.com/spf13/cobra" - flag "github.com/spf13/pflag" - "github.com/spf13/viper" - "k8s.io/client-go/tools/leaderelection" - log "k8s.io/klog/v2" - - "k8s.io/cloud-provider-openstack/pkg/autohealing/config" - "k8s.io/cloud-provider-openstack/pkg/autohealing/controller" -) - -var ( - cfgFile string - conf config.Config -) - -// rootCmd represents the base command when called without any subcommands -var rootCmd = &cobra.Command{ - Use: "magnum-auto-healer", - Short: "Auto healer for Kubernetes cluster.", - Long: "Auto healer is responsible for monitoring the nodes’ status periodically in the cloud environment, searching " + - "for unhealthy instances and triggering replacements when needed, maximizing the cluster’s efficiency and performance. " + - "OpenStack is supported by default.", - - Run: func(cmd *cobra.Command, args []string) { - autohealer := controller.NewController(conf) - - if !conf.LeaderElect { - autohealer.Start(context.TODO()) - panic("unreachable") - } - - lock, err := autohealer.GetLeaderElectionLock() - if err != nil { - log.Fatalf("failed to get resource lock for leader election, error: %v", err) - } - - // Try and become the leader and start autohealing loops - leaderelection.RunOrDie(context.TODO(), leaderelection.LeaderElectionConfig{ - Lock: lock, - LeaseDuration: 20 * time.Second, - RenewDeadline: 15 * time.Second, - RetryPeriod: 5 * time.Second, - Callbacks: leaderelection.LeaderCallbacks{ - OnStartedLeading: autohealer.Start, - OnStoppedLeading: func() { - log.Fatal("leaderelection lost") - }, - }, - Name: "k8s-auto-healer", - }) - - sigCh := make(chan os.Signal) - signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) - <-sigCh - }, -} - -// Execute adds all child commands to the root command sets flags appropriately. -// This is called by main.main(). It only needs to happen once to the rootCmd. -func Execute() { - if err := rootCmd.Execute(); err != nil { - fmt.Println(err) - os.Exit(-1) - } -} - -func init() { - cobra.OnInitialize(initConfig) - rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.kube_autohealer_config.yaml)") - - log.InitFlags(nil) - goflag.CommandLine.Parse(nil) - flag.CommandLine.AddGoFlagSet(goflag.CommandLine) -} - -// initConfig reads in config file and ENV variables if set. -func initConfig() { - if cfgFile != "" { - // Use config file from the flag. - viper.SetConfigFile(cfgFile) - } else { - // Find home directory. - home, err := homedir.Dir() - if err != nil { - fmt.Println(err) - os.Exit(1) - } - - // Search config in home directory with name ".kube_autohealer_config" (without extension). - viper.AddConfigPath(home) - viper.SetConfigName(".kube_autohealer_config") - } - - viper.AutomaticEnv() // read in environment variables that match - - // If a config file is found, read it in. - if err := viper.ReadInConfig(); err != nil { - log.Fatalf("Failed to read config file, error: %s", err) - } - - log.Infof("Using config file %s", viper.ConfigFileUsed()) - - conf = config.NewConfig() - if err := viper.Unmarshal(&conf); err != nil { - log.Fatalf("Unable to decode the configuration, error: %v", err) - } - if conf.ClusterName == "" { - log.Fatal("cluster-name is required in the configuration.") - } -} diff --git a/pkg/autohealing/config/config.go b/pkg/autohealing/config/config.go deleted file mode 100644 index 10c3b5b043..0000000000 --- a/pkg/autohealing/config/config.go +++ /dev/null @@ -1,94 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package config - -import ( - "time" - - openstack_provider "k8s.io/cloud-provider-openstack/pkg/cloudprovider/providers/openstack" -) - -// Config struct contains ingress controller configuration -type Config struct { - // (Optional) Emit an event without repairing nodes. Default: false - DryRun bool `mapstructure:"dry-run"` - - // (Required) Cluster identifier - ClusterName string `mapstructure:"cluster-name"` - - // (Optional) Cloud provider name. Default: openstack - CloudProvider string `mapstructure:"cloud-provider"` - - // (Optional) Interval of the nodes monitoring check. Default: 30s - MonitorInterval time.Duration `mapstructure:"monitor-interval"` - - // (Optional) If master nodes monitoring is enabled. Default: true - MasterMonitorEnabled bool `mapstructure:"master-monitor-enabled"` - - // (Optional) If worker nodes monitoring is enabled. Default: true - WorkerMonitorEnabled bool `mapstructure:"worker-monitor-enabled"` - - // (Optional) Kubernetes related configuration. - Kubernetes kubeConfig `mapstructure:"kubernetes"` - - // (Required) OpenStack related configuration. - OpenStack openstack_provider.AuthOpts `mapstructure:"openstack"` - - // (Optional) Healthcheck configuration for master and worker. - HealthCheck healthCheck `mapstructure:"healthcheck"` - - // (Optional) Start a leader election client and gain leadership before executing the main loop. Enable this when running replicated components for high availability. Default: true - LeaderElect bool `mapstructure:"leader-elect"` - - // (Optional) How long after new node added that the node will be checked for health status. Default: 10m - CheckDelayAfterAdd time.Duration `mapstructure:"check-delay-after-add"` -} - -type healthCheck struct { - Master []Check `mapstructure:"master"` - Worker []Check `mapstructure:"worker"` -} - -type Check struct { - // (Required) Health check plugin type. - Type string `mapstructure:"type"` - - // (Required) Customized health check parameters defined by individual health check plugin. - Params map[string]interface{} `mapstructure:"params"` -} - -// Configuration for connecting to Kubernetes API server, either api_host or kubeconfig should be configured. -type kubeConfig struct { - // (Optional) Kubernetes API server host address. - ApiserverHost string `mapstructure:"api-host"` - - // (Optional) Kubeconfig file used to connect to Kubernetes cluster. - KubeConfig string `mapstructure:"kubeconfig"` -} - -// NewConfig defines the default values for Config -func NewConfig() Config { - return Config{ - DryRun: false, - CloudProvider: "openstack", - MonitorInterval: 30 * time.Second, - MasterMonitorEnabled: true, - WorkerMonitorEnabled: true, - LeaderElect: true, - CheckDelayAfterAdd: 10 * time.Minute, - } -} diff --git a/pkg/autohealing/controller/controller.go b/pkg/autohealing/controller/controller.go deleted file mode 100644 index d2db36d7a1..0000000000 --- a/pkg/autohealing/controller/controller.go +++ /dev/null @@ -1,409 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "context" - "fmt" - "os" - "sync" - "time" - - apiv1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apimachinery/pkg/util/uuid" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/kubernetes/scheme" - typev1 "k8s.io/client-go/kubernetes/typed/core/v1" - restclient "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" - "k8s.io/client-go/tools/leaderelection/resourcelock" - "k8s.io/client-go/tools/record" - log "k8s.io/klog/v2" - - "k8s.io/cloud-provider-openstack/pkg/autohealing/cloudprovider" - _ "k8s.io/cloud-provider-openstack/pkg/autohealing/cloudprovider/register" - "k8s.io/cloud-provider-openstack/pkg/autohealing/config" - "k8s.io/cloud-provider-openstack/pkg/autohealing/healthcheck" -) - -// EventType type of event associated with an informer -type EventType string - -const ( - // High enough QPS to fit all expected use cases. QPS=0 is not set here, because - // client code is overriding it. - defaultQPS = 1e6 - // High enough Burst to fit all expected use cases. Burst=0 is not set here, because - // client code is overriding it. - defaultBurst = 1e6 - - // CreateEvent event associated with new objects in an informer - CreateEvent EventType = "CREATE" - // UpdateEvent event associated with an object update in an informer - UpdateEvent EventType = "UPDATE" - // DeleteEvent event associated when an object is removed from an informer - DeleteEvent EventType = "DELETE" - - // LabelNodeRoleMaster specifies that a node is a master - // Related discussion: https://github.com/kubernetes/kubernetes/pull/39112 - LabelNodeRoleMaster = "node-role.kubernetes.io/master" -) - -var ( - masterUnhealthyNodes []healthcheck.NodeInfo - workerUnhealthyNodes []healthcheck.NodeInfo -) - -// Event holds the context of an event -type Event struct { - Type EventType - Obj interface{} -} - -func createKubeClients(apiserverHost string, kubeConfig string) (*kubernetes.Clientset, *kubernetes.Clientset, error) { - cfg, err := clientcmd.BuildConfigFromFlags(apiserverHost, kubeConfig) - if err != nil { - return nil, nil, err - } - - cfg.QPS = defaultQPS - cfg.Burst = defaultBurst - cfg.ContentType = "application/vnd.kubernetes.protobuf" - cfg.Timeout = 5 * time.Second - - log.V(4).Info("Creating kubernetes API clients") - - client, err := kubernetes.NewForConfig(cfg) - if err != nil { - return nil, nil, err - } - leaderElectionClient, err := kubernetes.NewForConfig(restclient.AddUserAgent(cfg, "leader-election")) - if err != nil { - return nil, nil, err - } - - v, err := client.Discovery().ServerVersion() - if err != nil { - return nil, nil, err - } - log.V(4).Infof("Kubernetes API client created, server version: %s", fmt.Sprintf("v%v.%v", v.Major, v.Minor)) - - return client, leaderElectionClient, nil -} - -// NewController creates a new autohealer controller. -func NewController(conf config.Config) *Controller { - // initialize k8s clients - kubeClient, leaderElectionClient, err := createKubeClients(conf.Kubernetes.ApiserverHost, conf.Kubernetes.KubeConfig) - if err != nil { - log.Fatalf("failed to initialize kubernetes client, error: %v", err) - } - - // initialize cloud provider - provider, err := cloudprovider.GetCloudProvider(conf.CloudProvider, conf, kubeClient) - if err != nil { - log.Fatalf("Failed to get the cloud provider %s: %v", conf.CloudProvider, err) - } - - log.Infof("Using cloud provider: %s", provider.GetName()) - - // event - eventBroadcaster := record.NewBroadcaster() - eventBroadcaster.StartLogging(log.V(4).Infof) - eventBroadcaster.StartRecordingToSink(&typev1.EventSinkImpl{ - Interface: kubeClient.CoreV1().Events(""), - }) - recorder := eventBroadcaster.NewRecorder(scheme.Scheme, apiv1.EventSource{Component: "openstack-ingress-controller"}) - - // Initialize the configured health checkers - var workerCheckers []healthcheck.HealthCheck - var masterCheckers []healthcheck.HealthCheck - for _, item := range conf.HealthCheck.Worker { - checker, err := healthcheck.GetHealthChecker(item.Type, item.Params) - if err != nil { - log.Fatalf("failed to get %s type health check for worker node, error: %v", item.Type, err) - } - if !checker.IsWorkerSupported() { - log.Warningf("Plugin type %s does not support worker node health check, will skip", item.Type) - continue - } - workerCheckers = append(workerCheckers, checker) - } - for _, item := range conf.HealthCheck.Master { - checker, err := healthcheck.GetHealthChecker(item.Type, item.Params) - if err != nil { - log.Fatalf("failed to get %s type health check for master node, error: %v", item.Type, err) - } - if !checker.IsMasterSupported() { - log.Warningf("Plugin type %s does not support master node health check, will skip", item.Type) - continue - } - masterCheckers = append(masterCheckers, checker) - } - - controller := &Controller{ - config: conf, - recorder: recorder, - provider: provider, - kubeClient: kubeClient, - leaderElectionClient: leaderElectionClient, - masterCheckers: masterCheckers, - workerCheckers: workerCheckers, - } - - return controller -} - -// Controller ... -type Controller struct { - provider cloudprovider.CloudProvider - recorder record.EventRecorder - kubeClient kubernetes.Interface - leaderElectionClient kubernetes.Interface - config config.Config - workerCheckers []healthcheck.HealthCheck - masterCheckers []healthcheck.HealthCheck -} - -// UpdateNodeAnnotation updates the specified node annotation, if value equals empty string, the annotation will be -// removed. This implements the interface healthcheck.NodeController -func (c *Controller) UpdateNodeAnnotation(node healthcheck.NodeInfo, annotation string, value string) error { - n, err := c.kubeClient.CoreV1().Nodes().Get(context.TODO(), node.KubeNode.Name, metav1.GetOptions{}) - if err != nil { - return err - } - - if value == "" { - delete(n.Annotations, annotation) - } else { - n.Annotations[annotation] = value - } - - if _, err := c.kubeClient.CoreV1().Nodes().Update(context.TODO(), n, metav1.UpdateOptions{}); err != nil { - return err - } - - return nil -} - -func (c *Controller) GetLeaderElectionLock() (resourcelock.Interface, error) { - // Identity used to distinguish between multiple cloud controller manager instances - id, err := os.Hostname() - if err != nil { - return nil, err - } - // add a uniquifier so that two processes on the same host don't accidentally both become active - id = id + "_" + string(uuid.NewUUID()) - - rl, err := resourcelock.New( - "configmaps", - "kube-system", - "magnum-auto-healer", - c.leaderElectionClient.CoreV1(), - c.leaderElectionClient.CoordinationV1(), - resourcelock.ResourceLockConfig{ - Identity: id, - EventRecorder: c.recorder, - }) - if err != nil { - return nil, err - } - - return rl, nil -} - -// getUnhealthyMasterNodes returns the master nodes that need to be repaired. -func (c *Controller) getUnhealthyMasterNodes() ([]healthcheck.NodeInfo, error) { - var nodes []healthcheck.NodeInfo - - // If no checkers defined, skip - if len(c.masterCheckers) == 0 { - log.V(3).Info("No health check defined for master node, skip.") - return nodes, nil - } - - // Get all the master nodes need to check - nodeList, err := c.kubeClient.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) - if err != nil { - return nil, err - } - for _, node := range nodeList.Items { - if _, hasMasterRoleLabel := node.Labels[LabelNodeRoleMaster]; hasMasterRoleLabel { - if time.Now().Before(node.ObjectMeta.GetCreationTimestamp().Add(c.config.CheckDelayAfterAdd)) { - log.V(4).Infof("The node %s is created less than the configured check delay, skip", node.Name) - continue - } - - nodes = append(nodes, healthcheck.NodeInfo{KubeNode: node, IsWorker: false}) - } - } - - // Do health check - unhealthyNodes := healthcheck.CheckNodes(c.masterCheckers, nodes, c) - - return unhealthyNodes, nil -} - -// getUnhealthyWorkerNodes returns the nodes that need to be repaired. -func (c *Controller) getUnhealthyWorkerNodes() ([]healthcheck.NodeInfo, error) { - var nodes []healthcheck.NodeInfo - - // If no checkers defined, skip. - if len(c.workerCheckers) == 0 { - log.V(3).Info("No health check defined for worker node, skip.") - return nodes, nil - } - - // Get all the worker nodes. - nodeList, err := c.kubeClient.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) - if err != nil { - return nil, err - } - for _, node := range nodeList.Items { - if _, hasMasterRoleLabel := node.Labels[LabelNodeRoleMaster]; hasMasterRoleLabel { - continue - } - if len(node.Status.Conditions) == 0 { - continue - } - if time.Now().Before(node.ObjectMeta.GetCreationTimestamp().Add(c.config.CheckDelayAfterAdd)) { - log.V(4).Infof("The node %s is created less than the configured check delay, skip", node.Name) - continue - } - nodes = append(nodes, healthcheck.NodeInfo{KubeNode: node, IsWorker: true}) - } - - // Do health check - unhealthyNodes := healthcheck.CheckNodes(c.workerCheckers, nodes, c) - - return unhealthyNodes, nil -} - -func (c *Controller) repairNodes(unhealthyNodes []healthcheck.NodeInfo) { - unhealthyNodeNames := sets.NewString() - for _, n := range unhealthyNodes { - unhealthyNodeNames.Insert(n.KubeNode.Name) - } - - // Trigger unhealthy nodes repair. - if len(unhealthyNodes) > 0 { - if !c.provider.Enabled() { - // The cloud provider doesn't allow to trigger node repair. - log.Infof("Auto healing is ignored for nodes %s", unhealthyNodeNames.List()) - } else { - log.Infof("Starting to repair nodes %s, dryrun: %t", unhealthyNodeNames.List(), c.config.DryRun) - - if !c.config.DryRun { - // Cordon the nodes before repair. - for _, node := range unhealthyNodes { - nodeName := node.KubeNode.Name - newNode := node.KubeNode.DeepCopy() - newNode.Spec.Unschedulable = true - - if _, err := c.kubeClient.CoreV1().Nodes().Update(context.TODO(), newNode, metav1.UpdateOptions{}); err != nil { - log.Errorf("Failed to cordon node %s, error: %v", nodeName, err) - } else { - log.Infof("Node %s is cordoned", nodeName) - } - } - - // Start to repair all the unhealthy nodes. - if err := c.provider.Repair(unhealthyNodes); err != nil { - log.Errorf("Failed to repair the nodes %s, error: %v", unhealthyNodeNames.List(), err) - } - } - } - } -} - -// startMasterMonitor checks if there are failed master nodes and triggers the repair action. This function is supposed -// to be running in a goroutine. -func (c *Controller) startMasterMonitor(wg *sync.WaitGroup) { - log.V(3).Info("Starting to check master nodes.") - defer wg.Done() - - // Get all the unhealthy master nodes. - unhealthyNodes, err := c.getUnhealthyMasterNodes() - if err != nil { - log.Errorf("Failed to get unhealthy master nodes, error: %v", err) - return - } - - masterUnhealthyNodes = append(masterUnhealthyNodes, unhealthyNodes...) - - c.repairNodes(unhealthyNodes) - - if len(unhealthyNodes) == 0 { - log.V(3).Info("Master nodes are healthy") - } - - log.V(3).Info("Finished checking master nodes.") -} - -// startWorkerMonitor checks if there are failed worker nodes and triggers the repair action. This function is supposed -// to be running in a goroutine. -func (c *Controller) startWorkerMonitor(wg *sync.WaitGroup) { - log.V(3).Info("Starting to check worker nodes.") - defer wg.Done() - - // Get all the unhealthy worker nodes. - unhealthyNodes, err := c.getUnhealthyWorkerNodes() - if err != nil { - log.Errorf("Failed to get unhealthy worker nodes, error: %v", err) - return - } - - workerUnhealthyNodes = append(workerUnhealthyNodes, unhealthyNodes...) - - c.repairNodes(unhealthyNodes) - - if len(unhealthyNodes) == 0 { - log.V(3).Info("Worker nodes are healthy") - } - - log.V(3).Info("Finished checking worker nodes.") -} - -// Start starts the autohealing controller. -func (c *Controller) Start(ctx context.Context) { - log.Info("Starting autohealing controller") - - ticker := time.NewTicker(c.config.MonitorInterval) - defer ticker.Stop() - - var wg sync.WaitGroup - for { - masterUnhealthyNodes = []healthcheck.NodeInfo{} - workerUnhealthyNodes = []healthcheck.NodeInfo{} - select { - case <-ticker.C: - if c.config.MasterMonitorEnabled { - wg.Add(1) - go c.startMasterMonitor(&wg) - } - if c.config.WorkerMonitorEnabled { - wg.Add(1) - go c.startWorkerMonitor(&wg) - } - - wg.Wait() - c.provider.UpdateHealthStatus(masterUnhealthyNodes, workerUnhealthyNodes) - } - } -} diff --git a/pkg/autohealing/healthcheck/healthcheck.go b/pkg/autohealing/healthcheck/healthcheck.go deleted file mode 100644 index 7f6202d034..0000000000 --- a/pkg/autohealing/healthcheck/healthcheck.go +++ /dev/null @@ -1,91 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package healthcheck - -import ( - apiv1 "k8s.io/api/core/v1" - log "k8s.io/klog/v2" -) - -var ( - checkPlugins = make(map[string]registerPlugin) -) - -type registerPlugin func(config interface{}) (HealthCheck, error) - -// NodeInfo is a wrapper of Node, may contains more information in future. -type NodeInfo struct { - KubeNode apiv1.Node - IsWorker bool - FailedCheck string -} - -type HealthCheck interface { - // Check checks the node health, returns false if the node is unhealthy. The plugin should deal with any error happened. - Check(node NodeInfo, controller NodeController) bool - - // IsMasterSupported checks if the health check plugin supports master node. - IsMasterSupported() bool - - // IsWorkerSupported checks if the health check plugin supports worker node. - IsWorkerSupported() bool - - // GetName returns name of the health check plugin - GetName() string -} - -// NodeController is to avoid circle reference. -type NodeController interface { - // UpdateNodeAnnotation updates the specified node annotation, if value equals empty string, the annotation will be - // removed. - UpdateNodeAnnotation(node NodeInfo, annotation string, value string) error -} - -func registerHealthCheck(name string, register registerPlugin) { - if _, found := checkPlugins[name]; found { - log.Fatalf("Health check plugin %s is already registered.", name) - } - - log.Infof("Registered health check plugin %s", name) - checkPlugins[name] = register -} - -func GetHealthChecker(name string, config interface{}) (HealthCheck, error) { - c, found := checkPlugins[name] - if !found { - return nil, nil - } - return c(config) -} - -// CheckNodes goes through the health checkers, returns the unhealthy nodes. -func CheckNodes(checkers []HealthCheck, nodes []NodeInfo, controller NodeController) []NodeInfo { - var unhealthyNodes []NodeInfo - - // Check the health for each node. - for _, node := range nodes { - for _, checker := range checkers { - if !checker.Check(node, controller) { - node.FailedCheck = checker.GetName() - unhealthyNodes = append(unhealthyNodes, node) - break - } - } - } - - return unhealthyNodes -} diff --git a/pkg/autohealing/healthcheck/plugin_endpoint.go b/pkg/autohealing/healthcheck/plugin_endpoint.go deleted file mode 100644 index 8a02e6c54c..0000000000 --- a/pkg/autohealing/healthcheck/plugin_endpoint.go +++ /dev/null @@ -1,216 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package healthcheck - -import ( - "crypto/tls" - "fmt" - "io/ioutil" - "net/http" - "strings" - "time" - - "github.com/mitchellh/mapstructure" - log "k8s.io/klog/v2" - - "k8s.io/cloud-provider-openstack/pkg/autohealing/utils" -) - -const ( - EndpointType = "Endpoint" - TokenPath = "/var/run/secrets/kubernetes.io/serviceaccount/token" - TimeLayout = "2006-01-02 15:04:05" -) - -type EndpointCheck struct { - // (Optional) URL scheme, case insensitive, e.g. HTTP, HTTPS. Default: ["HTTPS"]. - Protocol string `mapstructure:"protocol"` - - // (Optional) Service port. Default: 6443 - Port int `mapstructure:"port"` - - // (Optional) The endpoints for health check. Default: ["/healthz"]. - Endpoints []string `mapstructure:"endpoints"` - - // (Optional) How long to wait before a unhealthy worker node should be repaired. Default: 300s - UnhealthyDuration time.Duration `mapstructure:"unhealthy-duration"` - - // (Optional) The node annotation which records the node unhealthy time. Default: autohealing.openstack.org/unhealthy-timestamp - UnhealthyAnnotation string `mapstructure:"unhealthy-annotation"` - - // (Optional) The accepted HTTP response codes. Default: [200]. - OKCodes []int `mapstructure:"ok-codes"` - - // (Optional) If token is required to access the endpoint. Default: false - RequireToken bool `mapstructure:"require-token"` - - // (Optional) Token to use in the request header. Default: read from TokenPath file - Token string `mapstructure:"token"` -} - -// GetName returns name of the health check -func (check *EndpointCheck) GetName() string { - return "EndpointCheck" -} - -// IsMasterSupported checks if the health check plugin supports master node. -func (check *EndpointCheck) IsMasterSupported() bool { - return true -} - -// IsWorkerSupported checks if the health check plugin supports worker node. -func (check *EndpointCheck) IsWorkerSupported() bool { - return false -} - -// checkDuration checks if the node should be marked as healthy or not. -func (check *EndpointCheck) checkDuration(node NodeInfo, controller NodeController, checkRet bool) bool { - name := node.KubeNode.Name - - if checkRet { - // Remove the annotation - if err := controller.UpdateNodeAnnotation(node, check.UnhealthyAnnotation, ""); err != nil { - log.Errorf("Failed to remove the node annotation(will skip the check) for %s, error: %v", name, err) - } - return true - } - - now := time.Now() - var unhealthyStartTime *time.Time - - // Get the current annotation value - if timeStr, isPresent := node.KubeNode.Annotations[check.UnhealthyAnnotation]; isPresent { - if timeStr != "" { - startTime, err := time.Parse(TimeLayout, timeStr) - if err != nil { - unhealthyStartTime = nil - } else { - unhealthyStartTime = &startTime - } - } - } - - if unhealthyStartTime == nil { - // Set the annotation value - if err := controller.UpdateNodeAnnotation(node, check.UnhealthyAnnotation, now.Format(TimeLayout)); err != nil { - log.Errorf("Failed to set the node annotation(will skip the check) for %s, error: %v", name, err) - } - return true - } - - if now.Sub(*unhealthyStartTime) >= check.UnhealthyDuration { - // Need repair - return false - } - // Keep the annotation value - return true -} - -// Check checks the node health, returns false if the node is unhealthy. Update the node cache accordingly. -func (check *EndpointCheck) Check(node NodeInfo, controller NodeController) bool { - nodeName := node.KubeNode.Name - ip := "" - for _, addr := range node.KubeNode.Status.Addresses { - if addr.Type == "InternalIP" { - ip = addr.Address - break - } - } - - if ip == "" { - log.Warningf("Cannot find IP address for node %s, skip the check", nodeName) - return true - } - - var client *http.Client - protocol := strings.ToLower(check.Protocol) - if protocol == "https" { - tr := &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - } - client = &http.Client{Transport: tr, Timeout: time.Second * 5} - } else if protocol == "http" { - client = &http.Client{Timeout: time.Second * 5} - } - - for _, endpoint := range check.Endpoints { - url := fmt.Sprintf("%s://%s:%d/%s", strings.ToLower(check.Protocol), ip, check.Port, strings.TrimLeft(endpoint, "/")) - req, err := http.NewRequest("GET", url, nil) - if err != nil { - log.Errorf("Node %s, failed to get request %s, error: %v", nodeName, url, err) - return check.checkDuration(node, controller, false) - } - - if check.RequireToken { - if check.Token == "" { - b, err := ioutil.ReadFile(TokenPath) - if err != nil { - log.Warningf("Node %s, failed to get token from %s, skip the check", nodeName, TokenPath) - return true - } - req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", string(b))) - } else { - req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", check.Token)) - } - } - - resp, err := client.Do(req) - if err != nil { - log.Errorf("Node %s, failed to read response for url %s, error: %v", nodeName, url, err) - return check.checkDuration(node, controller, false) - } - resp.Body.Close() - - if !utils.ContainsInt(check.OKCodes, resp.StatusCode) { - log.V(4).Infof("Node %s, return code for url %s is %d, expected: %d", nodeName, url, resp.StatusCode, check.OKCodes) - return check.checkDuration(node, controller, false) - } - } - - return check.checkDuration(node, controller, true) -} - -func newEndpointCheck(config interface{}) (HealthCheck, error) { - check := EndpointCheck{ - Protocol: "https", - Port: 6443, - UnhealthyDuration: 300 * time.Second, - Endpoints: []string{"/healthz"}, - OKCodes: []int{200}, - RequireToken: false, - UnhealthyAnnotation: "autohealing.openstack.org/unhealthy-timestamp", - } - - decConfig := mapstructure.DecoderConfig{ - DecodeHook: mapstructure.StringToTimeDurationHookFunc(), - Result: &check, - } - decoder, err := mapstructure.NewDecoder(&decConfig) - if err != nil { - return nil, err - } - err = decoder.Decode(config) - if err != nil { - return nil, fmt.Errorf("failed to get configuration for health check plugin %s, error: %v", NodeConditionType, err) - } - - return &check, nil -} - -func init() { - registerHealthCheck(EndpointType, newEndpointCheck) -} diff --git a/pkg/autohealing/healthcheck/plugin_nodecondition.go b/pkg/autohealing/healthcheck/plugin_nodecondition.go deleted file mode 100644 index 6c2458662c..0000000000 --- a/pkg/autohealing/healthcheck/plugin_nodecondition.go +++ /dev/null @@ -1,116 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package healthcheck - -import ( - "fmt" - "time" - - "github.com/mitchellh/mapstructure" - log "k8s.io/klog/v2" - - "k8s.io/cloud-provider-openstack/pkg/autohealing/utils" -) - -const ( - NodeConditionType = "NodeCondition" -) - -type NodeConditionCheck struct { - // (Optional) The condition types(case sensitive) specified in Node.spec.status.conditions items. Default: ["Ready"]. - Types []string `mapstructure:"types"` - - // (Optional) How long to wait before a unhealthy worker node should be repaired. Default: 300s - UnhealthyDuration time.Duration `mapstructure:"unhealthy-duration"` - - // (Optional) The accepted healthy values(case sensitive) for the type. Default: ["True"]. OKValues could not be used together with ErrorValues. - OKValues []string `mapstructure:"ok-values"` - - // (Optional) The unhealthy values(case sensitive) for the type. Default: []. ErrorValues could not be used together with OKValues. - ErrorValues []string `mapstructure:"error-values"` -} - -// Check checks the node health, returns false if the node is unhealthy. -func (check *NodeConditionCheck) Check(node NodeInfo, controller NodeController) bool { - nodeName := node.KubeNode.Name - - for _, cond := range node.KubeNode.Status.Conditions { - if utils.Contains(check.Types, string(cond.Type)) { - unhealthyDuration := time.Now().Sub(cond.LastTransitionTime.Time) - - if len(check.ErrorValues) > 0 { - if utils.Contains(check.ErrorValues, string(cond.Status)) { - if unhealthyDuration >= check.UnhealthyDuration { - return false - } - log.Warningf("Node %s is unhealthy, %s: %s", nodeName, string(cond.Type), string(cond.Status)) - } - } else if len(check.OKValues) > 0 { - if !utils.Contains(check.OKValues, string(cond.Status)) { - if unhealthyDuration >= check.UnhealthyDuration { - return false - } - log.Warningf("Node %s is unhealthy, %s: %s", nodeName, string(cond.Type), string(cond.Status)) - } - } - } - } - - return true -} - -// GetName returns name of the health check -func (check *NodeConditionCheck) GetName() string { - return "NodeConditionCheck" -} - -// IsMasterSupported checks if the health check plugin supports master node. -func (check *NodeConditionCheck) IsMasterSupported() bool { - return true -} - -// IsWorkerSupported checks if the health check plugin supports worker node. -func (check *NodeConditionCheck) IsWorkerSupported() bool { - return true -} - -func newNodeConditionCheck(config interface{}) (HealthCheck, error) { - check := NodeConditionCheck{ - UnhealthyDuration: 300 * time.Second, - Types: []string{"Ready"}, - OKValues: []string{"True"}, - } - - decConfig := mapstructure.DecoderConfig{ - DecodeHook: mapstructure.StringToTimeDurationHookFunc(), - Result: &check, - } - decoder, err := mapstructure.NewDecoder(&decConfig) - if err != nil { - return nil, err - } - err = decoder.Decode(config) - if err != nil { - return nil, fmt.Errorf("failed to get configuration for health check plugin %s, error: %v", NodeConditionType, err) - } - - return &check, nil -} - -func init() { - registerHealthCheck(NodeConditionType, newNodeConditionCheck) -} diff --git a/pkg/autohealing/utils/utils.go b/pkg/autohealing/utils/utils.go deleted file mode 100644 index f806878854..0000000000 --- a/pkg/autohealing/utils/utils.go +++ /dev/null @@ -1,37 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package utils - -// Contains searches if a string list contains the given string or not. -func Contains(list []string, strToSearch string) bool { - for _, item := range list { - if item == strToSearch { - return true - } - } - return false -} - -// ContainsInt searches if a int list contains the given int or not. -func ContainsInt(list []int, toSearch int) bool { - for _, item := range list { - if item == toSearch { - return true - } - } - return false -} diff --git a/pkg/cloudprovider/.import-restrictions b/pkg/cloudprovider/.import-restrictions deleted file mode 100644 index 0cff927521..0000000000 --- a/pkg/cloudprovider/.import-restrictions +++ /dev/null @@ -1,23 +0,0 @@ -{ - "Rules": [{ - "SelectorRegexp": "^k8s[.]io", - "AllowedPrefixes": [ - "k8s.io/api", - "k8s.io/client-go", - "k8s.io/cloud-provider", - "k8s.io/klog", - "k8s.io/utils" - ], - "ForbiddenPrefixes": [ - "k8s.io/kubernetes" - ] - }, - { - "SelectorRegexp": "^sigs[.]k8s[.]io", - "AllowedPrefixes": [ - "sigs.k8s.io/yaml", - "sigs.k8s.io/structured-merge-diff" - ] - } - ] -} diff --git a/pkg/cloudprovider/providers/openstack/OWNERS b/pkg/cloudprovider/providers/openstack/OWNERS deleted file mode 100644 index 83e574a1ab..0000000000 --- a/pkg/cloudprovider/providers/openstack/OWNERS +++ /dev/null @@ -1,8 +0,0 @@ -approvers: -- anguslees -- dims -- FengyunPan2 -reviewers: -- anguslees -- dims -- FengyunPan2 diff --git a/pkg/cloudprovider/providers/openstack/doc.go b/pkg/cloudprovider/providers/openstack/doc.go deleted file mode 100644 index 47b354a77e..0000000000 --- a/pkg/cloudprovider/providers/openstack/doc.go +++ /dev/null @@ -1,17 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package openstack // import "k8s.io/cloud-provider-openstack/pkg/cloudprovider/providers/openstack" diff --git a/pkg/cloudprovider/providers/openstack/openstack.go b/pkg/cloudprovider/providers/openstack/openstack.go deleted file mode 100644 index a8862e8d60..0000000000 --- a/pkg/cloudprovider/providers/openstack/openstack.go +++ /dev/null @@ -1,1038 +0,0 @@ -/* -Copyright 2014 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package openstack - -import ( - "context" - "crypto/tls" - "crypto/x509" - "errors" - "fmt" - "io" - "io/ioutil" - "net" - "net/http" - "os" - "regexp" - "runtime" - "sort" - "strings" - "time" - - "k8s.io/cloud-provider-openstack/pkg/util" - - "github.com/gophercloud/gophercloud" - "github.com/gophercloud/gophercloud/openstack" - "github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/attachinterfaces" - "github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/availabilityzones" - "github.com/gophercloud/gophercloud/openstack/compute/v2/servers" - "github.com/gophercloud/gophercloud/openstack/identity/v3/extensions/trusts" - tokens3 "github.com/gophercloud/gophercloud/openstack/identity/v3/tokens" - "github.com/gophercloud/gophercloud/pagination" - "github.com/gophercloud/utils/client" - "github.com/gophercloud/utils/openstack/clientconfig" - "github.com/mitchellh/mapstructure" - "github.com/spf13/pflag" - gcfg "gopkg.in/gcfg.v1" - - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" - netutil "k8s.io/apimachinery/pkg/util/net" - certutil "k8s.io/client-go/util/cert" - cloudprovider "k8s.io/cloud-provider" - "k8s.io/cloud-provider-openstack/pkg/util/metadata" - "k8s.io/cloud-provider-openstack/pkg/version" - "k8s.io/klog/v2" -) - -const ( - // ProviderName is the name of the openstack provider - ProviderName = "openstack" - - // TypeHostName is the name type of openstack instance - TypeHostName = "hostname" - availabilityZone = "availability_zone" - defaultTimeOut = 60 * time.Second -) - -// ErrNotFound is used to inform that the object is missing -var ErrNotFound = errors.New("failed to find object") - -// ErrMultipleResults is used when we unexpectedly get back multiple results -var ErrMultipleResults = errors.New("multiple results where only one expected") - -// ErrNoAddressFound is used when we cannot find an ip address for the host -var ErrNoAddressFound = errors.New("no address found for host") - -// ErrIPv6SupportDisabled is used when one tries to use IPv6 Addresses when -// IPv6 support is disabled by config -var ErrIPv6SupportDisabled = errors.New("IPv6 support is disabled") - -// userAgentData is used to add extra information to the gophercloud user-agent -var userAgentData []string - -// AddExtraFlags is called by the main package to add component specific command line flags -func AddExtraFlags(fs *pflag.FlagSet) { - fs.StringArrayVar(&userAgentData, "user-agent", nil, "Extra data to add to gophercloud user-agent. Use multiple times to add more than one component.") -} - -// MyDuration is the encoding.TextUnmarshaler interface for time.Duration -type MyDuration struct { - time.Duration -} - -// UnmarshalText is used to convert from text to Duration -func (d *MyDuration) UnmarshalText(text []byte) error { - res, err := time.ParseDuration(string(text)) - if err != nil { - return err - } - d.Duration = res - return nil -} - -// LoadBalancer is used for creating and maintaining load balancers -type LoadBalancer struct { - network *gophercloud.ServiceClient - compute *gophercloud.ServiceClient - lb *gophercloud.ServiceClient - opts LoadBalancerOpts -} - -// LoadBalancerOpts have the options to talk to Neutron LBaaSV2 or Octavia -type LoadBalancerOpts struct { - LBVersion string `gcfg:"lb-version"` // overrides autodetection. Only support v2. - UseOctavia bool `gcfg:"use-octavia"` // uses Octavia V2 service catalog endpoint - SubnetID string `gcfg:"subnet-id"` // overrides autodetection. - NetworkID string `gcfg:"network-id"` // If specified, will create virtual ip from a subnet in network which has available IP addresses - FloatingNetworkID string `gcfg:"floating-network-id"` // If specified, will create floating ip for loadbalancer, or do not create floating ip. - FloatingSubnetID string `gcfg:"floating-subnet-id"` // If specified, will create floating ip for loadbalancer in this particular floating pool subnetwork. - LBClasses map[string]*LBClass // Predefined named Floating networks and subnets - LBMethod string `gcfg:"lb-method"` // default to ROUND_ROBIN. - LBProvider string `gcfg:"lb-provider"` - CreateMonitor bool `gcfg:"create-monitor"` - MonitorDelay MyDuration `gcfg:"monitor-delay"` - MonitorTimeout MyDuration `gcfg:"monitor-timeout"` - MonitorMaxRetries uint `gcfg:"monitor-max-retries"` - ManageSecurityGroups bool `gcfg:"manage-security-groups"` - NodeSecurityGroupIDs []string // Do not specify, get it automatically when enable manage-security-groups. TODO(FengyunPan): move it into cache - InternalLB bool `gcfg:"internal-lb"` // default false - CascadeDelete bool `gcfg:"cascade-delete"` // applicable only if use-octavia is set to True - FlavorID string `gcfg:"flavor-id"` -} - -// LBClass defines the corresponding floating network, floating subnet or internal subnet ID -type LBClass struct { - FloatingNetworkID string `gcfg:"floating-network-id,omitempty"` - FloatingSubnetID string `gcfg:"floating-subnet-id,omitempty"` - SubnetID string `gcfg:"subnet-id,omitempty"` - NetworkID string `gcfg:"network-id,omitempty"` -} - -// BlockStorageOpts is used to talk to Cinder service -type BlockStorageOpts struct { - BSVersion string `gcfg:"bs-version"` // overrides autodetection. v1 or v2. Defaults to auto - TrustDevicePath bool `gcfg:"trust-device-path"` // See Issue #33128 - IgnoreVolumeAZ bool `gcfg:"ignore-volume-az"` -} - -// NetworkingOpts is used for networking settings -type NetworkingOpts struct { - IPv6SupportDisabled bool `gcfg:"ipv6-support-disabled"` - PublicNetworkName []string `gcfg:"public-network-name"` - InternalNetworkName []string `gcfg:"internal-network-name"` -} - -// RouterOpts is used for Neutron routes -type RouterOpts struct { - RouterID string `gcfg:"router-id"` // required -} - -// MetadataOpts is used for configuring how to talk to metadata service or config drive -type MetadataOpts struct { - SearchOrder string `gcfg:"search-order"` - RequestTimeout MyDuration `gcfg:"request-timeout"` -} - -type ServerAttributesExt struct { - servers.Server - availabilityzones.ServerAvailabilityZoneExt -} - -// OpenStack is an implementation of cloud provider Interface for OpenStack. -type OpenStack struct { - provider *gophercloud.ProviderClient - region string - lbOpts LoadBalancerOpts - bsOpts BlockStorageOpts - routeOpts RouterOpts - metadataOpts MetadataOpts - networkingOpts NetworkingOpts - // InstanceID of the server where this OpenStack object is instantiated. - localInstanceID string -} - -type AuthOpts struct { - AuthURL string `gcfg:"auth-url" mapstructure:"auth-url" name:"os-authURL" dependsOn:"os-password|os-trustID|os-applicationCredentialSecret"` - UserID string `gcfg:"user-id" mapstructure:"user-id" name:"os-userID" value:"optional" dependsOn:"os-password"` - Username string `name:"os-userName" value:"optional" dependsOn:"os-password"` - Password string `name:"os-password" value:"optional" dependsOn:"os-domainID|os-domainName,os-projectID|os-projectName,os-userID|os-userName"` - TenantID string `gcfg:"tenant-id" mapstructure:"project-id" name:"os-projectID" value:"optional" dependsOn:"os-password"` - TenantName string `gcfg:"tenant-name" mapstructure:"project-name" name:"os-projectName" value:"optional" dependsOn:"os-password"` - TrustID string `gcfg:"trust-id" mapstructure:"trust-id" name:"os-trustID" value:"optional"` - DomainID string `gcfg:"domain-id" mapstructure:"domain-id" name:"os-domainID" value:"optional" dependsOn:"os-password"` - DomainName string `gcfg:"domain-name" mapstructure:"domain-name" name:"os-domainName" value:"optional" dependsOn:"os-password"` - TenantDomainID string `gcfg:"tenant-domain-id" mapstructure:"project-domain-id" name:"os-projectDomainID" value:"optional"` - TenantDomainName string `gcfg:"tenant-domain-name" mapstructure:"project-domain-name" name:"os-projectDomainName" value:"optional"` - UserDomainID string `gcfg:"user-domain-id" mapstructure:"user-domain-id" name:"os-userDomainID" value:"optional"` - UserDomainName string `gcfg:"user-domain-name" mapstructure:"user-domain-name" name:"os-userDomainName" value:"optional"` - Region string `name:"os-region"` - CAFile string `gcfg:"ca-file" mapstructure:"ca-file" name:"os-certAuthorityPath" value:"optional"` - - // Manila only options - TLSInsecure string `name:"os-TLSInsecure" value:"optional" matches:"^true|false$"` - // backward compatibility with the manila-csi-plugin - CAFileContents string `name:"os-certAuthority" value:"optional"` - TrusteeID string `name:"os-trusteeID" value:"optional" dependsOn:"os-trustID"` - TrusteePassword string `name:"os-trusteePassword" value:"optional" dependsOn:"os-trustID"` - - UseClouds bool `gcfg:"use-clouds" mapstructure:"use-clouds" name:"os-useClouds" value:"optional"` - CloudsFile string `gcfg:"clouds-file,omitempty" mapstructure:"clouds-file,omitempty" name:"os-cloudsFile" value:"optional"` - Cloud string `gcfg:"cloud,omitempty" mapstructure:"cloud,omitempty" name:"os-cloud" value:"optional"` - - ApplicationCredentialID string `gcfg:"application-credential-id" mapstructure:"application-credential-id" name:"os-applicationCredentialID" value:"optional"` - ApplicationCredentialName string `gcfg:"application-credential-name" mapstructure:"application-credential-name" name:"os-applicationCredentialName" value:"optional"` - ApplicationCredentialSecret string `gcfg:"application-credential-secret" mapstructure:"application-credential-secret" name:"os-applicationCredentialSecret" value:"optional"` -} - -// Config is used to read and store information from the cloud configuration file -type Config struct { - Global AuthOpts - LoadBalancer LoadBalancerOpts - LoadBalancerClass map[string]*LBClass - BlockStorage BlockStorageOpts - Route RouterOpts - Metadata MetadataOpts - Networking NetworkingOpts -} - -func LogCfg(cfg Config) { - klog.V(5).Infof("AuthURL: %s", cfg.Global.AuthURL) - klog.V(5).Infof("UserID: %s", cfg.Global.UserID) - klog.V(5).Infof("Username: %s", cfg.Global.Username) - klog.V(5).Infof("TenantID: %s", cfg.Global.TenantID) - klog.V(5).Infof("TenantName: %s", cfg.Global.TenantName) - klog.V(5).Infof("TrustID: %s", cfg.Global.TrustID) - klog.V(5).Infof("DomainID: %s", cfg.Global.DomainID) - klog.V(5).Infof("DomainName: %s", cfg.Global.DomainName) - klog.V(5).Infof("TenantDomainID: %s", cfg.Global.TenantDomainID) - klog.V(5).Infof("TenantDomainName: %s", cfg.Global.TenantDomainName) - klog.V(5).Infof("UserDomainID: %s", cfg.Global.UserDomainID) - klog.V(5).Infof("UserDomainName: %s", cfg.Global.UserDomainName) - klog.V(5).Infof("Region: %s", cfg.Global.Region) - klog.V(5).Infof("CAFile: %s", cfg.Global.CAFile) - klog.V(5).Infof("UseClouds: %t", cfg.Global.UseClouds) - klog.V(5).Infof("CloudsFile: %s", cfg.Global.CloudsFile) - klog.V(5).Infof("Cloud: %s", cfg.Global.Cloud) - klog.V(5).Infof("ApplicationCredentialID: %s", cfg.Global.ApplicationCredentialID) - klog.V(5).Infof("ApplicationCredentialName: %s", cfg.Global.ApplicationCredentialName) -} - -type Logger struct{} - -func (l Logger) Printf(format string, args ...interface{}) { - debugger := klog.V(6).Enabled() - - // extra check in case, when verbosity has been changed dynamically - if debugger { - var skip int - var found bool - var gc = "/github.com/gophercloud/gophercloud" - - // detect the depth of the actual function, which calls gophercloud code - // 10 is the common depth from the logger to "github.com/gophercloud/gophercloud" - for i := 10; i <= 20; i++ { - if _, file, _, ok := runtime.Caller(i); ok && !found && strings.Contains(file, gc) { - found = true - continue - } else if ok && found && !strings.Contains(file, gc) { - skip = i - break - } else if !ok { - break - } - } - - for _, v := range strings.Split(fmt.Sprintf(format, args...), "\n") { - klog.InfoDepth(skip, v) - } - } -} - -func init() { - RegisterMetrics() - - cloudprovider.RegisterCloudProvider(ProviderName, func(config io.Reader) (cloudprovider.Interface, error) { - cfg, err := ReadConfig(config) - if err != nil { - klog.Warningf("failed to read config: %v", err) - return nil, err - } - cloud, err := NewOpenStack(cfg) - if err != nil { - klog.Warningf("New openstack client created failed with config: %v", err) - } - return cloud, err - }) -} - -func (cfg AuthOpts) ToAuthOptions() gophercloud.AuthOptions { - opts := clientconfig.ClientOpts{ - // this is needed to disable the clientconfig.AuthOptions func env detection - EnvPrefix: "_", - Cloud: cfg.Cloud, - AuthInfo: &clientconfig.AuthInfo{ - AuthURL: cfg.AuthURL, - UserID: cfg.UserID, - Username: cfg.Username, - Password: cfg.Password, - ProjectID: cfg.TenantID, - ProjectName: cfg.TenantName, - DomainID: cfg.DomainID, - DomainName: cfg.DomainName, - ProjectDomainID: cfg.TenantDomainID, - ProjectDomainName: cfg.TenantDomainName, - UserDomainID: cfg.UserDomainID, - UserDomainName: cfg.UserDomainName, - ApplicationCredentialID: cfg.ApplicationCredentialID, - ApplicationCredentialName: cfg.ApplicationCredentialName, - ApplicationCredentialSecret: cfg.ApplicationCredentialSecret, - }, - } - - ao, err := clientconfig.AuthOptions(&opts) - if err != nil { - klog.V(1).Infof("Error parsing auth: %s", err) - return gophercloud.AuthOptions{} - } - - // Persistent service, so we need to be able to renew tokens. - ao.AllowReauth = true - - return *ao -} - -func (cfg AuthOpts) ToAuth3Options() tokens3.AuthOptions { - ao := cfg.ToAuthOptions() - - var scope tokens3.Scope - if ao.Scope != nil { - scope.ProjectID = ao.Scope.ProjectID - scope.ProjectName = ao.Scope.ProjectName - scope.DomainID = ao.Scope.DomainID - scope.DomainName = ao.Scope.DomainName - } - - return tokens3.AuthOptions{ - IdentityEndpoint: ao.IdentityEndpoint, - UserID: ao.UserID, - Username: ao.Username, - Password: ao.Password, - DomainID: ao.DomainID, - DomainName: ao.DomainName, - ApplicationCredentialID: ao.ApplicationCredentialID, - ApplicationCredentialName: ao.ApplicationCredentialName, - ApplicationCredentialSecret: ao.ApplicationCredentialSecret, - Scope: scope, - AllowReauth: ao.AllowReauth, - } -} - -// ReadConfig reads values from the cloud.conf -func ReadConfig(config io.Reader) (Config, error) { - if config == nil { - return Config{}, fmt.Errorf("no OpenStack cloud provider config file given") - } - var cfg Config - - // Set default values explicitly - cfg.LoadBalancer.UseOctavia = true - cfg.LoadBalancer.InternalLB = false - cfg.LoadBalancer.LBProvider = "amphora" - cfg.LoadBalancer.LBMethod = "ROUND_ROBIN" - cfg.LoadBalancer.CreateMonitor = false - cfg.LoadBalancer.ManageSecurityGroups = false - cfg.LoadBalancer.MonitorDelay = MyDuration{5 * time.Second} - cfg.LoadBalancer.MonitorTimeout = MyDuration{3 * time.Second} - cfg.LoadBalancer.MonitorMaxRetries = 1 - cfg.LoadBalancer.CascadeDelete = true - - err := gcfg.FatalOnly(gcfg.ReadInto(&cfg, config)) - if err != nil { - return Config{}, err - } - - klog.V(5).Infof("Config, loaded from the config file:") - LogCfg(cfg) - - if cfg.Global.UseClouds { - if cfg.Global.CloudsFile != "" { - os.Setenv("OS_CLIENT_CONFIG_FILE", cfg.Global.CloudsFile) - } - err = ReadClouds(&cfg) - if err != nil { - return Config{}, err - } - klog.V(5).Infof("Config, loaded from the %s:", cfg.Global.CloudsFile) - LogCfg(cfg) - } - // Set the default values for search order if not set - if cfg.Metadata.SearchOrder == "" { - cfg.Metadata.SearchOrder = fmt.Sprintf("%s,%s", metadata.ConfigDriveID, metadata.MetadataID) - } - - return cfg, err -} - -// replaceEmpty is a helper function to replace empty fields with another field -func replaceEmpty(a string, b string) string { - if a == "" { - return b - } - return a -} - -// ReadClouds reads Reads clouds.yaml to generate a Config -// Allows the cloud-config to have priority -func ReadClouds(cfg *Config) error { - co := new(clientconfig.ClientOpts) - if cfg.Global.Cloud != "" { - co.Cloud = cfg.Global.Cloud - } - cloud, err := clientconfig.GetCloudFromYAML(co) - if err != nil && err.Error() != "unable to load clouds.yaml: no clouds.yaml file found" { - return err - } - - cfg.Global.AuthURL = replaceEmpty(cfg.Global.AuthURL, cloud.AuthInfo.AuthURL) - cfg.Global.UserID = replaceEmpty(cfg.Global.UserID, cloud.AuthInfo.UserID) - cfg.Global.Username = replaceEmpty(cfg.Global.Username, cloud.AuthInfo.Username) - cfg.Global.Password = replaceEmpty(cfg.Global.Password, cloud.AuthInfo.Password) - cfg.Global.TenantID = replaceEmpty(cfg.Global.TenantID, cloud.AuthInfo.ProjectID) - cfg.Global.TenantName = replaceEmpty(cfg.Global.TenantName, cloud.AuthInfo.ProjectName) - cfg.Global.DomainID = replaceEmpty(cfg.Global.DomainID, cloud.AuthInfo.DomainID) - cfg.Global.DomainName = replaceEmpty(cfg.Global.DomainName, cloud.AuthInfo.DomainName) - cfg.Global.TenantDomainID = replaceEmpty(cfg.Global.TenantDomainID, cloud.AuthInfo.ProjectDomainID) - cfg.Global.TenantDomainName = replaceEmpty(cfg.Global.TenantDomainName, cloud.AuthInfo.ProjectDomainName) - cfg.Global.UserDomainID = replaceEmpty(cfg.Global.UserDomainID, cloud.AuthInfo.UserDomainID) - cfg.Global.UserDomainName = replaceEmpty(cfg.Global.UserDomainName, cloud.AuthInfo.UserDomainName) - cfg.Global.Region = replaceEmpty(cfg.Global.Region, cloud.RegionName) - cfg.Global.CAFile = replaceEmpty(cfg.Global.CAFile, cloud.CACertFile) - cfg.Global.ApplicationCredentialID = replaceEmpty(cfg.Global.ApplicationCredentialID, cloud.AuthInfo.ApplicationCredentialID) - cfg.Global.ApplicationCredentialName = replaceEmpty(cfg.Global.ApplicationCredentialName, cloud.AuthInfo.ApplicationCredentialName) - cfg.Global.ApplicationCredentialSecret = replaceEmpty(cfg.Global.ApplicationCredentialSecret, cloud.AuthInfo.ApplicationCredentialSecret) - - return nil -} - -// caller is a tiny helper for conditional unwind logic -type caller bool - -func newCaller() caller { return caller(true) } -func (c *caller) disarm() { *c = false } - -func (c *caller) call(f func()) { - if *c { - f() - } -} - -func readInstanceID(searchOrder string) (string, error) { - // First, try to get data from metadata service because local - // data might be changed by accident - md, err := metadata.Get(searchOrder) - if err == nil { - return md.UUID, nil - } - - // Try to find instance ID on the local filesystem (created by cloud-init) - const instanceIDFile = "/var/lib/cloud/data/instance-id" - idBytes, err := ioutil.ReadFile(instanceIDFile) - if err == nil { - instanceID := string(idBytes) - instanceID = strings.TrimSpace(instanceID) - klog.V(3).Infof("Got instance id from %s: %s", instanceIDFile, instanceID) - if instanceID != "" && instanceID != "iid-datasource-none" { - return instanceID, nil - } - } - - return "", err -} - -// check opts for OpenStack -func checkOpenStackOpts(openstackOpts *OpenStack) error { - return checkMetadataSearchOrder(openstackOpts.metadataOpts.SearchOrder) -} - -// NewOpenStackClient creates a new instance of the openstack client -func NewOpenStackClient(cfg *AuthOpts, userAgent string, extraUserAgent ...string) (*gophercloud.ProviderClient, error) { - provider, err := openstack.NewClient(cfg.AuthURL) - if err != nil { - return nil, err - } - - ua := gophercloud.UserAgent{} - ua.Prepend(fmt.Sprintf("%s/%s", userAgent, version.Version)) - for _, data := range extraUserAgent { - ua.Prepend(data) - } - provider.UserAgent = ua - klog.V(4).Infof("Using user-agent %s", ua.Join()) - - var caPool *x509.CertPool - if cfg.CAFile != "" { - // read and parse CA certificate from file - caPool, err = certutil.NewPool(cfg.CAFile) - if err != nil { - return nil, fmt.Errorf("failed to read and parse %s certificate: %s", cfg.CAFile, err) - } - } else if cfg.CAFileContents != "" { - // parse CA certificate from the contents - caPool = x509.NewCertPool() - if ok := caPool.AppendCertsFromPEM([]byte(cfg.CAFileContents)); !ok { - return nil, fmt.Errorf("failed to parse os-certAuthority certificate") - } - } - - config := &tls.Config{} - config.InsecureSkipVerify = cfg.TLSInsecure == "true" - - if caPool != nil { - config.RootCAs = caPool - } - - provider.HTTPClient.Transport = netutil.SetOldTransportDefaults(&http.Transport{TLSClientConfig: config}) - - if klog.V(6).Enabled() { - provider.HTTPClient.Transport = &client.RoundTripper{ - Rt: provider.HTTPClient.Transport, - Logger: &Logger{}, - } - } - - if cfg.TrustID != "" { - opts := cfg.ToAuth3Options() - - // support for the legacy manila auth - // if TrusteeID and TrusteePassword were defined, then use them - opts.UserID = replaceEmpty(cfg.TrusteeID, opts.UserID) - opts.Password = replaceEmpty(cfg.TrusteePassword, opts.Password) - - authOptsExt := trusts.AuthOptsExt{ - TrustID: cfg.TrustID, - AuthOptionsBuilder: &opts, - } - err = openstack.AuthenticateV3(provider, authOptsExt, gophercloud.EndpointOpts{}) - - return provider, err - } - - opts := cfg.ToAuthOptions() - err = openstack.Authenticate(provider, opts) - - return provider, err -} - -// NewOpenStack creates a new new instance of the openstack struct from a config struct -func NewOpenStack(cfg Config) (*OpenStack, error) { - provider, err := NewOpenStackClient(&cfg.Global, "openstack-cloud-controller-manager", userAgentData...) - if err != nil { - return nil, err - } - - if cfg.Metadata.RequestTimeout == (MyDuration{}) { - cfg.Metadata.RequestTimeout.Duration = time.Duration(defaultTimeOut) - } - provider.HTTPClient.Timeout = cfg.Metadata.RequestTimeout.Duration - - os := OpenStack{ - provider: provider, - region: cfg.Global.Region, - lbOpts: cfg.LoadBalancer, - bsOpts: cfg.BlockStorage, - routeOpts: cfg.Route, - metadataOpts: cfg.Metadata, - networkingOpts: cfg.Networking, - } - - // ini file doesn't support maps so we are reusing top level sub sections - // and copy the resulting map to corresponding loadbalancer section - os.lbOpts.LBClasses = cfg.LoadBalancerClass - - err = checkOpenStackOpts(&os) - if err != nil { - return nil, err - } - - return &os, nil -} - -// Initialize passes a Kubernetes clientBuilder interface to the cloud provider -func (os *OpenStack) Initialize(clientBuilder cloudprovider.ControllerClientBuilder, stop <-chan struct{}) { -} - -// mapNodeNameToServerName maps a k8s NodeName to an OpenStack Server Name -// This is a simple string cast. -func mapNodeNameToServerName(nodeName types.NodeName) string { - return string(nodeName) -} - -// GetNodeNameByID maps instanceid to types.NodeName -func (os *OpenStack) GetNodeNameByID(instanceID string) (types.NodeName, error) { - client, err := os.NewComputeV2() - var nodeName types.NodeName - if err != nil { - return nodeName, err - } - - server, err := servers.Get(client, instanceID).Extract() - if err != nil { - return nodeName, err - } - nodeName = mapServerToNodeName(server) - return nodeName, nil -} - -// mapServerToNodeName maps an OpenStack Server to a k8s NodeName -func mapServerToNodeName(server *servers.Server) types.NodeName { - // Node names are always lowercase, and (at least) - // routecontroller does case-sensitive string comparisons - // assuming this - return types.NodeName(strings.ToLower(server.Name)) -} - -func foreachServer(client *gophercloud.ServiceClient, opts servers.ListOptsBuilder, handler func(*servers.Server) (bool, error)) error { - pager := servers.List(client, opts) - - err := pager.EachPage(func(page pagination.Page) (bool, error) { - s, err := servers.ExtractServers(page) - if err != nil { - return false, err - } - for _, server := range s { - ok, err := handler(&server) - if !ok || err != nil { - return false, err - } - } - return true, nil - }) - return err -} - -func getServerByName(client *gophercloud.ServiceClient, name types.NodeName) (*ServerAttributesExt, error) { - opts := servers.ListOpts{ - Name: fmt.Sprintf("^%s$", regexp.QuoteMeta(mapNodeNameToServerName(name))), - } - - pager := servers.List(client, opts) - - var s []ServerAttributesExt - serverList := make([]ServerAttributesExt, 0, 1) - - err := pager.EachPage(func(page pagination.Page) (bool, error) { - if err := servers.ExtractServersInto(page, &s); err != nil { - return false, err - } - serverList = append(serverList, s...) - if len(serverList) > 1 { - return false, ErrMultipleResults - } - return true, nil - }) - if err != nil { - return nil, err - } - - if len(serverList) == 0 { - return nil, ErrNotFound - } - - return &serverList[0], nil -} - -// IP addresses order: -// * interfaces private IPs -// * access IPs -// * metadata hostname -// * server object Addresses (floating type) -func nodeAddresses(srv *servers.Server, interfaces []attachinterfaces.Interface, networkingOpts NetworkingOpts) ([]v1.NodeAddress, error) { - addrs := []v1.NodeAddress{} - - // parse private IP addresses first in an ordered manner - for _, iface := range interfaces { - for _, fixedIP := range iface.FixedIPs { - if iface.PortState == "ACTIVE" { - isIPv6 := net.ParseIP(fixedIP.IPAddress).To4() == nil - if !(isIPv6 && networkingOpts.IPv6SupportDisabled) { - AddToNodeAddresses(&addrs, - v1.NodeAddress{ - Type: v1.NodeInternalIP, - Address: fixedIP.IPAddress, - }, - ) - } - } - } - } - - // process public IP addresses - if srv.AccessIPv4 != "" { - AddToNodeAddresses(&addrs, - v1.NodeAddress{ - Type: v1.NodeExternalIP, - Address: srv.AccessIPv4, - }, - ) - } - - if srv.AccessIPv6 != "" && !networkingOpts.IPv6SupportDisabled { - AddToNodeAddresses(&addrs, - v1.NodeAddress{ - Type: v1.NodeExternalIP, - Address: srv.AccessIPv6, - }, - ) - } - - if srv.Metadata[TypeHostName] != "" { - AddToNodeAddresses(&addrs, - v1.NodeAddress{ - Type: v1.NodeHostName, - Address: srv.Metadata[TypeHostName], - }, - ) - } - - // process the rest - type Address struct { - IPType string `mapstructure:"OS-EXT-IPS:type"` - Addr string - } - - var addresses map[string][]Address - err := mapstructure.Decode(srv.Addresses, &addresses) - if err != nil { - return nil, err - } - - var networks []string - for k := range addresses { - networks = append(networks, k) - } - sort.Strings(networks) - - for _, network := range networks { - for _, props := range addresses[network] { - var addressType v1.NodeAddressType - if props.IPType == "floating" { - addressType = v1.NodeExternalIP - } else if util.Contains(networkingOpts.PublicNetworkName, network) { - addressType = v1.NodeExternalIP - // removing already added address to avoid listing it as both ExternalIP and InternalIP - // may happen due to listing "private" network as "public" in CCM's config - RemoveFromNodeAddresses(&addrs, - v1.NodeAddress{ - Address: props.Addr, - }, - ) - } else { - if len(networkingOpts.InternalNetworkName) == 0 || util.Contains(networkingOpts.InternalNetworkName, network) { - addressType = v1.NodeInternalIP - } else { - klog.V(5).Infof("Node '%s' address '%s' ignored due to 'internal-network-name' option", srv.Name, props.Addr) - RemoveFromNodeAddresses(&addrs, - v1.NodeAddress{ - Address: props.Addr, - }, - ) - continue - } - } - - isIPv6 := net.ParseIP(props.Addr).To4() == nil - if !(isIPv6 && networkingOpts.IPv6SupportDisabled) { - AddToNodeAddresses(&addrs, - v1.NodeAddress{ - Type: addressType, - Address: props.Addr, - }, - ) - } - } - } - - return addrs, nil -} - -func getAddressesByName(client *gophercloud.ServiceClient, name types.NodeName, networkingOpts NetworkingOpts) ([]v1.NodeAddress, error) { - srv, err := getServerByName(client, name) - if err != nil { - return nil, err - } - - interfaces, err := getAttachedInterfacesByID(client, srv.ID) - if err != nil { - return nil, err - } - - return nodeAddresses(&srv.Server, interfaces, networkingOpts) -} - -func getAddressByName(client *gophercloud.ServiceClient, name types.NodeName, needIPv6 bool, networkingOpts NetworkingOpts) (string, error) { - if needIPv6 && networkingOpts.IPv6SupportDisabled { - return "", ErrIPv6SupportDisabled - } - - addrs, err := getAddressesByName(client, name, networkingOpts) - if err != nil { - return "", err - } else if len(addrs) == 0 { - return "", ErrNoAddressFound - } - - for _, addr := range addrs { - isIPv6 := net.ParseIP(addr.Address).To4() == nil - if (addr.Type == v1.NodeInternalIP) && (isIPv6 == needIPv6) { - return addr.Address, nil - } - } - - for _, addr := range addrs { - isIPv6 := net.ParseIP(addr.Address).To4() == nil - if (addr.Type == v1.NodeExternalIP) && (isIPv6 == needIPv6) { - return addr.Address, nil - } - } - // It should never return an address from a different IP Address family than the one needed - return "", ErrNoAddressFound -} - -// getAttachedInterfacesByID returns the node interfaces of the specified instance. -func getAttachedInterfacesByID(client *gophercloud.ServiceClient, serviceID string) ([]attachinterfaces.Interface, error) { - var interfaces []attachinterfaces.Interface - - pager := attachinterfaces.List(client, serviceID) - err := pager.EachPage(func(page pagination.Page) (bool, error) { - s, err := attachinterfaces.ExtractInterfaces(page) - if err != nil { - return false, err - } - interfaces = append(interfaces, s...) - return true, nil - }) - if err != nil { - return interfaces, err - } - - return interfaces, nil -} - -// Clusters is a no-op -func (os *OpenStack) Clusters() (cloudprovider.Clusters, bool) { - return nil, false -} - -// ProviderName returns the cloud provider ID. -func (os *OpenStack) ProviderName() string { - return ProviderName -} - -// HasClusterID returns true if the cluster has a clusterID -func (os *OpenStack) HasClusterID() bool { - return true -} - -// LoadBalancer initializes a LbaasV2 object -func (os *OpenStack) LoadBalancer() (cloudprovider.LoadBalancer, bool) { - klog.V(4).Info("openstack.LoadBalancer() called") - - network, err := os.NewNetworkV2() - if err != nil { - klog.Errorf("Failed to create an OpenStack Network client: %v", err) - return nil, false - } - - compute, err := os.NewComputeV2() - if err != nil { - klog.Errorf("Failed to create an OpenStack Compute client: %v", err) - return nil, false - } - - lb, err := os.NewLoadBalancerV2() - if err != nil { - klog.Errorf("Failed to create an OpenStack LoadBalancer client: %v", err) - return nil, false - } - - // LBaaS v1 is deprecated in the OpenStack Liberty release. - // Currently kubernetes OpenStack cloud provider just support LBaaS v2. - lbVersion := os.lbOpts.LBVersion - if lbVersion != "" && lbVersion != "v2" { - klog.Warningf("Config error: currently only support LBaaS v2, unrecognised lb-version \"%v\"", lbVersion) - return nil, false - } - - klog.V(1).Info("Claiming to support LoadBalancer") - - return &LbaasV2{LoadBalancer{network, compute, lb, os.lbOpts}}, true -} - -// Zones indicates that we support zones -func (os *OpenStack) Zones() (cloudprovider.Zones, bool) { - klog.V(1).Info("Claiming to support Zones") - return os, true -} - -// GetZone returns the current zone -func (os *OpenStack) GetZone(ctx context.Context) (cloudprovider.Zone, error) { - md, err := metadata.Get(os.metadataOpts.SearchOrder) - if err != nil { - return cloudprovider.Zone{}, err - } - - zone := cloudprovider.Zone{ - FailureDomain: md.AvailabilityZone, - Region: os.region, - } - klog.V(4).Infof("Current zone is %v", zone) - return zone, nil -} - -// GetZoneByProviderID implements Zones.GetZoneByProviderID -// This is particularly useful in external cloud providers where the kubelet -// does not initialize node data. -func (os *OpenStack) GetZoneByProviderID(ctx context.Context, providerID string) (cloudprovider.Zone, error) { - instanceID, err := instanceIDFromProviderID(providerID) - if err != nil { - return cloudprovider.Zone{}, err - } - - compute, err := os.NewComputeV2() - if err != nil { - return cloudprovider.Zone{}, err - } - - var serverWithAttributesExt ServerAttributesExt - if err := servers.Get(compute, instanceID).ExtractInto(&serverWithAttributesExt); err != nil { - return cloudprovider.Zone{}, err - } - - zone := cloudprovider.Zone{ - FailureDomain: serverWithAttributesExt.AvailabilityZone, - Region: os.region, - } - klog.V(4).Infof("The instance %s in zone %v", serverWithAttributesExt.Name, zone) - return zone, nil -} - -// GetZoneByNodeName implements Zones.GetZoneByNodeName -// This is particularly useful in external cloud providers where the kubelet -// does not initialize node data. -func (os *OpenStack) GetZoneByNodeName(ctx context.Context, nodeName types.NodeName) (cloudprovider.Zone, error) { - compute, err := os.NewComputeV2() - if err != nil { - return cloudprovider.Zone{}, err - } - - srv, err := getServerByName(compute, nodeName) - if err != nil { - if err == ErrNotFound { - return cloudprovider.Zone{}, cloudprovider.InstanceNotFound - } - return cloudprovider.Zone{}, err - } - - zone := cloudprovider.Zone{ - FailureDomain: srv.AvailabilityZone, - Region: os.region, - } - klog.V(4).Infof("The instance %s in zone %v", srv.Name, zone) - return zone, nil -} - -// Routes initializes routes support -func (os *OpenStack) Routes() (cloudprovider.Routes, bool) { - klog.V(4).Info("openstack.Routes() called") - - network, err := os.NewNetworkV2() - if err != nil { - klog.Errorf("Failed to create an OpenStack Network client: %v", err) - return nil, false - } - - netExts, err := networkExtensions(network) - if err != nil { - klog.Warningf("Failed to list neutron extensions: %v", err) - return nil, false - } - - if !netExts["extraroute"] { - klog.V(3).Info("Neutron extraroute extension not found, required for Routes support") - return nil, false - } - - compute, err := os.NewComputeV2() - if err != nil { - klog.Errorf("Failed to create an OpenStack Compute client: %v", err) - return nil, false - } - - r, err := NewRoutes(compute, network, os.routeOpts, os.networkingOpts) - if err != nil { - klog.Warningf("Error initialising Routes support: %v", err) - return nil, false - } - - klog.V(1).Info("Claiming to support Routes") - return r, true -} - -func checkMetadataSearchOrder(order string) error { - if order == "" { - return errors.New("invalid value in section [Metadata] with key `search-order`. Value cannot be empty") - } - - elements := strings.Split(order, ",") - if len(elements) > 2 { - return errors.New("invalid value in section [Metadata] with key `search-order`. Value cannot contain more than 2 elements") - } - - for _, id := range elements { - id = strings.TrimSpace(id) - switch id { - case metadata.ConfigDriveID: - case metadata.MetadataID: - default: - return fmt.Errorf("invalid element %q found in section [Metadata] with key `search-order`."+ - "Supported elements include %q and %q", id, metadata.ConfigDriveID, metadata.MetadataID) - } - } - - return nil -} diff --git a/pkg/cloudprovider/providers/openstack/openstack_client.go b/pkg/cloudprovider/providers/openstack/openstack_client.go deleted file mode 100644 index 0a858201aa..0000000000 --- a/pkg/cloudprovider/providers/openstack/openstack_client.go +++ /dev/null @@ -1,76 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package openstack - -import ( - "fmt" - - "github.com/gophercloud/gophercloud" - "github.com/gophercloud/gophercloud/openstack" -) - -// NewNetworkV2 creates a ServiceClient that may be used with the neutron v2 API -func (os *OpenStack) NewNetworkV2() (*gophercloud.ServiceClient, error) { - network, err := openstack.NewNetworkV2(os.provider, gophercloud.EndpointOpts{ - Region: os.region, - }) - if err != nil { - return nil, fmt.Errorf("failed to find network v2 endpoint for region %s: %v", os.region, err) - } - return network, nil -} - -// NewComputeV2 creates a ServiceClient that may be used with the nova v2 API -func (os *OpenStack) NewComputeV2() (*gophercloud.ServiceClient, error) { - compute, err := openstack.NewComputeV2(os.provider, gophercloud.EndpointOpts{ - Region: os.region, - }) - if err != nil { - return nil, fmt.Errorf("failed to find compute v2 endpoint for region %s: %v", os.region, err) - } - return compute, nil -} - -// NewBlockStorageV3 creates a ServiceClient that may be used with the Cinder v3 API -func (os *OpenStack) NewBlockStorageV3() (*gophercloud.ServiceClient, error) { - storage, err := openstack.NewBlockStorageV3(os.provider, gophercloud.EndpointOpts{ - Region: os.region, - }) - if err != nil { - return nil, fmt.Errorf("unable to initialize cinder v3 client for region %s: %v", os.region, err) - } - return storage, nil -} - -// NewLoadBalancerV2 creates a ServiceClient that may be used with the Neutron LBaaS v2 API -func (os *OpenStack) NewLoadBalancerV2() (*gophercloud.ServiceClient, error) { - var lb *gophercloud.ServiceClient - var err error - if os.lbOpts.UseOctavia { - lb, err = openstack.NewLoadBalancerV2(os.provider, gophercloud.EndpointOpts{ - Region: os.region, - }) - } else { - lb, err = openstack.NewNetworkV2(os.provider, gophercloud.EndpointOpts{ - Region: os.region, - }) - } - if err != nil { - return nil, fmt.Errorf("failed to find load-balancer v2 endpoint for region %s: %v", os.region, err) - } - return lb, nil -} diff --git a/pkg/cloudprovider/providers/openstack/openstack_instances.go b/pkg/cloudprovider/providers/openstack/openstack_instances.go deleted file mode 100644 index 0d78690a89..0000000000 --- a/pkg/cloudprovider/providers/openstack/openstack_instances.go +++ /dev/null @@ -1,285 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package openstack - -import ( - "context" - "fmt" - "regexp" - "strings" - - "github.com/gophercloud/gophercloud" - "github.com/gophercloud/gophercloud/openstack/compute/v2/servers" - "k8s.io/klog/v2" - - "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" - cloudprovider "k8s.io/cloud-provider" - "k8s.io/cloud-provider-openstack/pkg/util/errors" - "k8s.io/cloud-provider-openstack/pkg/util/metadata" -) - -// Instances encapsulates an implementation of Instances for OpenStack. -type Instances struct { - compute *gophercloud.ServiceClient - opts MetadataOpts - networkingOpts NetworkingOpts -} - -const ( - instanceShutoff = "SHUTOFF" -) - -// Instances returns an implementation of Instances for OpenStack. -func (os *OpenStack) Instances() (cloudprovider.Instances, bool) { - klog.V(4).Info("openstack.Instances() called") - - compute, err := os.NewComputeV2() - if err != nil { - klog.Errorf("unable to access compute v2 API : %v", err) - return nil, false - } - - klog.V(4).Info("Claiming to support Instances") - - return &Instances{ - compute: compute, - opts: os.metadataOpts, - networkingOpts: os.networkingOpts, - }, true -} - -// CurrentNodeName implements Instances.CurrentNodeName -// Note this is *not* necessarily the same as hostname. -func (i *Instances) CurrentNodeName(ctx context.Context, hostname string) (types.NodeName, error) { - md, err := metadata.Get(i.opts.SearchOrder) - if err != nil { - return "", err - } - return types.NodeName(md.Name), nil -} - -// AddSSHKeyToAllInstances is not implemented for OpenStack -func (i *Instances) AddSSHKeyToAllInstances(ctx context.Context, user string, keyData []byte) error { - return cloudprovider.NotImplemented -} - -// NodeAddresses implements Instances.NodeAddresses -func (i *Instances) NodeAddresses(ctx context.Context, name types.NodeName) ([]v1.NodeAddress, error) { - klog.V(4).Infof("NodeAddresses(%v) called", name) - - addrs, err := getAddressesByName(i.compute, name, i.networkingOpts) - if err != nil { - return nil, err - } - - klog.V(4).Infof("NodeAddresses(%v) => %v", name, addrs) - return addrs, nil -} - -// NodeAddressesByProviderID returns the node addresses of an instances with the specified unique providerID -// This method will not be called from the node that is requesting this ID. i.e. metadata service -// and other local methods cannot be used here -func (i *Instances) NodeAddressesByProviderID(ctx context.Context, providerID string) ([]v1.NodeAddress, error) { - instanceID, err := instanceIDFromProviderID(providerID) - - if err != nil { - return []v1.NodeAddress{}, err - } - - server, err := servers.Get(i.compute, instanceID).Extract() - - if err != nil { - return []v1.NodeAddress{}, err - } - - interfaces, err := getAttachedInterfacesByID(i.compute, server.ID) - if err != nil { - return []v1.NodeAddress{}, err - } - - addresses, err := nodeAddresses(server, interfaces, i.networkingOpts) - if err != nil { - return []v1.NodeAddress{}, err - } - - return addresses, nil -} - -// InstanceExistsByProviderID returns true if the instance with the given provider id still exist. -// If false is returned with no error, the instance will be immediately deleted by the cloud controller manager. -func (i *Instances) InstanceExistsByProviderID(ctx context.Context, providerID string) (bool, error) { - instanceID, err := instanceIDFromProviderID(providerID) - if err != nil { - return false, err - } - - _, err = servers.Get(i.compute, instanceID).Extract() - if err != nil { - if errors.IsNotFound(err) { - return false, nil - } - return false, err - } - - return true, nil -} - -// InstanceShutdownByProviderID returns true if the instances is in safe state to detach volumes -func (i *Instances) InstanceShutdownByProviderID(ctx context.Context, providerID string) (bool, error) { - instanceID, err := instanceIDFromProviderID(providerID) - if err != nil { - return false, err - } - - server, err := servers.Get(i.compute, instanceID).Extract() - if err != nil { - return false, err - } - - // SHUTOFF is the only state where we can detach volumes immediately - if server.Status == instanceShutoff { - return true, nil - } - return false, nil -} - -// InstanceID returns the kubelet's cloud provider ID. -func (os *OpenStack) InstanceID() (string, error) { - if len(os.localInstanceID) == 0 { - id, err := readInstanceID(os.metadataOpts.SearchOrder) - if err != nil { - return "", err - } - os.localInstanceID = id - } - return os.localInstanceID, nil -} - -// InstanceID returns the cloud provider ID of the specified instance. -func (i *Instances) InstanceID(ctx context.Context, name types.NodeName) (string, error) { - srv, err := getServerByName(i.compute, name) - if err != nil { - if err == ErrNotFound { - return "", cloudprovider.InstanceNotFound - } - return "", err - } - // In the future it is possible to also return an endpoint as: - // / - return "/" + srv.ID, nil -} - -// InstanceTypeByProviderID returns the cloudprovider instance type of the node with the specified unique providerID -// This method will not be called from the node that is requesting this ID. i.e. metadata service -// and other local methods cannot be used here -func (i *Instances) InstanceTypeByProviderID(ctx context.Context, providerID string) (string, error) { - instanceID, err := instanceIDFromProviderID(providerID) - - if err != nil { - return "", err - } - - server, err := servers.Get(i.compute, instanceID).Extract() - - if err != nil { - return "", err - } - - return srvInstanceType(server) -} - -// InstanceType returns the type of the specified instance. -func (i *Instances) InstanceType(ctx context.Context, name types.NodeName) (string, error) { - srv, err := getServerByName(i.compute, name) - - if err != nil { - return "", err - } - - return srvInstanceType(&srv.Server) -} - -func srvInstanceType(srv *servers.Server) (string, error) { - keys := []string{"name", "id", "original_name"} - for _, key := range keys { - val, found := srv.Flavor[key] - if found { - flavor, ok := val.(string) - if ok { - return flavor, nil - } - } - } - return "", fmt.Errorf("flavor name/id not found") -} - -// If Instances.InstanceID or cloudprovider.GetInstanceProviderID is changed, the regexp should be changed too. -var providerIDRegexp = regexp.MustCompile(`^` + ProviderName + `:///([^/]+)$`) - -// instanceIDFromProviderID splits a provider's id and return instanceID. -// A providerID is build out of '${ProviderName}:///${instance-id}'which contains ':///'. -// See cloudprovider.GetInstanceProviderID and Instances.InstanceID. -func instanceIDFromProviderID(providerID string) (instanceID string, err error) { - - // https://github.com/kubernetes/kubernetes/issues/85731 - if providerID != "" && !strings.Contains(providerID, "://") { - providerID = ProviderName + "://" + providerID - } - - matches := providerIDRegexp.FindStringSubmatch(providerID) - if len(matches) != 2 { - return "", fmt.Errorf("ProviderID \"%s\" didn't match expected format \"openstack:///InstanceID\"", providerID) - } - return matches[1], nil -} - -// AddToNodeAddresses appends the NodeAddresses to the passed-by-pointer slice, -// only if they do not already exist -func AddToNodeAddresses(addresses *[]v1.NodeAddress, addAddresses ...v1.NodeAddress) { - for _, add := range addAddresses { - exists := false - for _, existing := range *addresses { - if existing.Address == add.Address && existing.Type == add.Type { - exists = true - break - } - } - if !exists { - *addresses = append(*addresses, add) - } - } -} - -// RemoveFromNodeAddresses removes the NodeAddresses from the passed-by-pointer -// slice if they already exist. -func RemoveFromNodeAddresses(addresses *[]v1.NodeAddress, removeAddresses ...v1.NodeAddress) { - var indexesToRemove []int - for _, remove := range removeAddresses { - for i := len(*addresses) - 1; i >= 0; i-- { - existing := (*addresses)[i] - if existing.Address == remove.Address && (existing.Type == remove.Type || remove.Type == "") { - indexesToRemove = append(indexesToRemove, i) - } - } - } - for _, i := range indexesToRemove { - if i < len(*addresses) { - *addresses = append((*addresses)[:i], (*addresses)[i+1:]...) - } - } -} diff --git a/pkg/cloudprovider/providers/openstack/openstack_loadbalancer.go b/pkg/cloudprovider/providers/openstack/openstack_loadbalancer.go deleted file mode 100644 index 2ecd5364da..0000000000 --- a/pkg/cloudprovider/providers/openstack/openstack_loadbalancer.go +++ /dev/null @@ -1,2808 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package openstack - -import ( - "context" - "fmt" - "net" - "reflect" - "regexp" - "strconv" - "strings" - "time" - - "github.com/gophercloud/gophercloud" - "github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/listeners" - "github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/loadbalancers" - v2monitors "github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/monitors" - v2pools "github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/pools" - "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions" - neutrontags "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/attributestags" - "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/external" - "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/floatingips" - "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/security/groups" - "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/security/rules" - "github.com/gophercloud/gophercloud/openstack/networking/v2/networks" - neutronports "github.com/gophercloud/gophercloud/openstack/networking/v2/ports" - "github.com/gophercloud/gophercloud/openstack/networking/v2/subnets" - "github.com/gophercloud/gophercloud/pagination" - secgroups "github.com/gophercloud/utils/openstack/networking/v2/extensions/security/groups" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apimachinery/pkg/util/wait" - cloudprovider "k8s.io/cloud-provider" - klog "k8s.io/klog/v2" - - cpoutil "k8s.io/cloud-provider-openstack/pkg/util" - cpoerrors "k8s.io/cloud-provider-openstack/pkg/util/errors" - netsets "k8s.io/cloud-provider-openstack/pkg/util/net/sets" - openstackutil "k8s.io/cloud-provider-openstack/pkg/util/openstack" -) - -// Note: when creating a new Loadbalancer (VM), it can take some time before it is ready for use, -// this timeout is used for waiting until the Loadbalancer provisioning status goes to ACTIVE state. -const ( - defaultLoadBalancerSourceRanges = "0.0.0.0/0" - // loadbalancerActive* is configuration of exponential backoff for - // going into ACTIVE loadbalancer provisioning status. Starting with 1 - // seconds, multiplying by 1.2 with each step and taking 19 steps at maximum - // it will time out after 128s, which roughly corresponds to 120s - loadbalancerActiveInitDelay = 1 * time.Second - loadbalancerActiveFactor = 1.2 - loadbalancerActiveSteps = 19 - - // loadbalancerDelete* is configuration of exponential backoff for - // waiting for delete operation to complete. Starting with 1 - // seconds, multiplying by 1.2 with each step and taking 13 steps at maximum - // it will time out after 32s, which roughly corresponds to 30s - loadbalancerDeleteInitDelay = 1 * time.Second - loadbalancerDeleteFactor = 1.2 - loadbalancerDeleteSteps = 13 - - activeStatus = "ACTIVE" - errorStatus = "ERROR" - - annotationXForwardedFor = "X-Forwarded-For" - - // ServiceAnnotationLoadBalancerInternal defines whether or not to create an internal loadbalancer. Default: false. - ServiceAnnotationLoadBalancerInternal = "service.beta.kubernetes.io/openstack-internal-load-balancer" - ServiceAnnotationLoadBalancerConnLimit = "loadbalancer.openstack.org/connection-limit" - ServiceAnnotationLoadBalancerFloatingNetworkID = "loadbalancer.openstack.org/floating-network-id" - ServiceAnnotationLoadBalancerFloatingSubnet = "loadbalancer.openstack.org/floating-subnet" - ServiceAnnotationLoadBalancerFloatingSubnetID = "loadbalancer.openstack.org/floating-subnet-id" - ServiceAnnotationLoadBalancerClass = "loadbalancer.openstack.org/class" - ServiceAnnotationLoadBalancerKeepFloatingIP = "loadbalancer.openstack.org/keep-floatingip" - ServiceAnnotationLoadBalancerPortID = "loadbalancer.openstack.org/port-id" - ServiceAnnotationLoadBalancerProxyEnabled = "loadbalancer.openstack.org/proxy-protocol" - ServiceAnnotationLoadBalancerSubnetID = "loadbalancer.openstack.org/subnet-id" - ServiceAnnotationLoadBalancerNetworkID = "loadbalancer.openstack.org/network-id" - ServiceAnnotationLoadBalancerTimeoutClientData = "loadbalancer.openstack.org/timeout-client-data" - ServiceAnnotationLoadBalancerTimeoutMemberConnect = "loadbalancer.openstack.org/timeout-member-connect" - ServiceAnnotationLoadBalancerTimeoutMemberData = "loadbalancer.openstack.org/timeout-member-data" - ServiceAnnotationLoadBalancerTimeoutTCPInspect = "loadbalancer.openstack.org/timeout-tcp-inspect" - ServiceAnnotationLoadBalancerXForwardedFor = "loadbalancer.openstack.org/x-forwarded-for" - ServiceAnnotationLoadBalancerFlavorID = "loadbalancer.openstack.org/flavor-id" - // ServiceAnnotationLoadBalancerEnableHealthMonitor defines whether or not to create health monitor for the load balancer - // pool, if not specified, use 'create-monitor' config. The health monitor can be created or deleted dynamically. - ServiceAnnotationLoadBalancerEnableHealthMonitor = "loadbalancer.openstack.org/enable-health-monitor" -) - -// LbaasV2 is a LoadBalancer implementation for Neutron LBaaS v2 API -type LbaasV2 struct { - LoadBalancer -} - -// serviceConfig contains configurations for creating a Service. -type serviceConfig struct { - internal bool - connLimit int - configClassName string - lbNetworkID string - lbSubnetID string - lbMemberSubnetID string - lbPublicNetworkID string - lbPublicSubnetID string - keepClientIP bool - enableProxyProtocol bool - timeoutClientData int - timeoutMemberConnect int - timeoutMemberData int - timeoutTCPInspect int - allowedCIDR []string - enableMonitor bool - flavorID string -} - -type listenerKey struct { - Protocol listeners.Protocol - Port int -} - -func networkExtensions(client *gophercloud.ServiceClient) (map[string]bool, error) { - seen := make(map[string]bool) - - pager := extensions.List(client) - err := pager.EachPage(func(page pagination.Page) (bool, error) { - exts, err := extensions.ExtractExtensions(page) - if err != nil { - return false, err - } - for _, ext := range exts { - seen[ext.Alias] = true - } - return true, nil - }) - - return seen, err -} - -func getLoadBalancers(client *gophercloud.ServiceClient, opts loadbalancers.ListOpts) ([]loadbalancers.LoadBalancer, error) { - allPages, err := loadbalancers.List(client, opts).AllPages() - if err != nil { - return nil, err - } - allLoadbalancers, err := loadbalancers.ExtractLoadBalancers(allPages) - if err != nil { - return nil, err - } - - return allLoadbalancers, nil -} - -// getLoadbalancerByName get the load balancer which is in valid status by the given name/legacy name. -func getLoadbalancerByName(client *gophercloud.ServiceClient, name string, legacyName string) (*loadbalancers.LoadBalancer, error) { - var validLBs []loadbalancers.LoadBalancer - - opts := loadbalancers.ListOpts{ - Name: name, - } - allLoadbalancers, err := getLoadBalancers(client, opts) - if err != nil { - return nil, err - } - - if len(allLoadbalancers) == 0 { - if len(legacyName) > 0 { - // Backoff to get load balnacer by legacy name. - opts := loadbalancers.ListOpts{ - Name: legacyName, - } - allLoadbalancers, err = getLoadBalancers(client, opts) - if err != nil { - return nil, err - } - } else { - return nil, ErrNotFound - } - } - - for _, lb := range allLoadbalancers { - // All the ProvisioningStatus could be found here https://developer.openstack.org/api-ref/load-balancer/v2/index.html#provisioning-status-codes - if lb.ProvisioningStatus != "DELETED" && lb.ProvisioningStatus != "PENDING_DELETE" { - validLBs = append(validLBs, lb) - } - } - - if len(validLBs) > 1 { - return nil, ErrMultipleResults - } - if len(validLBs) == 0 { - return nil, ErrNotFound - } - - return &validLBs[0], nil -} - -func getListenersByLoadBalancerID(client *gophercloud.ServiceClient, id string) ([]listeners.Listener, error) { - var existingListeners []listeners.Listener - err := listeners.List(client, listeners.ListOpts{LoadbalancerID: id}).EachPage(func(page pagination.Page) (bool, error) { - listenerList, err := listeners.ExtractListeners(page) - if err != nil { - return false, err - } - for _, l := range listenerList { - for _, lb := range l.Loadbalancers { - if lb.ID == id { - existingListeners = append(existingListeners, l) - break - } - } - } - - return true, nil - }) - if err != nil { - return nil, err - } - - return existingListeners, nil -} - -// get listener for a port or nil if does not exist -func getListenerForPort(existingListeners []listeners.Listener, port corev1.ServicePort) *listeners.Listener { - for _, l := range existingListeners { - if listeners.Protocol(l.Protocol) == toListenersProtocol(port.Protocol) && l.ProtocolPort == int(port.Port) { - return &l - } - } - - return nil -} - -// Check if a member exists for node -func memberExists(members []v2pools.Member, addr string, port int) bool { - for _, member := range members { - if member.Address == addr && member.ProtocolPort == port { - return true - } - } - - return false -} - -func popListener(existingListeners []listeners.Listener, id string) []listeners.Listener { - for i, existingListener := range existingListeners { - if existingListener.ID == id { - existingListeners[i] = existingListeners[len(existingListeners)-1] - existingListeners = existingListeners[:len(existingListeners)-1] - break - } - } - - return existingListeners -} - -func popMember(members []v2pools.Member, addr string, port int) []v2pools.Member { - for i, member := range members { - if member.Address == addr && member.ProtocolPort == port { - members[i] = members[len(members)-1] - members = members[:len(members)-1] - } - } - - return members -} - -func getSecurityGroupName(service *corev1.Service) string { - securityGroupName := fmt.Sprintf("lb-sg-%s-%s-%s", service.UID, service.Namespace, service.Name) - //OpenStack requires that the name of a security group is shorter than 255 bytes. - if len(securityGroupName) > 255 { - securityGroupName = securityGroupName[:255] - } - - return securityGroupName -} - -func getSecurityGroupRules(client *gophercloud.ServiceClient, opts rules.ListOpts) ([]rules.SecGroupRule, error) { - - pager := rules.List(client, opts) - - var securityRules []rules.SecGroupRule - - err := pager.EachPage(func(page pagination.Page) (bool, error) { - ruleList, err := rules.ExtractRules(page) - if err != nil { - return false, err - } - securityRules = append(securityRules, ruleList...) - return true, nil - }) - - if err != nil { - return nil, err - } - - return securityRules, nil -} - -func waitLoadbalancerActiveProvisioningStatus(client *gophercloud.ServiceClient, loadbalancerID string) (string, error) { - backoff := wait.Backoff{ - Duration: loadbalancerActiveInitDelay, - Factor: loadbalancerActiveFactor, - Steps: loadbalancerActiveSteps, - } - - var provisioningStatus string - err := wait.ExponentialBackoff(backoff, func() (bool, error) { - loadbalancer, err := loadbalancers.Get(client, loadbalancerID).Extract() - if err != nil { - return false, err - } - provisioningStatus = loadbalancer.ProvisioningStatus - if loadbalancer.ProvisioningStatus == activeStatus { - return true, nil - } else if loadbalancer.ProvisioningStatus == errorStatus { - return true, fmt.Errorf("loadbalancer has gone into ERROR state") - } else { - return false, nil - } - - }) - - if err == wait.ErrWaitTimeout { - err = fmt.Errorf("loadbalancer failed to go into ACTIVE provisioning status within allotted time") - } - return provisioningStatus, err -} - -func waitLoadbalancerDeleted(client *gophercloud.ServiceClient, loadbalancerID string) error { - backoff := wait.Backoff{ - Duration: loadbalancerDeleteInitDelay, - Factor: loadbalancerDeleteFactor, - Steps: loadbalancerDeleteSteps, - } - err := wait.ExponentialBackoff(backoff, func() (bool, error) { - _, err := loadbalancers.Get(client, loadbalancerID).Extract() - if err != nil { - if cpoerrors.IsNotFound(err) { - return true, nil - } - return false, err - } - return false, nil - }) - - if err == wait.ErrWaitTimeout { - err = fmt.Errorf("loadbalancer failed to delete within the allotted time") - } - - return err -} - -func toRuleProtocol(protocol corev1.Protocol) rules.RuleProtocol { - switch protocol { - case corev1.ProtocolTCP: - return rules.ProtocolTCP - case corev1.ProtocolUDP: - return rules.ProtocolUDP - default: - return rules.RuleProtocol(strings.ToLower(string(protocol))) - } -} - -func toListenersProtocol(protocol corev1.Protocol) listeners.Protocol { - switch protocol { - case corev1.ProtocolTCP: - return listeners.ProtocolTCP - default: - return listeners.Protocol(protocol) - } -} - -func createNodeSecurityGroup(client *gophercloud.ServiceClient, nodeSecurityGroupID string, port int, protocol corev1.Protocol, lbSecGroup string) error { - v4NodeSecGroupRuleCreateOpts := rules.CreateOpts{ - Direction: rules.DirIngress, - PortRangeMax: port, - PortRangeMin: port, - Protocol: toRuleProtocol(protocol), - RemoteGroupID: lbSecGroup, - SecGroupID: nodeSecurityGroupID, - EtherType: rules.EtherType4, - } - - v6NodeSecGroupRuleCreateOpts := rules.CreateOpts{ - Direction: rules.DirIngress, - PortRangeMax: port, - PortRangeMin: port, - Protocol: toRuleProtocol(protocol), - RemoteGroupID: lbSecGroup, - SecGroupID: nodeSecurityGroupID, - EtherType: rules.EtherType6, - } - - _, err := rules.Create(client, v4NodeSecGroupRuleCreateOpts).Extract() - - if err != nil { - return err - } - - _, err = rules.Create(client, v6NodeSecGroupRuleCreateOpts).Extract() - - if err != nil { - return err - } - return nil -} - -// This function is deprecated. -func (lbaas *LbaasV2) createLoadBalancer(service *corev1.Service, name, clusterName string, lbClass *LBClass, internalAnnotation bool, vipPort string) (*loadbalancers.LoadBalancer, error) { - createOpts := loadbalancers.CreateOpts{ - Name: name, - Description: fmt.Sprintf("Kubernetes external service %s/%s from cluster %s", service.Namespace, service.Name, clusterName), - Provider: lbaas.opts.LBProvider, - } - - if vipPort != "" { - createOpts.VipPortID = vipPort - } else { - if lbClass != nil && lbClass.SubnetID != "" { - createOpts.VipSubnetID = lbClass.SubnetID - } else { - createOpts.VipSubnetID = lbaas.opts.SubnetID - } - - if lbClass != nil && lbClass.NetworkID != "" { - createOpts.VipNetworkID = lbClass.NetworkID - } else if lbaas.opts.NetworkID != "" { - createOpts.VipNetworkID = lbaas.opts.NetworkID - } else { - klog.V(4).Infof("network-id parameter not passed, it will be inferred from subnet-id") - } - } - - loadBalancerIP := service.Spec.LoadBalancerIP - if loadBalancerIP != "" && internalAnnotation { - createOpts.VipAddress = loadBalancerIP - } - - loadbalancer, err := loadbalancers.Create(lbaas.lb, createOpts).Extract() - if err != nil { - return nil, fmt.Errorf("error creating loadbalancer %v: %v", createOpts, err) - } - - // when NetworkID is specified, subnet will be selected by the backend for allocating virtual IP - if (lbClass != nil && lbClass.NetworkID != "") || lbaas.opts.NetworkID != "" { - lbaas.opts.SubnetID = loadbalancer.VipSubnetID - } - - return loadbalancer, nil -} - -func (lbaas *LbaasV2) createOctaviaLoadBalancer(service *corev1.Service, name, clusterName string, svcConf *serviceConfig) (*loadbalancers.LoadBalancer, error) { - createOpts := loadbalancers.CreateOpts{ - Name: name, - Description: fmt.Sprintf("Kubernetes external service %s/%s from cluster %s", service.Namespace, service.Name, clusterName), - Provider: lbaas.opts.LBProvider, - } - - if svcConf.flavorID != "" { - createOpts.FlavorID = svcConf.flavorID - } - - vipPort := getStringFromServiceAnnotation(service, ServiceAnnotationLoadBalancerPortID, "") - lbClass := lbaas.opts.LBClasses[svcConf.configClassName] - - if vipPort != "" { - createOpts.VipPortID = vipPort - } else { - if lbClass != nil && lbClass.SubnetID != "" { - createOpts.VipSubnetID = lbClass.SubnetID - } else { - createOpts.VipSubnetID = svcConf.lbSubnetID - } - - if lbClass != nil && lbClass.NetworkID != "" { - createOpts.VipNetworkID = lbClass.NetworkID - } else if svcConf.lbNetworkID != "" { - createOpts.VipNetworkID = svcConf.lbNetworkID - } else { - klog.V(4).Infof("network-id parameter not passed, it will be inferred from subnet-id") - } - } - - // For external load balancer, the LoadBalancerIP is a public IP address. - loadBalancerIP := service.Spec.LoadBalancerIP - if loadBalancerIP != "" && svcConf.internal { - createOpts.VipAddress = loadBalancerIP - } - - loadbalancer, err := loadbalancers.Create(lbaas.lb, createOpts).Extract() - if err != nil { - return nil, fmt.Errorf("error creating loadbalancer %v: %v", createOpts, err) - } - - // In case subnet ID is not configured - if lbaas.opts.SubnetID == "" { - lbaas.opts.SubnetID = loadbalancer.VipSubnetID - svcConf.lbMemberSubnetID = loadbalancer.VipSubnetID - } - - return loadbalancer, nil -} - -// GetLoadBalancer returns whether the specified load balancer exists and its status -func (lbaas *LbaasV2) GetLoadBalancer(ctx context.Context, clusterName string, service *corev1.Service) (*corev1.LoadBalancerStatus, bool, error) { - name := lbaas.GetLoadBalancerName(ctx, clusterName, service) - legacyName := lbaas.GetLoadBalancerLegacyName(ctx, clusterName, service) - loadbalancer, err := getLoadbalancerByName(lbaas.lb, name, legacyName) - if err == ErrNotFound { - return nil, false, nil - } - if loadbalancer == nil { - return nil, false, err - } - - status := &corev1.LoadBalancerStatus{} - - portID := loadbalancer.VipPortID - if portID != "" { - floatIP, err := openstackutil.GetFloatingIPByPortID(lbaas.network, portID) - if err != nil { - return nil, false, fmt.Errorf("failed when trying to get floating IP for port %s: %v", portID, err) - } - if floatIP != nil { - status.Ingress = []corev1.LoadBalancerIngress{{IP: floatIP.FloatingIP}} - } else { - status.Ingress = []corev1.LoadBalancerIngress{{IP: loadbalancer.VipAddress}} - } - } - - return status, true, nil -} - -// GetLoadBalancerName returns the constructed load balancer name. -func (lbaas *LbaasV2) GetLoadBalancerName(ctx context.Context, clusterName string, service *corev1.Service) string { - name := fmt.Sprintf("kube_service_%s_%s_%s", clusterName, service.Namespace, service.Name) - return cutString(name) -} - -// GetLoadBalancerLegacyName returns the legacy load balancer name for backward compatibility. -func (lbaas *LbaasV2) GetLoadBalancerLegacyName(ctx context.Context, clusterName string, service *corev1.Service) string { - return cloudprovider.DefaultLoadBalancerName(service) -} - -// cutString makes sure the string length doesn't exceed 255, which is usually the maximum string length in OpenStack. -func cutString(original string) string { - ret := original - if len(original) > 255 { - ret = original[:255] - } - return ret -} - -// The LB needs to be configured with instance addresses on the same -// subnet as the LB (aka opts.SubnetID). Currently we're just -// guessing that the node's InternalIP is the right address. -// In case no InternalIP can be found, ExternalIP is tried. -// If neither InternalIP nor ExternalIP can be found an error is -// returned. -func nodeAddressForLB(node *corev1.Node) (string, error) { - addrs := node.Status.Addresses - if len(addrs) == 0 { - return "", ErrNoAddressFound - } - - allowedAddrTypes := []corev1.NodeAddressType{corev1.NodeInternalIP, corev1.NodeExternalIP} - - for _, allowedAddrType := range allowedAddrTypes { - for _, addr := range addrs { - if addr.Type == allowedAddrType { - return addr.Address, nil - } - } - } - - return "", ErrNoAddressFound -} - -//getStringFromServiceAnnotation searches a given v1.Service for a specific annotationKey and either returns the annotation's value or a specified defaultSetting -func getStringFromServiceAnnotation(service *corev1.Service, annotationKey string, defaultSetting string) string { - if annotationValue, ok := service.Annotations[annotationKey]; ok { - //if there is an annotation for this setting, set the "setting" var to it - // annotationValue can be empty, it is working as designed - // it makes possible for instance provisioning loadbalancer without floatingip - klog.V(4).Infof("Found a Service Annotation: %v = %v", annotationKey, annotationValue) - return annotationValue - } - //if there is no annotation, set "settings" var to the value from cloud config - klog.V(4).Infof("Could not find a Service Annotation; falling back on cloud-config setting: %v = %v", annotationKey, defaultSetting) - return defaultSetting -} - -//getIntFromServiceAnnotation searches a given v1.Service for a specific annotationKey and either returns the annotation's integer value or a specified defaultSetting -func getIntFromServiceAnnotation(service *corev1.Service, annotationKey string, defaultSetting int) int { - if annotationValue, ok := service.Annotations[annotationKey]; ok { - returnValue, err := strconv.Atoi(annotationValue) - if err != nil { - klog.Warningf("Could not parse int value from %q, failing back to default %s = %v, %v", annotationValue, annotationKey, defaultSetting, err) - return defaultSetting - } - - klog.V(4).Infof("Found a Service Annotation: %v = %v", annotationKey, annotationValue) - return returnValue - } - klog.V(4).Infof("Could not find a Service Annotation; falling back to default setting: %v = %v", annotationKey, defaultSetting) - return defaultSetting -} - -//getBoolFromServiceAnnotation searches a given v1.Service for a specific annotationKey and either returns the annotation's boolean value or a specified defaultSetting -func getBoolFromServiceAnnotation(service *corev1.Service, annotationKey string, defaultSetting bool) (bool, error) { - klog.V(4).Infof("getBoolFromServiceAnnotation(%v, %v, %v)", service, annotationKey, defaultSetting) - if annotationValue, ok := service.Annotations[annotationKey]; ok { - returnValue := false - switch annotationValue { - case "true": - returnValue = true - case "false": - returnValue = false - default: - return returnValue, fmt.Errorf("unknown %s annotation: %v, specify \"true\" or \"false\" ", annotationKey, annotationValue) - } - - klog.V(4).Infof("Found a Service Annotation: %v = %v", annotationKey, returnValue) - return returnValue, nil - } - klog.V(4).Infof("Could not find a Service Annotation; falling back to default setting: %v = %v", annotationKey, defaultSetting) - return defaultSetting, nil -} - -// getSubnetIDForLB returns subnet-id for a specific node -func getSubnetIDForLB(compute *gophercloud.ServiceClient, node corev1.Node) (string, error) { - ipAddress, err := nodeAddressForLB(&node) - if err != nil { - return "", err - } - - instanceID := node.Spec.ProviderID - if ind := strings.LastIndex(instanceID, "/"); ind >= 0 { - instanceID = instanceID[(ind + 1):] - } - - interfaces, err := getAttachedInterfacesByID(compute, instanceID) - if err != nil { - return "", err - } - - for _, intf := range interfaces { - for _, fixedIP := range intf.FixedIPs { - if fixedIP.IPAddress == ipAddress { - return fixedIP.SubnetID, nil - } - } - } - - return "", ErrNotFound -} - -// getPorts gets all the filtered ports. -func getPorts(network *gophercloud.ServiceClient, listOpts neutronports.ListOpts) ([]neutronports.Port, error) { - allPages, err := neutronports.List(network, listOpts).AllPages() - if err != nil { - return []neutronports.Port{}, err - } - allPorts, err := neutronports.ExtractPorts(allPages) - if err != nil { - return []neutronports.Port{}, err - } - - return allPorts, nil -} - -// applyNodeSecurityGroupIDForLB associates the security group with all the ports on the nodes. -func applyNodeSecurityGroupIDForLB(compute *gophercloud.ServiceClient, network *gophercloud.ServiceClient, nodes []*corev1.Node, sg string) error { - for _, node := range nodes { - nodeName := types.NodeName(node.Name) - srv, err := getServerByName(compute, nodeName) - if err != nil { - return err - } - - listOpts := neutronports.ListOpts{DeviceID: srv.ID} - allPorts, err := getPorts(network, listOpts) - if err != nil { - return err - } - - for _, port := range allPorts { - newSGs := append(port.SecurityGroups, sg) - updateOpts := neutronports.UpdateOpts{SecurityGroups: &newSGs} - res := neutronports.Update(network, port.ID, updateOpts) - if res.Err != nil { - return fmt.Errorf("failed to update security group for port %s: %v", port.ID, res.Err) - } - // Add the security group ID as a tag to the port in order to find all these ports when removing the security group. - if err := neutrontags.Add(network, "ports", port.ID, sg).ExtractErr(); err != nil { - return fmt.Errorf("failed to add tag %s to port %s: %v", sg, port.ID, res.Err) - } - } - } - - return nil -} - -// disassociateSecurityGroupForLB removes the given security group from the ports -func disassociateSecurityGroupForLB(network *gophercloud.ServiceClient, sg string) error { - // Find all the ports that have the security group associated. - listOpts := neutronports.ListOpts{TagsAny: sg} - allPorts, err := getPorts(network, listOpts) - if err != nil { - return err - } - - // Disassocate security group and remove the tag. - for _, port := range allPorts { - existingSGs := sets.NewString() - for _, sgID := range port.SecurityGroups { - existingSGs.Insert(sgID) - } - existingSGs.Delete(sg) - - // Update port security groups - newSGs := existingSGs.List() - updateOpts := neutronports.UpdateOpts{SecurityGroups: &newSGs} - res := neutronports.Update(network, port.ID, updateOpts) - if res.Err != nil { - return fmt.Errorf("failed to update security group for port %s: %v", port.ID, res.Err) - } - // Remove the security group ID tag from the port. - if err := neutrontags.Delete(network, "ports", port.ID, sg).ExtractErr(); err != nil { - return fmt.Errorf("failed to remove tag %s to port %s: %v", sg, port.ID, res.Err) - } - } - - return nil -} - -// getNodeSecurityGroupIDForLB lists node-security-groups for specific nodes -func getNodeSecurityGroupIDForLB(compute *gophercloud.ServiceClient, network *gophercloud.ServiceClient, nodes []*corev1.Node) ([]string, error) { - secGroupIDs := sets.NewString() - - for _, node := range nodes { - nodeName := types.NodeName(node.Name) - srv, err := getServerByName(compute, nodeName) - if err != nil { - return []string{}, err - } - - // Get the security groups of all the ports on the worker nodes. In future, we could filter the ports by some way. - // case 0: node1:SG1 node2:SG1 return SG1 - // case 1: node1:SG1 node2:SG2 return SG1,SG2 - // case 2: node1:SG1,SG2 node2:SG3,SG4 return SG1,SG2,SG3,SG4 - // case 3: node1:SG1,SG2 node2:SG2,SG3 return SG1,SG2,SG3 - listOpts := neutronports.ListOpts{DeviceID: srv.ID} - allPorts, err := getPorts(network, listOpts) - if err != nil { - return []string{}, err - } - - for _, port := range allPorts { - for _, sg := range port.SecurityGroups { - secGroupIDs.Insert(sg) - } - } - } - - return secGroupIDs.List(), nil -} - -// isSecurityGroupNotFound return true while 'err' is object of gophercloud.ErrResourceNotFound -func isSecurityGroupNotFound(err error) bool { - errType := reflect.TypeOf(err).String() - errTypeSlice := strings.Split(errType, ".") - errTypeValue := "" - if len(errTypeSlice) != 0 { - errTypeValue = errTypeSlice[len(errTypeSlice)-1] - } - if errTypeValue == "ErrResourceNotFound" { - return true - } - - return false -} - -// getFloatingNetworkIDForLB returns a floating-network-id for cluster. -func getFloatingNetworkIDForLB(client *gophercloud.ServiceClient) (string, error) { - var floatingNetworkIds []string - - type NetworkWithExternalExt struct { - networks.Network - external.NetworkExternalExt - } - - err := networks.List(client, networks.ListOpts{}).EachPage(func(page pagination.Page) (bool, error) { - var externalNetwork []NetworkWithExternalExt - err := networks.ExtractNetworksInto(page, &externalNetwork) - if err != nil { - return false, err - } - - for _, externalNet := range externalNetwork { - if externalNet.External { - floatingNetworkIds = append(floatingNetworkIds, externalNet.ID) - } - } - - if len(floatingNetworkIds) > 1 { - return false, ErrMultipleResults - } - return true, nil - }) - if err != nil { - if cpoerrors.IsNotFound(err) { - return "", ErrNotFound - } - - if err == ErrMultipleResults { - klog.V(4).Infof("find multiple external networks, pick the first one when there are no explicit configuration.") - return floatingNetworkIds[0], nil - } - return "", err - } - - if len(floatingNetworkIds) == 0 { - return "", ErrNotFound - } - - return floatingNetworkIds[0], nil -} - -func (lbaas *LbaasV2) deleteListeners(lbID string, listenerList []listeners.Listener) error { - for _, listener := range listenerList { - klog.V(2).Infof("Deleting obsolete listener %s:", listener.ID) - - pool, err := openstackutil.GetPoolByListener(lbaas.lb, lbID, listener.ID) - if err != nil && err != openstackutil.ErrNotFound { - return fmt.Errorf("error getting pool for obsolete listener %s: %v", listener.ID, err) - } - if pool != nil { - klog.V(2).Infof("Deleting obsolete pool %s for listener %s", pool.ID, listener.ID) - - // Delete pool automatically deletes all its members. - err = v2pools.Delete(lbaas.lb, pool.ID).ExtractErr() - if err != nil && !cpoerrors.IsNotFound(err) { - return fmt.Errorf("error deleting obsolete pool %s for listener %s: %v", pool.ID, listener.ID, err) - } - provisioningStatus, err := waitLoadbalancerActiveProvisioningStatus(lbaas.lb, lbID) - if err != nil { - return fmt.Errorf("timeout when waiting for loadbalancer to be ACTIVE after deleting pool, current provisioning status %s", provisioningStatus) - } - } - - err = listeners.Delete(lbaas.lb, listener.ID).ExtractErr() - if err != nil && !cpoerrors.IsNotFound(err) { - return fmt.Errorf("failed to delete obsolete listener %s: %v", listener.ID, err) - } - provisioningStatus, err := waitLoadbalancerActiveProvisioningStatus(lbaas.lb, lbID) - if err != nil { - return fmt.Errorf("timeout when waiting for loadbalancer to be ACTIVE after deleting listener, current provisioning status %s", provisioningStatus) - } - - klog.V(2).Infof("Deleted obsolete listener %s for load balancer %s", listener.ID, lbID) - } - - return nil -} - -// Priority of choosing VIP port floating IP: -// 1. The floating IP that is already attached to the VIP port. -// 2. Floating IP specified in Spec.LoadBalancerIP -// 3. Create a new one -func (lbaas *LbaasV2) getServiceAddress(clusterName string, service *corev1.Service, lb *loadbalancers.LoadBalancer, svcConf *serviceConfig) (string, error) { - if svcConf.internal { - return lb.VipAddress, nil - } - - var floatIP *floatingips.FloatingIP - serviceName := fmt.Sprintf("%s/%s", service.Namespace, service.Name) - - // first attempt: fetch floating IP attached to load balancer's VIP port - portID := lb.VipPortID - floatIP, err := openstackutil.GetFloatingIPByPortID(lbaas.network, portID) - if err != nil { - return "", fmt.Errorf("failed when getting floating IP for port %s: %v", portID, err) - } - - // second attempt: fetch floating IP specified in service Spec.LoadBalancerIP - // if found, assosiate floating IP with loadbalancer's VIP port - loadBalancerIP := service.Spec.LoadBalancerIP - if floatIP == nil && loadBalancerIP != "" { - opts := floatingips.ListOpts{ - FloatingIP: loadBalancerIP, - } - existingIPs, err := openstackutil.GetFloatingIPs(lbaas.network, opts) - if err != nil { - return "", fmt.Errorf("failed when trying to get existing floating IP %s, error: %v", loadBalancerIP, err) - } - - if len(existingIPs) > 0 { - floatingip := existingIPs[0] - if len(floatingip.PortID) == 0 { - floatUpdateOpts := floatingips.UpdateOpts{ - PortID: &portID, - } - floatIP, err = floatingips.Update(lbaas.network, floatingip.ID, floatUpdateOpts).Extract() - if err != nil { - return "", fmt.Errorf("error updating LB floatingip %+v: %v", floatUpdateOpts, err) - } - } else { - return "", fmt.Errorf("floating IP %s is not available", loadBalancerIP) - } - } - } - - // third attempt: create a new floating IP - if floatIP == nil { - if svcConf.lbPublicNetworkID != "" { - klog.V(2).Infof("Creating floating IP %s for loadbalancer %s", loadBalancerIP, lb.ID) - - floatIPOpts := floatingips.CreateOpts{ - FloatingNetworkID: svcConf.lbPublicNetworkID, - PortID: portID, - Description: fmt.Sprintf("Floating IP for Kubernetes external service %s from cluster %s", serviceName, clusterName), - } - if svcConf.lbPublicSubnetID != "" { - floatIPOpts.SubnetID = svcConf.lbPublicSubnetID - } - if loadBalancerIP != "" { - floatIPOpts.FloatingIP = loadBalancerIP - } - - klog.V(4).Infof("Creating floating ip with opts %+v", floatIPOpts) - - floatIP, err = floatingips.Create(lbaas.network, floatIPOpts).Extract() - if err != nil { - return "", fmt.Errorf("error creating LB floatingip: %v", err) - } - } else { - klog.Warningf("Floating network configuration not provided for Service %s, forcing to ensure an internal load balancer service", serviceName) - } - } - - if floatIP != nil { - return floatIP.FloatingIP, nil - } - - return lb.VipAddress, nil -} - -func (lbaas *LbaasV2) ensureOctaviaHealthMonitor(lbID string, pool *v2pools.Pool, port corev1.ServicePort, svcConf *serviceConfig) error { - monitorID := pool.MonitorID - - if monitorID == "" && svcConf.enableMonitor { - klog.V(4).Infof("Creating monitor for pool %s", pool.ID) - - monitorProtocol := string(port.Protocol) - if port.Protocol == corev1.ProtocolUDP { - monitorProtocol = "UDP-CONNECT" - } - monitor, err := v2monitors.Create(lbaas.lb, v2monitors.CreateOpts{ - PoolID: pool.ID, - Type: monitorProtocol, - Delay: int(lbaas.opts.MonitorDelay.Duration.Seconds()), - Timeout: int(lbaas.opts.MonitorTimeout.Duration.Seconds()), - MaxRetries: int(lbaas.opts.MonitorMaxRetries), - }).Extract() - if err != nil { - return fmt.Errorf("failed to create healthmonitor for pool %s: %v", pool.ID, err) - } - provisioningStatus, err := waitLoadbalancerActiveProvisioningStatus(lbaas.lb, lbID) - if err != nil { - return fmt.Errorf("timeout when waiting for loadbalancer %s to be ACTIVE after creating health monitor, current provisioning status %s", lbID, provisioningStatus) - } - monitorID = monitor.ID - } else if monitorID != "" && !svcConf.enableMonitor { - klog.Infof("Deleting health monitor %s for pool %s", monitorID, pool.ID) - - err := v2monitors.Delete(lbaas.lb, monitorID).ExtractErr() - if err != nil { - return fmt.Errorf("failed to delete health monitor %s for pool %s, error: %v", monitorID, pool.ID, err) - } - } - - if monitorID != "" { - klog.Infof("Health monitor %s for pool %s created.", monitorID, pool.ID) - } - - return nil -} - -// Make sure the pool is created for the Service, nodes are added as pool members. -func (lbaas *LbaasV2) ensureOctaviaPool(lbID string, listener *listeners.Listener, service *corev1.Service, port corev1.ServicePort, nodes []*corev1.Node, svcConf *serviceConfig) (*v2pools.Pool, error) { - var members []v2pools.BatchUpdateMemberOpts - newMembers := sets.NewString() - - for _, node := range nodes { - addr, err := nodeAddressForLB(node) - if err != nil { - if err == ErrNotFound { - // Node failure, do not create member - klog.Warningf("Failed to get the address of node %s for creating member: %v", node.Name, err) - continue - } else { - return nil, fmt.Errorf("error getting address of node %s: %v", node.Name, err) - } - } - - member := v2pools.BatchUpdateMemberOpts{ - Address: addr, - ProtocolPort: int(port.NodePort), - Name: &node.Name, - SubnetID: &svcConf.lbMemberSubnetID, - } - members = append(members, member) - newMembers.Insert(fmt.Sprintf("%s-%d", addr, member.ProtocolPort)) - } - - pool, err := openstackutil.GetPoolByListener(lbaas.lb, lbID, listener.ID) - if err != nil && err != openstackutil.ErrNotFound { - return nil, fmt.Errorf("error getting pool for listener %s: %v", listener.ID, err) - } - if pool == nil { - // By default, use the protocol of the listerner - poolProto := v2pools.Protocol(listener.Protocol) - - if svcConf.enableProxyProtocol { - poolProto = v2pools.ProtocolPROXY - } else if svcConf.keepClientIP && poolProto != v2pools.ProtocolHTTP { - klog.V(4).Infof("Forcing to use %q protocol for pool because annotation %q is set", v2pools.ProtocolHTTP, ServiceAnnotationLoadBalancerXForwardedFor) - poolProto = v2pools.ProtocolHTTP - } - - affinity := service.Spec.SessionAffinity - var persistence *v2pools.SessionPersistence - switch affinity { - case corev1.ServiceAffinityNone: - persistence = nil - case corev1.ServiceAffinityClientIP: - persistence = &v2pools.SessionPersistence{Type: "SOURCE_IP"} - } - - lbmethod := v2pools.LBMethod(lbaas.opts.LBMethod) - createOpt := v2pools.CreateOpts{ - Protocol: poolProto, - LBMethod: lbmethod, - ListenerID: listener.ID, - Persistence: persistence, - } - - klog.V(2).Infof("Creating pool for listener %s using protocol %s", listener.ID, poolProto) - - pool, err = v2pools.Create(lbaas.lb, createOpt).Extract() - if err != nil { - return nil, fmt.Errorf("error creating pool for listener %s: %v", listener.ID, err) - } - - provisioningStatus, err := waitLoadbalancerActiveProvisioningStatus(lbaas.lb, lbID) - if err != nil { - return nil, fmt.Errorf("timeout when waiting for loadbalancer %s to be ACTIVE after creating pool, current provisioning status %s", lbID, provisioningStatus) - } - } - - klog.V(2).Infof("Pool %s created for listener %s", pool.ID, listener.ID) - - curMembers := sets.NewString() - poolMembers, err := openstackutil.GetMembersbyPool(lbaas.lb, pool.ID) - if err != nil { - klog.Errorf("failed to get members in the pool %s: %v", pool.ID, err) - } - for _, m := range poolMembers { - curMembers.Insert(fmt.Sprintf("%s-%d", m.Address, m.ProtocolPort)) - } - - if !curMembers.Equal(newMembers) { - klog.V(2).Infof("Updating %d members for pool %s", len(members), pool.ID) - if err := openstackutil.BatchUpdatePoolMembers(lbaas.lb, lbID, pool.ID, members); err != nil { - return nil, err - } - klog.V(2).Infof("Successfully updated %d members for pool %s", len(members), pool.ID) - } - - return pool, nil -} - -// Make sure the listener is created for Service -func (lbaas *LbaasV2) ensureOctaviaListener(lbID string, oldListeners []listeners.Listener, service *corev1.Service, port corev1.ServicePort, svcConf *serviceConfig) (*listeners.Listener, error) { - // Get all listeners by "port&protocol". - lbListeners := make(map[listenerKey]*listeners.Listener) - for _, l := range oldListeners { - key := listenerKey{Protocol: listeners.Protocol(l.Protocol), Port: l.ProtocolPort} - lbListeners[key] = &l - } - - proto := toListenersProtocol(port.Protocol) - if svcConf.keepClientIP { - proto = listeners.ProtocolHTTP - } - listener, ok := lbListeners[listenerKey{ - Protocol: proto, - Port: int(port.Port), - }] - - if !ok { - listenerProtocol := listeners.Protocol(port.Protocol) - listenerCreateOpt := listeners.CreateOpts{ - Protocol: listenerProtocol, - ProtocolPort: int(port.Port), - ConnLimit: &svcConf.connLimit, - LoadbalancerID: lbID, - TimeoutClientData: &svcConf.timeoutClientData, - TimeoutMemberConnect: &svcConf.timeoutMemberConnect, - TimeoutMemberData: &svcConf.timeoutMemberData, - TimeoutTCPInspect: &svcConf.timeoutTCPInspect, - } - - if svcConf.keepClientIP { - if listenerCreateOpt.Protocol != listeners.ProtocolHTTP { - klog.V(4).Infof("Forcing to use %q protocol for listener because %q annotation is set", listeners.ProtocolHTTP, ServiceAnnotationLoadBalancerXForwardedFor) - listenerCreateOpt.Protocol = listeners.ProtocolHTTP - } - listenerCreateOpt.InsertHeaders = map[string]string{annotationXForwardedFor: "true"} - } - - if len(svcConf.allowedCIDR) > 0 { - listenerCreateOpt.AllowedCIDRs = svcConf.allowedCIDR - } - - klog.V(2).Infof("Creating listener for port %d using protocol %s", int(port.Port), listenerProtocol) - - var err error - listener, err = openstackutil.CreateListener(lbaas.lb, lbID, listenerCreateOpt) - if err != nil { - return nil, fmt.Errorf("failed to create listener for loadbalancer %s: %v", lbID, err) - } - - klog.V(4).Infof("Listener %s created for loadbalancer %s", listener.ID, lbID) - } else { - listenerChanged := false - updateOpts := listeners.UpdateOpts{} - - if svcConf.connLimit != listener.ConnLimit { - updateOpts.ConnLimit = &svcConf.connLimit - listenerChanged = true - } - updateOpts.InsertHeaders = &listener.InsertHeaders - listenerKeepClientIP := listener.InsertHeaders[annotationXForwardedFor] == "true" - if svcConf.keepClientIP != listenerKeepClientIP { - if svcConf.keepClientIP { - (*updateOpts.InsertHeaders)[annotationXForwardedFor] = "true" - } else { - delete(*updateOpts.InsertHeaders, annotationXForwardedFor) - } - listenerChanged = true - } - if svcConf.timeoutClientData != listener.TimeoutClientData { - updateOpts.TimeoutClientData = &svcConf.timeoutClientData - listenerChanged = true - } - if svcConf.timeoutMemberConnect != listener.TimeoutMemberConnect { - updateOpts.TimeoutMemberConnect = &svcConf.timeoutMemberConnect - listenerChanged = true - } - if svcConf.timeoutMemberData != listener.TimeoutMemberData { - updateOpts.TimeoutMemberData = &svcConf.timeoutMemberData - listenerChanged = true - } - if svcConf.timeoutTCPInspect != listener.TimeoutTCPInspect { - updateOpts.TimeoutTCPInspect = &svcConf.timeoutTCPInspect - listenerChanged = true - } - if openstackutil.IsOctaviaFeatureSupported(lbaas.lb, openstackutil.OctaviaFeatureVIPACL) { - if !cpoutil.StringListEqual(svcConf.allowedCIDR, listener.AllowedCIDRs) { - updateOpts.AllowedCIDRs = &svcConf.allowedCIDR - listenerChanged = true - } - } - - if listenerChanged { - if err := openstackutil.UpdateListener(lbaas.lb, lbID, listener.ID, updateOpts); err != nil { - return nil, fmt.Errorf("failed to update listener %s of loadbalancer %s: %v", listener.ID, lbID, err) - } - klog.V(2).Infof("Listener %s updated for loadbalancer %s", listener.ID, lbID) - } - } - - return listener, nil -} - -func (lbaas *LbaasV2) checkService(service *corev1.Service, nodes []*corev1.Node, svcConf *serviceConfig) error { - serviceName := fmt.Sprintf("%s/%s", service.Namespace, service.Name) - - if len(nodes) == 0 { - return fmt.Errorf("there are no available nodes for LoadBalancer service %s", serviceName) - } - ports := service.Spec.Ports - if len(ports) == 0 { - return fmt.Errorf("no service ports provided") - } - - // If in the config file internal-lb=true, user is not allowed to create external service. - if lbaas.opts.InternalLB { - svcConf.internal = true - } else { - internal, err := getBoolFromServiceAnnotation(service, ServiceAnnotationLoadBalancerInternal, lbaas.opts.InternalLB) - if err != nil { - return err - } - svcConf.internal = internal - } - - svcConf.connLimit = getIntFromServiceAnnotation(service, ServiceAnnotationLoadBalancerConnLimit, -1) - - svcConf.lbNetworkID = getStringFromServiceAnnotation(service, ServiceAnnotationLoadBalancerNetworkID, lbaas.opts.NetworkID) - svcConf.lbSubnetID = getStringFromServiceAnnotation(service, ServiceAnnotationLoadBalancerSubnetID, lbaas.opts.SubnetID) - if lbaas.opts.SubnetID != "" { - svcConf.lbMemberSubnetID = lbaas.opts.SubnetID - } else { - svcConf.lbMemberSubnetID = svcConf.lbSubnetID - } - if len(svcConf.lbNetworkID) == 0 && len(svcConf.lbSubnetID) == 0 { - subnetID, err := getSubnetIDForLB(lbaas.compute, *nodes[0]) - if err != nil { - return fmt.Errorf("failed to get subnet to create load balancer for service %s: %v", serviceName, err) - } - svcConf.lbSubnetID = subnetID - svcConf.lbMemberSubnetID = subnetID - lbaas.opts.SubnetID = subnetID - } - - if !svcConf.internal { - var lbClass *LBClass - var floatingNetworkID string - var floatingSubnetID string - - klog.V(4).Infof("Ensure an external loadbalancer service") - - svcConf.configClassName = getStringFromServiceAnnotation(service, ServiceAnnotationLoadBalancerClass, "") - if svcConf.configClassName != "" { - lbClass = lbaas.opts.LBClasses[svcConf.configClassName] - if lbClass == nil { - return fmt.Errorf("invalid loadbalancer class %q", svcConf.configClassName) - } - - klog.V(4).Infof("Found loadbalancer class %q with %+v", svcConf.configClassName, lbClass) - - // Get floating network id and floating subnet id from loadbalancer class - if lbClass.FloatingNetworkID != "" { - floatingNetworkID = lbClass.FloatingNetworkID - } - - if lbClass.FloatingSubnetID != "" { - floatingSubnetID = lbClass.FloatingSubnetID - } - } - - if floatingNetworkID == "" { - floatingNetworkID = getStringFromServiceAnnotation(service, ServiceAnnotationLoadBalancerFloatingNetworkID, lbaas.opts.FloatingNetworkID) - if floatingNetworkID == "" { - var err error - floatingNetworkID, err = getFloatingNetworkIDForLB(lbaas.network) - if err != nil { - klog.Warningf("Failed to find floating-network-id for Service %s: %v", serviceName, err) - } - } - } - - if floatingSubnetID == "" { - floatingSubnetID = getStringFromServiceAnnotation(service, ServiceAnnotationLoadBalancerFloatingSubnetID, lbaas.opts.FloatingSubnetID) - - if floatingSubnetID == "" { - floatingSubnetName := getStringFromServiceAnnotation(service, ServiceAnnotationLoadBalancerFloatingSubnet, "") - if floatingSubnetName != "" { - lbSubnet, err := lbaas.getSubnet(floatingSubnetName) - if err != nil || lbSubnet == nil { - klog.Warningf("Failed to find floating-subnet-id for Service %s: %v", serviceName, err) - } else { - floatingSubnetID = lbSubnet.ID - } - } - } - } - - // check subnets belongs to network - if floatingNetworkID != "" && floatingSubnetID != "" { - subnet, err := subnets.Get(lbaas.network, floatingSubnetID).Extract() - if err != nil { - return fmt.Errorf("failed to find subnet %q: %v", floatingSubnetID, err) - } - - if subnet.NetworkID != floatingNetworkID { - return fmt.Errorf("floating IP subnet %q doesn't belong to the network %q", floatingSubnetID, floatingSubnetID) - } - } - - svcConf.lbPublicNetworkID = floatingNetworkID - svcConf.lbPublicSubnetID = floatingSubnetID - } else { - klog.V(4).Infof("Ensure an internal loadbalancer service.") - } - - keepClientIP, err := getBoolFromServiceAnnotation(service, ServiceAnnotationLoadBalancerXForwardedFor, false) - if err != nil { - return err - } - useProxyProtocol, err := getBoolFromServiceAnnotation(service, ServiceAnnotationLoadBalancerProxyEnabled, false) - if err != nil { - return err - } - if useProxyProtocol && keepClientIP { - return fmt.Errorf("annotation %s and %s cannot be used together", ServiceAnnotationLoadBalancerProxyEnabled, ServiceAnnotationLoadBalancerXForwardedFor) - } - svcConf.keepClientIP = keepClientIP - svcConf.enableProxyProtocol = useProxyProtocol - - svcConf.timeoutClientData = getIntFromServiceAnnotation(service, ServiceAnnotationLoadBalancerTimeoutClientData, 50000) - svcConf.timeoutMemberConnect = getIntFromServiceAnnotation(service, ServiceAnnotationLoadBalancerTimeoutMemberConnect, 5000) - svcConf.timeoutMemberData = getIntFromServiceAnnotation(service, ServiceAnnotationLoadBalancerTimeoutMemberData, 50000) - svcConf.timeoutTCPInspect = getIntFromServiceAnnotation(service, ServiceAnnotationLoadBalancerTimeoutTCPInspect, 0) - - var listenerAllowedCIDRs []string - sourceRanges, err := GetLoadBalancerSourceRanges(service) - if err != nil { - return fmt.Errorf("failed to get source ranges for loadbalancer service %s: %v", serviceName, err) - } - if openstackutil.IsOctaviaFeatureSupported(lbaas.lb, openstackutil.OctaviaFeatureVIPACL) { - klog.V(4).Info("LoadBalancerSourceRanges is suppported") - listenerAllowedCIDRs = sourceRanges.StringSlice() - } else { - klog.Warning("LoadBalancerSourceRanges is ignored") - } - svcConf.allowedCIDR = listenerAllowedCIDRs - - if openstackutil.IsOctaviaFeatureSupported(lbaas.lb, openstackutil.OctaviaFeatureFlavors) { - svcConf.flavorID = getStringFromServiceAnnotation(service, ServiceAnnotationLoadBalancerFlavorID, lbaas.opts.FlavorID) - } - - enableHealthMonitor, err := getBoolFromServiceAnnotation(service, ServiceAnnotationLoadBalancerEnableHealthMonitor, lbaas.opts.CreateMonitor) - if err != nil { - return err - } - svcConf.enableMonitor = enableHealthMonitor - - return nil -} - -func (lbaas *LbaasV2) ensureOctaviaLoadBalancer(ctx context.Context, clusterName string, service *corev1.Service, nodes []*corev1.Node) (*corev1.LoadBalancerStatus, error) { - svcConf := new(serviceConfig) - - if err := lbaas.checkService(service, nodes, svcConf); err != nil { - return nil, err - } - - serviceName := fmt.Sprintf("%s/%s", service.Namespace, service.Name) - - // Use more meaningful name for the load balancer but still need to check the legacy name for backward compatibility. - name := lbaas.GetLoadBalancerName(ctx, clusterName, service) - legacyName := lbaas.GetLoadBalancerLegacyName(ctx, clusterName, service) - loadbalancer, err := getLoadbalancerByName(lbaas.lb, name, legacyName) - if err != nil { - if err != ErrNotFound { - return nil, fmt.Errorf("error getting loadbalancer for Service %s: %v", serviceName, err) - } - - klog.V(2).Infof("Creating loadbalancer %s", name) - loadbalancer, err = lbaas.createOctaviaLoadBalancer(service, name, clusterName, svcConf) - if err != nil { - return nil, fmt.Errorf("error creating loadbalancer %s: %v", name, err) - } - } else { - klog.V(2).Infof("LoadBalancer %s(%s) already exists", loadbalancer.Name, loadbalancer.ID) - } - - provisioningStatus, err := waitLoadbalancerActiveProvisioningStatus(lbaas.lb, loadbalancer.ID) - if err != nil { - return nil, fmt.Errorf("timeout when waiting for loadbalancer %s to be ACTIVE, current provisioning status %s", loadbalancer.ID, provisioningStatus) - } - - oldListeners, err := getListenersByLoadBalancerID(lbaas.lb, loadbalancer.ID) - if err != nil { - return nil, fmt.Errorf("failed to get listeners for load balancer %s: %v", loadbalancer.Name, err) - } - - for _, port := range service.Spec.Ports { - listener, err := lbaas.ensureOctaviaListener(loadbalancer.ID, oldListeners, service, port, svcConf) - if err != nil { - return nil, err - } - - // After all ports have been processed, remaining listeners are removed as obsolete. - // Pop valid listener. - oldListeners = popListener(oldListeners, listener.ID) - - pool, err := lbaas.ensureOctaviaPool(loadbalancer.ID, listener, service, port, nodes, svcConf) - if err != nil { - return nil, err - } - - if err := lbaas.ensureOctaviaHealthMonitor(loadbalancer.ID, pool, port, svcConf); err != nil { - return nil, err - } - } - - if err := lbaas.deleteListeners(loadbalancer.ID, oldListeners); err != nil { - return nil, err - } - - addr, err := lbaas.getServiceAddress(clusterName, service, loadbalancer, svcConf) - if err != nil { - return nil, err - } - - status := &corev1.LoadBalancerStatus{ - Ingress: []corev1.LoadBalancerIngress{{IP: addr}}, - } - - return status, nil -} - -// EnsureLoadBalancer creates a new load balancer or updates the existing one. -func (lbaas *LbaasV2) EnsureLoadBalancer(ctx context.Context, clusterName string, apiService *corev1.Service, nodes []*corev1.Node) (*corev1.LoadBalancerStatus, error) { - serviceName := fmt.Sprintf("%s/%s", apiService.Namespace, apiService.Name) - klog.V(4).Infof("EnsureLoadBalancer(%s, %s)", clusterName, serviceName) - - if lbaas.opts.UseOctavia { - return lbaas.ensureOctaviaLoadBalancer(ctx, clusterName, apiService, nodes) - } - - // Following code is just for legacy Neutron-LBaaS support which has been deprecated since OpenStack stable/queens - // and not recommended to use in production. No new features should be added. - - if len(nodes) == 0 { - return nil, fmt.Errorf("there are no available nodes for LoadBalancer service %s", serviceName) - } - - lbaas.opts.NetworkID = getStringFromServiceAnnotation(apiService, ServiceAnnotationLoadBalancerNetworkID, lbaas.opts.NetworkID) - lbaas.opts.SubnetID = getStringFromServiceAnnotation(apiService, ServiceAnnotationLoadBalancerSubnetID, lbaas.opts.SubnetID) - if len(lbaas.opts.SubnetID) == 0 && len(lbaas.opts.NetworkID) == 0 { - // Get SubnetID automatically. - // The LB needs to be configured with instance addresses on the same subnet, so get SubnetID by one node. - subnetID, err := getSubnetIDForLB(lbaas.compute, *nodes[0]) - if err != nil { - klog.Warningf("Failed to find subnet-id for loadbalancer service %s/%s: %v", apiService.Namespace, apiService.Name, err) - return nil, fmt.Errorf("no subnet-id for service %s/%s : subnet-id not set in cloud provider config, "+ - "and failed to find subnet-id from OpenStack: %v", apiService.Namespace, apiService.Name, err) - } - lbaas.opts.SubnetID = subnetID - } - - ports := apiService.Spec.Ports - if len(ports) == 0 { - return nil, fmt.Errorf("no ports provided to openstack load balancer") - } - - internalAnnotation, err := getBoolFromServiceAnnotation(apiService, ServiceAnnotationLoadBalancerInternal, lbaas.opts.InternalLB) - if err != nil { - return nil, err - } - - var lbClass *LBClass - var floatingNetworkID string - var floatingSubnetID string - if !internalAnnotation { - klog.V(4).Infof("Ensure an external loadbalancer service") - - class := getStringFromServiceAnnotation(apiService, ServiceAnnotationLoadBalancerClass, "") - if class != "" { - lbClass = lbaas.opts.LBClasses[class] - if lbClass == nil { - return nil, fmt.Errorf("invalid loadbalancer class %q", class) - } - - klog.V(4).Infof("found loadbalancer class %q with %+v", class, lbClass) - - // read floating network id and floating subnet id from loadbalancer class - if lbClass.FloatingNetworkID != "" { - floatingNetworkID = lbClass.FloatingNetworkID - } - - if lbClass.FloatingSubnetID != "" { - floatingSubnetID = lbClass.FloatingSubnetID - } - } - - if floatingNetworkID == "" { - floatingNetworkID = getStringFromServiceAnnotation(apiService, ServiceAnnotationLoadBalancerFloatingNetworkID, lbaas.opts.FloatingNetworkID) - if floatingNetworkID == "" { - var err error - floatingNetworkID, err = getFloatingNetworkIDForLB(lbaas.network) - if err != nil { - klog.Warningf("Failed to find floating-network-id for Service %s: %v", serviceName, err) - } - } - } - - if floatingSubnetID == "" { - floatingSubnetName := getStringFromServiceAnnotation(apiService, ServiceAnnotationLoadBalancerFloatingSubnet, "") - if floatingSubnetName != "" { - lbSubnet, err := lbaas.getSubnet(floatingSubnetName) - if err != nil || lbSubnet == nil { - klog.Warningf("Failed to find floating-subnet-id for Service %s: %v", serviceName, err) - } else { - floatingSubnetID = lbSubnet.ID - } - } - - if floatingSubnetID == "" { - floatingSubnetID = getStringFromServiceAnnotation(apiService, ServiceAnnotationLoadBalancerFloatingSubnetID, lbaas.opts.FloatingSubnetID) - } - } - - // check subnets belongs to network - if floatingNetworkID != "" && floatingSubnetID != "" { - subnet, err := subnets.Get(lbaas.network, floatingSubnetID).Extract() - if err != nil { - return nil, fmt.Errorf("failed to find subnet %q: %v", floatingSubnetID, err) - } - - if subnet.NetworkID != floatingNetworkID { - return nil, fmt.Errorf("FloatingSubnet %q doesn't belong to FloatingNetwork %q", floatingSubnetID, floatingSubnetID) - } - } - } else { - klog.V(4).Infof("Ensure an internal loadbalancer service.") - } - - var keepClientIP bool - var useProxyProtocol bool - var timeoutClientData int - var timeoutMemberConnect int - var timeoutMemberData int - var timeoutTCPInspect int - if !lbaas.opts.UseOctavia { - // Check for TCP protocol on each port - for _, port := range ports { - if port.Protocol != corev1.ProtocolTCP { - return nil, fmt.Errorf("only TCP LoadBalancer is supported for openstack load balancers") - } - } - } else { - //setting http headers and proxy protocol is only supported by Octavia - keepClientIP, err = getBoolFromServiceAnnotation(apiService, ServiceAnnotationLoadBalancerXForwardedFor, false) - if err != nil { - return nil, err - } - - useProxyProtocol, err = getBoolFromServiceAnnotation(apiService, ServiceAnnotationLoadBalancerProxyEnabled, false) - if err != nil { - return nil, err - } - - if useProxyProtocol && keepClientIP { - return nil, fmt.Errorf("annotation %s and %s cannot be used together", ServiceAnnotationLoadBalancerProxyEnabled, ServiceAnnotationLoadBalancerXForwardedFor) - } - - timeoutClientData = getIntFromServiceAnnotation(apiService, ServiceAnnotationLoadBalancerTimeoutClientData, 50000) - timeoutMemberConnect = getIntFromServiceAnnotation(apiService, ServiceAnnotationLoadBalancerTimeoutMemberConnect, 5000) - timeoutMemberData = getIntFromServiceAnnotation(apiService, ServiceAnnotationLoadBalancerTimeoutMemberData, 50000) - timeoutTCPInspect = getIntFromServiceAnnotation(apiService, ServiceAnnotationLoadBalancerTimeoutTCPInspect, 0) - } - - var listenerAllowedCIDRs []string - sourceRanges, err := GetLoadBalancerSourceRanges(apiService) - if err != nil { - return nil, fmt.Errorf("failed to get source ranges for loadbalancer service %s: %v", serviceName, err) - } - if lbaas.opts.UseOctavia && openstackutil.IsOctaviaFeatureSupported(lbaas.lb, openstackutil.OctaviaFeatureVIPACL) { - klog.V(4).Info("loadBalancerSourceRanges is suppported") - listenerAllowedCIDRs = sourceRanges.StringSlice() - } else if !IsAllowAll(sourceRanges) && !lbaas.opts.ManageSecurityGroups { - return nil, fmt.Errorf("source range restrictions are not supported for openstack load balancers without managing security groups") - } - - affinity := apiService.Spec.SessionAffinity - var persistence *v2pools.SessionPersistence - switch affinity { - case corev1.ServiceAffinityNone: - persistence = nil - case corev1.ServiceAffinityClientIP: - persistence = &v2pools.SessionPersistence{Type: "SOURCE_IP"} - default: - return nil, fmt.Errorf("unsupported load balancer affinity: %v", affinity) - } - - // Use more meaningful name for the load balancer but still need to check the legacy name for backward compatibility. - name := lbaas.GetLoadBalancerName(ctx, clusterName, apiService) - legacyName := lbaas.GetLoadBalancerLegacyName(ctx, clusterName, apiService) - loadbalancer, err := getLoadbalancerByName(lbaas.lb, name, legacyName) - if err != nil { - if err != ErrNotFound { - return nil, fmt.Errorf("error getting loadbalancer for Service %s: %v", serviceName, err) - } - - klog.V(2).Infof("Creating loadbalancer %s", name) - - portID := "" - if lbClass == nil { - portID = getStringFromServiceAnnotation(apiService, ServiceAnnotationLoadBalancerPortID, "") - } - loadbalancer, err = lbaas.createLoadBalancer(apiService, name, clusterName, lbClass, internalAnnotation, portID) - if err != nil { - return nil, fmt.Errorf("error creating loadbalancer %s: %v", name, err) - } - } else { - klog.V(2).Infof("LoadBalancer %s already exists", loadbalancer.Name) - } - - provisioningStatus, err := waitLoadbalancerActiveProvisioningStatus(lbaas.lb, loadbalancer.ID) - if err != nil { - return nil, fmt.Errorf("timeout when waiting for loadbalancer to be ACTIVE, current provisioning status %s", provisioningStatus) - } - - oldListeners, err := getListenersByLoadBalancerID(lbaas.lb, loadbalancer.ID) - if err != nil { - return nil, fmt.Errorf("error getting LB %s listeners: %v", loadbalancer.Name, err) - } - for portIndex, port := range ports { - listener := getListenerForPort(oldListeners, port) - climit := getStringFromServiceAnnotation(apiService, ServiceAnnotationLoadBalancerConnLimit, "-1") - connLimit := -1 - tmp, err := strconv.Atoi(climit) - if err != nil { - klog.V(4).Infof("Could not parse int value from \"%s\" error \"%v\" failing back to default", climit, err) - } else { - connLimit = tmp - } - - if listener == nil { - listenerProtocol := listeners.Protocol(port.Protocol) - listenerCreateOpt := listeners.CreateOpts{ - Name: cutString(fmt.Sprintf("listener_%d_%s", portIndex, name)), - Protocol: listenerProtocol, - ProtocolPort: int(port.Port), - ConnLimit: &connLimit, - LoadbalancerID: loadbalancer.ID, - } - - if lbaas.opts.UseOctavia { - if keepClientIP { - if listenerCreateOpt.Protocol != listeners.ProtocolHTTP { - klog.V(4).Infof("Forcing to use %q protocol for listener because %q "+ - "annotation is set", listeners.ProtocolHTTP, ServiceAnnotationLoadBalancerXForwardedFor) - listenerCreateOpt.Protocol = listeners.ProtocolHTTP - } - listenerCreateOpt.InsertHeaders = map[string]string{"X-Forwarded-For": "true"} - } - - listenerCreateOpt.TimeoutClientData = &timeoutClientData - listenerCreateOpt.TimeoutMemberData = &timeoutMemberData - listenerCreateOpt.TimeoutMemberConnect = &timeoutMemberConnect - listenerCreateOpt.TimeoutTCPInspect = &timeoutTCPInspect - - if len(listenerAllowedCIDRs) > 0 { - listenerCreateOpt.AllowedCIDRs = listenerAllowedCIDRs - } - } - - klog.V(4).Infof("Creating listener for port %d using protocol: %s", int(port.Port), listenerProtocol) - - listener, err = openstackutil.CreateListener(lbaas.lb, loadbalancer.ID, listenerCreateOpt) - if err != nil { - return nil, fmt.Errorf("failed to create listener for loadbalancer %s: %v", loadbalancer.ID, err) - } - - klog.V(4).Infof("Listener %s created for loadbalancer %s", listener.ID, loadbalancer.ID) - } else { - listenerChanged := false - updateOpts := listeners.UpdateOpts{} - - if connLimit != listener.ConnLimit { - updateOpts.ConnLimit = &connLimit - listenerChanged = true - } - - if lbaas.opts.UseOctavia { - if openstackutil.IsOctaviaFeatureSupported(lbaas.lb, openstackutil.OctaviaFeatureVIPACL) { - if !cpoutil.StringListEqual(listenerAllowedCIDRs, listener.AllowedCIDRs) { - updateOpts.AllowedCIDRs = &listenerAllowedCIDRs - listenerChanged = true - } - } - } - - if listenerChanged { - if err := openstackutil.UpdateListener(lbaas.lb, loadbalancer.ID, listener.ID, updateOpts); err != nil { - return nil, fmt.Errorf("failed to update listener %s of loadbalancer %s: %v", listener.ID, loadbalancer.ID, err) - } - - klog.V(4).Infof("Listener %s updated for loadbalancer %s", listener.ID, loadbalancer.ID) - } - } - - // After all ports have been processed, remaining listeners are removed as obsolete. - // Pop valid listeners. - oldListeners = popListener(oldListeners, listener.ID) - - pool, err := openstackutil.GetPoolByListener(lbaas.lb, loadbalancer.ID, listener.ID) - if err != nil && err != openstackutil.ErrNotFound { - return nil, fmt.Errorf("error getting pool for listener %s: %v", listener.ID, err) - } - if pool == nil { - // Use the protocol of the listerner - poolProto := v2pools.Protocol(listener.Protocol) - - if lbaas.opts.UseOctavia { - if useProxyProtocol { - poolProto = v2pools.ProtocolPROXY - } else if keepClientIP && poolProto != v2pools.ProtocolHTTP { - klog.V(4).Infof("Forcing to use %q protocol for pool because %q "+ - "annotation is set", v2pools.ProtocolHTTP, ServiceAnnotationLoadBalancerXForwardedFor) - poolProto = v2pools.ProtocolHTTP - } - } - - lbmethod := v2pools.LBMethod(lbaas.opts.LBMethod) - createOpt := v2pools.CreateOpts{ - Name: cutString(fmt.Sprintf("pool_%d_%s", portIndex, name)), - Protocol: poolProto, - LBMethod: lbmethod, - ListenerID: listener.ID, - Persistence: persistence, - } - - klog.V(4).Infof("Creating pool for listener %s using protocol %s", listener.ID, poolProto) - - pool, err = v2pools.Create(lbaas.lb, createOpt).Extract() - if err != nil { - return nil, fmt.Errorf("error creating pool for listener %s: %v", listener.ID, err) - } - provisioningStatus, err := waitLoadbalancerActiveProvisioningStatus(lbaas.lb, loadbalancer.ID) - if err != nil { - return nil, fmt.Errorf("timeout when waiting for loadbalancer to be ACTIVE after creating pool, current provisioning status %s", provisioningStatus) - } - - } - - klog.V(4).Infof("Pool created for listener %s: %s", listener.ID, pool.ID) - - members, err := openstackutil.GetMembersbyPool(lbaas.lb, pool.ID) - if err != nil && !cpoerrors.IsNotFound(err) { - return nil, fmt.Errorf("error getting pool members %s: %v", pool.ID, err) - } - for _, node := range nodes { - addr, err := nodeAddressForLB(node) - if err != nil { - if err == ErrNotFound { - // Node failure, do not create member - klog.Warningf("Failed to create LB pool member for node %s: %v", node.Name, err) - continue - } else { - return nil, fmt.Errorf("error getting address for node %s: %v", node.Name, err) - } - } - - if !memberExists(members, addr, int(port.NodePort)) { - klog.V(4).Infof("Creating member for pool %s", pool.ID) - _, err := v2pools.CreateMember(lbaas.lb, pool.ID, v2pools.CreateMemberOpts{ - Name: cutString(fmt.Sprintf("member_%d_%s_%s", portIndex, node.Name, name)), - ProtocolPort: int(port.NodePort), - Address: addr, - SubnetID: lbaas.opts.SubnetID, - }).Extract() - if err != nil { - return nil, fmt.Errorf("error creating LB pool member for node: %s, %v", node.Name, err) - } - - provisioningStatus, err := waitLoadbalancerActiveProvisioningStatus(lbaas.lb, loadbalancer.ID) - if err != nil { - return nil, fmt.Errorf("timeout when waiting for loadbalancer to be ACTIVE after creating member, current provisioning status %s", provisioningStatus) - } - } else { - // After all members have been processed, remaining members are deleted as obsolete. - members = popMember(members, addr, int(port.NodePort)) - } - - klog.V(4).Infof("Ensured pool %s has member for %s at %s", pool.ID, node.Name, addr) - } - - // Delete obsolete members for this pool - for _, member := range members { - klog.V(4).Infof("Deleting obsolete member %s for pool %s address %s", member.ID, pool.ID, member.Address) - err := v2pools.DeleteMember(lbaas.lb, pool.ID, member.ID).ExtractErr() - if err != nil && !cpoerrors.IsNotFound(err) { - return nil, fmt.Errorf("error deleting obsolete member %s for pool %s address %s: %v", member.ID, pool.ID, member.Address, err) - } - provisioningStatus, err := waitLoadbalancerActiveProvisioningStatus(lbaas.lb, loadbalancer.ID) - if err != nil { - return nil, fmt.Errorf("timeout when waiting for loadbalancer to be ACTIVE after deleting member, current provisioning status %s", provisioningStatus) - } - } - - monitorID := pool.MonitorID - enableHealthMonitor, err := getBoolFromServiceAnnotation(apiService, ServiceAnnotationLoadBalancerEnableHealthMonitor, lbaas.opts.CreateMonitor) - if err != nil { - return nil, err - } - if monitorID == "" && enableHealthMonitor { - klog.V(4).Infof("Creating monitor for pool %s", pool.ID) - monitorProtocol := string(port.Protocol) - if port.Protocol == corev1.ProtocolUDP { - monitorProtocol = "UDP-CONNECT" - } - monitor, err := v2monitors.Create(lbaas.lb, v2monitors.CreateOpts{ - Name: cutString(fmt.Sprintf("monitor_%d_%s)", portIndex, name)), - PoolID: pool.ID, - Type: monitorProtocol, - Delay: int(lbaas.opts.MonitorDelay.Duration.Seconds()), - Timeout: int(lbaas.opts.MonitorTimeout.Duration.Seconds()), - MaxRetries: int(lbaas.opts.MonitorMaxRetries), - }).Extract() - if err != nil { - return nil, fmt.Errorf("error creating LB pool healthmonitor: %v", err) - } - provisioningStatus, err := waitLoadbalancerActiveProvisioningStatus(lbaas.lb, loadbalancer.ID) - if err != nil { - return nil, fmt.Errorf("timeout when waiting for loadbalancer to be ACTIVE after creating monitor, current provisioning status %s", provisioningStatus) - } - monitorID = monitor.ID - } else if monitorID != "" && !enableHealthMonitor { - klog.Infof("Deleting health monitor %s for pool %s", monitorID, pool.ID) - err := v2monitors.Delete(lbaas.lb, monitorID).ExtractErr() - if err != nil { - return nil, fmt.Errorf("failed to delete health monitor %s for pool %s, error: %v", monitorID, pool.ID, err) - } - } - } - - // All remaining listeners are obsolete, delete - for _, listener := range oldListeners { - klog.V(4).Infof("Deleting obsolete listener %s:", listener.ID) - // get pool for listener - pool, err := openstackutil.GetPoolByListener(lbaas.lb, loadbalancer.ID, listener.ID) - if err != nil && err != openstackutil.ErrNotFound { - return nil, fmt.Errorf("error getting pool for obsolete listener %s: %v", listener.ID, err) - } - if pool != nil { - // get and delete monitor - monitorID := pool.MonitorID - if monitorID != "" { - klog.V(4).Infof("Deleting obsolete monitor %s for pool %s", monitorID, pool.ID) - err = v2monitors.Delete(lbaas.lb, monitorID).ExtractErr() - if err != nil && !cpoerrors.IsNotFound(err) { - return nil, fmt.Errorf("error deleting obsolete monitor %s for pool %s: %v", monitorID, pool.ID, err) - } - provisioningStatus, err := waitLoadbalancerActiveProvisioningStatus(lbaas.lb, loadbalancer.ID) - if err != nil { - return nil, fmt.Errorf("timeout when waiting for loadbalancer to be ACTIVE after deleting monitor, current provisioning status %s", provisioningStatus) - } - } - // get and delete pool members - members, err := openstackutil.GetMembersbyPool(lbaas.lb, pool.ID) - if err != nil && !cpoerrors.IsNotFound(err) { - return nil, fmt.Errorf("error getting members for pool %s: %v", pool.ID, err) - } - if members != nil { - for _, member := range members { - klog.V(4).Infof("Deleting obsolete member %s for pool %s address %s", member.ID, pool.ID, member.Address) - err := v2pools.DeleteMember(lbaas.lb, pool.ID, member.ID).ExtractErr() - if err != nil && !cpoerrors.IsNotFound(err) { - return nil, fmt.Errorf("error deleting obsolete member %s for pool %s address %s: %v", member.ID, pool.ID, member.Address, err) - } - provisioningStatus, err := waitLoadbalancerActiveProvisioningStatus(lbaas.lb, loadbalancer.ID) - if err != nil { - return nil, fmt.Errorf("timeout when waiting for loadbalancer to be ACTIVE after deleting member, current provisioning status %s", provisioningStatus) - } - } - } - klog.V(4).Infof("Deleting obsolete pool %s for listener %s", pool.ID, listener.ID) - // delete pool - err = v2pools.Delete(lbaas.lb, pool.ID).ExtractErr() - if err != nil && !cpoerrors.IsNotFound(err) { - return nil, fmt.Errorf("error deleting obsolete pool %s for listener %s: %v", pool.ID, listener.ID, err) - } - provisioningStatus, err := waitLoadbalancerActiveProvisioningStatus(lbaas.lb, loadbalancer.ID) - if err != nil { - return nil, fmt.Errorf("timeout when waiting for loadbalancer to be ACTIVE after deleting pool, current provisioning status %s", provisioningStatus) - } - } - // delete listener - err = listeners.Delete(lbaas.lb, listener.ID).ExtractErr() - if err != nil && !cpoerrors.IsNotFound(err) { - return nil, fmt.Errorf("error deleteting obsolete listener: %v", err) - } - provisioningStatus, err := waitLoadbalancerActiveProvisioningStatus(lbaas.lb, loadbalancer.ID) - if err != nil { - return nil, fmt.Errorf("timeout when waiting for loadbalancer to be ACTIVE after deleting listener, current provisioning status %s", provisioningStatus) - } - klog.V(2).Infof("Deleted obsolete listener: %s", listener.ID) - } - - // Priority of choosing VIP port floating IP: - // 1. The floating IP that is already attached to the VIP port. - // 2. Floating IP specified in Spec.LoadBalancerIP - // 3. Create a new one - var floatIP *floatingips.FloatingIP - if !internalAnnotation { - - // first attempt: fetch floating IP attached to load balancer's VIP port - portID := loadbalancer.VipPortID - floatIP, err = openstackutil.GetFloatingIPByPortID(lbaas.network, portID) - if err != nil { - return nil, fmt.Errorf("failed when getting floating IP for port %s: %v", portID, err) - } - - // second attempt: fetch floating IP specified in service Spec.LoadBalancerIP - // if found, assosiate floating IP with loadbalancer's VIP port - loadBalancerIP := apiService.Spec.LoadBalancerIP - if floatIP == nil && loadBalancerIP != "" { - - opts := floatingips.ListOpts{ - FloatingIP: loadBalancerIP, - } - existingIPs, err := openstackutil.GetFloatingIPs(lbaas.network, opts) - if err != nil { - return nil, fmt.Errorf("failed when trying to get existing floating IP %s, error: %v", loadBalancerIP, err) - } - - if len(existingIPs) > 0 { - floatingip := existingIPs[0] - if len(floatingip.PortID) == 0 { - floatUpdateOpts := floatingips.UpdateOpts{ - PortID: &portID, - } - floatIP, err = floatingips.Update(lbaas.network, floatingip.ID, floatUpdateOpts).Extract() - if err != nil { - return nil, fmt.Errorf("error updating LB floatingip %+v: %v", floatUpdateOpts, err) - } - } else { - return nil, fmt.Errorf("floating IP %s is not available", loadBalancerIP) - } - } - } - - // third attempt: create a new floating IP - if floatIP == nil { - if floatingNetworkID != "" { - klog.V(4).Infof("Creating floating IP %s for loadbalancer %s", loadBalancerIP, loadbalancer.ID) - floatIPOpts := floatingips.CreateOpts{ - FloatingNetworkID: floatingNetworkID, - PortID: portID, - Description: fmt.Sprintf("Floating IP for Kubernetes external service %s from cluster %s", serviceName, clusterName), - } - - if floatingSubnetID != "" { - floatIPOpts.SubnetID = floatingSubnetID - } - - if loadBalancerIP != "" { - floatIPOpts.FloatingIP = loadBalancerIP - } - - klog.V(4).Infof("creating floating ip with opts %+v", floatIPOpts) - floatIP, err = floatingips.Create(lbaas.network, floatIPOpts).Extract() - if err != nil { - return nil, fmt.Errorf("error creating LB floatingip %+v: %v", floatIPOpts, err) - } - } else { - klog.Warningf("Failed to find floating network information, for Service %s,"+ - "forcing to ensure an internal load balancer service", serviceName) - } - } - } - - status := &corev1.LoadBalancerStatus{} - - if floatIP != nil { - status.Ingress = []corev1.LoadBalancerIngress{{IP: floatIP.FloatingIP}} - } else { - status.Ingress = []corev1.LoadBalancerIngress{{IP: loadbalancer.VipAddress}} - } - - if lbaas.opts.ManageSecurityGroups { - err := lbaas.ensureSecurityGroup(clusterName, apiService, nodes, loadbalancer) - if err != nil { - return status, fmt.Errorf("failed when reconciling security groups for LB service %v/%v: %v", apiService.Namespace, apiService.Name, err) - } - } - - return status, nil -} - -func (lbaas *LbaasV2) getSubnet(subnet string) (*subnets.Subnet, error) { - if subnet == "" { - return nil, nil - } - - allPages, err := subnets.List(lbaas.network, subnets.ListOpts{Name: subnet}).AllPages() - if err != nil { - return nil, fmt.Errorf("error listing subnets: %v", err) - } - subs, err := subnets.ExtractSubnets(allPages) - if err != nil { - return nil, fmt.Errorf("error extracting subnets from pages: %v", err) - } - - if len(subs) == 0 { - return nil, fmt.Errorf("could not find subnet %s", subnet) - } - if len(subs) == 1 { - return &subs[0], nil - } - return nil, fmt.Errorf("find multiple subnets with name %s", subnet) -} - -// ensureSecurityGroup ensures security group exist for specific loadbalancer service. -// Creating security group for specific loadbalancer service when it does not exist. -func (lbaas *LbaasV2) ensureSecurityGroup(clusterName string, apiService *corev1.Service, nodes []*corev1.Node, loadbalancer *loadbalancers.LoadBalancer) error { - // find node-security-group for service - var err error - if len(lbaas.opts.NodeSecurityGroupIDs) == 0 && !lbaas.opts.UseOctavia { - lbaas.opts.NodeSecurityGroupIDs, err = getNodeSecurityGroupIDForLB(lbaas.compute, lbaas.network, nodes) - if err != nil { - return fmt.Errorf("failed to find node-security-group for loadbalancer service %s/%s: %v", apiService.Namespace, apiService.Name, err) - } - - klog.V(4).Infof("find node-security-group %v for loadbalancer service %s/%s", lbaas.opts.NodeSecurityGroupIDs, apiService.Namespace, apiService.Name) - } - - // get service ports - ports := apiService.Spec.Ports - if len(ports) == 0 { - return fmt.Errorf("no ports provided to openstack load balancer") - } - - // get service source ranges - sourceRanges, err := GetLoadBalancerSourceRanges(apiService) - if err != nil { - return fmt.Errorf("failed to get source ranges for loadbalancer service %s/%s: %v", apiService.Namespace, apiService.Name, err) - } - - // ensure security group for LB - lbSecGroupName := getSecurityGroupName(apiService) - lbSecGroupID, err := secgroups.IDFromName(lbaas.network, lbSecGroupName) - if err != nil { - // If the security group of LB not exist, create it later - if isSecurityGroupNotFound(err) { - lbSecGroupID = "" - } else { - return fmt.Errorf("error occurred finding security group: %s: %v", lbSecGroupName, err) - } - } - if len(lbSecGroupID) == 0 { - // create security group - lbSecGroupCreateOpts := groups.CreateOpts{ - Name: lbSecGroupName, - Description: fmt.Sprintf("Security Group for %s/%s Service LoadBalancer in cluster %s", apiService.Namespace, apiService.Name, clusterName), - } - - lbSecGroup, err := groups.Create(lbaas.network, lbSecGroupCreateOpts).Extract() - if err != nil { - return fmt.Errorf("failed to create Security Group for loadbalancer service %s/%s: %v", apiService.Namespace, apiService.Name, err) - } - lbSecGroupID = lbSecGroup.ID - - if !lbaas.opts.UseOctavia { - //add rule in security group - for _, port := range ports { - for _, sourceRange := range sourceRanges.StringSlice() { - ethertype := rules.EtherType4 - network, _, err := net.ParseCIDR(sourceRange) - - if err != nil { - return fmt.Errorf("error parsing source range %s as a CIDR: %v", sourceRange, err) - } - - if network.To4() == nil { - ethertype = rules.EtherType6 - } - - lbSecGroupRuleCreateOpts := rules.CreateOpts{ - Direction: rules.DirIngress, - PortRangeMax: int(port.Port), - PortRangeMin: int(port.Port), - Protocol: toRuleProtocol(port.Protocol), - RemoteIPPrefix: sourceRange, - SecGroupID: lbSecGroup.ID, - EtherType: ethertype, - } - - _, err = rules.Create(lbaas.network, lbSecGroupRuleCreateOpts).Extract() - - if err != nil { - return fmt.Errorf("error occurred creating rule for SecGroup %s: %v", lbSecGroup.ID, err) - } - } - } - - lbSecGroupRuleCreateOpts := rules.CreateOpts{ - Direction: rules.DirIngress, - PortRangeMax: 4, // ICMP: Code - Values for ICMP "Destination Unreachable: Fragmentation Needed and Don't Fragment was Set" - PortRangeMin: 3, // ICMP: Type - Protocol: rules.ProtocolICMP, - RemoteIPPrefix: "0.0.0.0/0", // The Fragmentation packet can come from anywhere along the path back to the sourceRange - we need to all this from all - SecGroupID: lbSecGroup.ID, - EtherType: rules.EtherType4, - } - - _, err = rules.Create(lbaas.network, lbSecGroupRuleCreateOpts).Extract() - - if err != nil { - return fmt.Errorf("error occurred creating rule for SecGroup %s: %v", lbSecGroup.ID, err) - } - - lbSecGroupRuleCreateOpts = rules.CreateOpts{ - Direction: rules.DirIngress, - PortRangeMax: 0, // ICMP: Code - Values for ICMP "Packet Too Big" - PortRangeMin: 2, // ICMP: Type - Protocol: rules.ProtocolICMP, - RemoteIPPrefix: "::/0", // The Fragmentation packet can come from anywhere along the path back to the sourceRange - we need to all this from all - SecGroupID: lbSecGroup.ID, - EtherType: rules.EtherType6, - } - - _, err = rules.Create(lbaas.network, lbSecGroupRuleCreateOpts).Extract() - if err != nil { - return fmt.Errorf("error occurred creating rule for SecGroup %s: %v", lbSecGroup.ID, err) - } - - // get security groups of port - portID := loadbalancer.VipPortID - port, err := getPortByID(lbaas.network, portID) - if err != nil { - return err - } - - // ensure the vip port has the security groups - found := false - for _, portSecurityGroups := range port.SecurityGroups { - if portSecurityGroups == lbSecGroup.ID { - found = true - break - } - } - - // update loadbalancer vip port - if !found { - port.SecurityGroups = append(port.SecurityGroups, lbSecGroup.ID) - updateOpts := neutronports.UpdateOpts{SecurityGroups: &port.SecurityGroups} - res := neutronports.Update(lbaas.network, portID, updateOpts) - if res.Err != nil { - msg := fmt.Sprintf("Error occurred updating port %s for loadbalancer service %s/%s: %v", portID, apiService.Namespace, apiService.Name, res.Err) - return fmt.Errorf(msg) - } - } - } - } - - // ensure rules for node security group - for _, port := range ports { - // If Octavia is used, the VIP port security group is already taken good care of, we only need to allow ingress - // traffic from Octavia amphorae to the node port on the worker nodes. - if lbaas.opts.UseOctavia { - subnet, err := subnets.Get(lbaas.network, lbaas.opts.SubnetID).Extract() - if err != nil { - return fmt.Errorf("failed to find subnet %s from openstack: %v", lbaas.opts.SubnetID, err) - } - - sgListopts := rules.ListOpts{ - Direction: string(rules.DirIngress), - Protocol: string(port.Protocol), - PortRangeMax: int(port.NodePort), - PortRangeMin: int(port.NodePort), - RemoteIPPrefix: subnet.CIDR, - SecGroupID: lbSecGroupID, - } - sgRules, err := getSecurityGroupRules(lbaas.network, sgListopts) - if err != nil && !cpoerrors.IsNotFound(err) { - return fmt.Errorf("failed to find security group rules in %s: %v", lbSecGroupID, err) - } - if len(sgRules) != 0 { - continue - } - - // The Octavia amphorae and worker nodes are supposed to be in the same subnet. We allow the ingress traffic - // from the amphorae to the specific node port on the nodes. - sgRuleCreateOpts := rules.CreateOpts{ - Direction: rules.DirIngress, - PortRangeMax: int(port.NodePort), - PortRangeMin: int(port.NodePort), - Protocol: toRuleProtocol(port.Protocol), - RemoteIPPrefix: subnet.CIDR, - SecGroupID: lbSecGroupID, - EtherType: rules.EtherType4, - } - if _, err = rules.Create(lbaas.network, sgRuleCreateOpts).Extract(); err != nil { - return fmt.Errorf("failed to create rule for security group %s: %v", lbSecGroupID, err) - } - - if err := applyNodeSecurityGroupIDForLB(lbaas.compute, lbaas.network, nodes, lbSecGroupID); err != nil { - return err - } - } else { - for _, nodeSecurityGroupID := range lbaas.opts.NodeSecurityGroupIDs { - opts := rules.ListOpts{ - Direction: string(rules.DirIngress), - SecGroupID: nodeSecurityGroupID, - RemoteGroupID: lbSecGroupID, - PortRangeMax: int(port.NodePort), - PortRangeMin: int(port.NodePort), - Protocol: string(port.Protocol), - } - secGroupRules, err := getSecurityGroupRules(lbaas.network, opts) - if err != nil && !cpoerrors.IsNotFound(err) { - msg := fmt.Sprintf("Error finding rules for remote group id %s in security group id %s: %v", lbSecGroupID, nodeSecurityGroupID, err) - return fmt.Errorf(msg) - } - if len(secGroupRules) != 0 { - // Do not add rule when find rules for remote group in the Node Security Group - continue - } - - // Add the rules in the Node Security Group - err = createNodeSecurityGroup(lbaas.network, nodeSecurityGroupID, int(port.NodePort), port.Protocol, lbSecGroupID) - if err != nil { - return fmt.Errorf("error occurred creating security group for loadbalancer service %s/%s: %v", apiService.Namespace, apiService.Name, err) - } - } - } - } - - return nil -} - -func (lbaas *LbaasV2) updateOctaviaLoadBalancer(ctx context.Context, clusterName string, service *corev1.Service, nodes []*corev1.Node) error { - svcConf := new(serviceConfig) - ports := service.Spec.Ports - if len(ports) == 0 { - return fmt.Errorf("no ports provided to openstack load balancer") - } - - serviceName := fmt.Sprintf("%s/%s", service.Namespace, service.Name) - klog.V(2).Infof("Updating %d nodes for Service %s in cluster %s", len(nodes), serviceName, clusterName) - - // Find subnet ID for creating members - if lbaas.opts.SubnetID != "" { - svcConf.lbMemberSubnetID = lbaas.opts.SubnetID - } else { - svcConf.configClassName = getStringFromServiceAnnotation(service, ServiceAnnotationLoadBalancerClass, "") - if svcConf.configClassName != "" { - lbClass := lbaas.opts.LBClasses[svcConf.configClassName] - if lbClass == nil { - return fmt.Errorf("invalid loadbalancer class %q", svcConf.configClassName) - } - - if lbClass.SubnetID != "" { - svcConf.lbMemberSubnetID = lbClass.SubnetID - } - } else { - svcConf.lbMemberSubnetID = getStringFromServiceAnnotation(service, ServiceAnnotationLoadBalancerSubnetID, lbaas.opts.SubnetID) - if len(svcConf.lbMemberSubnetID) == 0 && len(nodes) > 0 { - subnetID, err := getSubnetIDForLB(lbaas.compute, *nodes[0]) - if err != nil { - return fmt.Errorf("no subnet-id found for service %s: %v", serviceName, err) - } - svcConf.lbMemberSubnetID = subnetID - } - } - } - - // This affects the protocol of listener and pool - keepClientIP, err := getBoolFromServiceAnnotation(service, ServiceAnnotationLoadBalancerXForwardedFor, false) - if err != nil { - return err - } - useProxyProtocol, err := getBoolFromServiceAnnotation(service, ServiceAnnotationLoadBalancerProxyEnabled, false) - if err != nil { - return err - } - if useProxyProtocol && keepClientIP { - return fmt.Errorf("annotation %s and %s cannot be used together", ServiceAnnotationLoadBalancerProxyEnabled, ServiceAnnotationLoadBalancerXForwardedFor) - } - svcConf.keepClientIP = keepClientIP - svcConf.enableProxyProtocol = useProxyProtocol - - // Get load balancer - name := lbaas.GetLoadBalancerName(ctx, clusterName, service) - legacyName := lbaas.GetLoadBalancerLegacyName(ctx, clusterName, service) - loadbalancer, err := getLoadbalancerByName(lbaas.lb, name, legacyName) - if err != nil { - return err - } - if loadbalancer == nil { - return fmt.Errorf("loadbalancer does not exist for Service %s", serviceName) - } - - // Get all listeners for this loadbalancer, by "port&protocol". - lbListeners := make(map[listenerKey]listeners.Listener) - allListeners, err := getListenersByLoadBalancerID(lbaas.lb, loadbalancer.ID) - if err != nil { - return fmt.Errorf("error getting listeners for LB %s: %v", loadbalancer.ID, err) - } - for _, l := range allListeners { - key := listenerKey{Protocol: listeners.Protocol(l.Protocol), Port: l.ProtocolPort} - lbListeners[key] = l - } - - // Update pool members for each listener. - for _, port := range ports { - proto := toListenersProtocol(port.Protocol) - if svcConf.keepClientIP { - proto = listeners.ProtocolHTTP - } - listener, ok := lbListeners[listenerKey{ - Protocol: proto, - Port: int(port.Port), - }] - if !ok { - return fmt.Errorf("loadbalancer %s does not contain required listener for port %d and protocol %s", loadbalancer.ID, port.Port, port.Protocol) - } - - _, err := lbaas.ensureOctaviaPool(loadbalancer.ID, &listener, service, port, nodes, svcConf) - if err != nil { - return err - } - } - - return nil -} - -// UpdateLoadBalancer updates hosts under the specified load balancer. -func (lbaas *LbaasV2) UpdateLoadBalancer(ctx context.Context, clusterName string, service *corev1.Service, nodes []*corev1.Node) error { - if lbaas.opts.UseOctavia { - return lbaas.updateOctaviaLoadBalancer(ctx, clusterName, service, nodes) - } - - // Following code is just for legacy Neutron-LBaaS support which has been deprecated since OpenStack stable/queens - // and not recommended to use in production. No new features should be added. - - serviceName := fmt.Sprintf("%s/%s", service.Namespace, service.Name) - klog.V(4).Infof("UpdateLoadBalancer(%v, %s, %v)", clusterName, serviceName, nodes) - - lbaas.opts.SubnetID = getStringFromServiceAnnotation(service, ServiceAnnotationLoadBalancerSubnetID, lbaas.opts.SubnetID) - if len(lbaas.opts.SubnetID) == 0 && len(nodes) > 0 { - // Get SubnetID automatically. - // The LB needs to be configured with instance addresses on the same subnet, so get SubnetID by one node. - subnetID, err := getSubnetIDForLB(lbaas.compute, *nodes[0]) - if err != nil { - klog.Warningf("Failed to find subnet-id for loadbalancer service %s/%s: %v", service.Namespace, service.Name, err) - return fmt.Errorf("no subnet-id for service %s/%s : subnet-id not set in cloud provider config, "+ - "and failed to find subnet-id from OpenStack: %v", service.Namespace, service.Name, err) - } - lbaas.opts.SubnetID = subnetID - } - - ports := service.Spec.Ports - if len(ports) == 0 { - return fmt.Errorf("no ports provided to openstack load balancer") - } - - name := lbaas.GetLoadBalancerName(ctx, clusterName, service) - legacyName := lbaas.GetLoadBalancerLegacyName(ctx, clusterName, service) - loadbalancer, err := getLoadbalancerByName(lbaas.lb, name, legacyName) - if err != nil { - return err - } - if loadbalancer == nil { - return fmt.Errorf("loadbalancer does not exist for Service %s", serviceName) - } - - // Get all listeners for this loadbalancer, by "port key". - type portKey struct { - Protocol listeners.Protocol - Port int - } - var listenerIDs []string - lbListeners := make(map[portKey]listeners.Listener) - allListeners, err := getListenersByLoadBalancerID(lbaas.lb, loadbalancer.ID) - if err != nil { - return fmt.Errorf("error getting listeners for LB %s: %v", loadbalancer.ID, err) - } - for _, l := range allListeners { - key := portKey{Protocol: listeners.Protocol(l.Protocol), Port: l.ProtocolPort} - lbListeners[key] = l - listenerIDs = append(listenerIDs, l.ID) - } - - // Get all pools for this loadbalancer, by listener ID. - lbPools := make(map[string]v2pools.Pool) - for _, listenerID := range listenerIDs { - pool, err := openstackutil.GetPoolByListener(lbaas.lb, loadbalancer.ID, listenerID) - if err != nil { - return fmt.Errorf("error getting pool for listener %s: %v", listenerID, err) - } - lbPools[listenerID] = *pool - } - - // Compose Set of member (addresses) that _should_ exist - addrs := make(map[string]*corev1.Node) - for _, node := range nodes { - addr, err := nodeAddressForLB(node) - if err != nil { - return err - } - addrs[addr] = node - } - - // Check for adding/removing members associated with each port - for portIndex, port := range ports { - // Get listener associated with this port - listener, ok := lbListeners[portKey{ - Protocol: toListenersProtocol(port.Protocol), - Port: int(port.Port), - }] - if !ok { - return fmt.Errorf("loadbalancer %s does not contain required listener for port %d and protocol %s", loadbalancer.ID, port.Port, port.Protocol) - } - - // Get pool associated with this listener - pool, ok := lbPools[listener.ID] - if !ok { - return fmt.Errorf("loadbalancer %s does not contain required pool for listener %s", loadbalancer.ID, listener.ID) - } - - // Find existing pool members (by address) for this port - getMembers, err := openstackutil.GetMembersbyPool(lbaas.lb, pool.ID) - if err != nil { - return fmt.Errorf("error getting pool members %s: %v", pool.ID, err) - } - members := make(map[string]v2pools.Member) - for _, member := range getMembers { - members[member.Address] = member - } - - // Add any new members for this port - for addr, node := range addrs { - if _, ok := members[addr]; ok && members[addr].ProtocolPort == int(port.NodePort) { - // Already exists, do not create member - continue - } - _, err := v2pools.CreateMember(lbaas.lb, pool.ID, v2pools.CreateMemberOpts{ - Name: cutString(fmt.Sprintf("member_%d_%s_%s_", portIndex, node.Name, loadbalancer.Name)), - Address: addr, - ProtocolPort: int(port.NodePort), - SubnetID: lbaas.opts.SubnetID, - }).Extract() - if err != nil { - return err - } - provisioningStatus, err := waitLoadbalancerActiveProvisioningStatus(lbaas.lb, loadbalancer.ID) - if err != nil { - return fmt.Errorf("timeout when waiting for loadbalancer to be ACTIVE after creating member, current provisioning status %s", provisioningStatus) - } - } - - // Remove any old members for this port - for _, member := range members { - if _, ok := addrs[member.Address]; ok && member.ProtocolPort == int(port.NodePort) { - // Still present, do not delete member - continue - } - err = v2pools.DeleteMember(lbaas.lb, pool.ID, member.ID).ExtractErr() - if err != nil && !cpoerrors.IsNotFound(err) { - return err - } - provisioningStatus, err := waitLoadbalancerActiveProvisioningStatus(lbaas.lb, loadbalancer.ID) - if err != nil { - return fmt.Errorf("timeout when waiting for loadbalancer to be ACTIVE after deleting member, current provisioning status %s", provisioningStatus) - } - } - } - - if lbaas.opts.ManageSecurityGroups { - err := lbaas.updateSecurityGroup(clusterName, service, nodes, loadbalancer) - if err != nil { - return fmt.Errorf("failed to update Security Group for loadbalancer service %s: %v", serviceName, err) - } - } - - return nil -} - -// updateSecurityGroup updating security group for specific loadbalancer service. -func (lbaas *LbaasV2) updateSecurityGroup(clusterName string, apiService *corev1.Service, nodes []*corev1.Node, loadbalancer *loadbalancers.LoadBalancer) error { - originalNodeSecurityGroupIDs := lbaas.opts.NodeSecurityGroupIDs - - var err error - lbaas.opts.NodeSecurityGroupIDs, err = getNodeSecurityGroupIDForLB(lbaas.compute, lbaas.network, nodes) - if err != nil { - return fmt.Errorf("failed to find node-security-group for loadbalancer service %s/%s: %v", apiService.Namespace, apiService.Name, err) - } - klog.V(4).Infof("find node-security-group %v for loadbalancer service %s/%s", lbaas.opts.NodeSecurityGroupIDs, apiService.Namespace, apiService.Name) - - original := sets.NewString(originalNodeSecurityGroupIDs...) - current := sets.NewString(lbaas.opts.NodeSecurityGroupIDs...) - removals := original.Difference(current) - - // Generate Name - lbSecGroupName := getSecurityGroupName(apiService) - lbSecGroupID, err := secgroups.IDFromName(lbaas.network, lbSecGroupName) - if err != nil { - return fmt.Errorf("error occurred finding security group: %s: %v", lbSecGroupName, err) - } - - ports := apiService.Spec.Ports - if len(ports) == 0 { - return fmt.Errorf("no ports provided to openstack load balancer") - } - - for _, port := range ports { - for removal := range removals { - // Delete the rules in the Node Security Group - opts := rules.ListOpts{ - Direction: string(rules.DirIngress), - SecGroupID: removal, - RemoteGroupID: lbSecGroupID, - PortRangeMax: int(port.NodePort), - PortRangeMin: int(port.NodePort), - Protocol: string(port.Protocol), - } - secGroupRules, err := getSecurityGroupRules(lbaas.network, opts) - if err != nil && !cpoerrors.IsNotFound(err) { - return fmt.Errorf("error finding rules for remote group id %s in security group id %s: %v", lbSecGroupID, removal, err) - } - - for _, rule := range secGroupRules { - res := rules.Delete(lbaas.network, rule.ID) - if res.Err != nil && !cpoerrors.IsNotFound(res.Err) { - return fmt.Errorf("error occurred deleting security group rule: %s: %v", rule.ID, res.Err) - } - } - } - - for _, nodeSecurityGroupID := range lbaas.opts.NodeSecurityGroupIDs { - opts := rules.ListOpts{ - Direction: string(rules.DirIngress), - SecGroupID: nodeSecurityGroupID, - RemoteGroupID: lbSecGroupID, - PortRangeMax: int(port.NodePort), - PortRangeMin: int(port.NodePort), - Protocol: string(port.Protocol), - } - secGroupRules, err := getSecurityGroupRules(lbaas.network, opts) - if err != nil && !cpoerrors.IsNotFound(err) { - return fmt.Errorf("error finding rules for remote group id %s in security group id %s: %v", lbSecGroupID, nodeSecurityGroupID, err) - } - if len(secGroupRules) != 0 { - // Do not add rule when find rules for remote group in the Node Security Group - continue - } - - // Add the rules in the Node Security Group - err = createNodeSecurityGroup(lbaas.network, nodeSecurityGroupID, int(port.NodePort), port.Protocol, lbSecGroupID) - if err != nil { - return fmt.Errorf("error occurred creating security group for loadbalancer service %s/%s: %v", apiService.Namespace, apiService.Name, err) - } - } - } - - return nil -} - -// EnsureLoadBalancerDeleted deletes the specified load balancer -func (lbaas *LbaasV2) EnsureLoadBalancerDeleted(ctx context.Context, clusterName string, service *corev1.Service) error { - serviceName := fmt.Sprintf("%s/%s", service.Namespace, service.Name) - klog.V(4).Infof("EnsureLoadBalancerDeleted(%s, %s)", clusterName, serviceName) - - name := lbaas.GetLoadBalancerName(ctx, clusterName, service) - legacyName := lbaas.GetLoadBalancerLegacyName(ctx, clusterName, service) - loadbalancer, err := getLoadbalancerByName(lbaas.lb, name, legacyName) - if err != nil && err != ErrNotFound { - return err - } - if loadbalancer == nil { - return nil - } - - keepFloatingAnnotation, err := getBoolFromServiceAnnotation(service, ServiceAnnotationLoadBalancerKeepFloatingIP, false) - if err != nil { - return err - } - - if !keepFloatingAnnotation { - if loadbalancer.VipPortID != "" { - portID := loadbalancer.VipPortID - fip, err := openstackutil.GetFloatingIPByPortID(lbaas.network, portID) - if err != nil { - return fmt.Errorf("failed to get floating IP for loadbalancer VIP port %s: %v", portID, err) - } - - // Delete the floating IP only if it was created dynamically by the controller manager. - if fip != nil { - matched, err := regexp.Match("Floating IP for Kubernetes external service", []byte(fip.Description)) - if err != nil { - return err - } - if matched { - if err := floatingips.Delete(lbaas.network, fip.ID).ExtractErr(); err != nil { - return fmt.Errorf("failed to delete floating IP %s for loadbalancer VIP port %s: %v", fip.FloatingIP, portID, err) - } - } - } - } - } - - // delete the loadbalancer and all its sub-resources. - if lbaas.opts.UseOctavia && lbaas.opts.CascadeDelete { - deleteOpts := loadbalancers.DeleteOpts{Cascade: true} - if err := loadbalancers.Delete(lbaas.lb, loadbalancer.ID, deleteOpts).ExtractErr(); err != nil { - return fmt.Errorf("failed to delete loadbalancer %s: %v", loadbalancer.ID, err) - } - } else { - // get all listeners associated with this loadbalancer - listenerList, err := getListenersByLoadBalancerID(lbaas.lb, loadbalancer.ID) - if err != nil { - return fmt.Errorf("error getting LB %s listeners: %v", loadbalancer.ID, err) - } - - // get all pools (and health monitors) associated with this loadbalancer - var poolIDs []string - var monitorIDs []string - for _, listener := range listenerList { - pool, err := openstackutil.GetPoolByListener(lbaas.lb, loadbalancer.ID, listener.ID) - if err != nil && err != openstackutil.ErrNotFound { - return fmt.Errorf("error getting pool for listener %s: %v", listener.ID, err) - } - if pool != nil { - poolIDs = append(poolIDs, pool.ID) - // If create-monitor of cloud-config is false, pool has not monitor. - if pool.MonitorID != "" { - monitorIDs = append(monitorIDs, pool.MonitorID) - } - } - } - - // delete all monitors - for _, monitorID := range monitorIDs { - err := v2monitors.Delete(lbaas.lb, monitorID).ExtractErr() - if err != nil && !cpoerrors.IsNotFound(err) { - return err - } - provisioningStatus, err := waitLoadbalancerActiveProvisioningStatus(lbaas.lb, loadbalancer.ID) - if err != nil { - return fmt.Errorf("timeout when waiting for loadbalancer to be ACTIVE after deleting monitor, current provisioning status %s", provisioningStatus) - } - } - - // delete all members and pools - for _, poolID := range poolIDs { - // get members for current pool - membersList, err := openstackutil.GetMembersbyPool(lbaas.lb, poolID) - if err != nil && !cpoerrors.IsNotFound(err) { - return fmt.Errorf("error getting pool members %s: %v", poolID, err) - } - // delete all members for this pool - for _, member := range membersList { - err := v2pools.DeleteMember(lbaas.lb, poolID, member.ID).ExtractErr() - if err != nil && !cpoerrors.IsNotFound(err) { - return err - } - provisioningStatus, err := waitLoadbalancerActiveProvisioningStatus(lbaas.lb, loadbalancer.ID) - if err != nil { - return fmt.Errorf("timeout when waiting for loadbalancer to be ACTIVE after deleting member, current provisioning status %s", provisioningStatus) - } - } - - // delete pool - err = v2pools.Delete(lbaas.lb, poolID).ExtractErr() - if err != nil && !cpoerrors.IsNotFound(err) { - return err - } - provisioningStatus, err := waitLoadbalancerActiveProvisioningStatus(lbaas.lb, loadbalancer.ID) - if err != nil { - return fmt.Errorf("timeout when waiting for loadbalancer to be ACTIVE after deleting pool, current provisioning status %s", provisioningStatus) - } - } - - // delete all listeners - for _, listener := range listenerList { - err := listeners.Delete(lbaas.lb, listener.ID).ExtractErr() - if err != nil && !cpoerrors.IsNotFound(err) { - return err - } - provisioningStatus, err := waitLoadbalancerActiveProvisioningStatus(lbaas.lb, loadbalancer.ID) - if err != nil { - return fmt.Errorf("timeout when waiting for loadbalancer to be ACTIVE after deleting listener, current provisioning status %s", provisioningStatus) - } - } - - // delete loadbalancer - err = loadbalancers.Delete(lbaas.lb, loadbalancer.ID, loadbalancers.DeleteOpts{}).ExtractErr() - if err != nil && !cpoerrors.IsNotFound(err) { - return err - } - err = waitLoadbalancerDeleted(lbaas.lb, loadbalancer.ID) - if err != nil { - return fmt.Errorf("failed to delete loadbalancer: %v", err) - } - } - - // Delete the Security Group - if lbaas.opts.ManageSecurityGroups { - err := lbaas.EnsureSecurityGroupDeleted(clusterName, service) - if err != nil { - return fmt.Errorf("failed to delete Security Group for loadbalancer service %s: %v", serviceName, err) - } - } - - return nil -} - -// EnsureSecurityGroupDeleted deleting security group for specific loadbalancer service. -func (lbaas *LbaasV2) EnsureSecurityGroupDeleted(clusterName string, service *corev1.Service) error { - // Generate Name - lbSecGroupName := getSecurityGroupName(service) - lbSecGroupID, err := secgroups.IDFromName(lbaas.network, lbSecGroupName) - if err != nil { - if isSecurityGroupNotFound(err) { - // It is OK when the security group has been deleted by others. - return nil - } - return fmt.Errorf("error occurred finding security group: %s: %v", lbSecGroupName, err) - } - - if lbaas.opts.UseOctavia { - // Disassociate the security group from the neutron ports on the nodes. - if err := disassociateSecurityGroupForLB(lbaas.network, lbSecGroupID); err != nil { - return fmt.Errorf("failed to disassociate security group %s: %v", lbSecGroupID, err) - } - } - - lbSecGroup := groups.Delete(lbaas.network, lbSecGroupID) - if lbSecGroup.Err != nil && !cpoerrors.IsNotFound(lbSecGroup.Err) { - return lbSecGroup.Err - } - - if len(lbaas.opts.NodeSecurityGroupIDs) == 0 { - // Just happen when nodes have not Security Group, or should not happen - // UpdateLoadBalancer and EnsureLoadBalancer can set lbaas.opts.NodeSecurityGroupIDs when it is empty - // And service controller call UpdateLoadBalancer to set lbaas.opts.NodeSecurityGroupIDs when controller manager service is restarted. - klog.Warningf("Can not find node-security-group from all the nodes of this cluster when delete loadbalancer service %s/%s", - service.Namespace, service.Name) - } else { - // Delete the rules in the Node Security Group - for _, nodeSecurityGroupID := range lbaas.opts.NodeSecurityGroupIDs { - opts := rules.ListOpts{ - SecGroupID: nodeSecurityGroupID, - RemoteGroupID: lbSecGroupID, - } - secGroupRules, err := getSecurityGroupRules(lbaas.network, opts) - - if err != nil && !cpoerrors.IsNotFound(err) { - msg := fmt.Sprintf("error finding rules for remote group id %s in security group id %s: %v", lbSecGroupID, nodeSecurityGroupID, err) - return fmt.Errorf(msg) - } - - for _, rule := range secGroupRules { - res := rules.Delete(lbaas.network, rule.ID) - if res.Err != nil && !cpoerrors.IsNotFound(res.Err) { - return fmt.Errorf("error occurred deleting security group rule: %s: %v", rule.ID, res.Err) - } - } - } - } - - return nil -} - -// IsAllowAll checks whether the netsets.IPNet allows traffic from 0.0.0.0/0 -func IsAllowAll(ipnets netsets.IPNet) bool { - for _, s := range ipnets.StringSlice() { - if s == "0.0.0.0/0" { - return true - } - } - return false -} - -// GetLoadBalancerSourceRanges first try to parse and verify LoadBalancerSourceRanges field from a service. -// If the field is not specified, turn to parse and verify the AnnotationLoadBalancerSourceRangesKey annotation from a service, -// extracting the source ranges to allow, and if not present returns a default (allow-all) value. -func GetLoadBalancerSourceRanges(service *corev1.Service) (netsets.IPNet, error) { - var ipnets netsets.IPNet - var err error - // if SourceRange field is specified, ignore sourceRange annotation - if len(service.Spec.LoadBalancerSourceRanges) > 0 { - specs := service.Spec.LoadBalancerSourceRanges - ipnets, err = netsets.ParseIPNets(specs...) - - if err != nil { - return nil, fmt.Errorf("service.Spec.LoadBalancerSourceRanges: %v is not valid. Expecting a list of IP ranges. For example, 10.0.0.0/24. Error msg: %v", specs, err) - } - } else { - val := service.Annotations[corev1.AnnotationLoadBalancerSourceRangesKey] - val = strings.TrimSpace(val) - if val == "" { - val = defaultLoadBalancerSourceRanges - } - specs := strings.Split(val, ",") - ipnets, err = netsets.ParseIPNets(specs...) - if err != nil { - return nil, fmt.Errorf("%s: %s is not valid. Expecting a comma-separated list of source IP ranges. For example, 10.0.0.0/24,192.168.2.0/24", corev1.AnnotationLoadBalancerSourceRangesKey, val) - } - } - return ipnets, nil -} diff --git a/pkg/cloudprovider/providers/openstack/openstack_metrics.go b/pkg/cloudprovider/providers/openstack/openstack_metrics.go deleted file mode 100644 index e483875d46..0000000000 --- a/pkg/cloudprovider/providers/openstack/openstack_metrics.go +++ /dev/null @@ -1,57 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package openstack - -import ( - "github.com/prometheus/client_golang/prometheus" - "k8s.io/klog/v2" -) - -const ( - openstackSubsystem = "openstack" - openstackOperationKey = "cloudprovider_openstack_api_request_duration_seconds" - openstackOperationErrorKey = "cloudprovider_openstack_api_request_errors" -) - -var ( - openstackOperationsLatency = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Subsystem: openstackSubsystem, - Name: openstackOperationKey, - Help: "Latency of openstack api call", - }, - []string{"request"}, - ) - - openstackAPIRequestErrors = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Subsystem: openstackSubsystem, - Name: openstackOperationErrorKey, - Help: "Cumulative number of openstack Api call errors", - }, - []string{"request"}, - ) -) - -func RegisterMetrics() { - if err := prometheus.Register(openstackOperationsLatency); err != nil { - klog.V(5).Infof("unable to register for latency metrics") - } - if err := prometheus.Register(openstackAPIRequestErrors); err != nil { - klog.V(5).Infof("unable to register for error metrics") - } -} diff --git a/pkg/cloudprovider/providers/openstack/openstack_routes.go b/pkg/cloudprovider/providers/openstack/openstack_routes.go deleted file mode 100644 index a566c5487f..0000000000 --- a/pkg/cloudprovider/providers/openstack/openstack_routes.go +++ /dev/null @@ -1,348 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package openstack - -import ( - "context" - "errors" - "net" - - "github.com/gophercloud/gophercloud" - "github.com/gophercloud/gophercloud/openstack/compute/v2/servers" - "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/routers" - neutronports "github.com/gophercloud/gophercloud/openstack/networking/v2/ports" - - "k8s.io/apimachinery/pkg/types" - cloudprovider "k8s.io/cloud-provider" - "k8s.io/klog/v2" -) - -var errNoRouterID = errors.New("router-id not set in cloud provider config") - -// Routes implements the cloudprovider.Routes for OpenStack clouds -type Routes struct { - compute *gophercloud.ServiceClient - network *gophercloud.ServiceClient - opts RouterOpts - networkingOpts NetworkingOpts -} - -// NewRoutes creates a new instance of Routes -func NewRoutes(compute *gophercloud.ServiceClient, network *gophercloud.ServiceClient, opts RouterOpts, networkingOpts NetworkingOpts) (cloudprovider.Routes, error) { - if opts.RouterID == "" { - return nil, errNoRouterID - } - - return &Routes{ - compute: compute, - network: network, - opts: opts, - networkingOpts: networkingOpts, - }, nil -} - -// ListRoutes lists all managed routes that belong to the specified clusterName -func (r *Routes) ListRoutes(ctx context.Context, clusterName string) ([]*cloudprovider.Route, error) { - klog.V(4).Infof("ListRoutes(%v)", clusterName) - - nodeNamesByAddr := make(map[string]types.NodeName) - err := foreachServer(r.compute, servers.ListOpts{}, func(srv *servers.Server) (bool, error) { - interfaces, err := getAttachedInterfacesByID(r.compute, srv.ID) - if err != nil { - return false, err - } - - addrs, err := nodeAddresses(srv, interfaces, r.networkingOpts) - if err != nil { - return false, err - } - - name := mapServerToNodeName(srv) - for _, addr := range addrs { - nodeNamesByAddr[addr.Address] = name - } - - return true, nil - }) - if err != nil { - return nil, err - } - - router, err := routers.Get(r.network, r.opts.RouterID).Extract() - if err != nil { - return nil, err - } - - var routes []*cloudprovider.Route - for _, item := range router.Routes { - nodeName, foundNode := nodeNamesByAddr[item.NextHop] - if !foundNode { - nodeName = types.NodeName(item.NextHop) - } - route := cloudprovider.Route{ - Name: item.DestinationCIDR, - TargetNode: nodeName, //contains the nexthop address if node was not found - Blackhole: !foundNode, - DestinationCIDR: item.DestinationCIDR, - } - routes = append(routes, &route) - } - - return routes, nil -} - -func updateRoutes(network *gophercloud.ServiceClient, router *routers.Router, newRoutes []routers.Route) (func(), error) { - origRoutes := router.Routes // shallow copy - - _, err := routers.Update(network, router.ID, routers.UpdateOpts{ - Routes: newRoutes, - }).Extract() - if err != nil { - return nil, err - } - - unwinder := func() { - klog.V(4).Infof("Reverting routes change to router %v", router.ID) - _, err := routers.Update(network, router.ID, routers.UpdateOpts{ - Routes: origRoutes, - }).Extract() - if err != nil { - klog.Warningf("Unable to reset routes during error unwind: %v", err) - } - } - - return unwinder, nil -} - -func updateAllowedAddressPairs(network *gophercloud.ServiceClient, port *neutronports.Port, newPairs []neutronports.AddressPair) (func(), error) { - origPairs := port.AllowedAddressPairs // shallow copy - - _, err := neutronports.Update(network, port.ID, neutronports.UpdateOpts{ - AllowedAddressPairs: &newPairs, - }).Extract() - if err != nil { - return nil, err - } - - unwinder := func() { - klog.V(4).Infof("Reverting allowed-address-pairs change to port %v", port.ID) - _, err := neutronports.Update(network, port.ID, neutronports.UpdateOpts{ - AllowedAddressPairs: &origPairs, - }).Extract() - if err != nil { - klog.Warningf("Unable to reset allowed-address-pairs during error unwind: %v", err) - } - } - - return unwinder, nil -} - -// CreateRoute creates the described managed route -func (r *Routes) CreateRoute(ctx context.Context, clusterName string, nameHint string, route *cloudprovider.Route) error { - klog.V(4).Infof("CreateRoute(%v, %v, %v)", clusterName, nameHint, route) - - onFailure := newCaller() - - ip, _, _ := net.ParseCIDR(route.DestinationCIDR) - isCIDRv6 := ip.To4() == nil - addr, err := getAddressByName(r.compute, route.TargetNode, isCIDRv6, r.networkingOpts) - - if err != nil { - return err - } - - klog.V(4).Infof("Using nexthop %v for node %v", addr, route.TargetNode) - - router, err := routers.Get(r.network, r.opts.RouterID).Extract() - if err != nil { - return err - } - - routes := router.Routes - - for _, item := range routes { - if item.DestinationCIDR == route.DestinationCIDR && item.NextHop == addr { - klog.V(4).Infof("Skipping existing route: %v", route) - return nil - } - } - - routes = append(routes, routers.Route{ - DestinationCIDR: route.DestinationCIDR, - NextHop: addr, - }) - - unwind, err := updateRoutes(r.network, router, routes) - if err != nil { - return err - } - defer onFailure.call(unwind) - - // get the port of addr on target node. - portID, err := getPortIDByIP(r.compute, route.TargetNode, addr) - if err != nil { - return err - } - port, err := getPortByID(r.network, portID) - if err != nil { - return err - } - - found := false - for _, item := range port.AllowedAddressPairs { - if item.IPAddress == route.DestinationCIDR { - klog.V(4).Infof("Found existing allowed-address-pair: %v", item) - found = true - break - } - } - - if !found { - newPairs := append(port.AllowedAddressPairs, neutronports.AddressPair{ - IPAddress: route.DestinationCIDR, - }) - unwind, err := updateAllowedAddressPairs(r.network, port, newPairs) - if err != nil { - return err - } - defer onFailure.call(unwind) - } - - klog.V(4).Infof("Route created: %v", route) - onFailure.disarm() - return nil -} - -// DeleteRoute deletes the specified managed route -func (r *Routes) DeleteRoute(ctx context.Context, clusterName string, route *cloudprovider.Route) error { - klog.V(4).Infof("DeleteRoute(%v, %v)", clusterName, route) - - onFailure := newCaller() - - ip, _, _ := net.ParseCIDR(route.DestinationCIDR) - isCIDRv6 := ip.To4() == nil - var addr string - - // Blackhole routes are orphaned and have no counterpart in OpenStack - if !route.Blackhole { - var err error - addr, err = getAddressByName(r.compute, route.TargetNode, isCIDRv6, r.networkingOpts) - if err != nil { - return err - } - } - - router, err := routers.Get(r.network, r.opts.RouterID).Extract() - if err != nil { - return err - } - - routes := router.Routes - index := -1 - for i, item := range routes { - if item.DestinationCIDR == route.DestinationCIDR && (item.NextHop == addr || route.Blackhole && item.NextHop == string(route.TargetNode)) { - index = i - break - } - } - - if index == -1 { - klog.V(4).Infof("Skipping non-existent route: %v", route) - return nil - } - - // Delete element `index` - routes[index] = routes[len(routes)-1] - routes = routes[:len(routes)-1] - - unwind, err := updateRoutes(r.network, router, routes) - // If this was a blackhole route we are done, there are no ports to update - if err != nil || route.Blackhole { - return err - } - defer onFailure.call(unwind) - - // get the port of addr on target node. - portID, err := getPortIDByIP(r.compute, route.TargetNode, addr) - if err != nil { - return err - } - port, err := getPortByID(r.network, portID) - if err != nil { - return err - } - - addrPairs := port.AllowedAddressPairs - index = -1 - for i, item := range addrPairs { - if item.IPAddress == route.DestinationCIDR { - index = i - break - } - } - - if index != -1 { - // Delete element `index` - addrPairs[index] = addrPairs[len(addrPairs)-1] - addrPairs = addrPairs[:len(addrPairs)-1] - - unwind, err := updateAllowedAddressPairs(r.network, port, addrPairs) - if err != nil { - return err - } - defer onFailure.call(unwind) - } - - klog.V(4).Infof("Route deleted: %v", route) - onFailure.disarm() - return nil -} - -func getPortIDByIP(compute *gophercloud.ServiceClient, targetNode types.NodeName, ipAddress string) (string, error) { - srv, err := getServerByName(compute, targetNode) - if err != nil { - return "", err - } - - interfaces, err := getAttachedInterfacesByID(compute, srv.ID) - if err != nil { - return "", err - } - - for _, intf := range interfaces { - for _, fixedIP := range intf.FixedIPs { - if fixedIP.IPAddress == ipAddress { - return intf.PortID, nil - } - } - } - - return "", ErrNotFound -} - -func getPortByID(client *gophercloud.ServiceClient, portID string) (*neutronports.Port, error) { - targetPort, err := neutronports.Get(client, portID).Extract() - if err != nil { - return nil, err - } - - if targetPort == nil { - return nil, ErrNotFound - } - - return targetPort, nil -} diff --git a/pkg/cloudprovider/providers/openstack/openstack_routes_test.go b/pkg/cloudprovider/providers/openstack/openstack_routes_test.go deleted file mode 100644 index 6da1b14157..0000000000 --- a/pkg/cloudprovider/providers/openstack/openstack_routes_test.go +++ /dev/null @@ -1,123 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package openstack - -import ( - "context" - "net" - "testing" - - "github.com/gophercloud/gophercloud/openstack/compute/v2/servers" - "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/routers" - "k8s.io/apimachinery/pkg/types" - cloudprovider "k8s.io/cloud-provider" -) - -func TestRoutes(t *testing.T) { - const clusterName = "ignored" - - cfg := ConfigFromEnv() - testConfigFromEnv(t, &cfg) - - os, err := NewOpenStack(cfg) - if err != nil { - t.Fatalf("Failed to construct/authenticate OpenStack: %s", err) - } - - vms := getServers(os) - _, err = os.InstanceID() - if err != nil || len(vms) == 0 { - t.Skipf("Please run this test in an OpenStack vm or create at least one VM in OpenStack before you run this test.") - } - - // We know we have at least one vm. - servername := vms[0].Name - - // Pick the first router and server to try a test with - os.routeOpts.RouterID = getRouters(os)[0].ID - - r, ok := os.Routes() - if !ok { - t.Skip("Routes() returned false - perhaps your stack does not support Neutron extraroute extension?") - } - - newroute := cloudprovider.Route{ - DestinationCIDR: "10.164.2.0/24", - TargetNode: types.NodeName(servername), - } - err = r.CreateRoute(context.TODO(), clusterName, "myhint", &newroute) - if err != nil { - t.Fatalf("CreateRoute error: %v", err) - } - - routelist, err := r.ListRoutes(context.TODO(), clusterName) - if err != nil { - t.Fatalf("ListRoutes() error: %v", err) - } - for _, route := range routelist { - _, cidr, err := net.ParseCIDR(route.DestinationCIDR) - if err != nil { - t.Logf("Ignoring route %s, unparsable CIDR: %v", route.Name, err) - continue - } - t.Logf("%s via %s", cidr, route.TargetNode) - } - - err = r.DeleteRoute(context.TODO(), clusterName, &newroute) - if err != nil { - t.Fatalf("DeleteRoute error: %v", err) - } -} - -func getServers(os *OpenStack) []servers.Server { - c, err := os.NewComputeV2() - opts := servers.ListOpts{ - Status: "ACTIVE", - } - allPages, err := servers.List(c, opts).AllPages() - if err != nil { - panic(err) - } - allServers, err := servers.ExtractServers(allPages) - if err != nil { - panic(err) - } - if len(allServers) == 0 { - panic("No servers to test with") - } - return allServers -} - -func getRouters(os *OpenStack) []routers.Router { - listOpts := routers.ListOpts{} - n, err := os.NewNetworkV2() - if err != nil { - panic(err) - } - allPages, err := routers.List(n, listOpts).AllPages() - if err != nil { - panic(err) - } - allRouters, err := routers.ExtractRouters(allPages) - if err != nil { - panic(err) - } - if len(allRouters) == 0 { - panic("No routers to test with") - } - return allRouters -} diff --git a/pkg/cloudprovider/providers/openstack/openstack_test.go b/pkg/cloudprovider/providers/openstack/openstack_test.go deleted file mode 100644 index 3bf90509f5..0000000000 --- a/pkg/cloudprovider/providers/openstack/openstack_test.go +++ /dev/null @@ -1,1198 +0,0 @@ -/* -Copyright 2014 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package openstack - -import ( - "context" - "fmt" - "io/ioutil" - "os" - "path/filepath" - "reflect" - "regexp" - "strings" - "testing" - "time" - - "github.com/gophercloud/gophercloud" - "github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/attachinterfaces" - "github.com/gophercloud/gophercloud/openstack/compute/v2/servers" - "github.com/spf13/pflag" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "k8s.io/cloud-provider-openstack/pkg/util/metadata" -) - -const ( - testClusterName = "testCluster" - - // volumeStatus* is configuration of exponential backoff for - // waiting for specified volume status. Starting with 1 - // seconds, multiplying by 1.2 with each step and taking 13 steps at maximum - // it will time out after 32s, which roughly corresponds to 30s - volumeStatusInitDelay = 1 * time.Second - volumeStatusFactor = 1.2 - volumeStatusSteps = 13 -) - -// ConfigFromEnv allows setting up credentials etc using the -// standard OS_* OpenStack client environment variables. USed only in tests -func ConfigFromEnv() Config { - var cfg Config - - cfg.Global.AuthURL = os.Getenv("OS_AUTH_URL") - cfg.Global.UserID = os.Getenv("OS_USER_ID") - cfg.Global.Username = os.Getenv("OS_USERNAME") - cfg.Global.Password = os.Getenv("OS_PASSWORD") - - cfg.Global.TenantID = os.Getenv("OS_TENANT_ID") - if cfg.Global.TenantID == "" { - cfg.Global.TenantID = os.Getenv("OS_PROJECT_ID") - } - cfg.Global.TenantName = os.Getenv("OS_TENANT_NAME") - if cfg.Global.TenantName == "" { - cfg.Global.TenantName = os.Getenv("OS_PROJECT_NAME") - } - - cfg.Global.TrustID = os.Getenv("OS_TRUST_ID") - cfg.Global.DomainID = os.Getenv("OS_DOMAIN_ID") - cfg.Global.DomainName = os.Getenv("OS_DOMAIN_NAME") - cfg.Global.TenantDomainID = os.Getenv("OS_PROJECT_DOMAIN_ID") - cfg.Global.TenantDomainName = os.Getenv("OS_PROJECT_DOMAIN_NAME") - cfg.Global.UserDomainID = os.Getenv("OS_USER_DOMAIN_ID") - cfg.Global.UserDomainName = os.Getenv("OS_USER_DOMAIN_NAME") - cfg.Global.Region = os.Getenv("OS_REGION_NAME") - cfg.Global.ApplicationCredentialID = os.Getenv("OS_APPLICATION_CREDENTIAL_ID") - cfg.Global.ApplicationCredentialName = os.Getenv("OS_APPLICATION_CREDENTIAL_NAME") - cfg.Global.ApplicationCredentialSecret = os.Getenv("OS_APPLICATION_CREDENTIAL_SECRET") - - // Set default values for config params - cfg.BlockStorage.BSVersion = "auto" - cfg.BlockStorage.TrustDevicePath = false - cfg.BlockStorage.IgnoreVolumeAZ = false - cfg.Metadata.SearchOrder = fmt.Sprintf("%s,%s", metadata.ConfigDriveID, metadata.MetadataID) - cfg.Networking.IPv6SupportDisabled = false - cfg.Networking.PublicNetworkName = []string{"public"} - cfg.LoadBalancer.InternalLB = false - - return cfg -} - -func TestReadConfig(t *testing.T) { - _, err := ReadConfig(nil) - if err == nil { - t.Errorf("Should fail when no config is provided: %s", err) - } - - cfg, err := ReadConfig(strings.NewReader(` - [Global] - auth-url = http://auth.url - user-id = user - password = mypass - tenant-name = demo - tenant-domain-name = Default - region = RegionOne - [LoadBalancer] - create-monitor = yes - monitor-delay = 1m - monitor-timeout = 30s - monitor-max-retries = 3 - [BlockStorage] - bs-version = auto - trust-device-path = yes - ignore-volume-az = yes - [Metadata] - search-order = configDrive, metadataService - `)) - if err != nil { - t.Fatalf("Should succeed when a valid config is provided: %s", err) - } - if cfg.Global.AuthURL != "http://auth.url" { - t.Errorf("incorrect authurl: %s", cfg.Global.AuthURL) - } - - if cfg.Global.UserID != "user" { - t.Errorf("incorrect userid: %s", cfg.Global.UserID) - } - - if cfg.Global.Password != "mypass" { - t.Errorf("incorrect password: %s", cfg.Global.Password) - } - - // config file wins over environment variable - if cfg.Global.TenantName != "demo" { - t.Errorf("incorrect tenant name: %s", cfg.Global.TenantName) - } - - if cfg.Global.TenantDomainName != "Default" { - t.Errorf("incorrect tenant domain name: %s", cfg.Global.TenantDomainName) - } - - if cfg.Global.Region != "RegionOne" { - t.Errorf("incorrect region: %s", cfg.Global.Region) - } - - if !cfg.LoadBalancer.CreateMonitor { - t.Errorf("incorrect lb.createmonitor: %t", cfg.LoadBalancer.CreateMonitor) - } - if cfg.LoadBalancer.MonitorDelay.Duration != 1*time.Minute { - t.Errorf("incorrect lb.monitordelay: %s", cfg.LoadBalancer.MonitorDelay) - } - if cfg.LoadBalancer.MonitorTimeout.Duration != 30*time.Second { - t.Errorf("incorrect lb.monitortimeout: %s", cfg.LoadBalancer.MonitorTimeout) - } - if cfg.LoadBalancer.MonitorMaxRetries != 3 { - t.Errorf("incorrect lb.monitormaxretries: %d", cfg.LoadBalancer.MonitorMaxRetries) - } - if cfg.BlockStorage.TrustDevicePath != true { - t.Errorf("incorrect bs.trustdevicepath: %v", cfg.BlockStorage.TrustDevicePath) - } - if cfg.BlockStorage.BSVersion != "auto" { - t.Errorf("incorrect bs.bs-version: %v", cfg.BlockStorage.BSVersion) - } - if cfg.BlockStorage.IgnoreVolumeAZ != true { - t.Errorf("incorrect bs.IgnoreVolumeAZ: %v", cfg.BlockStorage.IgnoreVolumeAZ) - } - if cfg.Metadata.SearchOrder != "configDrive, metadataService" { - t.Errorf("incorrect md.search-order: %v", cfg.Metadata.SearchOrder) - } -} - -func TestReadClouds(t *testing.T) { - - dir, err := filepath.Abs(filepath.Dir(os.Args[0])) - if err != nil { - t.Error(err) - } - - cloudFile := dir + "/test_clouds.yaml" - _ = os.Remove(cloudFile) - - var cloud = ` -clouds: - default: - auth: - domain_name: default - auth_url: http://not-auth.url - project_name: demo - username: admin - user_id: user - region_name: RegionOne - identity_api_version: 3 -` - data := []byte(cloud) - err = ioutil.WriteFile(cloudFile, data, 0644) - if err != nil { - t.Error(err) - } - defer os.Remove(cloudFile) - - cfg, err := ReadConfig(strings.NewReader(` - [Global] - auth-url = http://auth.url - trust-id = mytrust - password = mypass - use-clouds = true - clouds-file = ` + cloudFile + ` - [LoadBalancer] - create-monitor = yes - monitor-delay = 1m - monitor-timeout = 30s - monitor-max-retries = 3 - [BlockStorage] - bs-version = auto - trust-device-path = yes - ignore-volume-az = yes - [Metadata] - search-order = configDrive, metadataService -`)) - - if err != nil { - t.Fatalf("Should succeed when a valid config is provided: %s", err) - } - - // config has priority - if cfg.Global.AuthURL != "http://auth.url" { - t.Errorf("incorrect IdentityEndpoint: %s", cfg.Global.AuthURL) - } - - if cfg.Global.UserID != "user" { - t.Errorf("incorrect user-id: %s", cfg.Global.UserID) - } - - if cfg.Global.Username != "admin" { - t.Errorf("incorrect user-name: %s", cfg.Global.Username) - } - - if cfg.Global.Password != "mypass" { - t.Errorf("incorrect password: %s", cfg.Global.Password) - } - - if cfg.Global.Region != "RegionOne" { - t.Errorf("incorrect region: %s", cfg.Global.Region) - } - - if cfg.Global.TenantName != "demo" { - t.Errorf("incorrect tenant name: %s", cfg.Global.TenantName) - } - - if cfg.Global.TrustID != "mytrust" { - t.Errorf("incorrect tenant name: %s", cfg.Global.TrustID) - } - - // Make non-global sections dont get overwritten - if cfg.BlockStorage.TrustDevicePath != true { - t.Errorf("incorrect bs.trustdevicepath: %v", cfg.BlockStorage.TrustDevicePath) - } - if cfg.BlockStorage.BSVersion != "auto" { - t.Errorf("incorrect bs.bs-version: %v", cfg.BlockStorage.BSVersion) - } - if !cfg.LoadBalancer.CreateMonitor { - t.Errorf("incorrect lb.createmonitor: %t", cfg.LoadBalancer.CreateMonitor) - } -} - -func TestToAuthOptions(t *testing.T) { - cfg := Config{} - cfg.Global.Username = "user" - cfg.Global.Password = "pass" - cfg.Global.DomainName = "local" - cfg.Global.AuthURL = "http://auth.url" - cfg.Global.UserID = "user" - - ao := cfg.Global.ToAuthOptions() - - if !ao.AllowReauth { - t.Errorf("Will need to be able to reauthenticate") - } - if ao.Username != cfg.Global.Username { - t.Errorf("Username %s != %s", ao.Username, cfg.Global.Username) - } - if ao.Password != cfg.Global.Password { - t.Errorf("Password %s != %s", ao.Password, cfg.Global.Password) - } - if ao.IdentityEndpoint != cfg.Global.AuthURL { - t.Errorf("IdentityEndpoint %s != %s", ao.IdentityEndpoint, cfg.Global.AuthURL) - } - if ao.UserID != cfg.Global.UserID { - t.Errorf("UserID %s != %s", ao.UserID, cfg.Global.UserID) - } - if ao.Scope.DomainName != cfg.Global.DomainName { - t.Errorf("DomainName %s != %s", ao.Scope.DomainName, cfg.Global.DomainName) - } - if ao.TenantID != cfg.Global.TenantID { - t.Errorf("TenantID %s != %s", ao.TenantID, cfg.Global.TenantID) - } - - // test setting of the DomainID - cfg.Global.DomainID = "2a73b8f597c04551a0fdc8e95544be8a" - - ao = cfg.Global.ToAuthOptions() - - if ao.Scope.DomainID != cfg.Global.DomainID { - t.Errorf("DomainID %s != %s", ao.Scope.DomainID, cfg.Global.DomainID) - } -} - -func TestCheckOpenStackOpts(t *testing.T) { - tests := []struct { - name string - openstackOpts *OpenStack - expectedError error - }{ - { - name: "test1", - openstackOpts: &OpenStack{ - metadataOpts: MetadataOpts{ - SearchOrder: metadata.ConfigDriveID, - }, - }, - expectedError: nil, - }, - { - name: "test2", - openstackOpts: &OpenStack{ - provider: nil, - metadataOpts: MetadataOpts{ - SearchOrder: "", - }, - }, - expectedError: fmt.Errorf("invalid value in section [Metadata] with key `search-order`. Value cannot be empty"), - }, - { - name: "test3", - openstackOpts: &OpenStack{ - provider: nil, - metadataOpts: MetadataOpts{ - SearchOrder: "value1,value2,value3", - }, - }, - expectedError: fmt.Errorf("invalid value in section [Metadata] with key `search-order`. Value cannot contain more than 2 elements"), - }, - { - name: "test4", - openstackOpts: &OpenStack{ - provider: nil, - metadataOpts: MetadataOpts{ - SearchOrder: "value1", - }, - }, - expectedError: fmt.Errorf("invalid element %q found in section [Metadata] with key `search-order`."+ - "Supported elements include %q and %q", "value1", metadata.ConfigDriveID, metadata.MetadataID), - }, - } - - for _, testcase := range tests { - err := checkOpenStackOpts(testcase.openstackOpts) - - if err == nil && testcase.expectedError == nil { - continue - } - if (err != nil && testcase.expectedError == nil) || (err == nil && testcase.expectedError != nil) || err.Error() != testcase.expectedError.Error() { - t.Errorf("%s failed: expected err=%q, got %q", - testcase.name, testcase.expectedError, err) - } - } -} - -func TestCaller(t *testing.T) { - called := false - myFunc := func() { called = true } - - c := newCaller() - c.call(myFunc) - - if !called { - t.Errorf("caller failed to call function in default case") - } - - c.disarm() - called = false - c.call(myFunc) - - if called { - t.Error("caller still called function when disarmed") - } - - // Confirm the "usual" deferred caller pattern works as expected - - called = false - successCase := func() { - c := newCaller() - defer c.call(func() { called = true }) - c.disarm() - } - if successCase(); called { - t.Error("Deferred success case still invoked unwind") - } - - called = false - failureCase := func() { - c := newCaller() - defer c.call(func() { called = true }) - } - if failureCase(); !called { - t.Error("Deferred failure case failed to invoke unwind") - } -} - -func TestNodeAddresses(t *testing.T) { - srv := servers.Server{ - Status: "ACTIVE", - HostID: "29d3c8c896a45aa4c34e52247875d7fefc3d94bbcc9f622b5d204362", - AccessIPv4: "50.56.176.99", - AccessIPv6: "2001:4800:790e:510:be76:4eff:fe04:82a8", - Addresses: map[string]interface{}{ - "private": []interface{}{ - map[string]interface{}{ - "OS-EXT-IPS-MAC:mac_addr": "fa:16:3e:7c:1b:2b", - "version": float64(4), - "addr": "10.0.0.32", - "OS-EXT-IPS:type": "fixed", - }, - map[string]interface{}{ - "version": float64(4), - "addr": "50.56.176.36", - "OS-EXT-IPS:type": "floating", - }, - map[string]interface{}{ - "version": float64(4), - "addr": "10.0.0.31", - // No OS-EXT-IPS:type - }, - }, - "public": []interface{}{ - map[string]interface{}{ - "version": float64(4), - "addr": "50.56.176.35", - }, - map[string]interface{}{ - "version": float64(6), - "addr": "2001:4800:780e:510:be76:4eff:fe04:84a8", - }, - }, - }, - Metadata: map[string]string{ - "name": "a1-yinvcez57-0-bvynoyawrhcg-kube-minion-fg5i4jwcc2yy", - TypeHostName: "a1-yinvcez57-0-bvynoyawrhcg-kube-minion-fg5i4jwcc2yy.novalocal", - }, - } - - networkingOpts := NetworkingOpts{ - PublicNetworkName: []string{"public"}, - } - - interfaces := []attachinterfaces.Interface{ - { - PortState: "ACTIVE", - FixedIPs: []attachinterfaces.FixedIP{ - { - IPAddress: "10.0.0.32", - }, - { - IPAddress: "10.0.0.31", - }, - }, - }, - } - - addrs, err := nodeAddresses(&srv, interfaces, networkingOpts) - if err != nil { - t.Fatalf("nodeAddresses returned error: %v", err) - } - - t.Logf("addresses are %v", addrs) - - want := []v1.NodeAddress{ - {Type: v1.NodeInternalIP, Address: "10.0.0.32"}, - {Type: v1.NodeInternalIP, Address: "10.0.0.31"}, - {Type: v1.NodeExternalIP, Address: "50.56.176.99"}, - {Type: v1.NodeExternalIP, Address: "2001:4800:790e:510:be76:4eff:fe04:82a8"}, - {Type: v1.NodeHostName, Address: "a1-yinvcez57-0-bvynoyawrhcg-kube-minion-fg5i4jwcc2yy.novalocal"}, - {Type: v1.NodeExternalIP, Address: "50.56.176.36"}, - {Type: v1.NodeExternalIP, Address: "50.56.176.35"}, - {Type: v1.NodeExternalIP, Address: "2001:4800:780e:510:be76:4eff:fe04:84a8"}, - } - - if !reflect.DeepEqual(want, addrs) { - t.Errorf("nodeAddresses returned incorrect value, want %v", want) - } -} - -func TestNodeAddressesCustomPublicNetwork(t *testing.T) { - srv := servers.Server{ - Status: "ACTIVE", - HostID: "29d3c8c896a45aa4c34e52247875d7fefc3d94bbcc9f622b5d204362", - AccessIPv4: "50.56.176.99", - AccessIPv6: "2001:4800:790e:510:be76:4eff:fe04:82a8", - Addresses: map[string]interface{}{ - "private": []interface{}{ - map[string]interface{}{ - "OS-EXT-IPS-MAC:mac_addr": "fa:16:3e:7c:1b:2b", - "version": float64(4), - "addr": "10.0.0.32", - "OS-EXT-IPS:type": "fixed", - }, - map[string]interface{}{ - "version": float64(4), - "addr": "50.56.176.36", - "OS-EXT-IPS:type": "floating", - }, - map[string]interface{}{ - "version": float64(4), - "addr": "10.0.0.31", - // No OS-EXT-IPS:type - }, - }, - "pub-net": []interface{}{ - map[string]interface{}{ - "version": float64(4), - "addr": "50.56.176.35", - }, - map[string]interface{}{ - "version": float64(6), - "addr": "2001:4800:780e:510:be76:4eff:fe04:84a8", - }, - }, - }, - } - - networkingOpts := NetworkingOpts{ - PublicNetworkName: []string{"pub-net"}, - } - - interfaces := []attachinterfaces.Interface{ - { - PortState: "ACTIVE", - FixedIPs: []attachinterfaces.FixedIP{ - { - IPAddress: "10.0.0.32", - }, - { - IPAddress: "10.0.0.31", - }, - }, - }, - } - - addrs, err := nodeAddresses(&srv, interfaces, networkingOpts) - if err != nil { - t.Fatalf("nodeAddresses returned error: %v", err) - } - - t.Logf("addresses are %v", addrs) - - want := []v1.NodeAddress{ - {Type: v1.NodeInternalIP, Address: "10.0.0.32"}, - {Type: v1.NodeInternalIP, Address: "10.0.0.31"}, - {Type: v1.NodeExternalIP, Address: "50.56.176.99"}, - {Type: v1.NodeExternalIP, Address: "2001:4800:790e:510:be76:4eff:fe04:82a8"}, - {Type: v1.NodeExternalIP, Address: "50.56.176.36"}, - {Type: v1.NodeExternalIP, Address: "50.56.176.35"}, - {Type: v1.NodeExternalIP, Address: "2001:4800:780e:510:be76:4eff:fe04:84a8"}, - } - - if !reflect.DeepEqual(want, addrs) { - t.Errorf("nodeAddresses returned incorrect value, want %v", want) - } -} - -func TestNodeAddressesCustomPublicNetworkWithIntersectingFixedIP(t *testing.T) { - srv := servers.Server{ - Status: "ACTIVE", - HostID: "29d3c8c896a45aa4c34e52247875d7fefc3d94bbcc9f622b5d204362", - AccessIPv4: "50.56.176.99", - AccessIPv6: "2001:4800:790e:510:be76:4eff:fe04:82a8", - Addresses: map[string]interface{}{ - "private": []interface{}{ - map[string]interface{}{ - "OS-EXT-IPS-MAC:mac_addr": "fa:16:3e:7c:1b:2b", - "version": float64(4), - "addr": "10.0.0.32", - "OS-EXT-IPS:type": "fixed", - }, - map[string]interface{}{ - "version": float64(4), - "addr": "10.0.0.31", - // No OS-EXT-IPS:type - }, - }, - "pub-net": []interface{}{ - map[string]interface{}{ - "version": float64(4), - "addr": "50.56.176.36", - }, - map[string]interface{}{ - "version": float64(6), - "addr": "2001:4800:780e:510:be76:4eff:fe04:84a8", - }, - }, - }, - } - - networkingOpts := NetworkingOpts{ - PublicNetworkName: []string{"pub-net"}, - } - - interfaces := []attachinterfaces.Interface{ - { - PortState: "ACTIVE", - FixedIPs: []attachinterfaces.FixedIP{ - { - IPAddress: "10.0.0.32", - }, - { - IPAddress: "10.0.0.31", - }, - // intersects with one of pub-net's addresses, shouldn't be counted as an InternalIP - { - IPAddress: "50.56.176.36", - }, - }, - }, - } - - addrs, err := nodeAddresses(&srv, interfaces, networkingOpts) - if err != nil { - t.Fatalf("nodeAddresses returned error: %v", err) - } - - t.Logf("addresses are %v", addrs) - - want := []v1.NodeAddress{ - {Type: v1.NodeInternalIP, Address: "10.0.0.32"}, - {Type: v1.NodeInternalIP, Address: "10.0.0.31"}, - {Type: v1.NodeExternalIP, Address: "50.56.176.99"}, - {Type: v1.NodeExternalIP, Address: "2001:4800:790e:510:be76:4eff:fe04:82a8"}, - {Type: v1.NodeExternalIP, Address: "50.56.176.36"}, - {Type: v1.NodeExternalIP, Address: "2001:4800:780e:510:be76:4eff:fe04:84a8"}, - } - - if !reflect.DeepEqual(want, addrs) { - t.Errorf("nodeAddresses returned incorrect value, want %v", want) - } -} - -func TestNodeAddressesMultipleCustomInternalNetworks(t *testing.T) { - srv := servers.Server{ - Status: "ACTIVE", - HostID: "29d3c8c896a45aa4c34e52247875d7fefc3d94bbcc9f622b5d204362", - AccessIPv4: "50.56.176.99", - AccessIPv6: "2001:4800:790e:510:be76:4eff:fe04:82a8", - Addresses: map[string]interface{}{ - "private": []interface{}{ - map[string]interface{}{ - "OS-EXT-IPS-MAC:mac_addr": "fa:16:3e:7c:1b:2b", - "version": float64(4), - "addr": "10.0.0.32", - "OS-EXT-IPS:type": "fixed", - }, - map[string]interface{}{ - "version": float64(4), - "addr": "50.56.176.36", - "OS-EXT-IPS:type": "floating", - }, - map[string]interface{}{ - "version": float64(4), - "addr": "10.0.0.31", - // No OS-EXT-IPS:type - }, - }, - "also-private": []interface{}{ - map[string]interface{}{ - "version": float64(4), - "addr": "10.0.0.64", - // No OS-EXT-IPS:type - }, - }, - "pub-net": []interface{}{ - map[string]interface{}{ - "version": float64(4), - "addr": "50.56.176.35", - }, - map[string]interface{}{ - "version": float64(6), - "addr": "2001:4800:780e:510:be76:4eff:fe04:84a8", - }, - }, - }, - } - - networkingOpts := NetworkingOpts{ - InternalNetworkName: []string{"private", "also-private"}, - } - - interfaces := []attachinterfaces.Interface{ - { - PortState: "ACTIVE", - FixedIPs: []attachinterfaces.FixedIP{ - { - IPAddress: "10.0.0.32", - }, - { - IPAddress: "10.0.0.31", - }, - }, - }, - } - - addrs, err := nodeAddresses(&srv, interfaces, networkingOpts) - if err != nil { - t.Fatalf("nodeAddresses returned error: %v", err) - } - - t.Logf("addresses are %v", addrs) - - want := []v1.NodeAddress{ - {Type: v1.NodeInternalIP, Address: "10.0.0.32"}, - {Type: v1.NodeInternalIP, Address: "10.0.0.31"}, - {Type: v1.NodeExternalIP, Address: "50.56.176.99"}, - {Type: v1.NodeExternalIP, Address: "2001:4800:790e:510:be76:4eff:fe04:82a8"}, - {Type: v1.NodeInternalIP, Address: "10.0.0.64"}, - {Type: v1.NodeExternalIP, Address: "50.56.176.36"}, - } - - if !reflect.DeepEqual(want, addrs) { - t.Errorf("nodeAddresses returned incorrect value, want %v", want) - } -} - -func TestNodeAddressesOneInternalNetwork(t *testing.T) { - srv := servers.Server{ - Status: "ACTIVE", - HostID: "29d3c8c896a45aa4c34e52247875d7fefc3d94bbcc9f622b5d204362", - AccessIPv4: "50.56.176.99", - AccessIPv6: "2001:4800:790e:510:be76:4eff:fe04:82a8", - Addresses: map[string]interface{}{ - "private": []interface{}{ - map[string]interface{}{ - "OS-EXT-IPS-MAC:mac_addr": "fa:16:3e:7c:1b:2b", - "version": float64(4), - "addr": "10.0.0.32", - "OS-EXT-IPS:type": "fixed", - }, - map[string]interface{}{ - "version": float64(4), - "addr": "50.56.176.36", - "OS-EXT-IPS:type": "floating", - }, - map[string]interface{}{ - "version": float64(4), - "addr": "10.0.0.31", - // No OS-EXT-IPS:type - }, - }, - "also-private": []interface{}{ - map[string]interface{}{ - "version": float64(4), - "addr": "10.0.0.64", - // No OS-EXT-IPS:type - }, - }, - "pub-net": []interface{}{ - map[string]interface{}{ - "version": float64(4), - "addr": "50.56.176.35", - }, - map[string]interface{}{ - "version": float64(6), - "addr": "2001:4800:780e:510:be76:4eff:fe04:84a8", - }, - }, - }, - } - - networkingOpts := NetworkingOpts{ - InternalNetworkName: []string{"also-private"}, - } - - interfaces := []attachinterfaces.Interface{ - { - PortState: "ACTIVE", - FixedIPs: []attachinterfaces.FixedIP{ - { - IPAddress: "10.0.0.32", - }, - { - IPAddress: "10.0.0.31", - }, - }, - }, - } - - addrs, err := nodeAddresses(&srv, interfaces, networkingOpts) - if err != nil { - t.Fatalf("nodeAddresses returned error: %v", err) - } - - t.Logf("addresses are %v", addrs) - - want := []v1.NodeAddress{ - {Type: v1.NodeExternalIP, Address: "50.56.176.99"}, - {Type: v1.NodeExternalIP, Address: "2001:4800:790e:510:be76:4eff:fe04:82a8"}, - {Type: v1.NodeInternalIP, Address: "10.0.0.64"}, - {Type: v1.NodeExternalIP, Address: "50.56.176.36"}, - } - - if !reflect.DeepEqual(want, addrs) { - t.Errorf("nodeAddresses returned incorrect value, want %v", want) - } -} - -func TestNodeAddressesIPv6Disabled(t *testing.T) { - srv := servers.Server{ - Status: "ACTIVE", - HostID: "29d3c8c896a45aa4c34e52247875d7fefc3d94bbcc9f622b5d204362", - AccessIPv4: "50.56.176.99", - AccessIPv6: "2001:4800:790e:510:be76:4eff:fe04:82a8", - Addresses: map[string]interface{}{ - "private": []interface{}{ - map[string]interface{}{ - "OS-EXT-IPS-MAC:mac_addr": "fa:16:3e:7c:1b:2b", - "version": float64(4), - "addr": "10.0.0.32", - "OS-EXT-IPS:type": "fixed", - }, - map[string]interface{}{ - "version": float64(4), - "addr": "50.56.176.36", - "OS-EXT-IPS:type": "floating", - }, - map[string]interface{}{ - "version": float64(4), - "addr": "10.0.0.31", - // No OS-EXT-IPS:type - }, - }, - "public": []interface{}{ - map[string]interface{}{ - "version": float64(4), - "addr": "50.56.176.35", - }, - map[string]interface{}{ - "version": float64(6), - "addr": "2001:4800:780e:510:be76:4eff:fe04:84a8", - }, - }, - }, - } - - networkingOpts := NetworkingOpts{ - PublicNetworkName: []string{"public"}, - IPv6SupportDisabled: true, - } - - interfaces := []attachinterfaces.Interface{ - { - PortState: "ACTIVE", - FixedIPs: []attachinterfaces.FixedIP{ - { - IPAddress: "10.0.0.32", - }, - { - IPAddress: "10.0.0.31", - }, - }, - }, - } - - addrs, err := nodeAddresses(&srv, interfaces, networkingOpts) - if err != nil { - t.Fatalf("nodeAddresses returned error: %v", err) - } - - t.Logf("addresses are %v", addrs) - - want := []v1.NodeAddress{ - {Type: v1.NodeInternalIP, Address: "10.0.0.32"}, - {Type: v1.NodeInternalIP, Address: "10.0.0.31"}, - {Type: v1.NodeExternalIP, Address: "50.56.176.99"}, - {Type: v1.NodeExternalIP, Address: "50.56.176.36"}, - {Type: v1.NodeExternalIP, Address: "50.56.176.35"}, - } - - if !reflect.DeepEqual(want, addrs) { - t.Errorf("nodeAddresses returned incorrect value, want %v", want) - } -} - -func TestNewOpenStack(t *testing.T) { - cfg := ConfigFromEnv() - testConfigFromEnv(t, &cfg) - - _, err := NewOpenStack(cfg) - if err != nil { - t.Fatalf("Failed to construct/authenticate OpenStack: %s", err) - } -} - -func TestLoadBalancer(t *testing.T) { - cfg := ConfigFromEnv() - testConfigFromEnv(t, &cfg) - - versions := []string{"v2"} - - for _, v := range versions { - t.Logf("Trying LBVersion = '%s'\n", v) - cfg.LoadBalancer.LBVersion = v - - os, err := NewOpenStack(cfg) - if err != nil { - t.Fatalf("Failed to construct/authenticate OpenStack: %s", err) - } - - lb, ok := os.LoadBalancer() - if !ok { - t.Fatalf("LoadBalancer() returned false - perhaps your stack doesn't support Neutron?") - } - - _, exists, err := lb.GetLoadBalancer(context.TODO(), testClusterName, &v1.Service{ObjectMeta: metav1.ObjectMeta{Name: "noexist"}}) - if err != nil { - t.Fatalf("GetLoadBalancer(\"noexist\") returned error: %s", err) - } - if exists { - t.Fatalf("GetLoadBalancer(\"noexist\") returned exists") - } - } -} - -var FakeMetadata = metadata.Metadata{ - UUID: "83679162-1378-4288-a2d4-70e13ec132aa", - Name: "test", - AvailabilityZone: "nova", -} - -func TestZones(t *testing.T) { - metadata.Set(&FakeMetadata) - defer metadata.Clear() - - os := OpenStack{ - provider: &gophercloud.ProviderClient{ - IdentityBase: "http://auth.url/", - }, - region: "myRegion", - } - - z, ok := os.Zones() - if !ok { - t.Fatalf("Zones() returned false") - } - - zone, err := z.GetZone(context.TODO()) - if err != nil { - t.Fatalf("GetZone() returned error: %s", err) - } - - if zone.Region != "myRegion" { - t.Fatalf("GetZone() returned wrong region (%s)", zone.Region) - } - - if zone.FailureDomain != "nova" { - t.Fatalf("GetZone() returned wrong failure domain (%s)", zone.FailureDomain) - } -} - -var diskPathRegexp = regexp.MustCompile("/dev/disk/(?:by-id|by-path)/") - -func TestInstanceIDFromProviderID(t *testing.T) { - testCases := []struct { - providerID string - instanceID string - fail bool - }{ - { - providerID: ProviderName + "://" + "/" + "7b9cf879-7146-417c-abfd-cb4272f0c935", - instanceID: "7b9cf879-7146-417c-abfd-cb4272f0c935", - fail: false, - }, - { - // https://github.com/kubernetes/kubernetes/issues/85731 - providerID: "/7b9cf879-7146-417c-abfd-cb4272f0c935", - instanceID: "7b9cf879-7146-417c-abfd-cb4272f0c935", - fail: false, - }, - { - providerID: "openstack://7b9cf879-7146-417c-abfd-cb4272f0c935", - instanceID: "", - fail: true, - }, - { - providerID: "7b9cf879-7146-417c-abfd-cb4272f0c935", - instanceID: "", - fail: true, - }, - { - providerID: "other-provider:///7b9cf879-7146-417c-abfd-cb4272f0c935", - instanceID: "", - fail: true, - }, - } - - for _, test := range testCases { - instanceID, err := instanceIDFromProviderID(test.providerID) - if (err != nil) != test.fail { - t.Errorf("expected err: %t, got err: %v", test.fail, err) - } - - if test.fail { - continue - } - - if instanceID != test.instanceID { - t.Errorf("%s yielded %s. expected %q", test.providerID, instanceID, test.instanceID) - } - } -} - -func TestToAuth3Options(t *testing.T) { - cfg := Config{} - cfg.Global.Username = "user" - cfg.Global.Password = "pass" - cfg.Global.DomainID = "2a73b8f597c04551a0fdc8e95544be8a" - cfg.Global.DomainName = "local" - cfg.Global.AuthURL = "http://auth.url" - cfg.Global.UserID = "user" - cfg.Global.TenantName = "demo" - cfg.Global.TenantDomainName = "Default" - - ao := cfg.Global.ToAuth3Options() - - if !ao.AllowReauth { - t.Errorf("Will need to be able to reauthenticate") - } - if ao.Username != cfg.Global.Username { - t.Errorf("Username %s != %s", ao.Username, cfg.Global.Username) - } - if ao.Password != cfg.Global.Password { - t.Errorf("Password %s != %s", ao.Password, cfg.Global.Password) - } - if ao.DomainID != cfg.Global.DomainID { - t.Errorf("DomainID %s != %s", ao.DomainID, cfg.Global.DomainID) - } - if ao.IdentityEndpoint != cfg.Global.AuthURL { - t.Errorf("IdentityEndpoint %s != %s", ao.IdentityEndpoint, cfg.Global.AuthURL) - } - if ao.UserID != cfg.Global.UserID { - t.Errorf("UserID %s != %s", ao.UserID, cfg.Global.UserID) - } - if ao.DomainName != cfg.Global.DomainName { - t.Errorf("DomainName %s != %s", ao.DomainName, cfg.Global.DomainName) - } - if ao.Scope.ProjectName != cfg.Global.TenantName { - t.Errorf("TenantName %s != %s", ao.Scope.ProjectName, cfg.Global.TenantName) - } - if ao.Scope.DomainName != cfg.Global.TenantDomainName { - t.Errorf("TenantDomainName %s != %s", ao.Scope.DomainName, cfg.Global.TenantDomainName) - } -} - -func TestToAuth3OptionsScope(t *testing.T) { - // Use Domain Name/ID if Tenant Domain Name/ID is not set - cfg := Config{} - cfg.Global.Username = "user" - cfg.Global.Password = "pass" - cfg.Global.DomainID = "2a73b8f597c04551a0fdc8e95544be8a" - cfg.Global.DomainName = "local" - cfg.Global.AuthURL = "http://auth.url" - cfg.Global.UserID = "user" - cfg.Global.TenantName = "demo" - - ao := cfg.Global.ToAuth3Options() - - if ao.Scope.ProjectName != cfg.Global.TenantName { - t.Errorf("TenantName %s != %s", ao.Scope.ProjectName, cfg.Global.TenantName) - } - if ao.Scope.DomainName != cfg.Global.DomainName { - t.Errorf("DomainName %s != %s", ao.Scope.DomainName, cfg.Global.DomainName) - } - if ao.Scope.DomainID != cfg.Global.DomainID { - t.Errorf("DomainID %s != %s", ao.Scope.DomainID, cfg.Global.DomainID) - } - - // Use Tenant Domain Name/ID if set - cfg = Config{} - cfg.Global.Username = "user" - cfg.Global.Password = "pass" - cfg.Global.DomainID = "2a73b8f597c04551a0fdc8e95544be8a" - cfg.Global.DomainName = "local" - cfg.Global.AuthURL = "http://auth.url" - cfg.Global.UserID = "user" - cfg.Global.TenantName = "demo" - cfg.Global.TenantDomainName = "Default" - cfg.Global.TenantDomainID = "default" - - ao = cfg.Global.ToAuth3Options() - - if ao.Scope.ProjectName != cfg.Global.TenantName { - t.Errorf("TenantName %s != %s", ao.Scope.ProjectName, cfg.Global.TenantName) - } - if ao.Scope.DomainName != cfg.Global.TenantDomainName { - t.Errorf("TenantDomainName %s != %s", ao.Scope.DomainName, cfg.Global.TenantDomainName) - } - if ao.Scope.DomainID != cfg.Global.TenantDomainID { - t.Errorf("TenantDomainID %s != %s", ao.Scope.DomainName, cfg.Global.TenantDomainID) - } - - // Do not use neither Domain Name nor ID, if Tenant ID was provided - cfg = Config{} - cfg.Global.Username = "user" - cfg.Global.Password = "pass" - cfg.Global.DomainID = "2a73b8f597c04551a0fdc8e95544be8a" - cfg.Global.DomainName = "local" - cfg.Global.AuthURL = "http://auth.url" - cfg.Global.UserID = "user" - cfg.Global.TenantID = "7808db451cfc43eaa9acda7d67da8cf1" - cfg.Global.TenantDomainName = "Default" - cfg.Global.TenantDomainID = "default" - - ao = cfg.Global.ToAuth3Options() - - if ao.Scope.ProjectName != "" { - t.Errorf("TenantName in the scope is not empty") - } - if ao.Scope.DomainName != "" { - t.Errorf("DomainName in the scope is not empty") - } - if ao.Scope.DomainID != "" { - t.Errorf("DomainID in the scope is not empty") - } -} - -func TestUserAgentFlag(t *testing.T) { - tests := []struct { - name string - shouldParse bool - flags []string - expected []string - }{ - {"no_flag", true, []string{}, nil}, - {"one_flag", true, []string{"--user-agent=cluster/abc-123"}, []string{"cluster/abc-123"}}, - {"multiple_flags", true, []string{"--user-agent=a/b", "--user-agent=c/d"}, []string{"a/b", "c/d"}}, - {"flag_with_space", true, []string{"--user-agent=a b"}, []string{"a b"}}, - {"flag_split_with_space", true, []string{"--user-agent=a", "b"}, []string{"a"}}, - {"empty_flag", false, []string{"--user-agent"}, nil}, - } - - for _, testCase := range tests { - userAgentData = []string{} - - t.Run(testCase.name, func(t *testing.T) { - fs := pflag.NewFlagSet("test", pflag.ContinueOnError) - AddExtraFlags(fs) - - err := fs.Parse(testCase.flags) - - if testCase.shouldParse && err != nil { - t.Errorf("Flags failed to parse") - } else if !testCase.shouldParse && err == nil { - t.Errorf("Flags should not have parsed") - } else if testCase.shouldParse { - if !reflect.DeepEqual(userAgentData, testCase.expected) { - t.Errorf("userAgentData %#v did not match expected value %#v", userAgentData, testCase.expected) - } - } - }) - } -} - -func clearEnviron(t *testing.T) []string { - env := os.Environ() - for _, pair := range env { - if strings.HasPrefix(pair, "OS_") { - i := strings.Index(pair, "=") + 1 - os.Unsetenv(pair[:i-1]) - } - } - return env -} -func resetEnviron(t *testing.T, items []string) { - for _, pair := range items { - if strings.HasPrefix(pair, "OS_") { - i := strings.Index(pair, "=") + 1 - if err := os.Setenv(pair[:i-1], pair[i:]); err != nil { - t.Errorf("Setenv(%q, %q) failed during reset: %v", pair[:i-1], pair[i:], err) - } - } - } -} - -func testConfigFromEnv(t *testing.T, cfg *Config) { - if cfg.Global.AuthURL == "" { - t.Skip("No config found in environment") - } -} diff --git a/pkg/csi/cinder/controllerserver.go b/pkg/csi/cinder/controllerserver.go deleted file mode 100644 index 4e5b134b17..0000000000 --- a/pkg/csi/cinder/controllerserver.go +++ /dev/null @@ -1,623 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cinder - -import ( - "fmt" - "strconv" - - "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/golang/protobuf/ptypes" - "github.com/gophercloud/gophercloud/openstack/blockstorage/v3/snapshots" - ossnapshots "github.com/gophercloud/gophercloud/openstack/blockstorage/v3/snapshots" - "github.com/gophercloud/gophercloud/openstack/blockstorage/v3/volumes" - "golang.org/x/net/context" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - - "k8s.io/cloud-provider-openstack/pkg/csi/cinder/openstack" - "k8s.io/cloud-provider-openstack/pkg/util" - cpoerrors "k8s.io/cloud-provider-openstack/pkg/util/errors" - "k8s.io/klog/v2" -) - -type controllerServer struct { - Driver *CinderDriver - Cloud openstack.IOpenStack -} - -func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) { - - // Volume Name - volName := req.GetName() - - if len(volName) == 0 { - return nil, status.Error(codes.InvalidArgument, "CreateVolume request missing Volume Name") - } - - if req.VolumeCapabilities == nil { - return nil, status.Error(codes.InvalidArgument, "CreateVolume request missing Volume capability") - } - - // Volume Size - Default is 1 GiB - volSizeBytes := int64(1 * 1024 * 1024 * 1024) - if req.GetCapacityRange() != nil { - volSizeBytes = int64(req.GetCapacityRange().GetRequiredBytes()) - } - volSizeGB := int(util.RoundUpSize(volSizeBytes, 1024*1024*1024)) - - // Volume Type - volType := req.GetParameters()["type"] - - var volAvailability string - if req.GetAccessibilityRequirements() != nil { - volAvailability = getAZFromTopology(req.GetAccessibilityRequirements()) - } - - if len(volAvailability) == 0 { - // Volume Availability - Default is nova - volAvailability = req.GetParameters()["availability"] - } - - cloud := cs.Cloud - - // Verify a volume with the provided name doesn't already exist for this tenant - volumes, err := cloud.GetVolumesByName(volName) - if err != nil { - klog.V(3).Infof("Failed to query for existing Volume during CreateVolume: %v", err) - } - - if len(volumes) == 1 { - if volSizeGB != volumes[0].Size { - return nil, status.Error(codes.AlreadyExists, "Volume Already exists with same name and different capacity") - } - - klog.V(4).Infof("Volume %s already exists in Availability Zone: %s of size %d GiB", volumes[0].ID, volumes[0].AvailabilityZone, volumes[0].Size) - return getCreateVolumeResponse(&volumes[0]), nil - } else if len(volumes) > 1 { - klog.V(3).Infof("found multiple existing volumes with selected name (%s) during create", volName) - return nil, status.Error(codes.Internal, "Multiple volumes reported by Cinder with same name") - - } - - // Volume Create - properties := map[string]string{"cinder.csi.openstack.org/cluster": cs.Driver.cluster} - content := req.GetVolumeContentSource() - var snapshotID string - var sourcevolID string - - if content != nil && content.GetSnapshot() != nil { - snapshotID = content.GetSnapshot().GetSnapshotId() - _, err := cloud.GetSnapshotByID(snapshotID) - if err != nil { - if cpoerrors.IsNotFound(err) { - return nil, status.Errorf(codes.NotFound, "VolumeContentSource Snapshot %s not found", snapshotID) - } - return nil, status.Errorf(codes.Internal, "Failed to retrieve the snapshot %s: %v", snapshotID, err) - } - } - - if content != nil && content.GetVolume() != nil { - sourcevolID = content.GetVolume().GetVolumeId() - _, err := cloud.GetVolume(sourcevolID) - if err != nil { - if cpoerrors.IsNotFound(err) { - return nil, status.Errorf(codes.NotFound, "Source Volume %s not found", sourcevolID) - } - return nil, status.Errorf(codes.Internal, "Failed to retrieve the source volume %s: %v", sourcevolID, err) - } - } - - vol, err := cloud.CreateVolume(volName, volSizeGB, volType, volAvailability, snapshotID, sourcevolID, &properties) - if err != nil { - klog.V(3).Infof("Failed to CreateVolume: %v", err) - return nil, status.Error(codes.Internal, fmt.Sprintf("CreateVolume failed with error %v", err)) - - } - - klog.V(4).Infof("Create volume %s in Availability Zone: %s of size %d GiB", vol.ID, vol.AvailabilityZone, vol.Size) - - return getCreateVolumeResponse(vol), nil -} - -func (cs *controllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) { - - // Volume Delete - volID := req.GetVolumeId() - if len(volID) == 0 { - return nil, status.Error(codes.InvalidArgument, "DeleteVolume Volume ID must be provided") - } - err := cs.Cloud.DeleteVolume(volID) - if err != nil { - if cpoerrors.IsNotFound(err) { - klog.V(3).Infof("Volume %s is already deleted.", volID) - return &csi.DeleteVolumeResponse{}, nil - } - klog.V(3).Infof("Failed to DeleteVolume: %v", err) - return nil, status.Error(codes.Internal, fmt.Sprintf("DeleteVolume failed with error %v", err)) - } - - klog.V(4).Infof("Delete volume %s", volID) - - return &csi.DeleteVolumeResponse{}, nil -} - -func (cs *controllerServer) ControllerPublishVolume(ctx context.Context, req *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error) { - - // Volume Attach - instanceID := req.GetNodeId() - volumeID := req.GetVolumeId() - volumeCapability := req.GetVolumeCapability() - - if len(volumeID) == 0 { - return nil, status.Error(codes.InvalidArgument, "[ControllerPublishVolume] Volume ID must be provided") - } - if len(instanceID) == 0 { - return nil, status.Error(codes.InvalidArgument, "[ControllerPublishVolume] Instance ID must be provided") - } - if volumeCapability == nil { - return nil, status.Error(codes.InvalidArgument, "[ControllerPublishVolume] Volume capability must be provided") - } - - _, err := cs.Cloud.GetVolume(volumeID) - if err != nil { - if cpoerrors.IsNotFound(err) { - return nil, status.Errorf(codes.NotFound, "[ControllerPublishVolume] Volume %s not found", volumeID) - } - return nil, status.Error(codes.Internal, fmt.Sprintf("[ControllerPublishVolume] get volume failed with error %v", err)) - } - - _, err = cs.Cloud.GetInstanceByID(instanceID) - if err != nil { - if cpoerrors.IsNotFound(err) { - return nil, status.Errorf(codes.NotFound, "[ControllerPublishVolume] Instance %s not found", instanceID) - } - return nil, status.Error(codes.Internal, fmt.Sprintf("[ControllerPublishVolume] GetInstanceByID failed with error %v", err)) - } - - _, err = cs.Cloud.AttachVolume(instanceID, volumeID) - if err != nil { - klog.V(3).Infof("Failed to AttachVolume: %v", err) - return nil, status.Error(codes.Internal, fmt.Sprintf("[ControllerPublishVolume] Attach Volume failed with error %v", err)) - - } - - err = cs.Cloud.WaitDiskAttached(instanceID, volumeID) - if err != nil { - klog.V(3).Infof("Failed to WaitDiskAttached: %v", err) - return nil, status.Error(codes.Internal, fmt.Sprintf("[ControllerPublishVolume] failed to attach volume: %v", err)) - } - - devicePath, err := cs.Cloud.GetAttachmentDiskPath(instanceID, volumeID) - if err != nil { - klog.V(3).Infof("Failed to GetAttachmentDiskPath: %v", err) - return nil, status.Error(codes.Internal, fmt.Sprintf("[ControllerPublishVolume] failed to get device path of attached volume : %v", err)) - } - - klog.V(4).Infof("ControllerPublishVolume %s on %s is successful", volumeID, instanceID) - - // Publish Volume Info - pvInfo := map[string]string{} - pvInfo["DevicePath"] = devicePath - - return &csi.ControllerPublishVolumeResponse{ - PublishContext: pvInfo, - }, nil -} - -func (cs *controllerServer) ControllerUnpublishVolume(ctx context.Context, req *csi.ControllerUnpublishVolumeRequest) (*csi.ControllerUnpublishVolumeResponse, error) { - - // Volume Detach - instanceID := req.GetNodeId() - volumeID := req.GetVolumeId() - - if len(volumeID) == 0 { - return nil, status.Error(codes.InvalidArgument, "[ControllerUnpublishVolume] Volume ID must be provided") - } - _, err := cs.Cloud.GetInstanceByID(instanceID) - if err != nil { - if cpoerrors.IsNotFound(err) { - klog.V(3).Infof("ControllerUnpublishVolume assuming volume %s is detached, because node %s does not exist", volumeID, instanceID) - return &csi.ControllerUnpublishVolumeResponse{}, nil - } - return nil, status.Error(codes.Internal, fmt.Sprintf("[ControllerUnpublishVolume] GetInstanceByID failed with error %v", err)) - } - - err = cs.Cloud.DetachVolume(instanceID, volumeID) - if err != nil { - if cpoerrors.IsNotFound(err) { - klog.V(3).Infof("ControllerUnpublishVolume assuming volume %s is detached, because it does not exist", volumeID) - return &csi.ControllerUnpublishVolumeResponse{}, nil - } - klog.V(3).Infof("Failed to DetachVolume: %v", err) - return nil, status.Error(codes.Internal, fmt.Sprintf("ControllerUnpublishVolume Detach Volume failed with error %v", err)) - } - - err = cs.Cloud.WaitDiskDetached(instanceID, volumeID) - if err != nil { - klog.V(3).Infof("Failed to WaitDiskDetached: %v", err) - if cpoerrors.IsNotFound(err) { - klog.V(3).Infof("ControllerUnpublishVolume assuming volume %s is detached, because it was deleted in the meanwhile", volumeID) - return &csi.ControllerUnpublishVolumeResponse{}, nil - } - return nil, status.Error(codes.Internal, fmt.Sprintf("ControllerUnpublishVolume failed with error %v", err)) - } - - klog.V(4).Infof("ControllerUnpublishVolume %s on %s", volumeID, instanceID) - - return &csi.ControllerUnpublishVolumeResponse{}, nil -} - -func (cs *controllerServer) ListVolumes(ctx context.Context, req *csi.ListVolumesRequest) (*csi.ListVolumesResponse, error) { - - if req.MaxEntries < 0 { - return nil, status.Error(codes.InvalidArgument, fmt.Sprintf( - "[ListVolumes] Invalid max entries request %v, must not be negative ", req.MaxEntries)) - } - maxEntries := int(req.MaxEntries) - - vlist, nextPageToken, err := cs.Cloud.ListVolumes(maxEntries, req.StartingToken) - if err != nil { - klog.V(3).Infof("Failed to ListVolumes: %v", err) - if cpoerrors.IsInvalidError(err) { - return nil, status.Errorf(codes.Aborted, "[ListVolumes] Invalid request: %v", err) - } - return nil, status.Error(codes.Internal, fmt.Sprintf("ListVolumes failed with error %v", err)) - } - - var ventries []*csi.ListVolumesResponse_Entry - for _, v := range vlist { - ventry := csi.ListVolumesResponse_Entry{ - Volume: &csi.Volume{ - VolumeId: v.ID, - CapacityBytes: int64(v.Size * 1024 * 1024 * 1024), - }, - } - - status := &csi.ListVolumesResponse_VolumeStatus{} - for _, attachment := range v.Attachments { - status.PublishedNodeIds = append(status.PublishedNodeIds, attachment.ServerID) - } - ventry.Status = status - - ventries = append(ventries, &ventry) - } - return &csi.ListVolumesResponse{ - Entries: ventries, - NextToken: nextPageToken, - }, nil -} - -func (cs *controllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateSnapshotRequest) (*csi.CreateSnapshotResponse, error) { - name := req.Name - volumeId := req.GetSourceVolumeId() - - if name == "" { - return nil, status.Error(codes.InvalidArgument, "Snapshot name must be provided in CreateSnapshot request") - } - - if volumeId == "" { - return nil, status.Error(codes.InvalidArgument, "VolumeID must be provided in CreateSnapshot request") - } - - // Verify a snapshot with the provided name doesn't already exist for this tenant - filters := map[string]string{} - filters["Name"] = name - snapshots, _, err := cs.Cloud.ListSnapshots(filters) - if err != nil { - klog.V(3).Infof("Failed to query for existing Snapshot during CreateSnapshot: %v", err) - } - var snap *ossnapshots.Snapshot - - if len(snapshots) == 1 { - snap = &snapshots[0] - - if snap.VolumeID != volumeId { - return nil, status.Error(codes.AlreadyExists, "Snapshot with given name already exists, with different source volume ID") - } - - klog.V(3).Infof("Found existing snapshot %s on %s", name, volumeId) - - } else if len(snapshots) > 1 { - klog.V(3).Infof("found multiple existing snapshots with selected name (%s) during create", name) - return nil, status.Error(codes.Internal, "Multiple snapshots reported by Cinder with same name") - - } else { - // TODO: Delegate the check to openstack itself and ignore the conflict - snap, err = cs.Cloud.CreateSnapshot(name, volumeId, &req.Parameters) - if err != nil { - klog.V(3).Infof("Failed to Create snapshot: %v", err) - return nil, status.Error(codes.Internal, fmt.Sprintf("CreateSnapshot failed with error %v", err)) - } - - klog.V(3).Infof("CreateSnapshot %s on %s", name, volumeId) - } - - ctime, err := ptypes.TimestampProto(snap.CreatedAt) - if err != nil { - klog.Errorf("Error to convert time to timestamp: %v", err) - } - - err = cs.Cloud.WaitSnapshotReady(snap.ID) - if err != nil { - klog.V(3).Infof("Failed to WaitSnapshotReady: %v", err) - return nil, status.Error(codes.Internal, fmt.Sprintf("CreateSnapshot failed with error %v", err)) - } - - return &csi.CreateSnapshotResponse{ - Snapshot: &csi.Snapshot{ - SnapshotId: snap.ID, - SizeBytes: int64(snap.Size * 1024 * 1024 * 1024), - SourceVolumeId: snap.VolumeID, - CreationTime: ctime, - ReadyToUse: true, - }, - }, nil -} - -func (cs *controllerServer) DeleteSnapshot(ctx context.Context, req *csi.DeleteSnapshotRequest) (*csi.DeleteSnapshotResponse, error) { - - id := req.GetSnapshotId() - - if id == "" { - return nil, status.Error(codes.InvalidArgument, "Snapshot ID must be provided in DeleteSnapshot request") - } - - // Delegate the check to openstack itself - err := cs.Cloud.DeleteSnapshot(id) - if err != nil { - if cpoerrors.IsNotFound(err) { - klog.V(3).Infof("Snapshot %s is already deleted.", id) - return &csi.DeleteSnapshotResponse{}, nil - } - klog.V(3).Infof("Failed to Delete snapshot: %v", err) - return nil, status.Error(codes.Internal, fmt.Sprintf("DeleteSnapshot failed with error %v", err)) - } - return &csi.DeleteSnapshotResponse{}, nil -} - -func (cs *controllerServer) ListSnapshots(ctx context.Context, req *csi.ListSnapshotsRequest) (*csi.ListSnapshotsResponse, error) { - - snapshotID := req.GetSnapshotId() - if len(snapshotID) != 0 { - snap, err := cs.Cloud.GetSnapshotByID(snapshotID) - if err != nil { - if cpoerrors.IsNotFound(err) { - klog.V(3).Infof("Snapshot %s not found", snapshotID) - return &csi.ListSnapshotsResponse{}, nil - } - return nil, status.Errorf(codes.Internal, "Failed to GetSnapshot %s : %v", snapshotID, err) - } - - ctime, err := ptypes.TimestampProto(snap.CreatedAt) - - entry := &csi.ListSnapshotsResponse_Entry{ - Snapshot: &csi.Snapshot{ - SizeBytes: int64(snap.Size * 1024 * 1024 * 1024), - SnapshotId: snap.ID, - SourceVolumeId: snap.VolumeID, - CreationTime: ctime, - ReadyToUse: true, - }, - } - - entries := []*csi.ListSnapshotsResponse_Entry{entry} - return &csi.ListSnapshotsResponse{ - Entries: entries, - }, err - - } - - filters := map[string]string{} - - var slist []snapshots.Snapshot - var err error - var nextPageToken string - - // Add the filters - if len(req.GetSourceVolumeId()) != 0 { - filters["VolumeID"] = req.GetSourceVolumeId() - } else { - filters["Limit"] = strconv.Itoa(int(req.MaxEntries)) - filters["Marker"] = req.StartingToken - } - - // Only retrieve snapshots that are available - filters["Status"] = "available" - slist, nextPageToken, err = cs.Cloud.ListSnapshots(filters) - if err != nil { - klog.V(3).Infof("Failed to ListSnapshots: %v", err) - return nil, status.Errorf(codes.Internal, "ListSnapshots failed with error %v", err) - } - - var sentries []*csi.ListSnapshotsResponse_Entry - for _, v := range slist { - ctime, err := ptypes.TimestampProto(v.CreatedAt) - if err != nil { - klog.Errorf("Error to convert time to timestamp: %v", err) - } - sentry := csi.ListSnapshotsResponse_Entry{ - Snapshot: &csi.Snapshot{ - SizeBytes: int64(v.Size * 1024 * 1024 * 1024), - SnapshotId: v.ID, - SourceVolumeId: v.VolumeID, - CreationTime: ctime, - ReadyToUse: true, - }, - } - sentries = append(sentries, &sentry) - } - return &csi.ListSnapshotsResponse{ - Entries: sentries, - NextToken: nextPageToken, - }, nil - -} - -// ControllerGetCapabilities implements the default GRPC callout. -// Default supports all capabilities -func (cs *controllerServer) ControllerGetCapabilities(ctx context.Context, req *csi.ControllerGetCapabilitiesRequest) (*csi.ControllerGetCapabilitiesResponse, error) { - klog.V(5).Infof("Using default ControllerGetCapabilities") - - return &csi.ControllerGetCapabilitiesResponse{ - Capabilities: cs.Driver.cscap, - }, nil -} - -func (cs *controllerServer) ValidateVolumeCapabilities(ctx context.Context, req *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error) { - - reqVolCap := req.GetVolumeCapabilities() - - if reqVolCap == nil || len(reqVolCap) == 0 { - return nil, status.Error(codes.InvalidArgument, "ValidateVolumeCapabilities Volume Capabilities must be provided") - } - volumeID := req.GetVolumeId() - - if len(volumeID) == 0 { - return nil, status.Error(codes.InvalidArgument, "ValidateVolumeCapabilities Volume ID must be provided") - } - - _, err := cs.Cloud.GetVolume(volumeID) - if err != nil { - if cpoerrors.IsNotFound(err) { - return nil, status.Error(codes.NotFound, fmt.Sprintf("ValidateVolumeCapabiltites Volume %s not found", volumeID)) - } - return nil, status.Error(codes.Internal, fmt.Sprintf("ValidateVolumeCapabiltites %v", err)) - } - - for _, cap := range reqVolCap { - if cap.GetAccessMode().GetMode() != cs.Driver.vcap[0].Mode { - return &csi.ValidateVolumeCapabilitiesResponse{Message: "Requested Volume Capabilty not supported"}, nil - } - } - - // Cinder CSI driver currently supports one mode only - resp := &csi.ValidateVolumeCapabilitiesResponse{ - Confirmed: &csi.ValidateVolumeCapabilitiesResponse_Confirmed{ - VolumeCapabilities: []*csi.VolumeCapability{ - { - AccessMode: cs.Driver.vcap[0], - }, - }, - }, - } - - return resp, nil -} - -func (cs *controllerServer) GetCapacity(ctx context.Context, req *csi.GetCapacityRequest) (*csi.GetCapacityResponse, error) { - return nil, status.Error(codes.Unimplemented, fmt.Sprintf("GetCapacity is not yet implemented")) -} - -func (cs *controllerServer) ControllerExpandVolume(ctx context.Context, req *csi.ControllerExpandVolumeRequest) (*csi.ControllerExpandVolumeResponse, error) { - klog.V(4).Infof("ControllerExpandVolume: called with args %+v", *req) - - volumeID := req.GetVolumeId() - if len(volumeID) == 0 { - return nil, status.Error(codes.InvalidArgument, "Volume ID not provided") - } - cap := req.GetCapacityRange() - if cap == nil { - return nil, status.Error(codes.InvalidArgument, "Capacity range not provided") - } - - volSizeBytes := int64(req.GetCapacityRange().GetRequiredBytes()) - volSizeGB := int(util.RoundUpSize(volSizeBytes, 1024*1024*1024)) - maxVolSize := cap.GetLimitBytes() - - if maxVolSize > 0 && maxVolSize < volSizeBytes { - return nil, status.Error(codes.OutOfRange, "After round-up, volume size exceeds the limit specified") - } - - _, err := cs.Cloud.GetVolume(volumeID) - if err != nil { - if cpoerrors.IsNotFound(err) { - return nil, status.Error(codes.NotFound, "Volume not found") - } - return nil, status.Error(codes.Internal, fmt.Sprintf("GetVolume failed with error %v", err)) - } - - err = cs.Cloud.ExpandVolume(volumeID, volSizeGB) - if err != nil { - return nil, status.Errorf(codes.Internal, fmt.Sprintf("Could not resize volume %q to size %v: %v", volumeID, volSizeGB, err)) - } - - klog.V(4).Infof("ControllerExpandVolume resized volume %v to size %v", volumeID, volSizeGB) - - return &csi.ControllerExpandVolumeResponse{ - CapacityBytes: volSizeBytes, - NodeExpansionRequired: true, - }, nil -} - -func getAZFromTopology(requirement *csi.TopologyRequirement) string { - for _, topology := range requirement.GetPreferred() { - zone, exists := topology.GetSegments()[topologyKey] - if exists { - return zone - } - } - - for _, topology := range requirement.GetRequisite() { - zone, exists := topology.GetSegments()[topologyKey] - if exists { - return zone - } - } - return "" -} - -func getCreateVolumeResponse(vol *volumes.Volume) *csi.CreateVolumeResponse { - - var volsrc *csi.VolumeContentSource - - if vol.SnapshotID != "" { - volsrc = &csi.VolumeContentSource{ - Type: &csi.VolumeContentSource_Snapshot{ - Snapshot: &csi.VolumeContentSource_SnapshotSource{ - SnapshotId: vol.SnapshotID, - }, - }, - } - } - - if vol.SourceVolID != "" { - volsrc = &csi.VolumeContentSource{ - Type: &csi.VolumeContentSource_Volume{ - Volume: &csi.VolumeContentSource_VolumeSource{ - VolumeId: vol.SourceVolID, - }, - }, - } - } - - resp := &csi.CreateVolumeResponse{ - Volume: &csi.Volume{ - VolumeId: vol.ID, - CapacityBytes: int64(vol.Size * 1024 * 1024 * 1024), - AccessibleTopology: []*csi.Topology{ - { - Segments: map[string]string{topologyKey: vol.AvailabilityZone}, - }, - }, - ContentSource: volsrc, - }, - } - - return resp - -} diff --git a/pkg/csi/cinder/controllerserver_test.go b/pkg/csi/cinder/controllerserver_test.go deleted file mode 100644 index dd68849a7e..0000000000 --- a/pkg/csi/cinder/controllerserver_test.go +++ /dev/null @@ -1,523 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cinder - -import ( - "testing" - - "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "k8s.io/cloud-provider-openstack/pkg/csi/cinder/openstack" -) - -var fakeCs *controllerServer -var osmock *openstack.OpenStackMock - -// Init Controller Server -func init() { - if fakeCs == nil { - osmock = new(openstack.OpenStackMock) - openstack.OsInstance = osmock - - d := NewDriver(FakeNodeID, FakeEndpoint, FakeCluster) - - fakeCs = NewControllerServer(d, openstack.OsInstance) - } -} - -// Test CreateVolume -func TestCreateVolume(t *testing.T) { - - // mock OpenStack - properties := map[string]string{"cinder.csi.openstack.org/cluster": FakeCluster} - // CreateVolume(name string, size int, vtype, availability string, snapshotID string, tags *map[string]string) (string, string, int, error) - osmock.On("CreateVolume", FakeVolName, mock.AnythingOfType("int"), FakeVolType, FakeAvailability, "", "", &properties).Return(&FakeVol, nil) - - osmock.On("GetVolumesByName", FakeVolName).Return(FakeVolListEmpty, nil) - // Init assert - assert := assert.New(t) - - // Fake request - fakeReq := &csi.CreateVolumeRequest{ - Name: FakeVolName, - VolumeCapabilities: []*csi.VolumeCapability{ - { - AccessType: &csi.VolumeCapability_Mount{ - Mount: &csi.VolumeCapability_MountVolume{}, - }, - }, - }, - AccessibilityRequirements: &csi.TopologyRequirement{ - Requisite: []*csi.Topology{ - { - Segments: map[string]string{"topology.cinder.csi.openstack.org/zone": FakeAvailability}, - }, - }, - }, - } - - // Invoke CreateVolume - actualRes, err := fakeCs.CreateVolume(FakeCtx, fakeReq) - if err != nil { - t.Errorf("failed to CreateVolume: %v", err) - } - - // Assert - assert.NotNil(actualRes.Volume) - assert.NotNil(actualRes.Volume.CapacityBytes) - assert.NotEqual(0, len(actualRes.Volume.VolumeId), "Volume Id is nil") - assert.NotNil(actualRes.Volume.AccessibleTopology) - assert.Equal(FakeAvailability, actualRes.Volume.AccessibleTopology[0].GetSegments()[topologyKey]) - -} - -func TestCreateVolumeFromSnapshot(t *testing.T) { - - properties := map[string]string{"cinder.csi.openstack.org/cluster": FakeCluster} - // CreateVolume(name string, size int, vtype, availability string, snapshotID string, tags *map[string]string) (string, string, int, error) - osmock.On("CreateVolume", FakeVolName, mock.AnythingOfType("int"), FakeVolType, "", FakeSnapshotID, "", &properties).Return(&FakeVolFromSnapshot, nil) - osmock.On("GetVolumesByName", FakeVolName).Return(FakeVolListEmpty, nil) - - // Init assert - assert := assert.New(t) - - src := &csi.VolumeContentSource{ - Type: &csi.VolumeContentSource_Snapshot{ - Snapshot: &csi.VolumeContentSource_SnapshotSource{ - SnapshotId: FakeSnapshotID, - }, - }, - } - - // Fake request - fakeReq := &csi.CreateVolumeRequest{ - Name: FakeVolName, - VolumeCapabilities: []*csi.VolumeCapability{ - { - AccessType: &csi.VolumeCapability_Mount{ - Mount: &csi.VolumeCapability_MountVolume{}, - }, - }, - }, - VolumeContentSource: src, - } - - // Invoke CreateVolume - actualRes, err := fakeCs.CreateVolume(FakeCtx, fakeReq) - if err != nil { - t.Errorf("failed to CreateVolume: %v", err) - } - - // Assert - assert.NotNil(actualRes.Volume) - - assert.NotNil(actualRes.Volume.CapacityBytes) - - assert.NotEqual(0, len(actualRes.Volume.VolumeId), "Volume Id is nil") - - assert.Equal(FakeSnapshotID, actualRes.Volume.ContentSource.GetSnapshot().SnapshotId) - -} - -func TestCreateVolumeFromSourceVolume(t *testing.T) { - - properties := map[string]string{"cinder.csi.openstack.org/cluster": FakeCluster} - // CreateVolume(name string, size int, vtype, availability string, snapshotID string, tags *map[string]string) (string, string, int, error) - osmock.On("CreateVolume", FakeVolName, mock.AnythingOfType("int"), FakeVolType, "", "", FakeVolID, &properties).Return(&FakeVolFromSourceVolume, nil) - osmock.On("GetVolumesByName", FakeVolName).Return(FakeVolListEmpty, nil) - - // Init assert - assert := assert.New(t) - - volsrc := &csi.VolumeContentSource{ - Type: &csi.VolumeContentSource_Volume{ - Volume: &csi.VolumeContentSource_VolumeSource{ - VolumeId: FakeVolID, - }, - }, - } - - // Fake request - fakeReq := &csi.CreateVolumeRequest{ - Name: FakeVolName, - VolumeCapabilities: []*csi.VolumeCapability{ - { - AccessType: &csi.VolumeCapability_Mount{ - Mount: &csi.VolumeCapability_MountVolume{}, - }, - }, - }, - VolumeContentSource: volsrc, - } - - // Invoke CreateVolume - actualRes, err := fakeCs.CreateVolume(FakeCtx, fakeReq) - if err != nil { - t.Errorf("failed to CreateVolume: %v", err) - } - - // Assert - assert.NotNil(actualRes.Volume) - - assert.NotNil(actualRes.Volume.CapacityBytes) - - assert.NotEqual(0, len(actualRes.Volume.VolumeId), "Volume Id is nil") - - assert.Equal(FakeVolID, actualRes.Volume.ContentSource.GetVolume().VolumeId) - -} - -// Test CreateVolumeDuplicate -func TestCreateVolumeDuplicate(t *testing.T) { - - // Init assert - assert := assert.New(t) - - osmock.On("GetVolumesByName", "fake-duplicate").Return(FakeVolList, nil) - - // Fake request - fakeReq := &csi.CreateVolumeRequest{ - Name: "fake-duplicate", - VolumeCapabilities: []*csi.VolumeCapability{ - { - AccessType: &csi.VolumeCapability_Mount{ - Mount: &csi.VolumeCapability_MountVolume{}, - }, - }, - }, - } - - // Invoke CreateVolume - actualRes, err := fakeCs.CreateVolume(FakeCtx, fakeReq) - if err != nil { - t.Errorf("failed to CreateVolume: %v", err) - } - - // Assert - assert.NotNil(actualRes.Volume) - assert.NotEqual(0, len(actualRes.Volume.VolumeId), "Volume Id is nil") - assert.Equal("nova", actualRes.Volume.AccessibleTopology[0].GetSegments()[topologyKey]) - assert.Equal(FakeVolID, actualRes.Volume.VolumeId) -} - -// Test DeleteVolume -func TestDeleteVolume(t *testing.T) { - - // DeleteVolume(volumeID string) error - osmock.On("DeleteVolume", FakeVolID).Return(nil) - - // Init assert - assert := assert.New(t) - - // Fake request - fakeReq := &csi.DeleteVolumeRequest{ - VolumeId: FakeVolID, - } - - // Expected Result - expectedRes := &csi.DeleteVolumeResponse{} - - // Invoke DeleteVolume - actualRes, err := fakeCs.DeleteVolume(FakeCtx, fakeReq) - if err != nil { - t.Errorf("failed to DeleteVolume: %v", err) - } - - // Assert - assert.Equal(expectedRes, actualRes) -} - -// Test ControllerPublishVolume -func TestControllerPublishVolume(t *testing.T) { - - // AttachVolume(instanceID, volumeID string) (string, error) - osmock.On("AttachVolume", FakeNodeID, FakeVolID).Return(FakeVolID, nil) - // WaitDiskAttached(instanceID string, volumeID string) error - osmock.On("WaitDiskAttached", FakeNodeID, FakeVolID).Return(nil) - // GetAttachmentDiskPath(instanceID, volumeID string) (string, error) - osmock.On("GetAttachmentDiskPath", FakeNodeID, FakeVolID).Return(FakeDevicePath, nil) - - // Init assert - assert := assert.New(t) - - // Fake request - fakeReq := &csi.ControllerPublishVolumeRequest{ - VolumeId: FakeVolID, - NodeId: FakeNodeID, - VolumeCapability: &csi.VolumeCapability{ - AccessType: &csi.VolumeCapability_Mount{ - Mount: &csi.VolumeCapability_MountVolume{}, - }, - }, - Readonly: false, - } - - // Expected Result - expectedRes := &csi.ControllerPublishVolumeResponse{ - PublishContext: map[string]string{ - "DevicePath": FakeDevicePath, - }, - } - - // Invoke ControllerPublishVolume - actualRes, err := fakeCs.ControllerPublishVolume(FakeCtx, fakeReq) - if err != nil { - t.Errorf("failed to ControllerPublishVolume: %v", err) - } - - // Assert - assert.Equal(expectedRes, actualRes) -} - -// Test ControllerUnpublishVolume -func TestControllerUnpublishVolume(t *testing.T) { - - // DetachVolume(instanceID, volumeID string) error - osmock.On("DetachVolume", FakeNodeID, FakeVolID).Return(nil) - // WaitDiskDetached(instanceID string, volumeID string) error - osmock.On("WaitDiskDetached", FakeNodeID, FakeVolID).Return(nil) - - // Init assert - assert := assert.New(t) - - // Fake request - fakeReq := &csi.ControllerUnpublishVolumeRequest{ - VolumeId: FakeVolID, - NodeId: FakeNodeID, - } - - // Expected Result - expectedRes := &csi.ControllerUnpublishVolumeResponse{} - - // Invoke ControllerUnpublishVolume - actualRes, err := fakeCs.ControllerUnpublishVolume(FakeCtx, fakeReq) - if err != nil { - t.Errorf("failed to ControllerUnpublishVolume: %v", err) - } - - // Assert - assert.Equal(expectedRes, actualRes) -} - -func TestListVolumes(t *testing.T) { - - osmock.On("ListVolumes", 2, FakeVolID).Return(FakeVolListMultiple, "", nil) - - // Init assert - assert := assert.New(t) - - fakeReq := &csi.ListVolumesRequest{MaxEntries: 2, StartingToken: FakeVolID} - - // Expected Result - expectedRes := &csi.ListVolumesResponse{ - Entries: []*csi.ListVolumesResponse_Entry{ - { - Volume: &csi.Volume{ - VolumeId: FakeVol1.ID, - CapacityBytes: int64(FakeVol1.Size * 1024 * 1024 * 1024), - }, - Status: &csi.ListVolumesResponse_VolumeStatus{ - PublishedNodeIds: []string{FakeNodeID}, - }, - }, - { - Volume: &csi.Volume{ - VolumeId: FakeVol3.ID, - CapacityBytes: int64(FakeVol3.Size * 1024 * 1024 * 1024), - }, - Status: &csi.ListVolumesResponse_VolumeStatus{}, - }, - }, - NextToken: "", - } - - // Invoke ListVolumes - actualRes, err := fakeCs.ListVolumes(FakeCtx, fakeReq) - if err != nil { - t.Errorf("failed to ListVolumes: %v", err) - } - - // Assert - assert.Equal(expectedRes, actualRes) -} - -// Test CreateSnapshot -func TestCreateSnapshot(t *testing.T) { - - osmock.On("CreateSnapshot", FakeSnapshotName, FakeVolID, &map[string]string{"tag": "tag1"}).Return(&FakeSnapshotRes, nil) - osmock.On("ListSnapshots", map[string]string{"Name": FakeSnapshotName}).Return(FakeSnapshotListEmpty, "", nil) - osmock.On("WaitSnapshotReady", FakeSnapshotID).Return(nil) - - // Init assert - assert := assert.New(t) - - // Fake request - fakeReq := &csi.CreateSnapshotRequest{ - Name: FakeSnapshotName, - SourceVolumeId: FakeVolID, - Parameters: map[string]string{"tag": "tag1"}, - } - - // Invoke CreateSnapshot - actualRes, err := fakeCs.CreateSnapshot(FakeCtx, fakeReq) - if err != nil { - t.Errorf("failed to CreateSnapshot: %v", err) - } - - // Assert - assert.Equal(FakeVolID, actualRes.Snapshot.SourceVolumeId) - - assert.NotNil(FakeSnapshotID, actualRes.Snapshot.SnapshotId) -} - -// Test DeleteSnapshot -func TestDeleteSnapshot(t *testing.T) { - - // DeleteSnapshot(volumeID string) error - osmock.On("DeleteSnapshot", FakeSnapshotID).Return(nil) - - // Init assert - assert := assert.New(t) - - // Fake request - fakeReq := &csi.DeleteSnapshotRequest{ - SnapshotId: FakeSnapshotID, - } - - // Expected Result - expectedRes := &csi.DeleteSnapshotResponse{} - - // Invoke DeleteSnapshot - actualRes, err := fakeCs.DeleteSnapshot(FakeCtx, fakeReq) - if err != nil { - t.Errorf("failed to DeleteVolume: %v", err) - } - - // Assert - assert.Equal(expectedRes, actualRes) -} - -func TestListSnapshots(t *testing.T) { - - osmock.On("ListSnapshots", map[string]string{"Limit": "1", "Marker": FakeVolID, "Status": "available"}).Return(FakeSnapshotsRes, "", nil) - assert := assert.New(t) - - fakeReq := &csi.ListSnapshotsRequest{MaxEntries: 1, StartingToken: FakeVolID} - actualRes, err := fakeCs.ListSnapshots(FakeCtx, fakeReq) - if err != nil { - t.Errorf("failed to ListSnapshots: %v", err) - } - - // Assert - assert.Equal(FakeVolID, actualRes.Entries[0].Snapshot.SourceVolumeId) - assert.NotNil(FakeSnapshotID, actualRes.Entries[0].Snapshot.SnapshotId) -} - -func TestControllerExpandVolume(t *testing.T) { - - // ExpandVolume(volumeID string, size int) - osmock.On("ExpandVolume", FakeVolName, 5).Return(nil) - - // Init assert - assert := assert.New(t) - - // Fake request - fakeReq := &csi.ControllerExpandVolumeRequest{ - VolumeId: FakeVolName, - CapacityRange: &csi.CapacityRange{ - RequiredBytes: 5 * 1024 * 1024 * 1024, - }, - } - - // Expected Result - expectedRes := &csi.ControllerExpandVolumeResponse{ - CapacityBytes: 5 * 1024 * 1024 * 1024, - NodeExpansionRequired: true, - } - - // Invoke ControllerExpandVolume - actualRes, err := fakeCs.ControllerExpandVolume(FakeCtx, fakeReq) - if err != nil { - t.Errorf("failed to ExpandVolume: %v", err) - } - - // Assert - assert.Equal(expectedRes, actualRes) - -} - -func TestValidateVolumeCapabilities(t *testing.T) { - - // GetVolume(volumeID string) - osmock.On("GetVolume", FakeVolID).Return(FakeVol1) - - // Init assert - assert := assert.New(t) - - // fake req - fakereq := &csi.ValidateVolumeCapabilitiesRequest{ - VolumeId: FakeVolID, - VolumeCapabilities: []*csi.VolumeCapability{ - { - AccessMode: &csi.VolumeCapability_AccessMode{ - Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, - }, - }, - }, - } - - // expected result - expectedRes := &csi.ValidateVolumeCapabilitiesResponse{ - Confirmed: &csi.ValidateVolumeCapabilitiesResponse_Confirmed{ - VolumeCapabilities: []*csi.VolumeCapability{ - { - AccessMode: &csi.VolumeCapability_AccessMode{ - Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, - }, - }, - }, - }, - } - - // Testing Negative case - fakereq2 := &csi.ValidateVolumeCapabilitiesRequest{ - VolumeId: FakeVolID, - VolumeCapabilities: []*csi.VolumeCapability{ - { - AccessMode: &csi.VolumeCapability_AccessMode{ - Mode: csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY, - }, - }, - }, - } - - expectedRes2 := &csi.ValidateVolumeCapabilitiesResponse{Message: "Requested Volume Capabilty not supported"} - - // Invoke ValidateVolumeCapabilties - actualRes, err := fakeCs.ValidateVolumeCapabilities(FakeCtx, fakereq) - actualRes2, err := fakeCs.ValidateVolumeCapabilities(FakeCtx, fakereq2) - - if err != nil { - t.Errorf("failed to ValidateVolumeCapabilties: %v", err) - } - - // assert - assert.Equal(expectedRes, actualRes) - assert.Equal(expectedRes2, actualRes2) - -} diff --git a/pkg/csi/cinder/driver.go b/pkg/csi/cinder/driver.go deleted file mode 100644 index 695823a2f0..0000000000 --- a/pkg/csi/cinder/driver.go +++ /dev/null @@ -1,157 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cinder - -import ( - "fmt" - - "github.com/container-storage-interface/spec/lib/go/csi" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - "k8s.io/cloud-provider-openstack/pkg/csi/cinder/openstack" - "k8s.io/cloud-provider-openstack/pkg/util/metadata" - "k8s.io/cloud-provider-openstack/pkg/util/mount" - "k8s.io/cloud-provider-openstack/pkg/version" - "k8s.io/klog/v2" -) - -const ( - driverName = "cinder.csi.openstack.org" - topologyKey = "topology." + driverName + "/zone" -) - -var ( - specVersion = "1.2.0" - // we used to use spec version as driver version, now separate them - Version = "1.2.1" -) - -type CinderDriver struct { - name string - nodeID string - fqVersion string //Fully qualified version in format {Version}@{CPO version} - endpoint string - cloudconfig string - cluster string - - ids *identityServer - cs *controllerServer - ns *nodeServer - - vcap []*csi.VolumeCapability_AccessMode - cscap []*csi.ControllerServiceCapability - nscap []*csi.NodeServiceCapability -} - -func NewDriver(nodeID, endpoint, cluster string) *CinderDriver { - - d := &CinderDriver{} - d.name = driverName - d.nodeID = nodeID - d.fqVersion = fmt.Sprintf("%s@%s", Version, version.Version) - d.endpoint = endpoint - d.cluster = cluster - - klog.Info("Driver: ", d.name) - klog.Info("Driver version: ", d.fqVersion) - klog.Info("CSI Spec version: ", specVersion) - - d.AddControllerServiceCapabilities( - []csi.ControllerServiceCapability_RPC_Type{ - csi.ControllerServiceCapability_RPC_LIST_VOLUMES, - csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME, - csi.ControllerServiceCapability_RPC_PUBLISH_UNPUBLISH_VOLUME, - csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT, - csi.ControllerServiceCapability_RPC_LIST_SNAPSHOTS, - csi.ControllerServiceCapability_RPC_EXPAND_VOLUME, - csi.ControllerServiceCapability_RPC_CLONE_VOLUME, - csi.ControllerServiceCapability_RPC_LIST_VOLUMES_PUBLISHED_NODES, - }) - d.AddVolumeCapabilityAccessModes([]csi.VolumeCapability_AccessMode_Mode{csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER}) - - d.AddNodeServiceCapabilities( - []csi.NodeServiceCapability_RPC_Type{ - csi.NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME, - csi.NodeServiceCapability_RPC_EXPAND_VOLUME, - csi.NodeServiceCapability_RPC_GET_VOLUME_STATS, - }) - - return d -} - -func (d *CinderDriver) AddControllerServiceCapabilities(cl []csi.ControllerServiceCapability_RPC_Type) { - var csc []*csi.ControllerServiceCapability - - for _, c := range cl { - klog.Infof("Enabling controller service capability: %v", c.String()) - csc = append(csc, NewControllerServiceCapability(c)) - } - - d.cscap = csc - - return -} - -func (d *CinderDriver) AddVolumeCapabilityAccessModes(vc []csi.VolumeCapability_AccessMode_Mode) []*csi.VolumeCapability_AccessMode { - var vca []*csi.VolumeCapability_AccessMode - for _, c := range vc { - klog.Infof("Enabling volume access mode: %v", c.String()) - vca = append(vca, NewVolumeCapabilityAccessMode(c)) - } - d.vcap = vca - return vca -} - -func (d *CinderDriver) AddNodeServiceCapabilities(nl []csi.NodeServiceCapability_RPC_Type) error { - var nsc []*csi.NodeServiceCapability - for _, n := range nl { - klog.Infof("Enabling node service capability: %v", n.String()) - nsc = append(nsc, NewNodeServiceCapability(n)) - } - d.nscap = nsc - return nil -} - -func (d *CinderDriver) ValidateControllerServiceRequest(c csi.ControllerServiceCapability_RPC_Type) error { - if c == csi.ControllerServiceCapability_RPC_UNKNOWN { - return nil - } - - for _, cap := range d.cscap { - if c == cap.GetRpc().GetType() { - return nil - } - } - return status.Error(codes.InvalidArgument, fmt.Sprintf("%s", c)) -} - -func (d *CinderDriver) GetVolumeCapabilityAccessModes() []*csi.VolumeCapability_AccessMode { - return d.vcap -} - -func (d *CinderDriver) SetupDriver(cloud openstack.IOpenStack, mount mount.IMount, metadata metadata.IMetadata) { - - d.ids = NewIdentityServer(d) - d.cs = NewControllerServer(d, cloud) - d.ns = NewNodeServer(d, mount, metadata, cloud) - -} - -func (d *CinderDriver) Run() { - - RunControllerandNodePublishServer(d.endpoint, d.ids, d.cs, d.ns) -} diff --git a/pkg/csi/cinder/driver_test.go b/pkg/csi/cinder/driver_test.go deleted file mode 100644 index 7ce0e1b843..0000000000 --- a/pkg/csi/cinder/driver_test.go +++ /dev/null @@ -1,66 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cinder - -import ( - "testing" - - "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/stretchr/testify/assert" -) - -const ( - fakeDriverName = "fake" -) - -var ( - vendorVersion = "1.2.1" -) - -func NewFakeDriver() *CinderDriver { - - driver := NewDriver(FakeNodeID, FakeEndpoint, FakeCluster) - - return driver -} -func TestValidateControllerServiceRequest(t *testing.T) { - d := NewFakeDriver() - - // Valid requests which require no capabilities - err := d.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_UNKNOWN) - assert.NoError(t, err) - - // Test controller service publish/unpublish is supported - err = d.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_PUBLISH_UNPUBLISH_VOLUME) - assert.NoError(t, err) - - // Test controller service create/delete is supported - err = d.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME) - assert.NoError(t, err) - - // Test controller service list volumes is supported - err = d.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_LIST_VOLUMES) - assert.NoError(t, err) - - // Test controller service create/delete snapshot is supported - err = d.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT) - assert.NoError(t, err) - - // Test controller service list snapshot is supported - err = d.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_LIST_SNAPSHOTS) - assert.NoError(t, err) -} diff --git a/pkg/csi/cinder/etc/cloud.conf b/pkg/csi/cinder/etc/cloud.conf deleted file mode 100644 index 3ddbd7203e..0000000000 --- a/pkg/csi/cinder/etc/cloud.conf +++ /dev/null @@ -1,6 +0,0 @@ -[Global] -username=user -password=pass -auth-url=https:///identity/v3 -tenant-id=c869168a828847f39f7f06edd7305637 -domain-id=2a73b8f597c04551a0fdc8e95544be8a diff --git a/pkg/csi/cinder/fake.go b/pkg/csi/cinder/fake.go deleted file mode 100644 index 09d3c466c3..0000000000 --- a/pkg/csi/cinder/fake.go +++ /dev/null @@ -1,120 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cinder - -import ( - "github.com/gophercloud/gophercloud/openstack/blockstorage/v3/snapshots" - "github.com/gophercloud/gophercloud/openstack/blockstorage/v3/volumes" - "golang.org/x/net/context" - "k8s.io/cloud-provider-openstack/pkg/util/mount" -) - -var FakeCluster = "cluster" -var FakeNodeID = "CSINodeID" -var FakeEndpoint = "tcp://127.0.0.1:10000" -var FakeConfig = "/etc/cloud.conf" -var FakeCtx = context.Background() -var FakeVolName = "CSIVolumeName" -var FakeVolID = "CSIVolumeID" -var FakeSnapshotName = "CSISnapshotName" -var FakeSnapshotID = "261a8b81-3660-43e5-bab8-6470b65ee4e8" -var FakeCapacityGiB = 1 -var FakeVolType = "" -var FakeAvailability = "nova" -var FakeDevicePath = "/dev/xxx" -var FakeTargetPath = "/mnt/cinder" -var FakeStagingTargetPath = "/mnt/globalmount" -var FakeAttachment = volumes.Attachment{ - ServerID: FakeNodeID, -} -var FakeVol = volumes.Volume{ - ID: FakeVolID, - Name: FakeVolName, - Size: FakeCapacityGiB, - AvailabilityZone: FakeAvailability, -} -var FakeVolFromSnapshot = volumes.Volume{ - ID: FakeVolID, - Name: FakeVolName, - Size: FakeCapacityGiB, - AvailabilityZone: FakeAvailability, - SnapshotID: FakeSnapshotID, -} - -var FakeVolFromSourceVolume = volumes.Volume{ - ID: "test-clone-id", - Name: FakeVolName, - Size: FakeCapacityGiB, - AvailabilityZone: FakeAvailability, - SourceVolID: FakeVolID, -} - -var FakeVol1 = volumes.Volume{ - ID: FakeVolID, - Name: "fake-duplicate", - Status: "available", - AvailabilityZone: FakeAvailability, - Size: FakeCapacityGiB, - Attachments: []volumes.Attachment{FakeAttachment}, -} -var FakeVol2 = volumes.Volume{ - ID: "261a8b81-3660-43e5-bab8-6470b65ee4e9", - Name: "fake-duplicate", - Status: "available", - AvailabilityZone: "", -} -var FakeVol3 = volumes.Volume{ - ID: "261a8b81-3660-43e5-bab8-6470b65ee4e9", - Name: "fake-3", - Status: "available", - Size: 2, - AvailabilityZone: "", -} -var FakeSnapshotRes = snapshots.Snapshot{ - ID: FakeSnapshotID, - Name: "fake-snapshot", - VolumeID: FakeVolID, - Size: 1, -} - -var FakeSnapshotsRes = []snapshots.Snapshot{FakeSnapshotRes} - -var FakeVolListMultiple = []volumes.Volume{FakeVol1, FakeVol3} -var FakeVolList = []volumes.Volume{FakeVol1} -var FakeVolListEmpty = []volumes.Volume{} -var FakeSnapshotListEmpty = []snapshots.Snapshot{} - -var FakeInstanceID = "321a8b81-3660-43e5-bab8-6470b65ee4e8" - -const FakeMaxVolume int64 = 256 - -var FakeFsStats = &mount.DeviceStats{ - Block: false, - - AvailableBytes: 2100, - TotalBytes: 2121, - UsedBytes: 21, - AvailableInodes: 150, - TotalInodes: 200, - UsedInodes: 50, -} - -var FakeBlockDeviceStats = &mount.DeviceStats{ - Block: true, - - TotalBytes: 536870912, -} diff --git a/pkg/csi/cinder/identityserver.go b/pkg/csi/cinder/identityserver.go deleted file mode 100644 index 99efa7637b..0000000000 --- a/pkg/csi/cinder/identityserver.go +++ /dev/null @@ -1,97 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cinder - -import ( - "github.com/container-storage-interface/spec/lib/go/csi" - "golang.org/x/net/context" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - "k8s.io/cloud-provider-openstack/pkg/csi/cinder/openstack" - "k8s.io/klog/v2" -) - -type identityServer struct { - Driver *CinderDriver -} - -func (ids *identityServer) GetPluginInfo(ctx context.Context, req *csi.GetPluginInfoRequest) (*csi.GetPluginInfoResponse, error) { - klog.V(5).Infof("Using default GetPluginInfo") - - if ids.Driver.name == "" { - return nil, status.Error(codes.Unavailable, "Driver name not configured") - } - - if ids.Driver.fqVersion == "" { - return nil, status.Error(codes.Unavailable, "Driver is missing version") - } - - return &csi.GetPluginInfoResponse{ - Name: ids.Driver.name, - VendorVersion: ids.Driver.fqVersion, - }, nil -} - -func (ids *identityServer) Probe(ctx context.Context, req *csi.ProbeRequest) (*csi.ProbeResponse, error) { - klog.V(5).Infof("Probe() called with req %+v", req) - oProvider, err := openstack.GetOpenStackProvider() - if err != nil { - klog.Errorf("Failed to GetOpenStackProvider: %v", err) - return nil, status.Error(codes.FailedPrecondition, "Failed to retrieve openstack provider") - } - if err := oProvider.CheckBlockStorageAPI(); err != nil { - klog.Errorf("Failed to query blockstorage API: %v", err) - return nil, status.Error(codes.FailedPrecondition, "Failed to communicate with Openstack BlockStorage API") - } - return &csi.ProbeResponse{}, nil -} - -func (ids *identityServer) GetPluginCapabilities(ctx context.Context, req *csi.GetPluginCapabilitiesRequest) (*csi.GetPluginCapabilitiesResponse, error) { - klog.V(5).Infof("GetPluginCapabilities called with req %+v", req) - return &csi.GetPluginCapabilitiesResponse{ - Capabilities: []*csi.PluginCapability{ - { - Type: &csi.PluginCapability_Service_{ - Service: &csi.PluginCapability_Service{ - Type: csi.PluginCapability_Service_CONTROLLER_SERVICE, - }, - }, - }, - { - Type: &csi.PluginCapability_Service_{ - Service: &csi.PluginCapability_Service{ - Type: csi.PluginCapability_Service_VOLUME_ACCESSIBILITY_CONSTRAINTS, - }, - }, - }, - { - Type: &csi.PluginCapability_VolumeExpansion_{ - VolumeExpansion: &csi.PluginCapability_VolumeExpansion{ - Type: csi.PluginCapability_VolumeExpansion_ONLINE, - }, - }, - }, - { - Type: &csi.PluginCapability_VolumeExpansion_{ - VolumeExpansion: &csi.PluginCapability_VolumeExpansion{ - Type: csi.PluginCapability_VolumeExpansion_OFFLINE, - }, - }, - }, - }, - }, nil -} diff --git a/pkg/csi/cinder/identityserver_test.go b/pkg/csi/cinder/identityserver_test.go deleted file mode 100644 index f1213ebb49..0000000000 --- a/pkg/csi/cinder/identityserver_test.go +++ /dev/null @@ -1,40 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cinder - -import ( - "context" - "fmt" - "testing" - - "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/stretchr/testify/assert" -) - -func TestGetPluginInfo(t *testing.T) { - d := NewDriver(FakeNodeID, FakeEndpoint, FakeCluster) - - ids := NewIdentityServer(d) - - req := csi.GetPluginInfoRequest{} - resp, err := ids.GetPluginInfo(context.Background(), &req) - assert.NoError(t, err) - assert.Equal(t, resp.GetName(), driverName) - - fqVersion := fmt.Sprintf("%s@", vendorVersion) - assert.Equal(t, resp.GetVendorVersion(), fqVersion) -} diff --git a/pkg/csi/cinder/nodeserver.go b/pkg/csi/cinder/nodeserver.go deleted file mode 100644 index e5d772bd6d..0000000000 --- a/pkg/csi/cinder/nodeserver.go +++ /dev/null @@ -1,551 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cinder - -import ( - "fmt" - "os" - "path/filepath" - "strconv" - "strings" - - "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/gophercloud/gophercloud/openstack/blockstorage/v3/volumes" - "golang.org/x/net/context" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - "k8s.io/klog/v2" - "k8s.io/kubernetes/pkg/util/resizefs" - utilpath "k8s.io/utils/path" - - "k8s.io/cloud-provider-openstack/pkg/csi/cinder/openstack" - "k8s.io/cloud-provider-openstack/pkg/util/blockdevice" - cpoerrors "k8s.io/cloud-provider-openstack/pkg/util/errors" - "k8s.io/cloud-provider-openstack/pkg/util/metadata" - "k8s.io/cloud-provider-openstack/pkg/util/mount" -) - -type nodeServer struct { - Driver *CinderDriver - Mount mount.IMount - Metadata metadata.IMetadata - Cloud openstack.IOpenStack -} - -func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { - klog.V(4).Infof("NodePublishVolume: called with args %+v", *req) - - volumeID := req.GetVolumeId() - source := req.GetStagingTargetPath() - targetPath := req.GetTargetPath() - volumeCapability := req.GetVolumeCapability() - - if len(volumeID) == 0 { - return nil, status.Error(codes.InvalidArgument, "NodePublishVolume Volume ID must be provided") - } - if len(targetPath) == 0 { - return nil, status.Error(codes.InvalidArgument, "NodePublishVolume Target Path must be provided") - } - if volumeCapability == nil { - return nil, status.Error(codes.InvalidArgument, "NodePublishVolume Volume Capability must be provided") - } - - ephemeralVolume := req.GetVolumeContext()["csi.storage.k8s.io/ephemeral"] == "true" - if ephemeralVolume { - return nodePublishEphermeral(req, ns) - } - - // In case of ephemeral volume staging path not provided - if len(source) == 0 { - return nil, status.Error(codes.InvalidArgument, "NodePublishVolume Staging Target Path must be provided") - } - _, err := ns.Cloud.GetVolume(volumeID) - if err != nil { - if cpoerrors.IsNotFound(err) { - return nil, status.Error(codes.NotFound, "Volume not found") - } - return nil, status.Error(codes.Internal, fmt.Sprintf("GetVolume failed with error %v", err)) - } - - mountOptions := []string{"bind"} - if req.GetReadonly() { - mountOptions = append(mountOptions, "ro") - } else { - mountOptions = append(mountOptions, "rw") - } - - if blk := volumeCapability.GetBlock(); blk != nil { - return nodePublishVolumeForBlock(req, ns, mountOptions) - } - - m := ns.Mount - // Verify whether mounted - notMnt, err := m.IsLikelyNotMountPointAttach(targetPath) - if err != nil { - return nil, status.Error(codes.Internal, err.Error()) - } - - // Volume Mount - if notMnt { - fsType := "ext4" - if mnt := volumeCapability.GetMount(); mnt != nil { - if mnt.FsType != "" { - fsType = mnt.FsType - } - } - // Mount - err = m.Mounter().Mount(source, targetPath, fsType, mountOptions) - if err != nil { - return nil, status.Error(codes.Internal, err.Error()) - } - } - - return &csi.NodePublishVolumeResponse{}, nil -} - -func nodePublishEphermeral(req *csi.NodePublishVolumeRequest, ns *nodeServer) (*csi.NodePublishVolumeResponse, error) { - - var size int - var err error - - volID := req.GetVolumeId() - volName := fmt.Sprintf("ephemeral-%s", volID) - properties := map[string]string{"cinder.csi.openstack.org/cluster": ns.Driver.cluster} - capacity, ok := req.GetVolumeContext()["capacity"] - volumeCapability := req.GetVolumeCapability() - - volAvailability, err := ns.Metadata.GetAvailabilityZone() - if err != nil { - return nil, status.Error(codes.Internal, fmt.Sprintf("retrieving availability zone from MetaData service failed with error %v", err)) - } - - size = 1 // default size is 1GB - if ok && strings.HasSuffix(capacity, "Gi") { - size, err = strconv.Atoi(strings.TrimSuffix(capacity, "Gi")) - if err != nil { - klog.V(3).Infof("Unable to parse capacity: %v", err) - return nil, status.Error(codes.Internal, fmt.Sprintf("Unable to parse capacity %v", err)) - } - } - - // TODO: Add Volume type support - evol, err := ns.Cloud.CreateVolume(volName, size, "", volAvailability, "", "", &properties) - - if err != nil { - klog.V(3).Infof("Failed to Create Ephermal Volume: %v", err) - return nil, status.Error(codes.Internal, fmt.Sprintf("Failed to create Ephermal Volume %v", err)) - } - - klog.V(4).Infof("Ephermeral Volume %s is created", evol.ID) - - // attach volume - // for attach volume we need to have information about node. - nodeID, err := ns.Metadata.GetInstanceID() - if err != nil { - klog.V(3).Infof("Ephermal Volume Attach: Failed to get Instance ID: %v", err) - return nil, status.Error(codes.Internal, fmt.Sprintf("Ephermal Volume Attach: Failed to get Instance ID: %v", err)) - } - - _, err = ns.Cloud.AttachVolume(nodeID, evol.ID) - if err != nil { - klog.V(3).Infof("Ephermal Volume Attach: %v", err) - return nil, status.Error(codes.Internal, fmt.Sprintf("Ephermal Volume Attach: Failed to Attach Volume: %v", err)) - } - - err = ns.Cloud.WaitDiskAttached(nodeID, evol.ID) - if err != nil { - klog.V(3).Infof("Ephermal Volume Attach: %v", err) - return nil, status.Error(codes.Internal, fmt.Sprintf("Ephermal Volume Attach: Failed to Attach Volume: %v", err)) - } - - m := ns.Mount - - devicePath, err := getDevicePath(evol.ID, m) - if devicePath == "" { - return nil, status.Error(codes.Internal, "Unable to find Device path for volume") - } - - targetPath := req.GetTargetPath() - - // Verify whether mounted - notMnt, err := m.IsLikelyNotMountPointAttach(targetPath) - if err != nil { - return nil, status.Error(codes.Internal, err.Error()) - } - - // Volume Mount - if notMnt { - // set default fstype is ext4 - fsType := "ext4" - var options []string - if mnt := volumeCapability.GetMount(); mnt != nil { - if mnt.FsType != "" { - fsType = mnt.FsType - } - mountFlags := mnt.GetMountFlags() - options = append(options, mountFlags...) - } - // Mount - err = m.Mounter().FormatAndMount(devicePath, targetPath, fsType, nil) - if err != nil { - return nil, status.Error(codes.Internal, err.Error()) - } - } - - return &csi.NodePublishVolumeResponse{}, nil - -} - -func nodePublishVolumeForBlock(req *csi.NodePublishVolumeRequest, ns *nodeServer, mountOptions []string) (*csi.NodePublishVolumeResponse, error) { - klog.V(4).Infof("NodePublishVolumeBlock: called with args %+v", *req) - - volumeID := req.GetVolumeId() - targetPath := req.GetTargetPath() - podVolumePath := filepath.Dir(targetPath) - - m := ns.Mount - - // Do not trust the path provided by cinder, get the real path on node - source, err := getDevicePath(volumeID, m) - if source == "" { - return nil, status.Error(codes.Internal, "Unable to find Device path for volume") - } - - exists, err := utilpath.Exists(utilpath.CheckFollowSymlink, podVolumePath) - if err != nil { - return nil, status.Error(codes.Internal, err.Error()) - } - if !exists { - if err := m.MakeDir(podVolumePath); err != nil { - return nil, status.Errorf(codes.Internal, "Could not create dir %q: %v", podVolumePath, err) - } - } - err = m.MakeFile(targetPath) - if err != nil { - return nil, status.Errorf(codes.Internal, "Error in making file %v", err) - } - - if err := m.Mounter().Mount(source, targetPath, "", mountOptions); err != nil { - if removeErr := os.Remove(targetPath); removeErr != nil { - return nil, status.Errorf(codes.Internal, "Could not remove mount target %q: %v", targetPath, err) - } - return nil, status.Errorf(codes.Internal, "Could not mount %q at %q: %v", source, targetPath, err) - } - - return &csi.NodePublishVolumeResponse{}, nil -} - -func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) { - klog.V(4).Infof("NodeUnPublishVolume: called with args %+v", *req) - - volumeID := req.GetVolumeId() - targetPath := req.GetTargetPath() - if len(targetPath) == 0 { - return nil, status.Error(codes.InvalidArgument, "[NodeUnpublishVolume] Target Path must be provided") - } - if len(volumeID) == 0 { - return nil, status.Error(codes.InvalidArgument, "[NodeUnpublishVolume] volumeID must be provided") - } - - ephemeralVolume := false - - vol, err := ns.Cloud.GetVolume(volumeID) - - if err != nil { - - if !cpoerrors.IsNotFound(err) { - return nil, status.Error(codes.Internal, fmt.Sprintf("GetVolume failed with error %v", err)) - } - - // if not found by id, try to search by name - volName := fmt.Sprintf("ephemeral-%s", volumeID) - - vols, err := ns.Cloud.GetVolumesByName(volName) - - //if volume not found then GetVolumesByName returns empty list - if err != nil { - return nil, status.Error(codes.Internal, fmt.Sprintf("GetVolume failed with error %v", err)) - } - if len(vols) > 0 { - vol = &vols[0] - ephemeralVolume = true - } else { - return nil, status.Error(codes.NotFound, fmt.Sprintf("Volume not found %s", volName)) - } - } - - err = ns.Mount.UnmountPath(targetPath) - if err != nil { - return nil, status.Errorf(codes.Internal, "Unmount of targetpath %s failed with error %v", targetPath, err) - } - - if ephemeralVolume { - return nodeUnpublishEphermeral(req, ns, vol) - } - - return &csi.NodeUnpublishVolumeResponse{}, nil - -} - -func nodeUnpublishEphermeral(req *csi.NodeUnpublishVolumeRequest, ns *nodeServer, vol *volumes.Volume) (*csi.NodeUnpublishVolumeResponse, error) { - volumeID := vol.ID - var instanceID string - - if len(vol.Attachments) > 0 { - instanceID = vol.Attachments[0].ServerID - } else { - return nil, status.Error(codes.FailedPrecondition, "Volume attachement not found in request") - } - - err := ns.Cloud.DetachVolume(instanceID, volumeID) - if err != nil { - klog.V(3).Infof("Failed to DetachVolume: %v", err) - return nil, status.Error(codes.Internal, err.Error()) - } - - err = ns.Cloud.WaitDiskDetached(instanceID, volumeID) - if err != nil { - klog.V(3).Infof("Failed to WaitDiskDetached: %v", err) - return nil, status.Error(codes.Internal, err.Error()) - } - - err = ns.Cloud.DeleteVolume(volumeID) - if err != nil { - klog.V(3).Infof("Failed to DeleteVolume: %v", err) - return nil, status.Error(codes.Internal, err.Error()) - } - - return &csi.NodeUnpublishVolumeResponse{}, nil -} - -func (ns *nodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) { - klog.V(4).Infof("NodeStageVolume: called with args %+v", *req) - - stagingTarget := req.GetStagingTargetPath() - volumeCapability := req.GetVolumeCapability() - volumeID := req.GetVolumeId() - - if len(volumeID) == 0 { - return nil, status.Error(codes.InvalidArgument, "Volume Id not provided") - } - - if len(stagingTarget) == 0 { - return nil, status.Error(codes.InvalidArgument, "Staging target not provided") - } - if volumeCapability == nil { - return nil, status.Error(codes.InvalidArgument, "NodeStageVolume Volume Capability must be provided") - } - - _, err := ns.Cloud.GetVolume(volumeID) - if err != nil { - if cpoerrors.IsNotFound(err) { - return nil, status.Error(codes.NotFound, "Volume not found") - } - return nil, status.Error(codes.Internal, fmt.Sprintf("GetVolume failed with error %v", err)) - } - - m := ns.Mount - // Do not trust the path provided by cinder, get the real path on node - devicePath, err := getDevicePath(volumeID, m) - if devicePath == "" { - return nil, status.Error(codes.Internal, "Unable to find Device path for volume") - } - - if blk := volumeCapability.GetBlock(); blk != nil { - // If block volume, do nothing - return &csi.NodeStageVolumeResponse{}, nil - } - - // Verify whether mounted - notMnt, err := m.IsLikelyNotMountPointAttach(stagingTarget) - if err != nil { - return nil, status.Error(codes.Internal, err.Error()) - } - - // Volume Mount - if notMnt { - // set default fstype is ext4 - fsType := "ext4" - var options []string - if mnt := volumeCapability.GetMount(); mnt != nil { - if mnt.FsType != "" { - fsType = mnt.FsType - } - mountFlags := mnt.GetMountFlags() - options = append(options, mountFlags...) - } - // Mount - err = m.Mounter().FormatAndMount(devicePath, stagingTarget, fsType, options) - if err != nil { - return nil, status.Error(codes.Internal, err.Error()) - } - } - - return &csi.NodeStageVolumeResponse{}, nil -} - -func (ns *nodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) { - klog.V(4).Infof("NodeUnstageVolume: called with args %+v", *req) - - volumeID := req.GetVolumeId() - if len(volumeID) == 0 { - return nil, status.Error(codes.InvalidArgument, "Volume Id not provided") - } - - stagingTargetPath := req.GetStagingTargetPath() - if len(stagingTargetPath) == 0 { - return nil, status.Error(codes.InvalidArgument, "NodeUnstageVolume Staging Target Path must be provided") - } - - _, err := ns.Cloud.GetVolume(volumeID) - if err != nil { - if cpoerrors.IsNotFound(err) { - klog.V(4).Infof("NodeUnstageVolume: Unable to find volume: %v", err) - return nil, status.Error(codes.NotFound, "Volume not found") - } - return nil, status.Error(codes.Internal, fmt.Sprintf("GetVolume failed with error %v", err)) - } - - err = ns.Mount.UnmountPath(stagingTargetPath) - if err != nil { - return nil, status.Errorf(codes.Internal, "Unmount of targetPath %s failed with error %v", stagingTargetPath, err) - } - - return &csi.NodeUnstageVolumeResponse{}, nil -} - -func (ns *nodeServer) NodeGetInfo(ctx context.Context, req *csi.NodeGetInfoRequest) (*csi.NodeGetInfoResponse, error) { - - nodeID, err := ns.Metadata.GetInstanceID() - if err != nil { - return nil, status.Error(codes.Internal, fmt.Sprintf("[NodeGetInfo] unable to retrieve instance id of node %v", err)) - } - - zone, err := ns.Metadata.GetAvailabilityZone() - if err != nil { - return nil, status.Error(codes.Internal, fmt.Sprintf("[NodeGetInfo] Unable to retrieve availability zone of node %v", err)) - } - topology := &csi.Topology{Segments: map[string]string{topologyKey: zone}} - - maxVolume := ns.Cloud.GetMaxVolLimit() - - return &csi.NodeGetInfoResponse{ - NodeId: nodeID, - AccessibleTopology: topology, - MaxVolumesPerNode: maxVolume, - }, nil -} - -func (ns *nodeServer) NodeGetCapabilities(ctx context.Context, req *csi.NodeGetCapabilitiesRequest) (*csi.NodeGetCapabilitiesResponse, error) { - klog.V(5).Infof("NodeGetCapabilities called with req: %#v", req) - - return &csi.NodeGetCapabilitiesResponse{ - Capabilities: ns.Driver.nscap, - }, nil -} - -func (ns *nodeServer) NodeGetVolumeStats(_ context.Context, req *csi.NodeGetVolumeStatsRequest) (*csi.NodeGetVolumeStatsResponse, error) { - klog.V(4).Infof("NodeGetVolumeStats: called with args %+v", *req) - - volumeID := req.GetVolumeId() - if len(volumeID) == 0 { - return nil, status.Error(codes.InvalidArgument, "Volume Id not provided") - } - - volumePath := req.GetVolumePath() - if len(volumePath) == 0 { - return nil, status.Error(codes.InvalidArgument, "Volume path not provided") - } - - exists, err := utilpath.Exists(utilpath.CheckFollowSymlink, req.VolumePath) - if err != nil { - return nil, status.Errorf(codes.Internal, "failed to check whether volumePath exists: %s", err) - } - if !exists { - return nil, status.Errorf(codes.NotFound, "target: %s not found", volumePath) - } - stats, err := ns.Mount.GetDeviceStats(volumePath) - if err != nil { - return nil, status.Errorf(codes.Internal, "failed to get stats by path: %s", err) - } - - if stats.Block { - return &csi.NodeGetVolumeStatsResponse{ - Usage: []*csi.VolumeUsage{ - { - Total: stats.TotalBytes, - Unit: csi.VolumeUsage_BYTES, - }, - }, - }, nil - } - - return &csi.NodeGetVolumeStatsResponse{ - Usage: []*csi.VolumeUsage{ - {Total: stats.TotalBytes, Available: stats.AvailableBytes, Used: stats.UsedBytes, Unit: csi.VolumeUsage_BYTES}, - {Total: stats.TotalInodes, Available: stats.AvailableInodes, Used: stats.UsedInodes, Unit: csi.VolumeUsage_INODES}, - }, - }, nil -} - -func (ns *nodeServer) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandVolumeRequest) (*csi.NodeExpandVolumeResponse, error) { - klog.V(4).Infof("NodeExpandVolume: called with args %+v", *req) - - volumeID := req.GetVolumeId() - if len(volumeID) == 0 { - return nil, status.Error(codes.InvalidArgument, "Volume ID not provided") - } - volumePath := req.GetVolumePath() - - args := []string{"-o", "source", "--noheadings", "--target", volumePath} - output, err := ns.Mount.Mounter().Exec.Command("findmnt", args...).CombinedOutput() - if err != nil { - return nil, status.Errorf(codes.Internal, "Could not determine device path: %v", err) - - } - devicePath := strings.TrimSpace(string(output)) - - if devicePath == "" { - return nil, status.Error(codes.Internal, "Unable to find Device path for volume") - } - - if ns.Cloud.GetBlockStorageOpts().RescanOnResize { - // comparing current volume size with the expected one - newSize := req.GetCapacityRange().GetRequiredBytes() - if err := blockdevice.RescanBlockDeviceGeometry(devicePath, volumePath, newSize); err != nil { - return nil, status.Errorf(codes.Internal, "Could not verify %q volume size: %v", volumeID, err) - } - } - - r := resizefs.NewResizeFs(ns.Mount.Mounter()) - if _, err := r.Resize(devicePath, volumePath); err != nil { - return nil, status.Errorf(codes.Internal, "Could not resize volume %q: %v", volumeID, err) - } - return &csi.NodeExpandVolumeResponse{}, nil -} - -func getDevicePath(volumeID string, m mount.IMount) (string, error) { - var devicePath string - devicePath, _ = m.GetDevicePath(volumeID) - if devicePath == "" { - // try to get from metadata service - devicePath = metadata.GetDevicePath(volumeID) - } - - return devicePath, nil - -} diff --git a/pkg/csi/cinder/nodeserver_test.go b/pkg/csi/cinder/nodeserver_test.go deleted file mode 100644 index 7417dbf725..0000000000 --- a/pkg/csi/cinder/nodeserver_test.go +++ /dev/null @@ -1,448 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cinder - -import ( - "fmt" - "io/ioutil" - "os" - "path/filepath" - "testing" - - "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/stretchr/testify/assert" - "k8s.io/cloud-provider-openstack/pkg/csi/cinder/openstack" - "k8s.io/cloud-provider-openstack/pkg/util/metadata" - "k8s.io/cloud-provider-openstack/pkg/util/mount" -) - -var fakeNs *nodeServer -var mmock *mount.MountMock -var metamock *metadata.MetadataMock -var omock *openstack.OpenStackMock - -// Init Node Server -func init() { - if fakeNs == nil { - - d := NewDriver(FakeNodeID, FakeEndpoint, FakeCluster) - - // mock MountMock - mmock = new(mount.MountMock) - mount.MInstance = mmock - - metamock = new(metadata.MetadataMock) - metadata.MetadataService = metamock - - omock = new(openstack.OpenStackMock) - openstack.OsInstance = omock - - fakeNs = NewNodeServer(d, mount.MInstance, metadata.MetadataService, openstack.OsInstance) - } -} - -// Test NodeGetInfo -func TestNodeGetInfo(t *testing.T) { - - metamock.On("GetInstanceID").Return(FakeNodeID, nil) - metamock.On("GetAvailabilityZone").Return(FakeAvailability, nil) - omock.On("GetMaxVolumeLimit").Return(FakeMaxVolume) - - // Init assert - assert := assert.New(t) - - // Expected Result - expectedRes := &csi.NodeGetInfoResponse{ - NodeId: FakeNodeID, - AccessibleTopology: &csi.Topology{Segments: map[string]string{topologyKey: FakeAvailability}}, - MaxVolumesPerNode: FakeMaxVolume, - } - - // Fake request - fakeReq := &csi.NodeGetInfoRequest{} - - // Invoke NodeGetId - actualRes, err := fakeNs.NodeGetInfo(FakeCtx, fakeReq) - if err != nil { - t.Errorf("failed to NodeGetInfo: %v", err) - } - - // Assert - assert.Equal(expectedRes, actualRes) -} - -// Test NodePublishVolume -func TestNodePublishVolume(t *testing.T) { - - mmock.On("ScanForAttach", FakeDevicePath).Return(nil) - mmock.On("IsLikelyNotMountPointAttach", FakeTargetPath).Return(true, nil) - omock.On("GetVolume", FakeVolID).Return(FakeVol, nil) - // Init assert - assert := assert.New(t) - - // Expected Result - expectedRes := &csi.NodePublishVolumeResponse{} - stdVolCap := &csi.VolumeCapability{ - AccessType: &csi.VolumeCapability_Mount{ - Mount: &csi.VolumeCapability_MountVolume{}, - }, - AccessMode: &csi.VolumeCapability_AccessMode{ - Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, - }, - } - // Fake request - fakeReq := &csi.NodePublishVolumeRequest{ - VolumeId: FakeVolID, - PublishContext: map[string]string{"DevicePath": FakeDevicePath}, - TargetPath: FakeTargetPath, - StagingTargetPath: FakeStagingTargetPath, - VolumeCapability: stdVolCap, - Readonly: false, - } - - // Invoke NodePublishVolume - actualRes, err := fakeNs.NodePublishVolume(FakeCtx, fakeReq) - if err != nil { - t.Errorf("failed to NodePublishVolume: %v", err) - } - - // Assert - assert.Equal(expectedRes, actualRes) -} - -func TestNodePublishVolumeEphermeral(t *testing.T) { - - properties := map[string]string{"cinder.csi.openstack.org/cluster": FakeCluster} - fvolName := fmt.Sprintf("ephemeral-%s", FakeVolID) - - omock.On("CreateVolume", fvolName, 2, "", "nova", "", "", &properties).Return(&FakeVol, nil) - - omock.On("AttachVolume", FakeNodeID, FakeVolID).Return(FakeVolID, nil) - omock.On("WaitDiskAttached", FakeNodeID, FakeVolID).Return(nil) - mmock.On("GetDevicePath", FakeVolID).Return(FakeDevicePath, nil) - mmock.On("IsLikelyNotMountPointAttach", FakeTargetPath).Return(true, nil) - metamock.On("GetAvailabilityZone").Return(FakeAvailability, nil) - - mount.MInstance = mmock - metadata.MetadataService = metamock - openstack.OsInstance = omock - - d := NewDriver(FakeNodeID, FakeEndpoint, FakeCluster) - fakeNse := NewNodeServer(d, mount.MInstance, metadata.MetadataService, openstack.OsInstance) - - // Init assert - assert := assert.New(t) - - // Expected Result - expectedRes := &csi.NodePublishVolumeResponse{} - stdVolCap := &csi.VolumeCapability{ - AccessType: &csi.VolumeCapability_Mount{ - Mount: &csi.VolumeCapability_MountVolume{}, - }, - AccessMode: &csi.VolumeCapability_AccessMode{ - Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, - }, - } - - // Fake request - fakeReq := &csi.NodePublishVolumeRequest{ - VolumeId: FakeVolID, - PublishContext: map[string]string{"DevicePath": FakeDevicePath}, - TargetPath: FakeTargetPath, - VolumeCapability: stdVolCap, - Readonly: false, - VolumeContext: map[string]string{"capacity": "2Gi", "csi.storage.k8s.io/ephemeral": "true"}, - } - - // Invoke NodePublishVolume - actualRes, err := fakeNse.NodePublishVolume(FakeCtx, fakeReq) - if err != nil { - t.Errorf("failed to NodePublishVolume: %v", err) - } - - // Assert - assert.Equal(expectedRes, actualRes) -} - -// Test NodeStageVolume -func TestNodeStageVolume(t *testing.T) { - - mmock.On("GetDevicePath", FakeVolID).Return(FakeDevicePath, nil) - mmock.On("IsLikelyNotMountPointAttach", FakeStagingTargetPath).Return(true, nil) - omock.On("GetVolume", FakeVolID).Return(FakeVol, nil) - - // Init assert - assert := assert.New(t) - - // Expected Result - expectedRes := &csi.NodeStageVolumeResponse{} - stdVolCap := &csi.VolumeCapability{ - AccessType: &csi.VolumeCapability_Mount{ - Mount: &csi.VolumeCapability_MountVolume{}, - }, - AccessMode: &csi.VolumeCapability_AccessMode{ - Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, - }, - } - - // Fake request - fakeReq := &csi.NodeStageVolumeRequest{ - VolumeId: FakeVolID, - PublishContext: map[string]string{"DevicePath": FakeDevicePath}, - StagingTargetPath: FakeStagingTargetPath, - VolumeCapability: stdVolCap, - } - - // Invoke NodeStageVolume - actualRes, err := fakeNs.NodeStageVolume(FakeCtx, fakeReq) - if err != nil { - t.Errorf("failed to NodeStageVolume: %v", err) - } - - // Assert - assert.Equal(expectedRes, actualRes) -} - -func TestNodeStageVolumeBlock(t *testing.T) { - - // Init assert - assert := assert.New(t) - - // Expected Result - expectedRes := &csi.NodeStageVolumeResponse{} - stdVolCap := &csi.VolumeCapability{ - AccessType: &csi.VolumeCapability_Block{ - Block: &csi.VolumeCapability_BlockVolume{}, - }, - AccessMode: &csi.VolumeCapability_AccessMode{ - Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, - }, - } - - // Fake request - fakeReq := &csi.NodeStageVolumeRequest{ - VolumeId: FakeVolID, - PublishContext: map[string]string{"DevicePath": FakeDevicePath}, - StagingTargetPath: FakeStagingTargetPath, - VolumeCapability: stdVolCap, - } - - // Invoke NodeStageVolume - actualRes, err := fakeNs.NodeStageVolume(FakeCtx, fakeReq) - if err != nil { - t.Errorf("failed to NodeStageVolume: %v", err) - } - - // Assert - assert.Equal(expectedRes, actualRes) -} - -// Test NodeUnpublishVolume -func TestNodeUnpublishVolume(t *testing.T) { - - mmock.On("UnmountPath", FakeTargetPath).Return(nil) - omock.On("GetVolume", FakeVolID).Return(FakeVol, nil) - - // Init assert - assert := assert.New(t) - - // Expected Result - expectedRes := &csi.NodeUnpublishVolumeResponse{} - - // Fake request - fakeReq := &csi.NodeUnpublishVolumeRequest{ - VolumeId: FakeVolID, - TargetPath: FakeTargetPath, - } - - // Invoke NodeUnpublishVolume - actualRes, err := fakeNs.NodeUnpublishVolume(FakeCtx, fakeReq) - if err != nil { - t.Errorf("failed to NodeUnpublishVolume: %v", err) - } - - // Assert - assert.Equal(expectedRes, actualRes) -} - -func TestNodeUnpublishVolumeEphermeral(t *testing.T) { - - mount.MInstance = mmock - metadata.MetadataService = metamock - openstack.OsInstance = omock - fvolName := fmt.Sprintf("ephemeral-%s", FakeVolID) - - mmock.On("UnmountPath", FakeTargetPath).Return(nil) - omock.On("GetVolumesByName", fvolName).Return(FakeVolList, nil) - omock.On("DetachVolume", FakeNodeID, FakeVolID).Return(nil) - omock.On("WaitDiskDetached", FakeNodeID, FakeVolID).Return(nil) - omock.On("DeleteVolume", FakeVolID).Return(nil) - - d := NewDriver(FakeNodeID, FakeEndpoint, FakeCluster) - fakeNse := NewNodeServer(d, mount.MInstance, metadata.MetadataService, openstack.OsInstance) - - // Init assert - assert := assert.New(t) - - // Expected Result - expectedRes := &csi.NodeUnpublishVolumeResponse{} - - // Fake request - fakeReq := &csi.NodeUnpublishVolumeRequest{ - VolumeId: FakeVolID, - TargetPath: FakeTargetPath, - } - - // Invoke NodeUnpublishVolume - actualRes, err := fakeNse.NodeUnpublishVolume(FakeCtx, fakeReq) - if err != nil { - t.Errorf("failed to NodeUnpublishVolume: %v", err) - } - - // Assert - assert.Equal(expectedRes, actualRes) -} - -// Test NodeUnstageVolume -func TestNodeUnstageVolume(t *testing.T) { - - mmock.On("UnmountPath", FakeStagingTargetPath).Return(nil) - omock.On("GetVolume", FakeVolID).Return(FakeVol, nil) - - // Init assert - assert := assert.New(t) - - // Expected Result - expectedRes := &csi.NodeUnstageVolumeResponse{} - - // Fake request - fakeReq := &csi.NodeUnstageVolumeRequest{ - VolumeId: FakeVolID, - StagingTargetPath: FakeStagingTargetPath, - } - - // Invoke NodeUnstageVolume - actualRes, err := fakeNs.NodeUnstageVolume(FakeCtx, fakeReq) - if err != nil { - t.Errorf("failed to NodeUnstageVolume: %v", err) - } - - // Assert - assert.Equal(expectedRes, actualRes) -} - -func TestNodeExpandVolume(t *testing.T) { - - // Init assert - assert := assert.New(t) - - // Fake request - fakeReq := &csi.NodeExpandVolumeRequest{ - VolumeId: FakeVolName, - VolumePath: FakeDevicePath, - } - - // Expected Result - expectedRes := &csi.NodeExpandVolumeResponse{} - - // Invoke NodeExpandVolume - actualRes, err := fakeNs.NodeExpandVolume(FakeCtx, fakeReq) - if err != nil { - t.Errorf("failed to ExpandVolume: %v", err) - } - - // Assert - assert.Equal(expectedRes, actualRes) - -} - -func TestNodeGetVolumeStatsBlock(t *testing.T) { - - // Init assert - assert := assert.New(t) - mmock.ExpectedCalls = nil - - // setup for test - tempDir, err := ioutil.TempDir("", "cpo-test") - if err != nil { - t.Fatalf("Failed to set up temp dir: %v", err) - } - defer os.RemoveAll(tempDir) - volumePath := filepath.Join(tempDir, FakeTargetPath) - err = os.MkdirAll(volumePath, 0750) - if err != nil { - t.Fatalf("Failed to set up volumepath: %v", err) - } - - // Fake request - fakeReq := &csi.NodeGetVolumeStatsRequest{ - VolumeId: FakeVolName, - VolumePath: volumePath, - } - - mmock.On("GetDeviceStats", volumePath).Return(FakeBlockDeviceStats, nil) - expectedBlockRes := &csi.NodeGetVolumeStatsResponse{ - Usage: []*csi.VolumeUsage{ - {Total: FakeBlockDeviceStats.TotalBytes, Unit: csi.VolumeUsage_BYTES}, - }, - } - - blockRes, err := fakeNs.NodeGetVolumeStats(FakeCtx, fakeReq) - - assert.NoError(err) - assert.Equal(expectedBlockRes, blockRes) - -} - -func TestNodeGetVolumeStatsFs(t *testing.T) { - - // Init assert - assert := assert.New(t) - mmock.ExpectedCalls = nil - - // setup for test - tempDir, err := ioutil.TempDir("", "cpo-test") - if err != nil { - t.Fatalf("Failed to set up temp dir: %v", err) - } - defer os.RemoveAll(tempDir) - volumePath := filepath.Join(tempDir, FakeTargetPath) - err = os.MkdirAll(volumePath, 0750) - if err != nil { - t.Fatalf("Failed to set up volumepath: %v", err) - } - - // Fake request - fakeReq := &csi.NodeGetVolumeStatsRequest{ - VolumeId: FakeVolName, - VolumePath: volumePath, - } - - mmock.On("GetDeviceStats", volumePath).Return(FakeFsStats, nil) - expectedFsRes := &csi.NodeGetVolumeStatsResponse{ - Usage: []*csi.VolumeUsage{ - {Total: FakeFsStats.TotalBytes, Available: FakeFsStats.AvailableBytes, Used: FakeFsStats.UsedBytes, Unit: csi.VolumeUsage_BYTES}, - {Total: FakeFsStats.TotalInodes, Available: FakeFsStats.AvailableInodes, Used: FakeFsStats.UsedInodes, Unit: csi.VolumeUsage_INODES}, - }, - } - - fsRes, err := fakeNs.NodeGetVolumeStats(FakeCtx, fakeReq) - - assert.NoError(err) - assert.Equal(expectedFsRes, fsRes) - -} diff --git a/pkg/csi/cinder/openstack/openstack.go b/pkg/csi/cinder/openstack/openstack.go deleted file mode 100644 index 7b3936e772..0000000000 --- a/pkg/csi/cinder/openstack/openstack.go +++ /dev/null @@ -1,185 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package openstack - -import ( - "fmt" - "os" - - "github.com/gophercloud/gophercloud" - "github.com/gophercloud/gophercloud/openstack" - "github.com/gophercloud/gophercloud/openstack/blockstorage/v3/snapshots" - "github.com/gophercloud/gophercloud/openstack/blockstorage/v3/volumes" - "github.com/gophercloud/gophercloud/openstack/compute/v2/servers" - "github.com/spf13/pflag" - gcfg "gopkg.in/gcfg.v1" - openstack_provider "k8s.io/cloud-provider-openstack/pkg/cloudprovider/providers/openstack" - md "k8s.io/cloud-provider-openstack/pkg/util/metadata" - "k8s.io/klog/v2" -) - -// userAgentData is used to add extra information to the gophercloud user-agent -var userAgentData []string - -// AddExtraFlags is called by the main package to add component specific command line flags -func AddExtraFlags(fs *pflag.FlagSet) { - fs.StringArrayVar(&userAgentData, "user-agent", nil, "Extra data to add to gophercloud user-agent. Use multiple times to add more than one component.") -} - -type IOpenStack interface { - CheckBlockStorageAPI() error - CreateVolume(name string, size int, vtype, availability string, snapshotID string, sourcevolID string, tags *map[string]string) (*volumes.Volume, error) - DeleteVolume(volumeID string) error - AttachVolume(instanceID, volumeID string) (string, error) - ListVolumes(limit int, startingToken string) ([]volumes.Volume, string, error) - WaitDiskAttached(instanceID string, volumeID string) error - DetachVolume(instanceID, volumeID string) error - WaitDiskDetached(instanceID string, volumeID string) error - GetAttachmentDiskPath(instanceID, volumeID string) (string, error) - GetVolume(volumeID string) (*volumes.Volume, error) - GetVolumesByName(name string) ([]volumes.Volume, error) - CreateSnapshot(name, volID string, tags *map[string]string) (*snapshots.Snapshot, error) - ListSnapshots(filters map[string]string) ([]snapshots.Snapshot, string, error) - DeleteSnapshot(snapID string) error - GetSnapshotByID(snapshotID string) (*snapshots.Snapshot, error) - WaitSnapshotReady(snapshotID string) error - GetInstanceByID(instanceID string) (*servers.Server, error) - ExpandVolume(volumeID string, size int) error - GetMaxVolLimit() int64 - GetMetadataOpts() openstack_provider.MetadataOpts - GetBlockStorageOpts() BlockStorageOpts -} - -type OpenStack struct { - compute *gophercloud.ServiceClient - blockstorage *gophercloud.ServiceClient - bsOpts BlockStorageOpts - epOpts gophercloud.EndpointOpts - metadataOpts openstack_provider.MetadataOpts -} - -type BlockStorageOpts struct { - NodeVolumeAttachLimit int64 `gcfg:"node-volume-attach-limit"` - RescanOnResize bool `gcfg:"rescan-on-resize"` -} - -type Config struct { - openstack_provider.Config - BlockStorage BlockStorageOpts -} - -func logcfg(cfg Config) { - openstack_provider.LogCfg(cfg.Config) - klog.Infof("Block storage opts: %v", cfg.BlockStorage) -} - -// GetConfigFromFile retrieves config options from file -func GetConfigFromFile(configFilePath string) (Config, error) { - var cfg Config - config, err := os.Open(configFilePath) - if err != nil { - klog.V(3).Infof("Failed to open OpenStack configuration file: %v", err) - return cfg, err - } - defer config.Close() - - err = gcfg.FatalOnly(gcfg.ReadInto(&cfg, config)) - if err != nil { - klog.V(3).Infof("Failed to read OpenStack configuration file: %v", err) - return cfg, err - } - - return cfg, nil -} - -const defaultMaxVolAttachLimit int64 = 256 - -var OsInstance IOpenStack = nil -var configFile = "/etc/cloud.conf" -var cfg Config - -func InitOpenStackProvider(cfgFile string) { - configFile = cfgFile - klog.V(2).Infof("InitOpenStackProvider configFile: %s", configFile) -} - -// CreateOpenStackProvider creates Openstack Instance -func CreateOpenStackProvider() (IOpenStack, error) { - // Get config from file - cfg, err := GetConfigFromFile(configFile) - if err != nil { - klog.Errorf("GetConfigFromFile %s failed with error: %v", configFile, err) - return nil, err - } - logcfg(cfg) - - provider, err := openstack_provider.NewOpenStackClient(&cfg.Config.Global, "cinder-csi-plugin", userAgentData...) - if err != nil { - return nil, err - } - - epOpts := gophercloud.EndpointOpts{ - Region: cfg.Global.Region, - } - - // Init Nova ServiceClient - computeclient, err := openstack.NewComputeV2(provider, epOpts) - if err != nil { - return nil, err - } - - // Init Cinder ServiceClient - blockstorageclient, err := openstack.NewBlockStorageV3(provider, epOpts) - if err != nil { - return nil, err - } - - // if no search order given, use default - if len(cfg.Config.Metadata.SearchOrder) == 0 { - cfg.Config.Metadata.SearchOrder = fmt.Sprintf("%s,%s", md.ConfigDriveID, md.MetadataID) - } - - // Init OpenStack - OsInstance = &OpenStack{ - compute: computeclient, - blockstorage: blockstorageclient, - bsOpts: cfg.BlockStorage, - epOpts: epOpts, - metadataOpts: cfg.Config.Metadata, - } - - return OsInstance, nil -} - -// GetOpenStackProvider returns Openstack Instance -func GetOpenStackProvider() (IOpenStack, error) { - if OsInstance != nil { - return OsInstance, nil - } - var err error - OsInstance, err = CreateOpenStackProvider() - if err != nil { - return nil, err - } - - return OsInstance, nil -} - -// GetMetadataOpts returns metadataopts -func (os *OpenStack) GetMetadataOpts() openstack_provider.MetadataOpts { - return os.metadataOpts -} diff --git a/pkg/csi/cinder/openstack/openstack_instances.go b/pkg/csi/cinder/openstack/openstack_instances.go deleted file mode 100644 index 83a87d3ffe..0000000000 --- a/pkg/csi/cinder/openstack/openstack_instances.go +++ /dev/null @@ -1,14 +0,0 @@ -package openstack - -import ( - "github.com/gophercloud/gophercloud/openstack/compute/v2/servers" -) - -// GetInstanceByID returns server with specified instanceID -func (os *OpenStack) GetInstanceByID(instanceID string) (*servers.Server, error) { - server, err := servers.Get(os.compute, instanceID).Extract() - if err != nil { - return nil, err - } - return server, nil -} diff --git a/pkg/csi/cinder/openstack/openstack_mock.go b/pkg/csi/cinder/openstack/openstack_mock.go deleted file mode 100644 index ec306a1037..0000000000 --- a/pkg/csi/cinder/openstack/openstack_mock.go +++ /dev/null @@ -1,385 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package openstack - -import ( - "github.com/gophercloud/gophercloud/openstack/blockstorage/v3/snapshots" - "github.com/gophercloud/gophercloud/openstack/blockstorage/v3/volumes" - "github.com/gophercloud/gophercloud/openstack/compute/v2/servers" - "github.com/stretchr/testify/mock" - cpo "k8s.io/cloud-provider-openstack/pkg/cloudprovider/providers/openstack" -) - -var fakeVol1 = volumes.Volume{ - ID: "261a8b81-3660-43e5-bab8-6470b65ee4e9", - Name: "fake-duplicate", - Status: "available", - AvailabilityZone: "nova", - Size: 1, -} - -var fakeVol2 = volumes.Volume{ - ID: "261a8b81-3660-43e5-bab8-6470b65ee4e9", - Name: "fake-duplicate", - Status: "available", - AvailabilityZone: "nova", - Size: 1, -} - -var fakeSnapshot = snapshots.Snapshot{ - ID: "261a8b81-3660-43e5-bab8-6470b65ee4e8", - Name: "fake-snapshot", - Status: "available", - Size: 1, - VolumeID: "CSIVolumeID", - Metadata: make(map[string]string), -} - -// OpenStackMock is an autogenerated mock type for the IOpenStack type -// ORIGINALLY GENERATED BY mockery with hand edits -type OpenStackMock struct { - mock.Mock -} - -// AttachVolume provides a mock function with given fields: instanceID, volumeID -func (_m *OpenStackMock) AttachVolume(instanceID string, volumeID string) (string, error) { - ret := _m.Called(instanceID, volumeID) - - var r0 string - if rf, ok := ret.Get(0).(func(string, string) string); ok { - r0 = rf(instanceID, volumeID) - } else { - r0 = ret.Get(0).(string) - } - - var r1 error - if rf, ok := ret.Get(1).(func(string, string) error); ok { - r1 = rf(instanceID, volumeID) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// CreateVolume provides a mock function with given fields: name, size, vtype, availability, tags -func (_m *OpenStackMock) CreateVolume(name string, size int, vtype string, availability string, snapshotID string, sourceVolID string, tags *map[string]string) (*volumes.Volume, error) { - ret := _m.Called(name, size, vtype, availability, snapshotID, sourceVolID, tags) - - var r0 *volumes.Volume - if rf, ok := ret.Get(0).(func(string, int, string, string, string, string, *map[string]string) *volumes.Volume); ok { - r0 = rf(name, size, vtype, availability, snapshotID, sourceVolID, tags) - } else { - r0 = ret.Get(0).(*volumes.Volume) - } - - var r1 error - if rf, ok := ret.Get(1).(func(string, int, string, string, string, string, *map[string]string) error); ok { - r1 = rf(name, size, vtype, availability, snapshotID, sourceVolID, tags) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// DeleteVolume provides a mock function with given fields: volumeID -func (_m *OpenStackMock) DeleteVolume(volumeID string) error { - ret := _m.Called(volumeID) - - var r0 error - if rf, ok := ret.Get(0).(func(string) error); ok { - r0 = rf(volumeID) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// GetVolume provides a mock function with given fields: volumeID -func (_m *OpenStackMock) GetVolume(volumeID string) (*volumes.Volume, error) { - return &fakeVol1, nil -} - -// CheckBlockStorageAPI -func (_m *OpenStackMock) CheckBlockStorageAPI() error { - return nil -} - -// DetachVolume provides a mock function with given fields: instanceID, volumeID -func (_m *OpenStackMock) DetachVolume(instanceID string, volumeID string) error { - ret := _m.Called(instanceID, volumeID) - - var r0 error - if rf, ok := ret.Get(0).(func(string, string) error); ok { - r0 = rf(instanceID, volumeID) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// GetAttachmentDiskPath provides a mock function with given fields: instanceID, volumeID -func (_m *OpenStackMock) GetAttachmentDiskPath(instanceID string, volumeID string) (string, error) { - ret := _m.Called(instanceID, volumeID) - - var r0 string - if rf, ok := ret.Get(0).(func(string, string) string); ok { - r0 = rf(instanceID, volumeID) - } else { - r0 = ret.Get(0).(string) - } - - var r1 error - if rf, ok := ret.Get(1).(func(string, string) error); ok { - r1 = rf(instanceID, volumeID) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// WaitDiskAttached provides a mock function with given fields: instanceID, volumeID -func (_m *OpenStackMock) WaitDiskAttached(instanceID string, volumeID string) error { - ret := _m.Called(instanceID, volumeID) - - var r0 error - if rf, ok := ret.Get(0).(func(string, string) error); ok { - r0 = rf(instanceID, volumeID) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// WaitDiskDetached provides a mock function with given fields: instanceID, volumeID -func (_m *OpenStackMock) WaitDiskDetached(instanceID string, volumeID string) error { - ret := _m.Called(instanceID, volumeID) - - var r0 error - if rf, ok := ret.Get(0).(func(string, string) error); ok { - r0 = rf(instanceID, volumeID) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// GetVolumesByName provides a mock function with given fields: name -func (_m *OpenStackMock) GetVolumesByName(name string) ([]volumes.Volume, error) { - - ret := _m.Called(name) - - var r0 []volumes.Volume - if rf, ok := ret.Get(0).(func(string) []volumes.Volume); ok { - r0 = rf(name) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]volumes.Volume) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(string) error); ok { - r1 = rf(name) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// ListSnapshots provides a mock function with given fields: limit, offset, filters -func (_m *OpenStackMock) ListSnapshots(filters map[string]string) ([]snapshots.Snapshot, string, error) { - ret := _m.Called(filters) - - var r0 []snapshots.Snapshot - if rf, ok := ret.Get(0).(func(map[string]string) []snapshots.Snapshot); ok { - r0 = rf(filters) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]snapshots.Snapshot) - } - } - var r1 string - - if rf, ok := ret.Get(1).(func(map[string]string) string); ok { - r1 = rf(filters) - } else { - if ret.Get(1) != nil { - r1 = ret.Get(1).(string) - } - } - - var r2 error - if rf, ok := ret.Get(2).(func(map[string]string) error); ok { - r2 = rf(filters) - } else { - r2 = ret.Error(2) - } - - return r0, r1, r2 -} - -// CreateSnapshot provides a mock function with given fields: name, volID, tags -func (_m *OpenStackMock) CreateSnapshot(name string, volID string, tags *map[string]string) (*snapshots.Snapshot, error) { - ret := _m.Called(name, volID, tags) - - var r0 *snapshots.Snapshot - if rf, ok := ret.Get(0).(func(string, string, *map[string]string) *snapshots.Snapshot); ok { - r0 = rf(name, volID, tags) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*snapshots.Snapshot) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(string, string, *map[string]string) error); ok { - r1 = rf(name, volID, tags) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// DeleteSnapshot provides a mock function with given fields: snapID -func (_m *OpenStackMock) DeleteSnapshot(snapID string) error { - ret := _m.Called(snapID) - - var r0 error - if rf, ok := ret.Get(0).(func(string) error); ok { - r0 = rf(snapID) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// ListVolumes provides a mock function without param -func (_m *OpenStackMock) ListVolumes(limit int, marker string) ([]volumes.Volume, string, error) { - ret := _m.Called(limit, marker) - - var r0 []volumes.Volume - if rf, ok := ret.Get(0).(func(int, string) []volumes.Volume); ok { - r0 = rf(limit, marker) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]volumes.Volume) - } - } - - var r1 string - - if rf, ok := ret.Get(1).(func(int, string) string); ok { - r1 = rf(limit, marker) - } else { - if ret.Get(1) != nil { - r1 = ret.Get(1).(string) - } - } - - var r2 error - if rf, ok := ret.Get(2).(func(int, string) error); ok { - r2 = rf(limit, marker) - } else { - r2 = ret.Error(2) - } - - return r0, r1, r2 -} - -func (_m *OpenStackMock) GetAvailabilityZone() (string, error) { - ret := _m.Called() - var r0 string - if rf, ok := ret.Get(0).(func() string); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(string) - } - - var r1 error - if rf, ok := ret.Get(1).(func() error); ok { - r1 = rf() - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -func (_m *OpenStackMock) GetInstanceID() (string, error) { - return "", nil -} - -func (_m *OpenStackMock) GetSnapshotByID(snapshotID string) (*snapshots.Snapshot, error) { - - return &fakeSnapshot, nil -} - -func (_m *OpenStackMock) WaitSnapshotReady(snapshotID string) error { - ret := _m.Called(snapshotID) - - var r0 error - if rf, ok := ret.Get(0).(func(string) error); ok { - r0 = rf(snapshotID) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -func (_m *OpenStackMock) GetMaxVolLimit() int64 { - return 256 -} - -func (_m *OpenStackMock) GetInstanceByID(instanceID string) (*servers.Server, error) { - return nil, nil -} - -// ExpandVolume provides a mock function with given fields: instanceID, volumeID -func (_m *OpenStackMock) ExpandVolume(volumeID string, size int) error { - ret := _m.Called(volumeID, size) - - var r0 error - if rf, ok := ret.Get(0).(func(string, int) error); ok { - r0 = rf(volumeID, size) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -func (_m *OpenStackMock) GetMetadataOpts() cpo.MetadataOpts { - var m cpo.MetadataOpts - m.SearchOrder = "configDrive" - return m -} - -// GetBlockStorageOpts provides a mock function to return BlockStorageOpts -func (_m *OpenStackMock) GetBlockStorageOpts() BlockStorageOpts { - return BlockStorageOpts{} -} diff --git a/pkg/csi/cinder/openstack/openstack_snapshots.go b/pkg/csi/cinder/openstack/openstack_snapshots.go deleted file mode 100644 index cd1beb095a..0000000000 --- a/pkg/csi/cinder/openstack/openstack_snapshots.go +++ /dev/null @@ -1,175 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package openstack snapshots provides an implementation of Cinder Snapshot features -// cinder functions using Gophercloud. -package openstack - -import ( - "fmt" - "net/url" - "strconv" - "time" - - "github.com/gophercloud/gophercloud/openstack/blockstorage/v3/snapshots" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/klog/v2" -) - -const ( - snapshotReadyStatus = "available" - snapReadyDuration = 1 * time.Second - snapReadyFactor = 1.2 - snapReadySteps = 10 - - snapshotDescription = "Created by OpenStack Cinder CSI driver" - snapshotForceCreate = "force-create" -) - -// CreateSnapshot issues a request to take a Snapshot of the specified Volume with the corresponding ID and -// returns the resultant gophercloud Snapshot Item upon success -func (os *OpenStack) CreateSnapshot(name, volID string, tags *map[string]string) (*snapshots.Snapshot, error) { - - force := false - // if no flag given, then force will be false by default - // if flag it given , check it - if item, ok := (*tags)[snapshotForceCreate]; ok { - var err error - force, err = strconv.ParseBool(item) - if err != nil { - klog.V(5).Infof("Make force create flag to false due to: %v", err) - } - - delete(*tags, snapshotForceCreate) - } - // Force the creation of snapshot even the Volume is in in-use state - opts := &snapshots.CreateOpts{ - VolumeID: volID, - Name: name, - Description: snapshotDescription, - Force: force, - } - if tags != nil { - opts.Metadata = *tags - } - // TODO: Do some check before really call openstack API on the input - - snap, err := snapshots.Create(os.blockstorage, opts).Extract() - if err != nil { - return &snapshots.Snapshot{}, err - } - // There's little value in rewrapping these gophercloud types into yet another abstraction/type, instead just - // return the gophercloud item - return snap, nil -} - -// ListSnapshots retrieves a list of active snapshots from Cinder for the corresponding Tenant. We also -// provide the ability to provide limit and offset to enable the consumer to provide accurate pagination. -// In addition the filters argument provides a mechanism for passing in valid filter strings to the list -// operation. Valid filter keys are: Name, Status, VolumeID, Limit, Marker (TenantID has no effect) -func (os *OpenStack) ListSnapshots(filters map[string]string) ([]snapshots.Snapshot, string, error) { - // Build the Opts - opts := snapshots.ListOpts{} - nextPageToken := "" - - for key, val := range filters { - switch key { - case "Status": - opts.Status = val - case "Name": - opts.Name = val - case "VolumeID": - opts.VolumeID = val - case "Marker": - opts.Marker = val - case "Limit": - opts.Limit, _ = strconv.Atoi(val) - default: - klog.V(3).Infof("Not a valid filter key %s", key) - } - } - - pages, err := snapshots.List(os.blockstorage, opts).AllPages() - if err != nil { - klog.V(3).Infof("Failed to retrieve snapshots: %v", err) - return nil, nextPageToken, err - } - snaps, err := snapshots.ExtractSnapshots(pages) - if err != nil { - klog.V(3).Infof("Failed to extract snapshot pages: %v", err) - return nil, nextPageToken, err - } - - nextPageURL, err := pages.NextPageURL() - if err != nil && nextPageURL != "" { - if queryParams, nerr := url.ParseQuery(nextPageURL); nerr != nil { - nextPageToken = queryParams.Get("marker") - } - } - return snaps, nextPageToken, nil - -} - -// DeleteSnapshot issues a request to delete the Snapshot with the specified ID from the Cinder backend -func (os *OpenStack) DeleteSnapshot(snapID string) error { - err := snapshots.Delete(os.blockstorage, snapID).ExtractErr() - if err != nil { - klog.V(3).Infof("Failed to delete snapshot: %v", err) - } - return err -} - -//GetSnapshotByID returns snapshot details by id -func (os *OpenStack) GetSnapshotByID(snapshotID string) (*snapshots.Snapshot, error) { - s, err := snapshots.Get(os.blockstorage, snapshotID).Extract() - if err != nil { - klog.V(3).Infof("Failed to get snapshot: %v", err) - return nil, err - } - return s, nil -} - -// WaitSnapshotReady waits till snapshot is ready -func (os *OpenStack) WaitSnapshotReady(snapshotID string) error { - backoff := wait.Backoff{ - Duration: snapReadyDuration, - Factor: snapReadyFactor, - Steps: snapReadySteps, - } - - err := wait.ExponentialBackoff(backoff, func() (bool, error) { - ready, err := os.snapshotIsReady(snapshotID) - if err != nil { - return false, err - } - return ready, nil - }) - - if err == wait.ErrWaitTimeout { - err = fmt.Errorf("Timeout, Snapshot %s is still not Ready %v", snapshotID, err) - } - - return err -} - -func (os *OpenStack) snapshotIsReady(snapshotID string) (bool, error) { - snap, err := os.GetSnapshotByID(snapshotID) - if err != nil { - return false, err - } - - return snap.Status == snapshotReadyStatus, nil -} diff --git a/pkg/csi/cinder/openstack/openstack_test.go b/pkg/csi/cinder/openstack/openstack_test.go deleted file mode 100644 index 4ccea8f555..0000000000 --- a/pkg/csi/cinder/openstack/openstack_test.go +++ /dev/null @@ -1,143 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package openstack - -import ( - "os" - "reflect" - "strings" - "testing" - - "github.com/spf13/pflag" - "github.com/stretchr/testify/assert" -) - -var fakeFileName = "cloud.conf" -var fakeUserName = "user" -var fakePassword = "pass" -var fakeAuthUrl = "https://169.254.169.254/identity/v3" -var fakeTenantID = "c869168a828847f39f7f06edd7305637" -var fakeDomainID = "2a73b8f597c04551a0fdc8e95544be8a" -var fakeRegion = "RegionOne" -var fakeCAfile = "fake-ca.crt" - -// Test GetConfigFromFile -func TestGetConfigFromFile(t *testing.T) { - // init file - var fakeFileContent = ` -[Global] -username=` + fakeUserName + ` -password=` + fakePassword + ` -auth-url=` + fakeAuthUrl + ` -tenant-id=` + fakeTenantID + ` -domain-id=` + fakeDomainID + ` -ca-file=` + fakeCAfile + ` -region=` + fakeRegion + ` -[BlockStorage] -rescan-on-resize=true` - - f, err := os.Create(fakeFileName) - if err != nil { - t.Errorf("failed to create file: %v", err) - } - - _, err = f.WriteString(fakeFileContent) - f.Close() - if err != nil { - t.Errorf("failed to write file: %v", err) - } - defer os.Remove(fakeFileName) - - // Init assert - assert := assert.New(t) - expectedOpts := Config{} - expectedOpts.Global.Username = fakeUserName - expectedOpts.Global.Password = fakePassword - expectedOpts.Global.DomainID = fakeDomainID - expectedOpts.Global.AuthURL = fakeAuthUrl - expectedOpts.Global.CAFile = fakeCAfile - expectedOpts.Global.TenantID = fakeTenantID - expectedOpts.Global.Region = fakeRegion - expectedOpts.BlockStorage.RescanOnResize = true - - // Invoke GetConfigFromFile - actualAuthOpts, err := GetConfigFromFile(fakeFileName) - if err != nil { - t.Errorf("failed to GetConfigFromFile: %v", err) - } - - // Assert - assert.Equal(expectedOpts, actualAuthOpts) -} - -func TestUserAgentFlag(t *testing.T) { - tests := []struct { - name string - shouldParse bool - flags []string - expected []string - }{ - {"no_flag", true, []string{}, nil}, - {"one_flag", true, []string{"--user-agent=cluster/abc-123"}, []string{"cluster/abc-123"}}, - {"multiple_flags", true, []string{"--user-agent=a/b", "--user-agent=c/d"}, []string{"a/b", "c/d"}}, - {"flag_with_space", true, []string{"--user-agent=a b"}, []string{"a b"}}, - {"flag_split_with_space", true, []string{"--user-agent=a", "b"}, []string{"a"}}, - {"empty_flag", false, []string{"--user-agent"}, nil}, - } - - for _, testCase := range tests { - userAgentData = []string{} - - t.Run(testCase.name, func(t *testing.T) { - fs := pflag.NewFlagSet("test", pflag.ContinueOnError) - AddExtraFlags(fs) - - err := fs.Parse(testCase.flags) - - if testCase.shouldParse && err != nil { - t.Errorf("Flags failed to parse") - } else if !testCase.shouldParse && err == nil { - t.Errorf("Flags should not have parsed") - } else if testCase.shouldParse { - if !reflect.DeepEqual(userAgentData, testCase.expected) { - t.Errorf("userAgentData %#v did not match expected value %#v", userAgentData, testCase.expected) - } - } - }) - } -} - -func clearEnviron(t *testing.T) []string { - env := os.Environ() - for _, pair := range env { - if strings.HasPrefix(pair, "OS_") { - i := strings.Index(pair, "=") + 1 - os.Unsetenv(pair[:i-1]) - } - } - return env -} -func resetEnviron(t *testing.T, items []string) { - for _, pair := range items { - if strings.HasPrefix(pair, "OS_") { - i := strings.Index(pair, "=") + 1 - if err := os.Setenv(pair[:i-1], pair[i:]); err != nil { - t.Errorf("Setenv(%q, %q) failed during reset: %v", pair[:i-1], pair[i:], err) - } - } - } -} diff --git a/pkg/csi/cinder/openstack/openstack_volumes.go b/pkg/csi/cinder/openstack/openstack_volumes.go deleted file mode 100644 index ce373725b9..0000000000 --- a/pkg/csi/cinder/openstack/openstack_volumes.go +++ /dev/null @@ -1,343 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package openstack - -import ( - "fmt" - "net/url" - "time" - - "github.com/gophercloud/gophercloud/openstack" - "github.com/gophercloud/gophercloud/openstack/blockstorage/apiversions" - volumeexpand "github.com/gophercloud/gophercloud/openstack/blockstorage/extensions/volumeactions" - "github.com/gophercloud/gophercloud/openstack/blockstorage/v3/volumes" - "github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/volumeattach" - "k8s.io/apimachinery/pkg/util/wait" - cpoerrors "k8s.io/cloud-provider-openstack/pkg/util/errors" - - "k8s.io/klog/v2" -) - -const ( - VolumeAvailableStatus = "available" - VolumeInUseStatus = "in-use" - VolumeDeletedStatus = "deleted" - VolumeErrorStatus = "error" - operationFinishInitDelay = 1 * time.Second - operationFinishFactor = 1.1 - operationFinishSteps = 10 - diskAttachInitDelay = 1 * time.Second - diskAttachFactor = 1.2 - diskAttachSteps = 15 - diskDetachInitDelay = 1 * time.Second - diskDetachFactor = 1.2 - diskDetachSteps = 13 - volumeDescription = "Created by OpenStack Cinder CSI driver" -) - -func (os *OpenStack) CheckBlockStorageAPI() error { - _, err := apiversions.List(os.blockstorage).AllPages() - if err != nil { - return err - } - return nil -} - -// CreateVolume creates a volume of given size -func (os *OpenStack) CreateVolume(name string, size int, vtype, availability string, snapshotID string, sourcevolID string, tags *map[string]string) (*volumes.Volume, error) { - opts := &volumes.CreateOpts{ - Name: name, - Size: size, - VolumeType: vtype, - AvailabilityZone: availability, - Description: volumeDescription, - SnapshotID: snapshotID, - SourceVolID: sourcevolID, - } - if tags != nil { - opts.Metadata = *tags - } - - vol, err := volumes.Create(os.blockstorage, opts).Extract() - if err != nil { - return nil, err - } - - return vol, nil -} - -// ListVolumes list all the volumes -func (os *OpenStack) ListVolumes(limit int, startingToken string) ([]volumes.Volume, string, error) { - nextPageToken := "" - opts := volumes.ListOpts{Limit: limit, Marker: startingToken} - pages, err := volumes.List(os.blockstorage, opts).AllPages() - if err != nil { - return nil, nextPageToken, err - } - vols, err := volumes.ExtractVolumes(pages) - if err != nil { - return nil, nextPageToken, err - } - nextPageURL, err := pages.NextPageURL() - if err != nil && nextPageURL != "" { - if queryParams, nerr := url.ParseQuery(nextPageURL); nerr != nil { - nextPageToken = queryParams.Get("marker") - } - } - return vols, nextPageToken, nil -} - -// GetVolumesByName is a wrapper around ListVolumes that creates a Name filter to act as a GetByName -// Returns a list of Volume references with the specified name -func (os *OpenStack) GetVolumesByName(n string) ([]volumes.Volume, error) { - opts := volumes.ListOpts{Name: n} - pages, err := volumes.List(os.blockstorage, opts).AllPages() - if err != nil { - return nil, err - } - - vols, err := volumes.ExtractVolumes(pages) - if err != nil { - return nil, err - } - - return vols, nil -} - -// DeleteVolume delete a volume -func (os *OpenStack) DeleteVolume(volumeID string) error { - used, err := os.diskIsUsed(volumeID) - if err != nil { - return err - } - if used { - return fmt.Errorf("Cannot delete the volume %q, it's still attached to a node", volumeID) - } - - err = volumes.Delete(os.blockstorage, volumeID, nil).ExtractErr() - return err -} - -// GetVolume retrieves Volume by its ID. -func (os *OpenStack) GetVolume(volumeID string) (*volumes.Volume, error) { - - vol, err := volumes.Get(os.blockstorage, volumeID).Extract() - if err != nil { - return nil, err - } - - return vol, nil -} - -// AttachVolume attaches given cinder volume to the compute -func (os *OpenStack) AttachVolume(instanceID, volumeID string) (string, error) { - volume, err := os.GetVolume(volumeID) - if err != nil { - return "", err - } - - if len(volume.Attachments) > 0 { - if instanceID == volume.Attachments[0].ServerID { - klog.V(4).Infof("Disk %s is already attached to instance %s", volumeID, instanceID) - return volume.ID, nil - } - return "", fmt.Errorf("disk %s is attached to a different instance (%s)", volumeID, volume.Attachments[0].ServerID) - } - - _, err = volumeattach.Create(os.compute, instanceID, &volumeattach.CreateOpts{ - VolumeID: volume.ID, - }).Extract() - - if err != nil { - return "", fmt.Errorf("failed to attach %s volume to %s compute: %v", volumeID, instanceID, err) - } - - return volume.ID, nil -} - -// WaitDiskAttached waits for attched -func (os *OpenStack) WaitDiskAttached(instanceID string, volumeID string) error { - backoff := wait.Backoff{ - Duration: diskAttachInitDelay, - Factor: diskAttachFactor, - Steps: diskAttachSteps, - } - - err := wait.ExponentialBackoff(backoff, func() (bool, error) { - attached, err := os.diskIsAttached(instanceID, volumeID) - if err != nil && !cpoerrors.IsNotFound(err) { - // if this is a race condition indicate the volume is deleted - // during sleep phase, ignore the error and return attach=false - return false, err - } - return attached, nil - }) - - if err == wait.ErrWaitTimeout { - err = fmt.Errorf("Volume %q failed to be attached within the alloted time", volumeID) - } - - return err -} - -// DetachVolume detaches given cinder volume from the compute -func (os *OpenStack) DetachVolume(instanceID, volumeID string) error { - volume, err := os.GetVolume(volumeID) - if err != nil { - return err - } - if volume.Status == VolumeAvailableStatus { - klog.V(2).Infof("volume: %s has been detached from compute: %s ", volume.ID, instanceID) - return nil - } - - if volume.Status != VolumeInUseStatus { - return fmt.Errorf("can not detach volume %s, its status is %s", volume.Name, volume.Status) - } - - if len(volume.Attachments) > 0 { - if volume.Attachments[0].ServerID != instanceID { - return fmt.Errorf("disk: %s is not attached to compute: %s", volume.Name, instanceID) - } - err = volumeattach.Delete(os.compute, instanceID, volume.ID).ExtractErr() - if err != nil { - return fmt.Errorf("failed to delete volume %s from compute %s attached %v", volume.ID, instanceID, err) - } - klog.V(2).Infof("Successfully detached volume: %s from compute: %s", volume.ID, instanceID) - - } else { - return fmt.Errorf("disk: %s has no attachments", volume.Name) - } - - return nil -} - -// WaitDiskDetached waits for detached -func (os *OpenStack) WaitDiskDetached(instanceID string, volumeID string) error { - backoff := wait.Backoff{ - Duration: diskDetachInitDelay, - Factor: diskDetachFactor, - Steps: diskDetachSteps, - } - - err := wait.ExponentialBackoff(backoff, func() (bool, error) { - attached, err := os.diskIsAttached(instanceID, volumeID) - if err != nil { - return false, err - } - return !attached, nil - }) - - if err == wait.ErrWaitTimeout { - err = fmt.Errorf("Volume %q failed to detach within the alloted time", volumeID) - } - - return err -} - -// GetAttachmentDiskPath gets device path of attached volume to the compute -func (os *OpenStack) GetAttachmentDiskPath(instanceID, volumeID string) (string, error) { - volume, err := os.GetVolume(volumeID) - if err != nil { - return "", err - } - if volume.Status != VolumeInUseStatus { - return "", fmt.Errorf("can not get device path of volume %s, its status is %s ", volume.Name, volume.Status) - } - - if len(volume.Attachments) > 0 && volume.Attachments[0].ServerID != "" { - if instanceID == volume.Attachments[0].ServerID { - return volume.Attachments[0].Device, nil - } - return "", fmt.Errorf("disk %q is attached to a different compute: %q, should be detached before proceeding", volumeID, volume.Attachments[0].ServerID) - } - return "", fmt.Errorf("volume %s has no ServerId", volumeID) -} - -// ExpandVolume expands the volume to new size -func (os *OpenStack) ExpandVolume(volumeID string, newSize int) error { - volume, err := os.GetVolume(volumeID) - if err != nil { - return err - } - - createOpts := volumeexpand.ExtendSizeOpts{ - NewSize: newSize, - } - - switch volume.Status { - case VolumeInUseStatus: - // Init a local thread safe copy of the Cinder ServiceClient - blockstorageClient, err := openstack.NewBlockStorageV3(os.blockstorage.ProviderClient, os.epOpts) - if err != nil { - return err - } - - // cinder online resize is available since 3.42 microversion - // https://docs.openstack.org/cinder/latest/contributor/api_microversion_history.html#id40 - blockstorageClient.Microversion = "3.42" - - return volumeexpand.ExtendSize(blockstorageClient, volumeID, createOpts).ExtractErr() - case VolumeAvailableStatus: - return volumeexpand.ExtendSize(os.blockstorage, volumeID, createOpts).ExtractErr() - } - - // cinder volume can not be expanded when volume status is not volumeInUseStatus or not volumeAvailableStatus - return fmt.Errorf("volume cannot be resized, when status is %s", volume.Status) -} - -//GetMaxVolLimit returns max vol limit -func (os *OpenStack) GetMaxVolLimit() int64 { - if os.bsOpts.NodeVolumeAttachLimit > 0 && os.bsOpts.NodeVolumeAttachLimit <= 256 { - return os.bsOpts.NodeVolumeAttachLimit - } - - return defaultMaxVolAttachLimit -} - -// diskIsAttached queries if a volume is attached to a compute instance -func (os *OpenStack) diskIsAttached(instanceID, volumeID string) (bool, error) { - volume, err := os.GetVolume(volumeID) - if err != nil { - return false, err - } - - if len(volume.Attachments) > 0 { - return instanceID == volume.Attachments[0].ServerID, nil - } - - return false, nil -} - -// diskIsUsed returns true a disk is attached to any node. -func (os *OpenStack) diskIsUsed(volumeID string) (bool, error) { - volume, err := os.GetVolume(volumeID) - if err != nil { - return false, err - } - - if len(volume.Attachments) > 0 { - return volume.Attachments[0].ServerID != "", nil - } - - return false, nil -} - -// GetBlockStorageOpts returns OpenStack block storage options -func (os *OpenStack) GetBlockStorageOpts() BlockStorageOpts { - return os.bsOpts -} diff --git a/pkg/csi/cinder/server.go b/pkg/csi/cinder/server.go deleted file mode 100644 index 0a93680b52..0000000000 --- a/pkg/csi/cinder/server.go +++ /dev/null @@ -1,112 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cinder - -import ( - "net" - "os" - "sync" - - "google.golang.org/grpc" - "k8s.io/klog/v2" - - "github.com/container-storage-interface/spec/lib/go/csi" -) - -// NonBlockingGRPCServer defines Non blocking GRPC server interfaces -type NonBlockingGRPCServer interface { - // Start services at the endpoint - Start(endpoint string, ids csi.IdentityServer, cs csi.ControllerServer, ns csi.NodeServer) - // Waits for the service to stop - Wait() - // Stops the service gracefully - Stop() - // Stops the service forcefully - ForceStop() -} - -func NewNonBlockingGRPCServer() NonBlockingGRPCServer { - return &nonBlockingGRPCServer{} -} - -// NonBlocking server -type nonBlockingGRPCServer struct { - wg sync.WaitGroup - server *grpc.Server -} - -func (s *nonBlockingGRPCServer) Start(endpoint string, ids csi.IdentityServer, cs csi.ControllerServer, ns csi.NodeServer) { - - s.wg.Add(1) - - go s.serve(endpoint, ids, cs, ns) - - return -} - -func (s *nonBlockingGRPCServer) Wait() { - s.wg.Wait() -} - -func (s *nonBlockingGRPCServer) Stop() { - s.server.GracefulStop() -} - -func (s *nonBlockingGRPCServer) ForceStop() { - s.server.Stop() -} - -func (s *nonBlockingGRPCServer) serve(endpoint string, ids csi.IdentityServer, cs csi.ControllerServer, ns csi.NodeServer) { - - proto, addr, err := ParseEndpoint(endpoint) - if err != nil { - klog.Fatal(err.Error()) - } - - if proto == "unix" { - addr = "/" + addr - if err := os.Remove(addr); err != nil && !os.IsNotExist(err) { - klog.Fatalf("Failed to remove %s, error: %s", addr, err.Error()) - } - } - - listener, err := net.Listen(proto, addr) - if err != nil { - klog.Fatalf("Failed to listen: %v", err) - } - - opts := []grpc.ServerOption{ - grpc.UnaryInterceptor(logGRPC), - } - server := grpc.NewServer(opts...) - s.server = server - - if ids != nil { - csi.RegisterIdentityServer(server, ids) - } - if cs != nil { - csi.RegisterControllerServer(server, cs) - } - if ns != nil { - csi.RegisterNodeServer(server, ns) - } - - klog.Infof("Listening for connections on address: %#v", listener.Addr()) - - server.Serve(listener) - -} diff --git a/pkg/csi/cinder/utils.go b/pkg/csi/cinder/utils.go deleted file mode 100644 index 5268c8779f..0000000000 --- a/pkg/csi/cinder/utils.go +++ /dev/null @@ -1,89 +0,0 @@ -package cinder - -import ( - "fmt" - "strings" - - "github.com/container-storage-interface/spec/lib/go/csi" - "golang.org/x/net/context" - "google.golang.org/grpc" - "k8s.io/cloud-provider-openstack/pkg/csi/cinder/openstack" - "k8s.io/cloud-provider-openstack/pkg/util/metadata" - "k8s.io/cloud-provider-openstack/pkg/util/mount" - "k8s.io/klog/v2" -) - -func NewControllerServiceCapability(cap csi.ControllerServiceCapability_RPC_Type) *csi.ControllerServiceCapability { - return &csi.ControllerServiceCapability{ - Type: &csi.ControllerServiceCapability_Rpc{ - Rpc: &csi.ControllerServiceCapability_RPC{ - Type: cap, - }, - }, - } -} - -func NewNodeServiceCapability(cap csi.NodeServiceCapability_RPC_Type) *csi.NodeServiceCapability { - return &csi.NodeServiceCapability{ - Type: &csi.NodeServiceCapability_Rpc{ - Rpc: &csi.NodeServiceCapability_RPC{ - Type: cap, - }, - }, - } -} - -func NewVolumeCapabilityAccessMode(mode csi.VolumeCapability_AccessMode_Mode) *csi.VolumeCapability_AccessMode { - return &csi.VolumeCapability_AccessMode{Mode: mode} -} - -func NewControllerServer(d *CinderDriver, cloud openstack.IOpenStack) *controllerServer { - return &controllerServer{ - Driver: d, - Cloud: cloud, - } -} - -func NewIdentityServer(d *CinderDriver) *identityServer { - return &identityServer{ - Driver: d, - } -} - -func NewNodeServer(d *CinderDriver, mount mount.IMount, metadata metadata.IMetadata, cloud openstack.IOpenStack) *nodeServer { - return &nodeServer{ - Driver: d, - Mount: mount, - Metadata: metadata, - Cloud: cloud, - } -} - -func RunControllerandNodePublishServer(endpoint string, ids csi.IdentityServer, cs csi.ControllerServer, ns csi.NodeServer) { - - s := NewNonBlockingGRPCServer() - s.Start(endpoint, ids, cs, ns) - s.Wait() -} - -func ParseEndpoint(ep string) (string, string, error) { - if strings.HasPrefix(strings.ToLower(ep), "unix://") || strings.HasPrefix(strings.ToLower(ep), "tcp://") { - s := strings.SplitN(ep, "://", 2) - if s[1] != "" { - return s[0], s[1], nil - } - } - return "", "", fmt.Errorf("Invalid endpoint: %v", ep) -} - -func logGRPC(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { - klog.V(3).Infof("GRPC call: %s", info.FullMethod) - klog.V(5).Infof("GRPC request: %+v", req) - resp, err := handler(ctx, req) - if err != nil { - klog.Errorf("GRPC error: %v", err) - } else { - klog.V(5).Infof("GRPC response: %+v", resp) - } - return resp, err -} diff --git a/pkg/csi/cinder/utils_test.go b/pkg/csi/cinder/utils_test.go deleted file mode 100644 index 95f1a047bd..0000000000 --- a/pkg/csi/cinder/utils_test.go +++ /dev/null @@ -1,76 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cinder - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestParseEndpoint(t *testing.T) { - - //Valid unix domain socket endpoint - sockType, addr, err := ParseEndpoint("unix://fake.sock") - assert.NoError(t, err) - assert.Equal(t, sockType, "unix") - assert.Equal(t, addr, "fake.sock") - - sockType, addr, err = ParseEndpoint("unix:///fakedir/fakedir/fake.sock") - assert.NoError(t, err) - assert.Equal(t, sockType, "unix") - assert.Equal(t, addr, "/fakedir/fakedir/fake.sock") - - //Valid unix domain socket with uppercase - sockType, addr, err = ParseEndpoint("UNIX://fake.sock") - assert.NoError(t, err) - assert.Equal(t, sockType, "UNIX") - assert.Equal(t, addr, "fake.sock") - - //Valid TCP endpoint with ip - sockType, addr, err = ParseEndpoint("tcp://127.0.0.1:80") - assert.NoError(t, err) - assert.Equal(t, sockType, "tcp") - assert.Equal(t, addr, "127.0.0.1:80") - - //Valid TCP endpoint with uppercase - sockType, addr, err = ParseEndpoint("TCP://127.0.0.1:80") - assert.NoError(t, err) - assert.Equal(t, sockType, "TCP") - assert.Equal(t, addr, "127.0.0.1:80") - - //Valid TCP endpoint with hostname - sockType, addr, err = ParseEndpoint("tcp://fakehost:80") - assert.NoError(t, err) - assert.Equal(t, sockType, "tcp") - assert.Equal(t, addr, "fakehost:80") - - _, _, err = ParseEndpoint("unix:/fake.sock/") - assert.NotNil(t, err) - - _, _, err = ParseEndpoint("fake.sock") - assert.NotNil(t, err) - - _, _, err = ParseEndpoint("unix://") - assert.NotNil(t, err) - - _, _, err = ParseEndpoint("://") - assert.NotNil(t, err) - - _, _, err = ParseEndpoint("") - assert.NotNil(t, err) -} diff --git a/pkg/csi/manila/OWNERS b/pkg/csi/manila/OWNERS deleted file mode 100644 index 132d49af6f..0000000000 --- a/pkg/csi/manila/OWNERS +++ /dev/null @@ -1,10 +0,0 @@ -approvers: -- gman0 -- gouthampacha -- tombarron -reviewers: -- gman0 -- gouthampacha -- tombarron -- Fedosin -- tsmetana diff --git a/pkg/csi/manila/adapters.go b/pkg/csi/manila/adapters.go deleted file mode 100644 index a41186e691..0000000000 --- a/pkg/csi/manila/adapters.go +++ /dev/null @@ -1,37 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package manila - -import ( - "strings" - - "k8s.io/cloud-provider-openstack/pkg/csi/manila/shareadapters" - "k8s.io/klog/v2" -) - -func getShareAdapter(proto string) shareadapters.ShareAdapter { - switch strings.ToUpper(proto) { - case "CEPHFS": - return &shareadapters.Cephfs{} - case "NFS": - return &shareadapters.NFS{} - default: - klog.Fatalf("unknown share adapter %s", proto) - } - - return nil -} diff --git a/pkg/csi/manila/capabilities/manilacapabilities.go b/pkg/csi/manila/capabilities/manilacapabilities.go deleted file mode 100644 index a665e62412..0000000000 --- a/pkg/csi/manila/capabilities/manilacapabilities.go +++ /dev/null @@ -1,70 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package capabilities - -import ( - "fmt" - "strconv" - - "k8s.io/cloud-provider-openstack/pkg/csi/manila/manilaclient" -) - -type ( - ManilaCapability int - ManilaCapabilities map[ManilaCapability]bool -) - -const ( - ManilaCapabilityNone ManilaCapability = iota - ManilaCapabilitySnapshot - ManilaCapabilityShareFromSnapshot - - extraSpecSnapshotSupport = "snapshot_support" - extraSpecCreateShareFromSnapshotSupport = "create_share_from_snapshot_support" -) - -func GetManilaCapabilities(shareType string, manilaClient manilaclient.Interface) (ManilaCapabilities, error) { - shareTypes, err := manilaClient.GetShareTypes() - if err != nil { - return nil, err - } - - for _, t := range shareTypes { - if t.Name == shareType || t.ID == shareType { - return readManilaCaps(t.ExtraSpecs), nil - } - } - - return nil, fmt.Errorf("unknown share type %s", shareType) -} - -func readManilaCaps(extraSpecs map[string]interface{}) ManilaCapabilities { - strToBool := func(ss interface{}) bool { - var b bool - if ss != nil { - if str, ok := ss.(string); ok { - b, _ = strconv.ParseBool(str) - } - } - return b - } - - return ManilaCapabilities{ - ManilaCapabilitySnapshot: strToBool(extraSpecs[extraSpecSnapshotSupport]), - ManilaCapabilityShareFromSnapshot: strToBool(extraSpecs[extraSpecCreateShareFromSnapshotSupport]), - } -} diff --git a/pkg/csi/manila/compatibility/compatibility.go b/pkg/csi/manila/compatibility/compatibility.go deleted file mode 100644 index f588ae4664..0000000000 --- a/pkg/csi/manila/compatibility/compatibility.go +++ /dev/null @@ -1,44 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package compatibility - -import ( - "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/gophercloud/gophercloud/openstack/sharedfilesystems/v2/shares" - "k8s.io/cloud-provider-openstack/pkg/csi/manila/capabilities" - "k8s.io/cloud-provider-openstack/pkg/csi/manila/csiclient" - "k8s.io/cloud-provider-openstack/pkg/csi/manila/manilaclient" - "k8s.io/cloud-provider-openstack/pkg/csi/manila/options" -) - -type Layer interface { - SupplementCapability(compatOpts *options.CompatibilityOptions, dstShare *shares.Share, dstShareAccessRight *shares.AccessRight, req *csi.CreateVolumeRequest, fwdEndpoint string, manilaClient manilaclient.Interface, csiClientBuilder csiclient.Builder) error -} - -// Certain share protocols may not support certain Manila capabilities -// in a given share type. This map forms a compatibility layer which -// fills in the feature gap with in-driver functionality. -var compatCaps = map[string]map[capabilities.ManilaCapability]Layer{} - -func FindCompatibilityLayer(shareProto string, wantsCap capabilities.ManilaCapability, shareTypeCaps capabilities.ManilaCapabilities) Layer { - if layers, ok := compatCaps[shareProto]; ok { - if hasCapability := shareTypeCaps[wantsCap]; !hasCapability { - if compatCapability, ok := layers[wantsCap]; ok { - return compatCapability - } - } - } - - return nil -} diff --git a/pkg/csi/manila/controllerserver.go b/pkg/csi/manila/controllerserver.go deleted file mode 100644 index 9dd3d745b1..0000000000 --- a/pkg/csi/manila/controllerserver.go +++ /dev/null @@ -1,420 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package manila - -import ( - "context" - "strings" - "sync" - - "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/golang/protobuf/ptypes" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/cloud-provider-openstack/pkg/csi/manila/capabilities" - "k8s.io/cloud-provider-openstack/pkg/csi/manila/options" - "k8s.io/cloud-provider-openstack/pkg/csi/manila/shareadapters" - clouderrors "k8s.io/cloud-provider-openstack/pkg/util/errors" - "k8s.io/klog/v2" -) - -type controllerServer struct { - d *Driver -} - -var ( - pendingVolumes = sync.Map{} - pendingSnapshots = sync.Map{} -) - -func getVolumeCreator(source *csi.VolumeContentSource, shareOpts *options.ControllerVolumeContext, compatOpts *options.CompatibilityOptions, shareTypeCaps capabilities.ManilaCapabilities) (volumeCreator, error) { - if source == nil { - return &blankVolume{}, nil - } - - if source.GetVolume() != nil { - return nil, status.Error(codes.Unimplemented, "volume cloning is not supported yet") - } - - if source.GetSnapshot() != nil { - if tryCompatForVolumeSource(compatOpts, shareOpts, source, shareTypeCaps) != nil { - klog.Infof("share type %s does not advertise create_share_from_snapshot_support capability, compatibility mode is available", shareOpts.Type) - return &blankVolume{}, nil - } - - return &volumeFromSnapshot{}, nil - } - - return nil, status.Error(codes.InvalidArgument, "invalid volume content source") -} - -func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) { - if err := validateCreateVolumeRequest(req); err != nil { - return nil, status.Error(codes.InvalidArgument, err.Error()) - } - - // Configuration - - params := req.GetParameters() - if params == nil { - params = make(map[string]string) - } - - params["protocol"] = cs.d.shareProto - - shareOpts, err := options.NewControllerVolumeContext(params) - if err != nil { - return nil, status.Errorf(codes.InvalidArgument, "invalid volume parameters: %v", err) - } - - osOpts, err := options.NewOpenstackOptions(req.GetSecrets()) - if err != nil { - return nil, status.Errorf(codes.InvalidArgument, "invalid OpenStack secrets: %v", err) - } - - // Check for pending CreateVolume for this volume name - if _, isPending := pendingVolumes.LoadOrStore(req.GetName(), true); isPending { - return nil, status.Errorf(codes.Aborted, "a volume named %s is already being created", req.GetName()) - } - defer pendingVolumes.Delete(req.GetName()) - - manilaClient, err := cs.d.manilaClientBuilder.New(osOpts) - if err != nil { - return nil, status.Errorf(codes.Unauthenticated, "failed to create Manila v2 client: %v", err) - } - - shareTypeCaps, err := capabilities.GetManilaCapabilities(shareOpts.Type, manilaClient) - if err != nil { - return nil, status.Errorf(codes.Internal, "failed to get Manila capabilities for share type %s: %v", shareOpts.Type, err) - } - - requestedSize := req.GetCapacityRange().GetRequiredBytes() - if requestedSize == 0 { - // At least 1GiB - requestedSize = 1 * bytesInGiB - } - - sizeInGiB := bytesToGiB(requestedSize) - - // Retrieve an existing share or create a new one - - volCreator, err := getVolumeCreator(req.GetVolumeContentSource(), shareOpts, cs.d.compatOpts, shareTypeCaps) - if err != nil { - return nil, err - } - - share, err := volCreator.create(req, req.GetName(), sizeInGiB, manilaClient, shareOpts) - if err != nil { - return nil, err - } - - if err = verifyVolumeCompatibility(sizeInGiB, req, share, shareOpts, cs.d.compatOpts, shareTypeCaps); err != nil { - return nil, status.Errorf(codes.AlreadyExists, "a share named %s already exists, but is incompatible with the request: %v", req.GetName(), err) - } - - // Grant access to the share - - ad := getShareAdapter(shareOpts.Protocol) - - klog.V(4).Infof("creating an access rule for share %s", share.ID) - - accessRight, err := ad.GetOrGrantAccess(&shareadapters.GrantAccessArgs{Share: share, ManilaClient: manilaClient, Options: shareOpts}) - if err != nil { - if err == wait.ErrWaitTimeout { - return nil, status.Errorf(codes.DeadlineExceeded, "deadline exceeded while waiting for access rights %s for share %s to become available", accessRight.ID, share.ID) - } - - return nil, status.Errorf(codes.Internal, "failed to grant access for share %s: %v", share.ID, err) - } - - // Check if compatibility layer is needed and can be used - if compatLayer := tryCompatForVolumeSource(cs.d.compatOpts, shareOpts, req.GetVolumeContentSource(), shareTypeCaps); compatLayer != nil { - if err = compatLayer.SupplementCapability(cs.d.compatOpts, share, accessRight, req, cs.d.fwdEndpoint, manilaClient, cs.d.csiClientBuilder); err != nil { - // An error occurred, the user must clean the share manually - // TODO needs proper monitoring - return nil, err - } - } - - var accessibleTopology []*csi.Topology - if cs.d.withTopology { - // All requisite/preferred topologies are considered valid. Nodes from those zones are required to be able to reach the storage. - // The operator is responsible for making sure that provided topology keys are valid and present on the nodes of the cluster. - accessibleTopology = req.GetAccessibilityRequirements().GetPreferred() - } - - return &csi.CreateVolumeResponse{ - Volume: &csi.Volume{ - VolumeId: share.ID, - ContentSource: req.GetVolumeContentSource(), - AccessibleTopology: accessibleTopology, - CapacityBytes: int64(sizeInGiB) * bytesInGiB, - VolumeContext: map[string]string{ - "shareID": share.ID, - "shareAccessID": accessRight.ID, - "cephfs-mounter": shareOpts.CephfsMounter, - }, - }, - }, nil -} - -func (cs *controllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) { - if err := validateDeleteVolumeRequest(req); err != nil { - return nil, status.Error(codes.InvalidArgument, err.Error()) - } - - osOpts, err := options.NewOpenstackOptions(req.GetSecrets()) - if err != nil { - return nil, status.Errorf(codes.InvalidArgument, "invalid OpenStack secrets: %v", err) - } - - manilaClient, err := cs.d.manilaClientBuilder.New(osOpts) - if err != nil { - return nil, status.Errorf(codes.Unauthenticated, "failed to create Manila v2 client: %v", err) - } - - if err := deleteShare(req.GetVolumeId(), manilaClient); err != nil { - return nil, status.Errorf(codes.Internal, "failed to delete share %s: %v", req.GetVolumeId(), err) - } - - return &csi.DeleteVolumeResponse{}, nil -} - -func (cs *controllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateSnapshotRequest) (*csi.CreateSnapshotResponse, error) { - if cs.d.shareProto == "CEPHFS" { - // Restoring shares from CephFS snapshots needs special handling that's not implemented yet. - // TODO: Creating CephFS snapshots is forbidden until CephFS restoration is in place. - return nil, status.Errorf(codes.InvalidArgument, "the driver doesn't support snapshotting CephFS shares yet") - } - - if err := validateCreateSnapshotRequest(req); err != nil { - return nil, status.Error(codes.InvalidArgument, err.Error()) - } - - // Configuration - - osOpts, err := options.NewOpenstackOptions(req.GetSecrets()) - if err != nil { - return nil, status.Errorf(codes.InvalidArgument, "invalid OpenStack secrets: %v", err) - } - - // Check for pending CreateSnapshots for this snapshot name - if _, isPending := pendingSnapshots.LoadOrStore(req.GetName(), true); isPending { - return nil, status.Errorf(codes.Aborted, "a snapshot named %s is already being created", req.GetName()) - } - defer pendingSnapshots.Delete(req.GetName()) - - manilaClient, err := cs.d.manilaClientBuilder.New(osOpts) - if err != nil { - return nil, status.Errorf(codes.Unauthenticated, "failed to create Manila v2 client: %v", err) - } - - // Retrieve the source share - - sourceShare, err := manilaClient.GetShareByID(req.GetSourceVolumeId()) - if err != nil { - if clouderrors.IsNotFound(err) { - return nil, status.Errorf(codes.NotFound, "failed to create a snapshot (%s) for share %s because the share doesn't exist: %v", req.GetName(), req.GetSourceVolumeId(), err) - } - - return nil, status.Errorf(codes.Internal, "failed to retrieve source share %s when creating a snapshot (%s): %v", req.GetSourceVolumeId(), req.GetName(), err) - } - - if strings.ToUpper(sourceShare.ShareProto) != cs.d.shareProto { - return nil, status.Errorf(codes.InvalidArgument, "share protocol mismatch: requested a snapshot of %s share %s, but share protocol selector is set to %s", - sourceShare.ShareProto, req.GetSourceVolumeId(), cs.d.shareProto) - } - - // Retrieve an existing snapshot or create a new one - - klog.V(4).Infof("creating a snapshot (%s) of share %s", req.GetName(), req.GetSourceVolumeId()) - - snapshot, err := getOrCreateSnapshot(req.GetName(), sourceShare.ID, manilaClient) - if err != nil { - if err == wait.ErrWaitTimeout { - return nil, status.Errorf(codes.DeadlineExceeded, "deadline exceeded while waiting for snapshot %s of share %s to become available", snapshot.ID, req.GetSourceVolumeId()) - } - - if clouderrors.IsNotFound(err) { - return nil, status.Errorf(codes.NotFound, "failed to create a snapshot (%s) for share %s because the share doesn't exist: %v", req.GetName(), req.GetSourceVolumeId(), err) - } - - return nil, status.Errorf(codes.Internal, "failed to create a snapshot (%s) of share %s: %v", req.GetName(), req.GetSourceVolumeId(), err) - } - - if err = verifySnapshotCompatibility(snapshot, req); err != nil { - return nil, status.Errorf(codes.AlreadyExists, "a snapshot named %s already exists, but is incompatible with the request: %v", req.GetName(), err) - } - - // Check for snapshot status, determine whether it's ready - - var readyToUse bool - - switch snapshot.Status { - case snapshotCreating: - readyToUse = false - case snapshotAvailable: - readyToUse = true - case snapshotError: - // An error occurred, try to roll-back the snapshot - tryDeleteSnapshot(snapshot, manilaClient) - - manilaErrMsg, err := lastResourceError(snapshot.ID, manilaClient) - if err != nil { - return nil, status.Errorf(codes.Internal, "snapshot %s of share %s is in error state, error description could not be retrieved: %v", snapshot.ID, req.GetSourceVolumeId(), err.Error()) - } - - return nil, status.Errorf(manilaErrMsg.errCode.toRpcErrorCode(), "snapshot %s of share %s is in error state: %s", snapshot.ID, req.GetSourceVolumeId(), manilaErrMsg.message) - default: - return nil, status.Errorf(codes.Internal, "an error occurred while creating a snapshot (%s) of share %s: snapshot is in an unexpected state: wanted creating/available, got %s", - req.GetName(), req.GetSourceVolumeId(), snapshot.Status) - } - - // Parse CreatedAt timestamp - ctime, err := ptypes.TimestampProto(snapshot.CreatedAt) - if err != nil { - klog.Warningf("couldn't parse timestamp %v from snapshot %s: %v", snapshot.CreatedAt, snapshot.ID, err) - } - - return &csi.CreateSnapshotResponse{ - Snapshot: &csi.Snapshot{ - SnapshotId: snapshot.ID, - SourceVolumeId: req.GetSourceVolumeId(), - SizeBytes: int64(sourceShare.Size) * bytesInGiB, - CreationTime: ctime, - ReadyToUse: readyToUse, - }, - }, nil -} - -func (cs *controllerServer) DeleteSnapshot(ctx context.Context, req *csi.DeleteSnapshotRequest) (*csi.DeleteSnapshotResponse, error) { - if cs.d.shareProto == "CEPHFS" { - // Restoring shares from CephFS snapshots needs special handling that's not implemented yet. - // TODO: Deleting CephFS snapshots is forbidden until CephFS restoration is in place. - return nil, status.Errorf(codes.InvalidArgument, "the driver doesn't support CephFS snapshots yet") - } - - if err := validateDeleteSnapshotRequest(req); err != nil { - return nil, status.Error(codes.InvalidArgument, err.Error()) - } - - osOpts, err := options.NewOpenstackOptions(req.GetSecrets()) - if err != nil { - return nil, status.Errorf(codes.InvalidArgument, "invalid OpenStack secrets: %v", err) - } - - manilaClient, err := cs.d.manilaClientBuilder.New(osOpts) - if err != nil { - return nil, status.Errorf(codes.Unauthenticated, "failed to create Manila v2 client: %v", err) - } - - if err := deleteSnapshot(req.GetSnapshotId(), manilaClient); err != nil { - return nil, status.Errorf(codes.Internal, "failed to delete snapshot %s: %v", req.GetSnapshotId(), err) - } - - return &csi.DeleteSnapshotResponse{}, nil -} - -func (cs *controllerServer) ControllerGetCapabilities(ctx context.Context, req *csi.ControllerGetCapabilitiesRequest) (*csi.ControllerGetCapabilitiesResponse, error) { - return &csi.ControllerGetCapabilitiesResponse{ - Capabilities: cs.d.cscaps, - }, nil -} - -func (cs *controllerServer) ValidateVolumeCapabilities(ctx context.Context, req *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error) { - if err := validateValidateVolumeCapabilitiesRequest(req); err != nil { - return nil, status.Error(codes.InvalidArgument, err.Error()) - } - - osOpts, err := options.NewOpenstackOptions(req.GetSecrets()) - if err != nil { - return nil, status.Errorf(codes.InvalidArgument, "invalid OpenStack secrets: %v", err) - } - - for _, volCap := range req.GetVolumeCapabilities() { - if volCap.GetBlock() != nil { - return &csi.ValidateVolumeCapabilitiesResponse{Message: "block access type is not allowed"}, nil - } - - if volCap.GetMount() == nil { - return &csi.ValidateVolumeCapabilitiesResponse{Message: "volume must be accessible via filesystem API"}, nil - } - - if volCap.GetAccessMode().GetMode() == csi.VolumeCapability_AccessMode_UNKNOWN { - return &csi.ValidateVolumeCapabilitiesResponse{Message: "unknown volume access mode"}, nil - } - } - - manilaClient, err := cs.d.manilaClientBuilder.New(osOpts) - if err != nil { - return nil, status.Errorf(codes.Unauthenticated, "failed to create Manila v2 client: %v", err) - } - - share, err := manilaClient.GetShareByID(req.GetVolumeId()) - if err != nil { - if clouderrors.IsNotFound(err) { - return nil, status.Errorf(codes.NotFound, "share %s not found: %v", req.GetVolumeId(), err) - } - - return nil, status.Errorf(codes.Internal, "failed to retrieve share %s: %v", req.GetVolumeId(), err) - } - - if share.Status != shareAvailable { - if share.Status == shareCreating { - return nil, status.Errorf(codes.Unavailable, "share %s is in transient creating state", share.ID) - } - - return nil, status.Errorf(codes.FailedPrecondition, "share %s is in an unexpected state: wanted %s, got %s", share.ID, shareAvailable, share.Status) - } - - if !compareProtocol(share.ShareProto, cs.d.shareProto) { - return nil, status.Errorf(codes.InvalidArgument, "share protocol mismatch: wanted %s, got %s", cs.d.shareProto, share.ShareProto) - } - - return &csi.ValidateVolumeCapabilitiesResponse{ - Confirmed: &csi.ValidateVolumeCapabilitiesResponse_Confirmed{ - VolumeContext: req.GetVolumeContext(), - VolumeCapabilities: req.GetVolumeCapabilities(), - Parameters: req.GetParameters(), - }, - }, nil -} - -func (cs *controllerServer) ControllerPublishVolume(ctx context.Context, req *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error) { - return nil, status.Error(codes.Unimplemented, "") -} - -func (cs *controllerServer) ControllerUnpublishVolume(ctx context.Context, req *csi.ControllerUnpublishVolumeRequest) (*csi.ControllerUnpublishVolumeResponse, error) { - return nil, status.Error(codes.Unimplemented, "") -} - -func (cs *controllerServer) ListVolumes(context.Context, *csi.ListVolumesRequest) (*csi.ListVolumesResponse, error) { - return nil, status.Error(codes.Unimplemented, "") -} - -func (cs *controllerServer) GetCapacity(context.Context, *csi.GetCapacityRequest) (*csi.GetCapacityResponse, error) { - return nil, status.Error(codes.Unimplemented, "") -} - -func (cs *controllerServer) ListSnapshots(context.Context, *csi.ListSnapshotsRequest) (*csi.ListSnapshotsResponse, error) { - return nil, status.Error(codes.Unimplemented, "") -} - -func (cs *controllerServer) ControllerExpandVolume(ctx context.Context, req *csi.ControllerExpandVolumeRequest) (*csi.ControllerExpandVolumeResponse, error) { - return nil, status.Error(codes.Unimplemented, "") -} diff --git a/pkg/csi/manila/csiclient.go b/pkg/csi/manila/csiclient.go deleted file mode 100644 index 855886921e..0000000000 --- a/pkg/csi/manila/csiclient.go +++ /dev/null @@ -1,48 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package manila - -import ( - "context" - - "github.com/container-storage-interface/spec/lib/go/csi" - "k8s.io/cloud-provider-openstack/pkg/csi/manila/csiclient" -) - -type csiNodeCapabilitySet map[csi.NodeServiceCapability_RPC_Type]bool - -func csiNodeGetCapabilities(ctx context.Context, nodeClient csiclient.Node) (csiNodeCapabilitySet, error) { - rsp, err := nodeClient.GetCapabilities(ctx) - if err != nil { - return nil, err - } - - caps := csiNodeCapabilitySet{} - for _, cap := range rsp.GetCapabilities() { - if cap == nil { - continue - } - rpc := cap.GetRpc() - if rpc == nil { - continue - } - t := rpc.GetType() - caps[t] = true - } - - return caps, nil -} diff --git a/pkg/csi/manila/csiclient/builder.go b/pkg/csi/manila/csiclient/builder.go deleted file mode 100644 index a8d31a0dde..0000000000 --- a/pkg/csi/manila/csiclient/builder.go +++ /dev/null @@ -1,109 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package csiclient - -import ( - "context" - "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/kubernetes-csi/csi-lib-utils/protosanitizer" - "google.golang.org/grpc" - "k8s.io/klog/v2" - "sync/atomic" - "time" -) - -var ( - grpcCallCounter uint64 - - dialOptions = []grpc.DialOption{ - grpc.WithInsecure(), - grpc.WithBackoffMaxDelay(time.Second), - grpc.WithBlock(), - grpc.WithUnaryInterceptor(func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { - callID := atomic.AddUint64(&grpcCallCounter, 1) - - klog.V(3).Infof("[ID:%d] FWD GRPC call: %s", callID, method) - klog.V(5).Infof("[ID:%d] FWD GRPC request: %s", callID, protosanitizer.StripSecrets(req)) - - err := invoker(ctx, method, req, reply, cc, opts...) - if err != nil { - klog.Infof("[ID:%d] FWD GRPC error: %v", callID, err) - } else { - klog.V(5).Infof("[ID:%d] FWD GRPC response: %s", callID, protosanitizer.StripSecrets(reply)) - } - - return err - }), - } - - _ Builder = &ClientBuilder{} -) - -func NewNodeSvcClient(conn *grpc.ClientConn) *NodeSvcClient { - return &NodeSvcClient{cl: csi.NewNodeClient(conn)} -} - -func NewIdentitySvcClient(conn *grpc.ClientConn) *IdentitySvcClient { - return &IdentitySvcClient{cl: csi.NewIdentityClient(conn)} -} - -func NewConnection(endpoint string) (*grpc.ClientConn, error) { - var ( - conn *grpc.ClientConn - err error - ) - - dialFinished := make(chan bool) - go func() { - conn, err = grpc.Dial(endpoint, dialOptions...) - close(dialFinished) - }() - - ticker := time.NewTicker(time.Second) - defer ticker.Stop() - - for { - select { - case <-ticker.C: - klog.Warningf("still connecting to %s", endpoint) - case <-dialFinished: - return conn, err - } - } -} - -func NewConnectionWithContext(ctx context.Context, endpoint string) (*grpc.ClientConn, error) { - return grpc.DialContext(ctx, endpoint, dialOptions...) -} - -type ClientBuilder struct{} - -func (b ClientBuilder) NewConnection(endpoint string) (*grpc.ClientConn, error) { - return NewConnection(endpoint) -} - -func (b ClientBuilder) NewConnectionWithContext(ctx context.Context, endpoint string) (*grpc.ClientConn, error) { - return NewConnectionWithContext(ctx, endpoint) -} - -func (b ClientBuilder) NewNodeServiceClient(conn *grpc.ClientConn) Node { - return NewNodeSvcClient(conn) -} - -func (b ClientBuilder) NewIdentityServiceClient(conn *grpc.ClientConn) Identity { - return NewIdentitySvcClient(conn) -} diff --git a/pkg/csi/manila/csiclient/client.go b/pkg/csi/manila/csiclient/client.go deleted file mode 100644 index e4974748a8..0000000000 --- a/pkg/csi/manila/csiclient/client.go +++ /dev/null @@ -1,70 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package csiclient - -import ( - "context" - "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/kubernetes-csi/csi-lib-utils/connection" - "google.golang.org/grpc" - "time" -) - -var ( - _ Node = &NodeSvcClient{} - _ Identity = &IdentitySvcClient{} -) - -type NodeSvcClient struct { - cl csi.NodeClient -} - -func (c *NodeSvcClient) GetCapabilities(ctx context.Context) (*csi.NodeGetCapabilitiesResponse, error) { - return c.cl.NodeGetCapabilities(ctx, &csi.NodeGetCapabilitiesRequest{}) -} - -func (c *NodeSvcClient) StageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) { - return c.cl.NodeStageVolume(ctx, req) -} - -func (c *NodeSvcClient) UnstageVolume(ctx context.Context, req *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) { - return c.cl.NodeUnstageVolume(ctx, req) -} - -func (c *NodeSvcClient) PublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { - return c.cl.NodePublishVolume(ctx, req) -} - -func (c *NodeSvcClient) UnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) { - return c.cl.NodeUnpublishVolume(ctx, req) -} - -type IdentitySvcClient struct { - cl csi.IdentityClient -} - -func (c IdentitySvcClient) Probe(ctx context.Context, req *csi.ProbeRequest) (*csi.ProbeResponse, error) { - return c.cl.Probe(ctx, req) -} - -func (c IdentitySvcClient) ProbeForever(conn *grpc.ClientConn, singleProbeTimeout time.Duration) error { - return connection.ProbeForever(conn, singleProbeTimeout) -} - -func (c IdentitySvcClient) GetPluginInfo(ctx context.Context) (*csi.GetPluginInfoResponse, error) { - return c.cl.GetPluginInfo(ctx, &csi.GetPluginInfoRequest{}) -} diff --git a/pkg/csi/manila/csiclient/interface.go b/pkg/csi/manila/csiclient/interface.go deleted file mode 100644 index bf395fe251..0000000000 --- a/pkg/csi/manila/csiclient/interface.go +++ /dev/null @@ -1,48 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package csiclient - -import ( - "context" - "github.com/container-storage-interface/spec/lib/go/csi" - "google.golang.org/grpc" - "time" -) - -type Node interface { - GetCapabilities(ctx context.Context) (*csi.NodeGetCapabilitiesResponse, error) - - StageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) - UnstageVolume(ctx context.Context, req *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) - - PublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) - UnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) -} - -type Identity interface { - GetPluginInfo(ctx context.Context) (*csi.GetPluginInfoResponse, error) - Probe(ctx context.Context, req *csi.ProbeRequest) (*csi.ProbeResponse, error) - ProbeForever(conn *grpc.ClientConn, singleProbeTimeout time.Duration) error -} - -type Builder interface { - NewConnection(endpoint string) (*grpc.ClientConn, error) - NewConnectionWithContext(ctx context.Context, endpoint string) (*grpc.ClientConn, error) - - NewNodeServiceClient(conn *grpc.ClientConn) Node - NewIdentityServiceClient(conn *grpc.ClientConn) Identity -} diff --git a/pkg/csi/manila/driver.go b/pkg/csi/manila/driver.go deleted file mode 100644 index d7bb4d1c07..0000000000 --- a/pkg/csi/manila/driver.go +++ /dev/null @@ -1,331 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package manila - -import ( - "context" - "fmt" - "net" - "os" - "strings" - "sync" - "sync/atomic" - "time" - - "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/kubernetes-csi/csi-lib-utils/protosanitizer" - "google.golang.org/grpc" - "k8s.io/cloud-provider-openstack/pkg/csi/manila/csiclient" - "k8s.io/cloud-provider-openstack/pkg/csi/manila/manilaclient" - "k8s.io/cloud-provider-openstack/pkg/csi/manila/options" - "k8s.io/cloud-provider-openstack/pkg/version" - "k8s.io/klog/v2" -) - -type DriverOpts struct { - DriverName string - NodeID string - NodeAZ string - WithTopology bool - ShareProto string - - ServerCSIEndpoint string - FwdCSIEndpoint string - - ManilaClientBuilder manilaclient.Builder - CSIClientBuilder csiclient.Builder - - CompatOpts *options.CompatibilityOptions -} - -type Driver struct { - nodeID string - nodeAZ string - withTopology bool - name string - fqVersion string // Fully qualified version in format {driverVersion}@{CPO version} - shareProto string - - serverEndpoint string - fwdEndpoint string - - compatOpts *options.CompatibilityOptions - - ids *identityServer - cs *controllerServer - ns *nodeServer - - vcaps []*csi.VolumeCapability_AccessMode - cscaps []*csi.ControllerServiceCapability - nscaps []*csi.NodeServiceCapability - - manilaClientBuilder manilaclient.Builder - csiClientBuilder csiclient.Builder -} - -type nonBlockingGRPCServer struct { - wg sync.WaitGroup - server *grpc.Server -} - -const ( - specVersion = "1.2.0" - driverVersion = "0.9.0" - topologyKey = "topology.manila.csi.openstack.org/zone" -) - -var ( - serverGRPCEndpointCallCounter uint64 -) - -func argNotEmpty(val, name string) error { - if val == "" { - return fmt.Errorf("%s is missing", name) - } - - return nil -} - -func NewDriver(o *DriverOpts) (*Driver, error) { - for k, v := range map[string]string{"node ID": o.NodeID, "driver name": o.DriverName, "driver endpoint": o.ServerCSIEndpoint, "FWD endpoint": o.FwdCSIEndpoint, "share protocol selector": o.ShareProto} { - if err := argNotEmpty(v, k); err != nil { - return nil, err - } - } - - d := &Driver{ - fqVersion: fmt.Sprintf("%s@%s", driverVersion, version.Version), - nodeID: o.NodeID, - nodeAZ: o.NodeAZ, - withTopology: o.WithTopology, - name: o.DriverName, - serverEndpoint: o.ServerCSIEndpoint, - fwdEndpoint: o.FwdCSIEndpoint, - shareProto: strings.ToUpper(o.ShareProto), - compatOpts: o.CompatOpts, - manilaClientBuilder: o.ManilaClientBuilder, - csiClientBuilder: o.CSIClientBuilder, - } - - klog.Info("Driver: ", d.name) - klog.Info("Driver version: ", d.fqVersion) - klog.Info("CSI spec version: ", specVersion) - - getShareAdapter(d.shareProto) // The program will terminate with a non-zero exit code if the share protocol selector is wrong - klog.Infof("Operating on %s shares", d.shareProto) - - if d.withTopology { - klog.Infof("Topology awareness enabled, node availability zone: %s", d.nodeAZ) - } else { - klog.Info("Topology awareness disabled") - } - - serverProto, serverAddr, err := parseGRPCEndpoint(o.ServerCSIEndpoint) - if err != nil { - return nil, fmt.Errorf("failed to parse server endpoint address %s: %v", o.ServerCSIEndpoint, err) - } - - fwdProto, fwdAddr, err := parseGRPCEndpoint(o.FwdCSIEndpoint) - if err != nil { - return nil, fmt.Errorf("failed to parse proxy client address %s: %v", o.FwdCSIEndpoint, err) - } - - d.serverEndpoint = endpointAddress(serverProto, serverAddr) - d.fwdEndpoint = endpointAddress(fwdProto, fwdAddr) - - d.addControllerServiceCapabilities([]csi.ControllerServiceCapability_RPC_Type{ - csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME, - csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT, - }) - - d.addVolumeCapabilityAccessModes([]csi.VolumeCapability_AccessMode_Mode{ - csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER, - csi.VolumeCapability_AccessMode_MULTI_NODE_SINGLE_WRITER, - csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY, - csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, - csi.VolumeCapability_AccessMode_SINGLE_NODE_READER_ONLY, - }) - - var supportsNodeStage bool - - nodeCapsMap, err := d.initProxiedDriver() - if err != nil { - return nil, fmt.Errorf("failed to initialize proxied CSI driver: %v", err) - } - var nscaps []csi.NodeServiceCapability_RPC_Type - for c := range nodeCapsMap { - nscaps = append(nscaps, c) - - if c == csi.NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME { - supportsNodeStage = true - } - } - - d.addNodeServiceCapabilities(nscaps) - - d.ids = &identityServer{d: d} - d.cs = &controllerServer{d: d} - d.ns = &nodeServer{d: d, supportsNodeStage: supportsNodeStage, nodeStageCache: make(map[volumeID]stageCacheEntry)} - - return d, nil -} - -func (d *Driver) Run() { - s := nonBlockingGRPCServer{} - s.start(d.serverEndpoint, d.ids, d.cs, d.ns) - s.wait() -} - -func (d *Driver) addControllerServiceCapabilities(cs []csi.ControllerServiceCapability_RPC_Type) { - var caps []*csi.ControllerServiceCapability - - for _, c := range cs { - klog.Infof("Enabling controller service capability: %v", c.String()) - csc := &csi.ControllerServiceCapability{ - Type: &csi.ControllerServiceCapability_Rpc{ - Rpc: &csi.ControllerServiceCapability_RPC{ - Type: c, - }, - }, - } - - caps = append(caps, csc) - } - - d.cscaps = caps -} - -func (d *Driver) addVolumeCapabilityAccessModes(vs []csi.VolumeCapability_AccessMode_Mode) { - var caps []*csi.VolumeCapability_AccessMode - - for _, c := range vs { - klog.Infof("Enabling volume access mode: %v", c.String()) - caps = append(caps, &csi.VolumeCapability_AccessMode{Mode: c}) - } - - d.vcaps = caps -} - -func (d *Driver) addNodeServiceCapabilities(ns []csi.NodeServiceCapability_RPC_Type) { - var caps []*csi.NodeServiceCapability - - for _, c := range ns { - klog.Infof("Enabling node service capability: %v", c.String()) - nsc := &csi.NodeServiceCapability{ - Type: &csi.NodeServiceCapability_Rpc{ - Rpc: &csi.NodeServiceCapability_RPC{ - Type: c, - }, - }, - } - - caps = append(caps, nsc) - } - - d.nscaps = caps -} - -func (d *Driver) initProxiedDriver() (csiNodeCapabilitySet, error) { - conn, err := d.csiClientBuilder.NewConnection(d.fwdEndpoint) - if err != nil { - return nil, fmt.Errorf("connecting to %s endpoint failed: %v", d.fwdEndpoint, err) - } - - identityClient := d.csiClientBuilder.NewIdentityServiceClient(conn) - - if err = identityClient.ProbeForever(conn, time.Second*5); err != nil { - return nil, fmt.Errorf("probe failed: %v", err) - } - - ctx, cancel := context.WithTimeout(context.Background(), time.Second*15) - defer cancel() - - pluginInfo, err := identityClient.GetPluginInfo(ctx) - if err != nil { - return nil, fmt.Errorf("failed to get plugin info of the proxied driver: %v", err) - } - - klog.Infof("proxying CSI driver %s version %s", pluginInfo.GetName(), pluginInfo.GetVendorVersion()) - - nodeCaps, err := csiNodeGetCapabilities(ctx, d.csiClientBuilder.NewNodeServiceClient(conn)) - if err != nil { - return nil, fmt.Errorf("failed to get node capabilities: %v", err) - } - - return nodeCaps, nil -} - -func (s *nonBlockingGRPCServer) start(endpoint string, ids *identityServer, cs *controllerServer, ns *nodeServer) { - s.wg.Add(1) - go s.serve(endpoint, ids, cs, ns) -} - -func (s *nonBlockingGRPCServer) wait() { - s.wg.Wait() -} - -func (s *nonBlockingGRPCServer) stop() { - s.server.GracefulStop() -} - -func (s *nonBlockingGRPCServer) forceStop() { - s.server.Stop() -} - -func (s *nonBlockingGRPCServer) serve(endpoint string, ids *identityServer, cs *controllerServer, ns *nodeServer) { - proto, addr, err := parseGRPCEndpoint(endpoint) - if err != nil { - klog.Fatalf("couldn't parse GRPC server endpoint address %s: %v", endpoint, err) - } - - if proto == "unix" { - if err = os.Remove(addr); err != nil && !os.IsNotExist(err) { - klog.Fatalf("failed to remove an existing socket file %s: %v", addr, err) - } - } - - listener, err := net.Listen(proto, addr) - if err != nil { - klog.Fatalf("listen failed for GRPC server: %v", err) - } - - server := grpc.NewServer(grpc.UnaryInterceptor(func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { - callID := atomic.AddUint64(&serverGRPCEndpointCallCounter, 1) - - klog.V(3).Infof("[ID:%d] GRPC call: %s", callID, info.FullMethod) - klog.V(5).Infof("[ID:%d] GRPC request: %s", callID, protosanitizer.StripSecrets(req)) - resp, err := handler(ctx, req) - if err != nil { - klog.Errorf("[ID:%d] GRPC error: %v", callID, err) - } else { - klog.V(5).Infof("[ID:%d] GRPC response: %s", callID, protosanitizer.StripSecrets(resp)) - } - return resp, err - })) - - s.server = server - - csi.RegisterIdentityServer(server, ids) - csi.RegisterControllerServer(server, cs) - csi.RegisterNodeServer(server, ns) - - klog.Infof("listening for connections on %#v", listener.Addr()) - - if err := server.Serve(listener); err != nil { - klog.Fatalf("GRPC server failure: %v", err) - } -} diff --git a/pkg/csi/manila/identityserver.go b/pkg/csi/manila/identityserver.go deleted file mode 100644 index 6be4fb4a10..0000000000 --- a/pkg/csi/manila/identityserver.go +++ /dev/null @@ -1,74 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package manila - -import ( - "context" - "github.com/container-storage-interface/spec/lib/go/csi" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" -) - -type identityServer struct { - d *Driver -} - -func (ids *identityServer) GetPluginInfo(ctx context.Context, req *csi.GetPluginInfoRequest) (*csi.GetPluginInfoResponse, error) { - if ids.d.name == "" { - return nil, status.Error(codes.Unavailable, "Driver name not configured") - } - - return &csi.GetPluginInfoResponse{ - Name: ids.d.name, - VendorVersion: ids.d.fqVersion, - }, nil -} - -func (ids *identityServer) Probe(ctx context.Context, req *csi.ProbeRequest) (*csi.ProbeResponse, error) { - csiConn, err := ids.d.csiClientBuilder.NewConnectionWithContext(ctx, ids.d.fwdEndpoint) - if err != nil { - return nil, status.Error(codes.FailedPrecondition, fmtGrpcConnError(ids.d.fwdEndpoint, err)) - } - - return ids.d.csiClientBuilder.NewIdentityServiceClient(csiConn).Probe(ctx, req) -} - -func (ids *identityServer) GetPluginCapabilities(ctx context.Context, req *csi.GetPluginCapabilitiesRequest) (*csi.GetPluginCapabilitiesResponse, error) { - caps := []*csi.PluginCapability{ - { - Type: &csi.PluginCapability_Service_{ - Service: &csi.PluginCapability_Service{ - Type: csi.PluginCapability_Service_CONTROLLER_SERVICE, - }, - }, - }, - } - - if ids.d.withTopology { - caps = append(caps, &csi.PluginCapability{ - Type: &csi.PluginCapability_Service_{ - Service: &csi.PluginCapability_Service{ - Type: csi.PluginCapability_Service_VOLUME_ACCESSIBILITY_CONSTRAINTS, - }, - }, - }) - } - - return &csi.GetPluginCapabilitiesResponse{ - Capabilities: caps, - }, nil -} diff --git a/pkg/csi/manila/manilaclient/builder.go b/pkg/csi/manila/manilaclient/builder.go deleted file mode 100644 index 9f1b8b14bb..0000000000 --- a/pkg/csi/manila/manilaclient/builder.go +++ /dev/null @@ -1,124 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package manilaclient - -import ( - "fmt" - "regexp" - "strconv" - "strings" - - "github.com/gophercloud/gophercloud" - "github.com/gophercloud/gophercloud/openstack" - "github.com/gophercloud/gophercloud/openstack/sharedfilesystems/apiversions" - openstack_provider "k8s.io/cloud-provider-openstack/pkg/cloudprovider/providers/openstack" -) - -type ClientBuilder struct { - UserAgent string - ExtraUserAgentData []string -} - -func (cb *ClientBuilder) New(o *openstack_provider.AuthOpts) (Interface, error) { - return New(o, cb.UserAgent, cb.ExtraUserAgentData) -} - -const ( - minimumManilaVersion = "2.37" -) - -var ( - manilaMicroversionRegexp = regexp.MustCompile(`^(\d+)\.(\d+)$`) -) - -func splitManilaMicroversion(microversion string) (major, minor int) { - if err := validateManilaMicroversion(microversion); err != nil { - return - } - - parts := strings.Split(microversion, ".") - major, _ = strconv.Atoi(parts[0]) - minor, _ = strconv.Atoi(parts[1]) - - return -} - -func validateManilaMicroversion(microversion string) error { - if !manilaMicroversionRegexp.MatchString(microversion) { - return fmt.Errorf("invalid microversion format in %q", microversion) - } - - return nil -} - -func compareManilaVersionsLessThan(a, b string) bool { - aMaj, aMin := splitManilaMicroversion(a) - bMaj, bMin := splitManilaMicroversion(b) - - return aMaj < bMaj || (aMaj == bMaj && aMin < bMin) -} - -func validateManilaClient(c *gophercloud.ServiceClient) error { - serverVersion, err := apiversions.Get(c, "v2").Extract() - if err != nil { - return fmt.Errorf("failed to get Manila v2 API microversions: %v", err) - } - - if err = validateManilaMicroversion(serverVersion.MinVersion); err != nil { - return fmt.Errorf("server's minimum microversion is invalid: %v", err) - } - - if err = validateManilaMicroversion(serverVersion.Version); err != nil { - return fmt.Errorf("server's maximum microversion is invalid: %v", err) - } - - if compareManilaVersionsLessThan(c.Microversion, serverVersion.MinVersion) { - return fmt.Errorf("client's microversion %s is lower than server's minimum microversion %s", c.Microversion, serverVersion.MinVersion) - } - - if compareManilaVersionsLessThan(serverVersion.Version, c.Microversion) { - return fmt.Errorf("client's microversion %s is higher than server's highest supported microversion %s", c.Microversion, serverVersion.Version) - } - - return nil -} - -func New(o *openstack_provider.AuthOpts, userAgent string, extraUserAgentData []string) (*Client, error) { - // Authenticate and create Manila v2 client - provider, err := openstack_provider.NewOpenStackClient(o, userAgent, extraUserAgentData...) - if err != nil { - return nil, fmt.Errorf("failed to authenticate: %v", err) - } - - client, err := openstack.NewSharedFileSystemV2(provider, gophercloud.EndpointOpts{Region: o.Region}) - if err != nil { - return nil, fmt.Errorf("failed to create Manila v2 client: %v", err) - } - - // Check client's and server's versions for compatibility - - client.Microversion = minimumManilaVersion - if err = validateManilaClient(client); err != nil { - return nil, fmt.Errorf("Manila v2 client validation failed: %v", err) - } - - return &Client{c: client}, nil -} - -func NewFromServiceClient(c *gophercloud.ServiceClient) *Client { - return &Client{c: c} -} diff --git a/pkg/csi/manila/manilaclient/client.go b/pkg/csi/manila/manilaclient/client.go deleted file mode 100644 index 4f507551bc..0000000000 --- a/pkg/csi/manila/manilaclient/client.go +++ /dev/null @@ -1,116 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package manilaclient - -import ( - "github.com/gophercloud/gophercloud" - "github.com/gophercloud/gophercloud/openstack/sharedfilesystems/v2/messages" - "github.com/gophercloud/gophercloud/openstack/sharedfilesystems/v2/shares" - "github.com/gophercloud/gophercloud/openstack/sharedfilesystems/v2/sharetypes" - "github.com/gophercloud/gophercloud/openstack/sharedfilesystems/v2/snapshots" - shares_utils "github.com/gophercloud/utils/openstack/sharedfilesystems/v2/shares" - sharetypes_utils "github.com/gophercloud/utils/openstack/sharedfilesystems/v2/sharetypes" - snapshots_utils "github.com/gophercloud/utils/openstack/sharedfilesystems/v2/snapshots" -) - -type Client struct { - c *gophercloud.ServiceClient -} - -func (c Client) GetShareByID(shareID string) (*shares.Share, error) { - return shares.Get(c.c, shareID).Extract() -} - -func (c Client) GetShareByName(shareName string) (*shares.Share, error) { - shareID, err := shares_utils.IDFromName(c.c, shareName) - if err != nil { - return nil, err - } - - return shares.Get(c.c, shareID).Extract() -} - -func (c Client) CreateShare(opts shares.CreateOptsBuilder) (*shares.Share, error) { - return shares.Create(c.c, opts).Extract() -} - -func (c Client) DeleteShare(shareID string) error { - return shares.Delete(c.c, shareID).ExtractErr() -} - -func (c Client) GetExportLocations(shareID string) ([]shares.ExportLocation, error) { - return shares.ListExportLocations(c.c, shareID).Extract() -} - -func (c Client) SetShareMetadata(shareID string, opts shares.SetMetadataOptsBuilder) (map[string]string, error) { - return shares.SetMetadata(c.c, shareID, opts).Extract() -} - -func (c Client) GetAccessRights(shareID string) ([]shares.AccessRight, error) { - return shares.ListAccessRights(c.c, shareID).Extract() -} - -func (c Client) GrantAccess(shareID string, opts shares.GrantAccessOptsBuilder) (*shares.AccessRight, error) { - return shares.GrantAccess(c.c, shareID, opts).Extract() -} - -func (c Client) GetSnapshotByID(snapID string) (*snapshots.Snapshot, error) { - return snapshots.Get(c.c, snapID).Extract() -} - -func (c Client) GetSnapshotByName(snapName string) (*snapshots.Snapshot, error) { - snapID, err := snapshots_utils.IDFromName(c.c, snapName) - if err != nil { - return nil, err - } - - return snapshots.Get(c.c, snapID).Extract() -} - -func (c Client) CreateSnapshot(opts snapshots.CreateOptsBuilder) (*snapshots.Snapshot, error) { - return snapshots.Create(c.c, opts).Extract() -} - -func (c Client) DeleteSnapshot(snapID string) error { - return snapshots.Delete(c.c, snapID).ExtractErr() -} - -func (c Client) GetExtraSpecs(shareTypeID string) (sharetypes.ExtraSpecs, error) { - return sharetypes.GetExtraSpecs(c.c, shareTypeID).Extract() -} - -func (c Client) GetShareTypes() ([]sharetypes.ShareType, error) { - allPages, err := sharetypes.List(c.c, sharetypes.ListOpts{}).AllPages() - if err != nil { - return nil, err - } - - return sharetypes.ExtractShareTypes(allPages) -} - -func (c Client) GetShareTypeIDFromName(shareTypeName string) (string, error) { - return sharetypes_utils.IDFromName(c.c, shareTypeName) -} - -func (c Client) GetUserMessages(opts messages.ListOptsBuilder) ([]messages.Message, error) { - allPages, err := messages.List(c.c, opts).AllPages() - if err != nil { - return nil, err - } - - return messages.ExtractMessages(allPages) -} diff --git a/pkg/csi/manila/manilaclient/interface.go b/pkg/csi/manila/manilaclient/interface.go deleted file mode 100644 index fd89f7a177..0000000000 --- a/pkg/csi/manila/manilaclient/interface.go +++ /dev/null @@ -1,54 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package manilaclient - -import ( - "github.com/gophercloud/gophercloud/openstack/sharedfilesystems/v2/messages" - "github.com/gophercloud/gophercloud/openstack/sharedfilesystems/v2/shares" - "github.com/gophercloud/gophercloud/openstack/sharedfilesystems/v2/sharetypes" - "github.com/gophercloud/gophercloud/openstack/sharedfilesystems/v2/snapshots" - openstack_provider "k8s.io/cloud-provider-openstack/pkg/cloudprovider/providers/openstack" -) - -type Interface interface { - GetShareByID(shareID string) (*shares.Share, error) - GetShareByName(shareName string) (*shares.Share, error) - CreateShare(opts shares.CreateOptsBuilder) (*shares.Share, error) - DeleteShare(shareID string) error - - GetExportLocations(shareID string) ([]shares.ExportLocation, error) - - SetShareMetadata(shareID string, opts shares.SetMetadataOptsBuilder) (map[string]string, error) - - GetAccessRights(shareID string) ([]shares.AccessRight, error) - GrantAccess(shareID string, opts shares.GrantAccessOptsBuilder) (*shares.AccessRight, error) - - GetSnapshotByID(snapID string) (*snapshots.Snapshot, error) - GetSnapshotByName(snapName string) (*snapshots.Snapshot, error) - CreateSnapshot(opts snapshots.CreateOptsBuilder) (*snapshots.Snapshot, error) - DeleteSnapshot(snapID string) error - - GetExtraSpecs(shareTypeID string) (sharetypes.ExtraSpecs, error) - GetShareTypes() ([]sharetypes.ShareType, error) - GetShareTypeIDFromName(shareTypeName string) (string, error) - - GetUserMessages(opts messages.ListOptsBuilder) ([]messages.Message, error) -} - -type Builder interface { - New(o *openstack_provider.AuthOpts) (Interface, error) -} diff --git a/pkg/csi/manila/nodeserver.go b/pkg/csi/manila/nodeserver.go deleted file mode 100644 index f45a67e7d0..0000000000 --- a/pkg/csi/manila/nodeserver.go +++ /dev/null @@ -1,338 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package manila - -import ( - "context" - "strings" - "sync" - - "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/gophercloud/gophercloud/openstack/sharedfilesystems/v2/shares" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - openstack_provider "k8s.io/cloud-provider-openstack/pkg/cloudprovider/providers/openstack" - "k8s.io/cloud-provider-openstack/pkg/csi/manila/options" - "k8s.io/cloud-provider-openstack/pkg/csi/manila/shareadapters" - clouderrors "k8s.io/cloud-provider-openstack/pkg/util/errors" - "k8s.io/klog/v2" -) - -type nodeServer struct { - d *Driver - - supportsNodeStage bool - // The result of NodeStageVolume is stashed away for NodePublishVolume(s) that will follow - nodeStageCache map[volumeID]stageCacheEntry - nodeStageCacheMtx sync.RWMutex -} - -type stageCacheEntry struct { - volumeContext map[string]string - stageSecret map[string]string - publishSecret map[string]string -} - -func (ns *nodeServer) buildVolumeContext(volID volumeID, shareOpts *options.NodeVolumeContext, osOpts *openstack_provider.AuthOpts) ( - volumeContext map[string]string, accessRight *shares.AccessRight, err error, -) { - manilaClient, err := ns.d.manilaClientBuilder.New(osOpts) - if err != nil { - return nil, nil, status.Errorf(codes.Unauthenticated, "failed to create Manila v2 client: %v", err) - } - - // Retrieve the share by its ID or name - - var share *shares.Share - - if shareOpts.ShareID != "" { - share, err = manilaClient.GetShareByID(shareOpts.ShareID) - if err != nil { - if clouderrors.IsNotFound(err) { - return nil, nil, status.Errorf(codes.NotFound, "share %s not found: %v", shareOpts.ShareID, err) - } - - return nil, nil, status.Errorf(codes.Internal, "failed to retrieve share %s: %v", shareOpts.ShareID, err) - } - } else { - share, err = manilaClient.GetShareByName(shareOpts.ShareName) - if err != nil { - if clouderrors.IsNotFound(err) { - return nil, nil, status.Errorf(codes.NotFound, "no share named %s found: %v", shareOpts.ShareName, err) - } - - return nil, nil, status.Errorf(codes.Internal, "failed to retrieve share named %s: %v", shareOpts.ShareName, err) - } - } - - // Verify the plugin supports this share - - if strings.ToLower(share.ShareProto) != strings.ToLower(ns.d.shareProto) { - return nil, nil, status.Errorf(codes.InvalidArgument, - "wrong share protocol %s for volume %s (share ID %s), the plugin is set to operate in %s", - share.ShareProto, volID, share.ID, ns.d.shareProto) - } - - if share.Status != shareAvailable { - if share.Status == shareCreating { - return nil, nil, status.Errorf(codes.Unavailable, "share %s for volume %s is in transient creating state", share.ID, volID) - } - - return nil, nil, status.Errorf(codes.FailedPrecondition, "invalid share status for volume %s (share ID %s): expected 'available', got '%s'", - volID, share.ID, share.Status) - } - - // Get the access right for this share - - accessRights, err := manilaClient.GetAccessRights(share.ID) - if err != nil { - return nil, nil, status.Errorf(codes.Internal, "failed to list access rights for volume %s (share ID %s): %v", - volID, share.ID, err) - } - - for i := range accessRights { - if accessRights[i].ID == shareOpts.ShareAccessID { - accessRight = &accessRights[i] - break - } - } - - if accessRight == nil { - return nil, nil, status.Errorf(codes.InvalidArgument, "cannot find access right %s for volume %s (share ID %s)", - shareOpts.ShareAccessID, volID, share.ID) - } - - // Retrieve list of all export locations for this share. - // Share adapter will try to choose the correct one for mounting. - - availableExportLocations, err := manilaClient.GetExportLocations(share.ID) - if err != nil { - return nil, nil, status.Errorf(codes.Internal, "failed to list export locations for volume %s: %v", volID, err) - } - - // Build volume context for fwd plugin - - sa := getShareAdapter(ns.d.shareProto) - - volumeContext, err = sa.BuildVolumeContext(&shareadapters.VolumeContextArgs{Locations: availableExportLocations, Options: shareOpts}) - if err != nil { - return nil, nil, status.Errorf(codes.InvalidArgument, "failed to build volume context for volume %s: %v", volID, err) - } - - return -} - -func buildNodePublishSecret(accessRight *shares.AccessRight, sa shareadapters.ShareAdapter, volID volumeID) (map[string]string, error) { - secret, err := sa.BuildNodePublishSecret(&shareadapters.SecretArgs{AccessRight: accessRight}) - if err != nil { - return nil, status.Errorf(codes.InvalidArgument, "failed to build publish secret for volume %s: %v", volID, err) - } - - return secret, nil -} - -func buildNodeStageSecret(accessRight *shares.AccessRight, sa shareadapters.ShareAdapter, volID volumeID) (map[string]string, error) { - secret, err := sa.BuildNodeStageSecret(&shareadapters.SecretArgs{AccessRight: accessRight}) - if err != nil { - return nil, status.Errorf(codes.InvalidArgument, "failed to build stage secret for volume %s: %v", volID, err) - } - - return secret, nil -} - -func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { - if err := validateNodePublishVolumeRequest(req); err != nil { - return nil, status.Error(codes.InvalidArgument, err.Error()) - } - - // Configuration - - shareOpts, err := options.NewNodeVolumeContext(req.GetVolumeContext()) - if err != nil { - return nil, status.Errorf(codes.InvalidArgument, "invalid volume context: %v", err) - } - - osOpts, err := options.NewOpenstackOptions(req.GetSecrets()) - if err != nil { - return nil, status.Errorf(codes.InvalidArgument, "invalid OpenStack secrets: %v", err) - } - - volID := volumeID(req.GetVolumeId()) - - var ( - accessRight *shares.AccessRight - volumeCtx, secret map[string]string - ) - - if ns.supportsNodeStage { - // STAGE_UNSTAGE_VOLUME capability is enabled, NodeStageVolume should've already built the staging data - - ns.nodeStageCacheMtx.RLock() - cacheEntry, ok := ns.nodeStageCache[volID] - ns.nodeStageCacheMtx.RUnlock() - - if ok { - volumeCtx, secret = cacheEntry.volumeContext, cacheEntry.publishSecret - } else { - klog.Warningf("STAGE_UNSTAGE_VOLUME capability is enabled, but node stage cache doesn't contain an entry for %s - this is most likely a bug! Rebuilding staging data anyway...", volID) - volumeCtx, accessRight, err = ns.buildVolumeContext(volID, shareOpts, osOpts) - if err == nil { - secret, err = buildNodePublishSecret(accessRight, getShareAdapter(ns.d.shareProto), volID) - } - } - } else { - volumeCtx, accessRight, err = ns.buildVolumeContext(volID, shareOpts, osOpts) - if err == nil { - secret, err = buildNodePublishSecret(accessRight, getShareAdapter(ns.d.shareProto), volID) - } - } - - if err != nil { - return nil, err - } - - // Forward the RPC - - csiConn, err := ns.d.csiClientBuilder.NewConnectionWithContext(ctx, ns.d.fwdEndpoint) - if err != nil { - return nil, status.Error(codes.Unavailable, fmtGrpcConnError(ns.d.fwdEndpoint, err)) - } - - req.Secrets = secret - req.VolumeContext = volumeCtx - - return ns.d.csiClientBuilder.NewNodeServiceClient(csiConn).PublishVolume(ctx, req) -} - -func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) { - if err := validateNodeUnpublishVolumeRequest(req); err != nil { - return nil, status.Error(codes.InvalidArgument, err.Error()) - } - - csiConn, err := ns.d.csiClientBuilder.NewConnectionWithContext(ctx, ns.d.fwdEndpoint) - if err != nil { - return nil, status.Error(codes.Unavailable, fmtGrpcConnError(ns.d.fwdEndpoint, err)) - } - - return ns.d.csiClientBuilder.NewNodeServiceClient(csiConn).UnpublishVolume(ctx, req) -} - -func (ns *nodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) { - if err := validateNodeStageVolumeRequest(req); err != nil { - return nil, status.Error(codes.InvalidArgument, err.Error()) - } - - // Configuration - - var ( - accessRight *shares.AccessRight - volumeCtx map[string]string - stageSecret, publishSecret map[string]string - err error - ) - - shareOpts, err := options.NewNodeVolumeContext(req.GetVolumeContext()) - if err != nil { - return nil, status.Errorf(codes.InvalidArgument, "invalid volume context: %v", err) - } - - osOpts, err := options.NewOpenstackOptions(req.GetSecrets()) - if err != nil { - return nil, status.Errorf(codes.InvalidArgument, "invalid OpenStack secrets: %v", err) - } - - volID := volumeID(req.GetVolumeId()) - - ns.nodeStageCacheMtx.Lock() - if cacheEntry, ok := ns.nodeStageCache[volID]; ok { - volumeCtx, stageSecret = cacheEntry.volumeContext, cacheEntry.stageSecret - } else { - volumeCtx, accessRight, err = ns.buildVolumeContext(volID, shareOpts, osOpts) - - if err == nil { - stageSecret, err = buildNodeStageSecret(accessRight, getShareAdapter(ns.d.shareProto), volID) - } - - if err == nil { - publishSecret, err = buildNodePublishSecret(accessRight, getShareAdapter(ns.d.shareProto), volID) - } - - ns.nodeStageCache[volID] = stageCacheEntry{volumeContext: volumeCtx, stageSecret: stageSecret, publishSecret: publishSecret} - } - ns.nodeStageCacheMtx.Unlock() - - if err != nil { - return nil, err - } - - // Forward the RPC - - csiConn, err := ns.d.csiClientBuilder.NewConnectionWithContext(ctx, ns.d.fwdEndpoint) - if err != nil { - return nil, status.Error(codes.Unavailable, fmtGrpcConnError(ns.d.fwdEndpoint, err)) - } - - req.Secrets = stageSecret - req.VolumeContext = volumeCtx - - return ns.d.csiClientBuilder.NewNodeServiceClient(csiConn).StageVolume(ctx, req) -} - -func (ns *nodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) { - if err := validateNodeUnstageVolumeRequest(req); err != nil { - return nil, status.Error(codes.InvalidArgument, err.Error()) - } - - ns.nodeStageCacheMtx.Lock() - delete(ns.nodeStageCache, volumeID(req.VolumeId)) - ns.nodeStageCacheMtx.Unlock() - - csiConn, err := ns.d.csiClientBuilder.NewConnectionWithContext(ctx, ns.d.fwdEndpoint) - if err != nil { - return nil, status.Error(codes.Unavailable, fmtGrpcConnError(ns.d.fwdEndpoint, err)) - } - - return ns.d.csiClientBuilder.NewNodeServiceClient(csiConn).UnstageVolume(ctx, req) -} - -func (ns *nodeServer) NodeGetInfo(ctx context.Context, req *csi.NodeGetInfoRequest) (*csi.NodeGetInfoResponse, error) { - nodeInfo := &csi.NodeGetInfoResponse{ - NodeId: ns.d.nodeID, - } - - if ns.d.withTopology { - nodeInfo.AccessibleTopology = &csi.Topology{ - Segments: map[string]string{topologyKey: ns.d.nodeAZ}, - } - } - - return nodeInfo, nil -} - -func (ns *nodeServer) NodeGetCapabilities(ctx context.Context, req *csi.NodeGetCapabilitiesRequest) (*csi.NodeGetCapabilitiesResponse, error) { - return &csi.NodeGetCapabilitiesResponse{ - Capabilities: ns.d.nscaps, - }, nil -} - -func (ns *nodeServer) NodeGetVolumeStats(ctx context.Context, req *csi.NodeGetVolumeStatsRequest) (*csi.NodeGetVolumeStatsResponse, error) { - return nil, status.Error(codes.Unimplemented, "") -} - -func (ns *nodeServer) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandVolumeRequest) (*csi.NodeExpandVolumeResponse, error) { - return nil, status.Error(codes.Unimplemented, "") -} diff --git a/pkg/csi/manila/options/compatibilityoptions.go b/pkg/csi/manila/options/compatibilityoptions.go deleted file mode 100644 index 0ed4d6cfd2..0000000000 --- a/pkg/csi/manila/options/compatibilityoptions.go +++ /dev/null @@ -1,31 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import "k8s.io/cloud-provider-openstack/pkg/csi/manila/validator" - -type CompatibilityOptions struct { - CreateShareFromSnapshotEnabled string `name:"CreateShareFromSnapshotEnabled" value:"default:false" matches:"^true|false$"` - CreateShareFromSnapshotRetries string `name:"CreateShareFromSnapshotRetries" value:"default:10" matches:"^[0-9]+$"` - CreateShareFromSnapshotBackoffInterval string `name:"CreateShareFromSnapshotBackoffInterval" value:"default:5" matches:"^[0-9]+$"` -} - -var ( - compatOptionsValidator = validator.New(&CompatibilityOptions{}) -) - -func NewCompatibilityOptions(data map[string]string) (*CompatibilityOptions, error) { - opts := &CompatibilityOptions{} - return opts, compatOptionsValidator.Populate(data, opts) -} diff --git a/pkg/csi/manila/options/openstackoptions.go b/pkg/csi/manila/options/openstackoptions.go deleted file mode 100644 index a6f5e0a954..0000000000 --- a/pkg/csi/manila/options/openstackoptions.go +++ /dev/null @@ -1,35 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - cpo "k8s.io/cloud-provider-openstack/pkg/cloudprovider/providers/openstack" - "k8s.io/cloud-provider-openstack/pkg/csi/manila/validator" -) - -var ( - osOptionsValidator = validator.New(&cpo.AuthOpts{}) -) - -func NewOpenstackOptions(data map[string]string) (*cpo.AuthOpts, error) { - opts := &cpo.AuthOpts{} - if err := osOptionsValidator.Populate(data, opts); err != nil { - return nil, err - } - - return opts, nil -} diff --git a/pkg/csi/manila/options/shareoptions.go b/pkg/csi/manila/options/shareoptions.go deleted file mode 100644 index 806c869d2e..0000000000 --- a/pkg/csi/manila/options/shareoptions.go +++ /dev/null @@ -1,66 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "k8s.io/cloud-provider-openstack/pkg/csi/manila/validator" -) - -type ControllerVolumeContext struct { - Protocol string `name:"protocol" matches:"^(?i)CEPHFS|NFS$"` - Type string `name:"type" value:"default:default"` - ShareNetworkID string `name:"shareNetworkID" value:"optional"` - AvailabilityZone string `name:"availability" value:"optional"` - - // Adapter options - - CephfsMounter string `name:"cephfs-mounter" value:"default:fuse" matches:"^kernel|fuse$"` - NFSShareClient string `name:"nfs-shareClient" value:"default:0.0.0.0/0"` -} - -type NodeVolumeContext struct { - ShareID string `name:"shareID" value:"optionalIf:shareName=." precludes:"shareName"` - ShareName string `name:"shareName" value:"optionalIf:shareID=." precludes:"shareID"` - ShareAccessID string `name:"shareAccessID"` - - // Adapter options - - CephfsMounter string `name:"cephfs-mounter" value:"default:fuse" matches:"^kernel|fuse$"` -} - -var ( - controllerVolumeCtxValidator = validator.New(&ControllerVolumeContext{}) - nodeVolumeCtxValidator = validator.New(&NodeVolumeContext{}) -) - -func NewControllerVolumeContext(data map[string]string) (*ControllerVolumeContext, error) { - opts := &ControllerVolumeContext{} - if err := controllerVolumeCtxValidator.Populate(data, opts); err != nil { - return nil, err - } - - return opts, nil -} - -func NewNodeVolumeContext(data map[string]string) (*NodeVolumeContext, error) { - opts := &NodeVolumeContext{} - if err := nodeVolumeCtxValidator.Populate(data, opts); err != nil { - return nil, err - } - - return opts, nil -} diff --git a/pkg/csi/manila/runtimeconfig/nfsconfig.go b/pkg/csi/manila/runtimeconfig/nfsconfig.go deleted file mode 100644 index 7867acf977..0000000000 --- a/pkg/csi/manila/runtimeconfig/nfsconfig.go +++ /dev/null @@ -1,26 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package runtimeconfig - -type NfsConfig struct { - // When mounting an NFS share, select an export location with matching IP address. - // No match between this address and at least a single export location for this share - // will result in an error. - // Expects a CIDR-formatted address. If prefix is not provided, - // /32 or /128 prefix is assumed for IPv4 and IPv6 respectively. - MatchExportLocationAddress string `json:"matchExportLocationAddress,omitempty"` -} diff --git a/pkg/csi/manila/runtimeconfig/runtimeconfig.go b/pkg/csi/manila/runtimeconfig/runtimeconfig.go deleted file mode 100644 index d7032005bc..0000000000 --- a/pkg/csi/manila/runtimeconfig/runtimeconfig.go +++ /dev/null @@ -1,52 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package runtimeconfig - -import ( - "encoding/json" - "io/ioutil" - "os" -) - -var ( - // Path to the runtime config file - RuntimeConfigFilename string -) - -type RuntimeConfig struct { - Nfs *NfsConfig `json:"nfs,omitempty"` -} - -func Get() (*RuntimeConfig, error) { - // File contents are deliberately not cached - // as they may change over time. - data, err := ioutil.ReadFile(RuntimeConfigFilename) - if err != nil { - if os.IsNotExist(err) { - return nil, nil - } - - return nil, err - } - - var cfg *RuntimeConfig - if err = json.Unmarshal(data, &cfg); err != nil { - return nil, err - } - - return cfg, nil -} diff --git a/pkg/csi/manila/share.go b/pkg/csi/manila/share.go deleted file mode 100644 index 92dd6e6ab6..0000000000 --- a/pkg/csi/manila/share.go +++ /dev/null @@ -1,152 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package manila - -import ( - "fmt" - "time" - - "github.com/gophercloud/gophercloud/openstack/sharedfilesystems/v2/shares" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/cloud-provider-openstack/pkg/csi/manila/manilaclient" - clouderrors "k8s.io/cloud-provider-openstack/pkg/util/errors" - "k8s.io/klog/v2" -) - -const ( - waitForAvailableShareTimeout = 3 - waitForAvailableShareRetries = 10 - - shareCreating = "creating" - shareDeleting = "deleting" - shareError = "error" - shareErrorDeleting = "error_deleting" - shareAvailable = "available" - - shareDescription = "provisioned-by=manila.csi.openstack.org" -) - -// getOrCreateShare first retrieves an existing share with name=shareName, or creates a new one if it doesn't exist yet. -// Once the share is created, an exponential back-off is used to wait till the status of the share is "available". -func getOrCreateShare(shareName string, createOpts *shares.CreateOpts, manilaClient manilaclient.Interface) (*shares.Share, manilaError, error) { - var ( - share *shares.Share - err error - ) - - // First, check if the share already exists or needs to be created - - if share, err = manilaClient.GetShareByName(shareName); err != nil { - if clouderrors.IsNotFound(err) { - // It doesn't exist, create it - - var createErr error - if share, createErr = manilaClient.CreateShare(createOpts); createErr != nil { - return nil, 0, createErr - } - } else { - // Something else is wrong - return nil, 0, fmt.Errorf("failed to probe for a share named %s: %v", shareName, err) - } - } else { - klog.V(4).Infof("a share named %s already exists", shareName) - } - - // It exists, wait till it's Available - - if share.Status == shareAvailable { - return share, 0, nil - } - - return waitForShareStatus(share.ID, shareCreating, shareAvailable, false, manilaClient) -} - -func deleteShare(shareID string, manilaClient manilaclient.Interface) error { - if err := manilaClient.DeleteShare(shareID); err != nil { - if clouderrors.IsNotFound(err) { - klog.V(4).Infof("share %s not found, assuming it to be already deleted", shareID) - } else { - return err - } - } - - return nil -} - -func tryDeleteShare(share *shares.Share, manilaClient manilaclient.Interface) { - if share == nil { - return - } - - if err := manilaClient.DeleteShare(share.ID); err != nil { - // TODO failure to delete a share in an error state needs proper monitoring support - klog.Errorf("couldn't delete share %s in a roll-back procedure: %v", share.ID, err) - return - } - - _, _, err := waitForShareStatus(share.ID, shareDeleting, "", true, manilaClient) - if err != nil && err != wait.ErrWaitTimeout { - klog.Errorf("couldn't retrieve share %s in a roll-back procedure: %v", share.ID, err) - } -} - -func waitForShareStatus(shareID, currentStatus, desiredStatus string, successOnNotFound bool, manilaClient manilaclient.Interface) (*shares.Share, manilaError, error) { - var ( - backoff = wait.Backoff{ - Duration: time.Second * waitForAvailableShareTimeout, - Factor: 1.2, - Steps: waitForAvailableShareRetries, - } - - share *shares.Share - manilaErrCode manilaError - err error - ) - - return share, manilaErrCode, wait.ExponentialBackoff(backoff, func() (bool, error) { - share, err = manilaClient.GetShareByID(shareID) - - if err != nil { - if clouderrors.IsNotFound(err) && successOnNotFound { - return true, nil - } - - return false, err - } - - var isAvailable bool - - switch share.Status { - case currentStatus: - isAvailable = false - case desiredStatus: - isAvailable = true - case shareError: - manilaErrMsg, err := lastResourceError(shareID, manilaClient) - if err != nil { - return false, fmt.Errorf("share %s is in error state, error description could not be retrieved: %v", shareID, err) - } - - manilaErrCode = manilaErrMsg.errCode - return false, fmt.Errorf("share %s is in error state: %s", shareID, manilaErrMsg.message) - default: - return false, fmt.Errorf("share %s is in an unexpected state: wanted either %s or %s, got %s", shareID, currentStatus, desiredStatus, share.Status) - } - - return isAvailable, nil - }) -} diff --git a/pkg/csi/manila/shareadapters/cephfs.go b/pkg/csi/manila/shareadapters/cephfs.go deleted file mode 100644 index f1a2c5ee71..0000000000 --- a/pkg/csi/manila/shareadapters/cephfs.go +++ /dev/null @@ -1,132 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package shareadapters - -import ( - "fmt" - "time" - - "github.com/gophercloud/gophercloud" - "github.com/gophercloud/gophercloud/openstack/sharedfilesystems/v2/shares" - "k8s.io/apimachinery/pkg/util/wait" - manilautil "k8s.io/cloud-provider-openstack/pkg/csi/manila/util" - "k8s.io/klog/v2" -) - -type Cephfs struct{} - -var _ ShareAdapter = &Cephfs{} - -func (Cephfs) GetOrGrantAccess(args *GrantAccessArgs) (accessRight *shares.AccessRight, err error) { - // First, check if the access right exists or needs to be created - - var rights []shares.AccessRight - - rights, err = args.ManilaClient.GetAccessRights(args.Share.ID) - if err != nil { - if _, ok := err.(gophercloud.ErrResourceNotFound); !ok { - return nil, fmt.Errorf("failed to list access rights: %v", err) - } - } else { - // Try to find the access right - - for _, r := range rights { - if r.AccessTo == args.Share.Name && r.AccessType == "cephx" && r.AccessLevel == "rw" { - klog.V(4).Infof("cephx access right for share %s already exists", args.Share.Name) - - accessRight = &r - break - } - } - } - - if accessRight == nil { - // Not found, create it - - accessRight, err = args.ManilaClient.GrantAccess(args.Share.ID, shares.GrantAccessOpts{ - AccessType: "cephx", - AccessLevel: "rw", - AccessTo: args.Share.Name, - }) - - if err != nil { - return - } - } - - if accessRight.AccessKey != "" { - // The access right is ready - return - } - - // Wait till a ceph key is assigned to the access right - - backoff := wait.Backoff{ - Duration: time.Second * 5, - Factor: 1.2, - Steps: 10, - } - - return accessRight, wait.ExponentialBackoff(backoff, func() (bool, error) { - rights, err := args.ManilaClient.GetAccessRights(args.Share.ID) - if err != nil { - return false, err - } - - var accessRight *shares.AccessRight - - for i := range rights { - if rights[i].AccessTo == args.Share.Name { - accessRight = &rights[i] - break - } - } - - if accessRight == nil { - return false, fmt.Errorf("cannot find the access right we've just created") - } - - return accessRight.AccessKey != "", nil - }) -} - -func (Cephfs) BuildVolumeContext(args *VolumeContextArgs) (volumeContext map[string]string, err error) { - chosenExportLocationIdx, err := manilautil.FindExportLocation(args.Locations, manilautil.AnyExportLocation) - if err != nil { - return nil, fmt.Errorf("failed to choose an export location: %v", err) - } - - monitors, rootPath, err := splitExportLocationPath(args.Locations[chosenExportLocationIdx].Path) - - return map[string]string{ - "monitors": monitors, - "rootPath": rootPath, - "mounter": args.Options.CephfsMounter, - "provisionVolume": "false", - }, err -} - -func (Cephfs) BuildNodeStageSecret(args *SecretArgs) (secret map[string]string, err error) { - return map[string]string{ - "userID": args.AccessRight.AccessTo, - "userKey": args.AccessRight.AccessKey, - }, nil -} - -func (Cephfs) BuildNodePublishSecret(args *SecretArgs) (secret map[string]string, err error) { - return nil, nil -} diff --git a/pkg/csi/manila/shareadapters/nfs.go b/pkg/csi/manila/shareadapters/nfs.go deleted file mode 100644 index 63f771a6a9..0000000000 --- a/pkg/csi/manila/shareadapters/nfs.go +++ /dev/null @@ -1,165 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package shareadapters - -import ( - "fmt" - "net" - "strings" - - "github.com/gophercloud/gophercloud" - "github.com/gophercloud/gophercloud/openstack/sharedfilesystems/v2/shares" - "k8s.io/cloud-provider-openstack/pkg/csi/manila/runtimeconfig" - manilautil "k8s.io/cloud-provider-openstack/pkg/csi/manila/util" - "k8s.io/klog/v2" -) - -type NFS struct{} - -var _ ShareAdapter = &NFS{} - -func (NFS) GetOrGrantAccess(args *GrantAccessArgs) (*shares.AccessRight, error) { - // First, check if the access right exists or needs to be created - - rights, err := args.ManilaClient.GetAccessRights(args.Share.ID) - if err != nil { - if _, ok := err.(gophercloud.ErrResourceNotFound); !ok { - return nil, fmt.Errorf("failed to list access rights: %v", err) - } - } - - // Try to find the access right - - for _, r := range rights { - if r.AccessTo == args.Options.NFSShareClient && r.AccessType == "ip" && r.AccessLevel == "rw" { - klog.V(4).Infof("IP access right for share %s already exists", args.Share.Name) - return &r, nil - } - } - - // Not found, create it - - return args.ManilaClient.GrantAccess(args.Share.ID, shares.GrantAccessOpts{ - AccessType: "ip", - AccessLevel: "rw", - AccessTo: args.Options.NFSShareClient, - }) -} - -func (NFS) BuildVolumeContext(args *VolumeContextArgs) (volumeContext map[string]string, err error) { - chosenExportLocationIdx, err := nfsChooseExportLocation(args.Locations) - if err != nil { - return nil, fmt.Errorf("failed to choose an export location: %v", err) - } - - server, share, err := splitExportLocationPath(args.Locations[chosenExportLocationIdx].Path) - - return map[string]string{ - "server": server, - "share": share, - }, err -} - -func (NFS) BuildNodeStageSecret(args *SecretArgs) (secret map[string]string, err error) { - return nil, nil -} - -func (NFS) BuildNodePublishSecret(args *SecretArgs) (secret map[string]string, err error) { - return nil, nil -} - -// Tries to choose a suitable export location from the given list. -// Returns index into `locs`. -// Runtime config for NFS is probed first to see if it contains any export location filters. -// Those are then used for selecting the location. If none are defined, the function -// falls back to using manilautil.AnyExportLocation filter. -func nfsChooseExportLocation(locs []shares.ExportLocation) (chosenExportLocationIdx int, err error) { - var conf *runtimeconfig.RuntimeConfig - - if conf, err = runtimeconfig.Get(); err != nil { - return -1, fmt.Errorf("failed to read runtime config file %s: %v", runtimeconfig.RuntimeConfigFilename, err) - } - - if conf != nil { - if chosenExportLocationIdx, err = nfsMatchExportLocationFromConfig(locs, conf); err != nil { - return -1, err - } - - if chosenExportLocationIdx != -1 { - return chosenExportLocationIdx, err - } - - // If we got here, it either means there's no NFS config, - // or it doesn't contain any configuration for export locations. - // Fall through and choose any suitable location. - } - - return manilautil.FindExportLocation(locs, manilautil.AnyExportLocation) -} - -func nfsMatchExportLocationFromConfig(locs []shares.ExportLocation, conf *runtimeconfig.RuntimeConfig) (idx int, err error) { - if conf.Nfs != nil { - if conf.Nfs.MatchExportLocationAddress != "" { - return nfsMatchExportLocationAddress(locs, conf.Nfs.MatchExportLocationAddress) - } - } - - // No NFS export location filters specified - return -1, nil -} - -// Selects an export location with a matching address -func nfsMatchExportLocationAddress(locs []shares.ExportLocation, matchAddress string) (idx int, err error) { - if ip := net.ParseIP(matchAddress); ip != nil { - // `matchAddress` is a valid IP, but does not have a prefix. - // This means we're looking for an exact match in export location addresses. - - // Heuristic to check whether this is an IPv4 or IPv6 address - if strings.Contains(matchAddress, ".") { - // IPv4 - matchAddress += "/32" - } else { - // IPv6 - matchAddress += "/128" - } - } - - _, netIP, err := net.ParseCIDR(matchAddress) - if err != nil { - return -1, fmt.Errorf("matchExportLocationAddress filter '%s' is not a CIDR-formatted IP address", matchAddress) - } - - idx, err = manilautil.FindExportLocation(locs, func(i int) (bool, error) { - addr, _, err := splitExportLocationPath(locs[i].Path) - if err != nil { - return false, err - } - - hostIP := net.ParseIP(addr) - if hostIP == nil { - return false, fmt.Errorf("IP '%s' in export location path %s is invalid", addr, locs[i].Path) - } - - return netIP.Contains(hostIP), nil - }) - - if err != nil { - return -1, fmt.Errorf("matchExportLocationAddress filter '%s': %v", matchAddress, err) - } - - return idx, nil -} diff --git a/pkg/csi/manila/shareadapters/shareadapter.go b/pkg/csi/manila/shareadapters/shareadapter.go deleted file mode 100644 index f6cf115374..0000000000 --- a/pkg/csi/manila/shareadapters/shareadapter.go +++ /dev/null @@ -1,57 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package shareadapters - -import ( - "github.com/gophercloud/gophercloud/openstack/sharedfilesystems/v2/shares" - "k8s.io/cloud-provider-openstack/pkg/csi/manila/manilaclient" - "k8s.io/cloud-provider-openstack/pkg/csi/manila/options" -) - -type GrantAccessArgs struct { - ManilaClient manilaclient.Interface - Share *shares.Share - Options *options.ControllerVolumeContext -} - -type VolumeContextArgs struct { - // Share adapters are responsible for choosing - // an export location when building a volume context. - Locations []shares.ExportLocation - - Options *options.NodeVolumeContext -} - -type SecretArgs struct { - AccessRight *shares.AccessRight -} - -type ShareAdapter interface { - // GetOrGrantAccess first tries to retrieve an access right for args.Share. - // An access right is created for the share in case it doesn't exist yet. - // Returns an existing or new access right for args.Share. - GetOrGrantAccess(args *GrantAccessArgs) (accessRight *shares.AccessRight, err error) - - // BuildVolumeContext builds a volume context map that's passed to NodeStageVolumeRequest and NodePublishVolumeRequest - BuildVolumeContext(args *VolumeContextArgs) (volumeContext map[string]string, err error) - - // Builds secret map for NodeStageVolumeRequest - BuildNodeStageSecret(args *SecretArgs) (secret map[string]string, err error) - - // Builds secret map for NodePublishVolumeRequest - BuildNodePublishSecret(args *SecretArgs) (secret map[string]string, err error) -} diff --git a/pkg/csi/manila/shareadapters/util.go b/pkg/csi/manila/shareadapters/util.go deleted file mode 100644 index 8dba5d0b93..0000000000 --- a/pkg/csi/manila/shareadapters/util.go +++ /dev/null @@ -1,35 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package shareadapters - -import ( - "fmt" - "strings" -) - -func splitExportLocationPath(exportLocationPath string) (address, location string, err error) { - delimPos := strings.LastIndexByte(exportLocationPath, ':') - if delimPos <= 0 { - err = fmt.Errorf("failed to parse address and location from export location '%s'", exportLocationPath) - return - } - - address = exportLocationPath[:delimPos] - location = exportLocationPath[delimPos+1:] - - return -} diff --git a/pkg/csi/manila/snapshot.go b/pkg/csi/manila/snapshot.go deleted file mode 100644 index f7270217b6..0000000000 --- a/pkg/csi/manila/snapshot.go +++ /dev/null @@ -1,150 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package manila - -import ( - "fmt" - "time" - - "github.com/gophercloud/gophercloud/openstack/sharedfilesystems/v2/snapshots" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/cloud-provider-openstack/pkg/csi/manila/manilaclient" - clouderrors "k8s.io/cloud-provider-openstack/pkg/util/errors" - "k8s.io/klog/v2" -) - -const ( - snapshotCreating = "creating" - snapshotDeleting = "deleting" - snapshotError = "error" - snapshotErrorDeleting = "error_deleting" - snapshotAvailable = "available" - - snapshotDescription = "snapshotted-by=manila.csi.openstack.org" -) - -// getOrCreateSnapshot retrieves an existing snapshot with name=snapName, or creates a new one if it doesn't exist yet. -// Instead of waiting for the snapshot to become available (as getOrCreateShare does), CSI's ready_to_use flag is used to signal readiness -func getOrCreateSnapshot(snapName, sourceShareID string, manilaClient manilaclient.Interface) (*snapshots.Snapshot, error) { - var ( - snapshot *snapshots.Snapshot - err error - ) - - // First, check if the snapshot already exists or needs to be created - - if snapshot, err = manilaClient.GetSnapshotByName(snapName); err != nil { - if clouderrors.IsNotFound(err) { - // It doesn't exist, create it - - opts := snapshots.CreateOpts{ - ShareID: sourceShareID, - Name: snapName, - Description: snapshotDescription, - } - - var createErr error - if snapshot, createErr = manilaClient.CreateSnapshot(opts); createErr != nil { - return nil, createErr - } - - } else { - // Something else is wrong - return nil, fmt.Errorf("failed to probe for a snapshot named %s: %v", snapName, err) - } - } else { - klog.V(4).Infof("a snapshot named %s already exists", snapName) - } - - return snapshot, nil -} - -func deleteSnapshot(snapID string, manilaClient manilaclient.Interface) error { - if err := manilaClient.DeleteSnapshot(snapID); err != nil { - if clouderrors.IsNotFound(err) { - klog.V(4).Infof("snapshot %s not found, assuming it to be already deleted", snapID) - } else { - return err - } - } - - return nil -} - -func tryDeleteSnapshot(snapshot *snapshots.Snapshot, manilaClient manilaclient.Interface) { - if snapshot == nil { - return - } - - if err := deleteSnapshot(snapshot.ID, manilaClient); err != nil { - // TODO failure to delete a snapshot in an error state needs proper monitoring support - klog.Errorf("couldn't delete snapshot %s in a roll-back procedure: %v", snapshot.ID, err) - return - } - - _, _, err := waitForSnapshotStatus(snapshot.ID, snapshotDeleting, "", true, manilaClient) - if err != nil && err != wait.ErrWaitTimeout { - klog.Errorf("couldn't retrieve snapshot %s in a roll-back procedure: %v", snapshot.ID, err) - } -} - -func waitForSnapshotStatus(snapshotID, currentStatus, desiredStatus string, successOnNotFound bool, manilaClient manilaclient.Interface) (*snapshots.Snapshot, manilaError, error) { - var ( - backoff = wait.Backoff{ - Duration: time.Second * waitForAvailableShareTimeout, - Factor: 1.2, - Steps: waitForAvailableShareRetries, - } - - snapshot *snapshots.Snapshot - manilaErrCode manilaError - err error - ) - - return snapshot, manilaErrCode, wait.ExponentialBackoff(backoff, func() (bool, error) { - snapshot, err = manilaClient.GetSnapshotByID(snapshotID) - - if err != nil { - if clouderrors.IsNotFound(err) && successOnNotFound { - return true, nil - } - - return false, err - } - - var isAvailable bool - - switch snapshot.Status { - case currentStatus: - isAvailable = false - case desiredStatus: - isAvailable = true - case shareError: - manilaErrMsg, err := lastResourceError(snapshotID, manilaClient) - if err != nil { - return false, fmt.Errorf("snapshot %s is in error state, error description could not be retrieved: %v", snapshotID, err) - } - - manilaErrCode = manilaErrMsg.errCode - return false, fmt.Errorf("snapshot %s is in error state: %s", snapshotID, manilaErrMsg.message) - default: - return false, fmt.Errorf("snapshot %s is in an unexpected state: wanted either %s or %s, got %s", snapshotID, currentStatus, desiredStatus, snapshot.Status) - } - - return isAvailable, nil - }) -} diff --git a/pkg/csi/manila/util.go b/pkg/csi/manila/util.go deleted file mode 100644 index 642d49e54e..0000000000 --- a/pkg/csi/manila/util.go +++ /dev/null @@ -1,372 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package manila - -import ( - "errors" - "fmt" - "strconv" - "strings" - - "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/gophercloud/gophercloud/openstack/sharedfilesystems/v2/messages" - "github.com/gophercloud/gophercloud/openstack/sharedfilesystems/v2/shares" - "github.com/gophercloud/gophercloud/openstack/sharedfilesystems/v2/snapshots" - "google.golang.org/grpc/codes" - "k8s.io/cloud-provider-openstack/pkg/csi/manila/capabilities" - "k8s.io/cloud-provider-openstack/pkg/csi/manila/compatibility" - "k8s.io/cloud-provider-openstack/pkg/csi/manila/manilaclient" - "k8s.io/cloud-provider-openstack/pkg/csi/manila/options" - "k8s.io/klog/v2" -) - -type ( - volumeID string - snapshotID string -) - -const ( - bytesInGiB = 1024 * 1024 * 1024 -) - -type manilaError int - -func (c manilaError) toRpcErrorCode() codes.Code { - switch c { - case manilaErrNoValidHost: - return codes.OutOfRange - case manilaErrUnexpectedNetwork: - return codes.InvalidArgument - case manilaErrAvailability: - return codes.ResourceExhausted - case manilaErrCapabilities: - return codes.InvalidArgument - case manilaErrCapacity: - return codes.OutOfRange - default: - return codes.Internal - } -} - -const ( - manilaErrNoValidHost manilaError = iota + 1 - manilaErrUnexpectedNetwork - manilaErrAvailability - manilaErrCapabilities - manilaErrCapacity -) - -var ( - manilaErrorCodesMap = map[string]manilaError{ - "002": manilaErrNoValidHost, - "003": manilaErrUnexpectedNetwork, - "007": manilaErrAvailability, - "008": manilaErrCapabilities, - "009": manilaErrCapacity, - } -) - -type manilaErrorMessage struct { - errCode manilaError - message string -} - -func parseGRPCEndpoint(endpoint string) (proto, addr string, err error) { - const ( - unixScheme = "unix://" - tcpScheme = "tcp://" - ) - - if strings.HasPrefix(endpoint, "/") { - return "unix", endpoint, nil - } - - if strings.HasPrefix(endpoint, unixScheme) { - pos := len(unixScheme) - if endpoint[pos] != '/' { - // endpoint seems to be "unix://absolute/path/to/somewhere" - // we're missing one '/'...compensate by decrementing pos - pos-- - } - - return "unix", endpoint[pos:], nil - } - - if strings.HasPrefix(endpoint, tcpScheme) { - return "tcp", endpoint[len(tcpScheme):], nil - } - - return "", "", errors.New("endpoint uses unsupported scheme") -} - -func endpointAddress(proto, addr string) string { - return fmt.Sprintf("%s://%s", proto, addr) -} - -func fmtGrpcConnError(fwdEndpoint string, err error) string { - return fmt.Sprintf("connecting to fwd plugin at %s failed: %v", fwdEndpoint, err) -} - -func bytesToGiB(sizeInBytes int64) int { - sizeInGiB := int(sizeInBytes / bytesInGiB) - - if int64(sizeInGiB)*bytesInGiB < sizeInBytes { - // Round up - return sizeInGiB + 1 - } - - return sizeInGiB -} - -func lastResourceError(resourceID string, manilaClient manilaclient.Interface) (manilaErrorMessage, error) { - msgs, err := manilaClient.GetUserMessages(&messages.ListOpts{ - ResourceID: resourceID, - MessageLevel: "ERROR", - Limit: 1, - SortDir: "desc", - SortKey: "created_at", - }) - - if err != nil { - return manilaErrorMessage{}, err - } - - if msgs != nil && len(msgs) == 1 { - return manilaErrorMessage{errCode: manilaErrorCodesMap[msgs[0].DetailID], message: msgs[0].UserMessage}, nil - } - - return manilaErrorMessage{message: "unknown error"}, nil -} - -func compareProtocol(protoA, protoB string) bool { - return strings.ToUpper(protoA) == strings.ToUpper(protoB) -} - -func tryCompatForVolumeSource(compatOpts *options.CompatibilityOptions, shareOpts *options.ControllerVolumeContext, source *csi.VolumeContentSource, shareTypeCaps capabilities.ManilaCapabilities) compatibility.Layer { - if source != nil { - if source.GetSnapshot() != nil && shareTypeCaps[capabilities.ManilaCapabilitySnapshot] { - if createShareFromSnapshotEnabled, _ := strconv.ParseBool(compatOpts.CreateShareFromSnapshotEnabled); createShareFromSnapshotEnabled { - return compatibility.FindCompatibilityLayer(shareOpts.Protocol, capabilities.ManilaCapabilityShareFromSnapshot, shareTypeCaps) - } - } - } - - return nil -} - -// -// Controller service request validation -// - -func validateCreateVolumeRequest(req *csi.CreateVolumeRequest) error { - if req.GetName() == "" { - return errors.New("volume name cannot be empty") - } - - reqCaps := req.GetVolumeCapabilities() - if reqCaps == nil { - return errors.New("volume capabilities cannot be empty") - } - - for _, cap := range reqCaps { - if cap.GetBlock() != nil { - return errors.New("block access type not allowed") - } - } - - if req.GetSecrets() == nil || len(req.GetSecrets()) == 0 { - return errors.New("secrets cannot be nil or empty") - } - - return nil -} - -func validateDeleteVolumeRequest(req *csi.DeleteVolumeRequest) error { - if req.GetVolumeId() == "" { - return errors.New("volume ID cannot be empty") - } - - if req.GetSecrets() == nil || len(req.GetSecrets()) == 0 { - return errors.New("secrets cannot be nil or empty") - } - - return nil -} - -func validateCreateSnapshotRequest(req *csi.CreateSnapshotRequest) error { - if req.GetName() == "" { - return errors.New("snapshot name cannot be empty") - } - - if req.GetSourceVolumeId() == "" { - return errors.New("source volume ID cannot be empty") - } - - if req.GetSecrets() == nil || len(req.GetSecrets()) == 0 { - return errors.New("secrets cannot be nil or empty") - } - - if req.GetParameters() != nil { - klog.Info("parameters in CreateSnapshot requests are ignored") - } - - return nil -} - -func validateDeleteSnapshotRequest(req *csi.DeleteSnapshotRequest) error { - if req.GetSnapshotId() == "" { - return errors.New("snapshot ID cannot be empty") - } - - if req.GetSecrets() == nil || len(req.GetSecrets()) == 0 { - return errors.New("secrets cannot be nil or empty") - } - - return nil -} - -func coalesceValue(v string) string { - if v == "" { - return "" - } - - return v -} - -func verifyVolumeCompatibility(sizeInGiB int, req *csi.CreateVolumeRequest, share *shares.Share, shareOpts *options.ControllerVolumeContext, compatOpts *options.CompatibilityOptions, shareTypeCaps capabilities.ManilaCapabilities) error { - if share.Size != sizeInGiB { - return fmt.Errorf("size mismatch: wanted %d, got %d", share.Size, sizeInGiB) - } - - if share.ShareProto != shareOpts.Protocol { - return fmt.Errorf("share protocol mismatch: wanted %s, got %s", coalesceValue(share.ShareProto), coalesceValue(shareOpts.Protocol)) - } - - // FIXME shareOpts.Type may be either type name or type ID - /* - if share.ShareType != shareOpts.Type { - return fmt.Errorf("share type mismatch: wanted %s, got %s", shareOpts.Type, share.ShareType) - } - */ - - if share.ShareNetworkID != shareOpts.ShareNetworkID { - return fmt.Errorf("share network ID mismatch: wanted %s, got %s", coalesceValue(share.ShareNetworkID), coalesceValue(shareOpts.ShareNetworkID)) - } - - var reqSrcSnapID string - if req.GetVolumeContentSource() != nil && req.GetVolumeContentSource().GetSnapshot() != nil { - reqSrcSnapID = req.GetVolumeContentSource().GetSnapshot().GetSnapshotId() - } - - if share.SnapshotID != reqSrcSnapID { - return fmt.Errorf("source snapshot ID mismatch: wanted %s, got %s", coalesceValue(share.SnapshotID), coalesceValue(reqSrcSnapID)) - } - - return nil -} - -func verifySnapshotCompatibility(snapshot *snapshots.Snapshot, req *csi.CreateSnapshotRequest) error { - if snapshot.ShareID != req.GetSourceVolumeId() { - return fmt.Errorf("source share ID mismatch: wanted %s, got %s", snapshot.ID, req.GetSourceVolumeId()) - } - - return nil -} - -func validateValidateVolumeCapabilitiesRequest(req *csi.ValidateVolumeCapabilitiesRequest) error { - if req.GetVolumeId() == "" { - return errors.New("volume ID missing in request") - } - - if req.GetVolumeCapabilities() == nil || len(req.GetVolumeCapabilities()) == 0 { - return errors.New("volume capabilities cannot be nil or empty") - } - - if req.GetSecrets() == nil || len(req.GetSecrets()) == 0 { - return errors.New("stage secrets cannot be nil or empty") - } - - return nil -} - -// -// Node service request validation -// - -func validateNodeStageVolumeRequest(req *csi.NodeStageVolumeRequest) error { - if req.GetVolumeCapability() == nil { - return errors.New("volume capability missing in request") - } - - if req.GetVolumeId() == "" { - return errors.New("volume ID missing in request") - } - - if req.GetVolumeContext() == nil || len(req.GetVolumeContext()) == 0 { - return errors.New("volume context cannot be nil or empty") - } - - if req.GetSecrets() == nil || len(req.GetSecrets()) == 0 { - return errors.New("stage secrets cannot be nil or empty") - } - - return nil -} - -func validateNodeUnstageVolumeRequest(req *csi.NodeUnstageVolumeRequest) error { - if req.GetStagingTargetPath() == "" { - return errors.New("staging path missing in request") - } - - if req.GetVolumeId() == "" { - return errors.New("volume ID missing in request") - } - - return nil -} - -func validateNodePublishVolumeRequest(req *csi.NodePublishVolumeRequest) error { - if req.GetVolumeCapability() == nil { - return errors.New("volume capability missing in request") - } - - if req.GetVolumeId() == "" { - return errors.New("volume ID missing in request") - } - - if req.GetVolumeContext() == nil || len(req.GetSecrets()) == 0 { - return errors.New("volume context cannot be nil or empty") - } - - if req.GetSecrets() == nil || len(req.GetSecrets()) == 0 { - return errors.New("node publish secrets cannot be nil or empty") - } - - return nil -} - -func validateNodeUnpublishVolumeRequest(req *csi.NodeUnpublishVolumeRequest) error { - if req.GetTargetPath() == "" { - return errors.New("target path missing in request") - } - - if req.GetVolumeId() == "" { - return errors.New("volume ID missing in request") - } - - return nil -} diff --git a/pkg/csi/manila/util/exportlocation.go b/pkg/csi/manila/util/exportlocation.go deleted file mode 100644 index 902210e8af..0000000000 --- a/pkg/csi/manila/util/exportlocation.go +++ /dev/null @@ -1,68 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "errors" - "strings" - - "github.com/gophercloud/gophercloud/openstack/sharedfilesystems/v2/shares" -) - -// Predicate type for filtering out export locations. -// It's supplied with the index into the export location slice that's being scanned. -// Should return (true, nil) when search conditions are satisfied, -// and (false, nil) when they are not. -type ExportLocationPredicate func(index int) (match bool, err error) - -// Predicate matching any export location -func AnyExportLocation(int) (bool, error) { return true, nil } - -// Searches for an export location. -// Returns index of an export location from the `locs` slice that satisfies following rules: -// 1. Location is not admin-only and is not empty -// 2. Location satisfies ExportLocationPredicate -// Search is biased: -// 1. Location.Preferred == true is preferred over Location.Preferred == false -// 2. Locations with lower index are preferred over those with higher index -func FindExportLocation(locs []shares.ExportLocation, pred ExportLocationPredicate) (index int, err error) { - const invalidIdx = -1 - firstMatchNotPreferred := invalidIdx - - for i := range locs { - if locs[i].IsAdminOnly || strings.TrimSpace(locs[i].Path) == "" { - continue - } - - if hasMatch, err := pred(i); err != nil { - return i, err - } else if hasMatch { - if locs[i].Preferred { - // This export location is a match and it is Preferred. - // Can't get better than that! - return i, nil - } - - if firstMatchNotPreferred == invalidIdx { - firstMatchNotPreferred = i - } - } - } - - if firstMatchNotPreferred == invalidIdx { - err = errors.New("no match, or no suitable non-admin export locations available") - } - - return firstMatchNotPreferred, err -} diff --git a/pkg/csi/manila/util/exportlocation_test.go b/pkg/csi/manila/util/exportlocation_test.go deleted file mode 100644 index 0f383d0f5b..0000000000 --- a/pkg/csi/manila/util/exportlocation_test.go +++ /dev/null @@ -1,127 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "testing" - - "github.com/gophercloud/gophercloud/openstack/sharedfilesystems/v2/shares" -) - -// Tests FindExportLocation with AnyExportLocation predicate -func TestFindExportLocationAny(t *testing.T) { - ts := []struct { - locs []shares.ExportLocation - expectedMatchIdx int - }{ - { - locs: []shares.ExportLocation{ - { - Path: "loc-0", - IsAdminOnly: true, - Preferred: true, - }, - { - Path: "loc-1", - IsAdminOnly: true, - Preferred: false, - }, - }, - // Expected result `-1` because all locs are admin-only - expectedMatchIdx: -1, - }, - { - locs: []shares.ExportLocation{ - { - Path: "loc-0", - IsAdminOnly: true, - Preferred: true, - }, - { - Path: "loc-1", - IsAdminOnly: false, - Preferred: false, - }, - { - Path: "loc-2", - IsAdminOnly: true, - Preferred: true, - }, - }, - // Expected result `1` because locs[1] is the only non-admin location - expectedMatchIdx: 1, - }, - { - locs: []shares.ExportLocation{ - { - Path: "loc-0", - IsAdminOnly: true, - Preferred: true, - }, - { - Path: "loc-1", - IsAdminOnly: false, - Preferred: false, - }, - { - Path: "loc-2", - IsAdminOnly: false, - Preferred: false, - }, - }, - // Expected result `1` because locs[1] is the first non-admin location - expectedMatchIdx: 1, - }, - { - locs: []shares.ExportLocation{ - { - Path: "loc-0", - IsAdminOnly: true, - Preferred: true, - }, - { - Path: "loc-1", - IsAdminOnly: false, - Preferred: false, - }, - { - Path: "loc-2", - IsAdminOnly: false, - Preferred: true, - }, - { - Path: "loc-3", - IsAdminOnly: false, - Preferred: true, - }, - }, - // Expected result `2` because locs[2] is both non-admin and Preferred, - // and goes before locs[3] which is also non-admin and Preferred, - // but its index is higher. - expectedMatchIdx: 2, - }, - } - - for i := range ts { - result, err := FindExportLocation(ts[i].locs, AnyExportLocation) - - if err != nil && result != -1 { - t.Errorf("test %d: unexpected error: %v", i, err) - } - - if result != ts[i].expectedMatchIdx { - t.Errorf("test %d: returned an incorrect index: got %d, expected %d", i, result, ts[i].expectedMatchIdx) - } - } -} diff --git a/pkg/csi/manila/validator/dependsexpr.go b/pkg/csi/manila/validator/dependsexpr.go deleted file mode 100644 index d12823cf05..0000000000 --- a/pkg/csi/manila/validator/dependsexpr.go +++ /dev/null @@ -1,110 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package validator - -import ( - "fmt" - "reflect" - "strings" -) - -const ( - dependsTag = "dependsOn" - - depsGroupDelim = "," - depsExclGroupDelim = "|" -) - -type ( - exclusiveDependsExpression []fieldIndex - dependsExpression []exclusiveDependsExpression - dependenciesMap map[fieldIndex]dependsExpression -) - -func parseDepsExpr(deps string, nameIdxMap nameIndexMap, s structName, f fieldName) dependsExpression { - groups := strings.Split(deps, depsGroupDelim) - depsExpr := make(dependsExpression, len(groups)) - - for i := range groups { - exclGroup := strings.Split(groups[i], depsExclGroupDelim) - depsExpr[i] = make(exclusiveDependsExpression, len(exclGroup)) - - for j := range depsExpr[i] { - fIdx, ok := nameIdxMap[fieldName(exclGroup[j])] - if !ok { - panic(fmt.Sprintf("invalid dependsOn expression in %s.%s: no field named '%s' found", - s, f, fieldName(exclGroup[j]))) - } - - depsExpr[i][j] = fIdx - } - } - - return depsExpr -} - -func buildDependenciesMap(t reflect.Type, idxNameMap indexNameMap, nameIdxMap nameIndexMap) dependenciesMap { - m := make(dependenciesMap) - - for i := 0; i < t.NumField(); i++ { - ft := t.Field(i) - - deps, ok := ft.Tag.Lookup(dependsTag) - if !ok { - continue - } - - fIdx := fieldIndex(i) - m[fIdx] = parseDepsExpr(deps, nameIdxMap, structName(t.Name()), idxNameMap[fIdx]) - } - - return m -} - -func dependenciesUnsatisfiedError(i fieldIndex, exclDeps exclusiveDependsExpression, idxNameMap indexNameMap) error { - if len(exclDeps) == 1 { - return fmt.Errorf("parameter '%s' requires '%s'", idxNameMap[i], idxNameMap[exclDeps[0]]) - } - - names := make([]fieldName, len(exclDeps)) - for i := range names { - names[i] = idxNameMap[exclDeps[i]] - } - - return fmt.Errorf("parameter '%s' requires exactly one of %v parameters", idxNameMap[i], names) -} - -func dependenciesSatisfied(fIdx fieldIndex, v reflect.Value, deps dependsExpression, idxNameMap indexNameMap) error { - for i := range deps { - var values int - - for j := range deps[i] { - if v.Field(int(deps[i][j])).String() != "" { - values++ - if values > 1 { - break - } - } - } - - if values != 1 { - return dependenciesUnsatisfiedError(fIdx, deps[i], idxNameMap) - } - } - - return nil -} diff --git a/pkg/csi/manila/validator/identity.go b/pkg/csi/manila/validator/identity.go deleted file mode 100644 index 34d10eebfa..0000000000 --- a/pkg/csi/manila/validator/identity.go +++ /dev/null @@ -1,55 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package validator - -import ( - "fmt" - "reflect" -) - -const ( - nameFieldTag = "name" -) - -type ( - fieldIndex int - fieldName string - structName string - - nameIndexMap map[fieldName]fieldIndex - indexNameMap map[fieldIndex]fieldName -) - -func buildNameIdxMap(t reflect.Type) (nameIndexMap, indexNameMap) { - nameIdxMap := make(nameIndexMap) - idxNameMap := make(indexNameMap) - - for i := 0; i < t.NumField(); i++ { - ft := t.Field(i) - tags := ft.Tag - - name, ok := tags.Lookup(nameFieldTag) - if !ok { - panic(fmt.Sprintf("missing name tag for field %s in struct %s", ft.Name, t.Name())) - } - - nameIdxMap[fieldName(name)] = fieldIndex(i) - idxNameMap[fieldIndex(i)] = fieldName(name) - } - - return nameIdxMap, idxNameMap -} diff --git a/pkg/csi/manila/validator/matchexpr.go b/pkg/csi/manila/validator/matchexpr.go deleted file mode 100644 index e81ba0690c..0000000000 --- a/pkg/csi/manila/validator/matchexpr.go +++ /dev/null @@ -1,51 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package validator - -import ( - "fmt" - "reflect" - "regexp" -) - -const ( - matchTag = "matches" -) - -type ( - matchRegexpMap map[fieldIndex]*regexp.Regexp -) - -func buildMatchRegexMap(t reflect.Type) matchRegexpMap { - m := make(matchRegexpMap) - - for i := 0; i < t.NumField(); i++ { - if pattern, ok := t.Field(i).Tag.Lookup(matchTag); ok { - m[fieldIndex(i)] = regexp.MustCompile(pattern) - } - } - - return m -} - -func valueSatisfiesPattern(value string, rx *regexp.Regexp, fName fieldName) error { - if !rx.MatchString(value) { - return fmt.Errorf("parameter '%s' does not match pattern %s", fName, rx.String()) - } - - return nil -} diff --git a/pkg/csi/manila/validator/precludesexpr.go b/pkg/csi/manila/validator/precludesexpr.go deleted file mode 100644 index c499936463..0000000000 --- a/pkg/csi/manila/validator/precludesexpr.go +++ /dev/null @@ -1,79 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package validator - -import ( - "fmt" - "reflect" - "strings" -) - -const ( - precludesTag = "precludes" - - preclsDelim = "," -) - -type ( - precludesExpression []fieldIndex - preclusionsMap map[fieldIndex]precludesExpression -) - -func parsePreclsExpr(precls string, nameIdxMap nameIndexMap, s structName, f fieldName) precludesExpression { - ps := strings.Split(precls, preclsDelim) - preclsExpr := make(precludesExpression, len(ps)) - - for i := range ps { - fIdx, ok := nameIdxMap[fieldName(ps[i])] - if !ok { - panic(fmt.Sprintf("invalid precludes expression in '%s.%s': no field named '%s' found", - s, f, fieldName(ps[i]))) - } - - preclsExpr[i] = fIdx - } - - return preclsExpr -} - -func buildPreclusionsMap(t reflect.Type, idxNameMap indexNameMap, nameIdxMap nameIndexMap) preclusionsMap { - m := make(preclusionsMap) - - for i := 0; i < t.NumField(); i++ { - ft := t.Field(i) - - precls, ok := ft.Tag.Lookup(precludesTag) - if !ok { - continue - } - - fIdx := fieldIndex(i) - m[fIdx] = parsePreclsExpr(precls, nameIdxMap, structName(t.Name()), idxNameMap[fIdx]) - } - - return m -} - -func preclusionsSatisfied(fIdx fieldIndex, v reflect.Value, precls precludesExpression, idxNameMap indexNameMap) error { - for i := range precls { - if v.Field(int(precls[i])).String() != "" { - return fmt.Errorf("parameter '%s' forbids '%s' to be set", idxNameMap[fIdx], idxNameMap[precls[i]]) - } - } - - return nil -} diff --git a/pkg/csi/manila/validator/util.go b/pkg/csi/manila/validator/util.go deleted file mode 100644 index e40de22634..0000000000 --- a/pkg/csi/manila/validator/util.go +++ /dev/null @@ -1,28 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package validator - -import "strings" - -func splitOne(value string, delim byte) (left string, right string, ok bool) { - delimIdx := strings.IndexByte(value, delim) - if delimIdx <= 0 { - return "", "", false - } - - return value[:delimIdx], value[delimIdx+1:], true -} diff --git a/pkg/csi/manila/validator/validator.go b/pkg/csi/manila/validator/validator.go deleted file mode 100644 index 96d9981d0a..0000000000 --- a/pkg/csi/manila/validator/validator.go +++ /dev/null @@ -1,147 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package validator - -import ( - "fmt" - "reflect" -) - -// Validator validates input data and stores it in the output struct. -// The output struct may contain only string fields, where each field -// must be decorated with a `name` tag. An input value with matching name -// is then stored in its corresponding struct field. -// By default, the input data has to contain a value for each struct field. -// -// Available tags: -// * name:"FIELD-NAME" : key of the value in the input map -// * value: modifies value requirements: -// value:"required" : field is required -// value:"optional" : field is optional -// value:"requiredIf:FIELD-NAME=REGEXP-PATTERN" : field is required if the value of FIELD-NAME matches REGEXP-PATTERN -// value:"optionalIf:FIELD-NAME=REGEXP-PATTERN" : field is optional if the value of FIELD-NAME matches REGEXP-PATTERN -// value:"default:VALUE" : field value defaults to VALUE -// * dependsOn:"FIELD-NAMES|,..." : if this field is not empty, the specified fields are required to be present -// operator ',' acts as AND -// operator '|' acts as XOR -// e.g.: dependsOn:"f1|f2|f3,f4,f5" : if this field is not empty, exactly one of {f1,f2,f3} is required to be present, -// and f4 and f5 is required to be present -// * precludes:"FIELD-NAMES,..." : if this field is not empty, all specified fields are required to be empty -// * matches:"REGEXP-PATTERN" : if this field is not empty, it's required to match REGEXP-PATTERN -type Validator struct { - t reflect.Type - - nameIdxMap nameIndexMap - idxNameMap indexNameMap - - valueExprs *valueExpressions - depsMap dependenciesMap - preclsMap preclusionsMap - matchMap matchRegexpMap -} - -// New creates a new instance of Validator. -// stringStruct is the struct used for validation, -// must contain only string fields, must be the same type used in Validator.Populate. -// May panic on invalid struct tags. -func New(stringStruct interface{}) *Validator { - t := reflect.TypeOf(stringStruct).Elem() - - nameIdxMap, idxNameMap := buildNameIdxMap(t) - - return &Validator{ - t: t, - nameIdxMap: nameIdxMap, - idxNameMap: idxNameMap, - valueExprs: buildValueExpressions(t, idxNameMap, nameIdxMap), - depsMap: buildDependenciesMap(t, idxNameMap, nameIdxMap), - preclsMap: buildPreclusionsMap(t, idxNameMap, nameIdxMap), - matchMap: buildMatchRegexMap(t), - } -} - -// Populate validates input data and populates the output struct. -func (v *Validator) Populate(data map[string]string, out interface{}) error { - if reflect.TypeOf(out).Elem() != v.t { - panic("destination type mismatch") - } - - vOut := reflect.ValueOf(out).Elem() - - // Initialize unset entries with their default values - - for fIdx, defaultValue := range v.valueExprs.defaultVal { - name := string(v.idxNameMap[fIdx]) - if _, ok := data[name]; !ok { - data[name] = defaultValue - } - } - - // Populate the values - - for fIdx, fName := range v.idxNameMap { - value, ok := data[string(fName)] - if ok { - if value == "" { - return fmt.Errorf("parameter '%s' cannot be empty", fName) - } - - // The value is present, populate the field and continue with the next one - - vOut.Field(int(fIdx)).SetString(value) - continue - } - - // Value not present, determine whether this field is required - - if requiresValue(fIdx, v.valueExprs, data) { - return fmt.Errorf("missing required parameter '%s'", fName) - } - } - - // Ensure requirements - - for fIdx, deps := range v.depsMap { - if vOut.Field(int(fIdx)).String() != "" { - if err := dependenciesSatisfied(fIdx, vOut, deps, v.idxNameMap); err != nil { - return err - } - } - } - - // Ensure preclusions - - for fIdx, precls := range v.preclsMap { - if vOut.Field(int(fIdx)).String() != "" { - if err := preclusionsSatisfied(fIdx, vOut, precls, v.idxNameMap); err != nil { - return err - } - } - } - - // Ensure the values match their regexp patterns - - for fIdx, rx := range v.matchMap { - if vOut.Field(int(fIdx)).String() != "" { - if err := valueSatisfiesPattern(vOut.Field(int(fIdx)).String(), rx, v.idxNameMap[fIdx]); err != nil { - return err - } - } - } - - return nil -} diff --git a/pkg/csi/manila/validator/validator_test.go b/pkg/csi/manila/validator/validator_test.go deleted file mode 100644 index 1d38db7575..0000000000 --- a/pkg/csi/manila/validator/validator_test.go +++ /dev/null @@ -1,195 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package validator - -import "testing" - -func TestValueRequired(t *testing.T) { - type s1 struct { - A string `name:"a"` - } - - if New(&s1{}).Populate(map[string]string{}, &s1{}) == nil { - t.Error(`value:"required" should be enabled by default`) - } - - type s2 struct { - A string `name:"a" value:"required"` - } - - if New(&s2{}).Populate(map[string]string{}, &s2{}) == nil { - t.Error(`value:"required" violated, should fail on missing parameter`) - } -} - -func TestValueOptional(t *testing.T) { - type s struct { - A string `name:"a" value:"optional"` - } - - if New(&s{}).Populate(map[string]string{}, &s{}) != nil { - t.Error(`value:"optional" violated, should permit missing parameter`) - } -} - -func TestValueRequiredIf(t *testing.T) { - type s struct { - A string `name:"a" value:"optional"` - B string `name:"b" value:"requiredIf:a=^FOO$"` - } - - v := New(&s{}) - - if v.Populate(map[string]string{}, &s{}) != nil { - t.Error(`value:"optional" + value:"requiredIf:a=^FOO$" violated, should permit missing parameter`) - } - - if v.Populate(map[string]string{"a": "xxx"}, &s{}) != nil { - t.Error(`value:"optional" + value:"requiredIf:a=^FOO$" violated, requiredIf should not match`) - } - - if v.Populate(map[string]string{"a": "FOO"}, &s{}) == nil { - t.Error(`value:"optional" + value:"requiredIf:a=^FOO$" violated, requiredIf should match`) - } - - if v.Populate(map[string]string{"a": "FOO", "b": "BAR"}, &s{}) != nil { - t.Error(`value:"optional" + value:"requiredIf:a=^FOO$" violated, should succeed`) - } -} - -func TestValueOptionalIf(t *testing.T) { - type s struct { - A string `name:"a" value:"optional"` - B string `name:"b" value:"optionalIf:a=^FOO$"` - } - - v := New(&s{}) - - if v.Populate(map[string]string{}, &s{}) == nil { - t.Error(`value:"optional" + value:"optionalIf:a=^FOO$" violated, should fail on missing parameter`) - } - - if v.Populate(map[string]string{"a": "xxx"}, &s{}) == nil { - t.Error(`value:"optional" + value:"optionalIf:a=^FOO$" violated, should fail on mis-matched parameter`) - } - - if v.Populate(map[string]string{"a": "FOO"}, &s{}) != nil { - t.Error(`value:"optional" + value:"optionalIf:a=^FOO$" violated, should succeed on matching parameter`) - } -} - -func TestValueDefault(t *testing.T) { - type s struct { - A string `name:"a" value:"default:FOO"` - } - - o := &s{} - v := New(o) - - if v.Populate(map[string]string{}, o) != nil { - t.Error(`value:"default:FOO" should succeed`) - } - - if o.A != "FOO" { - t.Errorf(`value:"default:FOO" : expected value "FOO", got "%s"`, o.A) - } - - if v.Populate(map[string]string{"a": "xxx"}, o) != nil { - t.Error(`setting a value should succeed`) - } - - if o.A != "xxx" { - t.Errorf(`value:"default:FOO" with value "xxx": expected value "xxx", got "%s"`, o.A) - } -} - -func TestDependsOn(t *testing.T) { - type s struct { - A string `name:"a" value:"optional"` - B string `name:"b" value:"optional"` - C string `name:"c" value:"optional"` - D string `name:"d" value:"optional" dependsOn:"a|b,c"` - } - - v := New(&s{}) - - if v.Populate(map[string]string{}, &s{}) != nil { - t.Error(`value:"optional" dependsOn:"a|b,c" violated, should permit missing parameter`) - } - - if v.Populate(map[string]string{"d": "ddd"}, &s{}) == nil { - t.Error(`value:"optional" dependsOn:"a|b,c" violated, should fail on unsatisfied dependencies`) - } - - if v.Populate(map[string]string{"d": "ddd", "c": "ccc"}, &s{}) == nil { - t.Error(`value:"optional" dependsOn:"a|b,c" violated, should fail on unsatisfied dependencies`) - } - - if v.Populate(map[string]string{"d": "ddd", "c": "ccc", "b": "bbb", "a": "aaa"}, &s{}) == nil { - t.Error(`value:"optional" dependsOn:"a|b,c" violated, should fail on unsatisfied dependencies`) - } - - if v.Populate(map[string]string{"d": "ddd", "c": "ccc", "a": "aaa"}, &s{}) != nil { - t.Error(`value:"optional" dependsOn:"a|b,c" violated, should succeed on satisfied dependencies`) - } - - if v.Populate(map[string]string{"d": "ddd", "c": "ccc", "b": "bbb"}, &s{}) != nil { - t.Error(`value:"optional" dependsOn:"a|b,c" violated, should succeed on satisfied dependencies`) - } -} - -func TestPrecludes(t *testing.T) { - type s struct { - A string `name:"a" value:"optional"` - B string `name:"b" value:"optional"` - C string `name:"c" precludes:"a,b"` - } - - v := New(&s{}) - - if v.Populate(map[string]string{"c": "ccc", "a": "aaa"}, &s{}) == nil { - t.Error(`precludes:"a,b" violated, should fail on unsatisfied dependencies`) - } - - if v.Populate(map[string]string{"c": "ccc", "b": "bbb"}, &s{}) == nil { - t.Error(`precludes:"a,b" violated, should fail on unsatisfied dependencies`) - } - - if v.Populate(map[string]string{"c": "ccc", "a": "aaa", "b": "bbb"}, &s{}) == nil { - t.Error(`precludes:"a,b" violated, should fail on unsatisfied dependencies`) - } - - if v.Populate(map[string]string{"c": "ccc"}, &s{}) != nil { - t.Error(`precludes:"a,b" violated, should succeed on satisfied dependencies`) - } -} - -func TestMatches(t *testing.T) { - type s struct { - A string `name:"a" matches:"^(?i)true|false$"` - } - - v := New(&s{}) - - if v.Populate(map[string]string{"a": "xxx"}, &s{}) == nil { - t.Error(`matches:"true|false" violated, should fail on mis-matched parameter`) - } - - if v.Populate(map[string]string{"a": "false"}, &s{}) != nil { - t.Error(`matches:"true|false" violated, should succeed on matching parameter`) - } -} diff --git a/pkg/csi/manila/validator/valueexpr.go b/pkg/csi/manila/validator/valueexpr.go deleted file mode 100644 index e7414818ef..0000000000 --- a/pkg/csi/manila/validator/valueexpr.go +++ /dev/null @@ -1,187 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package validator - -import ( - "fmt" - "reflect" - "regexp" -) - -const ( - valueTag = "value" - valueRequiredTag = "required" - valueRequiredIfTag = "requiredIf:" - valueOptionalTag = "optional" - valueOptionalIfTag = "optionalIf:" - valueDefaultTag = "default:" - valueCoalesceTag = "coalesce:" - - valueExprDelim = '=' -) - -type valueExprType int - -const ( - valueRequired valueExprType = iota + 1 - valueRequiredIf - valueOptional - valueOptionalIf - valueDefault -) - -type ( - valueExpression struct { - exprType valueExprType - fName fieldName - arg string - } - - valueConditionalExpression struct { - r *regexp.Regexp - condField fieldName - } - - valueRequiredMap map[fieldIndex]bool - valueConditionalMap map[fieldIndex]valueConditionalExpression - valueDefaultMap map[fieldIndex]string - - valueExpressions struct { - required valueRequiredMap - condRequired valueConditionalMap - condOptional valueConditionalMap - defaultVal valueDefaultMap - } -) - -func newValueExpressions() *valueExpressions { - return &valueExpressions{ - required: make(valueRequiredMap), - condRequired: make(valueConditionalMap), - condOptional: make(valueConditionalMap), - defaultVal: make(valueDefaultMap), - } -} - -func checkValueExpr(value, valueExprTag string) bool { - return len(value) > len(valueExprTag) && value[:len(valueExprTag)] == valueExprTag -} - -func invalidValueExprError(msg, value, sName string, fName fieldName) error { - return fmt.Errorf("invalid value expression '%s' in '%s.%s': %s", value, sName, fName, msg) -} - -func parseCondValueExpr(value, tag, sName string, nameIdxMap nameIndexMap, fName fieldName) (fieldName, string) { - n, pattern, ok := splitOne(value[len(tag):], valueExprDelim) - targetField := fieldName(n) - - if targetField == fName { - panic(invalidValueExprError(fmt.Sprintf("field '%s' is not allowed in the expression", fName), - value, sName, fName)) - } - - if !ok { - panic(invalidValueExprError("parsing failed", value, sName, fName)) - } - - if _, ok := nameIdxMap[targetField]; !ok { - panic(invalidValueExprError(fmt.Sprintf("no field named '%s' found", targetField), value, sName, fName)) - } - - return targetField, pattern -} - -func parseValueExpr(value string, selfStructName string, fName fieldName, nameIdxMap nameIndexMap) valueExpression { - var expr valueExpression - - if value == valueOptionalTag { - expr.exprType = valueOptional - } else if value == valueRequiredTag { - expr.exprType = valueRequired - } else if checkValueExpr(value, valueDefaultTag) { - expr.exprType = valueDefault - expr.arg = value[len(valueDefaultTag):] - } else if checkValueExpr(value, valueRequiredIfTag) { - expr.exprType = valueRequiredIf - expr.fName, expr.arg = parseCondValueExpr(value, valueRequiredIfTag, selfStructName, nameIdxMap, fName) - } else if checkValueExpr(value, valueOptionalIfTag) { - expr.exprType = valueOptionalIf - expr.fName, expr.arg = parseCondValueExpr(value, valueOptionalIfTag, selfStructName, nameIdxMap, fName) - } else { - panic(invalidValueExprError(fmt.Sprintf("unrecognized value expression"), value, selfStructName, fName)) - } - - return expr -} - -func buildValueExpressions(t reflect.Type, idxNameMap indexNameMap, nameIdxMap nameIndexMap) *valueExpressions { - vs := newValueExpressions() - - for i := 0; i < t.NumField(); i++ { - ft := t.Field(i) - - value, ok := ft.Tag.Lookup(valueTag) - if !ok { - continue - } - - fIdx := fieldIndex(i) - - expr := parseValueExpr(value, t.Name(), idxNameMap[fieldIndex(i)], nameIdxMap) - switch expr.exprType { - case valueOptional: - vs.required[fIdx] = false - case valueRequired: - vs.required[fIdx] = true - case valueDefault: - vs.defaultVal[fIdx] = expr.arg - case valueRequiredIf: - vs.condRequired[fIdx] = valueConditionalExpression{condField: expr.fName, r: regexp.MustCompile(expr.arg)} - case valueOptionalIf: - vs.condOptional[fIdx] = valueConditionalExpression{condField: expr.fName, r: regexp.MustCompile(expr.arg)} - } - } - - return vs -} - -func resolveConditionalReq(condExpr valueConditionalExpression, data map[string]string) bool { - condValue, ok := data[string(condExpr.condField)] - if !ok { - return false - } - - return condExpr.r.MatchString(condValue) -} - -func requiresValue(i fieldIndex, ve *valueExpressions, data map[string]string) bool { - if required, ok := ve.required[i]; ok { - return required - } - - if condExpr, ok := ve.condRequired[i]; ok { - return resolveConditionalReq(condExpr, data) - } - - if condExpr, ok := ve.condOptional[i]; ok { - return !resolveConditionalReq(condExpr, data) - } - - // By default, all fields are required - - return true -} diff --git a/pkg/csi/manila/volumesource.go b/pkg/csi/manila/volumesource.go deleted file mode 100644 index be1b738a7d..0000000000 --- a/pkg/csi/manila/volumesource.go +++ /dev/null @@ -1,126 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package manila - -import ( - "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/gophercloud/gophercloud/openstack/sharedfilesystems/v2/shares" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/cloud-provider-openstack/pkg/csi/manila/manilaclient" - "k8s.io/cloud-provider-openstack/pkg/csi/manila/options" - clouderrors "k8s.io/cloud-provider-openstack/pkg/util/errors" - "k8s.io/klog/v2" -) - -type volumeCreator interface { - create(req *csi.CreateVolumeRequest, shareName string, sizeInGiB int, manilaClient manilaclient.Interface, shareOpts *options.ControllerVolumeContext) (*shares.Share, error) -} - -type blankVolume struct{} - -func (blankVolume) create(req *csi.CreateVolumeRequest, shareName string, sizeInGiB int, manilaClient manilaclient.Interface, shareOpts *options.ControllerVolumeContext) (*shares.Share, error) { - klog.V(4).Infof("creating a new share (%s) in AZ %s", shareName, coalesceValue(shareOpts.AvailabilityZone)) - - createOpts := &shares.CreateOpts{ - AvailabilityZone: shareOpts.AvailabilityZone, - ShareProto: shareOpts.Protocol, - ShareType: shareOpts.Type, - ShareNetworkID: shareOpts.ShareNetworkID, - Name: shareName, - Description: shareDescription, - Size: sizeInGiB, - } - - share, manilaErrCode, err := getOrCreateShare(shareName, createOpts, manilaClient) - if err != nil { - if err == wait.ErrWaitTimeout { - return nil, status.Errorf(codes.DeadlineExceeded, "deadline exceeded while waiting for share %s to become available", share.ID) - } - - if manilaErrCode != 0 { - // An error has occurred, try to roll-back the share - tryDeleteShare(share, manilaClient) - } - - return nil, status.Errorf(manilaErrCode.toRpcErrorCode(), "failed to create a share (%s): %v", shareName, err) - } - - return share, err -} - -type volumeFromSnapshot struct{} - -func (volumeFromSnapshot) create(req *csi.CreateVolumeRequest, shareName string, sizeInGiB int, manilaClient manilaclient.Interface, shareOpts *options.ControllerVolumeContext) (*shares.Share, error) { - snapshotSource := req.GetVolumeContentSource().GetSnapshot() - - if shareOpts.Protocol == "CEPHFS" { - // TODO: Restoring shares from CephFS snapshots needs special handling. - return nil, status.Errorf(codes.InvalidArgument, "restoring CephFS snapshots is not supported yet") - } - - if snapshotSource.GetSnapshotId() == "" { - return nil, status.Error(codes.InvalidArgument, "snapshot ID cannot be empty") - } - - klog.V(4).Infof("restoring snapshot %s into a share (%s) in AZ %s", snapshotSource.GetSnapshotId(), shareName, coalesceValue(shareOpts.AvailabilityZone)) - - snapshot, err := manilaClient.GetSnapshotByID(snapshotSource.GetSnapshotId()) - if err != nil { - if clouderrors.IsNotFound(err) { - return nil, status.Errorf(codes.NotFound, "source snapshot %s not found: %v", snapshotSource.GetSnapshotId(), err) - } - - return nil, status.Errorf(codes.Internal, "failed to retrieve snapshot %s: %v", snapshotSource.GetSnapshotId(), err) - } - - if snapshot.Status != snapshotAvailable { - if snapshot.Status == snapshotCreating { - return nil, status.Errorf(codes.Unavailable, "snapshot %s is in transient creating state", snapshot.ID) - } - - return nil, status.Errorf(codes.FailedPrecondition, "snapshot %s is in invalid state: expected 'available', got '%s'", snapshot.ID, snapshot.Status) - } - - createOpts := &shares.CreateOpts{ - AvailabilityZone: shareOpts.AvailabilityZone, - SnapshotID: snapshot.ID, - ShareProto: shareOpts.Protocol, - ShareType: shareOpts.Type, - ShareNetworkID: shareOpts.ShareNetworkID, - Name: shareName, - Description: shareDescription, - Size: sizeInGiB, - } - - share, manilaErrCode, err := getOrCreateShare(shareName, createOpts, manilaClient) - if err != nil { - if err == wait.ErrWaitTimeout { - return nil, status.Errorf(codes.DeadlineExceeded, "deadline exceeded while waiting for share %s to become available", share.ID) - } - - if manilaErrCode != 0 { - // An error has occurred, try to roll-back the share - tryDeleteShare(share, manilaClient) - } - - return nil, status.Errorf(manilaErrCode.toRpcErrorCode(), "failed to restore snapshot %s into a share (%s): %v", snapshotSource.GetSnapshotId(), shareName, err) - } - - return share, err -} diff --git a/pkg/identity/keystone/authenticator.go b/pkg/identity/keystone/authenticator.go deleted file mode 100644 index a3ca04155c..0000000000 --- a/pkg/identity/keystone/authenticator.go +++ /dev/null @@ -1,144 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package keystone - -import ( - "fmt" - - "github.com/gophercloud/gophercloud" - "github.com/gophercloud/gophercloud/openstack/identity/v3/groups" - "github.com/gophercloud/gophercloud/openstack/identity/v3/tokens" - "github.com/gophercloud/gophercloud/openstack/identity/v3/users" - "k8s.io/apiserver/pkg/authentication/user" -) - -type tokenInfo struct { - userName string - userID string - roles []string - projectName string - projectID string - domainName string - domainID string -} - -type IKeystone interface { - GetTokenInfo(string) (*tokenInfo, error) - GetGroups(string, string) ([]string, error) -} - -type Keystoner struct { - client *gophercloud.ServiceClient -} - -func NewKeystoner(client *gophercloud.ServiceClient) *Keystoner { - return &Keystoner{ - client: client, - } -} - -func (k *Keystoner) GetTokenInfo(token string) (*tokenInfo, error) { - k.client.ProviderClient.SetToken(token) - ret := tokens.Get(k.client, token) - - tokenUser, err := ret.ExtractUser() - if err != nil { - return nil, fmt.Errorf("failed to extract user information from Keystone response: %v", err) - } - - project, err := ret.ExtractProject() - if err != nil { - return nil, fmt.Errorf("failed to extract project information from Keystone response: %v", err) - } - - roles, err := ret.ExtractRoles() - if err != nil { - return nil, fmt.Errorf("failed to extract roles information from Keystone response: %v", err) - } - - var userRoles []string - for _, role := range roles { - userRoles = append(userRoles, role.Name) - } - - return &tokenInfo{ - userName: tokenUser.Name, - userID: tokenUser.ID, - projectName: project.Name, - projectID: project.ID, - roles: userRoles, - domainID: tokenUser.Domain.ID, - domainName: tokenUser.Domain.Name, - }, nil -} - -func (k *Keystoner) GetGroups(token string, userID string) ([]string, error) { - var userGroups []string - - k.client.ProviderClient.SetToken(token) - allGroupPages, err := users.ListGroups(k.client, userID).AllPages() - if err != nil { - return userGroups, fmt.Errorf("failed to get user groups from Keystone: %v", err) - } - - allGroups, err := groups.ExtractGroups(allGroupPages) - if err != nil { - return userGroups, fmt.Errorf("failed to extract user groups from Keystone response: %v", err) - } - - for _, g := range allGroups { - userGroups = append(userGroups, g.Name) - } - - return userGroups, nil -} - -// Authenticator contacts openstack keystone to validate user's token passed in the request. -type Authenticator struct { - keystoner IKeystone -} - -// AuthenticateToken checks the token via Keystone call -func (a *Authenticator) AuthenticateToken(token string) (user.Info, bool, error) { - tokenInfo, err := a.keystoner.GetTokenInfo(token) - if err != nil { - return nil, false, fmt.Errorf("failed to authenticate: %v", err) - } - - userGroups, err := a.keystoner.GetGroups(token, tokenInfo.userID) - if err != nil { - return nil, false, fmt.Errorf("failed to authenticate: %v", err) - } - - extra := map[string][]string{ - Roles: tokenInfo.roles, - ProjectID: {tokenInfo.projectID}, - ProjectName: {tokenInfo.projectName}, - DomainID: {tokenInfo.domainID}, - DomainName: {tokenInfo.domainName}, - } - - userGroups = append(userGroups, tokenInfo.projectID) - authenticatedUser := &user.DefaultInfo{ - Name: tokenInfo.userName, - UID: tokenInfo.userID, - Groups: userGroups, - Extra: extra, - } - - return authenticatedUser, true, nil -} diff --git a/pkg/identity/keystone/authenticator_test.go b/pkg/identity/keystone/authenticator_test.go deleted file mode 100644 index 5bb8d21c54..0000000000 --- a/pkg/identity/keystone/authenticator_test.go +++ /dev/null @@ -1,65 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package keystone - -import ( - "testing" - - th "github.com/gophercloud/gophercloud/testhelper" - "k8s.io/apiserver/pkg/authentication/user" -) - -func TestAuthenticateToken(t *testing.T) { - keystone := &MockIKeystone{} - keystone. - On("GetTokenInfo", "token"). - Return(&tokenInfo{ - userName: "user-name", - userID: "user-id", - projectID: "project-id", - projectName: "project-name", - domainName: "domain-name", - domainID: "domain-id", - roles: []string{"role1", "role2"}, - }, nil). - Once() - keystone. - On("GetGroups", "token", "user-id"). - Return([]string{"group1", "group2"}, nil). - Once() - - a := &Authenticator{ - keystoner: keystone, - } - userInfo, allowed, err := a.AuthenticateToken("token") - - th.AssertNoErr(t, err) - th.AssertEquals(t, true, allowed) - - expectedUserInfo := &user.DefaultInfo{ - Name: "user-name", - UID: "user-id", - Groups: []string{"group1", "group2"}, - Extra: map[string][]string{ - Roles: {"role1", "role2"}, - ProjectID: {"project-id"}, - ProjectName: {"project-name"}, - DomainID: {"domain-id"}, - DomainName: {"domain-name"}, - }, - } - th.AssertDeepEquals(t, expectedUserInfo, userInfo) - - keystone.AssertExpectations(t) -} diff --git a/pkg/identity/keystone/authorizer.go b/pkg/identity/keystone/authorizer.go deleted file mode 100644 index 1cf2942d20..0000000000 --- a/pkg/identity/keystone/authorizer.go +++ /dev/null @@ -1,388 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package keystone - -import ( - "encoding/json" - "fmt" - "sort" - "strings" - "sync" - - "github.com/gophercloud/gophercloud" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/klog/v2" - - "k8s.io/apiserver/pkg/authorization/authorizer" -) - -// Authorizer contacts openstack keystone to check whether the user can perform -// requested operations. -// The keystone endpoint and policy list are passed during apiserver startup -type Authorizer struct { - authURL string - client *gophercloud.ServiceClient - pl policyList - mu sync.Mutex -} - -func findString(a string, list []string) bool { - sort.Strings(list) - index := sort.SearchStrings(list, a) - if index < len(list) && list[index] == a { - return true - } - return false -} - -// getAllowed gets the allowed resources based on the definition. -func getAllowed(definition string, str string) (sets.String, error) { - allowed := sets.NewString() - - if definition == str || definition == "*" || str == "" { - allowed.Insert(str) - } else if strings.Index(definition, "!") == 0 && strings.Index(definition, "[") != 1 { - // "!namespace" - if definition[1:] == str || definition[1:] == "*" { - return nil, fmt.Errorf("") - } - allowed.Insert(str) - } else if strings.Index(definition, "[") == 0 && strings.Index(definition, "]") == (len(definition)-1) { - // "['namespace1', 'namespace2']" - var items []string - if err := json.Unmarshal([]byte(strings.Replace(definition, "'", "\"", -1)), &items); err != nil { - klog.V(4).Infof("Skip the permission definition %s", definition) - return nil, fmt.Errorf("") - } - for _, val := range items { - if val == "*" { - allowed.Insert(str) - continue - } - allowed.Insert(val) - } - } else if strings.Index(definition, "!") == 0 && strings.Index(definition, "[") == 1 && strings.Index(definition, "]") == (len(definition)-1) { - // "!['namespace1', 'namespace2']" - var items []string - if err := json.Unmarshal([]byte(strings.Replace(definition[1:], "'", "\"", -1)), &items); err != nil { - klog.V(4).Infof("Skip the permission definition %s", definition) - return nil, fmt.Errorf("") - } - found := false - for _, val := range items { - if val == str || val == "*" { - found = true - } - } - - if found { - return nil, fmt.Errorf("") - } - allowed.Insert(str) - } - - return allowed, nil -} - -func resourcePermissionAllowed(permissionSpec map[string][]string, attr authorizer.Attributes) bool { - ns := attr.GetNamespace() - res := attr.GetResource() - verb := attr.GetVerb() - klog.V(4).Infof("Request namespace: %s, resource: %s, verb: %s", ns, res, verb) - - for key, value := range permissionSpec { - klog.V(4).Infof("Evaluating %s: %s", key, value) - - allowedVerbs := sets.NewString() - for _, val := range value { - allowedVerbs.Insert(strings.ToLower(val)) - } - if allowedVerbs.Has("*") { - allowedVerbs.Insert(verb) - } - - keyList := strings.Split(key, "/") - if len(keyList) != 2 { - // Ignore this spec - klog.V(4).Infof("Skip the permission definition %s", key) - continue - } - nsDef := strings.ToLower(strings.TrimSpace(keyList[0])) - resDef := strings.ToLower(strings.TrimSpace(keyList[1])) - - allowedNamespaces, err := getAllowed(nsDef, ns) - if err != nil { - continue - } - - allowedResources, err := getAllowed(resDef, res) - if err != nil { - continue - } - - klog.V(4).Infof("allowedNamespaces: %s, allowedResources: %s, allowedVerbs: %s", allowedNamespaces.List(), allowedResources.List(), allowedVerbs.List()) - - if allowedNamespaces.Has(ns) && allowedResources.Has(res) && allowedVerbs.Has(verb) { - return true - } - } - - return false -} - -func nonResourcePermissionAllowed(permissionSpec map[string][]string, attr authorizer.Attributes) bool { - path := attr.GetPath() - verb := attr.GetVerb() - - for key, value := range permissionSpec { - allowedVerbs := sets.NewString() - for _, val := range value { - allowedVerbs.Insert(val) - } - if allowedVerbs.Has("*") { - allowedVerbs.Insert(verb) - } - - if key == path && allowedVerbs.Has(verb) { - return true - } - } - - return false -} - -func resourceMatches(p policy, a authorizer.Attributes) bool { - if *p.ResourceSpec.APIGroup != "*" && *p.ResourceSpec.APIGroup != a.GetAPIGroup() { - return false - } - if *p.ResourceSpec.Namespace != "*" && *p.ResourceSpec.Namespace != a.GetNamespace() { - return false - } - if len(a.GetSubresource()) > 0 { - if !findString("*", p.ResourceSpec.Resources) && - !findString(strings.Join([]string{a.GetResource(), "*"}, "/"), p.ResourceSpec.Resources) && - !findString(strings.Join([]string{a.GetResource(), a.GetSubresource()}, "/"), p.ResourceSpec.Resources) { - return false - } - } else { - if !findString("*", p.ResourceSpec.Resources) && - !findString(a.GetResource(), p.ResourceSpec.Resources) { - return false - } - } - if !findString("*", p.ResourceSpec.Verbs) && !findString(a.GetVerb(), p.ResourceSpec.Verbs) { - return false - } - allowed := match(p.Match, a) - if !allowed { - return false - } - output, err := json.MarshalIndent(p, "", " ") - if err == nil { - klog.V(6).Infof("matched rule : %s", string(output)) - } - return true -} - -func nonResourceMatches(p policy, a authorizer.Attributes) bool { - if findString("", p.NonResourceSpec.Verbs) { - klog.Infof("verb is empty. skipping : %#v", p) - return false - } - - if p.NonResourceSpec.NonResourcePath == nil { - klog.Infof("path should be set. skipping : %#v", p) - return false - } - - if !findString("*", p.NonResourceSpec.Verbs) && !findString(a.GetVerb(), p.NonResourceSpec.Verbs) { - return false - } - if *p.NonResourceSpec.NonResourcePath != "*" && *p.NonResourceSpec.NonResourcePath != a.GetPath() && - // Allow a trailing * subpath match - !(strings.HasSuffix(*p.NonResourceSpec.NonResourcePath, "*") && - strings.HasPrefix(a.GetPath(), strings.TrimRight(*p.NonResourceSpec.NonResourcePath, "*"))) { - return false - } - allowed := match(p.Match, a) - if !allowed { - return false - } - output, err := json.MarshalIndent(p, "", " ") - if err == nil { - klog.V(6).Infof("matched rule : %s", string(output)) - } - return true -} - -func match(match []policyMatch, attributes authorizer.Attributes) bool { - user := attributes.GetUser() - var find = false - types := []string{TypeGroup, TypeProject, TypeRole, TypeUser} - - for _, m := range match { - if !findString(m.Type, types) { - klog.Warningf("unknown type %s", m.Type) - return false - } - if findString("*", m.Values) { - continue - } - - find = false - - if m.Type == TypeGroup { - for _, group := range user.GetGroups() { - if findString(group, m.Values) { - find = true - break - } - } - if !find { - return false - } - } else if m.Type == TypeUser { - if !findString(user.GetName(), m.Values) && !findString(user.GetUID(), m.Values) { - return false - } - } else if m.Type == TypeProject { - if val, ok := user.GetExtra()[ProjectID]; ok { - for _, item := range val { - if findString(item, m.Values) { - find = true - break - } - } - if find { - continue - } - } - - if val, ok := user.GetExtra()[ProjectName]; ok { - for _, item := range val { - if findString(item, m.Values) { - find = true - break - } - } - if find { - continue - } - } - return false - } else if m.Type == TypeRole { - if val, ok := user.GetExtra()[Roles]; ok { - for _, item := range val { - if findString(item, m.Values) { - find = true - break - } - } - if find { - continue - } - } - return false - } else { - klog.Infof("unknown type %s. skipping.", m.Type) - } - } - - return true -} - -// Authorize checks whether the user can perform an operation -func (a *Authorizer) Authorize(attributes authorizer.Attributes) (authorized authorizer.Decision, reason string, err error) { - a.mu.Lock() - defer a.mu.Unlock() - - // Get roles and projects from the request. - user := attributes.GetUser() - userRoles := sets.NewString() - if val, ok := user.GetExtra()[Roles]; ok { - for _, role := range val { - userRoles.Insert(role) - } - } - - // We support both project name and project ID. - userProjects := sets.NewString() - if val, ok := user.GetExtra()[ProjectName]; ok { - for _, project := range val { - userProjects.Insert(project) - } - } - if val, ok := user.GetExtra()[ProjectID]; ok { - for _, project := range val { - userProjects.Insert(project) - } - } - - klog.V(4).Infof("Request userRoles: %s, userProjects: %s", userRoles.List(), userProjects.List()) - - // The permission is whitelist. Make sure we go through all the policies that match the user roles and projects. If - // the operation is allowed explicitly, stop the loop and return "allowed". - for _, p := range a.pl { - policyRoles := sets.NewString() - policyProjects := sets.NewString() - - if p.Users != nil { - if val, ok := p.Users["roles"]; ok { - for _, role := range val { - policyRoles.Insert(role) - } - } - if val, ok := p.Users["projects"]; ok { - for _, project := range val { - policyProjects.Insert(project) - } - } - - klog.V(4).Infof("policyRoles: %s, policyProjects: %s", policyRoles.List(), policyProjects.List()) - - if !userRoles.IsSuperset(policyRoles) || !policyProjects.HasAny(userProjects.List()...) { - continue - } - } - - // ResourcePermissionsSpec and NonResourcePermissionsSpec take precedence over ResourceSpec and NonResourceSpec - if attributes.IsResourceRequest() { - if p.ResourcePermissionsSpec != nil { - if resourcePermissionAllowed(p.ResourcePermissionsSpec, attributes) { - return authorizer.DecisionAllow, "", nil - } - } else if p.ResourceSpec != nil { - if resourceMatches(*p, attributes) { - return authorizer.DecisionAllow, "", nil - } - } - } else { - if p.NonResourcePermissionsSpec != nil { - if nonResourcePermissionAllowed(p.NonResourcePermissionsSpec, attributes) { - return authorizer.DecisionAllow, "", nil - } - } else if p.NonResourceSpec != nil { - if nonResourceMatches(*p, attributes) { - return authorizer.DecisionAllow, "", nil - } - } - } - } - - klog.V(4).Infof("Authorization failed, user: %#v, attributes: %#v\n", attributes.GetUser(), attributes) - return authorizer.DecisionDeny, "No policy matched.", nil -} diff --git a/pkg/identity/keystone/authorizer_test.go b/pkg/identity/keystone/authorizer_test.go deleted file mode 100644 index a92789af8e..0000000000 --- a/pkg/identity/keystone/authorizer_test.go +++ /dev/null @@ -1,380 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package keystone - -import ( - "os" - "testing" - - "github.com/gophercloud/gophercloud" - "github.com/gophercloud/gophercloud/openstack" - th "github.com/gophercloud/gophercloud/testhelper" - - "k8s.io/apiserver/pkg/authentication/user" - "k8s.io/apiserver/pkg/authorization/authorizer" -) - -func TestAuthorizer(t *testing.T) { - provider, err := openstack.NewClient("127.0.0.1") - th.AssertNoErr(t, err) - client := &gophercloud.ServiceClient{ - ProviderClient: provider, - Endpoint: "127.0.0.1", - } - - // Read policies from an example file - path, err := os.Getwd() - th.AssertNoErr(t, err) - path += "/authorizer_test_policy.json" - policy, err := newFromFile(path) - th.AssertNoErr(t, err) - - a := &Authorizer{authURL: "127.0.0.1", client: client, pl: policy} - - // Create two users with different permissions - user1 := &user.DefaultInfo{ - Name: "user1", - Groups: []string{"group1"}, - Extra: map[string][]string{ - ProjectName: {"project1"}, - Roles: {"role1"}, - }, - } - user2 := &user.DefaultInfo{ - Name: "user2", - Groups: []string{"group2"}, - Extra: map[string][]string{ - ProjectName: {"project2"}, - Roles: {"role2"}, - }, - } - - // TODO(Fedosin): The tests below cover all main cases. Nevertheless, the list can (and should) be expanded - // with additional tests for other scenarios. - - // Resource user match - attrs := authorizer.AttributesRecord{User: user1, ResourceRequest: true, Verb: "get", Resource: "user_resource1"} - decision, _, _ := a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionAllow, decision) - - attrs = authorizer.AttributesRecord{User: user1, ResourceRequest: true, Verb: "list", Resource: "user_resource2"} - decision, _, _ = a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionAllow, decision) - - attrs = authorizer.AttributesRecord{User: user1, ResourceRequest: true, Verb: "patch", Resource: "user_resource1"} - decision, _, _ = a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionDeny, decision) - - attrs = authorizer.AttributesRecord{User: user2, ResourceRequest: true, Verb: "get", Resource: "user_resource1"} - decision, _, _ = a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionDeny, decision) - - // Resource group match - attrs = authorizer.AttributesRecord{User: user1, ResourceRequest: true, Verb: "get", Resource: "group_resource"} - decision, _, _ = a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionAllow, decision) - - attrs = authorizer.AttributesRecord{User: user1, ResourceRequest: true, Verb: "patch", Resource: "group_resource"} - decision, _, _ = a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionDeny, decision) - - attrs = authorizer.AttributesRecord{User: user2, ResourceRequest: true, Verb: "get", Resource: "group_resource"} - decision, _, _ = a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionDeny, decision) - - // Resource group and role match - attrs = authorizer.AttributesRecord{User: user1, ResourceRequest: true, Verb: "get", Resource: "group_role_resource"} - decision, _, _ = a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionDeny, decision) - - // Resource project match - attrs = authorizer.AttributesRecord{User: user1, ResourceRequest: true, Verb: "get", Resource: "project_resource"} - decision, _, _ = a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionAllow, decision) - - attrs = authorizer.AttributesRecord{User: user1, ResourceRequest: true, Verb: "patch", Resource: "project_resource"} - decision, _, _ = a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionDeny, decision) - - attrs = authorizer.AttributesRecord{User: user2, ResourceRequest: true, Verb: "get", Resource: "project_resource"} - decision, _, _ = a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionDeny, decision) - - // Resource role match - attrs = authorizer.AttributesRecord{User: user1, ResourceRequest: true, Verb: "get", Resource: "role_resource"} - decision, _, _ = a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionAllow, decision) - - attrs = authorizer.AttributesRecord{User: user1, ResourceRequest: true, Verb: "patch", Resource: "role_resource"} - decision, _, _ = a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionDeny, decision) - - attrs = authorizer.AttributesRecord{User: user2, ResourceRequest: true, Verb: "get", Resource: "role_resource"} - decision, _, _ = a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionDeny, decision) - - // Core api group resource match - attrs = authorizer.AttributesRecord{User: user1, ResourceRequest: true, Verb: "get", Resource: "core_resource"} - decision, _, _ = a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionAllow, decision) - - attrs = authorizer.AttributesRecord{User: user1, ResourceRequest: true, Verb: "get", Resource: "core_resource", APIGroup: "NonCoreAPIGroup"} - decision, _, _ = a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionDeny, decision) - - // Nonresource user match - attrs = authorizer.AttributesRecord{User: user1, ResourceRequest: false, Verb: "get", Path: "/user"} - decision, _, _ = a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionAllow, decision) - - attrs = authorizer.AttributesRecord{User: user1, ResourceRequest: false, Verb: "patch", Path: "/user"} - decision, _, _ = a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionDeny, decision) - - attrs = authorizer.AttributesRecord{User: user2, ResourceRequest: false, Verb: "get", Path: "/user"} - decision, _, _ = a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionDeny, decision) - - // Nonresource group match - attrs = authorizer.AttributesRecord{User: user1, ResourceRequest: false, Verb: "get", Path: "/group"} - decision, _, _ = a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionAllow, decision) - - attrs = authorizer.AttributesRecord{User: user1, ResourceRequest: false, Verb: "patch", Path: "/group"} - decision, _, _ = a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionDeny, decision) - - attrs = authorizer.AttributesRecord{User: user2, ResourceRequest: false, Verb: "get", Path: "/group"} - decision, _, _ = a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionDeny, decision) - - // Nonresource project match - attrs = authorizer.AttributesRecord{User: user1, ResourceRequest: false, Verb: "get", Path: "/project"} - decision, _, _ = a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionAllow, decision) - - attrs = authorizer.AttributesRecord{User: user1, ResourceRequest: false, Verb: "patch", Path: "/project"} - decision, _, _ = a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionDeny, decision) - - attrs = authorizer.AttributesRecord{User: user2, ResourceRequest: false, Verb: "get", Path: "/project"} - decision, _, _ = a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionDeny, decision) - - // Nonresource role match - attrs = authorizer.AttributesRecord{User: user1, ResourceRequest: false, Verb: "get", Path: "/role"} - decision, _, _ = a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionAllow, decision) - - attrs = authorizer.AttributesRecord{User: user1, ResourceRequest: false, Verb: "patch", Path: "/role"} - decision, _, _ = a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionDeny, decision) - - attrs = authorizer.AttributesRecord{User: user2, ResourceRequest: false, Verb: "get", Path: "/role"} - decision, _, _ = a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionDeny, decision) - - // Unknown match type - attrs = authorizer.AttributesRecord{User: user1, ResourceRequest: true, Verb: "get", Resource: "unknown_type_resource"} - decision, _, _ = a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionDeny, decision) - - // Allow subresource with specific value, e.i. "user_resource1/subresource1" - attrs = authorizer.AttributesRecord{ - User: user1, - ResourceRequest: true, - Verb: "get", - Resource: "user_resource1", - Subresource: "subresource1", - } - decision, _, _ = a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionAllow, decision) - - // Deny subresource with specific value, e.i. "user_resource1/subresource2", it must not be present in policy.json - attrs = authorizer.AttributesRecord{ - User: user1, - ResourceRequest: true, - Verb: "get", - Resource: "user_resource1", - Subresource: "subresource2", - } - decision, _, _ = a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionDeny, decision) - - // Allow subresource with representing wildcard, e.i. "user_resource3/wildcard_resource", - // expected to be tested with policy where resources: ["user_resource3/*"] - attrs = authorizer.AttributesRecord{ - User: user1, - ResourceRequest: true, - Verb: "get", - Resource: "user_resource3", - Subresource: "wildcard_resource", - } - decision, _, _ = a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionAllow, decision) - - // Allow nonresource endpoint with specific value, e.i. "/api" - attrs = authorizer.AttributesRecord{ - User: user1, - ResourceRequest: false, - Verb: "get", - Path: "/api", - } - decision, _, _ = a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionAllow, decision) - - // Deny nonresource endpoint with specific value, e.i. "/imaginary_api/path/none", it must not be present in policy.json - attrs = authorizer.AttributesRecord{ - User: user1, - ResourceRequest: false, - Verb: "get", - Path: "/imaginary_api/path/none", - } - decision, _, _ = a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionDeny, decision) - - // Allow nonresource endpoint with specific prefix, e.i. "/api/represents/wildcard", - // expected to be tested with policy where path: ["/api/*"] - attrs = authorizer.AttributesRecord{ - User: user1, - ResourceRequest: false, - Verb: "get", - Path: "/api/represents/wildcard", - } - decision, _, _ = a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionAllow, decision) -} - -func TestAuthorizerVersion2(t *testing.T) { - provider, err := openstack.NewClient("127.0.0.1") - th.AssertNoErr(t, err) - client := &gophercloud.ServiceClient{ - ProviderClient: provider, - Endpoint: "127.0.0.1", - } - - path, err := os.Getwd() - th.AssertNoErr(t, err) - path += "/authorizer_test_policy_version2.json" - policy, err := newFromFile(path) - th.AssertNoErr(t, err) - - a := &Authorizer{authURL: "127.0.0.1", client: client, pl: policy} - - developer := &user.DefaultInfo{ - Name: "developer", - Groups: []string{"group1"}, - Extra: map[string][]string{ - ProjectName: {"demo"}, - Roles: {"developer"}, - }, - } - viewer := &user.DefaultInfo{ - Name: "viewer", - Groups: []string{"group2"}, - Extra: map[string][]string{ - ProjectName: {"demo"}, - ProjectID: {"ff9db8980cf24a74bc9dd796b6ce811f"}, - Roles: {"viewer"}, - }, - } - anotherviewer := &user.DefaultInfo{ - Name: "anotherviewer", - Groups: []string{"group"}, - Extra: map[string][]string{ - ProjectName: {"alt_demo"}, - ProjectID: {"cd08a539b7c845ddb92c5d08752101d1"}, - Roles: {"viewer"}, - }, - } - clusteradmin := &user.DefaultInfo{ - Name: "clusteradmin", - Groups: []string{"group2"}, - Extra: map[string][]string{ - ProjectName: {"demo"}, - Roles: {"clusteradmin"}, - }, - } - - // Test developer - attrs := authorizer.AttributesRecord{User: developer, ResourceRequest: true, Verb: "get", Namespace: "default", Resource: "pods"} - decision, _, _ := a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionAllow, decision) - - attrs = authorizer.AttributesRecord{User: developer, ResourceRequest: true, Verb: "get", Namespace: "default", Resource: "clusterroles"} - decision, _, _ = a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionAllow, decision) - - attrs = authorizer.AttributesRecord{User: developer, ResourceRequest: true, Verb: "get", Namespace: "", Resource: "clusterroles"} - decision, _, _ = a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionAllow, decision) - - attrs = authorizer.AttributesRecord{User: developer, ResourceRequest: true, Verb: "create", Namespace: "", Resource: "clusterrolebindings"} - decision, _, _ = a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionDeny, decision) - - // Test developer, resource name should be case insensitive. - attrs = authorizer.AttributesRecord{User: developer, ResourceRequest: true, Verb: "get", Namespace: "", Resource: "podsecuritypolicies"} - decision, _, _ = a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionAllow, decision) - - // Test viewer - attrs = authorizer.AttributesRecord{User: viewer, ResourceRequest: true, Verb: "get", Namespace: "default", Resource: "deployments"} - decision, _, _ = a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionAllow, decision) - - attrs = authorizer.AttributesRecord{User: viewer, ResourceRequest: true, Verb: "get", Namespace: "kube-system", Resource: "services"} - decision, _, _ = a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionDeny, decision) - - attrs = authorizer.AttributesRecord{User: viewer, ResourceRequest: true, Verb: "create", Namespace: "default", Resource: "pods"} - decision, _, _ = a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionDeny, decision) - - // Test anotherviewer, the result should be the same with viewer - attrs = authorizer.AttributesRecord{User: anotherviewer, ResourceRequest: true, Verb: "get", Namespace: "default", Resource: "deployments"} - decision, _, _ = a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionAllow, decision) - - attrs = authorizer.AttributesRecord{User: anotherviewer, ResourceRequest: true, Verb: "get", Namespace: "kube-system", Resource: "services"} - decision, _, _ = a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionDeny, decision) - - attrs = authorizer.AttributesRecord{User: anotherviewer, ResourceRequest: true, Verb: "create", Namespace: "default", Resource: "pods"} - decision, _, _ = a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionDeny, decision) - - attrs = authorizer.AttributesRecord{User: clusteradmin, ResourceRequest: true, Verb: "create", Namespace: "", Resource: "clusterroles"} - decision, _, _ = a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionAllow, decision) - - attrs = authorizer.AttributesRecord{User: clusteradmin, ResourceRequest: true, Verb: "get", Namespace: "kube-system", Resource: "secrets"} - decision, _, _ = a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionAllow, decision) - - attrs = authorizer.AttributesRecord{User: developer, ResourceRequest: false, Verb: "get", Path: "/healthz"} - decision, _, _ = a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionDeny, decision) - - attrs = authorizer.AttributesRecord{User: viewer, ResourceRequest: false, Verb: "get", Path: "/healthz"} - decision, _, _ = a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionDeny, decision) - - attrs = authorizer.AttributesRecord{User: clusteradmin, ResourceRequest: false, Verb: "get", Path: "/healthz"} - decision, _, _ = a.Authorize(attrs) - th.AssertEquals(t, authorizer.DecisionAllow, decision) -} diff --git a/pkg/identity/keystone/authorizer_test_policy.json b/pkg/identity/keystone/authorizer_test_policy.json deleted file mode 100644 index 97de66026c..0000000000 --- a/pkg/identity/keystone/authorizer_test_policy.json +++ /dev/null @@ -1,259 +0,0 @@ -[ - { - "resource": { - "verbs": [ - "get", - "list" - ], - "resources": [ - "user_resource1", - "user_resource2" - ], - "version": "*", - "namespace": "*" - }, - "match": [ - { - "type": "user", - "values": [ - "user1" - ] - }, - { - "type": "role", - "values": [ - "role1" - ] - } - ] - }, - { - "resource": { - "verbs": ["get"], - "resources": ["group_resource"], - "version": "*", - "namespace": "*" - }, - "match": [{ - "type": "group", - "values": ["group1"] - }] - }, - { - "resource": { - "verbs": [ - "get" - ], - "resources": [ - "group_role_resource" - ], - "version": "*", - "namespace": "*" - }, - "match": [ - { - "type": "group", - "values": [ - "group1" - ] - }, - { - "type": "role", - "values": [ - "role3" - ] - } - ] - }, - { - "resource": { - "verbs": ["get"], - "resources": ["project_resource"], - "version": "*", - "namespace": "*" - }, - "match": [{ - "type": "project", - "values": ["project1"] - }] - }, - { - "resource": { - "verbs": ["get"], - "resources": ["role_resource"], - "version": "*", - "namespace": "*" - }, - "match": [{ - "type": "role", - "values": ["role1"] - }] - }, - { - "resource": { - "verbs": ["get"], - "resources": ["core_resource"], - "version": "", - "namespace": "*" - }, - "match": [{ - "type": "role", - "values": ["role1"] - }] - }, - { - "nonresource": { - "verbs": ["get"], - "path": "/user" - }, - "match": [{ - "type": "user", - "values": ["user1"] - }] - }, - { - "nonresource": { - "verbs": ["get"], - "path": "/group" - }, - "match": [{ - "type": "group", - "values": ["group1"] - }] - }, - { - "nonresource": { - "verbs": ["get"], - "path": "/project" - }, - "match": [{ - "type": "project", - "values": ["project1"] - }] - }, - { - "nonresource": { - "verbs": ["get"], - "path": "/role" - }, - "match": [{ - "type": "role", - "values": ["role1"] - }] - }, - - { - "resource": { - "verbs": ["*"], - "resources": ["unknown_type_resource"], - "version": "*", - "namespace": "*" - }, - "match": [{ - "type": "unknown_type", - "values": ["*"] - }] - }, - { - "resource": { - "verbs": ["get"], - "resources": ["user_resource"], - "version": "*", - "namespace": "*" - }, - "nonresource": { - "verbs": ["get"], - "path": "/user" - }, - "match": [{ - "type": "user", - "values": ["user1"] - }] - }, - { - "resource": { - "verbs": [""], - "resources": ["user_resource"], - "version": "*", - "namespace": "*" - }, - "match": [{ - "type": "user", - "values": ["user1"] - }] - }, - { - "nonresource": { - "verbs": [""], - "path": "/user" - }, - "match": [{ - "type": "user", - "values": ["user1"] - }] - }, - { - "resource": { - "verbs": ["get"], - "version": "*", - "namespace": "*" - }, - "match": [{ - "type": "user", - "values": ["user1"] - }] - }, - { - "nonresource": { - "verbs": ["get"] - }, - "match": [{ - "type": "user", - "values": ["user1"] - }] - }, - { - "resource": { - "verbs": ["get"], - "resources": ["user_resource1/subresource1"], - "version": "*", - "namespace": "*" - }, - "match": [{ - "type": "user", - "values": ["user1"] - }] - }, - { - "resource": { - "verbs": ["get"], - "resources": ["user_resource3/*"], - "version": "*", - "namespace": "*" - }, - "match": [{ - "type": "user", - "values": ["user1"] - }] - }, - { - "nonresource": { - "verbs": ["get"], - "path": "/api" - }, - "match": [{ - "type": "user", - "values": ["user1"] - }] - }, - { - "nonresource": { - "verbs": ["get"], - "path": "/api/*" - }, - "match": [{ - "type": "user", - "values": ["user1"] - }] - } -] diff --git a/pkg/identity/keystone/authorizer_test_policy_version2.json b/pkg/identity/keystone/authorizer_test_policy_version2.json deleted file mode 100644 index 0e1010db20..0000000000 --- a/pkg/identity/keystone/authorizer_test_policy_version2.json +++ /dev/null @@ -1,33 +0,0 @@ -[ - { - "users": { - "roles": ["developer"], - "projects": ["demo"] - }, - "resource_permissions": { - "!kube-system/!['namespaces', 'clusterrolebindings', 'clusterroles', 'podsecuritypolicies', 'rolebindings', 'roles', 'podSecurityPolicies']": ["*"], - "!kube-system/['namespaces', 'clusterrolebindings', 'clusterroles', 'podsecuritypolicies', 'rolebindings', 'roles', 'podSecurityPolicies']": ["get", "list"] - } - }, - { - "users": { - "roles": ["viewer"], - "projects": ["demo", "cd08a539b7c845ddb92c5d08752101d1"] - }, - "resource_permissions": { - "default/['*']": ["get", "list"] - } - }, - { - "users": { - "roles": ["clusteradmin"], - "projects": ["demo"] - }, - "resource_permissions": { - "*/*": ["*"] - }, - "nonresource_permissions": { - "/healthz": ["get", "post"] - } - } -] diff --git a/pkg/identity/keystone/config.go b/pkg/identity/keystone/config.go deleted file mode 100644 index 2d4a87c162..0000000000 --- a/pkg/identity/keystone/config.go +++ /dev/null @@ -1,94 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package keystone - -import ( - "fmt" - "os" - - "github.com/spf13/pflag" - "k8s.io/klog/v2" -) - -// Config configures a keystone webhook server -type Config struct { - Address string - CertFile string - KeyFile string - KeystoneURL string - KeystoneCA string - PolicyFile string - PolicyConfigMapName string - SyncConfigFile string - SyncConfigMapName string - Kubeconfig string -} - -// NewConfig returns a Config -func NewConfig() *Config { - return &Config{ - Address: "0.0.0.0:8443", - CertFile: os.Getenv("TLS_CERT_FILE"), - KeyFile: os.Getenv("TLS_PRIVATE_KEY_FILE"), - KeystoneURL: os.Getenv("OS_AUTH_URL"), - KeystoneCA: os.Getenv("KEYSTONE_CA_FILE"), - PolicyFile: os.Getenv("KEYSTONE_POLICY_FILE"), - PolicyConfigMapName: os.Getenv("KEYSTONE_POLICY_CONFIGMAP_NAME"), - SyncConfigFile: os.Getenv("KEYSTONE_SYNC_CONFIG_FILE"), - SyncConfigMapName: os.Getenv("KEYSTONE_SYNC_CONFIGMAP_NAME"), - Kubeconfig: os.Getenv("KEYSTONE_KUBECONFIG_FILE"), - } -} - -// ValidateFlags validates whether flags are set up correctly -func (c *Config) ValidateFlags() error { - var errorsFound bool - - if c.KeystoneURL == "" { - errorsFound = true - klog.Errorf("please specify --keystone-url or set the OS_AUTH_URL environment variable.") - } - if c.CertFile == "" || c.KeyFile == "" { - errorsFound = true - klog.Errorf("Please specify --tls-cert-file and --tls-private-key-file arguments.") - } - if c.PolicyFile == "" && c.PolicyConfigMapName == "" { - klog.Warning("Argument --keystone-policy-file or --policy-configmap-name missing. Only keystone authentication will work. Use RBAC for authorization.") - } - if c.SyncConfigFile == "" && c.SyncConfigMapName == "" { - klog.Warning("Argument --sync-config-file or --sync-configmap-name missing. Data synchronization between Keystone and Kubernetes is disabled.") - } - - if errorsFound { - return fmt.Errorf("failed to validate the input parameters") - } - return nil -} - -// AddFlags adds flags for a specific AutoScaler to the specified FlagSet -func (c *Config) AddFlags(fs *pflag.FlagSet) { - fs.StringVar(&c.Address, "listen", c.Address, "
: to listen on") - fs.StringVar(&c.CertFile, "tls-cert-file", c.CertFile, "File containing the default x509 Certificate for HTTPS.") - fs.StringVar(&c.KeyFile, "tls-private-key-file", c.KeyFile, "File containing the default x509 private key matching --tls-cert-file.") - fs.StringVar(&c.KeystoneURL, "keystone-url", c.KeystoneURL, "URL for the OpenStack Keystone API") - fs.StringVar(&c.KeystoneCA, "keystone-ca-file", c.KeystoneCA, "File containing the certificate authority for Keystone Service.") - fs.StringVar(&c.PolicyFile, "keystone-policy-file", c.PolicyFile, "File containing the policy, if provided, it takes precedence over the policy configmap.") - fs.StringVar(&c.PolicyConfigMapName, "policy-configmap-name", c.PolicyConfigMapName, "ConfigMap in kube-system namespace containing the policy configuration, the ConfigMap data must contain the key 'policies'") - fs.StringVar(&c.SyncConfigFile, "sync-config-file", c.SyncConfigFile, "File containing config values for data synchronization beetween Keystone and Kubernetes.") - fs.StringVar(&c.SyncConfigMapName, "sync-configmap-name", "", "ConfigMap in kube-system namespace containing config values for data synchronization beetween Keystone and Kubernetes.") - fs.StringVar(&c.Kubeconfig, "kubeconfig", c.Kubeconfig, "Kubeconfig file used to connect to Kubernetes API to get policy configmap. If the service is running inside the pod, this option is not necessary, will use in-cluster config instead.") -} diff --git a/pkg/identity/keystone/keystone.go b/pkg/identity/keystone/keystone.go deleted file mode 100644 index d89e75602c..0000000000 --- a/pkg/identity/keystone/keystone.go +++ /dev/null @@ -1,613 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package keystone - -import ( - "context" - "crypto/tls" - "encoding/json" - "fmt" - "net/http" - "time" - - "github.com/gophercloud/gophercloud" - "github.com/gophercloud/gophercloud/openstack" - "github.com/gophercloud/gophercloud/openstack/utils" - "github.com/gorilla/mux" - "github.com/spf13/pflag" - "gopkg.in/yaml.v2" - apiv1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - netutil "k8s.io/apimachinery/pkg/util/net" - runtimeutil "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/apimachinery/pkg/util/wait" - k8suser "k8s.io/apiserver/pkg/authentication/user" - "k8s.io/apiserver/pkg/authorization/authorizer" - "k8s.io/client-go/informers" - "k8s.io/client-go/kubernetes" - corelisters "k8s.io/client-go/listers/core/v1" - "k8s.io/client-go/tools/cache" - "k8s.io/client-go/tools/clientcmd" - certutil "k8s.io/client-go/util/cert" - "k8s.io/client-go/util/workqueue" - "k8s.io/cloud-provider-openstack/pkg/version" - "k8s.io/klog/v2" -) - -const ( - maxRetries = 5 - cmNamespace = "kube-system" - Roles = "alpha.kubernetes.io/identity/roles" - ProjectID = "alpha.kubernetes.io/identity/project/id" - ProjectName = "alpha.kubernetes.io/identity/project/name" - DomainID = "alpha.kubernetes.io/identity/user/domain/id" - DomainName = "alpha.kubernetes.io/identity/user/domain/name" -) - -var userAgentData []string - -// AddExtraFlags is called by the main package to add component specific command line flags -func AddExtraFlags(fs *pflag.FlagSet) { - fs.StringArrayVar(&userAgentData, "user-agent", nil, "Extra data to add to gophercloud user-agent. Use multiple times to add more than one component.") -} - -type userInfo struct { - Username string `json:"username"` - UID string `json:"uid"` - Groups []string `json:"groups"` - Extra map[string][]string `json:"extra"` -} - -type status struct { - Authenticated bool `json:"authenticated"` - User userInfo `json:"user"` -} - -// KeystoneAuth manages authentication and authorization -type KeystoneAuth struct { - authn *Authenticator - authz *Authorizer - k8sClient *kubernetes.Clientset - syncer *Syncer - config *Config - stopCh chan struct{} - queue workqueue.RateLimitingInterface - informer informers.SharedInformerFactory - cmLister corelisters.ConfigMapLister - cmListerSynced cache.InformerSynced -} - -// Run starts the keystone webhook server. -func (k *KeystoneAuth) Run() { - defer close(k.stopCh) - - if k.k8sClient != nil { - defer k.queue.ShutDown() - go k.informer.Start(k.stopCh) - - // wait for the caches to synchronize before starting the worker - if !cache.WaitForCacheSync(k.stopCh, k.cmListerSynced) { - runtimeutil.HandleError(fmt.Errorf("timed out waiting for caches to sync")) - return - } - klog.Info("ConfigMaps synced and ready") - - go wait.Until(k.runWorker, time.Second, k.stopCh) - } - - r := mux.NewRouter() - r.HandleFunc("/webhook", k.Handler) - - klog.Infof("Starting webhook server...") - klog.Fatal(http.ListenAndServeTLS(k.config.Address, k.config.CertFile, k.config.KeyFile, r)) -} - -func (k *KeystoneAuth) enqueueConfigMap(obj interface{}) { - // obj could be an *v1.ConfigMap, or a DeletionFinalStateUnknown marker item. - key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj) - if err != nil { - klog.Errorf("Failed to get key for object: %v", err) - return - } - namespace, name, err := cache.SplitMetaNamespaceKey(key) - if err != nil { - klog.Errorf("Failed to get namespace and name for the key %s: %v", key, err) - return - } - - if namespace == cmNamespace && (name == k.config.PolicyConfigMapName || name == k.config.SyncConfigMapName) { - k.queue.Add(key) - } -} - -func (k *KeystoneAuth) runWorker() { - for k.processNextItem() { - // continue looping - } -} - -func (k *KeystoneAuth) processNextItem() bool { - key, quit := k.queue.Get() - - if quit { - return false - } - defer k.queue.Done(key) - - err := k.processItem(key.(string)) - if err == nil { - // No error, reset the ratelimit counters - k.queue.Forget(key) - } else if k.queue.NumRequeues(key) < maxRetries { - klog.Errorf("Failed to process key %s (will retry): %v", key, err) - k.queue.AddRateLimited(key) - } else { - // err != nil and too many retries - klog.Errorf("Failed to process key %s (giving up): %v", key, err) - k.queue.Forget(key) - runtimeutil.HandleError(err) - } - - return true -} - -func (k *KeystoneAuth) updatePolicies(cm *apiv1.ConfigMap, key string) { - klog.Info("ConfigMap created or updated, will update the authorization policy.") - - var policy policyList - if err := json.Unmarshal([]byte(cm.Data["policies"]), &policy); err != nil { - runtimeutil.HandleError(fmt.Errorf("failed to parse policies defined in the configmap %s: %v", key, err)) - } - if len(policy) > 0 { - if _, err := json.MarshalIndent(policy, "", " "); err != nil { - runtimeutil.HandleError(fmt.Errorf("failed to parse policies defined in the configmap %s: %v", key, err)) - } - } - - k.authz.mu.Lock() - k.authz.pl = policy - k.authz.mu.Unlock() - - klog.Infof("Authorization policy updated.") -} - -func (k *KeystoneAuth) updateSyncConfig(cm *apiv1.ConfigMap, key string) { - klog.Info("ConfigMap created or updated, will update the sync configuration.") - - var sc *syncConfig - newConfig := newSyncConfig() - sc = &newConfig - if err := yaml.Unmarshal([]byte(cm.Data["syncConfig"]), sc); err != nil { - runtimeutil.HandleError(fmt.Errorf("failed to parse sync config defined in the configmap %s: %v", key, err)) - } - - k.syncer.mu.Lock() - k.syncer.syncConfig = sc - k.syncer.mu.Unlock() - - klog.Infof("Sync configuration updated.") -} - -func (k *KeystoneAuth) processItem(key string) error { - namespace, name, err := cache.SplitMetaNamespaceKey(key) - if err != nil { - return err - } - - cm, err := k.cmLister.ConfigMaps(namespace).Get(name) - switch { - case errors.IsNotFound(err): - if name == k.config.PolicyConfigMapName { - klog.Infof("PolicyConfigmap %v has been deleted.", k.config.PolicyConfigMapName) - k.authz.mu.Lock() - k.authz.pl = make([]*policy, 0) - k.authz.mu.Unlock() - } - if name == k.config.SyncConfigMapName { - klog.Infof("SyncConfigmap %v has been deleted.", k.config.SyncConfigMapName) - k.syncer.mu.Lock() - sc := newSyncConfig() - k.syncer.syncConfig = &sc - k.syncer.mu.Unlock() - } - case err != nil: - return fmt.Errorf("error fetching object with key %s: %v", key, err) - default: - if name == k.config.PolicyConfigMapName { - k.updatePolicies(cm, key) - } - if name == k.config.SyncConfigMapName { - k.updateSyncConfig(cm, key) - } - } - - return nil -} - -// Handler serves the http requests -func (k *KeystoneAuth) Handler(w http.ResponseWriter, r *http.Request) { - var data map[string]interface{} - decoder := json.NewDecoder(r.Body) - defer r.Body.Close() - err := decoder.Decode(&data) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - var apiVersion = data["apiVersion"].(string) - var kind = data["kind"].(string) - - if apiVersion != "authentication.k8s.io/v1beta1" && apiVersion != "authorization.k8s.io/v1beta1" { - http.Error(w, fmt.Sprintf("unknown apiVersion %q", apiVersion), http.StatusBadRequest) - return - } - - if kind == "TokenReview" { - var token = data["spec"].(map[string]interface{})["token"].(string) - userInfo := k.authenticateToken(w, r, token, data) - - // Do synchronization - // In the case of unscoped tokens, when project id is not defined, we have to skip this part - if k.syncer.syncConfig != nil && len(k.syncer.syncConfig.DataTypesToSync) > 0 && userInfo != nil && len(userInfo.Extra[ProjectID]) != 0 { - err = k.syncer.syncData(userInfo) - if err != nil { - klog.Errorf("an error occurred during data synchronization: %v", err) - } - } - } else if kind == "SubjectAccessReview" { - k.authorizeToken(w, r, data) - } else { - http.Error(w, fmt.Sprintf("unknown kind/apiVersion %q %q", kind, apiVersion), http.StatusBadRequest) - } -} - -func (k *KeystoneAuth) authenticateToken(w http.ResponseWriter, r *http.Request, token string, data map[string]interface{}) *userInfo { - user, authenticated, err := k.authn.AuthenticateToken(token) - klog.V(4).Infof("authenticateToken : %v, %v, %v\n", token, user, err) - - if !authenticated { - var response status - response.Authenticated = false - data["status"] = response - - output, err := json.MarshalIndent(data, "", " ") - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return nil - } - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusUnauthorized) - w.Write(output) - return nil - } - - var info userInfo - info.Username = user.GetName() - info.UID = user.GetUID() - info.Groups = user.GetGroups() - info.Extra = user.GetExtra() - - var response status - response.Authenticated = true - - // Modify user info accoding to the sync configuration. - response.User = *k.syncer.syncRoles(&info) - - data["status"] = response - - output, err := json.MarshalIndent(data, "", " ") - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return nil - } - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - w.Write(output) - - return &info -} - -func (k *KeystoneAuth) authorizeToken(w http.ResponseWriter, r *http.Request, data map[string]interface{}) { - output, err := json.MarshalIndent(data, "", " ") - klog.V(4).Infof("authorizeToken data : %s\n", string(output)) - - spec := data["spec"].(map[string]interface{}) - - username := spec["user"] - usr := &k8suser.DefaultInfo{Name: username.(string)} - attrs := authorizer.AttributesRecord{User: usr} - - groups := spec["group"].([]interface{}) - for _, v := range groups { - usr.Groups = append(usr.Groups, v.(string)) - } - if extras, ok := spec["extra"].(map[string]interface{}); ok { - usr.Extra = make(map[string][]string, len(extras)) - for key, value := range extras { - for _, v := range value.([]interface{}) { - if data, ok := usr.Extra[key]; ok { - usr.Extra[key] = append(data, v.(string)) - } else { - usr.Extra[key] = []string{v.(string)} - } - } - } - } - - if resourceAttributes, ok := spec["resourceAttributes"]; ok { - v := resourceAttributes.(map[string]interface{}) - attrs.ResourceRequest = true - attrs.Verb = getField(v, "verb") - attrs.Namespace = getField(v, "namespace") - attrs.APIGroup = getField(v, "group") - attrs.APIVersion = getField(v, "version") - attrs.Resource = getField(v, "resource") - attrs.Subresource = getField(v, "subresource") - attrs.Name = getField(v, "name") - } else if nonResourceAttributes, ok := spec["nonResourceAttributes"]; ok { - v := nonResourceAttributes.(map[string]interface{}) - attrs.ResourceRequest = false - attrs.Verb = getField(v, "verb") - attrs.Path = getField(v, "path") - } else { - err := fmt.Errorf("unable to find attributes") - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - var allowed authorizer.Decision - if len(k.authz.pl) > 0 { - var reason string - allowed, reason, err = k.authz.Authorize(attrs) - klog.V(4).Infof("<<<< authorizeToken: %v, %v, %v\n", allowed, reason, err) - if err != nil { - http.Error(w, reason, http.StatusInternalServerError) - return - } - } else { - // The operator didn't set authorization policy, deny by default. - allowed = authorizer.DecisionDeny - } - - delete(data, "spec") - data["status"] = map[string]interface{}{ - "allowed": allowed == authorizer.DecisionAllow, - } - output, err = json.MarshalIndent(data, "", " ") - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - w.Write(output) -} - -// NewKeystoneAuth returns a new KeystoneAuth controller -func NewKeystoneAuth(c *Config) (*KeystoneAuth, error) { - keystoneClient, err := createKeystoneClient(c.KeystoneURL, c.KeystoneCA) - if err != nil { - return nil, fmt.Errorf("failed to initialize keystone client: %v", err) - } - - var k8sClient *kubernetes.Clientset - if c.PolicyConfigMapName != "" || c.SyncConfigMapName != "" || c.SyncConfigFile != "" { - k8sClient, err = createKubernetesClient(c.Kubeconfig) - if err != nil { - return nil, fmt.Errorf("failed to get kubernetes client: %v", err) - } - } - - // Get policy definition either from a policy file or the policy configmap. Policy file takes precedence - // over the configmap, but the policy definition will be refreshed based on the configmap change on-the-fly. It - // is possible that both are not provided, in this case, the keytone webhook authorization will always return deny. - var policy policyList - if c.PolicyConfigMapName != "" { - cm, err := k8sClient.CoreV1().ConfigMaps(cmNamespace).Get(context.TODO(), c.PolicyConfigMapName, metav1.GetOptions{}) - if err != nil { - return nil, fmt.Errorf("failed to get configmap %s: %v", c.PolicyConfigMapName, err) - } - - if err := json.Unmarshal([]byte(cm.Data["policies"]), &policy); err != nil { - return nil, fmt.Errorf("failed to parse policies defined in the configmap %s: %v", c.PolicyConfigMapName, err) - } - } - if c.PolicyFile != "" { - policy, err = newFromFile(c.PolicyFile) - if err != nil { - return nil, fmt.Errorf("failed to extract policy from policy file %s: %v", c.PolicyFile, err) - } - } - - if len(policy) > 0 { - output, err := json.MarshalIndent(policy, "", " ") - if err == nil { - klog.V(4).Infof("Policy %s", string(output)) - } else { - return nil, err - } - } - - // Get sync config either from a sync config file or the sync configmap. Sync config file takes precedence - // over the configmap, but the sync config definition will be refreshed based on the configmap change on-the-fly. It - // is possible that both are not provided, in this case, the keytone webhook authenticator will not synchronize data. - var sc *syncConfig - if c.SyncConfigMapName != "" { - cm, err := k8sClient.CoreV1().ConfigMaps(cmNamespace).Get(context.TODO(), c.SyncConfigMapName, metav1.GetOptions{}) - if err != nil { - klog.Errorf("configmap get err #%v ", err) - return nil, fmt.Errorf("failed to get configmap %s: %v", c.SyncConfigMapName, err) - } - - newConfig := newSyncConfig() - sc = &newConfig - if err := yaml.Unmarshal([]byte(cm.Data["syncConfig"]), sc); err != nil { - klog.Errorf("Unmarshal: %v", err) - return nil, fmt.Errorf("failed to parse sync config defined in the configmap %s: %v", c.SyncConfigMapName, err) - } - } - if c.SyncConfigFile != "" { - sc, err = newSyncConfigFromFile(c.SyncConfigFile) - if err != nil { - return nil, fmt.Errorf("failed to extract data from sync config file %s: %v", c.SyncConfigFile, err) - } - } - if sc != nil { - // Validate that config data is correct - sc.validate() - } - - keystoneAuth := &KeystoneAuth{ - authn: &Authenticator{keystoner: NewKeystoner(keystoneClient)}, - authz: &Authorizer{authURL: c.KeystoneURL, client: keystoneClient, pl: policy}, - syncer: &Syncer{k8sClient: k8sClient, syncConfig: sc}, - k8sClient: k8sClient, - config: c, - stopCh: make(chan struct{}), - } - - if k8sClient != nil { - queue := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()) - kubeInformerFactory := informers.NewSharedInformerFactory(k8sClient, time.Minute*5) - cmInformer := kubeInformerFactory.Core().V1().ConfigMaps() - cmInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: keystoneAuth.enqueueConfigMap, - UpdateFunc: func(old, new interface{}) { - newIng := new.(*apiv1.ConfigMap) - oldIng := old.(*apiv1.ConfigMap) - if newIng.ResourceVersion == oldIng.ResourceVersion { - // Periodic resync will send update events for all known ConfigMaps. - // Two different versions of the same ConfigMap will always have different RVs. - return - } - keystoneAuth.enqueueConfigMap(new) - }, - DeleteFunc: keystoneAuth.enqueueConfigMap, - }) - - keystoneAuth.informer = kubeInformerFactory - keystoneAuth.cmLister = cmInformer.Lister() - keystoneAuth.cmListerSynced = cmInformer.Informer().HasSynced - keystoneAuth.queue = queue - } - - return keystoneAuth, nil -} - -func getField(data map[string]interface{}, name string) string { - if v, ok := data[name]; ok { - return v.(string) - } - return "" -} - -// Construct a Keystone v3 client, bail out if we cannot find the v3 API endpoint -func createIdentityV3Provider(options gophercloud.AuthOptions, transport http.RoundTripper) (*gophercloud.ProviderClient, error) { - client, err := openstack.NewClient(options.IdentityEndpoint) - if err != nil { - return nil, err - } - - if transport != nil { - client.HTTPClient.Transport = transport - } - - versions := []*utils.Version{ - {ID: "v3", Priority: 30, Suffix: "/v3/"}, - } - chosen, _, err := utils.ChooseVersion(client, versions) - if err != nil { - return nil, fmt.Errorf("Unable to find identity API v3 version : %v", err) - } - - switch chosen.ID { - case "v3": - return client, nil - default: - // The switch statement must be out of date from the versions list. - return nil, fmt.Errorf("Unsupported identity API version: %s", chosen.ID) - } -} - -func createKubernetesClient(kubeConfig string) (*kubernetes.Clientset, error) { - klog.Info("Creating kubernetes API client.") - - cfg, err := clientcmd.BuildConfigFromFlags("", kubeConfig) - if err != nil { - return nil, err - } - - client, err := kubernetes.NewForConfig(cfg) - if err != nil { - return nil, err - } - - v, err := client.Discovery().ServerVersion() - if err != nil { - return nil, err - } - - klog.Infof("Kubernetes API client created, server version %s", fmt.Sprintf("v%v.%v", v.Major, v.Minor)) - return client, nil -} - -func createKeystoneClient(authURL string, caFile string) (*gophercloud.ServiceClient, error) { - // FIXME: Enable this check later - //if !strings.HasPrefix(authURL, "https") { - // return nil, errors.New("Auth URL should be secure and start with https") - //} - var transport http.RoundTripper - if authURL == "" { - return nil, fmt.Errorf("auth URL is empty") - } - if caFile != "" { - roots, err := certutil.NewPool(caFile) - if err != nil { - return nil, err - } - config := &tls.Config{} - config.RootCAs = roots - transport = netutil.SetOldTransportDefaults(&http.Transport{TLSClientConfig: config}) - } - opts := gophercloud.AuthOptions{IdentityEndpoint: authURL} - provider, err := createIdentityV3Provider(opts, transport) - if err != nil { - return nil, err - } - - userAgent := gophercloud.UserAgent{} - userAgent.Prepend(fmt.Sprintf("k8s-keystone-auth/%s", version.Version)) - for _, data := range userAgentData { - userAgent.Prepend(data) - } - provider.UserAgent = userAgent - klog.V(4).Infof("Using user-agent %s", userAgent.Join()) - - // We should use the V3 API - client, err := openstack.NewIdentityV3(provider, gophercloud.EndpointOpts{}) - if err != nil { - klog.Warningf("Failed: Unable to use keystone v3 identity service: %v", err) - return nil, fmt.Errorf("failed to authenticate") - } - - // Make sure we look under /v3 for resources - client.IdentityBase = client.IdentityEndpoint - client.Endpoint = client.IdentityEndpoint - return client, nil -} diff --git a/pkg/identity/keystone/keystone_test.go b/pkg/identity/keystone/keystone_test.go deleted file mode 100644 index 64da341b1a..0000000000 --- a/pkg/identity/keystone/keystone_test.go +++ /dev/null @@ -1,61 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package keystone - -import ( - "reflect" - "testing" - - "github.com/spf13/pflag" -) - -func TestUserAgentFlag(t *testing.T) { - tests := []struct { - name string - shouldParse bool - flags []string - expected []string - }{ - {"no_flag", true, []string{}, nil}, - {"one_flag", true, []string{"--user-agent=cluster/abc-123"}, []string{"cluster/abc-123"}}, - {"multiple_flags", true, []string{"--user-agent=a/b", "--user-agent=c/d"}, []string{"a/b", "c/d"}}, - {"flag_with_space", true, []string{"--user-agent=a b"}, []string{"a b"}}, - {"flag_split_with_space", true, []string{"--user-agent=a", "b"}, []string{"a"}}, - {"empty_flag", false, []string{"--user-agent"}, nil}, - } - - for _, testCase := range tests { - userAgentData = []string{} - - t.Run(testCase.name, func(t *testing.T) { - fs := pflag.NewFlagSet("test", pflag.ContinueOnError) - AddExtraFlags(fs) - - err := fs.Parse(testCase.flags) - - if testCase.shouldParse && err != nil { - t.Errorf("Flags failed to parse") - } else if !testCase.shouldParse && err == nil { - t.Errorf("Flags should not have parsed") - } else if testCase.shouldParse { - if !reflect.DeepEqual(userAgentData, testCase.expected) { - t.Errorf("userAgentData %#v did not match expected value %#v", userAgentData, testCase.expected) - } - } - }) - } -} diff --git a/pkg/identity/keystone/mock_IKeystone.go b/pkg/identity/keystone/mock_IKeystone.go deleted file mode 100644 index 6f1d457423..0000000000 --- a/pkg/identity/keystone/mock_IKeystone.go +++ /dev/null @@ -1,56 +0,0 @@ -// Code generated by mockery v1.0.0. DO NOT EDIT. - -package keystone - -import mock "github.com/stretchr/testify/mock" - -// MockIKeystone is an autogenerated mock type for the IKeystone type -type MockIKeystone struct { - mock.Mock -} - -// GetGroups provides a mock function with given fields: _a0, _a1 -func (_m *MockIKeystone) GetGroups(_a0 string, _a1 string) ([]string, error) { - ret := _m.Called(_a0, _a1) - - var r0 []string - if rf, ok := ret.Get(0).(func(string, string) []string); ok { - r0 = rf(_a0, _a1) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]string) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(string, string) error); ok { - r1 = rf(_a0, _a1) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// GetTokenInfo provides a mock function with given fields: _a0 -func (_m *MockIKeystone) GetTokenInfo(_a0 string) (*tokenInfo, error) { - ret := _m.Called(_a0) - - var r0 *tokenInfo - if rf, ok := ret.Get(0).(func(string) *tokenInfo); ok { - r0 = rf(_a0) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*tokenInfo) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(string) error); ok { - r1 = rf(_a0) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} diff --git a/pkg/identity/keystone/policy.go b/pkg/identity/keystone/policy.go deleted file mode 100644 index ca4a805100..0000000000 --- a/pkg/identity/keystone/policy.go +++ /dev/null @@ -1,101 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package keystone - -import ( - "bufio" - "encoding/json" - "os" -) - -type policy struct { - ResourceSpec *resourcePolicySpec `json:"resource,omitempty"` - - NonResourceSpec *nonResourcePolicySpec `json:"nonresource,omitempty"` - - Match []policyMatch `json:"match"` - - ResourcePermissionsSpec map[string][]string `json:"resource_permissions,omitempty"` - - NonResourcePermissionsSpec map[string][]string `json:"nonresource_permissions,omitempty"` - - Users map[string][]string `json:"users"` -} - -// Supported types for policy match. -const ( - TypeUser string = "user" - TypeGroup string = "group" - TypeProject string = "project" - TypeRole string = "role" -) - -type policyMatch struct { - Type string `json:"type"` - - Values []string `json:"values"` -} - -type resourcePolicySpec struct { - // Kubernetes resource API verb like: get, list, watch, create, update, delete, proxy. - // ["*"] matches all verbs. - Verbs []string `json:"verbs"` - - // Resources is the list of resource names. - // ["*"] matches all resources - Resources []string `json:"resources"` - - // APIGroup is the name of an API group. - // "*" matches all API groups - APIGroup *string `json:"version"` - - // Namespace is the name of a namespace. - // "*" matches all namespaces (including unnamespaced requests) - Namespace *string `json:"namespace"` -} - -type nonResourcePolicySpec struct { - // Kubernetes resource API verb like: get, list, watch, create, update, delete, proxy. - // "*" matches all verbs. - Verbs []string `json:"verbs"` - - // NonResourcePath matches non-resource request paths. - // "*" matches all paths - // "/foo/*" matches all subpaths of foo - NonResourcePath *string `json:"path"` -} - -type policyList []*policy - -// newFromFile loads a list of policies from a file -func newFromFile(path string) (policyList, error) { - file, err := os.Open(path) - if err != nil { - return nil, err - } - defer file.Close() - - var data policyList - - reader := bufio.NewReader(file) - decoder := json.NewDecoder(reader) - err = decoder.Decode(&data) - if err != nil { - return nil, err - } - return data, nil -} diff --git a/pkg/identity/keystone/sync.go b/pkg/identity/keystone/sync.go deleted file mode 100644 index 56e5302a4b..0000000000 --- a/pkg/identity/keystone/sync.go +++ /dev/null @@ -1,329 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package keystone - -import ( - "context" - "errors" - "fmt" - "io/ioutil" - "regexp" - "strings" - "sync" - - "gopkg.in/yaml.v2" - corev1 "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" - k8serrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - "k8s.io/klog/v2" - - cpoutil "k8s.io/cloud-provider-openstack/pkg/util" -) - -const ( - Projects = "projects" - RoleAssignments = "role_assignments" -) - -var allowedDataTypesToSync = []string{Projects, RoleAssignments} - -type roleMap struct { - KeystoneRole string `yaml:"keystone-role"` - Username string `yaml:"username"` - Groups []string `yaml:"groups"` -} - -// syncConfig contains configuration data for synchronization between Keystone and Kubernetes -type syncConfig struct { - // List containing possible data types to sync. Now only "projects" are supported. - DataTypesToSync []string `yaml:"data-types-to-sync"` - - // Format of automatically created namespace name. Can contain wildcards %i and %n, - // corresponding to project id and project name respectively. - NamespaceFormat string `yaml:"namespace-format"` - - // List of project ids to exclude from syncing. - ProjectBlackList []string `yaml:"projects-blacklist"` - - // List of project names to exclude from syncing. - ProjectNameBlackList []string `yaml:"projects-name-blacklist"` - - // List of role mappings that will apply to the user info after authentication. - RoleMaps []*roleMap `yaml:"role-mappings"` -} - -func (sc *syncConfig) validate() error { - if sc.NamespaceFormat != "" { - // Namespace name must contain keystone project id - if !strings.Contains(sc.NamespaceFormat, "%i") { - return fmt.Errorf("format string should comprise a %%i substring (keystone project id)") - } - - // By convention, the names should be up to maximum length of 63 characters and consist of - // lower and upper case alphanumeric characters, -, _ and . - ts := strings.Replace(sc.NamespaceFormat, "%i", "aa", -1) - ts = strings.Replace(ts, "%n", "aa", -1) - ts = strings.Replace(ts, "%d", "aa", -1) - - re := regexp.MustCompile("^[a-zA-Z0-9][a-zA-Z0-9_.-]*[a-zA-Z0-9]$") - if !re.MatchString(ts) { - return fmt.Errorf("namespace name must consist of alphanumeric characters, '-', '_' or '.', and must start and end with an alphanumeric character") - } - } - - // Check that only allowed data types are enabled for synchronization - for _, dt := range sc.DataTypesToSync { - var flag bool - for _, a := range allowedDataTypesToSync { - if a == dt { - flag = true - break - } - } - if !flag { - return fmt.Errorf( - "Unsupported data type to sync: %v. Available values: %v", - dt, - strings.Join(allowedDataTypesToSync, ","), - ) - } - } - - return nil -} - -// formatNamespaceName generates a namespace name, based on format string -func (sc *syncConfig) formatNamespaceName(id string, name string, domain string) string { - res := strings.Replace(sc.NamespaceFormat, "%i", id, -1) - res = strings.Replace(res, "%n", name, -1) - res = strings.Replace(res, "%d", domain, -1) - - if len(res) > 63 { - klog.Warningf("Generated namespace name '%v' exceeds the maximum possible length of 63 characters. Just Keystone project id '%v' will be used as the namespace name.", res, id) - return id - } - - return res -} - -// newSyncConfig defines the default values for syncConfig -func newSyncConfig() syncConfig { - return syncConfig{ - // by default namespace name is a string containing just keystone project id - NamespaceFormat: "%i", - } -} - -// newSyncConfigFromFile loads a sync config from a file -func newSyncConfigFromFile(path string) (*syncConfig, error) { - sc := newSyncConfig() - - yamlFile, err := ioutil.ReadFile(path) - if err != nil { - klog.Errorf("yamlFile get err #%v ", err) - return nil, err - } - err = yaml.Unmarshal(yamlFile, &sc) - if err != nil { - klog.Errorf("Unmarshal: %v", err) - return nil, err - } - - return &sc, nil -} - -// Syncer synchronizes auth data between Keystone and Kubernetes -type Syncer struct { - k8sClient *kubernetes.Clientset - syncConfig *syncConfig - mu sync.Mutex -} - -func (s *Syncer) syncData(u *userInfo) error { - s.mu.Lock() - defer s.mu.Unlock() - - for _, p := range s.syncConfig.ProjectBlackList { - if u.Extra[ProjectID][0] == p { - klog.Infof("Project %v is in black list. Skipping.", p) - return nil - } - } - - for _, p := range s.syncConfig.ProjectNameBlackList { - if u.Extra[ProjectName][0] == p { - klog.Infof("Project %v is in black list. Skipping.", p) - return nil - } - } - - if s.k8sClient == nil { - return errors.New("cannot sync data because k8s client is not initialized") - } - - namespaceName := s.syncConfig.formatNamespaceName( - u.Extra[ProjectID][0], - u.Extra[ProjectName][0], - u.Extra[DomainID][0], - ) - - // sync project data first - for _, dataType := range s.syncConfig.DataTypesToSync { - if dataType == Projects { - err := s.syncProjectData(u, namespaceName) - if err != nil { - return err - } - } - } - - for _, dataType := range s.syncConfig.DataTypesToSync { - if dataType == RoleAssignments { - err := s.syncRoleAssignmentsData(u, namespaceName) - if err != nil { - return err - } - } - } - - return nil -} - -func (s *Syncer) syncProjectData(u *userInfo, namespaceName string) error { - _, err := s.k8sClient.CoreV1().Namespaces().Get(context.TODO(), namespaceName, metav1.GetOptions{}) - - if k8serrors.IsNotFound(err) { - // The required namespace is not found. Create it then. - namespace := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: namespaceName, - }, - } - namespace, err = s.k8sClient.CoreV1().Namespaces().Create(context.TODO(), namespace, metav1.CreateOptions{}) - if err != nil { - klog.Warningf("Cannot create a namespace for the user: %v", err) - return errors.New("internal server error") - } - } else if err != nil { - // Some other error. - klog.Warningf("Cannot get a response from the server: %v", err) - return errors.New("internal server error") - } - - return nil -} - -func (s *Syncer) syncRoleAssignmentsData(u *userInfo, namespaceName string) error { - // TODO(mfedosin): add a field separator to filter out unnecessary roles bindings at an early stage - roleBindings, err := s.k8sClient.RbacV1().RoleBindings(namespaceName).List(context.TODO(), metav1.ListOptions{}) - if err != nil { - klog.Warningf("Cannot get a list of role bindings from the server: %v", err) - return errors.New("internal server error") - } - - // delete role bindings removed from Keystone - for _, roleBinding := range roleBindings.Items { - // parts[0] is a user id, parts[1] is a role name - parts := strings.SplitN(roleBinding.Name, "_", 2) - if len(parts) == 1 || parts[0] != u.UID { - // role binding is either created by an admin or belongs to a different user - continue - } - - var keepRoleBinding bool - for _, roleName := range u.Extra[Roles] { - roleBindingName := u.UID + "_" + roleName - if roleBinding.Name == roleBindingName { - keepRoleBinding = true - } - } - if !keepRoleBinding { - err = s.k8sClient.RbacV1().RoleBindings(namespaceName).Delete(context.TODO(), roleBinding.Name, metav1.DeleteOptions{}) - if err != nil { - klog.Warningf("Cannot delete a role binding from the server: %v", err) - return errors.New("internal server error") - } - } - - } - - // create new role bindings - for _, roleName := range u.Extra[Roles] { - roleBindingName := u.UID + "_" + roleName - - // check that role binding doesn't exist - var roleBindingExists bool - for _, roleBinding := range roleBindings.Items { - if roleBindingName == roleBinding.Name { - roleBindingExists = true - break - } - } - if roleBindingExists { - continue - } - - roleBinding := &rbacv1.RoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: roleBindingName, - }, - Subjects: []rbacv1.Subject{ - { - APIGroup: "rbac.authorization.k8s.io", - Kind: "User", - Name: u.Username, - }, - }, - RoleRef: rbacv1.RoleRef{ - APIGroup: "rbac.authorization.k8s.io", - Kind: "ClusterRole", - Name: roleName, - }, - } - roleBinding, err = s.k8sClient.RbacV1().RoleBindings(namespaceName).Create(context.TODO(), roleBinding, metav1.CreateOptions{}) - if err != nil { - klog.Warningf("Cannot create a role binding for the user: %v", err) - return errors.New("internal server error") - } - } - - return nil -} - -// syncRoles modifies the user attributes according to the config. -func (s *Syncer) syncRoles(user *userInfo) *userInfo { - if s.syncConfig == nil || len(s.syncConfig.RoleMaps) == 0 { - return user - } - - if roles, isPresent := user.Extra[Roles]; isPresent { - for _, roleMap := range s.syncConfig.RoleMaps { - if roleMap.KeystoneRole != "" && cpoutil.Contains(roles, roleMap.KeystoneRole) { - if len(roleMap.Groups) > 0 { - user.Groups = append(user.Groups, roleMap.Groups...) - } - if roleMap.Username != "" { - user.Username = roleMap.Username - } - } - } - } - - return user -} diff --git a/pkg/identity/keystone/sync_test.go b/pkg/identity/keystone/sync_test.go deleted file mode 100644 index 1d95530331..0000000000 --- a/pkg/identity/keystone/sync_test.go +++ /dev/null @@ -1,156 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package keystone - -import ( - "fmt" - "strings" - "testing" - - th "github.com/gophercloud/gophercloud/testhelper" -) - -func TestSyncConfigFromFile(t *testing.T) { - sc, err := newSyncConfigFromFile("sync_test.yaml") - th.AssertNoErr(t, err) - th.AssertEquals(t, "prefix-%d-%n-%i-suffix", sc.NamespaceFormat) - th.AssertEquals(t, "id1", sc.ProjectBlackList[0]) - th.AssertEquals(t, "id2", sc.ProjectBlackList[1]) - th.AssertEquals(t, "name1", sc.ProjectNameBlackList[0]) - th.AssertEquals(t, "name2", sc.ProjectNameBlackList[1]) - th.AssertEquals(t, 1, len(sc.RoleMaps)) - th.AssertEquals(t, "_member_", sc.RoleMaps[0].KeystoneRole) - th.AssertEquals(t, "myuser", sc.RoleMaps[0].Username) - th.AssertEquals(t, 1, len(sc.RoleMaps[0].Groups)) - th.AssertEquals(t, "mygroup", sc.RoleMaps[0].Groups[0]) -} - -func TestSyncConfigValidation(t *testing.T) { - // Default sync config - sc := newSyncConfig() - err := sc.validate() - th.AssertNoErr(t, err) - - // Forbidden characters in the format string - sc.NamespaceFormat = strings.Join([]string{"%i", "!@#$"}, "") - err = sc.validate() - th.AssertEquals( - t, - "namespace name must consist of alphanumeric characters, '-', '_' or '.', and must start and end with an alphanumeric character", - err.Error(), - ) - - // Format string starts with "-" - sc.NamespaceFormat = "-%i" - err = sc.validate() - th.AssertEquals( - t, - "namespace name must consist of alphanumeric characters, '-', '_' or '.', and must start and end with an alphanumeric character", - err.Error(), - ) - - // NamespaceFormat string doesn't have project id - sc.NamespaceFormat = "%n" - err = sc.validate() - th.AssertEquals(t, "format string should comprise a %i substring (keystone project id)", err.Error()) - - sc = newSyncConfig() - - // DataTypesToSync must contain only allowed types - sc.DataTypesToSync = []string{"not_allowed_type"} - err = sc.validate() - th.AssertEquals( - t, - fmt.Sprintf( - "Unsupported data type to sync: not_allowed_type. Available values: %v", - strings.Join(allowedDataTypesToSync, ","), - ), - err.Error(), - ) -} - -func TestSyncRoles(t *testing.T) { - sc, err := newSyncConfigFromFile("sync_test.yaml") - th.AssertNoErr(t, err) - - syncer := Syncer{ - k8sClient: nil, - syncConfig: sc, - } - - fakeName := "fake-user" - fakeID := "b4db78f0-4dd7-41cf-8475-203c34230dc0" - fakeGroups := []string{"_member_", "kube_viewer"} - user1 := &userInfo{ - Username: fakeName, - UID: fakeID, - Groups: []string{fakeID}, - Extra: map[string][]string{Roles: fakeGroups}, - } - - userModified := syncer.syncRoles(user1) - - th.AssertEquals(t, "myuser", userModified.Username) - expectedGroups := []string{fakeID, "mygroup"} - th.AssertDeepEquals(t, expectedGroups, userModified.Groups) -} - -func TestSyncRolesSkipNilConfig(t *testing.T) { - syncer := Syncer{ - k8sClient: nil, - syncConfig: nil, - } - - fakeName := "fake-user" - fakeID := "b4db78f0-4dd7-41cf-8475-203c34230dc0" - fakeGroups := []string{"_member_", "kube_viewer"} - user1 := &userInfo{ - Username: fakeName, - UID: fakeID, - Groups: []string{fakeID}, - Extra: map[string][]string{Roles: fakeGroups}, - } - - userModified := syncer.syncRoles(user1) - - th.AssertEquals(t, userModified, user1) -} - -func TestSyncRolesSkipm(t *testing.T) { - sc, err := newSyncConfigFromFile("sync_test.yaml") - th.AssertNoErr(t, err) - - sc.RoleMaps = []*roleMap{} - syncer := Syncer{ - k8sClient: nil, - syncConfig: sc, - } - - fakeName := "fake-user" - fakeID := "b4db78f0-4dd7-41cf-8475-203c34230dc0" - fakeGroups := []string{"_member_", "kube_viewer"} - user1 := &userInfo{ - Username: fakeName, - UID: fakeID, - Groups: []string{fakeID}, - Extra: map[string][]string{Roles: fakeGroups}, - } - - userModified := syncer.syncRoles(user1) - - th.AssertEquals(t, userModified, user1) -} diff --git a/pkg/identity/keystone/sync_test.yaml b/pkg/identity/keystone/sync_test.yaml deleted file mode 100644 index 94351e2441..0000000000 --- a/pkg/identity/keystone/sync_test.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# In format %d, %n and %i wildcards represent keystone domain id, project name and project id respectively -namespace-format: "prefix-%d-%n-%i-suffix" - -# List of Keystone project ids to omit from syncing -projects-blacklist: ["id1", "id2"] - -# List of Keystone project names to omit from syncing -projects-name-blacklist: ["name1", "name2"] - -# List of data types to synchronize -"data-types-to-sync": ["projects", "role_assignments"] - -"role-mappings": - - keystone-role: _member_ - username: myuser - groups: ["mygroup"] \ No newline at end of file diff --git a/pkg/identity/keystone/token_getter.go b/pkg/identity/keystone/token_getter.go deleted file mode 100644 index 6607e62a0e..0000000000 --- a/pkg/identity/keystone/token_getter.go +++ /dev/null @@ -1,126 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package keystone - -import ( - "crypto/tls" - "fmt" - "github.com/gophercloud/gophercloud" - "github.com/gophercloud/gophercloud/openstack" - tokens3 "github.com/gophercloud/gophercloud/openstack/identity/v3/tokens" - "github.com/gophercloud/utils/client" - "io/ioutil" - certutil "k8s.io/client-go/util/cert" - cpo "k8s.io/cloud-provider-openstack/pkg/cloudprovider/providers/openstack" - "k8s.io/cloud-provider-openstack/pkg/version" - "k8s.io/klog/v2" - "net/http" -) - -type Options struct { - AuthOptions gophercloud.AuthOptions - ClientCertPath string - ClientKeyPath string - ClientCAPath string -} - -// GetToken creates a token by authenticate with keystone. -func GetToken(options Options) (*tokens3.Token, error) { - var token *tokens3.Token - var setTransport bool - - // Create new identity client - provider, err := openstack.NewClient(options.AuthOptions.IdentityEndpoint) - if err != nil { - msg := fmt.Errorf("failed: Initializing openstack authentication client: %v", err) - return token, msg - } - tlsConfig := &tls.Config{} - setTransport = false - - userAgent := gophercloud.UserAgent{} - userAgent.Prepend(fmt.Sprintf("client-keystone-auth/%s", version.Version)) - provider.UserAgent = userAgent - - if options.ClientCertPath != "" && options.ClientKeyPath != "" { - clientCert, err := ioutil.ReadFile(options.ClientCertPath) - if err != nil { - msg := fmt.Errorf("failed: Cannot read cert file: %v", err) - return token, msg - } - - clientKey, err := ioutil.ReadFile(options.ClientKeyPath) - if err != nil { - msg := fmt.Errorf("failed: Cannot read key file: %v", err) - return token, msg - } - - cert, err := tls.X509KeyPair([]byte(clientCert), []byte(clientKey)) - if err != nil { - msg := fmt.Errorf("failed: Cannot create keypair:: %v", err) - return token, msg - } - tlsConfig.Certificates = []tls.Certificate{cert} - tlsConfig.BuildNameToCertificate() - setTransport = true - } - - if options.ClientCAPath != "" { - roots, err := certutil.NewPool(options.ClientCAPath) - if err != nil { - msg := fmt.Errorf("failed: Cannot read CA file: %v", err) - return token, msg - } - - tlsConfig.RootCAs = roots - setTransport = true - } - - if setTransport { - transport := &http.Transport{Proxy: http.ProxyFromEnvironment, TLSClientConfig: tlsConfig} - provider.HTTPClient.Transport = transport - } - - if klog.V(6).Enabled() { - if provider.HTTPClient.Transport == nil { - provider.HTTPClient.Transport = http.DefaultTransport - } - provider.HTTPClient.Transport = &client.RoundTripper{ - Rt: provider.HTTPClient.Transport, - Logger: &cpo.Logger{}, - } - } - - v3Client, err := openstack.NewIdentityV3(provider, gophercloud.EndpointOpts{}) - if err != nil { - msg := fmt.Errorf("failed: Initializing openstack authentication client: %v", err) - return token, msg - } - - // Issue new unscoped token - result := tokens3.Create(v3Client, &options.AuthOptions) - if result.Err != nil { - return token, result.Err - } - token, err = result.ExtractToken() - if err != nil { - msg := fmt.Errorf("failed: Cannot extract the token from the response") - return token, msg - } - - return token, nil -} diff --git a/pkg/identity/keystone/token_getter_test.go b/pkg/identity/keystone/token_getter_test.go deleted file mode 100644 index f32e341b05..0000000000 --- a/pkg/identity/keystone/token_getter_test.go +++ /dev/null @@ -1,110 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package keystone - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - "testing" - - "github.com/gophercloud/gophercloud" - th "github.com/gophercloud/gophercloud/testhelper" -) - -func TestTokenGetter(t *testing.T) { - th.SetupHTTP() - defer th.TeardownHTTP() - - const ID = "0123456789" - - th.Mux.HandleFunc("/v3/auth/tokens", func(w http.ResponseWriter, r *http.Request) { - w.Header().Add("X-Subject-Token", ID) - type AuthRequest struct { - Auth struct { - Identity struct { - Password struct { - User struct { - Domain struct{ Name string } - Name string - Password string - } - } - } - } - } - var x AuthRequest - body, _ := ioutil.ReadAll(r.Body) - json.Unmarshal(body, &x) - domainName := x.Auth.Identity.Password.User.Domain.Name - userName := x.Auth.Identity.Password.User.Name - password := x.Auth.Identity.Password.User.Password - if domainName == "default" && userName == "testuser" && password == "testpw" { - w.WriteHeader(http.StatusCreated) - resp := `{"token": { - "methods": [ - "password" - ], - "expires_at": "2015-11-09T01:42:57.527363Z", - "user": { - "domain": { - "id": "default", - "name": "Default" - }, - "id": "some_id", - "name": "admin", - "password_expires_at": null - }, - "audit_ids": [ - "lC2Wj1jbQe-dLjLyOx4qPQ" - ], - "issued_at": "2015-11-09T00:42:57.527404Z" - } - }` - fmt.Fprintf(w, resp) - } else { - w.WriteHeader(http.StatusUnauthorized) - } - }) - - // Correct password - options := Options{ - AuthOptions: gophercloud.AuthOptions{ - IdentityEndpoint: th.Endpoint(), - Username: "testuser", - Password: "testpw", - DomainName: "default", - }, - } - - token, err := GetToken(options) - th.AssertNoErr(t, err) - th.AssertEquals(t, "0123456789", token.ID) - th.AssertEquals(t, "2015-11-09 01:42:57.527363 +0000 UTC", token.ExpiresAt.String()) - - // Incorrect password - options.AuthOptions.Password = "wrongpw" - - token, err = GetToken(options) - if _, ok := err.(gophercloud.ErrDefault401); !ok { - t.FailNow() - } - - // Invalid auth data - options.AuthOptions.Password = "" - - token, err = GetToken(options) - th.AssertEquals(t, "You must provide a password to authenticate", err.Error()) -} diff --git a/pkg/ingress/cmd/root.go b/pkg/ingress/cmd/root.go deleted file mode 100644 index 4834f97ba2..0000000000 --- a/pkg/ingress/cmd/root.go +++ /dev/null @@ -1,118 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cmd - -import ( - "flag" - "fmt" - "os" - "os/signal" - "syscall" - - "github.com/mitchellh/go-homedir" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - "github.com/spf13/viper" - - "k8s.io/cloud-provider-openstack/pkg/ingress/config" - "k8s.io/cloud-provider-openstack/pkg/ingress/controller" - "k8s.io/klog/v2" -) - -var ( - cfgFile string - isDebug bool - conf config.Config -) - -// rootCmd represents the base command when called without any subcommands -var rootCmd = &cobra.Command{ - Use: "ingress-openstack", - Short: "Ingress controller for OpenStack", - Long: `Ingress controller for OpenStack`, - - Run: func(cmd *cobra.Command, args []string) { - osIngress := controller.NewController(conf) - osIngress.Start() - - sigterm := make(chan os.Signal, 1) - signal.Notify(sigterm, syscall.SIGTERM) - signal.Notify(sigterm, syscall.SIGINT) - <-sigterm - }, -} - -// Execute adds all child commands to the root command sets flags appropriately. -// This is called by main.main(). It only needs to happen once to the rootCmd. -func Execute() { - if err := rootCmd.Execute(); err != nil { - fmt.Println(err) - os.Exit(-1) - } -} - -func init() { - log.SetOutput(os.Stdout) - - cobra.OnInitialize(initConfig) - - rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.ingress_openstack.yaml)") - rootCmd.PersistentFlags().BoolVar(&isDebug, "debug", false, "Print more detailed information.") - - klogFlags := flag.NewFlagSet("klog", flag.ExitOnError) - klog.InitFlags(klogFlags) - rootCmd.PersistentFlags().AddGoFlagSet(klogFlags) -} - -// initConfig reads in config file and ENV variables if set. -func initConfig() { - if cfgFile != "" { - // Use config file from the flag. - viper.SetConfigFile(cfgFile) - } else { - // Find home directory. - home, err := homedir.Dir() - if err != nil { - fmt.Println(err) - os.Exit(1) - } - - // Search config in home directory with name ".ingress_openstack" (without extension). - viper.AddConfigPath(home) - viper.SetConfigName(".ingress_openstack") - } - - viper.AutomaticEnv() // read in environment variables that match - - // If a config file is found, read it in. - if err := viper.ReadInConfig(); err != nil { - log.WithFields(log.Fields{"error": err}).Fatal("Failed to read config file") - } - - log.WithFields(log.Fields{"file": viper.ConfigFileUsed()}).Info("Using config file") - - if err := viper.Unmarshal(&conf); err != nil { - log.WithFields(log.Fields{"error": err}).Fatal("Unable to decode the configuration") - } - if conf.ClusterName == "" { - log.Fatal("clusterName configuration is required") - } - - if isDebug { - log.SetLevel(log.DebugLevel) - } -} diff --git a/pkg/ingress/config/config.go b/pkg/ingress/config/config.go deleted file mode 100644 index 6d932f007a..0000000000 --- a/pkg/ingress/config/config.go +++ /dev/null @@ -1,56 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package config - -import ( - openstack_provider "k8s.io/cloud-provider-openstack/pkg/cloudprovider/providers/openstack" -) - -// Config struct contains ingress controller configuration -type Config struct { - ClusterName string `mapstructure:"cluster-name"` - Kubernetes kubeConfig `mapstructure:"kubernetes"` - OpenStack openstack_provider.AuthOpts `mapstructure:"openstack"` - Octavia octaviaConfig `mapstructure:"octavia"` -} - -// Configuration for connecting to Kubernetes API server, either api_host or kubeconfig should be configured. -type kubeConfig struct { - // (Optional)Kubernetes API server host address. - ApiserverHost string `mapstructure:"api-host"` - - // (Optional)Kubeconfig file used to connect to Kubernetes cluster. - KubeConfig string `mapstructure:"kubeconfig"` -} - -// Octavia service related configuration -type octaviaConfig struct { - // (Optional) Provider name for the load balancer. Default: octavia - // For more information: https://docs.openstack.org/octavia/latest/admin/providers.html - Provider string `mapstructure:"provider"` - - // (Required) Subnet ID to create the load balancer. - SubnetID string `mapstructure:"subnet-id"` - - // (Optional) Public network ID to create floating IP. - // If empty, no floating IP will be allocated to the load balancer vip. - FloatingIPNetwork string `mapstructure:"floating-network-id"` - - // (Optional) If the ingress controller should manage the security groups attached to the cluster nodes. - // Default is false. - ManageSecurityGroups bool `mapstructure:"manage-security-groups"` -} diff --git a/pkg/ingress/controller/controller.go b/pkg/ingress/controller/controller.go deleted file mode 100644 index b821e5aa4c..0000000000 --- a/pkg/ingress/controller/controller.go +++ /dev/null @@ -1,921 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "context" - "crypto" - "crypto/rand" - "crypto/x509" - "encoding/base64" - "encoding/pem" - "fmt" - "reflect" - "strconv" - "strings" - "time" - - "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/security/groups" - log "github.com/sirupsen/logrus" - apiv1 "k8s.io/api/core/v1" - nwv1beta1 "k8s.io/api/networking/v1beta1" - apimetav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/util/intstr" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/informers" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/kubernetes/scheme" - v1core "k8s.io/client-go/kubernetes/typed/core/v1" - corelisters "k8s.io/client-go/listers/core/v1" - extlisters "k8s.io/client-go/listers/networking/v1beta1" - "k8s.io/client-go/tools/cache" - "k8s.io/client-go/tools/clientcmd" - "k8s.io/client-go/tools/record" - "k8s.io/client-go/util/workqueue" - "k8s.io/klog/v2" - pkcs12 "software.sslmate.com/src/go-pkcs12" - - "k8s.io/cloud-provider-openstack/pkg/ingress/config" - "k8s.io/cloud-provider-openstack/pkg/ingress/controller/openstack" - "k8s.io/cloud-provider-openstack/pkg/ingress/utils" - openstackutil "k8s.io/cloud-provider-openstack/pkg/util/openstack" -) - -const ( - // High enough QPS to fit all expected use cases. QPS=0 is not set here, because - // client code is overriding it. - defaultQPS = 1e6 - // High enough Burst to fit all expected use cases. Burst=0 is not set here, because - // client code is overriding it. - defaultBurst = 1e6 - - maxRetries = 5 - - // CreateEvent event associated with new objects in an informer - CreateEvent EventType = "CREATE" - // UpdateEvent event associated with an object update in an informer - UpdateEvent EventType = "UPDATE" - // DeleteEvent event associated when an object is removed from an informer - DeleteEvent EventType = "DELETE" - - // IngressKey picks a specific "class" for the Ingress. - // The controller only processes Ingresses with this annotation either - // unset, or set to either the configured value or the empty string. - IngressKey = "kubernetes.io/ingress.class" - - // IngressClass specifies which Ingress class we accept - IngressClass = "openstack" - - // LabelNodeRoleMaster specifies that a node is a master - // It's copied over to kubeadm until it's merged in core: https://github.com/kubernetes/kubernetes/pull/39112 - LabelNodeRoleMaster = "node-role.kubernetes.io/master" - - // IngressAnnotationInternal is the annotation used on the Ingress - // to indicate that we want an internal loadbalancer service so that octavia-ingress-controller won't associate - // floating ip to the load balancer VIP. - // Default to true. - IngressAnnotationInternal = "octavia.ingress.kubernetes.io/internal" - - // IngressControllerTag is added to the related resources. - IngressControllerTag = "octavia.ingress.kubernetes.io" - - // IngressSecretCertName is certificate key name defined in the secret data. - IngressSecretCertName = "tls.crt" - // IngressSecretKeyName is private key name defined in the secret data. - IngressSecretKeyName = "tls.key" - - // BarbicanSecretNameTemplate is the name format string to create Barbican secret. - BarbicanSecretNameTemplate = "kube_ingress_%s_%s_%s_%s" -) - -// EventType type of event associated with an informer -type EventType string - -// Event holds the context of an event -type Event struct { - Type EventType - Obj interface{} -} - -// Controller ... -type Controller struct { - stopCh chan struct{} - knownNodes []*apiv1.Node - queue workqueue.RateLimitingInterface - informer informers.SharedInformerFactory - recorder record.EventRecorder - ingressLister extlisters.IngressLister - ingressListerSynced cache.InformerSynced - serviceLister corelisters.ServiceLister - serviceListerSynced cache.InformerSynced - nodeLister corelisters.NodeLister - nodeListerSynced cache.InformerSynced - osClient *openstack.OpenStack - kubeClient kubernetes.Interface - config config.Config - subnetCIDR string -} - -// IsValid returns true if the given Ingress either doesn't specify -// the ingress.class annotation, or it's set to the configured in the -// ingress controller. -func IsValid(ing *nwv1beta1.Ingress) bool { - ingress, ok := ing.GetAnnotations()[IngressKey] - if !ok { - log.WithFields(log.Fields{ - "ingress_name": ing.Name, "ingress_ns": ing.Namespace, - }).Info("annotation not present in ingress") - return false - } - - return ingress == IngressClass -} - -func createApiserverClient(apiserverHost string, kubeConfig string) (*kubernetes.Clientset, error) { - cfg, err := clientcmd.BuildConfigFromFlags(apiserverHost, kubeConfig) - if err != nil { - return nil, err - } - - cfg.QPS = defaultQPS - cfg.Burst = defaultBurst - cfg.ContentType = "application/vnd.kubernetes.protobuf" - - log.Debug("creating kubernetes API client") - - client, err := kubernetes.NewForConfig(cfg) - if err != nil { - return nil, err - } - - v, err := client.Discovery().ServerVersion() - if err != nil { - return nil, err - } - log.WithFields(log.Fields{ - "version": fmt.Sprintf("v%v.%v", v.Major, v.Minor), - }).Debug("kubernetes API client created") - - return client, nil -} - -type NodeConditionPredicate func(node *apiv1.Node) bool - -// listWithPredicate gets nodes that matches predicate function. -func listWithPredicate(nodeLister corelisters.NodeLister, predicate NodeConditionPredicate) ([]*apiv1.Node, error) { - nodes, err := nodeLister.List(labels.Everything()) - if err != nil { - return nil, err - } - - var filtered []*apiv1.Node - for i := range nodes { - if predicate(nodes[i]) { - filtered = append(filtered, nodes[i]) - } - } - - return filtered, nil -} - -func getNodeConditionPredicate() NodeConditionPredicate { - return func(node *apiv1.Node) bool { - // We add the master to the node list, but its unschedulable. So we use this to filter - // the master. - if node.Spec.Unschedulable { - return false - } - - // As of 1.6, we will taint the master, but not necessarily mark it unschedulable. - // Recognize nodes labeled as master, and filter them also, as we were doing previously. - if _, hasMasterRoleLabel := node.Labels[LabelNodeRoleMaster]; hasMasterRoleLabel { - return false - } - - // If we have no info, don't accept - if len(node.Status.Conditions) == 0 { - return false - } - for _, cond := range node.Status.Conditions { - // We consider the node for load balancing only when its NodeReady condition status - // is ConditionTrue - if cond.Type == apiv1.NodeReady && cond.Status != apiv1.ConditionTrue { - log.WithFields(log.Fields{"name": node.Name, "status": cond.Status}).Info("ignoring node") - return false - } - } - return true - } -} - -// NewController creates a new OpenStack Ingress controller. -func NewController(conf config.Config) *Controller { - // initialize k8s client - kubeClient, err := createApiserverClient(conf.Kubernetes.ApiserverHost, conf.Kubernetes.KubeConfig) - if err != nil { - log.WithFields(log.Fields{ - "api_server": conf.Kubernetes.ApiserverHost, - "kuberconfig": conf.Kubernetes.KubeConfig, - "error": err, - }).Fatal("failed to initialize kubernetes client") - } - - // initialize openstack client - var osClient *openstack.OpenStack - osClient, err = openstack.NewOpenStack(conf) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - }).Fatal("failed to initialize openstack client") - } - - kubeInformerFactory := informers.NewSharedInformerFactory(kubeClient, time.Second*30) - serviceInformer := kubeInformerFactory.Core().V1().Services() - nodeInformer := kubeInformerFactory.Core().V1().Nodes() - queue := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()) - - eventBroadcaster := record.NewBroadcaster() - eventBroadcaster.StartLogging(klog.Infof) - eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{ - Interface: kubeClient.CoreV1().Events(""), - }) - recorder := eventBroadcaster.NewRecorder(scheme.Scheme, apiv1.EventSource{Component: "openstack-ingress-controller"}) - - controller := &Controller{ - config: conf, - queue: queue, - stopCh: make(chan struct{}), - informer: kubeInformerFactory, - recorder: recorder, - serviceLister: serviceInformer.Lister(), - serviceListerSynced: serviceInformer.Informer().HasSynced, - nodeLister: nodeInformer.Lister(), - nodeListerSynced: nodeInformer.Informer().HasSynced, - knownNodes: []*apiv1.Node{}, - osClient: osClient, - kubeClient: kubeClient, - } - - ingInformer := kubeInformerFactory.Networking().V1beta1().Ingresses() - ingInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: func(obj interface{}) { - addIng := obj.(*nwv1beta1.Ingress) - key := fmt.Sprintf("%s/%s", addIng.Namespace, addIng.Name) - - if !IsValid(addIng) { - log.Infof("ignore ingress %s", key) - return - } - - recorder.Event(addIng, apiv1.EventTypeNormal, "Creating", fmt.Sprintf("Ingress %s", key)) - controller.queue.AddRateLimited(Event{Obj: addIng, Type: CreateEvent}) - }, - UpdateFunc: func(old, new interface{}) { - newIng := new.(*nwv1beta1.Ingress) - oldIng := old.(*nwv1beta1.Ingress) - if newIng.ResourceVersion == oldIng.ResourceVersion { - // Periodic resync will send update events for all known Ingresses. - // Two different versions of the same Ingress will always have different RVs. - return - } - - key := fmt.Sprintf("%s/%s", newIng.Namespace, newIng.Name) - validOld := IsValid(oldIng) - validCur := IsValid(newIng) - if !validOld && validCur { - recorder.Event(newIng, apiv1.EventTypeNormal, "Creating", fmt.Sprintf("Ingress %s", key)) - controller.queue.AddRateLimited(Event{Obj: newIng, Type: CreateEvent}) - } else if validOld && !validCur { - recorder.Event(newIng, apiv1.EventTypeNormal, "Deleting", fmt.Sprintf("Ingress %s", key)) - controller.queue.AddRateLimited(Event{Obj: newIng, Type: DeleteEvent}) - } else if validCur && !reflect.DeepEqual(newIng.Spec, oldIng.Spec) { - recorder.Event(newIng, apiv1.EventTypeNormal, "Updating", fmt.Sprintf("Ingress %s", key)) - controller.queue.AddRateLimited(Event{Obj: newIng, Type: UpdateEvent}) - } else { - return - } - }, - DeleteFunc: func(obj interface{}) { - delIng, ok := obj.(*nwv1beta1.Ingress) - if !ok { - // If we reached here it means the ingress was deleted but its final state is unrecorded. - tombstone, ok := obj.(cache.DeletedFinalStateUnknown) - if !ok { - log.Errorf("couldn't get object from tombstone %#v", obj) - return - } - delIng, ok = tombstone.Obj.(*nwv1beta1.Ingress) - if !ok { - log.Errorf("Tombstone contained object that is not an Ingress: %#v", obj) - return - } - } - - key := fmt.Sprintf("%s/%s", delIng.Namespace, delIng.Name) - if !IsValid(delIng) { - log.Infof("ignore ingress %s", key) - return - } - - recorder.Event(delIng, apiv1.EventTypeNormal, "Deleting", fmt.Sprintf("Ingress %s", key)) - controller.queue.AddRateLimited(Event{Obj: delIng, Type: DeleteEvent}) - }, - }) - - controller.ingressLister = ingInformer.Lister() - controller.ingressListerSynced = ingInformer.Informer().HasSynced - - return controller -} - -// Start starts the openstack ingress controller. -func (c *Controller) Start() { - defer close(c.stopCh) - defer utilruntime.HandleCrash() - defer c.queue.ShutDown() - - log.Debug("starting Ingress controller") - go c.informer.Start(c.stopCh) - - // wait for the caches to synchronize before starting the worker - if !cache.WaitForCacheSync(c.stopCh, c.ingressListerSynced, c.serviceListerSynced, c.nodeListerSynced) { - utilruntime.HandleError(fmt.Errorf("timed out waiting for caches to sync")) - return - } - log.Info("ingress controller synced and ready") - - readyWorkerNodes, err := listWithPredicate(c.nodeLister, getNodeConditionPredicate()) - if err != nil { - log.Errorf("Failed to retrieve current set of nodes from node lister: %v", err) - return - } - c.knownNodes = readyWorkerNodes - - // Get subnet CIDR. The subnet CIDR will be used as source IP range for related security group rules. - subnet, err := c.osClient.GetSubnet(c.config.Octavia.SubnetID) - if err != nil { - log.Errorf("Failed to retrieve the subnet %s: %v", c.config.Octavia.SubnetID, err) - return - } - c.subnetCIDR = subnet.CIDR - - go wait.Until(c.runWorker, time.Second, c.stopCh) - go wait.Until(c.nodeSyncLoop, 60*time.Second, c.stopCh) - - <-c.stopCh -} - -// nodeSyncLoop handles updating the hosts pointed to by all load -// balancers whenever the set of nodes in the cluster changes. -func (c *Controller) nodeSyncLoop() { - readyWorkerNodes, err := listWithPredicate(c.nodeLister, getNodeConditionPredicate()) - if err != nil { - log.Errorf("Failed to retrieve current set of nodes from node lister: %v", err) - return - } - if utils.NodeSlicesEqual(readyWorkerNodes, c.knownNodes) { - return - } - - log.Infof("Detected change in list of current cluster nodes. New node set: %v", utils.NodeNames(readyWorkerNodes)) - - // if no new nodes, then avoid update member - if len(readyWorkerNodes) == 0 { - c.knownNodes = readyWorkerNodes - log.Info("Finished to handle node change, it's [] now") - return - } - - ings := new(nwv1beta1.IngressList) - // NOTE(lingxiankong): only take ingresses without ip address into consideration - opts := apimetav1.ListOptions{} - if ings, err = c.kubeClient.NetworkingV1beta1().Ingresses("").List(context.TODO(), opts); err != nil { - log.Errorf("Failed to retrieve current set of ingresses: %v", err) - return - } - - // Update each valid ingress - for _, ing := range ings.Items { - if !IsValid(&ing) { - continue - } - - log.WithFields(log.Fields{"ingress": ing.Name, "namespace": ing.Namespace}).Debug("Starting to handle ingress") - - lbName := utils.GetResourceName(ing.Namespace, ing.Name, c.config.ClusterName) - loadbalancer, err := openstackutil.GetLoadbalancerByName(c.osClient.Octavia, lbName) - if err != nil { - if err != openstackutil.ErrNotFound { - log.WithFields(log.Fields{"name": lbName}).Errorf("Failed to retrieve loadbalancer from OpenStack: %v", err) - } - - // If lb doesn't exist or error occurred, continue - continue - } - - if err = c.osClient.UpdateLoadbalancerMembers(loadbalancer.ID, readyWorkerNodes); err != nil { - log.WithFields(log.Fields{"ingress": ing.Name}).Error("Failed to handle ingress") - continue - } - - log.WithFields(log.Fields{"ingress": ing.Name, "namespace": ing.Namespace}).Info("Finished to handle ingress") - } - - c.knownNodes = readyWorkerNodes - - log.Info("Finished to handle node change") -} - -func (c *Controller) runWorker() { - for c.processNextItem() { - // continue looping - } -} - -func (c *Controller) processNextItem() bool { - obj, quit := c.queue.Get() - - if quit { - return false - } - defer c.queue.Done(obj) - - err := c.processItem(obj.(Event)) - if err == nil { - // No error, reset the ratelimit counters - c.queue.Forget(obj) - } else if c.queue.NumRequeues(obj) < maxRetries { - log.WithFields(log.Fields{"obj": obj, "error": err}).Error("Failed to process obj (will retry)") - c.queue.AddRateLimited(obj) - } else { - // err != nil and too many retries - log.WithFields(log.Fields{"obj": obj, "error": err}).Error("Failed to process obj (giving up)") - c.queue.Forget(obj) - utilruntime.HandleError(err) - } - - return true -} - -func (c *Controller) processItem(event Event) error { - ing := event.Obj.(*nwv1beta1.Ingress) - key := fmt.Sprintf("%s/%s", ing.Namespace, ing.Name) - - switch event.Type { - case CreateEvent: - log.WithFields(log.Fields{"ingress": key}).Info("ingress created, will create openstack resources") - - if err := c.ensureIngress(ing); err != nil { - utilruntime.HandleError(fmt.Errorf("failed to create openstack resources for ingress %s: %v", key, err)) - c.recorder.Event(ing, apiv1.EventTypeWarning, "Failed", fmt.Sprintf("Failed to create openstack resources for ingress %s: %v", key, err)) - } else { - c.recorder.Event(ing, apiv1.EventTypeNormal, "Created", fmt.Sprintf("Ingress %s", key)) - } - case UpdateEvent: - log.WithFields(log.Fields{"ingress": key}).Info("ingress updated, will update openstack resources") - - if err := c.ensureIngress(ing); err != nil { - utilruntime.HandleError(fmt.Errorf("failed to update openstack resources for ingress %s: %v", key, err)) - c.recorder.Event(ing, apiv1.EventTypeWarning, "Failed", fmt.Sprintf("Failed to update openstack resources for ingress %s: %v", key, err)) - } else { - c.recorder.Event(ing, apiv1.EventTypeNormal, "Updated", fmt.Sprintf("Ingress %s", key)) - } - case DeleteEvent: - log.WithFields(log.Fields{"ingress": key}).Info("ingress has been deleted, will delete openstack resources") - - if err := c.deleteIngress(ing); err != nil { - utilruntime.HandleError(fmt.Errorf("failed to delete openstack resources for ingress %s: %v", key, err)) - c.recorder.Event(ing, apiv1.EventTypeWarning, "Failed", fmt.Sprintf("Failed to delete openstack resources for ingress %s: %v", key, err)) - } else { - c.recorder.Event(ing, apiv1.EventTypeNormal, "Deleted", fmt.Sprintf("Ingress %s", key)) - } - } - - return nil -} - -func (c *Controller) deleteIngress(ing *nwv1beta1.Ingress) error { - key := fmt.Sprintf("%s/%s", ing.Namespace, ing.Name) - lbName := utils.GetResourceName(ing.Namespace, ing.Name, c.config.ClusterName) - - // Delete Barbican secrets - if c.osClient.Barbican != nil { - nameFilter := fmt.Sprintf("kube_ingress_%s_%s_%s", c.config.ClusterName, ing.Namespace, ing.Name) - if err := openstackutil.DeleteSecrets(c.osClient.Barbican, nameFilter); err != nil { - return fmt.Errorf("failed to remove Barbican secrets: %v", err) - } - - log.WithFields(log.Fields{"ingress": key}).Info("Barbican secrets deleted") - } - - // If load balancer doesn't exist, assume it's already deleted. - loadbalancer, err := openstackutil.GetLoadbalancerByName(c.osClient.Octavia, lbName) - if err != nil { - if err != openstackutil.ErrNotFound { - return fmt.Errorf("error getting loadbalancer %s: %v", ing.Name, err) - } - - log.WithFields(log.Fields{"lbName": lbName, "ingressName": ing.Name, "namespace": ing.Namespace}).Info("loadbalancer for ingress deleted") - return nil - } - - // Delete the floating IP for the load balancer VIP. We don't check if the Ingress is internal or not, just delete - // any floating IPs associated with the load balancer VIP port. - log.WithFields(log.Fields{"ingress": key}).Debug("deleting floating IP") - - if _, err = c.osClient.EnsureFloatingIP(true, loadbalancer.VipPortID, "", ""); err != nil { - return fmt.Errorf("failed to delete floating IP: %v", err) - } - - log.WithFields(log.Fields{"ingress": key}).Info("floating IP deleted") - - // Delete security group managed for the Ingress backend service - if c.config.Octavia.ManageSecurityGroups { - sgTags := []string{IngressControllerTag, fmt.Sprintf("%s_%s", ing.Namespace, ing.Name)} - tagString := strings.Join(sgTags, ",") - opts := groups.ListOpts{Tags: tagString} - sgs, err := c.osClient.GetSecurityGroups(opts) - if err != nil { - return fmt.Errorf("failed to get security groups for ingress %s: %v", key, err) - } - - nodes, err := listWithPredicate(c.nodeLister, getNodeConditionPredicate()) - if err != nil { - return fmt.Errorf("failed to get nodes: %v", err) - } - - for _, sg := range sgs { - if err = c.osClient.EnsurePortSecurityGroup(true, sg.ID, nodes); err != nil { - return fmt.Errorf("failed to operate on the port security groups for ingress %s: %v", key, err) - } - if _, err = c.osClient.EnsureSecurityGroup(true, "", "", sgTags); err != nil { - return fmt.Errorf("failed to delete the security groups for ingress %s: %v", key, err) - } - } - - log.WithFields(log.Fields{"ingress": key}).Info("security group deleted") - } - - err = openstackutil.DeleteLoadbalancer(c.osClient.Octavia, loadbalancer.ID) - log.WithFields(log.Fields{"lbID": loadbalancer.ID}).Info("loadbalancer deleted") - - return err -} - -func (c *Controller) toBarbicanSecret(name string, namespace string, toSecretName string) (string, error) { - secret, err := c.kubeClient.CoreV1().Secrets(namespace).Get(context.TODO(), name, apimetav1.GetOptions{}) - if err != nil { - // TODO(lingxiankong): Creating secret on the fly not supported yet. - return "", err - } - - var pk crypto.PrivateKey - if keyBytes, isPresent := secret.Data[IngressSecretKeyName]; isPresent { - pk, err = privateKeyFromPEM(keyBytes) - if err != nil { - return "", err - } - } else { - return "", fmt.Errorf("%s key doesn't exist in the secret %s", IngressSecretKeyName, name) - } - - var cb []*x509.Certificate - if certBytes, isPresent := secret.Data[IngressSecretCertName]; isPresent { - cb, err = parsePEMBundle(certBytes) - if err != nil { - return "", err - } - } else { - return "", fmt.Errorf("%s key doesn't exist in the secret %s", IngressSecretCertName, name) - } - - var caCerts []*x509.Certificate - // TODO(lingxiankong): We assume the 'tls.cert' data only contains the end user certificate. - pfxData, err := pkcs12.Encode(rand.Reader, pk, cb[0], caCerts, "") - if err != nil { - return "", fmt.Errorf("failed to create PKCS#12 bundle: %v", err) - } - encoded := base64.StdEncoding.EncodeToString(pfxData) - - return openstackutil.EnsureSecret(c.osClient.Barbican, toSecretName, "application/octet-stream", encoded) -} - -func (c *Controller) ensureIngress(ing *nwv1beta1.Ingress) error { - ingName := ing.ObjectMeta.Name - ingNamespace := ing.ObjectMeta.Namespace - clusterName := c.config.ClusterName - - key := fmt.Sprintf("%s/%s", ingNamespace, ingName) - name := utils.GetResourceName(ingNamespace, ingName, clusterName) - - if len(ing.Spec.TLS) > 0 && c.osClient.Barbican == nil { - return fmt.Errorf("TLS Ingress not supported because of Key Manager service unavailable") - } - - lb, err := c.osClient.EnsureLoadBalancer(name, c.config.Octavia.SubnetID, ingNamespace, ingName, clusterName) - if err != nil { - return err - } - - if strings.Contains(lb.Description, ing.ResourceVersion) { - log.WithFields(log.Fields{"ingress": key}).Info("ingress not changed") - return nil - } - - var nodePorts []int - var sgID string - - if c.config.Octavia.ManageSecurityGroups { - log.WithFields(log.Fields{"ingress": key}).Info("ensuring security group") - - sgDescription := fmt.Sprintf("Security group created for Ingress %s from cluster %s", key, clusterName) - sgTags := []string{IngressControllerTag, fmt.Sprintf("%s_%s", ingNamespace, ingName)} - sgID, err = c.osClient.EnsureSecurityGroup(false, name, sgDescription, sgTags) - if err != nil { - return fmt.Errorf("failed to prepare the security group for the ingress %s: %v", key, err) - } - - log.WithFields(log.Fields{"sgID": sgID, "ingress": key}).Info("ensured security group") - } - - // Convert kubernetes secrets to barbican ones - var secretRefs []string - for _, tls := range ing.Spec.TLS { - secretName := fmt.Sprintf(BarbicanSecretNameTemplate, clusterName, ingNamespace, ingName, tls.SecretName) - secretRef, err := c.toBarbicanSecret(tls.SecretName, ingNamespace, secretName) - if err != nil { - return fmt.Errorf("failed to create Barbican secret: %v", err) - } - - log.WithFields(log.Fields{"secretName": secretName, "secretRef": secretRef, "ingress": key}).Info("secret created in Barbican") - - secretRefs = append(secretRefs, secretRef) - } - port := 80 - if len(secretRefs) > 0 { - port = 443 - } - - // Create listener - listener, err := c.osClient.EnsureListener(name, lb.ID, secretRefs) - if err != nil { - return err - } - - // get nodes information - nodeObjs, err := listWithPredicate(c.nodeLister, getNodeConditionPredicate()) - if err != nil { - return err - } - - // Add default pool for the listener if 'backend' is defined - if ing.Spec.Backend != nil { - serviceName := fmt.Sprintf("%s/%s", ingNamespace, ing.Spec.Backend.ServiceName) - nodePort, err := c.getServiceNodePort(serviceName, ing.Spec.Backend.ServicePort) - if err != nil { - return err - } - - nodePorts = append(nodePorts, nodePort) - - if _, err = c.osClient.EnsurePoolMembers(false, name, lb.ID, listener.ID, &nodePort, nodeObjs); err != nil { - return err - } - } else { - // Delete default pool and its members - if _, err = c.osClient.EnsurePoolMembers(true, name, lb.ID, "", nil, nil); err != nil { - return err - } - } - - // Delete all existing policies - existingPolicies, err := c.osClient.GetL7policies(listener.ID) - if err != nil { - return err - } - for _, p := range existingPolicies { - err = c.osClient.DeleteL7policy(p.ID, lb.ID) - if err != nil { - log.WithFields(log.Fields{"policyID": p.ID, "lbID": lb.ID}).Errorf("could not delete L7 policy: %v", err) - } - } - - // Delete all existing shared pools - existingSharedPools, err := c.osClient.GetPools(lb.ID, true) - if err != nil { - return err - } - for _, sp := range existingSharedPools { - err = c.osClient.DeletePool(sp.ID, lb.ID) - if err != nil { - log.WithFields(log.Fields{"poolID": sp.ID, "lbID": lb.ID}).Errorf("could not delete shared pool: %v", err) - } - } - - // Add l7 load balancing rules. Each host and path combination is mapped to a l7 policy in octavia, - // which contains two rules(with type 'HOST_NAME' and 'PATH' respectively) - for _, rule := range ing.Spec.Rules { - host := rule.Host - - for _, path := range rule.HTTP.Paths { - // make the pool name unique in the load balancer - poolName := utils.Hash(fmt.Sprintf("%s+%s", path.Backend.ServiceName, path.Backend.ServicePort.String())) - - serviceName := fmt.Sprintf("%s/%s", ingNamespace, path.Backend.ServiceName) - nodePort, err := c.getServiceNodePort(serviceName, path.Backend.ServicePort) - if err != nil { - return err - } - - nodePorts = append(nodePorts, nodePort) - - poolID, err := c.osClient.EnsurePoolMembers(false, poolName, lb.ID, "", &nodePort, nodeObjs) - if err != nil { - return err - } - - if err = c.osClient.CreatePolicyRules(lb.ID, listener.ID, *poolID, host, path.Path, port); err != nil { - return err - } - } - } - - if c.config.Octavia.ManageSecurityGroups { - log.WithFields(log.Fields{"ingress": key, "sgID": sgID}).Info("ensuring security group rules") - - if err := c.osClient.EnsureSecurityGroupRules(sgID, c.subnetCIDR, nodePorts); err != nil { - return fmt.Errorf("failed to ensure security group rules for Ingress %s: %v", ingName, err) - } - - if err := c.osClient.EnsurePortSecurityGroup(false, sgID, nodeObjs); err != nil { - return fmt.Errorf("failed to operate port security group for Ingress %s: %v", ingName, err) - } - - log.WithFields(log.Fields{"ingress": key, "sgID": sgID}).Info("ensured security group rules") - } - - internalSetting := getStringFromIngressAnnotation(ing, IngressAnnotationInternal, "true") - isInternal, err := strconv.ParseBool(internalSetting) - if err != nil { - return fmt.Errorf("unknown annotation %s: %v", IngressAnnotationInternal, err) - } - - address := lb.VipAddress - // Allocate floating ip for loadbalancer vip if the external network is configured and the Ingress is not internal. - if !isInternal && c.config.Octavia.FloatingIPNetwork != "" { - log.WithFields(log.Fields{"ingress": key}).Info("creating floating IP") - - description := fmt.Sprintf("Floating IP for Kubernetes ingress %s in namespace %s from cluster %s", ingName, ingNamespace, clusterName) - address, err = c.osClient.EnsureFloatingIP(false, lb.VipPortID, c.config.Octavia.FloatingIPNetwork, description) - if err != nil { - return fmt.Errorf("failed to create floating IP: %v", err) - } - - log.WithFields(log.Fields{"ingress": key, "fip": address}).Info("floating IP created") - } - - // Update ingress status - newIng, err := c.updateIngressStatus(ing, address) - if err != nil { - return err - } - c.recorder.Event(ing, apiv1.EventTypeNormal, "Updated", fmt.Sprintf("Successfully associated IP address %s to ingress %s", address, key)) - - // Add ingress resource version to the load balancer description - newDes := fmt.Sprintf("Kubernetes Ingress %s in namespace %s from cluster %s, version: %s", ingName, ingNamespace, clusterName, newIng.ResourceVersion) - if err = c.osClient.UpdateLoadBalancerDescription(lb.ID, newDes); err != nil { - return err - } - - log.WithFields(log.Fields{"ingress": key, "lbID": lb.ID}).Info("openstack resources for ingress created") - - return nil -} - -func (c *Controller) updateIngressStatus(ing *nwv1beta1.Ingress, vip string) (*nwv1beta1.Ingress, error) { - newState := new(apiv1.LoadBalancerStatus) - newState.Ingress = []apiv1.LoadBalancerIngress{{IP: vip}} - newIng := ing.DeepCopy() - newIng.Status.LoadBalancer = *newState - - newObj, err := c.kubeClient.NetworkingV1beta1().Ingresses(newIng.Namespace).UpdateStatus(context.TODO(), newIng, apimetav1.UpdateOptions{}) - if err != nil { - return nil, err - } - - return newObj, nil -} - -func (c *Controller) getService(key string) (*apiv1.Service, error) { - namespace, name, err := cache.SplitMetaNamespaceKey(key) - if err != nil { - return nil, err - } - - service, err := c.serviceLister.Services(namespace).Get(name) - if err != nil { - return nil, err - } - - return service, nil -} - -func (c *Controller) getServiceNodePort(name string, port intstr.IntOrString) (int, error) { - svc, err := c.getService(name) - if err != nil { - return 0, err - } - - var nodePort int - ports := svc.Spec.Ports - for _, p := range ports { - if port.Type == intstr.Int && int(p.Port) == port.IntValue() { - nodePort = int(p.NodePort) - break - } - if port.Type == intstr.String && p.Name == port.StrVal { - nodePort = int(p.NodePort) - break - } - } - - if nodePort == 0 { - return 0, fmt.Errorf("failed to find service node port") - } - - return nodePort, nil -} - -// getStringFromIngressAnnotation searches a given Ingress for a specific annotationKey and either returns the -// annotation's value or a specified defaultSetting -func getStringFromIngressAnnotation(ingress *nwv1beta1.Ingress, annotationKey string, defaultValue string) string { - if annotationValue, ok := ingress.Annotations[annotationKey]; ok { - return annotationValue - } - - return defaultValue -} - -// privateKeyFromPEM converts a PEM block into a crypto.PrivateKey. -func privateKeyFromPEM(pemData []byte) (crypto.PrivateKey, error) { - var result *pem.Block - rest := pemData - for { - result, rest = pem.Decode(rest) - if result == nil { - return nil, fmt.Errorf("Cannot decode supplied PEM data") - } - - switch result.Type { - case "RSA PRIVATE KEY": - return x509.ParsePKCS1PrivateKey(result.Bytes) - case "EC PRIVATE KEY": - return x509.ParseECPrivateKey(result.Bytes) - } - } -} - -// parsePEMBundle parses a certificate bundle from top to bottom and returns -// a slice of x509 certificates. This function will error if no certificates are found. -// -func parsePEMBundle(bundle []byte) ([]*x509.Certificate, error) { - var certificates []*x509.Certificate - var certDERBlock *pem.Block - - for { - certDERBlock, bundle = pem.Decode(bundle) - if certDERBlock == nil { - break - } - - if certDERBlock.Type == "CERTIFICATE" { - cert, err := x509.ParseCertificate(certDERBlock.Bytes) - if err != nil { - return nil, err - } - certificates = append(certificates, cert) - } - } - - if len(certificates) == 0 { - return nil, fmt.Errorf("no certificates were found while parsing the bundle") - } - - return certificates, nil -} diff --git a/pkg/ingress/controller/openstack/client.go b/pkg/ingress/controller/openstack/client.go deleted file mode 100644 index 97d202b2a2..0000000000 --- a/pkg/ingress/controller/openstack/client.go +++ /dev/null @@ -1,94 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package openstack - -import ( - "fmt" - - "github.com/gophercloud/gophercloud" - "github.com/gophercloud/gophercloud/openstack" - log "github.com/sirupsen/logrus" - - cpo "k8s.io/cloud-provider-openstack/pkg/cloudprovider/providers/openstack" - "k8s.io/cloud-provider-openstack/pkg/ingress/config" -) - -// OpenStack is an implementation of cloud provider Interface for OpenStack. -type OpenStack struct { - Octavia *gophercloud.ServiceClient - nova *gophercloud.ServiceClient - neutron *gophercloud.ServiceClient - Barbican *gophercloud.ServiceClient - config config.Config -} - -// NewOpenStack gets openstack struct -func NewOpenStack(cfg config.Config) (*OpenStack, error) { - provider, err := cpo.NewOpenStackClient(&cfg.OpenStack, "octavia-ingress-controller") - if err != nil { - return nil, err - } - - // get octavia service client - var lb *gophercloud.ServiceClient - lb, err = openstack.NewLoadBalancerV2(provider, gophercloud.EndpointOpts{ - Region: cfg.OpenStack.Region, - }) - if err != nil { - return nil, fmt.Errorf("failed to find octavia endpoint for region %s: %v", cfg.OpenStack.Region, err) - } - - // get neutron service client - var network *gophercloud.ServiceClient - network, err = openstack.NewNetworkV2(provider, gophercloud.EndpointOpts{ - Region: cfg.OpenStack.Region, - }) - if err != nil { - return nil, fmt.Errorf("failed to find neutron endpoint for region %s: %v", cfg.OpenStack.Region, err) - } - - // get nova service client - var compute *gophercloud.ServiceClient - compute, err = openstack.NewComputeV2(provider, gophercloud.EndpointOpts{ - Region: cfg.OpenStack.Region, - }) - if err != nil { - return nil, fmt.Errorf("failed to find compute v2 endpoint for region %s: %v", cfg.OpenStack.Region, err) - } - - // Get barbican service client. - var barbican *gophercloud.ServiceClient - barbican, err = openstack.NewKeyManagerV1(provider, gophercloud.EndpointOpts{ - Region: cfg.OpenStack.Region, - }) - if err != nil { - log.Warn("Barbican not suppported.") - barbican = nil - } - - os := OpenStack{ - Octavia: lb, - nova: compute, - neutron: network, - Barbican: barbican, - config: cfg, - } - - log.Debug("openstack client initialized") - - return &os, nil -} diff --git a/pkg/ingress/controller/openstack/neutron.go b/pkg/ingress/controller/openstack/neutron.go deleted file mode 100644 index 6b7f4a315c..0000000000 --- a/pkg/ingress/controller/openstack/neutron.go +++ /dev/null @@ -1,294 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package openstack - -import ( - "fmt" - "strconv" - "strings" - - neutrontags "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/attributestags" - "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/floatingips" - "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/security/groups" - "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/security/rules" - "github.com/gophercloud/gophercloud/openstack/networking/v2/ports" - "github.com/gophercloud/gophercloud/openstack/networking/v2/subnets" - log "github.com/sirupsen/logrus" - "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/sets" - - "k8s.io/cloud-provider-openstack/pkg/ingress/utils" -) - -func (os *OpenStack) getFloatingIPs(listOpts floatingips.ListOpts) ([]floatingips.FloatingIP, error) { - allPages, err := floatingips.List(os.neutron, listOpts).AllPages() - if err != nil { - return []floatingips.FloatingIP{}, err - } - allFIPs, err := floatingips.ExtractFloatingIPs(allPages) - if err != nil { - return []floatingips.FloatingIP{}, err - } - - return allFIPs, nil -} - -// GetSubnet get a subnet by the given ID. -func (os *OpenStack) GetSubnet(subnetID string) (*subnets.Subnet, error) { - subnet, err := subnets.Get(os.neutron, subnetID).Extract() - if err != nil { - return nil, err - } - return subnet, nil -} - -// getPorts gets all the filtered ports. -func (os *OpenStack) getPorts(listOpts ports.ListOpts) ([]ports.Port, error) { - allPages, err := ports.List(os.neutron, listOpts).AllPages() - if err != nil { - return []ports.Port{}, err - } - allPorts, err := ports.ExtractPorts(allPages) - if err != nil { - return []ports.Port{}, err - } - - return allPorts, nil -} - -// EnsureFloatingIP makes sure a floating IP is allocated for the port -func (os *OpenStack) EnsureFloatingIP(needDelete bool, portID string, floatingIPNetwork string, description string) (string, error) { - listOpts := floatingips.ListOpts{PortID: portID} - fips, err := os.getFloatingIPs(listOpts) - - // If needed, delete the floating IPs and return. - if needDelete { - for _, fip := range fips { - if err := floatingips.Delete(os.neutron, fip.ID).ExtractErr(); err != nil { - return "", err - } - } - - return "", nil - } - - if len(fips) > 1 { - return "", fmt.Errorf("more than one floating IPs for port %s found", portID) - } - - var fip *floatingips.FloatingIP - if len(fips) == 0 { - floatIPOpts := floatingips.CreateOpts{ - PortID: portID, - FloatingNetworkID: floatingIPNetwork, - Description: description, - } - fip, err = floatingips.Create(os.neutron, floatIPOpts).Extract() - if err != nil { - return "", err - } - } else { - fip = &fips[0] - } - - return fip.FloatingIP, nil -} - -// GetSecurityGroups gets all the filtered security groups. -func (os *OpenStack) GetSecurityGroups(listOpts groups.ListOpts) ([]groups.SecGroup, error) { - allPages, err := groups.List(os.neutron, listOpts).AllPages() - if err != nil { - return []groups.SecGroup{}, err - } - allSGs, err := groups.ExtractGroups(allPages) - if err != nil { - return []groups.SecGroup{}, err - } - - return allSGs, nil -} - -// EnsureSecurityGroup make sure the security group with given tags exists or not according to need_delete param. -// Make sure the EnsurePortSecurityGroup function is called before EnsureSecurityGroup if you want to delete the security group. -func (os *OpenStack) EnsureSecurityGroup(needDelete bool, name string, description string, tags []string) (string, error) { - tagsString := strings.Join(tags, ",") - listOpts := groups.ListOpts{Tags: tagsString} - allGroups, err := os.GetSecurityGroups(listOpts) - if err != nil { - return "", err - } - - // If needed, delete the security groups and return. - if needDelete { - for _, group := range allGroups { - if err := groups.Delete(os.neutron, group.ID).ExtractErr(); err != nil { - return "", err - } - } - return "", nil - } - - if len(allGroups) > 1 { - return "", fmt.Errorf("more than one security groups found") - } - - // Create security group and add tags. - var group *groups.SecGroup - if len(allGroups) == 0 { - createOpts := groups.CreateOpts{ - Name: name, - Description: description, - } - group, err = groups.Create(os.neutron, createOpts).Extract() - if err != nil { - return "", err - } - - // Do not use tags replace API until https://bugs.launchpad.net/neutron/+bug/1817238 is resolved. - //tagReplaceAllOpts := neutrontags.ReplaceAllOpts{Tags: tags} - //if _, err := neutrontags.ReplaceAll(os.neutron, "security_groups", group.ID, tagReplaceAllOpts).Extract(); err != nil { - // return "", fmt.Errorf("failed to add tags %s to security group %s: %v", tagsString, group.ID, err) - //} - - for _, t := range tags { - if err := neutrontags.Add(os.neutron, "security_groups", group.ID, t).ExtractErr(); err != nil { - return "", fmt.Errorf("failed to add tag %s to security group %s: %v", t, group.ID, err) - } - } - } else { - group = &allGroups[0] - } - - return group.ID, nil -} - -// EnsureSecurityGroupRules ensures the only dstPorts are allowed in the given security group. -func (os *OpenStack) EnsureSecurityGroupRules(sgID string, sourceIP string, dstPorts []int) error { - listOpts := rules.ListOpts{ - Protocol: "tcp", - SecGroupID: sgID, - RemoteIPPrefix: sourceIP, - } - allPages, err := rules.List(os.neutron, listOpts).AllPages() - if err != nil { - return err - } - allRules, err := rules.ExtractRules(allPages) - if err != nil { - return err - } - - if len(dstPorts) == 0 { - // Delete all the rules and return. - - for _, rule := range allRules { - if err := rules.Delete(os.neutron, rule.ID).ExtractErr(); err != nil { - return err - } - } - - log.WithFields(log.Fields{"sgID": sgID}).Debug("all the security group rules deleted") - return nil - } - - dstPortsSet := sets.NewString() - for _, p := range dstPorts { - dstPortsSet.Insert(strconv.Itoa(p)) - } - - // Because the security group is supposed to be managed by octavia-ingress-controller, we assume the `port_range_min` - // equals to `port_range_max`. - for _, rule := range allRules { - if !dstPortsSet.Has(strconv.Itoa(rule.PortRangeMin)) { - // Delete the rule - if err := rules.Delete(os.neutron, rule.ID).ExtractErr(); err != nil { - return err - } - } else { - dstPortsSet.Delete(strconv.Itoa(rule.PortRangeMin)) - } - } - - // Now, the ports left in dstPortsSet are all needed for creating rules. - newPorts := dstPortsSet.List() - for _, p := range newPorts { - newPort, err := strconv.Atoi(p) - if err != nil { - return err - } - createOpts := rules.CreateOpts{ - Direction: "ingress", - PortRangeMin: newPort, - PortRangeMax: newPort, - EtherType: rules.EtherType4, - Protocol: "tcp", - RemoteIPPrefix: sourceIP, - SecGroupID: sgID, - } - if _, err := rules.Create(os.neutron, createOpts).Extract(); err != nil { - return err - } - } - - return nil -} - -// EnsurePortSecurityGroup ensures the security group is attached to all the node ports or detached from all the ports -// according to needDelete param. -func (os *OpenStack) EnsurePortSecurityGroup(needDelete bool, sgID string, nodes []*v1.Node) error { - for _, node := range nodes { - instanceID, err := utils.GetNodeID(node) - if err != nil { - return err - } - listOpts := ports.ListOpts{DeviceID: instanceID} - allPorts, err := os.getPorts(listOpts) - if err != nil { - return err - } - - for _, port := range allPorts { - sgSet := utils.Convert2Set(port.SecurityGroups) - - if sgSet.Has(sgID) && needDelete { - // Remove sg from the port - sgSet.Delete(sgID) - newSGs := sgSet.List() - updateOpts := ports.UpdateOpts{SecurityGroups: &newSGs} - if _, err := ports.Update(os.neutron, port.ID, updateOpts).Extract(); err != nil { - return err - } - - log.WithFields(log.Fields{"sgID": sgID, "portID": port.ID}).Debug("security group detached from the port") - } - - if !sgSet.Has(sgID) && !needDelete { - // Add sg to the port - sgSet.Insert(sgID) - newSGs := sgSet.List() - updateOpts := ports.UpdateOpts{SecurityGroups: &newSGs} - if _, err := ports.Update(os.neutron, port.ID, updateOpts).Extract(); err != nil { - return err - } - - log.WithFields(log.Fields{"sgID": sgID, "portID": port.ID}).Debug("security group attached to the port") - } - } - } - - return nil -} diff --git a/pkg/ingress/controller/openstack/octavia.go b/pkg/ingress/controller/openstack/octavia.go deleted file mode 100644 index f0fb273d0c..0000000000 --- a/pkg/ingress/controller/openstack/octavia.go +++ /dev/null @@ -1,472 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package openstack - -import ( - "errors" - "fmt" - "strings" - "time" - - "github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/l7policies" - "github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/listeners" - "github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/loadbalancers" - "github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/pools" - "github.com/gophercloud/gophercloud/pagination" - log "github.com/sirupsen/logrus" - apiv1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/wait" - - cpoerrors "k8s.io/cloud-provider-openstack/pkg/util/errors" - openstackutil "k8s.io/cloud-provider-openstack/pkg/util/openstack" -) - -const ( - loadbalancerActiveInitDealy = 3 * time.Second - loadbalancerActiveFactor = 1 - loadbalancerActiveSteps = 240 - - activeStatus = "ACTIVE" - errorStatus = "ERROR" -) - -func getNodeAddressForLB(node *apiv1.Node) (string, error) { - addrs := node.Status.Addresses - if len(addrs) == 0 { - return "", errors.New("no address found for host") - } - - for _, addr := range addrs { - if addr.Type == apiv1.NodeInternalIP { - return addr.Address, nil - } - } - - return addrs[0].Address, nil -} - -func (os *OpenStack) waitLoadbalancerActiveProvisioningStatus(loadbalancerID string) (string, error) { - backoff := wait.Backoff{ - Duration: loadbalancerActiveInitDealy, - Factor: loadbalancerActiveFactor, - Steps: loadbalancerActiveSteps, - } - - var provisioningStatus string - err := wait.ExponentialBackoff(backoff, func() (bool, error) { - loadbalancer, err := loadbalancers.Get(os.Octavia, loadbalancerID).Extract() - if err != nil { - return false, err - } - provisioningStatus = loadbalancer.ProvisioningStatus - if loadbalancer.ProvisioningStatus == activeStatus { - return true, nil - } else if loadbalancer.ProvisioningStatus == errorStatus { - return true, fmt.Errorf("loadbalancer has gone into ERROR state") - } else { - return false, nil - } - - }) - - if err == wait.ErrWaitTimeout { - err = fmt.Errorf("loadbalancer failed to go into ACTIVE provisioning status within alloted time") - } - return provisioningStatus, err -} - -// GetPools retrives the pools belong to the loadbalancer. If shared is true, only return the shared pools. -func (os *OpenStack) GetPools(lbID string, shared bool) ([]pools.Pool, error) { - var lbPools []pools.Pool - - opts := pools.ListOpts{ - LoadbalancerID: lbID, - } - err := pools.List(os.Octavia, opts).EachPage(func(page pagination.Page) (bool, error) { - v, err := pools.ExtractPools(page) - if err != nil { - return false, err - } - for _, p := range v { - if shared && len(p.Listeners) != 0 { - continue - } - lbPools = append(lbPools, p) - } - - return true, nil - }) - if err != nil { - return nil, err - } - - return lbPools, nil -} - -// GetMembers retrieve all the members of the specified pool -func (os *OpenStack) GetMembers(poolID string) ([]pools.Member, error) { - var members []pools.Member - - opts := pools.ListMembersOpts{} - err := pools.ListMembers(os.Octavia, poolID, opts).EachPage(func(page pagination.Page) (bool, error) { - v, err := pools.ExtractMembers(page) - if err != nil { - return false, err - } - members = append(members, v...) - return true, nil - }) - if err != nil { - return nil, err - } - - return members, nil -} - -// DeletePool deletes a pool -func (os *OpenStack) DeletePool(poolID string, lbID string) error { - if err := pools.Delete(os.Octavia, poolID).ExtractErr(); err != nil { - return fmt.Errorf("failed to delete pool %s: %v", poolID, err) - } - - _, err := os.waitLoadbalancerActiveProvisioningStatus(lbID) - if err != nil { - return fmt.Errorf("failed to wait for loadbalancer to be active: %v", err) - } - - return nil -} - -// GetL7policies retrieves all l7 policies for the given listener. -func (os *OpenStack) GetL7policies(listenerID string) ([]l7policies.L7Policy, error) { - var policies []l7policies.L7Policy - opts := l7policies.ListOpts{ - ListenerID: listenerID, - } - err := l7policies.List(os.Octavia, opts).EachPage(func(page pagination.Page) (bool, error) { - v, err := l7policies.ExtractL7Policies(page) - if err != nil { - return false, err - } - policies = append(policies, v...) - return true, nil - }) - if err != nil { - return nil, err - } - - return policies, nil -} - -// DeleteL7policy deletes a l7 policy -func (os *OpenStack) DeleteL7policy(policyID string, lbID string) error { - if err := l7policies.Delete(os.Octavia, policyID).ExtractErr(); err != nil { - return fmt.Errorf("failed to delete l7 policy: %v", err) - } - - _, err := os.waitLoadbalancerActiveProvisioningStatus(lbID) - if err != nil { - return fmt.Errorf("failed to wait for loadbalancer to be active: %v", err) - } - - return nil -} - -// EnsureLoadBalancer creates a loadbalancer in octavia if it does not exist, wait for the loadbalancer to be ACTIVE. -func (os *OpenStack) EnsureLoadBalancer(name string, subnetID string, ingNamespace string, ingName string, clusterName string) (*loadbalancers.LoadBalancer, error) { - loadbalancer, err := openstackutil.GetLoadbalancerByName(os.Octavia, name) - if err != nil { - if err != openstackutil.ErrNotFound { - return nil, fmt.Errorf("error getting loadbalancer %s: %v", name, err) - } - - var provider string - if os.config.Octavia.Provider == "" { - provider = "octavia" - } else { - provider = os.config.Octavia.Provider - } - - createOpts := loadbalancers.CreateOpts{ - Name: name, - Description: fmt.Sprintf("Kubernetes ingress %s in namespace %s from cluster %s", ingName, ingNamespace, clusterName), - VipSubnetID: subnetID, - Provider: provider, - } - loadbalancer, err = loadbalancers.Create(os.Octavia, createOpts).Extract() - if err != nil { - return nil, fmt.Errorf("error creating loadbalancer %v: %v", createOpts, err) - } - - log.WithFields(log.Fields{"name": name, "ID": loadbalancer.ID}).Info("creating loadbalancer") - } else { - log.WithFields(log.Fields{"name": name}).Debug("loadbalancer exists") - } - - _, err = os.waitLoadbalancerActiveProvisioningStatus(loadbalancer.ID) - if err != nil { - return nil, fmt.Errorf("error creating loadbalancer: %v", err) - } - - return loadbalancer, nil -} - -// UpdateLoadBalancerDescription updates the load balancer description field. -func (os *OpenStack) UpdateLoadBalancerDescription(lbID string, newDescription string) error { - _, err := loadbalancers.Update(os.Octavia, lbID, loadbalancers.UpdateOpts{ - Description: &newDescription, - }).Extract() - if err != nil { - return fmt.Errorf("failed to update loadbalancer description: %v", err) - } - - log.WithFields(log.Fields{"lb": lbID}).Debug("loadbalancer description updated") - return nil -} - -// EnsureListener creates a loadbalancer listener in octavia if it does not exist, wait for the loadbalancer to be ACTIVE. -func (os *OpenStack) EnsureListener(name string, lbID string, secretRefs []string) (*listeners.Listener, error) { - listener, err := openstackutil.GetListenerByName(os.Octavia, name, lbID) - if err != nil { - if err != openstackutil.ErrNotFound { - return nil, fmt.Errorf("error getting listener %s: %v", name, err) - } - - log.WithFields(log.Fields{"lb": lbID, "listenerName": name}).Debug("creating listener") - - opts := listeners.CreateOpts{ - Name: name, - Protocol: "HTTP", - ProtocolPort: 80, // Ingress Controller only supports http/https for now - LoadbalancerID: lbID, - } - if len(secretRefs) > 0 { - opts.DefaultTlsContainerRef = secretRefs[0] - opts.SniContainerRefs = secretRefs - opts.ProtocolPort = 443 - opts.Protocol = "TERMINATED_HTTPS" - } - listener, err = listeners.Create(os.Octavia, opts).Extract() - if err != nil { - return nil, fmt.Errorf("error creating listener: %v", err) - } - - log.WithFields(log.Fields{"lb": lbID, "listenerName": name}).Info("listener created") - } - - _, err = os.waitLoadbalancerActiveProvisioningStatus(lbID) - if err != nil { - return nil, fmt.Errorf("error creating listener: %v", err) - } - - return listener, nil -} - -// EnsurePoolMembers ensure the pool and its members exist if deleted flag is not set, delete the pool and all its members otherwise. -func (os *OpenStack) EnsurePoolMembers(deleted bool, poolName string, lbID string, listenerID string, nodePort *int, nodes []*apiv1.Node) (*string, error) { - if deleted { - pool, err := openstackutil.GetPoolByName(os.Octavia, poolName, lbID) - if err != nil { - if err != openstackutil.ErrNotFound { - return nil, fmt.Errorf("error getting pool %s: %v", poolName, err) - } - return nil, nil - } - - // Delete the existing pool, members are deleted automatically - err = pools.Delete(os.Octavia, pool.ID).ExtractErr() - if err != nil && !cpoerrors.IsNotFound(err) { - return nil, fmt.Errorf("error deleting pool %s: %v", pool.ID, err) - } - - _, err = os.waitLoadbalancerActiveProvisioningStatus(lbID) - if err != nil { - return nil, fmt.Errorf("error waiting for loadbalancer %s to be active: %v", lbID, err) - } - - return nil, nil - } - - pool, err := openstackutil.GetPoolByName(os.Octavia, poolName, lbID) - if err != nil { - if err != openstackutil.ErrNotFound { - return nil, fmt.Errorf("error getting pool %s: %v", poolName, err) - } - - log.WithFields(log.Fields{"lb": lbID, "listenerID": listenerID, "poolName": poolName}).Debug("creating pool") - - // Create new pool - var opts pools.CreateOptsBuilder - if listenerID != "" { - opts = pools.CreateOpts{ - Name: poolName, - Protocol: "HTTP", - LBMethod: pools.LBMethodRoundRobin, - ListenerID: listenerID, - Persistence: nil, - } - } else { - opts = pools.CreateOpts{ - Name: poolName, - Protocol: "HTTP", - LBMethod: pools.LBMethodRoundRobin, - LoadbalancerID: lbID, - Persistence: nil, - } - } - pool, err = pools.Create(os.Octavia, opts).Extract() - if err != nil { - return nil, fmt.Errorf("error creating pool: %v", err) - } - - log.WithFields(log.Fields{"lb": lbID, "listenerID": listenerID, "poolName": poolName, "pooID": pool.ID}).Info("pool created") - - } - - _, err = os.waitLoadbalancerActiveProvisioningStatus(lbID) - if err != nil { - return nil, fmt.Errorf("error waiting for loadbalancer %s to be active: %v", lbID, err) - } - - // Batch update pool members - var members []pools.BatchUpdateMemberOpts - for _, node := range nodes { - addr, err := getNodeAddressForLB(node) - if err != nil { - // Node failure, do not create member - log.WithFields(log.Fields{"node": node.Name, "poolName": poolName, "pooID": pool.ID, "error": err}).Warn("failed to create LB pool member for node") - continue - } - - member := pools.BatchUpdateMemberOpts{ - Address: addr, - ProtocolPort: *nodePort, - } - members = append(members, member) - } - // only allow >= 1 members or it will lead to openstack octavia issue - if len(members) == 0 { - return nil, fmt.Errorf("error because no members in pool: %s", pool.ID) - } - - if err := pools.BatchUpdateMembers(os.Octavia, pool.ID, members).ExtractErr(); err != nil { - return nil, fmt.Errorf("error batch updating members for pool %s: %v", pool.ID, err) - } - _, err = os.waitLoadbalancerActiveProvisioningStatus(lbID) - if err != nil { - return nil, fmt.Errorf("error waiting for loadbalancer %s to be active: %v", lbID, err) - } - - log.WithFields(log.Fields{"lb": lbID, "listenerID": listenerID, "poolName": poolName, "pooID": pool.ID}).Info("pool members updated") - - return &pool.ID, nil -} - -// CreatePolicyRules creates l7 policy and its rules for the listener -func (os *OpenStack) CreatePolicyRules(lbID, listenerID, poolID, host, path string, port int) error { - log.WithFields(log.Fields{"lb": lbID, "listenerID": listenerID}).Debug("creating policy") - - policy, err := l7policies.Create(os.Octavia, l7policies.CreateOpts{ - ListenerID: listenerID, - Action: l7policies.ActionRedirectToPool, - Description: "Created by kubernetes ingress", - RedirectPoolID: poolID, - }).Extract() - if err != nil { - return fmt.Errorf("error creating l7policy: %v", err) - } - - _, err = os.waitLoadbalancerActiveProvisioningStatus(lbID) - if err != nil { - return fmt.Errorf("error waiting for loadbalancer %s to be active: %v", lbID, err) - } - - log.WithFields(log.Fields{"lb": lbID, "listenerID": listenerID, "policyID": policy.ID}).Info("policy created") - - if host != "" { - log.WithFields(log.Fields{"type": l7policies.TypeHostName, "host": host, "policyID": policy.ID, "listenerID": listenerID}).Debug("creating policy rule") - - // Create HOST_NAME type rule. Use REGEX type rule to support both host and host:port - _, err = l7policies.CreateRule(os.Octavia, policy.ID, l7policies.CreateRuleOpts{ - RuleType: l7policies.TypeHostName, - CompareType: l7policies.CompareTypeRegex, - Value: fmt.Sprintf("^%s(:%d)?$", strings.ReplaceAll(host, ".", "\\."), port), - }).Extract() - if err != nil { - return fmt.Errorf("error creating l7 rule: %v", err) - } - - _, err = os.waitLoadbalancerActiveProvisioningStatus(lbID) - if err != nil { - return fmt.Errorf("error waiting for loadbalancer %s to be active: %v", lbID, err) - } - - log.WithFields(log.Fields{"type": l7policies.TypeHostName, "host": host, "policyID": policy.ID, "listenerID": listenerID}).Info("policy rule created") - } - - if path != "" { - log.WithFields(log.Fields{"type": l7policies.TypePath, "path": path, "policyID": policy.ID, "listenerID": listenerID}).Debug("creating policy rule") - - // Create PATH type rule - _, err = l7policies.CreateRule(os.Octavia, policy.ID, l7policies.CreateRuleOpts{ - RuleType: l7policies.TypePath, - CompareType: l7policies.CompareTypeStartWith, - Value: path, - }).Extract() - if err != nil { - return fmt.Errorf("error creating l7 rule: %v", err) - } - - _, err = os.waitLoadbalancerActiveProvisioningStatus(lbID) - if err != nil { - return fmt.Errorf("error waiting for loadbalancer %s to be active: %v", lbID, err) - } - - log.WithFields(log.Fields{"type": l7policies.TypePath, "path": path, "policyID": policy.ID, "listenerID": listenerID}).Info("policy rule created") - } - - return nil -} - -// UpdateLoadbalancerMembers update members for all the pools in the specified load balancer. -func (os *OpenStack) UpdateLoadbalancerMembers(lbID string, nodes []*apiv1.Node) error { - lbPools, err := os.GetPools(lbID, false) - if err != nil { - return err - } - - for _, pool := range lbPools { - log.WithFields(log.Fields{"poolID": pool.ID}).Debug("Starting to update pool members") - - members, err := os.GetMembers(pool.ID) - if err != nil { - log.WithFields(log.Fields{"poolID": pool.ID}).Errorf("Failed to get pool members: %v", err) - continue - } - - // Members have the same ProtocolPort - nodePort := members[0].ProtocolPort - - if _, err = os.EnsurePoolMembers(false, pool.Name, lbID, "", &nodePort, nodes); err != nil { - return err - } - - log.WithFields(log.Fields{"poolID": pool.ID, "lbID": lbID}).Info("Finished to update pool members") - } - - return nil -} diff --git a/pkg/ingress/utils/utils.go b/pkg/ingress/utils/utils.go deleted file mode 100644 index 6f37bfe591..0000000000 --- a/pkg/ingress/utils/utils.go +++ /dev/null @@ -1,95 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package utils - -import ( - "crypto/sha256" - "fmt" - "regexp" - "sort" - - apiv1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/sets" -) - -// Hash gets the data hash. -func Hash(data string) string { - return fmt.Sprintf("%x", sha256.Sum256([]byte(data))) -} - -// GetResourceName get Ingress related resource name. -func GetResourceName(namespace, name, clusterName string) string { - // Keep consistent with the name of load balancer created for Service. - return fmt.Sprintf("kube_ingress_%s_%s_%s", clusterName, namespace, name) -} - -// NodeNames get all the node names. -func NodeNames(nodes []*apiv1.Node) []string { - ret := make([]string, len(nodes)) - for i, node := range nodes { - ret[i] = node.Name - } - return ret -} - -// NodeSlicesEqual check if two nodes equals to each other. -func NodeSlicesEqual(x, y []*apiv1.Node) bool { - if len(x) != len(y) { - return false - } - return stringSlicesEqual(NodeNames(x), NodeNames(y)) -} - -func stringSlicesEqual(x, y []string) bool { - if len(x) != len(y) { - return false - } - if !sort.StringsAreSorted(x) { - sort.Strings(x) - } - if !sort.StringsAreSorted(y) { - sort.Strings(y) - } - for i := range x { - if x[i] != y[i] { - return false - } - } - return true -} - -// GetNodeID get instance ID from the Node spec. -func GetNodeID(node *apiv1.Node) (string, error) { - var providerIDRegexp = regexp.MustCompile(`^openstack:///([^/]+)$`) - - matches := providerIDRegexp.FindStringSubmatch(node.Spec.ProviderID) - if len(matches) != 2 { - return "", fmt.Errorf("failed to find instance ID from node provider ID %s", node.Spec.ProviderID) - } - - return matches[1], nil -} - -// Convert2Set converts a string list to string set. -func Convert2Set(list []string) sets.String { - set := sets.NewString() - for _, s := range list { - set.Insert(s) - } - - return set -} diff --git a/pkg/kms/barbican/barbican.go b/pkg/kms/barbican/barbican.go deleted file mode 100644 index 06ddb27a7e..0000000000 --- a/pkg/kms/barbican/barbican.go +++ /dev/null @@ -1,53 +0,0 @@ -package barbican - -import ( - "github.com/gophercloud/gophercloud" - "github.com/gophercloud/gophercloud/openstack" - "github.com/gophercloud/gophercloud/openstack/keymanager/v1/secrets" - openstack_provider "k8s.io/cloud-provider-openstack/pkg/cloudprovider/providers/openstack" -) - -type BarbicanService interface { - GetSecret(keyID string) ([]byte, error) -} - -type KMSOpts struct { - KeyID string `gcfg:"key-id"` -} - -//Config to read config options -type Config struct { - Global openstack_provider.AuthOpts - KeyManager KMSOpts -} - -// Barbican is gophercloud service client -type Barbican struct { - Client *gophercloud.ServiceClient -} - -// NewBarbicanClient creates new BarbicanClient -func NewBarbicanClient(cfg Config) (*gophercloud.ServiceClient, error) { - provider, err := openstack_provider.NewOpenStackClient(&cfg.Global, "barbican-kms-plugin") - if err != nil { - return nil, err - } - - return openstack.NewKeyManagerV1(provider, gophercloud.EndpointOpts{ - Region: cfg.Global.Region, - }) -} - -// GetSecret gets unencrypted secret -func (barbican *Barbican) GetSecret(keyID string) ([]byte, error) { - opts := secrets.GetPayloadOpts{ - PayloadContentType: "application/octet-stream", - } - - key, err := secrets.GetPayload(barbican.Client, keyID, opts).Extract() - if err != nil { - return nil, err - } - - return key, nil -} diff --git a/pkg/kms/barbican/fake_barbican.go b/pkg/kms/barbican/fake_barbican.go deleted file mode 100644 index 4b2772cdb4..0000000000 --- a/pkg/kms/barbican/fake_barbican.go +++ /dev/null @@ -1,10 +0,0 @@ -package barbican - -import "encoding/hex" - -type FakeBarbican struct { -} - -func (client *FakeBarbican) GetSecret(keyID string) ([]byte, error) { - return hex.DecodeString("6368616e676520746869732070617373") -} diff --git a/pkg/kms/client/client.go b/pkg/kms/client/client.go deleted file mode 100644 index ace69eca1b..0000000000 --- a/pkg/kms/client/client.go +++ /dev/null @@ -1,54 +0,0 @@ -package main - -import ( - "fmt" - "golang.org/x/net/context" - "google.golang.org/grpc" - pb "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1" - "os" -) - -//This client is for test purpose only, Kubernetes api server will call to kms plugin grpc server - -func main() { - - connection, err := grpc.Dial("unix:///var/lib/kms/kms.sock", grpc.WithInsecure()) - defer connection.Close() - if err != nil { - fmt.Printf("\nConnection to KMS plugin failed, error: %v", err) - } - - kmsClient := pb.NewKeyManagementServiceClient(connection) - request := &pb.VersionRequest{Version: "v1beta1"} - _, err = kmsClient.Version(context.TODO(), request) - - if err != nil { - fmt.Printf("\nError in getting version from KMS Plugin: %v", err) - } - - secretBytes := []byte("mypassword") - - //Encryption Request to KMS Plugin - encRequest := &pb.EncryptRequest{ - Version: "v1beta1", - Plain: secretBytes} - encResponse, err := kmsClient.Encrypt(context.TODO(), encRequest) - - if err != nil { - fmt.Printf("\nEncrypt Request Failed: %v", err) - os.Exit(1) - } - - cipher := string(encResponse.Cipher) - fmt.Println("cipher:", cipher) - - //Decryption Request to KMS plugin - decRequest := &pb.DecryptRequest{ - Version: "v1beta1", - Cipher: encResponse.Cipher, - } - - decResponse, err := kmsClient.Decrypt(context.TODO(), decRequest) - - fmt.Printf("\n\ndecryption response %v", decResponse) -} diff --git a/pkg/kms/encryption/aescbc/aescbc.go b/pkg/kms/encryption/aescbc/aescbc.go deleted file mode 100644 index 8203053824..0000000000 --- a/pkg/kms/encryption/aescbc/aescbc.go +++ /dev/null @@ -1,72 +0,0 @@ -package aescbc - -import ( - "bytes" - "crypto/aes" - "crypto/cipher" - "crypto/rand" - "errors" - "io" - - "k8s.io/klog/v2" -) - -// Encrypt plain text -func Encrypt(data, key []byte) (ciphertext []byte, err error) { - - klog.V(3).Infof("aescbc encrypt") - - // NewCipher returns a new cipher block, the key argument should be AES key - // either 16, 24 or 32 bytes to select AES-128, AES-192, AES-256, 32 byte is preferred - block, err := aes.NewCipher(key) - if err != nil { - return nil, err - } - - // determine the padding size for block cipher - paddingSize := aes.BlockSize - (len(data) % aes.BlockSize) - plaintext := make([]byte, len(data)+paddingSize) - // copy data and padding - copy(plaintext, data) - copy(plaintext[len(data):], bytes.Repeat([]byte{byte(paddingSize)}, paddingSize)) - - // create slice to hold ciphertext, iv - ciphertext = make([]byte, aes.BlockSize+len(plaintext)) - iv := ciphertext[:aes.BlockSize] - if _, err := io.ReadFull(rand.Reader, iv); err != nil { - return nil, err - } - - mode := cipher.NewCBCEncrypter(block, iv) - mode.CryptBlocks(ciphertext[aes.BlockSize:], plaintext) - - klog.V(3).Infof("aescbc encrypt %s", string(ciphertext)) - - return -} - -// Decrypt plaintext -func Decrypt(data, key []byte) (plaintext []byte, err error) { - klog.V(3).Infof("aescbc decrypt") - block, err := aes.NewCipher(key) - if err != nil { - return nil, err - } - - iv := data[:aes.BlockSize] - ciphertext := data[aes.BlockSize:] - - if len(ciphertext)%aes.BlockSize != 0 { - return nil, errors.New("Invalid Data, not multiple of block size") - } - - mode := cipher.NewCBCDecrypter(block, iv) - mode.CryptBlocks(ciphertext, ciphertext) - - paddingLength := int(ciphertext[len(ciphertext)-1]) - dataLength := len(ciphertext) - paddingLength - plaintext = ciphertext[:dataLength] - klog.V(3).Infof("aescbc decrypt %s", string(plaintext)) - - return -} diff --git a/pkg/kms/encryption/aescbc/aescbc_test.go b/pkg/kms/encryption/aescbc/aescbc_test.go deleted file mode 100644 index f1d04c5759..0000000000 --- a/pkg/kms/encryption/aescbc/aescbc_test.go +++ /dev/null @@ -1,40 +0,0 @@ -package aescbc - -import ( - "bytes" - "crypto/rand" - "testing" -) - -var key []byte - -func init() { - // genereate key for encrypt decrypt operation - genKey() -} - -func TestEncryptDecrypt(t *testing.T) { - data := []byte("mypassword") - cipher, _ := Encrypt((data), key) - plain, _ := Decrypt(cipher, key) - if !bytes.Equal((data), plain) { - t.FailNow() - } -} - -// testKeyerror -func TestEncryptDecryptInvalidData(t *testing.T) { - data := []byte("mypassword") - cipher, err := Encrypt(data, key) - _, err = Decrypt(cipher[1:], key) - if err == nil { - t.FailNow() - } - t.Log(err) -} - -func genKey() { - key = make([]byte, 32) - _, _ = rand.Read(key) - -} diff --git a/pkg/kms/server/server.go b/pkg/kms/server/server.go deleted file mode 100644 index 90323a66e3..0000000000 --- a/pkg/kms/server/server.go +++ /dev/null @@ -1,137 +0,0 @@ -package server - -import ( - "fmt" - "net" - "os" - - "golang.org/x/net/context" - "golang.org/x/sys/unix" - "google.golang.org/grpc" - gcfg "gopkg.in/gcfg.v1" - pb "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1" - "k8s.io/cloud-provider-openstack/pkg/kms/barbican" - "k8s.io/cloud-provider-openstack/pkg/kms/encryption/aescbc" - "k8s.io/klog/v2" -) - -const ( - netProtocol = "unix" - version = "v1beta1" - runtimename = "barbican" - runtimeversion = "0.0.1" -) - -// KMSserver struct -type KMSserver struct { - cfg barbican.Config - barbican barbican.BarbicanService -} - -func initConfig(configFilePath string, cfg *barbican.Config) error { - config, err := os.Open(configFilePath) - defer config.Close() - if err != nil { - return err - } - err = gcfg.FatalOnly(gcfg.ReadInto(cfg, config)) - if err != nil { - return err - } - return nil -} - -// Run Grpc server for barbican KMS -func Run(configFilePath string, socketpath string, sigchan <-chan os.Signal) (err error) { - klog.Infof("Barbican KMS Plugin Starting Version: %s, RunTimeVersion: %s", version, runtimeversion) - s := new(KMSserver) - err = initConfig(configFilePath, &s.cfg) - if err != nil { - klog.V(4).Infof("Error in Getting Config File: %v", err) - return err - } - - client, err := barbican.NewBarbicanClient(s.cfg) - if err != nil { - klog.V(4).Infof("Failed to get Barbican client: %v", err) - return err - } - s.barbican = &barbican.Barbican{Client: client} - - // unlink the unix socket - if err = unix.Unlink(socketpath); err != nil { - klog.V(4).Infof("Error to unlink unix socket: %v", err) - } - - listener, err := net.Listen(netProtocol, socketpath) - if err != nil { - klog.Fatalf("Failed to Listen: %v", err) - return err - } - - gServer := grpc.NewServer() - pb.RegisterKeyManagementServiceServer(gServer, s) - - go gServer.Serve(listener) - - for { - sig := <-sigchan - if sig == unix.SIGINT || sig == unix.SIGTERM { - fmt.Println("force stop, shutting down grpc server") - gServer.GracefulStop() - return nil - } - } -} - -// Version returns KMS service version -func (s *KMSserver) Version(ctx context.Context, req *pb.VersionRequest) (*pb.VersionResponse, error) { - klog.V(4).Infof("Version Information Requested by Kubernetes api server") - - res := &pb.VersionResponse{ - Version: version, - RuntimeName: runtimename, - RuntimeVersion: runtimeversion, - } - - return res, nil -} - -// Decrypt decrypts the cipher -func (s *KMSserver) Decrypt(ctx context.Context, req *pb.DecryptRequest) (*pb.DecryptResponse, error) { - klog.V(4).Infof("Decrypt Request by Kubernetes api server") - - key, err := s.barbican.GetSecret(s.cfg.KeyManager.KeyID) - if err != nil { - klog.V(4).Infof("Failed to get key %v: ", err) - return nil, err - } - - plain, err := aescbc.Decrypt(req.Cipher, key) - if err != nil { - klog.V(4).Infof("Failed to decrypt data %v: ", err) - return nil, err - } - - return &pb.DecryptResponse{Plain: plain}, nil -} - -// Encrypt encrypts DEK -func (s *KMSserver) Encrypt(ctx context.Context, req *pb.EncryptRequest) (*pb.EncryptResponse, error) { - klog.V(4).Infof("Encrypt Request by Kubernetes api server") - - key, err := s.barbican.GetSecret(s.cfg.KeyManager.KeyID) - - if err != nil { - klog.V(4).Infof("Failed to get key %v: ", err) - return nil, err - } - - cipher, err := aescbc.Encrypt(req.Plain, key) - - if err != nil { - klog.V(4).Infof("Failed to encrypt data %v: ", err) - return nil, err - } - return &pb.EncryptResponse{Cipher: cipher}, nil -} diff --git a/pkg/kms/server/server_test.go b/pkg/kms/server/server_test.go deleted file mode 100644 index 3a895c2412..0000000000 --- a/pkg/kms/server/server_test.go +++ /dev/null @@ -1,40 +0,0 @@ -package server - -import ( - "bytes" - "testing" - - "golang.org/x/net/context" - pb "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1" - "k8s.io/cloud-provider-openstack/pkg/kms/barbican" -) - -var s = new(KMSserver) - -func TestInitConfig(t *testing.T) { -} - -func TestVersion(t *testing.T) { - req := &pb.VersionRequest{Version: "v1beta1"} - _, err := s.Version(context.TODO(), req) - if err != nil { - t.FailNow() - } -} - -func TestEncryptDecrypt(t *testing.T) { - s.barbican = &barbican.FakeBarbican{} - fakeData := []byte("fakedata") - encreq := &pb.EncryptRequest{Version: "v1beta1", Plain: fakeData} - encresp, err := s.Encrypt(context.TODO(), encreq) - if err != nil { - t.Log(err) - t.FailNow() - } - decreq := &pb.DecryptRequest{Version: "v1beta1", Cipher: encresp.Cipher} - decresp, err := s.Decrypt(context.TODO(), decreq) - if err != nil || !bytes.Equal(decresp.Plain, fakeData) { - t.Log(err) - t.FailNow() - } -} diff --git a/pkg/util/blockdevice/blockdevice_linux.go b/pkg/util/blockdevice/blockdevice_linux.go deleted file mode 100644 index c61c66ad39..0000000000 --- a/pkg/util/blockdevice/blockdevice_linux.go +++ /dev/null @@ -1,119 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package blockdevice - -import ( - "fmt" - "io" - "io/ioutil" - "os" - "path/filepath" - "strings" - - "golang.org/x/sys/unix" - - "k8s.io/klog/v2" -) - -// findBlockDeviceRescanPath Find the underlaying disk for a linked path such as /dev/disk/by-path/XXXX or /dev/mapper/XXXX -// will return /sys/devices/pci0000:00/0000:00:15.0/0000:03:00.0/host0/target0:0:1/0:0:1:0/rescan -func findBlockDeviceRescanPath(path string) (string, error) { - devicePath, err := filepath.EvalSymlinks(path) - if err != nil { - return "", err - } - // if path /dev/hdX split into "", "dev", "hdX" then we will - // return just the last part - parts := strings.Split(devicePath, "/") - if len(parts) == 3 && strings.HasPrefix(parts[1], "dev") { - return filepath.EvalSymlinks(filepath.Join("/sys/block", parts[2], "device", "rescan")) - } - return "", fmt.Errorf("illegal path for device " + devicePath) -} - -// IsBlockDevice checks whether device on the path is a block device -func IsBlockDevice(path string) (bool, error) { - var stat unix.Stat_t - err := unix.Stat(path, &stat) - if err != nil { - return false, fmt.Errorf("failed to stat() %q: %s", path, err) - } - - return (stat.Mode & unix.S_IFMT) == unix.S_IFBLK, nil -} - -// GetBlockDeviceSize returns the size of the block device by path -func GetBlockDeviceSize(path string) (int64, error) { - fd, err := os.Open(path) - if err != nil { - return 0, err - } - defer fd.Close() - pos, err := fd.Seek(0, io.SeekEnd) - if err != nil { - return 0, fmt.Errorf("error seeking to end of %s: %s", path, err) - } - return pos, nil -} - -func checkBlockDeviceSize(devicePath string, deviceMountPath string, newSize int64) error { - klog.V(4).Infof("Detecting %q volume size", deviceMountPath) - size, err := GetBlockDeviceSize(devicePath) - if err != nil { - return err - } - - klog.V(3).Infof("Detected %q volume size: %d", deviceMountPath, size) - - if size < newSize { - return fmt.Errorf("current volume size is less than expected one: %d < %d", size, newSize) - } - - return nil -} - -func RescanBlockDeviceGeometry(devicePath string, deviceMountPath string, newSize int64) error { - if newSize == 0 { - klog.Error("newSize is empty, skipping the block device rescan") - return nil - } - - // when block device size corresponds expectations, return nil - bdSizeErr := checkBlockDeviceSize(devicePath, deviceMountPath, newSize) - if bdSizeErr == nil { - return nil - } - - // don't fail if resolving doesn't work - blockDeviceRescanPath, err := findBlockDeviceRescanPath(devicePath) - if err != nil { - klog.Errorf("Error resolving block device path from %q: %v", devicePath, err) - // no need to run checkBlockDeviceSize second time here, return the saved error - return bdSizeErr - } - - klog.V(3).Infof("Resolved block device path from %q to %q", devicePath, blockDeviceRescanPath) - klog.V(4).Infof("Rescanning %q block device geometry", devicePath) - err = ioutil.WriteFile(blockDeviceRescanPath, []byte{'1'}, 0666) - if err != nil { - klog.Errorf("Error rescanning new block device geometry: %v", err) - // no need to run checkBlockDeviceSize second time here, return the saved error - return bdSizeErr - } - - return checkBlockDeviceSize(devicePath, deviceMountPath, newSize) -} diff --git a/pkg/util/blockdevice/blockdevice_unsupported.go b/pkg/util/blockdevice/blockdevice_unsupported.go deleted file mode 100644 index a19150574f..0000000000 --- a/pkg/util/blockdevice/blockdevice_unsupported.go +++ /dev/null @@ -1,35 +0,0 @@ -// +build !linux - -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package blockdevice - -import ( - "errors" -) - -func IsBlockDevice(path string) (bool, error) { - return false, errors.New("IsBlockDevice is not implemented for this OS") -} - -func GetBlockDeviceSize(path string) (int64, error) { - return -1, errors.New("GetBlockDeviceSize is not implemented for this OS") -} - -func RescanBlockDeviceGeometry(devicePath string, deviceMountPath string, newSize int64) error { - return errors.New("RescanBlockDeviceGeometry is not implemented for this OS") -} diff --git a/pkg/util/errors/errors.go b/pkg/util/errors/errors.go deleted file mode 100644 index 683fd43208..0000000000 --- a/pkg/util/errors/errors.go +++ /dev/null @@ -1,55 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package errors - -import ( - "net/http" - - "github.com/gophercloud/gophercloud" -) - -func IsNotFound(err error) bool { - if _, ok := err.(gophercloud.ErrDefault404); ok { - return true - } - - if _, ok := err.(gophercloud.ErrResourceNotFound); ok { - return true - } - - if errCode, ok := err.(gophercloud.ErrUnexpectedResponseCode); ok { - if errCode.Actual == http.StatusNotFound { - return true - } - } - - return false -} - -func IsInvalidError(err error) bool { - if _, ok := err.(gophercloud.ErrDefault400); ok { - return true - } - - if errCode, ok := err.(gophercloud.ErrUnexpectedResponseCode); ok { - if errCode.Actual == http.StatusBadRequest { - return true - } - } - - return false -} diff --git a/pkg/util/file/file.go b/pkg/util/file/file.go deleted file mode 100644 index 70d26c4ef7..0000000000 --- a/pkg/util/file/file.go +++ /dev/null @@ -1,57 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package file - -import ( - "os" -) - -// FileExists checks if specified file exists. -func FileExists(filename string) (bool, error) { - if _, err := os.Stat(filename); os.IsNotExist(err) { - return false, nil - } else if err != nil { - return false, err - } - return true, nil -} - -// FileOrSymlinkExists checks if specified file or symlink exists. -func FileOrSymlinkExists(filename string) (bool, error) { - if _, err := os.Lstat(filename); os.IsNotExist(err) { - return false, nil - } else if err != nil { - return false, err - } - return true, nil -} - -// ReadDirNoStat returns a string of files/directories contained -// in dirname without calling lstat on them. -func ReadDirNoStat(dirname string) ([]string, error) { - if dirname == "" { - dirname = "." - } - - f, err := os.Open(dirname) - if err != nil { - return nil, err - } - defer f.Close() - - return f.Readdirnames(-1) -} diff --git a/pkg/util/io/consistentread.go b/pkg/util/io/consistentread.go deleted file mode 100644 index 6e1f17b098..0000000000 --- a/pkg/util/io/consistentread.go +++ /dev/null @@ -1,45 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package io - -import ( - "bytes" - "fmt" - "io/ioutil" -) - -// ConsistentRead repeatedly reads a file until it gets the same content twice. -// This is useful when reading files in /proc that are larger than page size -// and kernel may modify them between individual read() syscalls. -func ConsistentRead(filename string, attempts int) ([]byte, error) { - oldContent, err := ioutil.ReadFile(filename) - if err != nil { - return nil, err - } - for i := 0; i < attempts; i++ { - newContent, err := ioutil.ReadFile(filename) - if err != nil { - return nil, err - } - if bytes.Compare(oldContent, newContent) == 0 { - return newContent, nil - } - // Files are different, continue reading - oldContent = newContent - } - return nil, fmt.Errorf("could not get consistent content of %s after %d attempts", filename, attempts) -} diff --git a/pkg/util/metadata/metadata.go b/pkg/util/metadata/metadata.go deleted file mode 100644 index fdc2cbf739..0000000000 --- a/pkg/util/metadata/metadata.go +++ /dev/null @@ -1,294 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package metadata - -import ( - "encoding/json" - "errors" - "fmt" - "io" - "io/ioutil" - "net/http" - "os" - "path/filepath" - "strings" - - "k8s.io/klog/v2" - - "k8s.io/cloud-provider-openstack/pkg/util/mount" - "k8s.io/utils/exec" -) - -const ( - // metadataURLTemplate allows building an OpenStack Metadata service URL. - // It's a hardcoded IPv4 link-local address as documented in "User Documentation" - // section "Metadata service - // https://docs.openstack.org/nova/latest/user/metadata-service.html - defaultMetadataVersion = "latest" - metadataURLTemplate = "http://169.254.169.254/openstack/%s/meta_data.json" - - // MetadataID is used as an identifier on the metadata search order configuration. - MetadataID = "metadataService" - - // Config drive is defined as an iso9660 or vfat (deprecated) drive - // with the "config-2" label. - //https://docs.openstack.org/nova/latest/user/config-drive.html - configDriveLabel = "config-2" - configDrivePathTemplate = "openstack/%s/meta_data.json" - - // ConfigDriveID is used as an identifier on the metadata search order configuration. - ConfigDriveID = "configDrive" -) - -// ErrBadMetadata is used to indicate a problem parsing data from metadata server -var ErrBadMetadata = errors.New("invalid OpenStack metadata, got empty uuid") - -// MetadataService instance of IMetadata -var MetadataService IMetadata - -// Metadata is fixed for the current host, so cache the value process-wide -var metadataCache *Metadata - -// DeviceMetadata is a single/simplified data structure for all kinds of device metadata types. -type DeviceMetadata struct { - Type string `json:"type"` - Bus string `json:"bus,omitempty"` - Serial string `json:"serial,omitempty"` - Address string `json:"address,omitempty"` - // .. and other fields. -} - -// Metadata has the information fetched from OpenStack metadata service or -// config drives. Assumes the "latest" meta_data.json format. -type Metadata struct { - UUID string `json:"uuid"` - Name string `json:"name"` - AvailabilityZone string `json:"availability_zone"` - Devices []DeviceMetadata `json:"devices,omitempty"` - // .. and other fields we don't care about. Expand as necessary. -} - -type metadataService struct { - searchOrder string -} - -// IMetadata implements GetInstanceID & GetAvailabilityZone -type IMetadata interface { - GetInstanceID() (string, error) - GetAvailabilityZone() (string, error) -} - -// GetMetadataProvider retrieves instance of IMetadata -func GetMetadataProvider(order string) IMetadata { - - if MetadataService == nil { - MetadataService = &metadataService{searchOrder: order} - } - return MetadataService -} - -// Set sets the value of metadatacache -func Set(value *Metadata) { - metadataCache = value -} - -// Clear clears the metadatacache -func Clear() { - metadataCache = nil -} - -// parseMetadata reads JSON from OpenStack metadata server and parses -// instance ID out of it. -func parseMetadata(r io.Reader) (*Metadata, error) { - var metadata Metadata - json := json.NewDecoder(r) - if err := json.Decode(&metadata); err != nil { - return nil, err - } - - if metadata.UUID == "" { - return nil, ErrBadMetadata - } - - return &metadata, nil -} - -func getMetadataURL(metadataVersion string) string { - return fmt.Sprintf(metadataURLTemplate, metadataVersion) -} - -func getConfigDrivePath(metadataVersion string) string { - return fmt.Sprintf(configDrivePathTemplate, metadataVersion) -} - -func getFromConfigDrive(metadataVersion string) (*Metadata, error) { - // Try to read instance UUID from config drive. - dev := "/dev/disk/by-label/" + configDriveLabel - if _, err := os.Stat(dev); os.IsNotExist(err) { - out, err := exec.New().Command( - "blkid", "-l", - "-t", "LABEL="+configDriveLabel, - "-o", "device", - ).CombinedOutput() - if err != nil { - return nil, fmt.Errorf("unable to run blkid: %v", err) - } - dev = strings.TrimSpace(string(out)) - } - - mntdir, err := ioutil.TempDir("", "configdrive") - if err != nil { - return nil, err - } - defer os.Remove(mntdir) - - klog.V(4).Infof("Attempting to mount configdrive %s on %s", dev, mntdir) - - mounter := mount.GetMountProvider().Mounter() - err = mounter.Mount(dev, mntdir, "iso9660", []string{"ro"}) - if err != nil { - err = mounter.Mount(dev, mntdir, "vfat", []string{"ro"}) - } - if err != nil { - return nil, fmt.Errorf("error mounting configdrive %s: %v", dev, err) - } - defer mounter.Unmount(mntdir) - - klog.V(4).Infof("Configdrive mounted on %s", mntdir) - - configDrivePath := getConfigDrivePath(metadataVersion) - f, err := os.Open( - filepath.Join(mntdir, configDrivePath)) - if err != nil { - return nil, fmt.Errorf("error reading %s on config drive: %v", configDrivePath, err) - } - defer f.Close() - - return parseMetadata(f) -} - -func getFromMetadataService(metadataVersion string) (*Metadata, error) { - // Try to get JSON from metadata server. - metadataURL := getMetadataURL(metadataVersion) - klog.V(4).Infof("Attempting to fetch metadata from %s", metadataURL) - resp, err := http.Get(metadataURL) - if err != nil { - return nil, fmt.Errorf("error fetching %s: %v", metadataURL, err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - err = fmt.Errorf("unexpected status code when reading metadata from %s: %s", metadataURL, resp.Status) - return nil, err - } - - return parseMetadata(resp.Body) -} - -// GetDevicePath retrieves device path from metadata service -func GetDevicePath(volumeID string) string { - // Nova Hyper-V hosts cannot override disk SCSI IDs. In order to locate - // volumes, we're querying the metadata service. Note that the Hyper-V - // driver will include device metadata for untagged volumes as well. - // - // We're avoiding using cached metadata (or the configdrive), - // relying on the metadata service. - instanceMetadata, err := getFromMetadataService(defaultMetadataVersion) - - if err != nil { - klog.V(4).Infof("Could not retrieve instance metadata. Error: %v", err) - return "" - } - - for _, device := range instanceMetadata.Devices { - if device.Type == "disk" && device.Serial == volumeID { - klog.V(4).Infof( - "Found disk metadata for volumeID %q. Bus: %q, Address: %q", - volumeID, device.Bus, device.Address) - - diskPattern := fmt.Sprintf("/dev/disk/by-path/*-%s-%s", device.Bus, device.Address) - diskPaths, err := filepath.Glob(diskPattern) - if err != nil { - klog.Errorf( - "could not retrieve disk path for volumeID: %q. Error filepath.Glob(%q): %v", - volumeID, diskPattern, err) - return "" - } - - if len(diskPaths) == 1 { - return diskPaths[0] - } - - klog.V(4).Infof("expecting to find one disk path for volumeID %q, found %d: %v", - volumeID, len(diskPaths), diskPaths) - - } - } - - klog.V(4).Infof("Could not retrieve device metadata for volumeID: %q", volumeID) - return "" -} - -// Get retrieves metadata from either config drive or metadata service. -// Search order depends on the order set in config file. -func Get(order string) (*Metadata, error) { - if metadataCache == nil { - var md *Metadata - var err error - - elements := strings.Split(order, ",") - for _, id := range elements { - id = strings.TrimSpace(id) - switch id { - case ConfigDriveID: - md, err = getFromConfigDrive(defaultMetadataVersion) - case MetadataID: - md, err = getFromMetadataService(defaultMetadataVersion) - default: - err = fmt.Errorf("%s is not a valid metadata search order option. Supported options are %s and %s", id, ConfigDriveID, MetadataID) - } - - if err == nil { - break - } - } - - if err != nil { - return nil, err - } - metadataCache = md - } - return metadataCache, nil -} - -// GetInstanceID return instance ID of the node -func (m *metadataService) GetInstanceID() (string, error) { - md, err := Get(m.searchOrder) - if err != nil { - return "", err - } - return md.UUID, nil -} - -// GetAvailabilityZone returns AZ of the node -func (m *metadataService) GetAvailabilityZone() (string, error) { - md, err := Get(m.searchOrder) - if err != nil { - return "", err - } - return md.AvailabilityZone, nil -} diff --git a/pkg/util/metadata/metadata_mock.go b/pkg/util/metadata/metadata_mock.go deleted file mode 100644 index b5c8111ee0..0000000000 --- a/pkg/util/metadata/metadata_mock.go +++ /dev/null @@ -1,50 +0,0 @@ -package metadata - -import mock "github.com/stretchr/testify/mock" - -// MetadataMock is an autogenerated mock type for the IMetadata type -type MetadataMock struct { - mock.Mock -} - -// GetAvailabilityZone provides a mock function with given fields: -func (_m *MetadataMock) GetAvailabilityZone() (string, error) { - ret := _m.Called() - - var r0 string - if rf, ok := ret.Get(0).(func() string); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(string) - } - - var r1 error - if rf, ok := ret.Get(1).(func() error); ok { - r1 = rf() - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// GetInstanceID provides a mock function with given fields: -func (_m *MetadataMock) GetInstanceID() (string, error) { - ret := _m.Called() - - var r0 string - if rf, ok := ret.Get(0).(func() string); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(string) - } - - var r1 error - if rf, ok := ret.Get(1).(func() error); ok { - r1 = rf() - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} diff --git a/pkg/util/metadata/metadata_test.go b/pkg/util/metadata/metadata_test.go deleted file mode 100644 index f0df823c91..0000000000 --- a/pkg/util/metadata/metadata_test.go +++ /dev/null @@ -1,107 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package metadata - -import ( - "strings" - "testing" -) - -var FakeMetadata = Metadata{ - UUID: "83679162-1378-4288-a2d4-70e13ec132aa", - Name: "test", - AvailabilityZone: "nova", -} - -func TestParseMetadata(t *testing.T) { - _, err := parseMetadata(strings.NewReader("bogus")) - if err == nil { - t.Errorf("Should fail when bad data is provided: %s", err) - } - - data := strings.NewReader(` -{ - "availability_zone": "nova", - "files": [ - { - "content_path": "/content/0000", - "path": "/etc/network/interfaces" - }, - { - "content_path": "/content/0001", - "path": "known_hosts" - } - ], - "hostname": "test.novalocal", - "launch_index": 0, - "name": "test", - "meta": { - "role": "webservers", - "essential": "false" - }, - "public_keys": { - "mykey": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQDBqUfVvCSez0/Wfpd8dLLgZXV9GtXQ7hnMN+Z0OWQUyebVEHey1CXuin0uY1cAJMhUq8j98SiW+cU0sU4J3x5l2+xi1bodDm1BtFWVeLIOQINpfV1n8fKjHB+ynPpe1F6tMDvrFGUlJs44t30BrujMXBe8Rq44cCk6wqyjATA3rQ== Generated by Nova\n" - }, - "uuid": "83679162-1378-4288-a2d4-70e13ec132aa", - "devices": [ - { - "bus": "scsi", - "serial": "6df1888b-f373-41cf-b960-3786e60a28ef", - "tags": ["fake_tag"], - "type": "disk", - "address": "0:0:0:0" - } - ] -} -`) - md, err := parseMetadata(data) - if err != nil { - t.Fatalf("Should succeed when provided with valid data: %s", err) - } - - if md.Name != "test" { - t.Errorf("incorrect name: %s", md.Name) - } - - if md.UUID != "83679162-1378-4288-a2d4-70e13ec132aa" { - t.Errorf("incorrect uuid: %s", md.UUID) - } - - if md.AvailabilityZone != "nova" { - t.Errorf("incorrect az: %s", md.AvailabilityZone) - } - - if len(md.Devices) != 1 { - t.Errorf("expecting to find 1 device, found %d", len(md.Devices)) - } - - if md.Devices[0].Bus != "scsi" { - t.Errorf("incorrect disk bus: %s", md.Devices[0].Bus) - } - - if md.Devices[0].Address != "0:0:0:0" { - t.Errorf("incorrect disk address: %s", md.Devices[0].Address) - } - - if md.Devices[0].Type != "disk" { - t.Errorf("incorrect device type: %s", md.Devices[0].Type) - } - - if md.Devices[0].Serial != "6df1888b-f373-41cf-b960-3786e60a28ef" { - t.Errorf("incorrect device serial: %s", md.Devices[0].Serial) - } -} diff --git a/pkg/util/mount/mount.go b/pkg/util/mount/mount.go deleted file mode 100644 index c580930840..0000000000 --- a/pkg/util/mount/mount.go +++ /dev/null @@ -1,281 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package mount - -import ( - "fmt" - "io/ioutil" - "os" - "path" - "strings" - "time" - - "golang.org/x/sys/unix" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/klog/v2" - "k8s.io/utils/exec" - "k8s.io/utils/mount" - - "k8s.io/cloud-provider-openstack/pkg/util/blockdevice" -) - -const ( - probeVolumeDuration = 1 * time.Second - probeVolumeTimeout = 60 * time.Second - operationFinishInitDelay = 1 * time.Second - operationFinishFactor = 1.1 - operationFinishSteps = 15 -) - -type IMount interface { - Mounter() *mount.SafeFormatAndMount - ScanForAttach(devicePath string) error - GetDevicePath(volumeID string) (string, error) - IsLikelyNotMountPointAttach(targetpath string) (bool, error) - UnmountPath(mountPath string) error - MakeFile(pathname string) error - MakeDir(pathname string) error - GetDeviceStats(path string) (*DeviceStats, error) -} - -type DeviceStats struct { - Block bool - - AvailableBytes int64 - TotalBytes int64 - UsedBytes int64 - AvailableInodes int64 - TotalInodes int64 - UsedInodes int64 -} - -type Mount struct { - BaseMounter *mount.SafeFormatAndMount -} - -var _ IMount = &Mount{} - -var MInstance IMount = nil - -func getBaseMounter() *mount.SafeFormatAndMount { - nMounter := mount.New("") - nExec := exec.New() - return &mount.SafeFormatAndMount{ - Interface: nMounter, - Exec: nExec, - } -} - -//GetMountProvider returns instance of Mounter -func GetMountProvider() IMount { - if MInstance == nil { - MInstance = &Mount{BaseMounter: getBaseMounter()} - } - return MInstance -} - -// Mounter returns instance of SafeFormatAndMount -func (m *Mount) Mounter() *mount.SafeFormatAndMount { - return m.BaseMounter -} - -// probeVolume probes volume in compute -func probeVolume() error { - // rescan scsi bus - scsiPath := "/sys/class/scsi_host/" - if dirs, err := ioutil.ReadDir(scsiPath); err == nil { - for _, f := range dirs { - name := scsiPath + f.Name() + "/scan" - data := []byte("- - -") - ioutil.WriteFile(name, data, 0666) - } - } - - executor := exec.New() - args := []string{"trigger"} - cmd := executor.Command("udevadm", args...) - _, err := cmd.CombinedOutput() - if err != nil { - klog.V(3).Infof("error running udevadm trigger %v\n", err) - return err - } - return nil -} - -// GetDevicePath returns the path of an attached block storage volume, specified by its id. -func (m *Mount) GetDevicePath(volumeID string) (string, error) { - backoff := wait.Backoff{ - Duration: operationFinishInitDelay, - Factor: operationFinishFactor, - Steps: operationFinishSteps, - } - - var devicePath string - err := wait.ExponentialBackoff(backoff, func() (bool, error) { - devicePath = m.getDevicePathBySerialID(volumeID) - if devicePath != "" { - return true, nil - } - // see issue https://github.com/kubernetes/cloud-provider-openstack/issues/705 - probeVolume() - return false, nil - }) - - if err == wait.ErrWaitTimeout { - return "", fmt.Errorf("Failed to find device for the volumeID: %q within the alloted time", volumeID) - } else if devicePath == "" { - return "", fmt.Errorf("Device path was empty for volumeID: %q", volumeID) - } - return devicePath, nil -} - -// GetDevicePathBySerialID returns the path of an attached block storage volume, specified by its id. -func (m *Mount) getDevicePathBySerialID(volumeID string) string { - // Build a list of candidate device paths. - // Certain Nova drivers will set the disk serial ID, including the Cinder volume id. - candidateDeviceNodes := []string{ - // KVM - fmt.Sprintf("virtio-%s", volumeID[:20]), - // KVM #852 - fmt.Sprintf("virtio-%s", volumeID), - // KVM virtio-scsi - fmt.Sprintf("scsi-0QEMU_QEMU_HARDDISK_%s", volumeID[:20]), - // KVM virtio-scsi #852 - fmt.Sprintf("scsi-0QEMU_QEMU_HARDDISK_%s", volumeID), - // ESXi - fmt.Sprintf("wwn-0x%s", strings.Replace(volumeID, "-", "", -1)), - } - - files, err := ioutil.ReadDir("/dev/disk/by-id/") - if err != nil { - klog.V(4).Infof("ReadDir failed with error %v", err) - } - - for _, f := range files { - for _, c := range candidateDeviceNodes { - if c == f.Name() { - klog.V(4).Infof("Found disk attached as %q; full devicepath: %s\n", - f.Name(), path.Join("/dev/disk/by-id/", f.Name())) - return path.Join("/dev/disk/by-id/", f.Name()) - } - } - } - - klog.V(4).Infof("Failed to find device for the volumeID: %q by serial ID", volumeID) - return "" -} - -// ScanForAttach -func (m *Mount) ScanForAttach(devicePath string) error { - ticker := time.NewTicker(probeVolumeDuration) - defer ticker.Stop() - timer := time.NewTimer(probeVolumeTimeout) - defer timer.Stop() - - for { - select { - case <-ticker.C: - klog.V(5).Infof("Checking Cinder disk %q is attached.", devicePath) - probeVolume() - - exists, err := mount.PathExists(devicePath) - if exists && err == nil { - return nil - } - klog.V(3).Infof("Could not find attached Cinder disk %s", devicePath) - case <-timer.C: - return fmt.Errorf("Could not find attached Cinder disk %s. Timeout waiting for mount paths to be created.", devicePath) - } - } -} - -// IsLikelyNotMountPointAttach -func (m *Mount) IsLikelyNotMountPointAttach(targetpath string) (bool, error) { - notMnt, err := m.BaseMounter.IsLikelyNotMountPoint(targetpath) - if err != nil { - if os.IsNotExist(err) { - err = os.MkdirAll(targetpath, 0750) - if err == nil { - notMnt = true - } - } - } - return notMnt, err -} - -// UnmountPath -func (m *Mount) UnmountPath(mountPath string) error { - return mount.CleanupMountPoint(mountPath, m.BaseMounter, false /* extensiveMountPointCheck */) -} - -// MakeDir creates dir -func (m *Mount) MakeDir(pathname string) error { - err := os.MkdirAll(pathname, os.FileMode(0755)) - if err != nil { - if !os.IsExist(err) { - return err - } - } - return nil -} - -// MakeFile creates an empty file -func (m *Mount) MakeFile(pathname string) error { - f, err := os.OpenFile(pathname, os.O_CREATE, os.FileMode(0644)) - defer f.Close() - if err != nil { - if !os.IsExist(err) { - return err - } - } - return nil -} - -func (m *Mount) GetDeviceStats(path string) (*DeviceStats, error) { - isBlock, err := blockdevice.IsBlockDevice(path) - - if isBlock { - size, err := blockdevice.GetBlockDeviceSize(path) - if err != nil { - return nil, err - } - - return &DeviceStats{ - Block: true, - TotalBytes: size, - }, nil - } - - var statfs unix.Statfs_t - // See http://man7.org/linux/man-pages/man2/statfs.2.html for details. - err = unix.Statfs(path, &statfs) - if err != nil { - return nil, err - } - - return &DeviceStats{ - Block: false, - - AvailableBytes: int64(statfs.Bavail) * int64(statfs.Bsize), - TotalBytes: int64(statfs.Blocks) * int64(statfs.Bsize), - UsedBytes: (int64(statfs.Blocks) - int64(statfs.Bfree)) * int64(statfs.Bsize), - - AvailableInodes: int64(statfs.Ffree), - TotalInodes: int64(statfs.Files), - UsedInodes: int64(statfs.Files) - int64(statfs.Ffree), - }, nil -} diff --git a/pkg/util/mount/mount_mock.go b/pkg/util/mount/mount_mock.go deleted file mode 100644 index e392e43e78..0000000000 --- a/pkg/util/mount/mount_mock.go +++ /dev/null @@ -1,190 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package mount - -import ( - mock "github.com/stretchr/testify/mock" - utilsexec "k8s.io/utils/exec" - exec "k8s.io/utils/exec/testing" - "k8s.io/utils/mount" -) - -// MountMock is an autogenerated mock type for the IMount type -// ORIGINALLY GENERATED BY mockery with hand edits -type MountMock struct { - mock.Mock -} - -func (_m *MountMock) GetDeviceStats(path string) (*DeviceStats, error) { - ret := _m.Called(path) - - return ret.Get(0).(*DeviceStats), ret.Error(1) -} - -// NewFakeMounter returns fake mounter instance -func NewFakeMounter() *mount.FakeMounter { - return &mount.FakeMounter{ - MountPoints: []mount.MountPoint{}, - } -} - -// NewFakeSafeFormatAndMounter returns base Fake mounter instance -func NewFakeSafeFormatAndMounter() *mount.SafeFormatAndMount { - return &mount.SafeFormatAndMount{ - Interface: NewFakeMounter(), - Exec: &exec.FakeExec{DisableScripts: true}, - } -} - -// GetInstanceID provides a mock function with given fields: -func (_m *MountMock) GetInstanceID() (string, error) { - ret := _m.Called() - - var r0 string - if rf, ok := ret.Get(0).(func() string); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(string) - } - - var r1 error - if rf, ok := ret.Get(1).(func() error); ok { - r1 = rf() - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// IsLikelyNotMountPointAttach provides a mock function with given fields: targetpath -func (_m *MountMock) IsLikelyNotMountPointAttach(targetpath string) (bool, error) { - ret := _m.Called(targetpath) - - var r0 bool - if rf, ok := ret.Get(0).(func(string) bool); ok { - r0 = rf(targetpath) - } else { - r0 = ret.Get(0).(bool) - } - - var r1 error - if rf, ok := ret.Get(1).(func(string) error); ok { - r1 = rf(targetpath) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// ScanForAttach provides a mock function with given fields: devicePath -func (_m *MountMock) ScanForAttach(devicePath string) error { - ret := _m.Called(devicePath) - - var r0 error - if rf, ok := ret.Get(0).(func(string) error); ok { - r0 = rf(devicePath) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// GetDevicePath provides a mock function with given fields: volumeID -func (_m *MountMock) GetDevicePath(volumeID string) (string, error) { - ret := _m.Called(volumeID) - - var r0 string - if rf, ok := ret.Get(0).(func(string) string); ok { - r0 = rf(volumeID) - } else { - r0 = ret.Get(0).(string) - } - - var r1 error - if rf, ok := ret.Get(1).(func(string) error); ok { - r1 = rf(volumeID) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// UnmountPath provides a mock function with given fields: mountPath -func (_m *MountMock) UnmountPath(mountPath string) error { - ret := _m.Called(mountPath) - - var r0 error - if rf, ok := ret.Get(0).(func(string) error); ok { - r0 = rf(mountPath) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// GetBaseMounter provides a mock function -func (_m *MountMock) Mounter() *mount.SafeFormatAndMount { - scripts := []struct { - cmd string - output []byte - err error - }{ - { - cmd: "findmnt", - output: []byte("devicepath"), - err: nil, - }, - { - cmd: "blkid", - output: []byte("UUID=\"1b47881a-1563-4896-a178-eec887b759de\" \n TYPE=\"ext4\""), - err: nil, - }, - } - - fakeexec := &exec.FakeExec{} - for _, s := range scripts { - fakeCmd := &exec.FakeCmd{ - CombinedOutputScript: []exec.FakeAction{ - func() ([]byte, []byte, error) { return s.output, nil, s.err }, - }, - } - cmdAction := func(cmd string, args ...string) utilsexec.Cmd { - command := exec.InitFakeCmd(fakeCmd, s.cmd) - return command - } - fakeexec.CommandScript = append(fakeexec.CommandScript, cmdAction) - } - - return &mount.SafeFormatAndMount{ - Interface: NewFakeMounter(), - Exec: fakeexec, - } -} - -func (_m *MountMock) MakeDir(pathname string) error { - return nil -} - -// MakeFile creates an empty file -func (_m *MountMock) MakeFile(pathname string) error { - return nil -} diff --git a/pkg/util/net/sets/ipnet.go b/pkg/util/net/sets/ipnet.go deleted file mode 100644 index 90ad58c635..0000000000 --- a/pkg/util/net/sets/ipnet.go +++ /dev/null @@ -1,121 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package sets - -import ( - "net" - "strings" -) - -// IPNet maps string to net.IPNet. -type IPNet map[string]*net.IPNet - -// ParseIPNets parses string slice to IPNet. -func ParseIPNets(specs ...string) (IPNet, error) { - ipnetset := make(IPNet) - for _, spec := range specs { - spec = strings.TrimSpace(spec) - _, ipnet, err := net.ParseCIDR(spec) - if err != nil { - return nil, err - } - k := ipnet.String() // In case of normalization - ipnetset[k] = ipnet - } - return ipnetset, nil -} - -// Insert adds items to the set. -func (s IPNet) Insert(items ...*net.IPNet) { - for _, item := range items { - s[item.String()] = item - } -} - -// Delete removes all items from the set. -func (s IPNet) Delete(items ...*net.IPNet) { - for _, item := range items { - delete(s, item.String()) - } -} - -// Has returns true if and only if item is contained in the set. -func (s IPNet) Has(item *net.IPNet) bool { - _, contained := s[item.String()] - return contained -} - -// HasAll returns true if and only if all items are contained in the set. -func (s IPNet) HasAll(items ...*net.IPNet) bool { - for _, item := range items { - if !s.Has(item) { - return false - } - } - return true -} - -// Difference returns a set of objects that are not in s2 -// For example: -// s1 = {a1, a2, a3} -// s2 = {a1, a2, a4, a5} -// s1.Difference(s2) = {a3} -// s2.Difference(s1) = {a4, a5} -func (s IPNet) Difference(s2 IPNet) IPNet { - result := make(IPNet) - for k, i := range s { - _, found := s2[k] - if found { - continue - } - result[k] = i - } - return result -} - -// StringSlice returns a []string with the String representation of each element in the set. -// Order is undefined. -func (s IPNet) StringSlice() []string { - a := make([]string, 0, len(s)) - for k := range s { - a = append(a, k) - } - return a -} - -// IsSuperset returns true if and only if s1 is a superset of s2. -func (s IPNet) IsSuperset(s2 IPNet) bool { - for k := range s2 { - _, found := s[k] - if !found { - return false - } - } - return true -} - -// Equal returns true if and only if s1 is equal (as a set) to s2. -// Two sets are equal if their membership is identical. -// (In practice, this means same elements, order doesn't matter) -func (s IPNet) Equal(s2 IPNet) bool { - return len(s) == len(s2) && s.IsSuperset(s2) -} - -// Len returns the size of the set. -func (s IPNet) Len() int { - return len(s) -} diff --git a/pkg/util/openstack/keymanager.go b/pkg/util/openstack/keymanager.go deleted file mode 100644 index 8e027b237e..0000000000 --- a/pkg/util/openstack/keymanager.go +++ /dev/null @@ -1,123 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package openstack - -import ( - "fmt" - "strings" - - "github.com/gophercloud/gophercloud" - "github.com/gophercloud/gophercloud/openstack/keymanager/v1/secrets" -) - -// EnsureSecret creates a secret if it doesn't exist. -func EnsureSecret(client *gophercloud.ServiceClient, name string, secretType string, payload string) (string, error) { - secret, err := GetSecret(client, name) - if err != nil { - if err == ErrNotFound { - // Create a new one - return CreateSecret(client, name, secretType, payload) - } - - return "", err - } - - return secret.SecretRef, nil -} - -// GetSecret returns the secret by name -func GetSecret(client *gophercloud.ServiceClient, name string) (*secrets.Secret, error) { - listOpts := secrets.ListOpts{ - Name: name, - } - allPages, err := secrets.List(client, listOpts).AllPages() - if err != nil { - return nil, err - } - allSecrets, err := secrets.ExtractSecrets(allPages) - if err != nil { - return nil, err - } - - if len(allSecrets) == 0 { - return nil, ErrNotFound - } - if len(allSecrets) > 1 { - return nil, ErrMultipleResults - } - - return &allSecrets[0], nil -} - -// CreateSecret creates a secret in Barbican, returns the secret url. -func CreateSecret(client *gophercloud.ServiceClient, name string, secretType string, payload string) (string, error) { - createOpts := secrets.CreateOpts{ - Name: name, - Algorithm: "aes", - Mode: "cbc", - BitLength: 256, - PayloadContentType: secretType, - PayloadContentEncoding: "base64", - Payload: payload, - SecretType: secrets.OpaqueSecret, - } - secret, err := secrets.Create(client, createOpts).Extract() - if err != nil { - return "", err - } - return secret.SecretRef, nil -} - -// ParseSecretID return secret ID from serectRef -func ParseSecretID(ref string) (string, error) { - parts := strings.Split(ref, "/") - if len(parts) < 2 { - return "", fmt.Errorf("Could not parse %s", ref) - } - - return parts[len(parts)-1], nil -} - -// DeleteSecrets deletes all the secrets that including the name string. -func DeleteSecrets(client *gophercloud.ServiceClient, partName string) error { - listOpts := secrets.ListOpts{ - SecretType: secrets.OpaqueSecret, - } - allPages, err := secrets.List(client, listOpts).AllPages() - if err != nil { - return err - } - allSecrets, err := secrets.ExtractSecrets(allPages) - if err != nil { - return err - } - - for _, s := range allSecrets { - if strings.Contains(s.Name, partName) { - secretID, err := ParseSecretID(s.SecretRef) - if err != nil { - return err - } - err = secrets.Delete(client, secretID).ExtractErr() - if err != nil { - return err - } - } - } - - return nil -} diff --git a/pkg/util/openstack/loadbalancer.go b/pkg/util/openstack/loadbalancer.go deleted file mode 100644 index 529e1d4f3a..0000000000 --- a/pkg/util/openstack/loadbalancer.go +++ /dev/null @@ -1,341 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package openstack - -import ( - "errors" - "fmt" - "time" - - "github.com/gophercloud/gophercloud" - "github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/apiversions" - "github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/listeners" - "github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/loadbalancers" - "github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/pools" - "github.com/gophercloud/gophercloud/pagination" - version "github.com/hashicorp/go-version" - "k8s.io/apimachinery/pkg/util/wait" - klog "k8s.io/klog/v2" - - cpoerrors "k8s.io/cloud-provider-openstack/pkg/util/errors" -) - -const ( - OctaviaFeatureTags = 0 - OctaviaFeatureVIPACL = 1 - OctaviaFeatureFlavors = 2 - - loadbalancerActiveInitDelay = 1 * time.Second - loadbalancerActiveFactor = 1.2 - loadbalancerActiveSteps = 19 - - activeStatus = "ACTIVE" - errorStatus = "ERROR" -) - -var ( - octaviaVersion string - - // ErrNotFound is used to inform that the object is missing - ErrNotFound = errors.New("failed to find object") - - // ErrMultipleResults is used when we unexpectedly get back multiple results - ErrMultipleResults = errors.New("multiple results where only one expected") -) - -// getOctaviaVersion returns the current Octavia API version. -func getOctaviaVersion(client *gophercloud.ServiceClient) (string, error) { - if octaviaVersion != "" { - return octaviaVersion, nil - } - - var defaultVer = "0.0" - allPages, err := apiversions.List(client).AllPages() - if err != nil { - return defaultVer, err - } - versions, err := apiversions.ExtractAPIVersions(allPages) - if err != nil { - return defaultVer, err - } - if len(versions) == 0 { - return defaultVer, fmt.Errorf("API versions for Octavia not found") - } - - klog.V(4).Infof("Found Octavia API versions: %v", versions) - - // The current version is always the last one in the list - octaviaVersion = versions[len(versions)-1].ID - klog.V(4).Infof("The current Octavia API version: %v", octaviaVersion) - - return octaviaVersion, nil -} - -// IsOctaviaFeatureSupported returns true if the given feature is supported in the deployed Octavia version. -func IsOctaviaFeatureSupported(client *gophercloud.ServiceClient, feature int) bool { - octaviaVer, err := getOctaviaVersion(client) - if err != nil { - klog.Warningf("Failed to get current Octavia API version: %v", err) - return false - } - - currentVer, _ := version.NewVersion(octaviaVer) - - switch feature { - case OctaviaFeatureVIPACL: - verACL, _ := version.NewVersion("v2.12") - if currentVer.GreaterThanOrEqual(verACL) { - return true - } - case OctaviaFeatureTags: - verTags, _ := version.NewVersion("v2.5") - if currentVer.GreaterThanOrEqual(verTags) { - return true - } - case OctaviaFeatureFlavors: - verFlavors, _ := version.NewVersion("v2.6") - if currentVer.GreaterThanOrEqual(verFlavors) { - return true - } - default: - klog.Warningf("Feature %d not recognized", feature) - } - - return false -} - -func waitLoadbalancerActive(client *gophercloud.ServiceClient, loadbalancerID string) error { - backoff := wait.Backoff{ - Duration: loadbalancerActiveInitDelay, - Factor: loadbalancerActiveFactor, - Steps: loadbalancerActiveSteps, - } - - err := wait.ExponentialBackoff(backoff, func() (bool, error) { - loadbalancer, err := loadbalancers.Get(client, loadbalancerID).Extract() - if err != nil { - return false, err - } - if loadbalancer.ProvisioningStatus == activeStatus { - return true, nil - } else if loadbalancer.ProvisioningStatus == errorStatus { - return true, fmt.Errorf("loadbalancer has gone into ERROR state") - } else { - return false, nil - } - - }) - - return err -} - -// UpdateListener updates a listener and wait for the lb active -func UpdateListener(client *gophercloud.ServiceClient, lbID string, listenerID string, opts listeners.UpdateOpts) error { - if _, err := listeners.Update(client, listenerID, opts).Extract(); err != nil { - return err - } - - if err := waitLoadbalancerActive(client, lbID); err != nil { - return fmt.Errorf("failed to wait for load balancer ACTIVE after updating listener: %v", err) - } - - return nil -} - -// CreateListener creates a new listener -func CreateListener(client *gophercloud.ServiceClient, lbID string, opts listeners.CreateOpts) (*listeners.Listener, error) { - listener, err := listeners.Create(client, opts).Extract() - if err != nil { - return nil, err - } - - if err := waitLoadbalancerActive(client, lbID); err != nil { - return nil, fmt.Errorf("failed to wait for load balancer ACTIVE after creating listener: %v", err) - } - - return listener, nil -} - -// GetLoadbalancerByName retrieves loadbalancer object -func GetLoadbalancerByName(client *gophercloud.ServiceClient, name string) (*loadbalancers.LoadBalancer, error) { - opts := loadbalancers.ListOpts{ - Name: name, - } - allPages, err := loadbalancers.List(client, opts).AllPages() - if err != nil { - return nil, err - } - loadbalancerList, err := loadbalancers.ExtractLoadBalancers(allPages) - if err != nil { - return nil, err - } - - if len(loadbalancerList) > 1 { - return nil, ErrMultipleResults - } - if len(loadbalancerList) == 0 { - return nil, ErrNotFound - } - - return &loadbalancerList[0], nil -} - -// GetListenerByName gets a listener by its name, raise error if not found or get multiple ones. -func GetListenerByName(client *gophercloud.ServiceClient, name string, lbID string) (*listeners.Listener, error) { - opts := listeners.ListOpts{ - Name: name, - LoadbalancerID: lbID, - } - pager := listeners.List(client, opts) - var listenerList []listeners.Listener - - err := pager.EachPage(func(page pagination.Page) (bool, error) { - v, err := listeners.ExtractListeners(page) - if err != nil { - return false, err - } - listenerList = append(listenerList, v...) - if len(listenerList) > 1 { - return false, ErrMultipleResults - } - return true, nil - }) - if err != nil { - if cpoerrors.IsNotFound(err) { - return nil, ErrNotFound - } - return nil, err - } - - if len(listenerList) == 0 { - return nil, ErrNotFound - } - - return &listenerList[0], nil -} - -// GetPoolByName gets a pool by its name, raise error if not found or get multiple ones. -func GetPoolByName(client *gophercloud.ServiceClient, name string, lbID string) (*pools.Pool, error) { - var listenerPools []pools.Pool - - opts := pools.ListOpts{ - Name: name, - LoadbalancerID: lbID, - } - err := pools.List(client, opts).EachPage(func(page pagination.Page) (bool, error) { - v, err := pools.ExtractPools(page) - if err != nil { - return false, err - } - listenerPools = append(listenerPools, v...) - if len(listenerPools) > 1 { - return false, ErrMultipleResults - } - return true, nil - }) - if err != nil { - if cpoerrors.IsNotFound(err) { - return nil, ErrNotFound - } - return nil, err - } - - if len(listenerPools) == 0 { - return nil, ErrNotFound - } else if len(listenerPools) > 1 { - return nil, ErrMultipleResults - } - - return &listenerPools[0], nil -} - -// GetPoolsByListener finds pool for a listener. A listener always has exactly one pool. -func GetPoolByListener(client *gophercloud.ServiceClient, lbID, listenerID string) (*pools.Pool, error) { - listenerPools := make([]pools.Pool, 0, 1) - err := pools.List(client, pools.ListOpts{LoadbalancerID: lbID}).EachPage(func(page pagination.Page) (bool, error) { - poolsList, err := pools.ExtractPools(page) - if err != nil { - return false, err - } - for _, p := range poolsList { - for _, l := range p.Listeners { - if l.ID == listenerID { - listenerPools = append(listenerPools, p) - } - } - } - if len(listenerPools) > 1 { - return false, ErrMultipleResults - } - return true, nil - }) - if err != nil { - if cpoerrors.IsNotFound(err) { - return nil, ErrNotFound - } - return nil, err - } - - if len(listenerPools) == 0 { - return nil, ErrNotFound - } - - return &listenerPools[0], nil -} - -// DeleteLoadbalancer deletes a loadbalancer with all its child objects. -func DeleteLoadbalancer(client *gophercloud.ServiceClient, lbID string) error { - err := loadbalancers.Delete(client, lbID, loadbalancers.DeleteOpts{Cascade: true}).ExtractErr() - if err != nil && !cpoerrors.IsNotFound(err) { - return fmt.Errorf("error deleting loadbalancer %s: %v", lbID, err) - } - - return nil -} - -// GetMembersbyPool get all the members in the pool. -func GetMembersbyPool(client *gophercloud.ServiceClient, poolID string) ([]pools.Member, error) { - var members []pools.Member - - err := pools.ListMembers(client, poolID, pools.ListMembersOpts{}).EachPage(func(page pagination.Page) (bool, error) { - membersList, err := pools.ExtractMembers(page) - if err != nil { - return false, err - } - members = append(members, membersList...) - - return true, nil - }) - if err != nil { - return nil, err - } - - return members, nil -} - -func BatchUpdatePoolMembers(client *gophercloud.ServiceClient, lbID, poolID string, opts []pools.BatchUpdateMemberOpts) error { - err := pools.BatchUpdateMembers(client, poolID, opts).ExtractErr() - if err != nil { - return err - } - - if err := waitLoadbalancerActive(client, lbID); err != nil { - return fmt.Errorf("failed to wait for load balancer ACTIVE after updating pool members for %s: %v", poolID, err) - } - - return nil -} diff --git a/pkg/util/openstack/network.go b/pkg/util/openstack/network.go deleted file mode 100644 index baab4f2778..0000000000 --- a/pkg/util/openstack/network.go +++ /dev/null @@ -1,55 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package openstack - -import ( - "github.com/gophercloud/gophercloud" - "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/floatingips" -) - -// GetFloatingIPs returns all the filtered floating IPs -func GetFloatingIPs(client *gophercloud.ServiceClient, opts floatingips.ListOpts) ([]floatingips.FloatingIP, error) { - var floatingIPList []floatingips.FloatingIP - - allPages, err := floatingips.List(client, opts).AllPages() - if err != nil { - return floatingIPList, err - } - floatingIPList, err = floatingips.ExtractFloatingIPs(allPages) - if err != nil { - return floatingIPList, err - } - - return floatingIPList, nil -} - -// GetFloatingIPByPortID get the floating IP of the given port. -func GetFloatingIPByPortID(client *gophercloud.ServiceClient, portID string) (*floatingips.FloatingIP, error) { - opt := floatingips.ListOpts{ - PortID: portID, - } - ips, err := GetFloatingIPs(client, opt) - if err != nil { - return nil, err - } - - if len(ips) == 0 { - return nil, nil - } - - return &ips[0], nil -} diff --git a/pkg/util/util.go b/pkg/util/util.go deleted file mode 100644 index dd5d71ab27..0000000000 --- a/pkg/util/util.go +++ /dev/null @@ -1,76 +0,0 @@ -package util - -import ( - "fmt" - - "k8s.io/apimachinery/pkg/api/resource" - "k8s.io/apimachinery/pkg/util/sets" -) - -const ( - GIB = 1024 * 1024 * 1024 -) - -// StringListEqual compares two string list, returns true if they have the same items, order doesn't matter -func StringListEqual(list1, list2 []string) bool { - if len(list1) == 0 && len(list2) == 0 { - return true - } - - if len(list1) != len(list2) { - return false - } - - s1 := sets.String{} - for _, s := range list1 { - s1.Insert(s) - } - s2 := sets.String{} - for _, s := range list2 { - s2.Insert(s) - } - - return s1.Equal(s2) -} - -// Contains searches if a string list contains the given string or not. -func Contains(list []string, strToSearch string) bool { - for _, item := range list { - if item == strToSearch { - return true - } - } - return false -} - -// RoundUpSize calculates how many allocation units are needed to accommodate -// a volume of given size. E.g. when user wants 1500MiB volume, while AWS EBS -// allocates volumes in gibibyte-sized chunks, -// RoundUpSize(1500 * 1024*1024, 1024*1024*1024) returns '2' -// (2 GiB is the smallest allocatable volume that can hold 1500MiB) -func RoundUpSize(volumeSizeBytes int64, allocationUnitBytes int64) int64 { - roundedUp := volumeSizeBytes / allocationUnitBytes - if volumeSizeBytes%allocationUnitBytes > 0 { - roundedUp++ - } - return roundedUp -} - -// RoundUpSizeInt calculates how many allocation units are needed to accommodate -// a volume of given size. It returns an int instead of an int64 and an error if -// there's overflow -func RoundUpSizeInt(volumeSizeBytes int64, allocationUnitBytes int64) (int, error) { - roundedUp := RoundUpSize(volumeSizeBytes, allocationUnitBytes) - roundedUpInt := int(roundedUp) - if int64(roundedUpInt) != roundedUp { - return 0, fmt.Errorf("capacity %v is too great, casting results in integer overflow", roundedUp) - } - return roundedUpInt, nil -} - -// RoundUpToGiBInt rounds up given quantity upto chunks of GiB. It returns an -// int instead of an int64 and an error if there's overflow -func RoundUpToGiBInt(size resource.Quantity) (int, error) { - requestBytes := size.Value() - return RoundUpSizeInt(requestBytes, GIB) -} diff --git a/pkg/version/version.go b/pkg/version/version.go deleted file mode 100644 index 0278ad2a34..0000000000 --- a/pkg/version/version.go +++ /dev/null @@ -1,30 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package version - -import ( - "fmt" - "os" -) - -// Version is set by the linker flags in the Makefile. -var Version string - -func PrintVersionAndExit() { - fmt.Printf("%s\n", Version) - os.Exit(0) -} diff --git a/release-procedure.md b/release-procedure.md deleted file mode 100644 index fbd161a005..0000000000 --- a/release-procedure.md +++ /dev/null @@ -1,53 +0,0 @@ -# Release Procedure - -Cloud Provider OpenStack Release is done in sync with kubernetes/kubernetes, Minor versions can be released intermittently for critical bug fixes. - -## Before Release - -Update cloud-provider-openstack to kubernetes/kubernetes latest release. Make Sure all CI check passed. - -## Making a Release - -1. Checkout the release branch - -``` -$ git fetch upstream -$ git checkout -b release-X.Y upstream/release-X.Y -``` - -2. Sign tag and push to upstream repo. You will need your GPG pass phrase. Check if you have a key using `gpg --list-secret-keys --keyid-format LONG` - -``` -$ git tag -s -m "Release for cloud-provider-openstack to support Kubernetes release x" vX.Y.Z -$ git push upstream vX.Y.Z -``` -3. Build and push images to dockerhub. Provide `ARCHS` if needs to push only for specific types of architecture (for example `ARCHS='arm64 amd64'` or `ARCHS='amd64'`). -Don't need to provide `DOCKER_USERNAME` and `DOCKER_PASSWORD` if you already logged in. - -``` -GOOS=linux DOCKER_USERNAME=user DOCKER_PASSWORD=my_password REGISTRY=docker.io/k8scloudprovider VERSION=vX.Y.Z make upload-images -``` - -4. Update manifests with new release images, create a PR against release branch to update. - -5. Create entry in GitHub -``` -https://github.com/kubernetes/cloud-provider-openstack/releases -``` -Click on "Draft a new release", fill out the tag, title and description field and click on "Publish release" - -4. Sign the tarball with your key and upload vX.Y.Z.tar.gz.asc and vX.Y.Z.zip.asc - -``` -gpg --armor --detach-sign vX.Y.Z.tar.gz -gpg --armor --detach-sign vX.Y.Z.zip -``` - -6 Make the binaries, sign them and upload them - -``` -VERSION=vX.Y.Z make dist -cd _dist -find . -name "*.tar.gz" -exec gpg --armor --detach-sign {} \; -find . -name "*.zip" -exec gpg --armor --detach-sign {} \; -``` diff --git a/snapcraft.yaml b/snapcraft.yaml deleted file mode 100644 index 4f736cbc1f..0000000000 --- a/snapcraft.yaml +++ /dev/null @@ -1,22 +0,0 @@ -name: client-keystone-auth -version: git -summary: Client plugin for kubectl to enable keystone authentication. -description: | - client-keystone-auth is a plugin for kubectl. It allows kubectl to get an - auth token from Keystone and use that to authenticate to the Kubernetes - api server. -grade: stable -confinement: strict - -parts: - client-keystone-auth: - source: . - plugin: go - go-packages: - - k8s.io/cloud-provider-openstack/cmd/client-keystone-auth - go-importpath: k8s.io/cloud-provider-openstack - -apps: - client-keystone-auth: - command: client-keystone-auth - plugs: [ network ] diff --git a/tests/e2e/cloudprovider/test-lb-service.sh b/tests/e2e/cloudprovider/test-lb-service.sh deleted file mode 100755 index 3e1ca074f9..0000000000 --- a/tests/e2e/cloudprovider/test-lb-service.sh +++ /dev/null @@ -1,207 +0,0 @@ -#!/usr/bin/env bash - -# This script is used for the openlab CI job cloud-provider-openstack-acceptance-test-lb-octavia. -# See https://github.com/theopenlab/openlab-zuul-jobs/blob/master/playbooks/cloud-provider-openstack-acceptance-test-lb-octavia/run.yaml#L104 -# -# Prerequisites: -# - This script is supposed to be running on a host has access to kubernetes cluster. -# - kubectl is ready to talk with the kubernetes cluster. -# - It's recommended to run the script on a host with as less proxies to the public as possible, otherwise the -# x-forwarded-for test will probably fail. -# - This script is not responsible for resource clean up if there is test case fails. - -TIMEOUT=${TIMEOUT:-300} -FLOATING_IP=${FLOATING_IP:-""} -NAMESPACE="octavia-lb-test" - -######################################################################## -## Name: wait_for_service -## Desc: Waits for a k8s service until it gets a valid IP address -## Params: -## - (required) A k8s service name -######################################################################## -function wait_for_service { - local service_name=$1 - - end=$(($(date +%s) + ${TIMEOUT})) - while true; do - ipaddr=$(kubectl -n $NAMESPACE describe service ${service_name} | grep 'LoadBalancer Ingress' | awk -F":" '{print $2}' | tr -d ' ') - if [ "x${ipaddr}" != "x" ]; then - printf "\n>>>>>>> Service ${service_name} is created successfully, IP: ${ipaddr}\n" - export ipaddr=${ipaddr} - break - fi - sleep 3 - now=$(date +%s) - [ $now -gt $end ] && printf "\n>>>>>>> FAIL: Failed to wait for the Service ${service_name} created in time\n" && exit -1 - done -} - -######################################################################## -## Name: wait_for_service_deleted -## Desc: Waits for a k8s service deleted. -## Params: -## - (required) A k8s service name -######################################################################## -function wait_for_service_deleted { - local service_name=$1 - - end=$(($(date +%s) + ${TIMEOUT})) - while true; do - svc=$(kubectl -n $NAMESPACE get service | grep ${service_name}) - if [[ "x${svc}" == "x" ]]; then - printf "\n>>>>>>> Service ${service_name} deleted\n" - break - fi - sleep 3 - now=$(date +%s) - [ $now -gt $end ] && printf "\n>>>>>>> FAIL: Failed to wait for the Service ${service_name} deleted\n" && exit -1 - done -} - -######################################################################## -## Name: create_namespace -## Desc: Makes sure the namespace exists. -## Params: None -######################################################################## -function create_namespace { - printf "\n>>>>>>> Create Namespace ${NAMESPACE}\n" - kubectl -n $NAMESPACE get deploy echoserver > /dev/null 2>&1 - cat <>>>>>> Create a Deployment\n" - cat <>>>>>> Create Service ${service}\n" - cat <>>>>>> Waiting for the Service ${service} creation finished\n" - wait_for_service ${service} - - printf "\n>>>>>>> Sending request to the Service ${service}\n" - podname=$(curl -s http://${ipaddr} | grep Hostname | awk -F':' '{print $2}' | cut -d ' ' -f2) - if [[ "$podname" =~ "echoserver" ]]; then - printf "\n>>>>>>> Expected: Get correct response from Service ${service}\n" - else - printf "\n>>>>>>> FAIL: Get incorrect response from Service ${service}, expected: echoserver, actual: $podname\n" - exit -1 - fi - - printf "\n>>>>>>> Delete Service ${service}\n" - kubectl -n $NAMESPACE delete service ${service} -} - -######################################################################## -## Name: test_forwarded -## Desc: Create a k8s service that gets the original client IP -## Params: None -######################################################################## -function test_forwarded { - local service="test-x-forwarded-for" - local public_ip=$(curl -s ifconfig.me) - local local_ip=$(ip route get 8.8.8.8 | head -1 | awk '{print $7}') - - printf "\n>>>>>>> Create the Service ${service}\n" - cat <>>>>>> Waiting for the Service ${service} creation finished\n" - wait_for_service ${service} - - printf "\n>>>>>>> Sending request to the Service ${service}\n" - orig_ip=$(curl -s http://${ipaddr} | grep x-forwarded-for | awk -F'=' '{print $2}') - if [[ "${orig_ip}" != "${local_ip}" && "${orig_ip}" != "${public_ip}" ]]; then - printf "\n>>>>>>> FAIL: Get incorrect response from Service ${service}, orig_ip: ${local_ip}, public_ip: ${public_ip}\n" - exit -1 - else - printf "\n>>>>>>> Expected: Get correct response from Service ${service}\n" - fi - - printf "\n>>>>>>> Delete Service ${service}\n" - kubectl -n $NAMESPACE delete service ${service} -} - -create_namespace -create_deployment -test_basic -test_forwarded - -printf "\n>>>>>>> Delete k8s resources\n" -kubectl -n ${NAMESPACE} delete deploy echoserver \ No newline at end of file diff --git a/tests/e2e/csi/cinder/cinder-testdriver.go b/tests/e2e/csi/cinder/cinder-testdriver.go deleted file mode 100644 index 5c619de56c..0000000000 --- a/tests/e2e/csi/cinder/cinder-testdriver.go +++ /dev/null @@ -1,101 +0,0 @@ -package test - -import ( - "fmt" - - storagev1 "k8s.io/api/storage/v1" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/kubernetes/test/e2e/framework" - "k8s.io/kubernetes/test/e2e/storage/testpatterns" - "k8s.io/kubernetes/test/e2e/storage/testsuites" -) - -type cinderDriver struct { - driverInfo testsuites.DriverInfo - manifests []string -} - -var Cinderdriver = InitCinderDriver - -type cinderVolume struct { - ID string - Name string - Status string - AvailabilityZone string - f *framework.Framework -} - -// initCinderDriver returns cinderDriver that implements TestDriver interface -func initCinderDriver(name string, manifests ...string) testsuites.TestDriver { - return &cinderDriver{ - driverInfo: testsuites.DriverInfo{ - Name: name, - MaxFileSize: testpatterns.FileSizeLarge, - SupportedFsType: sets.NewString( - "", // Default fsType - "ext2", - "ext3", - "ext4", - "xfs", - ), - Capabilities: map[testsuites.Capability]bool{ - testsuites.CapPersistence: true, - testsuites.CapFsGroup: true, - testsuites.CapExec: true, - testsuites.CapMultiPODs: true, - testsuites.CapBlock: true, - }, - }, - manifests: manifests, - } -} - -func InitCinderDriver() testsuites.TestDriver { - - return initCinderDriver("cinder.csi.openstack.org", - "cinder-csi-controllerplugin.yaml", - "cinder-csi-controllerplugin-rbac.yaml", - "cinder-csi-nodeplugin.yaml", - "cinder-csi-nodeplugin-rbac.yaml", - "csi-secret-cinderplugin.yaml") - -} - -var _ testsuites.TestDriver = &cinderDriver{} - -// var _ testsuites.PreprovisionedVolumeTestDriver = &cinderDriver{} -// var _ testsuites.PreprovisionedPVTestDriver = &cinderDriver{} -var _ testsuites.DynamicPVTestDriver = &cinderDriver{} - -func (d *cinderDriver) GetDriverInfo() *testsuites.DriverInfo { - return &d.driverInfo -} - -func (d *cinderDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) { -} - -func (d *cinderDriver) GetDynamicProvisionStorageClass(config *testsuites.PerTestConfig, fsType string) *storagev1.StorageClass { - provisioner := "cinder.csi.openstack.org" - parameters := map[string]string{} - if fsType != "" { - parameters["fsType"] = fsType - } - ns := config.Framework.Namespace.Name - suffix := fmt.Sprintf("%s-sc", d.driverInfo.Name) - - return testsuites.GetStorageClass(provisioner, parameters, nil, ns, suffix) -} - -func (d *cinderDriver) GetClaimSize() string { - return "2Gi" -} - -func (d *cinderDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTestConfig, func()) { - config := &testsuites.PerTestConfig{ - Driver: d, - Prefix: "cinder", - Framework: f, - } - - return config, func() {} -} diff --git a/tests/e2e/csi/cinder/csi-volumes.go b/tests/e2e/csi/cinder/csi-volumes.go deleted file mode 100644 index aad0ffd916..0000000000 --- a/tests/e2e/csi/cinder/csi-volumes.go +++ /dev/null @@ -1,33 +0,0 @@ -package test - -import ( - "path" - - . "github.com/onsi/ginkgo" - _ "github.com/onsi/gomega" - "k8s.io/kubernetes/test/e2e/framework" - "k8s.io/kubernetes/test/e2e/framework/testfiles" - "k8s.io/kubernetes/test/e2e/storage/testsuites" - "k8s.io/kubernetes/test/e2e/storage/utils" -) - -var CSITestSuites = []func() testsuites.TestSuite{ - testsuites.InitVolumesTestSuite, - testsuites.InitSubPathTestSuite, - testsuites.InitProvisioningTestSuite, - testsuites.InitVolumeModeTestSuite, - //testsuites.InitVolumeIOTestSuite, - //testsuites.InitSnapshottableTestSuite, - //testsuites.InitMultiVolumeTestSuite, -} - -// This executes testSuites for csi volumes. -var _ = utils.SIGDescribe("[cinder-csi-e2e] CSI Volumes", func() { - testfiles.AddFileSource(testfiles.RootFileSource{Root: path.Join(framework.TestContext.RepoRoot, "../../manifests/cinder-csi-plugin/")}) - - curDriver := Cinderdriver() - Context(testsuites.GetDriverNameWithFeatureTags(curDriver), func() { - testsuites.DefineTestSuite(curDriver, CSITestSuites) - }) - -}) diff --git a/tests/e2e/k8s-keystone-auth/test-authz.sh b/tests/e2e/k8s-keystone-auth/test-authz.sh deleted file mode 100755 index 7a0d4de82c..0000000000 --- a/tests/e2e/k8s-keystone-auth/test-authz.sh +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env bash - -OS_CONTEXT_NAME=${OS_CONTEXT_NAME:-""} -AUTH_POLICY_CONFIGMAP=${AUTH_POLICY_CONFIGMAP:-""} -ROLE_MAPPING_CONFIGMAP=${ROLE_MAPPING_CONFIGMAP:-""} - -function log { - local msg=$1 - printf "\n>>>>>>> ${FUNCNAME[1]}: ${msg}\n" -} - -######################################################################## -# Desc: Test authorization policy that grant users with member role in -# project demo can get/list pods in default namespace. -# Params: N/A -######################################################################## -function test_auth_policy { - log "Update configmap ${AUTH_POLICY_CONFIGMAP}" - - cat <= limit { - retToken = value.ID - break - } - vollist = append(vollist, *value) - count++ - - } - return vollist, retToken, nil - -} - -func (cloud *cloud) WaitDiskAttached(instanceID string, volumeID string) error { - return nil - -} - -func (cloud *cloud) DetachVolume(instanceID, volumeID string) error { - return nil - -} - -func (cloud *cloud) WaitDiskDetached(instanceID string, volumeID string) error { - return nil - -} - -func (cloud *cloud) GetAttachmentDiskPath(instanceID, volumeID string) (string, error) { - return cinder.FakeDevicePath, nil - -} - -func (cloud *cloud) GetVolumesByName(name string) ([]volumes.Volume, error) { - var vlist []volumes.Volume - for _, v := range cloud.volumes { - if v.Name == name { - vlist = append(vlist, *v) - - } - } - - return vlist, nil - -} - -func (cloud *cloud) GetVolume(volumeID string) (*volumes.Volume, error) { - vol, ok := cloud.volumes[volumeID] - - if !ok { - return nil, notFoundError() - } - - return vol, nil -} - -func notFoundError() error { - return gophercloud.ErrDefault404{} -} - -func invalidError() error { - return gophercloud.ErrDefault400{} -} - -func (cloud *cloud) CreateSnapshot(name, volID string, tags *map[string]string) (*snapshots.Snapshot, error) { - - snap := &snapshots.Snapshot{ - ID: randString(10), - Name: name, - Status: "Available", - VolumeID: volID, - CreatedAt: time.Now(), - } - - cloud.snapshots[snap.ID] = snap - return snap, nil -} - -func (cloud *cloud) ListSnapshots(filters map[string]string) ([]snapshots.Snapshot, string, error) { - var snaplist []snapshots.Snapshot - startingToken := filters["Marker"] - limitfilter := filters["Limit"] - limit, _ := strconv.Atoi(limitfilter) - name := filters["Name"] - volumeID := filters["VolumeID"] - - for _, value := range cloud.snapshots { - if volumeID != "" { - if value.VolumeID == volumeID { - snaplist = append(snaplist, *value) - break - } - } else if name != "" { - if value.Name == name { - snaplist = append(snaplist, *value) - break - } - } else { - snaplist = append(snaplist, *value) - } - } - - if startingToken != "" { - t, _ := strconv.Atoi(startingToken) - snaplist = snaplist[t:] - } - - retToken := "" - - if limit != 0 { - snaplist = snaplist[:limit] - retToken = limitfilter - } - - return snaplist, retToken, nil -} - -func (cloud *cloud) DeleteSnapshot(snapID string) error { - - delete(cloud.snapshots, snapID) - - return nil - -} - -func (cloud *cloud) GetSnapshotByID(snapshotID string) (*snapshots.Snapshot, error) { - - snap, ok := cloud.snapshots[snapshotID] - - if !ok { - return nil, notFoundError() - } - - return snap, nil -} - -func (cloud *cloud) WaitSnapshotReady(snapshotID string) error { - return nil -} - -func randString(n int) string { - const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" - b := make([]byte, n) - for i := range b { - b[i] = letterBytes[rand.Intn(len(letterBytes))] - } - return string(b) -} - -func (cloud *cloud) GetInstanceByID(instanceID string) (*servers.Server, error) { - if _, ok := cloud.instances[cinder.FakeInstanceID]; !ok { - cloud.instances[cinder.FakeInstanceID] = &servers.Server{} - } - inst, ok := cloud.instances[instanceID] - - if !ok { - return nil, gophercloud.ErrDefault404{} - } - - return inst, nil -} - -func (cloud *cloud) ExpandVolume(volumeID string, size int) error { - return nil -} - -func (cloud *cloud) GetMaxVolLimit() int64 { - return 256 -} - -func (cloud *cloud) GetMetadataOpts() cpo.MetadataOpts { - var m cpo.MetadataOpts - m.SearchOrder = fmt.Sprintf("%s,%s", "configDrive", "metadataService") - return m -} - -func (cloud *cloud) GetBlockStorageOpts() cco.BlockStorageOpts { - return cco.BlockStorageOpts{} -} diff --git a/tests/sanity/cinder/fakemetadata.go b/tests/sanity/cinder/fakemetadata.go deleted file mode 100644 index e52c22f712..0000000000 --- a/tests/sanity/cinder/fakemetadata.go +++ /dev/null @@ -1,16 +0,0 @@ -package sanity - -import "k8s.io/cloud-provider-openstack/pkg/csi/cinder" - -type fakemetadata struct { -} - -// fake metadata - -func (m *fakemetadata) GetInstanceID() (string, error) { - return cinder.FakeInstanceID, nil -} - -func (m *fakemetadata) GetAvailabilityZone() (string, error) { - return cinder.FakeAvailability, nil -} diff --git a/tests/sanity/cinder/fakemount.go b/tests/sanity/cinder/fakemount.go deleted file mode 100644 index 114a44b711..0000000000 --- a/tests/sanity/cinder/fakemount.go +++ /dev/null @@ -1,69 +0,0 @@ -package sanity - -import ( - "k8s.io/cloud-provider-openstack/pkg/csi/cinder" - cpomount "k8s.io/cloud-provider-openstack/pkg/util/mount" - exec "k8s.io/utils/exec/testing" - "k8s.io/utils/mount" -) - -type fakemount struct { - BaseMounter *mount.SafeFormatAndMount -} - -var _ cpomount.IMount = &fakemount{} -var ( - fakeMounter = &mount.FakeMounter{MountPoints: []mount.MountPoint{}} - fakeExec = &exec.FakeExec{DisableScripts: true} - mounter = &cpomount.Mount{BaseMounter: newFakeSafeFormatAndMounter()} -) - -//GetFakeMountProvider returns fake instance of Mounter -func GetFakeMountProvider() cpomount.IMount { - return &fakemount{BaseMounter: newFakeSafeFormatAndMounter()} -} - -// NewFakeSafeFormatAndMounter returns base Fake mounter instance -func newFakeSafeFormatAndMounter() *mount.SafeFormatAndMount { - return &mount.SafeFormatAndMount{ - Interface: fakeMounter, - Exec: fakeExec, - } -} - -func (m *fakemount) Mounter() *mount.SafeFormatAndMount { - return m.BaseMounter -} - -func (m *fakemount) ScanForAttach(devicePath string) error { - return nil -} - -func (m *fakemount) IsLikelyNotMountPointAttach(targetpath string) (bool, error) { - return mounter.IsLikelyNotMountPointAttach(targetpath) -} - -func (m *fakemount) UnmountPath(mountPath string) error { - return mounter.UnmountPath(mountPath) -} - -func (m *fakemount) GetInstanceID() (string, error) { - return cinder.FakeInstanceID, nil -} - -func (m *fakemount) GetDevicePath(volumeID string) (string, error) { - return cinder.FakeDevicePath, nil -} - -func (m *fakemount) MakeDir(pathname string) error { - return nil -} - -// MakeFile creates an empty file -func (m *fakemount) MakeFile(pathname string) error { - return nil -} - -func (m *fakemount) GetDeviceStats(path string) (*cpomount.DeviceStats, error) { - return cinder.FakeFsStats, nil -} diff --git a/tests/sanity/cinder/sanity_test.go b/tests/sanity/cinder/sanity_test.go deleted file mode 100644 index dc70a2440f..0000000000 --- a/tests/sanity/cinder/sanity_test.go +++ /dev/null @@ -1,44 +0,0 @@ -package sanity - -import ( - "io/ioutil" - "os" - "path" - "testing" - - "github.com/kubernetes-csi/csi-test/v3/pkg/sanity" - "k8s.io/cloud-provider-openstack/pkg/csi/cinder" - "k8s.io/cloud-provider-openstack/pkg/csi/cinder/openstack" -) - -// start sanity test for driver -func TestDriver(t *testing.T) { - basePath, err := ioutil.TempDir("", "cinder.csi.openstack.org") - if err != nil { - t.Fatal(err) - } - - defer os.RemoveAll(basePath) - - socket := path.Join(basePath, "csi.sock") - endpoint := "unix://" + socket - cluster := "kubernetes" - nodeID := "fake-node" - - d := cinder.NewDriver(nodeID, endpoint, cluster) - fakecloudprovider := getfakecloud() - openstack.OsInstance = fakecloudprovider - - fakemnt := GetFakeMountProvider() - fakemet := &fakemetadata{} - - d.SetupDriver(fakecloudprovider, fakemnt, fakemet) - - // TODO: Stop call - - go d.Run() - - config := sanity.NewTestConfig() - config.Address = endpoint - sanity.Test(t, config) -} diff --git a/tests/sanity/manila/fake-secrets.yaml b/tests/sanity/manila/fake-secrets.yaml deleted file mode 100644 index 4cb4a79885..0000000000 --- a/tests/sanity/manila/fake-secrets.yaml +++ /dev/null @@ -1,49 +0,0 @@ -CreateVolumeSecret: - os-authURL: fake-url - os-region: fake-region - os-userID: fake-user-id - os-password: fake-password - os-domainID: fake-domain-id - os-projectID: fake-project-id -DeleteVolumeSecret: - os-authURL: fake-url - os-region: fake-region - os-userID: fake-user-id - os-password: fake-password - os-domainID: fake-domain-id - os-projectID: fake-project-id -CreateSnapshotSecret: - os-authURL: fake-url - os-region: fake-region - os-userID: fake-user-id - os-password: fake-password - os-domainID: fake-domain-id - os-projectID: fake-project-id -DeleteSnapshotSecret: - os-authURL: fake-url - os-region: fake-region - os-userID: fake-user-id - os-password: fake-password - os-domainID: fake-domain-id - os-projectID: fake-project-id -ControllerValidateVolumeCapabilitiesSecret: - os-authURL: fake-url - os-region: fake-region - os-userID: fake-user-id - os-password: fake-password - os-domainID: fake-domain-id - os-projectID: fake-project-id -NodeStageVolumeSecret: - os-authURL: fake-url - os-region: fake-region - os-userID: fake-user-id - os-password: fake-password - os-domainID: fake-domain-id - os-projectID: fake-project-id -NodePublishVolumeSecret: - os-authURL: fake-url - os-region: fake-region - os-userID: fake-user-id - os-password: fake-password - os-domainID: fake-domain-id - os-projectID: fake-project-id diff --git a/tests/sanity/manila/fakecsiclient.go b/tests/sanity/manila/fakecsiclient.go deleted file mode 100644 index 543facf8e1..0000000000 --- a/tests/sanity/manila/fakecsiclient.go +++ /dev/null @@ -1,95 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package sanity - -import ( - "context" - "time" - - "github.com/container-storage-interface/spec/lib/go/csi" - "google.golang.org/grpc" - "k8s.io/cloud-provider-openstack/pkg/csi/manila/csiclient" -) - -var ( - _ csiclient.Builder = &fakeCSIClientBuilder{} - _ csiclient.Identity = &fakeIdentitySvcClient{} - _ csiclient.Node = &fakeNodeSvcClient{} -) - -type fakeIdentitySvcClient struct{} - -func (c fakeIdentitySvcClient) Probe(ctx context.Context, req *csi.ProbeRequest) (*csi.ProbeResponse, error) { - return &csi.ProbeResponse{}, nil -} - -func (c fakeIdentitySvcClient) GetPluginInfo(context.Context) (*csi.GetPluginInfoResponse, error) { - return &csi.GetPluginInfoResponse{ - Name: "fake-fwd-driver", - VendorVersion: "1.0.0", - }, nil -} - -func (c fakeIdentitySvcClient) ProbeForever(*grpc.ClientConn, time.Duration) error { return nil } - -type fakeNodeSvcClient struct{} - -func (c fakeNodeSvcClient) GetCapabilities(context.Context) (*csi.NodeGetCapabilitiesResponse, error) { - return &csi.NodeGetCapabilitiesResponse{ - Capabilities: []*csi.NodeServiceCapability{ - { - Type: &csi.NodeServiceCapability_Rpc{ - Rpc: &csi.NodeServiceCapability_RPC{ - Type: csi.NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME, - }, - }, - }, - }, - }, nil -} - -func (c fakeNodeSvcClient) StageVolume(context.Context, *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) { - return &csi.NodeStageVolumeResponse{}, nil -} - -func (c fakeNodeSvcClient) UnstageVolume(context.Context, *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) { - return &csi.NodeUnstageVolumeResponse{}, nil -} - -func (c fakeNodeSvcClient) PublishVolume(context.Context, *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { - return &csi.NodePublishVolumeResponse{}, nil -} - -func (c fakeNodeSvcClient) UnpublishVolume(context.Context, *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) { - return &csi.NodeUnpublishVolumeResponse{}, nil -} - -type fakeCSIClientBuilder struct{} - -func (b fakeCSIClientBuilder) NewConnection(string) (*grpc.ClientConn, error) { return nil, nil } - -func (b fakeCSIClientBuilder) NewConnectionWithContext(context.Context, string) (*grpc.ClientConn, error) { - return nil, nil -} - -func (b fakeCSIClientBuilder) NewNodeServiceClient(conn *grpc.ClientConn) csiclient.Node { - return &fakeNodeSvcClient{} -} - -func (b fakeCSIClientBuilder) NewIdentityServiceClient(conn *grpc.ClientConn) csiclient.Identity { - return &fakeIdentitySvcClient{} -} diff --git a/tests/sanity/manila/fakemanilaclient.go b/tests/sanity/manila/fakemanilaclient.go deleted file mode 100644 index 5baf30ed1f..0000000000 --- a/tests/sanity/manila/fakemanilaclient.go +++ /dev/null @@ -1,249 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package sanity - -import ( - "github.com/gophercloud/gophercloud/openstack/sharedfilesystems/v2/sharetypes" - "strconv" - - "github.com/gophercloud/gophercloud" - "github.com/gophercloud/gophercloud/openstack/sharedfilesystems/v2/messages" - "github.com/gophercloud/gophercloud/openstack/sharedfilesystems/v2/shares" - "github.com/gophercloud/gophercloud/openstack/sharedfilesystems/v2/snapshots" - openstack_provider "k8s.io/cloud-provider-openstack/pkg/cloudprovider/providers/openstack" - "k8s.io/cloud-provider-openstack/pkg/csi/manila/manilaclient" -) - -var ( - fakeShareID = 1 - fakeAccessRightID = 1 - fakeSnapshotID = 1 - - fakeShares = make(map[int]*shares.Share) - fakeAccessRights = make(map[int]*shares.AccessRight) - fakeSnapshots = make(map[int]*snapshots.Snapshot) -) - -type fakeManilaClientBuilder struct{} - -func (b fakeManilaClientBuilder) New(o *openstack_provider.AuthOpts) (manilaclient.Interface, error) { - return &fakeManilaClient{}, nil -} - -type fakeManilaClient struct{} - -func optsMapToStruct(optsMap map[string]interface{}, dst interface{}) error { - res := gophercloud.Result{Body: optsMap} - return res.ExtractInto(dst) -} - -func strToInt(s string) int { - n, _ := strconv.Atoi(s) - return n -} - -func intToStr(n int) string { - return strconv.Itoa(n) -} - -func shareExists(shareID string) bool { - _, ok := fakeShares[strToInt(shareID)] - return ok -} - -func (c fakeManilaClient) GetShareByID(shareID string) (*shares.Share, error) { - s, ok := fakeShares[strToInt(shareID)] - if !ok { - return nil, gophercloud.ErrResourceNotFound{} - } - - return s, nil -} - -func (c fakeManilaClient) GetShareByName(shareName string) (*shares.Share, error) { - var shareID string - for _, share := range fakeShares { - if share.Name == shareName { - shareID = share.ID - break - } - } - - if shareID == "" { - return nil, gophercloud.ErrResourceNotFound{} - } - - return c.GetShareByID(shareID) -} - -func (c fakeManilaClient) CreateShare(opts shares.CreateOptsBuilder) (*shares.Share, error) { - var res shares.CreateResult - res.Body = opts - - share := &shares.Share{} - if err := res.ExtractInto(share); err != nil { - return nil, err - } - - share.ID = intToStr(fakeShareID) - share.Status = "available" - fakeShares[fakeShareID] = share - fakeShareID++ - - return share, nil -} - -func (c fakeManilaClient) DeleteShare(shareID string) error { - id := strToInt(shareID) - if _, ok := fakeShares[id]; !ok { - return gophercloud.ErrResourceNotFound{} - } - - delete(fakeShares, id) - return nil -} - -func (c fakeManilaClient) GetExportLocations(shareID string) ([]shares.ExportLocation, error) { - if !shareExists(shareID) { - return nil, gophercloud.ErrResourceNotFound{} - } - - return []shares.ExportLocation{{Path: "fake-server:/fake-path"}}, nil -} - -func (c fakeManilaClient) SetShareMetadata(shareID string, opts shares.SetMetadataOptsBuilder) (map[string]string, error) { - return nil, nil -} - -func (c fakeManilaClient) GetExtraSpecs(shareTypeID string) (sharetypes.ExtraSpecs, error) { - return map[string]interface{}{"snapshot_support": "True", "create_share_from_snapshot_support": "True"}, nil -} - -func (c fakeManilaClient) GetShareTypes() ([]sharetypes.ShareType, error) { - return []sharetypes.ShareType{ - { - ID: "914dbaad-7242-4c34-a9ee-aa3831189972", - Name: "default", - IsPublic: true, - RequiredExtraSpecs: map[string]interface{}{"driver_handles_share_servers": "True"}, - ExtraSpecs: map[string]interface{}{"driver_handles_share_servers": "True", "snapshot_support": "True", "create_share_from_snapshot_support": "True"}, - }, - }, nil -} - -func (c fakeManilaClient) GetShareTypeIDFromName(shareTypeName string) (string, error) { - return "", nil -} - -func (c fakeManilaClient) GetAccessRights(shareID string) ([]shares.AccessRight, error) { - if !shareExists(shareID) { - return nil, gophercloud.ErrResourceNotFound{} - } - - var accessRights []shares.AccessRight - for _, r := range fakeAccessRights { - if r.ShareID == shareID { - accessRights = append(accessRights, *r) - } - } - - return accessRights, nil -} - -func (c fakeManilaClient) GrantAccess(shareID string, opts shares.GrantAccessOptsBuilder) (*shares.AccessRight, error) { - if !shareExists(shareID) { - return nil, gophercloud.ErrResourceNotFound{} - } - - optsMap, err := opts.ToGrantAccessMap() - if err != nil { - return nil, err - } - - accessRight := &shares.AccessRight{} - if err = optsMapToStruct(optsMap, accessRight); err != nil { - return nil, err - } - - accessRight.ID = intToStr(fakeAccessRightID) - accessRight.ShareID = shareID - fakeAccessRights[fakeAccessRightID] = accessRight - fakeAccessRightID++ - - return accessRight, nil -} - -func (c fakeManilaClient) GetSnapshotByID(snapID string) (*snapshots.Snapshot, error) { - s, ok := fakeSnapshots[strToInt(snapID)] - if !ok { - return nil, gophercloud.ErrDefault404{} - } - - return s, nil -} - -func (c fakeManilaClient) GetSnapshotByName(snapName string) (*snapshots.Snapshot, error) { - var snapID string - for _, snap := range fakeSnapshots { - if snap.Name == snapName { - snapID = snap.ID - break - } - } - - if snapID == "" { - return nil, gophercloud.ErrResourceNotFound{} - } - - return c.GetSnapshotByID(snapID) -} - -func (c fakeManilaClient) CreateSnapshot(opts snapshots.CreateOptsBuilder) (*snapshots.Snapshot, error) { - var res snapshots.CreateResult - res.Body = opts - - snap := &snapshots.Snapshot{} - if err := res.ExtractInto(snap); err != nil { - return nil, err - } - - snap.ID = intToStr(fakeSnapshotID) - snap.Status = "available" - - if !shareExists(snap.ShareID) { - return nil, gophercloud.ErrDefault404{} - } - - fakeSnapshots[fakeSnapshotID] = snap - fakeSnapshotID++ - - return snap, nil -} - -func (c fakeManilaClient) DeleteSnapshot(snapID string) error { - id := strToInt(snapID) - if _, ok := fakeSnapshots[id]; !ok { - return gophercloud.ErrResourceNotFound{} - } - - delete(fakeSnapshots, id) - return nil -} - -func (c fakeManilaClient) GetUserMessages(opts messages.ListOptsBuilder) ([]messages.Message, error) { - return nil, nil -} diff --git a/tests/sanity/manila/sanity_test.go b/tests/sanity/manila/sanity_test.go deleted file mode 100644 index 27b22f8727..0000000000 --- a/tests/sanity/manila/sanity_test.go +++ /dev/null @@ -1,67 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package sanity - -import ( - "io/ioutil" - "os" - "path" - "testing" - - "github.com/kubernetes-csi/csi-test/pkg/sanity" - "k8s.io/cloud-provider-openstack/pkg/csi/manila" - "k8s.io/cloud-provider-openstack/pkg/csi/manila/options" -) - -func TestDriver(t *testing.T) { - basePath, err := ioutil.TempDir("", "manila.csi.openstack.org") - if err != nil { - t.Fatalf("failed create base path in %s: %v", basePath, err) - } - - defer os.RemoveAll(basePath) - - endpoint := path.Join(basePath, "csi.sock") - fwdEndpoint := "unix:///fake-fwd-endpoint" - - d, err := manila.NewDriver( - &manila.DriverOpts{ - DriverName: "fake.manila.csi.openstack.org", - NodeID: "node", - WithTopology: true, - NodeAZ: "fake-az", - ShareProto: "NFS", - ServerCSIEndpoint: endpoint, - FwdCSIEndpoint: fwdEndpoint, - ManilaClientBuilder: &fakeManilaClientBuilder{}, - CSIClientBuilder: &fakeCSIClientBuilder{}, - CompatOpts: &options.CompatibilityOptions{}, - }) - - if err != nil { - t.Fatalf("failed to initialize CSI Manila driver: %v", err) - } - - go d.Run() - - sanity.Test(t, &sanity.Config{ - Address: endpoint, - SecretsFile: "fake-secrets.yaml", - TargetPath: path.Join(basePath, "target"), - StagingPath: path.Join(basePath, "staging"), - }) -} diff --git a/tools/install-distro-packages.sh b/tools/install-distro-packages.sh deleted file mode 100755 index f922a8f81b..0000000000 --- a/tools/install-distro-packages.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash -xe - -BASE_DIR=$(cd $(dirname $BASH_SOURCE)/.. && pwd) - -# Local version to install bindep packages -# Suitable for use for development - -function is_fedora { - [ -f /usr/bin/yum ] && cat /etc/*release | grep -q -e "Fedora" -} - -PACKAGES="" -if ! which virtualenv; then - PACKAGES="$PACKAGES virtualenv" -fi -if ! which make; then - PACKAGES="$PACKAGES make" -fi -if [[ -n $PACKAGES ]]; then - sudo apt-get -q --assume-yes install $PACKAGES -fi - -# Check for bindep -if ! which bindep; then - pushd $BASE_DIR >/dev/null - make bindep - popd >/dev/null -fi - -PACKAGES=$(make bindep || true) - -# inspired from project-config install-distro-packages.sh -if apt-get -v >/dev/null 2>&1 ; then - sudo apt-get -qq update - sudo PATH=/usr/sbin:/sbin:$PATH DEBIAN_FRONTEND=noninteractive \ - apt-get -q --option "Dpkg::Options::=--force-confold" \ - --assume-yes install $PACKAGES -elif emerge --version >/dev/null 2>&1 ; then - sudo emerge -uDNq --jobs=4 @world - sudo PATH=/usr/sbin:/sbin:$PATH emerge -q --jobs=4 $PACKAGES -else - is_fedora && YUM=dnf || YUM=yum - sudo PATH=/usr/sbin:/sbin:$PATH $YUM install -y $PACKAGES -fi diff --git a/tools/test-setup.sh b/tools/test-setup.sh deleted file mode 100755 index 6330972de5..0000000000 --- a/tools/test-setup.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash -xe -# test-setup.sh - Install required stuffs -# Used in both CI jobs and locally -# -# Install the following tools: -# * dep - -# Get OS -case $(uname -s) in - Darwin) - OS=darwin - ;; - Linux) - if LSB_RELEASE=$(which lsb_release); then - OS=$($LSB_RELEASE -s -c) - else - # No lsb-release, trya hack or two - if which dpkg 1>/dev/null; then - OS=debian - elif which yum 1>/dev/null || which dnf 1>/dev/null; then - OS=redhat - else - echo "Linux distro not yet supported" - exit 1 - fi - fi - ;; - *) - echo "Unsupported OS" - exit 1 - ;; -esac - -case $OS in - darwin) - if which brew 1>/dev/null; then - if ! which dep 1>/dev/null; then - brew install dep - fi - else - curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh - fi - ;; - xenial|zesty) - APT_GET="DEBIAN_FRONTEND=noninteractive \ - apt-get -q --option "Dpkg::Options::=--force-confold" \ - --assume-yes" - if ! which add-apt-repository 1>/dev/null; then - sudo $APT_GET install software-properties-common - fi - sudo add-apt-repository --yes ppa:gophers/archive - sudo apt-get update && sudo $APT_GET install golang-1.9-go - sudo ln -sf /usr/lib/go-1.9/bin/go /usr/local/bin - sudo ln -sf /usr/lib/go-1.9/bin/gofmt /usr/local/bin - curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh - ;; -esac From acaea34ffa6aa4e0a65025af6a5b47fe146c781a Mon Sep 17 00:00:00 2001 From: Bharat Kunwar Date: Mon, 7 Sep 2020 14:32:31 +0000 Subject: [PATCH 02/90] Remove index.yaml and add README.md GH action fails when there is an empty index.yaml file. It is better to have none at all so that it can create a valid index.yaml file for us. --- README.md | 14 ++++++++++++++ index.html | 28 ++++++++++++++++++++++++++++ index.yaml | 0 3 files changed, 42 insertions(+) create mode 100644 README.md create mode 100644 index.html delete mode 100644 index.yaml diff --git a/README.md b/README.md new file mode 100644 index 0000000000..01a50c7f1d --- /dev/null +++ b/README.md @@ -0,0 +1,14 @@ +# Cloud Provider OpenStack Helm Chart Repository + +## Add repo + + helm repo add cpo https://kubernetes.github.io/cloud-provider-openstack + helm repo update + +## Install Cinder CSI chart + + helm install cinder-csi cpo/openstack-cinder-csi + +## Install Manila CSI chart + + helm install manila-csi cpo/openstack-manila-csi diff --git a/index.html b/index.html new file mode 100644 index 0000000000..e4337d6da6 --- /dev/null +++ b/index.html @@ -0,0 +1,28 @@ + + + + + + +

+ Cloud Provider OpenStack Helm Chart Repository +

+

+ Add repo +

+ 
helm repo add cpo https://kubernetes.github.io/cloud-provider-openstack
+
+ 
helm repo update
+
+

+ Install Cinder CSI chart +

+ 
helm install cinder-csi cpo/openstack-cinder-csi
+
+

+ Install Manila CSI chart +

+ 
helm install manila-csi cpo/openstack-manila-csi
+
+ + \ No newline at end of file diff --git a/index.yaml b/index.yaml deleted file mode 100644 index e69de29bb2..0000000000 From 9c7b1219da32ca3c4dd6473130de80188a73f145 Mon Sep 17 00:00:00 2001 From: lingxiankong Date: Mon, 28 Sep 2020 10:25:35 +0000 Subject: [PATCH 03/90] Update index.yaml Signed-off-by: lingxiankong --- index.yaml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 index.yaml diff --git a/index.yaml b/index.yaml new file mode 100644 index 0000000000..a8a157ffea --- /dev/null +++ b/index.yaml @@ -0,0 +1,33 @@ +apiVersion: v1 +entries: + openstack-cinder-csi: + - apiVersion: v1 + appVersion: v1.19.0 + created: "2020-09-28T10:25:35.075557926Z" + description: Cinder CSI Chart for OpenStack + digest: 4059348056c4c64f4188f5935e5def80214514ff8198d5213c6d49bffa94409a + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.2.2/openstack-cinder-csi-1.2.2.tgz + version: 1.2.2 + openstack-manila-csi: + - apiVersion: v1 + appVersion: v1.19.0 + created: "2020-09-28T10:25:35.206069325Z" + description: Manila CSI Chart for OpenStack + digest: 1d8c90f04db976befe9e27a315e787ab66657da35cbbcd253582e179e30b344d + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-0.2.1/openstack-manila-csi-0.2.1.tgz + version: 0.2.1 +generated: "2020-09-28T10:25:34.941244248Z" From 78d66b829a0b0f5bd16a9a59f6e9f76db9b7beb2 Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Thu, 28 Jan 2021 20:14:02 +0000 Subject: [PATCH 04/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/index.yaml b/index.yaml index a8a157ffea..eea2956a09 100644 --- a/index.yaml +++ b/index.yaml @@ -1,6 +1,20 @@ apiVersion: v1 entries: openstack-cinder-csi: + - apiVersion: v1 + appVersion: v1.20.0 + created: "2021-01-28T20:14:01.824951121Z" + description: Cinder CSI Chart for OpenStack + digest: 388728a9be60169d20c39d101bbb182f1c260e621888f493df6d9609c6442bd7 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.2.13/openstack-cinder-csi-1.2.13.tgz + version: 1.2.13 - apiVersion: v1 appVersion: v1.19.0 created: "2020-09-28T10:25:35.075557926Z" @@ -16,6 +30,20 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.2.2/openstack-cinder-csi-1.2.2.tgz version: 1.2.2 openstack-manila-csi: + - apiVersion: v1 + appVersion: v1.20.0 + created: "2021-01-28T20:14:02.090577746Z" + description: Manila CSI Chart for OpenStack + digest: 035a1d0a6025bd5e10db287d64f3e01717e2df8b1fd8f954c3c697713778b051 + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-1.0.5/openstack-manila-csi-1.0.5.tgz + version: 1.0.5 - apiVersion: v1 appVersion: v1.19.0 created: "2020-09-28T10:25:35.206069325Z" From 4c0b37a9be86f9f4655275dcab31b972b7bebcce Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Fri, 12 Feb 2021 05:41:05 +0000 Subject: [PATCH 05/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/index.yaml b/index.yaml index eea2956a09..36d97adac2 100644 --- a/index.yaml +++ b/index.yaml @@ -1,6 +1,20 @@ apiVersion: v1 entries: openstack-cinder-csi: + - apiVersion: v1 + appVersion: v1.20.0 + created: "2021-02-12T05:41:05.906103382Z" + description: Cinder CSI Chart for OpenStack + digest: 6d046af0d1922e8889c30e9a76a586c39d8323272f699739a224cb13cf6ba991 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.2.14/openstack-cinder-csi-1.2.14.tgz + version: 1.2.14 - apiVersion: v1 appVersion: v1.20.0 created: "2021-01-28T20:14:01.824951121Z" From 73f95888d50bdb8d42e52cdca16fceea88a06816 Mon Sep 17 00:00:00 2001 From: lingxiankong Date: Thu, 29 Apr 2021 05:24:08 +0000 Subject: [PATCH 06/90] Update index.yaml Signed-off-by: lingxiankong --- index.yaml | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/index.yaml b/index.yaml index 36d97adac2..9785228637 100644 --- a/index.yaml +++ b/index.yaml @@ -1,6 +1,20 @@ apiVersion: v1 entries: openstack-cinder-csi: + - apiVersion: v1 + appVersion: latest + created: "2021-04-29T05:24:08.155307767Z" + description: Cinder CSI Chart for OpenStack + digest: 0941fad9901423f495c0e8639bd25d70066155058f56ca22bb18a5209f5ba546 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.3.7/openstack-cinder-csi-1.3.7.tgz + version: 1.3.7 - apiVersion: v1 appVersion: v1.20.0 created: "2021-02-12T05:41:05.906103382Z" @@ -43,7 +57,40 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.2.2/openstack-cinder-csi-1.2.2.tgz version: 1.2.2 + openstack-cloud-controller-manager: + - apiVersion: v1 + appVersion: latest + created: "2021-04-29T05:24:08.308219138Z" + description: Openstack Cloud Controller Manager Helm Chart + digest: 92e770f3104690ce6467cc219854411228235a0e4c4c93e9d6506dc85906ebf1 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: kubernetes@maurice-meyer.de + name: morremeyer + url: https://maurice-meyer.de + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-1.0.0/openstack-cloud-controller-manager-1.0.0.tgz + version: 1.0.0 openstack-manila-csi: + - apiVersion: v1 + appVersion: latest + created: "2021-04-29T05:24:08.443365397Z" + description: Manila CSI Chart for OpenStack + digest: 08dc9e2d5c6f317a329542e3dc051fe98b30086cdc22dbcf1f0b825a6d68ca09 + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-1.1.1/openstack-manila-csi-1.1.1.tgz + version: 1.1.1 - apiVersion: v1 appVersion: v1.20.0 created: "2021-01-28T20:14:02.090577746Z" From 7d197bec33b0ab831324e4a35590fc9309516076 Mon Sep 17 00:00:00 2001 From: lingxiankong Date: Thu, 20 May 2021 02:32:10 +0000 Subject: [PATCH 07/90] Update index.yaml Signed-off-by: lingxiankong --- index.yaml | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/index.yaml b/index.yaml index 9785228637..f64adb97a6 100644 --- a/index.yaml +++ b/index.yaml @@ -1,6 +1,20 @@ apiVersion: v1 entries: openstack-cinder-csi: + - apiVersion: v1 + appVersion: v1.21.0 + created: "2021-05-20T02:32:09.919675752Z" + description: Cinder CSI Chart for OpenStack + digest: 5d4d66cb80513cb41d273e6c68ec451ddc00bb43990d5feaa08117bf53fc9184 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.3.8/openstack-cinder-csi-1.3.8.tgz + version: 1.3.8 - apiVersion: v1 appVersion: latest created: "2021-04-29T05:24:08.155307767Z" @@ -58,6 +72,24 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.2.2/openstack-cinder-csi-1.2.2.tgz version: 1.2.2 openstack-cloud-controller-manager: + - apiVersion: v1 + appVersion: v1.21.0 + created: "2021-05-20T02:32:10.106942675Z" + description: Openstack Cloud Controller Manager Helm Chart + digest: 04598ab4447edb1b88d00c5ffa81a602d6ae892bb60d56affb8979b820b27dcf + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: kubernetes@maurice-meyer.de + name: morremeyer + url: https://maurice-meyer.de + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-1.0.1/openstack-cloud-controller-manager-1.0.1.tgz + version: 1.0.1 - apiVersion: v1 appVersion: latest created: "2021-04-29T05:24:08.308219138Z" @@ -77,6 +109,20 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-1.0.0/openstack-cloud-controller-manager-1.0.0.tgz version: 1.0.0 openstack-manila-csi: + - apiVersion: v1 + appVersion: v1.21.0 + created: "2021-05-20T02:32:10.303476154Z" + description: Manila CSI Chart for OpenStack + digest: e389cf9d9fd1b9c5e0acf5fa56a0873c30fa70ec9ceff97346017bd807f880cc + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-1.1.2/openstack-manila-csi-1.1.2.tgz + version: 1.1.2 - apiVersion: v1 appVersion: latest created: "2021-04-29T05:24:08.443365397Z" From 8e1ee5b77d463bc3005cc8ed8b2acdc4a9df6d68 Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Mon, 2 Aug 2021 00:39:42 +0000 Subject: [PATCH 08/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/index.yaml b/index.yaml index f64adb97a6..2ea1be07f2 100644 --- a/index.yaml +++ b/index.yaml @@ -72,6 +72,24 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.2.2/openstack-cinder-csi-1.2.2.tgz version: 1.2.2 openstack-cloud-controller-manager: + - apiVersion: v1 + appVersion: v1.21.0 + created: "2021-08-02T00:39:42.411975484Z" + description: Openstack Cloud Controller Manager Helm Chart + digest: eb8a30f58350a0a531136f95c85520ad034ea927475e8599484f80277f7f7164 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: kubernetes@maurice-meyer.de + name: morremeyer + url: https://maurice-meyer.de + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-1.0.2/openstack-cloud-controller-manager-1.0.2.tgz + version: 1.0.2 - apiVersion: v1 appVersion: v1.21.0 created: "2021-05-20T02:32:10.106942675Z" From 6757e77777f0f1e3d63cdc0e7f9c947d9a32fb54 Mon Sep 17 00:00:00 2001 From: lingxiankong Date: Tue, 17 Aug 2021 06:04:47 +0000 Subject: [PATCH 09/90] Update index.yaml Signed-off-by: lingxiankong --- index.yaml | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/index.yaml b/index.yaml index 2ea1be07f2..7d3815a678 100644 --- a/index.yaml +++ b/index.yaml @@ -1,6 +1,20 @@ apiVersion: v1 entries: openstack-cinder-csi: + - apiVersion: v1 + appVersion: latest + created: "2021-08-17T06:04:46.856933422Z" + description: Cinder CSI Chart for OpenStack + digest: 0c3a7e63656c8e8227d5ddef157a0f6c315ff6b074e907be04af3d1309ca9992 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.4.8/openstack-cinder-csi-1.4.8.tgz + version: 1.4.8 - apiVersion: v1 appVersion: v1.21.0 created: "2021-05-20T02:32:09.919675752Z" @@ -72,6 +86,24 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.2.2/openstack-cinder-csi-1.2.2.tgz version: 1.2.2 openstack-cloud-controller-manager: + - apiVersion: v1 + appVersion: latest + created: "2021-08-17T06:04:47.012650408Z" + description: Openstack Cloud Controller Manager Helm Chart + digest: d8578a0d574b5abd5c88fa055d8b22e17f293871411cc01c1b49427bfff3f794 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: kubernetes@maurice-meyer.de + name: morremeyer + url: https://maurice-meyer.de + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-1.1.1/openstack-cloud-controller-manager-1.1.1.tgz + version: 1.1.1 - apiVersion: v1 appVersion: v1.21.0 created: "2021-08-02T00:39:42.411975484Z" @@ -127,6 +159,20 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-1.0.0/openstack-cloud-controller-manager-1.0.0.tgz version: 1.0.0 openstack-manila-csi: + - apiVersion: v1 + appVersion: latest + created: "2021-08-17T06:04:47.162333595Z" + description: Manila CSI Chart for OpenStack + digest: d42da40407d3903163c8d5a7d3d8133d972b8a046301e22d1cfffd1b08c9558a + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-1.3.2/openstack-manila-csi-1.3.2.tgz + version: 1.3.2 - apiVersion: v1 appVersion: v1.21.0 created: "2021-05-20T02:32:10.303476154Z" From e5b66e4f98f28a157659ed502e09587f82695b9e Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Wed, 18 Aug 2021 09:20:30 +0000 Subject: [PATCH 10/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/index.yaml b/index.yaml index 7d3815a678..717bd81d93 100644 --- a/index.yaml +++ b/index.yaml @@ -1,6 +1,20 @@ apiVersion: v1 entries: openstack-cinder-csi: + - apiVersion: v1 + appVersion: v1.22.0 + created: "2021-08-18T09:20:29.81577218Z" + description: Cinder CSI Chart for OpenStack + digest: 5e7ab2c1ec52ba8bfb12787abdf7b97a43e7db15e9b9fd2919b429bb8844114f + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.4.9/openstack-cinder-csi-1.4.9.tgz + version: 1.4.9 - apiVersion: v1 appVersion: latest created: "2021-08-17T06:04:46.856933422Z" @@ -86,6 +100,24 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.2.2/openstack-cinder-csi-1.2.2.tgz version: 1.2.2 openstack-cloud-controller-manager: + - apiVersion: v1 + appVersion: v1.22.0 + created: "2021-08-18T09:20:29.942793176Z" + description: Openstack Cloud Controller Manager Helm Chart + digest: c29859721b4fa83cf17c39a37ed84d18693b5b85e45a073edd509fbc643c7497 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: kubernetes@maurice-meyer.de + name: morremeyer + url: https://maurice-meyer.de + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-1.1.2/openstack-cloud-controller-manager-1.1.2.tgz + version: 1.1.2 - apiVersion: v1 appVersion: latest created: "2021-08-17T06:04:47.012650408Z" @@ -159,6 +191,20 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-1.0.0/openstack-cloud-controller-manager-1.0.0.tgz version: 1.0.0 openstack-manila-csi: + - apiVersion: v1 + appVersion: v1.22.0 + created: "2021-08-18T09:20:30.085429572Z" + description: Manila CSI Chart for OpenStack + digest: 43e49c7b63f751083a7724d0175683597932602c6022b80134059a6ee6f00cf1 + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-1.3.3/openstack-manila-csi-1.3.3.tgz + version: 1.3.3 - apiVersion: v1 appVersion: latest created: "2021-08-17T06:04:47.162333595Z" From e3e2f0ad66a998e00def36efe1746d60514dff11 Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Fri, 14 Jan 2022 01:04:49 +0000 Subject: [PATCH 11/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/index.yaml b/index.yaml index 717bd81d93..4791b75435 100644 --- a/index.yaml +++ b/index.yaml @@ -1,6 +1,20 @@ apiVersion: v1 entries: openstack-cinder-csi: + - apiVersion: v1 + appVersion: v1.23.0 + created: "2022-01-14T01:04:49.561589819Z" + description: Cinder CSI Chart for OpenStack + digest: 1bd7fac743d4a07c3418c053c3fd30f87af6a69c7125550ceed0b39f1d8bd600 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.1.0/openstack-cinder-csi-2.1.0.tgz + version: 2.1.0 - apiVersion: v1 appVersion: v1.22.0 created: "2021-08-18T09:20:29.81577218Z" @@ -100,6 +114,24 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.2.2/openstack-cinder-csi-1.2.2.tgz version: 1.2.2 openstack-cloud-controller-manager: + - apiVersion: v1 + appVersion: v1.23.0 + created: "2022-01-14T01:04:49.755008788Z" + description: Openstack Cloud Controller Manager Helm Chart + digest: b670987cf38a2340c9d63c154a6db2dbf989d2b1c1267eb4a38cfa677f491fd7 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: kubernetes@maurice-meyer.de + name: morremeyer + url: https://maurice-meyer.de + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-1.2.0/openstack-cloud-controller-manager-1.2.0.tgz + version: 1.2.0 - apiVersion: v1 appVersion: v1.22.0 created: "2021-08-18T09:20:29.942793176Z" @@ -191,6 +223,20 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-1.0.0/openstack-cloud-controller-manager-1.0.0.tgz version: 1.0.0 openstack-manila-csi: + - apiVersion: v1 + appVersion: v1.23.0 + created: "2022-01-14T01:04:49.934796447Z" + description: Manila CSI Chart for OpenStack + digest: 15652a79d64ca5405026f97dfa109f78c7a2083e04dd5a0097a8d2e3eb557dc6 + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-1.4.0/openstack-manila-csi-1.4.0.tgz + version: 1.4.0 - apiVersion: v1 appVersion: v1.22.0 created: "2021-08-18T09:20:30.085429572Z" From 46e5af4d014e43bf626ee926d56969f90684918b Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Thu, 17 Feb 2022 11:04:01 +0000 Subject: [PATCH 12/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/index.yaml b/index.yaml index 4791b75435..6362569514 100644 --- a/index.yaml +++ b/index.yaml @@ -43,6 +43,20 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.4.8/openstack-cinder-csi-1.4.8.tgz version: 1.4.8 + - apiVersion: v1 + appVersion: v1.21.0 + created: "2022-02-17T11:04:01.496082638Z" + description: Cinder CSI Chart for OpenStack + digest: af0afb3ed0e07fbbc5332b0d8591fe0c658941c672d61a448714b2eb2ac529f9 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.3.9/openstack-cinder-csi-1.3.9.tgz + version: 1.3.9 - apiVersion: v1 appVersion: v1.21.0 created: "2021-05-20T02:32:09.919675752Z" From 2648c54a6092fe279664153b472a9ed9a73946a2 Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Fri, 20 May 2022 11:23:35 +0000 Subject: [PATCH 13/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/index.yaml b/index.yaml index 6362569514..eab62e14b9 100644 --- a/index.yaml +++ b/index.yaml @@ -1,6 +1,20 @@ apiVersion: v1 entries: openstack-cinder-csi: + - apiVersion: v1 + appVersion: v1.24.0 + created: "2022-05-20T11:23:35.097107107Z" + description: Cinder CSI Chart for OpenStack + digest: 712d56fe4db705c61bed4b8dbdb076fc5cc9d81a2091851b43b6fd562a6674b4 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.2.0/openstack-cinder-csi-2.2.0.tgz + version: 2.2.0 - apiVersion: v1 appVersion: v1.23.0 created: "2022-01-14T01:04:49.561589819Z" @@ -128,6 +142,24 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.2.2/openstack-cinder-csi-1.2.2.tgz version: 1.2.2 openstack-cloud-controller-manager: + - apiVersion: v1 + appVersion: v1.24.0 + created: "2022-05-20T11:23:35.299154735Z" + description: Openstack Cloud Controller Manager Helm Chart + digest: 2eeca5fbaa4ba207647c75576afb9deb67df08026ed33bcca2daeda82e9abbca + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: kubernetes@maurice-meyer.de + name: morremeyer + url: https://maurice-meyer.de + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-1.3.0/openstack-cloud-controller-manager-1.3.0.tgz + version: 1.3.0 - apiVersion: v1 appVersion: v1.23.0 created: "2022-01-14T01:04:49.755008788Z" @@ -237,6 +269,20 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-1.0.0/openstack-cloud-controller-manager-1.0.0.tgz version: 1.0.0 openstack-manila-csi: + - apiVersion: v1 + appVersion: v1.24.0 + created: "2022-05-20T11:23:35.472156988Z" + description: Manila CSI Chart for OpenStack + digest: ad142f8cc96565ed876865c8482410ab19354483d62b609802dd06d7f0e03f93 + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-1.5.0/openstack-manila-csi-1.5.0.tgz + version: 1.5.0 - apiVersion: v1 appVersion: v1.23.0 created: "2022-01-14T01:04:49.934796447Z" From 8e56ff1db11f0534d4a4bb8817dcc0fad4318ff5 Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Tue, 11 Oct 2022 11:47:20 +0000 Subject: [PATCH 14/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/index.yaml b/index.yaml index eab62e14b9..0a2cd0a92e 100644 --- a/index.yaml +++ b/index.yaml @@ -269,6 +269,20 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-1.0.0/openstack-cloud-controller-manager-1.0.0.tgz version: 1.0.0 openstack-manila-csi: + - apiVersion: v1 + appVersion: v1.25.0 + created: "2022-10-11T11:47:20.0657225Z" + description: Manila CSI Chart for OpenStack + digest: ec7ae2755b2ba7948b3b9b5ebe5f80fc46fab5094a3965fbd8d783e4bbad6067 + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-1.6.0/openstack-manila-csi-1.6.0.tgz + version: 1.6.0 - apiVersion: v1 appVersion: v1.24.0 created: "2022-05-20T11:23:35.472156988Z" From 17a50af29d6212a21d3c49a05a8bfe1690971456 Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Mon, 5 Dec 2022 10:41:23 +0000 Subject: [PATCH 15/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/index.yaml b/index.yaml index 0a2cd0a92e..abc51e744b 100644 --- a/index.yaml +++ b/index.yaml @@ -1,6 +1,20 @@ apiVersion: v1 entries: openstack-cinder-csi: + - apiVersion: v1 + appVersion: v1.25.0 + created: "2022-12-05T10:41:23.419537015Z" + description: Cinder CSI Chart for OpenStack + digest: 50b55721a36cd3728bc4189629b1a5dd11e067a73f4f7bdab5e00bc8738fd82e + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.3.0/openstack-cinder-csi-2.3.0.tgz + version: 2.3.0 - apiVersion: v1 appVersion: v1.24.0 created: "2022-05-20T11:23:35.097107107Z" @@ -142,6 +156,24 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.2.2/openstack-cinder-csi-1.2.2.tgz version: 1.2.2 openstack-cloud-controller-manager: + - apiVersion: v1 + appVersion: v1.25.0 + created: "2022-12-05T10:41:23.603847871Z" + description: Openstack Cloud Controller Manager Helm Chart + digest: 0d832a5ca4107bd6ceb1be1263d49dc4260d33e78f29b810d3435fddc78dc246 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: kubernetes@maurice-meyer.de + name: morremeyer + url: https://maurice-meyer.de + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-1.4.0/openstack-cloud-controller-manager-1.4.0.tgz + version: 1.4.0 - apiVersion: v1 appVersion: v1.24.0 created: "2022-05-20T11:23:35.299154735Z" From 5fa2c102dce53c6850c6d134e39c0273ecf110be Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Wed, 15 Mar 2023 09:38:33 +0000 Subject: [PATCH 16/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/index.yaml b/index.yaml index abc51e744b..67bf53c1ef 100644 --- a/index.yaml +++ b/index.yaml @@ -29,6 +29,20 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.2.0/openstack-cinder-csi-2.2.0.tgz version: 2.2.0 + - apiVersion: v1 + appVersion: v1.23.0 + created: "2023-03-15T09:38:33.716818561Z" + description: Cinder CSI Chart for OpenStack + digest: 7ef04d3c9addbc2d8bf61ffd7b03280d8f249d82047e97225c0f3d8821298004 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.1.1/openstack-cinder-csi-2.1.1.tgz + version: 2.1.1 - apiVersion: v1 appVersion: v1.23.0 created: "2022-01-14T01:04:49.561589819Z" From c32486d21e2790cca9ee651564eeb8c48743b7d3 Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Fri, 17 Mar 2023 12:01:36 +0000 Subject: [PATCH 17/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/index.yaml b/index.yaml index 67bf53c1ef..0c6cd16b2f 100644 --- a/index.yaml +++ b/index.yaml @@ -15,6 +15,20 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.3.0/openstack-cinder-csi-2.3.0.tgz version: 2.3.0 + - apiVersion: v1 + appVersion: v1.24.6 + created: "2023-03-17T12:01:36.738570211Z" + description: Cinder CSI Chart for OpenStack + digest: 49bfd83651278d8ad0ef05de79321fbb32b10582f7432ab400d73456e4ec34fb + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.2.1/openstack-cinder-csi-2.2.1.tgz + version: 2.2.1 - apiVersion: v1 appVersion: v1.24.0 created: "2022-05-20T11:23:35.097107107Z" From 20ecb05812779826f117aaee1b826108645b24a7 Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Tue, 21 Mar 2023 06:09:32 +0000 Subject: [PATCH 18/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/index.yaml b/index.yaml index 0c6cd16b2f..c45fc3976e 100644 --- a/index.yaml +++ b/index.yaml @@ -1,6 +1,20 @@ apiVersion: v1 entries: openstack-cinder-csi: + - apiVersion: v1 + appVersion: v1.26.2 + created: "2023-03-21T06:09:31.979572431Z" + description: Cinder CSI Chart for OpenStack + digest: ab73cfa624f7908dd59e09a867ff66d08fecd24a0df687a62bbf20ade3afbb89 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.26.0/openstack-cinder-csi-2.26.0.tgz + version: 2.26.0 - apiVersion: v1 appVersion: v1.25.0 created: "2022-12-05T10:41:23.419537015Z" @@ -184,6 +198,21 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.2.2/openstack-cinder-csi-1.2.2.tgz version: 1.2.2 openstack-cloud-controller-manager: + - apiVersion: v1 + appVersion: v1.26.2 + created: "2023-03-21T06:09:32.21423869Z" + description: Openstack Cloud Controller Manager Helm Chart + digest: c098dbe92b7fbfc7749ed35487d8a2f9845442f1ca037b0abc5349884a81b392 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.26.0/openstack-cloud-controller-manager-2.26.0.tgz + version: 2.26.0 - apiVersion: v1 appVersion: v1.25.0 created: "2022-12-05T10:41:23.603847871Z" @@ -329,6 +358,20 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-1.0.0/openstack-cloud-controller-manager-1.0.0.tgz version: 1.0.0 openstack-manila-csi: + - apiVersion: v1 + appVersion: v1.26.2 + created: "2023-03-21T06:09:32.435940246Z" + description: Manila CSI Chart for OpenStack + digest: 1ee20e1e497a3438155a554e60dde19f1269f5be65e6861af0b6a2c81401a10b + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.26.0/openstack-manila-csi-2.26.0.tgz + version: 2.26.0 - apiVersion: v1 appVersion: v1.25.0 created: "2022-10-11T11:47:20.0657225Z" From 6fe2f8f62fd02aa41ffc6d7c6cf55a2de1bd539e Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Tue, 21 Mar 2023 23:34:27 +0000 Subject: [PATCH 19/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/index.yaml b/index.yaml index c45fc3976e..a928e112ef 100644 --- a/index.yaml +++ b/index.yaml @@ -15,6 +15,20 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.26.0/openstack-cinder-csi-2.26.0.tgz version: 2.26.0 + - apiVersion: v1 + appVersion: v1.24.6 + created: "2023-03-21T23:34:27.204662517Z" + description: Cinder CSI Chart for OpenStack + digest: d5554c17cd0f44c2e442cfd721d89db9a5562c43cac07413d0ba27ff684140d0 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.24.0/openstack-cinder-csi-2.24.0.tgz + version: 2.24.0 - apiVersion: v1 appVersion: v1.25.0 created: "2022-12-05T10:41:23.419537015Z" @@ -213,6 +227,21 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.26.0/openstack-cloud-controller-manager-2.26.0.tgz version: 2.26.0 + - apiVersion: v1 + appVersion: v1.24.6 + created: "2023-03-21T23:34:27.344870891Z" + description: Openstack Cloud Controller Manager Helm Chart + digest: c58218556d5724d84e82856b0ec80d5ebae216760ace6dce7223116ad6ef8650 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.24.0/openstack-cloud-controller-manager-2.24.0.tgz + version: 2.24.0 - apiVersion: v1 appVersion: v1.25.0 created: "2022-12-05T10:41:23.603847871Z" @@ -372,6 +401,20 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.26.0/openstack-manila-csi-2.26.0.tgz version: 2.26.0 + - apiVersion: v1 + appVersion: v1.24.6 + created: "2023-03-21T23:34:27.505944358Z" + description: Manila CSI Chart for OpenStack + digest: fcea23824f5994f063c016efb4d78043025bf2b98ed17334fe89712e7eed4c93 + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.24.0/openstack-manila-csi-2.24.0.tgz + version: 2.24.0 - apiVersion: v1 appVersion: v1.25.0 created: "2022-10-11T11:47:20.0657225Z" From 05846859e1d5960d86195bae260c15eb541f03a4 Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Tue, 21 Mar 2023 23:38:24 +0000 Subject: [PATCH 20/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/index.yaml b/index.yaml index a928e112ef..c021509fb6 100644 --- a/index.yaml +++ b/index.yaml @@ -15,6 +15,20 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.26.0/openstack-cinder-csi-2.26.0.tgz version: 2.26.0 + - apiVersion: v1 + appVersion: v1.25.5 + created: "2023-03-21T23:38:24.347810069Z" + description: Cinder CSI Chart for OpenStack + digest: da0a97120edcaa3f9143e2d150aa9bb4175e7933751160ab5ecd1a21946797ea + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.25.0/openstack-cinder-csi-2.25.0.tgz + version: 2.25.0 - apiVersion: v1 appVersion: v1.24.6 created: "2023-03-21T23:34:27.204662517Z" @@ -227,6 +241,21 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.26.0/openstack-cloud-controller-manager-2.26.0.tgz version: 2.26.0 + - apiVersion: v1 + appVersion: v1.25.5 + created: "2023-03-21T23:38:24.535974955Z" + description: Openstack Cloud Controller Manager Helm Chart + digest: eb7167c5e4d0e225f4362c4e97f852d579d557a3baa3961babfaf7a9a1e56965 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.25.0/openstack-cloud-controller-manager-2.25.0.tgz + version: 2.25.0 - apiVersion: v1 appVersion: v1.24.6 created: "2023-03-21T23:34:27.344870891Z" @@ -401,6 +430,20 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.26.0/openstack-manila-csi-2.26.0.tgz version: 2.26.0 + - apiVersion: v1 + appVersion: v1.25.5 + created: "2023-03-21T23:38:24.714714252Z" + description: Manila CSI Chart for OpenStack + digest: 422394c6e40aa936b0b8561b710db6d1edc58020ef2ec79001f662b5d010746a + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.25.0/openstack-manila-csi-2.25.0.tgz + version: 2.25.0 - apiVersion: v1 appVersion: v1.24.6 created: "2023-03-21T23:34:27.505944358Z" From 37d2c827eb88c5e2db26ac40f09c8a548bf21184 Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Wed, 22 Mar 2023 08:18:58 +0000 Subject: [PATCH 21/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/index.yaml b/index.yaml index c021509fb6..01dc348572 100644 --- a/index.yaml +++ b/index.yaml @@ -1,6 +1,20 @@ apiVersion: v1 entries: openstack-cinder-csi: + - apiVersion: v1 + appVersion: v1.27.0-alpha.0 + created: "2023-03-22T08:18:58.141361749Z" + description: Cinder CSI Chart for OpenStack + digest: 0172040dc5c1f32dbb2241dac0632b85c272e1d2f56c9317b874bbc750c1a34c + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.27.0-alpha.0/openstack-cinder-csi-2.27.0-alpha.0.tgz + version: 2.27.0-alpha.0 - apiVersion: v1 appVersion: v1.26.2 created: "2023-03-21T06:09:31.979572431Z" @@ -226,6 +240,21 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.2.2/openstack-cinder-csi-1.2.2.tgz version: 1.2.2 openstack-cloud-controller-manager: + - apiVersion: v1 + appVersion: v1.27.0-alpha.0 + created: "2023-03-22T08:18:58.305150054Z" + description: Openstack Cloud Controller Manager Helm Chart + digest: 86cdc09bbc2c43925d3b6d822b7623711ed61fd389a708f68dd82ed6956cd4eb + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.27.0-alpha.0/openstack-cloud-controller-manager-2.27.0-alpha.0.tgz + version: 2.27.0-alpha.0 - apiVersion: v1 appVersion: v1.26.2 created: "2023-03-21T06:09:32.21423869Z" @@ -416,6 +445,20 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-1.0.0/openstack-cloud-controller-manager-1.0.0.tgz version: 1.0.0 openstack-manila-csi: + - apiVersion: v1 + appVersion: v1.27.0-alpha.0 + created: "2023-03-22T08:18:58.470330191Z" + description: Manila CSI Chart for OpenStack + digest: 8acac29b3435edd8c4fbef55f5e30c3d7e9a1afeee250e0ed28e1f1c23a7fd9b + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.27.0-alpha.0/openstack-manila-csi-2.27.0-alpha.0.tgz + version: 2.27.0-alpha.0 - apiVersion: v1 appVersion: v1.26.2 created: "2023-03-21T06:09:32.435940246Z" From 22dbfd00431fee70085630f8056b22daff44e7a4 Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Tue, 28 Mar 2023 19:06:06 +0000 Subject: [PATCH 22/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/index.yaml b/index.yaml index 01dc348572..52fe1c6dd8 100644 --- a/index.yaml +++ b/index.yaml @@ -1,6 +1,20 @@ apiVersion: v1 entries: openstack-cinder-csi: + - apiVersion: v1 + appVersion: v1.27.0-alpha.1 + created: "2023-03-28T19:06:06.273727217Z" + description: Cinder CSI Chart for OpenStack + digest: d1be98edfadb1e2a714e3d06068612567479ed2879b9d1015b5bb875124a6849 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.27.0-alpha.1/openstack-cinder-csi-2.27.0-alpha.1.tgz + version: 2.27.0-alpha.1 - apiVersion: v1 appVersion: v1.27.0-alpha.0 created: "2023-03-22T08:18:58.141361749Z" @@ -240,6 +254,21 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.2.2/openstack-cinder-csi-1.2.2.tgz version: 1.2.2 openstack-cloud-controller-manager: + - apiVersion: v1 + appVersion: v1.27.0-alpha.1 + created: "2023-03-28T19:06:06.477600444Z" + description: Openstack Cloud Controller Manager Helm Chart + digest: 680f3f988d79d042857e1e28c052b3dab651b811e082814dbfa2c6671e4a3092 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.27.0-alpha.1/openstack-cloud-controller-manager-2.27.0-alpha.1.tgz + version: 2.27.0-alpha.1 - apiVersion: v1 appVersion: v1.27.0-alpha.0 created: "2023-03-22T08:18:58.305150054Z" @@ -445,6 +474,20 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-1.0.0/openstack-cloud-controller-manager-1.0.0.tgz version: 1.0.0 openstack-manila-csi: + - apiVersion: v1 + appVersion: v1.27.0-alpha.1 + created: "2023-03-28T19:06:06.650516521Z" + description: Manila CSI Chart for OpenStack + digest: 9e99489e7eef0bf94d3fabe08c85319b822bebee1fefcd7bd2cb288055557d91 + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.27.0-alpha.1/openstack-manila-csi-2.27.0-alpha.1.tgz + version: 2.27.0-alpha.1 - apiVersion: v1 appVersion: v1.27.0-alpha.0 created: "2023-03-22T08:18:58.470330191Z" From e867d82264a70423961fac46f537ecbdfb2d8c26 Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Tue, 18 Apr 2023 04:13:18 +0000 Subject: [PATCH 23/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/index.yaml b/index.yaml index 52fe1c6dd8..f23c27fb92 100644 --- a/index.yaml +++ b/index.yaml @@ -1,6 +1,20 @@ apiVersion: v1 entries: openstack-cinder-csi: + - apiVersion: v1 + appVersion: v1.27.0-alpha.2 + created: "2023-04-18T04:13:18.034389497Z" + description: Cinder CSI Chart for OpenStack + digest: 859b87b6f74598d2ac0d2345a42ee16d164c5c4407a239c07f77e89f8625c4ef + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.27.0-alpha.2/openstack-cinder-csi-2.27.0-alpha.2.tgz + version: 2.27.0-alpha.2 - apiVersion: v1 appVersion: v1.27.0-alpha.1 created: "2023-03-28T19:06:06.273727217Z" From f87588124a0df615c61d5aadeef8a939724b2609 Mon Sep 17 00:00:00 2001 From: zetaab Date: Tue, 18 Apr 2023 19:44:51 +0000 Subject: [PATCH 24/90] Update index.yaml Signed-off-by: zetaab --- index.yaml | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/index.yaml b/index.yaml index f23c27fb92..ab12f92ef0 100644 --- a/index.yaml +++ b/index.yaml @@ -1,6 +1,20 @@ apiVersion: v1 entries: openstack-cinder-csi: + - apiVersion: v1 + appVersion: v1.27.0 + created: "2023-04-18T19:44:50.965957846Z" + description: Cinder CSI Chart for OpenStack + digest: 313b3f669c31627a0eb0ef912980ec8236534e377436d61296a484d87c0a781f + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.27.0/openstack-cinder-csi-2.27.0.tgz + version: 2.27.0 - apiVersion: v1 appVersion: v1.27.0-alpha.2 created: "2023-04-18T04:13:18.034389497Z" @@ -268,6 +282,21 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.2.2/openstack-cinder-csi-1.2.2.tgz version: 1.2.2 openstack-cloud-controller-manager: + - apiVersion: v1 + appVersion: v1.27.0 + created: "2023-04-18T19:44:51.130083675Z" + description: Openstack Cloud Controller Manager Helm Chart + digest: 33781916172e959348dfb2663de488271081e98632aa50ada8e872b989662680 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.27.0/openstack-cloud-controller-manager-2.27.0.tgz + version: 2.27.0 - apiVersion: v1 appVersion: v1.27.0-alpha.1 created: "2023-03-28T19:06:06.477600444Z" @@ -488,6 +517,20 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-1.0.0/openstack-cloud-controller-manager-1.0.0.tgz version: 1.0.0 openstack-manila-csi: + - apiVersion: v1 + appVersion: v1.27.0 + created: "2023-04-18T19:44:51.302201294Z" + description: Manila CSI Chart for OpenStack + digest: 76d3284a4529a65a4d8d56c427d5a359f2c8d8e34883f5fd330ddd9d3ddc4484 + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.27.0/openstack-manila-csi-2.27.0.tgz + version: 2.27.0 - apiVersion: v1 appVersion: v1.27.0-alpha.1 created: "2023-03-28T19:06:06.650516521Z" From c44c3c728933cdedf3f3627b6ee827173ccbfe8b Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Mon, 24 Apr 2023 06:27:32 +0000 Subject: [PATCH 25/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/index.yaml b/index.yaml index ab12f92ef0..f48b596896 100644 --- a/index.yaml +++ b/index.yaml @@ -1,6 +1,20 @@ apiVersion: v1 entries: openstack-cinder-csi: + - apiVersion: v1 + appVersion: v1.27.1 + created: "2023-04-24T06:27:31.974898619Z" + description: Cinder CSI Chart for OpenStack + digest: ab0ba4ca8533d6385b7ad6ef444cfdeed7cbd4e3a5aba645b52d68ebfe159310 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.27.1/openstack-cinder-csi-2.27.1.tgz + version: 2.27.1 - apiVersion: v1 appVersion: v1.27.0 created: "2023-04-18T19:44:50.965957846Z" @@ -282,6 +296,21 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.2.2/openstack-cinder-csi-1.2.2.tgz version: 1.2.2 openstack-cloud-controller-manager: + - apiVersion: v1 + appVersion: v1.27.1 + created: "2023-04-24T06:27:32.111610998Z" + description: Openstack Cloud Controller Manager Helm Chart + digest: fbb6d59f15ca8bafcf7219e6c83815e10a86a8b2e5aaf2ddc086f62a09da5e0c + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.27.1/openstack-cloud-controller-manager-2.27.1.tgz + version: 2.27.1 - apiVersion: v1 appVersion: v1.27.0 created: "2023-04-18T19:44:51.130083675Z" @@ -517,6 +546,20 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-1.0.0/openstack-cloud-controller-manager-1.0.0.tgz version: 1.0.0 openstack-manila-csi: + - apiVersion: v1 + appVersion: v1.27.1 + created: "2023-04-24T06:27:32.233205401Z" + description: Manila CSI Chart for OpenStack + digest: a4706bd7a705badf7dd02a9fe4baf6b814bb870796c71da86145842efdbc318d + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.27.1/openstack-manila-csi-2.27.1.tgz + version: 2.27.1 - apiVersion: v1 appVersion: v1.27.0 created: "2023-04-18T19:44:51.302201294Z" From 751188b7f0e4dd8b73a1dd44595474f4807a8677 Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Wed, 26 Apr 2023 13:54:33 +0000 Subject: [PATCH 26/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/index.yaml b/index.yaml index f48b596896..6a48c280cc 100644 --- a/index.yaml +++ b/index.yaml @@ -296,6 +296,21 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.2.2/openstack-cinder-csi-1.2.2.tgz version: 1.2.2 openstack-cloud-controller-manager: + - apiVersion: v1 + appVersion: v1.27.0 + created: "2023-04-26T13:54:33.711036846Z" + description: Openstack Cloud Controller Manager Helm Chart + digest: 359cdc038adc213a760f3e319977054c2ea66b11e69c5786a428731a4580449c + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.28.0-alpha.2/openstack-cloud-controller-manager-2.28.0-alpha.2.tgz + version: 2.28.0-alpha.2 - apiVersion: v1 appVersion: v1.27.1 created: "2023-04-24T06:27:32.111610998Z" From 50a170d8ea4104ddd52d53f67f5f65f1a4d49a56 Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Thu, 27 Apr 2023 18:48:42 +0000 Subject: [PATCH 27/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/index.yaml b/index.yaml index 6a48c280cc..700006b201 100644 --- a/index.yaml +++ b/index.yaml @@ -1,6 +1,20 @@ apiVersion: v1 entries: openstack-cinder-csi: + - apiVersion: v1 + appVersion: v1.27.1 + created: "2023-04-27T18:48:42.126589886Z" + description: Cinder CSI Chart for OpenStack + digest: 1dcd6a8b0347ffc0802ddaf125e624037acce71380290c5aca2013e51bf401c9 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.28.0-alpha.2/openstack-cinder-csi-2.28.0-alpha.2.tgz + version: 2.28.0-alpha.2 - apiVersion: v1 appVersion: v1.27.1 created: "2023-04-24T06:27:31.974898619Z" @@ -296,6 +310,21 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.2.2/openstack-cinder-csi-1.2.2.tgz version: 1.2.2 openstack-cloud-controller-manager: + - apiVersion: v1 + appVersion: v1.27.1 + created: "2023-04-27T18:48:42.342944475Z" + description: Openstack Cloud Controller Manager Helm Chart + digest: 2ce630c39c9ebc4058214878e0633024391663a9285626f2c39b50bb4211b4ad + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.28.0-alpha.3/openstack-cloud-controller-manager-2.28.0-alpha.3.tgz + version: 2.28.0-alpha.3 - apiVersion: v1 appVersion: v1.27.0 created: "2023-04-26T13:54:33.711036846Z" @@ -561,6 +590,20 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-1.0.0/openstack-cloud-controller-manager-1.0.0.tgz version: 1.0.0 openstack-manila-csi: + - apiVersion: v1 + appVersion: v1.27.1 + created: "2023-04-27T18:48:42.572605969Z" + description: Manila CSI Chart for OpenStack + digest: ba228aba5a09decf0e2f60a1e097e0dbc32a4390699dd2dc14fbc8ac005f9b98 + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.28.0-alpha.2/openstack-manila-csi-2.28.0-alpha.2.tgz + version: 2.28.0-alpha.2 - apiVersion: v1 appVersion: v1.27.1 created: "2023-04-24T06:27:32.233205401Z" From e826ead2729fb920af9f24bce8fb9b26189dc070 Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Sat, 13 May 2023 17:03:46 +0000 Subject: [PATCH 28/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/index.yaml b/index.yaml index 700006b201..7105692d68 100644 --- a/index.yaml +++ b/index.yaml @@ -310,6 +310,21 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.2.2/openstack-cinder-csi-1.2.2.tgz version: 1.2.2 openstack-cloud-controller-manager: + - apiVersion: v1 + appVersion: v1.27.1 + created: "2023-05-13T17:03:46.324627283Z" + description: Openstack Cloud Controller Manager Helm Chart + digest: a0af0284f63796503ffb8a53d39ca294c4cbf775a5b968b09152b30038b784c3 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.28.0-alpha.4/openstack-cloud-controller-manager-2.28.0-alpha.4.tgz + version: 2.28.0-alpha.4 - apiVersion: v1 appVersion: v1.27.1 created: "2023-04-27T18:48:42.342944475Z" From 8125132cc3c9caf23712ea6d8841a7d1b2d1276e Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Thu, 25 May 2023 11:11:11 +0000 Subject: [PATCH 29/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/index.yaml b/index.yaml index 7105692d68..84e36614e5 100644 --- a/index.yaml +++ b/index.yaml @@ -1,6 +1,20 @@ apiVersion: v1 entries: openstack-cinder-csi: + - apiVersion: v1 + appVersion: v1.27.1 + created: "2023-05-25T11:11:11.579254866Z" + description: Cinder CSI Chart for OpenStack + digest: b2a040a40c3341a35bae2a54e07c99dcf2f65a559f110952ecad086b06625fbb + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.28.0-alpha.3/openstack-cinder-csi-2.28.0-alpha.3.tgz + version: 2.28.0-alpha.3 - apiVersion: v1 appVersion: v1.27.1 created: "2023-04-27T18:48:42.126589886Z" @@ -605,6 +619,20 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-1.0.0/openstack-cloud-controller-manager-1.0.0.tgz version: 1.0.0 openstack-manila-csi: + - apiVersion: v1 + appVersion: v1.27.1 + created: "2023-05-25T11:11:11.749929733Z" + description: Manila CSI Chart for OpenStack + digest: 59d75e57b59bfc1b0af59fee87b24fb4dce75796e647508b037ab200c3a87729 + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.28.0-alpha.3/openstack-manila-csi-2.28.0-alpha.3.tgz + version: 2.28.0-alpha.3 - apiVersion: v1 appVersion: v1.27.1 created: "2023-04-27T18:48:42.572605969Z" From 522f6d15cbac0ae1b7456434103bcb2892c726a8 Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Wed, 31 May 2023 07:52:06 +0000 Subject: [PATCH 30/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/index.yaml b/index.yaml index 84e36614e5..6e241a20bc 100644 --- a/index.yaml +++ b/index.yaml @@ -324,6 +324,21 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.2.2/openstack-cinder-csi-1.2.2.tgz version: 1.2.2 openstack-cloud-controller-manager: + - apiVersion: v1 + appVersion: v1.27.1 + created: "2023-05-31T07:52:06.541041482Z" + description: Openstack Cloud Controller Manager Helm Chart + digest: bdf7ea2a15cf629d451250c71e1a8e5e3a3af6ed89e1d90a34e5e11a9270119a + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.28.0-alpha.5/openstack-cloud-controller-manager-2.28.0-alpha.5.tgz + version: 2.28.0-alpha.5 - apiVersion: v1 appVersion: v1.27.1 created: "2023-05-13T17:03:46.324627283Z" From 7bff7ae6dbf60b88a81f0843fe7e0a18eb7a3e3e Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Thu, 8 Jun 2023 01:26:35 +0000 Subject: [PATCH 31/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/index.yaml b/index.yaml index 6e241a20bc..10cc195c1d 100644 --- a/index.yaml +++ b/index.yaml @@ -1,6 +1,20 @@ apiVersion: v1 entries: openstack-cinder-csi: + - apiVersion: v1 + appVersion: v1.27.1 + created: "2023-06-08T01:26:35.010839496Z" + description: Cinder CSI Chart for OpenStack + digest: 5d56602c98b9ae1afddb2c7e4ca44cb14f9fc824adc6bbeda7b6f0797a471ede + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.28.0-alpha.4/openstack-cinder-csi-2.28.0-alpha.4.tgz + version: 2.28.0-alpha.4 - apiVersion: v1 appVersion: v1.27.1 created: "2023-05-25T11:11:11.579254866Z" From 5c17b74f4cd15b7a6f9cd9e73d0fd8540f1efd72 Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Wed, 14 Jun 2023 04:06:19 +0000 Subject: [PATCH 32/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/index.yaml b/index.yaml index 10cc195c1d..0be34a7f6c 100644 --- a/index.yaml +++ b/index.yaml @@ -127,6 +127,20 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.26.0/openstack-cinder-csi-2.26.0.tgz version: 2.26.0 + - apiVersion: v1 + appVersion: v1.25.6 + created: "2023-06-14T04:06:19.353092436Z" + description: Cinder CSI Chart for OpenStack + digest: 0eabc087f302c8f1c0ce3b54ea5c4ace3859e62251e3e457dcb06593da51ece7 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.25.1/openstack-cinder-csi-2.25.1.tgz + version: 2.25.1 - apiVersion: v1 appVersion: v1.25.5 created: "2023-03-21T23:38:24.347810069Z" @@ -473,6 +487,21 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.26.0/openstack-cloud-controller-manager-2.26.0.tgz version: 2.26.0 + - apiVersion: v1 + appVersion: v1.25.6 + created: "2023-06-14T04:06:19.48768388Z" + description: Openstack Cloud Controller Manager Helm Chart + digest: f100fb6400d2460ea6aba7fb4223161fba08e5eb96773007353eb42214a08433 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.25.1/openstack-cloud-controller-manager-2.25.1.tgz + version: 2.25.1 - apiVersion: v1 appVersion: v1.25.5 created: "2023-03-21T23:38:24.535974955Z" @@ -746,6 +775,20 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.26.0/openstack-manila-csi-2.26.0.tgz version: 2.26.0 + - apiVersion: v1 + appVersion: v1.25.6 + created: "2023-06-14T04:06:19.628236379Z" + description: Manila CSI Chart for OpenStack + digest: d91bcd167378497dbb6d905bd834c5263483b1b9e7cc563ebdcb98106f5a4ac6 + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.25.1/openstack-manila-csi-2.25.1.tgz + version: 2.25.1 - apiVersion: v1 appVersion: v1.25.5 created: "2023-03-21T23:38:24.714714252Z" From 552931ed7c03c9b646701cd470826d406ddb9d42 Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Wed, 21 Jun 2023 14:26:11 +0000 Subject: [PATCH 33/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/index.yaml b/index.yaml index 0be34a7f6c..a525fb8c61 100644 --- a/index.yaml +++ b/index.yaml @@ -352,6 +352,21 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.2.2/openstack-cinder-csi-1.2.2.tgz version: 1.2.2 openstack-cloud-controller-manager: + - apiVersion: v1 + appVersion: v1.27.1 + created: "2023-06-21T14:26:11.077266334Z" + description: Openstack Cloud Controller Manager Helm Chart + digest: 219c4d00b68bc4f54d6559882d2a592451857c6136b7cdef5f51fb10dd61c70c + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.28.0-alpha.6/openstack-cloud-controller-manager-2.28.0-alpha.6.tgz + version: 2.28.0-alpha.6 - apiVersion: v1 appVersion: v1.27.1 created: "2023-05-31T07:52:06.541041482Z" From 738838368c9becf77f0f8821c47cf8069dd3158e Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Mon, 10 Jul 2023 01:31:26 +0000 Subject: [PATCH 34/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/index.yaml b/index.yaml index a525fb8c61..3c9e96d08b 100644 --- a/index.yaml +++ b/index.yaml @@ -1,6 +1,20 @@ apiVersion: v1 entries: openstack-cinder-csi: + - apiVersion: v1 + appVersion: v1.27.1 + created: "2023-07-10T01:31:26.061567398Z" + description: Cinder CSI Chart for OpenStack + digest: 4b9871739bb01c5a8806a9c51d0c4334d05989ea093348e2dd66977d19a738f9 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.28.0-alpha.5/openstack-cinder-csi-2.28.0-alpha.5.tgz + version: 2.28.0-alpha.5 - apiVersion: v1 appVersion: v1.27.1 created: "2023-06-08T01:26:35.010839496Z" @@ -352,6 +366,21 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.2.2/openstack-cinder-csi-1.2.2.tgz version: 1.2.2 openstack-cloud-controller-manager: + - apiVersion: v1 + appVersion: v1.27.1 + created: "2023-07-10T01:31:26.219499002Z" + description: Openstack Cloud Controller Manager Helm Chart + digest: 98dee96a675495eab4af6d192b165f5b5934087420bbfbd73f8e6c24e680064d + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.28.0-alpha.7/openstack-cloud-controller-manager-2.28.0-alpha.7.tgz + version: 2.28.0-alpha.7 - apiVersion: v1 appVersion: v1.27.1 created: "2023-06-21T14:26:11.077266334Z" From 4262dd6f313adc4a49bd694bbd7a0d28e50d9e6b Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Fri, 18 Aug 2023 03:32:47 +0000 Subject: [PATCH 35/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/index.yaml b/index.yaml index 3c9e96d08b..f9b3e10949 100644 --- a/index.yaml +++ b/index.yaml @@ -456,6 +456,21 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.28.0-alpha.2/openstack-cloud-controller-manager-2.28.0-alpha.2.tgz version: 2.28.0-alpha.2 + - apiVersion: v1 + appVersion: v1.27.1 + created: "2023-08-18T03:32:47.424697743Z" + description: Openstack Cloud Controller Manager Helm Chart + digest: 3d6b894536015d94cae3d91980f861952d2a8b9b9c90b656da50b71b99c16ad4 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.27.2/openstack-cloud-controller-manager-2.27.2.tgz + version: 2.27.2 - apiVersion: v1 appVersion: v1.27.1 created: "2023-04-24T06:27:32.111610998Z" From b420b838f8d9a486dfe2873645237a8ef5400685 Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Mon, 21 Aug 2023 02:59:45 +0000 Subject: [PATCH 36/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/index.yaml b/index.yaml index f9b3e10949..e049fbbd84 100644 --- a/index.yaml +++ b/index.yaml @@ -456,21 +456,6 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.28.0-alpha.2/openstack-cloud-controller-manager-2.28.0-alpha.2.tgz version: 2.28.0-alpha.2 - - apiVersion: v1 - appVersion: v1.27.1 - created: "2023-08-18T03:32:47.424697743Z" - description: Openstack Cloud Controller Manager Helm Chart - digest: 3d6b894536015d94cae3d91980f861952d2a8b9b9c90b656da50b71b99c16ad4 - home: https://github.com/kubernetes/cloud-provider-openstack - icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png - maintainers: - - email: f.kloeker@telekom.de - name: eumel8 - url: https://www.telekom.com - name: openstack-cloud-controller-manager - urls: - - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.27.2/openstack-cloud-controller-manager-2.27.2.tgz - version: 2.27.2 - apiVersion: v1 appVersion: v1.27.1 created: "2023-04-24T06:27:32.111610998Z" @@ -736,6 +721,20 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-1.0.0/openstack-cloud-controller-manager-1.0.0.tgz version: 1.0.0 openstack-manila-csi: + - apiVersion: v1 + appVersion: v1.27.1 + created: "2023-08-21T02:59:45.885778032Z" + description: Manila CSI Chart for OpenStack + digest: d387b673142b7a16a9c39a611de8757f44f8fb0cf90923468cb361392fb0842a + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.28.0-alpha.4/openstack-manila-csi-2.28.0-alpha.4.tgz + version: 2.28.0-alpha.4 - apiVersion: v1 appVersion: v1.27.1 created: "2023-05-25T11:11:11.749929733Z" From 41db6e868117da66a28f6e71e833bed746086900 Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Fri, 25 Aug 2023 08:43:39 +0000 Subject: [PATCH 37/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/index.yaml b/index.yaml index e049fbbd84..89d880bf8f 100644 --- a/index.yaml +++ b/index.yaml @@ -1,6 +1,20 @@ apiVersion: v1 entries: openstack-cinder-csi: + - apiVersion: v1 + appVersion: v1.28.0 + created: "2023-08-25T08:43:38.62659305Z" + description: Cinder CSI Chart for OpenStack + digest: c967bfdb11f5212f1a05bf3ce45c98f4e48702ffadbbbbec93cf5a5fd0a45894 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.28.0/openstack-cinder-csi-2.28.0.tgz + version: 2.28.0 - apiVersion: v1 appVersion: v1.27.1 created: "2023-07-10T01:31:26.061567398Z" @@ -366,6 +380,21 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.2.2/openstack-cinder-csi-1.2.2.tgz version: 1.2.2 openstack-cloud-controller-manager: + - apiVersion: v1 + appVersion: v1.28.0 + created: "2023-08-25T08:43:38.881497024Z" + description: Openstack Cloud Controller Manager Helm Chart + digest: 32aece1e37f86477c3904b45e79d6e40855992e42a1cd8a74898e67a6f2b97b9 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.28.0/openstack-cloud-controller-manager-2.28.0.tgz + version: 2.28.0 - apiVersion: v1 appVersion: v1.27.1 created: "2023-07-10T01:31:26.219499002Z" @@ -721,6 +750,20 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-1.0.0/openstack-cloud-controller-manager-1.0.0.tgz version: 1.0.0 openstack-manila-csi: + - apiVersion: v1 + appVersion: v1.28.0 + created: "2023-08-25T08:43:39.113350347Z" + description: Manila CSI Chart for OpenStack + digest: d81954028ba375b2b2eed086217c1886d588e718ffeab82ecd21b055e5c68c3b + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.28.0/openstack-manila-csi-2.28.0.tgz + version: 2.28.0 - apiVersion: v1 appVersion: v1.27.1 created: "2023-08-21T02:59:45.885778032Z" From 26984914924cc72200c38dcf8e0877a6cf6078eb Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Fri, 25 Aug 2023 10:29:39 +0000 Subject: [PATCH 38/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/index.yaml b/index.yaml index 89d880bf8f..45ac5adc50 100644 --- a/index.yaml +++ b/index.yaml @@ -1,6 +1,20 @@ apiVersion: v1 entries: openstack-cinder-csi: + - apiVersion: v1 + appVersion: v1.28.0 + created: "2023-08-25T10:29:39.223752658Z" + description: Cinder CSI Chart for OpenStack + digest: 12979bdc3c1e5671536f0444507bdb78d5969cf96dbdfbf98576ad2b80e28fd3 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.29.0-alpha.0/openstack-cinder-csi-2.29.0-alpha.0.tgz + version: 2.29.0-alpha.0 - apiVersion: v1 appVersion: v1.28.0 created: "2023-08-25T08:43:38.62659305Z" @@ -380,6 +394,21 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.2.2/openstack-cinder-csi-1.2.2.tgz version: 1.2.2 openstack-cloud-controller-manager: + - apiVersion: v1 + appVersion: v1.28.0 + created: "2023-08-25T10:29:39.373826276Z" + description: Openstack Cloud Controller Manager Helm Chart + digest: 0e9d86fce30feb26cf61293e285ee7a6bc6847772ad56da2013980c378c54d23 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.29.0-alpha.0/openstack-cloud-controller-manager-2.29.0-alpha.0.tgz + version: 2.29.0-alpha.0 - apiVersion: v1 appVersion: v1.28.0 created: "2023-08-25T08:43:38.881497024Z" @@ -750,6 +779,20 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-1.0.0/openstack-cloud-controller-manager-1.0.0.tgz version: 1.0.0 openstack-manila-csi: + - apiVersion: v1 + appVersion: v1.28.0 + created: "2023-08-25T10:29:39.554160577Z" + description: Manila CSI Chart for OpenStack + digest: d1633c1e4733d5de088e8920a1a9c32cf67a11031943eb053420f80a2a2946fd + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.29.0-alpha.0/openstack-manila-csi-2.29.0-alpha.0.tgz + version: 2.29.0-alpha.0 - apiVersion: v1 appVersion: v1.28.0 created: "2023-08-25T08:43:39.113350347Z" From 95d3155c3768c5c5efc84f57d2335cc1f9da8b10 Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Wed, 30 Aug 2023 01:08:14 +0000 Subject: [PATCH 39/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/index.yaml b/index.yaml index 45ac5adc50..0579465786 100644 --- a/index.yaml +++ b/index.yaml @@ -85,6 +85,20 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.28.0-alpha.2/openstack-cinder-csi-2.28.0-alpha.2.tgz version: 2.28.0-alpha.2 + - apiVersion: v1 + appVersion: v1.27.2 + created: "2023-08-30T01:08:14.669589273Z" + description: Cinder CSI Chart for OpenStack + digest: 6bb561de1693a29f572b3053eb80d30e4bb704eacdf7c474b21af9091a878cf5 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.27.2/openstack-cinder-csi-2.27.2.tgz + version: 2.27.2 - apiVersion: v1 appVersion: v1.27.1 created: "2023-04-24T06:27:31.974898619Z" @@ -514,6 +528,21 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.28.0-alpha.2/openstack-cloud-controller-manager-2.28.0-alpha.2.tgz version: 2.28.0-alpha.2 + - apiVersion: v1 + appVersion: v1.27.2 + created: "2023-08-30T01:08:14.800470925Z" + description: Openstack Cloud Controller Manager Helm Chart + digest: 106508e84b0a7f02632cbafb4c0fae76bb29419225265c111672360cce1ff96b + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.27.3/openstack-cloud-controller-manager-2.27.3.tgz + version: 2.27.3 - apiVersion: v1 appVersion: v1.27.1 created: "2023-04-24T06:27:32.111610998Z" @@ -849,6 +878,20 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.28.0-alpha.2/openstack-manila-csi-2.28.0-alpha.2.tgz version: 2.28.0-alpha.2 + - apiVersion: v1 + appVersion: v1.27.2 + created: "2023-08-30T01:08:14.900510065Z" + description: Manila CSI Chart for OpenStack + digest: dd8fe838c3745349029548d74865d502fe4dcd9c8be6f29b517747b750134ff9 + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.27.2/openstack-manila-csi-2.27.2.tgz + version: 2.27.2 - apiVersion: v1 appVersion: v1.27.1 created: "2023-04-24T06:27:32.233205401Z" From 88dbdecb56a88ee2b8a22e0077e7bf36b4ce6109 Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Wed, 30 Aug 2023 06:26:12 +0000 Subject: [PATCH 40/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/index.yaml b/index.yaml index 0579465786..b76729dbeb 100644 --- a/index.yaml +++ b/index.yaml @@ -408,6 +408,21 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.2.2/openstack-cinder-csi-1.2.2.tgz version: 1.2.2 openstack-cloud-controller-manager: + - apiVersion: v1 + appVersion: v1.28.0 + created: "2023-08-30T06:26:12.876099419Z" + description: Openstack Cloud Controller Manager Helm Chart + digest: ef926e9d7f3b84a8212cc832bd0dfea6e178d9aee93e76aaa7e94ec639808008 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.29.0-alpha.1/openstack-cloud-controller-manager-2.29.0-alpha.1.tgz + version: 2.29.0-alpha.1 - apiVersion: v1 appVersion: v1.28.0 created: "2023-08-25T10:29:39.373826276Z" From 1116af791871efbb26d5ec677de247716d6282f0 Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Thu, 31 Aug 2023 08:49:06 +0000 Subject: [PATCH 41/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/index.yaml b/index.yaml index b76729dbeb..5cff4eb625 100644 --- a/index.yaml +++ b/index.yaml @@ -1,6 +1,20 @@ apiVersion: v1 entries: openstack-cinder-csi: + - apiVersion: v1 + appVersion: v1.28.0 + created: "2023-08-31T08:49:06.639039057Z" + description: Cinder CSI Chart for OpenStack + digest: 9f88426a74dc5b312d3d03f3142285cf0595a11f712daa0d665607f8361ddf74 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.29.0-alpha.1/openstack-cinder-csi-2.29.0-alpha.1.tgz + version: 2.29.0-alpha.1 - apiVersion: v1 appVersion: v1.28.0 created: "2023-08-25T10:29:39.223752658Z" From 2b3ccdd67216d3e304dfe0fa151a23c48e1c4436 Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Thu, 31 Aug 2023 12:43:07 +0000 Subject: [PATCH 42/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/index.yaml b/index.yaml index 5cff4eb625..3999dd894e 100644 --- a/index.yaml +++ b/index.yaml @@ -422,6 +422,21 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.2.2/openstack-cinder-csi-1.2.2.tgz version: 1.2.2 openstack-cloud-controller-manager: + - apiVersion: v1 + appVersion: v1.28.0 + created: "2023-08-31T12:43:07.823532739Z" + description: Openstack Cloud Controller Manager Helm Chart + digest: b4412037106e56be218d3dc5224935474229a7f3fae6456ece587fb390897c58 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.29.0-alpha.2/openstack-cloud-controller-manager-2.29.0-alpha.2.tgz + version: 2.29.0-alpha.2 - apiVersion: v1 appVersion: v1.28.0 created: "2023-08-30T06:26:12.876099419Z" From 0f8a1612c983fc16623b23cbea27846cd64e7414 Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Thu, 21 Sep 2023 14:52:50 +0000 Subject: [PATCH 43/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/index.yaml b/index.yaml index 3999dd894e..79e80180a3 100644 --- a/index.yaml +++ b/index.yaml @@ -572,6 +572,21 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.28.0-alpha.2/openstack-cloud-controller-manager-2.28.0-alpha.2.tgz version: 2.28.0-alpha.2 + - apiVersion: v1 + appVersion: v1.27.2 + created: "2023-09-21T14:52:50.746084861Z" + description: Openstack Cloud Controller Manager Helm Chart + digest: e150543d066905684f8f0377e5d4c688612253fc637d10a70db3b8177952ed60 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.27.4/openstack-cloud-controller-manager-2.27.4.tgz + version: 2.27.4 - apiVersion: v1 appVersion: v1.27.2 created: "2023-08-30T01:08:14.800470925Z" From 319863e2282283b9caa46f8105b33e8f5b8c96b1 Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Tue, 10 Oct 2023 06:49:55 +0000 Subject: [PATCH 44/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/index.yaml b/index.yaml index 79e80180a3..1d9a0ab698 100644 --- a/index.yaml +++ b/index.yaml @@ -1,6 +1,20 @@ apiVersion: v1 entries: openstack-cinder-csi: + - apiVersion: v1 + appVersion: v1.28.0 + created: "2023-10-10T06:49:54.972779008Z" + description: Cinder CSI Chart for OpenStack + digest: 767d597651e467c3c3be940beadad4f94c42bd3c58589ba0040fe7a29fad025a + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.29.0-alpha.2/openstack-cinder-csi-2.29.0-alpha.2.tgz + version: 2.29.0-alpha.2 - apiVersion: v1 appVersion: v1.28.0 created: "2023-08-31T08:49:06.639039057Z" @@ -422,6 +436,21 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.2.2/openstack-cinder-csi-1.2.2.tgz version: 1.2.2 openstack-cloud-controller-manager: + - apiVersion: v1 + appVersion: v1.28.0 + created: "2023-10-10T06:49:55.134746091Z" + description: Openstack Cloud Controller Manager Helm Chart + digest: a478db3569d11bc7348c681c4a17307869e21fbd53e68f1616b76ea9bf8d8fb5 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.29.0-alpha.3/openstack-cloud-controller-manager-2.29.0-alpha.3.tgz + version: 2.29.0-alpha.3 - apiVersion: v1 appVersion: v1.28.0 created: "2023-08-31T12:43:07.823532739Z" @@ -867,6 +896,20 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-1.0.0/openstack-cloud-controller-manager-1.0.0.tgz version: 1.0.0 openstack-manila-csi: + - apiVersion: v1 + appVersion: v1.28.0 + created: "2023-10-10T06:49:55.265824333Z" + description: Manila CSI Chart for OpenStack + digest: 83b73508ef0d1a75ff33275d2bd79ce12bf3684032abd0c4c278a27048307f9f + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.29.0-alpha.1/openstack-manila-csi-2.29.0-alpha.1.tgz + version: 2.29.0-alpha.1 - apiVersion: v1 appVersion: v1.28.0 created: "2023-08-25T10:29:39.554160577Z" From 9ad9f58ca6d14aa0bc696786fbf6976853288a89 Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Thu, 19 Oct 2023 17:14:37 +0000 Subject: [PATCH 45/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/index.yaml b/index.yaml index 1d9a0ab698..a174f5a5a1 100644 --- a/index.yaml +++ b/index.yaml @@ -43,6 +43,20 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.29.0-alpha.0/openstack-cinder-csi-2.29.0-alpha.0.tgz version: 2.29.0-alpha.0 + - apiVersion: v1 + appVersion: v1.28.1 + created: "2023-10-19T17:14:37.305956016Z" + description: Cinder CSI Chart for OpenStack + digest: c2d9706d356036c757c5b4c48ebb165fb033ef7aff5a18a44710fc65045a44dd + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.28.1/openstack-cinder-csi-2.28.1.tgz + version: 2.28.1 - apiVersion: v1 appVersion: v1.28.0 created: "2023-08-25T08:43:38.62659305Z" @@ -496,6 +510,21 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.29.0-alpha.0/openstack-cloud-controller-manager-2.29.0-alpha.0.tgz version: 2.29.0-alpha.0 + - apiVersion: v1 + appVersion: v1.28.1 + created: "2023-10-19T17:14:37.464848952Z" + description: Openstack Cloud Controller Manager Helm Chart + digest: cde5ff5d8f48870f99375a9191b322256a0a09a4874319ad09d3f7549b4e710b + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.28.1/openstack-cloud-controller-manager-2.28.1.tgz + version: 2.28.1 - apiVersion: v1 appVersion: v1.28.0 created: "2023-08-25T08:43:38.881497024Z" @@ -924,6 +953,20 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.29.0-alpha.0/openstack-manila-csi-2.29.0-alpha.0.tgz version: 2.29.0-alpha.0 + - apiVersion: v1 + appVersion: v1.28.1 + created: "2023-10-19T17:14:37.611148352Z" + description: Manila CSI Chart for OpenStack + digest: fecca8ca2b319201b3ecb95e117b13d1bf48f68647ee6202c6398e5eec1bfe0d + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.28.1/openstack-manila-csi-2.28.1.tgz + version: 2.28.1 - apiVersion: v1 appVersion: v1.28.0 created: "2023-08-25T08:43:39.113350347Z" From 12c48448464b0bea832f0bd01b6dc61b24d88ce8 Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Thu, 19 Oct 2023 17:17:07 +0000 Subject: [PATCH 46/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/index.yaml b/index.yaml index a174f5a5a1..75056145d3 100644 --- a/index.yaml +++ b/index.yaml @@ -127,6 +127,20 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.28.0-alpha.2/openstack-cinder-csi-2.28.0-alpha.2.tgz version: 2.28.0-alpha.2 + - apiVersion: v1 + appVersion: v1.27.3 + created: "2023-10-19T17:17:07.495614674Z" + description: Cinder CSI Chart for OpenStack + digest: 921113494e9a69703e5b149669de5a25adc781f50e243a9ffc86620f88b5029d + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.27.3/openstack-cinder-csi-2.27.3.tgz + version: 2.27.3 - apiVersion: v1 appVersion: v1.27.2 created: "2023-08-30T01:08:14.669589273Z" @@ -630,6 +644,21 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.28.0-alpha.2/openstack-cloud-controller-manager-2.28.0-alpha.2.tgz version: 2.28.0-alpha.2 + - apiVersion: v1 + appVersion: v1.27.3 + created: "2023-10-19T17:17:07.653372024Z" + description: Openstack Cloud Controller Manager Helm Chart + digest: 152e9d4a1f4e91cd94cfa9c0e34538a8b82df2096150342cbc8788168889adc8 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.27.5/openstack-cloud-controller-manager-2.27.5.tgz + version: 2.27.5 - apiVersion: v1 appVersion: v1.27.2 created: "2023-09-21T14:52:50.746084861Z" @@ -1023,6 +1052,20 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.28.0-alpha.2/openstack-manila-csi-2.28.0-alpha.2.tgz version: 2.28.0-alpha.2 + - apiVersion: v1 + appVersion: v1.27.3 + created: "2023-10-19T17:17:07.8048381Z" + description: Manila CSI Chart for OpenStack + digest: db76c91d961ed2a98bbf37520ebc547d840be20c543dbd04e1cd7e7665440f8c + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.27.3/openstack-manila-csi-2.27.3.tgz + version: 2.27.3 - apiVersion: v1 appVersion: v1.27.2 created: "2023-08-30T01:08:14.900510065Z" From ca3f437ca4e2d62c1f0418bcef6f84c6b922ce72 Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Tue, 24 Oct 2023 12:45:02 +0000 Subject: [PATCH 47/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/index.yaml b/index.yaml index 75056145d3..e7d7ce4a14 100644 --- a/index.yaml +++ b/index.yaml @@ -464,6 +464,21 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.2.2/openstack-cinder-csi-1.2.2.tgz version: 1.2.2 openstack-cloud-controller-manager: + - apiVersion: v1 + appVersion: v1.28.0 + created: "2023-10-24T12:45:02.686419065Z" + description: Openstack Cloud Controller Manager Helm Chart + digest: f4863f4f2703d1f8e45880d0353b06409f09ceb4f55218f61afb907bc7f441a6 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.29.0-alpha.4/openstack-cloud-controller-manager-2.29.0-alpha.4.tgz + version: 2.29.0-alpha.4 - apiVersion: v1 appVersion: v1.28.0 created: "2023-10-10T06:49:55.134746091Z" From e24b01c4eb4c33bdf2eeae7b650faad9ebe44f6c Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Thu, 26 Oct 2023 09:26:39 +0000 Subject: [PATCH 48/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/index.yaml b/index.yaml index e7d7ce4a14..0192d0d2e2 100644 --- a/index.yaml +++ b/index.yaml @@ -764,6 +764,21 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.27.0-alpha.0/openstack-cloud-controller-manager-2.27.0-alpha.0.tgz version: 2.27.0-alpha.0 + - apiVersion: v1 + appVersion: v1.26.4 + created: "2023-10-26T09:26:39.520496951Z" + description: Openstack Cloud Controller Manager Helm Chart + digest: 305877d662a62e29bccc0374812c7e19c432d015e96a8c794396aba7f4b733bd + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.26.5/openstack-cloud-controller-manager-2.26.5.tgz + version: 2.26.5 - apiVersion: v1 appVersion: v1.26.2 created: "2023-03-21T06:09:32.21423869Z" From c2cf37e141d208972aa039f0d77105f59f3dab4c Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Thu, 26 Oct 2023 09:32:30 +0000 Subject: [PATCH 49/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/index.yaml b/index.yaml index 0192d0d2e2..811cff2133 100644 --- a/index.yaml +++ b/index.yaml @@ -659,6 +659,21 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.28.0-alpha.2/openstack-cloud-controller-manager-2.28.0-alpha.2.tgz version: 2.28.0-alpha.2 + - apiVersion: v1 + appVersion: v1.27.3 + created: "2023-10-26T09:32:30.839469588Z" + description: Openstack Cloud Controller Manager Helm Chart + digest: 4baf9f89a10f249b6eadcf8cadf435b624cdb49d83ced14fb9fc874789840d98 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.27.6/openstack-cloud-controller-manager-2.27.6.tgz + version: 2.27.6 - apiVersion: v1 appVersion: v1.27.3 created: "2023-10-19T17:17:07.653372024Z" From 74d2ddc7e16e89721205318b160320d12c63f579 Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Thu, 26 Oct 2023 09:35:26 +0000 Subject: [PATCH 50/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/index.yaml b/index.yaml index 811cff2133..5baa8b808c 100644 --- a/index.yaml +++ b/index.yaml @@ -539,6 +539,21 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.29.0-alpha.0/openstack-cloud-controller-manager-2.29.0-alpha.0.tgz version: 2.29.0-alpha.0 + - apiVersion: v1 + appVersion: v1.28.1 + created: "2023-10-26T09:35:26.155662326Z" + description: Openstack Cloud Controller Manager Helm Chart + digest: 88a3fe29e5e940d589c486aa4b631b61ea820f4917fa2a9ef7912b02fceef720 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.28.2/openstack-cloud-controller-manager-2.28.2.tgz + version: 2.28.2 - apiVersion: v1 appVersion: v1.28.1 created: "2023-10-19T17:14:37.464848952Z" From 287f2eb39df15121c997e0e8ba49ef082c3a30bb Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Thu, 2 Nov 2023 01:58:29 +0000 Subject: [PATCH 51/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/index.yaml b/index.yaml index 5baa8b808c..d8cc4593ce 100644 --- a/index.yaml +++ b/index.yaml @@ -464,6 +464,21 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.2.2/openstack-cinder-csi-1.2.2.tgz version: 1.2.2 openstack-cloud-controller-manager: + - apiVersion: v1 + appVersion: v1.28.0 + created: "2023-11-02T01:58:29.915529333Z" + description: Openstack Cloud Controller Manager Helm Chart + digest: ec5c1e88c37f67ee57a76d203fb53c10f5eec7eb1bd7124db30983540947f061 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.29.0-alpha.5/openstack-cloud-controller-manager-2.29.0-alpha.5.tgz + version: 2.29.0-alpha.5 - apiVersion: v1 appVersion: v1.28.0 created: "2023-10-24T12:45:02.686419065Z" From 00a4f5dc77a9aa8bba8e72dd33308f847ce107f4 Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Tue, 14 Nov 2023 16:08:41 +0000 Subject: [PATCH 52/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/index.yaml b/index.yaml index d8cc4593ce..598a6dbee2 100644 --- a/index.yaml +++ b/index.yaml @@ -554,6 +554,21 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.29.0-alpha.0/openstack-cloud-controller-manager-2.29.0-alpha.0.tgz version: 2.29.0-alpha.0 + - apiVersion: v1 + appVersion: v1.28.1 + created: "2023-11-14T16:08:41.696630889Z" + description: Openstack Cloud Controller Manager Helm Chart + digest: 3f19563f86c8649da98d7bf46d8d08f7214ac73d6139929eb1b7173813b71d96 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.28.3/openstack-cloud-controller-manager-2.28.3.tgz + version: 2.28.3 - apiVersion: v1 appVersion: v1.28.1 created: "2023-10-26T09:35:26.155662326Z" From 9b5a837d8672e26400b102b3e7a71400a36fbbd1 Mon Sep 17 00:00:00 2001 From: zetaab Date: Thu, 28 Dec 2023 14:33:46 +0000 Subject: [PATCH 53/90] Update index.yaml Signed-off-by: zetaab --- index.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/index.yaml b/index.yaml index 598a6dbee2..36204b1ce7 100644 --- a/index.yaml +++ b/index.yaml @@ -1,6 +1,20 @@ apiVersion: v1 entries: openstack-cinder-csi: + - apiVersion: v1 + appVersion: v1.28.0 + created: "2023-12-28T14:33:46.862269317Z" + description: Cinder CSI Chart for OpenStack + digest: 8a55ec0e073cf0d253d3b8e41ae01d53c4666a157069e7d2ce42bdfd905e28ba + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.29.0-alpha.4/openstack-cinder-csi-2.29.0-alpha.4.tgz + version: 2.29.0-alpha.4 - apiVersion: v1 appVersion: v1.28.0 created: "2023-10-10T06:49:54.972779008Z" From 51c6d30dcc1cab398dce75ba91e19547f24c8a7d Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Thu, 18 Jan 2024 13:20:09 +0000 Subject: [PATCH 54/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/index.yaml b/index.yaml index 36204b1ce7..82e58d4109 100644 --- a/index.yaml +++ b/index.yaml @@ -1,6 +1,20 @@ apiVersion: v1 entries: openstack-cinder-csi: + - apiVersion: v1 + appVersion: v1.29.0 + created: "2024-01-18T13:20:08.72090912Z" + description: Cinder CSI Chart for OpenStack + digest: a8a1b0fc2f00bb45e2f43d968576c1d160fab43ea9d15578ae7505cc05ffd78d + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.29.0/openstack-cinder-csi-2.29.0.tgz + version: 2.29.0 - apiVersion: v1 appVersion: v1.28.0 created: "2023-12-28T14:33:46.862269317Z" @@ -478,6 +492,21 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.2.2/openstack-cinder-csi-1.2.2.tgz version: 1.2.2 openstack-cloud-controller-manager: + - apiVersion: v1 + appVersion: v1.29.0 + created: "2024-01-18T13:20:08.84727297Z" + description: Openstack Cloud Controller Manager Helm Chart + digest: dc7f6feccb4db317cbb2bc1f8a54e56ba6bf1c1af818df370b2baff70e25bd6a + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.29.0/openstack-cloud-controller-manager-2.29.0.tgz + version: 2.29.0 - apiVersion: v1 appVersion: v1.28.0 created: "2023-11-02T01:58:29.915529333Z" @@ -1058,6 +1087,20 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-1.0.0/openstack-cloud-controller-manager-1.0.0.tgz version: 1.0.0 openstack-manila-csi: + - apiVersion: v1 + appVersion: v1.29.0 + created: "2024-01-18T13:20:09.061477863Z" + description: Manila CSI Chart for OpenStack + digest: fb3fad7bf10c3c5b75b6d7a9d02894feacaf7f0560b46c7bb7e12b7ab8bb2324 + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.29.0/openstack-manila-csi-2.29.0.tgz + version: 2.29.0 - apiVersion: v1 appVersion: v1.28.0 created: "2023-10-10T06:49:55.265824333Z" From f68ea9691a726048e45b11b7f93e13e5ae2846dc Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Tue, 20 Feb 2024 20:19:57 +0000 Subject: [PATCH 55/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/index.yaml b/index.yaml index 82e58d4109..cc50ac97ed 100644 --- a/index.yaml +++ b/index.yaml @@ -492,6 +492,25 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.2.2/openstack-cinder-csi-1.2.2.tgz version: 1.2.2 openstack-cloud-controller-manager: + - apiVersion: v2 + appVersion: v1.29.0 + created: "2024-02-20T20:19:57.890192207Z" + dependencies: + - name: common + repository: https://charts.bitnami.com/bitnami + version: 2.14.1 + description: Openstack Cloud Controller Manager Helm Chart + digest: 3b9f233f8a377f0e0e22222363bcb0f262e236a41a7b6780c1b51e3a5a120eb1 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.29.1/openstack-cloud-controller-manager-2.29.1.tgz + version: 2.29.1 - apiVersion: v1 appVersion: v1.29.0 created: "2024-01-18T13:20:08.84727297Z" From 2ef118b4086d811927421cc23baf927fc295dc5d Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Mon, 4 Mar 2024 08:00:37 +0000 Subject: [PATCH 56/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/index.yaml b/index.yaml index cc50ac97ed..4ca602a5c5 100644 --- a/index.yaml +++ b/index.yaml @@ -71,6 +71,20 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.29.0-alpha.0/openstack-cinder-csi-2.29.0-alpha.0.tgz version: 2.29.0-alpha.0 + - apiVersion: v1 + appVersion: v1.28.2 + created: "2024-03-04T08:00:37.611246295Z" + description: Cinder CSI Chart for OpenStack + digest: d339c0540f3e6a80a253a8dc43f060a71d05d3b9a98844918da5d9c1a33561f1 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.28.2/openstack-cinder-csi-2.28.2.tgz + version: 2.28.2 - apiVersion: v1 appVersion: v1.28.1 created: "2023-10-19T17:14:37.305956016Z" @@ -616,6 +630,21 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.29.0-alpha.0/openstack-cloud-controller-manager-2.29.0-alpha.0.tgz version: 2.29.0-alpha.0 + - apiVersion: v1 + appVersion: v1.28.2 + created: "2024-03-04T08:00:37.704513425Z" + description: Openstack Cloud Controller Manager Helm Chart + digest: b0214e02a0839754481e5792d52be8515bbe2a1aeed39b627257ab6ae4cf8093 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.28.4/openstack-cloud-controller-manager-2.28.4.tgz + version: 2.28.4 - apiVersion: v1 appVersion: v1.28.1 created: "2023-11-14T16:08:41.696630889Z" @@ -1148,6 +1177,20 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.29.0-alpha.0/openstack-manila-csi-2.29.0-alpha.0.tgz version: 2.29.0-alpha.0 + - apiVersion: v1 + appVersion: v1.28.2 + created: "2024-03-04T08:00:37.81516022Z" + description: Manila CSI Chart for OpenStack + digest: ab40fe5ccb79e71524061f0952f4ecc06ba06fd580751dd2c8c0fc96f179c1bb + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.28.2/openstack-manila-csi-2.28.2.tgz + version: 2.28.2 - apiVersion: v1 appVersion: v1.28.1 created: "2023-10-19T17:14:37.611148352Z" From 67eaf1986a8f6ff3fb4a21695c5d4b065c882c31 Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Fri, 12 Apr 2024 09:25:12 +0000 Subject: [PATCH 57/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/index.yaml b/index.yaml index 4ca602a5c5..2d93a31421 100644 --- a/index.yaml +++ b/index.yaml @@ -506,6 +506,25 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.2.2/openstack-cinder-csi-1.2.2.tgz version: 1.2.2 openstack-cloud-controller-manager: + - apiVersion: v2 + appVersion: v1.29.0 + created: "2024-04-12T09:25:12.417854898Z" + dependencies: + - name: common + repository: https://charts.bitnami.com/bitnami + version: 2.14.1 + description: Openstack Cloud Controller Manager Helm Chart + digest: 427ae8d2cf407bef6bf3e4b9478e31218b5e90ac5f3d8e935d7a053398606b84 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.29.2/openstack-cloud-controller-manager-2.29.2.tgz + version: 2.29.2 - apiVersion: v2 appVersion: v1.29.0 created: "2024-02-20T20:19:57.890192207Z" From d6e2dc3712bb529f3c0293bc9adeacea07cfb76a Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Tue, 7 May 2024 12:36:28 +0000 Subject: [PATCH 58/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/index.yaml b/index.yaml index 2d93a31421..6838617756 100644 --- a/index.yaml +++ b/index.yaml @@ -1,6 +1,22 @@ apiVersion: v1 entries: openstack-cinder-csi: + - apiVersion: v1 + appVersion: v1.30.0 + created: "2024-05-07T12:36:28.468450186Z" + description: Cinder CSI Chart for OpenStack + digest: 9466c968704ce957cce4ce951b59bfb156be18ec06a30cc0faf9a04a64927fc6 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: mnasiadka@gmail.com + name: mnasiadka + - email: matt@stackhpc.com + name: mkjpryor + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.30.0/openstack-cinder-csi-2.30.0.tgz + version: 2.30.0 - apiVersion: v1 appVersion: v1.29.0 created: "2024-01-18T13:20:08.72090912Z" @@ -506,6 +522,25 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.2.2/openstack-cinder-csi-1.2.2.tgz version: 1.2.2 openstack-cloud-controller-manager: + - apiVersion: v2 + appVersion: v1.30.0 + created: "2024-05-07T12:36:28.609530575Z" + dependencies: + - name: common + repository: https://charts.bitnami.com/bitnami + version: 2.14.1 + description: Openstack Cloud Controller Manager Helm Chart + digest: f9676bfa1a2aa75e9fa2ceaa21752d7fcad37878afb01fbeb006d2f31997755f + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.30.0/openstack-cloud-controller-manager-2.30.0.tgz + version: 2.30.0 - apiVersion: v2 appVersion: v1.29.0 created: "2024-04-12T09:25:12.417854898Z" @@ -1154,6 +1189,22 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-1.0.0/openstack-cloud-controller-manager-1.0.0.tgz version: 1.0.0 openstack-manila-csi: + - apiVersion: v1 + appVersion: v1.30.0 + created: "2024-05-07T12:36:28.729351547Z" + description: Manila CSI Chart for OpenStack + digest: e100aabe256422443e1fe55f18c519140a07320f2d6d89767a41410ee99832ae + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: mnasiadka@gmail.com + name: mnasiadka + - email: matt@stackhpc.com + name: mkjpryor + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.30.0/openstack-manila-csi-2.30.0.tgz + version: 2.30.0 - apiVersion: v1 appVersion: v1.29.0 created: "2024-01-18T13:20:09.061477863Z" From 3b75fd076a7ac86490e32d58aa033eb4b0205261 Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Wed, 15 May 2024 01:17:54 +0000 Subject: [PATCH 59/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/index.yaml b/index.yaml index 6838617756..65d178581d 100644 --- a/index.yaml +++ b/index.yaml @@ -522,6 +522,25 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.2.2/openstack-cinder-csi-1.2.2.tgz version: 1.2.2 openstack-cloud-controller-manager: + - apiVersion: v2 + appVersion: v1.30.0 + created: "2024-05-15T01:17:54.841312376Z" + dependencies: + - name: common + repository: https://charts.bitnami.com/bitnami + version: 2.14.1 + description: Openstack Cloud Controller Manager Helm Chart + digest: b40de98eb170e54cbdb60320d255ec15936747561ec13b37bf2f3c3b4c2469a3 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.30.1/openstack-cloud-controller-manager-2.30.1.tgz + version: 2.30.1 - apiVersion: v2 appVersion: v1.30.0 created: "2024-05-07T12:36:28.609530575Z" From a74ab70892f5074e1c3a2f4343c07ceb366e1e7c Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Fri, 7 Jun 2024 08:50:15 +0000 Subject: [PATCH 60/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/index.yaml b/index.yaml index 65d178581d..a339ccaeea 100644 --- a/index.yaml +++ b/index.yaml @@ -1,6 +1,22 @@ apiVersion: v1 entries: openstack-cinder-csi: + - apiVersion: v1 + appVersion: v1.30.0 + created: "2024-06-07T08:50:14.903013455Z" + description: Cinder CSI Chart for OpenStack + digest: dab821c6f44ae1d2430d44bdbe384bb75387abf15e370c93c4276463ae8861a4 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: mnasiadka@gmail.com + name: mnasiadka + - email: matt@stackhpc.com + name: mkjpryor + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.30.1-alpha.1/openstack-cinder-csi-2.30.1-alpha.1.tgz + version: 2.30.1-alpha.1 - apiVersion: v1 appVersion: v1.30.0 created: "2024-05-07T12:36:28.468450186Z" @@ -1208,6 +1224,22 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-1.0.0/openstack-cloud-controller-manager-1.0.0.tgz version: 1.0.0 openstack-manila-csi: + - apiVersion: v1 + appVersion: v1.30.0 + created: "2024-06-07T08:50:15.100454082Z" + description: Manila CSI Chart for OpenStack + digest: 194556b6fcab3be5bef8203c4a7b039c4c829480bf4df1d6c0a323dac203db34 + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: mnasiadka@gmail.com + name: mnasiadka + - email: matt@stackhpc.com + name: mkjpryor + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.30.1-alpha.1/openstack-manila-csi-2.30.1-alpha.1.tgz + version: 2.30.1-alpha.1 - apiVersion: v1 appVersion: v1.30.0 created: "2024-05-07T12:36:28.729351547Z" From aa4a85cc0b3ab5298788063f4aa46e9a570723e8 Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Fri, 12 Jul 2024 09:05:46 +0000 Subject: [PATCH 61/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/index.yaml b/index.yaml index a339ccaeea..63d7640f3f 100644 --- a/index.yaml +++ b/index.yaml @@ -538,6 +538,25 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.2.2/openstack-cinder-csi-1.2.2.tgz version: 1.2.2 openstack-cloud-controller-manager: + - apiVersion: v2 + appVersion: v1.30.0 + created: "2024-07-12T09:05:46.164313145Z" + dependencies: + - name: common + repository: https://charts.bitnami.com/bitnami + version: 2.14.1 + description: Openstack Cloud Controller Manager Helm Chart + digest: 7e319d5c05486622c96447b35e6515d4e7f3448d87ab6bf21877ed1bcd1408c6 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.30.2/openstack-cloud-controller-manager-2.30.2.tgz + version: 2.30.2 - apiVersion: v2 appVersion: v1.30.0 created: "2024-05-15T01:17:54.841312376Z" From 2446943c7d7b619a84231f956101e854697a44bc Mon Sep 17 00:00:00 2001 From: zetaab Date: Fri, 6 Sep 2024 13:42:43 +0000 Subject: [PATCH 62/90] Update index.yaml Signed-off-by: zetaab --- index.yaml | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/index.yaml b/index.yaml index 63d7640f3f..14db75a313 100644 --- a/index.yaml +++ b/index.yaml @@ -1,6 +1,22 @@ apiVersion: v1 entries: openstack-cinder-csi: + - apiVersion: v1 + appVersion: v1.31.0 + created: "2024-09-06T13:42:43.115163803Z" + description: Cinder CSI Chart for OpenStack + digest: b292ea3156863e08ece2dd47ce00a213719ed8c5a594c9237142947dddc07949 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: mnasiadka@gmail.com + name: mnasiadka + - email: matt@stackhpc.com + name: mkjpryor + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.31.0/openstack-cinder-csi-2.31.0.tgz + version: 2.31.0 - apiVersion: v1 appVersion: v1.30.0 created: "2024-06-07T08:50:14.903013455Z" @@ -538,6 +554,25 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.2.2/openstack-cinder-csi-1.2.2.tgz version: 1.2.2 openstack-cloud-controller-manager: + - apiVersion: v2 + appVersion: v1.31.0 + created: "2024-09-06T13:42:43.276068564Z" + dependencies: + - name: common + repository: https://charts.bitnami.com/bitnami + version: 2.14.1 + description: Openstack Cloud Controller Manager Helm Chart + digest: 51b4477a56557edb6f557029a525f5bd3a90906377f6c2c4cc1973ecee7bc816 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.31.0/openstack-cloud-controller-manager-2.31.0.tgz + version: 2.31.0 - apiVersion: v2 appVersion: v1.30.0 created: "2024-07-12T09:05:46.164313145Z" @@ -1243,6 +1278,22 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-1.0.0/openstack-cloud-controller-manager-1.0.0.tgz version: 1.0.0 openstack-manila-csi: + - apiVersion: v1 + appVersion: v1.31.0 + created: "2024-09-06T13:42:43.416152638Z" + description: Manila CSI Chart for OpenStack + digest: b5faa781afb62ce9ec319112052ae4709e90163d9f8276737f66b0799e982f8e + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: mnasiadka@gmail.com + name: mnasiadka + - email: matt@stackhpc.com + name: mkjpryor + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.31.0/openstack-manila-csi-2.31.0.tgz + version: 2.31.0 - apiVersion: v1 appVersion: v1.30.0 created: "2024-06-07T08:50:15.100454082Z" @@ -1667,4 +1718,4 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-0.2.1/openstack-manila-csi-0.2.1.tgz version: 0.2.1 -generated: "2020-09-28T10:25:34.941244248Z" +generated: "2024-09-06T13:42:43.417097811Z" From beccec0caa2030987f8888010504539473247fb7 Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Thu, 19 Sep 2024 14:11:37 +0000 Subject: [PATCH 63/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/index.yaml b/index.yaml index 14db75a313..24de9d166a 100644 --- a/index.yaml +++ b/index.yaml @@ -1,6 +1,22 @@ apiVersion: v1 entries: openstack-cinder-csi: + - apiVersion: v1 + appVersion: v1.31.1 + created: "2024-09-19T14:11:37.185768477Z" + description: Cinder CSI Chart for OpenStack + digest: ad9b114155cc2811d59447d15364b1d64a5157c1c09d037dcc08b89265c9657c + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: mnasiadka@gmail.com + name: mnasiadka + - email: matt@stackhpc.com + name: mkjpryor + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.31.1/openstack-cinder-csi-2.31.1.tgz + version: 2.31.1 - apiVersion: v1 appVersion: v1.31.0 created: "2024-09-06T13:42:43.115163803Z" @@ -554,6 +570,25 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.2.2/openstack-cinder-csi-1.2.2.tgz version: 1.2.2 openstack-cloud-controller-manager: + - apiVersion: v2 + appVersion: v1.31.1 + created: "2024-09-19T14:11:37.373161835Z" + dependencies: + - name: common + repository: https://charts.bitnami.com/bitnami + version: 2.14.1 + description: Openstack Cloud Controller Manager Helm Chart + digest: e47cce29f526097ece38232f2d3f44b572529dae60a9456d9063a05c12ee336f + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.31.1/openstack-cloud-controller-manager-2.31.1.tgz + version: 2.31.1 - apiVersion: v2 appVersion: v1.31.0 created: "2024-09-06T13:42:43.276068564Z" @@ -1278,6 +1313,22 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-1.0.0/openstack-cloud-controller-manager-1.0.0.tgz version: 1.0.0 openstack-manila-csi: + - apiVersion: v1 + appVersion: v1.31.1 + created: "2024-09-19T14:11:37.505548082Z" + description: Manila CSI Chart for OpenStack + digest: b2cbdc523788fe51097ef308fe4673a3323b6162c6a515cc711dd69391051e82 + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: mnasiadka@gmail.com + name: mnasiadka + - email: matt@stackhpc.com + name: mkjpryor + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.31.1/openstack-manila-csi-2.31.1.tgz + version: 2.31.1 - apiVersion: v1 appVersion: v1.31.0 created: "2024-09-06T13:42:43.416152638Z" From 2f73d962ecba0f408a2685c5e9561c4213899307 Mon Sep 17 00:00:00 2001 From: kayrus Date: Thu, 19 Sep 2024 15:04:23 +0000 Subject: [PATCH 64/90] Update index.yaml Signed-off-by: kayrus --- index.yaml | 45 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/index.yaml b/index.yaml index 24de9d166a..b8ab9a3b5b 100644 --- a/index.yaml +++ b/index.yaml @@ -135,6 +135,20 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.29.0-alpha.0/openstack-cinder-csi-2.29.0-alpha.0.tgz version: 2.29.0-alpha.0 + - apiVersion: v1 + appVersion: v1.28.3 + created: "2024-09-19T15:04:22.651579357Z" + description: Cinder CSI Chart for OpenStack + digest: 4530cdc47a854e75b61065572ef685c380e28fa2def89fed4ac4ed2f0c087ce5 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.28.3/openstack-cinder-csi-2.28.3.tgz + version: 2.28.3 - apiVersion: v1 appVersion: v1.28.2 created: "2024-03-04T08:00:37.611246295Z" @@ -808,6 +822,21 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.29.0-alpha.0/openstack-cloud-controller-manager-2.29.0-alpha.0.tgz version: 2.29.0-alpha.0 + - apiVersion: v1 + appVersion: v1.28.3 + created: "2024-09-19T15:04:22.903803218Z" + description: Openstack Cloud Controller Manager Helm Chart + digest: 056067c48882c39b0e10098e1729a2f71a3ff31136f919e4580f28559e26bedc + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.28.5/openstack-cloud-controller-manager-2.28.5.tgz + version: 2.28.5 - apiVersion: v1 appVersion: v1.28.2 created: "2024-03-04T08:00:37.704513425Z" @@ -1419,6 +1448,20 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.29.0-alpha.0/openstack-manila-csi-2.29.0-alpha.0.tgz version: 2.29.0-alpha.0 + - apiVersion: v1 + appVersion: v1.28.3 + created: "2024-09-19T15:04:23.048125148Z" + description: Manila CSI Chart for OpenStack + digest: 911d607e0be54c12899b6f08ee45e8c4ece04ad4befd972aed8046ca379444a1 + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.28.3/openstack-manila-csi-2.28.3.tgz + version: 2.28.3 - apiVersion: v1 appVersion: v1.28.2 created: "2024-03-04T08:00:37.81516022Z" @@ -1769,4 +1812,4 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-0.2.1/openstack-manila-csi-0.2.1.tgz version: 0.2.1 -generated: "2024-09-06T13:42:43.417097811Z" +generated: "2024-09-19T15:04:23.049061686Z" From a9bcf33ed6a9c8c20b076b48b059c9393169b976 Mon Sep 17 00:00:00 2001 From: kayrus Date: Thu, 19 Sep 2024 15:38:09 +0000 Subject: [PATCH 65/90] Update index.yaml Signed-off-by: kayrus --- index.yaml | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/index.yaml b/index.yaml index b8ab9a3b5b..5222331db2 100644 --- a/index.yaml +++ b/index.yaml @@ -65,6 +65,20 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.30.0/openstack-cinder-csi-2.30.0.tgz version: 2.30.0 + - apiVersion: v1 + appVersion: v1.29.1 + created: "2024-09-19T15:38:09.582944539Z" + description: Cinder CSI Chart for OpenStack + digest: dbe396845e66de1e792ebc12741955387a7aabd27ef3acfd227487edaf9ea0bf + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: brtknr@bath.edu + name: brtknr + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.29.2/openstack-cinder-csi-2.29.2.tgz + version: 2.29.2 - apiVersion: v1 appVersion: v1.29.0 created: "2024-01-18T13:20:08.72090912Z" @@ -679,6 +693,21 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.30.0/openstack-cloud-controller-manager-2.30.0.tgz version: 2.30.0 + - apiVersion: v1 + appVersion: v1.29.1 + created: "2024-09-19T15:38:09.737854602Z" + description: Openstack Cloud Controller Manager Helm Chart + digest: 7bc5c1262fa75eae4f1adbeb9a050cc36feb9fed40197610101b7c219b92f96c + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.29.3/openstack-cloud-controller-manager-2.29.3.tgz + version: 2.29.3 - apiVersion: v2 appVersion: v1.29.0 created: "2024-04-12T09:25:12.417854898Z" @@ -1812,4 +1841,4 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-0.2.1/openstack-manila-csi-0.2.1.tgz version: 0.2.1 -generated: "2024-09-19T15:04:23.049061686Z" +generated: "2024-09-19T15:38:09.738649315Z" From 26d11015f5f0dc4c735e494388a48b6ca907f004 Mon Sep 17 00:00:00 2001 From: kayrus Date: Thu, 19 Sep 2024 15:38:33 +0000 Subject: [PATCH 66/90] Update index.yaml Signed-off-by: kayrus --- index.yaml | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/index.yaml b/index.yaml index 5222331db2..bc8b9f4817 100644 --- a/index.yaml +++ b/index.yaml @@ -636,6 +636,25 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.31.0/openstack-cloud-controller-manager-2.31.0.tgz version: 2.31.0 + - apiVersion: v2 + appVersion: v1.30.1 + created: "2024-09-19T15:38:32.903794082Z" + dependencies: + - name: common + repository: https://charts.bitnami.com/bitnami + version: 2.14.1 + description: Openstack Cloud Controller Manager Helm Chart + digest: 7b04b5b5459a52aeb9500e7947d06bb5e85722f14a28d5c3dd6a9b09ddb8aa8d + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.30.3/openstack-cloud-controller-manager-2.30.3.tgz + version: 2.30.3 - apiVersion: v2 appVersion: v1.30.0 created: "2024-07-12T09:05:46.164313145Z" @@ -1841,4 +1860,4 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-0.2.1/openstack-manila-csi-0.2.1.tgz version: 0.2.1 -generated: "2024-09-19T15:38:09.738649315Z" +generated: "2024-09-19T15:38:32.904619271Z" From f3859d18adf5aba620da7dde43e9181d1609fd17 Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Mon, 14 Oct 2024 18:46:35 +0000 Subject: [PATCH 67/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/index.yaml b/index.yaml index bc8b9f4817..2db3387bfc 100644 --- a/index.yaml +++ b/index.yaml @@ -1,6 +1,22 @@ apiVersion: v1 entries: openstack-cinder-csi: + - apiVersion: v1 + appVersion: v1.31.0 + created: "2024-10-14T18:46:35.132522737Z" + description: Cinder CSI Chart for OpenStack + digest: 7c89289000a9f3c3cbf4c2c827d8bb7d120277827530794a18f1a03dfeeb5cc7 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: mnasiadka@gmail.com + name: mnasiadka + - email: matt@stackhpc.com + name: mkjpryor + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.31.2/openstack-cinder-csi-2.31.2.tgz + version: 2.31.2 - apiVersion: v1 appVersion: v1.31.1 created: "2024-09-19T14:11:37.185768477Z" @@ -1860,4 +1876,4 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-0.2.1/openstack-manila-csi-0.2.1.tgz version: 0.2.1 -generated: "2024-09-19T15:38:32.904619271Z" +generated: "2024-10-14T18:46:35.133291393Z" From 5129563a1701a9542c328eaa0cd2ff02a722e24e Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Sat, 23 Nov 2024 14:07:17 +0000 Subject: [PATCH 68/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/index.yaml b/index.yaml index 2db3387bfc..e4f0a33183 100644 --- a/index.yaml +++ b/index.yaml @@ -49,6 +49,22 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.31.0/openstack-cinder-csi-2.31.0.tgz version: 2.31.0 + - apiVersion: v1 + appVersion: v1.30.2 + created: "2024-11-23T14:07:17.332925614Z" + description: Cinder CSI Chart for OpenStack + digest: 3b6efadfbaa112a1738aeb3670e59e47743267a379b2db88c9ce4b3ad4ac125b + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: mnasiadka@gmail.com + name: mnasiadka + - email: matt@stackhpc.com + name: mkjpryor + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.30.2/openstack-cinder-csi-2.30.2.tgz + version: 2.30.2 - apiVersion: v1 appVersion: v1.30.0 created: "2024-06-07T08:50:14.903013455Z" @@ -652,6 +668,25 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.31.0/openstack-cloud-controller-manager-2.31.0.tgz version: 2.31.0 + - apiVersion: v2 + appVersion: v1.30.2 + created: "2024-11-23T14:07:17.444776298Z" + dependencies: + - name: common + repository: https://charts.bitnami.com/bitnami + version: 2.14.1 + description: Openstack Cloud Controller Manager Helm Chart + digest: 1a3caf217127b8160184c9f94feb773e398150de9aecfcd8338f521d5b621cbf + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.30.4/openstack-cloud-controller-manager-2.30.4.tgz + version: 2.30.4 - apiVersion: v2 appVersion: v1.30.1 created: "2024-09-19T15:38:32.903794082Z" @@ -1438,6 +1473,22 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.31.0/openstack-manila-csi-2.31.0.tgz version: 2.31.0 + - apiVersion: v1 + appVersion: v1.30.2 + created: "2024-11-23T14:07:17.541506055Z" + description: Manila CSI Chart for OpenStack + digest: 5d6a97580ff3d44e2f9ccd376cdb7e2f2ee68fac7295534a6dd9633c36d68539 + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: mnasiadka@gmail.com + name: mnasiadka + - email: matt@stackhpc.com + name: mkjpryor + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.30.2/openstack-manila-csi-2.30.2.tgz + version: 2.30.2 - apiVersion: v1 appVersion: v1.30.0 created: "2024-06-07T08:50:15.100454082Z" @@ -1876,4 +1927,4 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-0.2.1/openstack-manila-csi-0.2.1.tgz version: 0.2.1 -generated: "2024-10-14T18:46:35.133291393Z" +generated: "2024-11-23T14:07:17.542228592Z" From 79d8822aa416a0a1cae7ca5bfff2b92c165cb6c9 Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Tue, 17 Dec 2024 15:31:09 +0000 Subject: [PATCH 69/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/index.yaml b/index.yaml index e4f0a33183..61fc14d5e8 100644 --- a/index.yaml +++ b/index.yaml @@ -1441,6 +1441,22 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-1.0.0/openstack-cloud-controller-manager-1.0.0.tgz version: 1.0.0 openstack-manila-csi: + - apiVersion: v1 + appVersion: v1.31.2 + created: "2024-12-17T15:31:08.697073461Z" + description: Manila CSI Chart for OpenStack + digest: bf60bcd06212c5e270cabc955cc1726f2a15e339db03338a9fcd608bd912b1eb + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: mnasiadka@gmail.com + name: mnasiadka + - email: matt@stackhpc.com + name: mkjpryor + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.31.4/openstack-manila-csi-2.31.4.tgz + version: 2.31.4 - apiVersion: v1 appVersion: v1.31.1 created: "2024-09-19T14:11:37.505548082Z" @@ -1927,4 +1943,4 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-0.2.1/openstack-manila-csi-0.2.1.tgz version: 0.2.1 -generated: "2024-11-23T14:07:17.542228592Z" +generated: "2024-12-17T15:31:08.697958348Z" From becf56eeec415a57aed9f458f5b7be92296168f9 Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Thu, 23 Jan 2025 08:13:24 +0000 Subject: [PATCH 70/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/index.yaml b/index.yaml index 61fc14d5e8..b044ee59fa 100644 --- a/index.yaml +++ b/index.yaml @@ -1,6 +1,22 @@ apiVersion: v1 entries: openstack-cinder-csi: + - apiVersion: v1 + appVersion: v1.31.2 + created: "2025-01-23T08:13:23.695726105Z" + description: Cinder CSI Chart for OpenStack + digest: 6fab438d59b77ca317e84c59f136719465b22d45a08b9147e4a8d6542ab4efef + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: mnasiadka@gmail.com + name: mnasiadka + - email: matt@stackhpc.com + name: mkjpryor + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.32.0-alpha.0/openstack-cinder-csi-2.32.0-alpha.0.tgz + version: 2.32.0-alpha.0 - apiVersion: v1 appVersion: v1.31.0 created: "2024-10-14T18:46:35.132522737Z" @@ -630,6 +646,25 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.2.2/openstack-cinder-csi-1.2.2.tgz version: 1.2.2 openstack-cloud-controller-manager: + - apiVersion: v2 + appVersion: v1.31.2 + created: "2025-01-23T08:13:23.880096274Z" + dependencies: + - name: common + repository: https://charts.bitnami.com/bitnami + version: 2.14.1 + description: Openstack Cloud Controller Manager Helm Chart + digest: 69837b8c42129c63631f8b4286e3389dbf570bc4435703cc0092c055245f694b + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.32.0-alpha.0/openstack-cloud-controller-manager-2.32.0-alpha.0.tgz + version: 2.32.0-alpha.0 - apiVersion: v2 appVersion: v1.31.1 created: "2024-09-19T14:11:37.373161835Z" @@ -1441,6 +1476,22 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-1.0.0/openstack-cloud-controller-manager-1.0.0.tgz version: 1.0.0 openstack-manila-csi: + - apiVersion: v1 + appVersion: v1.31.2 + created: "2025-01-23T08:13:24.076860389Z" + description: Manila CSI Chart for OpenStack + digest: d82c407ef1cfc5ab3e9586e39b0a3233c69a1e9475f1055945c2365c3da4e330 + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: mnasiadka@gmail.com + name: mnasiadka + - email: matt@stackhpc.com + name: mkjpryor + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.32.0-alpha.0/openstack-manila-csi-2.32.0-alpha.0.tgz + version: 2.32.0-alpha.0 - apiVersion: v1 appVersion: v1.31.2 created: "2024-12-17T15:31:08.697073461Z" @@ -1943,4 +1994,4 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-0.2.1/openstack-manila-csi-0.2.1.tgz version: 0.2.1 -generated: "2024-12-17T15:31:08.697958348Z" +generated: "2025-01-23T08:13:24.077764536Z" From 30c5a395f3a4f9691e1541f6ffee86a28fded8b5 Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Wed, 26 Feb 2025 08:14:55 +0000 Subject: [PATCH 71/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/index.yaml b/index.yaml index b044ee59fa..e1f08efb91 100644 --- a/index.yaml +++ b/index.yaml @@ -1,6 +1,22 @@ apiVersion: v1 entries: openstack-cinder-csi: + - apiVersion: v1 + appVersion: v1.32.0 + created: "2025-02-26T08:14:55.252483342Z" + description: Cinder CSI Chart for OpenStack + digest: 43ab2b09d4278899ada52bb4e34a2d8ddc3357a9c410444762547b217f9596be + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: mnasiadka@gmail.com + name: mnasiadka + - email: matt@stackhpc.com + name: mkjpryor + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.32.0/openstack-cinder-csi-2.32.0.tgz + version: 2.32.0 - apiVersion: v1 appVersion: v1.31.2 created: "2025-01-23T08:13:23.695726105Z" @@ -646,6 +662,25 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.2.2/openstack-cinder-csi-1.2.2.tgz version: 1.2.2 openstack-cloud-controller-manager: + - apiVersion: v2 + appVersion: v1.32.0 + created: "2025-02-26T08:14:55.36844509Z" + dependencies: + - name: common + repository: https://charts.bitnami.com/bitnami + version: 2.14.1 + description: Openstack Cloud Controller Manager Helm Chart + digest: 99fe60e79df237564ac1a03d88bac688a9b8728ef21199c54cd487704bcd61a8 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.32.0/openstack-cloud-controller-manager-2.32.0.tgz + version: 2.32.0 - apiVersion: v2 appVersion: v1.31.2 created: "2025-01-23T08:13:23.880096274Z" @@ -1476,6 +1511,22 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-1.0.0/openstack-cloud-controller-manager-1.0.0.tgz version: 1.0.0 openstack-manila-csi: + - apiVersion: v1 + appVersion: v1.32.0 + created: "2025-02-26T08:14:55.475769248Z" + description: Manila CSI Chart for OpenStack + digest: cf4763e2d7477a15db51670431ebf104bed595cc8cfeb08e4f38fba29058e2d3 + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: mnasiadka@gmail.com + name: mnasiadka + - email: matt@stackhpc.com + name: mkjpryor + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.32.0/openstack-manila-csi-2.32.0.tgz + version: 2.32.0 - apiVersion: v1 appVersion: v1.31.2 created: "2025-01-23T08:13:24.076860389Z" @@ -1994,4 +2045,4 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-0.2.1/openstack-manila-csi-0.2.1.tgz version: 0.2.1 -generated: "2025-01-23T08:13:24.077764536Z" +generated: "2025-02-26T08:14:55.476605607Z" From 9820acd8be769b3c717c8230d91801b352b22b56 Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Wed, 26 Feb 2025 11:58:54 +0000 Subject: [PATCH 72/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/index.yaml b/index.yaml index e1f08efb91..4a93d75180 100644 --- a/index.yaml +++ b/index.yaml @@ -1,6 +1,22 @@ apiVersion: v1 entries: openstack-cinder-csi: + - apiVersion: v1 + appVersion: v1.32.0 + created: "2025-02-26T11:58:54.096447458Z" + description: Cinder CSI Chart for OpenStack + digest: e1fdc99684cc89a6dea1bb35d281225d74b561d124c17bf8681ce054a181122c + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: mnasiadka@gmail.com + name: mnasiadka + - email: matt@stackhpc.com + name: mkjpryor + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.33.0-alpha.0/openstack-cinder-csi-2.33.0-alpha.0.tgz + version: 2.33.0-alpha.0 - apiVersion: v1 appVersion: v1.32.0 created: "2025-02-26T08:14:55.252483342Z" @@ -662,6 +678,25 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.2.2/openstack-cinder-csi-1.2.2.tgz version: 1.2.2 openstack-cloud-controller-manager: + - apiVersion: v2 + appVersion: v1.32.0 + created: "2025-02-26T11:58:54.219329833Z" + dependencies: + - name: common + repository: https://charts.bitnami.com/bitnami + version: 2.14.1 + description: Openstack Cloud Controller Manager Helm Chart + digest: c43a8b1f29ef8507ce6fcac5414f621f4e03f1a3faf300500ecd6d86da126cf7 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.33.0-alpha.0/openstack-cloud-controller-manager-2.33.0-alpha.0.tgz + version: 2.33.0-alpha.0 - apiVersion: v2 appVersion: v1.32.0 created: "2025-02-26T08:14:55.36844509Z" @@ -1511,6 +1546,22 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-1.0.0/openstack-cloud-controller-manager-1.0.0.tgz version: 1.0.0 openstack-manila-csi: + - apiVersion: v1 + appVersion: v1.32.0 + created: "2025-02-26T11:58:54.420351499Z" + description: Manila CSI Chart for OpenStack + digest: 2a0ed31d5621f3faa4b3a3e38c4cdef5d2e8daca50e32a256cd5ce632eb34d4c + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: mnasiadka@gmail.com + name: mnasiadka + - email: matt@stackhpc.com + name: mkjpryor + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.33.0-alpha.0/openstack-manila-csi-2.33.0-alpha.0.tgz + version: 2.33.0-alpha.0 - apiVersion: v1 appVersion: v1.32.0 created: "2025-02-26T08:14:55.475769248Z" @@ -2045,4 +2096,4 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-0.2.1/openstack-manila-csi-0.2.1.tgz version: 0.2.1 -generated: "2025-02-26T08:14:55.476605607Z" +generated: "2025-02-26T11:58:54.421247034Z" From 03f2aab2c2d6aa76e1a6ff68111a263753c91570 Mon Sep 17 00:00:00 2001 From: kayrus Date: Wed, 26 Feb 2025 15:54:49 +0000 Subject: [PATCH 73/90] Update index.yaml Signed-off-by: kayrus --- index.yaml | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/index.yaml b/index.yaml index 4a93d75180..11a8a43211 100644 --- a/index.yaml +++ b/index.yaml @@ -49,6 +49,22 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.32.0-alpha.0/openstack-cinder-csi-2.32.0-alpha.0.tgz version: 2.32.0-alpha.0 + - apiVersion: v1 + appVersion: v1.31.3 + created: "2025-02-26T15:54:48.649280258Z" + description: Cinder CSI Chart for OpenStack + digest: 7321d7ae1a4cb3d14086cf1ff698d5cf51f565d9e7eb9d4565a937a9812e9c83 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: mnasiadka@gmail.com + name: mnasiadka + - email: matt@stackhpc.com + name: mkjpryor + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.31.7/openstack-cinder-csi-2.31.7.tgz + version: 2.31.7 - apiVersion: v1 appVersion: v1.31.0 created: "2024-10-14T18:46:35.132522737Z" @@ -735,6 +751,25 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.32.0-alpha.0/openstack-cloud-controller-manager-2.32.0-alpha.0.tgz version: 2.32.0-alpha.0 + - apiVersion: v2 + appVersion: v1.31.3 + created: "2025-02-26T15:54:48.800722188Z" + dependencies: + - name: common + repository: https://charts.bitnami.com/bitnami + version: 2.14.1 + description: Openstack Cloud Controller Manager Helm Chart + digest: 214236f4f964b3b4c368b2b6cdd0c916eaadba74dc5668caa50bb00678a752fe + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.31.3/openstack-cloud-controller-manager-2.31.3.tgz + version: 2.31.3 - apiVersion: v2 appVersion: v1.31.1 created: "2024-09-19T14:11:37.373161835Z" @@ -1610,6 +1645,22 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.31.4/openstack-manila-csi-2.31.4.tgz version: 2.31.4 + - apiVersion: v1 + appVersion: v1.31.3 + created: "2025-02-26T15:54:48.977063631Z" + description: Manila CSI Chart for OpenStack + digest: d7d4ed09472450e4db8a28271301823a250b8276ffa6d9965c308084226df878 + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: mnasiadka@gmail.com + name: mnasiadka + - email: matt@stackhpc.com + name: mkjpryor + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.31.3/openstack-manila-csi-2.31.3.tgz + version: 2.31.3 - apiVersion: v1 appVersion: v1.31.1 created: "2024-09-19T14:11:37.505548082Z" @@ -2096,4 +2147,4 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-0.2.1/openstack-manila-csi-0.2.1.tgz version: 0.2.1 -generated: "2025-02-26T11:58:54.421247034Z" +generated: "2025-02-26T15:54:48.977887593Z" From be7d43d7a7746a3dab500df40c396543a490a3d6 Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Wed, 26 Feb 2025 16:52:57 +0000 Subject: [PATCH 74/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/index.yaml b/index.yaml index 11a8a43211..90c239869e 100644 --- a/index.yaml +++ b/index.yaml @@ -113,6 +113,22 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.31.0/openstack-cinder-csi-2.31.0.tgz version: 2.31.0 + - apiVersion: v1 + appVersion: v1.30.3 + created: "2025-02-26T16:52:56.80894457Z" + description: Cinder CSI Chart for OpenStack + digest: 2e4a0e6b92264c441bb1a9016755620f367aa43a48967c7d2235a65cdc829eec + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: mnasiadka@gmail.com + name: mnasiadka + - email: matt@stackhpc.com + name: mkjpryor + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.30.3/openstack-cinder-csi-2.30.3.tgz + version: 2.30.3 - apiVersion: v1 appVersion: v1.30.2 created: "2024-11-23T14:07:17.332925614Z" @@ -808,6 +824,25 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.31.0/openstack-cloud-controller-manager-2.31.0.tgz version: 2.31.0 + - apiVersion: v2 + appVersion: v1.30.3 + created: "2025-02-26T16:52:56.906853643Z" + dependencies: + - name: common + repository: https://charts.bitnami.com/bitnami + version: 2.14.1 + description: Openstack Cloud Controller Manager Helm Chart + digest: 92d8aa7db0225e8e6e0d40652008653bf188b8f60917f294d9749bdf49f1eb0a + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.30.5/openstack-cloud-controller-manager-2.30.5.tgz + version: 2.30.5 - apiVersion: v2 appVersion: v1.30.2 created: "2024-11-23T14:07:17.444776298Z" @@ -1693,6 +1728,22 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.31.0/openstack-manila-csi-2.31.0.tgz version: 2.31.0 + - apiVersion: v1 + appVersion: v1.30.3 + created: "2025-02-26T16:52:57.02212979Z" + description: Manila CSI Chart for OpenStack + digest: 6de4a4f7014e585962e7555a8f330e22f6a90892c9fd1f053ed3c2691703287e + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: mnasiadka@gmail.com + name: mnasiadka + - email: matt@stackhpc.com + name: mkjpryor + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.30.3/openstack-manila-csi-2.30.3.tgz + version: 2.30.3 - apiVersion: v1 appVersion: v1.30.2 created: "2024-11-23T14:07:17.541506055Z" @@ -2147,4 +2198,4 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-0.2.1/openstack-manila-csi-0.2.1.tgz version: 0.2.1 -generated: "2025-02-26T15:54:48.977887593Z" +generated: "2025-02-26T16:52:57.023101753Z" From c09405c2054fb1d8a07c72e512873e707241856e Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Wed, 14 May 2025 13:19:41 +0000 Subject: [PATCH 75/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/index.yaml b/index.yaml index 90c239869e..f1ca124c43 100644 --- a/index.yaml +++ b/index.yaml @@ -1616,6 +1616,22 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-1.0.0/openstack-cloud-controller-manager-1.0.0.tgz version: 1.0.0 openstack-manila-csi: + - apiVersion: v1 + appVersion: v1.32.0 + created: "2025-05-14T13:19:40.859801685Z" + description: Manila CSI Chart for OpenStack + digest: 1a6b4ef0c5222cabdac20383a9eef0f6b3bc575b556699e76c8f233d7ee62105 + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: mnasiadka@gmail.com + name: mnasiadka + - email: matt@stackhpc.com + name: mkjpryor + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.33.0-alpha.1/openstack-manila-csi-2.33.0-alpha.1.tgz + version: 2.33.0-alpha.1 - apiVersion: v1 appVersion: v1.32.0 created: "2025-02-26T11:58:54.420351499Z" @@ -2198,4 +2214,4 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-0.2.1/openstack-manila-csi-0.2.1.tgz version: 0.2.1 -generated: "2025-02-26T16:52:57.023101753Z" +generated: "2025-05-14T13:19:40.860435199Z" From 7caca482632639441f583211a205234e5f713c76 Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Fri, 16 May 2025 11:23:27 +0000 Subject: [PATCH 76/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/index.yaml b/index.yaml index f1ca124c43..d6057ba12a 100644 --- a/index.yaml +++ b/index.yaml @@ -1616,6 +1616,22 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-1.0.0/openstack-cloud-controller-manager-1.0.0.tgz version: 1.0.0 openstack-manila-csi: + - apiVersion: v1 + appVersion: v1.32.0 + created: "2025-05-16T11:23:27.273028902Z" + description: Manila CSI Chart for OpenStack + digest: 9412412ff583036b974c75bd6be042ed4e1c6cacb73bd39d6b508cb38ac09262 + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: mnasiadka@gmail.com + name: mnasiadka + - email: matt@stackhpc.com + name: mkjpryor + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.33.0-alpha.2/openstack-manila-csi-2.33.0-alpha.2.tgz + version: 2.33.0-alpha.2 - apiVersion: v1 appVersion: v1.32.0 created: "2025-05-14T13:19:40.859801685Z" @@ -2214,4 +2230,4 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-0.2.1/openstack-manila-csi-0.2.1.tgz version: 0.2.1 -generated: "2025-05-14T13:19:40.860435199Z" +generated: "2025-05-16T11:23:27.274013631Z" From 64ee592484e8cdf30cbca363d2880f826f642ee9 Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Mon, 19 May 2025 07:15:35 +0000 Subject: [PATCH 77/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/index.yaml b/index.yaml index d6057ba12a..623a0c879d 100644 --- a/index.yaml +++ b/index.yaml @@ -1616,6 +1616,22 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-1.0.0/openstack-cloud-controller-manager-1.0.0.tgz version: 1.0.0 openstack-manila-csi: + - apiVersion: v1 + appVersion: v1.32.0 + created: "2025-05-19T07:15:33.667718747Z" + description: Manila CSI Chart for OpenStack + digest: 2b8d2355eadf3d1bdbf967616cfeeab87dc0a2315505337a5fa71c770ea12621 + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: mnasiadka@gmail.com + name: mnasiadka + - email: matt@stackhpc.com + name: mkjpryor + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.33.0-alpha.3/openstack-manila-csi-2.33.0-alpha.3.tgz + version: 2.33.0-alpha.3 - apiVersion: v1 appVersion: v1.32.0 created: "2025-05-16T11:23:27.273028902Z" @@ -2230,4 +2246,4 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-0.2.1/openstack-manila-csi-0.2.1.tgz version: 0.2.1 -generated: "2025-05-16T11:23:27.274013631Z" +generated: "2025-05-19T07:15:33.668319819Z" From 5ba5001b187fef4381778f0108e246f6e43b4cc2 Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Wed, 21 May 2025 20:32:53 +0000 Subject: [PATCH 78/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/index.yaml b/index.yaml index 623a0c879d..050fb03ede 100644 --- a/index.yaml +++ b/index.yaml @@ -1,6 +1,22 @@ apiVersion: v1 entries: openstack-cinder-csi: + - apiVersion: v1 + appVersion: v1.32.0 + created: "2025-05-21T20:32:52.675242108Z" + description: Cinder CSI Chart for OpenStack + digest: 21d3a87cd6e531646f3298e2664895704bb5d3e6f64c88055ab765d93f32d16e + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: mnasiadka@gmail.com + name: mnasiadka + - email: matt@stackhpc.com + name: mkjpryor + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.33.0-alpha.1/openstack-cinder-csi-2.33.0-alpha.1.tgz + version: 2.33.0-alpha.1 - apiVersion: v1 appVersion: v1.32.0 created: "2025-02-26T11:58:54.096447458Z" @@ -2246,4 +2262,4 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-0.2.1/openstack-manila-csi-0.2.1.tgz version: 0.2.1 -generated: "2025-05-19T07:15:33.668319819Z" +generated: "2025-05-21T20:32:52.675946443Z" From 6bf6baf314fd37d111efdac877a6d219c413eef9 Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Thu, 22 May 2025 09:10:53 +0000 Subject: [PATCH 79/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/index.yaml b/index.yaml index 050fb03ede..f7005e56e4 100644 --- a/index.yaml +++ b/index.yaml @@ -1,6 +1,22 @@ apiVersion: v1 entries: openstack-cinder-csi: + - apiVersion: v1 + appVersion: v1.32.0 + created: "2025-05-22T09:10:53.367684613Z" + description: Cinder CSI Chart for OpenStack + digest: 445bbcadc977f63dedfc84e9e75a9cb20a1296901a9eb964875eb3c81803ee7c + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: mnasiadka@gmail.com + name: mnasiadka + - email: matt@stackhpc.com + name: mkjpryor + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.33.0-alpha.2/openstack-cinder-csi-2.33.0-alpha.2.tgz + version: 2.33.0-alpha.2 - apiVersion: v1 appVersion: v1.32.0 created: "2025-05-21T20:32:52.675242108Z" @@ -2262,4 +2278,4 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-0.2.1/openstack-manila-csi-0.2.1.tgz version: 0.2.1 -generated: "2025-05-21T20:32:52.675946443Z" +generated: "2025-05-22T09:10:53.368092744Z" From e2c6f0eecf5a8be90e8b4b590e2e8a0006d11c2c Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Sun, 8 Jun 2025 11:16:50 +0000 Subject: [PATCH 80/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/index.yaml b/index.yaml index f7005e56e4..ca5961982d 100644 --- a/index.yaml +++ b/index.yaml @@ -1,6 +1,22 @@ apiVersion: v1 entries: openstack-cinder-csi: + - apiVersion: v1 + appVersion: v1.33.0 + created: "2025-06-08T11:16:49.230130922Z" + description: Cinder CSI Chart for OpenStack + digest: 47924e598703523ef70ef8f7926caabceda1009bdba44b53eb6f801eaa7f8bd4 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: mnasiadka@gmail.com + name: mnasiadka + - email: matt@stackhpc.com + name: mkjpryor + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.33.0/openstack-cinder-csi-2.33.0.tgz + version: 2.33.0 - apiVersion: v1 appVersion: v1.32.0 created: "2025-05-22T09:10:53.367684613Z" @@ -742,6 +758,25 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.2.2/openstack-cinder-csi-1.2.2.tgz version: 1.2.2 openstack-cloud-controller-manager: + - apiVersion: v2 + appVersion: v1.33.0 + created: "2025-06-08T11:16:49.43031341Z" + dependencies: + - name: common + repository: https://charts.bitnami.com/bitnami + version: 2.14.1 + description: Openstack Cloud Controller Manager Helm Chart + digest: 67e12f4d3981fffef2671c1d1f2b5f85507da1b1ad75e3b55f47966415e6ac11 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.33.0/openstack-cloud-controller-manager-2.33.0.tgz + version: 2.33.0 - apiVersion: v2 appVersion: v1.32.0 created: "2025-02-26T11:58:54.219329833Z" @@ -1648,6 +1683,22 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-1.0.0/openstack-cloud-controller-manager-1.0.0.tgz version: 1.0.0 openstack-manila-csi: + - apiVersion: v1 + appVersion: v1.33.0 + created: "2025-06-08T11:16:49.633581334Z" + description: Manila CSI Chart for OpenStack + digest: b1fbfbb9a97ec1ad792d0556d61de32b01e7c4c68f0a8521b7c8d4ca33b2f42a + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: mnasiadka@gmail.com + name: mnasiadka + - email: matt@stackhpc.com + name: mkjpryor + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.33.0/openstack-manila-csi-2.33.0.tgz + version: 2.33.0 - apiVersion: v1 appVersion: v1.32.0 created: "2025-05-19T07:15:33.667718747Z" @@ -2278,4 +2329,4 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-0.2.1/openstack-manila-csi-0.2.1.tgz version: 0.2.1 -generated: "2025-05-22T09:10:53.368092744Z" +generated: "2025-06-08T11:16:49.63468704Z" From 70c315453f3b5e49d5f7af9edc616fb52ecb930e Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Mon, 28 Jul 2025 08:54:56 +0000 Subject: [PATCH 81/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/index.yaml b/index.yaml index ca5961982d..54b9ffd9e8 100644 --- a/index.yaml +++ b/index.yaml @@ -1,6 +1,22 @@ apiVersion: v1 entries: openstack-cinder-csi: + - apiVersion: v1 + appVersion: v1.33.1 + created: "2025-07-28T08:54:55.678102936Z" + description: Cinder CSI Chart for OpenStack + digest: 07ea4fcc079a9283c2e3e5707c0e5a0d596277ed9f58667d0415176592e94940 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: mnasiadka@gmail.com + name: mnasiadka + - email: matt@stackhpc.com + name: mkjpryor + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.33.1/openstack-cinder-csi-2.33.1.tgz + version: 2.33.1 - apiVersion: v1 appVersion: v1.33.0 created: "2025-06-08T11:16:49.230130922Z" @@ -758,6 +774,25 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.2.2/openstack-cinder-csi-1.2.2.tgz version: 1.2.2 openstack-cloud-controller-manager: + - apiVersion: v2 + appVersion: v1.33.1 + created: "2025-07-28T08:54:55.836176428Z" + dependencies: + - name: common + repository: https://charts.bitnami.com/bitnami + version: 2.14.1 + description: Openstack Cloud Controller Manager Helm Chart + digest: 8e3723ed55e49bb8723eeddde586e6778c400ee9de546ede05c7dddf3caf60df + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.33.1/openstack-cloud-controller-manager-2.33.1.tgz + version: 2.33.1 - apiVersion: v2 appVersion: v1.33.0 created: "2025-06-08T11:16:49.43031341Z" @@ -1683,6 +1718,22 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-1.0.0/openstack-cloud-controller-manager-1.0.0.tgz version: 1.0.0 openstack-manila-csi: + - apiVersion: v1 + appVersion: v1.33.1 + created: "2025-07-28T08:54:55.955409474Z" + description: Manila CSI Chart for OpenStack + digest: 379560f27217736e5b3a28d713a2c41c6b6af70f471b4969fb022de8e3f860d9 + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: mnasiadka@gmail.com + name: mnasiadka + - email: matt@stackhpc.com + name: mkjpryor + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.33.1/openstack-manila-csi-2.33.1.tgz + version: 2.33.1 - apiVersion: v1 appVersion: v1.33.0 created: "2025-06-08T11:16:49.633581334Z" @@ -2329,4 +2380,4 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-0.2.1/openstack-manila-csi-0.2.1.tgz version: 0.2.1 -generated: "2025-06-08T11:16:49.63468704Z" +generated: "2025-07-28T08:54:55.956255117Z" From 00a07177b09f4ead71b1eaed532eb75c4e24d79a Mon Sep 17 00:00:00 2001 From: kayrus Date: Mon, 28 Jul 2025 09:02:33 +0000 Subject: [PATCH 82/90] Update index.yaml Signed-off-by: kayrus --- index.yaml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/index.yaml b/index.yaml index 54b9ffd9e8..fc43fcdba0 100644 --- a/index.yaml +++ b/index.yaml @@ -1846,6 +1846,22 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.32.0-alpha.0/openstack-manila-csi-2.32.0-alpha.0.tgz version: 2.32.0-alpha.0 + - apiVersion: v1 + appVersion: v1.31.4 + created: "2025-07-28T09:02:33.363901843Z" + description: Manila CSI Chart for OpenStack + digest: 3a6a755ee4045f2991b00601c7b03d9a156275c0fbb813cfc1b4b7d201fd75ad + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: mnasiadka@gmail.com + name: mnasiadka + - email: matt@stackhpc.com + name: mkjpryor + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.31.5/openstack-manila-csi-2.31.5.tgz + version: 2.31.5 - apiVersion: v1 appVersion: v1.31.2 created: "2024-12-17T15:31:08.697073461Z" @@ -2380,4 +2396,4 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-0.2.1/openstack-manila-csi-0.2.1.tgz version: 0.2.1 -generated: "2025-07-28T08:54:55.956255117Z" +generated: "2025-07-28T09:02:33.364789443Z" From 8b91a5dbfd5347c316cb5135b124817b4541351f Mon Sep 17 00:00:00 2001 From: kayrus Date: Mon, 28 Jul 2025 09:06:41 +0000 Subject: [PATCH 83/90] Update index.yaml Signed-off-by: kayrus --- index.yaml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/index.yaml b/index.yaml index fc43fcdba0..dc44ed368f 100644 --- a/index.yaml +++ b/index.yaml @@ -81,6 +81,22 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.33.0-alpha.0/openstack-cinder-csi-2.33.0-alpha.0.tgz version: 2.33.0-alpha.0 + - apiVersion: v1 + appVersion: v1.32.1 + created: "2025-07-28T09:06:41.169583245Z" + description: Cinder CSI Chart for OpenStack + digest: 34acfa269dfaf9701e55a63becaf8b12d159e5b1c4f3a4936d655261b48fbe93 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: mnasiadka@gmail.com + name: mnasiadka + - email: matt@stackhpc.com + name: mkjpryor + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.32.2/openstack-cinder-csi-2.32.2.tgz + version: 2.32.2 - apiVersion: v1 appVersion: v1.32.0 created: "2025-02-26T08:14:55.252483342Z" @@ -2396,4 +2412,4 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-0.2.1/openstack-manila-csi-0.2.1.tgz version: 0.2.1 -generated: "2025-07-28T09:02:33.364789443Z" +generated: "2025-07-28T09:06:41.170548414Z" From 35f07bf4320fa2b13e75accf27369a3375e11820 Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Mon, 13 Oct 2025 17:18:03 +0000 Subject: [PATCH 84/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/index.yaml b/index.yaml index dc44ed368f..b20c7cb190 100644 --- a/index.yaml +++ b/index.yaml @@ -1,6 +1,22 @@ apiVersion: v1 entries: openstack-cinder-csi: + - apiVersion: v1 + appVersion: v1.34.0 + created: "2025-10-13T17:18:02.572174606Z" + description: Cinder CSI Chart for OpenStack + digest: a5e2238ef925dbc8ce47f2a9f29dd3c79aa8fc6cb79f466a5134cff14bdb3361 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: mnasiadka@gmail.com + name: mnasiadka + - email: matt@stackhpc.com + name: mkjpryor + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.34.0/openstack-cinder-csi-2.34.0.tgz + version: 2.34.0 - apiVersion: v1 appVersion: v1.33.1 created: "2025-07-28T08:54:55.678102936Z" @@ -790,6 +806,25 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.2.2/openstack-cinder-csi-1.2.2.tgz version: 1.2.2 openstack-cloud-controller-manager: + - apiVersion: v2 + appVersion: v1.34.0 + created: "2025-10-13T17:18:02.695775179Z" + dependencies: + - name: common + repository: https://charts.bitnami.com/bitnami + version: 2.14.1 + description: Openstack Cloud Controller Manager Helm Chart + digest: a32574a96de7362a102961aa0b224f38552c8b49824d41541a080abfe3ff3368 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.34.0/openstack-cloud-controller-manager-2.34.0.tgz + version: 2.34.0 - apiVersion: v2 appVersion: v1.33.1 created: "2025-07-28T08:54:55.836176428Z" @@ -1734,6 +1769,22 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-1.0.0/openstack-cloud-controller-manager-1.0.0.tgz version: 1.0.0 openstack-manila-csi: + - apiVersion: v1 + appVersion: v1.34.0 + created: "2025-10-13T17:18:02.840898172Z" + description: Manila CSI Chart for OpenStack + digest: 2433afec32aa181dae758e1f81d08074827e0ee19b33836adeeafa6c59ce5fa7 + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: mnasiadka@gmail.com + name: mnasiadka + - email: matt@stackhpc.com + name: mkjpryor + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.34.0/openstack-manila-csi-2.34.0.tgz + version: 2.34.0 - apiVersion: v1 appVersion: v1.33.1 created: "2025-07-28T08:54:55.955409474Z" @@ -2412,4 +2463,4 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-0.2.1/openstack-manila-csi-0.2.1.tgz version: 0.2.1 -generated: "2025-07-28T09:06:41.170548414Z" +generated: "2025-10-13T17:18:02.842126803Z" From a3a2e20e49da5946c5b3a9f40e69c0d7c572ce76 Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Wed, 15 Oct 2025 15:42:08 +0000 Subject: [PATCH 85/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/index.yaml b/index.yaml index b20c7cb190..270be50941 100644 --- a/index.yaml +++ b/index.yaml @@ -1,6 +1,22 @@ apiVersion: v1 entries: openstack-cinder-csi: + - apiVersion: v1 + appVersion: v1.34.1 + created: "2025-10-15T15:42:07.233534841Z" + description: Cinder CSI Chart for OpenStack + digest: 4de112747178b9058e135a0facbda473b36b68b931455909df03ebb06d5f9bcd + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: mnasiadka@gmail.com + name: mnasiadka + - email: matt@stackhpc.com + name: mkjpryor + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.34.1/openstack-cinder-csi-2.34.1.tgz + version: 2.34.1 - apiVersion: v1 appVersion: v1.34.0 created: "2025-10-13T17:18:02.572174606Z" @@ -806,6 +822,25 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.2.2/openstack-cinder-csi-1.2.2.tgz version: 1.2.2 openstack-cloud-controller-manager: + - apiVersion: v2 + appVersion: v1.34.1 + created: "2025-10-15T15:42:07.536740095Z" + dependencies: + - name: common + repository: https://charts.bitnami.com/bitnami + version: 2.14.1 + description: Openstack Cloud Controller Manager Helm Chart + digest: 015b109401202bdbb46d3fd751d8b40b3991901def73682bb59d34a23f9d6103 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.34.1/openstack-cloud-controller-manager-2.34.1.tgz + version: 2.34.1 - apiVersion: v2 appVersion: v1.34.0 created: "2025-10-13T17:18:02.695775179Z" @@ -1769,6 +1804,22 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-1.0.0/openstack-cloud-controller-manager-1.0.0.tgz version: 1.0.0 openstack-manila-csi: + - apiVersion: v1 + appVersion: v1.34.1 + created: "2025-10-15T15:42:07.808684281Z" + description: Manila CSI Chart for OpenStack + digest: 37698555a3de28dde7453fb5a948a7d578a485137ed565cf278c9eae8a41d3aa + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: mnasiadka@gmail.com + name: mnasiadka + - email: matt@stackhpc.com + name: mkjpryor + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.34.1/openstack-manila-csi-2.34.1.tgz + version: 2.34.1 - apiVersion: v1 appVersion: v1.34.0 created: "2025-10-13T17:18:02.840898172Z" @@ -2463,4 +2514,4 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-0.2.1/openstack-manila-csi-0.2.1.tgz version: 0.2.1 -generated: "2025-10-13T17:18:02.842126803Z" +generated: "2025-10-15T15:42:07.809518408Z" From ed31fc55b9342de2e330123fecbe2e807b13fb51 Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Fri, 16 Jan 2026 08:51:34 +0000 Subject: [PATCH 86/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/index.yaml b/index.yaml index 270be50941..6839b8872c 100644 --- a/index.yaml +++ b/index.yaml @@ -822,6 +822,25 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.2.2/openstack-cinder-csi-1.2.2.tgz version: 1.2.2 openstack-cloud-controller-manager: + - apiVersion: v2 + appVersion: v1.34.1 + created: "2026-01-16T08:51:34.681853591Z" + dependencies: + - name: common + repository: https://charts.bitnami.com/bitnami + version: 2.14.1 + description: Openstack Cloud Controller Manager Helm Chart + digest: 412781f2d334f033dd92c2ecb9795471c853018daf2067141285c4d15a29e518 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.34.2/openstack-cloud-controller-manager-2.34.2.tgz + version: 2.34.2 - apiVersion: v2 appVersion: v1.34.1 created: "2025-10-15T15:42:07.536740095Z" @@ -2514,4 +2533,4 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-0.2.1/openstack-manila-csi-0.2.1.tgz version: 0.2.1 -generated: "2025-10-15T15:42:07.809518408Z" +generated: "2026-01-16T08:51:34.682167583Z" From 51438e984eb569c6cbe5c3270a45467ecd96260d Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Wed, 21 Jan 2026 13:43:42 +0000 Subject: [PATCH 87/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/index.yaml b/index.yaml index 6839b8872c..dd07833e3f 100644 --- a/index.yaml +++ b/index.yaml @@ -1,6 +1,22 @@ apiVersion: v1 entries: openstack-cinder-csi: + - apiVersion: v1 + appVersion: v1.34.1 + created: "2026-01-21T13:43:42.336955664Z" + description: Cinder CSI Chart for OpenStack + digest: 367f83412a0c7990a36df47cb5b650e88f5cb0774ebbcf81634cef088112a9a6 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: mnasiadka@gmail.com + name: mnasiadka + - email: matt@stackhpc.com + name: mkjpryor + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.34.3/openstack-cinder-csi-2.34.3.tgz + version: 2.34.3 - apiVersion: v1 appVersion: v1.34.1 created: "2025-10-15T15:42:07.233534841Z" @@ -2533,4 +2549,4 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-0.2.1/openstack-manila-csi-0.2.1.tgz version: 0.2.1 -generated: "2026-01-16T08:51:34.682167583Z" +generated: "2026-01-21T13:43:42.337730968Z" From 876b019be5e2f055d13143d8890b1e29a056f531 Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Wed, 21 Jan 2026 13:57:44 +0000 Subject: [PATCH 88/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/index.yaml b/index.yaml index dd07833e3f..5ce4cde73b 100644 --- a/index.yaml +++ b/index.yaml @@ -1839,6 +1839,22 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-1.0.0/openstack-cloud-controller-manager-1.0.0.tgz version: 1.0.0 openstack-manila-csi: + - apiVersion: v1 + appVersion: v1.34.1 + created: "2026-01-21T13:57:43.764651283Z" + description: Manila CSI Chart for OpenStack + digest: eeac5787ef4e595d60906dde74f7f75d0517f7f3564480658432b918d052ff8a + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: mnasiadka@gmail.com + name: mnasiadka + - email: matt@stackhpc.com + name: mkjpryor + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.34.2/openstack-manila-csi-2.34.2.tgz + version: 2.34.2 - apiVersion: v1 appVersion: v1.34.1 created: "2025-10-15T15:42:07.808684281Z" @@ -2549,4 +2565,4 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-0.2.1/openstack-manila-csi-0.2.1.tgz version: 0.2.1 -generated: "2026-01-21T13:43:42.337730968Z" +generated: "2026-01-21T13:57:43.765469882Z" From b0de2cefd230b36703750fbd6e8a9725d3b682fa Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Fri, 23 Jan 2026 16:51:55 +0000 Subject: [PATCH 89/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/index.yaml b/index.yaml index 5ce4cde73b..7350dd226a 100644 --- a/index.yaml +++ b/index.yaml @@ -1,6 +1,22 @@ apiVersion: v1 entries: openstack-cinder-csi: + - apiVersion: v1 + appVersion: v1.35.0 + created: "2026-01-23T16:51:54.309322732Z" + description: Cinder CSI Chart for OpenStack + digest: f32351c3efabec9bf1d99024b2f4e03b44d830725d22ef5604afbe36c0746a0c + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: mnasiadka@gmail.com + name: mnasiadka + - email: matt@stackhpc.com + name: mkjpryor + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.35.0/openstack-cinder-csi-2.35.0.tgz + version: 2.35.0 - apiVersion: v1 appVersion: v1.34.1 created: "2026-01-21T13:43:42.336955664Z" @@ -838,6 +854,25 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.2.2/openstack-cinder-csi-1.2.2.tgz version: 1.2.2 openstack-cloud-controller-manager: + - apiVersion: v2 + appVersion: v1.35.0 + created: "2026-01-23T16:51:54.512766199Z" + dependencies: + - name: common + repository: https://charts.bitnami.com/bitnami + version: 2.14.1 + description: Openstack Cloud Controller Manager Helm Chart + digest: d15c048a7fc843ad30fc3e91a089026ecd43eac12abd26a81fe37a1bfaea73fd + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.35.0/openstack-cloud-controller-manager-2.35.0.tgz + version: 2.35.0 - apiVersion: v2 appVersion: v1.34.1 created: "2026-01-16T08:51:34.681853591Z" @@ -1839,6 +1874,22 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-1.0.0/openstack-cloud-controller-manager-1.0.0.tgz version: 1.0.0 openstack-manila-csi: + - apiVersion: v1 + appVersion: v1.35.0 + created: "2026-01-23T16:51:54.774694597Z" + description: Manila CSI Chart for OpenStack + digest: a419e5f73d49fdad80e5b7e93fc2dce269336aa97c164ed24f9fff65794c9138 + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: mnasiadka@gmail.com + name: mnasiadka + - email: matt@stackhpc.com + name: mkjpryor + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.35.0/openstack-manila-csi-2.35.0.tgz + version: 2.35.0 - apiVersion: v1 appVersion: v1.34.1 created: "2026-01-21T13:57:43.764651283Z" @@ -2565,4 +2616,4 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-0.2.1/openstack-manila-csi-0.2.1.tgz version: 0.2.1 -generated: "2026-01-21T13:57:43.765469882Z" +generated: "2026-01-23T16:51:54.775282769Z" From e9b6bb4bfcf74e9dcc215c49db03038bedb44e4d Mon Sep 17 00:00:00 2001 From: k8s-ci-robot Date: Wed, 13 May 2026 13:12:25 +0000 Subject: [PATCH 90/90] Update index.yaml Signed-off-by: k8s-ci-robot --- index.yaml | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/index.yaml b/index.yaml index 7350dd226a..675848cac1 100644 --- a/index.yaml +++ b/index.yaml @@ -1,6 +1,22 @@ apiVersion: v1 entries: openstack-cinder-csi: + - apiVersion: v1 + appVersion: v1.36.0 + created: "2026-05-13T13:12:24.545552388Z" + description: Cinder CSI Chart for OpenStack + digest: be7006784e647493fb91c722926e2c61b5b29741ec26b58540def409e77bc11f + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: mnasiadka@gmail.com + name: mnasiadka + - email: matt@stackhpc.com + name: mkjpryor + name: openstack-cinder-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-2.36.0/openstack-cinder-csi-2.36.0.tgz + version: 2.36.0 - apiVersion: v1 appVersion: v1.35.0 created: "2026-01-23T16:51:54.309322732Z" @@ -854,6 +870,25 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cinder-csi-1.2.2/openstack-cinder-csi-1.2.2.tgz version: 1.2.2 openstack-cloud-controller-manager: + - apiVersion: v2 + appVersion: v1.36.0 + created: "2026-05-13T13:12:24.734844689Z" + dependencies: + - name: common + repository: https://charts.bitnami.com/bitnami + version: 2.14.1 + description: Openstack Cloud Controller Manager Helm Chart + digest: a75b3454cb34c9c59969f6680719e3e301f863a8cf7a5be5d1da08e88f8f6b21 + home: https://github.com/kubernetes/cloud-provider-openstack + icon: https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-images-prod/openstack-logo/OpenStack-Logo-Vertical.png + maintainers: + - email: f.kloeker@telekom.de + name: eumel8 + url: https://www.telekom.com + name: openstack-cloud-controller-manager + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-2.36.0/openstack-cloud-controller-manager-2.36.0.tgz + version: 2.36.0 - apiVersion: v2 appVersion: v1.35.0 created: "2026-01-23T16:51:54.512766199Z" @@ -1874,6 +1909,22 @@ entries: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-cloud-controller-manager-1.0.0/openstack-cloud-controller-manager-1.0.0.tgz version: 1.0.0 openstack-manila-csi: + - apiVersion: v1 + appVersion: v1.36.0 + created: "2026-05-13T13:12:24.957171787Z" + description: Manila CSI Chart for OpenStack + digest: fca7caeb49fafac68f0f72e7677ee02e7a1bedf8e87a68b65223eb9f4cc42b36 + home: http://github.com/kubernetes/cloud-provider-openstack + icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png + maintainers: + - email: mnasiadka@gmail.com + name: mnasiadka + - email: matt@stackhpc.com + name: mkjpryor + name: openstack-manila-csi + urls: + - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-2.36.0/openstack-manila-csi-2.36.0.tgz + version: 2.36.0 - apiVersion: v1 appVersion: v1.35.0 created: "2026-01-23T16:51:54.774694597Z" @@ -2616,4 +2667,4 @@ entries: urls: - https://github.com/kubernetes/cloud-provider-openstack/releases/download/openstack-manila-csi-0.2.1/openstack-manila-csi-0.2.1.tgz version: 0.2.1 -generated: "2026-01-23T16:51:54.775282769Z" +generated: "2026-05-13T13:12:24.957848477Z"