From 7a2dd4c3cf5891fc3d7627843b124dcdf4f8abf9 Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Wed, 8 Jun 2022 19:41:36 +0530 Subject: [PATCH] rbd: create token and use it for vault SA create the token if kubernetes version in 1.24+ and use it for vault sa. Signed-off-by: Madhu Rajanna Signed-off-by: Rakshith R --- .../templates/nodeplugin-clusterrole.yaml | 3 ++ .../templates/provisioner-clusterrole.yaml | 3 ++ .../rbd/kubernetes/csi-nodeplugin-rbac.yaml | 3 ++ .../rbd/kubernetes/csi-provisioner-rbac.yaml | 3 ++ internal/kms/vault_sa.go | 45 +++++++++++++++++-- internal/util/k8s/version.go | 45 +++++++++++++++++++ 6 files changed, 98 insertions(+), 4 deletions(-) create mode 100644 internal/util/k8s/version.go diff --git a/charts/ceph-csi-rbd/templates/nodeplugin-clusterrole.yaml b/charts/ceph-csi-rbd/templates/nodeplugin-clusterrole.yaml index 93ec30ed6..9e24d757e 100644 --- a/charts/ceph-csi-rbd/templates/nodeplugin-clusterrole.yaml +++ b/charts/ceph-csi-rbd/templates/nodeplugin-clusterrole.yaml @@ -31,4 +31,7 @@ rules: - apiGroups: ["storage.k8s.io"] resources: ["volumeattachments"] verbs: ["list", "get"] + - apiGroups: [""] + resources: ["serviceaccounts/token"] + verbs: ["create"] {{- end -}} diff --git a/charts/ceph-csi-rbd/templates/provisioner-clusterrole.yaml b/charts/ceph-csi-rbd/templates/provisioner-clusterrole.yaml index 9a4b1fe83..70c7670b2 100644 --- a/charts/ceph-csi-rbd/templates/provisioner-clusterrole.yaml +++ b/charts/ceph-csi-rbd/templates/provisioner-clusterrole.yaml @@ -70,5 +70,8 @@ rules: resources: ["csinodes"] verbs: ["get", "list", "watch"] {{- end }} + - apiGroups: [""] + resources: ["serviceaccounts/token"] + verbs: ["create"] {{- end -}} diff --git a/deploy/rbd/kubernetes/csi-nodeplugin-rbac.yaml b/deploy/rbd/kubernetes/csi-nodeplugin-rbac.yaml index 98ffbcaf8..baf5cdedc 100644 --- a/deploy/rbd/kubernetes/csi-nodeplugin-rbac.yaml +++ b/deploy/rbd/kubernetes/csi-nodeplugin-rbac.yaml @@ -30,6 +30,9 @@ rules: - apiGroups: ["storage.k8s.io"] resources: ["volumeattachments"] verbs: ["list", "get"] + - apiGroups: [""] + resources: ["serviceaccounts/token"] + verbs: ["create"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 diff --git a/deploy/rbd/kubernetes/csi-provisioner-rbac.yaml b/deploy/rbd/kubernetes/csi-provisioner-rbac.yaml index 0cc0b8282..0fd06e17d 100644 --- a/deploy/rbd/kubernetes/csi-provisioner-rbac.yaml +++ b/deploy/rbd/kubernetes/csi-provisioner-rbac.yaml @@ -63,6 +63,9 @@ rules: - apiGroups: [""] resources: ["serviceaccounts"] verbs: ["get"] + - apiGroups: [""] + resources: ["serviceaccounts/token"] + verbs: ["create"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 diff --git a/internal/kms/vault_sa.go b/internal/kms/vault_sa.go index 33c92f915..4a65768dd 100644 --- a/internal/kms/vault_sa.go +++ b/internal/kms/vault_sa.go @@ -23,9 +23,13 @@ import ( "io/ioutil" "os" + "github.com/ceph/ceph-csi/internal/util/k8s" + "github.com/libopenstorage/secrets/vault" + authenticationv1 "k8s.io/api/authentication/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" ) const ( @@ -35,6 +39,9 @@ const ( // should be available in the Tenants namespace. This ServiceAccount // will be used to connect to Hashicorp Vault. vaultTenantSAName = "ceph-csi-vault-sa" + // Kubernetes version which requires ServiceAccount token creation. + // Kubernetes 1.24 => 1 * 1000 + 24. + kubeMinVersionForCreateToken = 1024 ) /* @@ -292,9 +299,9 @@ func (kms *vaultTenantSA) getToken() (string, error) { } for _, secretRef := range sa.Secrets { - secret, err := c.CoreV1().Secrets(kms.Tenant).Get(context.TODO(), secretRef.Name, metav1.GetOptions{}) - if err != nil { - return "", fmt.Errorf("failed to get Secret %s/%s: %w", kms.Tenant, secretRef.Name, err) + secret, sErr := c.CoreV1().Secrets(kms.Tenant).Get(context.TODO(), secretRef.Name, metav1.GetOptions{}) + if sErr != nil { + return "", fmt.Errorf("failed to get Secret %s/%s: %w", kms.Tenant, secretRef.Name, sErr) } token, ok := secret.Data["token"] @@ -303,7 +310,7 @@ func (kms *vaultTenantSA) getToken() (string, error) { } } - return "", fmt.Errorf("failed to find token in ServiceAccount %s/%s", kms.Tenant, kms.tenantSAName) + return kms.createToken(sa, c) } // getTokenPath creates a temporary directory structure that contains the token @@ -327,3 +334,33 @@ func (kms *vaultTenantSA) getTokenPath() (string, error) { return dir + "/token", nil } + +// createToken creates required service account token for kubernetes 1.24+, +// else returns error. +// From kubernetes v1.24+, secret for service account tokens are not +// automatically created. Hence, use the create token api to fetch it. +// refer: https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.24.md \ +// #no-really-you-must-read-this-before-you-upgrade-1 . +func (kms *vaultTenantSA) createToken(sa *corev1.ServiceAccount, client *kubernetes.Clientset) (string, error) { + major, minor, err := k8s.GetServerVersion(client) + if err != nil { + return "", fmt.Errorf("failed to get server version: %w", err) + } + + if (major*1000 + minor) >= kubeMinVersionForCreateToken { + tokenRequest := &authenticationv1.TokenRequest{} + token, err := client.CoreV1().ServiceAccounts(kms.Tenant).CreateToken( + context.TODO(), + sa.Name, + tokenRequest, + metav1.CreateOptions{}, + ) + if err != nil { + return "", fmt.Errorf("failed to create token for service account %s/%s: %w", kms.Tenant, sa.Name, err) + } + + return token.Status.Token, nil + } + + return "", fmt.Errorf("failed to find token in ServiceAccount %s/%s", kms.Tenant, kms.tenantSAName) +} diff --git a/internal/util/k8s/version.go b/internal/util/k8s/version.go new file mode 100644 index 000000000..9a5e3c98d --- /dev/null +++ b/internal/util/k8s/version.go @@ -0,0 +1,45 @@ +/* +Copyright 2022 The Ceph-CSI Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package k8s + +import ( + "fmt" + "strconv" + + "k8s.io/client-go/kubernetes" +) + +// GetServerVersion returns kubernetes server major and minor version as +// integer. +func GetServerVersion(client *kubernetes.Clientset) (int, int, error) { + version, err := client.ServerVersion() + if err != nil { + return 0, 0, fmt.Errorf("failed to get ServerVersion: %w", err) + } + + major, err := strconv.Atoi(version.Major) + if err != nil { + return 0, 0, fmt.Errorf("failed to convert Kubernetes major version %q to int: %w", version.Major, err) + } + + minor, err := strconv.Atoi(version.Minor) + if err != nil { + return 0, 0, fmt.Errorf("failed to convert Kubernetes minor version %q to int: %w", version.Minor, err) + } + + return major, minor, nil +}