cleanup: move k8s functions to the util/k8s package

By placing the NewK8sClient() function in its own package, the KMS API
can be split from the "internal/util" package. Some of the KMS providers
use the NewK8sClient() function, and this causes circular dependencies
between "internal/utils" -> "internal/kms" -> "internal/utils", which
are not alowed in Go.

Updates: #852
Signed-off-by: Niels de Vos <ndevos@redhat.com>

internal/rbd/rbd_healer.go

@@ -21,6 +21,7 @@ import (
 	"sync"
 
 	"github.com/ceph/ceph-csi/internal/util"
+	kubeclient "github.com/ceph/ceph-csi/internal/util/k8s"
 	"github.com/ceph/ceph-csi/internal/util/log"
 
 	"github.com/container-storage-interface/spec/lib/go/csi"
@@ -128,7 +129,7 @@ func callNodeStageVolume(ns *NodeServer, c *k8s.Clientset, pv *v1.PersistentVolu
 
 // runVolumeHealer heal the volumes attached on a node.
 func runVolumeHealer(ns *NodeServer, conf *util.Config) error {
-	c := util.NewK8sClient()
+	c := kubeclient.NewK8sClient()
 	val, err := c.StorageV1().VolumeAttachments().List(context.TODO(), metav1.ListOptions{})
 	if err != nil {
 		log.ErrorLogMsg("list volumeAttachments failed, err: %v", err)

internal/rbd/rbd_util.go

@@ -29,6 +29,7 @@ import (
 	"time"
 
 	"github.com/ceph/ceph-csi/internal/util"
+	"github.com/ceph/ceph-csi/internal/util/k8s"
 	"github.com/ceph/ceph-csi/internal/util/log"
 
 	"github.com/ceph/go-ceph/rados"
@@ -1063,7 +1064,7 @@ func genVolFromVolID(
 	// be the same in the PV.Spec.CSI.VolumeHandle. Check the PV annotation for
 	// the new volumeHandle. If the new volumeHandle is found, generate the RBD
 	// volume structure from the new volumeHandle.
-	c := util.NewK8sClient()
+	c := k8s.NewK8sClient()
 	listOpt := metav1.ListOptions{
 		LabelSelector: PVReplicatedLabelKey,
 	}

internal/util/aws_metadata.go

@@ -22,6 +22,8 @@ import (
 	"errors"
 	"fmt"
 
+	"github.com/ceph/ceph-csi/internal/util/k8s"
+
 	"github.com/aws/aws-sdk-go/aws"
 	awsCreds "github.com/aws/aws-sdk-go/aws/credentials"
 	awsSession "github.com/aws/aws-sdk-go/aws/session"
@@ -123,7 +125,7 @@ func initAWSMetadataKMS(args KMSInitializerArgs) (EncryptionKMS, error) {
 }
 
 func (kms *AWSMetadataKMS) getSecrets() (map[string]interface{}, error) {
-	c := NewK8sClient()
+	c := k8s.NewK8sClient()
 	secret, err := c.CoreV1().Secrets(kms.namespace).Get(context.TODO(),
 		kms.secretName, metav1.GetOptions{})
 	if err != nil {

internal/util/k8s/client.go

@@ -14,20 +14,20 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package util
+package k8s
 
 import (
 	"os"
 
 	"github.com/ceph/ceph-csi/internal/util/log"
 
-	k8s "k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 )
 
 // NewK8sClient create kubernetes client.
-func NewK8sClient() *k8s.Clientset {
+func NewK8sClient() *kubernetes.Clientset {
 	var cfg *rest.Config
 	var err error
 	cPath := os.Getenv("KUBERNETES_CONFIG_PATH")
@@ -42,7 +42,7 @@ func NewK8sClient() *k8s.Clientset {
 			log.FatalLogMsg("Failed to get cluster config with error: %v\n", err)
 		}
 	}
-	client, err := k8s.NewForConfig(cfg)
+	client, err := kubernetes.NewForConfig(cfg)
 	if err != nil {
 		log.FatalLogMsg("Failed to create client with error: %v\n", err)
 	}

internal/util/kms.go

@@ -23,6 +23,8 @@ import (
 	"io/ioutil"
 	"os"
 
+	"github.com/ceph/ceph-csi/internal/util/k8s"
+
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -146,7 +148,7 @@ func getKMSConfigMap() (map[string]interface{}, error) {
 	}
 	cmName := getKMSConfigMapName()
 
-	c := NewK8sClient()
+	c := k8s.NewK8sClient()
 	cm, err := c.CoreV1().ConfigMaps(ns).Get(context.Background(),
 		cmName, metav1.GetOptions{})
 	if err != nil {

internal/util/secretskms.go

@@ -26,6 +26,8 @@ import (
 	"fmt"
 	"io"
 
+	"github.com/ceph/ceph-csi/internal/util/k8s"
+
 	"golang.org/x/crypto/scrypt"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -155,7 +157,7 @@ func (kms SecretsMetadataKMS) fetchEncryptionPassphrase(
 		secretNamespace = defaultNamespace
 	}
 
-	c := NewK8sClient()
+	c := k8s.NewK8sClient()
 	secret, err := c.CoreV1().Secrets(secretNamespace).Get(context.TODO(),
 		secretName, metav1.GetOptions{})
 	if err != nil {

internal/util/topology.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/ceph/ceph-csi/internal/util/k8s"
 	"github.com/ceph/ceph-csi/internal/util/log"
 
 	"github.com/container-storage-interface/spec/lib/go/csi"
@@ -34,7 +35,7 @@ const (
 )
 
 func k8sGetNodeLabels(nodeName string) (map[string]string, error) {
-	client := NewK8sClient()
+	client := k8s.NewK8sClient()
 	node, err := client.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{})
 	if err != nil {
 		return nil, fmt.Errorf("failed to get node %q information: %w", nodeName, err)

internal/util/vault_tokens.go

@@ -24,6 +24,8 @@ import (
 	"os"
 	"strconv"
 
+	"github.com/ceph/ceph-csi/internal/util/k8s"
+
 	"github.com/hashicorp/vault/api"
 	apierrs "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -438,7 +440,7 @@ func (vtc *vaultTenantConnection) initCertificates(config map[string]interface{}
 
 func (vtc *vaultTenantConnection) getK8sClient() *kubernetes.Clientset {
 	if vtc.client == nil {
-		vtc.client = NewK8sClient()
+		vtc.client = k8s.NewK8sClient()
 	}
 
 	return vtc.client