build: move e2e dependencies into e2e/go.mod

Several packages are only used while running the e2e suite. These
packages are less important to update, as the they can not influence the
final executable that is part of the Ceph-CSI container-image.

By moving these dependencies out of the main Ceph-CSI go.mod, it is
easier to identify if a reported CVE affects Ceph-CSI, or only the
testing (like most of the Kubernetes CVEs).

Signed-off-by: Niels de Vos <ndevos@ibm.com>

e2e/vendor/github.com/google/cadvisor/devicemapper/dmsetup_client.go

@@ -0,0 +1,64 @@
+// Copyright 2016 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package devicemapper
+
+import (
+	"os/exec"
+	"strconv"
+	"strings"
+
+	"k8s.io/klog/v2"
+)
+
+// DmsetupClient is a low-level client for interacting with device mapper via
+// the `dmsetup` utility, which is provided by the `device-mapper` package.
+type DmsetupClient interface {
+	// Table runs `dmsetup table` on the given device name and returns the
+	// output or an error.
+	Table(deviceName string) ([]byte, error)
+	// Message runs `dmsetup message` on the given device, passing the given
+	// message to the given sector, and returns the output or an error.
+	Message(deviceName string, sector int, message string) ([]byte, error)
+	// Status runs `dmsetup status` on the given device and returns the output
+	// or an error.
+	Status(deviceName string) ([]byte, error)
+}
+
+// NewDmSetupClient returns a new DmsetupClient.
+func NewDmsetupClient() DmsetupClient {
+	return &defaultDmsetupClient{}
+}
+
+// defaultDmsetupClient is a functional DmsetupClient
+type defaultDmsetupClient struct{}
+
+var _ DmsetupClient = &defaultDmsetupClient{}
+
+func (c *defaultDmsetupClient) Table(deviceName string) ([]byte, error) {
+	return c.dmsetup("table", deviceName)
+}
+
+func (c *defaultDmsetupClient) Message(deviceName string, sector int, message string) ([]byte, error) {
+	return c.dmsetup("message", deviceName, strconv.Itoa(sector), message)
+}
+
+func (c *defaultDmsetupClient) Status(deviceName string) ([]byte, error) {
+	return c.dmsetup("status", deviceName)
+}
+
+func (*defaultDmsetupClient) dmsetup(args ...string) ([]byte, error) {
+	klog.V(5).Infof("running dmsetup %v", strings.Join(args, " "))
+	return exec.Command("dmsetup", args...).Output()
+}

e2e/vendor/github.com/google/cadvisor/devicemapper/doc.go

@@ -0,0 +1,16 @@
+// Copyright 2016 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package devicemapper contains code for working with devicemapper
+package devicemapper

e2e/vendor/github.com/google/cadvisor/devicemapper/thin_ls_client.go

@@ -0,0 +1,93 @@
+// Copyright 2016 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package devicemapper
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"os/exec"
+	"strconv"
+	"strings"
+
+	"k8s.io/klog/v2"
+)
+
+// thinLsClient knows how to run a thin_ls very specific to CoW usage for
+// containers.
+type thinLsClient interface {
+	// ThinLs runs a thin ls on the given device, which is expected to be a
+	// metadata device. The caller must hold the metadata snapshot for the
+	// device.
+	ThinLs(deviceName string) (map[string]uint64, error)
+}
+
+// newThinLsClient returns a thinLsClient or an error if the thin_ls binary
+// couldn't be located.
+func newThinLsClient() (thinLsClient, error) {
+	thinLsPath, err := ThinLsBinaryPresent()
+	if err != nil {
+		return nil, fmt.Errorf("error creating thin_ls client: %v", err)
+	}
+
+	return &defaultThinLsClient{thinLsPath}, nil
+}
+
+// defaultThinLsClient is a functional thinLsClient
+type defaultThinLsClient struct {
+	thinLsPath string
+}
+
+var _ thinLsClient = &defaultThinLsClient{}
+
+func (c *defaultThinLsClient) ThinLs(deviceName string) (map[string]uint64, error) {
+	args := []string{"--no-headers", "-m", "-o", "DEV,EXCLUSIVE_BYTES", deviceName}
+	klog.V(4).Infof("running command: thin_ls %v", strings.Join(args, " "))
+
+	output, err := exec.Command(c.thinLsPath, args...).Output()
+	if err != nil {
+		return nil, fmt.Errorf("Error running command `thin_ls %v`: %v\noutput:\n\n%v", strings.Join(args, " "), err, string(output))
+	}
+
+	return parseThinLsOutput(output), nil
+}
+
+// parseThinLsOutput parses the output returned by thin_ls to build a map of
+// device id -> usage.
+func parseThinLsOutput(output []byte) map[string]uint64 {
+	cache := map[string]uint64{}
+
+	// parse output
+	scanner := bufio.NewScanner(bytes.NewReader(output))
+	for scanner.Scan() {
+		output := scanner.Text()
+		fields := strings.Fields(output)
+		if len(fields) != 2 {
+			continue
+		}
+
+		deviceID := fields[0]
+		usage, err := strconv.ParseUint(fields[1], 10, 64)
+		if err != nil {
+			klog.Warningf("unexpected error parsing thin_ls output: %v", err)
+			continue
+		}
+
+		cache[deviceID] = usage
+	}
+
+	return cache
+
+}

e2e/vendor/github.com/google/cadvisor/devicemapper/thin_pool_watcher.go

@@ -0,0 +1,179 @@
+// Copyright 2016 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package devicemapper
+
+import (
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"k8s.io/klog/v2"
+)
+
+// ThinPoolWatcher maintains a cache of device name -> usage stats for a
+// devicemapper thin-pool using thin_ls.
+type ThinPoolWatcher struct {
+	poolName       string
+	metadataDevice string
+	lock           *sync.RWMutex
+	cache          map[string]uint64
+	period         time.Duration
+	stopChan       chan struct{}
+	dmsetup        DmsetupClient
+	thinLsClient   thinLsClient
+}
+
+// NewThinPoolWatcher returns a new ThinPoolWatcher for the given devicemapper
+// thin pool name and metadata device or an error.
+func NewThinPoolWatcher(poolName, metadataDevice string) (*ThinPoolWatcher, error) {
+	thinLsClient, err := newThinLsClient()
+	if err != nil {
+		return nil, fmt.Errorf("encountered error creating thin_ls client: %v", err)
+	}
+
+	return &ThinPoolWatcher{poolName: poolName,
+		metadataDevice: metadataDevice,
+		lock:           &sync.RWMutex{},
+		cache:          make(map[string]uint64),
+		period:         15 * time.Second,
+		stopChan:       make(chan struct{}),
+		dmsetup:        NewDmsetupClient(),
+		thinLsClient:   thinLsClient,
+	}, nil
+}
+
+// Start starts the ThinPoolWatcher.
+func (w *ThinPoolWatcher) Start() {
+	err := w.Refresh()
+	if err != nil {
+		klog.Errorf("encountered error refreshing thin pool watcher: %v", err)
+	}
+
+	for {
+		select {
+		case <-w.stopChan:
+			return
+		case <-time.After(w.period):
+			start := time.Now()
+			err = w.Refresh()
+			if err != nil {
+				klog.Errorf("encountered error refreshing thin pool watcher: %v", err)
+			}
+
+			// print latency for refresh
+			duration := time.Since(start)
+			klog.V(5).Infof("thin_ls(%d) took %s", start.Unix(), duration)
+		}
+	}
+}
+
+// Stop stops the ThinPoolWatcher.
+func (w *ThinPoolWatcher) Stop() {
+	close(w.stopChan)
+}
+
+// GetUsage gets the cached usage value of the given device.
+func (w *ThinPoolWatcher) GetUsage(deviceID string) (uint64, error) {
+	w.lock.RLock()
+	defer w.lock.RUnlock()
+
+	v, ok := w.cache[deviceID]
+	if !ok {
+		return 0, fmt.Errorf("no cached value for usage of device %v", deviceID)
+	}
+
+	return v, nil
+}
+
+const (
+	reserveMetadataMessage = "reserve_metadata_snap"
+	releaseMetadataMessage = "release_metadata_snap"
+)
+
+// Refresh performs a `thin_ls` of the pool being watched and refreshes the
+// cached data with the result.
+func (w *ThinPoolWatcher) Refresh() error {
+	w.lock.Lock()
+	defer w.lock.Unlock()
+
+	currentlyReserved, err := w.checkReservation(w.poolName)
+	if err != nil {
+		err = fmt.Errorf("error determining whether snapshot is reserved: %v", err)
+		return err
+	}
+
+	if currentlyReserved {
+		klog.V(5).Infof("metadata for %v is currently reserved; releasing", w.poolName)
+		_, err = w.dmsetup.Message(w.poolName, 0, releaseMetadataMessage)
+		if err != nil {
+			err = fmt.Errorf("error releasing metadata snapshot for %v: %v", w.poolName, err)
+			return err
+		}
+	}
+
+	klog.V(5).Infof("reserving metadata snapshot for thin-pool %v", w.poolName)
+	// NOTE: "0" in the call below is for the 'sector' argument to 'dmsetup
+	// message'.  It's not needed for thin pools.
+	if output, err := w.dmsetup.Message(w.poolName, 0, reserveMetadataMessage); err != nil {
+		err = fmt.Errorf("error reserving metadata for thin-pool %v: %v output: %v", w.poolName, err, string(output))
+		return err
+	}
+	klog.V(5).Infof("reserved metadata snapshot for thin-pool %v", w.poolName)
+
+	defer func() {
+		klog.V(5).Infof("releasing metadata snapshot for thin-pool %v", w.poolName)
+		_, err := w.dmsetup.Message(w.poolName, 0, releaseMetadataMessage)
+		if err != nil {
+			klog.Warningf("Unable to release metadata snapshot for thin-pool %v: %s", w.poolName, err)
+		}
+	}()
+
+	klog.V(5).Infof("running thin_ls on metadata device %v", w.metadataDevice)
+	newCache, err := w.thinLsClient.ThinLs(w.metadataDevice)
+	if err != nil {
+		err = fmt.Errorf("error performing thin_ls on metadata device %v: %v", w.metadataDevice, err)
+		return err
+	}
+
+	w.cache = newCache
+	return nil
+}
+
+const (
+	thinPoolDmsetupStatusHeldMetadataRoot = 6
+	thinPoolDmsetupStatusMinFields        = thinPoolDmsetupStatusHeldMetadataRoot + 1
+)
+
+// checkReservation checks to see whether the thin device is currently holding
+// userspace metadata.
+func (w *ThinPoolWatcher) checkReservation(poolName string) (bool, error) {
+	klog.V(5).Infof("checking whether the thin-pool is holding a metadata snapshot")
+	output, err := w.dmsetup.Status(poolName)
+	if err != nil {
+		return false, err
+	}
+
+	// we care about the field at fields[thinPoolDmsetupStatusHeldMetadataRoot],
+	// so make sure we get enough fields
+	fields := strings.Fields(string(output))
+	if len(fields) < thinPoolDmsetupStatusMinFields {
+		return false, fmt.Errorf("unexpected output of dmsetup status command; expected at least %d fields, got %v; output: %v", thinPoolDmsetupStatusMinFields, len(fields), string(output))
+	}
+
+	heldMetadataRoot := fields[thinPoolDmsetupStatusHeldMetadataRoot]
+	currentlyReserved := heldMetadataRoot != "-"
+	return currentlyReserved, nil
+}

e2e/vendor/github.com/google/cadvisor/devicemapper/util.go

@@ -0,0 +1,50 @@
+// Copyright 2016 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package devicemapper
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+)
+
+// ThinLsBinaryPresent returns the location of the thin_ls binary in the mount
+// namespace cadvisor is running in or an error.  The locations checked are:
+//
+// - /sbin/
+// - /bin/
+// - /usr/sbin/
+// - /usr/bin/
+//
+// The thin_ls binary is provided by the device-mapper-persistent-data
+// package.
+func ThinLsBinaryPresent() (string, error) {
+	var (
+		thinLsPath string
+		err        error
+	)
+
+	for _, path := range []string{"/sbin", "/bin", "/usr/sbin/", "/usr/bin"} {
+		// try paths for non-containerized operation
+		// note: thin_ls is most likely a symlink to pdata_tools
+		thinLsPath = filepath.Join(path, "thin_ls")
+		_, err = os.Stat(thinLsPath)
+		if err == nil {
+			return thinLsPath, nil
+		}
+	}
+
+	return "", fmt.Errorf("unable to find thin_ls binary")
+}