ceph-csi/pkg/cephfs/util.go
ShyamsundarR c4a3675cec Move locks to more granular locking than CPU count based
As detailed in issue #279, current lock scheme has hash
buckets that are count of CPUs. This causes a lot of contention
when parallel requests are made to the CSI plugin. To reduce
lock contention, this commit introduces granular locks per
identifier.

The commit also changes the timeout for gRPC requests to Create
and Delete volumes, as the current timeout is 10s (kubernetes
documentation says 15s but code defaults are 10s). A virtual
setup takes about 12-15s to complete a request at times, that leads
to unwanted retries of the same request, hence the increased
timeout to enable operation completion with minimal retries.

Tests to create PVCs before and after these changes look like so,

Before:
Default master code + sidecar provisioner --timeout option set
to 30 seconds

20 PVCs
Creation: 3 runs, 396/391/400 seconds
Deletion: 3 runs, 218/271/118 seconds
  - Once was stalled for more than 8 minutes and cancelled the run

After:
Current commit + sidecar provisioner --timeout option set to 30 sec
20 PVCs
Creation: 3 runs, 42/59/65 seconds
Deletion: 3 runs, 32/32/31 seconds

Fixes: #279
Signed-off-by: ShyamsundarR <srangana@redhat.com>
2019-07-01 14:10:14 +00:00

191 lines
4.8 KiB
Go

/*
Copyright 2018 The Ceph-CSI Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cephfs
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"os"
"os/exec"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"k8s.io/klog"
"github.com/ceph/ceph-csi/pkg/util"
"github.com/container-storage-interface/spec/lib/go/csi"
"k8s.io/kubernetes/pkg/util/mount"
)
type volumeID string
func execCommand(program string, args ...string) (stdout, stderr []byte, err error) {
var (
cmd = exec.Command(program, args...) // nolint: gosec
sanitizedArgs = util.StripSecretInArgs(args)
stdoutBuf bytes.Buffer
stderrBuf bytes.Buffer
)
cmd.Stdout = &stdoutBuf
cmd.Stderr = &stderrBuf
klog.V(4).Infof("cephfs: EXEC %s %s", program, sanitizedArgs)
if err := cmd.Run(); err != nil {
return nil, nil, fmt.Errorf("an error occurred while running (%d) %s %v: %v: %s",
cmd.Process.Pid, program, sanitizedArgs, err, stderrBuf.Bytes())
}
return stdoutBuf.Bytes(), stderrBuf.Bytes(), nil
}
func execCommandErr(program string, args ...string) error {
_, _, err := execCommand(program, args...)
return err
}
//nolint: unparam
func execCommandJSON(v interface{}, program string, args ...string) error {
stdout, _, err := execCommand(program, args...)
if err != nil {
return err
}
if err = json.Unmarshal(stdout, v); err != nil {
return fmt.Errorf("failed to unmarshal JSON for %s %v: %s: %v", program, util.StripSecretInArgs(args), stdout, err)
}
return nil
}
// Used in isMountPoint()
var dummyMount = mount.New("")
func isMountPoint(p string) (bool, error) {
notMnt, err := dummyMount.IsLikelyNotMountPoint(p)
if err != nil {
return false, status.Error(codes.Internal, err.Error())
}
return !notMnt, nil
}
func pathExists(p string) bool {
_, err := os.Stat(p)
return err == nil
}
// Controller service request validation
func (cs *ControllerServer) validateCreateVolumeRequest(req *csi.CreateVolumeRequest) error {
if err := cs.Driver.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME); err != nil {
return fmt.Errorf("invalid CreateVolumeRequest: %v", err)
}
if req.GetName() == "" {
return status.Error(codes.InvalidArgument, "volume Name cannot be empty")
}
reqCaps := req.GetVolumeCapabilities()
if reqCaps == nil {
return status.Error(codes.InvalidArgument, "volume Capabilities cannot be empty")
}
for _, cap := range reqCaps {
if cap.GetBlock() != nil {
return status.Error(codes.Unimplemented, "block volume not supported")
}
}
return nil
}
func (cs *ControllerServer) validateDeleteVolumeRequest() error {
if err := cs.Driver.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME); err != nil {
return fmt.Errorf("invalid DeleteVolumeRequest: %v", err)
}
return nil
}
// Node service request validation
func validateNodeStageVolumeRequest(req *csi.NodeStageVolumeRequest) error {
if req.GetVolumeCapability() == nil {
return errors.New("volume capability missing in request")
}
if req.GetVolumeId() == "" {
return errors.New("volume ID missing in request")
}
if req.GetStagingTargetPath() == "" {
return errors.New("staging target path missing in request")
}
if req.GetSecrets() == nil || len(req.GetSecrets()) == 0 {
return errors.New("stage secrets cannot be nil or empty")
}
return nil
}
func validateNodeUnstageVolumeRequest(req *csi.NodeUnstageVolumeRequest) error {
if req.GetVolumeId() == "" {
return errors.New("volume ID missing in request")
}
if req.GetStagingTargetPath() == "" {
return errors.New("staging target path missing in request")
}
return nil
}
func validateNodePublishVolumeRequest(req *csi.NodePublishVolumeRequest) error {
if req.GetVolumeCapability() == nil {
return errors.New("volume capability missing in request")
}
if req.GetVolumeId() == "" {
return errors.New("volume ID missing in request")
}
if req.GetTargetPath() == "" {
return errors.New("target path missing in request")
}
if req.GetStagingTargetPath() == "" {
return errors.New("staging target path missing in request")
}
return nil
}
func validateNodeUnpublishVolumeRequest(req *csi.NodeUnpublishVolumeRequest) error {
if req.GetVolumeId() == "" {
return errors.New("volume ID missing in request")
}
if req.GetTargetPath() == "" {
return errors.New("target path missing in request")
}
return nil
}