mirror of
https://github.com/ceph/ceph-csi.git
synced 2025-06-13 10:33:35 +00:00
vendor files
This commit is contained in:
96
vendor/github.com/kubernetes-csi/drivers/pkg/csi-common/controllerserver-default.go
generated
vendored
Normal file
96
vendor/github.com/kubernetes-csi/drivers/pkg/csi-common/controllerserver-default.go
generated
vendored
Normal file
@ -0,0 +1,96 @@
|
||||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package csicommon
|
||||
|
||||
import (
|
||||
"github.com/container-storage-interface/spec/lib/go/csi"
|
||||
"github.com/golang/glog"
|
||||
"golang.org/x/net/context"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
)
|
||||
|
||||
type DefaultControllerServer struct {
|
||||
Driver *CSIDriver
|
||||
}
|
||||
|
||||
func (cs *DefaultControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) {
|
||||
return nil, status.Error(codes.Unimplemented, "")
|
||||
}
|
||||
|
||||
func (cs *DefaultControllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) {
|
||||
return nil, status.Error(codes.Unimplemented, "")
|
||||
}
|
||||
|
||||
func (cs *DefaultControllerServer) ControllerPublishVolume(ctx context.Context, req *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error) {
|
||||
return nil, status.Error(codes.Unimplemented, "")
|
||||
}
|
||||
|
||||
func (cs *DefaultControllerServer) ControllerUnpublishVolume(ctx context.Context, req *csi.ControllerUnpublishVolumeRequest) (*csi.ControllerUnpublishVolumeResponse, error) {
|
||||
return nil, status.Error(codes.Unimplemented, "")
|
||||
}
|
||||
|
||||
func (cs *DefaultControllerServer) ValidateVolumeCapabilities(ctx context.Context, req *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error) {
|
||||
glog.V(5).Infof("Using default ValidateVolumeCapabilities")
|
||||
|
||||
for _, c := range req.GetVolumeCapabilities() {
|
||||
found := false
|
||||
for _, c1 := range cs.Driver.vc {
|
||||
if c1.GetMode() == c.GetAccessMode().GetMode() {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return &csi.ValidateVolumeCapabilitiesResponse{
|
||||
Supported: false,
|
||||
Message: "Driver doesnot support mode:" + c.GetAccessMode().GetMode().String(),
|
||||
}, status.Error(codes.InvalidArgument, "Driver doesnot support mode:"+c.GetAccessMode().GetMode().String())
|
||||
}
|
||||
// TODO: Ignoring mount & block tyeps for now.
|
||||
}
|
||||
|
||||
return &csi.ValidateVolumeCapabilitiesResponse{
|
||||
Supported: true,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (cs *DefaultControllerServer) ListVolumes(ctx context.Context, req *csi.ListVolumesRequest) (*csi.ListVolumesResponse, error) {
|
||||
return nil, status.Error(codes.Unimplemented, "")
|
||||
}
|
||||
|
||||
func (cs *DefaultControllerServer) GetCapacity(ctx context.Context, req *csi.GetCapacityRequest) (*csi.GetCapacityResponse, error) {
|
||||
return nil, status.Error(codes.Unimplemented, "")
|
||||
}
|
||||
|
||||
func (cs *DefaultControllerServer) ControllerProbe(ctx context.Context, req *csi.ControllerProbeRequest) (*csi.ControllerProbeResponse, error) {
|
||||
glog.V(5).Infof("Using default ControllerProbe")
|
||||
|
||||
if err := cs.Driver.ValidateControllerServiceRequest(req.Version, csi.ControllerServiceCapability_RPC_UNKNOWN); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &csi.ControllerProbeResponse{}, nil
|
||||
}
|
||||
|
||||
// ControllerGetCapabilities implements the default GRPC callout.
|
||||
// Default supports all capabilities
|
||||
func (cs *DefaultControllerServer) ControllerGetCapabilities(ctx context.Context, req *csi.ControllerGetCapabilitiesRequest) (*csi.ControllerGetCapabilitiesResponse, error) {
|
||||
glog.V(5).Infof("Using default ControllerGetCapabilities")
|
||||
|
||||
return &csi.ControllerGetCapabilitiesResponse{
|
||||
Capabilities: cs.Driver.cap,
|
||||
}, nil
|
||||
}
|
137
vendor/github.com/kubernetes-csi/drivers/pkg/csi-common/driver.go
generated
vendored
Normal file
137
vendor/github.com/kubernetes-csi/drivers/pkg/csi-common/driver.go
generated
vendored
Normal file
@ -0,0 +1,137 @@
|
||||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package csicommon
|
||||
|
||||
import (
|
||||
"github.com/golang/glog"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
|
||||
"github.com/container-storage-interface/spec/lib/go/csi"
|
||||
)
|
||||
|
||||
type CSIDriver struct {
|
||||
name string
|
||||
nodeID string
|
||||
version *csi.Version
|
||||
supVers []*csi.Version
|
||||
cap []*csi.ControllerServiceCapability
|
||||
vc []*csi.VolumeCapability_AccessMode
|
||||
}
|
||||
|
||||
// Creates a NewCSIDriver object. Assumes vendor version is equal to driver version &
|
||||
// does not support optional driver plugin info manifest field. Refer to CSI spec for more details.
|
||||
func NewCSIDriver(name string, v *csi.Version, supVers []*csi.Version, nodeID string) *CSIDriver {
|
||||
if name == "" {
|
||||
glog.Errorf("Driver name missing")
|
||||
return nil
|
||||
}
|
||||
|
||||
if nodeID == "" {
|
||||
glog.Errorf("NodeID missing")
|
||||
return nil
|
||||
}
|
||||
|
||||
if v == nil {
|
||||
glog.Errorf("Version argument missing")
|
||||
return nil
|
||||
}
|
||||
|
||||
found := false
|
||||
for _, sv := range supVers {
|
||||
if sv.GetMajor() == v.GetMajor() && sv.GetMinor() == v.GetMinor() && sv.GetPatch() == v.GetPatch() {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
|
||||
if !found {
|
||||
supVers = append(supVers, v)
|
||||
}
|
||||
|
||||
driver := CSIDriver{
|
||||
name: name,
|
||||
version: v,
|
||||
supVers: supVers,
|
||||
nodeID: nodeID,
|
||||
}
|
||||
|
||||
return &driver
|
||||
}
|
||||
|
||||
func (d *CSIDriver) CheckVersion(v *csi.Version) error {
|
||||
if v == nil {
|
||||
return status.Error(codes.InvalidArgument, "Version missing")
|
||||
}
|
||||
|
||||
// Assumes always backward compatible
|
||||
for _, sv := range d.supVers {
|
||||
if v.Major == sv.Major && v.Minor <= sv.Minor {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
return status.Error(codes.InvalidArgument, "Unsupported version: "+GetVersionString(v))
|
||||
}
|
||||
|
||||
func (d *CSIDriver) ValidateControllerServiceRequest(v *csi.Version, c csi.ControllerServiceCapability_RPC_Type) error {
|
||||
if v == nil {
|
||||
return status.Error(codes.InvalidArgument, "Version not specified")
|
||||
}
|
||||
|
||||
if err := d.CheckVersion(v); err != nil {
|
||||
return status.Error(codes.InvalidArgument, "Unsupported version")
|
||||
}
|
||||
|
||||
if c == csi.ControllerServiceCapability_RPC_UNKNOWN {
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, cap := range d.cap {
|
||||
if c == cap.GetRpc().GetType() {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
return status.Error(codes.InvalidArgument, "Unsupported version: "+GetVersionString(v))
|
||||
}
|
||||
|
||||
func (d *CSIDriver) AddControllerServiceCapabilities(cl []csi.ControllerServiceCapability_RPC_Type) {
|
||||
var csc []*csi.ControllerServiceCapability
|
||||
|
||||
for _, c := range cl {
|
||||
glog.Infof("Enabling controller service capability: %v", c.String())
|
||||
csc = append(csc, NewControllerServiceCapability(c))
|
||||
}
|
||||
|
||||
d.cap = csc
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (d *CSIDriver) AddVolumeCapabilityAccessModes(vc []csi.VolumeCapability_AccessMode_Mode) []*csi.VolumeCapability_AccessMode {
|
||||
var vca []*csi.VolumeCapability_AccessMode
|
||||
for _, c := range vc {
|
||||
glog.Infof("Enabling volume access mode: %v", c.String())
|
||||
vca = append(vca, NewVolumeCapabilityAccessMode(c))
|
||||
}
|
||||
d.vc = vca
|
||||
return vca
|
||||
}
|
||||
|
||||
func (d *CSIDriver) GetVolumeCapabilityAccessModes() []*csi.VolumeCapability_AccessMode {
|
||||
return d.vc
|
||||
}
|
181
vendor/github.com/kubernetes-csi/drivers/pkg/csi-common/driver_test.go
generated
vendored
Normal file
181
vendor/github.com/kubernetes-csi/drivers/pkg/csi-common/driver_test.go
generated
vendored
Normal file
@ -0,0 +1,181 @@
|
||||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package csicommon
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/container-storage-interface/spec/lib/go/csi"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
)
|
||||
|
||||
const (
|
||||
fakeDriverName = "fake"
|
||||
fakeNodeID = "fakeNodeID"
|
||||
)
|
||||
|
||||
var (
|
||||
fakeVersion = csi.Version{Major: 5, Minor: 2, Patch: 0}
|
||||
fakeVersionsSupported = []*csi.Version{
|
||||
&csi.Version{
|
||||
Major: 4, Minor: 0, Patch: 0,
|
||||
},
|
||||
&csi.Version{
|
||||
Major: 4, Minor: 1, Patch: 0,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
func NewFakeDriver() *CSIDriver {
|
||||
fakeVersion = csi.Version{Major: 5, Minor: 2, Patch: 0}
|
||||
fakeVersionsSupported = []*csi.Version{
|
||||
&csi.Version{
|
||||
Major: 4, Minor: 0, Patch: 0,
|
||||
},
|
||||
&csi.Version{
|
||||
Major: 4, Minor: 1, Patch: 0,
|
||||
},
|
||||
}
|
||||
|
||||
driver := NewCSIDriver(fakeDriverName, &fakeVersion, fakeVersionsSupported, fakeNodeID)
|
||||
|
||||
return driver
|
||||
}
|
||||
|
||||
func TestNewFakeDriver(t *testing.T) {
|
||||
// Test New fake driver with invalid arguments.
|
||||
d := NewCSIDriver("", &fakeVersion, fakeVersionsSupported, fakeNodeID)
|
||||
assert.Nil(t, d)
|
||||
}
|
||||
|
||||
func TestCheckVersion(t *testing.T) {
|
||||
|
||||
driver := NewFakeDriver()
|
||||
|
||||
// Exact version
|
||||
v := csi.Version{
|
||||
Major: 5,
|
||||
Minor: 1,
|
||||
Patch: 0,
|
||||
}
|
||||
err := driver.CheckVersion(&v)
|
||||
assert.NoError(t, err)
|
||||
|
||||
//Supported version
|
||||
v = csi.Version{
|
||||
Major: 4,
|
||||
Minor: 0,
|
||||
Patch: 0,
|
||||
}
|
||||
err = driver.CheckVersion(&v)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Unsupported version
|
||||
v = csi.Version{
|
||||
Major: 6,
|
||||
Minor: 0,
|
||||
Patch: 0,
|
||||
}
|
||||
err = driver.CheckVersion(&v)
|
||||
s, ok := status.FromError(err)
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, s.Code(), codes.InvalidArgument)
|
||||
|
||||
// Supported minor version
|
||||
v = csi.Version{
|
||||
Major: 5,
|
||||
Minor: 1,
|
||||
Patch: 0,
|
||||
}
|
||||
err = driver.CheckVersion(&v)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Unsupported minor version
|
||||
v = csi.Version{
|
||||
Major: 5,
|
||||
Minor: 3,
|
||||
Patch: 0,
|
||||
}
|
||||
err = driver.CheckVersion(&v)
|
||||
s, ok = status.FromError(err)
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, s.Code(), codes.InvalidArgument)
|
||||
}
|
||||
|
||||
func TestGetVolumeCapabilityAccessModes(t *testing.T) {
|
||||
|
||||
d := NewFakeDriver()
|
||||
|
||||
// Test no volume access modes.
|
||||
// REVISIT: Do we need to support any default access modes.
|
||||
c := d.GetVolumeCapabilityAccessModes()
|
||||
assert.Zero(t, len(c))
|
||||
|
||||
// Test driver with access modes.
|
||||
d.AddVolumeCapabilityAccessModes([]csi.VolumeCapability_AccessMode_Mode{csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER})
|
||||
modes := d.GetVolumeCapabilityAccessModes()
|
||||
assert.Equal(t, 1, len(modes))
|
||||
assert.Equal(t, modes[0].GetMode(), csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER)
|
||||
}
|
||||
|
||||
func TestValidateControllerServiceRequest(t *testing.T) {
|
||||
d := NewFakeDriver()
|
||||
|
||||
v := csi.Version{
|
||||
Major: 5,
|
||||
Minor: 0,
|
||||
Patch: 0,
|
||||
}
|
||||
|
||||
// Valid requests which require no capabilities
|
||||
err := d.ValidateControllerServiceRequest(&v, csi.ControllerServiceCapability_RPC_UNKNOWN)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Test controller service publish/unpublish not supported
|
||||
err = d.ValidateControllerServiceRequest(&v, csi.ControllerServiceCapability_RPC_PUBLISH_UNPUBLISH_VOLUME)
|
||||
s, ok := status.FromError(err)
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, s.Code(), codes.InvalidArgument)
|
||||
|
||||
// Add controller service publish & unpublish request
|
||||
d.AddControllerServiceCapabilities(
|
||||
[]csi.ControllerServiceCapability_RPC_Type{
|
||||
csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME,
|
||||
csi.ControllerServiceCapability_RPC_PUBLISH_UNPUBLISH_VOLUME,
|
||||
csi.ControllerServiceCapability_RPC_GET_CAPACITY,
|
||||
csi.ControllerServiceCapability_RPC_LIST_VOLUMES,
|
||||
})
|
||||
|
||||
// Test controller service publish/unpublish is supported
|
||||
err = d.ValidateControllerServiceRequest(&v, csi.ControllerServiceCapability_RPC_PUBLISH_UNPUBLISH_VOLUME)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Test controller service create/delete is supported
|
||||
err = d.ValidateControllerServiceRequest(&v, csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Test controller service list volumes is supported
|
||||
err = d.ValidateControllerServiceRequest(&v, csi.ControllerServiceCapability_RPC_LIST_VOLUMES)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Test controller service get capacity is supported
|
||||
err = d.ValidateControllerServiceRequest(&v, csi.ControllerServiceCapability_RPC_GET_CAPACITY)
|
||||
assert.NoError(t, err)
|
||||
|
||||
}
|
58
vendor/github.com/kubernetes-csi/drivers/pkg/csi-common/identityserver-default.go
generated
vendored
Normal file
58
vendor/github.com/kubernetes-csi/drivers/pkg/csi-common/identityserver-default.go
generated
vendored
Normal file
@ -0,0 +1,58 @@
|
||||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package csicommon
|
||||
|
||||
import (
|
||||
"github.com/container-storage-interface/spec/lib/go/csi"
|
||||
"github.com/golang/glog"
|
||||
"golang.org/x/net/context"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
)
|
||||
|
||||
type DefaultIdentityServer struct {
|
||||
Driver *CSIDriver
|
||||
}
|
||||
|
||||
//GetSupportedVersions(context.Context, *GetSupportedVersionsRequest) (*GetSupportedVersionsResponse, error)
|
||||
//GetPluginInfo(context.Context, *GetPluginInfoRequest) (*GetPluginInfoResponse, error)
|
||||
func (ids *DefaultIdentityServer) GetSupportedVersions(ctx context.Context, req *csi.GetSupportedVersionsRequest) (*csi.GetSupportedVersionsResponse, error) {
|
||||
glog.V(5).Infof("Using default GetSupportedVersions")
|
||||
return &csi.GetSupportedVersionsResponse{
|
||||
SupportedVersions: ids.Driver.supVers,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (ids *DefaultIdentityServer) GetPluginInfo(ctx context.Context, req *csi.GetPluginInfoRequest) (*csi.GetPluginInfoResponse, error) {
|
||||
glog.V(5).Infof("Using default GetPluginInnfo")
|
||||
|
||||
if ids.Driver.name == "" {
|
||||
return nil, status.Error(codes.Unavailable, "Driver name not configured")
|
||||
}
|
||||
|
||||
err := ids.Driver.CheckVersion(req.GetVersion())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
version := GetVersionString(ids.Driver.version)
|
||||
|
||||
return &csi.GetPluginInfoResponse{
|
||||
Name: ids.Driver.name,
|
||||
VendorVersion: version,
|
||||
}, nil
|
||||
}
|
68
vendor/github.com/kubernetes-csi/drivers/pkg/csi-common/identityserver-default_test.go
generated
vendored
Normal file
68
vendor/github.com/kubernetes-csi/drivers/pkg/csi-common/identityserver-default_test.go
generated
vendored
Normal file
@ -0,0 +1,68 @@
|
||||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package csicommon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/container-storage-interface/spec/lib/go/csi"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
)
|
||||
|
||||
func TestGetSupportedVersions(t *testing.T) {
|
||||
d := NewFakeDriver()
|
||||
|
||||
ids := NewDefaultIdentityServer(d)
|
||||
|
||||
req := csi.GetSupportedVersionsRequest{}
|
||||
|
||||
// Test Get supported versions are valid.
|
||||
resp, err := ids.GetSupportedVersions(context.Background(), &req)
|
||||
assert.NoError(t, err)
|
||||
|
||||
for _, fv := range fakeVersionsSupported {
|
||||
found := false
|
||||
for _, rv := range resp.GetSupportedVersions() {
|
||||
if fv.GetMajor() == rv.GetMajor() && fv.GetMinor() == rv.GetMinor() && fv.GetPatch() == rv.GetPatch() {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
assert.True(t, found)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetPluginInfo(t *testing.T) {
|
||||
d := NewFakeDriver()
|
||||
|
||||
ids := NewDefaultIdentityServer(d)
|
||||
|
||||
// Test invalid request
|
||||
req := csi.GetPluginInfoRequest{}
|
||||
resp, err := ids.GetPluginInfo(context.Background(), &req)
|
||||
s, ok := status.FromError(err)
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, s.Code(), codes.InvalidArgument)
|
||||
|
||||
// Test valid request
|
||||
req.Version = &fakeVersion
|
||||
resp, err = ids.GetPluginInfo(context.Background(), &req)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, resp.GetName(), fakeDriverName)
|
||||
}
|
80
vendor/github.com/kubernetes-csi/drivers/pkg/csi-common/nodeserver-default.go
generated
vendored
Normal file
80
vendor/github.com/kubernetes-csi/drivers/pkg/csi-common/nodeserver-default.go
generated
vendored
Normal file
@ -0,0 +1,80 @@
|
||||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package csicommon
|
||||
|
||||
import (
|
||||
"github.com/container-storage-interface/spec/lib/go/csi"
|
||||
"github.com/golang/glog"
|
||||
"golang.org/x/net/context"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
)
|
||||
|
||||
type DefaultNodeServer struct {
|
||||
Driver *CSIDriver
|
||||
}
|
||||
|
||||
func (ns *DefaultNodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) {
|
||||
err := ns.Driver.CheckVersion(req.GetVersion())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return nil, status.Error(codes.Unimplemented, "")
|
||||
}
|
||||
|
||||
func (ns *DefaultNodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) {
|
||||
err := ns.Driver.CheckVersion(req.GetVersion())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return nil, status.Error(codes.Unimplemented, "")
|
||||
}
|
||||
|
||||
func (ns *DefaultNodeServer) GetNodeID(ctx context.Context, req *csi.GetNodeIDRequest) (*csi.GetNodeIDResponse, error) {
|
||||
glog.V(5).Infof("Using default GetNodeID")
|
||||
|
||||
err := ns.Driver.CheckVersion(req.GetVersion())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &csi.GetNodeIDResponse{
|
||||
NodeId: ns.Driver.nodeID,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (ns *DefaultNodeServer) NodeProbe(ctx context.Context, req *csi.NodeProbeRequest) (*csi.NodeProbeResponse, error) {
|
||||
glog.V(5).Infof("Using default NodeProbe")
|
||||
|
||||
err := ns.Driver.CheckVersion(req.GetVersion())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &csi.NodeProbeResponse{}, nil
|
||||
}
|
||||
|
||||
func (ns *DefaultNodeServer) NodeGetCapabilities(ctx context.Context, req *csi.NodeGetCapabilitiesRequest) (*csi.NodeGetCapabilitiesResponse, error) {
|
||||
glog.V(5).Infof("Using default NodeGetCapabilities")
|
||||
|
||||
err := ns.Driver.CheckVersion(req.GetVersion())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &csi.NodeGetCapabilitiesResponse{}, nil
|
||||
}
|
122
vendor/github.com/kubernetes-csi/drivers/pkg/csi-common/nodeserver-default_test.go
generated
vendored
Normal file
122
vendor/github.com/kubernetes-csi/drivers/pkg/csi-common/nodeserver-default_test.go
generated
vendored
Normal file
@ -0,0 +1,122 @@
|
||||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package csicommon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/container-storage-interface/spec/lib/go/csi"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
)
|
||||
|
||||
func TestGetNodeID(t *testing.T) {
|
||||
d := NewFakeDriver()
|
||||
|
||||
ns := NewDefaultNodeServer(d)
|
||||
|
||||
// Test invalid request
|
||||
req := csi.GetNodeIDRequest{}
|
||||
_, err := ns.GetNodeID(context.Background(), &req)
|
||||
s, ok := status.FromError(err)
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, s.Code(), codes.InvalidArgument)
|
||||
|
||||
// Test valid request
|
||||
req.Version = &fakeVersion
|
||||
resp, err := ns.GetNodeID(context.Background(), &req)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, resp.GetNodeId(), fakeNodeID)
|
||||
}
|
||||
|
||||
func TestNodeGetCapabilities(t *testing.T) {
|
||||
d := NewFakeDriver()
|
||||
|
||||
ns := NewDefaultNodeServer(d)
|
||||
|
||||
// Test invalid request
|
||||
req := csi.NodeGetCapabilitiesRequest{}
|
||||
_, err := ns.NodeGetCapabilities(context.Background(), &req)
|
||||
s, ok := status.FromError(err)
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, s.Code(), codes.InvalidArgument)
|
||||
|
||||
// Test valid request
|
||||
req.Version = &fakeVersion
|
||||
_, err = ns.NodeGetCapabilities(context.Background(), &req)
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestNodeProbe(t *testing.T) {
|
||||
d := NewFakeDriver()
|
||||
|
||||
ns := NewDefaultNodeServer(d)
|
||||
|
||||
// Test invalid request
|
||||
req := csi.NodeProbeRequest{}
|
||||
_, err := ns.NodeProbe(context.Background(), &req)
|
||||
s, ok := status.FromError(err)
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, s.Code(), codes.InvalidArgument)
|
||||
|
||||
// Test valid request
|
||||
req.Version = &fakeVersion
|
||||
_, err = ns.NodeProbe(context.Background(), &req)
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestNodePublishVolume(t *testing.T) {
|
||||
d := NewFakeDriver()
|
||||
|
||||
ns := NewDefaultNodeServer(d)
|
||||
|
||||
// Test invalid request
|
||||
req := csi.NodePublishVolumeRequest{}
|
||||
_, err := ns.NodePublishVolume(context.Background(), &req)
|
||||
s, ok := status.FromError(err)
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, s.Code(), codes.InvalidArgument)
|
||||
|
||||
// Test valid node publish request
|
||||
req.Version = &fakeVersion
|
||||
_, err = ns.NodePublishVolume(context.Background(), &req)
|
||||
s, ok = status.FromError(err)
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, s.Code(), codes.Unimplemented)
|
||||
}
|
||||
|
||||
func TestNodeUnpublishVolume(t *testing.T) {
|
||||
d := NewFakeDriver()
|
||||
|
||||
ns := NewDefaultNodeServer(d)
|
||||
|
||||
// Test invalid request
|
||||
req := csi.NodeUnpublishVolumeRequest{}
|
||||
_, err := ns.NodeUnpublishVolume(context.Background(), &req)
|
||||
s, ok := status.FromError(err)
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, s.Code(), codes.InvalidArgument)
|
||||
|
||||
// Test valid node publish request
|
||||
req.Version = &fakeVersion
|
||||
_, err = ns.NodeUnpublishVolume(context.Background(), &req)
|
||||
s, ok = status.FromError(err)
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, s.Code(), codes.Unimplemented)
|
||||
}
|
112
vendor/github.com/kubernetes-csi/drivers/pkg/csi-common/server.go
generated
vendored
Normal file
112
vendor/github.com/kubernetes-csi/drivers/pkg/csi-common/server.go
generated
vendored
Normal file
@ -0,0 +1,112 @@
|
||||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package csicommon
|
||||
|
||||
import (
|
||||
"net"
|
||||
"os"
|
||||
"sync"
|
||||
|
||||
"github.com/golang/glog"
|
||||
"google.golang.org/grpc"
|
||||
|
||||
"github.com/container-storage-interface/spec/lib/go/csi"
|
||||
)
|
||||
|
||||
// Defines Non blocking GRPC server interfaces
|
||||
type NonBlockingGRPCServer interface {
|
||||
// Start services at the endpoint
|
||||
Start(endpoint string, ids csi.IdentityServer, cs csi.ControllerServer, ns csi.NodeServer)
|
||||
// Waits for the service to stop
|
||||
Wait()
|
||||
// Stops the service gracefully
|
||||
Stop()
|
||||
// Stops the service forcefully
|
||||
ForceStop()
|
||||
}
|
||||
|
||||
func NewNonBlockingGRPCServer() NonBlockingGRPCServer {
|
||||
return &nonBlockingGRPCServer{}
|
||||
}
|
||||
|
||||
// NonBlocking server
|
||||
type nonBlockingGRPCServer struct {
|
||||
wg sync.WaitGroup
|
||||
server *grpc.Server
|
||||
}
|
||||
|
||||
func (s *nonBlockingGRPCServer) Start(endpoint string, ids csi.IdentityServer, cs csi.ControllerServer, ns csi.NodeServer) {
|
||||
|
||||
s.wg.Add(1)
|
||||
|
||||
go s.serve(endpoint, ids, cs, ns)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (s *nonBlockingGRPCServer) Wait() {
|
||||
s.wg.Wait()
|
||||
}
|
||||
|
||||
func (s *nonBlockingGRPCServer) Stop() {
|
||||
s.server.GracefulStop()
|
||||
}
|
||||
|
||||
func (s *nonBlockingGRPCServer) ForceStop() {
|
||||
s.server.Stop()
|
||||
}
|
||||
|
||||
func (s *nonBlockingGRPCServer) serve(endpoint string, ids csi.IdentityServer, cs csi.ControllerServer, ns csi.NodeServer) {
|
||||
|
||||
proto, addr, err := ParseEndpoint(endpoint)
|
||||
if err != nil {
|
||||
glog.Fatal(err.Error())
|
||||
}
|
||||
|
||||
if proto == "unix" {
|
||||
addr = "/" + addr
|
||||
if err := os.Remove(addr); err != nil && !os.IsNotExist(err) {
|
||||
glog.Fatalf("Failed to remove %s, error: %s", addr, err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
listener, err := net.Listen(proto, addr)
|
||||
if err != nil {
|
||||
glog.Fatalf("Failed to listen: %v", err)
|
||||
}
|
||||
|
||||
opts := []grpc.ServerOption{
|
||||
grpc.UnaryInterceptor(logGRPC),
|
||||
}
|
||||
server := grpc.NewServer(opts...)
|
||||
s.server = server
|
||||
|
||||
if ids != nil {
|
||||
csi.RegisterIdentityServer(server, ids)
|
||||
}
|
||||
if cs != nil {
|
||||
csi.RegisterControllerServer(server, cs)
|
||||
}
|
||||
if ns != nil {
|
||||
csi.RegisterNodeServer(server, ns)
|
||||
}
|
||||
|
||||
glog.Infof("Listening for connections on address: %#v", listener.Addr())
|
||||
|
||||
server.Serve(listener)
|
||||
|
||||
}
|
127
vendor/github.com/kubernetes-csi/drivers/pkg/csi-common/utils.go
generated
vendored
Normal file
127
vendor/github.com/kubernetes-csi/drivers/pkg/csi-common/utils.go
generated
vendored
Normal file
@ -0,0 +1,127 @@
|
||||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package csicommon
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/container-storage-interface/spec/lib/go/csi"
|
||||
"github.com/golang/glog"
|
||||
"golang.org/x/net/context"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
func ParseEndpoint(ep string) (string, string, error) {
|
||||
if strings.HasPrefix(strings.ToLower(ep), "unix://") || strings.HasPrefix(strings.ToLower(ep), "tcp://") {
|
||||
s := strings.SplitN(ep, "://", 2)
|
||||
if s[1] != "" {
|
||||
return s[0], s[1], nil
|
||||
}
|
||||
}
|
||||
return "", "", fmt.Errorf("Invalid endpoint: %v", ep)
|
||||
}
|
||||
|
||||
func GetVersionString(v *csi.Version) string {
|
||||
return fmt.Sprintf("%d.%d.%d", v.Major, v.Minor, v.Patch)
|
||||
}
|
||||
|
||||
func GetVersionFromString(v string) (*csi.Version, error) {
|
||||
var major, minor, patch uint32
|
||||
|
||||
n, err := fmt.Sscanf(v, "%d.%d.%d", &major, &minor, &patch)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if n != 3 {
|
||||
return nil, fmt.Errorf("Invalid format. Specify version in x.y.z format")
|
||||
}
|
||||
|
||||
return &csi.Version{
|
||||
Major: major,
|
||||
Minor: minor,
|
||||
Patch: patch,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func NewVolumeCapabilityAccessMode(mode csi.VolumeCapability_AccessMode_Mode) *csi.VolumeCapability_AccessMode {
|
||||
return &csi.VolumeCapability_AccessMode{Mode: mode}
|
||||
}
|
||||
|
||||
func NewDefaultNodeServer(d *CSIDriver) *DefaultNodeServer {
|
||||
return &DefaultNodeServer{
|
||||
Driver: d,
|
||||
}
|
||||
}
|
||||
|
||||
func NewDefaultIdentityServer(d *CSIDriver) *DefaultIdentityServer {
|
||||
return &DefaultIdentityServer{
|
||||
Driver: d,
|
||||
}
|
||||
}
|
||||
|
||||
func NewDefaultControllerServer(d *CSIDriver) *DefaultControllerServer {
|
||||
return &DefaultControllerServer{
|
||||
Driver: d,
|
||||
}
|
||||
}
|
||||
|
||||
func NewControllerServiceCapability(cap csi.ControllerServiceCapability_RPC_Type) *csi.ControllerServiceCapability {
|
||||
return &csi.ControllerServiceCapability{
|
||||
Type: &csi.ControllerServiceCapability_Rpc{
|
||||
Rpc: &csi.ControllerServiceCapability_RPC{
|
||||
Type: cap,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func RunNodePublishServer(endpoint string, d *CSIDriver, ns csi.NodeServer) {
|
||||
ids := NewDefaultIdentityServer(d)
|
||||
|
||||
s := NewNonBlockingGRPCServer()
|
||||
s.Start(endpoint, ids, nil, ns)
|
||||
s.Wait()
|
||||
}
|
||||
|
||||
func RunControllerPublishServer(endpoint string, d *CSIDriver, cs csi.ControllerServer) {
|
||||
ids := NewDefaultIdentityServer(d)
|
||||
|
||||
s := NewNonBlockingGRPCServer()
|
||||
s.Start(endpoint, ids, cs, nil)
|
||||
s.Wait()
|
||||
}
|
||||
|
||||
func RunControllerandNodePublishServer(endpoint string, d *CSIDriver, cs csi.ControllerServer, ns csi.NodeServer) {
|
||||
ids := NewDefaultIdentityServer(d)
|
||||
|
||||
s := NewNonBlockingGRPCServer()
|
||||
s.Start(endpoint, ids, cs, ns)
|
||||
s.Wait()
|
||||
}
|
||||
|
||||
func logGRPC(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
|
||||
glog.V(3).Infof("GRPC call: %s", info.FullMethod)
|
||||
glog.V(5).Infof("GRPC request: %+v", req)
|
||||
resp, err := handler(ctx, req)
|
||||
if err != nil {
|
||||
glog.Errorf("GRPC error: %v", err)
|
||||
} else {
|
||||
glog.V(5).Infof("GRPC response: %+v", resp)
|
||||
}
|
||||
return resp, err
|
||||
}
|
106
vendor/github.com/kubernetes-csi/drivers/pkg/csi-common/utils_test.go
generated
vendored
Normal file
106
vendor/github.com/kubernetes-csi/drivers/pkg/csi-common/utils_test.go
generated
vendored
Normal file
@ -0,0 +1,106 @@
|
||||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package csicommon
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/container-storage-interface/spec/lib/go/csi"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestGetVersionFromString(t *testing.T) {
|
||||
|
||||
//Invalid version
|
||||
_, err := GetVersionFromString("")
|
||||
assert.NotNil(t, err)
|
||||
|
||||
v, err := GetVersionFromString("1.2.3")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, v.GetMajor(), uint32(1))
|
||||
assert.Equal(t, v.GetMinor(), uint32(2))
|
||||
assert.Equal(t, v.GetPatch(), uint32(3))
|
||||
|
||||
// Invalid version
|
||||
_, err = GetVersionFromString("1.2")
|
||||
assert.NotNil(t, err)
|
||||
}
|
||||
|
||||
func TestGetVersionString(t *testing.T) {
|
||||
|
||||
v := &csi.Version{
|
||||
Major: 1,
|
||||
Minor: 0,
|
||||
Patch: 0,
|
||||
}
|
||||
//Invalid version
|
||||
vStr := GetVersionString(v)
|
||||
assert.Equal(t, vStr, "1.0.0")
|
||||
}
|
||||
|
||||
func TestParseEndpoint(t *testing.T) {
|
||||
|
||||
//Valid unix domain socket endpoint
|
||||
sockType, addr, err := ParseEndpoint("unix://fake.sock")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, sockType, "unix")
|
||||
assert.Equal(t, addr, "fake.sock")
|
||||
|
||||
sockType, addr, err = ParseEndpoint("unix:///fakedir/fakedir/fake.sock")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, sockType, "unix")
|
||||
assert.Equal(t, addr, "/fakedir/fakedir/fake.sock")
|
||||
|
||||
//Valid unix domain socket with uppercase
|
||||
sockType, addr, err = ParseEndpoint("UNIX://fake.sock")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, sockType, "UNIX")
|
||||
assert.Equal(t, addr, "fake.sock")
|
||||
|
||||
//Valid TCP endpoint with ip
|
||||
sockType, addr, err = ParseEndpoint("tcp://127.0.0.1:80")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, sockType, "tcp")
|
||||
assert.Equal(t, addr, "127.0.0.1:80")
|
||||
|
||||
//Valid TCP endpoint with uppercase
|
||||
sockType, addr, err = ParseEndpoint("TCP://127.0.0.1:80")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, sockType, "TCP")
|
||||
assert.Equal(t, addr, "127.0.0.1:80")
|
||||
|
||||
//Valid TCP endpoint with hostname
|
||||
sockType, addr, err = ParseEndpoint("tcp://fakehost:80")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, sockType, "tcp")
|
||||
assert.Equal(t, addr, "fakehost:80")
|
||||
|
||||
_, _, err = ParseEndpoint("unix:/fake.sock/")
|
||||
assert.NotNil(t, err)
|
||||
|
||||
_, _, err = ParseEndpoint("fake.sock")
|
||||
assert.NotNil(t, err)
|
||||
|
||||
_, _, err = ParseEndpoint("unix://")
|
||||
assert.NotNil(t, err)
|
||||
|
||||
_, _, err = ParseEndpoint("://")
|
||||
assert.NotNil(t, err)
|
||||
|
||||
_, _, err = ParseEndpoint("")
|
||||
assert.NotNil(t, err)
|
||||
}
|
Reference in New Issue
Block a user