vendor updates

This commit is contained in:
Serguei Bezverkhi
2018-03-06 17:33:18 -05:00
parent 4b3ebc171b
commit e9033989a0
5854 changed files with 248382 additions and 119809 deletions

View File

@ -2,6 +2,7 @@ package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)
@ -10,11 +11,13 @@ go_test(
size = "large",
srcs = [
"crd_test.go",
"kms_transformation_test.go",
"kube_apiserver_test.go",
"main_test.go",
"secrets_transformation_test.go",
"synthetic_master_test.go",
],
importpath = "k8s.io/kubernetes/test/integration/master",
embed = [":go_default_library"],
tags = ["integration"],
deps = [
"//cmd/kube-apiserver/app/testing:go_default_library",
@ -29,6 +32,7 @@ go_test(
"//vendor/k8s.io/api/apps/v1beta1:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/api/networking/v1:go_default_library",
"//vendor/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions:go_default_library",
"//vendor/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library",
"//vendor/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
@ -42,6 +46,10 @@ go_test(
"//vendor/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library",
"//vendor/k8s.io/apiserver/pkg/authorization/authorizerfactory:go_default_library",
"//vendor/k8s.io/apiserver/pkg/features:go_default_library",
"//vendor/k8s.io/apiserver/pkg/server/options/encryptionconfig:go_default_library",
"//vendor/k8s.io/apiserver/pkg/storage/value:go_default_library",
"//vendor/k8s.io/apiserver/pkg/storage/value/encrypt/aes:go_default_library",
"//vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
"//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
"//vendor/k8s.io/apiserver/pkg/util/feature/testing:go_default_library",
"//vendor/k8s.io/apiserver/plugin/pkg/authenticator/token/tokentest:go_default_library",
@ -49,6 +57,7 @@ go_test(
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
"//vendor/k8s.io/client-go/rest:go_default_library",
"//vendor/k8s.io/kube-aggregator/pkg/apis/apiregistration:go_default_library",
],
)
@ -64,3 +73,28 @@ filegroup(
srcs = [":package-srcs"],
tags = ["automanaged"],
)
go_library(
name = "go_default_library",
srcs = [
"kms_plugin_mock.go",
"transformation_testcase.go",
],
importpath = "k8s.io/kubernetes/test/integration/master",
deps = [
"//cmd/kube-apiserver/app/testing:go_default_library",
"//test/integration:go_default_library",
"//test/integration/framework:go_default_library",
"//vendor/github.com/coreos/etcd/clientv3:go_default_library",
"//vendor/github.com/ghodss/yaml:go_default_library",
"//vendor/golang.org/x/sys/unix:go_default_library",
"//vendor/google.golang.org/grpc:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apiserver/pkg/server/options/encryptionconfig:go_default_library",
"//vendor/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library",
"//vendor/k8s.io/apiserver/pkg/storage/value:go_default_library",
"//vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
],
)

View File

@ -0,0 +1,111 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package master
import (
"context"
"encoding/base64"
"fmt"
"net"
"os"
"golang.org/x/sys/unix"
"google.golang.org/grpc"
kmsapi "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1"
)
const (
kmsAPIVersion = "v1beta1"
sockFile = "/tmp/kms-provider.sock"
unixProtocol = "unix"
)
// base64Plugin gRPC sever for a mock KMS provider.
// Uses base64 to simulate encrypt and decrypt.
type base64Plugin struct {
grpcServer *grpc.Server
listener net.Listener
// Allow users of the plugin to sense requests that were passed to KMS.
encryptRequest chan *kmsapi.EncryptRequest
decryptRequest chan *kmsapi.DecryptRequest
}
func NewBase64Plugin() (*base64Plugin, error) {
if err := cleanSockFile(); err != nil {
return nil, err
}
listener, err := net.Listen(unixProtocol, sockFile)
if err != nil {
return nil, fmt.Errorf("failed to listen on the unix socket, error: %v", err)
}
server := grpc.NewServer()
result := &base64Plugin{
grpcServer: server,
listener: listener,
encryptRequest: make(chan *kmsapi.EncryptRequest, 1),
decryptRequest: make(chan *kmsapi.DecryptRequest, 1),
}
kmsapi.RegisterKeyManagementServiceServer(server, result)
return result, nil
}
func (s *base64Plugin) cleanUp() {
s.grpcServer.Stop()
s.listener.Close()
cleanSockFile()
}
var testProviderAPIVersion = kmsAPIVersion
func (s *base64Plugin) Version(ctx context.Context, request *kmsapi.VersionRequest) (*kmsapi.VersionResponse, error) {
return &kmsapi.VersionResponse{Version: testProviderAPIVersion, RuntimeName: "testKMS", RuntimeVersion: "0.0.1"}, nil
}
func (s *base64Plugin) Decrypt(ctx context.Context, request *kmsapi.DecryptRequest) (*kmsapi.DecryptResponse, error) {
s.decryptRequest <- request
buf := make([]byte, base64.StdEncoding.DecodedLen(len(request.Cipher)))
n, err := base64.StdEncoding.Decode(buf, request.Cipher)
if err != nil {
return nil, err
}
return &kmsapi.DecryptResponse{Plain: buf[:n]}, nil
}
func (s *base64Plugin) Encrypt(ctx context.Context, request *kmsapi.EncryptRequest) (*kmsapi.EncryptResponse, error) {
s.encryptRequest <- request
buf := make([]byte, base64.StdEncoding.EncodedLen(len(request.Plain)))
base64.StdEncoding.Encode(buf, request.Plain)
return &kmsapi.EncryptResponse{Cipher: buf}, nil
}
func cleanSockFile() error {
err := unix.Unlink(sockFile)
if err != nil && !os.IsNotExist(err) {
return fmt.Errorf("failed to delete the socket file, error: %v", err)
}
return nil
}

View File

@ -0,0 +1,163 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package master
import (
"bytes"
"context"
"crypto/aes"
"encoding/binary"
"fmt"
"strings"
"testing"
"time"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apiserver/pkg/storage/value"
aestransformer "k8s.io/apiserver/pkg/storage/value/encrypt/aes"
kmsapi "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1"
)
const (
kmsPrefix = "k8s:enc:kms:v1:grpc-kms-provider:"
dekKeySizeLen = 2
kmsConfigYAML = `
kind: EncryptionConfig
apiVersion: v1
resources:
- resources:
- secrets
providers:
- kms:
name: grpc-kms-provider
cachesize: 1000
endpoint: unix:///tmp/kms-provider.sock
`
)
// rawDEKKEKSecret provides operations for working with secrets transformed with Data Encryption Key(DEK) Key Encryption Kye(KEK) envelop.
type rawDEKKEKSecret []byte
func (r rawDEKKEKSecret) getDEKLen() int {
// DEK's length is stored in the two bytes that follow the prefix.
return int(binary.BigEndian.Uint16(r[len(kmsPrefix) : len(kmsPrefix)+dekKeySizeLen]))
}
func (r rawDEKKEKSecret) getDEK() []byte {
return r[len(kmsPrefix)+dekKeySizeLen : len(kmsPrefix)+dekKeySizeLen+r.getDEKLen()]
}
func (r rawDEKKEKSecret) getStartOfPayload() int {
return len(kmsPrefix) + dekKeySizeLen + r.getDEKLen()
}
func (r rawDEKKEKSecret) getPayload() []byte {
return r[r.getStartOfPayload():]
}
// TestKMSProvider is an integration test between KubAPI, ETCD and KMS Plugin
// Concretely, this test verifies the following integration contracts:
// 1. Raw records in ETCD that were processed by KMS Provider should be prefixed with k8s:enc:kms:v1:grpc-kms-provider-name:
// 2. Data Encryption Key (DEK) should be generated by envelopeTransformer and passed to KMS gRPC Plugin
// 3. KMS gRPC Plugin should encrypt the DEK with a Key Encryption Key (KEK) and pass it back to envelopeTransformer
// 4. The payload (ex. Secret) should be encrypted via AES CBC transform
// 5. Prefix-EncryptedDEK-EncryptedPayload structure should be deposited to ETCD
func TestKMSProvider(t *testing.T) {
pluginMock, err := NewBase64Plugin()
if err != nil {
t.Fatalf("failed to create mock of KMS Plugin: %v", err)
}
defer pluginMock.cleanUp()
go pluginMock.grpcServer.Serve(pluginMock.listener)
test, err := newTransformTest(t, kmsConfigYAML)
if err != nil {
t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s", kmsConfigYAML)
}
defer test.cleanUp()
secretETCDPath := test.getETCDPath()
var rawSecretAsSeenByETCD rawDEKKEKSecret
rawSecretAsSeenByETCD, err = test.getRawSecretFromETCD()
if err != nil {
t.Fatalf("failed to read %s from etcd: %v", secretETCDPath, err)
}
if !bytes.HasPrefix(rawSecretAsSeenByETCD, []byte(kmsPrefix)) {
t.Fatalf("expected secret to be prefixed with %s, but got %s", kmsPrefix, rawSecretAsSeenByETCD)
}
// Since Data Encryption Key (DEK) is randomly generated (per encryption operation), we need to ask KMS Mock for it.
dekPlainAsSeenByKMS, err := getDEKFromKMSPlugin(pluginMock)
if err != nil {
t.Fatalf("failed to get DEK from KMS: %v", err)
}
decryptResponse, err := pluginMock.Decrypt(context.Background(),
&kmsapi.DecryptRequest{Version: kmsAPIVersion, Cipher: rawSecretAsSeenByETCD.getDEK()})
if err != nil {
t.Fatalf("failed to decrypt DEK, %v", err)
}
dekPlainAsWouldBeSeenByETCD := decryptResponse.Plain
if !bytes.Equal(dekPlainAsSeenByKMS, dekPlainAsWouldBeSeenByETCD) {
t.Fatalf("expected dekPlainAsSeenByKMS %v to be passed to KMS Plugin, but got %s",
dekPlainAsSeenByKMS, dekPlainAsWouldBeSeenByETCD)
}
plainSecret, err := decryptPayload(dekPlainAsWouldBeSeenByETCD, rawSecretAsSeenByETCD, secretETCDPath)
if err != nil {
t.Fatalf("failed to transform from storage via AESCBC, err: %v", err)
}
if !strings.Contains(string(plainSecret), secretVal) {
t.Fatalf("expected %q after decryption, but got %q", secretVal, string(plainSecret))
}
// Secrets should be un-enveloped on direct reads from Kube API Server.
s, err := test.restClient.CoreV1().Secrets(testNamespace).Get(testSecret, metav1.GetOptions{})
if secretVal != string(s.Data[secretKey]) {
t.Fatalf("expected %s from KubeAPI, but got %s", secretVal, string(s.Data[secretKey]))
}
}
func getDEKFromKMSPlugin(pluginMock *base64Plugin) ([]byte, error) {
select {
case e := <-pluginMock.encryptRequest:
return e.Plain, nil
case <-time.After(1 * time.Microsecond):
return nil, fmt.Errorf("timed-out while getting encryption request from KMS Plugin Mock")
}
}
func decryptPayload(key []byte, secret rawDEKKEKSecret, secretETCDPath string) ([]byte, error) {
block, err := aes.NewCipher(key)
if err != nil {
return nil, fmt.Errorf("failed to initialize AES Cipher: %v", err)
}
// etcd path of the key is used as the authenticated context - need to pass it to decrypt
ctx := value.DefaultContext([]byte(secretETCDPath))
aescbcTransformer := aestransformer.NewCBCTransformer(block)
plainSecret, _, err := aescbcTransformer.TransformFromStorage(secret.getPayload(), ctx)
if err != nil {
return nil, fmt.Errorf("failed to transform from storage via AESCBC, err: %v", err)
}
return plainSecret, nil
}

View File

@ -17,21 +17,25 @@ limitations under the License.
package master
import (
"encoding/json"
"strings"
"testing"
appsv1beta1 "k8s.io/api/apps/v1beta1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/kube-aggregator/pkg/apis/apiregistration"
kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
"k8s.io/kubernetes/test/integration/framework"
)
func TestRun(t *testing.T) {
result := kubeapiservertesting.StartTestServerOrDie(t, nil, framework.SharedEtcd())
defer result.TearDownFn()
server := kubeapiservertesting.StartTestServerOrDie(t, nil, framework.SharedEtcd())
defer server.TearDownFn()
client, err := kubernetes.NewForConfig(result.ClientConfig)
client, err := kubernetes.NewForConfig(server.ClientConfig)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@ -72,3 +76,65 @@ func TestRun(t *testing.T) {
t.Fatalf("Failed to create deployment: %v", err)
}
}
// TestOpenAPIDelegationChainPlumbing is a smoke test that checks for
// the existence of some representative paths from the
// apiextensions-server and the kube-aggregator server, both part of
// the delegation chain in kube-apiserver.
func TestOpenAPIDelegationChainPlumbing(t *testing.T) {
server := kubeapiservertesting.StartTestServerOrDie(t, nil, framework.SharedEtcd())
defer server.TearDownFn()
kubeclient, err := kubernetes.NewForConfig(server.ClientConfig)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
result := kubeclient.RESTClient().Get().AbsPath("/swagger.json").Do()
status := 0
result.StatusCode(&status)
if status != 200 {
t.Fatalf("GET /swagger.json failed: expected status=%d, got=%d", 200, status)
}
raw, err := result.Raw()
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
type openAPISchema struct {
Paths map[string]interface{} `json:"paths"`
}
var doc openAPISchema
err = json.Unmarshal(raw, &doc)
if err != nil {
t.Fatalf("Failed to unmarshal: %v", err)
}
matchedExtension := false
extensionsPrefix := "/apis/" + apiextensions.GroupName
matchedRegistration := false
registrationPrefix := "/apis/" + apiregistration.GroupName
for path := range doc.Paths {
if strings.HasPrefix(path, extensionsPrefix) {
matchedExtension = true
}
if strings.HasPrefix(path, registrationPrefix) {
matchedRegistration = true
}
if matchedExtension && matchedRegistration {
return
}
}
if !matchedExtension {
t.Errorf("missing path: %q", extensionsPrefix)
}
if !matchedRegistration {
t.Errorf("missing path: %q", registrationPrefix)
}
}

View File

@ -0,0 +1,177 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package master
import (
"crypto/aes"
"crypto/cipher"
"encoding/base64"
"fmt"
"testing"
"k8s.io/apiserver/pkg/server/options/encryptionconfig"
"k8s.io/apiserver/pkg/storage/value"
aestransformer "k8s.io/apiserver/pkg/storage/value/encrypt/aes"
)
const (
aesGCMPrefix = "k8s:enc:aesgcm:v1:key1:"
aesCBCPrefix = "k8s:enc:aescbc:v1:key1:"
aesGCMConfigYAML = `
kind: EncryptionConfig
apiVersion: v1
resources:
- resources:
- secrets
providers:
- aesgcm:
keys:
- name: key1
secret: c2VjcmV0IGlzIHNlY3VyZQ==
`
aesCBCConfigYAML = `
kind: EncryptionConfig
apiVersion: v1
resources:
- resources:
- secrets
providers:
- aescbc:
keys:
- name: key1
secret: c2VjcmV0IGlzIHNlY3VyZQ==
`
identityConfigYAML = `
kind: EncryptionConfig
apiVersion: v1
resources:
- resources:
- secrets
providers:
- identity: {}
`
)
// TestSecretsShouldBeEnveloped is an integration test between KubeAPI and etcd that checks:
// 1. Secrets are encrypted on write
// 2. Secrets are decrypted on read
// when EncryptionConfig is passed to KubeAPI server.
func TestSecretsShouldBeTransformed(t *testing.T) {
var testCases = []struct {
transformerConfigContent string
transformerPrefix string
unSealFunc unSealSecret
}{
{aesGCMConfigYAML, aesGCMPrefix, unSealWithGCMTransformer},
{aesCBCConfigYAML, aesCBCPrefix, unSealWithCBCTransformer},
// TODO: add secretbox
}
for _, tt := range testCases {
test, err := newTransformTest(t, tt.transformerConfigContent)
if err != nil {
test.cleanUp()
t.Errorf("failed to setup test for envelop %s, error was %v", tt.transformerPrefix, err)
continue
}
test.run(tt.unSealFunc, tt.transformerPrefix)
test.cleanUp()
}
}
// Baseline (no enveloping) - use to contrast with enveloping benchmarks.
func BenchmarkBase(b *testing.B) {
runBenchmark(b, "")
}
// Identity transformer is a NOOP (crypto-wise) - use to contrast with AESGCM and AESCBC benchmark results.
func BenchmarkIdentityWrite(b *testing.B) {
runBenchmark(b, identityConfigYAML)
}
func BenchmarkAESGCMEnvelopeWrite(b *testing.B) {
runBenchmark(b, aesGCMConfigYAML)
}
func BenchmarkAESCBCEnvelopeWrite(b *testing.B) {
runBenchmark(b, aesCBCConfigYAML)
}
func runBenchmark(b *testing.B, transformerConfig string) {
b.StopTimer()
test, err := newTransformTest(b, transformerConfig)
defer test.cleanUp()
if err != nil {
b.Fatalf("failed to setup benchmark for config %s, error was %v", transformerConfig, err)
}
b.StartTimer()
test.benchmark(b)
b.StopTimer()
}
func unSealWithGCMTransformer(cipherText []byte, ctx value.Context,
transformerConfig encryptionconfig.ProviderConfig) ([]byte, error) {
block, err := newAESCipher(transformerConfig.AESGCM.Keys[0].Secret)
if err != nil {
return nil, fmt.Errorf("failed to create block cipher: %v", err)
}
gcmTransformer := aestransformer.NewGCMTransformer(block)
clearText, _, err := gcmTransformer.TransformFromStorage(cipherText, ctx)
if err != nil {
return nil, fmt.Errorf("failed to decypt secret: %v", err)
}
return clearText, nil
}
func unSealWithCBCTransformer(cipherText []byte, ctx value.Context,
transformerConfig encryptionconfig.ProviderConfig) ([]byte, error) {
block, err := newAESCipher(transformerConfig.AESCBC.Keys[0].Secret)
if err != nil {
return nil, err
}
cbcTransformer := aestransformer.NewCBCTransformer(block)
clearText, _, err := cbcTransformer.TransformFromStorage(cipherText, ctx)
if err != nil {
return nil, fmt.Errorf("failed to decypt secret: %v", err)
}
return clearText, nil
}
func newAESCipher(key string) (cipher.Block, error) {
k, err := base64.StdEncoding.DecodeString(key)
if err != nil {
return nil, fmt.Errorf("failed to decode config secret: %v", err)
}
block, err := aes.NewCipher(k)
if err != nil {
return nil, fmt.Errorf("failed to create AES cipher: %v", err)
}
return block, nil
}

View File

@ -134,7 +134,7 @@ func TestEmptyList(t *testing.T) {
func initStatusForbiddenMasterCongfig() *master.Config {
masterConfig := framework.NewIntegrationTestMasterConfig()
masterConfig.GenericConfig.Authorizer = authorizerfactory.NewAlwaysDenyAuthorizer()
masterConfig.GenericConfig.Authorization.Authorizer = authorizerfactory.NewAlwaysDenyAuthorizer()
return masterConfig
}
@ -143,8 +143,8 @@ func initUnauthorizedMasterCongfig() *master.Config {
tokenAuthenticator := tokentest.New()
tokenAuthenticator.Tokens[AliceToken] = &user.DefaultInfo{Name: "alice", UID: "1"}
tokenAuthenticator.Tokens[BobToken] = &user.DefaultInfo{Name: "bob", UID: "2"}
masterConfig.GenericConfig.Authenticator = group.NewGroupAdder(bearertoken.New(tokenAuthenticator), []string{user.AllAuthenticated})
masterConfig.GenericConfig.Authorizer = allowAliceAuthorizer{}
masterConfig.GenericConfig.Authentication.Authenticator = group.NewGroupAdder(bearertoken.New(tokenAuthenticator), []string{user.AllAuthenticated})
masterConfig.GenericConfig.Authorization.Authorizer = allowAliceAuthorizer{}
return masterConfig
}
@ -279,7 +279,7 @@ var deploymentExtensions string = `
"spec": {
"containers": [{
"name": "nginx",
"image": "gcr.io/google-containers/nginx:1.7.9"
"image": "k8s.gcr.io/nginx:1.7.9"
}]
}
}
@ -306,7 +306,7 @@ var deploymentApps string = `
"spec": {
"containers": [{
"name": "nginx",
"image": "gcr.io/google-containers/nginx:1.7.9"
"image": "k8s.gcr.io/nginx:1.7.9"
}]
}
}

View File

@ -0,0 +1,239 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package master
import (
"bytes"
"context"
"fmt"
"io/ioutil"
"os"
"path"
"strconv"
"strings"
"testing"
"github.com/coreos/etcd/clientv3"
"github.com/ghodss/yaml"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apiserver/pkg/server/options/encryptionconfig"
"k8s.io/apiserver/pkg/storage/storagebackend"
"k8s.io/apiserver/pkg/storage/value"
"k8s.io/client-go/kubernetes"
kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
"k8s.io/kubernetes/test/integration"
"k8s.io/kubernetes/test/integration/framework"
)
const (
secretKey = "api_key"
secretVal = "086a7ffc-0225-11e8-ba89-0ed5f89f718b"
encryptionConfigFileName = "encryption.conf"
testNamespace = "secret-encryption-test"
testSecret = "test-secret"
)
type unSealSecret func(cipherText []byte, ctx value.Context, config encryptionconfig.ProviderConfig) ([]byte, error)
type transformTest struct {
logger kubeapiservertesting.Logger
storageConfig *storagebackend.Config
configDir string
transformerConfig string
kubeAPIServer kubeapiservertesting.TestServer
restClient *kubernetes.Clientset
ns *corev1.Namespace
secret *corev1.Secret
}
func newTransformTest(l kubeapiservertesting.Logger, transformerConfigYAML string) (*transformTest, error) {
e := transformTest{
logger: l,
transformerConfig: transformerConfigYAML,
storageConfig: framework.SharedEtcd(),
}
var err error
if transformerConfigYAML != "" {
if e.configDir, err = e.createEncryptionConfig(); err != nil {
return nil, fmt.Errorf("error while creating KubeAPIServer encryption config: %v", err)
}
}
if e.kubeAPIServer, err = kubeapiservertesting.StartTestServer(l, e.getEncryptionOptions(), e.storageConfig); err != nil {
return nil, fmt.Errorf("failed to start KubeAPI server: %v", err)
}
if e.restClient, err = kubernetes.NewForConfig(e.kubeAPIServer.ClientConfig); err != nil {
return nil, fmt.Errorf("error while creating rest client: %v", err)
}
if e.ns, err = e.createNamespace(testNamespace); err != nil {
return nil, err
}
if e.secret, err = e.createSecret(testSecret, e.ns.Name); err != nil {
return nil, err
}
return &e, nil
}
func (e *transformTest) cleanUp() {
os.RemoveAll(e.configDir)
e.restClient.CoreV1().Namespaces().Delete(e.ns.Name, metav1.NewDeleteOptions(0))
e.kubeAPIServer.TearDownFn()
}
func (e *transformTest) run(unSealSecretFunc unSealSecret, expectedEnvelopePrefix string) {
response, err := e.readRawRecordFromETCD(e.getETCDPath())
if err != nil {
e.logger.Errorf("failed to read from etcd: %v", err)
return
}
if !bytes.HasPrefix(response.Kvs[0].Value, []byte(expectedEnvelopePrefix)) {
e.logger.Errorf("expected secret to be prefixed with %s, but got %s",
expectedEnvelopePrefix, response.Kvs[0].Value)
return
}
// etcd path of the key is used as the authenticated context - need to pass it to decrypt
ctx := value.DefaultContext([]byte(e.getETCDPath()))
// Envelope header precedes the payload
sealedData := response.Kvs[0].Value[len(expectedEnvelopePrefix):]
transformerConfig, err := e.getEncryptionConfig()
if err != nil {
e.logger.Errorf("failed to parse transformer config: %v", err)
}
v, err := unSealSecretFunc(sealedData, ctx, *transformerConfig)
if err != nil {
e.logger.Errorf("failed to unseal secret: %v", err)
return
}
if !strings.Contains(string(v), secretVal) {
e.logger.Errorf("expected %q after decryption, but got %q", secretVal, string(v))
}
// Secrets should be un-enveloped on direct reads from Kube API Server.
s, err := e.restClient.CoreV1().Secrets(testNamespace).Get(testSecret, metav1.GetOptions{})
if secretVal != string(s.Data[secretKey]) {
e.logger.Errorf("expected %s from KubeAPI, but got %s", secretVal, string(s.Data[secretKey]))
}
}
func (e *transformTest) benchmark(b *testing.B) {
for i := 0; i < b.N; i++ {
_, err := e.createSecret(e.secret.Name+strconv.Itoa(i), e.ns.Name)
if err != nil {
b.Fatalf("failed to create a secret: %v", err)
}
}
}
func (e *transformTest) getETCDPath() string {
return fmt.Sprintf("/%s/secrets/%s/%s", e.storageConfig.Prefix, e.ns.Name, e.secret.Name)
}
func (e *transformTest) getRawSecretFromETCD() ([]byte, error) {
secretETCDPath := e.getETCDPath()
etcdResponse, err := e.readRawRecordFromETCD(secretETCDPath)
if err != nil {
return nil, fmt.Errorf("failed to read %s from etcd: %v", secretETCDPath, err)
}
return etcdResponse.Kvs[0].Value, nil
}
func (e *transformTest) getEncryptionOptions() []string {
if e.transformerConfig != "" {
return []string{"--experimental-encryption-provider-config", path.Join(e.configDir, encryptionConfigFileName)}
}
return nil
}
func (e *transformTest) createEncryptionConfig() (string, error) {
tempDir, err := ioutil.TempDir("", "secrets-encryption-test")
if err != nil {
return "", fmt.Errorf("failed to create temp directory: %v", err)
}
encryptionConfig := path.Join(tempDir, encryptionConfigFileName)
if err := ioutil.WriteFile(encryptionConfig, []byte(e.transformerConfig), 0644); err != nil {
os.RemoveAll(tempDir)
return "", fmt.Errorf("error while writing encryption config: %v", err)
}
return tempDir, nil
}
func (e *transformTest) getEncryptionConfig() (*encryptionconfig.ProviderConfig, error) {
var config encryptionconfig.EncryptionConfig
err := yaml.Unmarshal([]byte(e.transformerConfig), &config)
if err != nil {
return nil, fmt.Errorf("failed to extract transformer key: %v", err)
}
return &config.Resources[0].Providers[0], nil
}
func (e *transformTest) createNamespace(name string) (*corev1.Namespace, error) {
ns := &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: name,
},
}
if _, err := e.restClient.CoreV1().Namespaces().Create(ns); err != nil {
return nil, fmt.Errorf("unable to create testing namespace %v", err)
}
return ns, nil
}
func (e *transformTest) createSecret(name, namespace string) (*corev1.Secret, error) {
secret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
},
Data: map[string][]byte{
secretKey: []byte(secretVal),
},
}
if _, err := e.restClient.CoreV1().Secrets(secret.Namespace).Create(secret); err != nil {
return nil, fmt.Errorf("error while writing secret: %v", err)
}
return secret, nil
}
func (e *transformTest) readRawRecordFromETCD(path string) (*clientv3.GetResponse, error) {
etcdClient, err := integration.GetEtcdKVClient(e.kubeAPIServer.ServerOpts.Etcd.StorageConfig)
if err != nil {
return nil, fmt.Errorf("failed to create etcd client: %v", err)
}
response, err := etcdClient.Get(context.Background(), path, clientv3.WithPrefix())
if err != nil {
return nil, fmt.Errorf("failed to retrieve secret from etcd %v", err)
}
return response, nil
}