fscrypt: fscrypt integration

Integrate google/fscrypt into Ceph CSI KMS and encryption setup. Adds
dependencies to google/fscrypt and pkg/xattr. Be as generic as
possible to support integration with both RBD and Ceph FS.

Add the following public functions:

InitializeNode: per-node initialization steps. Must be called
before Unlock at least once.

Unlock: All steps necessary to unlock an encrypted directory including
setting it up initially.

IsDirectoryUnlocked: Test if directory is really encrypted

Signed-off-by: Marcel Lauhoff <marcel.lauhoff@suse.com>
This commit is contained in:
Marcel Lauhoff 2022-08-12 16:30:35 +02:00 committed by mergify[bot]
parent 2cf8ecc6c7
commit cfea8d7562
4 changed files with 405 additions and 0 deletions

2
go.mod
View File

@ -15,6 +15,7 @@ require (
github.com/gemalto/kmip-go v0.0.8-0.20220721195433-3fe83e2d3f26
github.com/golang/protobuf v1.5.2
github.com/google/uuid v1.3.0
github.com/google/fscrypt v0.3.3
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
github.com/hashicorp/vault/api v1.7.2
@ -23,6 +24,7 @@ require (
github.com/libopenstorage/secrets v0.0.0-20210908194121-a1d19aa9713a
github.com/onsi/ginkgo/v2 v2.1.6
github.com/onsi/gomega v1.20.1
github.com/pkg/xattr v0.4.7
github.com/prometheus/client_golang v1.12.2
github.com/stretchr/testify v1.8.0
golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd

7
go.sum
View File

@ -485,6 +485,8 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ
github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA=
github.com/google/cadvisor v0.45.0/go.mod h1:vsMT3Uv2XjQ8M7WUtKARV74mU/HN64C4XtM1bJhUKcU=
github.com/google/cel-go v0.12.4/go.mod h1:Av7CU6r6X3YmcHR9GXqVDaEJYfEtSxl6wvIjUQTriCw=
github.com/google/fscrypt v0.3.3 h1:qwx9OCR/xZE68VGr/r0/yugFhlGpIOGsH9JHrttP7vc=
github.com/google/fscrypt v0.3.3/go.mod h1:H1JHtH8BVe0dYNhzx1Ztkn3azQ0OBdoOmM828vEWAXc=
github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54=
github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
@ -991,6 +993,8 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
github.com/pkg/xattr v0.4.7 h1:XoA3KzmFvyPlH4RwX5eMcgtzcaGBaSvgt3IoFQfbrmQ=
github.com/pkg/xattr v0.4.7/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/portworx/dcos-secrets v0.0.0-20180616013705-8e8ec3f66611/go.mod h1:4hklRW/4DQpLqkcXcjtNprbH2tz/sJaNtqinfPWl/LA=
@ -1143,6 +1147,7 @@ github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYp
github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
github.com/vmware/govmomi v0.20.3/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU=
github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad/go.mod h1:Hy8o65+MXnS6EwGElrSRjUzQDLXreJlzYLlWiHtt8hM=
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos=
@ -1503,6 +1508,7 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9w
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210422114643-f5beecf764ed/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@ -1551,6 +1557,7 @@ golang.org/x/tools v0.0.0-20190718200317-82a3ea8a504c/go.mod h1:jcCCGcm9btYwXyDq
golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191025023517-2077df36852e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=

View File

@ -0,0 +1,382 @@
/*
Copyright 2022 The Ceph-CSI Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package fscrypt
/*
#include <linux/fs.h>
*/
import "C"
import (
"context"
"errors"
"fmt"
"os"
"os/user"
"path"
"time"
"unsafe"
fscryptactions "github.com/google/fscrypt/actions"
fscryptcrypto "github.com/google/fscrypt/crypto"
fscryptfilesystem "github.com/google/fscrypt/filesystem"
fscryptmetadata "github.com/google/fscrypt/metadata"
"github.com/pkg/xattr"
"golang.org/x/sys/unix"
"github.com/ceph/ceph-csi/internal/kms"
"github.com/ceph/ceph-csi/internal/util"
"github.com/ceph/ceph-csi/internal/util/log"
)
const (
FscryptHashingTimeTarget = 1 * time.Second
FscryptProtectorPrefix = "ceph-csi"
FscryptSubdir = "ceph-csi-encrypted"
encryptionPassphraseSize = 64
)
func AppendEncyptedSubdirectory(dir string) string {
return path.Join(dir, FscryptSubdir)
}
// getPassphrase returns the passphrase from the configured Ceph CSI KMS to be used as a protector key in fscrypt.
func getPassphrase(ctx context.Context, encryption util.VolumeEncryption, volID string) (string, error) {
var (
passphrase string
err error
)
switch encryption.KMS.RequiresDEKStore() {
case kms.DEKStoreIntegrated:
passphrase, err = encryption.GetCryptoPassphrase(volID)
if err != nil {
log.ErrorLog(ctx, "fscrypt: failed to get passphrase from KMS: %v", err)
return "", err
}
case kms.DEKStoreMetadata:
passphrase, err = encryption.KMS.GetSecret(volID)
if err != nil {
log.ErrorLog(ctx, "fscrypt: failed to GetSecret: %v", err)
return "", err
}
}
return passphrase, nil
}
// createKeyFuncFromVolumeEncryption returns an fscrypt key function returning
// encryption keys form a VolumeEncryption struct.
func createKeyFuncFromVolumeEncryption(
ctx context.Context,
encryption util.VolumeEncryption,
volID string,
) (func(fscryptactions.ProtectorInfo, bool) (*fscryptcrypto.Key, error), error) {
passphrase, err := getPassphrase(ctx, encryption, volID)
if err != nil {
return nil, err
}
keyFunc := func(info fscryptactions.ProtectorInfo, retry bool) (*fscryptcrypto.Key, error) {
key, err := fscryptcrypto.NewBlankKey(32)
copy(key.Data(), passphrase)
return key, err
}
return keyFunc, nil
}
// unlockExisting tries to unlock an already set up fscrypt directory using keys from Ceph CSI.
func unlockExisting(
ctx context.Context,
fscryptContext *fscryptactions.Context,
encryptedPath string, protectorName string,
keyFn func(fscryptactions.ProtectorInfo, bool) (*fscryptcrypto.Key, error),
) error {
var err error
policy, err := fscryptactions.GetPolicyFromPath(fscryptContext, encryptedPath)
if err != nil {
log.ErrorLog(ctx, "fscrypt: policy get failed %v", err)
return err
}
optionFn := func(policyDescriptor string, options []*fscryptactions.ProtectorOption) (int, error) {
for idx, option := range options {
if option.Name() == protectorName {
return idx, nil
}
}
return 0, &fscryptactions.ErrNotProtected{PolicyDescriptor: policyDescriptor, ProtectorDescriptor: protectorName}
}
if err = policy.Unlock(optionFn, keyFn); err != nil {
log.ErrorLog(ctx, "fscrypt: unlock with protector error: %v", err)
return err
}
defer func() {
err = policy.Lock()
if err != nil {
log.ErrorLog(ctx, "fscrypt: failed to lock policy after use: %v", err)
}
}()
if err = policy.Provision(); err != nil {
log.ErrorLog(ctx, "fscrypt: provision fail %v", err)
return err
}
log.DebugLog(ctx, "fscrypt protector unlock: %s %+v", protectorName, policy)
return nil
}
func initializeAndUnlock(
ctx context.Context,
fscryptContext *fscryptactions.Context,
encryptedPath string, protectorName string,
keyFn func(fscryptactions.ProtectorInfo, bool) (*fscryptcrypto.Key, error),
) error {
var owner *user.User
var err error
if err = os.Mkdir(encryptedPath, 0o755); err != nil {
return err
}
protector, err := fscryptactions.CreateProtector(fscryptContext, protectorName, keyFn, owner)
if err != nil {
log.ErrorLog(ctx, "fscrypt: protector name=%s create failed: %v. reverting.", protectorName, err)
if revertErr := protector.Revert(); revertErr != nil {
return revertErr
}
return err
}
if err = protector.Unlock(keyFn); err != nil {
return err
}
log.DebugLog(ctx, "fscrypt protector unlock: %+v", protector)
var policy *fscryptactions.Policy
if policy, err = fscryptactions.CreatePolicy(fscryptContext, protector); err != nil {
return err
}
defer func() {
err = policy.Lock()
if err != nil {
log.ErrorLog(ctx, "fscrypt: failed to lock policy after init: %w")
err = policy.Revert()
if err != nil {
log.ErrorLog(ctx, "fscrypt: failed to revert policy after failed lock: %w")
}
}
}()
if err = policy.UnlockWithProtector(protector); err != nil {
log.ErrorLog(ctx, "fscrypt: Failed to unlock policy: %v", err)
return err
}
if err = policy.Provision(); err != nil {
log.ErrorLog(ctx, "fscrypt: Failed to provision policy: %v", err)
return err
}
if err = policy.Apply(encryptedPath); err != nil {
log.ErrorLog(ctx, "fscrypt: Failed to apply protector (see also kernel log): %w", err)
if err = policy.Deprovision(false); err != nil {
log.ErrorLog(ctx, "fscrypt: Policy cleanup response to failing apply failed: %w", err)
}
return err
}
return nil
}
// getInodeEncryptedAttribute returns the inode's encrypt attribute similar to lsattr(1)
func getInodeEncryptedAttribute(p string) (bool, error) {
file, err := os.Open(p)
if err != nil {
return false, err
}
defer file.Close()
var attr int
_, _, errno := unix.Syscall(unix.SYS_IOCTL, file.Fd(), unix.FS_IOC_GETFLAGS,
uintptr(unsafe.Pointer(&attr)))
if errno != 0 {
return false, fmt.Errorf("error calling ioctl_iflags: %w", errno)
}
if attr&C.FS_ENCRYPT_FL != 0 {
return true, nil
}
return false, nil
}
// IsDirectoryUnlockedFscrypt checks if a directory is an unlocked fscrypted directory.
func IsDirectoryUnlocked(directoryPath, filesystem string) error {
if _, err := fscryptmetadata.GetPolicy(directoryPath); err != nil {
return fmt.Errorf("no fscrypt policy set on directory %q: %w", directoryPath, err)
}
switch filesystem {
case "ceph":
_, err := xattr.Get(directoryPath, "ceph.fscrypt.auth")
if err != nil {
return fmt.Errorf("error reading ceph.fscrypt.auth xattr on %q: %w", directoryPath, err)
}
default:
encrypted, err := getInodeEncryptedAttribute(directoryPath)
if err != nil {
return err
}
if !encrypted {
return fmt.Errorf("path %s does not have the encrypted inode flag set. Encryption init must have failed",
directoryPath)
}
}
return nil
}
// InitializeNode performs once per nodeserver initialization
// required by the fscrypt library. Creates /etc/fscrypt.conf.
func InitializeNode(ctx context.Context) error {
err := fscryptactions.CreateConfigFile(FscryptHashingTimeTarget, 2)
if err != nil {
existsError := &fscryptactions.ErrConfigFileExists{}
if errors.As(err, &existsError) {
log.ErrorLog(ctx, "fscrypt: config file %q already exists. Skipping fscrypt node setup",
existsError.Path)
return nil
}
return fmt.Errorf("fscrypt node init failed to create node configuration (/etc/fscrypt.conf): %w",
err)
}
return nil
}
// FscryptUnlock unlocks possilby creating fresh fscrypt metadata
// iff a volume is encrypted. Otherwise return immediately Calling
// this function requires that InitializeFscrypt ran once on this node.
func Unlock(
ctx context.Context,
volEncryption *util.VolumeEncryption,
stagingTargetPath string, volID string,
) error {
fscryptContext, err := fscryptactions.NewContextFromMountpoint(stagingTargetPath, nil)
if err != nil {
log.ErrorLog(ctx, "fscrypt: failed to create context from mountpoint %v: %w", stagingTargetPath)
return err
}
fscryptContext.Config.UseFsKeyringForV1Policies = true
log.DebugLog(ctx, "fscrypt context: %+v", fscryptContext)
if err = fscryptContext.Mount.CheckSupport(); err != nil {
log.ErrorLog(ctx, "fscrypt: filesystem mount %s does not support fscrypt", fscryptContext.Mount)
return err
}
// A proper set up fscrypy directory requires metadata and a kernel policy:
// 1. Do we have a metadata directory (.fscrypt) set up?
metadataDirExists := false
if err = fscryptContext.Mount.Setup(0o755); err != nil {
alreadySetupErr := &fscryptfilesystem.ErrAlreadySetup{}
if errors.As(err, &alreadySetupErr) {
log.DebugLog(ctx, "fscrypt: metadata directory %q already set up", alreadySetupErr.Mount.Path)
metadataDirExists = true
} else {
log.ErrorLog(ctx, "fscrypt: mount setup failed: %v", err)
return err
}
}
encryptedPath := path.Join(stagingTargetPath, FscryptSubdir)
kernelPolicyExists := false
// 2. Ask the kernel if the directory has an fscrypt policy in place.
if _, err = fscryptmetadata.GetPolicy(encryptedPath); err == nil { // encrypted directory already set up
kernelPolicyExists = true
}
if metadataDirExists != kernelPolicyExists {
return fmt.Errorf("fscrypt: unsupported state metadata=%t kernel_policy=%t",
metadataDirExists, kernelPolicyExists)
}
keyFn, err := createKeyFuncFromVolumeEncryption(ctx, *volEncryption, volID)
if err != nil {
log.ErrorLog(ctx, "fscrypt: could not create key function: %v", err)
return err
}
protectorName := fmt.Sprintf("%s-%s", FscryptProtectorPrefix, volEncryption.GetID())
switch volEncryption.KMS.RequiresDEKStore() {
case kms.DEKStoreMetadata:
// Metadata style KMS use the KMS secret as a custom
// passphrase directly in fscrypt, circumenting key
// derivation on the CSI side to allow users to fall
// back on the fscrypt commandline tool easily
fscryptContext.Config.Source = fscryptmetadata.SourceType_custom_passphrase
case kms.DEKStoreIntegrated:
fscryptContext.Config.Source = fscryptmetadata.SourceType_raw_key
}
if kernelPolicyExists && metadataDirExists {
log.DebugLog(ctx, "fscrypt: Encrypted directory already set up, policy exists")
return unlockExisting(ctx, fscryptContext, encryptedPath, protectorName, keyFn)
}
if !kernelPolicyExists && !metadataDirExists {
log.DebugLog(ctx, "fscrypt: Creating new protector and policy")
if volEncryption.KMS.RequiresDEKStore() == kms.DEKStoreIntegrated {
if err := volEncryption.StoreNewCryptoPassphrase(volID, encryptionPassphraseSize); err != nil {
log.ErrorLog(ctx, "fscrypt: store new crypto passphrase failed: %v", err)
return err
}
}
return initializeAndUnlock(ctx, fscryptContext, encryptedPath, protectorName, keyFn)
}
return fmt.Errorf("unsupported")
}

14
vendor/modules.txt vendored
View File

@ -230,6 +230,15 @@ github.com/golang/protobuf/ptypes/wrappers
# github.com/golang/snappy v0.0.4
## explicit
github.com/golang/snappy
# github.com/google/fscrypt v0.3.3
## explicit; go 1.11
github.com/google/fscrypt/actions
github.com/google/fscrypt/crypto
github.com/google/fscrypt/filesystem
github.com/google/fscrypt/keyring
github.com/google/fscrypt/metadata
github.com/google/fscrypt/security
github.com/google/fscrypt/util
# github.com/google/gnostic v0.5.7-v3refs
## explicit; go 1.12
github.com/google/gnostic/compiler
@ -476,6 +485,9 @@ github.com/pierrec/lz4/internal/xxh32
# github.com/pkg/errors v0.9.1
## explicit
github.com/pkg/errors
# github.com/pkg/xattr v0.4.7
## explicit; go 1.14
github.com/pkg/xattr
# github.com/pmezard/go-difflib v1.0.0
## explicit
github.com/pmezard/go-difflib/difflib
@ -594,6 +606,7 @@ go.uber.org/zap/internal/exit
go.uber.org/zap/zapcore
# golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd
## explicit; go 1.17
golang.org/x/crypto/argon2
golang.org/x/crypto/blake2b
golang.org/x/crypto/blowfish
golang.org/x/crypto/chacha20
@ -602,6 +615,7 @@ golang.org/x/crypto/cryptobyte/asn1
golang.org/x/crypto/curve25519
golang.org/x/crypto/curve25519/internal/field
golang.org/x/crypto/ed25519
golang.org/x/crypto/hkdf
golang.org/x/crypto/internal/poly1305
golang.org/x/crypto/internal/subtle
golang.org/x/crypto/pbkdf2