// Copyright (c) 2015-2021 MinIO, Inc.//// This file is part of MinIO Object Storage stack//// This program is free software: you can redistribute it and/or modify// it under the terms of the GNU Affero General Public License as published by// the Free Software Foundation, either version 3 of the License, or// (at your option) any later version.//// This program is distributed in the hope that it will be useful// but WITHOUT ANY WARRANTY; without even the implied warranty of// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the// GNU Affero General Public License for more details.//// You should have received a copy of the GNU Affero General Public License// along with this program.  If not, see <http://www.gnu.org/licenses/>. package cmd import (	"bufio"	"context"	"crypto/hmac"	"crypto/rand"	"crypto/subtle"	"encoding/binary"	"encoding/hex"	"errors"	"fmt"	"io"	"net/http"	"path"	"strconv"	"strings" 	"github.com/minio/kes"	"github.com/minio/minio/internal/crypto"	"github.com/minio/minio/internal/etag"	"github.com/minio/minio/internal/fips"	"github.com/minio/minio/internal/hash/sha256"	xhttp "github.com/minio/minio/internal/http"	"github.com/minio/minio/internal/kms"	"github.com/minio/minio/internal/logger"	"github.com/minio/sio") var (	// AWS errors for invalid SSE-C requests.	errEncryptedObject      = errors.New("The object was stored using a form of SSE")	errInvalidSSEParameters = errors.New("The SSE-C key for key-rotation is not correct") // special access denied	errKMSNotConfigured     = errors.New("KMS not configured for a server side encrypted object")	errKMSKeyNotFound       = errors.New("Invalid KMS keyId")	// Additional MinIO errors for SSE-C requests.	errObjectTampered = errors.New("The requested object was modified and may be compromised")	// error returned when invalid encryption parameters are specified	errInvalidEncryptionParameters = errors.New("The encryption parameters are not applicable to this object")) const (	// SSECustomerKeySize is the size of valid client provided encryption keys in bytes.	// Currently AWS supports only AES256. So the SSE-C key size is fixed to 32 bytes.	SSECustomerKeySize = 32 	// SSEIVSize is the size of the IV data	SSEIVSize = 32 // 32 bytes 	// SSEDAREPackageBlockSize - SSE dare package block size.	SSEDAREPackageBlockSize = 64 * 1024 // 64KiB bytes 	// SSEDAREPackageMetaSize - SSE dare package meta padding bytes.	SSEDAREPackageMetaSize = 32 // 32 bytes ) // KMSKeyID returns in AWS compatible KMS KeyID() format.func (o *ObjectInfo) KMSKeyID() string { return kmsKeyIDFromMetadata(o.UserDefined) } // KMSKeyID returns in AWS compatible KMS KeyID() format.func (o *MultipartInfo) KMSKeyID() string { return kmsKeyIDFromMetadata(o.UserDefined) } // kmsKeyIDFromMetadata returns any AWS S3 KMS key ID in the// metadata, if any. It returns an empty ID if no key ID is// present.func kmsKeyIDFromMetadata(metadata map[string]string) string {	const ARNPrefix = "arn:aws:kms:"	if len(metadata) == 0 {		return ""	}	kmsID, ok := metadata[crypto.MetaKeyID]	if !ok {		return ""	}	if strings.HasPrefix(kmsID, ARNPrefix) {		return kmsID	}	return ARNPrefix + kmsID} // DecryptETags decryptes the ETag of all ObjectInfos using the KMS.//// It adjusts the size of all encrypted objects since encrypted// objects are slightly larger due to encryption overhead.// Further, it decrypts all single-part SSE-S3 encrypted objects// and formats ETags of SSE-C / SSE-KMS encrypted objects to// be AWS S3 compliant.//// DecryptETags uses a KMS bulk decryption API, if available, which// is more efficient than decrypting ETags sequentually.func DecryptETags(ctx context.Context, KMS kms.KMS, objects []ObjectInfo) error {	const BatchSize = 250 // We process the objects in batches - 250 is a reasonable default.	var (		metadata = make([]map[string]string, 0, BatchSize)		buckets  = make([]string, 0, BatchSize)		names    = make([]string, 0, BatchSize)	)	for len(objects) > 0 {		N := BatchSize		if len(objects) < BatchSize {			N = len(objects)		}		batch := objects[:N] 		// We have to decrypt only ETags of SSE-S3 single-part		// objects.		// Therefore, we remember which objects (there index)		// in the current batch are single-part SSE-S3 objects.		metadata = metadata[:0:N]		buckets = buckets[:0:N]		names = names[:0:N]		SSES3SinglePartObjects := make(map[int]bool)		for i, object := range batch {			if kind, ok := crypto.IsEncrypted(object.UserDefined); ok && kind == crypto.S3 && !crypto.IsMultiPart(object.UserDefined) {				SSES3SinglePartObjects[i] = true 				metadata = append(metadata, object.UserDefined)				buckets = append(buckets, object.Bucket)				names = append(names, object.Name)			}		} 		// If there are no SSE-S3 single-part objects		// we can skip the decryption process. However,		// we still have to adjust the size and ETag		// of SSE-C and SSE-KMS objects.		if len(SSES3SinglePartObjects) == 0 {			for i := range batch {				size, err := batch[i].GetActualSize()				if err != nil {					return err				}				batch[i].Size = size 				if _, ok := crypto.IsEncrypted(batch[i].UserDefined); ok {					ETag, err := etag.Parse(batch[i].ETag)					if err != nil {						return err					}					batch[i].ETag = ETag.Format().String()				}			}			objects = objects[N:]			continue		} 		// There is at least one SSE-S3 single-part object.		// For all SSE-S3 single-part objects we have to		// fetch their decryption keys. We do this using		// a Bulk-Decryption API call, if available.		keys, err := crypto.S3.UnsealObjectKeys(ctx, KMS, metadata, buckets, names)		if err != nil {			return err		} 		// Now, we have to decrypt the ETags of SSE-S3 single-part		// objects and adjust the size and ETags of all encrypted		// objects.		for i := range batch {			size, err := batch[i].GetActualSize()			if err != nil {				return err			}			batch[i].Size = size 			if _, ok := crypto.IsEncrypted(batch[i].UserDefined); ok {				ETag, err := etag.Parse(batch[i].ETag)				if err != nil {					return err				}				if SSES3SinglePartObjects[i] && ETag.IsEncrypted() {					ETag, err = etag.Decrypt(keys[0][:], ETag)					if err != nil {						return err					}					keys = keys[1:]				}				batch[i].ETag = ETag.Format().String()			}		}		objects = objects[N:]	}	return nil} // isMultipart returns true if the current object is// uploaded by the user using multipart mechanism:// initiate new multipart, upload part, complete uploadfunc (o *ObjectInfo) isMultipart() bool {	if len(o.Parts) == 0 {		return false	}	_, encrypted := crypto.IsEncrypted(o.UserDefined)	if encrypted {		if !crypto.IsMultiPart(o.UserDefined) {			return false		}		for _, part := range o.Parts {			_, err := sio.DecryptedSize(uint64(part.Size))			if err != nil {				return false			}		}	} 	// Further check if this object is uploaded using multipart mechanism	// by the user and it is not about Erasure internally splitting the	// object into parts in PutObject()	return !(o.backendType == BackendErasure && len(o.ETag) == 32)} // ParseSSECopyCustomerRequest parses the SSE-C header fields of the provided request.// It returns the client provided key on success.func ParseSSECopyCustomerRequest(h http.Header, metadata map[string]string) (key []byte, err error) {	if crypto.S3.IsEncrypted(metadata) && crypto.SSECopy.IsRequested(h) {		return nil, crypto.ErrIncompatibleEncryptionMethod	}	k, err := crypto.SSECopy.ParseHTTP(h)	return k[:], err} // ParseSSECustomerRequest parses the SSE-C header fields of the provided request.// It returns the client provided key on success.func ParseSSECustomerRequest(r *http.Request) (key []byte, err error) {	return ParseSSECustomerHeader(r.Header)} // ParseSSECustomerHeader parses the SSE-C header fields and returns// the client provided key on success.func ParseSSECustomerHeader(header http.Header) (key []byte, err error) {	if crypto.S3.IsRequested(header) && crypto.SSEC.IsRequested(header) {		return key, crypto.ErrIncompatibleEncryptionMethod	} 	k, err := crypto.SSEC.ParseHTTP(header)	return k[:], err} // This function rotates old to new key.func rotateKey(ctx context.Context, oldKey []byte, newKeyID string, newKey []byte, bucket, object string, metadata map[string]string, cryptoCtx kms.Context) error {	kind, _ := crypto.IsEncrypted(metadata)	switch kind {	case crypto.S3:		if GlobalKMS == nil {			return errKMSNotConfigured		}		keyID, kmsKey, sealedKey, err := crypto.S3.ParseMetadata(metadata)		if err != nil {			return err		}		oldKey, err := GlobalKMS.DecryptKey(keyID, kmsKey, kms.Context{bucket: path.Join(bucket, object)})		if err != nil {			return err		}		var objectKey crypto.ObjectKey		if err = objectKey.Unseal(oldKey, sealedKey, crypto.S3.String(), bucket, object); err != nil {			return err		} 		newKey, err := GlobalKMS.GenerateKey(ctx, "", kms.Context{bucket: path.Join(bucket, object)})		if err != nil {			return err		}		sealedKey = objectKey.Seal(newKey.Plaintext, crypto.GenerateIV(rand.Reader), crypto.S3.String(), bucket, object)		crypto.S3.CreateMetadata(metadata, newKey.KeyID, newKey.Ciphertext, sealedKey)		return nil	case crypto.S3KMS:		if GlobalKMS == nil {			return errKMSNotConfigured		}		objectKey, err := crypto.S3KMS.UnsealObjectKey(GlobalKMS, metadata, bucket, object)		if err != nil {			return err		} 		if len(cryptoCtx) == 0 {			_, _, _, cryptoCtx, err = crypto.S3KMS.ParseMetadata(metadata)			if err != nil {				return err			}		} 		// If the context does not contain the bucket key		// we must add it for key generation. However,		// the context must be stored exactly like the		// client provided it. Therefore, we create a copy		// of the client provided context and add the bucket		// key, if not present.		kmsCtx := kms.Context{}		for k, v := range cryptoCtx {			kmsCtx[k] = v		}		if _, ok := kmsCtx[bucket]; !ok {			kmsCtx[bucket] = path.Join(bucket, object)		}		newKey, err := GlobalKMS.GenerateKey(ctx, newKeyID, kmsCtx)		if err != nil {			return err		} 		sealedKey := objectKey.Seal(newKey.Plaintext, crypto.GenerateIV(rand.Reader), crypto.S3KMS.String(), bucket, object)		crypto.S3KMS.CreateMetadata(metadata, newKey.KeyID, newKey.Ciphertext, sealedKey, cryptoCtx)		return nil	case crypto.SSEC:		sealedKey, err := crypto.SSEC.ParseMetadata(metadata)		if err != nil {			return err		} 		var objectKey crypto.ObjectKey		if err = objectKey.Unseal(oldKey, sealedKey, crypto.SSEC.String(), bucket, object); err != nil {			if subtle.ConstantTimeCompare(oldKey, newKey) == 1 {				return errInvalidSSEParameters // AWS returns special error for equal but invalid keys.			}			return crypto.ErrInvalidCustomerKey // To provide strict AWS S3 compatibility we return: access denied. 		}		if subtle.ConstantTimeCompare(oldKey, newKey) == 1 && sealedKey.Algorithm == crypto.SealAlgorithm {			return nil // don't rotate on equal keys if seal algorithm is latest		}		sealedKey = objectKey.Seal(newKey, sealedKey.IV, crypto.SSEC.String(), bucket, object)		crypto.SSEC.CreateMetadata(metadata, sealedKey)		return nil	default:		return errObjectTampered	}} func newEncryptMetadata(ctx context.Context, kind crypto.Type, keyID string, key []byte, bucket, object string, metadata map[string]string, cryptoCtx kms.Context) (crypto.ObjectKey, error) {	var sealedKey crypto.SealedKey	switch kind {	case crypto.S3:		if GlobalKMS == nil {			return crypto.ObjectKey{}, errKMSNotConfigured		}		key, err := GlobalKMS.GenerateKey(ctx, "", kms.Context{bucket: path.Join(bucket, object)})		if err != nil {			return crypto.ObjectKey{}, err		} 		objectKey := crypto.GenerateKey(key.Plaintext, rand.Reader)		sealedKey = objectKey.Seal(key.Plaintext, crypto.GenerateIV(rand.Reader), crypto.S3.String(), bucket, object)		crypto.S3.CreateMetadata(metadata, key.KeyID, key.Ciphertext, sealedKey)		return objectKey, nil	case crypto.S3KMS:		if GlobalKMS == nil {			return crypto.ObjectKey{}, errKMSNotConfigured		} 		// If the context does not contain the bucket key		// we must add it for key generation. However,		// the context must be stored exactly like the		// client provided it. Therefore, we create a copy		// of the client provided context and add the bucket		// key, if not present.		kmsCtx := kms.Context{}		for k, v := range cryptoCtx {			kmsCtx[k] = v		}		if _, ok := kmsCtx[bucket]; !ok {			kmsCtx[bucket] = path.Join(bucket, object)		}		key, err := GlobalKMS.GenerateKey(ctx, keyID, kmsCtx)		if err != nil {			if errors.Is(err, kes.ErrKeyNotFound) {				return crypto.ObjectKey{}, errKMSKeyNotFound			}			return crypto.ObjectKey{}, err		} 		objectKey := crypto.GenerateKey(key.Plaintext, rand.Reader)		sealedKey = objectKey.Seal(key.Plaintext, crypto.GenerateIV(rand.Reader), crypto.S3KMS.String(), bucket, object)		crypto.S3KMS.CreateMetadata(metadata, key.KeyID, key.Ciphertext, sealedKey, cryptoCtx)		return objectKey, nil	case crypto.SSEC:		objectKey := crypto.GenerateKey(key, rand.Reader)		sealedKey = objectKey.Seal(key, crypto.GenerateIV(rand.Reader), crypto.SSEC.String(), bucket, object)		crypto.SSEC.CreateMetadata(metadata, sealedKey)		return objectKey, nil	default:		return crypto.ObjectKey{}, fmt.Errorf("encryption type '%v' not supported", kind)	}} func newEncryptReader(ctx context.Context, content io.Reader, kind crypto.Type, keyID string, key []byte, bucket, object string, metadata map[string]string, cryptoCtx kms.Context) (io.Reader, crypto.ObjectKey, error) {	objectEncryptionKey, err := newEncryptMetadata(ctx, kind, keyID, key, bucket, object, metadata, cryptoCtx)	if err != nil {		return nil, crypto.ObjectKey{}, err	} 	reader, err := sio.EncryptReader(content, sio.Config{Key: objectEncryptionKey[:], MinVersion: sio.Version20, CipherSuites: fips.DARECiphers()})	if err != nil {		return nil, crypto.ObjectKey{}, crypto.ErrInvalidCustomerKey	} 	return reader, objectEncryptionKey, nil} // set new encryption metadata from http request headers for SSE-C and generated key from KMS in the case of// SSE-S3func setEncryptionMetadata(r *http.Request, bucket, object string, metadata map[string]string) (err error) {	var (		key    []byte		keyID  string		kmsCtx kms.Context	)	kind, _ := crypto.IsRequested(r.Header)	switch kind {	case crypto.SSEC:		key, err = ParseSSECustomerRequest(r)		if err != nil {			return err		}	case crypto.S3KMS:		keyID, kmsCtx, err = crypto.S3KMS.ParseHTTP(r.Header)		if err != nil {			return err		}	}	_, err = newEncryptMetadata(r.Context(), kind, keyID, key, bucket, object, metadata, kmsCtx)	return} // EncryptRequest takes the client provided content and encrypts the data// with the client provided key. It also marks the object as client-side-encrypted// and sets the correct headers.func EncryptRequest(content io.Reader, r *http.Request, bucket, object string, metadata map[string]string) (io.Reader, crypto.ObjectKey, error) {	if r.ContentLength > encryptBufferThreshold {		// The encryption reads in blocks of 64KB.		// We add a buffer on bigger files to reduce the number of syscalls upstream.		content = bufio.NewReaderSize(content, encryptBufferSize)	} 	var (		key   []byte		keyID string		ctx   kms.Context		err   error	)	kind, _ := crypto.IsRequested(r.Header)	if kind == crypto.SSEC {		key, err = ParseSSECustomerRequest(r)		if err != nil {			return nil, crypto.ObjectKey{}, err		}	}	if kind == crypto.S3KMS {		keyID, ctx, err = crypto.S3KMS.ParseHTTP(r.Header)		if err != nil {			return nil, crypto.ObjectKey{}, err		}	}	return newEncryptReader(r.Context(), content, kind, keyID, key, bucket, object, metadata, ctx)} func decryptObjectInfo(key []byte, bucket, object string, metadata map[string]string) ([]byte, error) {	switch kind, _ := crypto.IsEncrypted(metadata); kind {	case crypto.S3:		var KMS kms.KMS = GlobalKMS		if isCacheEncrypted(metadata) {			KMS = globalCacheKMS		}		if KMS == nil {			return nil, errKMSNotConfigured		}		objectKey, err := crypto.S3.UnsealObjectKey(KMS, metadata, bucket, object)		if err != nil {			return nil, err		}		return objectKey[:], nil	case crypto.S3KMS:		if GlobalKMS == nil {			return nil, errKMSNotConfigured		}		objectKey, err := crypto.S3KMS.UnsealObjectKey(GlobalKMS, metadata, bucket, object)		if err != nil {			return nil, err		}		return objectKey[:], nil	case crypto.SSEC:		sealedKey, err := crypto.SSEC.ParseMetadata(metadata)		if err != nil {

Lines 1–500 of 1054. Load more