// Copyright (c) 2015-2021 MinIO, Inc.//// This file is part of MinIO Object Storage stack//// This program is free software: you can redistribute it and/or modify// it under the terms of the GNU Affero General Public License as published by// the Free Software Foundation, either version 3 of the License, or// (at your option) any later version.//// This program is distributed in the hope that it will be useful// but WITHOUT ANY WARRANTY; without even the implied warranty of// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the// GNU Affero General Public License for more details.//// You should have received a copy of the GNU Affero General Public License// along with this program.  If not, see <http://www.gnu.org/licenses/>. package cmd import (	"bytes"	"context"	"crypto/md5"	crand "crypto/rand"	"errors"	"fmt"	"io"	"io/ioutil"	"os"	"path/filepath"	"strconv"	"testing" 	humanize "github.com/dustin/go-humanize"	"github.com/minio/minio/internal/config/storageclass") func TestRepeatPutObjectPart(t *testing.T) {	ctx, cancel := context.WithCancel(context.Background())	defer cancel() 	var objLayer ObjectLayer	var disks []string	var err error	var opts ObjectOptions 	objLayer, disks, err = prepareErasure16(ctx)	if err != nil {		t.Fatal(err)	} 	// cleaning up of temporary test directories	defer objLayer.Shutdown(context.Background())	defer removeRoots(disks) 	err = objLayer.MakeBucketWithLocation(ctx, "bucket1", MakeBucketOptions{})	if err != nil {		t.Fatal(err)	} 	uploadID, err := objLayer.NewMultipartUpload(ctx, "bucket1", "mpartObj1", opts)	if err != nil {		t.Fatal(err)	}	fiveMBBytes := bytes.Repeat([]byte("a"), 5*humanize.MiByte)	md5Hex := getMD5Hash(fiveMBBytes)	_, err = objLayer.PutObjectPart(ctx, "bucket1", "mpartObj1", uploadID, 1, mustGetPutObjReader(t, bytes.NewReader(fiveMBBytes), 5*humanize.MiByte, md5Hex, ""), opts)	if err != nil {		t.Fatal(err)	}	// PutObjectPart should succeed even if part already exists. ref: https://github.com/minio/minio/issues/1930	_, err = objLayer.PutObjectPart(ctx, "bucket1", "mpartObj1", uploadID, 1, mustGetPutObjReader(t, bytes.NewReader(fiveMBBytes), 5*humanize.MiByte, md5Hex, ""), opts)	if err != nil {		t.Fatal(err)	}} func TestErasureDeleteObjectBasic(t *testing.T) {	testCases := []struct {		bucket      string		object      string		expectedErr error	}{		{".test", "dir/obj", BucketNameInvalid{Bucket: ".test"}},		{"----", "dir/obj", BucketNameInvalid{Bucket: "----"}},		{"bucket", "", ObjectNameInvalid{Bucket: "bucket", Object: ""}},		{"bucket", "doesnotexist", ObjectNotFound{Bucket: "bucket", Object: "doesnotexist"}},		{"bucket", "dir/doesnotexist", ObjectNotFound{Bucket: "bucket", Object: "dir/doesnotexist"}},		{"bucket", "dir", ObjectNotFound{Bucket: "bucket", Object: "dir"}},		{"bucket", "dir/", ObjectNotFound{Bucket: "bucket", Object: "dir/"}},		{"bucket", "dir/obj", nil},	} 	ctx, cancel := context.WithCancel(context.Background())	defer cancel() 	// Create an instance of xl backend	xl, fsDirs, err := prepareErasure16(ctx)	if err != nil {		t.Fatal(err)	}	defer xl.Shutdown(context.Background()) 	err = xl.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})	if err != nil {		t.Fatal(err)	} 	// Create object "dir/obj" under bucket "bucket" for Test 7 to pass	_, err = xl.PutObject(ctx, "bucket", "dir/obj", mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), ObjectOptions{})	if err != nil {		t.Fatalf("Erasure Object upload failed: <ERROR> %s", err)	}	for _, test := range testCases {		test := test		t.Run("", func(t *testing.T) {			_, err := xl.GetObjectInfo(ctx, "bucket", "dir/obj", ObjectOptions{})			if err != nil {				t.Fatal("dir/obj not found before last test")			}			_, actualErr := xl.DeleteObject(ctx, test.bucket, test.object, ObjectOptions{})			if test.expectedErr != nil && actualErr != test.expectedErr {				t.Errorf("Expected to fail with %s, but failed with %s", test.expectedErr, actualErr)			}			if test.expectedErr == nil && actualErr != nil {				t.Errorf("Expected to pass, but failed with %s", actualErr)			}		})	}	// Cleanup backend directories	removeRoots(fsDirs)} func TestDeleteObjectsVersioned(t *testing.T) {	ctx, cancel := context.WithCancel(context.Background())	defer cancel() 	obj, fsDirs, err := prepareErasure(ctx, 16)	if err != nil {		t.Fatal("Unable to initialize 'Erasure' object layer.", err)	}	// Remove all dirs.	for _, dir := range fsDirs {		defer os.RemoveAll(dir)	} 	type testCaseType struct {		bucket string		object string	} 	bucketName := "bucket"	testCases := []testCaseType{		{bucketName, "dir/obj1"},		{bucketName, "dir/obj1"},	} 	err = obj.MakeBucketWithLocation(ctx, bucketName, MakeBucketOptions{		VersioningEnabled: true,	})	if err != nil {		t.Fatal(err)	} 	names := make([]ObjectToDelete, len(testCases))	for i, testCase := range testCases {		objInfo, err := obj.PutObject(ctx, testCase.bucket, testCase.object,			mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), ObjectOptions{				Versioned: true,			})		if err != nil {			t.Fatalf("Erasure Object upload failed: <ERROR> %s", err)		}		names[i] = ObjectToDelete{			ObjectV: ObjectV{				ObjectName: objInfo.Name,				VersionID:  objInfo.VersionID,			},		} 	}	names = append(names, ObjectToDelete{		ObjectV: ObjectV{			ObjectName: "dir/obj1",			VersionID:  mustGetUUID(), // add a non-existent UUID.		},	}) 	_, delErrs := obj.DeleteObjects(ctx, bucketName, names, ObjectOptions{		Versioned: true,	})	for i := range delErrs {		if delErrs[i] != nil {			t.Errorf("Failed to remove object `%v` with the error: `%v`", names[i], delErrs[i])		}	} 	for i, test := range testCases {		_, statErr := obj.GetObjectInfo(ctx, test.bucket, test.object, ObjectOptions{			VersionID: names[i].ObjectV.VersionID,		})		switch statErr.(type) {		case VersionNotFound:		default:			t.Fatalf("Object %s is not removed", test.bucket+SlashSeparator+test.object)		}	} 	if _, err = ioutil.ReadFile(pathJoin(fsDirs[0], bucketName, "dir/obj1", "xl.meta")); err == nil {		t.Fatalf("xl.meta still present after removal")	}} func TestErasureDeleteObjectsErasureSet(t *testing.T) {	ctx, cancel := context.WithCancel(context.Background())	defer cancel()	var objs []*erasureObjects	for i := 0; i < 32; i++ {		obj, fsDirs, err := prepareErasure(ctx, 16)		if err != nil {			t.Fatal("Unable to initialize 'Erasure' object layer.", err)		}		// Remove all dirs.		for _, dir := range fsDirs {			defer os.RemoveAll(dir)		}		z := obj.(*erasureServerPools)		xl := z.serverPools[0].sets[0]		objs = append(objs, xl)	} 	erasureSets := &erasureSets{sets: objs, distributionAlgo: "CRCMOD"} 	type testCaseType struct {		bucket string		object string	} 	bucketName := "bucket"	testCases := []testCaseType{		{bucketName, "dir/obj1"},		{bucketName, "dir/obj2"},		{bucketName, "obj3"},		{bucketName, "obj_4"},	} 	err := erasureSets.MakeBucketWithLocation(ctx, bucketName, MakeBucketOptions{})	if err != nil {		t.Fatal(err)	} 	for _, testCase := range testCases {		_, err = erasureSets.PutObject(ctx, testCase.bucket, testCase.object,			mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), ObjectOptions{})		if err != nil {			t.Fatalf("Erasure Object upload failed: <ERROR> %s", err)		}	} 	toObjectNames := func(testCases []testCaseType) []ObjectToDelete {		names := make([]ObjectToDelete, len(testCases))		for i := range testCases {			names[i] = ObjectToDelete{				ObjectV: ObjectV{					ObjectName: testCases[i].object,				},			}		}		return names	} 	objectNames := toObjectNames(testCases)	_, delErrs := erasureSets.DeleteObjects(ctx, bucketName, objectNames, ObjectOptions{}) 	for i := range delErrs {		if delErrs[i] != nil {			t.Errorf("Failed to remove object `%v` with the error: `%v`", objectNames[i], delErrs[i])		}	} 	for _, test := range testCases {		_, statErr := erasureSets.GetObjectInfo(ctx, test.bucket, test.object, ObjectOptions{})		switch statErr.(type) {		case ObjectNotFound:		default:			t.Fatalf("Object %s is not removed", test.bucket+SlashSeparator+test.object)		}	}} func TestErasureDeleteObjectDiskNotFound(t *testing.T) {	ctx, cancel := context.WithCancel(context.Background())	defer cancel() 	// Create an instance of xl backend.	obj, fsDirs, err := prepareErasure16(ctx)	if err != nil {		t.Fatal(err)	}	// Cleanup backend directories	defer obj.Shutdown(context.Background())	defer removeRoots(fsDirs) 	z := obj.(*erasureServerPools)	xl := z.serverPools[0].sets[0] 	// Create "bucket"	err = obj.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})	if err != nil {		t.Fatal(err)	} 	bucket := "bucket"	object := "object"	opts := ObjectOptions{}	// Create object "obj" under bucket "bucket".	_, err = obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), opts)	if err != nil {		t.Fatal(err)	} 	erasureDisks := xl.getDisks()	z.serverPools[0].erasureDisksMu.Lock()	xl.getDisks = func() []StorageAPI {		for i := range erasureDisks[:6] {			erasureDisks[i] = newNaughtyDisk(erasureDisks[i], nil, errFaultyDisk)		}		return erasureDisks	} 	z.serverPools[0].erasureDisksMu.Unlock()	_, err = obj.DeleteObject(ctx, bucket, object, ObjectOptions{})	if !errors.Is(err, errErasureWriteQuorum) {		t.Fatal(err)	} 	// Create "obj" under "bucket".	_, err = obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), opts)	if err != nil {		t.Fatal(err)	} 	// Remove one more disk to 'lose' quorum, by taking 2 more drives offline.	erasureDisks = xl.getDisks()	z.serverPools[0].erasureDisksMu.Lock()	xl.getDisks = func() []StorageAPI {		erasureDisks[7] = nil		erasureDisks[8] = nil		return erasureDisks	} 	z.serverPools[0].erasureDisksMu.Unlock()	_, err = obj.DeleteObject(ctx, bucket, object, ObjectOptions{})	// since majority of disks are not available, metaquorum is not achieved and hence errErasureWriteQuorum error	if !errors.Is(err, errErasureWriteQuorum) {		t.Errorf("Expected deleteObject to fail with %v, but failed with %v", toObjectErr(errErasureWriteQuorum, bucket, object), err)	}} func TestErasureDeleteObjectDiskNotFoundErasure4(t *testing.T) {	ctx, cancel := context.WithCancel(context.Background())	defer cancel() 	// Create an instance of xl backend.	obj, fsDirs, err := prepareErasure16(ctx)	if err != nil {		t.Fatal(err)	}	// Cleanup backend directories	defer obj.Shutdown(context.Background())	defer removeRoots(fsDirs) 	z := obj.(*erasureServerPools)	xl := z.serverPools[0].sets[0] 	// Create "bucket"	err = obj.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})	if err != nil {		t.Fatal(err)	} 	bucket := "bucket"	object := "object"	opts := ObjectOptions{}	// Create object "obj" under bucket "bucket".	_, err = obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), opts)	if err != nil {		t.Fatal(err)	}	// Upload a good object	_, err = obj.DeleteObject(ctx, bucket, object, ObjectOptions{})	if err != nil {		t.Fatal(err)	} 	// Create "obj" under "bucket".	_, err = obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), opts)	if err != nil {		t.Fatal(err)	} 	// Remove disks to 'lose' quorum for object, by setting 5 to nil.	erasureDisks := xl.getDisks()	z.serverPools[0].erasureDisksMu.Lock()	xl.getDisks = func() []StorageAPI {		for i := range erasureDisks[:5] {			erasureDisks[i] = newNaughtyDisk(erasureDisks[i], nil, errFaultyDisk)		}		return erasureDisks	} 	z.serverPools[0].erasureDisksMu.Unlock()	_, err = obj.DeleteObject(ctx, bucket, object, ObjectOptions{})	// since majority of disks are not available, metaquorum is not achieved and hence errErasureWriteQuorum error	if !errors.Is(err, errErasureWriteQuorum) {		t.Errorf("Expected deleteObject to fail with %v, but failed with %v", toObjectErr(errErasureWriteQuorum, bucket, object), err)	}} func TestErasureDeleteObjectDiskNotFoundErr(t *testing.T) {	ctx, cancel := context.WithCancel(context.Background())	defer cancel() 	// Create an instance of xl backend.	obj, fsDirs, err := prepareErasure16(ctx)	if err != nil {		t.Fatal(err)	}	// Cleanup backend directories	defer obj.Shutdown(context.Background())	defer removeRoots(fsDirs) 	z := obj.(*erasureServerPools)	xl := z.serverPools[0].sets[0] 	// Create "bucket"	err = obj.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})	if err != nil {		t.Fatal(err)	} 	bucket := "bucket"	object := "object"	opts := ObjectOptions{}	// Create object "obj" under bucket "bucket".	_, err = obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), opts)	if err != nil {		t.Fatal(err)	}	// for a 16 disk setup, EC is 4, but will be upgraded up to 8.	// Remove 4 disks.	erasureDisks := xl.getDisks()	z.serverPools[0].erasureDisksMu.Lock()	xl.getDisks = func() []StorageAPI {		for i := range erasureDisks[:4] {			erasureDisks[i] = newNaughtyDisk(erasureDisks[i], nil, errFaultyDisk)		}		return erasureDisks	} 	z.serverPools[0].erasureDisksMu.Unlock()	_, err = obj.DeleteObject(ctx, bucket, object, ObjectOptions{})	if err != nil {		t.Fatal(err)	} 	// Create "obj" under "bucket".	_, err = obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), opts)	if err != nil {		t.Fatal(err)	} 	// Object was uploaded with 4 known bad drives, so we should still be able to lose 3 drives and still write to the object.	erasureDisks = xl.getDisks()	z.serverPools[0].erasureDisksMu.Lock()	xl.getDisks = func() []StorageAPI {		erasureDisks[7] = nil		erasureDisks[8] = nil		erasureDisks[9] = nil		return erasureDisks	} 	z.serverPools[0].erasureDisksMu.Unlock()	_, err = obj.DeleteObject(ctx, bucket, object, ObjectOptions{})	// since majority of disks are available, metaquorum achieved.	if err != nil {		t.Errorf("Expected deleteObject to not fail, but failed with %v", err)	}} func TestGetObjectNoQuorum(t *testing.T) {	ctx, cancel := context.WithCancel(context.Background())	defer cancel() 	// Create an instance of xl backend.	obj, fsDirs, err := prepareErasure16(ctx)	if err != nil {		t.Fatal(err)	}	// Cleanup backend directories.

Lines 1–500 of 1217. Load more