// Copyright (c) 2015-2021 MinIO, Inc.//// This file is part of MinIO Object Storage stack//// This program is free software: you can redistribute it and/or modify// it under the terms of the GNU Affero General Public License as published by// the Free Software Foundation, either version 3 of the License, or// (at your option) any later version.//// This program is distributed in the hope that it will be useful// but WITHOUT ANY WARRANTY; without even the implied warranty of// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the// GNU Affero General Public License for more details.//// You should have received a copy of the GNU Affero General Public License// along with this program.  If not, see <http://www.gnu.org/licenses/>. package cmd import (	"context"	"encoding/json"	"errors"	"fmt"	"io/fs"	"io/ioutil"	"reflect"	"sync" 	humanize "github.com/dustin/go-humanize"	"github.com/minio/minio/internal/color"	"github.com/minio/minio/internal/config"	"github.com/minio/minio/internal/config/storageclass"	xioutil "github.com/minio/minio/internal/ioutil"	"github.com/minio/minio/internal/logger"	"github.com/minio/minio/internal/sync/errgroup") const (	// Represents Erasure backend.	formatBackendErasure = "xl" 	// Represents Erasure backend - single drive	formatBackendErasureSingle = "xl-single" 	// formatErasureV1.Erasure.Version - version '1'.	formatErasureVersionV1 = "1" 	// formatErasureV2.Erasure.Version - version '2'.	formatErasureVersionV2 = "2" 	// formatErasureV3.Erasure.Version - version '3'.	formatErasureVersionV3 = "3" 	// Distribution algorithm used, legacy	formatErasureVersionV2DistributionAlgoV1 = "CRCMOD" 	// Distributed algorithm used, with N/2 default parity	formatErasureVersionV3DistributionAlgoV2 = "SIPMOD" 	// Distributed algorithm used, with EC:4 default parity	formatErasureVersionV3DistributionAlgoV3 = "SIPMOD+PARITY") // Offline disk UUID represents an offline disk.const offlineDiskUUID = "ffffffff-ffff-ffff-ffff-ffffffffffff" // Used to detect the version of "xl" format.type formatErasureVersionDetect struct {	Erasure struct {		Version string `json:"version"`	} `json:"xl"`} // Represents the V1 backend disk structure version// under `.minio.sys` and actual data namespace.// formatErasureV1 - structure holds format config version '1'.type formatErasureV1 struct {	formatMetaV1	Erasure struct {		Version string `json:"version"` // Version of 'xl' format.		Disk    string `json:"disk"`    // Disk field carries assigned disk uuid.		// JBOD field carries the input disk order generated the first		// time when fresh disks were supplied.		JBOD []string `json:"jbod"`	} `json:"xl"` // Erasure field holds xl format.} // Represents the V2 backend disk structure version// under `.minio.sys` and actual data namespace.// formatErasureV2 - structure holds format config version '2'.// The V2 format to support "large bucket" support where a bucket// can span multiple erasure sets.type formatErasureV2 struct {	formatMetaV1	Erasure struct {		Version string `json:"version"` // Version of 'xl' format.		This    string `json:"this"`    // This field carries assigned disk uuid.		// Sets field carries the input disk order generated the first		// time when fresh disks were supplied, it is a two dimensional		// array second dimension represents list of disks used per set.		Sets [][]string `json:"sets"`		// Distribution algorithm represents the hashing algorithm		// to pick the right set index for an object.		DistributionAlgo string `json:"distributionAlgo"`	} `json:"xl"`} // formatErasureV3 struct is same as formatErasureV2 struct except that formatErasureV3.Erasure.Version is "3" indicating// the simplified multipart backend which is a flat hierarchy now.// In .minio.sys/multipart we have:// sha256(bucket/object)/uploadID/[xl.meta, part.1, part.2 ....]type formatErasureV3 struct {	formatMetaV1	Erasure struct {		Version string `json:"version"` // Version of 'xl' format.		This    string `json:"this"`    // This field carries assigned disk uuid.		// Sets field carries the input disk order generated the first		// time when fresh disks were supplied, it is a two dimensional		// array second dimension represents list of disks used per set.		Sets [][]string `json:"sets"`		// Distribution algorithm represents the hashing algorithm		// to pick the right set index for an object.		DistributionAlgo string `json:"distributionAlgo"`	} `json:"xl"`} func (f *formatErasureV3) Drives() (drives int) {	for _, set := range f.Erasure.Sets {		drives += len(set)	}	return drives} func (f *formatErasureV3) Clone() *formatErasureV3 {	b, err := json.Marshal(f)	if err != nil {		panic(err)	}	var dst formatErasureV3	if err = json.Unmarshal(b, &dst); err != nil {		panic(err)	}	return &dst} // Returns formatErasure.Erasure.Versionfunc newFormatErasureV3(numSets int, setLen int) *formatErasureV3 {	format := &formatErasureV3{}	format.Version = formatMetaVersionV1	format.Format = formatBackendErasure	if setLen == 1 {		format.Format = formatBackendErasureSingle	}	format.ID = mustGetUUID()	format.Erasure.Version = formatErasureVersionV3	format.Erasure.DistributionAlgo = formatErasureVersionV3DistributionAlgoV3	format.Erasure.Sets = make([][]string, numSets) 	for i := 0; i < numSets; i++ {		format.Erasure.Sets[i] = make([]string, setLen)		for j := 0; j < setLen; j++ {			format.Erasure.Sets[i][j] = mustGetUUID()		}	}	return format} // Returns format Erasure version after reading `format.json`, returns// successfully the version only if the backend is Erasure.func formatGetBackendErasureVersion(b []byte) (string, error) {	meta := &formatMetaV1{}	if err := json.Unmarshal(b, meta); err != nil {		return "", err	}	if meta.Version != formatMetaVersionV1 {		return "", fmt.Errorf(`format.Version expected: %s, got: %s`, formatMetaVersionV1, meta.Version)	}	if meta.Format != formatBackendErasure && meta.Format != formatBackendErasureSingle {		return "", fmt.Errorf(`found backend type %s, expected %s or %s`, meta.Format, formatBackendErasure, formatBackendErasureSingle)	}	// Erasure backend found, proceed to detect version.	format := &formatErasureVersionDetect{}	if err := json.Unmarshal(b, format); err != nil {		return "", err	}	return format.Erasure.Version, nil} // Migrates all previous versions to latest version of `format.json`,// this code calls migration in sequence, such as V1 is migrated to V2// first before it V2 migrates to V3.nfunc formatErasureMigrate(export string) ([]byte, fs.FileInfo, error) {	formatPath := pathJoin(export, minioMetaBucket, formatConfigFile)	formatData, formatFi, err := xioutil.ReadFileWithFileInfo(formatPath)	if err != nil {		return nil, nil, err	} 	version, err := formatGetBackendErasureVersion(formatData)	if err != nil {		return nil, nil, fmt.Errorf("Disk %s: %w", export, err)	} 	migrate := func(formatPath string, formatData []byte) ([]byte, fs.FileInfo, error) {		if err = ioutil.WriteFile(formatPath, formatData, 0o666); err != nil {			return nil, nil, err		}		formatFi, err := Lstat(formatPath)		if err != nil {			return nil, nil, err		}		return formatData, formatFi, nil	} 	switch version {	case formatErasureVersionV1:		formatData, err = formatErasureMigrateV1ToV2(formatData, version)		if err != nil {			return nil, nil, fmt.Errorf("Disk %s: %w", export, err)		}		// Migrate successful v1 => v2, proceed to v2 => v3		version = formatErasureVersionV2		fallthrough	case formatErasureVersionV2:		formatData, err = formatErasureMigrateV2ToV3(formatData, export, version)		if err != nil {			return nil, nil, fmt.Errorf("Disk %s: %w", export, err)		}		// Migrate successful v2 => v3, v3 is latest		// version = formatXLVersionV3		return migrate(formatPath, formatData)	case formatErasureVersionV3:		// v3 is the latest version, return.		return formatData, formatFi, nil	}	return nil, nil, fmt.Errorf(`Disk %s: unknown format version %s`, export, version)} // Migrates version V1 of format.json to version V2 of format.json,// migration fails upon any error.func formatErasureMigrateV1ToV2(data []byte, version string) ([]byte, error) {	if version != formatErasureVersionV1 {		return nil, fmt.Errorf(`format version expected %s, found %s`, formatErasureVersionV1, version)	} 	formatV1 := &formatErasureV1{}	if err := json.Unmarshal(data, formatV1); err != nil {		return nil, err	} 	formatV2 := &formatErasureV2{}	formatV2.Version = formatMetaVersionV1	formatV2.Format = formatBackendErasure	formatV2.Erasure.Version = formatErasureVersionV2	formatV2.Erasure.DistributionAlgo = formatErasureVersionV2DistributionAlgoV1	formatV2.Erasure.This = formatV1.Erasure.Disk	formatV2.Erasure.Sets = make([][]string, 1)	formatV2.Erasure.Sets[0] = make([]string, len(formatV1.Erasure.JBOD))	copy(formatV2.Erasure.Sets[0], formatV1.Erasure.JBOD) 	return json.Marshal(formatV2)} // Migrates V2 for format.json to V3 (Flat hierarchy for multipart)func formatErasureMigrateV2ToV3(data []byte, export, version string) ([]byte, error) {	if version != formatErasureVersionV2 {		return nil, fmt.Errorf(`format version expected %s, found %s`, formatErasureVersionV2, version)	} 	formatV2 := &formatErasureV2{}	if err := json.Unmarshal(data, formatV2); err != nil {		return nil, err	} 	tmpOld := pathJoin(export, minioMetaTmpDeletedBucket, mustGetUUID())	if err := renameAll(pathJoin(export, minioMetaMultipartBucket),		tmpOld); err != nil && err != errFileNotFound {		logger.LogIf(GlobalContext, fmt.Errorf("unable to rename (%s -> %s) %w, drive may be faulty please investigate",			pathJoin(export, minioMetaMultipartBucket),			tmpOld,			osErrToFileErr(err)))	} 	// format-V2 struct is exactly same as format-V1 except that version is "3"	// which indicates the simplified multipart backend.	formatV3 := formatErasureV3{}	formatV3.Version = formatV2.Version	formatV3.Format = formatV2.Format	formatV3.Erasure = formatV2.Erasure	formatV3.Erasure.Version = formatErasureVersionV3 	return json.Marshal(formatV3)} // countErrs - count a specific error.func countErrs(errs []error, err error) int {	i := 0	for _, err1 := range errs {		if err1 == err || errors.Is(err1, err) {			i++		}	}	return i} // Does all errors indicate we need to initialize all disks?.func shouldInitErasureDisks(errs []error) bool {	return countErrs(errs, errUnformattedDisk) == len(errs)} // Check if unformatted disks are equal to write quorum.func quorumUnformattedDisks(errs []error) bool {	return countErrs(errs, errUnformattedDisk) >= (len(errs)/2)+1} // loadFormatErasureAll - load all format config from all input disks in parallel.func loadFormatErasureAll(storageDisks []StorageAPI, heal bool) ([]*formatErasureV3, []error) {	// Initialize list of errors.	g := errgroup.WithNErrs(len(storageDisks)) 	// Initialize format configs.	formats := make([]*formatErasureV3, len(storageDisks)) 	// Load format from each disk in parallel	for index := range storageDisks {		index := index		g.Go(func() error {			if storageDisks[index] == nil {				return errDiskNotFound			}			format, err := loadFormatErasure(storageDisks[index])			if err != nil {				return err			}			formats[index] = format			if !heal {				// If no healing required, make the disks valid and				// online.				storageDisks[index].SetDiskID(format.Erasure.This)			}			return nil		}, index)	} 	// Return all formats and errors if any.	return formats, g.Wait()} func saveFormatErasure(disk StorageAPI, format *formatErasureV3, heal bool) error {	if disk == nil || format == nil {		return errDiskNotFound	} 	diskID := format.Erasure.This 	if err := makeFormatErasureMetaVolumes(disk); err != nil {		return err	} 	// Marshal and write to disk.	formatBytes, err := json.Marshal(format)	if err != nil {		return err	} 	tmpFormat := mustGetUUID() 	// Purge any existing temporary file, okay to ignore errors here.	defer disk.Delete(context.TODO(), minioMetaBucket, tmpFormat, DeleteOptions{		Recursive: false,		Force:     false,	}) 	// write to unique file.	if err = disk.WriteAll(context.TODO(), minioMetaBucket, tmpFormat, formatBytes); err != nil {		return err	} 	// Rename file `uuid.json` --> `format.json`.	if err = disk.RenameFile(context.TODO(), minioMetaBucket, tmpFormat, minioMetaBucket, formatConfigFile); err != nil {		return err	} 	disk.SetDiskID(diskID)	if heal {		ctx := context.Background()		ht := newHealingTracker(disk)		return ht.save(ctx)	}	return nil} // loadFormatErasure - loads format.json from disk.func loadFormatErasure(disk StorageAPI) (format *formatErasureV3, err error) {	buf, err := disk.ReadAll(context.TODO(), minioMetaBucket, formatConfigFile)	if err != nil {		// 'file not found' and 'volume not found' as		// same. 'volume not found' usually means its a fresh disk.		if err == errFileNotFound || err == errVolumeNotFound {			return nil, errUnformattedDisk		}		return nil, err	} 	// Try to decode format json into formatConfigV1 struct.	format = &formatErasureV3{}	if err = json.Unmarshal(buf, format); err != nil {		return nil, err	} 	// Success.	return format, nil} // Valid formatErasure basic versions.func checkFormatErasureValue(formatErasure *formatErasureV3, disk StorageAPI) error {	// Validate format version and format type.	if formatErasure.Version != formatMetaVersionV1 {		return fmt.Errorf("Unsupported version of backend format [%s] found on %s", formatErasure.Version, disk)	}	if formatErasure.Format != formatBackendErasure && formatErasure.Format != formatBackendErasureSingle {		return fmt.Errorf("Unsupported backend format [%s] found on %s", formatErasure.Format, disk)	}	if formatErasure.Erasure.Version != formatErasureVersionV3 {		return fmt.Errorf("Unsupported Erasure backend format found [%s] on %s", formatErasure.Erasure.Version, disk)	}	return nil} // Check all format values.func checkFormatErasureValues(formats []*formatErasureV3, disks []StorageAPI, setDriveCount int) error {	for i, formatErasure := range formats {		if formatErasure == nil {			continue		}		if err := checkFormatErasureValue(formatErasure, disks[i]); err != nil {			return err		}		if len(formats) != len(formatErasure.Erasure.Sets)*len(formatErasure.Erasure.Sets[0]) {			return fmt.Errorf("%s disk is already being used in another erasure deployment. (Number of disks specified: %d but the number of disks found in the %s disk's format.json: %d)",				disks[i], len(formats), humanize.Ordinal(i+1), len(formatErasure.Erasure.Sets)*len(formatErasure.Erasure.Sets[0]))		}		// Only if custom erasure drive count is set, verify if the		// set_drive_count was manually set - we need to honor what is		// present on the drives.		if globalCustomErasureDriveCount && len(formatErasure.Erasure.Sets[0]) != setDriveCount {			return fmt.Errorf("%s disk is already formatted with %d drives per erasure set. This cannot be changed to %d, please revert your MINIO_ERASURE_SET_DRIVE_COUNT setting", disks[i], len(formatErasure.Erasure.Sets[0]), setDriveCount)		}	}	return nil} // Get Deployment ID for the Erasure sets from format.json.// This need not be in quorum. Even if one of the format.json// file has this value, we assume it is valid.// If more than one format.json's have different id, it is considered a corrupt// backend format.func formatErasureGetDeploymentID(refFormat *formatErasureV3, formats []*formatErasureV3) (string, error) {	var deploymentID string	for _, format := range formats {		if format == nil || format.ID == "" {			continue		}		if reflect.DeepEqual(format.Erasure.Sets, refFormat.Erasure.Sets) {			// Found an ID in one of the format.json file			// Set deploymentID for the first time.			if deploymentID == "" {				deploymentID = format.ID			} else if deploymentID != format.ID {				// DeploymentID found earlier doesn't match with the				// current format.json's ID.				return "", fmt.Errorf("Deployment IDs do not match expected %s, got %s: %w",					deploymentID, format.ID, errCorruptedFormat)			}		}	}	return deploymentID, nil} // formatErasureFixDeploymentID - Add deployment id if it is not present.func formatErasureFixDeploymentID(endpoints Endpoints, storageDisks []StorageAPI, refFormat *formatErasureV3) (err error) {	// Attempt to load all `format.json` from all disks.	formats, _ := loadFormatErasureAll(storageDisks, false)	for index := range formats {		// If the Erasure sets do not match, set those formats to nil,		// We do not have to update the ID on those format.json file.		if formats[index] != nil && !reflect.DeepEqual(formats[index].Erasure.Sets, refFormat.Erasure.Sets) {			formats[index] = nil		}	} 	refFormat.ID, err = formatErasureGetDeploymentID(refFormat, formats)	if err != nil {		return err	} 	// If ID is set, then some other node got the lock	// before this node could and generated an ID	// for the deployment. No need to generate one.

Lines 1–500 of 857. Load more