// Copyright (c) 2015-2021 MinIO, Inc.//// This file is part of MinIO Object Storage stack//// This program is free software: you can redistribute it and/or modify// it under the terms of the GNU Affero General Public License as published by// the Free Software Foundation, either version 3 of the License, or// (at your option) any later version.//// This program is distributed in the hope that it will be useful// but WITHOUT ANY WARRANTY; without even the implied warranty of// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the// GNU Affero General Public License for more details.//// You should have received a copy of the GNU Affero General Public License// along with this program.  If not, see <http://www.gnu.org/licenses/>. package cmd import (	"context"	"errors"	"fmt"	"io"	"net/http"	"path"	"path/filepath"	"sort"	"strings"	"time" 	"github.com/cespare/xxhash/v2"	"github.com/klauspost/compress/zstd"	"github.com/minio/madmin-go"	"github.com/minio/minio/internal/bucket/lifecycle"	"github.com/minio/minio/internal/hash"	"github.com/minio/minio/internal/logger"	"github.com/tinylib/msgp/msgp") //go:generate msgp -file $GOFILE -unexported // dataUsageHash is the hash type used.type dataUsageHash string // sizeHistogram is a size histogram.type sizeHistogram [dataUsageBucketLen]uint64 type dataUsageEntry struct {	Children dataUsageHashMap `msg:"ch"`	// These fields do no include any children.	Size             int64                `msg:"sz"`	Objects          uint64               `msg:"os"`	Versions         uint64               `msg:"vs"` // Versions that are not delete markers.	ObjSizes         sizeHistogram        `msg:"szs"`	ReplicationStats *replicationAllStats `msg:"rs,omitempty"`	AllTierStats     *allTierStats        `msg:"ats,omitempty"`	Compacted        bool                 `msg:"c"`} // allTierStats is a collection of per-tier stats across all configured remote// tiers.type allTierStats struct {	Tiers map[string]tierStats `msg:"ts"`} func newAllTierStats() *allTierStats {	return &allTierStats{		Tiers: make(map[string]tierStats),	}} func (ats *allTierStats) addSizes(sz sizeSummary) {	for tier, st := range sz.tiers {		ats.Tiers[tier] = ats.Tiers[tier].add(st)	}} func (ats *allTierStats) merge(other *allTierStats) {	for tier, st := range other.Tiers {		ats.Tiers[tier] = ats.Tiers[tier].add(st)	}} func (ats *allTierStats) adminStats(stats map[string]madmin.TierStats) map[string]madmin.TierStats {	if ats == nil {		return stats	} 	// Update stats for tiers as they become available.	for tier, st := range ats.Tiers {		stats[tier] = madmin.TierStats{			TotalSize:   st.TotalSize,			NumVersions: st.NumVersions,			NumObjects:  st.NumObjects,		}	}	return stats} // tierStats holds per-tier stats of a remote tier.type tierStats struct {	TotalSize   uint64 `msg:"ts"`	NumVersions int    `msg:"nv"`	NumObjects  int    `msg:"no"`} func (ts tierStats) add(u tierStats) tierStats {	ts.TotalSize += u.TotalSize	ts.NumVersions += u.NumVersions	ts.NumObjects += u.NumObjects	return ts} //msgp:tuple replicationStatsV1type replicationStatsV1 struct {	PendingSize          uint64	ReplicatedSize       uint64	FailedSize           uint64	ReplicaSize          uint64	FailedCount          uint64	PendingCount         uint64	MissedThresholdSize  uint64	AfterThresholdSize   uint64	MissedThresholdCount uint64	AfterThresholdCount  uint64} func (rsv1 replicationStatsV1) Empty() bool {	return rsv1.ReplicatedSize == 0 &&		rsv1.FailedSize == 0 &&		rsv1.FailedCount == 0} //msgp:tuple replicationStatstype replicationStats struct {	PendingSize          uint64	ReplicatedSize       uint64	FailedSize           uint64	FailedCount          uint64	PendingCount         uint64	MissedThresholdSize  uint64	AfterThresholdSize   uint64	MissedThresholdCount uint64	AfterThresholdCount  uint64} func (rs replicationStats) Empty() bool {	return rs.ReplicatedSize == 0 &&		rs.FailedSize == 0 &&		rs.FailedCount == 0} type replicationAllStats struct {	Targets     map[string]replicationStats `msg:"t,omitempty"`	ReplicaSize uint64                      `msg:"r,omitempty"`} //msgp:tuple replicationAllStatsV1type replicationAllStatsV1 struct {	Targets     map[string]replicationStats	ReplicaSize uint64 `msg:"ReplicaSize,omitempty"`} //msgp:encode ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4 dataUsageEntryV5 dataUsageEntryV6//msgp:marshal ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4 dataUsageEntryV5 dataUsageEntryV6 //msgp:tuple dataUsageEntryV2type dataUsageEntryV2 struct {	// These fields do no include any children.	Size     int64	Objects  uint64	ObjSizes sizeHistogram	Children dataUsageHashMap} //msgp:tuple dataUsageEntryV3type dataUsageEntryV3 struct {	// These fields do no include any children.	Size                   int64	ReplicatedSize         uint64	ReplicationPendingSize uint64	ReplicationFailedSize  uint64	ReplicaSize            uint64	Objects                uint64	ObjSizes               sizeHistogram	Children               dataUsageHashMap} //msgp:tuple dataUsageEntryV4type dataUsageEntryV4 struct {	Children dataUsageHashMap	// These fields do no include any children.	Size             int64	Objects          uint64	ObjSizes         sizeHistogram	ReplicationStats replicationStatsV1} //msgp:tuple dataUsageEntryV5type dataUsageEntryV5 struct {	Children dataUsageHashMap	// These fields do no include any children.	Size             int64	Objects          uint64	Versions         uint64 // Versions that are not delete markers.	ObjSizes         sizeHistogram	ReplicationStats *replicationStatsV1	Compacted        bool} //msgp:tuple dataUsageEntryV6type dataUsageEntryV6 struct {	Children dataUsageHashMap	// These fields do no include any children.	Size             int64	Objects          uint64	Versions         uint64 // Versions that are not delete markers.	ObjSizes         sizeHistogram	ReplicationStats *replicationAllStatsV1	Compacted        bool} // dataUsageCache contains a cache of data usage entries latest version.type dataUsageCache struct {	Info  dataUsageCacheInfo	Cache map[string]dataUsageEntry} //msgp:encode ignore dataUsageCacheV2 dataUsageCacheV3 dataUsageCacheV4 dataUsageCacheV5 dataUsageCacheV6//msgp:marshal ignore dataUsageCacheV2 dataUsageCacheV3 dataUsageCacheV4 dataUsageCacheV5 dataUsageCacheV6 // dataUsageCacheV2 contains a cache of data usage entries version 2.type dataUsageCacheV2 struct {	Info  dataUsageCacheInfo	Cache map[string]dataUsageEntryV2} // dataUsageCacheV3 contains a cache of data usage entries version 3.type dataUsageCacheV3 struct {	Info  dataUsageCacheInfo	Cache map[string]dataUsageEntryV3} // dataUsageCacheV4 contains a cache of data usage entries version 4.type dataUsageCacheV4 struct {	Info  dataUsageCacheInfo	Cache map[string]dataUsageEntryV4} // dataUsageCacheV5 contains a cache of data usage entries version 5.type dataUsageCacheV5 struct {	Info  dataUsageCacheInfo	Cache map[string]dataUsageEntryV5} // dataUsageCacheV6 contains a cache of data usage entries version 6.type dataUsageCacheV6 struct {	Info  dataUsageCacheInfo	Cache map[string]dataUsageEntryV6} //msgp:ignore dataUsageEntryInfotype dataUsageEntryInfo struct {	Name   string	Parent string	Entry  dataUsageEntry} type dataUsageCacheInfo struct {	// Name of the bucket. Also root element.	Name       string	NextCycle  uint32	LastUpdate time.Time	// indicates if the disk is being healed and scanner	// should skip healing the disk	SkipHealing bool	BloomFilter []byte `msg:"BloomFilter,omitempty"` 	// Active lifecycle, if any on the bucket	lifeCycle *lifecycle.Lifecycle `msg:"-"` 	// optional updates channel.	// If set updates will be sent regularly to this channel.	// Will not be closed when returned.	updates     chan<- dataUsageEntry `msg:"-"`	replication replicationConfig     `msg:"-"`} func (e *dataUsageEntry) addSizes(summary sizeSummary) {	e.Size += summary.totalSize	e.Versions += summary.versions	e.ObjSizes.add(summary.totalSize) 	if e.ReplicationStats == nil {		e.ReplicationStats = &replicationAllStats{			Targets: make(map[string]replicationStats),		}	} else if e.ReplicationStats.Targets == nil {		e.ReplicationStats.Targets = make(map[string]replicationStats)	}	e.ReplicationStats.ReplicaSize += uint64(summary.replicaSize) 	if summary.replTargetStats != nil {		for arn, st := range summary.replTargetStats {			tgtStat, ok := e.ReplicationStats.Targets[arn]			if !ok {				tgtStat = replicationStats{}			}			tgtStat.PendingSize += uint64(st.pendingSize)			tgtStat.FailedSize += uint64(st.failedSize)			tgtStat.ReplicatedSize += uint64(st.replicatedSize)			tgtStat.FailedCount += st.failedCount			tgtStat.PendingCount += st.pendingCount			e.ReplicationStats.Targets[arn] = tgtStat		}	}	if summary.tiers != nil {		if e.AllTierStats == nil {			e.AllTierStats = newAllTierStats()		}		e.AllTierStats.addSizes(summary)	}} // merge other data usage entry into this, excluding children.func (e *dataUsageEntry) merge(other dataUsageEntry) {	e.Objects += other.Objects	e.Versions += other.Versions	e.Size += other.Size	if other.ReplicationStats != nil {		if e.ReplicationStats == nil {			e.ReplicationStats = &replicationAllStats{Targets: make(map[string]replicationStats)}		} else if e.ReplicationStats.Targets == nil {			e.ReplicationStats.Targets = make(map[string]replicationStats)		}		e.ReplicationStats.ReplicaSize += other.ReplicationStats.ReplicaSize		for arn, stat := range other.ReplicationStats.Targets {			st := e.ReplicationStats.Targets[arn]			e.ReplicationStats.Targets[arn] = replicationStats{				PendingSize:    stat.PendingSize + st.PendingSize,				FailedSize:     stat.FailedSize + st.FailedSize,				ReplicatedSize: stat.ReplicatedSize + st.ReplicatedSize,				PendingCount:   stat.PendingCount + st.PendingCount,				FailedCount:    stat.FailedCount + st.FailedCount,			}		}	} 	for i, v := range other.ObjSizes[:] {		e.ObjSizes[i] += v	} 	if other.AllTierStats != nil {		if e.AllTierStats == nil {			e.AllTierStats = newAllTierStats()		}		e.AllTierStats.merge(other.AllTierStats)	}} // mod returns true if the hash mod cycles == cycle.// If cycles is 0 false is always returned.// If cycles is 1 true is always returned (as expected).func (h dataUsageHash) mod(cycle uint32, cycles uint32) bool {	if cycles <= 1 {		return cycles == 1	}	return uint32(xxhash.Sum64String(string(h)))%cycles == cycle%cycles} // modAlt returns true if the hash mod cycles == cycle.// This is out of sync with mod.// If cycles is 0 false is always returned.// If cycles is 1 true is always returned (as expected).func (h dataUsageHash) modAlt(cycle uint32, cycles uint32) bool {	if cycles <= 1 {		return cycles == 1	}	return uint32(xxhash.Sum64String(string(h))>>32)%(cycles) == cycle%cycles} // addChild will add a child based on its hash.// If it already exists it will not be added again.func (e *dataUsageEntry) addChild(hash dataUsageHash) {	if _, ok := e.Children[hash.Key()]; ok {		return	}	if e.Children == nil {		e.Children = make(dataUsageHashMap, 1)	}	e.Children[hash.Key()] = struct{}{}} // removeChild will remove a child based on its hash.func (e *dataUsageEntry) removeChild(hash dataUsageHash) {	if len(e.Children) > 0 {		delete(e.Children, hash.Key())	}} // Create a clone of the entry.func (e dataUsageEntry) clone() dataUsageEntry {	// We operate on a copy from the receiver.	if e.Children != nil {		ch := make(dataUsageHashMap, len(e.Children))		for k, v := range e.Children {			ch[k] = v		}		e.Children = ch	}	if e.ReplicationStats != nil {		// Copy to new struct		r := *e.ReplicationStats		e.ReplicationStats = &r	}	if e.AllTierStats != nil {		ats := newAllTierStats()		ats.merge(e.AllTierStats)		e.AllTierStats = ats	}	return e} // find a path in the cache.// Returns nil if not found.func (d *dataUsageCache) find(path string) *dataUsageEntry {	due, ok := d.Cache[hashPath(path).Key()]	if !ok {		return nil	}	return &due} // isCompacted returns whether an entry is compacted.// Returns false if not found.func (d *dataUsageCache) isCompacted(h dataUsageHash) bool {	due, ok := d.Cache[h.Key()]	if !ok {		return false	}	return due.Compacted} // findChildrenCopy returns a copy of the children of the supplied hash.func (d *dataUsageCache) findChildrenCopy(h dataUsageHash) dataUsageHashMap {	ch := d.Cache[h.String()].Children	res := make(dataUsageHashMap, len(ch))	for k := range ch {		res[k] = struct{}{}	}	return res} // searchParent will search for the parent of h.// This is an O(N*N) operation if there is no parent or it cannot be guessed.func (d *dataUsageCache) searchParent(h dataUsageHash) *dataUsageHash {	want := h.Key()	if idx := strings.LastIndexByte(want, '/'); idx >= 0 {		if v := d.find(want[:idx]); v != nil {			for child := range v.Children {				if child == want {					found := hashPath(want[:idx])					return &found				}			}		}	}	for k, v := range d.Cache {		for child := range v.Children {			if child == want {				found := dataUsageHash(k)				return &found			}		}	}	return nil} // deleteRecursive will delete an entry recursively, but not change its parent.func (d *dataUsageCache) deleteRecursive(h dataUsageHash) {	if existing, ok := d.Cache[h.String()]; ok {		// Delete first if there should be a loop.		delete(d.Cache, h.Key())		for child := range existing.Children {			d.deleteRecursive(dataUsageHash(child))		}	}} // keepBuckets will keep only the buckets specified specified by delete all others.func (d *dataUsageCache) keepBuckets(b []BucketInfo) {	lu := make(map[dataUsageHash]struct{})	for _, v := range b {		lu[hashPath(v.Name)] = struct{}{}	}	d.keepRootChildren(lu)} // keepRootChildren will keep the root children specified by delete all others.

Lines 1–500 of 1341. Load more