Add input option for a file list to display total distribution.
Showing the changes to docs/debugging (1 of 1 file). Show all
Unified · Split
| @@ -18,11 +18,16 @@ · 17 unmodified lines | ||
| 18 | 18 | package main |
| 19 | 19 | |
| 20 | 20 | import ( |
| 21 | + "bufio" | |
| 22 | + "bytes" | |
| 21 | 23 | "encoding/binary" |
| 22 | 24 | "flag" |
| 23 | 25 | "fmt" |
| 24 | 26 | "hash/crc32" |
| 27 | + "io/ioutil" | |
| 25 | 28 | "log" |
| 29 | + "os" | |
| 30 | + "strings" | |
| 26 | 31 | |
| 27 | 32 | "github.com/dchest/siphash" |
| 28 | 33 | "github.com/google/uuid" |
| @@ -64,22 +69,22 @@ func hashOrder(key string, cardinality int) []int { · 35 unmodified lines | ||
| 64 | 69 | } |
| 65 | 70 | |
| 66 | 71 | var ( |
| 67 | − object, deploymentID string | |
| 68 | − setCount, shards int | |
| 72 | + file, object, deploymentID, prefix string | |
| 73 | + setCount, shards int | |
| 74 | + verbose bool | |
| 69 | 75 | ) |
| 70 | 76 | |
| 71 | 77 | func main() { |
| 78 | + flag.StringVar(&file, "file", "", "Read all objects from file, newline separated") | |
| 79 | + flag.StringVar(&prefix, "prefix", "", "Add prefix to all objects") | |
| 72 | 80 | flag.StringVar(&object, "object", "", "Select an object") |
| 73 | 81 | flag.StringVar(&deploymentID, "deployment-id", "", "MinIO deployment ID, obtained from 'format.json'") |
| 74 | 82 | flag.IntVar(&setCount, "set-count", 0, "Total set count") |
| 75 | 83 | flag.IntVar(&shards, "shards", 0, "Total shards count") |
| 84 | + flag.BoolVar(&verbose, "v", false, "Display all objects") | |
| 76 | 85 | |
| 77 | 86 | flag.Parse() |
| 78 | 87 | |
| 79 | − if object == "" { | |
| 80 | − log.Fatalln("object name is mandatory") | |
| 81 | − } | |
| 82 | − | |
| 83 | 88 | if deploymentID == "" { |
| 84 | 89 | log.Fatalln("deployment ID is mandatory") |
| 85 | 90 | } |
| @@ -88,12 +93,39 @@ func main() { · 2 unmodified lines | ||
| 88 | 93 | log.Fatalln("set count cannot be zero") |
| 89 | 94 | } |
| 90 | 95 | |
| 91 | − if shards == 0 { | |
| 92 | − log.Fatalln("total shards cannot be zero") | |
| 96 | + id := uuid.MustParse(deploymentID) | |
| 97 | + | |
| 98 | + if file != "" { | |
| 99 | + distrib := make([][]string, setCount) | |
| 100 | + b, err := ioutil.ReadFile(file) | |
| 101 | + if err != nil { | |
| 102 | + log.Fatalln(err) | |
| 103 | + } | |
| 104 | + b = bytes.ReplaceAll(b, []byte("\r"), []byte{}) | |
| 105 | + sc := bufio.NewScanner(bytes.NewBuffer(b)) | |
| 106 | + for sc.Scan() { | |
| 107 | + object = strings.TrimSpace(sc.Text()) | |
| 108 | + set := sipHashMod(prefix+object, setCount, id) | |
| 109 | + distrib[set] = append(distrib[set], prefix+object) | |
| 110 | + } | |
| 111 | + for set, files := range distrib { | |
| 112 | + fmt.Println("Set:", set+1, "Objects:", len(files)) | |
| 113 | + if !verbose { | |
| 114 | + continue | |
| 115 | + } | |
| 116 | + for _, s := range files { | |
| 117 | + fmt.Printf("\t%s\n", s) | |
| 118 | + } | |
| 119 | + } | |
| 120 | + os.Exit(0) | |
| 93 | 121 | } |
| 94 | 122 | |
| 95 | − id := uuid.MustParse(deploymentID) | |
| 123 | + if object == "" { | |
| 124 | + log.Fatalln("object name is mandatory") | |
| 125 | + } | |
| 96 | 126 | |
| 97 | − fmt.Println("Erasure distribution for the object", hashOrder(object, shards)) | |
| 98 | − fmt.Println("Erasure setNumber for the object", sipHashMod(object, setCount, id)+1) | |
| 127 | + if shards != 0 { | |
| 128 | + fmt.Println("Erasure distribution for the object", hashOrder(prefix+object, shards)) | |
| 129 | + } | |
| 130 | + fmt.Println("Erasure setNumber for the object", sipHashMod(prefix+object, setCount, id)+1) | |
| 99 | 131 | } |