Add input option for a file list to display total distribution.
Showing the changes to docs/debugging (1 of 1 file). Show all
Unified · Split
| @@ -18,11 +18,16 @@ · 17 unmodified lines | |||
| 18 | package main | 18 | package main |
| 19 | 19 | ||
| 20 | import ( | 20 | import ( |
| 21 | "bufio" | ||
| 22 | "bytes" | ||
| 21 | "encoding/binary" | 23 | "encoding/binary" |
| 22 | "flag" | 24 | "flag" |
| 23 | "fmt" | 25 | "fmt" |
| 24 | "hash/crc32" | 26 | "hash/crc32" |
| 27 | "io/ioutil" | ||
| 25 | "log" | 28 | "log" |
| 29 | "os" | ||
| 30 | "strings" | ||
| 26 | 31 | ||
| 27 | "github.com/dchest/siphash" | 32 | "github.com/dchest/siphash" |
| 28 | "github.com/google/uuid" | 33 | "github.com/google/uuid" |
| @@ -64,22 +69,22 @@ func hashOrder(key string, cardinality int) []int { · 35 unmodified lines | |||
| 64 | } | 69 | } |
| 65 | 70 | ||
| 66 | var ( | 71 | var ( |
| 67 | object, deploymentID string | 72 | file, object, deploymentID, prefix string |
| 68 | setCount, shards int | 73 | setCount, shards int |
| 74 | verbose bool | ||
| 69 | ) | 75 | ) |
| 70 | 76 | ||
| 71 | func main() { | 77 | func main() { |
| 78 | flag.StringVar(&file, "file", "", "Read all objects from file, newline separated") | ||
| 79 | flag.StringVar(&prefix, "prefix", "", "Add prefix to all objects") | ||
| 72 | flag.StringVar(&object, "object", "", "Select an object") | 80 | flag.StringVar(&object, "object", "", "Select an object") |
| 73 | flag.StringVar(&deploymentID, "deployment-id", "", "MinIO deployment ID, obtained from 'format.json'") | 81 | flag.StringVar(&deploymentID, "deployment-id", "", "MinIO deployment ID, obtained from 'format.json'") |
| 74 | flag.IntVar(&setCount, "set-count", 0, "Total set count") | 82 | flag.IntVar(&setCount, "set-count", 0, "Total set count") |
| 75 | flag.IntVar(&shards, "shards", 0, "Total shards count") | 83 | flag.IntVar(&shards, "shards", 0, "Total shards count") |
| 84 | flag.BoolVar(&verbose, "v", false, "Display all objects") | ||
| 76 | 85 | ||
| 77 | flag.Parse() | 86 | flag.Parse() |
| 78 | 87 | ||
| 79 | if object == "" { | ||
| 80 | log.Fatalln("object name is mandatory") | ||
| 81 | } | ||
| 82 | |||
| 83 | if deploymentID == "" { | 88 | if deploymentID == "" { |
| 84 | log.Fatalln("deployment ID is mandatory") | 89 | log.Fatalln("deployment ID is mandatory") |
| 85 | } | 90 | } |
| @@ -88,12 +93,39 @@ func main() { · 2 unmodified lines | |||
| 88 | log.Fatalln("set count cannot be zero") | 93 | log.Fatalln("set count cannot be zero") |
| 89 | } | 94 | } |
| 90 | 95 | ||
| 91 | if shards == 0 { | 96 | id := uuid.MustParse(deploymentID) |
| 92 | log.Fatalln("total shards cannot be zero") | 97 | |
| 98 | if file != "" { | ||
| 99 | distrib := make([][]string, setCount) | ||
| 100 | b, err := ioutil.ReadFile(file) | ||
| 101 | if err != nil { | ||
| 102 | log.Fatalln(err) | ||
| 103 | } | ||
| 104 | b = bytes.ReplaceAll(b, []byte("\r"), []byte{}) | ||
| 105 | sc := bufio.NewScanner(bytes.NewBuffer(b)) | ||
| 106 | for sc.Scan() { | ||
| 107 | object = strings.TrimSpace(sc.Text()) | ||
| 108 | set := sipHashMod(prefix+object, setCount, id) | ||
| 109 | distrib[set] = append(distrib[set], prefix+object) | ||
| 110 | } | ||
| 111 | for set, files := range distrib { | ||
| 112 | fmt.Println("Set:", set+1, "Objects:", len(files)) | ||
| 113 | if !verbose { | ||
| 114 | continue | ||
| 115 | } | ||
| 116 | for _, s := range files { | ||
| 117 | fmt.Printf("\t%s\n", s) | ||
| 118 | } | ||
| 119 | } | ||
| 120 | os.Exit(0) | ||
| 93 | } | 121 | } |
| 94 | 122 | ||
| 95 | id := uuid.MustParse(deploymentID) | 123 | if object == "" { |
| 124 | log.Fatalln("object name is mandatory") | ||
| 125 | } | ||
| 96 | 126 | ||
| 97 | fmt.Println("Erasure distribution for the object", hashOrder(object, shards)) | 127 | if shards != 0 { |
| 98 | fmt.Println("Erasure setNumber for the object", sipHashMod(object, setCount, id)+1) | 128 | fmt.Println("Erasure distribution for the object", hashOrder(prefix+object, shards)) |
| 129 | } | ||
| 130 | fmt.Println("Erasure setNumber for the object", sipHashMod(prefix+object, setCount, id)+1) | ||
| 99 | } | 131 | } |