Bonus: - heal buckets in-case during startup the new pools have bucket missing.
Unified · Split
| @@ -94,7 +94,7 @@ func loadHealingTracker(ctx context.Context, disk StorageAPI) (*healingTracker, · 93 unmodified lines | ||
| 94 | 94 | return nil, err |
| 95 | 95 | } |
| 96 | 96 | b, err := disk.ReadAll(ctx, minioMetaBucket, |
| 97 | − pathJoin(bucketMetaPrefix, slashSeparator, healingTrackerFilename)) | |
| 97 | + pathJoin(bucketMetaPrefix, healingTrackerFilename)) | |
| 98 | 98 | if err != nil { |
| 99 | 99 | return nil, err |
| 100 | 100 | } |
| @@ -155,14 +155,14 @@ func (h *healingTracker) save(ctx context.Context) error { · 54 unmodified lines | ||
| 155 | 155 | } |
| 156 | 156 | globalBackgroundHealState.updateHealStatus(h) |
| 157 | 157 | return h.disk.WriteAll(ctx, minioMetaBucket, |
| 158 | − pathJoin(bucketMetaPrefix, slashSeparator, healingTrackerFilename), | |
| 158 | + pathJoin(bucketMetaPrefix, healingTrackerFilename), | |
| 159 | 159 | htrackerBytes) |
| 160 | 160 | } |
| 161 | 161 | |
| 162 | 162 | // delete the tracker on disk. |
| 163 | 163 | func (h *healingTracker) delete(ctx context.Context) error { |
| 164 | 164 | return h.disk.Delete(ctx, minioMetaBucket, |
| 165 | − pathJoin(bucketMetaPrefix, slashSeparator, healingTrackerFilename), | |
| 165 | + pathJoin(bucketMetaPrefix, healingTrackerFilename), | |
| 166 | 166 | DeleteOptions{ |
| 167 | 167 | Recursive: false, |
| 168 | 168 | Force: false, |
| @@ -365,12 +365,15 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint · 196 unmodified lines | ||
| 365 | 365 | } |
| 366 | 366 | |
| 367 | 367 | // Start or resume healing of this erasure set |
| 368 | − err = z.serverPools[poolIdx].sets[setIdx].healErasureSet(ctx, tracker.QueuedBuckets, tracker) | |
| 369 | − if err != nil { | |
| 368 | + if err = z.serverPools[poolIdx].sets[setIdx].healErasureSet(ctx, tracker.QueuedBuckets, tracker); err != nil { | |
| 370 | 369 | return err |
| 371 | 370 | } |
| 372 | 371 | |
| 373 | − logger.Info("Healing disk '%s' is complete (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed) | |
| 372 | + if tracker.ItemsFailed > 0 { | |
| 373 | + logger.Info("Healing disk '%s' failed (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed) | |
| 374 | + } else { | |
| 375 | + logger.Info("Healing disk '%s' complete (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed) | |
| 376 | + } | |
| 374 | 377 | |
| 375 | 378 | if serverDebugLog { |
| 376 | 379 | tracker.printTo(os.Stdout) |
| @@ -378,6 +381,7 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint · 1 unmodified line | ||
| 378 | 381 | } |
| 379 | 382 | |
| 380 | 383 | logger.LogIf(ctx, tracker.delete(ctx)) |
| 384 | + | |
| 381 | 385 | return nil |
| 382 | 386 | } |
| 383 | 387 | |
| @@ -394,12 +398,11 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools) { · 10 unmodified lines | ||
| 394 | 398 | case <-ctx.Done(): |
| 395 | 399 | return |
| 396 | 400 | case <-diskCheckTimer.C: |
| 397 | − | |
| 398 | 401 | healDisks := globalBackgroundHealState.getHealLocalDiskEndpoints() |
| 399 | 402 | if len(healDisks) == 0 { |
| 400 | 403 | // Reset for next interval. |
| 401 | 404 | diskCheckTimer.Reset(defaultMonitorNewDiskInterval) |
| 402 | − break | |
| 405 | + continue | |
| 403 | 406 | } |
| 404 | 407 | |
| 405 | 408 | // Reformat disks immediately |
| @@ -408,7 +411,7 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools) { · 2 unmodified lines | ||
| 408 | 411 | logger.LogIf(ctx, err) |
| 409 | 412 | // Reset for next interval. |
| 410 | 413 | diskCheckTimer.Reset(defaultMonitorNewDiskInterval) |
| 411 | − break | |
| 414 | + continue | |
| 412 | 415 | } |
| 413 | 416 | |
| 414 | 417 | for _, disk := range healDisks { |
| @@ -31,6 +31,7 @@ import ( · 30 unmodified lines | ||
| 31 | 31 | "time" |
| 32 | 32 | |
| 33 | 33 | "github.com/dustin/go-humanize" |
| 34 | + "github.com/minio/madmin-go" | |
| 34 | 35 | "github.com/minio/minio/internal/bucket/lifecycle" |
| 35 | 36 | "github.com/minio/minio/internal/hash" |
| 36 | 37 | "github.com/minio/minio/internal/logger" |
| @@ -967,15 +968,9 @@ func (z *erasureServerPools) getDecommissionPoolSpaceInfo(idx int) (pi poolSpace · 930 unmodified lines | ||
| 967 | 968 | if idx+1 > len(z.serverPools) { |
| 968 | 969 | return pi, errInvalidArgument |
| 969 | 970 | } |
| 971 | + | |
| 970 | 972 | info, _ := z.serverPools[idx].StorageInfo(context.Background()) |
| 971 | 973 | info.Backend = z.BackendInfo() |
| 972 | − for _, disk := range info.Disks { | |
| 973 | − if disk.Healing { | |
| 974 | − return pi, decomError{ | |
| 975 | − Err: fmt.Sprintf("%s drive is healing, decommission will not be started", disk.Endpoint), | |
| 976 | − } | |
| 977 | − } | |
| 978 | − } | |
| 979 | 974 | |
| 980 | 975 | usableTotal := int64(GetTotalUsableCapacity(info.Disks, info)) |
| 981 | 976 | usableFree := int64(GetTotalUsableCapacityFree(info.Disks, info)) |
| @@ -1112,6 +1107,13 @@ func (z *erasureServerPools) StartDecommission(ctx context.Context, idx int) (er · 130 unmodified lines | ||
| 1112 | 1107 | return err |
| 1113 | 1108 | } |
| 1114 | 1109 | |
| 1110 | + // Make sure to heal the buckets to ensure the new | |
| 1111 | + // pool has the new buckets, this is to avoid | |
| 1112 | + // failures later. | |
| 1113 | + for _, bucket := range buckets { | |
| 1114 | + z.HealBucket(ctx, bucket.Name, madmin.HealOpts{}) | |
| 1115 | + } | |
| 1116 | + | |
| 1115 | 1117 | decomBuckets := make([]decomBucketInfo, len(buckets)) |
| 1116 | 1118 | for i := range buckets { |
| 1117 | 1119 | decomBuckets[i] = decomBucketInfo{ |