Bonus: - heal buckets in-case during startup the new pools have bucket missing.
Unified · Split
| @@ -94,7 +94,7 @@ func loadHealingTracker(ctx context.Context, disk StorageAPI) (*healingTracker, · 93 unmodified lines | |||
| 94 | return nil, err | 94 | return nil, err |
| 95 | } | 95 | } |
| 96 | b, err := disk.ReadAll(ctx, minioMetaBucket, | 96 | b, err := disk.ReadAll(ctx, minioMetaBucket, |
| 97 | pathJoin(bucketMetaPrefix, slashSeparator, healingTrackerFilename)) | 97 | pathJoin(bucketMetaPrefix, healingTrackerFilename)) |
| 98 | if err != nil { | 98 | if err != nil { |
| 99 | return nil, err | 99 | return nil, err |
| 100 | } | 100 | } |
| @@ -155,14 +155,14 @@ func (h *healingTracker) save(ctx context.Context) error { · 54 unmodified lines | |||
| 155 | } | 155 | } |
| 156 | globalBackgroundHealState.updateHealStatus(h) | 156 | globalBackgroundHealState.updateHealStatus(h) |
| 157 | return h.disk.WriteAll(ctx, minioMetaBucket, | 157 | return h.disk.WriteAll(ctx, minioMetaBucket, |
| 158 | pathJoin(bucketMetaPrefix, slashSeparator, healingTrackerFilename), | 158 | pathJoin(bucketMetaPrefix, healingTrackerFilename), |
| 159 | htrackerBytes) | 159 | htrackerBytes) |
| 160 | } | 160 | } |
| 161 | 161 | ||
| 162 | // delete the tracker on disk. | 162 | // delete the tracker on disk. |
| 163 | func (h *healingTracker) delete(ctx context.Context) error { | 163 | func (h *healingTracker) delete(ctx context.Context) error { |
| 164 | return h.disk.Delete(ctx, minioMetaBucket, | 164 | return h.disk.Delete(ctx, minioMetaBucket, |
| 165 | pathJoin(bucketMetaPrefix, slashSeparator, healingTrackerFilename), | 165 | pathJoin(bucketMetaPrefix, healingTrackerFilename), |
| 166 | DeleteOptions{ | 166 | DeleteOptions{ |
| 167 | Recursive: false, | 167 | Recursive: false, |
| 168 | Force: false, | 168 | Force: false, |
| @@ -365,12 +365,15 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint · 196 unmodified lines | |||
| 365 | } | 365 | } |
| 366 | 366 | ||
| 367 | // Start or resume healing of this erasure set | 367 | // Start or resume healing of this erasure set |
| 368 | err = z.serverPools[poolIdx].sets[setIdx].healErasureSet(ctx, tracker.QueuedBuckets, tracker) | 368 | if err = z.serverPools[poolIdx].sets[setIdx].healErasureSet(ctx, tracker.QueuedBuckets, tracker); err != nil { |
| 369 | if err != nil { | ||
| 370 | return err | 369 | return err |
| 371 | } | 370 | } |
| 372 | 371 | ||
| 373 | logger.Info("Healing disk '%s' is complete (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed) | 372 | if tracker.ItemsFailed > 0 { |
| 373 | logger.Info("Healing disk '%s' failed (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed) | ||
| 374 | } else { | ||
| 375 | logger.Info("Healing disk '%s' complete (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed) | ||
| 376 | } | ||
| 374 | 377 | ||
| 375 | if serverDebugLog { | 378 | if serverDebugLog { |
| 376 | tracker.printTo(os.Stdout) | 379 | tracker.printTo(os.Stdout) |
| @@ -378,6 +381,7 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint · 1 unmodified line | |||
| 378 | } | 381 | } |
| 379 | 382 | ||
| 380 | logger.LogIf(ctx, tracker.delete(ctx)) | 383 | logger.LogIf(ctx, tracker.delete(ctx)) |
| 384 | |||
| 381 | return nil | 385 | return nil |
| 382 | } | 386 | } |
| 383 | 387 | ||
| @@ -394,12 +398,11 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools) { · 10 unmodified lines | |||
| 394 | case <-ctx.Done(): | 398 | case <-ctx.Done(): |
| 395 | return | 399 | return |
| 396 | case <-diskCheckTimer.C: | 400 | case <-diskCheckTimer.C: |
| 397 | |||
| 398 | healDisks := globalBackgroundHealState.getHealLocalDiskEndpoints() | 401 | healDisks := globalBackgroundHealState.getHealLocalDiskEndpoints() |
| 399 | if len(healDisks) == 0 { | 402 | if len(healDisks) == 0 { |
| 400 | // Reset for next interval. | 403 | // Reset for next interval. |
| 401 | diskCheckTimer.Reset(defaultMonitorNewDiskInterval) | 404 | diskCheckTimer.Reset(defaultMonitorNewDiskInterval) |
| 402 | break | 405 | continue |
| 403 | } | 406 | } |
| 404 | 407 | ||
| 405 | // Reformat disks immediately | 408 | // Reformat disks immediately |
| @@ -408,7 +411,7 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools) { · 2 unmodified lines | |||
| 408 | logger.LogIf(ctx, err) | 411 | logger.LogIf(ctx, err) |
| 409 | // Reset for next interval. | 412 | // Reset for next interval. |
| 410 | diskCheckTimer.Reset(defaultMonitorNewDiskInterval) | 413 | diskCheckTimer.Reset(defaultMonitorNewDiskInterval) |
| 411 | break | 414 | continue |
| 412 | } | 415 | } |
| 413 | 416 | ||
| 414 | for _, disk := range healDisks { | 417 | for _, disk := range healDisks { |
| @@ -31,6 +31,7 @@ import ( · 30 unmodified lines | |||
| 31 | "time" | 31 | "time" |
| 32 | 32 | ||
| 33 | "github.com/dustin/go-humanize" | 33 | "github.com/dustin/go-humanize" |
| 34 | "github.com/minio/madmin-go" | ||
| 34 | "github.com/minio/minio/internal/bucket/lifecycle" | 35 | "github.com/minio/minio/internal/bucket/lifecycle" |
| 35 | "github.com/minio/minio/internal/hash" | 36 | "github.com/minio/minio/internal/hash" |
| 36 | "github.com/minio/minio/internal/logger" | 37 | "github.com/minio/minio/internal/logger" |
| @@ -967,15 +968,9 @@ func (z *erasureServerPools) getDecommissionPoolSpaceInfo(idx int) (pi poolSpace · 930 unmodified lines | |||
| 967 | if idx+1 > len(z.serverPools) { | 968 | if idx+1 > len(z.serverPools) { |
| 968 | return pi, errInvalidArgument | 969 | return pi, errInvalidArgument |
| 969 | } | 970 | } |
| 971 | |||
| 970 | info, _ := z.serverPools[idx].StorageInfo(context.Background()) | 972 | info, _ := z.serverPools[idx].StorageInfo(context.Background()) |
| 971 | info.Backend = z.BackendInfo() | 973 | info.Backend = z.BackendInfo() |
| 972 | for _, disk := range info.Disks { | ||
| 973 | if disk.Healing { | ||
| 974 | return pi, decomError{ | ||
| 975 | Err: fmt.Sprintf("%s drive is healing, decommission will not be started", disk.Endpoint), | ||
| 976 | } | ||
| 977 | } | ||
| 978 | } | ||
| 979 | 974 | ||
| 980 | usableTotal := int64(GetTotalUsableCapacity(info.Disks, info)) | 975 | usableTotal := int64(GetTotalUsableCapacity(info.Disks, info)) |
| 981 | usableFree := int64(GetTotalUsableCapacityFree(info.Disks, info)) | 976 | usableFree := int64(GetTotalUsableCapacityFree(info.Disks, info)) |
| @@ -1112,6 +1107,13 @@ func (z *erasureServerPools) StartDecommission(ctx context.Context, idx int) (er · 130 unmodified lines | |||
| 1112 | return err | 1107 | return err |
| 1113 | } | 1108 | } |
| 1114 | 1109 | ||
| 1110 | // Make sure to heal the buckets to ensure the new | ||
| 1111 | // pool has the new buckets, this is to avoid | ||
| 1112 | // failures later. | ||
| 1113 | for _, bucket := range buckets { | ||
| 1114 | z.HealBucket(ctx, bucket.Name, madmin.HealOpts{}) | ||
| 1115 | } | ||
| 1116 | |||
| 1115 | decomBuckets := make([]decomBucketInfo, len(buckets)) | 1117 | decomBuckets := make([]decomBucketInfo, len(buckets)) |
| 1116 | for i := range buckets { | 1118 | for i := range buckets { |
| 1117 | decomBuckets[i] = decomBucketInfo{ | 1119 | decomBuckets[i] = decomBucketInfo{ |