History

reject resync start on misconfigured replication rules (#15041)

we expect resync to start on buckets with replication rule ExistingObjects enabled, if not we reject such calls.

Harshavardhana committed 4 years ago

Commit 48e367f1 parent fd024928 files · +519 −383Browse files

Unified · Split

8 files
  1. Makefile +1 −0
  2. cmd/api-errors.go +4 −4
  3. cmd/apierrorcode_string.go +3 −3
  4. cmd/bucket-handlers.go +0 −371
  5. cmd/bucket-replication-handlers.go +404 −0
  6. cmd/bucket-replication.go +2 −5
  7. docs/bucket/replication/setup_2site_existing_replication.sh +93 −0
  8. internal/bucket/replication/replication.go +12 −0
Makefile+1−0
@@ -58,6 +58,7 @@ test-iam: build ## verify IAM (external IDP, etcd backends) · 57 unmodified lines
5858 test-replication: install ## verify multi site replication
5959 @echo "Running tests for replicating three sites"
6060 @(env bash $(PWD)/docs/bucket/replication/setup_3site_replication.sh)
61+ @(env bash $(PWD)/docs/bucket/replication/setup_2site_existing_replication.sh)
6162
6263 test-site-replication-ldap: install ## verify automatic site replication
6364 @echo "Running tests for automatic site replication of IAM (with LDAP)"
cmd/api-errors.go+4−4
@@ -131,7 +131,7 @@ const ( · 130 unmodified lines
131131 ErrReplicationNeedsVersioningError
132132 ErrReplicationBucketNeedsVersioningError
133133 ErrReplicationDenyEditError
134− ErrReplicationNoMatchingRuleError
134+ ErrReplicationNoExistingObjects
135135 ErrObjectRestoreAlreadyInProgress
136136 ErrNoSuchKey
137137 ErrNoSuchUpload
@@ -893,9 +893,9 @@ var errorCodes = errorCodeMap{ · 755 unmodified lines
893893 Description: "Bandwidth limit for remote target must be atleast 100MBps",
894894 HTTPStatusCode: http.StatusBadRequest,
895895 },
896− ErrReplicationNoMatchingRuleError: {
897− Code: "XMinioReplicationNoMatchingRule",
898− Description: "No matching replication rule found for this object prefix",
896+ ErrReplicationNoExistingObjects: {
897+ Code: "XMinioReplicationNoExistingObjects",
898+ Description: "No matching ExistingsObjects rule enabled",
899899 HTTPStatusCode: http.StatusBadRequest,
900900 },
901901 ErrReplicationDenyEditError: {
cmd/apierrorcode_string.go+3−3
@@ -66,7 +66,7 @@ func _() { · 65 unmodified lines
6666 _ = x[ErrReplicationNeedsVersioningError-55]
6767 _ = x[ErrReplicationBucketNeedsVersioningError-56]
6868 _ = x[ErrReplicationDenyEditError-57]
69− _ = x[ErrReplicationNoMatchingRuleError-58]
69+ _ = x[ErrReplicationNoExistingObjects-58]
7070 _ = x[ErrObjectRestoreAlreadyInProgress-59]
7171 _ = x[ErrNoSuchKey-60]
7272 _ = x[ErrNoSuchUpload-61]
@@ -299,9 +299,9 @@ func _() { · 226 unmodified lines
299299 _ = x[ErrPostPolicyConditionInvalidFormat-288]
300300 }
301301
302−const _APIErrorCode_name = "NoneAccessDeniedBadDigestEntityTooSmallEntityTooLargePolicyTooLargeIncompleteBodyInternalErrorInvalidAccessKeyIDAccessKeyDisabledInvalidBucketNameInvalidDigestInvalidRangeInvalidRangePartNumberInvalidCopyPartRangeInvalidCopyPartRangeSourceInvalidMaxKeysInvalidEncodingMethodInvalidMaxUploadsInvalidMaxPartsInvalidPartNumberMarkerInvalidPartNumberInvalidRequestBodyInvalidCopySourceInvalidMetadataDirectiveInvalidCopyDestInvalidPolicyDocumentInvalidObjectStateMalformedXMLMissingContentLengthMissingContentMD5MissingRequestBodyErrorMissingSecurityHeaderNoSuchBucketNoSuchBucketPolicyNoSuchBucketLifecycleNoSuchLifecycleConfigurationInvalidLifecycleWithObjectLockNoSuchBucketSSEConfigNoSuchCORSConfigurationNoSuchWebsiteConfigurationReplicationConfigurationNotFoundErrorRemoteDestinationNotFoundErrorReplicationDestinationMissingLockRemoteTargetNotFoundErrorReplicationRemoteConnectionErrorReplicationBandwidthLimitErrorBucketRemoteIdenticalToSourceBucketRemoteAlreadyExistsBucketRemoteLabelInUseBucketRemoteArnTypeInvalidBucketRemoteArnInvalidBucketRemoteRemoveDisallowedRemoteTargetNotVersionedErrorReplicationSourceNotVersionedErrorReplicationNeedsVersioningErrorReplicationBucketNeedsVersioningErrorReplicationDenyEditErrorReplicationNoMatchingRuleErrorObjectRestoreAlreadyInProgressNoSuchKeyNoSuchUploadInvalidVersionIDNoSuchVersionNotImplementedPreconditionFailedRequestTimeTooSkewedSignatureDoesNotMatchMethodNotAllowedInvalidPartInvalidPartOrderAuthorizationHeaderMalformedMalformedPOSTRequestPOSTFileRequiredSignatureVersionNotSupportedBucketNotEmptyAllAccessDisabledMalformedPolicyMissingFieldsMissingCredTagCredMalformedInvalidRegionInvalidServiceS3InvalidServiceSTSInvalidRequestVersionMissingSignTagMissingSignHeadersTagMalformedDateMalformedPresignedDateMalformedCredentialDateMalformedCredentialRegionMalformedExpiresNegativeExpiresAuthHeaderEmptyExpiredPresignRequestRequestNotReadyYetUnsignedHeadersMissingDateHeaderInvalidQuerySignatureAlgoInvalidQueryParamsBucketAlreadyOwnedByYouInvalidDurationBucketAlreadyExistsMetadataTooLargeUnsupportedMetadataMaximumExpiresSlowDownInvalidPrefixMarkerBadRequestKeyTooLongErrorInvalidBucketObjectLockConfigurationObjectLockConfigurationNotFoundObjectLockConfigurationNotAllowedNoSuchObjectLockConfigurationObjectLockedInvalidRetentionDatePastObjectLockRetainDateUnknownWORMModeDirectiveBucketTaggingNotFoundObjectLockInvalidHeadersInvalidTagDirectiveInvalidEncryptionMethodInsecureSSECustomerRequestSSEMultipartEncryptedSSEEncryptedObjectInvalidEncryptionParametersInvalidSSECustomerAlgorithmInvalidSSECustomerKeyMissingSSECustomerKeyMissingSSECustomerKeyMD5SSECustomerKeyMD5MismatchInvalidSSECustomerParametersIncompatibleEncryptionMethodKMSNotConfiguredKMSKeyNotFoundExceptionNoAccessKeyInvalidTokenEventNotificationARNNotificationRegionNotificationOverlappingFilterNotificationFilterNameInvalidFilterNamePrefixFilterNameSuffixFilterValueInvalidOverlappingConfigsUnsupportedNotificationContentSHA256MismatchReadQuorumWriteQuorumStorageFullRequestBodyParseObjectExistsAsDirectoryInvalidObjectNameInvalidObjectNamePrefixSlashInvalidResourceNameServerNotInitializedOperationTimedOutClientDisconnectedOperationMaxedOutInvalidRequestTransitionStorageClassNotFoundErrorInvalidStorageClassBackendDownMalformedJSONAdminNoSuchUserAdminNoSuchGroupAdminGroupNotEmptyAdminNoSuchPolicyAdminInvalidArgumentAdminInvalidAccessKeyAdminInvalidSecretKeyAdminConfigNoQuorumAdminConfigTooLargeAdminConfigBadJSONAdminConfigDuplicateKeysAdminCredentialsMismatchInsecureClientRequestObjectTamperedSiteReplicationInvalidRequestSiteReplicationPeerRespSiteReplicationBackendIssueSiteReplicationServiceAccountErrorSiteReplicationBucketConfigErrorSiteReplicationBucketMetaErrorSiteReplicationIAMErrorSiteReplicationConfigMissingAdminBucketQuotaExceededAdminNoSuchQuotaConfigurationHealNotImplementedHealNoSuchProcessHealInvalidClientTokenHealMissingBucketHealAlreadyRunningHealOverlappingPathsIncorrectContinuationTokenEmptyRequestBodyUnsupportedFunctionInvalidExpressionTypeBusyUnauthorizedAccessExpressionTooLongIllegalSQLFunctionArgumentInvalidKeyPathInvalidCompressionFormatInvalidFileHeaderInfoInvalidJSONTypeInvalidQuoteFieldsInvalidRequestParameterInvalidDataTypeInvalidTextEncodingInvalidDataSourceInvalidTableAliasMissingRequiredParameterObjectSerializationConflictUnsupportedSQLOperationUnsupportedSQLStructureUnsupportedSyntaxUnsupportedRangeHeaderLexerInvalidCharLexerInvalidOperatorLexerInvalidLiteralLexerInvalidIONLiteralParseExpectedDatePartParseExpectedKeywordParseExpectedTokenTypeParseExpected2TokenTypesParseExpectedNumberParseExpectedRightParenBuiltinFunctionCallParseExpectedTypeNameParseExpectedWhenClauseParseUnsupportedTokenParseUnsupportedLiteralsGroupByParseExpectedMemberParseUnsupportedSelectParseUnsupportedCaseParseUnsupportedCaseClauseParseUnsupportedAliasParseUnsupportedSyntaxParseUnknownOperatorParseMissingIdentAfterAtParseUnexpectedOperatorParseUnexpectedTermParseUnexpectedTokenParseUnexpectedKeywordParseExpectedExpressionParseExpectedLeftParenAfterCastParseExpectedLeftParenValueConstructorParseExpectedLeftParenBuiltinFunctionCallParseExpectedArgumentDelimiterParseCastArityParseInvalidTypeParamParseEmptySelectParseSelectMissingFromParseExpectedIdentForGroupNameParseExpectedIdentForAliasParseUnsupportedCallWithStarParseNonUnaryAgregateFunctionCallParseMalformedJoinParseExpectedIdentForAtParseAsteriskIsNotAloneInSelectListParseCannotMixSqbAndWildcardInSelectListParseInvalidContextForWildcardInSelectListIncorrectSQLFunctionArgumentTypeValueParseFailureEvaluatorInvalidArgumentsIntegerOverflowLikeInvalidInputsCastFailedInvalidCastEvaluatorInvalidTimestampFormatPatternEvaluatorInvalidTimestampFormatPatternSymbolForParsingEvaluatorTimestampFormatPatternDuplicateFieldsEvaluatorTimestampFormatPatternHourClockAmPmMismatchEvaluatorUnterminatedTimestampFormatPatternTokenEvaluatorInvalidTimestampFormatPatternTokenEvaluatorInvalidTimestampFormatPatternSymbolEvaluatorBindingDoesNotExistMissingHeadersInvalidColumnIndexAdminConfigNotificationTargetsFailedAdminProfilerNotEnabledInvalidDecompressedSizeAddUserInvalidArgumentAdminResourceInvalidArgumentAdminAccountNotEligibleAccountNotEligibleAdminServiceAccountNotFoundPostPolicyConditionInvalidFormat"
302+const _APIErrorCode_name = "NoneAccessDeniedBadDigestEntityTooSmallEntityTooLargePolicyTooLargeIncompleteBodyInternalErrorInvalidAccessKeyIDAccessKeyDisabledInvalidBucketNameInvalidDigestInvalidRangeInvalidRangePartNumberInvalidCopyPartRangeInvalidCopyPartRangeSourceInvalidMaxKeysInvalidEncodingMethodInvalidMaxUploadsInvalidMaxPartsInvalidPartNumberMarkerInvalidPartNumberInvalidRequestBodyInvalidCopySourceInvalidMetadataDirectiveInvalidCopyDestInvalidPolicyDocumentInvalidObjectStateMalformedXMLMissingContentLengthMissingContentMD5MissingRequestBodyErrorMissingSecurityHeaderNoSuchBucketNoSuchBucketPolicyNoSuchBucketLifecycleNoSuchLifecycleConfigurationInvalidLifecycleWithObjectLockNoSuchBucketSSEConfigNoSuchCORSConfigurationNoSuchWebsiteConfigurationReplicationConfigurationNotFoundErrorRemoteDestinationNotFoundErrorReplicationDestinationMissingLockRemoteTargetNotFoundErrorReplicationRemoteConnectionErrorReplicationBandwidthLimitErrorBucketRemoteIdenticalToSourceBucketRemoteAlreadyExistsBucketRemoteLabelInUseBucketRemoteArnTypeInvalidBucketRemoteArnInvalidBucketRemoteRemoveDisallowedRemoteTargetNotVersionedErrorReplicationSourceNotVersionedErrorReplicationNeedsVersioningErrorReplicationBucketNeedsVersioningErrorReplicationDenyEditErrorReplicationNoExistingObjectsObjectRestoreAlreadyInProgressNoSuchKeyNoSuchUploadInvalidVersionIDNoSuchVersionNotImplementedPreconditionFailedRequestTimeTooSkewedSignatureDoesNotMatchMethodNotAllowedInvalidPartInvalidPartOrderAuthorizationHeaderMalformedMalformedPOSTRequestPOSTFileRequiredSignatureVersionNotSupportedBucketNotEmptyAllAccessDisabledMalformedPolicyMissingFieldsMissingCredTagCredMalformedInvalidRegionInvalidServiceS3InvalidServiceSTSInvalidRequestVersionMissingSignTagMissingSignHeadersTagMalformedDateMalformedPresignedDateMalformedCredentialDateMalformedCredentialRegionMalformedExpiresNegativeExpiresAuthHeaderEmptyExpiredPresignRequestRequestNotReadyYetUnsignedHeadersMissingDateHeaderInvalidQuerySignatureAlgoInvalidQueryParamsBucketAlreadyOwnedByYouInvalidDurationBucketAlreadyExistsMetadataTooLargeUnsupportedMetadataMaximumExpiresSlowDownInvalidPrefixMarkerBadRequestKeyTooLongErrorInvalidBucketObjectLockConfigurationObjectLockConfigurationNotFoundObjectLockConfigurationNotAllowedNoSuchObjectLockConfigurationObjectLockedInvalidRetentionDatePastObjectLockRetainDateUnknownWORMModeDirectiveBucketTaggingNotFoundObjectLockInvalidHeadersInvalidTagDirectiveInvalidEncryptionMethodInsecureSSECustomerRequestSSEMultipartEncryptedSSEEncryptedObjectInvalidEncryptionParametersInvalidSSECustomerAlgorithmInvalidSSECustomerKeyMissingSSECustomerKeyMissingSSECustomerKeyMD5SSECustomerKeyMD5MismatchInvalidSSECustomerParametersIncompatibleEncryptionMethodKMSNotConfiguredKMSKeyNotFoundExceptionNoAccessKeyInvalidTokenEventNotificationARNNotificationRegionNotificationOverlappingFilterNotificationFilterNameInvalidFilterNamePrefixFilterNameSuffixFilterValueInvalidOverlappingConfigsUnsupportedNotificationContentSHA256MismatchReadQuorumWriteQuorumStorageFullRequestBodyParseObjectExistsAsDirectoryInvalidObjectNameInvalidObjectNamePrefixSlashInvalidResourceNameServerNotInitializedOperationTimedOutClientDisconnectedOperationMaxedOutInvalidRequestTransitionStorageClassNotFoundErrorInvalidStorageClassBackendDownMalformedJSONAdminNoSuchUserAdminNoSuchGroupAdminGroupNotEmptyAdminNoSuchPolicyAdminInvalidArgumentAdminInvalidAccessKeyAdminInvalidSecretKeyAdminConfigNoQuorumAdminConfigTooLargeAdminConfigBadJSONAdminConfigDuplicateKeysAdminCredentialsMismatchInsecureClientRequestObjectTamperedSiteReplicationInvalidRequestSiteReplicationPeerRespSiteReplicationBackendIssueSiteReplicationServiceAccountErrorSiteReplicationBucketConfigErrorSiteReplicationBucketMetaErrorSiteReplicationIAMErrorSiteReplicationConfigMissingAdminBucketQuotaExceededAdminNoSuchQuotaConfigurationHealNotImplementedHealNoSuchProcessHealInvalidClientTokenHealMissingBucketHealAlreadyRunningHealOverlappingPathsIncorrectContinuationTokenEmptyRequestBodyUnsupportedFunctionInvalidExpressionTypeBusyUnauthorizedAccessExpressionTooLongIllegalSQLFunctionArgumentInvalidKeyPathInvalidCompressionFormatInvalidFileHeaderInfoInvalidJSONTypeInvalidQuoteFieldsInvalidRequestParameterInvalidDataTypeInvalidTextEncodingInvalidDataSourceInvalidTableAliasMissingRequiredParameterObjectSerializationConflictUnsupportedSQLOperationUnsupportedSQLStructureUnsupportedSyntaxUnsupportedRangeHeaderLexerInvalidCharLexerInvalidOperatorLexerInvalidLiteralLexerInvalidIONLiteralParseExpectedDatePartParseExpectedKeywordParseExpectedTokenTypeParseExpected2TokenTypesParseExpectedNumberParseExpectedRightParenBuiltinFunctionCallParseExpectedTypeNameParseExpectedWhenClauseParseUnsupportedTokenParseUnsupportedLiteralsGroupByParseExpectedMemberParseUnsupportedSelectParseUnsupportedCaseParseUnsupportedCaseClauseParseUnsupportedAliasParseUnsupportedSyntaxParseUnknownOperatorParseMissingIdentAfterAtParseUnexpectedOperatorParseUnexpectedTermParseUnexpectedTokenParseUnexpectedKeywordParseExpectedExpressionParseExpectedLeftParenAfterCastParseExpectedLeftParenValueConstructorParseExpectedLeftParenBuiltinFunctionCallParseExpectedArgumentDelimiterParseCastArityParseInvalidTypeParamParseEmptySelectParseSelectMissingFromParseExpectedIdentForGroupNameParseExpectedIdentForAliasParseUnsupportedCallWithStarParseNonUnaryAgregateFunctionCallParseMalformedJoinParseExpectedIdentForAtParseAsteriskIsNotAloneInSelectListParseCannotMixSqbAndWildcardInSelectListParseInvalidContextForWildcardInSelectListIncorrectSQLFunctionArgumentTypeValueParseFailureEvaluatorInvalidArgumentsIntegerOverflowLikeInvalidInputsCastFailedInvalidCastEvaluatorInvalidTimestampFormatPatternEvaluatorInvalidTimestampFormatPatternSymbolForParsingEvaluatorTimestampFormatPatternDuplicateFieldsEvaluatorTimestampFormatPatternHourClockAmPmMismatchEvaluatorUnterminatedTimestampFormatPatternTokenEvaluatorInvalidTimestampFormatPatternTokenEvaluatorInvalidTimestampFormatPatternSymbolEvaluatorBindingDoesNotExistMissingHeadersInvalidColumnIndexAdminConfigNotificationTargetsFailedAdminProfilerNotEnabledInvalidDecompressedSizeAddUserInvalidArgumentAdminResourceInvalidArgumentAdminAccountNotEligibleAccountNotEligibleAdminServiceAccountNotFoundPostPolicyConditionInvalidFormat"
303303
304−var _APIErrorCode_index = [...]uint16{0, 4, 16, 25, 39, 53, 67, 81, 94, 112, 129, 146, 159, 171, 193, 213, 239, 253, 274, 291, 306, 329, 346, 364, 381, 405, 420, 441, 459, 471, 491, 508, 531, 552, 564, 582, 603, 631, 661, 682, 705, 731, 768, 798, 831, 856, 888, 918, 947, 972, 994, 1020, 1042, 1070, 1099, 1133, 1164, 1201, 1225, 1255, 1285, 1294, 1306, 1322, 1335, 1349, 1367, 1387, 1408, 1424, 1435, 1451, 1479, 1499, 1515, 1543, 1557, 1574, 1589, 1602, 1616, 1629, 1642, 1658, 1675, 1696, 1710, 1731, 1744, 1766, 1789, 1814, 1830, 1845, 1860, 1881, 1899, 1914, 1931, 1956, 1974, 1997, 2012, 2031, 2047, 2066, 2080, 2088, 2107, 2117, 2132, 2168, 2199, 2232, 2261, 2273, 2293, 2317, 2341, 2362, 2386, 2405, 2428, 2454, 2475, 2493, 2520, 2547, 2568, 2589, 2613, 2638, 2666, 2694, 2710, 2733, 2744, 2756, 2773, 2788, 2806, 2835, 2852, 2868, 2884, 2902, 2920, 2943, 2964, 2974, 2985, 2996, 3012, 3035, 3052, 3080, 3099, 3119, 3136, 3154, 3171, 3185, 3220, 3239, 3250, 3263, 3278, 3294, 3312, 3329, 3349, 3370, 3391, 3410, 3429, 3447, 3471, 3495, 3516, 3530, 3559, 3582, 3609, 3643, 3675, 3705, 3728, 3756, 3780, 3809, 3827, 3844, 3866, 3883, 3901, 3921, 3947, 3963, 3982, 4003, 4007, 4025, 4042, 4068, 4082, 4106, 4127, 4142, 4160, 4183, 4198, 4217, 4234, 4251, 4275, 4302, 4325, 4348, 4365, 4387, 4403, 4423, 4442, 4464, 4485, 4505, 4527, 4551, 4570, 4612, 4633, 4656, 4677, 4708, 4727, 4749, 4769, 4795, 4816, 4838, 4858, 4882, 4905, 4924, 4944, 4966, 4989, 5020, 5058, 5099, 5129, 5143, 5164, 5180, 5202, 5232, 5258, 5286, 5319, 5337, 5360, 5395, 5435, 5477, 5509, 5526, 5551, 5566, 5583, 5593, 5604, 5642, 5696, 5742, 5794, 5842, 5885, 5929, 5957, 5971, 5989, 6025, 6048, 6071, 6093, 6121, 6144, 6162, 6189, 6221}
304+var _APIErrorCode_index = [...]uint16{0, 4, 16, 25, 39, 53, 67, 81, 94, 112, 129, 146, 159, 171, 193, 213, 239, 253, 274, 291, 306, 329, 346, 364, 381, 405, 420, 441, 459, 471, 491, 508, 531, 552, 564, 582, 603, 631, 661, 682, 705, 731, 768, 798, 831, 856, 888, 918, 947, 972, 994, 1020, 1042, 1070, 1099, 1133, 1164, 1201, 1225, 1253, 1283, 1292, 1304, 1320, 1333, 1347, 1365, 1385, 1406, 1422, 1433, 1449, 1477, 1497, 1513, 1541, 1555, 1572, 1587, 1600, 1614, 1627, 1640, 1656, 1673, 1694, 1708, 1729, 1742, 1764, 1787, 1812, 1828, 1843, 1858, 1879, 1897, 1912, 1929, 1954, 1972, 1995, 2010, 2029, 2045, 2064, 2078, 2086, 2105, 2115, 2130, 2166, 2197, 2230, 2259, 2271, 2291, 2315, 2339, 2360, 2384, 2403, 2426, 2452, 2473, 2491, 2518, 2545, 2566, 2587, 2611, 2636, 2664, 2692, 2708, 2731, 2742, 2754, 2771, 2786, 2804, 2833, 2850, 2866, 2882, 2900, 2918, 2941, 2962, 2972, 2983, 2994, 3010, 3033, 3050, 3078, 3097, 3117, 3134, 3152, 3169, 3183, 3218, 3237, 3248, 3261, 3276, 3292, 3310, 3327, 3347, 3368, 3389, 3408, 3427, 3445, 3469, 3493, 3514, 3528, 3557, 3580, 3607, 3641, 3673, 3703, 3726, 3754, 3778, 3807, 3825, 3842, 3864, 3881, 3899, 3919, 3945, 3961, 3980, 4001, 4005, 4023, 4040, 4066, 4080, 4104, 4125, 4140, 4158, 4181, 4196, 4215, 4232, 4249, 4273, 4300, 4323, 4346, 4363, 4385, 4401, 4421, 4440, 4462, 4483, 4503, 4525, 4549, 4568, 4610, 4631, 4654, 4675, 4706, 4725, 4747, 4767, 4793, 4814, 4836, 4856, 4880, 4903, 4922, 4942, 4964, 4987, 5018, 5056, 5097, 5127, 5141, 5162, 5178, 5200, 5230, 5256, 5284, 5317, 5335, 5358, 5393, 5433, 5475, 5507, 5524, 5549, 5564, 5581, 5591, 5602, 5640, 5694, 5740, 5792, 5840, 5883, 5927, 5955, 5969, 5987, 6023, 6046, 6069, 6091, 6119, 6142, 6160, 6187, 6219}
305305
306306 func (i APIErrorCode) String() string {
307307 if i < 0 || i >= APIErrorCode(len(_APIErrorCode_index)-1) {
cmd/bucket-handlers.go+0−371
@@ -21,7 +21,6 @@ import ( · 20 unmodified lines
2121 "bytes"
2222 "context"
2323 "encoding/base64"
24− "encoding/json"
2524 "encoding/xml"
2625 "fmt"
2726 "io"
@@ -33,7 +32,6 @@ import ( · 5 unmodified lines
3332 "strconv"
3433 "strings"
3534 "sync"
36− "time"
3735
3836 "github.com/google/uuid"
3937 "github.com/gorilla/mux"
@@ -1596,372 +1594,3 @@ func (api objectAPIHandlers) DeleteBucketTaggingHandler(w http.ResponseWriter, r · 1556 unmodified lines
15961594 // Write success response.
15971595 writeSuccessResponseHeadersOnly(w)
15981596 }
1599−
1600−// PutBucketReplicationConfigHandler - PUT Bucket replication configuration.
1601−// ----------
1602−// Add a replication configuration on the specified bucket as specified in https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketReplication.html
1603−func (api objectAPIHandlers) PutBucketReplicationConfigHandler(w http.ResponseWriter, r *http.Request) {
1604− ctx := newContext(r, w, "PutBucketReplicationConfig")
1605− defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
1606−
1607− vars := mux.Vars(r)
1608− bucket := vars["bucket"]
1609− objectAPI := api.ObjectAPI()
1610− if objectAPI == nil {
1611− writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
1612− return
1613− }
1614− if globalIsGateway {
1615− writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
1616− return
1617− }
1618− if s3Error := checkRequestAuthType(ctx, r, policy.PutReplicationConfigurationAction, bucket, ""); s3Error != ErrNone {
1619− writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
1620− return
1621− }
1622− // Check if bucket exists.
1623− if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
1624− writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
1625− return
1626− }
1627−
1628− if versioned := globalBucketVersioningSys.Enabled(bucket); !versioned {
1629− writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationNeedsVersioningError), r.URL)
1630− return
1631− }
1632− replicationConfig, err := replication.ParseConfig(io.LimitReader(r.Body, r.ContentLength))
1633− if err != nil {
1634− apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
1635− apiErr.Description = err.Error()
1636− writeErrorResponse(ctx, w, apiErr, r.URL)
1637− return
1638− }
1639− sameTarget, apiErr := validateReplicationDestination(ctx, bucket, replicationConfig, true)
1640− if apiErr != noError {
1641− writeErrorResponse(ctx, w, apiErr, r.URL)
1642− return
1643− }
1644− // Validate the received bucket replication config
1645− if err = replicationConfig.Validate(bucket, sameTarget); err != nil {
1646− writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
1647− return
1648− }
1649− configData, err := xml.Marshal(replicationConfig)
1650− if err != nil {
1651− writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
1652− return
1653− }
1654− if err = globalBucketMetadataSys.Update(ctx, bucket, bucketReplicationConfig, configData); err != nil {
1655− writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
1656− return
1657− }
1658−
1659− // Write success response.
1660− writeSuccessResponseHeadersOnly(w)
1661−}
1662−
1663−// GetBucketReplicationConfigHandler - GET Bucket replication configuration.
1664−// ----------
1665−// Gets the replication configuration for a bucket.
1666−func (api objectAPIHandlers) GetBucketReplicationConfigHandler(w http.ResponseWriter, r *http.Request) {
1667− ctx := newContext(r, w, "GetBucketReplicationConfig")
1668−
1669− defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
1670−
1671− vars := mux.Vars(r)
1672− bucket := vars["bucket"]
1673−
1674− objectAPI := api.ObjectAPI()
1675− if objectAPI == nil {
1676− writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
1677− return
1678− }
1679−
1680− // check if user has permissions to perform this operation
1681− if s3Error := checkRequestAuthType(ctx, r, policy.GetReplicationConfigurationAction, bucket, ""); s3Error != ErrNone {
1682− writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
1683− return
1684− }
1685− // Check if bucket exists.
1686− if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
1687− writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
1688− return
1689− }
1690−
1691− config, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket)
1692− if err != nil {
1693− writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
1694− return
1695− }
1696− configData, err := xml.Marshal(config)
1697− if err != nil {
1698− writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
1699− return
1700− }
1701−
1702− // Write success response.
1703− writeSuccessResponseXML(w, configData)
1704−}
1705−
1706−// DeleteBucketReplicationConfigHandler - DELETE Bucket replication config.
1707−// ----------
1708−func (api objectAPIHandlers) DeleteBucketReplicationConfigHandler(w http.ResponseWriter, r *http.Request) {
1709− ctx := newContext(r, w, "DeleteBucketReplicationConfig")
1710− defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
1711− vars := mux.Vars(r)
1712− bucket := vars["bucket"]
1713−
1714− objectAPI := api.ObjectAPI()
1715− if objectAPI == nil {
1716− writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
1717− return
1718− }
1719−
1720− if s3Error := checkRequestAuthType(ctx, r, policy.PutReplicationConfigurationAction, bucket, ""); s3Error != ErrNone {
1721− writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
1722− return
1723− }
1724− // Check if bucket exists.
1725− if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
1726− writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
1727− return
1728− }
1729− if globalSiteReplicationSys.isEnabled() {
1730− writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationDenyEditError), r.URL)
1731− return
1732− }
1733− if err := globalBucketMetadataSys.Update(ctx, bucket, bucketReplicationConfig, nil); err != nil {
1734− writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
1735− return
1736− }
1737−
1738− // Write success response.
1739− writeSuccessResponseHeadersOnly(w)
1740−}
1741−
1742−// GetBucketReplicationMetricsHandler - GET Bucket replication metrics.
1743−// ----------
1744−// Gets the replication metrics for a bucket.
1745−func (api objectAPIHandlers) GetBucketReplicationMetricsHandler(w http.ResponseWriter, r *http.Request) {
1746− ctx := newContext(r, w, "GetBucketReplicationMetrics")
1747−
1748− defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
1749−
1750− vars := mux.Vars(r)
1751− bucket := vars["bucket"]
1752−
1753− objectAPI := api.ObjectAPI()
1754− if objectAPI == nil {
1755− writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
1756− return
1757− }
1758−
1759− // check if user has permissions to perform this operation
1760− if s3Error := checkRequestAuthType(ctx, r, policy.GetReplicationConfigurationAction, bucket, ""); s3Error != ErrNone {
1761− writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
1762− return
1763− }
1764−
1765− // Check if bucket exists.
1766− if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
1767− writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
1768− return
1769− }
1770− var usageInfo BucketUsageInfo
1771− dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
1772− if err == nil && !dataUsageInfo.LastUpdate.IsZero() {
1773− usageInfo = dataUsageInfo.BucketsUsage[bucket]
1774− }
1775−
1776− w.Header().Set(xhttp.ContentType, string(mimeJSON))
1777−
1778− enc := json.NewEncoder(w)
1779− if err = enc.Encode(getLatestReplicationStats(bucket, usageInfo)); err != nil {
1780− writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
1781− return
1782− }
1783−}
1784−
1785−// ResetBucketReplicationStartHandler - starts a replication reset for all objects in a bucket which
1786−// qualify for replication and re-sync the object(s) to target, provided ExistingObjectReplication is
1787−// enabled for the qualifying rule. This API is a MinIO only extension provided for situations where
1788−// remote target is entirely lost,and previously replicated objects need to be re-synced. If resync is
1789−// already in progress it returns an error
1790−func (api objectAPIHandlers) ResetBucketReplicationStartHandler(w http.ResponseWriter, r *http.Request) {
1791− ctx := newContext(r, w, "ResetBucketReplicationStart")
1792− defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
1793−
1794− vars := mux.Vars(r)
1795− bucket := vars["bucket"]
1796− durationStr := r.URL.Query().Get("older-than")
1797− arn := r.URL.Query().Get("arn")
1798− resetID := r.URL.Query().Get("reset-id")
1799− if resetID == "" {
1800− resetID = mustGetUUID()
1801− }
1802− var (
1803− days time.Duration
1804− err error
1805− )
1806− if durationStr != "" {
1807− days, err = time.ParseDuration(durationStr)
1808− if err != nil {
1809− writeErrorResponse(ctx, w, toAPIError(ctx, InvalidArgument{
1810− Bucket: bucket,
1811− Err: fmt.Errorf("invalid query parameter older-than %s for %s : %w", durationStr, bucket, err),
1812− }), r.URL)
1813− }
1814− }
1815− resetBeforeDate := UTCNow().AddDate(0, 0, -1*int(days/24))
1816−
1817− objectAPI := api.ObjectAPI()
1818− if objectAPI == nil {
1819− writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
1820− return
1821− }
1822−
1823− if s3Error := checkRequestAuthType(ctx, r, policy.ResetBucketReplicationStateAction, bucket, ""); s3Error != ErrNone {
1824− writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
1825− return
1826− }
1827−
1828− // Check if bucket exists.
1829− if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
1830− writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
1831− return
1832− }
1833−
1834− config, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket)
1835− if err != nil {
1836− writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
1837− return
1838− }
1839− if !config.HasActiveRules("", true) {
1840− writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationNoMatchingRuleError), r.URL)
1841− return
1842− }
1843− tgtArns := config.FilterTargetArns(
1844− replication.ObjectOpts{
1845− OpType: replication.ResyncReplicationType,
1846− TargetArn: arn,
1847− })
1848−
1849− if len(tgtArns) == 0 {
1850− writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrBadRequest, InvalidArgument{
1851− Bucket: bucket,
1852− Err: fmt.Errorf("Remote target ARN %s missing or ineligible for replication resync", arn),
1853− }), r.URL)
1854− return
1855− }
1856−
1857− if len(tgtArns) > 1 && arn == "" {
1858− writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrBadRequest, InvalidArgument{
1859− Bucket: bucket,
1860− Err: fmt.Errorf("ARN should be specified for replication reset"),
1861− }), r.URL)
1862− return
1863− }
1864− var rinfo ResyncTargetsInfo
1865− target := globalBucketTargetSys.GetRemoteBucketTargetByArn(ctx, bucket, tgtArns[0])
1866− target.ResetBeforeDate = UTCNow().AddDate(0, 0, -1*int(days/24))
1867− target.ResetID = resetID
1868− rinfo.Targets = append(rinfo.Targets, ResyncTarget{Arn: tgtArns[0], ResetID: target.ResetID})
1869− if err = globalBucketTargetSys.SetTarget(ctx, bucket, &target, true); err != nil {
1870− switch err.(type) {
1871− case BucketRemoteConnectionErr:
1872− writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationRemoteConnectionError, err), r.URL)
1873− default:
1874− writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
1875− }
1876− }
1877− if err := startReplicationResync(ctx, bucket, arn, resetID, resetBeforeDate, objectAPI); err != nil {
1878− writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrBadRequest, InvalidArgument{
1879− Bucket: bucket,
1880− Err: err,
1881− }), r.URL)
1882− return
1883− }
1884−
1885− data, err := json.Marshal(rinfo)
1886− if err != nil {
1887− writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
1888− return
1889− }
1890− // Write success response.
1891− writeSuccessResponseJSON(w, data)
1892−}
1893−
1894−// ResetBucketReplicationStatusHandler - returns the status of replication reset.
1895−// This API is a MinIO only extension
1896−func (api objectAPIHandlers) ResetBucketReplicationStatusHandler(w http.ResponseWriter, r *http.Request) {
1897− ctx := newContext(r, w, "ResetBucketReplicationStatus")
1898− defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
1899−
1900− vars := mux.Vars(r)
1901− bucket := vars["bucket"]
1902− arn := r.URL.Query().Get("arn")
1903− var err error
1904−
1905− objectAPI := api.ObjectAPI()
1906− if objectAPI == nil {
1907− writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
1908− return
1909− }
1910−
1911− if s3Error := checkRequestAuthType(ctx, r, policy.ResetBucketReplicationStateAction, bucket, ""); s3Error != ErrNone {
1912− writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
1913− return
1914− }
1915−
1916− // Check if bucket exists.
1917− if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
1918− writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
1919− return
1920− }
1921−
1922− if _, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket); err != nil {
1923− writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
1924− return
1925− }
1926−
1927− globalReplicationPool.resyncState.RLock()
1928− brs, ok := globalReplicationPool.resyncState.statusMap[bucket]
1929− if !ok {
1930− brs, err = loadBucketResyncMetadata(ctx, bucket, objectAPI)
1931− if err != nil {
1932− writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrBadRequest, InvalidArgument{
1933− Bucket: bucket,
1934− Err: fmt.Errorf("No replication resync status available for %s", arn),
1935− }), r.URL)
1936− }
1937− return
1938− }
1939−
1940− var rinfo ResyncTargetsInfo
1941− for tarn, st := range brs.TargetsMap {
1942− if arn != "" && tarn != arn {
1943− continue
1944− }
1945− rinfo.Targets = append(rinfo.Targets, ResyncTarget{
1946− Arn: tarn,
1947− ResetID: st.ResyncID,
1948− StartTime: st.StartTime,
1949− EndTime: st.EndTime,
1950− ResyncStatus: st.ResyncStatus.String(),
1951− ReplicatedSize: st.ReplicatedSize,
1952− ReplicatedCount: st.ReplicatedCount,
1953− FailedSize: st.FailedSize,
1954− FailedCount: st.FailedCount,
1955− Bucket: st.Bucket,
1956− Object: st.Object,
1957− })
1958− }
1959− globalReplicationPool.resyncState.RUnlock()
1960− data, err := json.Marshal(rinfo)
1961− if err != nil {
1962− writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
1963− return
1964− }
1965− // Write success response.
1966− writeSuccessResponseJSON(w, data)
1967−}
cmd/bucket-replication-handlers.go+404−0
@@ -0,0 +1,404 @@
1+// Copyright (c) 2015-2022 MinIO, Inc.
2+//
3+// This file is part of MinIO Object Storage stack
4+//
5+// This program is free software: you can redistribute it and/or modify
6+// it under the terms of the GNU Affero General Public License as published by
7+// the Free Software Foundation, either version 3 of the License, or
8+// (at your option) any later version.
9+//
10+// This program is distributed in the hope that it will be useful
11+// but WITHOUT ANY WARRANTY; without even the implied warranty of
12+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13+// GNU Affero General Public License for more details.
14+//
15+// You should have received a copy of the GNU Affero General Public License
16+// along with this program. If not, see <http://www.gnu.org/licenses/>.
17+
18+package cmd
19+
20+import (
21+ "encoding/json"
22+ "encoding/xml"
23+ "fmt"
24+ "io"
25+ "net/http"
26+ "time"
27+
28+ "github.com/gorilla/mux"
29+ "github.com/minio/minio/internal/bucket/replication"
30+ xhttp "github.com/minio/minio/internal/http"
31+ "github.com/minio/minio/internal/logger"
32+ "github.com/minio/pkg/bucket/policy"
33+)
34+
35+// PutBucketReplicationConfigHandler - PUT Bucket replication configuration.
36+// ----------
37+// Add a replication configuration on the specified bucket as specified in https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketReplication.html
38+func (api objectAPIHandlers) PutBucketReplicationConfigHandler(w http.ResponseWriter, r *http.Request) {
39+ ctx := newContext(r, w, "PutBucketReplicationConfig")
40+ defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
41+
42+ vars := mux.Vars(r)
43+ bucket := vars["bucket"]
44+ objectAPI := api.ObjectAPI()
45+ if objectAPI == nil {
46+ writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
47+ return
48+ }
49+ if globalIsGateway {
50+ writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
51+ return
52+ }
53+ if s3Error := checkRequestAuthType(ctx, r, policy.PutReplicationConfigurationAction, bucket, ""); s3Error != ErrNone {
54+ writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
55+ return
56+ }
57+ // Check if bucket exists.
58+ if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
59+ writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
60+ return
61+ }
62+
63+ if versioned := globalBucketVersioningSys.Enabled(bucket); !versioned {
64+ writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationNeedsVersioningError), r.URL)
65+ return
66+ }
67+ replicationConfig, err := replication.ParseConfig(io.LimitReader(r.Body, r.ContentLength))
68+ if err != nil {
69+ apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
70+ apiErr.Description = err.Error()
71+ writeErrorResponse(ctx, w, apiErr, r.URL)
72+ return
73+ }
74+ sameTarget, apiErr := validateReplicationDestination(ctx, bucket, replicationConfig, true)
75+ if apiErr != noError {
76+ writeErrorResponse(ctx, w, apiErr, r.URL)
77+ return
78+ }
79+ // Validate the received bucket replication config
80+ if err = replicationConfig.Validate(bucket, sameTarget); err != nil {
81+ writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
82+ return
83+ }
84+ configData, err := xml.Marshal(replicationConfig)
85+ if err != nil {
86+ writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
87+ return
88+ }
89+ if err = globalBucketMetadataSys.Update(ctx, bucket, bucketReplicationConfig, configData); err != nil {
90+ writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
91+ return
92+ }
93+
94+ // Write success response.
95+ writeSuccessResponseHeadersOnly(w)
96+}
97+
98+// GetBucketReplicationConfigHandler - GET Bucket replication configuration.
99+// ----------
100+// Gets the replication configuration for a bucket.
101+func (api objectAPIHandlers) GetBucketReplicationConfigHandler(w http.ResponseWriter, r *http.Request) {
102+ ctx := newContext(r, w, "GetBucketReplicationConfig")
103+
104+ defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
105+
106+ vars := mux.Vars(r)
107+ bucket := vars["bucket"]
108+
109+ objectAPI := api.ObjectAPI()
110+ if objectAPI == nil {
111+ writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
112+ return
113+ }
114+
115+ // check if user has permissions to perform this operation
116+ if s3Error := checkRequestAuthType(ctx, r, policy.GetReplicationConfigurationAction, bucket, ""); s3Error != ErrNone {
117+ writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
118+ return
119+ }
120+ // Check if bucket exists.
121+ if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
122+ writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
123+ return
124+ }
125+
126+ config, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket)
127+ if err != nil {
128+ writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
129+ return
130+ }
131+ configData, err := xml.Marshal(config)
132+ if err != nil {
133+ writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
134+ return
135+ }
136+
137+ // Write success response.
138+ writeSuccessResponseXML(w, configData)
139+}
140+
141+// DeleteBucketReplicationConfigHandler - DELETE Bucket replication config.
142+// ----------
143+func (api objectAPIHandlers) DeleteBucketReplicationConfigHandler(w http.ResponseWriter, r *http.Request) {
144+ ctx := newContext(r, w, "DeleteBucketReplicationConfig")
145+ defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
146+ vars := mux.Vars(r)
147+ bucket := vars["bucket"]
148+
149+ objectAPI := api.ObjectAPI()
150+ if objectAPI == nil {
151+ writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
152+ return
153+ }
154+
155+ if s3Error := checkRequestAuthType(ctx, r, policy.PutReplicationConfigurationAction, bucket, ""); s3Error != ErrNone {
156+ writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
157+ return
158+ }
159+ // Check if bucket exists.
160+ if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
161+ writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
162+ return
163+ }
164+ if globalSiteReplicationSys.isEnabled() {
165+ writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationDenyEditError), r.URL)
166+ return
167+ }
168+ if err := globalBucketMetadataSys.Update(ctx, bucket, bucketReplicationConfig, nil); err != nil {
169+ writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
170+ return
171+ }
172+
173+ // Write success response.
174+ writeSuccessResponseHeadersOnly(w)
175+}
176+
177+// GetBucketReplicationMetricsHandler - GET Bucket replication metrics.
178+// ----------
179+// Gets the replication metrics for a bucket.
180+func (api objectAPIHandlers) GetBucketReplicationMetricsHandler(w http.ResponseWriter, r *http.Request) {
181+ ctx := newContext(r, w, "GetBucketReplicationMetrics")
182+
183+ defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
184+
185+ vars := mux.Vars(r)
186+ bucket := vars["bucket"]
187+
188+ objectAPI := api.ObjectAPI()
189+ if objectAPI == nil {
190+ writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
191+ return
192+ }
193+
194+ // check if user has permissions to perform this operation
195+ if s3Error := checkRequestAuthType(ctx, r, policy.GetReplicationConfigurationAction, bucket, ""); s3Error != ErrNone {
196+ writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
197+ return
198+ }
199+
200+ // Check if bucket exists.
201+ if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
202+ writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
203+ return
204+ }
205+ var usageInfo BucketUsageInfo
206+ dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
207+ if err == nil && !dataUsageInfo.LastUpdate.IsZero() {
208+ usageInfo = dataUsageInfo.BucketsUsage[bucket]
209+ }
210+
211+ w.Header().Set(xhttp.ContentType, string(mimeJSON))
212+
213+ enc := json.NewEncoder(w)
214+ if err = enc.Encode(getLatestReplicationStats(bucket, usageInfo)); err != nil {
215+ writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
216+ return
217+ }
218+}
219+
220+// ResetBucketReplicationStartHandler - starts a replication reset for all objects in a bucket which
221+// qualify for replication and re-sync the object(s) to target, provided ExistingObjectReplication is
222+// enabled for the qualifying rule. This API is a MinIO only extension provided for situations where
223+// remote target is entirely lost,and previously replicated objects need to be re-synced. If resync is
224+// already in progress it returns an error
225+func (api objectAPIHandlers) ResetBucketReplicationStartHandler(w http.ResponseWriter, r *http.Request) {
226+ ctx := newContext(r, w, "ResetBucketReplicationStart")
227+ defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
228+
229+ vars := mux.Vars(r)
230+ bucket := vars["bucket"]
231+ durationStr := r.URL.Query().Get("older-than")
232+ arn := r.URL.Query().Get("arn")
233+ resetID := r.URL.Query().Get("reset-id")
234+ if resetID == "" {
235+ resetID = mustGetUUID()
236+ }
237+ var (
238+ days time.Duration
239+ err error
240+ )
241+ if durationStr != "" {
242+ days, err = time.ParseDuration(durationStr)
243+ if err != nil {
244+ writeErrorResponse(ctx, w, toAPIError(ctx, InvalidArgument{
245+ Bucket: bucket,
246+ Err: fmt.Errorf("invalid query parameter older-than %s for %s : %w", durationStr, bucket, err),
247+ }), r.URL)
248+ }
249+ }
250+ resetBeforeDate := UTCNow().AddDate(0, 0, -1*int(days/24))
251+
252+ objectAPI := api.ObjectAPI()
253+ if objectAPI == nil {
254+ writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
255+ return
256+ }
257+
258+ if s3Error := checkRequestAuthType(ctx, r, policy.ResetBucketReplicationStateAction, bucket, ""); s3Error != ErrNone {
259+ writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
260+ return
261+ }
262+
263+ // Check if bucket exists.
264+ if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
265+ writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
266+ return
267+ }
268+
269+ config, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket)
270+ if err != nil {
271+ writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
272+ return
273+ }
274+ if !config.HasExistingObjectReplication(arn) {
275+ writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationNoExistingObjects), r.URL)
276+ return
277+ }
278+
279+ tgtArns := config.FilterTargetArns(
280+ replication.ObjectOpts{
281+ OpType: replication.ResyncReplicationType,
282+ TargetArn: arn,
283+ })
284+
285+ if len(tgtArns) == 0 {
286+ writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrBadRequest, InvalidArgument{
287+ Bucket: bucket,
288+ Err: fmt.Errorf("Remote target ARN %s missing or ineligible for replication resync", arn),
289+ }), r.URL)
290+ return
291+ }
292+
293+ if len(tgtArns) > 1 && arn == "" {
294+ writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrBadRequest, InvalidArgument{
295+ Bucket: bucket,
296+ Err: fmt.Errorf("ARN should be specified for replication reset"),
297+ }), r.URL)
298+ return
299+ }
300+ var rinfo ResyncTargetsInfo
301+ target := globalBucketTargetSys.GetRemoteBucketTargetByArn(ctx, bucket, tgtArns[0])
302+ target.ResetBeforeDate = UTCNow().AddDate(0, 0, -1*int(days/24))
303+ target.ResetID = resetID
304+ rinfo.Targets = append(rinfo.Targets, ResyncTarget{Arn: tgtArns[0], ResetID: target.ResetID})
305+ if err = globalBucketTargetSys.SetTarget(ctx, bucket, &target, true); err != nil {
306+ switch err.(type) {
307+ case BucketRemoteConnectionErr:
308+ writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationRemoteConnectionError, err), r.URL)
309+ default:
310+ writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
311+ }
312+ }
313+ if err := startReplicationResync(ctx, bucket, arn, resetID, resetBeforeDate, objectAPI); err != nil {
314+ writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrBadRequest, InvalidArgument{
315+ Bucket: bucket,
316+ Err: err,
317+ }), r.URL)
318+ return
319+ }
320+
321+ data, err := json.Marshal(rinfo)
322+ if err != nil {
323+ writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
324+ return
325+ }
326+ // Write success response.
327+ writeSuccessResponseJSON(w, data)
328+}
329+
330+// ResetBucketReplicationStatusHandler - returns the status of replication reset.
331+// This API is a MinIO only extension
332+func (api objectAPIHandlers) ResetBucketReplicationStatusHandler(w http.ResponseWriter, r *http.Request) {
333+ ctx := newContext(r, w, "ResetBucketReplicationStatus")
334+ defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
335+
336+ vars := mux.Vars(r)
337+ bucket := vars["bucket"]
338+ arn := r.URL.Query().Get("arn")
339+ var err error
340+
341+ objectAPI := api.ObjectAPI()
342+ if objectAPI == nil {
343+ writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
344+ return
345+ }
346+
347+ if s3Error := checkRequestAuthType(ctx, r, policy.ResetBucketReplicationStateAction, bucket, ""); s3Error != ErrNone {
348+ writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
349+ return
350+ }
351+
352+ // Check if bucket exists.
353+ if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
354+ writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
355+ return
356+ }
357+
358+ if _, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket); err != nil {
359+ writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
360+ return
361+ }
362+
363+ globalReplicationPool.resyncState.RLock()
364+ brs, ok := globalReplicationPool.resyncState.statusMap[bucket]
365+ globalReplicationPool.resyncState.RUnlock()
366+ if !ok {
367+ brs, err = loadBucketResyncMetadata(ctx, bucket, objectAPI)
368+ if err != nil {
369+ writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrBadRequest, InvalidArgument{
370+ Bucket: bucket,
371+ Err: fmt.Errorf("No replication resync status available for %s", arn),
372+ }), r.URL)
373+ return
374+ }
375+ }
376+
377+ var rinfo ResyncTargetsInfo
378+ for tarn, st := range brs.TargetsMap {
379+ if arn != "" && tarn != arn {
380+ continue
381+ }
382+ rinfo.Targets = append(rinfo.Targets, ResyncTarget{
383+ Arn: tarn,
384+ ResetID: st.ResyncID,
385+ StartTime: st.StartTime,
386+ EndTime: st.EndTime,
387+ ResyncStatus: st.ResyncStatus.String(),
388+ ReplicatedSize: st.ReplicatedSize,
389+ ReplicatedCount: st.ReplicatedCount,
390+ FailedSize: st.FailedSize,
391+ FailedCount: st.FailedCount,
392+ Bucket: st.Bucket,
393+ Object: st.Object,
394+ })
395+ }
396+ data, err := json.Marshal(rinfo)
397+ if err != nil {
398+ writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
399+ return
400+ }
401+
402+ // Write success response.
403+ writeSuccessResponseJSON(w, data)
404+}
cmd/bucket-replication.go+2−5
@@ -1757,10 +1757,6 @@ func (c replicationConfig) Resync(ctx context.Context, oi ObjectInfo, dsc *Repli · 1756 unmodified lines
17571757 if c.Empty() {
17581758 return
17591759 }
1760− // existing object replication does not apply to un-versioned objects
1761− if oi.VersionID == "" {
1762− return
1763− }
17641760
17651761 // Now overlay existing object replication choices for target
17661762 if oi.DeleteMarker {
@@ -2006,6 +2002,7 @@ func resyncBucket(ctx context.Context, bucket, arn string, heal bool, objectAPI · 239 unmodified lines
20062002 st.EndTime = UTCNow()
20072003 st.ResyncStatus = resyncStatus
20082004 m.TargetsMap[arn] = st
2005+ globalReplicationPool.resyncState.statusMap[bucket] = m
20092006 globalReplicationPool.resyncState.Unlock()
20102007 }()
20112008 // Allocate new results channel to receive ObjectInfo.
@@ -2066,7 +2063,6 @@ func resyncBucket(ctx context.Context, bucket, arn string, heal bool, objectAPI · 54 unmodified lines
20662063 }
20672064
20682065 if roi.DeleteMarker || !roi.VersionPurgeStatus.Empty() {
2069−
20702066 versionID := ""
20712067 dmVersionID := ""
20722068 if roi.VersionPurgeStatus.Empty() {
@@ -2113,6 +2109,7 @@ func resyncBucket(ctx context.Context, bucket, arn string, heal bool, objectAPI · 40 unmodified lines
21132109 st.ReplicatedSize += roi.Size
21142110 }
21152111 m.TargetsMap[arn] = st
2112+ globalReplicationPool.resyncState.statusMap[bucket] = m
21162113 globalReplicationPool.resyncState.Unlock()
21172114 }
21182115 resyncStatus = ResyncCompleted
docs/bucket/replication/setup_2site_existing_replication.sh+93−0
@@ -0,0 +1,93 @@
1+#!/usr/bin/env bash
2+
3+trap 'catch $LINENO' ERR
4+
5+# shellcheck disable=SC2120
6+catch() {
7+ if [ $# -ne 0 ]; then
8+ echo "error on line $1"
9+ for site in sitea siteb; do
10+ echo "$site server logs ========="
11+ cat "/tmp/${site}_1.log"
12+ echo "==========================="
13+ cat "/tmp/${site}_2.log"
14+ done
15+ fi
16+
17+ echo "Cleaning up instances of MinIO"
18+ pkill minio
19+ pkill -9 minio
20+ rm -rf /tmp/multisitea
21+ rm -rf /tmp/multisiteb
22+ rm -rf /tmp/data
23+}
24+
25+catch
26+
27+set -e
28+export MINIO_CI_CD=1
29+export MINIO_BROWSER=off
30+export MINIO_ROOT_USER="minio"
31+export MINIO_ROOT_PASSWORD="minio123"
32+export MINIO_KMS_AUTO_ENCRYPTION=off
33+export MINIO_PROMETHEUS_AUTH_TYPE=public
34+export MINIO_KMS_SECRET_KEY=my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw=
35+unset MINIO_KMS_KES_CERT_FILE
36+unset MINIO_KMS_KES_KEY_FILE
37+unset MINIO_KMS_KES_ENDPOINT
38+unset MINIO_KMS_KES_KEY_NAME
39+
40+wget -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc \
41+ && chmod +x mc
42+
43+minio server --address 127.0.0.1:9001 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \
44+ "http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_1.log 2>&1 &
45+minio server --address 127.0.0.1:9002 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \
46+ "http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_2.log 2>&1 &
47+
48+minio server --address 127.0.0.1:9003 "http://127.0.0.1:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \
49+ "http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_1.log 2>&1 &
50+minio server --address 127.0.0.1:9004 "http://127.0.0.1:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \
51+ "http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_2.log 2>&1 &
52+
53+sleep 10s
54+
55+export MC_HOST_sitea=http://minio:minio123@127.0.0.1:9001
56+export MC_HOST_siteb=http://minio:minio123@127.0.0.1:9004
57+
58+./mc mb sitea/bucket
59+
60+## Create 100 files
61+mkdir -p /tmp/data
62+for i in $(seq 1 10); do
63+ echo "T" > /tmp/data/file_${i}.txt
64+done
65+
66+./mc mirror /tmp/data sitea/bucket/
67+./mc version enable sitea/bucket
68+
69+./mc mb siteb/bucket/
70+./mc version enable siteb/bucket/
71+
72+echo "adding replication config for site a -> site b"
73+remote_arn=$(./mc admin bucket remote add sitea/bucket/ \
74+ http://minio:minio123@127.0.0.1:9004/bucket \
75+ --service "replication" --json | jq -r ".RemoteARN")
76+echo "adding replication rule for a -> b : ${remote_arn}"
77+
78+./mc replicate add sitea/bucket/ \
79+ --remote-bucket "${remote_arn}"
80+sleep 1
81+
82+./mc replicate resync start sitea/bucket/ --remote-bucket "${remote_arn}"
83+sleep 10s ## sleep for 10s idea is that we give 100ms per object.
84+
85+count=$(./mc replicate resync status sitea/bucket --remote-bucket "${remote_arn}" --json | jq .resyncInfo.target[].replicationCount)
86+./mc ls --versions sitea/bucket
87+./mc ls --versions siteb/bucket
88+if [ $count -ne 10 ]; then
89+ echo "resync not complete after 100s unexpected failure"
90+ ./mc diff sitea/bucket siteb/bucket
91+fi
92+
93+catch
internal/bucket/replication/replication.go+12−0
@@ -146,6 +146,18 @@ type ObjectOpts struct { · 145 unmodified lines
146146 TargetArn string
147147 }
148148
149+// HasExistingObjectReplication returns true if any of the rule returns 'ExistingObjects' replication.
150+func (c Config) HasExistingObjectReplication(arn string) bool {
151+ for _, rule := range c.Rules {
152+ if rule.Destination.ARN == arn || c.RoleArn == arn {
153+ if rule.ExistingObjectReplication.Status == Enabled {
154+ return true
155+ }
156+ }
157+ }
158+ return false
159+}
160+
149161 // FilterActionableRules returns the rules actions that need to be executed
150162 // after evaluating prefix/tag filtering
151163 func (c Config) FilterActionableRules(obj ObjectOpts) []Rule {

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t