Skip to content

Commit 4ec715c

Browse files
symiousclaude
andauthored
HDDS-15879. [Ozone Versioning] [T1] Metadata foundation (#10781)
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
1 parent a581452 commit 4ec715c

25 files changed

Lines changed: 886 additions & 27 deletions

File tree

‎hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -180,6 +180,11 @@ public final class OzoneConsts {
180180

181181
public static final String OM_KEY_PREFIX = "/";
182182
public static final String DOUBLE_SLASH_OM_KEY_PREFIX = "//";
183+
/**
184+
* Separates the key name from the versionId in versionedKeyTable DB keys.
185+
* Not '/', since key names may contain it.
186+
*/
187+
public static final String OM_VERSIONED_KEY_SEPARATOR = "\0";
183188
public static final String OM_USER_PREFIX = "$";
184189
public static final String OM_S3_PREFIX = "S3:";
185190
public static final String OM_S3_CALLER_CONTEXT_PREFIX = "S3Auth:S3G|";
@@ -299,6 +304,7 @@ public final class OzoneConsts {
299304
public static final String STORAGE_TYPE = "storageType";
300305
public static final String RESOURCE_TYPE = "resourceType";
301306
public static final String IS_VERSION_ENABLED = "isVersionEnabled";
307+
public static final String VERSIONING_STATUS = "versioningStatus";
302308
public static final String CREATION_TIME = "creationTime";
303309
public static final String MODIFICATION_TIME = "modificationTime";
304310
public static final String DATA_SIZE = "dataSize";

‎hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,7 @@ public enum OzoneManagerVersion implements ComponentVersion {
9393

9494
S3_IF_NONE_MATCH_ETAG(104,
9595
"OzoneManager version that supports If-None-Match entity ETags for writes"),
96+
S3_OBJECT_VERSIONING(105, "OzoneManager version that supports S3 object versioning"),
9697

9798
UNKNOWN_VERSION(-1, "Used when a version cannot be deserialized to any version recognized by this" +
9899
" component, which may indicate it came from a component in a newer version");

‎hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,7 @@
136136
import org.apache.hadoop.ozone.om.helpers.BasicOmKeyInfo;
137137
import org.apache.hadoop.ozone.om.helpers.BucketEncryptionKeyInfo;
138138
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
139+
import org.apache.hadoop.ozone.om.helpers.BucketVersioningStatus;
139140
import org.apache.hadoop.ozone.om.helpers.CallerIdentityInfo;
140141
import org.apache.hadoop.ozone.om.helpers.DeleteTenantState;
141142
import org.apache.hadoop.ozone.om.helpers.ErrorInfo;
@@ -626,6 +627,10 @@ public void createBucket(
626627
Objects.requireNonNull(bucketArgs, "bucketArgs == null");
627628
verifyCountsQuota(bucketArgs.getQuotaInNamespace());
628629
verifySpaceQuota(bucketArgs.getQuotaInBytes());
630+
if (bucketArgs.getVersioning()) {
631+
throw new OMException("Bucket versioning cannot be set while the bucket is being created; create the bucket"
632+
+ " first and then set its versioning.", OMException.ResultCodes.INVALID_REQUEST);
633+
}
629634
if (!OzoneManagerVersion.ERASURE_CODED_STORAGE_SUPPORT.isSupportedBy(omVersion)) {
630635
if (bucketArgs.getDefaultReplicationConfig() != null &&
631636
bucketArgs.getDefaultReplicationConfig().getType()
@@ -661,7 +666,6 @@ public void createBucket(
661666
OmBucketInfo.Builder builder = OmBucketInfo.newBuilder();
662667
builder.setVolumeName(volumeName)
663668
.setBucketName(bucketName)
664-
.setIsVersionEnabled(isVersionEnabled)
665669
.addAllMetadata(bucketArgs.getMetadata())
666670
.setStorageType(storageType)
667671
.setSourceVolume(bucketArgs.getSourceVolume())
@@ -1231,10 +1235,14 @@ public void setBucketVersioning(
12311235
verifyVolumeName(volumeName);
12321236
verifyBucketName(bucketName);
12331237
Objects.requireNonNull(versioning, "versioning == null");
1238+
if (!OzoneManagerVersion.S3_OBJECT_VERSIONING.isSupportedBy(omVersion)) {
1239+
throw new IOException("Can not set bucket versioning, as OzoneManager does not support S3 object versioning.");
1240+
}
12341241
OmBucketArgs.Builder builder = OmBucketArgs.newBuilder();
12351242
builder.setVolumeName(volumeName)
12361243
.setBucketName(bucketName)
1237-
.setIsVersionEnabled(versioning);
1244+
.setVersioningStatus(versioning
1245+
? BucketVersioningStatus.ENABLED : BucketVersioningStatus.SUSPENDED);
12381246
ozoneManagerClient.setBucketProperty(builder.build());
12391247
}
12401248

‎hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/rpc/TestRpcClient.java‎

Lines changed: 36 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -686,7 +686,7 @@ public OMResponse submitRequest(OMRequest request) throws IOException {
686686
"key", 1L, "excluded-etag", replication, Collections.emptyMap(), Collections.emptyMap(), false));
687687
IOException completeError = assertThrows(IOException.class, () -> client.completeMultipartUpload("vol", "bucket",
688688
"key", "upload-id", Collections.singletonMap(1, "part-etag"), null, null, "excluded-etag"));
689-
if (version == OzoneManagerVersion.S3_IF_NONE_MATCH_ETAG) {
689+
if (version.compareTo(OzoneManagerVersion.S3_IF_NONE_MATCH_ETAG) >= 0) {
690690
assertEquals("captured", putError.getMessage());
691691
assertEquals("captured", streamError.getMessage());
692692
assertEquals("captured", completeError.getMessage());
@@ -702,6 +702,41 @@ public OMResponse submitRequest(OMRequest request) throws IOException {
702702
}
703703
}
704704

705+
@ParameterizedTest
706+
@EnumSource(value = OzoneManagerVersion.class, mode = EnumSource.Mode.EXCLUDE, names = "UNKNOWN_VERSION")
707+
public void testSetBucketVersioningRejectsOldVersion(OzoneManagerVersion version) throws IOException {
708+
List<OMRequest> requests = new ArrayList<>();
709+
OmTransport transport = new MockOmTransport() {
710+
@Override
711+
public OMResponse submitRequest(OMRequest request) throws IOException {
712+
if (request.getCmdType() == Type.ServiceList) {
713+
return OMResponse.newBuilder().setCmdType(Type.ServiceList)
714+
.setStatus(org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.OK)
715+
.setSuccess(true).setServiceListResponse(ServiceListResponse.newBuilder()
716+
.addServiceInfo(om(version).getProtobuf())).build();
717+
}
718+
if (request.getCmdType() == Type.SetBucketProperty) {
719+
requests.add(request);
720+
throw new IOException("captured");
721+
}
722+
return super.submitRequest(request);
723+
}
724+
};
725+
RpcClient client = createRpcClient(transport);
726+
try {
727+
IOException error = assertThrows(IOException.class, () -> client.setBucketVersioning("vol", "bucket", true));
728+
if (OzoneManagerVersion.S3_OBJECT_VERSIONING.isSupportedBy(version)) {
729+
assertEquals("captured", error.getMessage());
730+
assertThat(requests).hasSize(1);
731+
} else {
732+
assertThat(requests).isEmpty();
733+
assertThat(error.getMessage()).contains("does not support S3 object versioning");
734+
}
735+
} finally {
736+
client.close();
737+
}
738+
}
739+
705740
private static ServiceInfo om(OzoneManagerVersion version) {
706741
return new ServiceInfo.Builder()
707742
.setNodeType(HddsProtos.NodeType.OM)
Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.hadoop.ozone.om.helpers;
19+
20+
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.BucketVersioningStatusProto;
21+
22+
/**
23+
* S3-compatible bucket versioning state machine:
24+
* UNVERSIONED -&gt; ENABLED &lt;-&gt; SUSPENDED.
25+
* Once versioning has been enabled, a bucket can never return to UNVERSIONED.
26+
*/
27+
public enum BucketVersioningStatus {
28+
UNVERSIONED,
29+
ENABLED,
30+
SUSPENDED;
31+
32+
public static BucketVersioningStatus fromProto(BucketVersioningStatusProto proto) {
33+
switch (proto) {
34+
case VERSIONING_ENABLED:
35+
return ENABLED;
36+
case VERSIONING_SUSPENDED:
37+
return SUSPENDED;
38+
case UNVERSIONED:
39+
default:
40+
return UNVERSIONED;
41+
}
42+
}
43+
44+
public BucketVersioningStatusProto toProto() {
45+
switch (this) {
46+
case ENABLED:
47+
return BucketVersioningStatusProto.VERSIONING_ENABLED;
48+
case SUSPENDED:
49+
return BucketVersioningStatusProto.VERSIONING_SUSPENDED;
50+
default:
51+
return BucketVersioningStatusProto.UNVERSIONED;
52+
}
53+
}
54+
55+
/** The legacy isVersionEnabled flag value kept in sync with this status. */
56+
public boolean toVersionEnabledFlag() {
57+
return this == ENABLED;
58+
}
59+
60+
/**
61+
* Whether a bucket may transition from this status to {@code target}.
62+
* The only forbidden transition is back to UNVERSIONED after versioning
63+
* has been enabled or suspended (matches the S3 state machine).
64+
*/
65+
public boolean canTransitionTo(BucketVersioningStatus target) {
66+
return this == UNVERSIONED || target != UNVERSIONED;
67+
}
68+
}

‎hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketArgs.java‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,11 @@ public final class OmBucketArgs extends WithMetadata implements Auditable {
4444
* Bucket Version flag.
4545
*/
4646
private final Boolean isVersionEnabled;
47+
/**
48+
* S3-compatible versioning status; null when not being changed.
49+
* Takes precedence over isVersionEnabled when both are set.
50+
*/
51+
private final BucketVersioningStatus versioningStatus;
4752
/**
4853
* Type of storage to be used for this bucket.
4954
* [RAM_DISK, SSD, DISK, ARCHIVE]
@@ -73,6 +78,7 @@ private OmBucketArgs(Builder b) {
7378
this.volumeName = b.volumeName;
7479
this.bucketName = b.bucketName;
7580
this.isVersionEnabled = b.isVersionEnabled;
81+
this.versioningStatus = b.versioningStatus;
7682
this.storageType = b.storageType;
7783
this.ownerName = b.ownerName;
7884
this.defaultReplicationConfig = b.defaultReplicationConfig;
@@ -108,6 +114,14 @@ public Boolean getIsVersionEnabled() {
108114
return isVersionEnabled;
109115
}
110116

117+
/**
118+
* Returns the requested versioning status, or null if not being changed.
119+
* @return BucketVersioningStatus
120+
*/
121+
public BucketVersioningStatus getVersioningStatus() {
122+
return versioningStatus;
123+
}
124+
111125
/**
112126
* Returns the type of storage to be used.
113127
* @return StorageType
@@ -190,6 +204,9 @@ public Map<String, String> toAuditMap() {
190204
getMetadata().get(OzoneConsts.GDPR_FLAG));
191205
auditMap.put(OzoneConsts.IS_VERSION_ENABLED,
192206
String.valueOf(this.isVersionEnabled));
207+
if (this.versioningStatus != null) {
208+
auditMap.put(OzoneConsts.VERSIONING_STATUS, this.versioningStatus.name());
209+
}
193210
if (this.storageType != null) {
194211
auditMap.put(OzoneConsts.STORAGE_TYPE, this.storageType.name());
195212
}
@@ -227,6 +244,7 @@ public static class Builder extends WithMetadata.Builder {
227244
private String volumeName;
228245
private String bucketName;
229246
private Boolean isVersionEnabled;
247+
private BucketVersioningStatus versioningStatus;
230248
private StorageType storageType;
231249
private boolean quotaInBytesSet = false;
232250
private long quotaInBytes;
@@ -261,6 +279,11 @@ public Builder setIsVersionEnabled(Boolean versionFlag) {
261279
return this;
262280
}
263281

282+
public Builder setVersioningStatus(BucketVersioningStatus status) {
283+
this.versioningStatus = status;
284+
return this;
285+
}
286+
264287
@Deprecated
265288
public Builder setBucketEncryptionKey(BucketEncryptionKeyInfo info) {
266289
if (info == null || info.getKeyName() != null) {
@@ -339,6 +362,9 @@ public BucketArgs getProtobuf() {
339362
if (isVersionEnabled != null) {
340363
builder.setIsVersionEnabled(isVersionEnabled);
341364
}
365+
if (versioningStatus != null) {
366+
builder.setVersioningStatus(versioningStatus.toProto());
367+
}
342368
if (storageType != null) {
343369
builder.setStorageType(storageType.toProto());
344370
}
@@ -381,6 +407,10 @@ public static Builder builderFromProtobuf(BucketArgs bucketArgs) {
381407
if (bucketArgs.hasIsVersionEnabled()) {
382408
builder.setIsVersionEnabled(bucketArgs.getIsVersionEnabled());
383409
}
410+
if (bucketArgs.hasVersioningStatus()) {
411+
builder.setVersioningStatus(
412+
BucketVersioningStatus.fromProto(bucketArgs.getVersioningStatus()));
413+
}
384414
if (bucketArgs.hasStorageType()) {
385415
builder.setStorageType(StorageType.valueOf(bucketArgs.getStorageType()));
386416
}

0 commit comments

Comments
 (0)