diff --git a/google-cloud-storage/clirr-ignored-differences.xml b/google-cloud-storage/clirr-ignored-differences.xml index b3e4b99f72..ba32338734 100644 --- a/google-cloud-storage/clirr-ignored-differences.xml +++ b/google-cloud-storage/clirr-ignored-differences.xml @@ -15,6 +15,12 @@ * writeAndClose(*) + + 7013 + com/google/cloud/storage/BlobInfo$Builder + com.google.cloud.storage.BlobInfo$Builder setRetention(com.google.cloud.storage.BlobInfo$Retention) + + 7009 diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java index 4bba8aef9c..e6295b8c6a 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java @@ -526,6 +526,12 @@ Builder setRetentionExpirationTimeOffsetDateTime(OffsetDateTime retentionExpirat return this; } + @Override + public Builder setRetention(Retention retention) { + infoBuilder.setRetention(retention); + return this; + } + @Override public Blob build() { return new Blob(storage, infoBuilder); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobInfo.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobInfo.java index 288de2b395..e808f444ca 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobInfo.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobInfo.java @@ -22,7 +22,10 @@ import static com.google.common.base.Preconditions.checkNotNull; import com.google.api.client.util.Data; +import com.google.api.core.ApiFunction; import com.google.api.core.BetaApi; +import com.google.cloud.StringEnumType; +import com.google.cloud.StringEnumValue; import com.google.cloud.storage.Storage.BlobField; import com.google.cloud.storage.TransportCompatibility.Transport; import com.google.cloud.storage.UnifiedOpts.NamedField; @@ -104,6 +107,7 @@ public class BlobInfo implements Serializable { private final Boolean eventBasedHold; private final Boolean temporaryHold; private final OffsetDateTime retentionExpirationTime; + private final Retention retention; private final transient ImmutableSet modifiedFields; /** This class is meant for internal use only. Users are discouraged from using this class. */ @@ -168,6 +172,119 @@ public final boolean equals(Object o) { } } + /** + * Defines a blob's Retention policy. Can only be used on objects in a retention-enabled bucket. + */ + public static final class Retention implements Serializable { + + private static final long serialVersionUID = 5046718464542688444L; + + private Mode mode; + + private OffsetDateTime retainUntilTime; + + /** Returns the retention policy's Mode. Can be Locked or Unlocked. */ + public Mode getMode() { + return mode; + } + + /** Returns what time this object will be retained until, if the mode is Locked. */ + public OffsetDateTime getRetainUntilTime() { + return retainUntilTime; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof Retention)) { + return false; + } + Retention that = (Retention) o; + return Objects.equals(mode, that.mode) + && Objects.equals(retainUntilTime, that.retainUntilTime); + } + + @Override + public int hashCode() { + return Objects.hash(mode, retainUntilTime); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("mode", mode) + .add("retainUntilTime", retainUntilTime) + .toString(); + } + + public static Builder newBuilder() { + return new Builder(); + } + + public Builder toBuilder() { + return new Builder().setMode(this.mode).setRetainUntilTime(this.retainUntilTime); + } + + private Retention() {} + + public Retention(Builder builder) { + this.mode = builder.mode; + this.retainUntilTime = builder.retainUntilTime; + } + + public static final class Builder { + private Mode mode; + private OffsetDateTime retainUntilTime; + + /** Sets the retention policy's Mode. Can be Locked or Unlocked. */ + public Builder setMode(Mode mode) { + this.mode = mode; + return this; + } + + /** Sets what time this object will be retained until, if the mode is Locked. */ + public Builder setRetainUntilTime(OffsetDateTime retainUntilTime) { + this.retainUntilTime = retainUntilTime; + return this; + } + + public Retention build() { + return new Retention(this); + } + } + + public static final class Mode extends StringEnumValue { + private static final long serialVersionUID = 1973143582659557184L; + + private Mode(String constant) { + super(constant); + } + + private static final ApiFunction CONSTRUCTOR = Mode::new; + + private static final StringEnumType type = + new StringEnumType<>(Mode.class, CONSTRUCTOR); + + public static final Mode UNLOCKED = type.createAndRegister("Unlocked"); + + public static final Mode LOCKED = type.createAndRegister("Locked"); + + public static Mode valueOfStrict(String constant) { + return type.valueOfStrict(constant); + } + + public static Mode valueOf(String constant) { + return type.valueOf(constant); + } + + public static Mode[] values() { + return type.values(); + } + } + } + /** Builder for {@code BlobInfo}. */ public abstract static class Builder { @@ -408,6 +525,8 @@ Builder setRetentionExpirationTimeOffsetDateTime(OffsetDateTime retentionExpirat return setRetentionExpirationTime(millisOffsetDateTimeCodec.decode(retentionExpirationTime)); } + public abstract Builder setRetention(Retention retention); + /** Creates a {@code BlobInfo} object. */ public abstract BlobInfo build(); @@ -506,6 +625,7 @@ static final class BuilderImpl extends Builder { private Boolean eventBasedHold; private Boolean temporaryHold; private OffsetDateTime retentionExpirationTime; + private Retention retention; private final ImmutableSet.Builder modifiedFields = ImmutableSet.builder(); BuilderImpl(BlobId blobId) { @@ -543,6 +663,7 @@ static final class BuilderImpl extends Builder { eventBasedHold = blobInfo.eventBasedHold; temporaryHold = blobInfo.temporaryHold; retentionExpirationTime = blobInfo.retentionExpirationTime; + retention = blobInfo.retention; } @Override @@ -916,6 +1037,14 @@ Builder setRetentionExpirationTimeOffsetDateTime(OffsetDateTime retentionExpirat return this; } + @Override + public Builder setRetention(Retention retention) { + // todo: b/308194853 + modifiedFields.add(BlobField.RETENTION); + this.retention = retention; + return this; + } + @Override public BlobInfo build() { checkNotNull(blobId); @@ -1139,6 +1268,7 @@ Builder clearRetentionExpirationTime() { eventBasedHold = builder.eventBasedHold; temporaryHold = builder.temporaryHold; retentionExpirationTime = builder.retentionExpirationTime; + retention = builder.retention; modifiedFields = builder.modifiedFields.build(); } @@ -1532,6 +1662,11 @@ public OffsetDateTime getRetentionExpirationTimeOffsetDateTime() { return retentionExpirationTime; } + /** Returns the object's Retention policy. */ + public Retention getRetention() { + return retention; + } + /** Returns a builder for the current blob. */ public Builder toBuilder() { return new BuilderImpl(this); @@ -1581,6 +1716,7 @@ public int hashCode() { kmsKeyName, eventBasedHold, temporaryHold, + retention, retentionExpirationTime); } @@ -1622,7 +1758,8 @@ public boolean equals(Object o) { && Objects.equals(kmsKeyName, blobInfo.kmsKeyName) && Objects.equals(eventBasedHold, blobInfo.eventBasedHold) && Objects.equals(temporaryHold, blobInfo.temporaryHold) - && Objects.equals(retentionExpirationTime, blobInfo.retentionExpirationTime); + && Objects.equals(retentionExpirationTime, blobInfo.retentionExpirationTime) + && Objects.equals(retention, blobInfo.retention); } ImmutableSet getModifiedFields() { diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java index c8827c1fac..3c1652bc96 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java @@ -272,6 +272,16 @@ public static BlobTargetOption userProject(@NonNull String userProject) { return new BlobTargetOption(UnifiedOpts.userProject(userProject)); } + /** + * Returns an option for overriding an Unlocked Retention policy. This must be set to true in + * order to change a policy from Unlocked to Locked, to set it to null, or to reduce its + * retainUntilTime attribute. + */ + @TransportCompatibility({Transport.HTTP}) + public static BlobTargetOption overrideUnlockedRetention(boolean overrideUnlockedRetention) { + return new BlobTargetOption(UnifiedOpts.overrideUnlockedRetention(overrideUnlockedRetention)); + } + /** * Deduplicate any options which are the same parameter. The value which comes last in {@code * os} will be the value included in the return. @@ -732,6 +742,12 @@ public Builder setCustomPlacementConfig(CustomPlacementConfig customPlacementCon return this; } + @Override + Builder setObjectRetention(ObjectRetention objectRetention) { + infoBuilder.setObjectRetention(objectRetention); + return this; + } + @Override public Bucket build() { return new Bucket(storage, infoBuilder); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java index 5750b55591..9fb46b5bdb 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java @@ -27,8 +27,11 @@ import com.google.api.client.json.jackson2.JacksonFactory; import com.google.api.client.util.Data; import com.google.api.client.util.DateTime; +import com.google.api.core.ApiFunction; import com.google.api.core.BetaApi; import com.google.api.services.storage.model.Bucket.Lifecycle.Rule; +import com.google.cloud.StringEnumType; +import com.google.cloud.StringEnumValue; import com.google.cloud.storage.Acl.Entity; import com.google.cloud.storage.BlobInfo.ImmutableEmptyMap; import com.google.cloud.storage.Storage.BucketField; @@ -114,6 +117,8 @@ public class BucketInfo implements Serializable { private final String locationType; private final Logging logging; private final CustomPlacementConfig customPlacementConfig; + private final ObjectRetention objectRetention; + private final transient ImmutableSet modifiedFields; /** @@ -480,6 +485,95 @@ public Autoclass build() { } } + public static final class ObjectRetention implements Serializable { + + private static final long serialVersionUID = 3948199339534287669L; + private Mode mode; + + public Mode getMode() { + return mode; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof ObjectRetention)) { + return false; + } + ObjectRetention that = (ObjectRetention) o; + return Objects.equals(mode, that.mode); + } + + @Override + public int hashCode() { + return Objects.hash(mode); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this).add("mode", mode).toString(); + } + + private ObjectRetention() {} + + private ObjectRetention(Builder builder) { + this.mode = builder.mode; + } + + public static Builder newBuilder() { + return new Builder(); + } + + public Builder toBuilder() { + return new Builder().setMode(this.mode); + } + + public static final class Builder { + private Mode mode; + + /** Sets the object retention mode. Can be Enabled or Disabled. */ + public Builder setMode(Mode mode) { + this.mode = mode; + return this; + } + + public ObjectRetention build() { + return new ObjectRetention(this); + } + } + + public static final class Mode extends StringEnumValue { + private static final long serialVersionUID = 1973143582659557184L; + + private Mode(String constant) { + super(constant); + } + + private static final ApiFunction CONSTRUCTOR = Mode::new; + + private static final StringEnumType type = + new StringEnumType<>(Mode.class, CONSTRUCTOR); + + public static final Mode ENABLED = type.createAndRegister("Enabled"); + + public static final Mode DISABLED = type.createAndRegister("Disabled"); + + public static Mode valueOfStrict(String constant) { + return type.valueOfStrict(constant); + } + + public static Mode valueOf(String constant) { + return type.valueOf(constant); + } + + public static Mode[] values() { + return type.values(); + } + } + } + /** * The bucket's custom placement configuration for Custom Dual Regions. If using `location` is * also required. @@ -1589,6 +1683,8 @@ public Builder setRetentionPeriodDuration(Duration retentionPeriod) { public abstract Builder setCustomPlacementConfig(CustomPlacementConfig customPlacementConfig); + abstract Builder setObjectRetention(ObjectRetention objectRetention); + /** Creates a {@code BucketInfo} object. */ public abstract BucketInfo build(); @@ -1686,6 +1782,7 @@ static final class BuilderImpl extends Builder { private String locationType; private Logging logging; private CustomPlacementConfig customPlacementConfig; + private ObjectRetention objectRetention; private final ImmutableSet.Builder modifiedFields = ImmutableSet.builder(); BuilderImpl(String name) { @@ -1724,6 +1821,7 @@ static final class BuilderImpl extends Builder { locationType = bucketInfo.locationType; logging = bucketInfo.logging; customPlacementConfig = bucketInfo.customPlacementConfig; + objectRetention = bucketInfo.objectRetention; } @Override @@ -2080,6 +2178,15 @@ public Builder setCustomPlacementConfig(CustomPlacementConfig customPlacementCon return this; } + @Override + Builder setObjectRetention(ObjectRetention objectRetention) { + if (!Objects.equals(this.objectRetention, objectRetention)) { + modifiedFields.add(BucketField.OBJECT_RETENTION); + } + this.objectRetention = objectRetention; + return this; + } + @Override Builder setLocationType(String locationType) { if (!Objects.equals(this.locationType, locationType)) { @@ -2320,6 +2427,7 @@ private Builder clearDeleteLifecycleRules() { locationType = builder.locationType; logging = builder.logging; customPlacementConfig = builder.customPlacementConfig; + objectRetention = builder.objectRetention; modifiedFields = builder.modifiedFields.build(); } @@ -2655,6 +2763,11 @@ public CustomPlacementConfig getCustomPlacementConfig() { return customPlacementConfig; } + /** returns the Object Retention configuration */ + public ObjectRetention getObjectRetention() { + return objectRetention; + } + /** Returns a builder for the current bucket. */ public Builder toBuilder() { return new BuilderImpl(this); @@ -2691,6 +2804,7 @@ public int hashCode() { iamConfiguration, autoclass, locationType, + objectRetention, logging); } @@ -2731,6 +2845,7 @@ public boolean equals(Object o) { && Objects.equals(iamConfiguration, that.iamConfiguration) && Objects.equals(autoclass, that.autoclass) && Objects.equals(locationType, that.locationType) + && Objects.equals(objectRetention, that.objectRetention) && Objects.equals(logging, that.logging); } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageImpl.java index 516748dc22..a5613ff6ea 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageImpl.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageImpl.java @@ -135,6 +135,7 @@ import java.nio.file.OpenOption; import java.nio.file.Path; import java.nio.file.StandardOpenOption; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Objects; @@ -169,7 +170,12 @@ final class GrpcStorageImpl extends BaseService private static final Opts ALL_BLOB_FIELDS = Opts.from(UnifiedOpts.fields(ImmutableSet.copyOf(BlobField.values()))); private static final Opts ALL_BUCKET_FIELDS = - Opts.from(UnifiedOpts.fields(ImmutableSet.copyOf(BucketField.values()))); + // todo: b/308194853 + Opts.from( + UnifiedOpts.fields( + Arrays.stream(BucketField.values()) + .filter(f -> !f.equals(BucketField.OBJECT_RETENTION)) + .collect(ImmutableSet.toImmutableSet()))); final StorageClient storageClient; final WriterFactory writerFactory; diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/JsonConversions.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/JsonConversions.java index 8ba271c944..4bbf08d4b9 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/JsonConversions.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/JsonConversions.java @@ -53,6 +53,7 @@ import com.google.cloud.storage.Acl.Role; import com.google.cloud.storage.Acl.User; import com.google.cloud.storage.BlobInfo.CustomerEncryption; +import com.google.cloud.storage.BlobInfo.Retention; import com.google.cloud.storage.BucketInfo.Autoclass; import com.google.cloud.storage.BucketInfo.CustomPlacementConfig; import com.google.cloud.storage.BucketInfo.IamConfiguration; @@ -63,6 +64,7 @@ import com.google.cloud.storage.BucketInfo.LifecycleRule.LifecycleCondition; import com.google.cloud.storage.BucketInfo.LifecycleRule.SetStorageClassLifecycleAction; import com.google.cloud.storage.BucketInfo.Logging; +import com.google.cloud.storage.BucketInfo.ObjectRetention; import com.google.cloud.storage.BucketInfo.PublicAccessPrevention; import com.google.cloud.storage.Conversions.Codec; import com.google.cloud.storage.Cors.Origin; @@ -114,6 +116,9 @@ final class JsonConversions { Codec.of(this::iamConfigEncode, this::iamConfigDecode); private final Codec autoclassCodec = Codec.of(this::autoclassEncode, this::autoclassDecode); + + private final Codec objectRetentionCodec = + Codec.of(this::objectRetentionEncode, this::objectRetentionDecode); private final Codec lifecycleRuleCodec = Codec.of(this::lifecycleRuleEncode, this::lifecycleRuleDecode); private final Codec lifecycleConditionCodec = @@ -124,6 +129,9 @@ final class JsonConversions { private final Codec customerEncryptionCodec = Codec.of(this::customerEncryptionEncode, this::customerEncryptionDecode); + + private final Codec retentionCodec = + Codec.of(this::retentionEncode, this::retentionDecode); private final Codec blobIdCodec = Codec.of(this::blobIdEncode, this::blobIdDecode); private final Codec blobInfoCodec = @@ -238,6 +246,19 @@ private StorageObject blobInfoEncode(BlobInfo from) { from.getRetentionExpirationTimeOffsetDateTime(), dateTimeCodec::encode, to::setRetentionExpirationTime); + + // todo: clean this up once retention is enabled in grpc + // This is a workaround so that explicitly null retention objects are only included when the + // user set an existing policy to null, to avoid sending any retention objects to the test + // bench. + // We should clean this up once the test bench can handle the retention field. + // See also the comment in StorageImpl.update(BlobInfo blobInfo, BlobTargetOption... options) + // todo: b/308194853 + if (from.getModifiedFields().contains(Storage.BlobField.RETENTION) + && from.getRetention() == null) { + to.setRetention(Data.nullOf(StorageObject.Retention.class)); + } + ifNonNull(from.getRetention(), this::retentionEncode, to::setRetention); to.setKmsKeyName(from.getKmsKeyName()); to.setEventBasedHold(from.getEventBasedHold()); to.setTemporaryHold(from.getTemporaryHold()); @@ -306,6 +327,7 @@ private BlobInfo blobInfoDecode(StorageObject from) { from.getRetentionExpirationTime(), dateTimeCodec::decode, to::setRetentionExpirationTimeOffsetDateTime); + ifNonNull(from.getRetention(), this::retentionDecode, to::setRetention); return to.build(); } @@ -331,6 +353,20 @@ private CustomerEncryption customerEncryptionDecode(StorageObject.CustomerEncryp return new CustomerEncryption(from.getEncryptionAlgorithm(), from.getKeySha256()); } + private StorageObject.Retention retentionEncode(Retention from) { + StorageObject.Retention to = new StorageObject.Retention(); + ifNonNull(from.getMode(), Retention.Mode::toString, to::setMode); + ifNonNull(from.getRetainUntilTime(), dateTimeCodec::encode, to::setRetainUntilTime); + return to; + } + + private Retention retentionDecode(StorageObject.Retention from) { + Retention.Builder to = Retention.newBuilder(); + ifNonNull(from.getMode(), Retention.Mode::valueOf, to::setMode); + ifNonNull(from.getRetainUntilTime(), dateTimeCodec::decode, to::setRetainUntilTime); + return to.build(); + } + private Bucket bucketInfoEncode(BucketInfo from) { Bucket to = new Bucket(); ifNonNull(from.getProject(), projectNameCodec::encode, p -> to.set(PROJECT_ID_FIELD_NAME, p)); @@ -400,6 +436,7 @@ private Bucket bucketInfoEncode(BucketInfo from) { from.getCustomPlacementConfig(), this::customPlacementConfigEncode, to::setCustomPlacementConfig); + ifNonNull(from.getObjectRetention(), this::objectRetentionEncode, to::setObjectRetention); return to; } @@ -450,7 +487,7 @@ private BucketInfo bucketInfoDecode(com.google.api.services.storage.model.Bucket from.getCustomPlacementConfig(), this::customPlacementConfigDecode, to::setCustomPlacementConfig); - + ifNonNull(from.getObjectRetention(), this::objectRetentionDecode, to::setObjectRetention); return to.build(); } @@ -504,6 +541,18 @@ private Autoclass autoclassDecode(Bucket.Autoclass from) { return to.build(); } + private Bucket.ObjectRetention objectRetentionEncode(ObjectRetention from) { + Bucket.ObjectRetention to = new Bucket.ObjectRetention(); + ifNonNull(from.getMode(), ObjectRetention.Mode::toString, to::setMode); + return to; + } + + private ObjectRetention objectRetentionDecode(Bucket.ObjectRetention from) { + ObjectRetention.Builder to = ObjectRetention.newBuilder(); + ifNonNull(from.getMode(), ObjectRetention.Mode::valueOf, to::setMode); + return to.build(); + } + private UniformBucketLevelAccess ublaEncode(IamConfiguration from) { UniformBucketLevelAccess to = new UniformBucketLevelAccess(); to.setEnabled(from.isUniformBucketLevelAccessEnabled()); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java index abec9c2f9a..5ca31c042d 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java @@ -158,7 +158,9 @@ enum BucketField implements FieldSelector, NamedField { @TransportCompatibility({Transport.HTTP, Transport.GRPC}) CUSTOM_PLACEMENT_CONFIG("customPlacementConfig", "custom_placement_config"), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - AUTOCLASS("autoclass"); + AUTOCLASS("autoclass"), + @TransportCompatibility({Transport.HTTP}) + OBJECT_RETENTION("objectRetention"); static final List REQUIRED_FIELDS = ImmutableList.of(NAME); @@ -256,7 +258,9 @@ enum BlobField implements FieldSelector, NamedField { @TransportCompatibility({Transport.HTTP, Transport.GRPC}) TIME_STORAGE_CLASS_UPDATED("timeStorageClassUpdated", "update_storage_class_time"), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) - CUSTOMER_ENCRYPTION("customerEncryption", "customer_encryption"); + CUSTOMER_ENCRYPTION("customerEncryption", "customer_encryption"), + @TransportCompatibility({Transport.HTTP}) + RETENTION("retention"); static final List REQUIRED_FIELDS = ImmutableList.of(BUCKET, NAME); @@ -324,6 +328,16 @@ public static BucketTargetOption predefinedDefaultObjectAcl(@NonNull PredefinedA return new BucketTargetOption(UnifiedOpts.predefinedDefaultObjectAcl(acl)); } + /** + * Returns an option for enabling Object Retention on this bucket. Enabling this will create an + * ObjectRetention object in the created bucket (You must use this option, creating your own + * ObjectRetention object in the request won't work). + */ + @TransportCompatibility({Transport.HTTP}) + public static BucketTargetOption enableObjectRetention(boolean enable) { + return new BucketTargetOption(UnifiedOpts.enableObjectRetention(enable)); + } + /** * Returns an option for bucket's metageneration match. If this option is used the request will * fail if metageneration does not match. @@ -1025,6 +1039,16 @@ public static BlobTargetOption kmsKeyName(@NonNull String kmsKeyName) { return new BlobTargetOption(UnifiedOpts.kmsKeyName(kmsKeyName)); } + /** + * Returns an option for overriding an Unlocked Retention policy. This must be set to true in + * order to change a policy from Unlocked to Locked, to set it to null, or to reduce its + * retainUntilTime attribute. + */ + @TransportCompatibility({Transport.HTTP}) + public static BlobTargetOption overrideUnlockedRetention(boolean overrideUnlockedRetention) { + return new BlobTargetOption(UnifiedOpts.overrideUnlockedRetention(overrideUnlockedRetention)); + } + /** * Deduplicate any options which are the same parameter. The value which comes last in {@code * os} will be the value included in the return. diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java index 48167b5db1..1540f452d8 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java @@ -487,7 +487,17 @@ public Blob update(BlobInfo blobInfo, BlobTargetOption... options) { Opts opts = Opts.unwrap(options).resolveFrom(blobInfo); Map optionsMap = opts.getRpcOptions(); boolean unmodifiedBeforeOpts = blobInfo.getModifiedFields().isEmpty(); - BlobInfo updated = opts.blobInfoMapper().apply(blobInfo.toBuilder()).build(); + BlobInfo.Builder builder = blobInfo.toBuilder(); + + // This is a workaround until everything is in prod for both json and grpc. + // We need to make sure that the retention field is only included in the + // request if it was modified, so that we don't send a null object in a + // grpc or json request. + // todo: b/308194853 + if (blobInfo.getModifiedFields().contains(BlobField.RETENTION)) { + builder.setRetention(blobInfo.getRetention()); + } + BlobInfo updated = opts.blobInfoMapper().apply(builder).build(); boolean unmodifiedAfterOpts = updated.getModifiedFields().isEmpty(); if (unmodifiedBeforeOpts && unmodifiedAfterOpts) { return internalGetBlob(blobInfo.getBlobId(), optionsMap); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/UnifiedOpts.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/UnifiedOpts.java index 99a05342ef..7765596b33 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/UnifiedOpts.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/UnifiedOpts.java @@ -450,6 +450,14 @@ static PredefinedDefaultObjectAcl predefinedDefaultObjectAcl( return new PredefinedDefaultObjectAcl(predefinedAcl.getEntry()); } + static EnableObjectRetention enableObjectRetention(boolean enable) { + return new EnableObjectRetention(enable); + } + + static OverrideUnlockedRetention overrideUnlockedRetention(boolean overrideUnlockedRetention) { + return new OverrideUnlockedRetention(overrideUnlockedRetention); + } + static Prefix prefix(@NonNull String prefix) { requireNonNull(prefix, "prefix must be non null"); return new Prefix(prefix); @@ -1399,6 +1407,20 @@ public Mapper updateBucket() { } } + static final class EnableObjectRetention extends RpcOptVal implements BucketTargetOpt { + private static final long serialVersionUID = -2581147719605551578L; + + private EnableObjectRetention(boolean val) { + super(StorageRpc.Option.ENABLE_OBJECT_RETENTION, val); + } + + @Override + public Mapper updateBucket() { + return CrossTransportUtils.throwHttpJsonOnly( + Storage.BucketTargetOption.class, "enableObjectRetention(boolean)"); + } + } + static final class Prefix extends RpcOptVal implements BucketListOpt, ObjectListOpt { private static final long serialVersionUID = -3973478772547687371L; @@ -1623,6 +1645,22 @@ public String toString() { } } + static final class OverrideUnlockedRetention extends RpcOptVal + implements ObjectTargetOpt { + + private static final long serialVersionUID = -7764590745622588287L; + + private OverrideUnlockedRetention(boolean val) { + super(StorageRpc.Option.OVERRIDE_UNLOCKED_RETENTION, val); + } + + @Override + public Mapper updateObject() { + return CrossTransportUtils.throwHttpJsonOnly( + Storage.BlobTargetOption.class, "overrideUnlockedRetention(boolean)"); + } + } + static final class ShowDeletedKeys extends RpcOptVal<@NonNull Boolean> implements HmacKeyListOpt { private static final long serialVersionUID = -6604176744362903487L; diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java index 1402a4b572..04847ff0bd 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java @@ -364,6 +364,7 @@ public Bucket create(Bucket bucket, Map options) { .setPredefinedAcl(Option.PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(Option.PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .setUserProject(Option.USER_PROJECT.getString(options)) + .setEnableObjectRetention(Option.ENABLE_OBJECT_RETENTION.getBoolean(options)) .execute(); } catch (IOException ex) { span.setStatus(Status.UNKNOWN.withDescription(ex.getMessage())); @@ -622,6 +623,7 @@ private Storage.Objects.Patch patchCall(StorageObject storageObject, Map !f.equals(Storage.BucketField.OBJECT_RETENTION)) + .collect(ImmutableSet.toImmutableSet()) + .toArray(new Storage.BucketField[0]); + } } diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITAccessTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITAccessTest.java index ab09709fcc..2c5826687a 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITAccessTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITAccessTest.java @@ -45,6 +45,7 @@ import com.google.cloud.storage.Storage.BucketGetOption; import com.google.cloud.storage.Storage.BucketTargetOption; import com.google.cloud.storage.StorageException; +import com.google.cloud.storage.TestUtils; import com.google.cloud.storage.TransportCompatibility.Transport; import com.google.cloud.storage.it.runner.StorageITRunner; import com.google.cloud.storage.it.runner.annotations.Backend; @@ -150,7 +151,10 @@ public void bucket_defaultAcl_create() throws Exception { assertThat(actual.getEtag()).isNotEmpty(); Bucket bucketUpdated = - storage.get(bucket.getName(), BucketGetOption.fields(BucketField.values())); + storage.get( + bucket.getName(), + BucketGetOption.fields( + TestUtils.filterOutHttpOnlyBucketFields(BucketField.values()))); assertThat(bucketUpdated.getMetageneration()).isNotEqualTo(bucket.getMetageneration()); // etags change when updates happen, drop before our comparison @@ -203,7 +207,10 @@ public void bucket_defaultAcl_update() throws Exception { assertThat(actual.getEtag()).isNotEmpty(); Bucket bucketUpdated = - storage.get(bucket.getName(), BucketGetOption.fields(BucketField.values())); + storage.get( + bucket.getName(), + BucketGetOption.fields( + TestUtils.filterOutHttpOnlyBucketFields(BucketField.values()))); assertThat(bucketUpdated.getMetageneration()).isNotEqualTo(bucket.getMetageneration()); // etags change when updates happen, drop before our comparison @@ -255,7 +262,10 @@ public void bucket_defaultAcl_delete() throws Exception { assertThat(actual).isTrue(); Bucket bucketUpdated = - storage.get(bucket.getName(), BucketGetOption.fields(BucketField.values())); + storage.get( + bucket.getName(), + BucketGetOption.fields( + TestUtils.filterOutHttpOnlyBucketFields(BucketField.values()))); assertThat(bucketUpdated.getMetageneration()).isNotEqualTo(bucket.getMetageneration()); // etags change when deletes happen, drop before our comparison diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBlobReadMaskTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBlobReadMaskTest.java index 96b478d30f..c2594a484f 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBlobReadMaskTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBlobReadMaskTest.java @@ -198,7 +198,10 @@ public ImmutableList parameters() { new Args<>(BlobField.TIME_CREATED, LazyAssertion.equal()), new Args<>(BlobField.TIME_DELETED, LazyAssertion.equal()), new Args<>(BlobField.TIME_STORAGE_CLASS_UPDATED, LazyAssertion.equal()), - new Args<>(BlobField.UPDATED, LazyAssertion.equal())); + new Args<>(BlobField.UPDATED, LazyAssertion.equal()), + new Args<>( + BlobField.RETENTION, + LazyAssertion.skip("TODO: jesse fill in buganizer bug here"))); List argsDefined = args.stream().map(Args::getField).map(Enum::name).sorted().collect(Collectors.toList()); diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketAclTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketAclTest.java index a1545202e5..1b05a0a1d6 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketAclTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketAclTest.java @@ -34,6 +34,7 @@ import com.google.cloud.storage.Storage.BucketField; import com.google.cloud.storage.Storage.BucketGetOption; import com.google.cloud.storage.StorageException; +import com.google.cloud.storage.TestUtils; import com.google.cloud.storage.TransportCompatibility.Transport; import com.google.cloud.storage.it.runner.StorageITRunner; import com.google.cloud.storage.it.runner.annotations.Backend; @@ -127,7 +128,10 @@ public void bucket_acl_create() throws Exception { assertThat(actual.getEtag()).isNotEmpty(); Bucket bucketUpdated = - storage.get(bucket.getName(), BucketGetOption.fields(BucketField.values())); + storage.get( + bucket.getName(), + BucketGetOption.fields( + TestUtils.filterOutHttpOnlyBucketFields(BucketField.values()))); assertThat(bucketUpdated.getMetageneration()).isNotEqualTo(bucket.getMetageneration()); // etags change when updates happen, drop before our comparison @@ -177,7 +181,10 @@ public void bucket_acl_update() throws Exception { assertThat(actual.getEtag()).isNotEmpty(); Bucket bucketUpdated = - storage.get(bucket.getName(), BucketGetOption.fields(BucketField.values())); + storage.get( + bucket.getName(), + BucketGetOption.fields( + TestUtils.filterOutHttpOnlyBucketFields(BucketField.values()))); assertThat(bucketUpdated.getMetageneration()).isNotEqualTo(bucket.getMetageneration()); // etags change when updates happen, drop before our comparison @@ -220,7 +227,11 @@ public void bucket_acl_404_acl_update() throws Exception { assertThat(actual.getRole()).isEqualTo(readAll.getRole()); assertThat(actual.getEtag()).isNotEmpty(); - Bucket updated = storage.get(mgen1.getName(), BucketGetOption.fields(BucketField.values())); + Bucket updated = + storage.get( + mgen1.getName(), + BucketGetOption.fields( + TestUtils.filterOutHttpOnlyBucketFields(BucketField.values()))); assertThat(updated.getMetageneration()).isNotEqualTo(bucket.getMetageneration()); // etags change when updates happen, drop before our comparison @@ -254,7 +265,10 @@ public void bucket_acl_delete() throws Exception { assertThat(actual).isTrue(); Bucket bucketUpdated = - storage.get(bucket.getName(), BucketGetOption.fields(BucketField.values())); + storage.get( + bucket.getName(), + BucketGetOption.fields( + TestUtils.filterOutHttpOnlyBucketFields(BucketField.values()))); assertThat(bucketUpdated.getMetageneration()).isNotEqualTo(bucket.getMetageneration()); // etags change when deletes happen, drop before our comparison diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketReadMaskTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketReadMaskTest.java index d65c271d5e..93099d41e9 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketReadMaskTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketReadMaskTest.java @@ -25,6 +25,7 @@ import com.google.cloud.storage.Storage.BucketField; import com.google.cloud.storage.Storage.BucketGetOption; import com.google.cloud.storage.Storage.BucketListOption; +import com.google.cloud.storage.TestUtils; import com.google.cloud.storage.TransportCompatibility.Transport; import com.google.cloud.storage.it.ITBucketReadMaskTest.BucketReadMaskTestParameters; import com.google.cloud.storage.it.ReadMaskTestUtils.Args; @@ -138,7 +139,10 @@ public ImmutableList parameters() { args.stream().map(Args::getField).map(Enum::name).sorted().collect(Collectors.toList()); List definedFields = - Arrays.stream(BucketField.values()).map(Enum::name).sorted().collect(Collectors.toList()); + Arrays.stream(TestUtils.filterOutHttpOnlyBucketFields(BucketField.values())) + .map(Enum::name) + .sorted() + .collect(Collectors.toList()); assertThat(argsDefined).containsExactlyElementsIn(definedFields); return args; diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketTest.java index f5b2b00c53..e32be5c32c 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketTest.java @@ -33,6 +33,7 @@ import com.google.cloud.storage.BucketInfo; import com.google.cloud.storage.BucketInfo.Autoclass; import com.google.cloud.storage.BucketInfo.CustomPlacementConfig; +import com.google.cloud.storage.BucketInfo.ObjectRetention.Mode; import com.google.cloud.storage.Cors; import com.google.cloud.storage.HttpMethod; import com.google.cloud.storage.Rpo; @@ -43,6 +44,7 @@ import com.google.cloud.storage.Storage.BucketListOption; import com.google.cloud.storage.Storage.BucketTargetOption; import com.google.cloud.storage.StorageClass; +import com.google.cloud.storage.TestUtils; import com.google.cloud.storage.TransportCompatibility.Transport; import com.google.cloud.storage.it.runner.StorageITRunner; import com.google.cloud.storage.it.runner.annotations.Backend; @@ -55,6 +57,7 @@ import com.google.common.collect.ImmutableMap; import java.time.Duration; import java.time.OffsetDateTime; +import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -111,7 +114,10 @@ public void testGetBucketSelectedFields() { @Test public void testGetBucketAllSelectedFields() { Bucket remoteBucket = - storage.get(bucket.getName(), Storage.BucketGetOption.fields(BucketField.values())); + storage.get( + bucket.getName(), + Storage.BucketGetOption.fields( + TestUtils.filterOutHttpOnlyBucketFields(BucketField.values()))); assertEquals(bucket.getName(), remoteBucket.getName()); assertNotNull(remoteBucket.getCreateTime()); } @@ -420,6 +426,69 @@ public void testCreateBucketWithAutoclass() { } @Test + @CrossRun.Exclude(transports = Transport.GRPC) + public void testObjectRetention() { + String bucketName = generator.randomBucketName(); + + // Create a bucket with object retention enabled + storage.create( + BucketInfo.newBuilder(bucketName).build(), BucketTargetOption.enableObjectRetention(true)); + + try { + Bucket remoteBucket = storage.get(bucketName); + assertNotNull(remoteBucket.getObjectRetention()); + assertEquals(Mode.ENABLED, remoteBucket.getObjectRetention().getMode()); + + OffsetDateTime now = OffsetDateTime.now(); + + // Create an object with a retention policy configured + storage.create( + BlobInfo.newBuilder(bucketName, "retentionObject") + .setRetention( + BlobInfo.Retention.newBuilder() + .setMode(BlobInfo.Retention.Mode.UNLOCKED) + .setRetainUntilTime(now.plusDays(2)) + .build()) + .build()); + + Blob remoteBlob = storage.get(bucketName, "retentionObject"); + assertNotNull(remoteBlob.getRetention()); + assertEquals(BlobInfo.Retention.Mode.UNLOCKED, remoteBlob.getRetention().getMode()); + + // Reduce the retainUntilTime of an object's retention policy + remoteBlob + .toBuilder() + .setRetention( + BlobInfo.Retention.newBuilder() + .setMode(BlobInfo.Retention.Mode.UNLOCKED) + .setRetainUntilTime(now.plusHours(1)) + .build()) + .build() + .update(Storage.BlobTargetOption.overrideUnlockedRetention(true)); + + remoteBlob = storage.get(bucketName, "retentionObject"); + assertEquals( + now.plusHours(1).toInstant().truncatedTo(ChronoUnit.SECONDS), + remoteBlob + .getRetention() + .getRetainUntilTime() + .toInstant() + .truncatedTo(ChronoUnit.SECONDS)); + + // Remove an unlocked retention policy + remoteBlob + .toBuilder() + .setRetention(null) + .build() + .update(Storage.BlobTargetOption.overrideUnlockedRetention(true)); + + remoteBlob = storage.get(bucketName, "retentionObject"); + assertNull(remoteBlob.getRetention()); + } finally { + BucketCleaner.doCleanup(bucketName, storage); + } + } + public void testCreateBucketWithAutoclass_ARCHIVE() throws Exception { String bucketName = generator.randomBucketName(); Autoclass autoclass = diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITOptionRegressionTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITOptionRegressionTest.java index dc99678c6d..4dbe55adc2 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITOptionRegressionTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITOptionRegressionTest.java @@ -50,6 +50,7 @@ import com.google.cloud.storage.Storage.UpdateHmacKeyOption; import com.google.cloud.storage.StorageClass; import com.google.cloud.storage.StorageException; +import com.google.cloud.storage.TestUtils; import com.google.cloud.storage.TransportCompatibility.Transport; import com.google.cloud.storage.it.runner.StorageITRunner; import com.google.cloud.storage.it.runner.annotations.Backend; @@ -326,7 +327,9 @@ public void storage_BucketGetOption_fields_BucketField() { "updated", "versioning", "website"); - s.get(b.getName(), BucketGetOption.fields(BucketField.values())); + s.get( + b.getName(), + BucketGetOption.fields(TestUtils.filterOutHttpOnlyBucketFields(BucketField.values()))); requestAuditing.assertQueryParam("fields", expected, splitOnCommaToSet()); } @@ -734,7 +737,8 @@ public void storage_BlobGetOption_fields_BlobField() { "timeCreated", "timeDeleted", "timeStorageClassUpdated", - "updated"); + "updated", + "retention"); s.get(o.getBlobId(), BlobGetOption.fields(BlobField.values())); requestAuditing.assertQueryParam("fields", expected, splitOnCommaToSet()); } @@ -813,7 +817,7 @@ public void storage_BucketListOption_fields_BucketField() { "items/updated", "items/versioning", "items/website"); - s.list(BucketListOption.fields(BucketField.values())); + s.list(BucketListOption.fields(TestUtils.filterOutHttpOnlyBucketFields(BucketField.values()))); requestAuditing.assertQueryParam("fields", expected, splitOnCommaToSet()); } @@ -908,7 +912,8 @@ public void storage_BlobListOption_fields_BlobField() { "items/timeCreated", "items/timeDeleted", "items/timeStorageClassUpdated", - "items/updated"); + "items/updated", + "items/retention"); s.list(b.getName(), BlobListOption.fields(BlobField.values())); requestAuditing.assertQueryParam("fields", expected, splitOnCommaToSet()); }