Skip to content

Commit a179337

Browse files
cojencoparthea
cojenco
and
parthea
authored
feat: support object retention lock (#1188)
* feat: add support for object retention lock * add Retention config object in Blob * update tests * update test coverage * clarify docstrings --------- Co-authored-by: Anthonios Partheniou <partheniou@google.com>
1 parent 22f36da commit a179337

File tree

9 files changed

+308
-2
lines changed

9 files changed

+308
-2
lines changed

‎google/cloud/storage/_helpers.py

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -290,6 +290,7 @@ def patch(
290290
if_metageneration_not_match=None,
291291
timeout=_DEFAULT_TIMEOUT,
292292
retry=DEFAULT_RETRY_IF_METAGENERATION_SPECIFIED,
293+
override_unlocked_retention=False,
293294
):
294295
"""Sends all changed properties in a PATCH request.
295296
@@ -326,12 +327,21 @@ def patch(
326327
:type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
327328
:param retry:
328329
(Optional) How to retry the RPC. See: :ref:`configuring_retries`
330+
331+
:type override_unlocked_retention: bool
332+
:param override_unlocked_retention:
333+
(Optional) override_unlocked_retention must be set to True if the operation includes
334+
a retention property that changes the mode from Unlocked to Locked, reduces the
335+
retainUntilTime, or removes the retention configuration from the object. See:
336+
https://cloud.google.com/storage/docs/json_api/v1/objects/patch
329337
"""
330338
client = self._require_client(client)
331339
query_params = self._query_params
332340
# Pass '?projection=full' here because 'PATCH' documented not
333341
# to work properly w/ 'noAcl'.
334342
query_params["projection"] = "full"
343+
if override_unlocked_retention:
344+
query_params["overrideUnlockedRetention"] = override_unlocked_retention
335345
_add_generation_match_parameters(
336346
query_params,
337347
if_generation_match=if_generation_match,
@@ -361,6 +371,7 @@ def update(
361371
if_metageneration_not_match=None,
362372
timeout=_DEFAULT_TIMEOUT,
363373
retry=DEFAULT_RETRY_IF_METAGENERATION_SPECIFIED,
374+
override_unlocked_retention=False,
364375
):
365376
"""Sends all properties in a PUT request.
366377
@@ -397,11 +408,20 @@ def update(
397408
:type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
398409
:param retry:
399410
(Optional) How to retry the RPC. See: :ref:`configuring_retries`
411+
412+
:type override_unlocked_retention: bool
413+
:param override_unlocked_retention:
414+
(Optional) override_unlocked_retention must be set to True if the operation includes
415+
a retention property that changes the mode from Unlocked to Locked, reduces the
416+
retainUntilTime, or removes the retention configuration from the object. See:
417+
https://cloud.google.com/storage/docs/json_api/v1/objects/patch
400418
"""
401419
client = self._require_client(client)
402420

403421
query_params = self._query_params
404422
query_params["projection"] = "full"
423+
if override_unlocked_retention:
424+
query_params["overrideUnlockedRetention"] = override_unlocked_retention
405425
_add_generation_match_parameters(
406426
query_params,
407427
if_generation_match=if_generation_match,

‎google/cloud/storage/blob.py

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,7 @@
102102
"md5Hash",
103103
"metadata",
104104
"name",
105+
"retention",
105106
"storageClass",
106107
)
107108
_READ_LESS_THAN_SIZE = (
@@ -1700,6 +1701,7 @@ def _get_writable_metadata(self):
17001701
* ``md5Hash``
17011702
* ``metadata``
17021703
* ``name``
1704+
* ``retention``
17031705
* ``storageClass``
17041706
17051707
For now, we don't support ``acl``, access control lists should be
@@ -4667,6 +4669,16 @@ def custom_time(self, value):
46674669

46684670
self._patch_property("customTime", value)
46694671

4672+
@property
4673+
def retention(self):
4674+
"""Retrieve the retention configuration for this object.
4675+
4676+
:rtype: :class:`Retention`
4677+
:returns: an instance for managing the object's retention configuration.
4678+
"""
4679+
info = self._properties.get("retention", {})
4680+
return Retention.from_api_repr(info, self)
4681+
46704682

46714683
def _get_host_name(connection):
46724684
"""Returns the host name from the given connection.
@@ -4797,3 +4809,126 @@ def _add_query_parameters(base_url, name_value_pairs):
47974809
query = parse_qsl(query)
47984810
query.extend(name_value_pairs)
47994811
return urlunsplit((scheme, netloc, path, urlencode(query), frag))
4812+
4813+
4814+
class Retention(dict):
4815+
"""Map an object's retention configuration.
4816+
4817+
:type blob: :class:`Blob`
4818+
:params blob: blob for which this retention configuration applies to.
4819+
4820+
:type mode: str or ``NoneType``
4821+
:params mode:
4822+
(Optional) The mode of the retention configuration, which can be either Unlocked or Locked.
4823+
See: https://cloud.google.com/storage/docs/object-lock
4824+
4825+
:type retain_until_time: :class:`datetime.datetime` or ``NoneType``
4826+
:params retain_until_time:
4827+
(Optional) The earliest time that the object can be deleted or replaced, which is the
4828+
retention configuration set for this object.
4829+
4830+
:type retention_expiration_time: :class:`datetime.datetime` or ``NoneType``
4831+
:params retention_expiration_time:
4832+
(Optional) The earliest time that the object can be deleted, which depends on any
4833+
retention configuration set for the object and any retention policy set for the bucket
4834+
that contains the object. This value should normally only be set by the back-end API.
4835+
"""
4836+
4837+
def __init__(
4838+
self,
4839+
blob,
4840+
mode=None,
4841+
retain_until_time=None,
4842+
retention_expiration_time=None,
4843+
):
4844+
data = {"mode": mode}
4845+
if retain_until_time is not None:
4846+
retain_until_time = _datetime_to_rfc3339(retain_until_time)
4847+
data["retainUntilTime"] = retain_until_time
4848+
4849+
if retention_expiration_time is not None:
4850+
retention_expiration_time = _datetime_to_rfc3339(retention_expiration_time)
4851+
data["retentionExpirationTime"] = retention_expiration_time
4852+
4853+
super(Retention, self).__init__(data)
4854+
self._blob = blob
4855+
4856+
@classmethod
4857+
def from_api_repr(cls, resource, blob):
4858+
"""Factory: construct instance from resource.
4859+
4860+
:type blob: :class:`Blob`
4861+
:params blob: Blob for which this retention configuration applies to.
4862+
4863+
:type resource: dict
4864+
:param resource: mapping as returned from API call.
4865+
4866+
:rtype: :class:`Retention`
4867+
:returns: Retention configuration created from resource.
4868+
"""
4869+
instance = cls(blob)
4870+
instance.update(resource)
4871+
return instance
4872+
4873+
@property
4874+
def blob(self):
4875+
"""Blob for which this retention configuration applies to.
4876+
4877+
:rtype: :class:`Blob`
4878+
:returns: the instance's blob.
4879+
"""
4880+
return self._blob
4881+
4882+
@property
4883+
def mode(self):
4884+
"""The mode of the retention configuration. Options are 'Unlocked' or 'Locked'.
4885+
4886+
:rtype: string
4887+
:returns: The mode of the retention configuration, which can be either set to 'Unlocked' or 'Locked'.
4888+
"""
4889+
return self.get("mode")
4890+
4891+
@mode.setter
4892+
def mode(self, value):
4893+
self["mode"] = value
4894+
self.blob._patch_property("retention", self)
4895+
4896+
@property
4897+
def retain_until_time(self):
4898+
"""The earliest time that the object can be deleted or replaced, which is the
4899+
retention configuration set for this object.
4900+
4901+
:rtype: :class:`datetime.datetime` or ``NoneType``
4902+
:returns: Datetime object parsed from RFC3339 valid timestamp, or
4903+
``None`` if the blob's resource has not been loaded from
4904+
the server (see :meth:`reload`).
4905+
"""
4906+
value = self.get("retainUntilTime")
4907+
if value is not None:
4908+
return _rfc3339_nanos_to_datetime(value)
4909+
4910+
@retain_until_time.setter
4911+
def retain_until_time(self, value):
4912+
"""Set the retain_until_time for the object retention configuration.
4913+
4914+
:type value: :class:`datetime.datetime`
4915+
:param value: The earliest time that the object can be deleted or replaced.
4916+
"""
4917+
if value is not None:
4918+
value = _datetime_to_rfc3339(value)
4919+
self["retainUntilTime"] = value
4920+
self.blob._patch_property("retention", self)
4921+
4922+
@property
4923+
def retention_expiration_time(self):
4924+
"""The earliest time that the object can be deleted, which depends on any
4925+
retention configuration set for the object and any retention policy set for
4926+
the bucket that contains the object.
4927+
4928+
:rtype: :class:`datetime.datetime` or ``NoneType``
4929+
:returns:
4930+
(readonly) The earliest time that the object can be deleted.
4931+
"""
4932+
retention_expiration_time = self.get("retentionExpirationTime")
4933+
if retention_expiration_time is not None:
4934+
return _rfc3339_nanos_to_datetime(retention_expiration_time)

‎google/cloud/storage/bucket.py

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -917,6 +917,7 @@ def create(
917917
location=None,
918918
predefined_acl=None,
919919
predefined_default_object_acl=None,
920+
enable_object_retention=False,
920921
timeout=_DEFAULT_TIMEOUT,
921922
retry=DEFAULT_RETRY,
922923
):
@@ -956,6 +957,11 @@ def create(
956957
(Optional) Name of predefined ACL to apply to bucket's objects. See:
957958
https://cloud.google.com/storage/docs/access-control/lists#predefined-acl
958959
960+
:type enable_object_retention: bool
961+
:param enable_object_retention:
962+
(Optional) Whether object retention should be enabled on this bucket. See:
963+
https://cloud.google.com/storage/docs/object-lock
964+
959965
:type timeout: float or tuple
960966
:param timeout:
961967
(Optional) The amount of time, in seconds, to wait
@@ -974,6 +980,7 @@ def create(
974980
location=location,
975981
predefined_acl=predefined_acl,
976982
predefined_default_object_acl=predefined_default_object_acl,
983+
enable_object_retention=enable_object_retention,
977984
timeout=timeout,
978985
retry=retry,
979986
)
@@ -2750,6 +2757,18 @@ def autoclass_terminal_storage_class_update_time(self):
27502757
if timestamp is not None:
27512758
return _rfc3339_nanos_to_datetime(timestamp)
27522759

2760+
@property
2761+
def object_retention_mode(self):
2762+
"""Retrieve the object retention mode set on the bucket.
2763+
2764+
:rtype: str
2765+
:returns: When set to Enabled, retention configurations can be
2766+
set on objects in the bucket.
2767+
"""
2768+
object_retention = self._properties.get("objectRetention")
2769+
if object_retention is not None:
2770+
return object_retention.get("mode")
2771+
27532772
def configure_website(self, main_page_suffix=None, not_found_page=None):
27542773
"""Configure website-related properties.
27552774

‎google/cloud/storage/client.py

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -845,6 +845,7 @@ def create_bucket(
845845
data_locations=None,
846846
predefined_acl=None,
847847
predefined_default_object_acl=None,
848+
enable_object_retention=False,
848849
timeout=_DEFAULT_TIMEOUT,
849850
retry=DEFAULT_RETRY,
850851
):
@@ -883,6 +884,9 @@ def create_bucket(
883884
predefined_default_object_acl (str):
884885
(Optional) Name of predefined ACL to apply to bucket's objects. See:
885886
https://cloud.google.com/storage/docs/access-control/lists#predefined-acl
887+
enable_object_retention (bool):
888+
(Optional) Whether object retention should be enabled on this bucket. See:
889+
https://cloud.google.com/storage/docs/object-lock
886890
timeout (Optional[Union[float, Tuple[float, float]]]):
887891
The amount of time, in seconds, to wait for the server response.
888892
@@ -951,6 +955,9 @@ def create_bucket(
951955
if user_project is not None:
952956
query_params["userProject"] = user_project
953957

958+
if enable_object_retention:
959+
query_params["enableObjectRetention"] = enable_object_retention
960+
954961
properties = {key: bucket._properties[key] for key in bucket._changes}
955962
properties["name"] = bucket.name
956963

‎tests/system/test_blob.py

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1117,3 +1117,32 @@ def test_blob_update_storage_class_large_file(
11171117
blob.update_storage_class(constants.COLDLINE_STORAGE_CLASS)
11181118
blob.reload()
11191119
assert blob.storage_class == constants.COLDLINE_STORAGE_CLASS
1120+
1121+
1122+
def test_object_retention_lock(storage_client, buckets_to_delete, blobs_to_delete):
1123+
# Test bucket created with object retention enabled
1124+
new_bucket_name = _helpers.unique_name("object-retention")
1125+
created_bucket = _helpers.retry_429_503(storage_client.create_bucket)(
1126+
new_bucket_name, enable_object_retention=True
1127+
)
1128+
buckets_to_delete.append(created_bucket)
1129+
assert created_bucket.object_retention_mode == "Enabled"
1130+
1131+
# Test create object with object retention enabled
1132+
payload = b"Hello World"
1133+
mode = "Unlocked"
1134+
current_time = datetime.datetime.utcnow()
1135+
expiration_time = current_time + datetime.timedelta(seconds=10)
1136+
blob = created_bucket.blob("object-retention-lock")
1137+
blob.retention.mode = mode
1138+
blob.retention.retain_until_time = expiration_time
1139+
blob.upload_from_string(payload)
1140+
blobs_to_delete.append(blob)
1141+
blob.reload()
1142+
assert blob.retention.mode == mode
1143+
1144+
# Test patch object to disable object retention
1145+
blob.retention.mode = None
1146+
blob.retention.retain_until_time = None
1147+
blob.patch(override_unlocked_retention=True)
1148+
assert blob.retention.mode is None

‎tests/unit/test__helpers.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -353,12 +353,14 @@ def test_patch_w_metageneration_match_w_timeout_w_retry(self):
353353
retry = mock.Mock(spec=[])
354354
generation_number = 9
355355
metageneration_number = 6
356+
override_unlocked_retention = True
356357

357358
derived.patch(
358359
if_generation_match=generation_number,
359360
if_metageneration_match=metageneration_number,
360361
timeout=timeout,
361362
retry=retry,
363+
override_unlocked_retention=override_unlocked_retention,
362364
)
363365

364366
self.assertEqual(derived._properties, {"foo": "Foo"})
@@ -370,6 +372,7 @@ def test_patch_w_metageneration_match_w_timeout_w_retry(self):
370372
"projection": "full",
371373
"ifGenerationMatch": generation_number,
372374
"ifMetagenerationMatch": metageneration_number,
375+
"overrideUnlockedRetention": override_unlocked_retention,
373376
}
374377
client._patch_resource.assert_called_once_with(
375378
path,
@@ -454,10 +457,12 @@ def test_update_with_metageneration_not_match_w_timeout_w_retry(self):
454457
client = derived.client = mock.Mock(spec=["_put_resource"])
455458
client._put_resource.return_value = api_response
456459
timeout = 42
460+
override_unlocked_retention = True
457461

458462
derived.update(
459463
if_metageneration_not_match=generation_number,
460464
timeout=timeout,
465+
override_unlocked_retention=override_unlocked_retention,
461466
)
462467

463468
self.assertEqual(derived._properties, {"foo": "Foo"})
@@ -467,6 +472,7 @@ def test_update_with_metageneration_not_match_w_timeout_w_retry(self):
467472
expected_query_params = {
468473
"projection": "full",
469474
"ifMetagenerationNotMatch": generation_number,
475+
"overrideUnlockedRetention": override_unlocked_retention,
470476
}
471477
client._put_resource.assert_called_once_with(
472478
path,

0 commit comments

Comments
 (0)