feat(storage): Get/Insert/Update/Patch Object contexts

ci/cloudbuild/builds/lib/integration.sh

@@ -31,7 +31,7 @@ source module ci/lib/io.sh
 export PATH="${HOME}/.local/bin:${PATH}"
 python3 -m pip uninstall -y --quiet googleapis-storage-testbench
 python3 -m pip install --upgrade --user --quiet --disable-pip-version-check \
-  "git+https://github.com/googleapis/storage-testbench@v0.60.0"
+  "git+https://github.com/googleapis/storage-testbench@v0.61.0"
 
 # Some of the tests will need a valid roots.pem file.
 rm -f /dev/shm/roots.pem

google/cloud/storage/google_cloud_cpp_storage.bzl

@@ -116,6 +116,7 @@ google_cloud_cpp_storage_hdrs = [
     "notification_metadata.h",
     "notification_payload_format.h",
     "object_access_control.h",
+    "object_contexts.h",
     "object_metadata.h",
     "object_read_stream.h",
     "object_retention.h",
@@ -219,6 +220,7 @@ google_cloud_cpp_storage_srcs = [
     "list_objects_reader.cc",
     "notification_metadata.cc",
     "object_access_control.cc",
+    "object_contexts.cc",
     "object_metadata.cc",
     "object_read_stream.cc",
     "object_retention.cc",

google/cloud/storage/google_cloud_cpp_storage.cmake

@@ -197,6 +197,8 @@ add_library(
     notification_payload_format.h
     object_access_control.cc
     object_access_control.h
+    object_contexts.cc
+    object_contexts.h
     object_metadata.cc
     object_metadata.h
     object_read_stream.cc

google/cloud/storage/internal/grpc/object_metadata_parser.cc

@@ -171,6 +171,19 @@ storage::ObjectMetadata FromProto(google::storage::v2::Object object,
     metadata.set_hard_delete_time(
         google::cloud::internal::ToChronoTimePoint(object.hard_delete_time()));
   }
+  if (object.has_contexts()) {
+    storage::ObjectContexts contexts;
+    for (auto& kv : *object.mutable_contexts()->mutable_custom()) {
+      storage::ObjectCustomContextPayload payload;
+      payload.value = std::move(*kv.second.mutable_value());
+      payload.create_time =
+          google::cloud::internal::ToChronoTimePoint(kv.second.create_time());
+      payload.update_time =
+          google::cloud::internal::ToChronoTimePoint(kv.second.update_time());
+      contexts.upsert(std::move(kv.first), std::move(payload));
+    }
+    metadata.set_contexts(std::move(contexts));
+  }
   return metadata;
 }
 

google/cloud/storage/internal/grpc/object_metadata_parser_test.cc

@@ -79,6 +79,32 @@ TEST(GrpcClientFromProto, ObjectSimple) {
     }
     metadata: { key: "test-key-1" value: "test-value-1" }
     metadata: { key: "test-key-2" value: "test-value-2" }
+    contexts: {
+      custom: {
+        key: "custom-key-1" value: {
+          value: "custom-value-1"
+          create_time: {
+          seconds: 1565194924
+          nanos: 456789012
+        }
+        update_time: {
+          seconds: 1565194924
+          nanos: 456789012
+        }
+      }}
+      custom: {
+        key: "custom-key-2" value: {
+          value: "custom-value-2"
+          create_time: {
+          seconds: 1565194924
+          nanos: 456789012
+        }
+        update_time: {
+          seconds: 1709555696
+          nanos: 987654321
+        }
+      }}
+    }
     event_based_hold: true
     name: "test-object-name"
     bucket: "test-bucket"
@@ -141,6 +167,20 @@ TEST(GrpcClientFromProto, ObjectSimple) {
         "test-key-1": "test-value-1",
         "test-key-2": "test-value-2"
     },
+    "contexts": {
+        "custom": {
+            "custom-key-1": {
+                "value": "custom-value-1",
+                "createTime": "2019-08-07T16:22:04.456789012Z",
+                "updateTime": "2019-08-07T16:22:04.456789012Z"
+             },
+             "custom-key-2": {
+                "value": "custom-value-2",
+                "createTime": "2019-08-07T16:22:04.456789012Z",
+                "updateTime": "2024-03-04T12:34:56.987654321Z"
+             }
+        }
+    },
     "eventBasedHold": true,
     "name": "test-object-name",
     "id": "test-bucket/test-object-name/2345",

google/cloud/storage/internal/grpc/object_request_parser.cc

@@ -23,6 +23,7 @@
 #include "google/cloud/internal/make_status.h"
 #include "google/cloud/internal/time_utils.h"
 #include "absl/strings/str_cat.h"
+#include <google/storage/v2/storage.pb.h>
 #include <iterator>
 
 namespace google {
@@ -155,6 +156,15 @@ Status SetObjectMetadata(google::storage::v2::Object& resource,
     *resource.mutable_custom_time() =
         google::cloud::internal::ToProtoTimestamp(metadata.custom_time());
   }
+  if (metadata.has_contexts()) {
+    auto& custom_map = *resource.mutable_contexts()->mutable_custom();
+    for (auto const& kv : metadata.contexts().custom()) {
+      // In request, the create_time and update_time are ignored by the server,
+      // hence there is no need to parse them.
+      custom_map[kv.first].set_value(kv.second.value);
+    }
+  }
+
   return Status{};
 }
 
@@ -257,6 +267,48 @@ Status FinalizeChecksums(google::storage::v2::ObjectChecksums& checksums,
   return {};
 }
 
+void PatchGrpcMetadata(storage::ObjectMetadataPatchBuilder const& patch_builder,
+                       google::storage::v2::UpdateObjectRequest& result,
+                       google::storage::v2::Object& object) {
+  auto const& subpatch =
+      storage::internal::PatchBuilderDetails::GetMetadataSubPatch(
+          patch_builder);
+  if (subpatch.is_null()) {
+    object.clear_metadata();
+    result.mutable_update_mask()->add_paths("metadata");
+  } else {
+    for (auto const& kv : subpatch.items()) {
+      result.mutable_update_mask()->add_paths("metadata." + kv.key());
+      auto const& v = kv.value();
+      if (!v.is_string()) continue;
+      (*object.mutable_metadata())[kv.key()] = v.get<std::string>();
+    }
+  }
+}
+
+void PatchGrpcContexts(storage::ObjectMetadataPatchBuilder const& patch_builder,
+                       google::storage::v2::UpdateObjectRequest& result,
+                       google::storage::v2::Object& object) {
+  auto const& contexts_subpatch =
+      storage::internal::PatchBuilderDetails::GetCustomContextsSubPatch(
+          patch_builder);
+  if (contexts_subpatch.is_null()) {
+    object.clear_contexts();
+    result.mutable_update_mask()->add_paths("contexts.custom");
+  } else {
+    for (auto const& kv : contexts_subpatch.items()) {
+      result.mutable_update_mask()->add_paths("contexts.custom." + kv.key());
+      auto const& v = kv.value();
+      if (v.is_object() && v.contains("value")) {
+        std::string value_str = v["value"].get<std::string>();
+        auto& payload =
+            (*object.mutable_contexts()->mutable_custom())[kv.key()];
+        payload.set_value(std::move(value_str));
+      }
+    }
+  }
+}
+
 }  // namespace
 
 StatusOr<google::storage::v2::ComposeObjectRequest> ToProto(
@@ -276,6 +328,13 @@ StatusOr<google::storage::v2::ComposeObjectRequest> ToProto(
     for (auto const& kv : metadata.metadata()) {
       (*destination.mutable_metadata())[kv.first] = kv.second;
     }
+    if (metadata.has_contexts()) {
+      for (auto const& kv : metadata.contexts().custom()) {
+        auto& payload =
+            (*destination.mutable_contexts()->mutable_custom())[kv.first];
+        payload.set_value(kv.second.value);
+      }
+    }
     destination.set_content_encoding(metadata.content_encoding());
     destination.set_content_disposition(metadata.content_disposition());
     destination.set_cache_control(metadata.cache_control());
@@ -441,20 +500,8 @@ StatusOr<google::storage::v2::UpdateObjectRequest> ToProto(
     result.mutable_update_mask()->add_paths(field.grpc_name);
   }
 
-  auto const& subpatch =
-      storage::internal::PatchBuilderDetails::GetMetadataSubPatch(
-          request.patch());
-  if (subpatch.is_null()) {
-    object.clear_metadata();
-    result.mutable_update_mask()->add_paths("metadata");
-  } else {
-    for (auto const& kv : subpatch.items()) {
-      result.mutable_update_mask()->add_paths("metadata." + kv.key());
-      auto const& v = kv.value();
-      if (!v.is_string()) continue;
-      (*object.mutable_metadata())[kv.key()] = v.get<std::string>();
-    }
-  }
+  PatchGrpcMetadata(request.patch(), result, object);
+  PatchGrpcContexts(request.patch(), result, object);
 
   // We need to check each modifiable field.
   struct StringField {
@@ -510,6 +557,19 @@ StatusOr<google::storage::v2::UpdateObjectRequest> ToProto(
     (*object.mutable_metadata())[kv.first] = kv.second;
   }
 
+  if (request.metadata().has_contexts()) {
+    result.mutable_update_mask()->add_paths("contexts");
+    auto& custom_map = *object.mutable_contexts()->mutable_custom();
+
+    for (auto const& kv : request.metadata().contexts().custom()) {
+      google::storage::v2::ObjectCustomContextPayload& payload_ref =
+          custom_map[kv.first];
+      // In request, the create_time and update_time are ignored by
+      // the server, hence there is no need to parse them.
+      payload_ref.set_value(kv.second.value);
+    }
+  }
+
   if (request.metadata().has_custom_time()) {
     result.mutable_update_mask()->add_paths("custom_time");
     *object.mutable_custom_time() = google::cloud::internal::ToProtoTimestamp(

google/cloud/storage/internal/grpc/object_request_parser_test.cc

@@ -88,6 +88,12 @@ google::storage::v2::Object ExpectedFullObjectMetadata() {
     temporary_hold: true
     metadata: { key: "test-metadata-key1" value: "test-value1" }
     metadata: { key: "test-metadata-key2" value: "test-value2" }
+    contexts: {
+      custom: {
+        key: "custom-key-1"
+        value: { value: "custom-value-1" }
+      }
+    }
     event_based_hold: true
     custom_time { seconds: 1643126687 nanos: 123000000 }
   )pb";
@@ -113,6 +119,8 @@ storage::ObjectMetadata FullObjectMetadata() {
       .set_temporary_hold(true)
       .upsert_metadata("test-metadata-key1", "test-value1")
       .upsert_metadata("test-metadata-key2", "test-value2")
+      .set_contexts(storage::ObjectContexts().upsert(
+          "custom-key-1", {"custom-value-1", {}, {}}))
       .set_event_based_hold(true)
       .set_custom_time(std::chrono::system_clock::time_point{} +
                        std::chrono::seconds(1643126687) +
@@ -479,6 +487,7 @@ TEST(GrpcObjectRequestParser, PatchObjectRequestAllOptions) {
           .SetContentType("test-content-type")
           .SetMetadata("test-metadata-key1", "test-value1")
           .SetMetadata("test-metadata-key2", "test-value2")
+          .SetContext("custom-key-1", "custom-value-1")
           .SetTemporaryHold(true)
           .SetAcl({
               storage::ObjectAccessControl{}
@@ -505,12 +514,13 @@ TEST(GrpcObjectRequestParser, PatchObjectRequestAllOptions) {
   ASSERT_STATUS_OK(actual);
   // First check the paths. We do not care about their order, so checking them
   // with IsProtoEqual does not work.
-  EXPECT_THAT(actual->update_mask().paths(),
-              UnorderedElementsAre(
-                  "acl", "content_encoding", "content_disposition",
-                  "cache_control", "content_language", "content_type",
-                  "metadata.test-metadata-key1", "metadata.test-metadata-key2",
-                  "temporary_hold", "event_based_hold", "custom_time"));
+  EXPECT_THAT(
+      actual->update_mask().paths(),
+      UnorderedElementsAre(
+          "acl", "content_encoding", "content_disposition", "cache_control",
+          "content_language", "content_type", "metadata.test-metadata-key1",
+          "metadata.test-metadata-key2", "temporary_hold", "event_based_hold",
+          "custom_time", "contexts.custom.custom-key-1"));
   // Clear the paths, which we already compared, and compare the proto.
   actual->mutable_update_mask()->clear_paths();
   EXPECT_THAT(*actual, IsProtoEqual(expected));
@@ -535,6 +545,7 @@ TEST(GrpcObjectRequestParser, PatchObjectRequestAllResets) {
           .ResetContentType()
           .ResetEventBasedHold()
           .ResetMetadata()
+          .ResetContexts()
           .ResetTemporaryHold()
           .ResetCustomTime());
 
@@ -547,7 +558,7 @@ TEST(GrpcObjectRequestParser, PatchObjectRequestAllResets) {
       UnorderedElementsAre("acl", "content_encoding", "content_disposition",
                            "cache_control", "content_language", "content_type",
                            "metadata", "temporary_hold", "event_based_hold",
-                           "custom_time"));
+                           "custom_time", "contexts.custom"));
   // Clear the paths, which we already compared, and compare the proto.
   actual->mutable_update_mask()->clear_paths();
   EXPECT_THAT(*actual, IsProtoEqual(expected));
@@ -604,6 +615,64 @@ TEST(GrpcObjectRequestParser, PatchObjectRequestResetMetadata) {
   EXPECT_THAT(*actual, IsProtoEqual(expected));
 }
 
+TEST(GrpcObjectRequestParser, PatchObjectRequestContexts) {
+  auto constexpr kTextProto = R"pb(
+    object {
+      bucket: "projects/_/buckets/bucket-name"
+      name: "object-name"
+      contexts: {
+        custom: {
+          key: "custom-key-1"
+          value: { value: "custom-value-1" }
+        }
+      }
+    }
+    update_mask {}
+  )pb";
+  google::storage::v2::UpdateObjectRequest expected;
+  ASSERT_TRUE(TextFormat::ParseFromString(kTextProto, &expected));
+
+  storage::internal::PatchObjectRequest req(
+      "bucket-name", "object-name",
+      storage::ObjectMetadataPatchBuilder{}
+          .SetContext("custom-key-1", "custom-value-1")
+          .ResetContext("custom-key-2"));
+
+  auto actual = ToProto(req);
+  ASSERT_STATUS_OK(actual);
+  // First check the paths. We do not care about their order, so checking them
+  // with IsProtoEqual does not work.
+  EXPECT_THAT(actual->update_mask().paths(),
+              UnorderedElementsAre("contexts.custom.custom-key-1",
+                                   "contexts.custom.custom-key-2"));
+  // Clear the paths, which we already compared, and compare the proto.
+  actual->mutable_update_mask()->clear_paths();
+  EXPECT_THAT(*actual, IsProtoEqual(expected));
+}
+
+TEST(GrpcObjectRequestParser, PatchObjectRequestResetContexts) {
+  auto constexpr kTextProto = R"pb(
+    object { bucket: "projects/_/buckets/bucket-name" name: "object-name" }
+    update_mask {}
+  )pb";
+  google::storage::v2::UpdateObjectRequest expected;
+  ASSERT_TRUE(TextFormat::ParseFromString(kTextProto, &expected));
+
+  storage::internal::PatchObjectRequest req(
+      "bucket-name", "object-name",
+      storage::ObjectMetadataPatchBuilder{}.ResetContexts());
+
+  auto actual = ToProto(req);
+  ASSERT_STATUS_OK(actual);
+  // First check the paths. We do not care about their order, so checking them
+  // with IsProtoEqual does not work.
+  EXPECT_THAT(actual->update_mask().paths(),
+              UnorderedElementsAre("contexts.custom"));
+  // Clear the paths, which we already compared, and compare the proto.
+  actual->mutable_update_mask()->clear_paths();
+  EXPECT_THAT(*actual, IsProtoEqual(expected));
+}
+
 TEST(GrpcObjectRequestParser, UpdateObjectRequestAllOptions) {
   auto constexpr kTextProto = R"pb(
     predefined_acl: "projectPrivate"
@@ -643,7 +712,7 @@ TEST(GrpcObjectRequestParser, UpdateObjectRequestAllOptions) {
       UnorderedElementsAre("acl", "content_encoding", "content_disposition",
                            "cache_control", "content_language", "content_type",
                            "metadata", "temporary_hold", "event_based_hold",
-                           "custom_time"));
+                           "custom_time", "contexts"));
   // Clear the paths, which we already compared, and test the rest
   actual->mutable_update_mask()->clear_paths();
   EXPECT_THAT(*actual, IsProtoEqual(expected));