diff --git a/ci/cloudbuild/dockerfiles/demo-opensuse-leap.Dockerfile b/ci/cloudbuild/dockerfiles/demo-opensuse-leap.Dockerfile index 310ef24b41aad..6d258d8fa7fd0 100644 --- a/ci/cloudbuild/dockerfiles/demo-opensuse-leap.Dockerfile +++ b/ci/cloudbuild/dockerfiles/demo-opensuse-leap.Dockerfile @@ -27,16 +27,15 @@ ARG NCPU=4 # ```bash RUN zypper refresh && \ zypper install --allow-downgrade -y automake cmake curl \ - gcc8 gcc8-c++ git gzip libtool make patch tar wget + gcc9 gcc9-c++ git gzip libtool make patch tar wget # ``` # Install some of the dependencies for `google-cloud-cpp`. # ```bash RUN zypper refresh && \ - zypper install --allow-downgrade -y abseil-cpp-devel c-ares-devel \ - libcurl-devel libopenssl-devel nlohmann_json-devel \ - grpc-devel libprotobuf-devel + zypper install --allow-downgrade -y \ + libcurl-devel libopenssl-devel nlohmann_json-devel # ``` # The following steps will install libraries and tools in `/usr/local`. openSUSE @@ -52,19 +51,52 @@ ENV PATH=/usr/local/bin:${PATH} # Use the following environment variables to configure the compiler used by # CMake. -ENV CC=gcc-8 -ENV CXX=g++-8 +ENV CC=gcc-9 +ENV CXX=g++-9 -# #### opentelemetry-cpp +# #### Abseil + +# ```bash +WORKDIR /var/tmp/build/abseil-cpp +RUN curl -fsSL https://github.com/abseil/abseil-cpp/archive/20250814.2.tar.gz | \ + tar -xzf - --strip-components=1 && \ + cmake \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_CXX_STANDARD=17 \ + -DABSL_BUILD_TESTING=OFF \ + -DBUILD_SHARED_LIBS=yes \ + -S . -B cmake-out && \ + cmake --build cmake-out -- -j ${NCPU:-4} && \ + cmake --build cmake-out --target install -- -j ${NCPU:-4} && \ + ldconfig +# ``` + +# #### Protobuf + +# ```bash +WORKDIR /var/tmp/build/protobuf +RUN curl -fsSL https://github.com/protocolbuffers/protobuf/archive/v33.1.tar.gz | \ + tar -xzf - --strip-components=1 && \ + cmake \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_CXX_STANDARD=17 \ + -DBUILD_SHARED_LIBS=yes \ + -Dprotobuf_BUILD_TESTS=OFF \ + -Dprotobuf_ABSL_PROVIDER=package \ + -S . -B cmake-out && \ + cmake --build cmake-out --target install -- -j ${NCPU:-4} && \ + ldconfig && \ + ln -s /usr/local/bin/protoc /usr/bin/protoc +# ``` -# The project has a dependency on the OpenTelemetry library. +# #### opentelemetry-cpp # ```bash WORKDIR /var/tmp/build/opentelemetry-cpp RUN curl -fsSL https://github.com/open-telemetry/opentelemetry-cpp/archive/v1.24.0.tar.gz | \ tar -xzf - --strip-components=1 && \ cmake \ - -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_BUILD_TYPE=Debug \ -DCMAKE_CXX_STANDARD=17 \ -DBUILD_SHARED_LIBS=yes \ -DWITH_EXAMPLES=OFF \ @@ -77,6 +109,58 @@ RUN curl -fsSL https://github.com/open-telemetry/opentelemetry-cpp/archive/v1.24 ldconfig # ``` +# #### c-ares + +# ```bash +WORKDIR /var/tmp/build/c-ares +RUN curl -fsSL https://github.com/c-ares/c-ares/archive/refs/tags/cares-1_17_1.tar.gz | \ + tar -xzf - --strip-components=1 && \ + cmake \ + -DCMAKE_BUILD_TYPE=Debug \ + -DBUILD_SHARED_LIBS=yes \ + -S . -B cmake-out && \ + cmake --build cmake-out --target install && \ + ldconfig +# ``` + +# #### RE2 + +# ```bash +WORKDIR /var/tmp/build/re2 +RUN curl -fsSL https://github.com/google/re2/archive/2025-07-22.tar.gz | \ + tar -xzf - --strip-components=1 && \ + cmake -DCMAKE_BUILD_TYPE=Debug \ + -DBUILD_SHARED_LIBS=ON \ + -DRE2_BUILD_TESTING=OFF \ + -S . -B cmake-out && \ + cmake --build cmake-out --target install && \ + ldconfig +# ``` + +# #### gRPC + +# ```bash +WORKDIR /var/tmp/build/grpc +RUN curl -fsSL https://github.com/grpc/grpc/archive/v1.71.1.tar.gz | \ + tar -xzf - --strip-components=1 && \ + cmake \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_CXX_STANDARD=17 \ + -DBUILD_SHARED_LIBS=yes \ + -DgRPC_INSTALL=ON \ + -DgRPC_BUILD_TESTS=OFF \ + -DgRPC_ABSL_PROVIDER=package \ + -DgRPC_CARES_PROVIDER=package \ + -DgRPC_PROTOBUF_PROVIDER=package \ + -DgRPC_RE2_PROVIDER=package \ + -DgRPC_SSL_PROVIDER=package \ + -DgRPC_ZLIB_PROVIDER=package \ + -DgRPC_OPENTELEMETRY_PROVIDER=package \ + -S . -B cmake-out && \ + cmake --build cmake-out --target install -- -j ${NCPU:-4} && \ + ldconfig +# ``` + ## [DONE packaging.md] RUN zypper refresh && \ @@ -91,5 +175,5 @@ RUN curl -fsSL https://github.com/mozilla/sccache/releases/download/v0.10.0/scca # Update the ld.conf cache in case any libraries were installed in /usr/local/lib* RUN ldconfig /usr/local/lib* - ENV DEMO_CORD_WORKAROUND=OFF +RUN echo 'root:cloudcxx' | chpasswd diff --git a/doc/packaging.md b/doc/packaging.md index c189baf867784..394bd85bb7345 100644 --- a/doc/packaging.md +++ b/doc/packaging.md @@ -332,16 +332,15 @@ use GCC 8 or higher to compile `google-cloud-cpp`. ```bash sudo zypper refresh && \ sudo zypper install --allow-downgrade -y automake cmake curl \ - gcc8 gcc8-c++ git gzip libtool make patch tar wget + gcc9 gcc9-c++ git gzip libtool make patch tar wget ``` Install some of the dependencies for `google-cloud-cpp`. ```bash sudo zypper refresh && \ -sudo zypper install --allow-downgrade -y abseil-cpp-devel c-ares-devel \ - libcurl-devel libopenssl-devel nlohmann_json-devel \ - grpc-devel libprotobuf-devel +sudo zypper install --allow-downgrade -y \ + libcurl-devel libopenssl-devel nlohmann_json-devel ``` The following steps will install libraries and tools in `/usr/local`. openSUSE @@ -356,18 +355,51 @@ export PATH=/usr/local/bin:${PATH} ``` Use the following environment variables to configure the compiler used by CMake. -export CC=gcc-8 export CXX=g++-8 +export CC=gcc-9 export CXX=g++-9 -#### opentelemetry-cpp +#### Abseil -The project has a dependency on the OpenTelemetry library. +```bash +mkdir -p $HOME/Downloads/abseil-cpp && cd $HOME/Downloads/abseil-cpp +curl -fsSL https://github.com/abseil/abseil-cpp/archive/20250814.2.tar.gz | \ + tar -xzf - --strip-components=1 && \ + cmake \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_CXX_STANDARD=17 \ + -DABSL_BUILD_TESTING=OFF \ + -DBUILD_SHARED_LIBS=yes \ + -S . -B cmake-out && \ + cmake --build cmake-out -- -j ${NCPU:-4} && \ +sudo cmake --build cmake-out --target install -- -j ${NCPU:-4} && \ +sudo ldconfig +``` + +#### Protobuf + +```bash +mkdir -p $HOME/Downloads/protobuf && cd $HOME/Downloads/protobuf +curl -fsSL https://github.com/protocolbuffers/protobuf/archive/v33.1.tar.gz | \ + tar -xzf - --strip-components=1 && \ + cmake \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_CXX_STANDARD=17 \ + -DBUILD_SHARED_LIBS=yes \ + -Dprotobuf_BUILD_TESTS=OFF \ + -Dprotobuf_ABSL_PROVIDER=package \ + -S . -B cmake-out && \ +sudo cmake --build cmake-out --target install -- -j ${NCPU:-4} && \ +sudo ldconfig && \ + ln -s /usr/local/bin/protoc /usr/bin/protoc +``` + +#### opentelemetry-cpp ```bash mkdir -p $HOME/Downloads/opentelemetry-cpp && cd $HOME/Downloads/opentelemetry-cpp curl -fsSL https://github.com/open-telemetry/opentelemetry-cpp/archive/v1.24.0.tar.gz | \ tar -xzf - --strip-components=1 && \ cmake \ - -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_BUILD_TYPE=Debug \ -DCMAKE_CXX_STANDARD=17 \ -DBUILD_SHARED_LIBS=yes \ -DWITH_EXAMPLES=OFF \ @@ -380,6 +412,58 @@ sudo cmake --build cmake-out --target install -- -j ${NCPU:-4} && \ sudo ldconfig ``` +#### c-ares + +```bash +mkdir -p $HOME/Downloads/c-ares && cd $HOME/Downloads/c-ares +curl -fsSL https://github.com/c-ares/c-ares/archive/refs/tags/cares-1_17_1.tar.gz | \ + tar -xzf - --strip-components=1 && \ + cmake \ + -DCMAKE_BUILD_TYPE=Debug \ + -DBUILD_SHARED_LIBS=yes \ + -S . -B cmake-out && \ +sudo cmake --build cmake-out --target install && \ +sudo ldconfig +``` + +#### RE2 + +```bash +mkdir -p $HOME/Downloads/re2 && cd $HOME/Downloads/re2 +curl -fsSL https://github.com/google/re2/archive/2025-07-22.tar.gz | \ + tar -xzf - --strip-components=1 && \ + cmake -DCMAKE_BUILD_TYPE=Debug \ + -DBUILD_SHARED_LIBS=ON \ + -DRE2_BUILD_TESTING=OFF \ + -S . -B cmake-out && \ +sudo cmake --build cmake-out --target install && \ +sudo ldconfig +``` + +#### gRPC + +```bash +mkdir -p $HOME/Downloads/grpc && cd $HOME/Downloads/grpc +curl -fsSL https://github.com/grpc/grpc/archive/v1.71.1.tar.gz | \ + tar -xzf - --strip-components=1 && \ + cmake \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_CXX_STANDARD=17 \ + -DBUILD_SHARED_LIBS=yes \ + -DgRPC_INSTALL=ON \ + -DgRPC_BUILD_TESTS=OFF \ + -DgRPC_ABSL_PROVIDER=package \ + -DgRPC_CARES_PROVIDER=package \ + -DgRPC_PROTOBUF_PROVIDER=package \ + -DgRPC_RE2_PROVIDER=package \ + -DgRPC_SSL_PROVIDER=package \ + -DgRPC_ZLIB_PROVIDER=package \ + -DgRPC_OPENTELEMETRY_PROVIDER=package \ + -S . -B cmake-out && \ +sudo cmake --build cmake-out --target install -- -j ${NCPU:-4} && \ +sudo ldconfig +``` + #### Compile and install the main project We can now compile and install `google-cloud-cpp`: diff --git a/google/cloud/bigtable/CMakeLists.txt b/google/cloud/bigtable/CMakeLists.txt index 4449a71d74515..81fcec2454f85 100644 --- a/google/cloud/bigtable/CMakeLists.txt +++ b/google/cloud/bigtable/CMakeLists.txt @@ -214,6 +214,8 @@ add_library( internal/row_reader_impl.h internal/rpc_policy_parameters.h internal/rpc_policy_parameters.inc + internal/stub_manager.cc + internal/stub_manager.h internal/traced_row_reader.cc internal/traced_row_reader.h internal/tuple_utils.h @@ -461,6 +463,7 @@ if (BUILD_TESTING) internal/query_plan_test.cc internal/rate_limiter_test.cc internal/retry_traits_test.cc + internal/stub_manager_test.cc internal/traced_row_reader_test.cc internal/tuple_utils_test.cc mocks/mock_row_reader_test.cc diff --git a/google/cloud/bigtable/bigtable_client_unit_tests.bzl b/google/cloud/bigtable/bigtable_client_unit_tests.bzl index 6c95291b749a2..faf9fa9fb9441 100644 --- a/google/cloud/bigtable/bigtable_client_unit_tests.bzl +++ b/google/cloud/bigtable/bigtable_client_unit_tests.bzl @@ -63,6 +63,7 @@ bigtable_client_unit_tests = [ "internal/query_plan_test.cc", "internal/rate_limiter_test.cc", "internal/retry_traits_test.cc", + "internal/stub_manager_test.cc", "internal/traced_row_reader_test.cc", "internal/tuple_utils_test.cc", "mocks/mock_row_reader_test.cc", diff --git a/google/cloud/bigtable/google_cloud_cpp_bigtable.bzl b/google/cloud/bigtable/google_cloud_cpp_bigtable.bzl index 2ecc66c6fd37e..28109e02224c5 100644 --- a/google/cloud/bigtable/google_cloud_cpp_bigtable.bzl +++ b/google/cloud/bigtable/google_cloud_cpp_bigtable.bzl @@ -107,6 +107,7 @@ google_cloud_cpp_bigtable_hdrs = [ "internal/row_reader_impl.h", "internal/rpc_policy_parameters.h", "internal/rpc_policy_parameters.inc", + "internal/stub_manager.h", "internal/traced_row_reader.h", "internal/tuple_utils.h", "internal/unary_client_utils.h", @@ -213,6 +214,7 @@ google_cloud_cpp_bigtable_srcs = [ "internal/rate_limiter.cc", "internal/readrowsparser.cc", "internal/retry_traits.cc", + "internal/stub_manager.cc", "internal/traced_row_reader.cc", "mutation_batcher.cc", "mutations.cc", diff --git a/google/cloud/bigtable/internal/stub_manager.cc b/google/cloud/bigtable/internal/stub_manager.cc new file mode 100644 index 0000000000000..9cfa7832d4652 --- /dev/null +++ b/google/cloud/bigtable/internal/stub_manager.cc @@ -0,0 +1,51 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "google/cloud/bigtable/internal/stub_manager.h" +#include + +namespace google { +namespace cloud { +namespace bigtable_internal { +GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_BEGIN + +StubManager::StubManager(std::shared_ptr stub) + : stub_(std::move(stub)) {} + +StubManager::StubManager( + absl::flat_hash_map> + affinity_stubs, + StubCreationFn stub_creation_fn) + : stub_creation_fn_(std::move(stub_creation_fn)), + affinity_stubs_(std::move(affinity_stubs)) {} + +std::shared_ptr StubManager::GetStub( + std::string_view instance_name) { + if (stub_) return stub_; + + std::scoped_lock lk(mu_); + if (auto iter = affinity_stubs_.find(instance_name); + iter != affinity_stubs_.end()) { + return iter->second; + } + auto inserted = affinity_stubs_.emplace( + std::string{instance_name}, + stub_creation_fn_(instance_name, Priming::kNoPriming)); + return inserted.first->second; +} + +GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END +} // namespace bigtable_internal +} // namespace cloud +} // namespace google diff --git a/google/cloud/bigtable/internal/stub_manager.h b/google/cloud/bigtable/internal/stub_manager.h new file mode 100644 index 0000000000000..9a5b54c0b078f --- /dev/null +++ b/google/cloud/bigtable/internal/stub_manager.h @@ -0,0 +1,66 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_BIGTABLE_INTERNAL_STUB_MANAGER_H +#define GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_BIGTABLE_INTERNAL_STUB_MANAGER_H + +#include "google/cloud/bigtable/internal/bigtable_stub.h" +#include "google/cloud/version.h" +#include "absl/container/flat_hash_map.h" +#include +#include +#include +#include +#include + +namespace google { +namespace cloud { +namespace bigtable_internal { +GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_BEGIN + +// This class provides support for operating with a single BigtableStub which is +// not bound to any specific Bigtable instance, or with a collection of +// BigtableStubs of which each BigtableStub is bound to a specific instance. +class StubManager { + public: + enum class Priming { kNoPriming, kSynchronousPriming }; + + // This function must not send any RPCs as it is called while a lock is held. + // This means channels cannot be primed as part of construction, but instead + // must be scheduled to be primed asynchronously. + using StubCreationFn = std::function( + std::string_view instance_name, Priming priming)>; + + explicit StubManager(std::shared_ptr stub); + + StubManager(absl::flat_hash_map> + affinity_stubs, + StubCreationFn stub_creation_fn); + + std::shared_ptr GetStub(std::string_view instance_name); + + private: + std::mutex mu_; + StubCreationFn stub_creation_fn_; + std::shared_ptr stub_; + absl::flat_hash_map> + affinity_stubs_; +}; + +GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END +} // namespace bigtable_internal +} // namespace cloud +} // namespace google + +#endif // GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_BIGTABLE_INTERNAL_STUB_MANAGER_H diff --git a/google/cloud/bigtable/internal/stub_manager_test.cc b/google/cloud/bigtable/internal/stub_manager_test.cc new file mode 100644 index 0000000000000..97fb6562b76ef --- /dev/null +++ b/google/cloud/bigtable/internal/stub_manager_test.cc @@ -0,0 +1,128 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "google/cloud/bigtable/internal/stub_manager.h" +#include "google/cloud/bigtable/instance_resource.h" +#include "google/cloud/bigtable/table_resource.h" +#include "google/cloud/bigtable/testing/mock_bigtable_stub.h" +#include "google/cloud/testing_util/status_matchers.h" +#include + +namespace google { +namespace cloud { +namespace bigtable_internal { +GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_BEGIN +namespace { + +using ::google::cloud::bigtable::testing::MockBigtableStub; +using ::google::cloud::testing_util::IsOk; +using ::testing::Eq; +using ::testing::MockFunction; +using ::testing::StartsWith; + +TEST(StubManagerTest, NoAffinity) { + bigtable::InstanceResource instance(Project("my-project"), "my-instance"); + bigtable::TableResource table(instance, "my-table"); + std::string const expected_table_name = table.FullName(); + auto mock = std::make_shared(); + EXPECT_CALL(*mock, MutateRow) + .WillOnce([&](grpc::ClientContext&, Options const&, + google::bigtable::v2::MutateRowRequest const& request) { + EXPECT_THAT(request.table_name(), Eq(expected_table_name)); + return google::bigtable::v2::MutateRowResponse{}; + }); + + StubManager manager(mock); + + auto stub = manager.GetStub({}); + grpc::ClientContext context; + google::bigtable::v2::MutateRowRequest request; + request.set_table_name(expected_table_name); + auto result = stub->MutateRow(context, {}, request); + EXPECT_THAT(result, IsOk()); +} + +TEST(StubManagerTest, AffinityToExistingInstance) { + bigtable::InstanceResource instance(Project("my-project"), "a"); + bigtable::TableResource table(instance, "my-table"); + std::string const expected_table_name = table.FullName(); + + auto mock_stub = std::make_shared(); + EXPECT_CALL(*mock_stub, MutateRow) + .WillOnce([instance_name = instance.FullName()]( + grpc::ClientContext&, Options const&, + google::bigtable::v2::MutateRowRequest const& request) { + EXPECT_THAT(request.table_name(), StartsWith(instance_name)); + return google::bigtable::v2::MutateRowResponse{}; + }); + + absl::flat_hash_map> + affinity_stubs; + affinity_stubs[instance.FullName()] = mock_stub; + + MockFunction(std::string_view, + StubManager::Priming)> + stub_creation_fn; + EXPECT_CALL(stub_creation_fn, Call).Times(0); + + StubManager manager(affinity_stubs, stub_creation_fn.AsStdFunction()); + + auto stub = manager.GetStub(instance.FullName()); + grpc::ClientContext context; + google::bigtable::v2::MutateRowRequest request; + request.set_table_name(expected_table_name); + auto result = stub->MutateRow(context, {}, request); + EXPECT_THAT(result, IsOk()); +} + +TEST(StubManagerTest, AffinityToMissingInstance) { + bigtable::InstanceResource instance_a(Project("my-project"), "a"); + bigtable::TableResource table_a(instance_a, "my-table"); + auto mock_stub_a = std::make_shared(); + EXPECT_CALL(*mock_stub_a, MutateRow).Times(0); + absl::flat_hash_map> + affinity_stubs; + affinity_stubs[instance_a.FullName()] = mock_stub_a; + + auto stub_creation_fn = [](std::string_view instance_name, + StubManager::Priming) { + auto mock_stub = std::make_shared(); + EXPECT_CALL(*mock_stub, MutateRow) + .WillOnce([instance_name = std::string{instance_name}]( + grpc::ClientContext&, Options const&, + google::bigtable::v2::MutateRowRequest const& request) { + EXPECT_THAT(request.table_name(), StartsWith(instance_name)); + return google::bigtable::v2::MutateRowResponse{}; + }); + return mock_stub; + }; + + StubManager manager(affinity_stubs, stub_creation_fn); + + bigtable::InstanceResource instance_b(Project("my-project"), "b"); + bigtable::TableResource table_b(instance_b, "my-table"); + std::string const expected_table_name = table_b.FullName(); + auto stub = manager.GetStub(instance_b.FullName()); + grpc::ClientContext context; + google::bigtable::v2::MutateRowRequest request; + request.set_table_name(expected_table_name); + auto result = stub->MutateRow(context, {}, request); + EXPECT_THAT(result, IsOk()); +} + +} // namespace +GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END +} // namespace bigtable_internal +} // namespace cloud +} // namespace google