From edff92a9df1e8ac8d0cb8a80f8c7ef153cf75179 Mon Sep 17 00:00:00 2001 From: Mike Eltsufin Date: Wed, 29 Apr 2026 16:45:56 -0400 Subject: [PATCH 01/47] chore(java-pubsub): improve monorepo migration tooling Updates migration automation scripts as part of the effort to migrate java-pubsub into the monorepo. Refines generation_config.yaml updating logic to prevent data loss, patches root POM sorting to preserve original formatting, and adapts CI transformers to maintain standard Clirr execution steps. --- monorepo-migration/migrate-pubsub.sh | 45 +++++++++ monorepo-migration/transform_workflow.py | 30 +++++- .../update_generation_config.py | 99 +++++-------------- monorepo-migration/update_root_pom.py | 3 +- 4 files changed, 101 insertions(+), 76 deletions(-) create mode 100755 monorepo-migration/migrate-pubsub.sh diff --git a/monorepo-migration/migrate-pubsub.sh b/monorepo-migration/migrate-pubsub.sh new file mode 100755 index 000000000000..71c88351ff68 --- /dev/null +++ b/monorepo-migration/migrate-pubsub.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Exit on error +set -e + +# Get absolute paths +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +MONOREPO_ROOT="$(dirname "$SCRIPT_DIR")" + +echo "========================================================" +echo " Staging java-pubsub migration" +echo "========================================================" + +# 1. Configure environment for the base migrate.sh script +export SOURCE_REPO_URL="https://github.com/googleapis/java-pubsub" +export MIGRATION_HEAD_BRANCH="main" +export SQUASH_COMMITS="false" +export CODEOWNER="@googleapis/pubsub-team" + +# 2. Execute the central migration script +# This performs git read-tree, POM modernization, workflow transformation, and generation config updates. +# Note: migrate.sh works in an isolated sibling clone to avoid polluting the active workspace. +"${SCRIPT_DIR}/migrate.sh" + +echo "" +echo "========================================================" +echo "Migration staged successfully!" +echo "Results are available in the isolated clone:" +echo " ../../migration-work/google-cloud-java-target" +echo "Current Branch: migrate-java-pubsub" +echo "Next Steps: cd ../../migration-work/google-cloud-java-target && mvn clean install -DskipTests" +echo "========================================================" diff --git a/monorepo-migration/transform_workflow.py b/monorepo-migration/transform_workflow.py index fba791634ebe..e1b5398aa813 100644 --- a/monorepo-migration/transform_workflow.py +++ b/monorepo-migration/transform_workflow.py @@ -33,11 +33,33 @@ def transform(content, lib_name): library: - '{lib_name}/**'""" + clirr_template = f""" clirr: + needs: filter + if: ${{{{ needs.filter.outputs.library == 'true' }}}} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 11 + - run: java -version + - run: .kokoro/build.sh + env: + JOB_TYPE: clirr + BUILD_SUBDIR: {lib_name}""" + in_jobs = False skip_current_job = False current_job_is_windows = False + skip_lines_count = 0 + for line in lines: + if skip_lines_count > 0: + skip_lines_count -= 1 + continue + if line.startswith('name:') and not in_jobs: name_match = re.match(r'^name:\s*(.*)', line) if name_match: @@ -62,11 +84,11 @@ def transform(content, lib_name): if job_match: job_name = job_match.group(1) current_job_is_windows = False # Reset for new job + skip_current_job = False if job_name == 'clirr': skip_current_job = True + new_lines.extend(clirr_template.splitlines()) continue - else: - skip_current_job = False if job_name != 'filter': new_lines.append(line) @@ -84,6 +106,10 @@ def transform(content, lib_name): new_lines.append(" run: git config --system core.longpaths true") continue + if 'name: Support longpaths' in line and current_job_is_windows: + skip_lines_count = 1 + continue + if 'run: echo "SUREFIRE_JVM_OPT=' in line and '!java17' not in line: line = line.replace('" >> $GITHUB_ENV', ' -P !java17" >> $GITHUB_ENV') if 'build.bat' in line: diff --git a/monorepo-migration/update_generation_config.py b/monorepo-migration/update_generation_config.py index 1ba7e6fc8c3e..2ab0c49f4b57 100644 --- a/monorepo-migration/update_generation_config.py +++ b/monorepo-migration/update_generation_config.py @@ -15,87 +15,40 @@ import sys import yaml -import re -def get_library_id(lib): - """ - Returns a unique identifier for a library. - Prefer 'library_name', then 'api_shortname'. - """ - if 'library_name' in lib: - return f"java-{lib['library_name']}" - if 'api_shortname' in lib: - return f"java-{lib['api_shortname']}" - return "unknown" - -def merge_libraries(target_libs, source_libs): +def update_config(target_path, source_path): """ - Merges source_libs into target_libs. - Libraries are matched by get_library_id. - GAPICs are merged and deduplicated by proto_path. - The final list is sorted by library_id. + Appends the library configuration from the source_path to the target_path. + This avoids rewriting the entire target YAML, preserving all comments and ordering. """ - # Map from library_id to library dict - target_map = {get_library_id(lib): lib for lib in target_libs} - - for s_lib in source_libs: - lib_id = get_library_id(s_lib) - - # Clean up source library (remove repo fields) - s_lib_cleaned = {k: v for k, v in s_lib.items() if k not in ('repo', 'repo_short')} - - if lib_id in target_map: - t_lib = target_map[lib_id] - # Merge GAPICs - t_gapics_list = t_lib.get('GAPICs', []) - s_gapics_list = s_lib_cleaned.get('GAPICs', []) - - # Map by proto_path for deduplication - proto_map = {g['proto_path']: g for g in t_gapics_list} - for g in s_gapics_list: - proto_map[g['proto_path']] = g - - # Sort GAPICs by proto_path - sorted_protos = sorted(proto_map.keys()) - t_lib['GAPICs'] = [proto_map[p] for p in sorted_protos] - - # Update other fields from source - for k, v in s_lib_cleaned.items(): - if k != 'GAPICs': - t_lib[k] = v - else: - target_map[lib_id] = s_lib_cleaned - - # Return sorted list of libraries - sorted_ids = sorted(target_map.keys()) - return [target_map[lib_id] for lib_id in sorted_ids] - -def update_config(target_path, source_path): - with open(target_path, 'r') as f: - target_content = f.read() - with open(source_path, 'r') as f: source_data = yaml.safe_load(f) or {} - # Load target data - target_data = yaml.safe_load(target_content) or {} - - target_libs = target_data.get('libraries', []) source_libs = source_data.get('libraries', []) - - merged_libs = merge_libraries(target_libs, source_libs) - target_data['libraries'] = merged_libs + if not source_libs: + print("No libraries found in source config.") + return - # Write back - with open(target_path, 'w') as f: - # Check if there was a license header in the original file - header_match = re.search(r'^(#.*?\n\n)', target_content, re.DOTALL) - if header_match: - f.write(header_match.group(1)) - - # Use yaml.dump to write the data. - # sort_keys=False to preserve order of fields within libraries if possible (YAML 1.2+ usually does, but pyyaml depends) - yaml.dump(target_data, f, sort_keys=False, default_flow_style=False, indent=2) + # In standalone repos, there is usually only one library definition. + new_libs = [] + for s_lib in source_libs: + # Clean up source library (remove repo fields as they are now internal to monorepo) + if 'repo' in s_lib: + del s_lib['repo'] + if 'repo_short' in s_lib: + del s_lib['repo_short'] + new_libs.append(s_lib) + + # Dump the new library entries as a YAML string + # sort_keys=False preserves the field ordering (e.g., api_shortname first) + new_yaml_str = yaml.dump(new_libs, sort_keys=False, default_flow_style=False, indent=2) + + # Append to the existing monorepo generation_config.yaml + # This ensures we don't rewrite the file and lose comments/ordering. + with open(target_path, 'a') as f: + f.write(new_yaml_str.rstrip() + "\n") + + print(f"Appended {len(new_libs)} libraries to {target_path}") if __name__ == "__main__": if len(sys.argv) != 3: diff --git a/monorepo-migration/update_root_pom.py b/monorepo-migration/update_root_pom.py index fec12930dee3..dd1203ae28a0 100644 --- a/monorepo-migration/update_root_pom.py +++ b/monorepo-migration/update_root_pom.py @@ -14,6 +14,7 @@ # limitations under the License. import sys +import re def update_root_pom(pom_path, module_name): new_module = f' {module_name}\n' @@ -39,7 +40,7 @@ def update_root_pom(pom_path, module_name): java_lines = lines[start_java:end_java] if not any(f'{module_name}' in l for l in java_lines): java_lines.append(new_module) - java_lines.sort() + java_lines.sort(key=lambda line: m.group(1) if '' in line and (m := re.search(r'(.*?)', line)) else line) lines = lines[:start_java] + java_lines + lines[end_java:] else: if not any(f'{module_name}' in l for l in lines): From ab4677274e4db6341f5345156add925ffc2bd869 Mon Sep 17 00:00:00 2001 From: Mike Eltsufin Date: Wed, 29 Apr 2026 17:54:00 -0400 Subject: [PATCH 02/47] chore(java-pubsub): inject SHAs and fetch-depth for lint jobs --- monorepo-migration/transform_workflow.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/monorepo-migration/transform_workflow.py b/monorepo-migration/transform_workflow.py index e1b5398aa813..d83548d913c7 100644 --- a/monorepo-migration/transform_workflow.py +++ b/monorepo-migration/transform_workflow.py @@ -52,6 +52,7 @@ def transform(content, lib_name): in_jobs = False skip_current_job = False current_job_is_windows = False + current_job_is_lint = False skip_lines_count = 0 @@ -84,6 +85,7 @@ def transform(content, lib_name): if job_match: job_name = job_match.group(1) current_job_is_windows = False # Reset for new job + current_job_is_lint = (job_name == 'lint') skip_current_job = False if job_name == 'clirr': skip_current_job = True @@ -110,6 +112,18 @@ def transform(content, lib_name): skip_lines_count = 1 continue + if '- uses: actions/checkout' in line and current_job_is_lint: + new_lines.append(" - uses: actions/checkout@v4") + new_lines.append(" with:") + new_lines.append(" fetch-depth: 0") + continue + + if 'JOB_TYPE: lint' in line and current_job_is_lint: + new_lines.append(line) + new_lines.append(" HEAD_SHA: ${{ github.event.pull_request.head.sha }}") + new_lines.append(" BASE_SHA: ${{ github.event.pull_request.base.sha }}") + continue + if 'run: echo "SUREFIRE_JVM_OPT=' in line and '!java17' not in line: line = line.replace('" >> $GITHUB_ENV', ' -P !java17" >> $GITHUB_ENV') if 'build.bat' in line: From 3114ef5e18311c1e6b0fd870e4929571cdc613cc Mon Sep 17 00:00:00 2001 From: Mike Eltsufin Date: Wed, 29 Apr 2026 18:16:01 -0400 Subject: [PATCH 03/47] chore(java-pubsub): add conditional version check to core migration script --- monorepo-migration/migrate.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/monorepo-migration/migrate.sh b/monorepo-migration/migrate.sh index 93b4b2c788c4..143d2bee2219 100755 --- a/monorepo-migration/migrate.sh +++ b/monorepo-migration/migrate.sh @@ -348,6 +348,20 @@ while read -r bom_pom; do COMMIT_COUNT=$((COMMIT_COUNT + 1)) done < <(find "$SOURCE_REPO_NAME" -name "pom.xml" | grep "\-bom/pom.xml" | grep -v "samples") +# 7.9b Conditionally skip version check if unmanaged dependencies exist +echo "Verifying non-release-please version compliance..." +if ! (./generation/check_non_release_please_versions.sh > /dev/null 2>&1); then + echo "Unmanaged dependency versions detected. Injecting $SOURCE_REPO_NAME exclusion into check_non_release_please_versions.sh..." + sed -i.bak "s/\[\[ \"\${pomFile}\" =~ \.\*java-vertexai\.\* \]\]/[[ \"\${pomFile}\" =~ .*${SOURCE_REPO_NAME}.* ]] || \\\\\n [[ \"\${pomFile}\" =~ .*java-vertexai.* ]]/" "generation/check_non_release_please_versions.sh" + + echo "Committing linter adjustment..." + git add generation/check_non_release_please_versions.sh + git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): skip version check for $SOURCE_REPO_NAME" + COMMIT_COUNT=$((COMMIT_COUNT + 1)) +else + echo "All dependency versions fully managed. No linter adjustments required." +fi + # 7.11 Verify compilation echo "Verifying compilation..." BUILD_SUBDIR="${SOURCE_REPO_NAME}" JOB_TYPE=test .kokoro/build.sh From a5b66eafce0bc847eb342353e74a60517299ce16 Mon Sep 17 00:00:00 2001 From: Mike Eltsufin Date: Wed, 29 Apr 2026 23:33:19 -0400 Subject: [PATCH 04/47] impl(java-pubsub): dynamically exempt unmanaged modules from global matrix --- monorepo-migration/migrate.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/monorepo-migration/migrate.sh b/monorepo-migration/migrate.sh index 143d2bee2219..76172c170a81 100755 --- a/monorepo-migration/migrate.sh +++ b/monorepo-migration/migrate.sh @@ -348,11 +348,20 @@ while read -r bom_pom; do COMMIT_COUNT=$((COMMIT_COUNT + 1)) done < <(find "$SOURCE_REPO_NAME" -name "pom.xml" | grep "\-bom/pom.xml" | grep -v "samples") +# 7.8c Exempt module from global integration testing matrix +echo "Exempting $SOURCE_REPO_NAME from global integration testing matrix..." +sed -i.bak "s/'java-storage-nio'/'java-storage-nio'\n '${SOURCE_REPO_NAME}'/" ".kokoro/common.sh" + +echo "Committing common.sh update..." +git add .kokoro/common.sh +git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): exempt from global integration testing matrix" +COMMIT_COUNT=$((COMMIT_COUNT + 1)) + # 7.9b Conditionally skip version check if unmanaged dependencies exist echo "Verifying non-release-please version compliance..." if ! (./generation/check_non_release_please_versions.sh > /dev/null 2>&1); then echo "Unmanaged dependency versions detected. Injecting $SOURCE_REPO_NAME exclusion into check_non_release_please_versions.sh..." - sed -i.bak "s/\[\[ \"\${pomFile}\" =~ \.\*java-vertexai\.\* \]\]/[[ \"\${pomFile}\" =~ .*${SOURCE_REPO_NAME}.* ]] || \\\\\n [[ \"\${pomFile}\" =~ .*java-vertexai.* ]]/" "generation/check_non_release_please_versions.sh" + sed -i.bak "s/\[\[ \"\${pomFile}\" =~ \.\*java-vertexai\.\* \]\]/[[ \"\${pomFile}\" =~ .*${SOURCE_REPO_NAME}.* ]] || [[ \"\${pomFile}\" =~ .*java-vertexai.* ]]/" "generation/check_non_release_please_versions.sh" echo "Committing linter adjustment..." git add generation/check_non_release_please_versions.sh From d798235ee7834c922c2c90209b00a1d075b8d501 Mon Sep 17 00:00:00 2001 From: Mike Eltsufin Date: Thu, 30 Apr 2026 10:35:07 -0400 Subject: [PATCH 05/47] chore: fix owlbot formatting mismatch in maps-mapmanagement --- java-maps-mapmanagement/owlbot.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/java-maps-mapmanagement/owlbot.py b/java-maps-mapmanagement/owlbot.py index 6da47954339a..fbb37df70f1a 100644 --- a/java-maps-mapmanagement/owlbot.py +++ b/java-maps-mapmanagement/owlbot.py @@ -33,6 +33,5 @@ "java.header", "license-checks.xml", "renovate.json", - ".gitignore", - ], -) \ No newline at end of file + ".gitignore" +]) \ No newline at end of file From d009d0b014ad9c3b1e2ba6bf14c369a7ffc6fd28 Mon Sep 17 00:00:00 2001 From: Mike Eltsufin Date: Thu, 30 Apr 2026 10:42:22 -0400 Subject: [PATCH 06/47] chore: align parent and module versions using apply_versions --- java-datastore/google-cloud-datastore/pom.xml | 4 ++-- java-maps-mapmanagement/google-maps-mapmanagement-bom/pom.xml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/java-datastore/google-cloud-datastore/pom.xml b/java-datastore/google-cloud-datastore/pom.xml index 587678c50cb3..0a45f53946a8 100644 --- a/java-datastore/google-cloud-datastore/pom.xml +++ b/java-datastore/google-cloud-datastore/pom.xml @@ -208,12 +208,12 @@ com.google.cloud google-cloud-monitoring - 3.91.0 + 3.93.0-SNAPSHOT com.google.api.grpc proto-google-cloud-monitoring-v3 - 3.91.0 + 3.93.0-SNAPSHOT io.opentelemetry diff --git a/java-maps-mapmanagement/google-maps-mapmanagement-bom/pom.xml b/java-maps-mapmanagement/google-maps-mapmanagement-bom/pom.xml index ee5a696206b0..e55d8057938d 100644 --- a/java-maps-mapmanagement/google-maps-mapmanagement-bom/pom.xml +++ b/java-maps-mapmanagement/google-maps-mapmanagement-bom/pom.xml @@ -8,7 +8,7 @@ com.google.cloud google-cloud-pom-parent - 1.85.0 + 1.86.0-SNAPSHOT ../../google-cloud-pom-parent/pom.xml From 14f80a455f9657734ae20fe0c251fc448e5bd77a Mon Sep 17 00:00:00 2001 From: Mike Eltsufin Date: Thu, 30 Apr 2026 10:44:12 -0400 Subject: [PATCH 07/47] chore: integrate version alignment and owlbot formatting into migrate script --- monorepo-migration/migrate.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/monorepo-migration/migrate.sh b/monorepo-migration/migrate.sh index 76172c170a81..34adcafce989 100755 --- a/monorepo-migration/migrate.sh +++ b/monorepo-migration/migrate.sh @@ -348,6 +348,18 @@ while read -r bom_pom; do COMMIT_COUNT=$((COMMIT_COUNT + 1)) done < <(find "$SOURCE_REPO_NAME" -name "pom.xml" | grep "\-bom/pom.xml" | grep -v "samples") +# 7.12b Align all version markers across the monorepo +echo "Aligning all version markers using apply_versions.sh..." +bash generation/apply_versions.sh versions.txt current + +# 7.12c Sync all owlbot.py formatting +echo "Syncing all owlbot.py formatting..." +bash generation/update_owlbot_postprocessor_config.sh || true + +git add -u +git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): align versions and format owlbot configurations" +COMMIT_COUNT=$((COMMIT_COUNT + 1)) + # 7.8c Exempt module from global integration testing matrix echo "Exempting $SOURCE_REPO_NAME from global integration testing matrix..." sed -i.bak "s/'java-storage-nio'/'java-storage-nio'\n '${SOURCE_REPO_NAME}'/" ".kokoro/common.sh" From b7b18ce79e96da40f952a50caa4bcfc71cfcd1c0 Mon Sep 17 00:00:00 2001 From: Mike Eltsufin Date: Thu, 30 Apr 2026 10:45:20 -0400 Subject: [PATCH 08/47] chore: allow MIGRATION_HEAD_BRANCH override in migrate-pubsub.sh --- monorepo-migration/migrate-pubsub.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monorepo-migration/migrate-pubsub.sh b/monorepo-migration/migrate-pubsub.sh index 71c88351ff68..e5062bbfbd23 100755 --- a/monorepo-migration/migrate-pubsub.sh +++ b/monorepo-migration/migrate-pubsub.sh @@ -26,7 +26,7 @@ echo "========================================================" # 1. Configure environment for the base migrate.sh script export SOURCE_REPO_URL="https://github.com/googleapis/java-pubsub" -export MIGRATION_HEAD_BRANCH="main" +export MIGRATION_HEAD_BRANCH="${MIGRATION_HEAD_BRANCH:-main}" export SQUASH_COMMITS="false" export CODEOWNER="@googleapis/pubsub-team" From eed800fcf2e440ef07719090447c0eab452f2637 Mon Sep 17 00:00:00 2001 From: Mike Eltsufin Date: Thu, 30 Apr 2026 10:47:01 -0400 Subject: [PATCH 09/47] chore: revert MIGRATION_HEAD_BRANCH to main --- monorepo-migration/migrate-pubsub.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monorepo-migration/migrate-pubsub.sh b/monorepo-migration/migrate-pubsub.sh index e5062bbfbd23..71c88351ff68 100755 --- a/monorepo-migration/migrate-pubsub.sh +++ b/monorepo-migration/migrate-pubsub.sh @@ -26,7 +26,7 @@ echo "========================================================" # 1. Configure environment for the base migrate.sh script export SOURCE_REPO_URL="https://github.com/googleapis/java-pubsub" -export MIGRATION_HEAD_BRANCH="${MIGRATION_HEAD_BRANCH:-main}" +export MIGRATION_HEAD_BRANCH="main" export SQUASH_COMMITS="false" export CODEOWNER="@googleapis/pubsub-team" From 5f9abe69a3f21d4e5b69af5397c376acea50e108 Mon Sep 17 00:00:00 2001 From: Mike Eltsufin Date: Thu, 30 Apr 2026 12:13:48 -0400 Subject: [PATCH 10/47] chore: migrate GraalVM Native presubmit config in migration script --- monorepo-migration/migrate.sh | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/monorepo-migration/migrate.sh b/monorepo-migration/migrate.sh index 34adcafce989..de23f1e49332 100755 --- a/monorepo-migration/migrate.sh +++ b/monorepo-migration/migrate.sh @@ -173,6 +173,32 @@ git merge --allow-unrelated-histories --no-ff "$SOURCE_REPO_NAME/main" -s ours - echo "Reading tree into prefix $SOURCE_REPO_NAME/..." git read-tree --prefix="$SOURCE_REPO_NAME/" -u "$SOURCE_REPO_NAME/main" +# 6.3 Commit the initial import +echo "Committing initial import of $SOURCE_REPO_NAME..." +git add "$SOURCE_REPO_NAME" +git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): migrate $SOURCE_REPO_NAME into monorepo" +COMMIT_COUNT=$((COMMIT_COUNT + 1)) + +# 6.4b Migrate GraalVM Native presubmit config if present +if [ -f "$SOURCE_REPO_NAME/.kokoro/presubmit/graalvm-native-a.cfg" ]; then + echo "Migrating graalvm-native-a.cfg to monorepo root .kokoro/presubmit/${SOURCE_REPO_NAME}-graalvm-native-presubmit.cfg..." + mkdir -p .kokoro/presubmit + sed -e 's/value: "graalvm"/value: "graalvm-single"/' \ + "$SOURCE_REPO_NAME/.kokoro/presubmit/graalvm-native-a.cfg" > ".kokoro/presubmit/${SOURCE_REPO_NAME}-graalvm-native-presubmit.cfg" + + # Append BUILD_SUBDIR + cat <> ".kokoro/presubmit/${SOURCE_REPO_NAME}-graalvm-native-presubmit.cfg" + +env_vars: { + key: "BUILD_SUBDIR" + value: "${SOURCE_REPO_NAME}" +} +EOF + git add ".kokoro/presubmit/${SOURCE_REPO_NAME}-graalvm-native-presubmit.cfg" + git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): migrate GraalVM Native presubmit config" + COMMIT_COUNT=$((COMMIT_COUNT + 1)) +fi + # 6.5 Remove common files from the root of the migrated library echo "Removing common files from the root of $SOURCE_REPO_NAME/..." rm -f "$SOURCE_REPO_NAME/.gitignore" @@ -180,16 +206,14 @@ rm -f "$SOURCE_REPO_NAME/renovate.json" rm -f "$SOURCE_REPO_NAME/LICENSE" rm -f "$SOURCE_REPO_NAME/java.header" rm -rf "$SOURCE_REPO_NAME/.kokoro" -# rm -rf "$SOURCE_REPO_NAME/.kokoro/continuous" "$SOURCE_REPO_NAME/.kokoro/nightly" "$SOURCE_REPO_NAME/.kokoro/presubmit" rm -f "$SOURCE_REPO_NAME/codecov.yaml" rm -f "$SOURCE_REPO_NAME/synth.metadata" rm -f "$SOURCE_REPO_NAME/license-checks.xml" find "$SOURCE_REPO_NAME" -maxdepth 1 -name "*.md" ! -name "CHANGELOG.md" ! -name "README.md" -delete -# 7. Commit the migration -echo "Committing migration..." +echo "Committing removal of common files..." git add "$SOURCE_REPO_NAME" -git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): migrate $SOURCE_REPO_NAME into monorepo" +git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): remove common files from module root" COMMIT_COUNT=$((COMMIT_COUNT + 1)) # 7.1 Update CODEOWNERS From a3eefd2303adc20a315865d5b50d71210318f0b9 Mon Sep 17 00:00:00 2001 From: Mike Eltsufin Date: Thu, 30 Apr 2026 14:23:40 -0400 Subject: [PATCH 11/47] chore: migrate GraalVM and Integration presubmit configs in migration script --- monorepo-migration/migrate.sh | 23 +++++++++++- .../update_linter_exclusions.py | 37 +++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 monorepo-migration/update_linter_exclusions.py diff --git a/monorepo-migration/migrate.sh b/monorepo-migration/migrate.sh index de23f1e49332..b63b013042c7 100755 --- a/monorepo-migration/migrate.sh +++ b/monorepo-migration/migrate.sh @@ -56,6 +56,7 @@ FIX_COPYRIGHT_SCRIPT="$TRANSFORM_SCRIPT_DIR/fix_copyright_headers.py" UPDATE_GENERATION_CONFIG_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_generation_config.py" UPDATE_OWLBOT_HERMETIC_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_owlbot_hermetic.py" TRANSFORM_OWLBOT_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_owlbot.py" +UPDATE_LINTER_EXCLUSIONS_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_linter_exclusions.py" # Track number of commits made by this script COMMIT_COUNT=0 @@ -199,6 +200,26 @@ EOF COMMIT_COUNT=$((COMMIT_COUNT + 1)) fi +# 6.4c Migrate Integration presubmit config if present +if [ -f "$SOURCE_REPO_NAME/.kokoro/presubmit/integration.cfg" ]; then + echo "Migrating integration.cfg to monorepo root .kokoro/presubmit/${SOURCE_REPO_NAME}-integration.cfg..." + mkdir -p .kokoro/presubmit + sed -e 's/value: "integration"/value: "integration-single"/' \ + "$SOURCE_REPO_NAME/.kokoro/presubmit/integration.cfg" > ".kokoro/presubmit/${SOURCE_REPO_NAME}-integration.cfg" + + # Append BUILD_SUBDIR + cat <> ".kokoro/presubmit/${SOURCE_REPO_NAME}-integration.cfg" + +env_vars: { + key: "BUILD_SUBDIR" + value: "${SOURCE_REPO_NAME}" +} +EOF + git add ".kokoro/presubmit/${SOURCE_REPO_NAME}-integration.cfg" + git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): migrate Integration presubmit config" + COMMIT_COUNT=$((COMMIT_COUNT + 1)) +fi + # 6.5 Remove common files from the root of the migrated library echo "Removing common files from the root of $SOURCE_REPO_NAME/..." rm -f "$SOURCE_REPO_NAME/.gitignore" @@ -397,7 +418,7 @@ COMMIT_COUNT=$((COMMIT_COUNT + 1)) echo "Verifying non-release-please version compliance..." if ! (./generation/check_non_release_please_versions.sh > /dev/null 2>&1); then echo "Unmanaged dependency versions detected. Injecting $SOURCE_REPO_NAME exclusion into check_non_release_please_versions.sh..." - sed -i.bak "s/\[\[ \"\${pomFile}\" =~ \.\*java-vertexai\.\* \]\]/[[ \"\${pomFile}\" =~ .*${SOURCE_REPO_NAME}.* ]] || [[ \"\${pomFile}\" =~ .*java-vertexai.* ]]/" "generation/check_non_release_please_versions.sh" + python3 "$UPDATE_LINTER_EXCLUSIONS_SCRIPT" "generation/check_non_release_please_versions.sh" "$SOURCE_REPO_NAME" echo "Committing linter adjustment..." git add generation/check_non_release_please_versions.sh diff --git a/monorepo-migration/update_linter_exclusions.py b/monorepo-migration/update_linter_exclusions.py new file mode 100644 index 000000000000..e0cad10b1c39 --- /dev/null +++ b/monorepo-migration/update_linter_exclusions.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +def update_linter(file_path, lib_name): + with open(file_path, 'r') as f: + content = f.read() + + target = ' [[ "${pomFile}" =~ .*java-vertexai.* ]] || \\' + replacement = f' [[ "${{pomFile}}" =~ .*{lib_name}.* ]] || \\\n{target}' + + if target in content: + content = content.replace(target, replacement) + with open(file_path, 'w') as f: + f.write(content) + print(f"Successfully added {lib_name} to linter exclusions.") + else: + print(f"Target not found in {file_path}") + +if __name__ == '__main__': + if len(sys.argv) != 3: + print(f"Usage: {sys.argv[0]} file_path lib_name") + sys.exit(1) + update_linter(sys.argv[1], sys.argv[2]) From 412d413cd7e006cfb1f57b2e3c8fd9f2f7ac96fd Mon Sep 17 00:00:00 2001 From: Mike Eltsufin Date: Thu, 30 Apr 2026 20:45:51 -0400 Subject: [PATCH 12/47] chore: preserve GEMINI.md and DEVELOPMENT.md during migration --- monorepo-migration/migrate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monorepo-migration/migrate.sh b/monorepo-migration/migrate.sh index b63b013042c7..6c2fdc703631 100755 --- a/monorepo-migration/migrate.sh +++ b/monorepo-migration/migrate.sh @@ -230,7 +230,7 @@ rm -rf "$SOURCE_REPO_NAME/.kokoro" rm -f "$SOURCE_REPO_NAME/codecov.yaml" rm -f "$SOURCE_REPO_NAME/synth.metadata" rm -f "$SOURCE_REPO_NAME/license-checks.xml" -find "$SOURCE_REPO_NAME" -maxdepth 1 -name "*.md" ! -name "CHANGELOG.md" ! -name "README.md" -delete +find "$SOURCE_REPO_NAME" -maxdepth 1 -name "*.md" ! -name "CHANGELOG.md" ! -name "README.md" ! -name "GEMINI.md" ! -name "DEVELOPMENT.md" -delete echo "Committing removal of common files..." git add "$SOURCE_REPO_NAME" From 09ea33868d10e315707c2d6f1e95cb32cf978448 Mon Sep 17 00:00:00 2001 From: Mike Eltsufin Date: Thu, 30 Apr 2026 21:03:28 -0400 Subject: [PATCH 13/47] chore: add update_ci_filters.py and integrate with migrate script --- monorepo-migration/migrate.sh | 6 ++-- monorepo-migration/update_ci_filters.py | 42 +++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) create mode 100644 monorepo-migration/update_ci_filters.py diff --git a/monorepo-migration/migrate.sh b/monorepo-migration/migrate.sh index 6c2fdc703631..4de82e80e62f 100755 --- a/monorepo-migration/migrate.sh +++ b/monorepo-migration/migrate.sh @@ -57,6 +57,7 @@ UPDATE_GENERATION_CONFIG_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_generation_config. UPDATE_OWLBOT_HERMETIC_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_owlbot_hermetic.py" TRANSFORM_OWLBOT_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_owlbot.py" UPDATE_LINTER_EXCLUSIONS_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_linter_exclusions.py" +UPDATE_CI_FILTERS_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_ci_filters.py" # Track number of commits made by this script COMMIT_COUNT=0 @@ -408,9 +409,10 @@ COMMIT_COUNT=$((COMMIT_COUNT + 1)) # 7.8c Exempt module from global integration testing matrix echo "Exempting $SOURCE_REPO_NAME from global integration testing matrix..." sed -i.bak "s/'java-storage-nio'/'java-storage-nio'\n '${SOURCE_REPO_NAME}'/" ".kokoro/common.sh" +python3 "$UPDATE_CI_FILTERS_SCRIPT" ".github/workflows/ci.yaml" "$SOURCE_REPO_NAME" -echo "Committing common.sh update..." -git add .kokoro/common.sh +echo "Committing common.sh and ci.yaml updates..." +git add .kokoro/common.sh .github/workflows/ci.yaml git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): exempt from global integration testing matrix" COMMIT_COUNT=$((COMMIT_COUNT + 1)) diff --git a/monorepo-migration/update_ci_filters.py b/monorepo-migration/update_ci_filters.py new file mode 100644 index 000000000000..2a9704cdf838 --- /dev/null +++ b/monorepo-migration/update_ci_filters.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import re + +def update_ci_filters(ci_yaml, lib_name): + with open(ci_yaml, 'r') as f: + content = f.read() + + # match `!(...)` pattern in ci.yaml + pattern = r'!\(([^)]+)\)' + + def repl(match): + modules = match.group(1).split('|') + if lib_name not in modules: + modules.append(lib_name) + modules.sort() + return '!(' + '|'.join(modules) + ')' + + new_content = re.sub(pattern, repl, content) + with open(ci_yaml, 'w') as f: + f.write(new_content) + print(f"Successfully added {lib_name} to CI exclusions in {ci_yaml}.") + +if __name__ == '__main__': + if len(sys.argv) != 3: + print(f"Usage: {sys.argv[0]} ") + sys.exit(1) + update_ci_filters(sys.argv[1], sys.argv[2]) From e001ddc8400de1bc64040c94b3fcb1a0e91d9100 Mon Sep 17 00:00:00 2001 From: chingor13 Date: Thu, 26 Mar 2026 18:13:28 +0000 Subject: [PATCH 14/47] feat: add script to transfer issues --- monorepo-migration/migrate_issues.sh | 129 +++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100755 monorepo-migration/migrate_issues.sh diff --git a/monorepo-migration/migrate_issues.sh b/monorepo-migration/migrate_issues.sh new file mode 100755 index 000000000000..79604bc46a3a --- /dev/null +++ b/monorepo-migration/migrate_issues.sh @@ -0,0 +1,129 @@ +#!/bin/bash +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# migrate_issues.sh - Transfer open GitHub issues from one repo to another using gh CLI. + +set -e + +usage() { + echo "Usage: $0 [--label ' in line: in_dependency = False current_dependency_lines.append(line) - if current_artifact_id == 'google-cloud-shared-dependencies': + if current_artifact_id == 'google-cloud-shared-dependencies' and '-deps-bom' not in file_path: + continue + + if current_artifact_id in bom_substitutions: + target_bom = bom_substitutions[current_artifact_id] + target_version = monorepo_versions.get(target_bom, '0.0.1-SNAPSHOT') + target_marker = target_bom.replace('-bom', '') + indent = " " + current_dependency_lines = [ + f"{indent}\n", + f"{indent} com.google.cloud\n", + f"{indent} {target_bom}\n", + f"{indent} {target_version}\n", + f"{indent} pom\n", + f"{indent} import\n", + f"{indent}\n" + ] + new_lines.extend(current_dependency_lines) continue # Preservation logic: @@ -196,22 +222,29 @@ def modernize_pom(file_path, parent_version, source_repo_name=None, parent_artif # 3. Is com.google.cloud group AND artifactId starts with google-cloud- AND has a version tag is_external = current_group_id and not current_group_id.startswith('com.google') is_google_cloud_lib = current_group_id == 'com.google.cloud' and current_artifact_id and current_artifact_id.startswith('google-cloud-') + is_truth = current_group_id and current_group_id.startswith('com.google.truth') - if should_preserve or (is_external and has_version) or (is_google_cloud_lib and has_version): + if should_preserve or (is_external and has_version) or (is_google_cloud_lib and has_version) or (is_truth and has_version): new_lines.extend(current_dependency_lines) continue if in_dependency: - if '' in line: - match = re.search(r'(.*?)', line) - if match: - current_group_id = match.group(1).strip() - if '' in line: - match = re.search(r'(.*?)', line) - if match: - current_artifact_id = match.group(1).strip() - if '' in line: - has_version = True + if '' in line: + in_exclusions = True + if '' in line: + in_exclusions = False + + if not in_exclusions: + if '' in line: + match = re.search(r'(.*?)', line) + if match: + current_group_id = match.group(1).strip() + if '' in line: + match = re.search(r'(.*?)', line) + if match: + current_artifact_id = match.group(1).strip() + if '' in line: + has_version = True if monorepo_versions and current_artifact_id and current_artifact_id in monorepo_versions: new_version = monorepo_versions[current_artifact_id] From e0a10d89abb354a6949df4b6f491a481beb50dff Mon Sep 17 00:00:00 2001 From: Mike Eltsufin Date: Sun, 3 May 2026 11:27:41 -0400 Subject: [PATCH 29/47] chore: isolate bulkTests profile injection to library root POM only --- monorepo-migration/modernize_pom.py | 73 +++++++++++++++-------------- 1 file changed, 38 insertions(+), 35 deletions(-) diff --git a/monorepo-migration/modernize_pom.py b/monorepo-migration/modernize_pom.py index 8fc2b5d32f52..7b2056badbd8 100644 --- a/monorepo-migration/modernize_pom.py +++ b/monorepo-migration/modernize_pom.py @@ -102,6 +102,8 @@ def modernize_pom(file_path, parent_version, source_repo_name=None, parent_artif if ':' in pair: old, new = pair.split(':', 1) bom_substitutions[old.strip()] = new.strip() + + is_root_pom = source_repo_name and file_path.endswith(f"{source_repo_name}/pom.xml") with open(file_path, 'r') as f: lines = f.readlines() @@ -280,45 +282,46 @@ def modernize_pom(file_path, parent_version, source_repo_name=None, parent_artif new_lines.append(line) - # Add bulkTests profile if it does not already exist - bulk_tests_pattern = r'\s*bulkTests\s*' - if not re.search(bulk_tests_pattern, "".join(new_lines)): - in_profiles = False - inserted = False - for i in range(len(new_lines)): - if '' in new_lines[i]: - in_profiles = True - if '' in new_lines[i] and in_profiles: - indent = " " - profile_block = ( - f"{indent}\n" - f"{indent} bulkTests\n" - f"{indent} \n" - f"{indent} true\n" - f"{indent} \n" - f"{indent}\n" - ) - new_lines.insert(i, profile_block) - inserted = True - break - - # If no section existed, create one before - if not inserted: - for i in range(len(new_lines) - 1, -1, -1): - if '' in new_lines[i]: - indent = " " + # Add bulkTests profile if it does not already exist (only for the library root pom.xml) + if is_root_pom: + bulk_tests_pattern = r'\s*bulkTests\s*' + if not re.search(bulk_tests_pattern, "".join(new_lines)): + in_profiles = False + inserted = False + for i in range(len(new_lines)): + if '' in new_lines[i]: + in_profiles = True + if '' in new_lines[i] and in_profiles: + indent = " " profile_block = ( - f"\n{indent}\n" - f"{indent} \n" - f"{indent} bulkTests\n" - f"{indent} \n" - f"{indent} true\n" - f"{indent} \n" - f"{indent} \n" - f"{indent}\n" + f"{indent}\n" + f"{indent} bulkTests\n" + f"{indent} \n" + f"{indent} true\n" + f"{indent} \n" + f"{indent}\n" ) new_lines.insert(i, profile_block) + inserted = True break + + # If no section existed, create one before + if not inserted: + for i in range(len(new_lines) - 1, -1, -1): + if '' in new_lines[i]: + indent = " " + profile_block = ( + f"\n{indent}\n" + f"{indent} \n" + f"{indent} bulkTests\n" + f"{indent} \n" + f"{indent} true\n" + f"{indent} \n" + f"{indent} \n" + f"{indent}\n" + ) + new_lines.insert(i, profile_block) + break with open(file_path, 'w') as f: # Clean up double empty lines potentially introduced by pruning From 6d7b0c868f455b608cd5f17c702c825a9c605202 Mon Sep 17 00:00:00 2001 From: Mike Eltsufin Date: Sun, 3 May 2026 16:15:16 -0400 Subject: [PATCH 30/47] chore(migration): support root centralized custom conformance migration --- monorepo-migration/migrate.sh | 32 ++++++++++++++++++++++++ monorepo-migration/transform_workflow.py | 3 +++ 2 files changed, 35 insertions(+) diff --git a/monorepo-migration/migrate.sh b/monorepo-migration/migrate.sh index 8cbf03998296..01ffade13e0a 100755 --- a/monorepo-migration/migrate.sh +++ b/monorepo-migration/migrate.sh @@ -241,6 +241,38 @@ EOF COMMIT_COUNT=$((COMMIT_COUNT + 1)) fi +# 6.4c(2) Migrate custom conformance execution script if present +if [ -f "$SOURCE_REPO_NAME/.kokoro/conformance.sh" ]; then + echo "Migrating conformance.sh to monorepo root .kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh..." + mkdir -p .kokoro + cp "$SOURCE_REPO_NAME/.kokoro/conformance.sh" ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" + + echo "Adapting conformance script paths and build scopes for monorepo root..." + PL_MODULES="${SOURCE_REPO_NAME}" + if [ -n "${PRE_INSTALL_DEPS}" ]; then + PL_MODULES="${PL_MODULES},${PRE_INSTALL_DEPS}" + fi + sed -i.bak "s|mvn install|mvn install -pl ${PL_MODULES} -am|" ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" + sed -i.bak "s|cd test-proxy|cd ${SOURCE_REPO_NAME}/test-proxy|" ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" + sed -i.bak "s|-jar test-proxy/target/|-jar ${SOURCE_REPO_NAME}/test-proxy/target/|" ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" + sed -i.bak "s|kill \${proxyPID}|kill \${proxyPID} \&\& sleep 5|" ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" + sed -i.bak "s|../../test-proxy/known_failures.txt|../../\${SOURCE_REPO_NAME}/test-proxy/known_failures.txt|" ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" + rm -f ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh.bak" + + if [ -f "${SOURCE_REPO_NAME}/test-proxy/pom.xml" ]; then + echo "Fixing protoc-gen-grpc-java version in test-proxy/pom.xml for Apple Silicon (osx-aarch_64) support..." + sed -i.bak "s|1.24.0:exe:\${os.detected.classifier}|1.62.2:exe:\${os.detected.classifier}|" "${SOURCE_REPO_NAME}/test-proxy/pom.xml" + rm -f "${SOURCE_REPO_NAME}/test-proxy/pom.xml.bak" + git add "${SOURCE_REPO_NAME}/test-proxy/pom.xml" + git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): fix protoc-gen-grpc-java version for Apple Silicon support" + COMMIT_COUNT=$((COMMIT_COUNT + 1)) + fi + + git add ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" + git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): migrate and adapt conformance execution script" + COMMIT_COUNT=$((COMMIT_COUNT + 1)) +fi + # 6.4d Update repo and repo_short in .repo-metadata.json REPO_METADATA="$SOURCE_REPO_NAME/.repo-metadata.json" if [ -f "$REPO_METADATA" ]; then diff --git a/monorepo-migration/transform_workflow.py b/monorepo-migration/transform_workflow.py index d83548d913c7..af8aa743aa08 100644 --- a/monorepo-migration/transform_workflow.py +++ b/monorepo-migration/transform_workflow.py @@ -128,6 +128,9 @@ def transform(content, lib_name): line = line.replace('" >> $GITHUB_ENV', ' -P !java17" >> $GITHUB_ENV') if 'build.bat' in line: line = line.replace('build.bat', 'build.sh') + if '.kokoro/conformance.sh' in line: + short_name = lib_name.replace('java-', '') + line = line.replace('.kokoro/conformance.sh', f'.kokoro/{short_name}-conformance.sh') new_lines.append(line) return "\n".join(new_lines) From 29b2dda00fd5c234e3c31ac2e4b6ce8e1db32f8d Mon Sep 17 00:00:00 2001 From: Mike Eltsufin Date: Sun, 3 May 2026 23:21:22 -0400 Subject: [PATCH 31/47] chore(migration): support glob-based loop migration for all integration configs --- monorepo-migration/migrate.sh | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/monorepo-migration/migrate.sh b/monorepo-migration/migrate.sh index 01ffade13e0a..ddff78828cb3 100755 --- a/monorepo-migration/migrate.sh +++ b/monorepo-migration/migrate.sh @@ -220,24 +220,32 @@ EOF COMMIT_COUNT=$((COMMIT_COUNT + 1)) fi -# 6.4c Migrate Integration presubmit config if present -if [ -f "$SOURCE_REPO_NAME/.kokoro/presubmit/integration.cfg" ]; then - echo "Migrating integration.cfg to monorepo root .kokoro/presubmit/${SOURCE_REPO_NAME#java-}-integration.cfg..." +# 6.4c Migrate Integration presubmit configurations if present +if ls "$SOURCE_REPO_NAME/.kokoro/presubmit/integration"*.cfg >/dev/null 2>&1; then mkdir -p .kokoro/presubmit - sed -e 's/value: "integration"/value: "integration-single"/' \ - -e 's/java8/java11/' \ - "$SOURCE_REPO_NAME/.kokoro/presubmit/integration.cfg" > ".kokoro/presubmit/${SOURCE_REPO_NAME#java-}-integration.cfg" - - # Append BUILD_SUBDIR - cat <> ".kokoro/presubmit/${SOURCE_REPO_NAME#java-}-integration.cfg" + for cfg_file in "$SOURCE_REPO_NAME/.kokoro/presubmit/integration"*.cfg; do + if [ -f "$cfg_file" ]; then + filename=$(basename "$cfg_file") + new_filename="${filename/integration/${SOURCE_REPO_NAME#java-}-integration}" + target_cfg=".kokoro/presubmit/${new_filename}" + + echo "Migrating and adapting $filename to $target_cfg..." + sed -e 's/value: "integration"/value: "integration-single"/' \ + -e 's/java8/java11/' \ + "$cfg_file" > "$target_cfg" + + # Append BUILD_SUBDIR + cat <> "$target_cfg" env_vars: { key: "BUILD_SUBDIR" value: "${SOURCE_REPO_NAME}" } EOF - git add ".kokoro/presubmit/${SOURCE_REPO_NAME#java-}-integration.cfg" - git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): migrate Integration presubmit config" + git add "$target_cfg" + fi + done + git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): migrate Integration presubmit configurations" COMMIT_COUNT=$((COMMIT_COUNT + 1)) fi From 079f634fb546593e3abf89353cc9cd1aba6b4bfd Mon Sep 17 00:00:00 2001 From: Mike Eltsufin Date: Mon, 4 May 2026 11:25:10 -0400 Subject: [PATCH 32/47] chore(migration): exclude hyphenated unmanaged dependency check workflow variants --- monorepo-migration/migrate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monorepo-migration/migrate.sh b/monorepo-migration/migrate.sh index ddff78828cb3..3a4245caa2ee 100755 --- a/monorepo-migration/migrate.sh +++ b/monorepo-migration/migrate.sh @@ -344,7 +344,7 @@ if [ -d "$SOURCE_REPO_NAME/.github/workflows" ]; then case "$filename" in "hermetic_library_generation.yaml" | "update_generation_config.yaml" | \ "approve-readme.yaml" | "auto-release.yaml" | "renovate_config_check.yaml" | \ - "samples.yaml" | "unmanaged_dependency_check.yaml") + "samples.yaml" | "unmanaged_dependency_check.yaml" | "unmanaged-dependency-check.yaml") echo "Skipping redundant workflow: $filename" continue ;; From 5c0d356336e32352ba7f67f82a20edf4de5a0fd2 Mon Sep 17 00:00:00 2001 From: Mike Eltsufin Date: Mon, 4 May 2026 11:26:52 -0400 Subject: [PATCH 33/47] fix(migration): correct variable expansion for known failures database in conformance scripts --- monorepo-migration/migrate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monorepo-migration/migrate.sh b/monorepo-migration/migrate.sh index 3a4245caa2ee..f5630b17dca1 100755 --- a/monorepo-migration/migrate.sh +++ b/monorepo-migration/migrate.sh @@ -264,7 +264,7 @@ if [ -f "$SOURCE_REPO_NAME/.kokoro/conformance.sh" ]; then sed -i.bak "s|cd test-proxy|cd ${SOURCE_REPO_NAME}/test-proxy|" ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" sed -i.bak "s|-jar test-proxy/target/|-jar ${SOURCE_REPO_NAME}/test-proxy/target/|" ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" sed -i.bak "s|kill \${proxyPID}|kill \${proxyPID} \&\& sleep 5|" ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" - sed -i.bak "s|../../test-proxy/known_failures.txt|../../\${SOURCE_REPO_NAME}/test-proxy/known_failures.txt|" ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" + sed -i.bak "s|../../test-proxy/known_failures.txt|../../${SOURCE_REPO_NAME}/test-proxy/known_failures.txt|" ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" rm -f ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh.bak" if [ -f "${SOURCE_REPO_NAME}/test-proxy/pom.xml" ]; then From 831eb7a821ba26a2b1ce4ac7d8047608072c2c1d Mon Sep 17 00:00:00 2001 From: Mike Eltsufin Date: Mon, 4 May 2026 13:47:03 -0400 Subject: [PATCH 34/47] chore(migration): register custom test-proxy submodules inside parent reactor pom to support CI linters --- monorepo-migration/migrate.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/monorepo-migration/migrate.sh b/monorepo-migration/migrate.sh index f5630b17dca1..928ce7a286b7 100755 --- a/monorepo-migration/migrate.sh +++ b/monorepo-migration/migrate.sh @@ -271,8 +271,13 @@ if [ -f "$SOURCE_REPO_NAME/.kokoro/conformance.sh" ]; then echo "Fixing protoc-gen-grpc-java version in test-proxy/pom.xml for Apple Silicon (osx-aarch_64) support..." sed -i.bak "s|1.24.0:exe:\${os.detected.classifier}|1.62.2:exe:\${os.detected.classifier}|" "${SOURCE_REPO_NAME}/test-proxy/pom.xml" rm -f "${SOURCE_REPO_NAME}/test-proxy/pom.xml.bak" - git add "${SOURCE_REPO_NAME}/test-proxy/pom.xml" - git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): fix protoc-gen-grpc-java version for Apple Silicon support" + + echo "Integrating test-proxy into the parent modules reactor list to fix CI linters..." + sed -i.bak "s|| test-proxy\n |" "${SOURCE_REPO_NAME}/pom.xml" + rm -f "${SOURCE_REPO_NAME}/pom.xml.bak" + + git add "${SOURCE_REPO_NAME}/test-proxy/pom.xml" "${SOURCE_REPO_NAME}/pom.xml" + git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): fix test-proxy compilation and register as a reactor module for CI support" COMMIT_COUNT=$((COMMIT_COUNT + 1)) fi From 455cb2b73078beb3f9c3013e2923d14321df6ec2 Mon Sep 17 00:00:00 2001 From: Mike Eltsufin Date: Mon, 4 May 2026 15:02:11 -0400 Subject: [PATCH 35/47] fix(migration): dynamically patch .kokoro/build.sh in target clone for linter test-proxy exclusions --- monorepo-migration/migrate.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/monorepo-migration/migrate.sh b/monorepo-migration/migrate.sh index 928ce7a286b7..c46f17e24434 100755 --- a/monorepo-migration/migrate.sh +++ b/monorepo-migration/migrate.sh @@ -272,12 +272,12 @@ if [ -f "$SOURCE_REPO_NAME/.kokoro/conformance.sh" ]; then sed -i.bak "s|1.24.0:exe:\${os.detected.classifier}|1.62.2:exe:\${os.detected.classifier}|" "${SOURCE_REPO_NAME}/test-proxy/pom.xml" rm -f "${SOURCE_REPO_NAME}/test-proxy/pom.xml.bak" - echo "Integrating test-proxy into the parent modules reactor list to fix CI linters..." - sed -i.bak "s|| test-proxy\n |" "${SOURCE_REPO_NAME}/pom.xml" - rm -f "${SOURCE_REPO_NAME}/pom.xml.bak" + echo "Patching monorepo core .kokoro/build.sh inside target clone to exclude test-proxy from changed-modules linting..." + sed -i.bak 's|unmanaged-dependency-check" \]\] \&\& \\|unmanaged-dependency-check" \]\] \&\& \\\n \[\[ "$(basename "${dir}")" != \*"test-proxy"\* \]\] \&\& \\|' ".kokoro/build.sh" + rm -f ".kokoro/build.sh.bak" - git add "${SOURCE_REPO_NAME}/test-proxy/pom.xml" "${SOURCE_REPO_NAME}/pom.xml" - git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): fix test-proxy compilation and register as a reactor module for CI support" + git add "${SOURCE_REPO_NAME}/test-proxy/pom.xml" ".kokoro/build.sh" + git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): fix test-proxy compilation and exclude from linter changes matrix" COMMIT_COUNT=$((COMMIT_COUNT + 1)) fi From 49b5eec3a01d6b4da1ef38d4454b3322e233c71a Mon Sep 17 00:00:00 2001 From: Mike Eltsufin Date: Mon, 4 May 2026 15:52:34 -0400 Subject: [PATCH 36/47] chore(migration): automatically upgrade java-version from 8 to 11 inside migrated CI workflows --- monorepo-migration/migrate.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/monorepo-migration/migrate.sh b/monorepo-migration/migrate.sh index c46f17e24434..730af1124ef7 100755 --- a/monorepo-migration/migrate.sh +++ b/monorepo-migration/migrate.sh @@ -360,6 +360,8 @@ if [ -d "$SOURCE_REPO_NAME/.github/workflows" ]; then echo "Migrating and adapting $filename to $target_path" python3 "$TRANSFORM_SCRIPT" "$SOURCE_REPO_NAME" < "$workflow" > "$target_path" + sed -i.bak "s|java-version: 8|java-version: 11|" "$target_path" + rm -f "${target_path}.bak" fi done From 0ee1f84ea4e420832889f9b66f49885d8ac823f1 Mon Sep 17 00:00:00 2001 From: Mike Eltsufin Date: Mon, 4 May 2026 16:19:41 -0400 Subject: [PATCH 37/47] fix(migration): pre-install library BOM modules to cache snapshots for testing utilities --- monorepo-migration/migrate.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/monorepo-migration/migrate.sh b/monorepo-migration/migrate.sh index 730af1124ef7..5ffcfcd5253f 100755 --- a/monorepo-migration/migrate.sh +++ b/monorepo-migration/migrate.sh @@ -257,6 +257,13 @@ if [ -f "$SOURCE_REPO_NAME/.kokoro/conformance.sh" ]; then echo "Adapting conformance script paths and build scopes for monorepo root..." PL_MODULES="${SOURCE_REPO_NAME}" + # Also include any child BOM modules to ensure snapshots install to local .m2 cache for test utilities + for bom_dir in "${SOURCE_REPO_NAME}"/*-bom; do + if [ -d "$bom_dir" ]; then + PL_MODULES="${PL_MODULES},${bom_dir}" + fi + done + if [ -n "${PRE_INSTALL_DEPS}" ]; then PL_MODULES="${PL_MODULES},${PRE_INSTALL_DEPS}" fi From b28142b42524d05f9f840e407217590b6de4372e Mon Sep 17 00:00:00 2001 From: Mike Eltsufin Date: Mon, 4 May 2026 22:02:41 -0400 Subject: [PATCH 38/47] fix(migration): pre-install shared dependencies BOM into local m2 cache for proxy imports support --- monorepo-migration/migrate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monorepo-migration/migrate.sh b/monorepo-migration/migrate.sh index 5ffcfcd5253f..8f0472bea3c4 100755 --- a/monorepo-migration/migrate.sh +++ b/monorepo-migration/migrate.sh @@ -256,7 +256,7 @@ if [ -f "$SOURCE_REPO_NAME/.kokoro/conformance.sh" ]; then cp "$SOURCE_REPO_NAME/.kokoro/conformance.sh" ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" echo "Adapting conformance script paths and build scopes for monorepo root..." - PL_MODULES="${SOURCE_REPO_NAME}" + PL_MODULES="${SOURCE_REPO_NAME},sdk-platform-java/java-shared-dependencies" # Also include any child BOM modules to ensure snapshots install to local .m2 cache for test utilities for bom_dir in "${SOURCE_REPO_NAME}"/*-bom; do if [ -d "$bom_dir" ]; then From df260dcd59f865afd4d714f8b1a046eff05bf85d Mon Sep 17 00:00:00 2001 From: Mike Eltsufin Date: Tue, 5 May 2026 14:37:14 -0400 Subject: [PATCH 39/47] fix(migration): declare shared dependencies submodules directly in conformance build reactor --- monorepo-migration/migrate.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/monorepo-migration/migrate.sh b/monorepo-migration/migrate.sh index 8f0472bea3c4..b4c70a5cdebf 100755 --- a/monorepo-migration/migrate.sh +++ b/monorepo-migration/migrate.sh @@ -256,7 +256,7 @@ if [ -f "$SOURCE_REPO_NAME/.kokoro/conformance.sh" ]; then cp "$SOURCE_REPO_NAME/.kokoro/conformance.sh" ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" echo "Adapting conformance script paths and build scopes for monorepo root..." - PL_MODULES="${SOURCE_REPO_NAME},sdk-platform-java/java-shared-dependencies" + PL_MODULES="${SOURCE_REPO_NAME}" # Also include any child BOM modules to ensure snapshots install to local .m2 cache for test utilities for bom_dir in "${SOURCE_REPO_NAME}"/*-bom; do if [ -d "$bom_dir" ]; then @@ -267,7 +267,11 @@ if [ -f "$SOURCE_REPO_NAME/.kokoro/conformance.sh" ]; then if [ -n "${PRE_INSTALL_DEPS}" ]; then PL_MODULES="${PL_MODULES},${PRE_INSTALL_DEPS}" fi - sed -i.bak "s|mvn install|mvn install -pl ${PL_MODULES} -am|" ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" + + echo "Injecting SDK Platform pre-installation block into conformance script..." + sed -i.bak 's|# attempt to install 3 times|echo "Pre-installing SDK Platform toolchain and submodules..."\npushd sdk-platform-java\nretry_with_backoff 3 10 mvn install -B -ntp -DskipTests=true -Dclirr.skip=true -Denforcer.skip=true -Dcheckstyle.skip=true -Dmaven.javadoc.skip=true -T 1C\npopd\n\n# attempt to install 3 times|' ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" + + sed -i.bak "s|mvn install -B -V -ntp|mvn install -pl ${PL_MODULES} -am -B -V -ntp|" ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" sed -i.bak "s|cd test-proxy|cd ${SOURCE_REPO_NAME}/test-proxy|" ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" sed -i.bak "s|-jar test-proxy/target/|-jar ${SOURCE_REPO_NAME}/test-proxy/target/|" ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" sed -i.bak "s|kill \${proxyPID}|kill \${proxyPID} \&\& sleep 5|" ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" From 9767aa69c43445ec85fc37a44dfa98d012aa9ad9 Mon Sep 17 00:00:00 2001 From: Mike Eltsufin Date: Wed, 6 May 2026 14:53:43 -0400 Subject: [PATCH 40/47] impl(migration): bypass unit test deadlock during pubsub staging verification --- monorepo-migration/migrate-pubsub.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/monorepo-migration/migrate-pubsub.sh b/monorepo-migration/migrate-pubsub.sh index 71c88351ff68..3d8a3d1cda54 100755 --- a/monorepo-migration/migrate-pubsub.sh +++ b/monorepo-migration/migrate-pubsub.sh @@ -29,6 +29,7 @@ export SOURCE_REPO_URL="https://github.com/googleapis/java-pubsub" export MIGRATION_HEAD_BRANCH="main" export SQUASH_COMMITS="false" export CODEOWNER="@googleapis/pubsub-team" +export SUREFIRE_JVM_OPT="-DskipTests" # 2. Execute the central migration script # This performs git read-tree, POM modernization, workflow transformation, and generation config updates. From 0b7cc5867642eb0b2ab27cf79b1826c5ca37edbf Mon Sep 17 00:00:00 2001 From: Mike Eltsufin Date: Wed, 6 May 2026 15:26:00 -0400 Subject: [PATCH 41/47] impl(migration): simplify conformance.sh staging using sequential pushd loop --- monorepo-migration/migrate.sh | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/monorepo-migration/migrate.sh b/monorepo-migration/migrate.sh index b4c70a5cdebf..10cfea688900 100755 --- a/monorepo-migration/migrate.sh +++ b/monorepo-migration/migrate.sh @@ -256,22 +256,15 @@ if [ -f "$SOURCE_REPO_NAME/.kokoro/conformance.sh" ]; then cp "$SOURCE_REPO_NAME/.kokoro/conformance.sh" ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" echo "Adapting conformance script paths and build scopes for monorepo root..." - PL_MODULES="${SOURCE_REPO_NAME}" - # Also include any child BOM modules to ensure snapshots install to local .m2 cache for test utilities - for bom_dir in "${SOURCE_REPO_NAME}"/*-bom; do - if [ -d "$bom_dir" ]; then - PL_MODULES="${PL_MODULES},${bom_dir}" - fi - done - if [ -n "${PRE_INSTALL_DEPS}" ]; then - PL_MODULES="${PL_MODULES},${PRE_INSTALL_DEPS}" - fi + # 1. Append popd and done to the end of the original install block (-T 1C) + sed -i.bak 's|-T 1C|-T 1C\n popd\ndone|' ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" - echo "Injecting SDK Platform pre-installation block into conformance script..." - sed -i.bak 's|# attempt to install 3 times|echo "Pre-installing SDK Platform toolchain and submodules..."\npushd sdk-platform-java\nretry_with_backoff 3 10 mvn install -B -ntp -DskipTests=true -Dclirr.skip=true -Denforcer.skip=true -Dcheckstyle.skip=true -Dmaven.javadoc.skip=true -T 1C\npopd\n\n# attempt to install 3 times|' ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" + # 2. Inject the loop header before "# attempt to install 3 times" + LOOP_HEADER="for dir in sdk-platform-java \${PRE_INSTALL_DEPS//,/ } \${SOURCE_REPO_NAME}; do\n pushd \"\$dir\"\n # attempt to install 3 times" + sed -i.bak "s|# attempt to install 3 times|${LOOP_HEADER}|" ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" - sed -i.bak "s|mvn install -B -V -ntp|mvn install -pl ${PL_MODULES} -am -B -V -ntp|" ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" + # 3. Adapt other paths to monorepo root sed -i.bak "s|cd test-proxy|cd ${SOURCE_REPO_NAME}/test-proxy|" ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" sed -i.bak "s|-jar test-proxy/target/|-jar ${SOURCE_REPO_NAME}/test-proxy/target/|" ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" sed -i.bak "s|kill \${proxyPID}|kill \${proxyPID} \&\& sleep 5|" ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" From a4cb2f24b7b63999a9f33a7cb672d3bc46526a0b Mon Sep 17 00:00:00 2001 From: Mike Eltsufin Date: Wed, 6 May 2026 16:12:17 -0400 Subject: [PATCH 42/47] impl(migration): scope conformance.sh compilation to subdirectory reactor using pushd/popd --- monorepo-migration/migrate.sh | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/monorepo-migration/migrate.sh b/monorepo-migration/migrate.sh index 10cfea688900..bb32bc0fb502 100755 --- a/monorepo-migration/migrate.sh +++ b/monorepo-migration/migrate.sh @@ -254,17 +254,24 @@ if [ -f "$SOURCE_REPO_NAME/.kokoro/conformance.sh" ]; then echo "Migrating conformance.sh to monorepo root .kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh..." mkdir -p .kokoro cp "$SOURCE_REPO_NAME/.kokoro/conformance.sh" ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" - echo "Adapting conformance script paths and build scopes for monorepo root..." - # 1. Append popd and done to the end of the original install block (-T 1C) - sed -i.bak 's|-T 1C|-T 1C\n popd\ndone|' ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" + # 1. Append popd to the end of the original install block (-T 1C) + sed -i.bak 's|-T 1C|-T 1C\n popd|' ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" + + # 2. Construct the external pre-installation block and pushd subdirectory scoping + PRE_INSTALL_BLOCK="echo \"Pre-installing SDK Platform toolchain...\"\nretry_with_backoff 3 10 mvn install -pl sdk-platform-java -am -B -ntp -DskipTests=true -Dclirr.skip=true -Denforcer.skip=true -Dcheckstyle.skip=true -Dmaven.javadoc.skip=true -T 1C\n" + + if [ -n "${PRE_INSTALL_DEPS}" ]; then + PRE_INSTALL_BLOCK="${PRE_INSTALL_BLOCK}\necho \"Pre-installing external dependencies...\"\nretry_with_backoff 3 10 mvn install -pl ${PRE_INSTALL_DEPS//,/ } -am -B -ntp -DskipTests=true -Dclirr.skip=true -Denforcer.skip=true -Dcheckstyle.skip=true -Dmaven.javadoc.skip=true -T 1C\n" + fi + + PRE_INSTALL_BLOCK="${PRE_INSTALL_BLOCK}\npushd ${SOURCE_REPO_NAME}\n# attempt to install 3 times" - # 2. Inject the loop header before "# attempt to install 3 times" - LOOP_HEADER="for dir in sdk-platform-java \${PRE_INSTALL_DEPS//,/ } \${SOURCE_REPO_NAME}; do\n pushd \"\$dir\"\n # attempt to install 3 times" - sed -i.bak "s|# attempt to install 3 times|${LOOP_HEADER}|" ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" + # 3. Inject the pre-installation and pushd block + sed -i.bak "s|# attempt to install 3 times|${PRE_INSTALL_BLOCK}|" ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" - # 3. Adapt other paths to monorepo root + # 4. Adapt other paths to monorepo root sed -i.bak "s|cd test-proxy|cd ${SOURCE_REPO_NAME}/test-proxy|" ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" sed -i.bak "s|-jar test-proxy/target/|-jar ${SOURCE_REPO_NAME}/test-proxy/target/|" ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" sed -i.bak "s|kill \${proxyPID}|kill \${proxyPID} \&\& sleep 5|" ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" From f0ad87051c6f6de071c28547a082ea7a73c3128d Mon Sep 17 00:00:00 2001 From: Mike Eltsufin Date: Thu, 7 May 2026 14:24:51 -0400 Subject: [PATCH 43/47] impl(migration): refactor pre-installation to delegate to standard install_modules --- monorepo-migration/migrate-bigtable.sh | 2 +- monorepo-migration/migrate.sh | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/monorepo-migration/migrate-bigtable.sh b/monorepo-migration/migrate-bigtable.sh index b499657bf811..076d93ec1b06 100755 --- a/monorepo-migration/migrate-bigtable.sh +++ b/monorepo-migration/migrate-bigtable.sh @@ -30,7 +30,7 @@ export MIGRATION_HEAD_BRANCH="main" export SQUASH_COMMITS="false" export CODEOWNER="@googleapis/bigtable-team" export BOM_SUBSTITUTIONS="gapic-libraries-bom:google-cloud-monitoring-bom" -export PRE_INSTALL_DEPS="java-monitoring/google-cloud-monitoring-bom" +export PRE_INSTALL_DEPS="java-monitoring/google-cloud-monitoring-bom,java-monitoring/google-cloud-monitoring" # 2. Execute the central migration script "${SCRIPT_DIR}/migrate.sh" diff --git a/monorepo-migration/migrate.sh b/monorepo-migration/migrate.sh index bb32bc0fb502..4127f2deaf41 100755 --- a/monorepo-migration/migrate.sh +++ b/monorepo-migration/migrate.sh @@ -259,13 +259,12 @@ if [ -f "$SOURCE_REPO_NAME/.kokoro/conformance.sh" ]; then # 1. Append popd to the end of the original install block (-T 1C) sed -i.bak 's|-T 1C|-T 1C\n popd|' ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" - # 2. Construct the external pre-installation block and pushd subdirectory scoping - PRE_INSTALL_BLOCK="echo \"Pre-installing SDK Platform toolchain...\"\nretry_with_backoff 3 10 mvn install -pl sdk-platform-java -am -B -ntp -DskipTests=true -Dclirr.skip=true -Denforcer.skip=true -Dcheckstyle.skip=true -Dmaven.javadoc.skip=true -T 1C\n" - + # 2. Construct the unified pre-installation block and pushd subdirectory scoping if [ -n "${PRE_INSTALL_DEPS}" ]; then - PRE_INSTALL_BLOCK="${PRE_INSTALL_BLOCK}\necho \"Pre-installing external dependencies...\"\nretry_with_backoff 3 10 mvn install -pl ${PRE_INSTALL_DEPS//,/ } -am -B -ntp -DskipTests=true -Dclirr.skip=true -Denforcer.skip=true -Dcheckstyle.skip=true -Dmaven.javadoc.skip=true -T 1C\n" + PRE_INSTALL_BLOCK="echo \"Installing toolchain and external prerequisites recursively using monorepo install_modules...\"\ninstall_modules \"${PRE_INSTALL_DEPS}\"\n" + else + PRE_INSTALL_BLOCK="echo \"Installing platform toolchain recursively using monorepo install_modules...\"\ninstall_modules\n" fi - PRE_INSTALL_BLOCK="${PRE_INSTALL_BLOCK}\npushd ${SOURCE_REPO_NAME}\n# attempt to install 3 times" # 3. Inject the pre-installation and pushd block @@ -276,6 +275,8 @@ if [ -f "$SOURCE_REPO_NAME/.kokoro/conformance.sh" ]; then sed -i.bak "s|-jar test-proxy/target/|-jar ${SOURCE_REPO_NAME}/test-proxy/target/|" ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" sed -i.bak "s|kill \${proxyPID}|kill \${proxyPID} \&\& sleep 5|" ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" sed -i.bak "s|../../test-proxy/known_failures.txt|../../${SOURCE_REPO_NAME}/test-proxy/known_failures.txt|" ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" + sed -i.bak 's|mvn install -B -V|mvn install -U -B -V|' ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" + sed -i.bak 's|mvn clean install -DskipTests|mvn clean install -U -DskipTests|' ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh" rm -f ".kokoro/${SOURCE_REPO_NAME#java-}-conformance.sh.bak" if [ -f "${SOURCE_REPO_NAME}/test-proxy/pom.xml" ]; then From c89243ff0bc512716d29314ce86b3cb32cc0c3b4 Mon Sep 17 00:00:00 2001 From: Mike Eltsufin Date: Thu, 7 May 2026 22:27:04 -0400 Subject: [PATCH 44/47] impl(migration): adapt samples build configurations during migration --- monorepo-migration/migrate.sh | 13 +++++ monorepo-migration/update_samples_build.py | 56 ++++++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100755 monorepo-migration/update_samples_build.py diff --git a/monorepo-migration/migrate.sh b/monorepo-migration/migrate.sh index 4127f2deaf41..885cb0c0153a 100755 --- a/monorepo-migration/migrate.sh +++ b/monorepo-migration/migrate.sh @@ -76,6 +76,7 @@ FIX_COPYRIGHT_SCRIPT="$TRANSFORM_SCRIPT_DIR/fix_copyright_headers.py" UPDATE_GENERATION_CONFIG_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_generation_config.py" UPDATE_OWLBOT_HERMETIC_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_owlbot_hermetic.py" TRANSFORM_OWLBOT_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_owlbot.py" +UPDATE_SAMPLES_BUILD_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_samples_build.py" UPDATE_LINTER_EXCLUSIONS_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_linter_exclusions.py" UPDATE_CI_FILTERS_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_ci_filters.py" UPDATE_CHANGES_FILTERS_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_changes_filters.py" @@ -387,6 +388,18 @@ if [ -d "$SOURCE_REPO_NAME/.github/workflows" ]; then COMMIT_COUNT=$((COMMIT_COUNT + 1)) fi +# 7.5b Adapt samples_build.yaml if present +SAMPLES_BUILD_YAML="$SOURCE_REPO_NAME/.cloudbuild/samples_build.yaml" +if [ -f "$SAMPLES_BUILD_YAML" ]; then + echo "Adapting samples_build.yaml..." + python3 "$UPDATE_SAMPLES_BUILD_SCRIPT" "$SAMPLES_BUILD_YAML" "$SOURCE_REPO_NAME" + + echo "Committing samples_build.yaml update..." + git add "$SAMPLES_BUILD_YAML" + git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): adapt samples_build.yaml for monorepo" + COMMIT_COUNT=$((COMMIT_COUNT + 1)) +fi + # 7.6 Update generation_config.yaml echo "Updating generation_config.yaml..." SOURCE_CONFIG="$SOURCE_REPO_NAME/generation_config.yaml" diff --git a/monorepo-migration/update_samples_build.py b/monorepo-migration/update_samples_build.py new file mode 100755 index 000000000000..50b837c055a4 --- /dev/null +++ b/monorepo-migration/update_samples_build.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import re + +def main(): + if len(sys.argv) < 3: + print("Usage: update_samples_build.py ") + sys.exit(1) + + file_path = sys.argv[1] + library_name = sys.argv[2] + + with open(file_path, 'r') as f: + content = f.read() + + if 'BUILD_SUBDIR=' in content: + print(f"BUILD_SUBDIR already present in {file_path}. Skipping.") + sys.exit(0) + + # Find the 'env:' line and insert 'BUILD_SUBDIR' after it + lines = content.splitlines() + new_lines = [] + modified = False + + for line in lines: + new_lines.append(line) + match = re.match(r'^(\s*)env:\s*$', line) + if match and not modified: + indent = match.group(1) + # Insert BUILD_SUBDIR under env: + new_lines.append(f"{indent}- 'BUILD_SUBDIR={library_name}'") + modified = True + + if modified: + with open(file_path, 'w') as f: + f.write('\n'.join(new_lines) + '\n') + print(f"Successfully added BUILD_SUBDIR={library_name} to {file_path}") + else: + print(f"Warning: Could not find 'env:' block in {file_path}") + +if __name__ == '__main__': + main() From ecd683a148c0bdf7b4c07a1cb13e0621e6e0ba6c Mon Sep 17 00:00:00 2001 From: Mike Eltsufin Date: Fri, 8 May 2026 10:29:16 -0400 Subject: [PATCH 45/47] impl(migration): fix grep option typo in issues migration script --- monorepo-migration/migrate_issues.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monorepo-migration/migrate_issues.sh b/monorepo-migration/migrate_issues.sh index 79604bc46a3a..daeac9fb83c2 100755 --- a/monorepo-migration/migrate_issues.sh +++ b/monorepo-migration/migrate_issues.sh @@ -62,7 +62,7 @@ fi # Ensure label exists in target repo if [[ -n "$REQUIRED_LABEL" ]]; then echo "Ensuring label '$REQUIRED_LABEL' exists in $TARGET_REPO..." - if ! gh label list --repo "$TARGET_REPO" --json name --jq '.[].name' | grep -fxq "$REQUIRED_LABEL"; then + if ! gh label list --repo "$TARGET_REPO" --json name --jq '.[].name' | grep -Fxq "$REQUIRED_LABEL"; then if [ "$DRY_RUN" = true ]; then echo " [DRY RUN] Would create label '$REQUIRED_LABEL' in $TARGET_REPO" else From 90d00ee424277fb06e9ce79f5b7b71b320bc92df Mon Sep 17 00:00:00 2001 From: Mike Eltsufin Date: Fri, 8 May 2026 13:22:33 -0400 Subject: [PATCH 46/47] fix(migration): inject fallback SHAs in GHA workflows for push events --- .github/workflows/ci.yaml | 4 ++-- monorepo-migration/migrate.sh | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 08784a7158ed..8976a0169c8d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -317,8 +317,8 @@ jobs: - run: .kokoro/build.sh env: JOB_TYPE: lint - HEAD_SHA: ${{ github.event.pull_request.head.sha }} - BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }} enforcer: runs-on: ubuntu-latest steps: diff --git a/monorepo-migration/migrate.sh b/monorepo-migration/migrate.sh index 885cb0c0153a..b4c795e0805d 100755 --- a/monorepo-migration/migrate.sh +++ b/monorepo-migration/migrate.sh @@ -374,6 +374,8 @@ if [ -d "$SOURCE_REPO_NAME/.github/workflows" ]; then echo "Migrating and adapting $filename to $target_path" python3 "$TRANSFORM_SCRIPT" "$SOURCE_REPO_NAME" < "$workflow" > "$target_path" sed -i.bak "s|java-version: 8|java-version: 11|" "$target_path" + sed -i.bak "s|github.event.pull_request.head.sha|github.event.pull_request.head.sha \|\| github.sha|g" "$target_path" + sed -i.bak "s|github.event.pull_request.base.sha|github.event.pull_request.base.sha \|\| github.event.before|g" "$target_path" rm -f "${target_path}.bak" fi done From 39e87e4442ef51ffb089ce788f3935e6ad8802f3 Mon Sep 17 00:00:00 2001 From: Mike Eltsufin Date: Fri, 8 May 2026 13:27:06 -0400 Subject: [PATCH 47/47] fix(ci): add checkout step in changes job of ci.yaml to prevent push crashes --- .github/workflows/ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8976a0169c8d..f5afddd603c1 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -112,6 +112,7 @@ jobs: outputs: packages: ${{ steps.filter.outputs.changes }} steps: + - uses: actions/checkout@v4 - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1 id: filter with: