From 37cb7cb9d96bfe8be13308a5b10060c00e1a5687 Mon Sep 17 00:00:00 2001 From: chingor13 Date: Wed, 11 Feb 2026 05:38:15 +0000 Subject: [PATCH 01/22] build: create split integration Kokoro test config --- monorepo-migration/extract_sm_keys.py | 24 ++++++++++ monorepo-migration/migrate.sh | 66 +++++++++++++++++++++++++++ 2 files changed, 90 insertions(+) create mode 100644 monorepo-migration/extract_sm_keys.py diff --git a/monorepo-migration/extract_sm_keys.py b/monorepo-migration/extract_sm_keys.py new file mode 100644 index 000000000000..0fa58eda99b0 --- /dev/null +++ b/monorepo-migration/extract_sm_keys.py @@ -0,0 +1,24 @@ +import re +import sys + +def extract_sm_keys(file_path): + try: + with open(file_path, 'r') as f: + content = f.read() + + # Regex to match key: "SECRET_MANAGER_KEYS" and its value + match = re.search(r'key:\s*"SECRET_MANAGER_KEYS"\s*value:\s*"([^"]+)"', content, re.MULTILINE) + if match: + return match.group(1).strip() + except Exception as e: + print(f"Error reading {file_path}: {e}", file=sys.stderr) + return None + +if __name__ == "__main__": + if len(sys.argv) < 2: + print("Usage: extract_sm_keys.py ", file=sys.stderr) + sys.exit(1) + + keys = extract_sm_keys(sys.argv[1]) + if keys: + print(keys) diff --git a/monorepo-migration/migrate.sh b/monorepo-migration/migrate.sh index 93b4b2c788c4..cc1c499db9b1 100755 --- a/monorepo-migration/migrate.sh +++ b/monorepo-migration/migrate.sh @@ -56,6 +56,7 @@ FIX_COPYRIGHT_SCRIPT="$TRANSFORM_SCRIPT_DIR/fix_copyright_headers.py" UPDATE_GENERATION_CONFIG_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_generation_config.py" UPDATE_OWLBOT_HERMETIC_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_owlbot_hermetic.py" TRANSFORM_OWLBOT_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_owlbot.py" +EXTRACT_SM_KEYS_SCRIPT="$TRANSFORM_SCRIPT_DIR/extract_sm_keys.py" # Track number of commits made by this script COMMIT_COUNT=0 @@ -181,6 +182,71 @@ rm -f "$SOURCE_REPO_NAME/LICENSE" rm -f "$SOURCE_REPO_NAME/java.header" rm -rf "$SOURCE_REPO_NAME/.kokoro" # rm -rf "$SOURCE_REPO_NAME/.kokoro/continuous" "$SOURCE_REPO_NAME/.kokoro/nightly" "$SOURCE_REPO_NAME/.kokoro/presubmit" + +# 6.6 Create split integration config if needed +SOURCE_INTEGRATION_CFG="$SOURCE_DIR/.kokoro/presubmit/integration.cfg" +if [ -f "$SOURCE_INTEGRATION_CFG" ]; then + echo "Creating split integration config for $SOURCE_REPO_NAME..." + SHORT_NAME="${SOURCE_REPO_NAME#java-}" + TARGET_INTEGRATION_CFG=".kokoro/presubmit/${SHORT_NAME}-integration.cfg" + + # Extract SECRET_MANAGER_KEYS from source config + SM_KEYS=$(python3 "$EXTRACT_SM_KEYS_SCRIPT" "$SOURCE_INTEGRATION_CFG") + + cat < "$TARGET_INTEGRATION_CFG" +# Format: //devtools/kokoro/config/proto/build.proto + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/java11" +} + +env_vars: { + key: "JOB_TYPE" + value: "integration-single" +} + +# TODO: remove this after we've migrated all tests and scripts +env_vars: { + key: "GCLOUD_PROJECT" + value: "cloud-java-ci-test" +} + +env_vars: { + key: "GOOGLE_CLOUD_PROJECT" + value: "cloud-java-ci-test" +} + +env_vars: { + key: "GOOGLE_APPLICATION_CREDENTIALS" + value: "secret_manager/cloud-java-ci-it-service-account" +} +EOF + + if [ -n "$SM_KEYS" ]; then + cat <> "$TARGET_INTEGRATION_CFG" + +env_vars: { + key: "SECRET_MANAGER_KEYS" + value: "$SM_KEYS" +} +EOF + fi + + cat <> "$TARGET_INTEGRATION_CFG" + +env_vars: { + key: "BUILD_SUBDIR" + value: "$SOURCE_REPO_NAME" +} +EOF + + echo "Committing split integration config..." + git add "$TARGET_INTEGRATION_CFG" + git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): create split integration config" + COMMIT_COUNT=$((COMMIT_COUNT + 1)) +fi rm -f "$SOURCE_REPO_NAME/codecov.yaml" rm -f "$SOURCE_REPO_NAME/synth.metadata" rm -f "$SOURCE_REPO_NAME/license-checks.xml" From 595bc9bfaf4d4cc1f89298844cfdce558470a3a0 Mon Sep 17 00:00:00 2001 From: chingor13 Date: Wed, 11 Feb 2026 05:53:13 +0000 Subject: [PATCH 02/22] add module to excluded_modules for partial tests --- monorepo-migration/migrate.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/monorepo-migration/migrate.sh b/monorepo-migration/migrate.sh index cc1c499db9b1..4b2a403bad23 100755 --- a/monorepo-migration/migrate.sh +++ b/monorepo-migration/migrate.sh @@ -247,6 +247,19 @@ EOF git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): create split integration config" COMMIT_COUNT=$((COMMIT_COUNT + 1)) fi + +# 6.7 Update excluded_modules in .kokoro/common.sh +COMMON_SH=".kokoro/common.sh" +if [ -f "$COMMON_SH" ]; then + echo "Updating excluded_modules in $COMMON_SH..." + # Insert the new module name before the closing parenthesis of the excluded_modules array + sed -i "/^excluded_modules=(/,/^)/ s/^)/ '$SOURCE_REPO_NAME'\n)/" "$COMMON_SH" + + echo "Committing excluded_modules update..." + git add "$COMMON_SH" + git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): add to excluded_modules in .kokoro/common.sh" + COMMIT_COUNT=$((COMMIT_COUNT + 1)) +fi rm -f "$SOURCE_REPO_NAME/codecov.yaml" rm -f "$SOURCE_REPO_NAME/synth.metadata" rm -f "$SOURCE_REPO_NAME/license-checks.xml" From 3ef8419432c47571439d704cc6d71086b48550bd Mon Sep 17 00:00:00 2001 From: chingor13 Date: Wed, 11 Feb 2026 06:09:21 +0000 Subject: [PATCH 03/22] conditionally skip tests --- monorepo-migration/migrate.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/monorepo-migration/migrate.sh b/monorepo-migration/migrate.sh index 4b2a403bad23..245cd71d8a49 100755 --- a/monorepo-migration/migrate.sh +++ b/monorepo-migration/migrate.sh @@ -429,8 +429,9 @@ done < <(find "$SOURCE_REPO_NAME" -name "pom.xml" | grep "\-bom/pom.xml" | grep # 7.11 Verify compilation echo "Verifying compilation..." -BUILD_SUBDIR="${SOURCE_REPO_NAME}" JOB_TYPE=test .kokoro/build.sh -# (cd "$SOURCE_REPO_NAME" && mvn compile -DskipTests -T 1C) +if [[ "${SKIP_TESTS:-false}" != "true" ]]; then + BUILD_SUBDIR="${SOURCE_REPO_NAME}" JOB_TYPE=test .kokoro/build.sh +fi # 7.13 Squash commits if [ "${SQUASH_COMMITS:-false}" = "true" ]; then From 9caadaea2033e580e06036415709d025179d42b8 Mon Sep 17 00:00:00 2001 From: chingor13 Date: Wed, 11 Feb 2026 06:41:53 +0000 Subject: [PATCH 04/22] prune managed dependencies --- monorepo-migration/modernize_pom.py | 75 ++++++++++++++++++++++++++++- 1 file changed, 73 insertions(+), 2 deletions(-) diff --git a/monorepo-migration/modernize_pom.py b/monorepo-migration/modernize_pom.py index bb8268c42470..6c7cbaf0ae53 100644 --- a/monorepo-migration/modernize_pom.py +++ b/monorepo-migration/modernize_pom.py @@ -15,6 +15,56 @@ import sys import re import os +import subprocess +import tempfile +import xml.etree.ElementTree as ET + +def get_managed_dependencies(pom_path): + """Runs mvn help:effective-pom and extracts non-snapshot managed dependencies.""" + managed_deps = {} + if not os.path.exists(pom_path): + return managed_deps + + with tempfile.NamedTemporaryFile(suffix='.xml', delete=True) as tmp: + try: + # We use -Doutput to write to the temp file + # We use -DskipTests for speed, and -B for batch mode + cmd = [ + 'mvn', 'help:effective-pom', + '-f', pom_path, + '-Doutput=' + tmp.name, + '-DskipTests', + '-B', + '-q' + ] + print(f"Resolving managed dependencies from {pom_path}...") + subprocess.run(cmd, check=True) + + tree = ET.parse(tmp.name) + root = tree.getroot() + + # Namespace handling for Maven POM + ns = {'mvn': 'http://maven.apache.org/POM/4.0.0'} + + # Use XPath to find managed dependencies + deps_selection = root.findall('.//mvn:dependencyManagement/mvn:dependencies/mvn:dependency', ns) + for dep in deps_selection: + gid_node = dep.find('mvn:groupId', ns) + aid_node = dep.find('mvn:artifactId', ns) + ver_node = dep.find('mvn:version', ns) + + if gid_node is not None and aid_node is not None and ver_node is not None: + group_id = gid_node.text.strip() + artifact_id = aid_node.text.strip() + version = ver_node.text.strip() + + if version and '-SNAPSHOT' not in version: + managed_deps[(group_id, artifact_id)] = version + except Exception as e: + print(f"Warning: Failed to get managed dependencies from {pom_path}: {e}") + + return managed_deps + def parse_pom_for_version(file_path): """Extracts artifactId and version from a pom.xml file.""" @@ -94,7 +144,7 @@ def get_monorepo_versions(monorepo_root='.'): versions[artifactId] = version return versions -def modernize_pom(file_path, parent_version, source_repo_name=None, parent_artifactId='google-cloud-jar-parent', relative_path='../google-cloud-jar-parent/pom.xml', monorepo_versions=None): +def modernize_pom(file_path, parent_version, source_repo_name=None, parent_artifactId='google-cloud-jar-parent', relative_path='../google-cloud-jar-parent/pom.xml', monorepo_versions=None, parent_managed_deps=None): with open(file_path, 'r') as f: lines = f.readlines() @@ -190,6 +240,22 @@ def modernize_pom(file_path, parent_version, source_repo_name=None, parent_artif if current_artifact_id == 'google-cloud-shared-dependencies': continue + # Prune if already managed by parent mit same version + if parent_managed_deps and (current_group_id, current_artifact_id) in parent_managed_deps: + managed_version = parent_managed_deps[(current_group_id, current_artifact_id)] + + # Extract current version to compare + current_version = None + for d_line in current_dependency_lines: + v_match = re.search(r'(.*?)', d_line) + if v_match: + current_version = v_match.group(1).strip() + break + + if current_version == managed_version: + continue + + # Preservation logic: # 1. Has x-version-update comment # 2. Is NOT com.google group AND has a version tag @@ -266,7 +332,12 @@ def modernize_pom(file_path, parent_version, source_repo_name=None, parent_artif source_repo = sys.argv[3] if len(sys.argv) > 3 else None parent_artifactId = sys.argv[4] if len(sys.argv) > 4 else 'google-cloud-jar-parent' relative_path = sys.argv[5] if len(sys.argv) > 5 else '../google-cloud-jar-parent/pom.xml' - modernize_pom(sys.argv[1], sys.argv[2], source_repo, parent_artifactId, relative_path, monorepo_versions) + + parent_pom_path = os.path.join(monorepo_root, 'google-cloud-jar-parent', 'pom.xml') + parent_managed_deps = get_managed_dependencies(parent_pom_path) + print(f"Loaded {len(parent_managed_deps)} managed dependencies from parent.") + + modernize_pom(sys.argv[1], sys.argv[2], source_repo, parent_artifactId, relative_path, monorepo_versions, parent_managed_deps) else: print("Usage: python3 modernize_pom.py [source_repo_name] [parent_artifactId] [relative_path]") sys.exit(1) From 4bf8432c71d088bfa9e1b26a76c91f030baa8d61 Mon Sep 17 00:00:00 2001 From: chingor13 Date: Thu, 12 Feb 2026 17:24:50 +0000 Subject: [PATCH 05/22] fix dependencies in other submodules --- monorepo-migration/migrate.sh | 18 +++++- monorepo-migration/modernize_pom.py | 85 +++++++++++++++++------------ 2 files changed, 64 insertions(+), 39 deletions(-) diff --git a/monorepo-migration/migrate.sh b/monorepo-migration/migrate.sh index 245cd71d8a49..d76a7bd2804c 100755 --- a/monorepo-migration/migrate.sh +++ b/monorepo-migration/migrate.sh @@ -148,7 +148,8 @@ fi # 2.5 Create a new feature branch for the migration -BRANCH_NAME="migrate-$SOURCE_REPO_NAME" +BRANCH_NAME_SUFFIX="${BRANCH_NAME_SUFFIX:-}" +BRANCH_NAME="migrate-${SOURCE_REPO_NAME}${BRANCH_NAME_SUFFIX}" echo "Creating feature branch: $BRANCH_NAME" if git rev-parse --verify "$BRANCH_NAME" >/dev/null 2>&1; then git branch -D "$BRANCH_NAME" @@ -405,7 +406,7 @@ fi # 7.11 Modernize root pom.xml echo "Modernizing root pom.xml..." PARENT_VERSION=$(grep -m 1 ".*{x-version-update:google-cloud-java:current}" google-cloud-jar-parent/pom.xml | sed -E 's/.*(.*)<\/version>.*/\1/') -python3 "$MODERNIZE_POM_SCRIPT" "$SOURCE_REPO_NAME/pom.xml" "$PARENT_VERSION" "$SOURCE_REPO_NAME" +python3 "$MODERNIZE_POM_SCRIPT" "$SOURCE_REPO_NAME/pom.xml" "$PARENT_VERSION" --source-repo "$SOURCE_REPO_NAME" echo "Committing root pom.xml modernization..." git add "$SOURCE_REPO_NAME/pom.xml" @@ -419,7 +420,7 @@ echo "Modernizing BOM pom.xml..." while read -r bom_pom; do echo "Modernizing BOM: $bom_pom" # BOMs should inherit from google-cloud-pom-parent - python3 "$MODERNIZE_POM_SCRIPT" "$bom_pom" "$PARENT_VERSION" "$SOURCE_REPO_NAME" "google-cloud-pom-parent" "../../google-cloud-pom-parent/pom.xml" + python3 "$MODERNIZE_POM_SCRIPT" "$bom_pom" "$PARENT_VERSION" --source-repo "$SOURCE_REPO_NAME" --parent-artifactId "google-cloud-pom-parent" --relative-path "../../google-cloud-pom-parent/pom.xml" echo "Committing BOM pom.xml modernization for $bom_pom..." git add "$bom_pom" @@ -427,6 +428,17 @@ while read -r bom_pom; do COMMIT_COUNT=$((COMMIT_COUNT + 1)) done < <(find "$SOURCE_REPO_NAME" -name "pom.xml" | grep "\-bom/pom.xml" | grep -v "samples") +# 7.12b Modernize other pom.xml files +echo "Modernizing other pom.xml files..." +while read -r other_pom; do + echo "Modernizing submodule POM: $other_pom" + # Preserve the existing parent, but update everything else + python3 "$MODERNIZE_POM_SCRIPT" "$other_pom" "$PARENT_VERSION" --source-repo "$SOURCE_REPO_NAME" --keep-parent + + echo "Committing submodule pom.xml modernization for $other_pom..." + git add "$other_pom" && git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): modernize submodule pom.xml" && COMMIT_COUNT=$((COMMIT_COUNT + 1)) || true +done < <(find "$SOURCE_REPO_NAME" -name "pom.xml" | grep -v "\-bom/pom.xml" | grep -v "samples" | grep -v "test_data") + # 7.11 Verify compilation echo "Verifying compilation..." if [[ "${SKIP_TESTS:-false}" != "true" ]]; then diff --git a/monorepo-migration/modernize_pom.py b/monorepo-migration/modernize_pom.py index 6c7cbaf0ae53..395c943910f2 100644 --- a/monorepo-migration/modernize_pom.py +++ b/monorepo-migration/modernize_pom.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import argparse import sys import re import os @@ -144,7 +145,7 @@ def get_monorepo_versions(monorepo_root='.'): versions[artifactId] = version return versions -def modernize_pom(file_path, parent_version, source_repo_name=None, parent_artifactId='google-cloud-jar-parent', relative_path='../google-cloud-jar-parent/pom.xml', monorepo_versions=None, parent_managed_deps=None): +def modernize_pom(file_path, parent_version, source_repo_name=None, parent_artifactId='google-cloud-jar-parent', relative_path='../google-cloud-jar-parent/pom.xml', monorepo_versions=None, parent_managed_deps=None, keep_parent=False): with open(file_path, 'r') as f: lines = f.readlines() @@ -188,21 +189,22 @@ def modernize_pom(file_path, parent_version, source_repo_name=None, parent_artif ) # Parent section modernization - if '' in line and not in_parent: - in_parent = True - indent = line[:line.find('<')] - new_lines.append(f"{indent}\n") - new_lines.append(f"{indent} com.google.cloud\n") - new_lines.append(f"{indent} {parent_artifactId}\n") - new_lines.append(f"{indent} {parent_version}\n") - new_lines.append(f"{indent} {relative_path}\n") - continue - if '' in line and in_parent: - in_parent = False - new_lines.append(line) - continue - if in_parent: - continue # skip original parent content + if not keep_parent: + if '' in line and not in_parent: + in_parent = True + indent = line[:line.find('<')] + new_lines.append(f"{indent}\n") + new_lines.append(f"{indent} com.google.cloud\n") + new_lines.append(f"{indent} {parent_artifactId}\n") + new_lines.append(f"{indent} {parent_version}\n") + new_lines.append(f"{indent} {relative_path}\n") + continue + if '' in line and in_parent: + in_parent = False + new_lines.append(line) + continue + if in_parent: + continue # skip original parent content # Dependency Management pruning if '' in line: @@ -320,25 +322,36 @@ def modernize_pom(file_path, parent_version, source_repo_name=None, parent_artif f.write(content) if __name__ == "__main__": - if len(sys.argv) > 2: - # Monorepo root is likely the parent of the directory containing this script - script_dir = os.path.dirname(os.path.abspath(__file__)) - monorepo_root = os.path.dirname(script_dir) - - print(f"Scanning monorepo at {monorepo_root} for versions...") - monorepo_versions = get_monorepo_versions(monorepo_root) - print(f"Found {len(monorepo_versions)} artifacts.") - - source_repo = sys.argv[3] if len(sys.argv) > 3 else None - parent_artifactId = sys.argv[4] if len(sys.argv) > 4 else 'google-cloud-jar-parent' - relative_path = sys.argv[5] if len(sys.argv) > 5 else '../google-cloud-jar-parent/pom.xml' - - parent_pom_path = os.path.join(monorepo_root, 'google-cloud-jar-parent', 'pom.xml') - parent_managed_deps = get_managed_dependencies(parent_pom_path) - print(f"Loaded {len(parent_managed_deps)} managed dependencies from parent.") + parser = argparse.ArgumentParser(description="Modernize a pom.xml file for the monorepo.") + parser.add_argument("file_path", help="Path to the pom.xml file to modernize.") + parser.add_argument("parent_version", help="Version of the parent POM.") + parser.add_argument("--source-repo", help="Name of the source repository (e.g., java-logging).") + parser.add_argument("--parent-artifactId", default="google-cloud-jar-parent", help="Artifact ID of the parent POM.") + parser.add_argument("--relative-path", default="../google-cloud-jar-parent/pom.xml", help="Relative path to the parent POM.") + parser.add_argument("--keep-parent", action="store_true", help="Keep the existing parent section.") + + args = parser.parse_args() + + # Monorepo root is likely the parent of the directory containing this script + script_dir = os.path.dirname(os.path.abspath(__file__)) + monorepo_root = os.path.dirname(script_dir) + + print(f"Scanning monorepo at {monorepo_root} for versions...") + monorepo_versions = get_monorepo_versions(monorepo_root) + print(f"Found {len(monorepo_versions)} artifacts.") + + parent_pom_path = os.path.join(monorepo_root, 'google-cloud-jar-parent', 'pom.xml') + parent_managed_deps = get_managed_dependencies(parent_pom_path) + print(f"Loaded {len(parent_managed_deps)} managed dependencies from parent.") - modernize_pom(sys.argv[1], sys.argv[2], source_repo, parent_artifactId, relative_path, monorepo_versions, parent_managed_deps) - else: - print("Usage: python3 modernize_pom.py [source_repo_name] [parent_artifactId] [relative_path]") - sys.exit(1) + modernize_pom( + args.file_path, + args.parent_version, + args.source_repo, + args.parent_artifactId, + args.relative_path, + monorepo_versions, + parent_managed_deps, + args.keep_parent + ) From 34912be30c7dbd6c987d002c13de1bc2de5fe567 Mon Sep 17 00:00:00 2001 From: chingor13 Date: Thu, 12 Feb 2026 19:02:41 +0000 Subject: [PATCH 06/22] skip managed dependencies versions check for new split repos --- monorepo-migration/migrate.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/monorepo-migration/migrate.sh b/monorepo-migration/migrate.sh index d76a7bd2804c..445fdfef8aa6 100755 --- a/monorepo-migration/migrate.sh +++ b/monorepo-migration/migrate.sh @@ -346,6 +346,19 @@ if [ -f "$SOURCE_CONFIG" ]; then COMMIT_COUNT=$((COMMIT_COUNT + 1)) fi +# 7.6b Update generation/check_non_release_please_versions.sh +CHECK_VERSIONS_SCRIPT="generation/check_non_release_please_versions.sh" +if [ -f "$CHECK_VERSIONS_SCRIPT" ]; then + echo "Updating exclusions in $CHECK_VERSIONS_SCRIPT..." + # Insert the new module name before the .github exclusion + sed -i "/\.github\*\./ i \ [[ \"\${pomFile}\" =~ .*$SOURCE_REPO_NAME.* ]] || \\\\" "$CHECK_VERSIONS_SCRIPT" + + echo "Committing $CHECK_VERSIONS_SCRIPT update..." + git add "$CHECK_VERSIONS_SCRIPT" + git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): add to exclusions in $CHECK_VERSIONS_SCRIPT" + COMMIT_COUNT=$((COMMIT_COUNT + 1)) +fi + # 7.7 Consolidate versions.txt echo "Consolidating versions.txt..." SOURCE_VERSIONS="$SOURCE_REPO_NAME/versions.txt" From f2114b037f5a1d0077201f5eeafc413d589b0985 Mon Sep 17 00:00:00 2001 From: chingor13 Date: Thu, 12 Feb 2026 21:03:44 +0000 Subject: [PATCH 07/22] keep integration configs --- monorepo-migration/extract_sm_keys.py | 24 ------------- monorepo-migration/migrate.sh | 49 +++------------------------ 2 files changed, 5 insertions(+), 68 deletions(-) delete mode 100644 monorepo-migration/extract_sm_keys.py diff --git a/monorepo-migration/extract_sm_keys.py b/monorepo-migration/extract_sm_keys.py deleted file mode 100644 index 0fa58eda99b0..000000000000 --- a/monorepo-migration/extract_sm_keys.py +++ /dev/null @@ -1,24 +0,0 @@ -import re -import sys - -def extract_sm_keys(file_path): - try: - with open(file_path, 'r') as f: - content = f.read() - - # Regex to match key: "SECRET_MANAGER_KEYS" and its value - match = re.search(r'key:\s*"SECRET_MANAGER_KEYS"\s*value:\s*"([^"]+)"', content, re.MULTILINE) - if match: - return match.group(1).strip() - except Exception as e: - print(f"Error reading {file_path}: {e}", file=sys.stderr) - return None - -if __name__ == "__main__": - if len(sys.argv) < 2: - print("Usage: extract_sm_keys.py ", file=sys.stderr) - sys.exit(1) - - keys = extract_sm_keys(sys.argv[1]) - if keys: - print(keys) diff --git a/monorepo-migration/migrate.sh b/monorepo-migration/migrate.sh index 445fdfef8aa6..b74bc982221b 100755 --- a/monorepo-migration/migrate.sh +++ b/monorepo-migration/migrate.sh @@ -56,7 +56,6 @@ FIX_COPYRIGHT_SCRIPT="$TRANSFORM_SCRIPT_DIR/fix_copyright_headers.py" UPDATE_GENERATION_CONFIG_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_generation_config.py" UPDATE_OWLBOT_HERMETIC_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_owlbot_hermetic.py" TRANSFORM_OWLBOT_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_owlbot.py" -EXTRACT_SM_KEYS_SCRIPT="$TRANSFORM_SCRIPT_DIR/extract_sm_keys.py" # Track number of commits made by this script COMMIT_COUNT=0 @@ -191,56 +190,18 @@ if [ -f "$SOURCE_INTEGRATION_CFG" ]; then SHORT_NAME="${SOURCE_REPO_NAME#java-}" TARGET_INTEGRATION_CFG=".kokoro/presubmit/${SHORT_NAME}-integration.cfg" - # Extract SECRET_MANAGER_KEYS from source config - SM_KEYS=$(python3 "$EXTRACT_SM_KEYS_SCRIPT" "$SOURCE_INTEGRATION_CFG") + cp "$SOURCE_INTEGRATION_CFG" "$TARGET_INTEGRATION_CFG" - cat < "$TARGET_INTEGRATION_CFG" -# Format: //devtools/kokoro/config/proto/build.proto - -# Configure the docker image for kokoro-trampoline. -env_vars: { - key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/java11" -} - -env_vars: { - key: "JOB_TYPE" - value: "integration-single" -} - -# TODO: remove this after we've migrated all tests and scripts -env_vars: { - key: "GCLOUD_PROJECT" - value: "cloud-java-ci-test" -} - -env_vars: { - key: "GOOGLE_CLOUD_PROJECT" - value: "cloud-java-ci-test" -} - -env_vars: { - key: "GOOGLE_APPLICATION_CREDENTIALS" - value: "secret_manager/cloud-java-ci-it-service-account" -} -EOF - - if [ -n "$SM_KEYS" ]; then - cat <> "$TARGET_INTEGRATION_CFG" - -env_vars: { - key: "SECRET_MANAGER_KEYS" - value: "$SM_KEYS" -} -EOF - fi + # Replace JOB_TYPE with integration-single. Robustly handle multi-line or single-line. + perl -0777 -i -pe 's/(key:\s*"JOB_TYPE"\s*value:\s*")[^"]*(")/$1integration-single$2/g' "$TARGET_INTEGRATION_CFG" + # Append BUILD_SUBDIR cat <> "$TARGET_INTEGRATION_CFG" - env_vars: { key: "BUILD_SUBDIR" value: "$SOURCE_REPO_NAME" } + EOF echo "Committing split integration config..." From d2f742e5ad87c9ca114d694fc2920b234063a2ca Mon Sep 17 00:00:00 2001 From: chingor13 Date: Tue, 17 Feb 2026 23:29:24 +0000 Subject: [PATCH 08/22] clean dependencies --- monorepo-migration/modernize_pom.py | 148 ++++++++++++++-------------- 1 file changed, 74 insertions(+), 74 deletions(-) diff --git a/monorepo-migration/modernize_pom.py b/monorepo-migration/modernize_pom.py index 395c943910f2..310468e13011 100644 --- a/monorepo-migration/modernize_pom.py +++ b/monorepo-migration/modernize_pom.py @@ -216,93 +216,93 @@ def modernize_pom(file_path, parent_version, source_repo_name=None, parent_artif new_lines.append(line) continue - if in_dep_mgmt: - if '' in line: - in_dependencies = True - new_lines.append(line) - continue - if '' in line: - in_dependencies = False - new_lines.append(line) + # if in_dep_mgmt: + if '' in line: + in_dependencies = True + new_lines.append(line) + continue + if '' in line: + in_dependencies = False + new_lines.append(line) + continue + + if in_dependencies: + if '' in line: + in_dependency = True + current_dependency_lines = [line] + should_preserve = False + current_group_id = None + current_artifact_id = None + has_version = False continue + if '' in line: + in_dependency = False + current_dependency_lines.append(line) - if in_dependencies: - if '' in line: - in_dependency = True - current_dependency_lines = [line] - should_preserve = False - current_group_id = None - current_artifact_id = None - has_version = False + if current_artifact_id == 'google-cloud-shared-dependencies': continue - if '' in line: - in_dependency = False - current_dependency_lines.append(line) - if current_artifact_id == 'google-cloud-shared-dependencies': + # Prune if already managed by parent mit same version + if in_dep_mgmt and parent_managed_deps and (current_group_id, current_artifact_id) in parent_managed_deps: + managed_version = parent_managed_deps[(current_group_id, current_artifact_id)] + + # Extract current version to compare + current_version = None + for d_line in current_dependency_lines: + v_match = re.search(r'(.*?)', d_line) + if v_match: + current_version = v_match.group(1).strip() + break + + if current_version == managed_version: continue - # Prune if already managed by parent mit same version - if parent_managed_deps and (current_group_id, current_artifact_id) in parent_managed_deps: - managed_version = parent_managed_deps[(current_group_id, current_artifact_id)] - - # Extract current version to compare - current_version = None - for d_line in current_dependency_lines: - v_match = re.search(r'(.*?)', d_line) - if v_match: - current_version = v_match.group(1).strip() - break - - if current_version == managed_version: - continue - - # Preservation logic: - # 1. Has x-version-update comment - # 2. Is NOT com.google group AND has a version tag - # 3. Is com.google.cloud group AND artifactId starts with google-cloud- AND has a version tag - is_external = current_group_id and not current_group_id.startswith('com.google') - is_google_cloud_lib = current_group_id == 'com.google.cloud' and current_artifact_id and current_artifact_id.startswith('google-cloud-') - - if should_preserve or (is_external and has_version) or (is_google_cloud_lib and has_version): - new_lines.extend(current_dependency_lines) - continue + # Preservation logic: + # 1. Has x-version-update comment + # 2. Is NOT com.google group AND has a version tag + # 3. Is com.google.cloud group AND artifactId starts with google-cloud- AND has a version tag + is_external = current_group_id and not current_group_id.startswith('com.google') + is_google_cloud_lib = current_group_id == 'com.google.cloud' and current_artifact_id and current_artifact_id.startswith('google-cloud-') + + if not in_dep_mgmt or should_preserve or (is_external and has_version) or (is_google_cloud_lib and has_version): + new_lines.extend(current_dependency_lines) + continue - if in_dependency: - if '' in line: - match = re.search(r'(.*?)', line) - if match: - current_group_id = match.group(1).strip() - if '' in line: - match = re.search(r'(.*?)', line) - if match: - current_artifact_id = match.group(1).strip() + if in_dependency: + if '' in line: + match = re.search(r'(.*?)', line) + if match: + current_group_id = match.group(1).strip() + if '' in line: + match = re.search(r'(.*?)', line) + if match: + current_artifact_id = match.group(1).strip() + if '' in line: + has_version = True + + if monorepo_versions and current_artifact_id and current_artifact_id in monorepo_versions: + new_version = monorepo_versions[current_artifact_id] + indent = line[:line.find('<')] if '' in line: - has_version = True - - if monorepo_versions and current_artifact_id and current_artifact_id in monorepo_versions: - new_version = monorepo_versions[current_artifact_id] - indent = line[:line.find('<')] - if '' in line: - marker_artifact = current_artifact_id.replace('-bom', '') - current_dependency_lines.append(f"{indent}{new_version}\n") - should_preserve = True - continue - - if current_artifact_id and current_artifact_id.startswith('google-api-services-'): - should_preserve = True - - current_dependency_lines.append(line) - if '{x-version-update:' in line: + marker_artifact = current_artifact_id.replace('-bom', '') + current_dependency_lines.append(f"{indent}{new_version}\n") should_preserve = True - continue + continue - # Prune comments and extra whitespace in depMgmt for a cleaner result - if not line.strip(): - new_lines.append(line) + if current_artifact_id and current_artifact_id.startswith('google-api-services-'): + should_preserve = True + + current_dependency_lines.append(line) + if '{x-version-update:' in line: + should_preserve = True continue + # Prune comments and extra whitespace in depMgmt for a cleaner result + if not line.strip(): + new_lines.append(line) + continue + # Reporting section removal if '' in line: in_reporting = True From 24639f437c885e599afb1ed625f845dbd6767a97 Mon Sep 17 00:00:00 2001 From: chingor13 Date: Tue, 17 Feb 2026 23:43:18 +0000 Subject: [PATCH 09/22] fix build subdir spacing --- monorepo-migration/migrate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monorepo-migration/migrate.sh b/monorepo-migration/migrate.sh index b74bc982221b..178b753710b9 100755 --- a/monorepo-migration/migrate.sh +++ b/monorepo-migration/migrate.sh @@ -197,11 +197,11 @@ if [ -f "$SOURCE_INTEGRATION_CFG" ]; then # Append BUILD_SUBDIR cat <> "$TARGET_INTEGRATION_CFG" + env_vars: { key: "BUILD_SUBDIR" value: "$SOURCE_REPO_NAME" } - EOF echo "Committing split integration config..." From 377a9c03af9049f708e6b10a9264dc7d5db21410 Mon Sep 17 00:00:00 2001 From: chingor13 Date: Thu, 19 Feb 2026 18:26:23 +0000 Subject: [PATCH 10/22] fix commit message when squashing commits --- monorepo-migration/migrate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monorepo-migration/migrate.sh b/monorepo-migration/migrate.sh index 178b753710b9..74958fdfbeb8 100755 --- a/monorepo-migration/migrate.sh +++ b/monorepo-migration/migrate.sh @@ -436,7 +436,7 @@ if [ "${SQUASH_COMMITS:-false}" = "true" ]; then # then commit --amend adds those staged changes to C1. git reset --soft "HEAD~$((COMMIT_COUNT - 1))" - git commit --amend --no-edit --no-gpg-sign + git commit --amend --no-edit --no-gpg-sign -m "chore($SOURCE_REPO_NAME): migrate $SOURCE_REPO_NAME into monorepo" echo "Squashed everything into one commit." fi fi From 8844b8fc8c201c35224428ad65a6a6d6705f475a Mon Sep 17 00:00:00 2001 From: chingor13 Date: Thu, 19 Feb 2026 18:36:36 +0000 Subject: [PATCH 11/22] rename customNative profile --- monorepo-migration/modernize_pom.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/monorepo-migration/modernize_pom.py b/monorepo-migration/modernize_pom.py index 310468e13011..0c84f8b89543 100644 --- a/monorepo-migration/modernize_pom.py +++ b/monorepo-migration/modernize_pom.py @@ -188,6 +188,10 @@ def modernize_pom(file_path, parent_version, source_repo_name=None, parent_artif line ) + # Profile renaming + if 'customNative' in line: + line = line.replace('customNative', 'native') + # Parent section modernization if not keep_parent: if '' in line and not in_parent: From 6052f5b3eb7091b40042cc6fa46c19baa92251ff Mon Sep 17 00:00:00 2001 From: chingor13 Date: Thu, 19 Feb 2026 18:54:18 +0000 Subject: [PATCH 12/22] create split graalvm config --- monorepo-migration/migrate.sh | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/monorepo-migration/migrate.sh b/monorepo-migration/migrate.sh index 74958fdfbeb8..3cadb14bb1d5 100755 --- a/monorepo-migration/migrate.sh +++ b/monorepo-migration/migrate.sh @@ -210,6 +210,33 @@ EOF COMMIT_COUNT=$((COMMIT_COUNT + 1)) fi +# 6.6b Create split GraalVM config if needed +SOURCE_GRAALVM_CFG="$SOURCE_DIR/.kokoro/presubmit/graalvm-native-a.cfg" +if [ -f "$SOURCE_GRAALVM_CFG" ]; then + echo "Creating split GraalVM config for $SOURCE_REPO_NAME..." + SHORT_NAME="${SOURCE_REPO_NAME#java-}" + TARGET_GRAALVM_CFG=".kokoro/presubmit/${SHORT_NAME}-graalvm-native-presubmit.cfg" + + cp "$SOURCE_GRAALVM_CFG" "$TARGET_GRAALVM_CFG" + + # Replace JOB_TYPE with graalvm-single. Robustly handle multi-line or single-line. + perl -0777 -i -pe 's/(key:\s*"JOB_TYPE"\s*value:\s*")[^"]*(")/$1graalvm-single$2/g' "$TARGET_GRAALVM_CFG" + + # Append BUILD_SUBDIR + cat <> "$TARGET_GRAALVM_CFG" + +env_vars: { + key: "BUILD_SUBDIR" + value: "$SOURCE_REPO_NAME" +} +EOF + + echo "Committing split GraalVM config..." + git add "$TARGET_GRAALVM_CFG" + git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): create split GraalVM config" + COMMIT_COUNT=$((COMMIT_COUNT + 1)) +fi + # 6.7 Update excluded_modules in .kokoro/common.sh COMMON_SH=".kokoro/common.sh" if [ -f "$COMMON_SH" ]; then From 7e54affe542a6dd26ce46ca55e6c49e2bc677568 Mon Sep 17 00:00:00 2001 From: chingor13 Date: Thu, 19 Feb 2026 19:29:07 +0000 Subject: [PATCH 13/22] conditionally add integration test args --- monorepo-migration/migrate.sh | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/monorepo-migration/migrate.sh b/monorepo-migration/migrate.sh index 3cadb14bb1d5..b09e9ab52294 100755 --- a/monorepo-migration/migrate.sh +++ b/monorepo-migration/migrate.sh @@ -204,6 +204,16 @@ env_vars: { } EOF + if [ -n "$INTEGRATION_TEST_ARGS" ]; then + cat <> "$TARGET_INTEGRATION_CFG" + +env_vars: { + key: "INTEGRATION_TEST_ARGS" + value: "$INTEGRATION_TEST_ARGS" +} +EOF + fi + echo "Committing split integration config..." git add "$TARGET_INTEGRATION_CFG" git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): create split integration config" @@ -231,6 +241,16 @@ env_vars: { } EOF + if [ -n "$INTEGRATION_TEST_ARGS" ]; then + cat <> "$TARGET_GRAALVM_CFG" + +env_vars: { + key: "INTEGRATION_TEST_ARGS" + value: "$INTEGRATION_TEST_ARGS" +} +EOF + fi + echo "Committing split GraalVM config..." git add "$TARGET_GRAALVM_CFG" git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): create split GraalVM config" From 525d84c1f4f662801382f689766114b9ff18a80f Mon Sep 17 00:00:00 2001 From: chingor13 Date: Thu, 19 Feb 2026 21:39:40 +0000 Subject: [PATCH 14/22] accept EXTRA_KOKORO_ENVS --- monorepo-migration/migrate.sh | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/monorepo-migration/migrate.sh b/monorepo-migration/migrate.sh index b09e9ab52294..62d8bdf48519 100755 --- a/monorepo-migration/migrate.sh +++ b/monorepo-migration/migrate.sh @@ -214,6 +214,23 @@ env_vars: { EOF fi + if [ -n "$EXTRA_KOKORO_ENVS" ]; then + OLD_IFS="$IFS" + IFS=',' + for env in $EXTRA_KOKORO_ENVS; do + key="${env%%=*}" + value="${env#*=}" + cat <> "$TARGET_INTEGRATION_CFG" + +env_vars: { + key: "$key" + value: "$value" +} +EOF + done + IFS="$OLD_IFS" + fi + echo "Committing split integration config..." git add "$TARGET_INTEGRATION_CFG" git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): create split integration config" @@ -251,6 +268,23 @@ env_vars: { EOF fi + if [ -n "$EXTRA_KOKORO_ENVS" ]; then + OLD_IFS="$IFS" + IFS=',' + for env in $EXTRA_KOKORO_ENVS; do + key="${env%%=*}" + value="${env#*=}" + cat <> "$TARGET_GRAALVM_CFG" + +env_vars: { + key: "$key" + value: "$value" +} +EOF + done + IFS="$OLD_IFS" + fi + echo "Committing split GraalVM config..." git add "$TARGET_GRAALVM_CFG" git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): create split GraalVM config" From cd30111b8dfcecf081478a6595a181dcd6cd6451 Mon Sep 17 00:00:00 2001 From: chingor13 Date: Thu, 19 Feb 2026 22:29:58 +0000 Subject: [PATCH 15/22] skip gapic-libraries-bom dependency management, manage all intra monorepo dependency versions --- monorepo-migration/modernize_pom.py | 38 ++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/monorepo-migration/modernize_pom.py b/monorepo-migration/modernize_pom.py index 0c84f8b89543..0efb0f1c6af1 100644 --- a/monorepo-migration/modernize_pom.py +++ b/monorepo-migration/modernize_pom.py @@ -241,6 +241,31 @@ def modernize_pom(file_path, parent_version, source_repo_name=None, parent_artif continue if '' in line: in_dependency = False + + # Monorepo version alignment and annotation + if monorepo_versions and current_artifact_id and current_artifact_id in monorepo_versions: + new_version = monorepo_versions[current_artifact_id] + marker_artifact = current_artifact_id.replace('-bom', '') + annotation = f"" + version_line_content = f"{new_version}{annotation}\n" + + if has_version: + # Replace existing version line + for i, d_line in enumerate(current_dependency_lines): + if '' in d_line: + indent = d_line[:d_line.find('<')] + current_dependency_lines[i] = f"{indent}{version_line_content}" + break + else: + # Add missing version line before + indent = " " # Default indent + if len(current_dependency_lines) > 0: + first_line = current_dependency_lines[0] + indent = first_line[:first_line.find('<')] + " " + current_dependency_lines.append(f"{indent}{version_line_content}") + + should_preserve = True + current_dependency_lines.append(line) if current_artifact_id == 'google-cloud-shared-dependencies': @@ -269,6 +294,10 @@ def modernize_pom(file_path, parent_version, source_repo_name=None, parent_artif is_external = current_group_id and not current_group_id.startswith('com.google') is_google_cloud_lib = current_group_id == 'com.google.cloud' and current_artifact_id and current_artifact_id.startswith('google-cloud-') + # Skip gapic-libraries-bom in dependencyManagement + if in_dep_mgmt and current_artifact_id == 'gapic-libraries-bom': + continue + if not in_dep_mgmt or should_preserve or (is_external and has_version) or (is_google_cloud_lib and has_version): new_lines.extend(current_dependency_lines) continue @@ -285,15 +314,6 @@ def modernize_pom(file_path, parent_version, source_repo_name=None, parent_artif if '' in line: has_version = True - if monorepo_versions and current_artifact_id and current_artifact_id in monorepo_versions: - new_version = monorepo_versions[current_artifact_id] - indent = line[:line.find('<')] - if '' in line: - marker_artifact = current_artifact_id.replace('-bom', '') - current_dependency_lines.append(f"{indent}{new_version}\n") - should_preserve = True - continue - if current_artifact_id and current_artifact_id.startswith('google-api-services-'): should_preserve = True From d9bdaaa17f551faa3e629c480fa3faa21584dd97 Mon Sep 17 00:00:00 2001 From: chingor13 Date: Fri, 20 Feb 2026 16:47:31 +0000 Subject: [PATCH 16/22] skip managing boms for artifacts managed in monorepo --- monorepo-migration/modernize_pom.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/monorepo-migration/modernize_pom.py b/monorepo-migration/modernize_pom.py index 0efb0f1c6af1..6f085fa5d429 100644 --- a/monorepo-migration/modernize_pom.py +++ b/monorepo-migration/modernize_pom.py @@ -298,6 +298,10 @@ def modernize_pom(file_path, parent_version, source_repo_name=None, parent_artif if in_dep_mgmt and current_artifact_id == 'gapic-libraries-bom': continue + # Skip boms for artifacts managed in the monorepo as their individual versions will be updated via x-version-update + if in_dep_mgmt and current_artifact_id.endswith('-bom') and current_artifact_id in monorepo_versions: + continue + if not in_dep_mgmt or should_preserve or (is_external and has_version) or (is_google_cloud_lib and has_version): new_lines.extend(current_dependency_lines) continue From 7bbd4261c1dd8a66ccc8a79ce80720a64a637692 Mon Sep 17 00:00:00 2001 From: chingor13 Date: Mon, 23 Feb 2026 19:04:59 +0000 Subject: [PATCH 17/22] update .repo-metadata repo fields --- monorepo-migration/migrate.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/monorepo-migration/migrate.sh b/monorepo-migration/migrate.sh index 62d8bdf48519..fea58bbd8d22 100755 --- a/monorepo-migration/migrate.sh +++ b/monorepo-migration/migrate.sh @@ -29,6 +29,7 @@ check_command() { check_command git check_command python3 check_command mvn +check_command jq # Configuration MONOREPO_URL="https://github.com/googleapis/google-cloud-java" @@ -303,6 +304,19 @@ if [ -f "$COMMON_SH" ]; then git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): add to excluded_modules in .kokoro/common.sh" COMMIT_COUNT=$((COMMIT_COUNT + 1)) fi + +# 6.8 Update .repo-metadata.json if it exists +REPO_METADATA="$SOURCE_REPO_NAME/.repo-metadata.json" +if [ -f "$REPO_METADATA" ]; then + echo "Updating $REPO_METADATA..." + # Update "repo" to googleapis/google-cloud-java and "repo_short" to google-cloud-java + jq '.repo = "googleapis/google-cloud-java" | .repo_short = "google-cloud-java"' "$REPO_METADATA" > "${REPO_METADATA}.tmp" && mv "${REPO_METADATA}.tmp" "$REPO_METADATA" + + echo "Committing $REPO_METADATA update..." + git add "$REPO_METADATA" + git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): update .repo-metadata.json" + COMMIT_COUNT=$((COMMIT_COUNT + 1)) +fi rm -f "$SOURCE_REPO_NAME/codecov.yaml" rm -f "$SOURCE_REPO_NAME/synth.metadata" rm -f "$SOURCE_REPO_NAME/license-checks.xml" From 8217b8ea387489b59d094c79237366bb929892d7 Mon Sep 17 00:00:00 2001 From: chingor13 Date: Tue, 24 Feb 2026 19:06:07 +0000 Subject: [PATCH 18/22] move google-api-services dependency management --- monorepo-migration/migrate.sh | 31 +++++- monorepo-migration/modernize_pom.py | 62 +++++++++++- .../update_dependency_management.py | 96 +++++++++++++++++++ 3 files changed, 183 insertions(+), 6 deletions(-) create mode 100644 monorepo-migration/update_dependency_management.py diff --git a/monorepo-migration/migrate.sh b/monorepo-migration/migrate.sh index fea58bbd8d22..85d77499bf72 100755 --- a/monorepo-migration/migrate.sh +++ b/monorepo-migration/migrate.sh @@ -57,6 +57,7 @@ FIX_COPYRIGHT_SCRIPT="$TRANSFORM_SCRIPT_DIR/fix_copyright_headers.py" UPDATE_GENERATION_CONFIG_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_generation_config.py" UPDATE_OWLBOT_HERMETIC_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_owlbot_hermetic.py" TRANSFORM_OWLBOT_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_owlbot.py" +UPDATE_DEP_MGMT_SCRIPT="$TRANSFORM_SCRIPT_DIR/update_dependency_management.py" # Track number of commits made by this script COMMIT_COUNT=0 @@ -472,10 +473,34 @@ fi # git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): update copyright headers to 2026 Google LLC" # COMMIT_COUNT=$((COMMIT_COUNT + 1)) +# 7.10 Modernize pom.xml files and extract google-api-services dependencies +modernize_and_extract() { + local pom_file="$1" + shift + local cmd_output + cmd_output=$(python3 "$MODERNIZE_POM_SCRIPT" "$pom_file" "$@" 2>&1) + echo "$cmd_output" | grep -v "EXTRACT_DEP:" || true + + # Process extracted dependencies + echo "$cmd_output" | grep "EXTRACT_DEP:" | while read -r line; do + dep_info="${line#EXTRACT_DEP:}" + IFS=':' read -r gid aid ver <<< "$dep_info" + echo "Extracting $gid:$aid:$ver to parent POM..." + python3 "$UPDATE_DEP_MGMT_SCRIPT" "google-cloud-jar-parent/pom.xml" "$gid" "$aid" "$ver" + + # Commit parent POM change if anything changed + if git diff --name-only | grep -q "google-cloud-jar-parent/pom.xml"; then + git add "google-cloud-jar-parent/pom.xml" + git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): manage $aid in parent POM" + COMMIT_COUNT=$((COMMIT_COUNT + 1)) + fi + done +} + # 7.11 Modernize root pom.xml echo "Modernizing root pom.xml..." PARENT_VERSION=$(grep -m 1 ".*{x-version-update:google-cloud-java:current}" google-cloud-jar-parent/pom.xml | sed -E 's/.*(.*)<\/version>.*/\1/') -python3 "$MODERNIZE_POM_SCRIPT" "$SOURCE_REPO_NAME/pom.xml" "$PARENT_VERSION" --source-repo "$SOURCE_REPO_NAME" +modernize_and_extract "$SOURCE_REPO_NAME/pom.xml" "$PARENT_VERSION" --source-repo "$SOURCE_REPO_NAME" echo "Committing root pom.xml modernization..." git add "$SOURCE_REPO_NAME/pom.xml" @@ -489,7 +514,7 @@ echo "Modernizing BOM pom.xml..." while read -r bom_pom; do echo "Modernizing BOM: $bom_pom" # BOMs should inherit from google-cloud-pom-parent - python3 "$MODERNIZE_POM_SCRIPT" "$bom_pom" "$PARENT_VERSION" --source-repo "$SOURCE_REPO_NAME" --parent-artifactId "google-cloud-pom-parent" --relative-path "../../google-cloud-pom-parent/pom.xml" + modernize_and_extract "$bom_pom" "$PARENT_VERSION" --source-repo "$SOURCE_REPO_NAME" --parent-artifactId "google-cloud-pom-parent" --relative-path "../../google-cloud-pom-parent/pom.xml" echo "Committing BOM pom.xml modernization for $bom_pom..." git add "$bom_pom" @@ -502,7 +527,7 @@ echo "Modernizing other pom.xml files..." while read -r other_pom; do echo "Modernizing submodule POM: $other_pom" # Preserve the existing parent, but update everything else - python3 "$MODERNIZE_POM_SCRIPT" "$other_pom" "$PARENT_VERSION" --source-repo "$SOURCE_REPO_NAME" --keep-parent + modernize_and_extract "$other_pom" "$PARENT_VERSION" --source-repo "$SOURCE_REPO_NAME" --keep-parent echo "Committing submodule pom.xml modernization for $other_pom..." git add "$other_pom" && git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): modernize submodule pom.xml" && COMMIT_COUNT=$((COMMIT_COUNT + 1)) || true diff --git a/monorepo-migration/modernize_pom.py b/monorepo-migration/modernize_pom.py index 6f085fa5d429..3e1485825854 100644 --- a/monorepo-migration/modernize_pom.py +++ b/monorepo-migration/modernize_pom.py @@ -67,6 +67,32 @@ def get_managed_dependencies(pom_path): return managed_deps +def get_pom_properties(pom_path): + """Extracts all properties from the section of a pom.xml.""" + properties = {} + if not os.path.exists(pom_path): + return properties + + try: + # We'll use a simple regex-based parser to avoid complex namespace issues with ET + # and to handle property names that might not be valid XML tags if not careful + with open(pom_path, 'r') as f: + content = f.read() + + props_match = re.search(r'(.*?)', content, re.DOTALL) + if props_match: + props_content = props_match.group(1) + # Find all tag pairs inside properties + for match in re.finditer(r'<([^/ >]+)>([^<]*)', props_content): + tag = match.group(1) + value = match.group(2).strip() + properties[tag] = value + except Exception as e: + print(f"Warning: Failed to get properties from {pom_path}: {e}") + + return properties + + def parse_pom_for_version(file_path): """Extracts artifactId and version from a pom.xml file.""" artifact_id = None @@ -150,6 +176,10 @@ def modernize_pom(file_path, parent_version, source_repo_name=None, parent_artif lines = f.readlines() new_lines = [] + + # Pre-parse properties for resolution + pom_properties = get_pom_properties(file_path) + in_parent = False in_dep_mgmt = False in_dependencies = False @@ -238,10 +268,17 @@ def modernize_pom(file_path, parent_version, source_repo_name=None, parent_artif current_group_id = None current_artifact_id = None has_version = False + extracted_dep = None continue if '' in line: in_dependency = False + if extracted_dep: + # Remove version tag from current_dependency_lines if it was extracted + # We do this before further checks + current_dependency_lines = [l for l in current_dependency_lines if '' not in l] + has_version = False # It's now managed + should_preserve = True # We want to keep the dependency entry # Monorepo version alignment and annotation if monorepo_versions and current_artifact_id and current_artifact_id in monorepo_versions: new_version = monorepo_versions[current_artifact_id] @@ -317,9 +354,28 @@ def modernize_pom(file_path, parent_version, source_repo_name=None, parent_artif current_artifact_id = match.group(1).strip() if '' in line: has_version = True - - if current_artifact_id and current_artifact_id.startswith('google-api-services-'): - should_preserve = True + match = re.search(r'(.*?)', line) + if match: + current_version = match.group(1).strip() + + # Resolve version if it's a property + resolved_version = current_version + if current_version.startswith("${") and current_version.endswith("}"): + prop_name = current_version[2:-1] + if prop_name in pom_properties: + resolved_version = pom_properties[prop_name] + + if current_artifact_id and current_artifact_id.startswith('google-api-services-'): + if not parent_managed_deps or (current_group_id, current_artifact_id) not in parent_managed_deps: + # Only extract if not already managed + extracted_dep = (current_group_id, current_artifact_id, resolved_version) + print(f"EXTRACT_DEP:{current_group_id}:{current_artifact_id}:{resolved_version}") + else: + # Even if already managed, we should remove the local version + extracted_dep = (current_group_id, current_artifact_id, resolved_version) + + # if current_artifact_id and current_artifact_id.startswith('google-api-services-'): + # should_preserve = True current_dependency_lines.append(line) if '{x-version-update:' in line: diff --git a/monorepo-migration/update_dependency_management.py b/monorepo-migration/update_dependency_management.py new file mode 100644 index 000000000000..e353a550b4eb --- /dev/null +++ b/monorepo-migration/update_dependency_management.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import os +import re + +def update_dependency_management(pom_path, group_id, artifact_id, version): + with open(pom_path, 'r') as f: + content = f.read() + + # Find + dm_match = re.search(r'.*?', content, re.DOTALL) + if not dm_match: + print(f"Error: section not found in {pom_path}") + sys.exit(1) + + dm_content = dm_match.group(0) + + # Find inside + deps_match = re.search(r'(.*?)', dm_content, re.DOTALL) + if not deps_match: + print(f"Error: section not found inside in {pom_path}") + sys.exit(1) + + inner_deps_content = deps_match.group(1) + + # Check if dependency already exists + dep_pattern = fr'{re.escape(artifact_id)}' + if dep_pattern in inner_deps_content: + # Check if version matches + if fr'{re.escape(version)}' in inner_deps_content: + print(f"Dependency {group_id}:{artifact_id}:{version} already exists in {pom_path}") + return + else: + # Update version? Requirement says "move it", but if it exists with different version, maybe we should update it? + # For now, let's just append if it's not EXACTLY same. + # Actually, usually there should be only one. + # Let's replace if artifactId matches but version is different. + + new_inner_deps = re.sub( + fr'(\s*{re.escape(group_id)}\s*{re.escape(artifact_id)}\s*).*?(.*?)', + fr'\g<1>{version}\g<2>', + inner_deps_content, + flags=re.DOTALL + ) + if new_inner_deps != inner_deps_content: + print(f"Updated {group_id}:{artifact_id} to version {version} in {pom_path}") + new_dm_content = dm_content.replace(inner_deps_content, new_inner_deps) + new_content = content.replace(dm_content, new_dm_content) + with open(pom_path, 'w') as f: + f.write(new_content) + return + + # If not found or not updated, append it + new_dep = f""" + {group_id} + {artifact_id} + {version} + +""" + # Find the last and insert after it, or just before + last_dep_end = inner_deps_content.rfind('') + if last_dep_end != -1: + # Check if there is anything after last that is not just whitespace + insert_pos = last_dep_end + len('') + # Ensure we insert before + new_inner_deps = inner_deps_content[:insert_pos] + "\n" + new_dep + inner_deps_content[insert_pos:] + else: + # No dependencies found, just insert at beginning + new_inner_deps = "\n" + new_dep + inner_deps_content + + new_dm_content = dm_content.replace(inner_deps_content, new_inner_deps) + new_content = content.replace(dm_content, new_dm_content) + + with open(pom_path, 'w') as f: + f.write(new_content) + print(f"Added {group_id}:{artifact_id}:{version} to {pom_path}") + +if __name__ == "__main__": + if len(sys.argv) != 5: + print("Usage: update_dependency_management.py ") + sys.exit(1) + update_dependency_management(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]) From a46f85a1c84797e9b5c6db65e9043f3fd902c325 Mon Sep 17 00:00:00 2001 From: chingor13 Date: Tue, 24 Feb 2026 20:30:37 +0000 Subject: [PATCH 19/22] ensure annotation-api in maven-dependency-plugin configuration --- monorepo-migration/modernize_pom.py | 68 +++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/monorepo-migration/modernize_pom.py b/monorepo-migration/modernize_pom.py index 3e1485825854..86f83d53030f 100644 --- a/monorepo-migration/modernize_pom.py +++ b/monorepo-migration/modernize_pom.py @@ -185,6 +185,13 @@ def modernize_pom(file_path, parent_version, source_repo_name=None, parent_artif in_dependencies = False in_dependency = False in_reporting = False + in_plugins = False + in_plugin = False + in_configuration = False + in_ignored_deps = False + current_plugin_artifactId = None + has_javax_annotation = False + ignored_deps_lines = [] current_dependency_lines = [] should_preserve = False current_group_id = None @@ -387,6 +394,67 @@ def modernize_pom(file_path, parent_version, source_repo_name=None, parent_artif new_lines.append(line) continue + # Plugin management (for maven-dependency-plugin configuration) + if '' in line: + in_plugins = True + if '' in line: + in_plugins = False + + if in_plugins: + if '' in line: + in_plugin = True + current_plugin_artifactId = None + has_javax_annotation = False + if '' in line: + in_plugin = False + + if in_plugin: + if '' in line and not in_configuration: + match = re.search(r'(.*?)', line) + if match: + current_plugin_artifactId = match.group(1).strip() + + if current_plugin_artifactId == 'maven-dependency-plugin': + if '' in line: + in_configuration = True + if '' in line: + in_configuration = False + + if in_configuration: + if '' in line: + in_ignored_deps = True + ignored_deps_lines = [line] + continue + + if in_ignored_deps: + if 'javax.annotation:javax.annotation-api' in line: + has_javax_annotation = True + + if '' in line: + if not has_javax_annotation: + # Try to find a good indentation + indent = " " # Default + if len(ignored_deps_lines) > 1: + last_line = ignored_deps_lines[-1] + match = re.search(r'^(\s+)', last_line) + if match: + indent = match.group(1) + elif len(ignored_deps_lines) == 1: + # Only the start tag + match = re.search(r'^(\s+)', ignored_deps_lines[0]) + if match: + indent = match.group(1) + " " + + ignored_deps_lines.append(f"{indent}javax.annotation:javax.annotation-api\n") + + new_lines.extend(ignored_deps_lines) + new_lines.append(line) + in_ignored_deps = False + continue + else: + ignored_deps_lines.append(line) + continue + # Reporting section removal if '' in line: in_reporting = True From 4ff3cf763b9e46e57d51828d09c73ddad7193c23 Mon Sep 17 00:00:00 2001 From: chingor13 Date: Tue, 24 Feb 2026 20:49:34 +0000 Subject: [PATCH 20/22] add TEST_WITH_EXISTING env which allows easy testing with existing kokoro job --- monorepo-migration/migrate.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/monorepo-migration/migrate.sh b/monorepo-migration/migrate.sh index 85d77499bf72..91cb62649bbd 100755 --- a/monorepo-migration/migrate.sh +++ b/monorepo-migration/migrate.sh @@ -233,6 +233,12 @@ EOF IFS="$OLD_IFS" fi + if [ "${TEST_WITH_EXISTING:-false}" = "true" ]; then + echo "Copying to logging-integration.cfg for testing..." + cp "$TARGET_INTEGRATION_CFG" ".kokoro/presubmit/logging-integration.cfg" + git add ".kokoro/presubmit/logging-integration.cfg" + fi + echo "Committing split integration config..." git add "$TARGET_INTEGRATION_CFG" git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): create split integration config" @@ -287,6 +293,12 @@ EOF IFS="$OLD_IFS" fi + if [ "${TEST_WITH_EXISTING:-false}" = "true" ]; then + echo "Copying to logging-graalvm-native-presubmit.cfg for testing..." + cp "$TARGET_GRAALVM_CFG" ".kokoro/presubmit/logging-graalvm-native-presubmit.cfg" + git add ".kokoro/presubmit/logging-graalvm-native-presubmit.cfg" + fi + echo "Committing split GraalVM config..." git add "$TARGET_GRAALVM_CFG" git commit -n --no-gpg-sign -m "chore($SOURCE_REPO_NAME): create split GraalVM config" From 7e00fb1b095b160dad0df3973fbec9a2c97c8997 Mon Sep 17 00:00:00 2001 From: chingor13 Date: Tue, 24 Feb 2026 21:17:36 +0000 Subject: [PATCH 21/22] preserve appengine version management --- monorepo-migration/modernize_pom.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monorepo-migration/modernize_pom.py b/monorepo-migration/modernize_pom.py index 86f83d53030f..18213553b7e8 100644 --- a/monorepo-migration/modernize_pom.py +++ b/monorepo-migration/modernize_pom.py @@ -381,8 +381,8 @@ def modernize_pom(file_path, parent_version, source_repo_name=None, parent_artif # Even if already managed, we should remove the local version extracted_dep = (current_group_id, current_artifact_id, resolved_version) - # if current_artifact_id and current_artifact_id.startswith('google-api-services-'): - # should_preserve = True + if current_artifact_id and current_artifact_id == 'appengine-api-1.0-sdk': + should_preserve = True current_dependency_lines.append(line) if '{x-version-update:' in line: From 5621824cf8a87ac0fdfb58241e92cbc2cfa62eb4 Mon Sep 17 00:00:00 2001 From: chingor13 Date: Wed, 25 Feb 2026 17:55:00 +0000 Subject: [PATCH 22/22] consolidate native test profiles --- monorepo-migration/modernize_pom.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/monorepo-migration/modernize_pom.py b/monorepo-migration/modernize_pom.py index 18213553b7e8..897032a80691 100644 --- a/monorepo-migration/modernize_pom.py +++ b/monorepo-migration/modernize_pom.py @@ -228,6 +228,8 @@ def modernize_pom(file_path, parent_version, source_repo_name=None, parent_artif # Profile renaming if 'customNative' in line: line = line.replace('customNative', 'native') + if 'native-test' in line: + line = line.replace('native-test', 'native') # Parent section modernization if not keep_parent: