Skip to content

Commit 85e66df

Browse files
diegomarquezpgcf-owl-bot[bot]
diegomarquezp
and
gcf-owl-bot[bot]
authored
feat: enable hermetic library generation (#1697)
* feat: enable hermetic library generation * fix config yaml syntax * do not map runners home folder * use copyright update comittish * correct proto_path * update protoc * preserve pr_description * update gapic_generator_version to 2.41.0 * generalize path to firestore/bundle in owlbot yaml * infer image tag from config yaml * correct workflow name * update config scripts and yamls * remove old update_googleapis_committish workflow * sync config structure with that of google-cloud-java * remove quotes from config yamls * fix typo in update_generation_config.yaml * correct * quote codeowners_team in generation config * update generator version * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
1 parent d885a9c commit 85e66df

7 files changed

+347
-4
lines changed

‎.github/.OwlBot.yaml ‎.github/.OwlBot-hermetic.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -29,5 +29,5 @@ deep-copy-regex:
2929
dest: "/owl-bot-staging/$1/grpc-google-cloud-firestore-admin-$1/src"
3030
- source: "/google/firestore/admin/(v\\d)/.*-java/gapic-google-.*/src"
3131
dest: "/owl-bot-staging/$1/google-cloud-firestore-admin/src"
32-
- source: "/google/firestore/bundle/google-cloud-firestore-bundle-v1-java/proto-google-cloud-firestore-bundle-v1-java/src"
32+
- source: "/google/firestore/bundle/.*-java/proto-google-.*/src"
3333
dest: "/owl-bot-staging/v1/proto-google-cloud-firestore-bundle-v1/src"

‎.github/scripts/hermetic_library_generation.sh

+117
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
#!/bin/bash
2+
set -e
3+
# This script should be run at the root of the repository.
4+
# This script is used to, when a pull request changes the generation
5+
# configuration (generation_config.yaml by default):
6+
# 1. Find whether the last commit in this pull request contains changes to
7+
# the generation configuration and exit early if it doesn't have such a change
8+
# since the generation result would be the same.
9+
# 2. Compare generation configurations in the current branch (with which the
10+
# pull request associated) and target branch (into which the pull request is
11+
# merged);
12+
# 3. Generate changed libraries using library_generation image;
13+
# 4. Commit the changes to the pull request, if any.
14+
# 5. Edit the PR body with generated pull request description, if applicable.
15+
16+
# The following commands need to be installed before running the script:
17+
# 1. git
18+
# 2. gh
19+
# 3. docker
20+
21+
# The parameters of this script is:
22+
# 1. target_branch, the branch into which the pull request is merged.
23+
# 2. current_branch, the branch with which the pull request is associated.
24+
# 3. [optional] generation_config, the path to the generation configuration,
25+
# the default value is generation_config.yaml in the repository root.
26+
while [[ $# -gt 0 ]]; do
27+
key="$1"
28+
case "${key}" in
29+
--target_branch)
30+
target_branch="$2"
31+
shift
32+
;;
33+
--current_branch)
34+
current_branch="$2"
35+
shift
36+
;;
37+
--generation_config)
38+
generation_config="$2"
39+
shift
40+
;;
41+
*)
42+
echo "Invalid option: [$1]"
43+
exit 1
44+
;;
45+
esac
46+
shift
47+
done
48+
49+
if [ -z "${target_branch}" ]; then
50+
echo "missing required argument --target_branch"
51+
exit 1
52+
fi
53+
54+
if [ -z "${current_branch}" ]; then
55+
echo "missing required argument --current_branch"
56+
exit 1
57+
fi
58+
59+
if [ -z "${generation_config}" ]; then
60+
generation_config=generation_config.yaml
61+
echo "Using default generation config: ${generation_config}"
62+
fi
63+
64+
workspace_name="/workspace"
65+
baseline_generation_config="baseline_generation_config.yaml"
66+
message="chore: generate libraries at $(date)"
67+
68+
git checkout "${target_branch}"
69+
git checkout "${current_branch}"
70+
# if the last commit doesn't contain changes to generation configuration,
71+
# do not generate again as the result will be the same.
72+
change_of_last_commit="$(git diff-tree --no-commit-id --name-only HEAD~1..HEAD -r)"
73+
if [[ ! ("${change_of_last_commit}" == *"${generation_config}"*) ]]; then
74+
echo "The last commit doesn't contain any changes to the generation_config.yaml, skipping the whole generation process." || true
75+
exit 0
76+
fi
77+
# copy generation configuration from target branch to current branch.
78+
git show "${target_branch}":"${generation_config}" > "${baseline_generation_config}"
79+
config_diff=$(diff "${generation_config}" "${baseline_generation_config}" || true)
80+
81+
# parse image tag from the generation configuration.
82+
image_tag=$(grep "gapic_generator_version" "${generation_config}" | cut -d ':' -f 2 | xargs)
83+
84+
# run hermetic code generation docker image.
85+
docker run \
86+
--rm \
87+
-u "$(id -u):$(id -g)" \
88+
-v "$(pwd):${workspace_name}" \
89+
gcr.io/cloud-devrel-public-resources/java-library-generation:"${image_tag}" \
90+
--baseline-generation-config-path="${workspace_name}/${baseline_generation_config}" \
91+
--current-generation-config-path="${workspace_name}/${generation_config}"
92+
93+
94+
# commit the change to the pull request.
95+
if [[ $(basename $(pwd)) == "google-cloud-java" ]]; then
96+
git add java-* pom.xml gapic-libraries-bom/pom.xml versions.txt
97+
else
98+
# The image leaves intermediate folders and files it works with. Here we remove them
99+
rm -rdf output googleapis "${baseline_generation_config}"
100+
git add --all -- ':!pr_description.txt'
101+
fi
102+
changed_files=$(git diff --cached --name-only)
103+
if [[ "${changed_files}" == "" ]]; then
104+
echo "There is no generated code change with the generation config change ${config_diff}."
105+
echo "Skip committing to the pull request."
106+
exit 0
107+
fi
108+
109+
echo "Configuration diff:"
110+
echo "${config_diff}"
111+
git commit -m "${message}"
112+
git push
113+
# set pr body if pr_description.txt is generated.
114+
if [[ -f "pr_description.txt" ]]; then
115+
pr_num=$(gh pr list -s open -H "${current_branch}" -q . --json number | jq ".[] | .number")
116+
gh pr edit "${pr_num}" --body "$(cat pr_description.txt)"
117+
fi

‎.github/scripts/update_generation_config.sh

+121
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
#!/bin/bash
2+
set -e
3+
# This script should be run at the root of the repository.
4+
# This script is used to update googleapis_commitish, gapic_generator_version,
5+
# and libraries_bom_version in generation configuration at the time of running
6+
# and create a pull request.
7+
8+
# The following commands need to be installed before running the script:
9+
# 1. git
10+
# 2. gh
11+
# 3. jq
12+
13+
# Utility functions
14+
# Get the latest released version of a Maven artifact.
15+
function get_latest_released_version() {
16+
local group_id=$1
17+
local artifact_id=$2
18+
latest=$(curl -s "https://search.maven.org/solrsearch/select?q=g:${group_id}+AND+a:${artifact_id}&core=gav&rows=500&wt=json" | jq -r '.response.docs[] | select(.v | test("^[0-9]+(\\.[0-9]+)*$")) | .v' | sort -V | tail -n 1)
19+
echo "${latest}"
20+
}
21+
22+
# Update a key to a new value in the generation config.
23+
function update_config() {
24+
local key_word=$1
25+
local new_value=$2
26+
local file=$3
27+
echo "Update ${key_word} to ${new_value} in ${file}"
28+
sed -i -e "s/^${key_word}.*$/${key_word}: ${new_value}/" "${file}"
29+
}
30+
31+
# The parameters of this script is:
32+
# 1. base_branch, the base branch of the result pull request.
33+
# 2. repo, organization/repo-name, e.g., googleapis/google-cloud-java
34+
# 3. [optional] generation_config, the path to the generation configuration,
35+
# the default value is generation_config.yaml in the repository root.
36+
while [[ $# -gt 0 ]]; do
37+
key="$1"
38+
case "${key}" in
39+
--base_branch)
40+
base_branch="$2"
41+
shift
42+
;;
43+
--repo)
44+
repo="$2"
45+
shift
46+
;;
47+
--generation_config)
48+
generation_config="$2"
49+
shift
50+
;;
51+
*)
52+
echo "Invalid option: [$1]"
53+
exit 1
54+
;;
55+
esac
56+
shift
57+
done
58+
59+
if [ -z "${base_branch}" ]; then
60+
echo "missing required argument --base_branch"
61+
exit 1
62+
fi
63+
64+
if [ -z "${repo}" ]; then
65+
echo "missing required argument --repo"
66+
exit 1
67+
fi
68+
69+
if [ -z "${generation_config}" ]; then
70+
generation_config="generation_config.yaml"
71+
echo "Use default generation config: ${generation_config}"
72+
fi
73+
74+
current_branch="generate-libraries-${base_branch}"
75+
title="chore: Update generation configuration at $(date)"
76+
77+
# try to find a open pull request associated with the branch
78+
pr_num=$(gh pr list -s open -H "${current_branch}" -q . --json number | jq ".[] | .number")
79+
# create a branch if there's no open pull request associated with the
80+
# branch; otherwise checkout the pull request.
81+
if [ -z "${pr_num}" ]; then
82+
git checkout -b "${current_branch}"
83+
else
84+
gh pr checkout "${pr_num}"
85+
fi
86+
87+
mkdir tmp-googleapis
88+
# use partial clone because only commit history is needed.
89+
git clone --filter=blob:none https://github.com/googleapis/googleapis.git tmp-googleapis
90+
pushd tmp-googleapis
91+
git pull
92+
latest_commit=$(git rev-parse HEAD)
93+
popd
94+
rm -rf tmp-googleapis
95+
update_config "googleapis_commitish" "${latest_commit}" "${generation_config}"
96+
97+
# update gapic-generator-java version to the latest
98+
latest_version=$(get_latest_released_version "com.google.api" "gapic-generator-java")
99+
update_config "gapic_generator_version" "${latest_version}" "${generation_config}"
100+
101+
# update libraries-bom version to the latest
102+
latest_version=$(get_latest_released_version "com.google.cloud" "libraries-bom")
103+
update_config "libraries_bom_version" "${latest_version}" "${generation_config}"
104+
105+
git add "${generation_config}"
106+
changed_files=$(git diff --cached --name-only)
107+
if [[ "${changed_files}" == "" ]]; then
108+
echo "The latest generation config is not changed."
109+
echo "Skip committing to the pull request."
110+
exit 0
111+
fi
112+
git commit -m "${title}"
113+
if [ -z "${pr_num}" ]; then
114+
git remote add remote_repo https://cloud-java-bot:"${GH_TOKEN}@github.com/${repo}.git"
115+
git fetch -q --unshallow remote_repo
116+
git push -f remote_repo "${current_branch}"
117+
gh pr create --title "${title}" --head "${current_branch}" --body "${title}" --base "${base_branch}"
118+
else
119+
git push
120+
gh pr edit "${pr_num}" --title "${title}" --body "${title}"
121+
fi

‎.github/workflows/hermetic_library_generation.yaml

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
# Copyright 2024 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
# GitHub action job to test core java library features on
15+
# downstream client libraries before they are released.
16+
name: Hermetic library generation upon generation config change through pull requests
17+
on:
18+
pull_request:
19+
20+
jobs:
21+
library_generation:
22+
runs-on: ubuntu-latest
23+
steps:
24+
- uses: actions/checkout@v4
25+
with:
26+
fetch-depth: 0
27+
token: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }}
28+
- name: Generate changed libraries
29+
shell: bash
30+
run: |
31+
set -x
32+
[ -z "$(git config user.email)" ] && git config --global user.email "cloud-java-bot@google.com"
33+
[ -z "$(git config user.name)" ] && git config --global user.name "cloud-java-bot"
34+
bash .github/scriptes/hermetic_library_generation.sh \
35+
--target_branch ${{ github.base_ref }} \
36+
--current_branch ${{ github.head_ref }}
37+
env:
38+
GH_TOKEN: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }}

‎.github/workflows/update_generation_config.yaml

+42
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
# Copyright 2024 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
# GitHub action job to test core java library features on
15+
# downstream client libraries before they are released.
16+
name: Update generation configuration
17+
on:
18+
schedule:
19+
- cron: '0 2 * * *'
20+
workflow_dispatch:
21+
22+
jobs:
23+
update-generation-config:
24+
runs-on: ubuntu-22.04
25+
env:
26+
# the branch into which the pull request is merged
27+
base_branch: main
28+
steps:
29+
- uses: actions/checkout@v4
30+
with:
31+
token: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }}
32+
- name: Update params in generation config to latest
33+
shell: bash
34+
run: |
35+
set -x
36+
[ -z "$(git config user.email)" ] && git config --global user.email "cloud-java-bot@google.com"
37+
[ -z "$(git config user.name)" ] && git config --global user.name "cloud-java-bot"
38+
bash .github/scripts/update_generation_config.sh \
39+
--base_branch "${base_branch}"\
40+
--repo ${{ github.repository }}
41+
env:
42+
GH_TOKEN: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }}

‎README.md

+3-3
Original file line numberDiff line numberDiff line change
@@ -57,13 +57,13 @@ implementation 'com.google.cloud:google-cloud-firestore'
5757
If you are using Gradle without BOM, add this to your dependencies:
5858

5959
```Groovy
60-
implementation 'com.google.cloud:google-cloud-firestore:3.21.6'
60+
implementation 'com.google.cloud:google-cloud-firestore:3.22.0'
6161
```
6262

6363
If you are using SBT, add this to your dependencies:
6464

6565
```Scala
66-
libraryDependencies += "com.google.cloud" % "google-cloud-firestore" % "3.21.6"
66+
libraryDependencies += "com.google.cloud" % "google-cloud-firestore" % "3.22.0"
6767
```
6868
<!-- {x-version-update-end} -->
6969

@@ -222,7 +222,7 @@ Java is a registered trademark of Oracle and/or its affiliates.
222222
[kokoro-badge-link-5]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-firestore/java11.html
223223
[stability-image]: https://img.shields.io/badge/stability-stable-green
224224
[maven-version-image]: https://img.shields.io/maven-central/v/com.google.cloud/google-cloud-firestore.svg
225-
[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-firestore/3.21.6
225+
[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-firestore/3.22.0
226226
[authentication]: https://github.com/googleapis/google-cloud-java#authentication
227227
[auth-scopes]: https://developers.google.com/identity/protocols/oauth2/scopes
228228
[predefined-iam-roles]: https://cloud.google.com/iam/docs/understanding-roles#predefined_roles

‎generation_config.yaml

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
gapic_generator_version: 2.42.0
2+
googleapis_commitish: 6f289d775912966eb0cf04bda91e5e355c998d30
3+
libraries_bom_version: 26.38.0
4+
libraries:
5+
- api_shortname: firestore
6+
name_pretty: Cloud Firestore
7+
product_documentation: https://cloud.google.com/firestore
8+
client_documentation: https://cloud.google.com/java/docs/reference/google-cloud-firestore/latest/history
9+
issue_tracker: https://issuetracker.google.com/savedsearches/5337669
10+
release_level: stable
11+
language: java
12+
repo: googleapis/java-firestore
13+
repo_short: java-firestore
14+
distribution_name: com.google.cloud:google-cloud-firestore
15+
codeowner_team: '@googleapis/api-firestore @googleapis/firestore-dpe'
16+
api_id: firestore.googleapis.com
17+
library_type: GAPIC_COMBO
18+
api_description: is a fully-managed NoSQL document database for mobile, web, and server development from Firebase and Google Cloud Platform. It's backed by a multi-region replicated database that ensures once data is committed, it's durable even in the face of unexpected disasters. Not only that, but despite being a distributed database, it's also strongly consistent and offers seamless integration with other Firebase and Google Cloud Platform products, including Google Cloud Functions.
19+
transport: grpc
20+
excluded_poms: google-cloud-firestore,google-cloud-firestore-bom
21+
recommended_package: com.google.cloud.firestore
22+
GAPICs:
23+
- proto_path: google/firestore/v1
24+
- proto_path: google/firestore/admin/v1
25+
- proto_path: google/firestore/bundle

0 commit comments

Comments
 (0)