Skip to content

Commit

Permalink
Add GCE and discogapic to batch gen script (googleapis#3441)
Browse files Browse the repository at this point in the history
  • Loading branch information
@andreamlin @yihanzhen
andreamlin authored and yihanzhen committed Jul 11, 2018
1 parent 6d60d15 commit 4ee3038
Show file tree
Hide file tree
Showing 2 changed files with 77 additions and 43 deletions.
26 changes: 20 additions & 6 deletions utilities/batch_generate_apis.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,24 +14,26 @@

# Instructions:
#
# Check out the googleapis repo somewhere locally (e.g. from
# https://github.com/googleapis/googleapis):
# Check out the googleapis and discovery-artifact-manager repo somewhere locally
# (e.g. from https://github.com/googleapis/googleapis):
#
# $ git checkout https://github.com/googleapis/googleapis.git
# $ git checkout https://github.com/googleapis/discovery-artifact-manager.git
#
# Run this script:
#
# $ python utilities/batch_generate_apis.py PATH_TO_GOOGLEAPIS
# $ python utilities/batch_generate_apis.py PATH_TO_GOOGLEAPIS PATH_TO_DISCOVERY_ARTIFACT_MANAGER

import argparse
import os

import generate_api


def run(googleapis):
def run_gapic_gen(googleapis):
def generate(artman_yaml):
generate_api.run_generate_api(os.path.join(googleapis, artman_yaml))
generate_api.run_generate_api(os.path.join(googleapis, artman_yaml),
generate_api.JAVA_GAPIC)

# TODO Needs to have java_proto called instead of java_grpc
#generate('google/datastore/artman_datastore.yaml')
Expand Down Expand Up @@ -73,13 +75,25 @@ def generate(artman_yaml):
generate('google/cloud/websecurityscanner/artman_websecurityscanner_v1alpha.yaml')


def run_discogapic_gen(discovery_repo):
def generate(artman_yaml):
# Run java_discogapic task. No proto or grpc libraries are generated.
generate_api.run_generate_api(os.path.join(discovery_repo, artman_yaml),
generate_api.JAVA_DISCOGAPIC)

generate('gapic/google/compute/artman_compute.yaml')


def main():
# TODO Make the docker image the default, add --local option
parser = argparse.ArgumentParser(description='Batch generate all APIs.')
parser.add_argument('googleapis', help='The path to the googleapis repo')
parser.add_argument('discovery_repo',
help='The path to the discovery-artifact-manager repo')
args = parser.parse_args()

run(args.googleapis)
run_gapic_gen(args.googleapis)
run_discogapic_gen(args.discovery_repo)

if __name__ == '__main__':
main()
94 changes: 57 additions & 37 deletions utilities/generate_api.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,9 @@
# Instructions:
#
# Find the artman config file the describes the API you want to generate a client for.
# Specifiy the artman ARTIFACT_TYPE to generate, e.g. "java_gapic"
#
# $ python utilities/generate_api.py PATH_TO_ARTMAN_CONFIG_FILE
# $ python utilities/generate_api.py PATH_TO_ARTMAN_CONFIG_FILE ARTIFACT_TYPE

import argparse
import io
Expand All @@ -35,19 +36,32 @@
'spanner-admin-database': 'google-cloud-spanner'
}

JAVA_GAPIC="java_gapic"
JAVA_DISCOGAPIC="java_discogapic"

def run_generate_api(config_path, noisy=False):
googleapis_index = config_path.rfind('/google/')
if googleapis_index == -1:
raise ValueError('Didn\'t find /googleapis/ in config file path; need absolute path to the artman config file.')
root_dir = config_path[0:googleapis_index]
api_dir = config_path[googleapis_index+1:]
def run_generate_api(config_path, artifact_type, noisy=False):
""" Generate an API client library.
:param config_path: (str) Path to directory containing artman config file.
:param artifact_type: (str) artman target, e.g "java_gapic".
:param noisy: (bool) if console output should be verbose.
"""
api_repo_index = config_path.rfind('/google/')
if artifact_type == JAVA_DISCOGAPIC:
api_repo_index = config_path.rfind('/gapic/')
if api_repo_index == -1:
raise ValueError('Didn\'t find the API repo in config file path; need absolute path to the artman config file.')
root_dir = config_path[0:api_repo_index]
api_dir = config_path[api_repo_index+1:]

extra_options = []
if noisy:
extra_options = ['-v']

subprocess.check_call(['artman', '--config', api_dir, '--local', '--root-dir', root_dir] + extra_options + ['generate', 'java_gapic'])
subprocess.check_call(
['artman', '--config', api_dir, '--local', '--root-dir', root_dir]
+ extra_options + ['generate', artifact_type])

with io.open(config_path, encoding='UTF-8') as config_file:
artman_config_data = yaml.load(config_file, Loader=yaml.Loader)
Expand All @@ -60,38 +74,42 @@ def run_generate_api(config_path, noisy=False):
proto_dirname = 'proto-{}'.format(api_full_name)
grpc_dirname = 'grpc-{}'.format(api_full_name)
gapic_dirname = 'gapic-{}'.format(api_full_name)
proto_dir = os.path.join('artman-genfiles', 'java', proto_dirname)
grpc_dir = os.path.join('artman-genfiles', 'java', grpc_dirname)

gapic_dir = os.path.join('artman-genfiles', 'java', gapic_dirname)
if not os.path.exists(proto_dir):
raise ValueError('generated proto dir doesn\'t exist: {}'.format(proto_dir))
if not os.path.exists(grpc_dir):
raise ValueError('generated grpc dir doesn\'t exist: {}'.format(grpc_dir))
if not os.path.exists(gapic_dir):
raise ValueError('generated gapic dir doesn\'t exist: {}'.format(gapic_dir))

target_proto_dir = os.path.join('google-api-grpc', proto_dirname)
target_grpc_dir = os.path.join('google-api-grpc', grpc_dirname)
if os.path.exists(target_proto_dir):
print('{} already exists, removing & replacing it.'.format(target_proto_dir))
if os.path.exists(target_grpc_dir):
print('{} already exists, removing & replacing it.'.format(target_grpc_dir))

print('-- ignore any pathspec errors that follow')

if os.path.exists(target_proto_dir):
shutil.rmtree(target_proto_dir)
shutil.copytree(proto_dir, target_proto_dir)
os.remove(os.path.join(target_proto_dir, 'LICENSE'))
os.remove(os.path.join(target_proto_dir, 'build.gradle'))
subprocess.call(['git', 'checkout', os.path.join(target_proto_dir, 'pom.xml')])

if os.path.exists(target_grpc_dir):
shutil.rmtree(target_grpc_dir)
shutil.copytree(grpc_dir, target_grpc_dir)
os.remove(os.path.join(target_grpc_dir, 'LICENSE'))
os.remove(os.path.join(target_grpc_dir, 'build.gradle'))
subprocess.call(['git', 'checkout', os.path.join(target_grpc_dir, 'pom.xml')])
if artifact_type != JAVA_DISCOGAPIC:
proto_dir = os.path.join('artman-genfiles', 'java', proto_dirname)
grpc_dir = os.path.join('artman-genfiles', 'java', grpc_dirname)

if not os.path.exists(proto_dir):
raise ValueError('generated proto dir doesn\'t exist: {}'.format(proto_dir))
if not os.path.exists(grpc_dir):
raise ValueError('generated grpc dir doesn\'t exist: {}'.format(grpc_dir))

target_proto_dir = os.path.join('google-api-grpc', proto_dirname)
target_grpc_dir = os.path.join('google-api-grpc', grpc_dirname)
if os.path.exists(target_proto_dir):
print('{} already exists, removing & replacing it.'.format(target_proto_dir))
if os.path.exists(target_grpc_dir):
print('{} already exists, removing & replacing it.'.format(target_grpc_dir))

print('-- ignore any pathspec errors that follow')

if os.path.exists(target_proto_dir):
shutil.rmtree(target_proto_dir)
shutil.copytree(proto_dir, target_proto_dir)
os.remove(os.path.join(target_proto_dir, 'LICENSE'))
os.remove(os.path.join(target_proto_dir, 'build.gradle'))
subprocess.call(['git', 'checkout', os.path.join(target_proto_dir, 'pom.xml')])

if os.path.exists(target_grpc_dir):
shutil.rmtree(target_grpc_dir)
shutil.copytree(grpc_dir, target_grpc_dir)
os.remove(os.path.join(target_grpc_dir, 'LICENSE'))
os.remove(os.path.join(target_grpc_dir, 'build.gradle'))
subprocess.call(['git', 'checkout', os.path.join(target_grpc_dir, 'pom.xml')])

api_unversioned_name = '{}-{}'.format(org_name, api_name)
if api_name in dir_overrides:
Expand All @@ -113,11 +131,13 @@ def run_generate_api(config_path, noisy=False):
def main():
parser = argparse.ArgumentParser(description='Regenerate a single API.')
parser.add_argument('config_file', help='The artman config file for the API')
parser.add_argument('artifact_type', help='The artman artifact type',
default="java_gapic")
parser.add_argument('--quiet', action="store_true", default=False,
help='Don\'t print informational instructions')
args = parser.parse_args()

run_generate_api(args.config_file, not args.quiet)
run_generate_api(args.config_file, args.artifact_type, not args.quiet)

if __name__ == '__main__':
main()

0 comments on commit 4ee3038

Please sign in to comment.