_endpointscfg_impl.py

#!/usr/bin/python
# Copyright 2017 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
r"""External script for generating Cloud Endpoints related files.

The gen_discovery_doc subcommand takes a list of fully qualified ProtoRPC
service names and calls a cloud service which generates a discovery document in
REST or RPC style.

Example:
  endpointscfg.py gen_discovery_doc -o . -f rest postservice.GreetingsV1

The gen_client_lib subcommand takes a discovery document and calls a cloud
service to generate a client library for a target language (currently just Java)

Example:
  endpointscfg.py gen_client_lib java -o . greetings-v0.1.discovery

The get_client_lib subcommand does both of the above commands at once.

Example:
  endpointscfg.py get_client_lib java -o . postservice.GreetingsV1

The gen_api_config command outputs an .api configuration file for a service.

Example:
  endpointscfg.py gen_api_config -o . -a /path/to/app \
    --hostname myhost.appspot.com postservice.GreetingsV1
"""

from __future__ import absolute_import

import argparse
import collections
import contextlib
import logging
import os
import re
import sys
import urllib
import urllib2

import yaml
from google.appengine.ext import testbed

from . import api_config
from . import discovery_generator
from . import openapi_generator
from . import remote

# Conditional import, pylint: disable=g-import-not-at-top
try:
  import json
except ImportError:
  # If we can't find json packaged with Python import simplejson, which is
  # packaged with the SDK.
  import simplejson as json


CLIENT_LIBRARY_BASE = 'https://google-api-client-libraries.appspot.com/generate'
_VISIBLE_COMMANDS = ('get_client_lib', 'get_discovery_doc', 'get_openapi_spec')


class ServerRequestException(Exception):
  """Exception for problems with the request to a server."""

  def __init__(self, http_error):
    """Create a ServerRequestException from a given urllib2.HTTPError.

    Args:
      http_error: The HTTPError that the ServerRequestException will be
        based on.
    """
    error_details = None
    error_response = None
    if http_error.fp:
      try:
        error_response = http_error.fp.read()
        error_body = json.loads(error_response)
        error_details = ['%s: %s' % (detail['message'], detail['debug_info'])
                         for detail in error_body['error']['errors']]
      except (ValueError, TypeError, KeyError):
        pass
    if error_details:
      error_details_str = ', '.join(error_details)
      error_message = ('HTTP %s (%s) error when communicating with URL: %s.  '
                       'Details: %s' % (http_error.code, http_error.reason,
                                        http_error.filename, error_details_str))
    else:
      error_message = ('HTTP %s (%s) error when communicating with URL: %s. '
                       'Response: %s' % (http_error.code, http_error.reason,
                                         http_error.filename,
                                         error_response))
    super(ServerRequestException, self).__init__(error_message)


class _EndpointsParser(argparse.ArgumentParser):
  """Create a subclass of argparse.ArgumentParser for Endpoints."""

  def error(self, message):
    """Override superclass to support customized error message.

    Error message needs to be rewritten in order to display visible commands
    only, when invalid command is called by user. Otherwise, hidden commands
    will be displayed in stderr, which is not expected.

    Refer the following argparse python documentation for detailed method
    information:
      http://docs.python.org/2/library/argparse.html#exiting-methods

    Args:
      message: original error message that will be printed to stderr
    """
    # subcommands_quoted is the same as subcommands, except each value is
    # surrounded with double quotes. This is done to match the standard
    # output of the ArgumentParser, while hiding commands we don't want users
    # to use, as they are no longer documented and only here for legacy use.
    subcommands_quoted = ', '.join(
        [repr(command) for command in _VISIBLE_COMMANDS])
    subcommands = ', '.join(_VISIBLE_COMMANDS)
    message = re.sub(
        r'(argument {%s}: invalid choice: .*) \(choose from (.*)\)$'
        % subcommands, r'\1 (choose from %s)' % subcommands_quoted, message)
    super(_EndpointsParser, self).error(message)


def _WriteFile(output_path, name, content):
  """Write given content to a file in a given directory.

  Args:
    output_path: The directory to store the file in.
    name: The name of the file to store the content in.
    content: The content to write to the file.close

  Returns:
    The full path to the written file.
  """
  path = os.path.join(output_path, name)
  with open(path, 'wb') as f:
    f.write(content)
  return path


def GenApiConfig(service_class_names, config_string_generator=None,
                 hostname=None, application_path=None, **additional_kwargs):
  """Write an API configuration for endpoints annotated ProtoRPC services.

  Args:
    service_class_names: A list of fully qualified ProtoRPC service classes.
    config_string_generator: A generator object that produces API config strings
      using its pretty_print_config_to_json method.
    hostname: A string hostname which will be used as the default version
      hostname. If no hostname is specificied in the @endpoints.api decorator,
      this value is the fallback.
    application_path: A string with the path to the AppEngine application.

  Raises:
    TypeError: If any service classes don't inherit from remote.Service.
    messages.DefinitionNotFoundError: If a service can't be found.

  Returns:
    A map from service names to a string containing the API configuration of the
      service in JSON format.
  """
  # First, gather together all the different APIs implemented by these
  # classes.  There may be fewer APIs than service classes.  Each API is
  # uniquely identified by (name, version).  Order needs to be preserved here,
  # so APIs that were listed first are returned first.
  api_service_map = collections.OrderedDict()
  resolved_services = []

  for service_class_name in service_class_names:
    module_name, base_service_class_name = service_class_name.rsplit('.', 1)
    module = __import__(module_name, fromlist=base_service_class_name)
    service = getattr(module, base_service_class_name)
    if hasattr(service, 'get_api_classes'):
      resolved_services.extend(service.get_api_classes())
    elif (not isinstance(service, type) or
          not issubclass(service, remote.Service)):
      raise TypeError('%s is not a subclass of endpoints.remote.Service' % service_class_name)
    else:
      resolved_services.append(service)

  for resolved_service in resolved_services:
    services = api_service_map.setdefault(
        (resolved_service.api_info.name, resolved_service.api_info.api_version), [])
    services.append(resolved_service)

  # If hostname isn't specified in the API or on the command line, we'll
  # try to build it from information in app.yaml.
  app_yaml_hostname = _GetAppYamlHostname(application_path)

  service_map = collections.OrderedDict()
  config_string_generator = (
      config_string_generator or api_config.ApiConfigGenerator())
  for api_info, services in api_service_map.iteritems():
    assert services, 'An API must have at least one ProtoRPC service'
    # Only override hostname if None.  Hostname will be the same for all
    # services within an API, since it's stored in common info.
    hostname = services[0].api_info.hostname or hostname or app_yaml_hostname

    # Map each API by name-version.
    service_map['%s-%s' % api_info] = (
        config_string_generator.pretty_print_config_to_json(
            services, hostname=hostname, **additional_kwargs))

  return service_map


def _GetAppYamlHostname(application_path, open_func=open):
  """Build the hostname for this app based on the name in app.yaml.

  Args:
    application_path: A string with the path to the AppEngine application.  This
      should be the directory containing the app.yaml file.
    open_func: Function to call to open a file.  Used to override the default
      open function in unit tests.

  Returns:
    A hostname, usually in the form of "myapp.appspot.com", based on the
    application name in the app.yaml file.  If the file can't be found or
    there's a problem building the name, this will return None.
  """
  try:
    app_yaml_file = open_func(os.path.join(application_path or '.', 'app.yaml'))
    config = yaml.safe_load(app_yaml_file.read())
  except IOError:
    # Couldn't open/read app.yaml.
    return None

  application = config.get('application')
  if not application:
    return None

  if ':' in application:
    # Don't try to deal with alternate domains.
    return None

  # If there's a prefix ending in a '~', strip it.
  tilde_index = application.rfind('~')
  if tilde_index >= 0:
    application = application[tilde_index + 1:]
    if not application:
      return None

  return '%s.appspot.com' % application


def _GenDiscoveryDoc(service_class_names,
                     output_path, hostname=None,
                     application_path=None):
  """Write discovery documents generated from the service classes to file.

  Args:
    service_class_names: A list of fully qualified ProtoRPC service names.
    output_path: The directory to output the discovery docs to.
    hostname: A string hostname which will be used as the default version
      hostname. If no hostname is specificied in the @endpoints.api decorator,
      this value is the fallback. Defaults to None.
    application_path: A string containing the path to the AppEngine app.

  Returns:
    A list of discovery doc filenames.
  """
  output_files = []
  service_configs = GenApiConfig(
      service_class_names, hostname=hostname,
      config_string_generator=discovery_generator.DiscoveryGenerator(),
      application_path=application_path)
  for api_name_version, config in service_configs.iteritems():
    discovery_name = api_name_version + '.discovery'
    output_files.append(_WriteFile(output_path, discovery_name, config))

  return output_files


def _GenOpenApiSpec(service_class_names, output_path, hostname=None,
                    application_path=None, x_google_api_name=False):
  """Write openapi documents generated from the service classes to file.

  Args:
    service_class_names: A list of fully qualified ProtoRPC service names.
    output_path: The directory to which to output the OpenAPI specs.
    hostname: A string hostname which will be used as the default version
      hostname. If no hostname is specified in the @endpoints.api decorator,
      this value is the fallback. Defaults to None.
    application_path: A string containing the path to the AppEngine app.

  Returns:
    A list of OpenAPI spec filenames.
  """
  output_files = []
  service_configs = GenApiConfig(
      service_class_names, hostname=hostname,
      config_string_generator=openapi_generator.OpenApiGenerator(),
      application_path=application_path,
      x_google_api_name=x_google_api_name)
  for api_name_version, config in service_configs.iteritems():
    openapi_name = api_name_version.replace('-', '') + 'openapi.json'
    output_files.append(_WriteFile(output_path, openapi_name, config))

  return output_files


def _GenClientLib(discovery_path, language, output_path, build_system):
  """Write a client library from a discovery doc.

  Args:
    discovery_path: Path to the discovery doc used to generate the client
      library.
    language: The client library language to generate. (java)
    output_path: The directory to output the client library zip to.
    build_system: The target build system for the client library language.

  Raises:
    IOError: If reading the discovery doc fails.
    ServerRequestException: If fetching the generated client library fails.

  Returns:
    The path to the zipped client library.
  """
  with open(discovery_path) as f:
    discovery_doc = f.read()

  client_name = re.sub(r'\.discovery$', '.zip',
                       os.path.basename(discovery_path))

  return _GenClientLibFromContents(discovery_doc, language, output_path,
                                   build_system, client_name)


def _GenClientLibFromContents(discovery_doc, language, output_path,
                              build_system, client_name):
  """Write a client library from a discovery doc.

  Args:
    discovery_doc: A string, the contents of the discovery doc used to
      generate the client library.
    language: A string, the client library language to generate. (java)
    output_path: A string, the directory to output the client library zip to.
    build_system: A string, the target build system for the client language.
    client_name: A string, the filename used to save the client lib.

  Raises:
    IOError: If reading the discovery doc fails.
    ServerRequestException: If fetching the generated client library fails.

  Returns:
    The path to the zipped client library.
  """

  body = urllib.urlencode({'lang': language, 'content': discovery_doc,
                           'layout': build_system})
  request = urllib2.Request(CLIENT_LIBRARY_BASE, body)
  try:
    with contextlib.closing(urllib2.urlopen(request)) as response:
      content = response.read()
      return _WriteFile(output_path, client_name, content)
  except urllib2.HTTPError, error:
    raise ServerRequestException(error)


def _GetClientLib(service_class_names, language, output_path, build_system,
                  hostname=None, application_path=None):
  """Fetch client libraries from a cloud service.

  Args:
    service_class_names: A list of fully qualified ProtoRPC service names.
    language: The client library language to generate. (java)
    output_path: The directory to output the discovery docs to.
    build_system: The target build system for the client library language.
    hostname: A string hostname which will be used as the default version
      hostname. If no hostname is specificied in the @endpoints.api decorator,
      this value is the fallback. Defaults to None.
    application_path: A string containing the path to the AppEngine app.

  Returns:
    A list of paths to client libraries.
  """
  client_libs = []
  service_configs = GenApiConfig(
      service_class_names, hostname=hostname,
      config_string_generator=discovery_generator.DiscoveryGenerator(),
      application_path=application_path)
  for api_name_version, config in service_configs.iteritems():
    client_name = api_name_version + '.zip'
    client_libs.append(
        _GenClientLibFromContents(config, language, output_path,
                                  build_system, client_name))
  return client_libs


def _GenApiConfigCallback(args, api_func=GenApiConfig):
  """Generate an api file.

  Args:
    args: An argparse.Namespace object to extract parameters from.
    api_func: A function that generates and returns an API configuration
      for a list of services.
  """
  service_configs = api_func(args.service,
                             hostname=args.hostname,
                             application_path=args.application)

  for api_name_version, config in service_configs.iteritems():
    _WriteFile(args.output, api_name_version + '.api', config)


def _GetClientLibCallback(args, client_func=_GetClientLib):
  """Generate discovery docs and client libraries to files.

  Args:
    args: An argparse.Namespace object to extract parameters from.
    client_func: A function that generates client libraries and stores them to
      files, accepting a list of service names, a client library language,
      an output directory, a build system for the client library language, and
      a hostname.
  """
  client_paths = client_func(
      args.service, args.language, args.output, args.build_system,
      hostname=args.hostname, application_path=args.application)

  for client_path in client_paths:
    print 'API client library written to %s' % client_path


def _GenDiscoveryDocCallback(args, discovery_func=_GenDiscoveryDoc):
  """Generate discovery docs to files.

  Args:
    args: An argparse.Namespace object to extract parameters from
    discovery_func: A function that generates discovery docs and stores them to
      files, accepting a list of service names, a discovery doc format, and an
      output directory.
  """
  discovery_paths = discovery_func(args.service, args.output,
                                   hostname=args.hostname,
                                   application_path=args.application)
  for discovery_path in discovery_paths:
    print 'API discovery document written to %s' % discovery_path


def _GenOpenApiSpecCallback(args, openapi_func=_GenOpenApiSpec):
  """Generate OpenAPI (Swagger) specs to files.

  Args:
    args: An argparse.Namespace object to extract parameters from
    openapi_func: A function that generates OpenAPI specs and stores them to
      files, accepting a list of service names and an output directory.
  """
  openapi_paths = openapi_func(args.service, args.output,
                               hostname=args.hostname,
                               application_path=args.application,
                               x_google_api_name=args.x_google_api_name)
  for openapi_path in openapi_paths:
    print 'OpenAPI spec written to %s' % openapi_path


def _GenClientLibCallback(args, client_func=_GenClientLib):
  """Generate a client library to file.

  Args:
    args: An argparse.Namespace object to extract parameters from
    client_func: A function that generates client libraries and stores them to
      files, accepting a path to a discovery doc, a client library language, an
      output directory, and a build system for the client library language.
  """
  client_path = client_func(args.discovery_doc[0], args.language, args.output,
                            args.build_system)
  print 'API client library written to %s' % client_path


def MakeParser(prog):
  """Create an argument parser.

  Args:
    prog: The name of the program to use when outputting help text.

  Returns:
    An argparse.ArgumentParser built to specification.
  """

  def AddStandardOptions(parser, *args):
    """Add common endpoints options to a parser.

    Args:
      parser: The parser to add options to.
      *args: A list of option names to add. Possible names are: application,
        format, output, language, service, and discovery_doc.
    """
    if 'application' in args:
      parser.add_argument('-a', '--application', default='.',
                          help='The path to the Python App Engine App')
    if 'format' in args:
      # This used to be a valid option, allowing the user to select 'rest' or 'rpc',
      # but now 'rest' is the only valid type. The argument remains so scripts using it
      # won't break.
      parser.add_argument('-f', '--format', default='rest',
                          choices=['rest'],
                          help='The requested API protocol type (ignored)')
    if 'hostname' in args:
      help_text = ('Default application hostname, if none is specified '
                   'for API service.')
      parser.add_argument('--hostname', help=help_text)
    if 'output' in args:
      parser.add_argument('-o', '--output', default='.',
                          help='The directory to store output files')
    if 'language' in args:
      parser.add_argument('language',
                          help='The target output programming language')
    if 'service' in args:
      parser.add_argument('service', nargs='+',
                          help='Fully qualified service class name')
    if 'discovery_doc' in args:
      parser.add_argument('discovery_doc', nargs=1,
                          help='Path to the discovery document')
    if 'build_system' in args:
      parser.add_argument('-bs', '--build_system', default='default',
                          help='The target build system')

  parser = _EndpointsParser(prog=prog)
  subparsers = parser.add_subparsers(
      title='subcommands', metavar='{%s}' % ', '.join(_VISIBLE_COMMANDS))

  get_client_lib = subparsers.add_parser(
      'get_client_lib', help=('Generates discovery documents and client '
                              'libraries from service classes'))
  get_client_lib.set_defaults(callback=_GetClientLibCallback)
  AddStandardOptions(get_client_lib, 'application', 'hostname', 'output',
                     'language', 'service', 'build_system')

  get_discovery_doc = subparsers.add_parser(
      'get_discovery_doc',
      help='Generates discovery documents from service classes')
  get_discovery_doc.set_defaults(callback=_GenDiscoveryDocCallback)
  AddStandardOptions(get_discovery_doc, 'application', 'format', 'hostname',
                     'output', 'service')

  get_openapi_spec = subparsers.add_parser(
      'get_openapi_spec',
      help='Generates OpenAPI (Swagger) specs from service classes')
  get_openapi_spec.set_defaults(callback=_GenOpenApiSpecCallback)
  AddStandardOptions(get_openapi_spec, 'application', 'hostname', 'output',
                     'service')
  get_openapi_spec.add_argument('--x-google-api-name', action='store_true',
                                help="Add the 'x-google-api-name' field to the generated spec")

  # Create an alias for get_openapi_spec called get_swagger_spec to support
  # the old-style naming. This won't be a visible command, but it will still
  # function to support legacy scripts.
  get_swagger_spec = subparsers.add_parser(
      'get_swagger_spec',
      help='Generates OpenAPI (Swagger) specs from service classes')
  get_swagger_spec.set_defaults(callback=_GenOpenApiSpecCallback)
  AddStandardOptions(get_swagger_spec, 'application', 'hostname', 'output',
                     'service')

  # By removing the help attribute, the following three actions won't be
  # displayed in usage message
  gen_api_config = subparsers.add_parser('gen_api_config')
  gen_api_config.set_defaults(callback=_GenApiConfigCallback)
  AddStandardOptions(gen_api_config, 'application', 'hostname', 'output',
                     'service')

  gen_discovery_doc = subparsers.add_parser('gen_discovery_doc')
  gen_discovery_doc.set_defaults(callback=_GenDiscoveryDocCallback)
  AddStandardOptions(gen_discovery_doc, 'application', 'format', 'hostname',
                     'output', 'service')

  gen_client_lib = subparsers.add_parser('gen_client_lib')
  gen_client_lib.set_defaults(callback=_GenClientLibCallback)
  AddStandardOptions(gen_client_lib, 'output', 'language', 'discovery_doc',
                     'build_system')

  return parser


def _SetupStubs():
  tb = testbed.Testbed()
  tb.setup_env(CURRENT_VERSION_ID='1.0')
  tb.activate()
  for k, v in testbed.INIT_STUB_METHOD_NAMES.iteritems():
    # The old stub initialization code didn't support the image service at all
    # so we just ignore it here.
    if k != 'images':
      getattr(tb, v)()


def main(argv):
  logging.basicConfig()
  # silence warnings from endpoints.apiserving; they're not relevant
  # to command-line operation.
  logging.getLogger('endpoints.apiserving').setLevel(logging.ERROR)

  _SetupStubs()

  parser = MakeParser(argv[0])
  args = parser.parse_args(argv[1:])

  # Handle the common "application" argument here, since most of the handlers
  # use this.
  application_path = getattr(args, 'application', None)
  if application_path is not None:
    sys.path.insert(0, os.path.abspath(application_path))

  args.callback(args)