From c7af2ea4958b5e184a0239608a767dc3760177cf Mon Sep 17 00:00:00 2001 From: ed lane Date: Tue, 21 Aug 2018 11:52:09 -0600 Subject: [PATCH 01/29] First commit --- README.md | 54 +++++++++ __init__.py | 0 cliapi.py | 194 +++++++++++++++++++++++++++++++++ cliapi_lib.py | 84 ++++++++++++++ design.md | 61 +++++++++++ providers/__init__.py.deleteme | 0 providers/azure.py | 66 +++++++++++ providers/test.py | 63 +++++++++++ what_cloud.py | 15 +++ 9 files changed, 537 insertions(+) create mode 100644 README.md create mode 100644 __init__.py create mode 100755 cliapi.py create mode 100644 cliapi_lib.py create mode 100644 design.md create mode 100644 providers/__init__.py.deleteme create mode 100644 providers/azure.py create mode 100644 providers/test.py create mode 100644 what_cloud.py diff --git a/README.md b/README.md new file mode 100644 index 0000000..acec6f2 --- /dev/null +++ b/README.md @@ -0,0 +1,54 @@ +# GUIDE FOR DEVELOPERS: +# cliapi +**cliapi** ( pronounced _calliope_ ): A Python framework for creating Unix, "getopt()" style CLI scripts +composed from a set of APIs. + +...at its core, **cliapi** is _a data driven, state machine for dynamically generating CLIs_. + +**cliapi Features**: +- entire CLI is generated by applying python decorators together with a "dictionary with backing API store". +This decorator is applied for each API supported by a "plugin provider". + +- multiple data queries within the same CLI will result in "at most" a single API call. + +- consistent "command line behavior" is enforced by the cliapi framework across all plugin providers. +Several common commands are automatically inherited by all the providers: + - "**--list-providers**" Lists all the available plugin providers. + - "**--provider=**" specify a particular provider plugin for all CLI commands. + - "**--all**" Lists all data returned by all APIs supported by a single provider. + - "**--list-apis**" Lists all the APIs available by a particular plugin provider. + - "**--query=**" extracts specified API data using pythonic dictionary syntax. +- other behaviors enforced by the cliapi framework: + - error handling and help is also consistent across all plugin providers. + - multiple queries in same command will return a JSON list in "query order" by default + - single query will return a single JSON element. + +**Key cliapi developer concepts:** +- **provider**: an associated set of APIs accessed from the CLI in a particular context or cloud + environment e.g. "azure", "gce", "ec2", ... but can essentially be any mix of apis + +- **api**: an API for interfacing to remote or local services. If the interface can be implemented as +a python function with an (_*args, **kwargs_) style calling convention AND it returns a JSON serializable +object, THEN it can easily become a configurable CLI query. With the cliapi decorator, both required and +optional parameters are expressible through the CLI. Help is also handled by the cliapi framework. + +- **scoops**: a dictionary which maps a CLI query name to a particular API data scoop. "scoops" are really just +"_sandboxed_ python _eval()_" statements. This allows scoops to be expressed as Python slices, +comprehensions, ect. It also allows restricted ad-hoc queries on the command line +when a specific value is desired but is not currently supported as an option in the CLI. + +- **fetchers**: a dictionary which maps from a particular API name to the actual python function which +provides the _backing store_ for the contents of the top-level API dictionary. + + +**Default Cliapi Directory structure** +``` +├── cliapi_lib.py +├── readme.md +├── cliapi.py +├── __init__.py +├── providers +│   ├── azure.py +│   └── test.py +└── what_cloud.py +``` diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cliapi.py b/cliapi.py new file mode 100755 index 0000000..35c83d2 --- /dev/null +++ b/cliapi.py @@ -0,0 +1,194 @@ +#!/usr/bin/python3 + +import os +if 'PYCHARM_DEBUG_ME' in os.environ: + import pydevd + # connect pycharm debugger with: + # bash> PYCHARM_DEBUGME= cliapi .... + pydevd.settrace('localhost', port=8282, stdoutToServer=True, + stderrToServer=True) + +import sys +import importlib +import pkgutil +import getopt +import json + +import providers + +prefix = providers.__name__ + '.' + +# this will scan the "providers" directory for all candidate provider modules... +all_providers = [mod for _, mod, _ in pkgutil.iter_modules(providers.__path__, prefix)] +valid_providers = {} + +# import all valid providers found in the "providers" directory... +for pro in all_providers: + module_name = pro.split('.')[-1] + try: + vp = importlib.import_module(pro) + valid_providers[module_name] = vp + except Exception as e: + pass + +# TODO: does "xml_out" really need to be implemented in a modern Devops world??? +cli_options = ['help', + 'provider=', + 'list-providers', + "list-apis", + 'query=', 'all', + 'pycharm-debug', + ] + # 'xml-out'] + + +def _cli_parse(argv): + cli_tree = dict() + for arg in argv: + kv = arg.split('=') + if len(kv)>1: + cli_tree[kv[0]] = kv[1] + else: + cli_tree[kv[0]] = None + return cli_tree + + +def _print_help(provider): + # Help for Common CLI options... + help = { + 'help': 'help for this CLI command', + 'provider=': 'specify name of provider module', + 'list-providers': 'list all available providers', + 'xml-out': 'output in xml format', + 'list-apis': 'list all available APIs for specified provider', + 'query=': 'specify a python dictionary style query command', + 'all': 'output all API results for specified API options or defaults', + } + + vpp = valid_providers[provider].provider + # update help with options from this provider... + help.update(vp.help) + indent2_format = ' --{:<15} {:<15}' + print("usage: {} [display option#1]... [API option#1]... [CLI option]".format(sys.argv[0])) + print("\n***[ {} ]*** provider Display options:".format(provider)) + for key in vpp.scoops.keys(): + # list display options for this provider... + print (indent2_format.format(key, vpp.help.get(key, ''))) + print("\n***[ {} ]*** provider API config options:".format(provider)) + for k, v in vpp.options.items(): + # list API configuration options for this provider... + if v.startswith('$'): + opt = v[1:] + print (indent2_format.format(opt + '=', vpp.help.get(opt, ''))) + print("\nCommon CLI options:") + for key in cli_options: + # list of common CLI options supported by this module, across all providers + print (indent2_format.format(key, vpp.help.get(key, ''))) + + exit(0) + + +def _sandbox_eval(vpp, lookup): + return eval('vpp' + lookup, + {'__builtins__': None}, + {'vpp': vpp}) + + +def main(): + # must do our own parsing here since we don't know the + # valid options until we establish the provider + # -- getopt() does not allow this usage... + ct = _cli_parse(sys.argv[1:]) + + if '--list-providers' in ct: + print('valid providers =\n', json.dumps(list(valid_providers.keys()), indent=2)) + exit(0) + + if '--provider' in ct: + # using supplied provider... + provider = ct['--provider'] + else: + # using the first valid provider... + provider = list(valid_providers.keys())[0] + + cmd_dict = {} + vpp = valid_providers[provider].provider + + all_opts = list(vpp.scoops.keys()) + options = [] + for k, v in vpp.options.items(): # process args... + if v.startswith('$'): + options.append(v[1:] + '=') + all_opts += options + + # add "cli_options" to allowed CLI options... + all_opts += cli_options + + try: + # now we have enough info to use getopt() for cli parsing... + optlist, arg = getopt.gnu_getopt(sys.argv[1:], '', all_opts) + except getopt.GetoptError as e: + # error -- print help and exit + print(e.msg) + cmd_dict['help'] = None + else: + # build a dictionary of the actual supplied CLI options... + for opt in optlist: + cmd_dict[opt[0][2:]] = opt[1] + + vpp.template.update(cmd_dict) + + # use defaults if option NOT provided on CLI... + for default in options: + ds = default.split('=') + try: + if not cmd_dict.get(ds[0]): + # override with default + cmd_dict[ds[0]] = ds[1] + except: + # no option to override or none specified -- ignore this... + pass + + if 'help' in cmd_dict: + _print_help(provider) + exit(0) + + if 'list-apis' in cmd_dict: + print(json.dumps(list(vpp.fetchers.keys()), indent=2)) + exit(0) + + if 'all' in cmd_dict: + # return a composite dictionary of all API calls... + for fetch in vpp.fetchers: + # fetch all the APIs + scoop = "['" + fetch + "']" + # must populate dictionary with a fake query... + query = _sandbox_eval(vpp, scoop) + # return all the API results as a dictionary + data = vpp + + else: + data = [] + for key, value in cmd_dict.items(): + # return all specified "data scoops"... + if key in vpp.scoops: + query = vpp.scoops[key] + try: + data.append(_sandbox_eval(vpp, query)) + except KeyError as e: + # error -- print help and exit + print('required option --{} missing'.format(e.args[0])) + _print_help(provider) + # cmd_dict['help'] = None + # exit(-1) + elif key == 'query': + data.append(_sandbox_eval(vpp, query)) + if len(data) == 1: + # a single value was requested so no list is required... + data = data[0] + + print(json.dumps(data, indent=2)) + + +if __name__ == '__main__': + main() diff --git a/cliapi_lib.py b/cliapi_lib.py new file mode 100644 index 0000000..b07a0c9 --- /dev/null +++ b/cliapi_lib.py @@ -0,0 +1,84 @@ +import inspect +import importlib +from string import Template + +class Provider(dict): + def __init__(self, fetchers={}): + self.fetchers = {} + self.scoops = {} + self.options = {} + self.help = {} + self.template = {} + + return super().__init__({}) + + def __getitem__(self, item): + + try: + result = super().__getitem__(item) + except KeyError: + # restrict python's eval() function -- No builtins, only "item" is accessible + # api = eval('[' + item.replace(']', '],') + ']', + # {'__builtins__': None}, + # {'item': item})[0][0] + fetcher = self.fetchers[item].split('.') + provider = importlib.import_module('.'.join(fetcher[0:2])) + + super().__setitem__(item, eval('provider.' + + Template(fetcher[2]).substitute(self.template))) + # restrict python's eval() functon -- No builtins, only "self" is accessible + result = self.get(item) + # result = eval("self" + item, {'__builtins__': None}, {'self': self}) + return result + + +def cliapi_decor(prov, api_alias=None, scoops={}, options={}, help={}): + def decorate_it(func): + if api_alias is None: + alias = func.__name__ + else: + alias = api_alias + # prov.scoops=alias + prov.scoops.update(scoops) + prov.options.update(options) + prov.help.update(help) + argspec = inspect.getargspec(func) + if argspec.defaults is None: + default_count = 0 + else: + default_count = len(argspec.defaults) + last_arg = len(argspec.args) - default_count + argspec_string = '(' + for i, arg in enumerate(argspec.args): + if i < last_arg: + # substitute CLI options... + arg = prov.options.get(arg, arg) + argspec_string += '\'' + arg + '\', ' + else: + default = argspec.defaults[i - last_arg] + # substitute CLI options... + # op_default = prov.options.get(arg, default) + op_default = prov.options.get(arg, default) + if op_default.startswith('$'): + prov.template[op_default[1:]] = default + argspec_string += arg + '=\'' + op_default + '\', ' + + prov.fetchers.update({alias : + func.__module__ + '.' + + func.__name__ + + argspec_string + ')' + }) + + def wrap_it(*args, **kwargs): + if len(args) == 0: + a = () + else: + a = args + if len(kwargs) == 0: + b = {} + else: + b = kwargs + return func(*a, **b) + return wrap_it + + return decorate_it diff --git a/design.md b/design.md new file mode 100644 index 0000000..57cb55d --- /dev/null +++ b/design.md @@ -0,0 +1,61 @@ +#Design Considerations: +**Task:** port azuremetadata.pl (perl) to python + +**Requirements:** +(Or rather, the Methodology used to Generate requirements): +1. compare azuremetadata CLI with the other metadata CLI modules for other cloud providers (ec2 and gce) +2. Identify and interview current users of metadata modules (Sean) + +**Observations:** +- The 3 CLI metadata modules appear to have been written at 3 different times by 3 different +authors. +- Subtle and not so subtle differences exist in the CLI behaviors and output formats. +This requires developers to implement separate handlers and be aware of the idiosyncrasies +of each implementation. +- The GCE and EC2 modules both use getopt() and are largely identical API except for +returned results. +- Azure is expectedly weird with different API, results, and configuration parameters +required (...and because it was written in Perl) +- It looks like the list of available and supported CLI query options for each cloud provider +has been added over time. +To allow a new item to be queried from the CLI, the cloud module must be revised first +in order to access it. +- There is a practice of adding other SUSE-cloud-provider specific functions along +with "metadata" rather than breaking them into separate CLIs. e.g "cloud-service" and +"billing-tag",... + +**Proposition 1:** + +Implement azuremetadata entirely using Salt's metadata grains module +- extend grains module to support azure cloud. (~40 LOC) +- add "cloud-service" or other SUSE required functionality (~20 LOC) +- replace other parts of Enceladus by Salt when duplicate functionality exists. +**(I am informed that "advocating this proposition would be my last official act")** + +**alternative #2 to Proposition 1:** +- Allow salt grains to be called directly from CLI. This would allow salt grains to be called +as an executable module via **_salt-call_** (new functionality ~10 LOC) +- support other SUSE specific functions as salt +This suffers from many of the objections of alternative #1 and would break consumers of +the old interface. + +**Proposition 2:** + +- Port "as is" and create yet another slightly inconsistent CLI, but coded in Python instead. + +**alternative #2 to Proposition 2:** + +- Choose an existing cloud CLI (gce or ec2) and emulate that one as much as possible that same +behavior for **azuremetadata.py** +- Try to reconcile all the behavior differences in the CLIs and bring them back into compatibility (at +least for one instance in time before the inevitable "code drift" takes hold) + + +**Proposition 3:** + +- Create a common, extensible framework which will generate a CLI given a set of APIs described as +python functions. +- Enable backward compatibility by supporting all existing CLI options. +- Enable forward flexibility by adding a query-language extension with allows recently +added or previously unanticipated metadata API values to be queried. + diff --git a/providers/__init__.py.deleteme b/providers/__init__.py.deleteme new file mode 100644 index 0000000..e69de29 diff --git a/providers/azure.py b/providers/azure.py new file mode 100644 index 0000000..c48a1b3 --- /dev/null +++ b/providers/azure.py @@ -0,0 +1,66 @@ +from cliapi_lib import Provider, cliapi_decor +from what_cloud import determine_provider + +import json +import os +import uuid +import urllib.request, urllib.error, urllib.parse +import xml.etree.ElementTree as ET + +#----- raises an error on import if not running in a "proper" cloud... +try: + cp = determine_provider() + if cp != 'azure': + raise Exception("not a valid provider") +except Exception as e: + raise e + +provider = Provider() + +scoops = { + 'instance-name': "['meta_data']['compute']['name']", + 'mac': "['meta_data']['network']['interface'][0]['macAddress']", + 'location': "meta_data['compute']['location']", + 'external-ip': "['meta_data']['network']['interface'][0]" + "['ipv4']['ipAddress'][0]['publicIpAddress']", + 'internal-ip': "['meta_data']['network']['interface'][0]" + "['ipv4']['ipAddress'][0]['privateIpAddress']", + 'cloud-service': "['get_cloud_service']" +} + +help = { + 'instance-name': 'name of instance', + 'location': 'region location', + 'mac': 'the MAC address for this interface', + 'api_version': 'azure metadata api version, default = \'2017-08-01\'', + 'cloud-service': 'what' +} + +options = dict(api_version='$api_version') + +@cliapi_decor(provider, api_alias='meta_data', scoops=scoops, help=help, options=options) +def get_meta_data_azure(api_version='2017-08-01'): + HEADERS = {'Metadata': 'true'} + IP = '169.254.169.254' + url = 'http://{0}/metadata/instance?api-version={1}'.format(IP, api_version) + req = urllib.request.Request(url, headers=HEADERS) + content = urllib.request.urlopen(req).read() + result = json.loads(content.decode('utf-8')) + return result + + +@cliapi_decor(provider, api_alias='cloud-service') +def get_cloud_service(): + # tree = ET.parse('/home/lane/Downloads/SharedConfig.xml') + tree = ET.parse('/var/lib/waagent/SharedConfig.xml') + root = tree.getroot() + cloud_service = root.find('Deployment').find('Service').get('name') + '.cloudapp.net' + return cloud_service + + +@cliapi_decor(provider, api_alias='tag') +def read_billing_guid(device='/dev/sda'): + fd = os.open(device, os.O_RDONLY) + os.lseek(fd, 65536, os.SEEK_SET) + uuid_string = str(uuid.UUID(bytes_le=os.read(fd, 16))) + return uuid_string diff --git a/providers/test.py b/providers/test.py new file mode 100644 index 0000000..a1b6def --- /dev/null +++ b/providers/test.py @@ -0,0 +1,63 @@ + +from cliapi_lib import Provider, cliapi_decor + +provider = Provider() + +scoops = { + 'test': "['test']", + 'meta-data': "['meta_data']", +} + +help = { + 'smurf': 'required, what is smurf name?', + 'lorax': 'who is the lorax?', + 'dweebville': 'where is dweebville?', +} + +options = dict(arg1='$smurf', key1='$dufus', key2='$dweebville',) + +@cliapi_decor(provider, api_alias='test', scoops=scoops, help=help, options=options) +def dork(arg1, key1='foo', key2='bar'): + return [arg1, key1, key2] + + +scoops = { + 'instance-name': "['meta_data']['compute']['name']", + 'mac': "['meta_data']['network']['interface'][0]['macAddress']", + 'location': "['meta_data']['compute']['location']", + 'external-ip': "['meta_data']['network']['interface'][0]" + "['ipv4']['ipAddress'][0]['publicIpAddress']", + 'internal-ip': "['meta_data']['network']['interface'][0]" + "['ipv4']['ipAddress'][0]['privateIpAddress']", +} + +help = { + 'instance-name': 'name of instance', + 'location': 'region location', + 'mac': 'the MAC address for this interface', +} + +@cliapi_decor(provider, api_alias='meta_data', scoops=scoops, help=help) +def get_meta_data_mock(): + return {"compute": {"location": "westus", + "name": "ed-sle12sp3byos", "offer": "SLES-BYOS", + "osType": "Linux", "placementGroupId": "", + "platformFaultDomain": "0", + "platformUpdateDomain": "0", "publisher": "SUSE", + "resourceGroupName": "ed_lane", "sku": "12-SP3", + "subscriptionId": "ce73a2b0-d2e7-4ff6-b987-b32d6908de4e", + "tags": "", "version": "2018.02.21", "vmId": "ce01dc32-6d0a-40bd-9534-a3509f768a53", + "vmSize": "Standard_B1ms"}, + "network": {"interface": [{"ipv4": {"ipAddress": + [{"privateIpAddress": "172.16.3.8", + "publicIpAddress": "104.42.34.116"}], + "subnet": [{"address": "172.16.3.0", "prefix": "24"}]}, + "ipv6": {"ipAddress": []}, + "macAddress": "000D3A3AE8A5"}]}} + + +options = dict(api_version='$api_version') + +@cliapi_decor(provider, api_alias='some_stuff', scoops=scoops, help=help, options=options) +def get_stuff(api_version='2017-08-01'): + return api_version diff --git a/what_cloud.py b/what_cloud.py new file mode 100644 index 0000000..6ae194a --- /dev/null +++ b/what_cloud.py @@ -0,0 +1,15 @@ +from os import popen + + +def determine_provider(): + result = popen('/usr/sbin/dmidecode -t system').read() + output = result.lower() + if 'amazon' in output: + provider = 'ec2' + elif 'microsoft' in output: + provider = 'azure' + elif 'google' in output: + provider = 'gce' + else: + raise Exception('Provider not found.') + return provider From c22d1c2c45c13fe7da322df920a0ae561d265841 Mon Sep 17 00:00:00 2001 From: ed lane Date: Tue, 21 Aug 2018 13:38:46 -0600 Subject: [PATCH 02/29] delete spurious file "__init__.py.deleteme" --- providers/__init__.py.deleteme | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 providers/__init__.py.deleteme diff --git a/providers/__init__.py.deleteme b/providers/__init__.py.deleteme deleted file mode 100644 index e69de29..0000000 From ae01e74e6caac95e9de89752fb8ab22847f47b20 Mon Sep 17 00:00:00 2001 From: ed lane Date: Tue, 21 Aug 2018 17:02:04 -0600 Subject: [PATCH 03/29] directory restructuring --- __init__.py => cliapi/__init__.py | 0 cliapi.py => cliapi/cliapi.py | 0 cliapi_lib.py => cliapi/cliapi_lib.py | 0 {providers => cliapi/providers}/azure.py | 0 {providers => cliapi/providers}/test.py | 0 what_cloud.py => cliapi/what_cloud.py | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename __init__.py => cliapi/__init__.py (100%) rename cliapi.py => cliapi/cliapi.py (100%) rename cliapi_lib.py => cliapi/cliapi_lib.py (100%) rename {providers => cliapi/providers}/azure.py (100%) rename {providers => cliapi/providers}/test.py (100%) rename what_cloud.py => cliapi/what_cloud.py (100%) diff --git a/__init__.py b/cliapi/__init__.py similarity index 100% rename from __init__.py rename to cliapi/__init__.py diff --git a/cliapi.py b/cliapi/cliapi.py similarity index 100% rename from cliapi.py rename to cliapi/cliapi.py diff --git a/cliapi_lib.py b/cliapi/cliapi_lib.py similarity index 100% rename from cliapi_lib.py rename to cliapi/cliapi_lib.py diff --git a/providers/azure.py b/cliapi/providers/azure.py similarity index 100% rename from providers/azure.py rename to cliapi/providers/azure.py diff --git a/providers/test.py b/cliapi/providers/test.py similarity index 100% rename from providers/test.py rename to cliapi/providers/test.py diff --git a/what_cloud.py b/cliapi/what_cloud.py similarity index 100% rename from what_cloud.py rename to cliapi/what_cloud.py From d0e04262c3b778869251a7f7dea361684293378e Mon Sep 17 00:00:00 2001 From: ed lane Date: Tue, 21 Aug 2018 15:12:52 -0600 Subject: [PATCH 04/29] Create LICENSE adding an Apache 2.0 license which allows for a subsequent fork to a different licences -- Namely GPL-3.0 --- LICENSE | 201 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 201 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. From a15262ba724d1972089c3ea36ce482f1354f313b Mon Sep 17 00:00:00 2001 From: ed lane Date: Wed, 22 Aug 2018 16:36:19 -0600 Subject: [PATCH 05/29] first pass at getting "setup.py" to install --- cliapi/cliapi.py | 2 +- cliapi/providers/azure.py | 2 +- cliapi/providers/test.py | 2 +- setup.py | 200 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 203 insertions(+), 3 deletions(-) create mode 100644 setup.py diff --git a/cliapi/cliapi.py b/cliapi/cliapi.py index 35c83d2..70266fa 100755 --- a/cliapi/cliapi.py +++ b/cliapi/cliapi.py @@ -14,7 +14,7 @@ import getopt import json -import providers +import cliapi.providers as providers prefix = providers.__name__ + '.' diff --git a/cliapi/providers/azure.py b/cliapi/providers/azure.py index c48a1b3..a453623 100644 --- a/cliapi/providers/azure.py +++ b/cliapi/providers/azure.py @@ -1,4 +1,4 @@ -from cliapi_lib import Provider, cliapi_decor +from cliapi.cliapi_lib import Provider, cliapi_decor from what_cloud import determine_provider import json diff --git a/cliapi/providers/test.py b/cliapi/providers/test.py index a1b6def..5be8693 100644 --- a/cliapi/providers/test.py +++ b/cliapi/providers/test.py @@ -1,5 +1,5 @@ -from cliapi_lib import Provider, cliapi_decor +from cliapi.cliapi_lib import Provider, cliapi_decor provider = Provider() diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..90020c0 --- /dev/null +++ b/setup.py @@ -0,0 +1,200 @@ +"""A setuptools based setup module. + +See: +https://packaging.python.org/en/latest/distributing.html +https://github.com/pypa/sampleproject +""" + +# Always prefer setuptools over distutils +from setuptools import setup, find_packages +from os import path +# io.open is needed for projects that support Python 2.7 +# It ensures open() defaults to text mode with universal newlines, +# and accepts an argument to specify the text encoding +# Python 3 only projects can skip this import +from io import open + +here = path.abspath(path.dirname(__file__)) + +# Get the long description from the README file +with open(path.join(here, 'README.md'), encoding='utf-8') as f: + long_description = f.read() + +# Arguments marked as "Required" below must be included for upload to PyPI. +# Fields marked as "Optional" may be commented out. + +setup( + # This is the name of your project. The first time you publish this + # package, this name will be registered for you. It will determine how + # users can install this project, e.g.: + # + # $ pip install sampleproject + # + # And where it will live on PyPI: https://pypi.org/project/sampleproject/ + # + # There are some restrictions on what makes a valid project name + # specification here: + # https://packaging.python.org/specifications/core-metadata/#name + name='cliapi', # Required + + # Versions should comply with PEP 440: + # https://www.python.org/dev/peps/pep-0440/ + # + # For a discussion on single-sourcing the version across setup.py and the + # project code, see + # https://packaging.python.org/en/latest/single_source_version.html + version='0.1', # Required + + # This is a one-line description or tagline of what your project does. This + # corresponds to the "Summary" metadata field: + # https://packaging.python.org/specifications/core-metadata/#summary + description='a dynamic getopts()-style CLI generator', # Required + + # This is an optional longer description of your project that represents + # the body of text which users will see when they visit PyPI. + # + # Often, this is the same as your README, so you can just read it in from + # that file directly (as we have already done above) + # + # This field corresponds to the "Description" metadata field: + # https://packaging.python.org/specifications/core-metadata/#description-optional + long_description=long_description, # Optional + + # Denotes that our long_description is in Markdown; valid values are + # text/plain, text/x-rst, and text/markdown + # + # Optional if long_description is written in reStructuredText (rst) but + # required for plain-text or Markdown; if unspecified, "applications should + # attempt to render [the long_description] as text/x-rst; charset=UTF-8 and + # fall back to text/plain if it is not valid rst" (see link below) + # + # This field corresponds to the "Description-Content-Type" metadata field: + # https://packaging.python.org/specifications/core-metadata/#description-content-type-optional + long_description_content_type='text/markdown', # Optional (see note above) + + # This should be a valid link to your project's main homepage. + # + # This field corresponds to the "Home-Page" metadata field: + # https://packaging.python.org/specifications/core-metadata/#home-page-optional + url='https://github.com/edlane/cliapi', # Optional + + # This should be your name or the name of the organization which owns the + # project. + author='ed lane', # Optional + + # This should be a valid email address corresponding to the author listed + # above. + author_email='ed.lane@suse.com', # Optional + + # Classifiers help users find your project by categorizing it. + # + # For a list of valid classifiers, see https://pypi.org/classifiers/ + classifiers=[ # Optional + # How mature is this project? Common values are + # 3 - Alpha + # 4 - Beta + # 5 - Production/Stable + 'Development Status :: 3 - Alpha', + + # Indicate who your project is intended for + 'Intended Audience :: Developers', + 'Topic :: Software Development :: CLI coding tools', + + # Pick your license as you wish + 'License :: OSI Approved :: Apache 2.0', + + # Specify the Python versions you support here. In particular, ensure + # that you indicate whether you support Python 2, Python 3 or both. + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + ], + + # This field adds keywords for your project which will appear on the + # project page. What does your project relate to? + # + # Note that this is a string of words separated by whitespace, not a list. + keywords='CLI coding tools', # Optional + + # You can just specify package directories manually here if your project is + # simple. Or you can use find_packages(). + # + # Alternatively, if you just want to distribute a single Python file, use + # the `py_modules` argument instead as follows, which will expect a file + # called `my_module.py` to exist: + # + # py_modules=["cliapi.py", "cliapi_lib.py"], + # py_modules=["my_module"], + # + packages=find_packages(exclude=['contrib', 'docs', 'tests']), # Required + + # This field lists other packages that your project depends on to run. + # Any package you put here will be installed by pip when your project is + # installed, so they must be valid existing projects. + # + # For an analysis of "install_requires" vs pip's requirements files see: + # https://packaging.python.org/en/latest/requirements.html + # install_requires=['peppercorn'], # Optional + # install_requires=['cliapi'], # Optional + + # List additional groups of dependencies here (e.g. development + # dependencies). Users will be able to install these using the "extras" + # syntax, for example: + # + # $ pip install sampleproject[dev] + # + # Similar to `install_requires` above, these must be valid existing + # projects. + # extras_require={ # Optional + # 'dev': ['check-manifest'], + # 'test': ['coverage'], + # }, + + # If there are data files included in your packages that need to be + # installed, specify them here. + # + # If using Python 2.6 or earlier, then these have to be included in + # MANIFEST.in as well. + # package_data={ # Optional + # 'sample': ['package_data.dat'], + # }, + + # Although 'package_data' is the preferred approach, in some case you may + # need to place data files outside of your packages. See: + # http://docs.python.org/3.4/distutils/setupscript.html#installing-additional-files + # + # In this case, 'data_file' will be installed into '/my_data' + # data_files=[('my_data', ['data/data_file'])], # Optional + + # To provide executable scripts, use entry points in preference to the + # "scripts" keyword. Entry points provide cross-platform support and allow + # `pip` to create the appropriate form of executable for the target + # platform. + # + # For example, the following would provide a command called `sample` which + # executes the function `main` from this package when invoked: + entry_points={ # Optional + 'console_scripts': [ + 'cliapi=cliapi.cliapi:main', + ], + }, + + # List additional URLs that are relevant to your project as a dict. + # + # This field corresponds to the "Project-URL" metadata fields: + # https://packaging.python.org/specifications/core-metadata/#project-url-multiple-use + # + # Examples listed include a pattern for specifying where the package tracks + # issues, where the source is hosted, where to say thanks to the package + # maintainers, and where to support the project financially. The key is + # what's used to render the link text on PyPI. + # project_urls={ # Optional + # 'Bug Reports': 'https://github.com/pypa/sampleproject/issues', + # 'Funding': 'https://donate.pypi.org', + # 'Say Thanks!': 'http://saythanks.io/to/example', + # 'Source': 'https://github.com/pypa/sampleproject/', + # }, +) From a719188b977f9f565febb180b5a5f36c5a20b970 Mon Sep 17 00:00:00 2001 From: ed lane Date: Thu, 23 Aug 2018 08:00:27 -0600 Subject: [PATCH 06/29] changes to prop #1 --- design.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/design.md b/design.md index 57cb55d..7ff83d8 100644 --- a/design.md +++ b/design.md @@ -26,7 +26,8 @@ with "metadata" rather than breaking them into separate CLIs. e.g "cloud-service **Proposition 1:** -Implement azuremetadata entirely using Salt's metadata grains module +This is mostly a solved problem with Salt's grain module [https://github.com/saltstack/salt/blob/develop/salt/grains/metadata.py] +- Implement azuremetadata entirely using Salt's metadata grains module - extend grains module to support azure cloud. (~40 LOC) - add "cloud-service" or other SUSE required functionality (~20 LOC) - replace other parts of Enceladus by Salt when duplicate functionality exists. From ea8c7326aa6d1b08fb5efdc67ffdf6385bd5b30f Mon Sep 17 00:00:00 2001 From: ed lane Date: Thu, 23 Aug 2018 13:45:03 -0600 Subject: [PATCH 07/29] more fixes for directory restructuring --- README.md | 123 +++++++++++++++++++++++++++++++++----- cliapi/cliapi.py | 48 +++++++++++---- cliapi/cliapi_lib.py | 4 +- cliapi/providers/azure.py | 2 +- design.md | 29 +++++---- 5 files changed, 164 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index acec6f2..af10282 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,19 @@ # GUIDE FOR DEVELOPERS: # cliapi **cliapi** ( pronounced _calliope_ ): A Python framework for creating Unix, "getopt()" style CLI scripts -composed from a set of APIs. +composed from an arbitrary set of APIs. ...at its core, **cliapi** is _a data driven, state machine for dynamically generating CLIs_. **cliapi Features**: -- entire CLI is generated by applying python decorators together with a "dictionary with backing API store". -This decorator is applied for each API supported by a "plugin provider". +- entire CLI is generated by applying python decorators to API calls together with a dictionary with +a _backing-API-store_. +This decorator is applied to each API that is supported by a "plugin provider". - multiple data queries within the same CLI will result in "at most" a single API call. - consistent "command line behavior" is enforced by the cliapi framework across all plugin providers. -Several common commands are automatically inherited by all the providers: +Several common commands are automatically inherited by all plugin providers: - "**--list-providers**" Lists all the available plugin providers. - "**--provider=**" specify a particular provider plugin for all CLI commands. - "**--all**" Lists all data returned by all APIs supported by a single provider. @@ -20,7 +21,7 @@ Several common commands are automatically inherited by all the providers: - "**--query=**" extracts specified API data using pythonic dictionary syntax. - other behaviors enforced by the cliapi framework: - error handling and help is also consistent across all plugin providers. - - multiple queries in same command will return a JSON list in "query order" by default + - multiple queries in same command will return a JSON list in "option order" by default - single query will return a single JSON element. **Key cliapi developer concepts:** @@ -34,21 +35,111 @@ optional parameters are expressible through the CLI. Help is also handled by th - **scoops**: a dictionary which maps a CLI query name to a particular API data scoop. "scoops" are really just "_sandboxed_ python _eval()_" statements. This allows scoops to be expressed as Python slices, -comprehensions, ect. It also allows restricted ad-hoc queries on the command line -when a specific value is desired but is not currently supported as an option in the CLI. +comprehensions, ect. It also allows restricted ad-hoc queries to be provided on the command line +when a return value is not currently supported as an option in the CLI. - **fetchers**: a dictionary which maps from a particular API name to the actual python function which provides the _backing store_ for the contents of the top-level API dictionary. -**Default Cliapi Directory structure** +**example #1** - help (provider and common) ``` -├── cliapi_lib.py -├── readme.md -├── cliapi.py -├── __init__.py -├── providers -│   ├── azure.py -│   └── test.py -└── what_cloud.py +ed-sle12sp3byos:/home/lane/cliapi # cliapi --help --provider=azure +usage: /usr/bin/cliapi [display option#1]... [API option#1]... [CLI option] + +***[ azure ]*** provider Display options: + --internal-ip + --cloud-service what + --instance-name name of instance + --external-ip + --mac the MAC address for this interface + --location region location + +***[ azure ]*** provider API config options: + --api_version= azure metadata api version, default = '2017-08-01' + +Common CLI options: + --help help for this CLI command + --provider= specify name of provider module + --list-providers list all available providers + --list-apis list all available APIs for specified provider + --query= specify a python dictionary style query command + --all output all API results for specified API options or defaults + --pycharm-debug +``` + +**example #2** - a predefined query option ``` +ed-sle12sp3byos:/home/lane/cliapi # cliapi --internal-ip --provider=azure +"172.16.3.8" +``` + +**example #3** - all values returned by all provider APIs +``` +ed-sle12sp3byos:/home/lane/cliapi # cliapi --all --provider=azure +{ + "tag": "391e4f53-b82d-5af5-8f58-dc1035e46e5e", + "meta_data": { + "network": { + "interface": [ + { + "ipv6": { + "ipAddress": [] + }, + "ipv4": { + "ipAddress": [ + { + "publicIpAddress": "40.112.253.198", + "privateIpAddress": "172.16.3.8" + } + ], + "subnet": [ + { + "address": "172.16.3.0", + "prefix": "24" + } + ] + }, + "macAddress": "000D3A3AE8A5" + } + ] + }, + "compute": { + "offer": "SLES-BYOS", + "publisher": "SUSE", + "subscriptionId": "ce73a2b0-d2e7-4ff6-b987-b32d6908de4e", + "name": "ed-sle12sp3byos", + "platformUpdateDomain": "0", + "version": "2018.02.21", + "placementGroupId": "", + "sku": "12-SP3", + "tags": "", + "vmId": "ce01dc32-6d0a-40bd-9534-a3509f768a53", + "resourceGroupName": "ed_lane", + "osType": "Linux", + "location": "westus", + "platformFaultDomain": "0", + "vmSize": "Standard_B1ms" + } + }, + "cloud-service": "__ed-sle12sp3byosService.cloudapp.net" +} + +``` + +**example #4** - a sandboxed python query +``` +lane@suse-laptop:~/develop/garage/cliapi> cliapi --query="['meta_data']['compute']['offer']" --provider=test +"SLES-BYOS" + +``` + +**example #5** - multiple queries +``` +lane@suse-laptop:~/develop/garage/cliapi> cliapi --location --query="['meta_data']['compute']['offer']" --provider=test +[ + "westus", + "SLES-BYOS" +] + +``` \ No newline at end of file diff --git a/cliapi/cliapi.py b/cliapi/cliapi.py index 70266fa..ceb52e4 100755 --- a/cliapi/cliapi.py +++ b/cliapi/cliapi.py @@ -1,10 +1,23 @@ #!/usr/bin/python3 +# Guide for cliapi developers found here: +# https://github.com/edlane/cliapi/blob/master/README.md + import os if 'PYCHARM_DEBUG_ME' in os.environ: import pydevd - # connect pycharm debugger with: - # bash> PYCHARM_DEBUGME= cliapi .... + # we want debugging enabled up top because much of the cliapi framework + # "import-meta-magic" happens before __main__() has ever been called... + # + # Remote debugging session enabled in cloud environments + # by establishing a reverse ssh tunnel with bash: + # + # local_host> ssh -R 8282:localhost:8282 lane@remote_host + # + # Then connect back to your local pycharm debugger with: + # + # remote_host> PYCHARM_DEBUG_ME= cliapi [various options]... + # pydevd.settrace('localhost', port=8282, stdoutToServer=True, stderrToServer=True) @@ -15,6 +28,7 @@ import json import cliapi.providers as providers +from cliapi.cliapi_lib import Provider prefix = providers.__name__ + '.' @@ -31,7 +45,7 @@ except Exception as e: pass -# TODO: does "xml_out" really need to be implemented in a modern Devops world??? +# TODO: does "xml-out" really need to be implemented in a modern Devops world??? cli_options = ['help', 'provider=', 'list-providers', @@ -65,9 +79,14 @@ def _print_help(provider): 'all': 'output all API results for specified API options or defaults', } - vpp = valid_providers[provider].provider - # update help with options from this provider... - help.update(vp.help) + if provider == None: + vpp = Provider() + # no provider supplied so at least print out the "common" help... + vpp.help = help + else: + vpp = valid_providers[provider].provider + # update help with options from this provider... + vpp.help.update(help) indent2_format = ' --{:<15} {:<15}' print("usage: {} [display option#1]... [API option#1]... [CLI option]".format(sys.argv[0])) print("\n***[ {} ]*** provider Display options:".format(provider)) @@ -104,16 +123,21 @@ def main(): print('valid providers =\n', json.dumps(list(valid_providers.keys()), indent=2)) exit(0) + cmd_dict = {} if '--provider' in ct: # using supplied provider... provider = ct['--provider'] else: - # using the first valid provider... - provider = list(valid_providers.keys())[0] + try: + # use the first valid provider in list... + provider = list(valid_providers.keys())[0] + except Exception as e: + # error -- print help and exit + print('No plugin providers found') + _print_help(None) + exit(-1) - cmd_dict = {} vpp = valid_providers[provider].provider - all_opts = list(vpp.scoops.keys()) options = [] for k, v in vpp.options.items(): # process args... @@ -138,7 +162,7 @@ def main(): vpp.template.update(cmd_dict) - # use defaults if option NOT provided on CLI... + # use defaults if option NOT supplied in CLI... for default in options: ds = default.split('=') try: @@ -182,7 +206,7 @@ def main(): # cmd_dict['help'] = None # exit(-1) elif key == 'query': - data.append(_sandbox_eval(vpp, query)) + data.append(_sandbox_eval(vpp, cmd_dict['query'])) if len(data) == 1: # a single value was requested so no list is required... data = data[0] diff --git a/cliapi/cliapi_lib.py b/cliapi/cliapi_lib.py index b07a0c9..9802d87 100644 --- a/cliapi/cliapi_lib.py +++ b/cliapi/cliapi_lib.py @@ -22,10 +22,10 @@ def __getitem__(self, item): # {'__builtins__': None}, # {'item': item})[0][0] fetcher = self.fetchers[item].split('.') - provider = importlib.import_module('.'.join(fetcher[0:2])) + provider = importlib.import_module('.'.join(fetcher[0:3])) super().__setitem__(item, eval('provider.' + - Template(fetcher[2]).substitute(self.template))) + Template(fetcher[3]).substitute(self.template))) # restrict python's eval() functon -- No builtins, only "self" is accessible result = self.get(item) # result = eval("self" + item, {'__builtins__': None}, {'self': self}) diff --git a/cliapi/providers/azure.py b/cliapi/providers/azure.py index a453623..089653e 100644 --- a/cliapi/providers/azure.py +++ b/cliapi/providers/azure.py @@ -1,5 +1,5 @@ from cliapi.cliapi_lib import Provider, cliapi_decor -from what_cloud import determine_provider +from cliapi.what_cloud import determine_provider import json import os diff --git a/design.md b/design.md index 7ff83d8..7093c5a 100644 --- a/design.md +++ b/design.md @@ -21,17 +21,24 @@ has been added over time. To allow a new item to be queried from the CLI, the cloud module must be revised first in order to access it. - There is a practice of adding other SUSE-cloud-provider specific functions along -with "metadata" rather than breaking them into separate CLIs. e.g "cloud-service" and -"billing-tag",... +with the "metadata" rather than breaking them into separate CLIs. e.g "cloud-service", +"billing-tag", etc. +- The meta-data collection of CLIs is primarily to support automation pipelines elsewhere in +Enceladus and as such should be expected to have a more consistent interface than tools +intended to be consumed only by humans. Behavior across all the meta-data CLIs should be +more of an API than an ad-hoc query tool. **Proposition 1:** -This is mostly a solved problem with Salt's grain module [https://github.com/saltstack/salt/blob/develop/salt/grains/metadata.py] -- Implement azuremetadata entirely using Salt's metadata grains module -- extend grains module to support azure cloud. (~40 LOC) -- add "cloud-service" or other SUSE required functionality (~20 LOC) -- replace other parts of Enceladus by Salt when duplicate functionality exists. -**(I am informed that "advocating this proposition would be my last official act")** +- This is mostly a solved problem with Salt's grain module [https://github.com/saltstack/salt/blob/develop/salt/grains/metadata.py] +so implement azuremetadata entirely using Salt's metadata grains module +- extend grains module to support azure cloud. (currently missing ~40 LOC) +- add "cloud-service" or other SUSE required functionality as custom grains (~20 LOC) +- replace other parts of Enceladus by Salt automation when duplicate functionality exists. +- develop custom Salt modules, runners, grains, states, beacons, .... as appropriate to +implement Enceladus functionality. +**(I am advised that "advocating this proposition would be my last official act" -- so +dropping this proposal immediately")** **alternative #2 to Proposition 1:** - Allow salt grains to be called directly from CLI. This would allow salt grains to be called @@ -48,14 +55,14 @@ the old interface. - Choose an existing cloud CLI (gce or ec2) and emulate that one as much as possible that same behavior for **azuremetadata.py** -- Try to reconcile all the behavior differences in the CLIs and bring them back into compatibility (at +- For extra credit: Try to reconcile all the behavior differences in the CLIs and bring them back into compatibility (at least for one instance in time before the inevitable "code drift" takes hold) **Proposition 3:** -- Create a common, extensible framework which will generate a CLI given a set of APIs described as -python functions. +- Create a common, extensible framework which will generate CLIs given a set of APIs +described as python functions using introspection. - Enable backward compatibility by supporting all existing CLI options. - Enable forward flexibility by adding a query-language extension with allows recently added or previously unanticipated metadata API values to be queried. From 312a05c674ea78ff824e1b8e9a63f8aa80b7f328 Mon Sep 17 00:00:00 2001 From: ed lane Date: Thu, 23 Aug 2018 14:25:56 -0600 Subject: [PATCH 08/29] format fix --- design.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/design.md b/design.md index 7093c5a..f064b03 100644 --- a/design.md +++ b/design.md @@ -1,4 +1,4 @@ -#Design Considerations: +# Design Considerations: **Task:** port azuremetadata.pl (perl) to python **Requirements:** From 69a15ed7f5a73e92733ca4b0fc7e59f4c5852c2e Mon Sep 17 00:00:00 2001 From: ed lane Date: Thu, 23 Aug 2018 14:38:30 -0600 Subject: [PATCH 09/29] correct URL markdown --- design.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/design.md b/design.md index f064b03..b6cba5b 100644 --- a/design.md +++ b/design.md @@ -30,7 +30,7 @@ more of an API than an ad-hoc query tool. **Proposition 1:** -- This is mostly a solved problem with Salt's grain module [https://github.com/saltstack/salt/blob/develop/salt/grains/metadata.py] +- This is mostly a solved problem with Salt's grain module so implement azuremetadata entirely using Salt's metadata grains module - extend grains module to support azure cloud. (currently missing ~40 LOC) - add "cloud-service" or other SUSE required functionality as custom grains (~20 LOC) From a9c5668c583c8cb47a6af3141dfb305d23f5f95b Mon Sep 17 00:00:00 2001 From: ed lane Date: Mon, 27 Aug 2018 15:12:26 -0600 Subject: [PATCH 10/29] various documentation updates --- README.md | 48 ++++++++++++++++++++++++------- design.md | 85 ++++++++++++++++++++++++++++++++++--------------------- 2 files changed, 90 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index af10282..9f9e9f1 100644 --- a/README.md +++ b/README.md @@ -1,30 +1,38 @@ -# GUIDE FOR DEVELOPERS: -# cliapi +## GUIDE FOR CLIAPI DEVELOPERS: **cliapi** ( pronounced _calliope_ ): A Python framework for creating Unix, "getopt()" style CLI scripts composed from an arbitrary set of APIs. ...at its core, **cliapi** is _a data driven, state machine for dynamically generating CLIs_. -**cliapi Features**: +### cliapi features: - entire CLI is generated by applying python decorators to API calls together with a dictionary with a _backing-API-store_. This decorator is applied to each API that is supported by a "plugin provider". +For each cloud provider (eg. Azure or GCE) a supporting "plugin" module is added to the +`cliapi/providers` directory (see **`cliapi directory layout`** below) -- multiple data queries within the same CLI will result in "at most" a single API call. - -- consistent "command line behavior" is enforced by the cliapi framework across all plugin providers. +- consistent command line behavior is enforced by the cliapi framework across all plugin providers. Several common commands are automatically inherited by all plugin providers: +(this allows a level of "discoverablility" to the CLI across all plugin providers) - "**--list-providers**" Lists all the available plugin providers. - "**--provider=**" specify a particular provider plugin for all CLI commands. - "**--all**" Lists all data returned by all APIs supported by a single provider. - "**--list-apis**" Lists all the APIs available by a particular plugin provider. - - "**--query=**" extracts specified API data using pythonic dictionary syntax. + - "**--query=**" extracts specified API data using a python-dictionary-restricted syntax. - other behaviors enforced by the cliapi framework: - error handling and help is also consistent across all plugin providers. - multiple queries in same command will return a JSON list in "option order" by default - single query will return a single JSON element. -**Key cliapi developer concepts:** +- multiple data queries within the same CLI will result in "at most" a single API call. + +- presumably the plugin developer will rarely need to change the cliapi framework involving +python meta-programming. +Most plugins can be supported with a single module, a few dictionary initializers, +and some function decorators. + +### Key cliapi developer concepts: + - **provider**: an associated set of APIs accessed from the CLI in a particular context or cloud environment e.g. "azure", "gce", "ec2", ... but can essentially be any mix of apis @@ -41,8 +49,25 @@ when a return value is not currently supported as an option in the CLI. - **fetchers**: a dictionary which maps from a particular API name to the actual python function which provides the _backing store_ for the contents of the top-level API dictionary. +### cliapi directory layout: +``` +├── cliapi root of cliapi package +│   ├── __init__.py +│   ├── cliapi_lib.py framework meta classes and decorators +│   ├── cliapi.py main() and the CLI generation code +│   └── what_cloud.py module (useful for detecting which cloud plugins are valid) +│   ├── providers directory for plugin providers +│   │   ├── __init__.py +│   │   ├── azure.py plugin for Azure APIs +│   │   └── test.py plugin for Test APIs (an included "test" API for hacking) +├── design.md design considerations for cliapi +├── LICENSE provisional license (Apache2 is mutable into any other license) +├── README.md this document +└── setup.py Python installation script +``` -**example #1** - help (provider and common) +### Examples: +**example #1** - help (provider=azure and cliapi common) ``` ed-sle12sp3byos:/home/lane/cliapi # cliapi --help --provider=azure usage: /usr/bin/cliapi [display option#1]... [API option#1]... [CLI option] @@ -134,7 +159,7 @@ lane@suse-laptop:~/develop/garage/cliapi> cliapi --query="['meta_data']['compute ``` -**example #5** - multiple queries +**example #5** - multiple queries (single CLI call) ``` lane@suse-laptop:~/develop/garage/cliapi> cliapi --location --query="['meta_data']['compute']['offer']" --provider=test [ @@ -142,4 +167,5 @@ lane@suse-laptop:~/develop/garage/cliapi> cliapi --location --query="['meta_data "SLES-BYOS" ] -``` \ No newline at end of file +``` + diff --git a/design.md b/design.md index b6cba5b..082631a 100644 --- a/design.md +++ b/design.md @@ -12,54 +12,58 @@ authors. - Subtle and not so subtle differences exist in the CLI behaviors and output formats. This requires developers to implement separate handlers and be aware of the idiosyncrasies of each implementation. -- The GCE and EC2 modules both use getopt() and are largely identical API except for -returned results. -- Azure is expectedly weird with different API, results, and configuration parameters +- The GCE and EC2 modules both use getopt() and are largely familiar API except for +the returned results. +- Azure (the target of this port) is expectedly weird with different API, results, and configuration parameters required (...and because it was written in Perl) - It looks like the list of available and supported CLI query options for each cloud provider -has been added over time. -To allow a new item to be queried from the CLI, the cloud module must be revised first -in order to access it. -- There is a practice of adding other SUSE-cloud-provider specific functions along -with the "metadata" rather than breaking them into separate CLIs. e.g "cloud-service", -"billing-tag", etc. -- The meta-data collection of CLIs is primarily to support automation pipelines elsewhere in -Enceladus and as such should be expected to have a more consistent interface than tools -intended to be consumed only by humans. Behavior across all the meta-data CLIs should be -more of an API than an ad-hoc query tool. - -**Proposition 1:** - -- This is mostly a solved problem with Salt's grain module -so implement azuremetadata entirely using Salt's metadata grains module -- extend grains module to support azure cloud. (currently missing ~40 LOC) -- add "cloud-service" or other SUSE required functionality as custom grains (~20 LOC) +has been added to over time. +To allow a new item to be queried from the CLI, the cloud module must also +be revised in order to access it. +- There is also a practice of adding other SUSE specific functions into the "cloud metadata" +rather than breaking them into separate CLIs. e.g "cloud-service", "billing-tag", etc. +- The meta-data collection of CLIs seems primarily to support automation pipelines elsewhere +in Enceladus and not meant exclusively for humans. +As such these CLIs are expected to have consistent behavior across all the meta-data CLIs +approaching that of an API rather than an ad-hoc query tool. Current best practice for API +publishers would seem to favor a "discoverable-style" APIs over traditional "man-page" described +ones". + +**Proposal 1:** + +- This is a mostly solved problem with Salt's grain module +so implement azuremetadata entirely using Salt's metadata grains module +- **Note:** Currently the Salt grains metadata module needs to be extended to support Azure. +(estimate ~40 LOC for Azure support) +- add a "cloud-service" and other SUSE required functionality as custom grains modules (~20 LOC) - replace other parts of Enceladus by Salt automation when duplicate functionality exists. -- develop custom Salt modules, runners, grains, states, beacons, .... as appropriate to -implement Enceladus functionality. -**(I am advised that "advocating this proposition would be my last official act" -- so +- develop custom Salt modules, runners, grains, states, beacons, .... to +implement Enceladus functionality where appropriate. +**(I am advised that "advocating this proposal would be _GREATLY OPPOSED_" -- so dropping this proposal immediately")** -**alternative #2 to Proposition 1:** +**alternative #2 to Proposal 1:** - Allow salt grains to be called directly from CLI. This would allow salt grains to be called as an executable module via **_salt-call_** (new functionality ~10 LOC) -- support other SUSE specific functions as salt -This suffers from many of the objections of alternative #1 and would break consumers of +- support other SUSE specific functions as separate **salt-call**-_able_ modules +rather than separate CLI programs. +This suffers from many of the objections of alternative #1 and would break existing consumers of the old interface. -**Proposition 2:** +**Proposal 2:** -- Port "as is" and create yet another slightly inconsistent CLI, but coded in Python instead. +- Port azuremetadata.pl "as is" and create yet another slightly inconsistent CLI, but coded in Python instead. -**alternative #2 to Proposition 2:** +**alternative #2 to Proposal 2:** - Choose an existing cloud CLI (gce or ec2) and emulate that one as much as possible that same behavior for **azuremetadata.py** -- For extra credit: Try to reconcile all the behavior differences in the CLIs and bring them back into compatibility (at -least for one instance in time before the inevitable "code drift" takes hold) +- For extra credit: Try to reconcile all the behavior differences in the various CLIs and +bring them back into compliance (at +least for one instance in time before the inevitable "code drift" takes hold) -**Proposition 3:** +**Proposal 3:** - Create a common, extensible framework which will generate CLIs given a set of APIs described as python functions using introspection. @@ -67,3 +71,20 @@ described as python functions using introspection. - Enable forward flexibility by adding a query-language extension with allows recently added or previously unanticipated metadata API values to be queried. + +### LOC Implementation Comparison for Proposal #3 above: +**Note:** Admittedly comparing developer editable lines of code between different implementations +is an unprecise activity but here is a worksheet used to prepare this report anyway... + + +https://docs.google.com/spreadsheets/d/1i_phns6QS3eCmsWFXWxxcafuNYXOEqG0ffxt6NHZysk/edit?usp=sharing + +**Total Dev Edited LOC for Existing Implementation:** +``` +~2177 Total LOC = ec2metadata + azuremetadata(perl) + gcemetadata +``` + +**Total Dev Edited LOC for Proposal #3 Implementation from above** +``` +~881 Total LOC = cliapi(framework actual) + azure(plugin actual) + gce(plugin estimate) + ec2(plugin estimate) +``` From c23fbca56bbbf20cda86da8dfccb25417f971cd7 Mon Sep 17 00:00:00 2001 From: ed lane Date: Mon, 27 Aug 2018 15:29:25 -0600 Subject: [PATCH 11/29] README fixups. --- README.md | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 9f9e9f1..caaea34 100644 --- a/README.md +++ b/README.md @@ -5,11 +5,14 @@ composed from an arbitrary set of APIs. ...at its core, **cliapi** is _a data driven, state machine for dynamically generating CLIs_. ### cliapi features: -- entire CLI is generated by applying python decorators to API calls together with a dictionary with -a _backing-API-store_. +- entire CLI is generated by applying python decorators to API calls together with a dictionary which +a _backing-API-store_. This decorator is applied to each API that is supported by a "plugin provider". For each cloud provider (eg. Azure or GCE) a supporting "plugin" module is added to the `cliapi/providers` directory (see **`cliapi directory layout`** below) +Presumably the plugin developer will rarely need to change the cliapi framework +itself which involves a little python meta-programming (1 decorator + 1 dictionary +overriding class). - consistent command line behavior is enforced by the cliapi framework across all plugin providers. Several common commands are automatically inherited by all plugin providers: @@ -26,11 +29,6 @@ Several common commands are automatically inherited by all plugin providers: - multiple data queries within the same CLI will result in "at most" a single API call. -- presumably the plugin developer will rarely need to change the cliapi framework involving -python meta-programming. -Most plugins can be supported with a single module, a few dictionary initializers, -and some function decorators. - ### Key cliapi developer concepts: - **provider**: an associated set of APIs accessed from the CLI in a particular context or cloud From 1cefc1a56b35feb07b74cb8052cb91ea41573afd Mon Sep 17 00:00:00 2001 From: ed lane Date: Mon, 27 Aug 2018 15:33:56 -0600 Subject: [PATCH 12/29] more README.md fixes --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index caaea34..a114821 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ composed from an arbitrary set of APIs. ### cliapi features: - entire CLI is generated by applying python decorators to API calls together with a dictionary which -a _backing-API-store_. +specifies a _backing-API-store_. This decorator is applied to each API that is supported by a "plugin provider". For each cloud provider (eg. Azure or GCE) a supporting "plugin" module is added to the `cliapi/providers` directory (see **`cliapi directory layout`** below) From 02d0dd16891d539a0be23a7cdb3a90659c1bdd68 Mon Sep 17 00:00:00 2001 From: ed lane Date: Mon, 27 Aug 2018 15:41:41 -0600 Subject: [PATCH 13/29] spelling and format fixes --- design.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/design.md b/design.md index 082631a..c4e82fe 100644 --- a/design.md +++ b/design.md @@ -68,23 +68,23 @@ least for one instance in time before the inevitable "code drift" takes hold) - Create a common, extensible framework which will generate CLIs given a set of APIs described as python functions using introspection. - Enable backward compatibility by supporting all existing CLI options. -- Enable forward flexibility by adding a query-language extension with allows recently +- Enable forward flexibility by adding a query-language extension which allows recently added or previously unanticipated metadata API values to be queried. ### LOC Implementation Comparison for Proposal #3 above: **Note:** Admittedly comparing developer editable lines of code between different implementations -is an unprecise activity but here is a worksheet used to prepare this report anyway... +is an unprecise and debatable activity but here is a worksheet used to prepare this report anyway... https://docs.google.com/spreadsheets/d/1i_phns6QS3eCmsWFXWxxcafuNYXOEqG0ffxt6NHZysk/edit?usp=sharing **Total Dev Edited LOC for Existing Implementation:** -``` -~2177 Total LOC = ec2metadata + azuremetadata(perl) + gcemetadata -``` + +**~2177 Total LOC** = ec2metadata + azuremetadata(perl) + gcemetadata + **Total Dev Edited LOC for Proposal #3 Implementation from above** -``` -~881 Total LOC = cliapi(framework actual) + azure(plugin actual) + gce(plugin estimate) + ec2(plugin estimate) -``` + +**~881 Total LOC** = cliapi(framework actual) + azure(plugin actual) + gce(plugin estimate) + ec2(plugin estimate) + From 00d3ec340cd4fa27458f57ddad3abc4467e92288 Mon Sep 17 00:00:00 2001 From: ed lane Date: Mon, 27 Aug 2018 16:07:20 -0600 Subject: [PATCH 14/29] more guidance to hackers --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a114821..2e9ce65 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,8 @@ provides the _backing store_ for the contents of the top-level API dictionary. │   ├── providers directory for plugin providers │   │   ├── __init__.py │   │   ├── azure.py plugin for Azure APIs +│   │   ├── ...your plugin goes here +│   │   ├── ... ...additional plugins are automatically discovered │   │   └── test.py plugin for Test APIs (an included "test" API for hacking) ├── design.md design considerations for cliapi ├── LICENSE provisional license (Apache2 is mutable into any other license) @@ -150,7 +152,7 @@ ed-sle12sp3byos:/home/lane/cliapi # cliapi --all --provider=azure ``` -**example #4** - a sandboxed python query +**example #4** - an ad-hock restricted python dictionary syntax query ``` lane@suse-laptop:~/develop/garage/cliapi> cliapi --query="['meta_data']['compute']['offer']" --provider=test "SLES-BYOS" From fa10c807c25a29af8723ca0835fc305b2768b810 Mon Sep 17 00:00:00 2001 From: ed lane Date: Wed, 29 Aug 2018 07:40:16 -0600 Subject: [PATCH 15/29] bad spelling --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2e9ce65..f805efc 100644 --- a/README.md +++ b/README.md @@ -152,7 +152,7 @@ ed-sle12sp3byos:/home/lane/cliapi # cliapi --all --provider=azure ``` -**example #4** - an ad-hock restricted python dictionary syntax query +**example #4** - an ad-hoc restricted python dictionary syntax query ``` lane@suse-laptop:~/develop/garage/cliapi> cliapi --query="['meta_data']['compute']['offer']" --provider=test "SLES-BYOS" From 7e1cf62edd9a3eaa028a822e41b76e6b4207ab34 Mon Sep 17 00:00:00 2001 From: ed lane Date: Wed, 29 Aug 2018 07:43:39 -0600 Subject: [PATCH 16/29] add syntax error reporting for bad query options --- cliapi/cliapi.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/cliapi/cliapi.py b/cliapi/cliapi.py index ceb52e4..029ad64 100755 --- a/cliapi/cliapi.py +++ b/cliapi/cliapi.py @@ -201,12 +201,18 @@ def main(): data.append(_sandbox_eval(vpp, query)) except KeyError as e: # error -- print help and exit - print('required option --{} missing'.format(e.args[0])) + print('required option -- "{}" missing'.format(e.args[0])) _print_help(provider) # cmd_dict['help'] = None # exit(-1) elif key == 'query': - data.append(_sandbox_eval(vpp, cmd_dict['query'])) + query = cmd_dict['query'] + try: + data.append(_sandbox_eval(vpp, query)) + except SyntaxError as e: + # error in query + print('error in query -- "{}" syntax error'.format(query)) + _print_help(provider) if len(data) == 1: # a single value was requested so no list is required... data = data[0] From b555a0c8bf9361e2476470cc34a64e6883e8b3b8 Mon Sep 17 00:00:00 2001 From: ed lane Date: Wed, 29 Aug 2018 09:50:09 -0600 Subject: [PATCH 17/29] more, better docs --- README.md | 44 ++++++++++++++++++++++++++------------------ design.md | 4 ++-- 2 files changed, 28 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index f805efc..2f580e0 100644 --- a/README.md +++ b/README.md @@ -56,10 +56,10 @@ provides the _backing store_ for the contents of the top-level API dictionary. │   └── what_cloud.py module (useful for detecting which cloud plugins are valid) │   ├── providers directory for plugin providers │   │   ├── __init__.py -│   │   ├── azure.py plugin for Azure APIs +│   │   ├── azure.py plugin for Azure/SUSE APIs │   │   ├── ...your plugin goes here -│   │   ├── ... ...additional plugins are automatically discovered -│   │   └── test.py plugin for Test APIs (an included "test" API for hacking) +│   │   ├── ... ...additional plugins are automatically discovered by cliapi +│   │   └── test.py ..."because it's not a framework without at least 2 plugins" ├── design.md design considerations for cliapi ├── LICENSE provisional license (Apache2 is mutable into any other license) ├── README.md this document @@ -67,9 +67,10 @@ provides the _backing store_ for the contents of the top-level API dictionary. ``` ### Examples: -**example #1** - help (provider=azure and cliapi common) + +**example #1** - Common help AND plugin provider help (default provider = azure) ``` -ed-sle12sp3byos:/home/lane/cliapi # cliapi --help --provider=azure +ed-sle12sp3byos:/home/lane/cliapi # cliapi --help usage: /usr/bin/cliapi [display option#1]... [API option#1]... [CLI option] ***[ azure ]*** provider Display options: @@ -92,16 +93,16 @@ Common CLI options: --all output all API results for specified API options or defaults --pycharm-debug ``` - -**example #2** - a predefined query option +--- +**example #2** - a predefined query option (default provider = azure) ``` -ed-sle12sp3byos:/home/lane/cliapi # cliapi --internal-ip --provider=azure +ed-sle12sp3byos:/home/lane/cliapi # cliapi --internal-ip "172.16.3.8" ``` - -**example #3** - all values returned by all provider APIs +--- +**example #3** - all values returned by all APIs (default provider = azure) ``` -ed-sle12sp3byos:/home/lane/cliapi # cliapi --all --provider=azure +ed-sle12sp3byos:/home/lane/cliapi # cliapi --all { "tag": "391e4f53-b82d-5af5-8f58-dc1035e46e5e", "meta_data": { @@ -149,23 +150,30 @@ ed-sle12sp3byos:/home/lane/cliapi # cliapi --all --provider=azure }, "cloud-service": "__ed-sle12sp3byosService.cloudapp.net" } - ``` - -**example #4** - an ad-hoc restricted python dictionary syntax query +--- +**example #4** - list of APIs supported by plugin provider (default provider = azure) +``` +ed-sle12sp3byos:/home/lane/cliapi # cliapi --list-apis +[ + "tag", + "cloud-service", + "meta_data" +] +``` +--- +**example #5** - an ad-hoc restricted python dictionary syntax query (provider='test') ``` lane@suse-laptop:~/develop/garage/cliapi> cliapi --query="['meta_data']['compute']['offer']" --provider=test "SLES-BYOS" - ``` - -**example #5** - multiple queries (single CLI call) +--- +**example #6** - mixed multiple queries with a single CLI call (provider='test') ``` lane@suse-laptop:~/develop/garage/cliapi> cliapi --location --query="['meta_data']['compute']['offer']" --provider=test [ "westus", "SLES-BYOS" ] - ``` diff --git a/design.md b/design.md index c4e82fe..3a2439b 100644 --- a/design.md +++ b/design.md @@ -81,10 +81,10 @@ https://docs.google.com/spreadsheets/d/1i_phns6QS3eCmsWFXWxxcafuNYXOEqG0ffxt6NHZ **Total Dev Edited LOC for Existing Implementation:** -**~2177 Total LOC** = ec2metadata + azuremetadata(perl) + gcemetadata +**`~2177 Total LOC`** = ec2metadata + azuremetadata(perl) + gcemetadata **Total Dev Edited LOC for Proposal #3 Implementation from above** -**~881 Total LOC** = cliapi(framework actual) + azure(plugin actual) + gce(plugin estimate) + ec2(plugin estimate) +**`~881 Total LOC`** = cliapi(framework actual) + azure(plugin actual) + gce(plugin estimate) + ec2(plugin estimate) From 269c21f3276796e612742728ba6e4f11668dcaa8 Mon Sep 17 00:00:00 2001 From: ed lane Date: Wed, 29 Aug 2018 10:35:48 -0600 Subject: [PATCH 18/29] more, better docs --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2f580e0..c3f98a6 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,7 @@ ed-sle12sp3byos:/home/lane/cliapi # cliapi --internal-ip ``` ed-sle12sp3byos:/home/lane/cliapi # cliapi --all { - "tag": "391e4f53-b82d-5af5-8f58-dc1035e46e5e", + "tag": "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx", "meta_data": { "network": { "interface": [ @@ -133,7 +133,7 @@ ed-sle12sp3byos:/home/lane/cliapi # cliapi --all "compute": { "offer": "SLES-BYOS", "publisher": "SUSE", - "subscriptionId": "ce73a2b0-d2e7-4ff6-b987-b32d6908de4e", + "subscriptionId": "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx", "name": "ed-sle12sp3byos", "platformUpdateDomain": "0", "version": "2018.02.21", From 100400f2cded024f0af591b643c3bcc57720ea4a Mon Sep 17 00:00:00 2001 From: ed lane Date: Wed, 29 Aug 2018 17:30:02 -0600 Subject: [PATCH 19/29] removing an evil eval() --- cliapi/cliapi.py | 32 +++++++++++++++++--------------- cliapi/cliapi_lib.py | 42 +++++++++++++++++++++++------------------- 2 files changed, 40 insertions(+), 34 deletions(-) diff --git a/cliapi/cliapi.py b/cliapi/cliapi.py index 029ad64..f4b09b8 100755 --- a/cliapi/cliapi.py +++ b/cliapi/cliapi.py @@ -51,7 +51,6 @@ 'list-providers', "list-apis", 'query=', 'all', - 'pycharm-debug', ] # 'xml-out'] @@ -197,22 +196,25 @@ def main(): # return all specified "data scoops"... if key in vpp.scoops: query = vpp.scoops[key] - try: - data.append(_sandbox_eval(vpp, query)) - except KeyError as e: - # error -- print help and exit - print('required option -- "{}" missing'.format(e.args[0])) - _print_help(provider) - # cmd_dict['help'] = None - # exit(-1) elif key == 'query': query = cmd_dict['query'] - try: - data.append(_sandbox_eval(vpp, query)) - except SyntaxError as e: - # error in query - print('error in query -- "{}" syntax error'.format(query)) - _print_help(provider) + else: + # ignore option, continue processing... + continue + try: + data.append(_sandbox_eval(vpp, query)) + except KeyError as e: + # error -- print help and exit + print('required option -- "{}" missing'.format(e.args[0])) + except SyntaxError as e: + # error in query + print('error in query -- "{}" syntax error'.format(query)) + else: + # no errors, continue processing... + continue + # must have encounered an error print help and abort... + _print_help(provider) + if len(data) == 1: # a single value was requested so no list is required... data = data[0] diff --git a/cliapi/cliapi_lib.py b/cliapi/cliapi_lib.py index 9802d87..7fa3b23 100644 --- a/cliapi/cliapi_lib.py +++ b/cliapi/cliapi_lib.py @@ -3,7 +3,7 @@ from string import Template class Provider(dict): - def __init__(self, fetchers={}): + def __init__(self): self.fetchers = {} self.scoops = {} self.options = {} @@ -17,18 +17,22 @@ def __getitem__(self, item): try: result = super().__getitem__(item) except KeyError: - # restrict python's eval() function -- No builtins, only "item" is accessible - # api = eval('[' + item.replace(']', '],') + ']', - # {'__builtins__': None}, - # {'item': item})[0][0] - fetcher = self.fetchers[item].split('.') - provider = importlib.import_module('.'.join(fetcher[0:3])) + fetcher = self.fetchers[item] + fetcher_function = fetcher[0].split('.') + # fetcher = self.fetchers[item].split('.') + provider = importlib.import_module('.'.join(fetcher_function[0:3])) - super().__setitem__(item, eval('provider.' + - Template(fetcher[3]).substitute(self.template))) - # restrict python's eval() functon -- No builtins, only "self" is accessible + arg_list = list() + kwarg_dict = dict() + for arg in fetcher[1][0]: + # build the *args tuple... + arg_list.append(Template(arg).substitute(self.template)) + for key, value in fetcher[1][1].items(): + # build the **kwargs dictionary... + kwarg_dict[key] = Template(value).substitute(self.template) + function = getattr(provider, fetcher_function[3]) + super().__setitem__(item, function(*arg_list, **kwarg_dict)) result = self.get(item) - # result = eval("self" + item, {'__builtins__': None}, {'self': self}) return result @@ -48,25 +52,25 @@ def decorate_it(func): else: default_count = len(argspec.defaults) last_arg = len(argspec.args) - default_count - argspec_string = '(' + func_args = [] + func_kwargs = {} for i, arg in enumerate(argspec.args): if i < last_arg: # substitute CLI options... arg = prov.options.get(arg, arg) - argspec_string += '\'' + arg + '\', ' + func_args.append(arg) else: default = argspec.defaults[i - last_arg] # substitute CLI options... - # op_default = prov.options.get(arg, default) op_default = prov.options.get(arg, default) if op_default.startswith('$'): prov.template[op_default[1:]] = default - argspec_string += arg + '=\'' + op_default + '\', ' + func_kwargs[arg] = op_default - prov.fetchers.update({alias : - func.__module__ + '.' + - func.__name__ + - argspec_string + ')' + prov.fetchers.update({alias: + (func.__module__ + '.' + + func.__name__, + (func_args, func_kwargs)) }) def wrap_it(*args, **kwargs): From 159fe9a0c4fd59aea242fde119859e692f287e04 Mon Sep 17 00:00:00 2001 From: ed lane Date: Wed, 29 Aug 2018 17:41:54 -0600 Subject: [PATCH 20/29] '--pycharm-debug' is no longer a CLI option -- now exclusively enabled by env variable 'PYCHARM_DEBUG_ME' --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index c3f98a6..61c399c 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,6 @@ Common CLI options: --list-apis list all available APIs for specified provider --query= specify a python dictionary style query command --all output all API results for specified API options or defaults - --pycharm-debug ``` --- **example #2** - a predefined query option (default provider = azure) From f6b07e3f148eae2721639d932ad2d28229270689 Mon Sep 17 00:00:00 2001 From: ed lane Date: Fri, 31 Aug 2018 10:20:49 -0600 Subject: [PATCH 21/29] better clarification of python eval() sandboxing --- README.md | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 61c399c..296038c 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -## GUIDE FOR CLIAPI DEVELOPERS: +## Guide for CLIAPI Plugin Developers: **cliapi** ( pronounced _calliope_ ): A Python framework for creating Unix, "getopt()" style CLI scripts composed from an arbitrary set of APIs. @@ -29,7 +29,10 @@ Several common commands are automatically inherited by all plugin providers: - multiple data queries within the same CLI will result in "at most" a single API call. -### Key cliapi developer concepts: +### Key cliapi plugin developer concepts: + +An example cliapi plugin provider for Azuremeta API + SUSE extensions can be found here: +https://github.com/edlane/cliapi/blob/master/cliapi/providers/azure.py - **provider**: an associated set of APIs accessed from the CLI in a particular context or cloud environment e.g. "azure", "gce", "ec2", ... but can essentially be any mix of apis @@ -40,9 +43,12 @@ object, THEN it can easily become a configurable CLI query. With the cliapi dec optional parameters are expressible through the CLI. Help is also handled by the cliapi framework. - **scoops**: a dictionary which maps a CLI query name to a particular API data scoop. "scoops" are really just -"_sandboxed_ python _eval()_" statements. This allows scoops to be expressed as Python slices, -comprehensions, ect. It also allows restricted ad-hoc queries to be provided on the command line -when a return value is not currently supported as an option in the CLI. +"_sandboxed_ python _eval()_" statements. This allows predefined scoops to be expressed as CLI options. +It also allows for restricted ad-hoc queries to be provided on the command line when a return value is not +currently supported as an option in the CLI. The query uses Python's dictionary lookup syntax for slice +slice**s** (See examples using **--query=** below). +Python's _eval()_ function allows limiting access to a single data structure and "no builtins" through +this facility: - **fetchers**: a dictionary which maps from a particular API name to the actual python function which provides the _backing store_ for the contents of the top-level API dictionary. @@ -151,7 +157,16 @@ ed-sle12sp3byos:/home/lane/cliapi # cliapi --all } ``` --- -**example #4** - list of APIs supported by plugin provider (default provider = azure) +**example #4** - list of valid plugin providers +``` +ed-sle12sp3byos:/home/lane/cliapi # cliapi --list-providers --provider=azure +[ + "test", + "azure" +] +``` +--- +**example #5** - list of APIs supported by plugin provider (default provider = azure) ``` ed-sle12sp3byos:/home/lane/cliapi # cliapi --list-apis [ @@ -161,13 +176,13 @@ ed-sle12sp3byos:/home/lane/cliapi # cliapi --list-apis ] ``` --- -**example #5** - an ad-hoc restricted python dictionary syntax query (provider='test') +**example #6** - an ad-hoc restricted python dictionary syntax query (provider='test') ``` lane@suse-laptop:~/develop/garage/cliapi> cliapi --query="['meta_data']['compute']['offer']" --provider=test "SLES-BYOS" ``` --- -**example #6** - mixed multiple queries with a single CLI call (provider='test') +**example #7** - mixed multiple queries with a single CLI call (provider='test') ``` lane@suse-laptop:~/develop/garage/cliapi> cliapi --location --query="['meta_data']['compute']['offer']" --provider=test [ From fe6fb1f539915a414d557c692532b660c1235b05 Mon Sep 17 00:00:00 2001 From: ed lane Date: Fri, 31 Aug 2018 10:22:56 -0600 Subject: [PATCH 22/29] better error handling and help --- cliapi/cliapi.py | 53 +++++++++++++++++++++++++++++----------- cliapi/providers/test.py | 14 +++++------ 2 files changed, 46 insertions(+), 21 deletions(-) diff --git a/cliapi/cliapi.py b/cliapi/cliapi.py index f4b09b8..7acd50e 100755 --- a/cliapi/cliapi.py +++ b/cliapi/cliapi.py @@ -86,6 +86,10 @@ def _print_help(provider): vpp = valid_providers[provider].provider # update help with options from this provider... vpp.help.update(help) + required = list() + for fetcher in vpp.fetchers.items(): + # create a list of required options for fast lookup... + required += fetcher[1][1][0] indent2_format = ' --{:<15} {:<15}' print("usage: {} [display option#1]... [API option#1]... [CLI option]".format(sys.argv[0])) print("\n***[ {} ]*** provider Display options:".format(provider)) @@ -97,7 +101,12 @@ def _print_help(provider): # list API configuration options for this provider... if v.startswith('$'): opt = v[1:] - print (indent2_format.format(opt + '=', vpp.help.get(opt, ''))) + help = vpp.help.get(opt,'') + if v in required: + # if this option is required, say so... + help = 'REQUIRED, ' + help + + print (indent2_format.format(opt + '=', help)) print("\nCommon CLI options:") for key in cli_options: # list of common CLI options supported by this module, across all providers @@ -114,12 +123,12 @@ def _sandbox_eval(vpp, lookup): def main(): # must do our own parsing here since we don't know the - # valid options until we establish the provider + # valid options until we establish the plugin provider # -- getopt() does not allow this usage... ct = _cli_parse(sys.argv[1:]) if '--list-providers' in ct: - print('valid providers =\n', json.dumps(list(valid_providers.keys()), indent=2)) + print(json.dumps(list(valid_providers.keys()), indent=2)) exit(0) cmd_dict = {} @@ -144,7 +153,7 @@ def main(): options.append(v[1:] + '=') all_opts += options - # add "cli_options" to allowed CLI options... + # add "plugin cli_options" to the "common" CLI options... all_opts += cli_options try: @@ -166,7 +175,7 @@ def main(): ds = default.split('=') try: if not cmd_dict.get(ds[0]): - # override with default + # override with default... cmd_dict[ds[0]] = ds[1] except: # no option to override or none specified -- ignore this... @@ -183,11 +192,17 @@ def main(): if 'all' in cmd_dict: # return a composite dictionary of all API calls... for fetch in vpp.fetchers: - # fetch all the APIs + # prefetch all the APIs... scoop = "['" + fetch + "']" - # must populate dictionary with a fake query... - query = _sandbox_eval(vpp, scoop) - # return all the API results as a dictionary + # must populate dictionary by walking the top-level API... + try: + query = _sandbox_eval(vpp, scoop) + except Exception as e: + # error caused by plugin developer... + print(e.args[0]) + _print_help(provider) + exit(-1) + # return all the API results from dictionary... data = vpp else: @@ -199,24 +214,34 @@ def main(): elif key == 'query': query = cmd_dict['query'] else: - # ignore option, continue processing... + # Not a scoop or a query? + # then ignore option, continue processing... continue try: + # accumulate all requested scoops data.append(_sandbox_eval(vpp, query)) except KeyError as e: # error -- print help and exit - print('required option -- "{}" missing'.format(e.args[0])) + if key == 'query': + # a bogus CLI option was input by user... + print(('bad query -- "{}", no such item').format(query)) + else: + # a bogus scoop option was specified by plugin developer... + print(('option {} -- "{}", bad plugin query').format(key, query)) + except AssertionError as e: + # a required option is missing... + print(e.args[0]) except SyntaxError as e: - # error in query + # a syntax error detected in query... print('error in query -- "{}" syntax error'.format(query)) else: # no errors, continue processing... continue - # must have encounered an error print help and abort... + # must have encountered an error, print help and abort... _print_help(provider) if len(data) == 1: - # a single value was requested so no list is required... + # a single value was returned so no list is returned... data = data[0] print(json.dumps(data, indent=2)) diff --git a/cliapi/providers/test.py b/cliapi/providers/test.py index 5be8693..2f2e621 100644 --- a/cliapi/providers/test.py +++ b/cliapi/providers/test.py @@ -1,5 +1,5 @@ -from cliapi.cliapi_lib import Provider, cliapi_decor +from cliapi.cliapi_lib import Provider, cliapi_assembler provider = Provider() @@ -9,18 +9,17 @@ } help = { - 'smurf': 'required, what is smurf name?', + 'smurf': 'what is smurf name?', 'lorax': 'who is the lorax?', 'dweebville': 'where is dweebville?', } options = dict(arg1='$smurf', key1='$dufus', key2='$dweebville',) -@cliapi_decor(provider, api_alias='test', scoops=scoops, help=help, options=options) -def dork(arg1, key1='foo', key2='bar'): +@cliapi_assembler(provider, api_alias='test', scoops=scoops, help=help, options=options) +def foo(arg1, key1='bar', key2='baz'): return [arg1, key1, key2] - scoops = { 'instance-name': "['meta_data']['compute']['name']", 'mac': "['meta_data']['network']['interface'][0]['macAddress']", @@ -29,6 +28,7 @@ def dork(arg1, key1='foo', key2='bar'): "['ipv4']['ipAddress'][0]['publicIpAddress']", 'internal-ip': "['meta_data']['network']['interface'][0]" "['ipv4']['ipAddress'][0]['privateIpAddress']", + 'always-fail':"['intentionally-fail']", } help = { @@ -37,7 +37,7 @@ def dork(arg1, key1='foo', key2='bar'): 'mac': 'the MAC address for this interface', } -@cliapi_decor(provider, api_alias='meta_data', scoops=scoops, help=help) +@cliapi_assembler(provider, api_alias='meta_data', scoops=scoops, help=help) def get_meta_data_mock(): return {"compute": {"location": "westus", "name": "ed-sle12sp3byos", "offer": "SLES-BYOS", @@ -58,6 +58,6 @@ def get_meta_data_mock(): options = dict(api_version='$api_version') -@cliapi_decor(provider, api_alias='some_stuff', scoops=scoops, help=help, options=options) +@cliapi_assembler(provider, api_alias='some_stuff', scoops=scoops, help=help, options=options) def get_stuff(api_version='2017-08-01'): return api_version From 455d791d3ebcf617e70814842cfd9fb562aa69a9 Mon Sep 17 00:00:00 2001 From: ed lane Date: Fri, 31 Aug 2018 10:26:02 -0600 Subject: [PATCH 23/29] added lots more comments to cliapi_lib.py --- cliapi/cliapi_lib.py | 68 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 53 insertions(+), 15 deletions(-) diff --git a/cliapi/cliapi_lib.py b/cliapi/cliapi_lib.py index 7fa3b23..117d731 100644 --- a/cliapi/cliapi_lib.py +++ b/cliapi/cliapi_lib.py @@ -3,6 +3,14 @@ from string import Template class Provider(dict): + # The Provider class overrides Python's dictionary with + # an API-backing-store. + # When a KeyError is encountered during lookup, + # a fetcher, or "API call", is prepared with the appropriate CLI + # options substituted as python function arguments. + # CLI "help" and predefined dictionary lookups, or "scoops", + # are also assembled in this class... + def __init__(self): self.fetchers = {} self.scoops = {} @@ -17,35 +25,59 @@ def __getitem__(self, item): try: result = super().__getitem__(item) except KeyError: + # this dictionary item does not currently exist + # so fetch it using the API which resolves to + # the top level dictionary with that API key name... fetcher = self.fetchers[item] fetcher_function = fetcher[0].split('.') - # fetcher = self.fetchers[item].split('.') + + # setup for calling the appropriate plugin API... provider = importlib.import_module('.'.join(fetcher_function[0:3])) arg_list = list() kwarg_dict = dict() - for arg in fetcher[1][0]: - # build the *args tuple... - arg_list.append(Template(arg).substitute(self.template)) - for key, value in fetcher[1][1].items(): - # build the **kwargs dictionary... - kwarg_dict[key] = Template(value).substitute(self.template) - function = getattr(provider, fetcher_function[3]) - super().__setitem__(item, function(*arg_list, **kwarg_dict)) + try: + # replace API args and kwargs with CLI provided options... + for arg in fetcher[1][0]: + # build the *args tuple... + arg_list.append(Template(arg).substitute(self.template)) + for key, value in fetcher[1][1].items(): + # build the **kwargs dictionary... + kwarg_dict[key] = Template(value).substitute(self.template) + function = getattr(provider, fetcher_function[3]) + # fault-in the API values... + super().__setitem__(item, function(*arg_list, **kwarg_dict)) + except Exception as e: + raise AssertionError('required option {}, missing'.format(e)) + # rerun the dictionary lookup and return results... result = self.get(item) return result -def cliapi_decor(prov, api_alias=None, scoops={}, options={}, help={}): - def decorate_it(func): +def cliapi_assembler(prov, api_alias=None, scoops={}, options={}, help={}): + # Compile each API supported by the plugin provider as a python + # function and assemble into the various dictionaries of + # the "Provider" class. + # These dictionaries are later used by the "cliapi engine" + # in "cliapi.main()" to create a Unix, getopt()-style CLI... + + def assemble_it(func): + # REMEMBER: All this work happens at "import time"... if api_alias is None: alias = func.__name__ else: alias = api_alias - # prov.scoops=alias + # assemble more scoops... prov.scoops.update(scoops) + # assemble more plugin options... prov.options.update(options) + # assemble more help... prov.help.update(help) + + # Here is some meta-magic... + # use introspection to extract calling requirements + # for this particular "API fetcher" and save in fetchers + # dictionary for later when it is actually called... argspec = inspect.getargspec(func) if argspec.defaults is None: default_count = 0 @@ -67,13 +99,19 @@ def decorate_it(func): prov.template[op_default[1:]] = default func_kwargs[arg] = op_default + # Register this function as an "API fetcher"... prov.fetchers.update({alias: (func.__module__ + '.' + func.__name__, (func_args, func_kwargs)) }) - def wrap_it(*args, **kwargs): + def do_it(*args, **kwargs): + # REMEMBER: All this work happens at call time... + # TODO: this is a hack + # at a "Universal Python Function Wrapper". + # There is probably a better way to do this but it + # eludes me at the moment... if len(args) == 0: a = () else: @@ -83,6 +121,6 @@ def wrap_it(*args, **kwargs): else: b = kwargs return func(*a, **b) - return wrap_it + return do_it - return decorate_it + return assemble_it From b5f9b460fa26ba7b0d836647da4c26c5096f7cb0 Mon Sep 17 00:00:00 2001 From: ed lane Date: Fri, 31 Aug 2018 10:29:00 -0600 Subject: [PATCH 24/29] fix 'location' bug and rename cliapi decorator --- cliapi/providers/azure.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cliapi/providers/azure.py b/cliapi/providers/azure.py index 089653e..db26025 100644 --- a/cliapi/providers/azure.py +++ b/cliapi/providers/azure.py @@ -1,4 +1,4 @@ -from cliapi.cliapi_lib import Provider, cliapi_decor +from cliapi.cliapi_lib import Provider, cliapi_assembler from cliapi.what_cloud import determine_provider import json @@ -7,8 +7,8 @@ import urllib.request, urllib.error, urllib.parse import xml.etree.ElementTree as ET -#----- raises an error on import if not running in a "proper" cloud... try: + # ----- raises an error on import if not running in a "proper" cloud... cp = determine_provider() if cp != 'azure': raise Exception("not a valid provider") @@ -20,7 +20,7 @@ scoops = { 'instance-name': "['meta_data']['compute']['name']", 'mac': "['meta_data']['network']['interface'][0]['macAddress']", - 'location': "meta_data['compute']['location']", + 'location': "['meta_data']['compute']['location']", 'external-ip': "['meta_data']['network']['interface'][0]" "['ipv4']['ipAddress'][0]['publicIpAddress']", 'internal-ip': "['meta_data']['network']['interface'][0]" @@ -38,7 +38,7 @@ options = dict(api_version='$api_version') -@cliapi_decor(provider, api_alias='meta_data', scoops=scoops, help=help, options=options) +@cliapi_assembler(provider, api_alias='meta_data', scoops=scoops, help=help, options=options) def get_meta_data_azure(api_version='2017-08-01'): HEADERS = {'Metadata': 'true'} IP = '169.254.169.254' @@ -49,7 +49,7 @@ def get_meta_data_azure(api_version='2017-08-01'): return result -@cliapi_decor(provider, api_alias='cloud-service') +@cliapi_assembler(provider, api_alias='cloud-service') def get_cloud_service(): # tree = ET.parse('/home/lane/Downloads/SharedConfig.xml') tree = ET.parse('/var/lib/waagent/SharedConfig.xml') @@ -58,7 +58,7 @@ def get_cloud_service(): return cloud_service -@cliapi_decor(provider, api_alias='tag') +@cliapi_assembler(provider, api_alias='tag') def read_billing_guid(device='/dev/sda'): fd = os.open(device, os.O_RDONLY) os.lseek(fd, 65536, os.SEEK_SET) From c40f8f54b08bf9a561657036cd8bfb9a252a84de Mon Sep 17 00:00:00 2001 From: ed lane Date: Fri, 31 Aug 2018 14:34:24 -0600 Subject: [PATCH 25/29] display 'REQUIRED' and 'defaults=' in help --- cliapi/cliapi.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/cliapi/cliapi.py b/cliapi/cliapi.py index 7acd50e..15d0af0 100755 --- a/cliapi/cliapi.py +++ b/cliapi/cliapi.py @@ -102,9 +102,15 @@ def _print_help(provider): if v.startswith('$'): opt = v[1:] help = vpp.help.get(opt,'') + if help != '': + # help message is provided by plugin provider... + help = ''.join((', ', help) ) if v in required: - # if this option is required, say so... - help = 'REQUIRED, ' + help + # if this option is required, then say so... + help = ''.join(('-REQUIRED-', help)) + elif opt in vpp.template: + # otherwise display the default... + help = ''.join(('default=\'', vpp.template[opt], '\'', help)) print (indent2_format.format(opt + '=', help)) print("\nCommon CLI options:") @@ -112,7 +118,7 @@ def _print_help(provider): # list of common CLI options supported by this module, across all providers print (indent2_format.format(key, vpp.help.get(key, ''))) - exit(0) + exit(-1) def _sandbox_eval(vpp, lookup): From eb167132ae2fb9b0ecea1f2be3b0e9f59d8a0379 Mon Sep 17 00:00:00 2001 From: ed lane Date: Fri, 31 Aug 2018 14:44:17 -0600 Subject: [PATCH 26/29] plugin help cleanup --- cliapi/providers/azure.py | 2 +- cliapi/providers/test.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/cliapi/providers/azure.py b/cliapi/providers/azure.py index db26025..62a0aca 100644 --- a/cliapi/providers/azure.py +++ b/cliapi/providers/azure.py @@ -32,7 +32,7 @@ 'instance-name': 'name of instance', 'location': 'region location', 'mac': 'the MAC address for this interface', - 'api_version': 'azure metadata api version, default = \'2017-08-01\'', + 'api_version': 'azure metadata api version', 'cloud-service': 'what' } diff --git a/cliapi/providers/test.py b/cliapi/providers/test.py index 2f2e621..eab3837 100644 --- a/cliapi/providers/test.py +++ b/cliapi/providers/test.py @@ -10,7 +10,6 @@ help = { 'smurf': 'what is smurf name?', - 'lorax': 'who is the lorax?', 'dweebville': 'where is dweebville?', } From 8ab6fb846a1809564cede3f77cde8da5571c92ae Mon Sep 17 00:00:00 2001 From: ed lane Date: Fri, 31 Aug 2018 14:55:06 -0600 Subject: [PATCH 27/29] update examples for current code base --- README.md | 50 +++++++++++++++++++++++++------------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 296038c..d13f78b 100644 --- a/README.md +++ b/README.md @@ -81,14 +81,14 @@ usage: /usr/bin/cliapi [display option#1]... [API option#1]... [CLI option] ***[ azure ]*** provider Display options: --internal-ip + --location region location --cloud-service what --instance-name name of instance - --external-ip --mac the MAC address for this interface - --location region location + --external-ip ***[ azure ]*** provider API config options: - --api_version= azure metadata api version, default = '2017-08-01' + --api_version= default='2017-08-01', azure metadata api version Common CLI options: --help help for this CLI command @@ -109,8 +109,24 @@ ed-sle12sp3byos:/home/lane/cliapi # cliapi --internal-ip ``` ed-sle12sp3byos:/home/lane/cliapi # cliapi --all { - "tag": "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx", "meta_data": { + "compute": { + "location": "westus", + "vmSize": "Standard_B1ms", + "osType": "Linux", + "platformUpdateDomain": "0", + "sku": "12-SP3", + "name": "ed-sle12sp3byos", + "placementGroupId": "", + "resourceGroupName": "ed_lane", + "offer": "SLES-BYOS", + "vmId": "ce01dc32-6d0a-40bd-9534-a3509f768a53", + "tags": "", + "subscriptionId": "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx", + "version": "2018.02.21", + "publisher": "SUSE", + "platformFaultDomain": "0" + }, "network": { "interface": [ { @@ -126,40 +142,24 @@ ed-sle12sp3byos:/home/lane/cliapi # cliapi --all ], "subnet": [ { - "address": "172.16.3.0", - "prefix": "24" + "prefix": "24", + "address": "172.16.3.0" } ] }, "macAddress": "000D3A3AE8A5" } ] - }, - "compute": { - "offer": "SLES-BYOS", - "publisher": "SUSE", - "subscriptionId": "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx", - "name": "ed-sle12sp3byos", - "platformUpdateDomain": "0", - "version": "2018.02.21", - "placementGroupId": "", - "sku": "12-SP3", - "tags": "", - "vmId": "ce01dc32-6d0a-40bd-9534-a3509f768a53", - "resourceGroupName": "ed_lane", - "osType": "Linux", - "location": "westus", - "platformFaultDomain": "0", - "vmSize": "Standard_B1ms" } }, - "cloud-service": "__ed-sle12sp3byosService.cloudapp.net" + "cloud-service": "__ed-sle12sp3byosService.cloudapp.net", + "tag": "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" } ``` --- **example #4** - list of valid plugin providers ``` -ed-sle12sp3byos:/home/lane/cliapi # cliapi --list-providers --provider=azure +ed-sle12sp3byos:/home/lane/cliapi # cliapi --list-providers [ "test", "azure" From 09a67d49d9b4e3670afc92a81c2afbc7893e9808 Mon Sep 17 00:00:00 2001 From: ed lane Date: Fri, 31 Aug 2018 15:06:12 -0600 Subject: [PATCH 28/29] change decorator name from 'cliapi_assembler' to 'cliapi_compile' just because it sounds better --- cliapi/cliapi_lib.py | 2 +- cliapi/providers/azure.py | 8 ++++---- cliapi/providers/test.py | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cliapi/cliapi_lib.py b/cliapi/cliapi_lib.py index 117d731..0136191 100644 --- a/cliapi/cliapi_lib.py +++ b/cliapi/cliapi_lib.py @@ -54,7 +54,7 @@ def __getitem__(self, item): return result -def cliapi_assembler(prov, api_alias=None, scoops={}, options={}, help={}): +def cliapi_compile(prov, api_alias=None, scoops={}, options={}, help={}): # Compile each API supported by the plugin provider as a python # function and assemble into the various dictionaries of # the "Provider" class. diff --git a/cliapi/providers/azure.py b/cliapi/providers/azure.py index 62a0aca..3bb7ee6 100644 --- a/cliapi/providers/azure.py +++ b/cliapi/providers/azure.py @@ -1,4 +1,4 @@ -from cliapi.cliapi_lib import Provider, cliapi_assembler +from cliapi.cliapi_lib import Provider, cliapi_compile from cliapi.what_cloud import determine_provider import json @@ -38,7 +38,7 @@ options = dict(api_version='$api_version') -@cliapi_assembler(provider, api_alias='meta_data', scoops=scoops, help=help, options=options) +@cliapi_compile(provider, api_alias='meta_data', scoops=scoops, help=help, options=options) def get_meta_data_azure(api_version='2017-08-01'): HEADERS = {'Metadata': 'true'} IP = '169.254.169.254' @@ -49,7 +49,7 @@ def get_meta_data_azure(api_version='2017-08-01'): return result -@cliapi_assembler(provider, api_alias='cloud-service') +@cliapi_compile(provider, api_alias='cloud-service') def get_cloud_service(): # tree = ET.parse('/home/lane/Downloads/SharedConfig.xml') tree = ET.parse('/var/lib/waagent/SharedConfig.xml') @@ -58,7 +58,7 @@ def get_cloud_service(): return cloud_service -@cliapi_assembler(provider, api_alias='tag') +@cliapi_compile(provider, api_alias='tag') def read_billing_guid(device='/dev/sda'): fd = os.open(device, os.O_RDONLY) os.lseek(fd, 65536, os.SEEK_SET) diff --git a/cliapi/providers/test.py b/cliapi/providers/test.py index eab3837..4f4619b 100644 --- a/cliapi/providers/test.py +++ b/cliapi/providers/test.py @@ -1,5 +1,5 @@ -from cliapi.cliapi_lib import Provider, cliapi_assembler +from cliapi.cliapi_lib import Provider, cliapi_compile provider = Provider() @@ -15,7 +15,7 @@ options = dict(arg1='$smurf', key1='$dufus', key2='$dweebville',) -@cliapi_assembler(provider, api_alias='test', scoops=scoops, help=help, options=options) +@cliapi_compile(provider, api_alias='test', scoops=scoops, help=help, options=options) def foo(arg1, key1='bar', key2='baz'): return [arg1, key1, key2] @@ -36,7 +36,7 @@ def foo(arg1, key1='bar', key2='baz'): 'mac': 'the MAC address for this interface', } -@cliapi_assembler(provider, api_alias='meta_data', scoops=scoops, help=help) +@cliapi_compile(provider, api_alias='meta_data', scoops=scoops, help=help) def get_meta_data_mock(): return {"compute": {"location": "westus", "name": "ed-sle12sp3byos", "offer": "SLES-BYOS", @@ -57,6 +57,6 @@ def get_meta_data_mock(): options = dict(api_version='$api_version') -@cliapi_assembler(provider, api_alias='some_stuff', scoops=scoops, help=help, options=options) +@cliapi_compile(provider, api_alias='some_stuff', scoops=scoops, help=help, options=options) def get_stuff(api_version='2017-08-01'): return api_version From 62dfe63c9013cd0cb37e1996964b51f02f39b42f Mon Sep 17 00:00:00 2001 From: ed lane Date: Wed, 17 Oct 2018 19:11:26 -0600 Subject: [PATCH 29/29] azuremetadata rework for SUSE/Enceladus suitibility --- README.md | 168 ++++++++------- cliapi/cliapi.py | 204 ++++++++++-------- cliapi/cliapi_lib.py | 33 +-- .../providers/{azure.py => azuremetadata.py} | 22 +- cliapi/providers/test.py | 27 +-- design.md | 90 -------- 6 files changed, 238 insertions(+), 306 deletions(-) rename cliapi/providers/{azure.py => azuremetadata.py} (64%) delete mode 100644 design.md diff --git a/README.md b/README.md index d13f78b..97bfbd7 100644 --- a/README.md +++ b/README.md @@ -21,8 +21,20 @@ Several common commands are automatically inherited by all plugin providers: - "**--provider=**" specify a particular provider plugin for all CLI commands. - "**--all**" Lists all data returned by all APIs supported by a single provider. - "**--list-apis**" Lists all the APIs available by a particular plugin provider. - - "**--query=**" extracts specified API data using a python-dictionary-restricted syntax. + - "**--item=**" Display an item from API by it's right-most JSON path specifier. + For example: + - "**--item=location**" will return the value for **meta_data.compute.location** + - "**--item=privateIpAddress**" will return a list of all keys ending in "**privateIpAddress**" + to get a particular element from a list the item should be referenced using it's unique right-most JSON path name, **i.e**: + ``` + root # azuremetadata --item=interface[0].macAddress + "000D3A3AE8A5" + ``` - other behaviors enforced by the cliapi framework: + - flexible provider handling based on the way cliapi is invoked: + - if invoked using "cliapi" name then API provider defaults to first valid plugin imported. + - if invoked using _non-cliapi_ name then API provider defaults to the plugin provider with same _non-cliapi_ name. + - the default provider can be over ridden using "--provider=" config option. - error handling and help is also consistent across all plugin providers. - multiple queries in same command will return a JSON list in "option order" by default - single query will return a single JSON element. @@ -42,14 +54,6 @@ a python function with an (_*args, **kwargs_) style calling convention AND it re object, THEN it can easily become a configurable CLI query. With the cliapi decorator, both required and optional parameters are expressible through the CLI. Help is also handled by the cliapi framework. -- **scoops**: a dictionary which maps a CLI query name to a particular API data scoop. "scoops" are really just -"_sandboxed_ python _eval()_" statements. This allows predefined scoops to be expressed as CLI options. -It also allows for restricted ad-hoc queries to be provided on the command line when a return value is not -currently supported as an option in the CLI. The query uses Python's dictionary lookup syntax for slice -slice**s** (See examples using **--query=** below). -Python's _eval()_ function allows limiting access to a single data structure and "no builtins" through -this facility: - - **fetchers**: a dictionary which maps from a particular API name to the actual python function which provides the _backing store_ for the contents of the top-level API dictionary. @@ -62,7 +66,7 @@ provides the _backing store_ for the contents of the top-level API dictionary. │   └── what_cloud.py module (useful for detecting which cloud plugins are valid) │   ├── providers directory for plugin providers │   │   ├── __init__.py -│   │   ├── azure.py plugin for Azure/SUSE APIs +│   │   ├── azuremetadata.py plugin for Azure/SUSE APIs │   │   ├── ...your plugin goes here │   │   ├── ... ...additional plugins are automatically discovered by cliapi │   │   └── test.py ..."because it's not a framework without at least 2 plugins" @@ -74,101 +78,118 @@ provides the _backing store_ for the contents of the top-level API dictionary. ### Examples: -**example #1** - Common help AND plugin provider help (default provider = azure) -``` -ed-sle12sp3byos:/home/lane/cliapi # cliapi --help -usage: /usr/bin/cliapi [display option#1]... [API option#1]... [CLI option] - -***[ azure ]*** provider Display options: - --internal-ip - --location region location - --cloud-service what - --instance-name name of instance - --mac the MAC address for this interface - --external-ip - -***[ azure ]*** provider API config options: - --api_version= default='2017-08-01', azure metadata api version +**example #1** - Common help AND plugin provider help +``` +ed-sle12sp3byos:/home/lane/cliapi # azuremetadata --help +usage: ./azuremetadata --item=[API display item #1]... [API config option#1]... [API required options]... [Common CLI options]... + +***[ azuremetadata ]*** provider Display options: + cloud-service + privateIpAddress + publicIpAddress + address + prefix + macAddress + osType + version + placementGroupId + vmId + sku + platformUpdateDomain + subscriptionId + offer + name + tags + resourceGroupName + vmSize + publisher + platformFaultDomain + location + tag + +***[ azuremetadata ]*** provider API config options: + --api_version= default='2017-08-01' Common CLI options: + --list-apis list all available APIs for specified provider + --item= value to return from an API using right-most path name specifier + --list-providers list all available providers --help help for this CLI command + --all output all API results for a specific provider --provider= specify name of provider module - --list-providers list all available providers - --list-apis list all available APIs for specified provider - --query= specify a python dictionary style query command - --all output all API results for specified API options or defaults + ``` --- -**example #2** - a predefined query option (default provider = azure) +**example #2** - a single display item ``` -ed-sle12sp3byos:/home/lane/cliapi # cliapi --internal-ip +ed-sle12sp3byos:/home/lane/cliapi # azuremetadata --item=privateIpAddress "172.16.3.8" ``` --- -**example #3** - all values returned by all APIs (default provider = azure) +**example #3** - all values returned by azuremetadata APIs ``` -ed-sle12sp3byos:/home/lane/cliapi # cliapi --all +ed-sle12sp3byos:/home/lane/cliapi # ./azuremetadata --all { "meta_data": { - "compute": { - "location": "westus", - "vmSize": "Standard_B1ms", - "osType": "Linux", - "platformUpdateDomain": "0", - "sku": "12-SP3", - "name": "ed-sle12sp3byos", - "placementGroupId": "", - "resourceGroupName": "ed_lane", - "offer": "SLES-BYOS", - "vmId": "ce01dc32-6d0a-40bd-9534-a3509f768a53", - "tags": "", - "subscriptionId": "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx", - "version": "2018.02.21", - "publisher": "SUSE", - "platformFaultDomain": "0" - }, "network": { "interface": [ { - "ipv6": { - "ipAddress": [] - }, + "macAddress": "000D3A3AE8A5", "ipv4": { - "ipAddress": [ + "subnet": [ { - "publicIpAddress": "40.112.253.198", - "privateIpAddress": "172.16.3.8" + "address": "172.16.3.0", + "prefix": "24" } ], - "subnet": [ + "ipAddress": [ { - "prefix": "24", - "address": "172.16.3.0" + "privateIpAddress": "172.16.3.8", + "publicIpAddress": "40.112.253.198" } ] }, - "macAddress": "000D3A3AE8A5" + "ipv6": { + "ipAddress": [] + } } ] + }, + "compute": { + "offer": "SLES-BYOS", + "placementGroupId": "", + "publisher": "SUSE", + "sku": "12-SP3", + "osType": "Linux", + "platformFaultDomain": "0", + "platformUpdateDomain": "0", + "tags": "", + "vmSize": "Standard_B1ms", + "location": "westus", + "name": "ed-sle12sp3byos", + "subscriptionId": "ce73a2b0-d2e7-4ff6-b987-b32d6908de4e", + "version": "2018.02.21", + "vmId": "ce01dc32-6d0a-40bd-9534-a3509f768a53", + "resourceGroupName": "ed_lane" } }, "cloud-service": "__ed-sle12sp3byosService.cloudapp.net", - "tag": "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" + "tag": "391e4f53-b82d-5af5-8f58-dc1035e46e5e" } ``` --- **example #4** - list of valid plugin providers ``` -ed-sle12sp3byos:/home/lane/cliapi # cliapi --list-providers +ed-sle12sp3byos:/home/lane/cliapi # azuremetadata --list-providers [ "test", - "azure" + "azuremetadata" ] ``` --- -**example #5** - list of APIs supported by plugin provider (default provider = azure) +**example #5** - list of APIs supported by plugin provider ``` -ed-sle12sp3byos:/home/lane/cliapi # cliapi --list-apis +ed-sle12sp3byos:/home/lane/cliapi # azuremetadata --list-apis [ "tag", "cloud-service", @@ -176,18 +197,17 @@ ed-sle12sp3byos:/home/lane/cliapi # cliapi --list-apis ] ``` --- -**example #6** - an ad-hoc restricted python dictionary syntax query (provider='test') -``` -lane@suse-laptop:~/develop/garage/cliapi> cliapi --query="['meta_data']['compute']['offer']" --provider=test -"SLES-BYOS" -``` ---- -**example #7** - mixed multiple queries with a single CLI call (provider='test') +**example #6** - return several display items in single call ``` -lane@suse-laptop:~/develop/garage/cliapi> cliapi --location --query="['meta_data']['compute']['offer']" --provider=test +ed-sle12sp3byos:/home/lane/cliapi # ./azuremetadata --item=privateIpAddress --item=publisher --item=vmSize --item=location --item=name --item=offer --item=interface[0].macAddress [ + "172.16.3.8", + "SUSE", + "Standard_B1ms", "westus", - "SLES-BYOS" + "ed-sle12sp3byos", + "SLES-BYOS", + "000D3A3AE8A5" ] ``` diff --git a/cliapi/cliapi.py b/cliapi/cliapi.py index 15d0af0..5679b68 100755 --- a/cliapi/cliapi.py +++ b/cliapi/cliapi.py @@ -4,8 +4,10 @@ # https://github.com/edlane/cliapi/blob/master/README.md import os + if 'PYCHARM_DEBUG_ME' in os.environ: import pydevd + # we want debugging enabled up top because much of the cliapi framework # "import-meta-magic" happens before __main__() has ever been called... # @@ -26,6 +28,7 @@ import pkgutil import getopt import json +from collections import defaultdict import cliapi.providers as providers from cliapi.cliapi_lib import Provider @@ -43,68 +46,68 @@ vp = importlib.import_module(pro) valid_providers[module_name] = vp except Exception as e: + print("failure to import {}, {}".format(pro, e)) pass -# TODO: does "xml-out" really need to be implemented in a modern Devops world??? -cli_options = ['help', - 'provider=', - 'list-providers', - "list-apis", - 'query=', 'all', - ] - # 'xml-out'] +# Help for Common CLI options... +common_help = { + 'help': 'help for this CLI command', + 'provider=': 'specify name of provider module', + 'list-providers': 'list all available providers', + # TODO: does "xml-out" really need to be implemented in a modern Devops world??? + # 'xml-out': 'output in xml format', + 'list-apis': 'list all available APIs for specified provider', + 'all': 'output all API results for a specific provider', + 'item=': 'value to return from an API using right-most path name specifier' +} def _cli_parse(argv): cli_tree = dict() for arg in argv: kv = arg.split('=') - if len(kv)>1: + if len(kv) > 1: cli_tree[kv[0]] = kv[1] else: cli_tree[kv[0]] = None - return cli_tree + return cli_tree def _print_help(provider): + global common_help # Help for Common CLI options... - help = { - 'help': 'help for this CLI command', - 'provider=': 'specify name of provider module', - 'list-providers': 'list all available providers', - 'xml-out': 'output in xml format', - 'list-apis': 'list all available APIs for specified provider', - 'query=': 'specify a python dictionary style query command', - 'all': 'output all API results for specified API options or defaults', - } if provider == None: vpp = Provider() # no provider supplied so at least print out the "common" help... - vpp.help = help + vpp.help = common_help else: vpp = valid_providers[provider].provider - # update help with options from this provider... - vpp.help.update(help) required = list() for fetcher in vpp.fetchers.items(): # create a list of required options for fast lookup... - required += fetcher[1][1][0] + required += fetcher[Provider.FUNC_PARMS][Provider.FUNC_KWARGS][Provider.FUNC_REQUIRED] indent2_format = ' --{:<15} {:<15}' - print("usage: {} [display option#1]... [API option#1]... [CLI option]".format(sys.argv[0])) + print( + "usage: {} --item=[API display item #1]... [API config option#1]... [API required options]... [Common CLI options]...".format( + sys.argv[0])) print("\n***[ {} ]*** provider Display options:".format(provider)) - for key in vpp.scoops.keys(): - # list display options for this provider... - print (indent2_format.format(key, vpp.help.get(key, ''))) + try: + _fetch_all_apis(vpp) + except AssertionError as e: + print(e.args[0], 'required options must be provided to list all displayable API options') + + all_apis = _scan_apis(vpp) + _print_api_options(all_apis) print("\n***[ {} ]*** provider API config options:".format(provider)) for k, v in vpp.options.items(): # list API configuration options for this provider... if v.startswith('$'): opt = v[1:] - help = vpp.help.get(opt,'') + help = vpp.help.get(opt, '') if help != '': # help message is provided by plugin provider... - help = ''.join((', ', help) ) + help = ''.join((', ', help)) if v in required: # if this option is required, then say so... help = ''.join(('-REQUIRED-', help)) @@ -114,20 +117,60 @@ def _print_help(provider): print (indent2_format.format(opt + '=', help)) print("\nCommon CLI options:") - for key in cli_options: + for key in common_help.keys(): # list of common CLI options supported by this module, across all providers - print (indent2_format.format(key, vpp.help.get(key, ''))) + print (indent2_format.format(key, common_help.get(key, ''))) + - exit(-1) +def _fetch_all_apis(vpp): + # return a composite dictionary of all API calls... + for fetch in vpp.fetchers: + # prefetch all the APIs... + # must populate the vpp dictionary by walking the top-level API... + try: + vpp[fetch] + except Exception as e: + raise e + # return all the API results from dictionary... + return vpp + + +def _get_value_by_endswith(api_list, endswith): + for path in api_list: + if path[0].endswith(endswith): + return path[1] + + +def _scan_apis(vpp, path='', path_list=[]): + leaf = None + if isinstance(vpp, dict): + for element in vpp.keys(): + leaf = _scan_apis(vpp[element], path + '.' + element, path_list) + if isinstance(leaf, tuple): + path_list.append((list(leaf[1]))) + elif isinstance(vpp, list): + index = 0 + for element in vpp: + leaf = _scan_apis(element, path + '[' + str(index) + ']', path_list) + index += 1 + if isinstance(leaf, tuple): + path_list.append((list(leaf[1]))) + else: + # print(path, vpp ) + return path_list, (path, vpp) + return path_list -def _sandbox_eval(vpp, lookup): - return eval('vpp' + lookup, - {'__builtins__': None}, - {'vpp': vpp}) + +def _print_api_options(path_list): + for element in path_list: + split_element = element[0].split('.') + element_len = len(split_element) + print(2 * element_len * ' ', split_element[-1]) def main(): + global common_help # must do our own parsing here since we don't know the # valid options until we establish the plugin provider # -- getopt() does not allow this usage... @@ -137,34 +180,43 @@ def main(): print(json.dumps(list(valid_providers.keys()), indent=2)) exit(0) - cmd_dict = {} + cmd_dict = defaultdict(list) + _, cli_name = os.path.split(sys.argv[0]) if '--provider' in ct: # using supplied provider... provider = ct['--provider'] - else: + elif cli_name == 'cliapi': try: # use the first valid provider in list... provider = list(valid_providers.keys())[0] except Exception as e: # error -- print help and exit - print('No plugin providers found') + print('No valid plugin providers found') + _print_help(None) + exit(-1) + else: + # the name of calling program is used to specify the provider... + if cli_name in list(valid_providers.keys()): + provider = cli_name + else: + print('No plugin provider for {} was found'.format(cli_name)) _print_help(None) exit(-1) vpp = valid_providers[provider].provider - all_opts = list(vpp.scoops.keys()) + all_opts = [] options = [] - for k, v in vpp.options.items(): # process args... + for k, v in vpp.options.items(): # process args... if v.startswith('$'): options.append(v[1:] + '=') all_opts += options - # add "plugin cli_options" to the "common" CLI options... - all_opts += cli_options + # add "common" CLI options to the plugin's cli_options... + all_opts += common_help.keys() try: # now we have enough info to use getopt() for cli parsing... - optlist, arg = getopt.gnu_getopt(sys.argv[1:], '', all_opts) + optlist, arg = getopt.getopt(sys.argv[1:], '', all_opts) except getopt.GetoptError as e: # error -- print help and exit print(e.msg) @@ -172,7 +224,12 @@ def main(): else: # build a dictionary of the actual supplied CLI options... for opt in optlist: - cmd_dict[opt[0][2:]] = opt[1] + if opt[0] == '--item': + # allow for many "items" in defaultdict as a list... + # skipping the leading '--' + cmd_dict[opt[0][2:]].append(opt[1]) + else: + cmd_dict[opt[0][2:]] = opt[1] vpp.template.update(cmd_dict) @@ -196,55 +253,28 @@ def main(): exit(0) if 'all' in cmd_dict: - # return a composite dictionary of all API calls... - for fetch in vpp.fetchers: - # prefetch all the APIs... - scoop = "['" + fetch + "']" - # must populate dictionary by walking the top-level API... - try: - query = _sandbox_eval(vpp, scoop) - except Exception as e: - # error caused by plugin developer... - print(e.args[0]) - _print_help(provider) - exit(-1) - # return all the API results from dictionary... - data = vpp + data = _fetch_all_apis(vpp) else: data = [] for key, value in cmd_dict.items(): - # return all specified "data scoops"... - if key in vpp.scoops: - query = vpp.scoops[key] - elif key == 'query': - query = cmd_dict['query'] - else: - # Not a scoop or a query? - # then ignore option, continue processing... + if key == 'item': + try: + _fetch_all_apis(vpp) + except AssertionError as e: + _print_help(provider) + exit(-1) + all_apis = _scan_apis(vpp) + for item in cmd_dict['item']: + # ensure that last name is a complete item by prepending '.' ... + endswith = '.' + item + item = _get_value_by_endswith(all_apis, endswith) + data.append(item) continue - try: - # accumulate all requested scoops - data.append(_sandbox_eval(vpp, query)) - except KeyError as e: - # error -- print help and exit - if key == 'query': - # a bogus CLI option was input by user... - print(('bad query -- "{}", no such item').format(query)) - else: - # a bogus scoop option was specified by plugin developer... - print(('option {} -- "{}", bad plugin query').format(key, query)) - except AssertionError as e: - # a required option is missing... - print(e.args[0]) - except SyntaxError as e: - # a syntax error detected in query... - print('error in query -- "{}" syntax error'.format(query)) else: - # no errors, continue processing... + # Not an item? + # then ignore option, continue processing... continue - # must have encountered an error, print help and abort... - _print_help(provider) if len(data) == 1: # a single value was returned so no list is returned... diff --git a/cliapi/cliapi_lib.py b/cliapi/cliapi_lib.py index 0136191..c094a3d 100644 --- a/cliapi/cliapi_lib.py +++ b/cliapi/cliapi_lib.py @@ -2,18 +2,26 @@ import importlib from string import Template + class Provider(dict): # The Provider class overrides Python's dictionary with # an API-backing-store. # When a KeyError is encountered during lookup, # a fetcher, or "API call", is prepared with the appropriate CLI # options substituted as python function arguments. - # CLI "help" and predefined dictionary lookups, or "scoops", - # are also assembled in this class... + + # offsets needed by framework to access the values in the + # "fetcher" dictionary... + FUNC_FULL_NAME = 0 + FUNC_PARMS = 1 + FUNC_ARGS = 0 + FUNC_KWARGS = 1 + FUNC_CALLABLE = 2 + FUNC_NAME = 3 + FUNC_REQUIRED = 0 def __init__(self): self.fetchers = {} - self.scoops = {} self.options = {} self.help = {} self.template = {} @@ -29,7 +37,7 @@ def __getitem__(self, item): # so fetch it using the API which resolves to # the top level dictionary with that API key name... fetcher = self.fetchers[item] - fetcher_function = fetcher[0].split('.') + fetcher_function = fetcher[Provider.FUNC_FULL_NAME].split('.') # setup for calling the appropriate plugin API... provider = importlib.import_module('.'.join(fetcher_function[0:3])) @@ -38,14 +46,14 @@ def __getitem__(self, item): kwarg_dict = dict() try: # replace API args and kwargs with CLI provided options... - for arg in fetcher[1][0]: + for arg in fetcher[Provider.FUNC_PARMS][Provider.FUNC_ARGS]: # build the *args tuple... arg_list.append(Template(arg).substitute(self.template)) - for key, value in fetcher[1][1].items(): + for key, value in fetcher[Provider.FUNC_PARMS][Provider.FUNC_KWARGS].items(): # build the **kwargs dictionary... kwarg_dict[key] = Template(value).substitute(self.template) - function = getattr(provider, fetcher_function[3]) - # fault-in the API values... + function = getattr(provider, fetcher_function[Provider.FUNC_NAME]) + # fault-in the API values... super().__setitem__(item, function(*arg_list, **kwarg_dict)) except Exception as e: raise AssertionError('required option {}, missing'.format(e)) @@ -54,7 +62,7 @@ def __getitem__(self, item): return result -def cliapi_compile(prov, api_alias=None, scoops={}, options={}, help={}): +def cliapi_compile(prov, api_alias=None, options={}, help={}): # Compile each API supported by the plugin provider as a python # function and assemble into the various dictionaries of # the "Provider" class. @@ -67,8 +75,6 @@ def assemble_it(func): alias = func.__name__ else: alias = api_alias - # assemble more scoops... - prov.scoops.update(scoops) # assemble more plugin options... prov.options.update(options) # assemble more help... @@ -96,6 +102,7 @@ def assemble_it(func): # substitute CLI options... op_default = prov.options.get(arg, default) if op_default.startswith('$'): + # skipping the leading '$'... prov.template[op_default[1:]] = default func_kwargs[arg] = op_default @@ -103,7 +110,8 @@ def assemble_it(func): prov.fetchers.update({alias: (func.__module__ + '.' + func.__name__, - (func_args, func_kwargs)) + (func_args, func_kwargs), + func) }) def do_it(*args, **kwargs): @@ -121,6 +129,7 @@ def do_it(*args, **kwargs): else: b = kwargs return func(*a, **b) + return do_it return assemble_it diff --git a/cliapi/providers/azure.py b/cliapi/providers/azuremetadata.py similarity index 64% rename from cliapi/providers/azure.py rename to cliapi/providers/azuremetadata.py index 3bb7ee6..0013ce1 100644 --- a/cliapi/providers/azure.py +++ b/cliapi/providers/azuremetadata.py @@ -17,28 +17,10 @@ provider = Provider() -scoops = { - 'instance-name': "['meta_data']['compute']['name']", - 'mac': "['meta_data']['network']['interface'][0]['macAddress']", - 'location': "['meta_data']['compute']['location']", - 'external-ip': "['meta_data']['network']['interface'][0]" - "['ipv4']['ipAddress'][0]['publicIpAddress']", - 'internal-ip': "['meta_data']['network']['interface'][0]" - "['ipv4']['ipAddress'][0]['privateIpAddress']", - 'cloud-service': "['get_cloud_service']" -} - -help = { - 'instance-name': 'name of instance', - 'location': 'region location', - 'mac': 'the MAC address for this interface', - 'api_version': 'azure metadata api version', - 'cloud-service': 'what' -} - options = dict(api_version='$api_version') -@cliapi_compile(provider, api_alias='meta_data', scoops=scoops, help=help, options=options) + +@cliapi_compile(provider, api_alias='meta_data', options=options) def get_meta_data_azure(api_version='2017-08-01'): HEADERS = {'Metadata': 'true'} IP = '169.254.169.254' diff --git a/cliapi/providers/test.py b/cliapi/providers/test.py index 4f4619b..8265719 100644 --- a/cliapi/providers/test.py +++ b/cliapi/providers/test.py @@ -3,40 +3,21 @@ provider = Provider() -scoops = { - 'test': "['test']", - 'meta-data': "['meta_data']", -} help = { 'smurf': 'what is smurf name?', 'dweebville': 'where is dweebville?', } +# options = dict(arg1='hello', key1='$dufus', key2='$dweebville',) options = dict(arg1='$smurf', key1='$dufus', key2='$dweebville',) -@cliapi_compile(provider, api_alias='test', scoops=scoops, help=help, options=options) +@cliapi_compile(provider, api_alias='test', help=help, options=options) def foo(arg1, key1='bar', key2='baz'): return [arg1, key1, key2] -scoops = { - 'instance-name': "['meta_data']['compute']['name']", - 'mac': "['meta_data']['network']['interface'][0]['macAddress']", - 'location': "['meta_data']['compute']['location']", - 'external-ip': "['meta_data']['network']['interface'][0]" - "['ipv4']['ipAddress'][0]['publicIpAddress']", - 'internal-ip': "['meta_data']['network']['interface'][0]" - "['ipv4']['ipAddress'][0]['privateIpAddress']", - 'always-fail':"['intentionally-fail']", -} - -help = { - 'instance-name': 'name of instance', - 'location': 'region location', - 'mac': 'the MAC address for this interface', -} -@cliapi_compile(provider, api_alias='meta_data', scoops=scoops, help=help) +@cliapi_compile(provider, api_alias='meta_data') def get_meta_data_mock(): return {"compute": {"location": "westus", "name": "ed-sle12sp3byos", "offer": "SLES-BYOS", @@ -57,6 +38,6 @@ def get_meta_data_mock(): options = dict(api_version='$api_version') -@cliapi_compile(provider, api_alias='some_stuff', scoops=scoops, help=help, options=options) +@cliapi_compile(provider, api_alias='some_stuff', options=options) def get_stuff(api_version='2017-08-01'): return api_version diff --git a/design.md b/design.md deleted file mode 100644 index 3a2439b..0000000 --- a/design.md +++ /dev/null @@ -1,90 +0,0 @@ -# Design Considerations: -**Task:** port azuremetadata.pl (perl) to python - -**Requirements:** -(Or rather, the Methodology used to Generate requirements): -1. compare azuremetadata CLI with the other metadata CLI modules for other cloud providers (ec2 and gce) -2. Identify and interview current users of metadata modules (Sean) - -**Observations:** -- The 3 CLI metadata modules appear to have been written at 3 different times by 3 different -authors. -- Subtle and not so subtle differences exist in the CLI behaviors and output formats. -This requires developers to implement separate handlers and be aware of the idiosyncrasies -of each implementation. -- The GCE and EC2 modules both use getopt() and are largely familiar API except for -the returned results. -- Azure (the target of this port) is expectedly weird with different API, results, and configuration parameters -required (...and because it was written in Perl) -- It looks like the list of available and supported CLI query options for each cloud provider -has been added to over time. -To allow a new item to be queried from the CLI, the cloud module must also -be revised in order to access it. -- There is also a practice of adding other SUSE specific functions into the "cloud metadata" -rather than breaking them into separate CLIs. e.g "cloud-service", "billing-tag", etc. -- The meta-data collection of CLIs seems primarily to support automation pipelines elsewhere -in Enceladus and not meant exclusively for humans. -As such these CLIs are expected to have consistent behavior across all the meta-data CLIs -approaching that of an API rather than an ad-hoc query tool. Current best practice for API -publishers would seem to favor a "discoverable-style" APIs over traditional "man-page" described -ones". - -**Proposal 1:** - -- This is a mostly solved problem with Salt's grain module -so implement azuremetadata entirely using Salt's metadata grains module -- **Note:** Currently the Salt grains metadata module needs to be extended to support Azure. -(estimate ~40 LOC for Azure support) -- add a "cloud-service" and other SUSE required functionality as custom grains modules (~20 LOC) -- replace other parts of Enceladus by Salt automation when duplicate functionality exists. -- develop custom Salt modules, runners, grains, states, beacons, .... to -implement Enceladus functionality where appropriate. -**(I am advised that "advocating this proposal would be _GREATLY OPPOSED_" -- so -dropping this proposal immediately")** - -**alternative #2 to Proposal 1:** -- Allow salt grains to be called directly from CLI. This would allow salt grains to be called -as an executable module via **_salt-call_** (new functionality ~10 LOC) -- support other SUSE specific functions as separate **salt-call**-_able_ modules -rather than separate CLI programs. -This suffers from many of the objections of alternative #1 and would break existing consumers of -the old interface. - -**Proposal 2:** - -- Port azuremetadata.pl "as is" and create yet another slightly inconsistent CLI, but coded in Python instead. - -**alternative #2 to Proposal 2:** - -- Choose an existing cloud CLI (gce or ec2) and emulate that one as much as possible that same -behavior for **azuremetadata.py** -- For extra credit: Try to reconcile all the behavior differences in the various CLIs and -bring them back into compliance (at -least for one instance in time before the inevitable "code drift" takes hold) - - -**Proposal 3:** - -- Create a common, extensible framework which will generate CLIs given a set of APIs -described as python functions using introspection. -- Enable backward compatibility by supporting all existing CLI options. -- Enable forward flexibility by adding a query-language extension which allows recently -added or previously unanticipated metadata API values to be queried. - - -### LOC Implementation Comparison for Proposal #3 above: -**Note:** Admittedly comparing developer editable lines of code between different implementations -is an unprecise and debatable activity but here is a worksheet used to prepare this report anyway... - - -https://docs.google.com/spreadsheets/d/1i_phns6QS3eCmsWFXWxxcafuNYXOEqG0ffxt6NHZysk/edit?usp=sharing - -**Total Dev Edited LOC for Existing Implementation:** - -**`~2177 Total LOC`** = ec2metadata + azuremetadata(perl) + gcemetadata - - -**Total Dev Edited LOC for Proposal #3 Implementation from above** - -**`~881 Total LOC`** = cliapi(framework actual) + azure(plugin actual) + gce(plugin estimate) + ec2(plugin estimate) -