diff --git a/PKG-INFO b/PKG-INFO new file mode 100644 index 0000000..e4b38b4 --- /dev/null +++ b/PKG-INFO @@ -0,0 +1,34 @@ +Metadata-Version: 1.2 +Name: pyaws +Version: 0.4.1 +Summary: Python Utilities for Amazon Web Services +Home-page: http://pyaws.readthedocs.io +Author: Blake Huber +Author-email: blakeca00@gmail.com +License: GPL-3.0 +Description: + **pyaws** | Utilities Library for Amazon Web Services (AWS) + ----------------------------------------------------------- + + PACKAGE: pyaws + + ``pyaws``: reusable library of utility classes and functions common AWS use cases and capabilities: + + * uploading to s3 + * adding/ deleting resource tags + * adding data elements to dynamodb table + * Determining the latest Amazon Machine Image in a region for Windows, Linux, etc + + + +Keywords: Amazon Web Services AWS iam ec2 lambda rds s3 sts +Platform: UNKNOWN +Classifier: Topic :: System :: Systems Administration +Classifier: Topic :: Utilities +Classifier: Development Status :: 4 - Beta +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3) +Classifier: Operating System :: POSIX :: Linux +Requires-Python: >=3.6, <4 diff --git a/build/lib/pyaws/__init__.py b/build/lib/pyaws/__init__.py new file mode 100644 index 0000000..18b93c8 --- /dev/null +++ b/build/lib/pyaws/__init__.py @@ -0,0 +1,22 @@ +from pyaws._version import __version__ as version + + +__author__ = 'Blake Huber' +__version__ = version +__credits__ = [] +__license__ = "GPL-3.0" +__maintainer__ = "Blake Huber" +__email__ = "blakeca00@gmail.com" +__status__ = "Development" + + +# the following imports require __version__ # + +try: + import logging + + # shared, global logger object + logger = logging.getLogger(__version__) + +except Exception: + pass diff --git a/build/lib/pyaws/_version.py b/build/lib/pyaws/_version.py new file mode 100644 index 0000000..f0ede3d --- /dev/null +++ b/build/lib/pyaws/_version.py @@ -0,0 +1 @@ +__version__ = '0.4.1' diff --git a/build/lib/pyaws/awslambda/__init__.py b/build/lib/pyaws/awslambda/__init__.py new file mode 100644 index 0000000..e0c2535 --- /dev/null +++ b/build/lib/pyaws/awslambda/__init__.py @@ -0,0 +1,6 @@ +""" +Functional Utilities for AWS Lambda +""" + +from pyaws.awslambda.lambda_utils import * +from pyaws.awslambda.env import read_env_variable diff --git a/build/lib/pyaws/awslambda/env.py b/build/lib/pyaws/awslambda/env.py new file mode 100644 index 0000000..2a2c3b2 --- /dev/null +++ b/build/lib/pyaws/awslambda/env.py @@ -0,0 +1,90 @@ +""" + +lambda_utils (python3) + + Common functionality for use with AWS Lambda Service + +Author: + Blake Huber + Copyright Blake Huber, All Rights Reserved. + +License: + + MIT License. + Additional terms may be found in the complete license agreement: + https://opensource.org/licenses/MIT + + Project README: + https://github.com/fstab50/pyaws/blob/master/README.md +""" + +import os +import re +import inspect +from pyaws import logger + + +def read_env_variable(arg, default=None, patterns=None): + """ + Summary. + + Parse environment variables, validate characters, convert + type(s). default should be used to avoid conversion of an + variable and retain string type + + Usage: + >>> from lambda_utils import read_env_variable + >>> os.environ['DBUGMODE'] = 'True' + >>> myvar = read_env_variable('DBUGMODE') + >>> type(myvar) + True + + >>> from lambda_utils import read_env_variable + >>> os.environ['MYVAR'] = '1345' + >>> myvar = read_env_variable('MYVAR', 'default') + >>> type(myvar) + str + + Args: + :arg (str): Environment variable name (external name) + :default (str): Default if no variable found in the environment under + name in arg parameter + :patterns (None): Unused; not user callable. Used preservation of the + patterns tuple between calls during runtime + + Returns: + environment variable value, TYPE str + + """ + if patterns is None: + patterns = ( + (re.compile('^[-+]?[0-9]+$'), int), + (re.compile('\d+\.\d+'), float), + (re.compile(r'^(true|false)$', flags=re.IGNORECASE), lambda x: x.lower() == 'true'), + (re.compile('[a-z/]+', flags=re.IGNORECASE), str), + (re.compile('[a-z/]+\.[a-z/]+', flags=re.IGNORECASE), str), + ) + + if arg in os.environ: + var = os.environ[arg] + if var is None: + ex = KeyError('environment variable %s not set' % arg) + logger.exception(ex) + raise ex + else: + if default: + return str(var) # force default type (str) + else: + for pattern, func in patterns: + if pattern.match(var): + return func(var) + # type not identified + logger.warning( + '%s: failed to identify environment variable [%s] type. May contain \ + special characters' % (inspect.stack()[0][3], arg) + ) + return str(var) + else: + ex = KeyError('environment variable %s not set' % arg) + logger.exception(ex) + raise ex diff --git a/build/lib/pyaws/awslambda/lambda_utils.py b/build/lib/pyaws/awslambda/lambda_utils.py new file mode 100644 index 0000000..c6b33bf --- /dev/null +++ b/build/lib/pyaws/awslambda/lambda_utils.py @@ -0,0 +1,144 @@ +""" + +lambda_utils (python3) + + Common functionality for use with AWS Lambda Service + +Author: + Blake Huber + Copyright Blake Huber, All Rights Reserved. + +License: + + MIT License. + Additional terms may be found in the complete license agreement: + https://opensource.org/licenses/MIT + + Project README: + https://github.com/fstab50/pyaws/blob/master/README.md +""" + + +import os +import re +import json +import time +import inspect +import boto3 +from botocore.exceptions import ClientError +from pyaws import logger + + +def get_account_info(account_profile=None): + """ + Summary. + + Queries AWS iam and sts services to discover account id information + in the form of account name and account alias (if assigned) + + Returns: + TYPE: tuple + + Example usage: + + >>> account_number, account_name = lambda_utils.get_account_info() + >>> print(account_number, account_name) + 103562488773 tooling-prod + + """ + if account_profile: + session = boto3.Session(profile_name=account_profile) + sts_client = session.client('sts') + iam_client = session.client('iam') + else: + sts_client = boto3.client('sts') + iam_client = boto3.client('iam') + + try: + number = sts_client.get_caller_identity()['Account'] + name = iam_client.list_account_aliases()['AccountAliases'][0] + + except IndexError as e: + name = '' + logger.info('Error: %s. No account alias defined. account_name set to %s' % (e, name)) + return (number, name) + except ClientError as e: + logger.warning( + "%s: problem retrieving caller identity (Code: %s Message: %s)" % + (inspect.stack()[0][3], e.response['Error']['Code'], e.response['Error']['Message']) + ) + raise e + return (number, name) + + +def get_regions(): + """ + Summary. + + Returns list of region codes for all AWS regions worldwide + + Returns: + TYPE: list + + """ + try: + client = boto3.client('ec2') + region_response = client.describe_regions() + regions = [region['RegionName'] for region in region_response['Regions']] + + except ClientError as e: + logger.critical( + "%s: problem retrieving aws regions (Code: %s Message: %s)" % + (inspect.stack()[0][3], e.response['Error']['Code'], + e.response['Error']['Message'])) + raise e + return regions + + +def sns_notification(topic_arn, subject, message, account_id=None, account_name=None): + """ + Summary. + + Sends message to AWS sns service topic provided as a + parameter + + Args: + topic_arn (str): sns topic arn + subject (str): subject of sns message notification + message (str): message body + + Returns: + TYPE: Boolean | Success or Failure + + """ + if not (account_id or account_name): + account_id, account_name = get_account_info() + + # assemble msg + header = 'AWS Account: %s (%s) | %s' % \ + (str(account_name).upper(), str(account_id), subject) + msg = '\n%s\n\n%s' % (time.strftime('%c'), message) + msg_dict = {'default': msg} + + # client + region = (topic_arn.split('sns:', 1)[1]).split(":", 1)[0] + client = boto3.client('sns', region_name=region) + + try: + # sns publish + response = client.publish( + TopicArn=topic_arn, + Subject=header, + Message=json.dumps(msg_dict), + MessageStructure='json' + ) + if response['ResponseMetadata']['HTTPStatusCode'] == '200': + return True + else: + return False + except ClientError as e: + logger.exception( + '%s: problem sending sns msg (Code: %s Message: %s)' % + (inspect.stack()[0][3], e.response['Error']['Code'], + e.response['Error']['Message'])) + return False diff --git a/build/lib/pyaws/colors.py b/build/lib/pyaws/colors.py new file mode 100644 index 0000000..d7dd01f --- /dev/null +++ b/build/lib/pyaws/colors.py @@ -0,0 +1,97 @@ +""" +Summary: + ANSI color and formatting code class + See: http://www.lihaoyi.com/post/BuildyourownCommandLinewithANSIescapecodes.html#256-colors + + VERSION: 1.14 + +Args: + None + +Returns: + ansi codes + +Raises: + None. AttributeError if no code match returns the reset ansi codes + +""" + + +class Colors(): + """ + Class attributes provide different format variations + """ + # forground colors + AQUA = '\u001b[38;5;14m' + PURPLE = '\033[95m' + CYAN = '\033[96m' + DARK_CYAN = '\033[36m' + BLUE = '\033[94m' + DARK_BLUE = '\033[38;5;95;38;5;24m' + PURE_BLUE = '\u001b[38;5;27m' + GREEN = '\033[92m' + DARK_GREEN = '\u001b[38;5;2m' + YELLOW = '\033[93m' + RED = '\033[91m' + ORANGE = '\033[38;5;95;38;5;214m' + GOLD2 = '\033[38;5;95;38;5;136m' # yellow-orange, lighter than GOLD3 + GOLD3 = '\033[38;5;95;38;5;178m' # yellow-orange + WHITE = '\033[37m' + WHITE_GRAY = '\033[38;5;95;38;5;250m' # white-gray + LT1GRAY = '\033[38;5;95;38;5;245m' # light gray + LT2GRAY = '\033[38;5;95;38;5;246m' + LT3GRAY = '\u001b[38;5;249m' + DARK_GRAY1 = '\033[90m' + DARK_GRAY2 = '\033[38;5;95;38;5;8m' # darkest gray + + # bright colors + BRIGHT_BLUE = '\033[38;5;51m' + BRIGHT_CYAN = '\033[38;5;36m' + BRIGHT_GREEN = '\033[38;5;95;38;5;46m' + BRIGHT_PURPLE = '\033[38;5;68m' + BRIGHT_RED = '\u001b[31;1m' + BRIGHT_YELLOW = '\033[38;5;11m' + BRIGHT_YELLOW2 = '\033[38;5;95;38;5;226m' + BRIGHT_YELLOWGREEN = '\033[38;5;95;38;5;155m' + BRIGHT_WHITE = '\033[38;5;15m' + + # background colors + BKGND_BLACK = '\u001b[0m' + BKGND_GREY = '\u001b[40m' + BKGND_BLUE = '\u001b[44m' + BKGND_CYAN = '\u001b[46m' + BKGND_GREEN = '\u001b[42m' + BKGND_MAGENTA = '\u001b[45m' + BKGND_RED = '\u001b[41m' + BKGND_WHITE = '\u001b[47m' + BKGND_WHITE_BOLD = '\u001b[47;1m' + BKGND_YELLOW = '\u001b[43m' + + # background colors; bright + BKGND_BRIGHT_BLACK = '\u001b[40;1m' + BKGND_BRIGHT_BLUE = '\u001b[44;1m' + BKGND_BRIGHT_CYAN = '\u001b[46;1m' + BKGND_BRIGHT_GREEN = '\u001b[42;1m' + BKGND_BRIGHT_GRAY = '\u001b[38;1m' + BKGND_BRIGHT_MAGENTA = '\u001b[45;1m' + BKGND_BRIGHT_RED = '\u001b[41;1m' + BKGND_BRIGHT_WHITE = '\u001b[47;1m' + BKGND_BRIGHT_YELLOW = '\u001b[43;1m' + + # formats + BOLD = '\033[1m' + UNBOLD = '\033[22m' + UNDERLINE = '\033[4m' + ITALIC = '\e[3m' + END = '\033[0m' + REVERSE = '\033[;7m' + RESET = '\033[0;0m' + RESET_ALT = '\u001b[0m' + CLEARSCN = '\033[2J' + + # special formats + URL = UNDERLINE + CYAN + TITLE = UNDERLINE + BOLD + + #except AttributeError as e: + # logger.info('Ansi color code not found (%s), returning reset code' % str(e)) diff --git a/build/lib/pyaws/core/__init__.py b/build/lib/pyaws/core/__init__.py new file mode 100644 index 0000000..18f9100 --- /dev/null +++ b/build/lib/pyaws/core/__init__.py @@ -0,0 +1,10 @@ +""" +Common AWS Functionality required to support all services +""" + +try: + + from pyaws.core.oscodes_unix import exit_codes + +except Exception: + from pyaws.core.oscodes_win import exit_codes diff --git a/build/lib/pyaws/core/create_client.py b/build/lib/pyaws/core/create_client.py new file mode 100644 index 0000000..baf468f --- /dev/null +++ b/build/lib/pyaws/core/create_client.py @@ -0,0 +1,44 @@ +def create_client(service, account=None, role=None): + """ + Summary: + Creates the appropriate boto3 client for a particular AWS service + Args: + :type service: str + :param service: name of service at Amazon Web Services (AWS), + e.g. s3, ec2, etc + :type credentials: sts credentials object + :param credentials: authentication credentials to resource in AWS + :type role: str + :param role: IAM role designation used to access AWS resources + in an account + Returns: + Boto3 Client Object + """ + try: + if role and account: # create client for a different AWS account + account_obj = AssumeAWSRole(account, role) + if account_obj.status.get('STATUS') == 'SUCCESS': + credentials = account_obj.credentials + client = boto3.client( + service, + aws_access_key_id=credentials['AccessKeyId'], + aws_secret_access_key=credentials['SecretAccessKey'], + aws_session_token=credentials['SessionToken'] + ) + else: + logger.critical('failed to create client - Error: %s' % + str(account_obj.status.get('STATUS'))) + raise ClientError( + '%s: Problem creating client using role %s' % + (inspect.stack()[0][3], str(role)) + ) + else: + return boto3.client(service) # create client in the current AWS account + except ClientError as e: + logger.exception( + "%s: Problem creating client %s in account %s (Code: %s Message: %s)" % + (inspect.stack()[0][3], role, account, e.response['Error']['Code'], + e.response['Error']['Message']) + ) + raise + return client diff --git a/build/lib/pyaws/core/cross_account_utils.py b/build/lib/pyaws/core/cross_account_utils.py new file mode 100644 index 0000000..2953e09 --- /dev/null +++ b/build/lib/pyaws/core/cross_account_utils.py @@ -0,0 +1,110 @@ +import inspect +import boto3 +from botocore.exceptions import ClientError +import loggers +from _version import __version__ + + +# lambda custom log object +logger = loggers.getLogger(__version__) + + +class AssumeAWSRole(): + """ class def for assuming roles in AWS """ + def __init__(self, role_name=None, account=None, profile=None): + self.role = role_name + if account: + self.account_number = str(account) + self.credentials = self.assume_role(str(account), role_name) + elif profile: + self.profile = profile + r = sts_client.get_caller_identity() + self.account_number = r['Account'] + self.credentials = self.assume_role() + self.status = {} + + def assume_role(self, account=None, role=None): + """ + Summary: + Assumes a DynamoDB role in 'destination' AWS account + Args: + :type account: str + :param account: AWS account number + :type role: str + :param role: IAM role designation used to access AWS resources + in an account + :type profile: str + :param role: profile_name is an IAM user or IAM role name represented + in the local awscli configuration as a profile entry + Returns: dict (Credentials) + """ + if self.profile: + session = boto3.Session(profile_name=self.profile) + sts_client = session.client('sts') + else: + sts_client = boto3.client('sts') + + try: + # assume role in destination account + if account and role: + assumed_role = sts_client.assume_role( + RoleArn="arn:aws:iam::%s:role/%s" % (account, role), + RoleSessionName="AssumeAWSRoleSession" + ) + else: + assumed_role = sts_client.assume_role( + RoleArn="arn:aws:iam::%s:role/%s" % (self.account_number, self.role), + RoleSessionName="AssumeAWSRoleSession" + ) + except ClientError as e: + logger.exception( + "%s: Problem assuming role %s in account %s (Code: %s Message: %s)" % + (inspect.stack()[0][3], role, account, + e.response['Error']['Code'], e.response['Error']['Message']) + ) + if e.response['Error']['Code'] == 'AccessDenied': + self.status = {'STATUS': 'AccessDenied', 'FLAG': False} + return {} + else: + self.status = {'STATUS': 'ERROR', 'FLAG': False} + return {} + self.status = {'STATUS': 'SUCCESS', 'FLAG': True} + return assumed_role['Credentials'] + + def create_service_client(self, aws_service, account=None, role=None): + """ + Summary: + Creates the appropriate boto3 client for a particular AWS service + Args: + :type service: str + :param service: name of service at Amazon Web Services (AWS), + e.g. s3, ec2, etc + :type credentials: sts credentials object + :param credentials: authentication credentials to resource in AWS + :type role: str + :param role: IAM role designation used to access AWS resources + in an account + Returns: + Success | Failure, TYPE: bool + """ + try: + if role and account: # create client for a different AWS account + if self.status.get('STATUS') == 'SUCCESS': + client = boto3.client( + aws_service, + aws_access_key_id=self.credentials['AccessKeyId'], + aws_secret_access_key=self.credentials['SecretAccessKey'], + aws_session_token=self.credentials['SessionToken'] + ) + else: + self.status = {'STATUS': 'ERROR', 'FLAG': False} + return self.status + else: + return boto3.client(aws_service) # create client in the current AWS account + except ClientError as e: + logger.exception( + "%s: Problem creating %s client in account %s (Code: %s Message: %s)" % + (inspect.stack()[0][3], aws_service, self.account_number, + e.response['Error']['Code'], e.response['Error']['Message'])) + raise + return client diff --git a/build/lib/pyaws/core/loggers.py b/build/lib/pyaws/core/loggers.py new file mode 100644 index 0000000..1cdb94f --- /dev/null +++ b/build/lib/pyaws/core/loggers.py @@ -0,0 +1,22 @@ +""" +Example usage: + +>>> __version__ = '1.2.3' # Or wherever the version is stored +>>> logger = getLogger(__version__) +>>> logger.warn('omg what is happening') + - 1.2.3 - omg what is happening +""" +import logging + + + +def getLogger(*args, **kwargs): + """ custom format logger """ + logger = logging.getLogger(*args, **kwargs) + logger.propagate = False + if not logger.handlers: + handler = logging.StreamHandler() + handler.setFormatter(logging.Formatter('%(pathname)s - %(name)s - [%(levelname)s]: %(message)s')) + logger.addHandler(handler) + logger.setLevel(logging.DEBUG) + return logger diff --git a/build/lib/pyaws/core/oscodes_unix.py b/build/lib/pyaws/core/oscodes_unix.py new file mode 100644 index 0000000..4e4eaee --- /dev/null +++ b/build/lib/pyaws/core/oscodes_unix.py @@ -0,0 +1,121 @@ +""" +Standard OS Module Exit Codes + - See https://docs.python.org/3.6/library/os.html#process-management + +Module Attributes: + - exit_codes (dict): exit error codes for Unix, Linux + +""" + +import os + + +# --- exit codes for Unix, Linux, Linux-based operating systems ---------------- + + +exit_codes = { + 'EX_OK': { + 'Code': 0, + 'Reason': 'No error occurred' + }, + 'E_DEPENDENCY': { + 'Code': 1, + 'Reason': 'Missing required dependency' + }, + 'E_DIR': { + 'Code': 2, + 'Reason': 'Failure to create log dir, log file' + }, + 'E_ENVIRONMENT': { + 'Code': 3, + 'Reason': 'Incorrect shell, language interpreter, or operating system' + }, + 'EX_AWSCLI': { + 'Code': 4, + 'Reason': 'Value could not be determined from local awscli configuration' + }, + 'EX_NOPERM': { + 'Code': os.EX_NOPERM, + 'Reason': 'IAM user or role permissions do not allow this action' + }, + 'E_AUTHFAIL': { + 'Code': 5, + 'Reason': 'Authentication Fail' + }, + 'E_BADPROFILE': { + 'Code': 6, + 'Reason': 'Local profile variable not set or incorrect' + }, + 'E_USER_CANCEL': { + 'Code': 7, + 'Reason': 'User abort' + }, + 'E_BADARG': { + 'Code': 8, + 'Reason': 'Bad input parameter' + }, + 'E_EXPIRED_CREDS': { + 'Code': 9, + 'Reason': 'Credentials expired or otherwise no longer valid' + }, + 'E_MISC': { + 'Code': 9, + 'Reason': 'Unknown Error' + }, + 'EX_NOUSER': { + 'Code': os.EX_NOUSER, + 'Reason': 'specified user does not exist' + }, + 'EX_CONFIG': { + 'Code': os.EX_CONFIG, + 'Reason': 'Configuration or config parameter error' + }, + 'EX_CREATE_FAIL': { + 'Code': 21, + 'Reason': 'Keyset failed to create. Possible Permissions issue' + }, + 'EX_DELETE_FAIL': { + 'Code': 22, + 'Reason': 'Keyset failed to delete. Possible Permissions issue' + }, + 'EX_DATAERR': { + 'Code': os.EX_DATAERR, + 'Reason': 'Input data incorrect' + }, + 'EX_NOINPUT': { + 'Code': os.EX_NOINPUT, + 'Reason': 'Input file does not exist or not readable.' + }, + 'EX_UNAVAILABLE': { + 'Code': os.EX_UNAVAILABLE, + 'Reason': 'Required service or dependency unavailable.' + }, + 'EX_PROTOCOL': { + 'Code': os.EX_PROTOCOL, + 'Reason': 'Protocol exchange was illegal, invalid, or not understood.' + }, + 'EX_OSERR': { + 'Code': os.EX_OSERR, + 'Reason': 'Operating system error.' + }, + 'EX_OSFILE': { + 'Code': os.EX_OSFILE, + 'Reason': 'System file does not exist, or could not be opened' + }, + 'EX_IOERR': { + 'Code': os.EX_IOERR, + 'Reason': 'Exit code that means that an error occurred while doing I/O on some file.' + }, + 'EX_NOHOST': { + 'Code': os.EX_NOHOST, + 'Reason': 'Network host does not exist or not found' + }, + 'EX_SOFTWARE': { + 'Code': os.EX_SOFTWARE, + 'Reason': 'Internal software error detected.' + }, + 'EX_CANTCREAT': { + 'Code': os.EX_CANTCREAT, + 'Reason': 'User specified output file could not be created.' + } +} diff --git a/build/lib/pyaws/core/oscodes_win.py b/build/lib/pyaws/core/oscodes_win.py new file mode 100644 index 0000000..b2690a3 --- /dev/null +++ b/build/lib/pyaws/core/oscodes_win.py @@ -0,0 +1,80 @@ +""" +Standard OS Module Exit Codes + - See https://docs.python.org/3.6/library/os.html#process-management + +Module Attributes: + - exit_codes (dict): exist error codes for Microsoft Windows +""" + +import os + + +# --- exit codes for Microsoft Windows operating systems ----------------------- + + +exit_codes = { + 'EX_OK': { + 'Code': 0, + 'Reason': 'No error occurred' + }, + 'E_DEPENDENCY': { + 'Code': 1, + 'Reason': 'Missing required dependency' + }, + 'E_DIR': { + 'Code': 2, + 'Reason': 'Failure to create log dir, log file' + }, + 'E_ENVIRONMENT': { + 'Code': 3, + 'Reason': 'Incorrect shell, language interpreter, or operating system' + }, + 'EX_AWSCLI': { + 'Code': 4, + 'Reason': 'Value could not be determined from local awscli configuration' + }, + 'EX_NOPERM': { + 'Code': 77, + 'Reason': 'IAM user or role permissions do not allow this action' + }, + 'E_AUTHFAIL': { + 'Code': 5, + 'Reason': 'Authentication Fail' + }, + 'E_BADPROFILE': { + 'Code': 6, + 'Reason': 'Local profile variable not set or incorrect' + }, + 'E_USER_CANCEL': { + 'Code': 7, + 'Reason': 'User abort' + }, + 'E_BADARG': { + 'Code': 8, + 'Reason': 'Bad input parameter' + }, + 'E_EXPIRED_CREDS': { + 'Code': 9, + 'Reason': 'Credentials expired or otherwise no longer valid' + }, + 'E_MISC': { + 'Code': 9, + 'Reason': 'Unknown Error' + }, + 'EX_NOUSER': { + 'Code': 67, + 'Reason': 'specified user does not exist' + }, + 'EX_CONFIG': { + 'Code': 78, + 'Reason': 'Configuration or config parameter error' + }, + 'EX_CREATE_FAIL': { + 'Code': 21, + 'Reason': 'Keyset failed to create. Possible Permissions issue' + }, + 'EX_DELETE_FAIL': { + 'Code': 22, + 'Reason': 'Keyset failed to delete. Possible Permissions issue' + } +} diff --git a/build/lib/pyaws/dynamodb/__init__.py b/build/lib/pyaws/dynamodb/__init__.py new file mode 100644 index 0000000..29dd5fa --- /dev/null +++ b/build/lib/pyaws/dynamodb/__init__.py @@ -0,0 +1,4 @@ +""" +Functional Utilities for Amazon DynamoDB Database Service +""" +from pyaws.dynamodb.dynamodb import DynamoDBReader diff --git a/build/lib/pyaws/dynamodb/dynamodb.py b/build/lib/pyaws/dynamodb/dynamodb.py new file mode 100644 index 0000000..39258a9 --- /dev/null +++ b/build/lib/pyaws/dynamodb/dynamodb.py @@ -0,0 +1,137 @@ +""" +Summary: + Boto3 DynamoDB Reader Operations + +""" + +import boto3 +from botocore.exceptions import ClientError +from boto3.dynamodb.conditions import Key, Attr +from pyaws import logger + + +class DynamoDBReader(): + def __init__(self, aws_account_id, service_role, tablename, region): + """ + Reads DynamoDB table + """ + self.tablename = tablename + self.region = region + self.aws_account_id = aws_account_id + self.service_role = service_role + self.aws_credentials = self.assume_role(aws_account_id, service_role) + + + def boto_dynamodb_resource(self, region): + """ + Initiates boto resource to communicate with AWS API + """ + try: + dynamodb_resource = boto3.resource( + 'dynamodb', + aws_access_key_id=self.aws_credentials['AccessKeyId'], + aws_secret_access_key=self.aws_credentials['SecretAccessKey'], + aws_session_token=self.aws_credentials['SessionToken'], + region_name=region + ) + except ClientError as e: + logger.exception("Unknown problem creating boto3 resource (Code: %s Message: %s)" % + (e.response['Error']['Code'], e.response['Error']['Message'])) + return 1 + return dynamodb_resource + + def assume_role(self, aws_account_id, service_role): + """ + Summary. + + Assumes a DynamoDB role in 'destination' AWS account + + Args: + aws_account_id (str): 12 digit AWS Account number containing dynamodb table + service_role (str): IAM role dynamodb service containing permissions + allowing interaction with dynamodb + + Returns: + temporary credentials for service_role when assumed, TYPE: json + """ + session = boto3.Session() + sts_client = session.client('sts') + + try: + + # assume role in destination account + assumed_role = sts_client.assume_role( + RoleArn="arn:aws:iam::%s:role/%s" % (str(aws_account_id), service_role), + RoleSessionName="DynamoDBReaderSession" + ) + + except ClientError as e: + logger.exception( + "Couldn't assume role to read DynamoDB, account " + + str(aws_account_id) + " (switching role) (Code: %s Message: %s)" % + (e.response['Error']['Code'], e.response['Error']['Message'])) + raise e + return assumed_role['Credentials'] + + def query_dynamodb(self, partition_key, key_value): + """ + Queries DynamoDB table using partition key, + returns the item matching key value + """ + try: + resource_dynamodb = self.boto_dynamodb_resource(self.region) + table = resource_dynamodb.Table(self.tablename) + logger.info('Table %s: Table Item Count is: %s' % (table.table_name, table.item_count)) + + # query on parition key + response = table.query(KeyConditionExpression=Key(partition_key).eq(str(key_value))) + if response['Items']: + item = response['Items'][0]['Account Name'] + else: + item = 'unIdentified' + + except ClientError as e: + logger.exception("Couldn\'t query DynamoDB table (Code: %s Message: %s)" % + (e.response['Error']['Code'], e.response['Error']['Message'])) + return 1 + return item + + def scan_accounts(self, account_type): + """ + Read method for DynamoDB table + """ + + accounts, account_ids = [], [] + valid_mpc_pkgs = ['B', 'RA-PKG-B', 'RA-PKG-C', 'P', 'ATA', 'BUP', 'DXA'] + + types = [x.strip(' ') for x in account_type.split(',')] # parse types + + try: + resource_dynamodb = self.boto_dynamodb_resource(self.region) + table = resource_dynamodb.Table(self.tablename) + # scan table + if set(types).issubset(set(valid_mpc_pkgs)): + for type in types: + response = table.scan(FilterExpression=Attr('MPCPackage').eq(type)) + for account_dict in response['Items']: + accounts.append(account_dict) + elif types[0] == 'All': + # all valid_mpc_pkgs accounts (commercial accounts) + response = table.scan(FilterExpression=Attr('MPCPackage').ne("P")) + for account_dict in response['Items']: + accounts.append(account_dict) + + except ClientError as e: + logger.exception("Couldn\'t scan DynamoDB table (Code: %s Message: %s)" % + (e.response['Error']['Code'], e.response['Error']['Message'])) + return 1 + + if accounts: + for account in accounts: + account_info = {} + account_info['AccountName'] = account['Account Name'] + account_info['AccountId'] = account['Account ID'] + account_ids.append(account_info) + else: + logger.info('No items returned from DyanamoDB') + return account_ids diff --git a/build/lib/pyaws/dynamodb/table.py b/build/lib/pyaws/dynamodb/table.py new file mode 100644 index 0000000..b04a700 --- /dev/null +++ b/build/lib/pyaws/dynamodb/table.py @@ -0,0 +1,49 @@ + +import boto3 + +# Get the service resource. +dynamodb = boto3.resource('dynamodb') + + +def create_dynamodb_table(tablename, keys, attributes): + """ + Summary. + + Creates table with keys and attributes + + Args: + :keys (list): List of dictionaries specifying hash, range + :attributes (list): List of dictionaries specifying table attributes + + Returns: + table name (str) + """ + # Create the DynamoDB table. + table = dynamodb.create_table( + TableName='users', + KeySchema=[ + { + 'AttributeName': 'username', + 'KeyType': 'HASH' + }, + { + 'AttributeName': 'last_name', + 'KeyType': 'RANGE' + } + ], + AttributeDefinitions=[ + { + 'AttributeName': 'username', + 'AttributeType': 'S' + }, + { + 'AttributeName': 'last_name', + 'AttributeType': 'S' + }, + + ], + ProvisionedThroughput={ + 'ReadCapacityUnits': 5, + 'WriteCapacityUnits': 5 + } + ) diff --git a/build/lib/pyaws/ec2/__init__.py b/build/lib/pyaws/ec2/__init__.py new file mode 100644 index 0000000..9346627 --- /dev/null +++ b/build/lib/pyaws/ec2/__init__.py @@ -0,0 +1,5 @@ +""" +Functionality utilising Amazon EC2 Service +""" +from pyaws.ec2.state import running_instances, stopped_instances +from pyaws.ec2.ec2_utils import * diff --git a/build/lib/pyaws/ec2/ec2_utils.py b/build/lib/pyaws/ec2/ec2_utils.py new file mode 100644 index 0000000..673e855 --- /dev/null +++ b/build/lib/pyaws/ec2/ec2_utils.py @@ -0,0 +1,278 @@ +""" +Summary: + ec2_utils (python3) | Common EC2 functionality implemented by boto3 SDK + +Author: + Blake Huber + Copyright Blake Huber, All Rights Reserved. + +License: + Permission to use, copy, modify, and distribute this software and its + documentation for any purpose and without fee is hereby granted, + provided that the above copyright notice appear in all copies and that + both the copyright notice and this permission notice appear in + supporting documentation + + Additional terms may be found in the complete license agreement located at: + https://bitbucket.org/blakeca00/lambda-library-python/src/master/LICENSE.md + +""" +import os +import subprocess +import inspect +import boto3 +from botocore.exceptions import ClientError +from pyaws.session import boto3_session +from pyaws.core import loggers +from pyaws._version import __version__ + + +# global objects +REGION = os.environ['AWS_DEFAULT_REGION'] + +# lambda custom log object +logger = loggers.getLogger(__version__) + + +# -- declarations ------------------------------------------------------------- + + +def default_region(profile='default'): + """ + Summary: + Determines the default region of profilename present in the local awscli + configuration or set in the environment via 'AWS_DEFAULT_REGION' variable. + If all else fails, returns region code 'us-east-1' as a default region. + Args: + profile (str): profile_name of a valid profile from local awscli config + Returns: + AWS Region Code, TYPE str + """ + + stderr = ' 2>/dev/null' + region = subprocess.check_output( + 'aws configure get profile.{profile}.region {stderr}'.format( + profile=profile, stderr=stderr)) + + try: + if region: + return region + elif os.getenv('AWS_DEFAULT_REGION') is None: + os.environ['AWS_DEFAULT_REGION'] = 'us-east-1' + except Exception as e: + logger.exception( + '{i}: Unknown error while interrogating local awscli config: {e}'.format( + i=inspect.stack()[0][3], e=e) + ) + raise + return os.getenv('AWS_DEFAULT_REGION') + + +def get_instances(region, profile=None, pageSize=100): + """ + Returns: all EC2 instance Ids in a region + """ + ids = [] + try: + if profile: + session = boto3.Session(profile_name=profile, region_name=region) + client = session.client('ec2') + else: + client = boto3.client('ec2', region_name=region) + + # find ebs volumes associated with instances + paginator = client.get_paginator('describe_instances') + response_iterator = paginator.paginate(PaginationConfig={'PageSize': pageSize}) + + # collect all instances in region + for page in response_iterator: + # collect all instanceIds (not used) + for x in [x[0]['InstanceId'] for x in [x['Instances'] for x in page['Reservations']]]: + ids.append(x) + + except ClientError as e: + logger.critical( + "%s: problem retrieving instances in region %s (Code: %s Message: %s)" % + (inspect.stack()[0][3], str(region), e.response['Error']['Code'], + e.response['Error']['Message'])) + raise e + return ids + + +def get_regions(profile=None): + """ Return list of all regions """ + try: + if profile is None: + profile = 'default' + client = boto3_session(service='ec2', profile=profile) + + except ClientError as e: + logger.exception( + '%s: Boto error while retrieving regions (%s)' % + (inspect.stack()[0][3], str(e))) + raise e + return [x['RegionName'] for x in client.describe_regions()['Regions']] + + +def dns_hostname(instanceId, profile='default'): + """ + Summary: + Reverse DNS for EC2 instances public or private subnets + Really only useful when EC2 instance assigned non-AWS DNS name + Args: + ip_info (dict): + { + "Association": { + "IpOwnerId": "102512488663", + "PublicDnsName": "ec2-34-247-23-51.eu-west-1.compute.amazonaws.com", + "PublicIp": "34.247.23.51" + }, + "Primary": true, + "PrivateDnsName": "ip-172-31-28-93.eu-west-1.compute.internal", + "PrivateIpAddress": "172.31.28.93" + } + Returns: + hostname (tuple): First element of the following tuple: + ( + 'ec2-34-247-23-51.eu-west-1.compute.amazonaws.com', + ['34.247.23.51'], + 'ip-172-31-28-93.eu-west-1.compute.internal', + ['172.31.28.93'] + ) + """ + try: + session = boto3.Session(profile_name=profile) + client = session.client('ec2', region_name=default_region(profile)) + r = client.describe_instances(InstanceIds=[instanceId]) + # dict of ip information + ip_info = [x['PrivateIpAddresses'][0] for x in r['Reservations'][0]['Instances'][0]['NetworkInterfaces']][0] + private_name = r['Reservations'][0]['Instances'][0]['PrivateDnsName'] + public_name = r['Reservations'][0]['Instances'][0]['PublicDnsName'] + + """BELOW NEEDS DEBUGGING + if ip_info.get('Association'): + public_ip = ip_info['Association']['PublicIp'] + priv_ip = ip_info['PrivateIpAddress'] + else: + public_ip = '' + priv_ip = ip_info['PrivateIpAddress'] + return ( + socket.gethostbyaddr(public_ip), + [public_ip], + socket.gethostbyaddr(priv_ip), + [priv_ip] + ) + """ + except KeyError as e: + logger.exception('%s: KeyError parsing ip info (%s)' % (inspect.stack()[0][3], str(e))) + return ('', [], '', []) + except ClientError as e: + logger.exception('%s: Boto Error parsing ip info (%s)' % (inspect.stack()[0][3], str(e))) + return '' + except Exception: + logger.exception( + '%s: No dns info from reverse lookup - Unknown host' % inspect.stack()[0][3]) + return ('', [], '', [ip_info['PrivateIpAddress']]) + return public_name or private_name + + +def get_attached_ids(region, instanceId, profile=None): + """ + Summary: + Audits the entire namespace of an AWS Account (essentially an + entire region) for resource ids of the type requested + Args: + instanceId (str): a single EC2 instance Identifier + pageSize (int): paging is used, + Returns: + vids (str): ebs volume ids attached to instanceId + enids (str): elastic network_interface ids attached to instanceId + Raises: + botocore ClientError + """ + vids, eids = [], [] + + if profile: + session = boto3.Session(profile_name=profile, region_name=region) + ec2 = session.resource('ec2') + else: + ec2 = boto3.resource('ec2', region_name=region) + + try: + logger.info('%s: function start' % inspect.stack()[0][3]) + + base = ec2.instances.filter(InstanceIds=[instanceId]) + + for instance in base: + # get volume ids + for vol in instance.volumes.all(): + vids.append(vol.id) + # get network interfaces + for eni in instance.network_interfaces: + eids.append(eni.id) + + logger.info( + '%d volume(s), %d ENIs found for instance %s' % + (len(vids), len(eids), instanceId) + ) + except ClientError as e: + logger.exception( + '%s: Problem while retrieving list of volumes for region %s' % + (inspect.stack()[0][3], region)) + return [], [] + return vids, eids + + +def namespace_volumes_eids(region, profile=None, pageSize=200): + """ + Summary: + - Audits the entire namespace of an AWS Account (essentially an + entire region) for resource ids of the type requested (with paging) + Args: + pageSize (int): paging is used, + Returns: + vids (str): ebs volume ids attached to instanceId + enids (str): elastic network_interface ids attached to instanceId + Raises: + botocore ClientError + """ + + vids, enids = [], [] + + try: + logger.info('%s: function start' % inspect.stack()[0][3]) + + if profile: + session = boto3.Session(profile_name=profile, region_name=region) + client = session.client('ec2') + else: + client = boto3.client('ec2', region_name=region) + + # find ebs volumes associated with instances + paginator = client.get_paginator('describe_volumes') + response_iterator = paginator.paginate(PaginationConfig={'PageSize': pageSize}) + + # collect all instances in region + for page in response_iterator: + # collect all instanceIds (not used) + for z in [y['VolumeId'] for y in [x['Attachments'][0] for x in page['Volumes']]]: + vids.append(z) + + logger.info('%d volume(s) found in region %s' % (len(vids), region)) + + # find enis + paginator = client.get_paginator('describe_network_interfaces') + response_iterator = paginator.paginate(PaginationConfig={'PageSize': pageSize}) + + # collect all instances in region + for page in response_iterator: + # collect all instanceIds (not used) + for z in [x['NetworkInterfaceId'] for x in page['NetworkInterfaces']]: + enids.append(z) + + except ClientError as e: + logger.exception( + '%s: Problem while retrieving list of volumes for region %s' % + (inspect.stack()[0][3], REGION)) + return [], [] + return vids, enids diff --git a/build/lib/pyaws/ec2/snapshot_ops.py b/build/lib/pyaws/ec2/snapshot_ops.py new file mode 100644 index 0000000..cae5601 --- /dev/null +++ b/build/lib/pyaws/ec2/snapshot_ops.py @@ -0,0 +1,228 @@ +""" +Summary: + ec2_utils (python3) | Common EC2 functionality implemented by boto3 SDK + +Author: + Blake Huber + Copyright Blake Huber, All Rights Reserved. + +License: + Permission to use, copy, modify, and distribute this software and its + documentation for any purpose and without fee is hereby granted, + provided that the above copyright notice appear in all copies and that + both the copyright notice and this permission notice appear in + supporting documentation + + Additional terms may be found in the complete license agreement located at: + https://bitbucket.org/blakeca00/lambda-library-python/src/master/LICENSE.md + +""" +import os +import datetime +import inspect +import boto3 +from botocore.exceptions import ClientError +from pyaws import logd, __version__ + + +# global objects + + +# lambda custom log object +logger = logd.getLogger(__version__) + + +# -- declarations ------------------------------------------------------------- + + +def snapshot_metadata(snapshot_id, region, tags=False, profile=None): + """ + Creates EC2 Snapshot Object to extract detailed metadata + Returns: + metadata (TYPE dict) | contains snapshot attribute values + """ + metadata = {} + try: + if profile: + session = boto3.Session(profile_name=profile, region_name=region) + ec2 = session.resource('ec2') + else: + ec2 = boto3.resource('ec2', region_name=region) + snapshot_obj = ec2.Snapshot(snapshot_id) + metadata = { + 'SnapshotId': snapshot_id, + 'State': snapshot_obj.state, + 'Description': snapshot_obj.description, + 'Encrypted': snapshot_obj.encrypted, + 'KmsKeyId': snapshot_obj.kms_key_id, + 'VolumeId': snapshot_obj.volume_id, + 'VolumeSize': snapshot_obj.volume_size, + 'StartTime': snapshot_obj.start_time, + 'ProgressPct': snapshot_obj.progress.split('%')[0], + 'OwnerId': snapshot_obj.owner_id + } + if tags: + metadata['Tags'] = snapshot_obj.tags + except ClientError as e: + logger.critical( + "%s: problem retrieving metadata for snapshot %s (Code: %s Message: %s)" % + (inspect.stack()[0][3], snapshot_id, e.response['Error']['Code'], + e.response['Error']['Message'])) + raise e + return metadata + + +class SnapshotOperations(): + """ + Summary: + EC2 Snapshot Operations Class: List, Create, Delete, CreateVolume + """ + def __init__(self, region, profile=None): + self.snapshot_list = [] + if profile: + session = boto3.Session(profile_name=profile, region_name=region) + self.client = session.client('ec2') + self.sts_client = session.client('sts') + else: + self.client = boto3.client('ec2', region_name=region) + self.sts_client = boto3.client('sts', region_name=region) + self.acct_number = self.sts_client.get_caller_identity()['Account'] + + def list(self, account=None, volume_ids=None): + """ + Args: + AWS Account (str) + Returns: + List (TYPE list): all snapshots in region owned by account + Works for large numbers of snapshots (pagination) + """ + if not account: + account = self.acct_number + try: + paginator = self.client.get_paginator('describe_snapshots') + + if volume_ids: + response_iterator = paginator.paginate( + Filters=[ + { + 'Name': 'volume-id', + 'Values': volume_ids + }, + ], + PaginationConfig={'PageSize': 100} + ) + else: + response_iterator = paginator.paginate( + OwnerIds=[str(account)], + PaginationConfig={'PageSize': 100} + ) + # page thru, retrieve all snapshots + snapshot_ids, temp = [], [] + for page in response_iterator: + temp = [x['SnapshotId'] for x in page['Snapshots']] + for pageid in temp: + if pageid not in snapshot_ids: + snapshot_ids.append(pageid) + # persist at the instance level for future use + self.snapshot_list = [x for x in snapshot_ids] + except ClientError as e: + logger.critical( + "%s: Problem during snapshot retrieval operation (Code: %s Message: %s)" % + (inspect.stack()[0][3], e.response['Error']['Code'], + e.response['Error']['Message'])) + raise e + return snapshot_ids + + def create(self, volume_list): + """ + Returns: + :result (dict): {'SnapshotID': 'State'} + """ + result, snap_dict = [], {} + now = datetime.datetime.utcnow() + try: + for vol_id in volume_list: + now = now + datetime.timedelta(seconds=1) + description = 'Snap Date: ' + str(now.strftime("%Y-%m-%dT%H:%M:%SZ")) + r = self.client.create_snapshot( + VolumeId=vol_id, + Description=description + ) + logger.info( + 'Created snapshot %s from volume %s' % (r['SnapshotId'], vol_id) + ) + result.append( + {r['SnapshotId']: r['State'], 'Description': description} + ) + except ClientError as e: + logger.critical( + "%s: Problem creating snapshot from volume %s (Code: %s Message: %s)" % + (inspect.stack()[0][3], vol_id, e.response['Error']['Code'], + e.response['Error']['Message'])) + raise e + return result + + def delete(self, snapshot_list): + """ + Args: + snapshot_list (list): list of snapshot ids + Returns: + :result (dict): {'SnapshotID': 'State'} + """ + result = [] + try: + for snapshot_id in snapshot_list: + r = self.client.delete_snapshot(SnapshotId=snapshot_id) + logger.info('Deleted snapshot %s' % str(snapshot_id)) + result.append( + { + 'SnapshotId': snapshot_id, + 'HTTPStatusCode': r['ResponseMetadata']['HTTPStatusCode'] + } + ) + except ClientError as e: + logger.critical( + "%s: Problem deleting snapshot %s (Code: %s Message: %s)" % + (inspect.stack()[0][3], snapshot_id, e.response['Error']['Code'], + e.response['Error']['Message'])) + raise e + return result + + def create_volume(self, snapshot_id, az, vol_type='gp2', + size=None, encrypted=False, kms_key=''): + """ + Summary: + Creates new ebs volume from snapshot(s) + Args: + - snapshot_id (str, required) + - az (str, required): Availability Zone in which to create volume + - vol_type (str): 'standard', 'gp2', 'io2', + - size (str): Size in GB, required only if larger than orig volume size + - encrypted (bool) + - kms_key (str): KMS key id (required if encrypted = True) + Returns: + :result (dict) + """ + try: + r = self.client.create_volume( + SnapshotId=snapshot_id, + Size=size, + VolumeType=vol_type, + KmsKeyId=kms_key, + Encrypted=encrypted + ) + logger.info( + 'Creating volume %s from snapshot %s' % (r['VolumeId'], snapshot_id) + ) + except ClientError as e: + logger.critical( + "%s: Problem creating volume from snapshot %s (Code: %s Message: %s)" % + (inspect.stack()[0][3], snapshot_id, e.response['Error']['Code'], + e.response['Error']['Message'])) + raise e + return { + 'VolumeId': r['VolumeId'], + 'State': r['State'], + 'Source': snapshot_id, + 'Encrypted': encrypted + } diff --git a/build/lib/pyaws/ec2/state.py b/build/lib/pyaws/ec2/state.py new file mode 100644 index 0000000..17a68d3 --- /dev/null +++ b/build/lib/pyaws/ec2/state.py @@ -0,0 +1,107 @@ +""" +Summary: + ec2_utils (python3) | Common EC2 functionality implemented by boto3 SDK + +Author: + Blake Huber + Copyright Blake Huber, All Rights Reserved. + +License: + Permission to use, copy, modify, and distribute this software and its + documentation for any purpose and without fee is hereby granted, + provided that the above copyright notice appear in all copies and that + both the copyright notice and this permission notice appear in + supporting documentation + + Additional terms may be found in the complete license agreement located at: + https://bitbucket.org/blakeca00/lambda-library-python/src/master/LICENSE.md + +""" +import os +import inspect +import boto3 +from botocore.exceptions import ClientError, ProfileNotFound +from pyaws.utils import stdout_message +from pyaws import logger + + +# global objects +REGION = os.environ['AWS_DEFAULT_REGION'] + + +# -- declarations ------------------------------------------------------------- + + +def running_instances(region, profile=None, ids=False, debug=False): + """ + Summary. + Determines state of all ec2 machines in a region + + Returns: + :running ec2 instances, TYPE: ec2 objects + OR + :running ec2 instance ids, TYPE: str + """ + try: + if profile and profile != 'default': + session = boto3.Session(profile_name=profile) + ec2 = session.resource('ec2', region_name=region) + else: + ec2 = boto3.resource('ec2', region_name=region) + + instances = ec2.instances.all() + + if ids: + return [x.id for x in instances if x.state['Name'] == 'running'] + + except ClientError as e: + logger.exception( + "%s: IAM user or role not found (Code: %s Message: %s)" % + (inspect.stack()[0][3], e.response['Error']['Code'], + e.response['Error']['Message'])) + raise + except ProfileNotFound: + msg = ( + '%s: The profile (%s) was not found in your local config' % + (inspect.stack()[0][3], profile)) + stdout_message(msg, 'FAIL') + logger.warning(msg) + return [x for x in instances if x.state['Name'] == 'running'] + + +def stopped_instances(region, profile=None, ids=False, debug=False): + """ + Summary. + Determines state of all ec2 machines in a region + + Returns: + :stopped ec2 instances, TYPE: ec2 objects + OR + :stopped ec2 instance ids, TYPE: str + + """ + try: + if profile and profile != 'default': + session = boto3.Session(profile_name=profile) + ec2 = session.resource('ec2', region_name=region) + else: + ec2 = boto3.resource('ec2', region_name=region) + + instances = ec2.instances.all() + + if ids: + return [x.id for x in instances if x.state['Name'] == 'stopped'] + + except ClientError as e: + logger.exception( + "%s: IAM user or role not found (Code: %s Message: %s)" % + (inspect.stack()[0][3], e.response['Error']['Code'], + e.response['Error']['Message'])) + raise + except ProfileNotFound: + msg = ( + '%s: The profile (%s) was not found in your local config' % + (inspect.stack()[0][3], profile)) + stdout_message(msg, 'FAIL') + logger.warning(msg) + return [x for x in instances if x.state['Name'] == 'stopped'] diff --git a/build/lib/pyaws/environment.py b/build/lib/pyaws/environment.py new file mode 100644 index 0000000..3d7522f --- /dev/null +++ b/build/lib/pyaws/environment.py @@ -0,0 +1,68 @@ +""" +Pretest Setup | pytest + + Calls set_environment() on module import +""" +import os +import subprocess +import inspect +import logging + + +logger = logging.getLogger() +logger.setLevel(logging.INFO) + + +def awscli_region(profile_name): + """ + Summary: + Sets default AWS region + Args: + profile_name: a username in local awscli profile + Returns: + region (str): AWS region code | None + Raises: + Exception if profile_name not found in config + """ + awscli = 'aws' + + cmd = awscli + ' configure get ' + profile_name + '.region' + + try: + region = subprocess.check_output(cmd) + except Exception: + logger.exception( + '%s: Failed to identify AccessKeyId used in %s profile.' % + (inspect.stack()[0][3], profile_name)) + return None + return region + + +def set_default_region(profile=None): + """ + Sets AWS default region globally + """ + if os.getenv('AWS_DEFAULT_REGION'): + return os.getenv('AWS_DEFAULT_REGION') + elif profile is not None: + return awscli_region(profile_name=profile) + return awscli_region(profile_name='default') + + +def set_environment(): + """ + Sets global environment variables for testing + """ + # status + + logger.info('setting global environment variables') + + # set all env vars + os.environ['DBUGMODE'] = 'False' + os.environ['AWS_DEFAULT_REGION'] = set_default_region() or 'us-east-1' + + logger.info('AWS_DEFAULT_REGION determined as %s' % os.environ['AWS_DEFAULT_REGION']) + + +# execute on import +set_environment() diff --git a/build/lib/pyaws/helpers.py b/build/lib/pyaws/helpers.py new file mode 100644 index 0000000..44fee07 --- /dev/null +++ b/build/lib/pyaws/helpers.py @@ -0,0 +1,36 @@ +from contextlib import contextmanager +import inspect +import logging + +from botocore.exceptions import ClientError + +from pyaws import __version__ + +logger = logging.getLogger(__version__) + +@contextmanager +def handle_boto_error(message_template=None): + ''' + Context manager for wrapping a potentially-ClientError raising block of code + with a consistent error handler/reporter. + + Example usage: + + >>> from botocore.exceptions import ClientError + >>> with handle_boto_error(): + ... # do something that raises a ClientError + ... + + ''' + message_template = message_template or '{function}: boto3 error occured (Code: {code} Message: {message})' + try: + yield + except ClientError as e: + logger.exception( + message_template.format( + function=inspect.stack()[0][3], + code=e.response['Error']['Code'], + message=e.response['Error']['Message'] + ) + ) + raise \ No newline at end of file diff --git a/build/lib/pyaws/logd.py b/build/lib/pyaws/logd.py new file mode 100644 index 0000000..f409222 --- /dev/null +++ b/build/lib/pyaws/logd.py @@ -0,0 +1,146 @@ +""" +Project-level logging module + +""" +import os +import sys +import inspect +import logging +import logging.handlers +from pathlib2 import Path +from pyaws.statics import local_config + + +syslog = logging.getLogger() +syslog.setLevel(logging.DEBUG) + + +def mode_assignment(arg): + """ + Translates arg to enforce proper assignment + """ + arg = arg.upper() + stream_args = ('STREAM', 'CONSOLE', 'STDOUT') + try: + if arg in stream_args: + return 'STREAM' + else: + return arg + except Exception: + return None + + +def logging_prep(mode): + """ + Summary: + prerequisites for log file generation + Return: + Success | Failure, TYPE: bool + """ + try: + if mode == 'FILE': + + log_path = local_config['LOGGING']['LOG_PATH'] + # path: path to log dir + path, log_dirname = os.path.split(log_path) + + if not os.path.exists(path): + os.makedirs(path) + + if not os.path.exists(log_path): + Path(log_path).touch(mode=0o644, exist_ok=True) + + except OSError as e: + syslog.exception( + '{i}: Failure while seeding log file path: {e}'.format( + i=inspect.stack()[0][3], e=e)) + return False + return True + + +def getLogger(*args, **kwargs): + """ + Summary: + custom format logger + + Args: + mode (str): The Logger module supprts the following log modes: + + - log to console / stdout. Log_mode = 'stream' + - log to file + - log to system logger (syslog) + + Returns: + logger object | TYPE: logging + """ + + log_mode = local_config['LOGGING']['LOG_MODE'] + + # log format - file + file_format = '%(asctime)s - %(pathname)s - %(name)s - [%(levelname)s]: %(message)s' + + # log format - stream + stream_format = '%(pathname)s - %(name)s - [%(levelname)s]: %(message)s' + + # log format - syslog + syslog_format = '- %(pathname)s - %(name)s - [%(levelname)s]: %(message)s' + # set facility for syslog: + if local_config['LOGGING']['SYSLOG_FILE']: + syslog_facility = 'local7' + else: + syslog_facility = 'user' + + # all formats + asctime_format = "%Y-%m-%d %H:%M:%S" + + # objects + logger = logging.getLogger(*args, **kwargs) + logger.propagate = False + + try: + if not logger.handlers: + # branch on output format, default to stream + if mode_assignment(log_mode) == 'FILE': + + # file handler + if logging_prep(mode_assignment(log_mode)): + + f_handler = logging.FileHandler(local_config['LOGGING']['LOG_PATH']) + f_formatter = logging.Formatter(file_format, asctime_format) + f_handler.setFormatter(f_formatter) + logger.addHandler(f_handler) + logger.setLevel(logging.DEBUG) + + else: + syslog.warning('{i}: Log preparation fail - exit'.format( + i=inspect.stack()[0][3])) + sys.exit(1) + + elif mode_assignment(log_mode) == 'STREAM': + # stream handlers + s_handler = logging.StreamHandler() + s_formatter = logging.Formatter(stream_format) + s_handler.setFormatter(s_formatter) + logger.addHandler(s_handler) + logger.setLevel(logging.DEBUG) + + elif mode_assignment(log_mode) == 'SYSLOG': + sys_handler = logging.handlers.SysLogHandler(address='/dev/log', facility=syslog_facility) + sys_formatter = logging.Formatter(syslog_format) + sys_handler.setFormatter(sys_formatter) + logger.addHandler(sys_handler) + logger.setLevel(logging.DEBUG) + + else: + syslog.warning( + '%s: [WARNING]: log_mode value of (%s) unrecognized - not supported' % + (inspect.stack()[0][3], str(log_mode)) + ) + ex = Exception( + '%s: Unsupported mode indicated by log_mode value: %s' % + (inspect.stack()[0][3], str(log_mode)) + ) + raise ex + except OSError as e: + raise e + return logger diff --git a/build/lib/pyaws/s3/__init__.py b/build/lib/pyaws/s3/__init__.py new file mode 100644 index 0000000..224436e --- /dev/null +++ b/build/lib/pyaws/s3/__init__.py @@ -0,0 +1,3 @@ +""" +Functionality utilising Amazon EC2 Service +""" diff --git a/build/lib/pyaws/s3/check_authenticated_s3.py b/build/lib/pyaws/s3/check_authenticated_s3.py new file mode 100644 index 0000000..b663b02 --- /dev/null +++ b/build/lib/pyaws/s3/check_authenticated_s3.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 + +def authenticated_s3(profile): + """ + Summary: + Tests authentication status to AWS Account using s3 + Args: + :profile (str): iam user name from local awscli configuration + Returns: + TYPE: bool, True (Authenticated)| False (Unauthenticated) + """ + try: + s3_client = boto3_session(service='s3', profile=profile) + httpstatus = s3_client.list_buckets()['ResponseMetadata']['HTTPStatusCode'] + if httpstatus == 200: + return True + except Exception: + return False + return False diff --git a/build/lib/pyaws/s3/object_operations.py b/build/lib/pyaws/s3/object_operations.py new file mode 100644 index 0000000..ad4f200 --- /dev/null +++ b/build/lib/pyaws/s3/object_operations.py @@ -0,0 +1,37 @@ +""" +Summary. + + S3 put object Class + +""" + +import boto3 +from botocore.exceptions import ClientError +from pyaws.session import boto3_session +from pyaws import logger + + +class S3FileOperations(): + """ + Summary. + + put, delete, put-acl object operations in Amazon S3 + """ + + def __init__(self, bucket, profile=None): + self.client = boto3_session(service='s3', profile_name=profile) + self.bucket = bucket + + def put_fileobject(self, key, file, bucket=self.bucket): + r = self.client.put_object( + Bucket=bucket, Key=key, Body=file + ) + return r + + def put_object_acl(self, key, acl, bucket=self.bucket): + r = self.client.put_object_acl( + Bucket=bucket, + Key=key, + ACL=acl + ) + return r diff --git a/build/lib/pyaws/script_utils.py b/build/lib/pyaws/script_utils.py new file mode 100644 index 0000000..f3ee674 --- /dev/null +++ b/build/lib/pyaws/script_utils.py @@ -0,0 +1,302 @@ +""" +Command-line Interface (CLI) Utilities Module + +Module Functions: + - get_os: + Retrieve localhost os type, ancillary environment specifics + - awscli_defaults: + determine awscli config file locations on localhost + - import_file_object: + import text filesystem object and convert to object + - export_json_object: + write a json object to a filesystem object + - read_local_config: + parse local config file +""" +import sys +import os +import json +import platform +import logging +import inspect +import distro +from pyaws._version import __version__ + +# globals +MODULE_VERSION = '1.16' +logger = logging.getLogger(__version__) +logger.setLevel(logging.INFO) + + +def debug_mode(header, data_object, debug=False, halt=False): + """ debug output """ + if debug: + print('\n ' + str(header) + '\n') + try: + print(json.dumps(data_object, indent=4)) + except Exception: + print(data_object) + if halt: + sys.exit(0) + return True + + +def awscli_defaults(os_type=None): + """ + Summary: + Parse, update local awscli config credentials + Args: + :user (str): USERNAME, only required when run on windows os + Returns: + TYPE: dict object containing key, value pairs describing + os information + """ + + try: + if os_type is None: + os_type = platform.system() + + if os_type == 'Linux': + HOME = os.environ['HOME'] + awscli_credentials = HOME + '/.aws/credentials' + awscli_config = HOME + '/.aws/config' + elif os_type == 'Windows': + username = os.getenv('username') + awscli_credentials = 'C:\\Users\\' + username + '\\.aws\\credentials' + awscli_config = 'C:\\Users\\' + username + '\\.aws\\config' + elif os_type == 'Java': + logger.warning('Unsupported OS. No information') + HOME = os.environ['HOME'] + awscli_credentials = HOME + '/.aws/credentials' + awscli_config = HOME + '/.aws/config' + alt_credentials = os.getenv('AWS_SHARED_CREDENTIALS_FILE') + except OSError as e: + logger.exception( + '%s: problem determining local os environment %s' % + (inspect.stack()[0][3], str(e)) + ) + raise e + return { + 'awscli_defaults': { + 'awscli_credentials': awscli_credentials, + 'awscli_config': awscli_config, + 'alt_credentials': alt_credentials + } + } + + +def config_init(config_file, json_config_obj, config_dirname=None): + """ + Summary: + Creates local config from JSON seed template + Args: + :config_file (str): filesystem object containing json dict of config values + :json_config_obj (json): data to be written to config_file + :config_dirname (str): dir name containing config_file + Returns: + TYPE: bool, Success | Failure + """ + from libtools.io import export_json_object + + HOME = os.environ['HOME'] + # client config dir + if config_dirname: + dir_path = HOME + '/' + config_dirname + if not os.path.exists(dir_path): + os.mkdir(dir_path) + os.chmod(dir_path, 0o755) + else: + dir_path = HOME + # client config file + r = export_json_object( + dict_obj=json_config_obj, + filename=dir_path + '/' + config_file + ) + return r + + +def get_os(detailed=False): + """ + Summary: + Retrieve local operating system environment characteristics + Args: + :user (str): USERNAME, only required when run on windows os + Returns: + TYPE: dict object containing key, value pairs describing + os information + """ + try: + + os_type = platform.system() + + if os_type == 'Linux': + os_detail = platform.platform() + distribution = ' '.join(distro.linux_distribution()[:2]) + HOME = os.getenv('HOME') + username = os.getenv('USER') + elif os_type == 'Windows': + os_detail = platform.platform() + username = os.getenv('username') or os.getenv('USER') + HOME = 'C:\\Users\\' + username + else: + logger.warning('Unsupported OS. No information') + os_type = 'Java' + os_detail = 'unknown' + HOME = os.getenv('HOME') + username = os.getenv('USER') + + except OSError as e: + raise e + except Exception as e: + logger.exception( + '%s: problem determining local os environment %s' % + (inspect.stack()[0][3], str(e)) + ) + if detailed and os_type == 'Linux': + return { + 'os_type': os_type, + 'os_detail': os_detail, + 'linux_distribution': distribution, + 'username': username, + 'HOME': HOME + } + elif detailed: + return { + 'os_type': os_type, + 'os_detail': os_detail, + 'username': username, + 'HOME': HOME + } + return {'os_type': os_type} + + +def import_file_object(filename): + """ + Summary: + Imports block filesystem object + Args: + :filename (str): block filesystem object + Returns: + dictionary obj (valid json file), file data object + """ + try: + handle = open(filename, 'r') + file_obj = handle.read() + dict_obj = json.loads(file_obj) + + except OSError as e: + logger.critical( + 'import_file_object: %s error opening %s' % (str(e), str(filename)) + ) + raise e + except ValueError: + logger.info( + '%s: import_file_object: %s not json. file object returned' % + (inspect.stack()[0][3], str(filename)) + ) + return file_obj # reg file, not valid json + return dict_obj + + +def json_integrity(baseline, suspect): + """ + Summary: + Validates baseline dict against suspect dict to ensure contain USERNAME + k,v parameters. + Args: + baseline (dict): baseline json structure + suspect (dict): json object validated against baseline structure + Returns: + Success (matches baseline) | Failure (no match), TYPE: bool + """ + try: + for k,v in baseline.items(): + for ks, vs in suspect.items(): + keys_baseline = set(v.keys()) + keys_suspect = set(vs.keys()) + intersect_keys = keys_baseline.intersection(keys_suspect) + added = keys_baseline - keys_suspect + rm = keys_suspect - keys_baseline + logger.info('keys added: %s, keys removed %s' % (str(added), str(rm))) + if keys_baseline != keys_suspect: + return False + except KeyError as e: + logger.info( + 'KeyError parsing pre-existing config (%s). Replacing config file' % + str(e)) + return True + + +def json_integrity_multilevel(d1, d2): + """ still under development """ + keys = [x for x in d2] + for key in keys: + d1_keys = set(d1.keys()) + d2_keys = set(d2.keys()) + intersect_keys = d1_keys.intersection(d2_keys) + added = d1_keys - d2_keys + removed = d2_keys - d1_keys + modified = {o : (d1[o], d2[o]) for o in intersect_keys if d1[o] != d2[o]} + same = set(o for o in intersect_keys if d1[o] == d2[o]) + if added == removed == set(): + d1_values = [x for x in d1.values()][0] + print('d1_values: ' + str(d1_values)) + d2_values = [x for x in d2.values()][0] + print('d2_values: ' + str(d2_values)) + length = len(d2_values) + print('length = %d' % length) + pdb.set_trace() + if length > 1: + d1 = d1_values.items() + d2 = d2_values.items() + else: + return False + return True + + +def read_local_config(cfg): + """ Parses local config file for override values + + Args: + :local_file (str): filename of local config file + + Returns: + dict object of values contained in local config file + """ + try: + if os.path.exists(cfg): + config = import_file_object(cfg) + return config + else: + logger.warning( + '%s: local config file (%s) not found, cannot be read' % + (inspect.stack()[0][3], str(cfg))) + except OSError as e: + logger.warning( + 'import_file_object: %s error opening %s' % (str(e), str(cfg)) + ) + return {} + + +def os_parityPath(path): + """ + Converts unix paths to correct windows equivalents. + Unix native paths remain unchanged (no effect) + """ + path = os.path.normpath(os.path.expanduser(path)) + if path.startswith('\\'): + return 'C:' + path + return path + + +def directory_contents(directory): + """Returns full paths of all file objects contained in a directory""" + filepaths = [] + try: + for dirpath,_,filenames in os.walk(directory): + for f in filenames: + filepaths.append(os.path.abspath(os.path.join(dirpath, f))) + except OSError as e: + logger.exception( + '{}: Problem walking directory contents: {}'.format(inspect.stack()[0][3], e)) + return filepaths diff --git a/build/lib/pyaws/session.py b/build/lib/pyaws/session.py new file mode 100644 index 0000000..45c2871 --- /dev/null +++ b/build/lib/pyaws/session.py @@ -0,0 +1,208 @@ +import os +import inspect +import subprocess +import boto3 +from botocore.exceptions import ClientError, ProfileNotFound +from pyaws.utils import stdout_message +from pyaws import logger + +try: + from pyaws.core.oscodes_unix import exit_codes + splitchar = '/' # character for splitting paths (linux) +except Exception: + from pyaws.core.oscodes_win import exit_codes # non-specific os-safe codes + splitchar = '\\' # character for splitting paths (window + + +DEFAULT_REGION = os.environ['AWS_DEFAULT_REGION'] or 'us-east-1' + + +def _profile_prefix(profile, prefix='gcreds'): + """ + Summary: + Determines if temporary STS credentials provided via + local awscli config; + - if yes, returns profile with correct prefix + - if no, returns profile (profile_name) unaltered + - Note: Caller is parse_profiles(), Not to be called directly + Args: + profile (str): profile_name of a valid profile from local awscli config + prefix (str): prefix prepended to profile containing STS temporary credentials + Returns: + awscli profilename, TYPE str + """ + + stderr = ' 2>/dev/null' + tempProfile = prefix + '-' + profile + + try: + if subprocess.check_output( + 'aws configure get profile.{profile}.aws_access_key_id {stderr}'.format( + profile=profile, stderr=stderr)): + return profile + elif subprocess.check_output( + 'aws configure get profile.{tempProfile}.aws_access_key_id {stderr}'.format( + tempProfile=tempProfile, stderr=stderr)): + return tempProfile + except Exception as e: + logger.exception( + '{i}: Unknown error while interrogating local awscli config: {e}'.format( + i=inspect.stack()[0][3], e=e)) + raise + return None + + +def parse_profiles(profiles): + """ + Summary: + Parse awscli profile_names given as parameter in 1 of 2 forms: + 1. single profilename given + 2. list of profile_names + Also, function prepends profile_name(s) with a prefix when it detects + profile_name refers to temp credentials in the local awscli configuration + Args: + profiles (str or file): Profiles parameter can be either: + - a single profile_name (str) + - a file containing multiple profile_names, 1 per line + Returns: + - list of awscli profilenames, TYPE: list + OR + - single profilename, TYPE: str + """ + + profile_list = [] + + try: + if isinstance(profiles, list): + return [_profile_prefix(x.strip()) for x in profiles] + elif os.path.isfile(profiles): + with open(profiles) as f1: + for line in f1: + profile_list.append(_profile_prefix(line.strip())) + else: + return _profile_prefix(profiles.strip()) + except Exception as e: + logger.exception( + '{i}: Unknown error while converting profile_names from local awscli config: {e}'.format( + i=inspect.stack()[0][3], e=e)) + raise + return profile_list + + +def boto3_session(service, region=DEFAULT_REGION, profile=None): + """ + Summary: + Establishes boto3 sessions, client + Args: + :service (str): boto3 service abbreviation ('ec2', 's3', etc) + :profile (str): profile_name of an iam user from local awscli config + :region (str): AWS region code, optional + + Returns: + client (boto3 object) + + """ + fx = inspect.stack()[0][3] + + try: + + if (not profile or profile == 'default') and service != 'iam': + return boto3.client(service, region_name=region) + + elif (not profile or profile == 'default') and service == 'iam': + return boto3.client(service) + + elif profile and profile != 'default': + session = boto3.Session(profile_name=profile) + return session.client(service, region_name=region) + + except ClientError as e: + if e.response['Error']['Code'] == 'InvalidClientTokenId': + logger.warning( + '{}: Invalid credentials used by profile user {}'.format(fx, profile or 'default') + ) + + elif e.response['Error']['Code'] == 'ExpiredToken': + logger.info( + '%s: Expired temporary credentials detected for profile user (%s) [Code: %d]' + % (fx, profile, exit_codes['EX_CONFIG']['Code'])) + + except ProfileNotFound: + msg = ('{}: Profile name {} was not found in your local config.'.format(fx, profile)) + stdout_message(msg, 'WARN') + logger.warning(msg) + return None + return boto3.client(service, region_name=region) + + +def authenticated(profile): + """ + Tests generic authentication status to AWS Account + + Args: + :profile (str): iam user name from local awscli configuration + + Returns: + TYPE: bool, True (Authenticated)| False (Unauthenticated) + + """ + try: + sts_client = boto3_session(service='sts', profile=profile) + httpstatus = sts_client.get_caller_identity()['ResponseMetadata']['HTTPStatusCode'] + if httpstatus == 200: + return True + + except ClientError as e: + if e.response['Error']['Code'] == 'InvalidClientTokenId': + logger.warning( + '%s: Invalid credentials to authenticate for profile user (%s). Exit. [Code: %d]' + % (inspect.stack()[0][3], profile, exit_codes['EX_NOPERM']['Code'])) + + elif e.response['Error']['Code'] == 'ExpiredToken': + logger.info( + '%s: Expired temporary credentials detected for profile user (%s) [Code: %d]' + % (inspect.stack()[0][3], profile, exit_codes['EX_CONFIG']['Code'])) + + else: + logger.exception( + '%s: Unknown Boto3 problem. Error: %s' % + (inspect.stack()[0][3], e.response['Error']['Message'])) + except Exception as e: + fx = inspect.stack()[0][3] + logger.exception('{}: Unknown error: {}'.format(fx, e)) + return False + return False + + +def client_wrapper(service, profile='default', region=DEFAULT_REGION): + """ + Summary. + + Single caller boto3 service wrapper. Instantiates client object while + using temporary credientials for profile_name, if available in + local configuration. Tests authentication prior to returning any + client object. + + Args: + :service (str): boto3 service abbreviation ('ec2', 's3', etc) + :profile (str): profile_name of an iam user from local awscli config + :region (str): AWS region code, optional + + Returns: + client (boto3 object) + + """ + profile_name = _profile_prefix(profile) + + try: + + if authenticated(profile_name): + return boto3_session(service=service, profile=profile_name, region=region) + + except ClientError as e: + logger.exception( + "%s: Unknown boto3 failure while establishing session (Code: %s Message: %s)" % + (inspect.stack()[0][3], e.response['Error']['Code'], + e.response['Error']['Message']) + ) + return None diff --git a/build/lib/pyaws/statics.py b/build/lib/pyaws/statics.py new file mode 100644 index 0000000..1d07077 --- /dev/null +++ b/build/lib/pyaws/statics.py @@ -0,0 +1,71 @@ +""" +Summary: + pyaws Project-level Defaults and Settings + + - **Local Default Settings**: Local defaults for your specific installation are derived from settings found in: + +Module Attributes: + - user_home (TYPE str): + $HOME environment variable, present for most Unix and Unix-like POSIX systems + - config_dir (TYPE str): + directory name default for stsaval config files (.stsaval) + - config_path (TYPE str): + default for stsaval config files, includes config_dir (~/.stsaval) +""" +import os +import inspect +import logging +from pyaws.script_utils import get_os, os_parityPath +from pyaws._version import __version__ + +logger = logging.getLogger(__version__) +logger.setLevel(logging.INFO) + + +# -- project-level DEFAULTS ------------------------------------------------ + + +try: + + env_info = get_os(detailed=True) + OS = env_info['os_type'] + user_home = env_info['HOME'] + + if user_home is None: + user_home = os.getenv('HOME') or '/tmp' + +except KeyError as e: + logger.critical( + '%s: %s variable is required and not found in the environment' % + (inspect.stack()[0][3], str(e))) + raise e + +else: + # project + PACKAGE = 'pyaws' + LICENSE = 'MIT' + LICENSE_DESC = 'MIT' + + # logging parameters + enable_logging = True + log_mode = 'STREAM' # log to cloudwatch logs + log_filename = PACKAGE + '.log' + log_dir = os_parityPath(user_home + '/logs') + log_path = os_parityPath(log_dir + '/' + log_filename) + + local_config = { + "PROJECT": { + "PACKAGE": PACKAGE, + "CONFIG_VERSION": __version__, + "HOME": user_home, + + }, + "LOGGING": { + "ENABLE_LOGGING": enable_logging, + "LOG_FILENAME": log_filename, + "LOG_DIR": log_dir, + "LOG_PATH": log_path, + "LOG_MODE": log_mode, + "SYSLOG_FILE": False + } + } diff --git a/build/lib/pyaws/sts/__init__.py b/build/lib/pyaws/sts/__init__.py new file mode 100644 index 0000000..224436e --- /dev/null +++ b/build/lib/pyaws/sts/__init__.py @@ -0,0 +1,3 @@ +""" +Functionality utilising Amazon EC2 Service +""" diff --git a/build/lib/pyaws/sts/cross_account_utils.py b/build/lib/pyaws/sts/cross_account_utils.py new file mode 100644 index 0000000..d5e7c4d --- /dev/null +++ b/build/lib/pyaws/sts/cross_account_utils.py @@ -0,0 +1,123 @@ +""" + +cross_account_utils (python3) + + Common Secure Token Service (STS) functionality for use + with AWS' Lambda Service + +Author: + Blake Huber + Copyright Blake Huber, All Rights Reserved. + +License: + Permission to use, copy, modify, and distribute this software and its + documentation for any purpose and without fee is hereby granted, + provided that the above copyright notice appear in all copies and that + both the copyright notice and this permission notice appear in + supporting documentation + + Additional terms may be found in the complete license agreement: + https://bitbucket.org/blakeca00/lambda-library-python/src/master/LICENSE.md + +""" + +import inspect +import boto3 +from botocore.exceptions import ClientError +import loggers +from _version import __version__ + + +# lambda custom log object +logger = loggers.getLogger(__version__) + + +class AssumeAWSRole(): + """ class def for assuming roles in AWS """ + def __init__(self, account, role_name, profile=None): + self.role = role_name + self.account_number = str(account) + self.profile = profile + self.credentials = self.assume_role(role_name, self.account_number) + self.status = {'STATUS': ''} + + def assume_role(self, account, role): + """ + Summary: + Assumes a DynamoDB role in 'destination' AWS account + Args: + :type account: str + :param account: AWS account number + :type role: str + :param role: IAM role designation used to access AWS resources + in an account + :type profile: str + :param role: profile_name is an IAM user or IAM role name represented + in the local awscli configuration as a profile entry + Returns: dict (Credentials) + """ + if self.profile: + session = boto3.Session(profile_name=self.profile) + else: + session = boto3.Session() + sts_client = session.client('sts') + + try: + # assume role in destination account + assumed_role = sts_client.assume_role( + RoleArn="arn:aws:iam::%s:role/%s" % (account, role), + RoleSessionName="AssumeAWSRoleSession" + ) + self.status = {'STATUS': 'SUCCESS'} + except ClientError as e: + if e.response['Error']['Code'] == 'AccessDenied': + self.status = {'STATUS': 'AccessDenied'} + return {} + else: + logger.exception("Couldn't assume role %s in account %s (Code: %s Message: %s)" % + (self.role, self.account_number, e.response['Error']['Code'], + e.response['Error']['Message'])) + self.status = {'STATUS': 'ERROR'} + return {} + return assumed_role['Credentials'] + + def create_service_client(self, aws_service, credentials=None): + """ + Summary: + Creates the appropriate boto3 client for a particular AWS service + Args: + :type service: str + :param service: name of service at Amazon Web Services (AWS), + e.g. s3, ec2, etc + :type credentials: sts credentials object + :param credentials: authentication credentials to resource in AWS + :type role: str + :param role: IAM role designation used to access AWS resources + in an account + Returns: + Success | Failure, TYPE: bool + """ + try: + if role and account: # create client for a different AWS account + if self.status.get('STATUS') == 'SUCCESS': + client = boto3.client( + aws_service, + aws_access_key_id=self.credentials['AccessKeyId'], + aws_secret_access_key=self.credentials['SecretAccessKey'], + aws_session_token=self.credentials['SessionToken'] + ) + else: + logger.critical('failed to create client using AssumeAWSRole') + raise ClientError( + '%s: Problem creating client by assuming role %s in account %s' % + (inspect.stack()[0][3], str(role), str(account)) + ) + else: + return boto3.client(aws_service) # create client in the current AWS account + except ClientError as e: + logger.exception( + "%s: Problem creating %s client in account %s (Code: %s Message: %s)" % + (inspect.stack()[0][3], aws_service, self.account_number, + e.response['Error']['Code'], e.response['Error']['Message'])) + raise + return client diff --git a/build/lib/pyaws/tags/__init__.py b/build/lib/pyaws/tags/__init__.py new file mode 100644 index 0000000..db6bd29 --- /dev/null +++ b/build/lib/pyaws/tags/__init__.py @@ -0,0 +1,4 @@ +""" +Functional Utilities for working with Amazon Web Services' Resource Tags +""" +from pyaws.tags.tag_utils import * diff --git a/build/lib/pyaws/tags/bulk-modify-tags.py b/build/lib/pyaws/tags/bulk-modify-tags.py new file mode 100644 index 0000000..987e359 --- /dev/null +++ b/build/lib/pyaws/tags/bulk-modify-tags.py @@ -0,0 +1,334 @@ +#!/usr/bin/env python3 + +""" +Summary: + Script to copy tags on all EC2 instances in a region to their respective + EBS Volumes while eliminating or retaining certain tags as specified + +Args: + profiles (list): awscli profile roles. Denotes accounts in which to run + regions (list): AWS region codes + DEBUGMODE (bool): don't change tags, but print out tags to be copied + SUMMARY_REPORT (bool): gen summary report only + logger (logging object): logger + +Author: + Blake Huber, copyright 2017 + + (although it's hard to think someone would be desperate enough to rip + off such hastily written crap code. At some pt I'll clean it up, meanwhile + Use at your own risk) +""" + +import sys +import argparse +import inspect +import datetime +from time import sleep +import boto3 +from botocore.exceptions import ClientError +from libtools.io import export_json_object +from libtools import stdout_message +from pyaws import exit_codes, logger, local_config + + +DEBUGMODE = True # will not retag any resources +SUMMARY_REPORT = True # print summary report only +VALID_TYPES = ['instances', 'volumes', 'display'] + +regions = ['ap-southeast-1', 'eu-west-1', 'us-east-1'] + +profiles = [ + 'gcreds-phht-gen-ra1-pr', + 'gcreds-phht-gen-ra2-pr', + 'gcreds-phht-gen-ra3-pr', + 'gcreds-phht-gen-ra4-pr', + 'gcreds-phht-gen-ra5-pr', +] + +profiles = ['gcreds-phht-gen-ra3-pr'] + +# tags - to remove +TAGKEY_BACKUP = 'MPC-AWS-BACKUP' +TAGKEY_CPM = 'cpm backup' +TAGKEY_SNOW_CPM = 'MPC-SN-BACKUP' +NETWORKER = 'networker backup' +NAME = 'Name' + +# tags we should not copy from the ec2 instance to the ebs volume +NO_COPY_LIST = [TAGKEY_BACKUP, TAGKEY_CPM, TAGKEY_SNOW_CPM, NETWORKER, NAME] + +# tags on ebs volumes to preserve and ensure we do not overwrite or rm +PRESERVE_TAGS = ['Name'] + + +# -- declarations ------------------------------------------------------------- + + +def filter_tags(tag_list, *args): + """ + - Filters a tag set by exclusion + - variable tag keys given as parameters, tag keys corresponding to args + are excluded + + RETURNS + TYPE: list + """ + clean = tag_list.copy() + for tag in tag_list: + for arg in args: + if arg == tag['Key']: + clean.remove(tag) + return clean + + +def valid_tags(tag_list): + """ checks tags for invalid chars """ + for tag in tag_list: + if ':' in tag['Key']: + return False + return True + + +def pretty_print_tags(tag_list): + """ prints json tags with syntax highlighting """ + export_json_object(tag_list) + print('\n') + return + + +def select_tags(tag_list, key_list): + """ + Return selected tags from a list of many tags given a tag key + """ + select_list = [] + # select tags by keys in key_list + for tag in tag_list: + for key in key_list: + if key == tag['Key']: + select_list.append(tag) + # ensure only tag-appropriate k,v pairs in list + return [{'Key': x['Key'], 'Value': x['Value']} for x in select_list] + + +def get_instances(profile, rgn): + """ returns all EC2 instance Ids in a region """ + vm_ids = [] + session = boto3.Session(profile_name=profile, region_name=rgn) + client = session.client('ec2') + r = client.describe_instances() + for detail in [x['Instances'] for x in r['Reservations']]: + for instance in detail: + vm_ids.append(instance['InstanceId']) + return vm_ids + + +def get_volumes(profile, rgn): + """ + returns all EC2 volume Ids in a region + """ + session = boto3.Session(profile_name=profile, region_name=rgn) + client = session.client('ec2') + return [x['VolumeId'] for x in client.describe_volumes()['Volumes']] + + +def calc_runtime(start, end): + """ Calculates job runtime given start, end datetime stamps + Args: + - start (datetime object): job start timestamp + - end (datetime object): job end timestamp + """ + duration = end - start + if (duration.seconds / 60) < 1: + return (duration.seconds), 'seconds' + else: + return (duration.seconds / 60), 'minutes' + + +def display_valid(print_object): + """ + Help Function | Displays an attribute of this program + """ + if print_object in ('list', 'print', 'display'): + print('\n' + VALID_TYPES + '\n') + return True + + +def options(parser): + """ + Summary: + parse cli parameter options + Returns: + TYPE: argparse object, parser argument set + """ + parser.add_argument("-p", "--profile", nargs='?', default="default", + required=False, help="type (default: %(default)s)") + parser.add_argument("-t", "--type", nargs='?', default='list', type=str, + choices=VALID_TYPES, required=False) + parser.add_argument("-a", "--auto", dest='auto', action='store_true', required=False) + parser.add_argument("-d", "--debug", dest='debug', action='store_true', required=False) + parser.add_argument("-V", "--version", dest='version', action='store_true', required=False) + return parser.parse_args() + + +# -- main --------------------------------------------------------------------- + + +def main(): + """ copies ec2 instance tags to attached resources """ + for profile in profiles: + # derive account alias from profile + account = '-'.join(profile.split('-')[1:]) + for region in regions: + #instances = [] + session = boto3.Session(profile_name=profile, region_name=region) + client = session.client('ec2') + ec2 = session.resource('ec2') + instances = get_instances(profile, region) + volumes = get_volumes(profile, region) + # print summary + if SUMMARY_REPORT: + print('\nFor AWS Account %s, region %s, Found %d Instances\n' % (account, region, len(instances))) + continue + # copy tags + if instances: + try: + base = ec2.instances.filter(InstanceIds=instances) + ct = 0 + for instance in base: + ids, after_tags = [], [] + ct += 1 + if instance.tags: + # filter out tags to prohibited from copy + filtered_tags = filter_tags(instance.tags, *NO_COPY_LIST) + else: + # no tags on instance to copy + continue + + if not valid_tags(filtered_tags): + print('\nWARNING:') + logger.warning('Skipping instance ID %s, Invalid Tags\n' % instance.id) + continue + # collect attached resource ids to be tagged + for vol in instance.volumes.all(): + ids.append(vol.id) + for eni in instance.network_interfaces: + ids.append(eni.id) + logger.info('InstanceID %s, instance %d of %d:' % (instance.id, ct, len(instances))) + logger.info('Resource Ids to tag is:') + logger.info(str(ids) + '\n') + if DEBUGMODE: + # BEFORE tag copy + logger.info('BEFORE list of %d tags is:' % (len(instance.tags))) + pretty_print_tags(instance.tags) + + # AFTER tag copy | put Name tag back into apply tags, ie, after_tags + retain_tags = select_tags(instance.tags, PRESERVE_TAGS) + all_tags = retain_tags + filtered_tags + for tag in all_tags: + after_tags.append(tag) + logger.info('For InstanceID %s, the AFTER FILTERING list of %d tags is:' % (instance.id, len(after_tags))) + logger.info('Tags to apply are:') + pretty_print_tags(after_tags) + else: + logger.info('InstanceID %s, instance %d of %d:' % (instance.id, ct, len(instances))) + if filtered_tags: # we must have something to apply + # apply tags + for resourceId in ids: + # retain a copy of tags to preserve if is a volume + if resourceId.startswith('vol-'): + r = client.describe_tags( + Filters=[{ + 'Name': 'resource-id', + 'Values': [resourceId], + }, + ] + ) + retain_tags = select_tags(r['Tags'], PRESERVE_TAGS) + # add retained tags before appling to volume + if retain_tags: + for tag in retain_tags: + filtered_tags.append(tag) + # clear tags + print('\n') + logger.info('Clearing tags on resource: %s' % str(resourceId)) + client.delete_tags(Resources=[resourceId], Tags=[]) + + # create new tags + logger.info('Applying tags to resource %s\n' % resourceId) + ec2.create_tags(Resources=[resourceId], Tags=filtered_tags) + # delay to throttle API requests + sleep(1) + + except ClientError as e: + logger.exception( + "%s: Problem (Code: %s Message: %s)" % + (inspect.stack()[0][3], e.response['Error']['Code'], + e.response['Error']['Message']) + ) + raise + + +def init_cli(): + # parser = argparse.ArgumentParser(add_help=False, usage=help_menu()) + parser = argparse.ArgumentParser(add_help=False) + + try: + args = options(parser) + except Exception as e: + help_menu() + stdout_message(str(e), 'ERROR') + sys.exit(exit_codes['EX_OK']['Code']) + + if len(sys.argv) == 1: + help_menu() + sys.exit(exit_codes['EX_OK']['Code']) + + elif args.help: + help_menu() + sys.exit(exit_codes['EX_OK']['Code']) + + elif args.version: + package_version() + + elif args.configure: + r = option_configure(args.debug, local_config['PROJECT']['CONFIG_PATH']) + return r + else: + if precheck(): # if prereqs set, run + if authenticated(profile=args.profile): + # execute keyset operation + success = main( + operation=args.operation, + profile=args.profile, + user_name=args.username, + auto=args.auto, + debug=args.debug + ) + if success: + logger.info('IAM access keyset operation complete') + sys.exit(exit_codes['EX_OK']['Code']) + else: + stdout_message( + 'Authenication Failed to AWS Account for user %s' % args.profile, + prefix='AUTH', + severity='WARNING' + ) + sys.exit(exit_codes['E_AUTHFAIL']['Code']) + + failure = """ : Check of runtime parameters failed for unknown reason. + Please ensure local awscli is configured. Then run keyconfig to + configure keyup runtime parameters. Exiting. Code: """ + logger.warning(failure + 'Exit. Code: %s' % sys.exit(exit_codes['E_MISC']['Code'])) + print(failure) + + +if __name__ == '__main__': + start_time = datetime.datetime.now() + main() + end_time = datetime.datetime.now() + duration, unit = calc_runtime(start_time, end_time) + logger.info('Job Start: %s' % start_time.isoformat()) + logger.info('Job End: %s' % end_time.isoformat()) + logger.info('Job Completed. Duration: %d %s' % (duration, unit)) + sys.exit(0) diff --git a/build/lib/pyaws/tags/copy-tags-all-instances.py b/build/lib/pyaws/tags/copy-tags-all-instances.py new file mode 100644 index 0000000..8538d44 --- /dev/null +++ b/build/lib/pyaws/tags/copy-tags-all-instances.py @@ -0,0 +1,241 @@ +#!/usr/bin/env python3 + +""" +Summary: + Script to copy tags on all EC2 instances in a region to their respective + EBS Volumes while eliminating or retaining certain tags as specified + +Args: + profiles (list): awscli profile roles. Denotes accounts in which to run + regions (list): AWS region codes + DEBUGMODE (bool): don't change tags, but print out tags to be copied + SUMMARY_REPORT (bool): gen summary report only + logger (logging object): logger + +Author: + Blake Huber, copyright 2017 + + (although it's hard to think someone would be desperate enough to rip + off such hastily written crap code. At some pt I'll clean it up, meanwhile + Use at your own risk) +""" + +import sys +import loggers +import inspect +import datetime +from time import sleep +from libtools.js import export_json_object +import boto3 +from botocore.exceptions import ClientError + +logger = loggers.getLogger('1.0') +DEBUGMODE = True # will not retag any resources +SUMMARY_REPORT = False # print summary report only + +regions = ['ap-southeast-1', 'eu-west-1', 'us-east-1'] + +profiles = [ + 'gcreds-phht-gen-ra1-pr', + 'gcreds-phht-gen-ra2-pr', + 'gcreds-phht-gen-ra3-pr', + 'gcreds-phht-gen-ra4-pr', + 'gcreds-phht-gen-ra5-pr', +] + +profiles = ['gcreds-phht-gen-ra3-pr'] + +# tags - to remove +TAGKEY_BACKUP = 'MPC-AWS-BACKUP' +TAGKEY_CPM = 'cpm backup' +TAGKEY_SNOW_CPM = 'MPC-SN-BACKUP' +NETWORKER = 'networker backup' +NAME = 'Name' + +# tags we should not copy from the ec2 instance to the ebs volume +NO_COPY_LIST = [TAGKEY_BACKUP, TAGKEY_CPM, TAGKEY_SNOW_CPM, NETWORKER, NAME] + +# tags on ebs volumes to preserve and ensure we do not overwrite or rm +PRESERVE_TAGS = ['Name'] + +# -- declarations ------------------------------------------------------------- + + +def filter_tags(tag_list, *args): + """ + - Filters a tag set by exclusion + - variable tag keys given as parameters, tag keys corresponding to args + are excluded + + RETURNS + TYPE: list + """ + clean = tag_list.copy() + for tag in tag_list: + for arg in args: + if arg == tag['Key']: + clean.remove(tag) + return clean + + +def valid_tags(tag_list): + """ checks tags for invalid chars """ + for tag in tag_list: + if ':' in tag['Key']: + return False + return True + + +def pretty_print_tags(tag_list): + """ prints json tags with syntax highlighting """ + export_json_object(tag_list) + print('\n') + return + + +def select_tags(tag_list, key_list): + """ + Return selected tags from a list of many tags given a tag key + """ + select_list = [] + # select tags by keys in key_list + for tag in tag_list: + for key in key_list: + if key == tag['Key']: + select_list.append(tag) + # ensure only tag-appropriate k,v pairs in list + clean = [{'Key': x['Key'], 'Value': x['Value']} for x in select_list] + return clean + + +def get_instances(profile, rgn): + """ returns all EC2 instance Ids in a region """ + vm_ids = [] + session = boto3.Session(profile_name=profile, region_name=rgn) + client = session.client('ec2') + r = client.describe_instances() + for detail in [x['Instances'] for x in r['Reservations']]: + for instance in detail: + vm_ids.append(instance['InstanceId']) + return vm_ids + + +def calc_runtime(start, end): + """ Calculates job runtime given start, end datetime stamps + Args: + - start (datetime object): job start timestamp + - end (datetime object): job end timestamp + """ + duration = end - start + if (duration.seconds / 60) < 1: + return (duration.seconds), 'seconds' + else: + return (duration.seconds / 60), 'minutes' + + +# -- main --------------------------------------------------------------------- + + +def main(): + """ copies ec2 instance tags to attached resources """ + for profile in profiles: + # derive account alias from profile + account = '-'.join(profile.split('-')[1:]) + for region in regions: + #instances = [] + session = boto3.Session(profile_name=profile, region_name=region) + client = session.client('ec2') + ec2 = session.resource('ec2') + instances = get_instances(profile, region) + + if SUMMARY_REPORT: + print('\nFor AWS Account %s, region %s, Found %d Instances\n' % (account, region, len(instances))) + continue + # copy tags + if instances: + try: + base = ec2.instances.filter(InstanceIds=instances) + ct = 0 + for instance in base: + ids, after_tags = [], [] + ct += 1 + if instance.tags: + # filter out tags to prohibited from copy + filtered_tags = filter_tags(instance.tags, *NO_COPY_LIST) + else: + # no tags on instance to copy + continue + + if not valid_tags(filtered_tags): + print('\nWARNING:') + logger.warning('Skipping instance ID %s, Invalid Tags\n' % instance.id) + continue + # collect attached resource ids to be tagged + for vol in instance.volumes.all(): + ids.append(vol.id) + for eni in instance.network_interfaces: + ids.append(eni.id) + logger.info('InstanceID %s, instance %d of %d:' % (instance.id, ct, len(instances))) + logger.info('Resource Ids to tag is:') + logger.info(str(ids) + '\n') + if DEBUGMODE: + # BEFORE tag copy + logger.info('BEFORE list of %d tags is:' % (len(instance.tags))) + pretty_print_tags(instance.tags) + + # AFTER tag copy | put Name tag back into apply tags, ie, after_tags + retain_tags = select_tags(instance.tags, PRESERVE_TAGS) + all_tags = retain_tags + filtered_tags + for tag in all_tags: + after_tags.append(tag) + logger.info('For InstanceID %s, the AFTER FILTERING list of %d tags is:' % (instance.id, len(after_tags))) + logger.info('Tags to apply are:') + pretty_print_tags(after_tags) + else: + logger.info('InstanceID %s, instance %d of %d:' % (instance.id, ct, len(instances))) + if filtered_tags: # we must have something to apply + # apply tags + for resourceId in ids: + # retain a copy of tags to preserve if is a volume + if resourceId.startswith('vol-'): + r = client.describe_tags( + Filters=[{ + 'Name': 'resource-id', + 'Values': [resourceId], + }, + ] + ) + retain_tags = select_tags(r['Tags'], PRESERVE_TAGS) + # add retained tags before appling to volume + if retain_tags: + for tag in retain_tags: + filtered_tags.append(tag) + # clear tags + print('\n') + logger.info('Clearing tags on resource: %s' % str(resourceId)) + client.delete_tags(Resources=[resourceId], Tags=[]) + + # create new tags + logger.info('Applying tags to resource %s\n' % resourceId) + ec2.create_tags(Resources=[resourceId], Tags=filtered_tags) + # delay to throttle API requests + sleep(1) + + except ClientError as e: + logger.exception( + "%s: Problem (Code: %s Message: %s)" % + (inspect.stack()[0][3], e.response['Error']['Code'], + e.response['Error']['Message']) + ) + raise + + +if __name__ == '__main__': + start_time = datetime.datetime.now() + main() + end_time = datetime.datetime.now() + duration, unit = calc_runtime(start_time, end_time) + logger.info('Job Start: %s' % start_time.isoformat()) + logger.info('Job End: %s' % end_time.isoformat()) + logger.info('Job Completed. Duration: %d %s' % (duration, unit)) + sys.exit(0) diff --git a/build/lib/pyaws/tags/ec2-update-tags.py b/build/lib/pyaws/tags/ec2-update-tags.py new file mode 100644 index 0000000..c729da7 --- /dev/null +++ b/build/lib/pyaws/tags/ec2-update-tags.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python + +import sys +import json +import argparse +import boto3 + +# -- functions ---------------------------------------------------------------- + + +def get_regions(): + """Summary + + Returns: + TYPE: Description + """ + client = boto3.client('ec2') + region_response = client.describe_regions() + regions = [region['RegionName'] for region in region_response['Regions']] + return regions + + +def remove_tags(id_list, tag_list): + """ + Deletes tags on resource ids provided as parameter + """ + pass + + +def clean_list(list): + """ cleans a list of all extraneous characters """ + clean_list = [] + try: + for element in list: + clean_list.append(element.strip()) + except Exception: + return -1 + return clean_list + + +def remove_duplicates(list): + """ + Removes duplicate dict in a list of dict by enforcing unique keys + """ + + clean_list, key_list = [], [] + + try: + for dict in list: + if dict['Key'] not in key_list: + clean_list.append(dict) + key_list.append(dict['Key']) + + except KeyError: + # dedup list of items, not dict + for item in list: + if clean_list: + if item not in clean_list: + clean_list.append(item) + else: + clean_list.append(item) + return clean_list + except Exception: + return -1 + return clean_list + + +def remove_restricted(list): + """ + Remove restricted Amazon tags from list of tags + """ + + clean_list = [] + + try: + for dict in list: + if 'aws' not in dict['Key']: + clean_list.append(dict) + except Exception: + return -1 + return clean_list + + +# -- MAIN ---------------------------------------------------------------------- + +# global vars +global DBUGMODE +global REGION +global PREFIX +PREFIX = 'gcreds-' # temporary credential prefix in local awscli config +global PROFILE + +# parse options setup +parser = argparse.ArgumentParser(description='Required parameters:') +parser.add_argument('-a','--accts', dest='file', action='store', + help='Profile name from local awscli config') +parser.add_argument('-c','--convert', action='store_true', + help='optional flag required to execute tag conversion') +args = parser.parse_args() + +if len(sys.argv) == 1: + parser.print_help() + sys.exit(0) + +if args.convert: + DBUGMODE = False +else: + DBUGMODE = True + +try: + fileobj1 = open(args.file) + accounts = fileobj1.readlines() + print("\nContents of accounts list: ") + for acct in accounts: + print(acct.strip()) + print("Length of accounts list: " + str(len(accounts))) + +except IOError as e: + print('File passed as parameter cannot be opened') +except Exception: + print("Unexpected error:", sys.exc_info()[0]) + raise + +# clean lists +accounts = clean_list(accounts) + +# list iam users for each account +for PROFILE in accounts: + # + TMPPROFILE = PREFIX + PROFILE + # + regions = get_regions() + change_record = [] # list to track number of resources for which config updated + regions = ['eu-west-1'] + for region in regions: + session = boto3.Session(profile_name=TMPPROFILE, region_name=region) + #base = AWSEC2_resource(PROFILE, REGION) + ec2 = session.resource('ec2') + base = ec2.instances.all() + # the following used if converting for one or more specific instances: + #ids = ['i-bc816f59'] + #base = ec2.instances.filter(InstanceIds=ids) + + ct = 0 + for instance in base: + ct += 1 + print('\n---------------------------------------------------------------------------------') + print('\n ' + str(ct) + ' instances found for account [' + str(PROFILE) + + '] in region [' + str(region) + ']\n') + print('---------------------------------------------------------------------------------\n') + + for instance in base: + # initialize tag lists + delete_tags = [] # k,v pairs (tags) to be deleted + tags = instance.tags # k,v pairs (pre-conversion tags on instance) + + # calc columns + col_width = 0 + for t in tags: + tag_width = len(t['Key']) + 2 + if tag_width > col_width: + col_width = tag_width + #print('col_width is: ' + str(col_width)) + col_width = 30 + + print('\nEC2 INSTANCE ID [' + str(instance.id) + '] ----------------------------\n') + print('\n\tExisting tag set:\n') + tags.sort() + for t in tags: + s1 = 'Key: ' + str(t['Key']) + s2 = 'Value: ' + str(t['Value']) + # print in 2 columns + print '{0:30} {1}'.format(s1, s2) + print('\n') + + # remove restricted tags before we attempt to convert and reapply + tags = remove_restricted(tags) + + # convert tag keys + for t in tags: + if t.get('MPC-SN-NAME'): + delete_tags.append(t) + tags.remove(t) + #change_record.append(instance.id) + + elif 'CreationDate' in t['Key']: + d = t.copy() + delete_tags.append(d) + t['Key'] = 'MPC-AWS-CREATIONDATE' + #change_record.append(instance.id) + + elif 'BillGroup' in t['Key']: + d = t.copy() + delete_tags.append(d) + t['Key'] = 'MPC-AWS-BILLGROUP' + #change_record.append(instance.id) + + elif t['Key'] == 'TAG-BACKUP': + d = t.copy() + delete_tags.append(d) + t['Key'] = 'MPC-AWS-BACKUP' + #change_record.append(instance.id) + + # output results + print('\n\tConverted tag set to be applied (aws* restricted tags omitted):\n') + tags = remove_duplicates(tags) + change_record = remove_duplicates(change_record) + tags.sort() + for t in tags: + s1 = 'Key: ' + str(t['Key']) + s2 = 'Value: ' + str(t['Value']) + # print in 2 columns + print '{0:30} {1}'.format(s1, s2) + print('\n') + + print('\n\tTags to be deleted:\n') + delete_tags.sort() + for t in delete_tags: + s1 = 'Key: ' + str(t['Key']) + s2 = 'Value: ' + str(t['Value']) + # print in 2 columns + print '{0:30} {1}'.format(s1, s2) + print('\n') + + if DBUGMODE == False: + # apply new converted tags + response_create = instance.create_tags(Tags=tags) + print('\nresponse_create is:') + print(str(response_create) + '\n') + if delete_tags: + # WARNING: calling instance.delete_tags() with empty tag list + # results in deletion of ALL pre-existing tags, must ensure + # contains at least 1 tag before calling delete method + response_delete = instance.delete_tags(Tags=delete_tags) + print('\nresponse_delete is:') + print(str(response_delete) + '\n') + + if DBUGMODE == True: + print('\n-------------------------------------') + print(' Converted tags not applied. DBUGMODE') + print('-------------------------------------\n') + # footer + print('\n---------------------------------------------------------------------------------') + print('Summary:\nUpdated config for ' + str(len(change_record)) + + ' resources in account ' + str(PROFILE)) + print('---------------------------------------------------------------------------------\n') +# <-- end --> +exit (0) diff --git a/build/lib/pyaws/tags/tag_utils.py b/build/lib/pyaws/tags/tag_utils.py new file mode 100644 index 0000000..e1587f4 --- /dev/null +++ b/build/lib/pyaws/tags/tag_utils.py @@ -0,0 +1,308 @@ +""" +pyaws.tags: Tag Utilities +""" +import json +import inspect +from functools import reduce +from botocore.exceptions import ClientError +from pyaws.session import boto3_session +from pyaws import logger + + +def create_taglist(dict): + """ + Summary: + Transforms tag dictionary back into a properly formatted tag list + Returns: + tags, TYPE: list + """ + tags = [] + for k, v in dict.items(): + temp = {} + temp['Key'] = k + temp['Value'] = v + tags.append(temp) + return tags + + +def create_tagdict(tags): + """ + Summary. + Converts tag list to tag dict + Args: + :tags (list): k,v + pair + { + [ + 'Key': k1, + 'Value': v1 + ] + } + Returns: + :tags (dict): {k1: v1, k2: v2} + + """ + return {x['Key']: x['Value'] for x in tags} + + +def delete_tags(resourceIds, region, tags): + """ Removes tags from an EC2 resource """ + client = boto3_session('ec2', region) + try: + for resourceid in resourceIds: + response = client.delete_tags( + Resources=[resourceid], + Tags=tags + ) + if response['ResponseMetadata']['HTTPStatusCode'] == 200: + logger.info('Existing Tags deleted from vol id %s' % resourceid) + return True + else: + logger.warning('Problem deleting existing tags from vol id %s' % resourceid) + return False + except ClientError as e: + logger.critical( + "%s: Problem apply tags to ec2 instances (Code: %s Message: %s)" % + (inspect.stack()[0][3], e.response['Error']['Code'], e.response['Error']['Message'])) + return False + + +def divide_tags(tag_list, *args): + """ + Summary: + Identifys a specific tag in tag_list by Key. When found, + creates a new tag list containing tags with keys provided in *args. + tag_list is returned without matching tags + Args: + tag_list (list): tag list starting reference + matching (list): tags which have keys matching any of *args + residual (list): tag_list after any matching tags are removed + RETURNS + TYPE: matching tag list, residual tag list + """ + matching = [] + residual = {x['Key']: x['Value'] for x in tag_list.copy()} + tag_dict = {x['Key']: x['Value'] for x in tag_list} + for key in args: + for k,v in tag_dict.items(): + try: + if key == k: + matching.append({'Key': k, 'Value': v}) + else: + residual.pop(key) + except KeyError: + continue + return matching, [{'Key': k, 'Value': v} for k,v in residual.items()] + + +def exclude_tags(tag_list, *args): + """ + - Filters a tag set by Exclusion + - variable tag keys given as parameters, tag keys corresponding to args + are excluded + + RETURNS + TYPE: list + """ + clean = tag_list.copy() + + for tag in tag_list: + for arg in args: + if arg == tag['Key']: + clean.remove(tag) + return clean + + +def extract_tag(tag_list, key): + """ + Summary: + Search tag list for prescence of tag matching key parameter + Returns: + tag, TYPE: list + """ + if {x['Key']: x['Value'] for x in tag_list}.get(key): + return list(filter(lambda x: x['Key'] == key, tag_list))[0] + return [] + + +def include_tags(tag_list, *args): + """ + - Filters a tag set by Inclusion + - variable tag keys given as parameters, tag keys corresponding to args + are excluded + + RETURNS + TYPE: list + """ + targets = [] + + for tag in tag_list: + for arg in args: + if arg == tag['Key']: + targets.append(tag) + return targets + + +def filter_tags(tag_list, *args): + """DEPRECATED + - Filters a tag set by exclusion + - variable tag keys given as parameters, tag keys corresponding to args + are excluded + + RETURNS + TYPE: list + """ + clean = tag_list.copy() + + for tag in tag_list: + for arg in args: + if arg == tag['Key']: + clean.remove(tag) + return clean + + +def json_tags(resource_list, tag_list, mode=''): + """ + - Prints tag keys, values applied to resources + - output: cloudwatch logs + - mode: INFO, DBUG, or UNKN (unknown or not provided) + """ + if mode == 0: + mode_text = 'DBUG' + else: + mode_text = 'INFO' + + try: + for resource in resource_list: + if mode == 0: + logger.debug('DBUGMODE enabled - Print tags found on resource %s:' % str(resource)) + else: + logger.info('Tags found resource %s:' % str(resource)) + print(json.dumps(tag_list, indent=4, sort_keys=True)) + except Exception as e: + logger.critical( + "%s: problem printing tag keys or values to cw logs: %s" % + (inspect.stack()[0][3], str(e))) + return False + return True + + +def pretty_print_tags(tag_list): + """ prints json tags with syntax highlighting """ + json_str = json.dumps(tag_list, indent=4, sort_keys=True) + print(highlight( + json_str, lexers.JsonLexer(), formatters.TerminalFormatter() + )) + print('\n') + return True + + +def print_tags(resource_list, tag_list, mode=''): + """ + - Prints tag keys, values applied to resources + - output: cloudwatch logs + - mode: INFO, DBUG, or UNKN (unknown or not provided) + """ + if mode == 0: + mode_text = 'DBUG' + else: + mode_text = 'INFO' + + try: + for resource in resource_list: + logger.info('Tags successfully applied to resource: ' + str(resource)) + ct = 0 + for t in tag_list: + logger.info('tag' + str(ct) + ': ' + str(t['Key']) + ' : ' + str(t['Value'])) + ct += 1 + if mode == 0: + logger.debug('DBUGMODE = True, No tags applied') + + except Exception as e: + logger.critical( + "%s: problem printing tag keys or values to cw logs: %s" % + (inspect.stack()[0][3], str(e))) + return 1 + return 0 + + +def remove_duplicates(alist): + """ + Removes duplicate dict in a list of dict by enforcing unique keys + """ + + clean_list, key_list = [], [] + + try: + for dict in alist: + if dict['Key'] not in key_list: + clean_list.append(dict) + key_list.append(dict['Key']) + except KeyError: + # dedup list of items, not dict + return list(reduce(lambda r, x: r + [x] if x not in r else r, alist, [])) + except Exception as e: + raise e + return clean_list + + +def remove_restricted(list): + """ + Remove restricted (system) Amazon tags from list of tags + """ + + clean_list = [] + + try: + for dict in list: + if 'aws' not in dict['Key']: + clean_list.append(dict) + except Exception: + return -1 + return clean_list + + +def remove_bykeys(tag_list, *keys): + """ + Summary: + Removes tags from a tag list for specified keys + Returns: + tags (list) + """ + tag_dict = {x['Key']: x['Value'] for x in tag_list} + for key in keys: + if key in tag_dict: + tag_dict.pop(key) + return [{'Key': k, 'Value': v} for k,v in tag_dict.items()] + + +def select_tags(tag_list, key_list): + """ + Return selected tags from a list of many tags given a tag key + """ + select_list = [] + # select tags by keys in key_list + for tag in tag_list: + for key in key_list: + if key == tag['Key']: + select_list.append(tag) + # ensure only tag-appropriate k,v pairs in list + clean = [{'Key': x['Key'], 'Value': x['Value']} for x in select_list] + return clean + + +def valid_tags(tag_list, invalid): + """ + Summary: + checks tags for invalid chars + Args: + tag_list (list): starting list of tags + invalid (list): list of illegal characters that should not be present in keys in tag_list + Returns: + Success | Failure, TYPE: bool + """ + for tag in tag_list: + for char in invalid: + if char in tag['Key']: + return False + return True diff --git a/build/lib/pyaws/utils/__init__.py b/build/lib/pyaws/utils/__init__.py new file mode 100644 index 0000000..61b8558 --- /dev/null +++ b/build/lib/pyaws/utils/__init__.py @@ -0,0 +1,8 @@ +""" +Utilities Module - Scripts +""" + +from libtools import bool_convert, bool_assignment, ascii_lowercase +from libtools import range_test, range_bind, userchoice_mapping +from libtools import stdout_message +from libtools.io import export_json_object diff --git a/dist/pyaws-0.4.1-py2.7.egg b/dist/pyaws-0.4.1-py2.7.egg new file mode 100644 index 0000000..d220a21 Binary files /dev/null and b/dist/pyaws-0.4.1-py2.7.egg differ diff --git a/dist/pyaws-0.4.1-py3.10.egg b/dist/pyaws-0.4.1-py3.10.egg new file mode 100644 index 0000000..470bcb6 Binary files /dev/null and b/dist/pyaws-0.4.1-py3.10.egg differ diff --git a/future/lambda/get_price.py b/future/lambda/get_price.py index fdefcaf..552829a 100644 --- a/future/lambda/get_price.py +++ b/future/lambda/get_price.py @@ -1,236 +1,235 @@ -""" -Retrieve Amazon Web Services Pricing -""" -import argparse -import os -import sys -import json -import logging -import inspect -import requests -from functools import lru_cache -from itertools import chain -from pygments import highlight, lexers, formatters -import boto3 - - -# globals -PROFILE = 'default' -__version__ = '1.0 ' -logger = logging.getLogger(__version__) -logger.setLevel(logging.INFO) - -# set region default -default_region = os.getenv('AWS_DEFAULT_REGION', 'eu-west-1') -default_region = 'eu-west-1' - -RETURN_DATA = ['compute', 'transfer', 'request', 'requests', 'edge'] -INDEXURL = "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/index.json" -url_prefix = "https://pricing.us-east-1.amazonaws.com" - - - -def export_json_object(dict_obj, filename=None): - """ - Summary: - exports object to block filesystem object - - Args: - :dict_obj (dict): dictionary object - :filename (str): name of file to be exported (optional) - - Returns: - True | False Boolean export status - - """ - try: - if filename: - try: - with open(filename, 'w') as handle: - handle.write(json.dumps(dict_obj, indent=4, sort_keys=True)) - logger.info( - '%s: Wrote %s to local filesystem location' % - (inspect.stack()[0][3], filename)) - handle.close() - except TypeError as e: - logger.warning( - '%s: object in dict not serializable: %s' % - (inspect.stack()[0][3], str(e))) - else: - json_str = json.dumps(dict_obj, indent=4, sort_keys=True) - print(highlight(json_str, lexers.JsonLexer(), formatters.TerminalFormatter())) - logger.info('%s: successful export to stdout' % inspect.stack()[0][3]) - return True - except OSError as e: - logger.critical( - '%s: export_file_object: error writing to %s to filesystem. Error: %s' % - (inspect.stack()[0][3], filename, str(e))) - return False - else: - logger.info('export_file_object: successful export to %s' % filename) - return True - - -def get_regions(name): - session = boto3.Session(profile_name=name) - ec2 = session.client('ec2', region_name=default_region) - return [x['RegionName'] for x in ec2.describe_regions()['Regions'] if 'cn' not in x['RegionName']] - - -def global_index(service, url=INDEXURL): - """ - Retrieves master index file containing current price file urls for all AWS Services - """ - r = requests.get(INDEXURL) - f1 = json.loads(r.content) - return url_prefix + f1['offers'][service]['currentRegionIndexUrl'] - - -def region_index(region): - """ - Returns url of price file for specific region - """ - r2 = requests.get(global_index(service='AWSLambda')) - return url_prefix + json.loads(r2.content)['regions'][region]['currentVersionUrl'] - - -@lru_cache() -def price_data(region, sku=None): - """ - Summary: - all price data for an AWS service - Return: - data (json) - """ - r = requests.get(region_index(region)).json() - - products = [x for x in r['products'].values() if x['attributes']['servicecode'] == 'AWSLambda'] - skus = {x['sku'] for x in products} - - terms = list(chain( - *[ - y.values() for y in [x for x in r['terms'].values()] - ] - )) - - parsed = [] - - for term in terms: - if list(term.values())[0]['sku'] not in skus: - continue - parsed.append(term) - return products, skus, parsed - - -def help_menu(): - """ Displays command line parameter options """ - menu = ''' - help menu - --------- - -DESCRIPTION - - Code returns AWS Price data metrics for AWS Lambda - -OPTIONS - - $ python3 get_price.py [OPTIONS] - - [-e, --element ] - [-f, --filename ] - [-r, --region ] - [-d, --debug ] - [-h, --help ] - - -e, --element (string): Data Return Type. Data element - returned when one of the following specified: - - - compute : AWS Lambda Compute Price ($/GB-s) - - transfer : AWS Lambda Bandwidth Price ($/GB) - - request : Price per request ($/req) - - edge : Compute Price Lambda Edge ($/GB-s) - - If no --element specified, the entire pricing json object - for the region returned - - -f, --filename (string): Name of output file. Valid when - a data element is NOT specified and you want the entire - pricing json object returned and persisted to the - filesystem. No effect when --element given. - - -r, --region (string): Region for which you want to return - pricing. If no region parameter specified, defaults to - eu-west-1 - - -d, --debug: Debug mode, verbose output. - - -h, --help: Print this menu - ''' - print(menu) - return True - - -def main(region, dataType=None, output_path=None): - products, skus, response = price_data(region=region) - if dataType and dataType == 'compute': - for k,v in response[4]['SGGKTWDV7PGMPPSJ.JRTCKXETXF']['priceDimensions'].items(): - if isinstance(v, dict): - for key, value in v.items(): - if key == 'pricePerUnit': - return value['USD'] - elif dataType and dataType == 'transfer': - for k,v in response[1]['B5V2RNHAWGVJBZD3.JRTCKXETXF']['priceDimensions'].items(): - if isinstance(v, dict): - for key, value in v.items(): - if key == 'pricePerUnit': - return value['USD'] - elif dataType and dataType in ('request', 'requests'): - for k,v in response[0]['DDKXA6JP8NCVUFRZ.JRTCKXETXF']['priceDimensions'].items(): - if isinstance(v, dict): - for key, value in v.items(): - if key == 'pricePerUnit': - return value['USD'] - elif dataType and dataType == 'edge': - for k,v in response[-1]['WUTD23YJE55E5JCC.JRTCKXETXF']['priceDimensions'].items(): - if isinstance(v, dict): - for key, value in v.items(): - if key == 'pricePerUnit': - return value['USD'] - return export_json_object(dict_obj=response, filename=output_path) - - -def options(parser, help_menu=False): - """ - Summary: - parse cli parameter options - Returns: - TYPE: argparse object, parser argument set - """ - parser.add_argument("-e", "--element", nargs='?', default='list', type=str, - choices=RETURN_DATA, required=False) - parser.add_argument("-f", "--filename", nargs='?', default=None, - required=False, help="type (default: %(default)s)") - parser.add_argument("-r", "--region", nargs='?', default=default_region, type=str, - choices=get_regions(PROFILE), required=False) - parser.add_argument("-d", "--debug", dest='debug', action='store_true', required=False) - parser.add_argument("-h", "--help", dest='help', action='store_true', required=False) - return parser.parse_args() - - -def init_cli(): - # parser = argparse.ArgumentParser(add_help=False, usage=help_menu()) - parser = argparse.ArgumentParser(add_help=False) - - try: - args = options(parser) - except Exception as e: - logger.exception('Problem parsing provided parameters: %s' % str(e)) - if args.help: - return help_menu() - if args.debug: - print('\n\nParameters:\n\targs.region:\t%s\n\targs.element:\t%s\n' % (args.region, args.element)) - return main(region=args.region, dataType=args.element, output_path=args.filename) - - -if __name__ == '__main__': - sys.exit(init_cli()) +""" +Retrieve Amazon Web Services Pricing +""" +import argparse +import os +import sys +import json +import logging +import inspect +import requests +from functools import lru_cache +from itertools import chain +from pygments import highlight, lexers, formatters +import boto3 + + +# globals +PROFILE = 'default' +__version__ = '1.0 ' +logger = logging.getLogger(__version__) +logger.setLevel(logging.INFO) + +# set region default +default_region = os.getenv('AWS_DEFAULT_REGION', 'eu-west-1') +default_region = 'eu-west-1' + +RETURN_DATA = ['compute', 'transfer', 'request', 'requests', 'edge'] +INDEXURL = "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/index.json" +url_prefix = "https://pricing.us-east-1.amazonaws.com" + + +def export_json_object(dict_obj, filename=None): + """ + Summary: + exports object to block filesystem object + + Args: + :dict_obj (dict): dictionary object + :filename (str): name of file to be exported (optional) + + Returns: + True | False Boolean export status + + """ + try: + if filename: + try: + with open(filename, 'w') as handle: + handle.write(json.dumps(dict_obj, indent=4, sort_keys=True)) + logger.info( + '%s: Wrote %s to local filesystem location' % + (inspect.stack()[0][3], filename)) + handle.close() + except TypeError as e: + logger.warning( + '%s: object in dict not serializable: %s' % + (inspect.stack()[0][3], str(e))) + else: + json_str = json.dumps(dict_obj, indent=4, sort_keys=True) + print(highlight(json_str, lexers.JsonLexer(), formatters.TerminalFormatter())) + logger.info('%s: successful export to stdout' % inspect.stack()[0][3]) + return True + except OSError as e: + logger.critical( + '%s: export_file_object: error writing to %s to filesystem. Error: %s' % + (inspect.stack()[0][3], filename, str(e))) + return False + else: + logger.info('export_file_object: successful export to %s' % filename) + return True + + +def get_regions(name): + session = boto3.Session(profile_name=name) + ec2 = session.client('ec2', region_name=default_region) + return [x['RegionName'] for x in ec2.describe_regions()['Regions'] if 'cn' not in x['RegionName']] + + +def global_index(service, url=INDEXURL): + """ + Retrieves master index file containing current price file urls for all AWS Services + """ + r = requests.get(INDEXURL) + f1 = json.loads(r.content) + return url_prefix + f1['offers'][service]['currentRegionIndexUrl'] + + +def region_index(region): + """ + Returns url of price file for specific region + """ + r2 = requests.get(global_index(service='AWSLambda')) + return url_prefix + json.loads(r2.content)['regions'][region]['currentVersionUrl'] + + +@lru_cache() +def price_data(region, sku=None): + """ + Summary: + all price data for an AWS service + Return: + data (json) + """ + r = requests.get(region_index(region)).json() + + products = [x for x in r['products'].values() if x['attributes']['servicecode'] == 'AWSLambda'] + skus = {x['sku'] for x in products} + + terms = list(chain( + *[ + y.values() for y in [x for x in r['terms'].values()] + ] + )) + + parsed = [] + + for term in terms: + if list(term.values())[0]['sku'] not in skus: + continue + parsed.append(term) + return products, skus, parsed + + +def help_menu(): + """ Displays command line parameter options """ + menu = ''' + help menu + --------- + +DESCRIPTION + + Code returns AWS Price data metrics for AWS Lambda + +OPTIONS + + $ python3 get_price.py [OPTIONS] + + [-e, --element ] + [-f, --filename ] + [-r, --region ] + [-d, --debug ] + [-h, --help ] + + -e, --element (string): Data Return Type. Data element + returned when one of the following specified: + + - compute : AWS Lambda Compute Price ($/GB-s) + - transfer : AWS Lambda Bandwidth Price ($/GB) + - request : Price per request ($/req) + - edge : Compute Price Lambda Edge ($/GB-s) + + If no --element specified, the entire pricing json object + for the region returned + + -f, --filename (string): Name of output file. Valid when + a data element is NOT specified and you want the entire + pricing json object returned and persisted to the + filesystem. No effect when --element given. + + -r, --region (string): Region for which you want to return + pricing. If no region parameter specified, defaults to + eu-west-1 + + -d, --debug: Debug mode, verbose output. + + -h, --help: Print this menu + ''' + print(menu) + return True + + +def main(region, dataType=None, output_path=None): + products, skus, response = price_data(region=region) + if dataType and dataType == 'compute': + for k,v in response[4]['SGGKTWDV7PGMPPSJ.JRTCKXETXF']['priceDimensions'].items(): + if isinstance(v, dict): + for key, value in v.items(): + if key == 'pricePerUnit': + return value['USD'] + elif dataType and dataType == 'transfer': + for k,v in response[1]['B5V2RNHAWGVJBZD3.JRTCKXETXF']['priceDimensions'].items(): + if isinstance(v, dict): + for key, value in v.items(): + if key == 'pricePerUnit': + return value['USD'] + elif dataType and dataType in ('request', 'requests'): + for k,v in response[0]['DDKXA6JP8NCVUFRZ.JRTCKXETXF']['priceDimensions'].items(): + if isinstance(v, dict): + for key, value in v.items(): + if key == 'pricePerUnit': + return value['USD'] + elif dataType and dataType == 'edge': + for k,v in response[-1]['WUTD23YJE55E5JCC.JRTCKXETXF']['priceDimensions'].items(): + if isinstance(v, dict): + for key, value in v.items(): + if key == 'pricePerUnit': + return value['USD'] + return export_json_object(dict_obj=response, filename=output_path) + + +def options(parser, help_menu=False): + """ + Summary: + parse cli parameter options + Returns: + TYPE: argparse object, parser argument set + """ + parser.add_argument("-e", "--element", nargs='?', default='list', type=str, + choices=RETURN_DATA, required=False) + parser.add_argument("-f", "--filename", nargs='?', default=None, + required=False, help="type (default: %(default)s)") + parser.add_argument("-r", "--region", nargs='?', default=default_region, type=str, + choices=get_regions(PROFILE), required=False) + parser.add_argument("-d", "--debug", dest='debug', action='store_true', required=False) + parser.add_argument("-h", "--help", dest='help', action='store_true', required=False) + return parser.parse_args() + + +def init_cli(): + # parser = argparse.ArgumentParser(add_help=False, usage=help_menu()) + parser = argparse.ArgumentParser(add_help=False) + + try: + args = options(parser) + except Exception as e: + logger.exception('Problem parsing provided parameters: %s' % str(e)) + if args.help: + return help_menu() + if args.debug: + print('\n\nParameters:\n\targs.region:\t%s\n\targs.element:\t%s\n' % (args.region, args.element)) + return main(region=args.region, dataType=args.element, output_path=args.filename) + + +if __name__ == '__main__': + sys.exit(init_cli()) diff --git a/pyaws.egg-info/PKG-INFO b/pyaws.egg-info/PKG-INFO new file mode 100644 index 0000000..e4b38b4 --- /dev/null +++ b/pyaws.egg-info/PKG-INFO @@ -0,0 +1,34 @@ +Metadata-Version: 1.2 +Name: pyaws +Version: 0.4.1 +Summary: Python Utilities for Amazon Web Services +Home-page: http://pyaws.readthedocs.io +Author: Blake Huber +Author-email: blakeca00@gmail.com +License: GPL-3.0 +Description: + **pyaws** | Utilities Library for Amazon Web Services (AWS) + ----------------------------------------------------------- + + PACKAGE: pyaws + + ``pyaws``: reusable library of utility classes and functions common AWS use cases and capabilities: + + * uploading to s3 + * adding/ deleting resource tags + * adding data elements to dynamodb table + * Determining the latest Amazon Machine Image in a region for Windows, Linux, etc + + + +Keywords: Amazon Web Services AWS iam ec2 lambda rds s3 sts +Platform: UNKNOWN +Classifier: Topic :: System :: Systems Administration +Classifier: Topic :: Utilities +Classifier: Development Status :: 4 - Beta +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3) +Classifier: Operating System :: POSIX :: Linux +Requires-Python: >=3.6, <4 diff --git a/pyaws.egg-info/SOURCES.txt b/pyaws.egg-info/SOURCES.txt new file mode 100644 index 0000000..9702ba0 --- /dev/null +++ b/pyaws.egg-info/SOURCES.txt @@ -0,0 +1,42 @@ +DESCRIPTION.rst +LICENSE +MANIFEST.in +README.md +setup.cfg +setup.py +pyaws/__init__.py +pyaws/_version.py +pyaws/colors.py +pyaws/environment.py +pyaws/helpers.py +pyaws/logd.py +pyaws/script_utils.py +pyaws/session.py +pyaws/statics.py +pyaws/awslambda/__init__.py +pyaws/awslambda/env.py +pyaws/awslambda/lambda_utils.py +pyaws/core/__init__.py +pyaws/core/create_client.py +pyaws/core/cross_account_utils.py +pyaws/core/loggers.py +pyaws/core/oscodes_unix.py +pyaws/core/oscodes_win.py +pyaws/dynamodb/__init__.py +pyaws/dynamodb/dynamodb.py +pyaws/dynamodb/table.py +pyaws/ec2/__init__.py +pyaws/ec2/ec2_utils.py +pyaws/ec2/snapshot_ops.py +pyaws/ec2/state.py +pyaws/s3/__init__.py +pyaws/s3/check_authenticated_s3.py +pyaws/s3/object_operations.py +pyaws/sts/__init__.py +pyaws/sts/cross_account_utils.py +pyaws/tags/__init__.py +pyaws/tags/bulk-modify-tags.py +pyaws/tags/copy-tags-all-instances.py +pyaws/tags/ec2-update-tags.py +pyaws/tags/tag_utils.py +pyaws/utils/__init__.py \ No newline at end of file diff --git a/pyaws.egg-info/dependency_links.txt b/pyaws.egg-info/dependency_links.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/pyaws.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/pyaws.egg-info/entry_points.txt b/pyaws.egg-info/entry_points.txt new file mode 100644 index 0000000..1182316 --- /dev/null +++ b/pyaws.egg-info/entry_points.txt @@ -0,0 +1,3 @@ +[console_scripts] +pyconfig = pyaws.cli:option_configure + diff --git a/pyaws.egg-info/not-zip-safe b/pyaws.egg-info/not-zip-safe new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/pyaws.egg-info/not-zip-safe @@ -0,0 +1 @@ + diff --git a/pyaws.egg-info/requires.txt b/pyaws.egg-info/requires.txt new file mode 100644 index 0000000..d02aa89 --- /dev/null +++ b/pyaws.egg-info/requires.txt @@ -0,0 +1,4 @@ +awscli>=1.16.100boto3>=1.9.100 +botocore +libtools>=0.2.5 +distro>=1.4.0 diff --git a/pyaws.egg-info/top_level.txt b/pyaws.egg-info/top_level.txt new file mode 100644 index 0000000..fd52136 --- /dev/null +++ b/pyaws.egg-info/top_level.txt @@ -0,0 +1 @@ +pyaws diff --git a/pyaws/__init__.pyc b/pyaws/__init__.pyc new file mode 100644 index 0000000..4aeaef0 Binary files /dev/null and b/pyaws/__init__.pyc differ diff --git a/pyaws/__pycache__/__init__.cpython-310.pyc b/pyaws/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000..bd0597f Binary files /dev/null and b/pyaws/__pycache__/__init__.cpython-310.pyc differ diff --git a/pyaws/__pycache__/_version.cpython-310.pyc b/pyaws/__pycache__/_version.cpython-310.pyc new file mode 100644 index 0000000..d9bf3ca Binary files /dev/null and b/pyaws/__pycache__/_version.cpython-310.pyc differ diff --git a/pyaws/__pycache__/environment.cpython-310.pyc b/pyaws/__pycache__/environment.cpython-310.pyc new file mode 100644 index 0000000..e86cb7f Binary files /dev/null and b/pyaws/__pycache__/environment.cpython-310.pyc differ diff --git a/pyaws/__pycache__/script_utils.cpython-310.pyc b/pyaws/__pycache__/script_utils.cpython-310.pyc new file mode 100644 index 0000000..7c5ee02 Binary files /dev/null and b/pyaws/__pycache__/script_utils.cpython-310.pyc differ diff --git a/pyaws/__pycache__/statics.cpython-310.pyc b/pyaws/__pycache__/statics.cpython-310.pyc new file mode 100644 index 0000000..164e89e Binary files /dev/null and b/pyaws/__pycache__/statics.cpython-310.pyc differ diff --git a/pyaws/_version.pyc b/pyaws/_version.pyc new file mode 100644 index 0000000..37644a4 Binary files /dev/null and b/pyaws/_version.pyc differ diff --git a/pyaws/ec2/ec2_utils.py b/pyaws/ec2/ec2_utils.py index 4de945b..673e855 100644 --- a/pyaws/ec2/ec2_utils.py +++ b/pyaws/ec2/ec2_utils.py @@ -50,7 +50,9 @@ def default_region(profile='default'): """ stderr = ' 2>/dev/null' - region = subprocess.getoutput(f'aws configure get profile.{profile}.region {stderr}') + region = subprocess.check_output( + 'aws configure get profile.{profile}.region {stderr}'.format( + profile=profile, stderr=stderr)) try: if region: @@ -59,7 +61,8 @@ def default_region(profile='default'): os.environ['AWS_DEFAULT_REGION'] = 'us-east-1' except Exception as e: logger.exception( - f'{inspect.stack()[0][3]}: Unknown error while interrogating local awscli config: {e}' + '{i}: Unknown error while interrogating local awscli config: {e}'.format( + i=inspect.stack()[0][3], e=e) ) raise return os.getenv('AWS_DEFAULT_REGION') diff --git a/pyaws/environment.py b/pyaws/environment.py index 067a087..3d7522f 100644 --- a/pyaws/environment.py +++ b/pyaws/environment.py @@ -7,7 +7,6 @@ import subprocess import inspect import logging -from shutil import which logger = logging.getLogger() @@ -27,14 +26,10 @@ def awscli_region(profile_name): """ awscli = 'aws' - if not which(awscli): - print('Unable to locate awscli') - return None - else: - cmd = awscli + ' configure get ' + profile_name + '.region' + cmd = awscli + ' configure get ' + profile_name + '.region' try: - region = subprocess.getoutput(cmd) + region = subprocess.check_output(cmd) except Exception: logger.exception( '%s: Failed to identify AccessKeyId used in %s profile.' % diff --git a/pyaws/environment.pyc b/pyaws/environment.pyc new file mode 100644 index 0000000..4bd4c19 Binary files /dev/null and b/pyaws/environment.pyc differ diff --git a/pyaws/logd.py b/pyaws/logd.py index 96c76ef..f409222 100644 --- a/pyaws/logd.py +++ b/pyaws/logd.py @@ -7,7 +7,7 @@ import inspect import logging import logging.handlers -from pathlib import Path +from pathlib2 import Path from pyaws.statics import local_config @@ -51,7 +51,9 @@ def logging_prep(mode): Path(log_path).touch(mode=0o644, exist_ok=True) except OSError as e: - syslog.exception(f'{inspect.stack()[0][3]}: Failure while seeding log file path: {e}') + syslog.exception( + '{i}: Failure while seeding log file path: {e}'.format( + i=inspect.stack()[0][3], e=e)) return False return True @@ -110,7 +112,8 @@ def getLogger(*args, **kwargs): logger.setLevel(logging.DEBUG) else: - syslog.warning(f'{inspect.stack()[0][3]}: Log preparation fail - exit') + syslog.warning('{i}: Log preparation fail - exit'.format( + i=inspect.stack()[0][3])) sys.exit(1) elif mode_assignment(log_mode) == 'STREAM': diff --git a/pyaws/script_utils.pyc b/pyaws/script_utils.pyc new file mode 100644 index 0000000..0b42e0c Binary files /dev/null and b/pyaws/script_utils.pyc differ diff --git a/pyaws/session.py b/pyaws/session.py index 38410d0..16055e7 100644 --- a/pyaws/session.py +++ b/pyaws/session.py @@ -5,7 +5,6 @@ from botocore.exceptions import ClientError, ProfileNotFound from pyaws.utils import stdout_message from pyaws import logger -from pyaws._version import __version__ try: from pyaws.core.oscodes_unix import exit_codes @@ -37,14 +36,18 @@ def _profile_prefix(profile, prefix='gcreds'): tempProfile = prefix + '-' + profile try: - if subprocess.getoutput(f'aws configure get profile.{profile}.aws_access_key_id {stderr}'): + if subprocess.check_output( + 'aws configure get profile.{profile}.aws_access_key_id {stderr}'.format( + profile=profile, stderr=stderr)): return profile - elif subprocess.getoutput(f'aws configure get profile.{tempProfile}.aws_access_key_id {stderr}'): + elif subprocess.check_output( + 'aws configure get profile.{tempProfile}.aws_access_key_id {stderr}'.format( + tempProfile=tempProfile, stderr=stderr)): return tempProfile except Exception as e: logger.exception( - f'{inspect.stack()[0][3]}: Unknown error while interrogating local awscli config: {e}' - ) + '{i}: Unknown error while interrogating local awscli config: {e}'.format( + i=inspect.stack()[0][3], e=e)) raise return None @@ -80,8 +83,8 @@ def parse_profiles(profiles): return _profile_prefix(profiles.strip()) except Exception as e: logger.exception( - f'{inspect.stack()[0][3]}: Unknown error while converting profile_names from local awscli config: {e}' - ) + '{i}: Unknown error while converting profile_names from local awscli config: {e}'.format( + i=inspect.stack()[0][3], e=e)) raise return profile_list diff --git a/pyaws/statics.pyc b/pyaws/statics.pyc new file mode 100644 index 0000000..ab621a1 Binary files /dev/null and b/pyaws/statics.pyc differ diff --git a/pyaws/tags/bulk-modify-tags.py b/pyaws/tags/bulk-modify-tags.py index b041888..987e359 100644 --- a/pyaws/tags/bulk-modify-tags.py +++ b/pyaws/tags/bulk-modify-tags.py @@ -224,7 +224,8 @@ def main(): # AFTER tag copy | put Name tag back into apply tags, ie, after_tags retain_tags = select_tags(instance.tags, PRESERVE_TAGS) - for tag in (*retain_tags, *filtered_tags): + all_tags = retain_tags + filtered_tags + for tag in all_tags: after_tags.append(tag) logger.info('For InstanceID %s, the AFTER FILTERING list of %d tags is:' % (instance.id, len(after_tags))) logger.info('Tags to apply are:') diff --git a/pyaws/tags/copy-tags-all-instances.py b/pyaws/tags/copy-tags-all-instances.py index 9ff74f0..5ef3e6a 100644 --- a/pyaws/tags/copy-tags-all-instances.py +++ b/pyaws/tags/copy-tags-all-instances.py @@ -185,7 +185,12 @@ def main(): # AFTER tag copy | put Name tag back into apply tags, ie, after_tags retain_tags = select_tags(instance.tags, PRESERVE_TAGS) +<<<<<<< HEAD for tag in (*retain_tags, *filtered_tags): +======= + all_tags = retain_tags + filtered_tags + for tag in all_tags: +>>>>>>> 34de914 (Refactor for python2.7) after_tags.append(tag) logger.info('For InstanceID %s, the AFTER FILTERING list of %d tags is:' % (instance.id, len(after_tags))) logger.info('Tags to apply are:') diff --git a/pyaws/tags/tag_utils.py b/pyaws/tags/tag_utils.py index 150cffa..e1587f4 100644 --- a/pyaws/tags/tag_utils.py +++ b/pyaws/tags/tag_utils.py @@ -8,11 +8,6 @@ from pyaws.session import boto3_session from pyaws import logger -try: - from pyaws oscodes_unix import exit_codes -except Exception: - from pyaws oscodes_win import exit_codes # non-specific os-safe codes - def create_taglist(dict): """ diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..8bfd5a1 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,4 @@ +[egg_info] +tag_build = +tag_date = 0 + diff --git a/setup.py b/setup.py index 8042e23..968686c 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,8 @@ 'boto3>=1.9.100', 'botocore', 'libtools>=0.2.5', - 'distro>=1.4.0' + 'distro>=1.4.0', + 'pathlib2' ] @@ -79,13 +80,14 @@ def run(self): 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 2.7', 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)', 'Operating System :: POSIX :: Linux' ], keywords='Amazon Web Services AWS iam ec2 lambda rds s3 sts', packages=find_packages(exclude=['docs', 'scripts', 'assets']), install_requires=requires, - python_requires='>=3.6, <4', + python_requires='>=2.7, <4', entry_points={ 'console_scripts': [ 'pyconfig=pyaws.cli:option_configure'