initial commit

Author: celdering-trm

.github/workflows/test.yaml

@@ -0,0 +1,27 @@
+# A github actions workflow for testing the code
+name: Test
+
+on:
+  push:
+
+env:
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  AWS_DEFAULT_REGION: us-east-1
+  DB_HOST: ${{ secrets.DB_HOST }}
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v3
+        with:
+          python-version: 3.11
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+      
+      - name: Run code
+        run: python main.py

README.md

@@ -1 +1,26 @@
-# python-coding-interview-ext
+# Python Coding Interview
+
+## Instructions
+
+> We have a business need to build a scheduled task which will flag anomolous in a database given to us by a third party.
+> 
+> 
+> The main requirements given to our team are:
+> 
+> - We need to connect to a database and retrieve data
+> - We need to identify duplicates in the database and flag them as such
+> - We need to identify outlier datapoints in the database and flag them as such
+> - We do not control the data source, and we have read-only access to the database.
+> 
+> You should create a new branch and iterate on the code in this repository to meet the requirements above. Please leave comments in the code to explain your thought process and any changes you make.
+> Push commits to your branch in order to trigger the github actions workflow. You can view the results of the workflow in the [actions tab of the repository](https://github.com/trmlabs/python-coding-interview-ext/actions).
+> 
+
+### Advanced Instructions
+
+> If you have time, please also consider the following:
+>
+> - How would you test this code?
+> - How can the database connection handling be improved?
+> - How would you handle a large amount of data?
+>

main.py

@@ -0,0 +1,80 @@
+import os
+import json
+import boto3
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy.sql import text
+
+from botocore.exceptions import ClientError
+
+
+class Config:
+    def __init__(self):
+        self.secrets_manager = boto3.client('secretsmanager')
+
+    def get_secret_from_aws(self, secret_name):
+        try:
+            response = self.secrets_manager.get_secret_value(SecretId=secret_name)
+            if 'SecretString' in response:
+                return json.loads(response['SecretString'])
+            return None
+        except (ClientError,):
+            return None
+
+    def get_config_value(self, key, default_value, secret_name=None):
+        if secret_name:
+            aws_secret = self.get_secret_from_aws(secret_name)
+            if aws_secret and key in aws_secret:
+                return aws_secret
+            if key in os.environ:
+                return os.environ[key]
+            return default_value
+
+    def get_db_config(self):
+        return {
+            "name": self.get_config_value("DB_NAME", "postgres", "python_interview_dbname"),
+            "user": self.get_config_value("DB_USER", "postgres", "python_interview_user"),
+            "password": self.get_config_value("DB_PASSWORD", "password", "python_interview_password"),
+            "host": self.get_config_value("DB_HOST", "localhost", "python_interview_host"),
+            "port": self.get_config_value("DB_PORT", "5432", "python_interview_port")
+        }
+
+    def get_conn_string(self):
+        db_config = self.get_db_config()
+        return "postgresql://{user}:{password}@{host}:{port}/{name}".format(
+            name=db_config['name'],
+            user=db_config['user'],
+            password=db_config['password'],
+            host=db_config['host'],
+            port=db_config['port']
+        )
+
+
+class DatabaseConnection:
+    def __init__(self, conn_string):
+        self.engine = create_engine(conn_string)
+        self.session = sessionmaker(bind=self.engine)
+
+    def get_session(self):
+        new_session = self.session()
+        return new_session
+
+
+def dedupe_data(session, table_name):
+    records = session.execute(text(f"SELECT * FROM {table_name}")).fetchall()
+
+    duplicates = []
+    outliers = []
+    for record in records:
+        print(record)
+        # Insert your anomaly detection logic here
+
+
+if __name__ == "__main__":
+    config = Config()
+    conn_string = config.get_conn_string()
+    db = DatabaseConnection(conn_string)
+    session = db.get_session()
+
+    dedupe_data(session, "crypto_transactions")

requirements.in

@@ -0,0 +1,3 @@
+boto3
+sqlalchemy
+psycopg2-binary

requirements.txt

@@ -0,0 +1,30 @@
+#
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
+#
+#    pip-compile requirements.in
+#
+boto3==1.28.43
+    # via -r requirements.in
+botocore==1.31.43
+    # via
+    #   boto3
+    #   s3transfer
+jmespath==1.0.1
+    # via
+    #   boto3
+    #   botocore
+psycopg2-binary==2.9.7
+    # via -r requirements.in
+python-dateutil==2.8.2
+    # via botocore
+s3transfer==0.6.2
+    # via boto3
+six==1.16.0
+    # via python-dateutil
+sqlalchemy==2.0.20
+    # via -r requirements.in
+typing-extensions==4.7.1
+    # via sqlalchemy
+urllib3==1.26.16
+    # via botocore