From a63c6fb8a610a2f2b717bd957eaaf92bd5d4ca84 Mon Sep 17 00:00:00 2001 From: guzman-raphael Date: Mon, 18 May 2020 13:09:23 -0500 Subject: [PATCH 1/5] Add test updates, version check, and bump version. --- LNX-docker-compose.yml | 12 ++++++++---- datajoint_connection_hub/version.py | 2 +- local-docker-compose.yml | 19 +++++++++++++------ setup.py | 2 +- 4 files changed, 23 insertions(+), 12 deletions(-) diff --git a/LNX-docker-compose.yml b/LNX-docker-compose.yml index 8700cb3..24d18f0 100644 --- a/LNX-docker-compose.yml +++ b/LNX-docker-compose.yml @@ -44,13 +44,17 @@ services: condition: service_healthy fakeservices.datajoint.io: <<: *net - image: raphaelguzman/nginx:v0.0.2 + image: raphaelguzman/nginx:v0.0.6 environment: - ADD_db_TYPE=DATABASE - ADD_db_ENDPOINT=db:3306 - ADD_minio_TYPE=MINIO - ADD_minio_ENDPOINT=minio:9000 - - ADD_minio_PREFIX=/ + - ADD_minio_PORT=80 # allow unencrypted connections + - ADD_minio_PREFIX=/datajoint + - ADD_browser_TYPE=MINIOADMIN + - ADD_browser_ENDPOINT=minio:9000 + - ADD_browser_PORT=80 # allow unencrypted connections - ADD_hub_TYPE=REST - ADD_hub_ENDPOINT=hub:5000 - ADD_hub_PREFIX=/api @@ -80,10 +84,10 @@ services: - DJ_TEST_USER=datajoint - DJ_TEST_PASSWORD=datajoint # If running tests locally, make sure to add entry in /etc/hosts for 127.0.0.1 fakeservices.datajoint.io - - S3_ENDPOINT=fakeservices.datajoint.io:9000 + - S3_ENDPOINT=fakeservices.datajoint.io - S3_ACCESS_KEY=datajoint - S3_SECRET_KEY=datajoint - - S3_BUCKET=datajoint-test + - S3_BUCKET=datajoint.test - PYTHON_USER=dja - JUPYTER_PASSWORD=datajoint - DISPLAY diff --git a/datajoint_connection_hub/version.py b/datajoint_connection_hub/version.py index 81f0fde..b1a19e3 100644 --- a/datajoint_connection_hub/version.py +++ b/datajoint_connection_hub/version.py @@ -1 +1 @@ -__version__ = "0.0.4" +__version__ = "0.0.5" diff --git a/local-docker-compose.yml b/local-docker-compose.yml index 3ddfb68..bff9bce 100644 --- a/local-docker-compose.yml +++ b/local-docker-compose.yml @@ -10,10 +10,12 @@ services: - MYSQL_ROOT_PASSWORD=${DJ_PASS} # ports: # - "3306:3306" - # To persist MySQL data + # # To persist MySQL data # volumes: # - ./mysql/data:/var/lib/mysql # - ./mysql/my.cnf:/etc/mysql/my.cnf + # # Additional mounts may go here + # - ../datajoint-python/notebook/backups:/backups minio: <<: *net environment: @@ -44,13 +46,17 @@ services: condition: service_healthy fakeservices.datajoint.io: <<: *net - image: raphaelguzman/nginx:v0.0.2 + image: raphaelguzman/nginx:v0.0.6 environment: - ADD_db_TYPE=DATABASE - ADD_db_ENDPOINT=db:3306 - ADD_minio_TYPE=MINIO - ADD_minio_ENDPOINT=minio:9000 - - ADD_minio_PREFIX=/ + - ADD_minio_PORT=80 # allow unencrypted connections + - ADD_minio_PREFIX=/datajoint + - ADD_browser_TYPE=MINIOADMIN + - ADD_browser_ENDPOINT=minio:9000 + - ADD_browser_PORT=80 # allow unencrypted connections - ADD_hub_TYPE=REST - ADD_hub_ENDPOINT=hub:5000 - ADD_hub_PREFIX=/api @@ -82,10 +88,10 @@ services: - DJ_TEST_USER=datajoint - DJ_TEST_PASSWORD=datajoint # If running tests locally, make sure to add entry in /etc/hosts for 127.0.0.1 fakeservices.datajoint.io - - S3_ENDPOINT=fakeservices.datajoint.io:9000 + - S3_ENDPOINT=fakeservices.datajoint.io - S3_ACCESS_KEY=datajoint - S3_SECRET_KEY=datajoint - - S3_BUCKET=datajoint-test + - S3_BUCKET=datajoint.test - PYTHON_USER=dja - JUPYTER_PASSWORD=datajoint - DISPLAY @@ -118,6 +124,7 @@ services: - .:/src - /tmp/.X11-unix:/tmp/.X11-unix:rw - ~/Documents/keys/datajoint-dev.pem:/home/dja/keys/datajoint-dev.pem:ro - # Additional mounted notebooks may go here + # # Additional mounted notebooks may go here + # - ../datajoint-python/notebook:/home/dja/notebooks networks: main: diff --git a/setup.py b/setup.py index 9283c8c..f119638 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ "Operating System :: OS Independent", ], setup_requires=['setuptools_certificate'], - install_requires=['datajoint', 'requests'], + install_requires=['datajoint>=0.12.5b1', 'requests'], privkey_path='~/keys/datajoint-dev.pem', entry_points={ 'datajoint_plugins.connection': [ From 859ca7184f1ecc0188db569ab7b8621167052a43 Mon Sep 17 00:00:00 2001 From: guzman-raphael Date: Mon, 18 May 2020 13:22:51 -0500 Subject: [PATCH 2/5] Update travis and implement stage area. --- .travis.yml | 44 ++++++++++++++++++++------------------------ 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/.travis.yml b/.travis.yml index 75b843b..132af3c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,6 @@ +branches: + except: + - /^stage.*$/ sudo: required env: global: @@ -34,6 +37,14 @@ main: &main - ./serve.sh up TRAVIS jobs: include: + - stage: "Lint: Syntax" + language: python + install: + - pip install flake8 + script: + - | + flake8 datajoint_connection_hub --count --select=E9,F63,F7,F82 --show-source \ + --statistics - <<: *main env: - PY_VER: "3.8" @@ -54,31 +65,16 @@ jobs: env: - PY_VER: "3.8" - MYSQL_VER: "8.0" - - <<: *main - env: - - PY_VER: "3.7" - - MYSQL_VER: "8.0" - - <<: *main - env: - - PY_VER: "3.6" - - MYSQL_VER: "8.0" - - <<: *main - env: - - PY_VER: "3.5" - - MYSQL_VER: "8.0" - <<: *main env: - PY_VER: "3.8" - MYSQL_VER: "5.6" - - <<: *main - env: - - PY_VER: "3.7" - - MYSQL_VER: "5.6" - - <<: *main - env: - - PY_VER: "3.6" - - MYSQL_VER: "5.6" - - <<: *main - env: - - PY_VER: "3.5" - - MYSQL_VER: "5.6" + - stage: "Lint: Style" + language: python + install: + - pip install flake8 + script: + - | + flake8 --ignore=E121,E123,E126,E226,E24,E704,W503,W504,E722,F401,W605 \ + datajoint_connection_hub --count --max-complexity=62 --max-line-length=127 \ + --statistics From 1a42f931faf83723a785e121e8f3bebc9f143a43 Mon Sep 17 00:00:00 2001 From: guzman-raphael Date: Mon, 18 May 2020 13:50:45 -0500 Subject: [PATCH 3/5] Update get_host_hook import. --- datajoint_connection_hub/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/datajoint_connection_hub/__init__.py b/datajoint_connection_hub/__init__.py index 75e933b..24820c4 100644 --- a/datajoint_connection_hub/__init__.py +++ b/datajoint_connection_hub/__init__.py @@ -2,6 +2,7 @@ import requests from requests.adapters import HTTPAdapter from datajoint.errors import DataJointError +from datajoint.connection import get_host_hook import pymysql as client from .version import __version__ From 072708c0bcd4fb31998a9cab70f2f122cae16495 Mon Sep 17 00:00:00 2001 From: guzman-raphael Date: Mon, 18 May 2020 14:10:55 -0500 Subject: [PATCH 4/5] Fix style. --- datajoint_connection_hub/__init__.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/datajoint_connection_hub/__init__.py b/datajoint_connection_hub/__init__.py index 24820c4..4a00319 100644 --- a/datajoint_connection_hub/__init__.py +++ b/datajoint_connection_hub/__init__.py @@ -22,7 +22,7 @@ def get_host(host_input): if re.match(HUB_PROTOCOL, host_input) and len(hub_path) > 2: try: resp = session.get('{}{}{}{}'.format(REQUEST_PROTOCOL, - hub_path[0][1:], API_ROUTE, API_TARGETS['PIPELINE']), + hub_path[0][1:], API_ROUTE, API_TARGETS['PIPELINE']), params={'org_name': hub_path[1][1:], 'pipeline_name': hub_path[2][1:]}, timeout=10) if resp.status_code == 200: @@ -30,19 +30,20 @@ def get_host(host_input): elif resp.status_code == 204: raise DataJointError( 'DataJoint Hub database resource `{}/{}/{}` not found.'.format( - hub_path[0][1:], hub_path[1][1:], hub_path[2][1:])) + hub_path[0][1:], hub_path[1][1:], hub_path[2][1:])) elif resp.status_code == 404: raise DataJointError( 'DataJoint Hub endpoint `{}{}{}{}` unavailable.'.format( - REQUEST_PROTOCOL, hub_path[0][1:], API_ROUTE, API_TARGETS['PIPELINE'])) + REQUEST_PROTOCOL, hub_path[0][1:], API_ROUTE, + API_TARGETS['PIPELINE'])) except requests.exceptions.SSLError: raise DataJointError( 'TLS security violation on DataJoint Hub target `{}{}{}`.'.format( - REQUEST_PROTOCOL, hub_path[0][1:], API_ROUTE)) + REQUEST_PROTOCOL, hub_path[0][1:], API_ROUTE)) except requests.exceptions.ConnectionError: raise DataJointError( 'Unable to reach DataJoint Hub target `{}{}{}`.'.format( - REQUEST_PROTOCOL, hub_path[0][1:], API_ROUTE)) + REQUEST_PROTOCOL, hub_path[0][1:], API_ROUTE)) elif not re.match('.*/', host_input): return host_input else: @@ -55,8 +56,8 @@ def connect_host(connection_obj): except client.err.OperationalError: if not connection_obj.is_connected: target = [int(v) if i == 1 else v - for i, v in enumerate( - get_host_hook(connection_obj.conn_info['host_input']).split(':'))] + for i, v in enumerate( + get_host_hook(connection_obj.conn_info['host_input']).split(':'))] if (target[0] != connection_obj.conn_info['host'] or len(target) > 1 and target[1] != connection_obj.conn_info['port']): connection_obj.conn_info['host'] = target[0] From 4e81e00ba3a86ce0190e2cc75844814714cc665c Mon Sep 17 00:00:00 2001 From: guzman-raphael Date: Mon, 18 May 2020 14:15:19 -0500 Subject: [PATCH 5/5] Fix style2. --- datajoint_connection_hub/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datajoint_connection_hub/__init__.py b/datajoint_connection_hub/__init__.py index 4a00319..f7bf701 100644 --- a/datajoint_connection_hub/__init__.py +++ b/datajoint_connection_hub/__init__.py @@ -23,8 +23,8 @@ def get_host(host_input): try: resp = session.get('{}{}{}{}'.format(REQUEST_PROTOCOL, hub_path[0][1:], API_ROUTE, API_TARGETS['PIPELINE']), - params={'org_name': hub_path[1][1:], 'pipeline_name': hub_path[2][1:]}, - timeout=10) + params={'org_name': hub_path[1][1:], + 'pipeline_name': hub_path[2][1:]}, timeout=10) if resp.status_code == 200: return resp.json()[0]['database_dsn'] elif resp.status_code == 204: