From 6e0b82e98368454ad29d4aaeca5581cc206f01c6 Mon Sep 17 00:00:00 2001 From: Mayur Date: Thu, 22 Sep 2022 10:31:17 +0530 Subject: [PATCH 1/4] CI Fix --- .github/workflows/py-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/py-tests.yml b/.github/workflows/py-tests.yml index a42b9f9db290..70837216e244 100644 --- a/.github/workflows/py-tests.yml +++ b/.github/workflows/py-tests.yml @@ -48,7 +48,7 @@ jobs: - name: Install Ubuntu dependencies run: | sudo apt-get install -y unixodbc-dev python3-venv librdkafka-dev gcc libsasl2-dev build-essential libssl-dev libffi-dev \ - librdkafka-dev unixodbc-dev libevent-dev + unixodbc-dev libevent-dev python3-dev - name: Generate models run: | From 4e22b7426d60ce4e60b4600f204dfa0aa991cbd1 Mon Sep 17 00:00:00 2001 From: Mayur Date: Thu, 22 Sep 2022 10:44:56 +0530 Subject: [PATCH 2/4] remove 3.10 from pytest --- .github/workflows/py-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/py-tests.yml b/.github/workflows/py-tests.yml index 70837216e244..d9eda8a3e85a 100644 --- a/.github/workflows/py-tests.yml +++ b/.github/workflows/py-tests.yml @@ -27,7 +27,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - py-version: ['3.7', '3.8', '3.9', '3.10'] + py-version: ['3.7', '3.8', '3.9'] steps: - name: Checkout uses: actions/checkout@v2 From 6306351068a4c4fb4eae8a7af90e3bcbdd2076c5 Mon Sep 17 00:00:00 2001 From: Mayur Date: Thu, 22 Sep 2022 11:08:43 +0530 Subject: [PATCH 3/4] added seperate 3.10pytest --- .github/workflows/py-tests-3_10.yml | 69 +++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 .github/workflows/py-tests-3_10.yml diff --git a/.github/workflows/py-tests-3_10.yml b/.github/workflows/py-tests-3_10.yml new file mode 100644 index 000000000000..de01b0234c62 --- /dev/null +++ b/.github/workflows/py-tests-3_10.yml @@ -0,0 +1,69 @@ +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: py-tests-3.10 +on: + push: + branches: [main] + paths: + - ingestion/** + - openmetadata-service/** + pull_request_target: + branches: [main] + paths: + - ingestion/** + - openmetadata-service/** + +jobs: + py-run-tests: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Set up JDK 11 + uses: actions/setup-java@v2 + with: + java-version: '11' + distribution: 'adopt' + + - name: Set up Python 3.10 + uses: actions/setup-python@v2 + with: + python-version: "3.10" + + - name: Install Ubuntu dependencies + run: | + sudo apt-get install -y unixodbc-dev python3-venv librdkafka-dev gcc libsasl2-dev build-essential libssl-dev libffi-dev \ + unixodbc-dev libevent-dev python3-dev + - name: Generate models + run: | + python3 -m venv env + source env/bin/activate + sudo make install_antlr_cli + make install_dev generate + - name: Install open-metadata dependencies + run: | + source env/bin/activate + make install_all install_test + - name: Start Server and Ingest Sample Data + env: + INGESTION_DEPENDENCY: "mysql,elasticsearch" + run: ./docker/run_local_docker.sh -m no-ui + timeout-minutes: 30 + + - name: Run Python Tests & record coverage + run: | + source env/bin/activate + make coverage + \ No newline at end of file From 38aa7c34ff40033dc17f740484b27ba77e54a5ab Mon Sep 17 00:00:00 2001 From: Mayur Date: Thu, 22 Sep 2022 10:36:04 +0530 Subject: [PATCH 4/4] Bigquery Exception Handling --- .../ingestion/source/database/bigquery.py | 35 ++++++++++--------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/ingestion/src/metadata/ingestion/source/database/bigquery.py b/ingestion/src/metadata/ingestion/source/database/bigquery.py index b0147f7ba9cd..50f809b6e0d2 100644 --- a/ingestion/src/metadata/ingestion/source/database/bigquery.py +++ b/ingestion/src/metadata/ingestion/source/database/bigquery.py @@ -134,24 +134,27 @@ def yield_tag(self, _: str) -> Iterable[OMetaTagAndCategory]: :param _: :return: """ - taxonomies = PolicyTagManagerClient().list_taxonomies( - parent=f"projects/{self.project_id}/locations/{self.service_connection.taxonomyLocation}" - ) - for taxonomy in taxonomies: - policiy_tags = PolicyTagManagerClient().list_policy_tags( - parent=taxonomy.name + try: + taxonomies = PolicyTagManagerClient().list_taxonomies( + parent=f"projects/{self.project_id}/locations/{self.service_connection.taxonomyLocation}" ) - for tag in policiy_tags: - yield OMetaTagAndCategory( - category_name=CreateTagCategoryRequest( - name=self.service_connection.tagCategoryName, - description="", - categoryType="Classification", - ), - category_details=CreateTagRequest( - name=tag.display_name, description="Bigquery Policy Tag" - ), + for taxonomy in taxonomies: + policiy_tags = PolicyTagManagerClient().list_policy_tags( + parent=taxonomy.name ) + for tag in policiy_tags: + yield OMetaTagAndCategory( + category_name=CreateTagCategoryRequest( + name=self.service_connection.tagCategoryName, + description="", + categoryType="Classification", + ), + category_details=CreateTagRequest( + name=tag.display_name, description="Bigquery Policy Tag" + ), + ) + except Exception as err: + logger.error(err) def get_tag_labels(self, table_name: str) -> Optional[List[TagLabel]]: """