Bigquery Exception Handling

.github/workflows/py-tests-3_10.yml

@@ -0,0 +1,69 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+name: py-tests-3.10
+on:
+  push:
+    branches: [main]
+    paths:
+      - ingestion/**
+      - openmetadata-service/**
+  pull_request_target:
+    branches: [main]
+    paths:
+      - ingestion/**
+      - openmetadata-service/**
+
+jobs:
+  py-run-tests:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+        ref: ${{ github.event.pull_request.head.sha }}
+
+    - name: Set up JDK 11
+      uses: actions/setup-java@v2
+      with:
+        java-version: '11'
+        distribution: 'adopt'
+
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v2
+      with:
+        python-version: "3.10"
+
+    - name: Install Ubuntu dependencies
+      run: |
+        sudo apt-get install -y unixodbc-dev python3-venv librdkafka-dev gcc libsasl2-dev build-essential libssl-dev libffi-dev \
+        unixodbc-dev libevent-dev python3-dev
+    - name: Generate models
+      run: |
+        python3 -m venv env
+        source env/bin/activate
+        sudo make install_antlr_cli
+        make install_dev generate
+    - name: Install open-metadata dependencies
+      run: |
+        source env/bin/activate
+        make install_all install_test
+    - name: Start Server and Ingest Sample Data
+      env:
+        INGESTION_DEPENDENCY: "mysql,elasticsearch"
+      run: ./docker/run_local_docker.sh -m no-ui
+      timeout-minutes: 30
+
+    - name: Run Python Tests & record coverage
+      run: |
+        source env/bin/activate
+        make coverage
+

.github/workflows/py-tests.yml

@@ -27,7 +27,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        py-version: ['3.7', '3.8', '3.9', '3.10']
+        py-version: ['3.7', '3.8', '3.9']
     steps:
     - name: Checkout
       uses: actions/checkout@v2
@@ -48,7 +48,7 @@ jobs:
     - name: Install Ubuntu dependencies
       run: |
         sudo apt-get install -y unixodbc-dev python3-venv librdkafka-dev gcc libsasl2-dev build-essential libssl-dev libffi-dev \
-        librdkafka-dev unixodbc-dev libevent-dev
+        unixodbc-dev libevent-dev python3-dev
 
     - name: Generate models
       run: |

ingestion/src/metadata/ingestion/source/database/bigquery.py

@@ -134,24 +134,27 @@ def yield_tag(self, _: str) -> Iterable[OMetaTagAndCategory]:
         :param _:
         :return:
         """
-        taxonomies = PolicyTagManagerClient().list_taxonomies(
-            parent=f"projects/{self.project_id}/locations/{self.service_connection.taxonomyLocation}"
-        )
-        for taxonomy in taxonomies:
-            policiy_tags = PolicyTagManagerClient().list_policy_tags(
-                parent=taxonomy.name
+        try:
+            taxonomies = PolicyTagManagerClient().list_taxonomies(
+                parent=f"projects/{self.project_id}/locations/{self.service_connection.taxonomyLocation}"
             )
-            for tag in policiy_tags:
-                yield OMetaTagAndCategory(
-                    category_name=CreateTagCategoryRequest(
-                        name=self.service_connection.tagCategoryName,
-                        description="",
-                        categoryType="Classification",
-                    ),
-                    category_details=CreateTagRequest(
-                        name=tag.display_name, description="Bigquery Policy Tag"
-                    ),
+            for taxonomy in taxonomies:
+                policiy_tags = PolicyTagManagerClient().list_policy_tags(
+                    parent=taxonomy.name
                 )
+                for tag in policiy_tags:
+                    yield OMetaTagAndCategory(
+                        category_name=CreateTagCategoryRequest(
+                            name=self.service_connection.tagCategoryName,
+                            description="",
+                            categoryType="Classification",
+                        ),
+                        category_details=CreateTagRequest(
+                            name=tag.display_name, description="Bigquery Policy Tag"
+                        ),
+                    )
+        except Exception as err:
+            logger.error(err)
 
     def get_tag_labels(self, table_name: str) -> Optional[List[TagLabel]]:
         """