Skip to content

Commit 048f7b9

Browse files
ksy36ksy36
committed
Fixes #135 - Add a job to fetch Webcompat Knowldegebase related bugs from bugzilla and store them in BQ
1 parent 9f7333e commit 048f7b9

14 files changed

Lines changed: 917 additions & 0 deletions

File tree

‎.circleci/config.yml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -279,6 +279,22 @@ jobs:
279279
command: docker run app:build pytest
280280

281281

282+
build-job-webcompat-kb:
283+
docker:
284+
- image: << pipeline.parameters.git-image >>
285+
steps:
286+
- checkout
287+
- compare-branch:
288+
pattern: ^jobs/webcompat-kb/
289+
- setup_remote_docker:
290+
version: << pipeline.parameters.docker-version >>
291+
- run:
292+
name: Build Docker image
293+
command: docker build -t app:build jobs/webcompat-kb/
294+
- run:
295+
name: Test Code
296+
command: docker run app:build pytest --flake8 --black
297+
282298
workflows:
283299
docker-etl:
284300
jobs:
@@ -475,3 +491,17 @@ workflows:
475491
branches:
476492
only: main
477493

494+
495+
job-webcompat-kb:
496+
jobs:
497+
- build-job-webcompat-kb
498+
- gcp-gcr/build-and-push-image:
499+
context: data-eng-airflow-gcr
500+
docker-context: jobs/webcompat-kb/
501+
path: jobs/webcompat-kb/
502+
image: webcompat-kb_docker_etl
503+
requires:
504+
- build-job-webcompat-kb
505+
filters:
506+
branches:
507+
only: main

‎jobs/webcompat-kb/.dockerignore‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
.ci_job.yaml
2+
.ci_workflow.yaml
3+
.DS_Store
4+
*.pyc
5+
.pytest_cache/
6+
__pycache__/
7+
venv/

‎jobs/webcompat-kb/.flake8‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
[flake8]
2+
max-line-length = 88

‎jobs/webcompat-kb/.gitignore‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
.DS_Store
2+
*.pyc
3+
__pycache__/
4+
venv/

‎jobs/webcompat-kb/Dockerfile‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
FROM python:3.8
2+
MAINTAINER <kberezina@mozilla.com>
3+
4+
# https://github.com/mozilla-services/Dockerflow/blob/master/docs/building-container.md
5+
ARG USER_ID="10001"
6+
ARG GROUP_ID="app"
7+
ARG HOME="/app"
8+
9+
ENV HOME=${HOME}
10+
RUN groupadd --gid ${USER_ID} ${GROUP_ID} && \
11+
useradd --create-home --uid ${USER_ID} --gid ${GROUP_ID} --home-dir ${HOME} ${GROUP_ID}
12+
13+
WORKDIR ${HOME}
14+
15+
RUN pip install --upgrade pip
16+
17+
COPY requirements.txt requirements.txt
18+
RUN pip install -r requirements.txt
19+
20+
COPY . .
21+
22+
RUN pip install .
23+
24+
# Drop root and change ownership of the application folder to the user
25+
RUN chown -R ${USER_ID}:${GROUP_ID} ${HOME}
26+
USER ${USER_ID}

‎jobs/webcompat-kb/README.md‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
# Web compatibility Knowledge Base bugs import from bugzilla
2+
3+
This job fetches bugzilla bugs from Web Compatibility > Knowledge Base component,
4+
as well as their core bugs dependencies and breakage reports and puts them into BQ.
5+
6+
## Usage
7+
8+
This script is intended to be run in a docker container.
9+
Build the docker image with:
10+
11+
```sh
12+
docker build -t webcompat-kb .
13+
```
14+
15+
To run locally, first install dependencies in `jobs/webcompat-kb`:
16+
17+
```sh
18+
pip install -r requirements.txt
19+
```
20+
21+
And then run the script after authentication with gcloud:
22+
23+
```sh
24+
gcloud auth application-default login
25+
python3 webcompat_kb/main.py --bq_project_id=<your_project_id> --bq_dataset_id=<your_dataset_id>
26+
```
27+
28+
## Development
29+
30+
Run tests with:
31+
32+
```sh
33+
pytest
34+
```
35+
36+
`flake8` and `black` are included for code linting and formatting:
37+
38+
```sh
39+
pytest --black --flake8
40+
```
41+
42+
or
43+
44+
```sh
45+
flake8 webcompat_kb/ tests/
46+
black --diff webcompat_kb/ tests/
47+
```

‎jobs/webcompat-kb/ci_job.yaml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
build-job-webcompat-kb:
2+
docker:
3+
- image: << pipeline.parameters.git-image >>
4+
steps:
5+
- checkout
6+
- compare-branch:
7+
pattern: ^jobs/webcompat-kb/
8+
- setup_remote_docker:
9+
version: << pipeline.parameters.docker-version >>
10+
- run:
11+
name: Build Docker image
12+
command: docker build -t app:build jobs/webcompat-kb/
13+
- run:
14+
name: Test Code
15+
command: docker run app:build pytest --flake8 --black

‎jobs/webcompat-kb/ci_workflow.yaml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
job-webcompat-kb:
2+
jobs:
3+
- build-job-webcompat-kb
4+
- gcp-gcr/build-and-push-image:
5+
context: data-eng-airflow-gcr
6+
docker-context: jobs/webcompat-kb/
7+
path: jobs/webcompat-kb/
8+
image: webcompat-kb_docker_etl
9+
requires:
10+
- build-job-webcompat-kb
11+
filters:
12+
branches:
13+
only: main

‎jobs/webcompat-kb/pytest.ini‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
[pytest]
2+
testpaths =
3+
tests

‎jobs/webcompat-kb/requirements.txt‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
click==8.0.4
2+
google-cloud-bigquery==3.11.4
3+
pytest==6.0.2
4+
pytest-black==0.3.11
5+
pytest-flake8==1.0.6

0 commit comments

Comments
 (0)