Skip to content

Commit 09e650e

Browse files
PowermacintoshPowermacintosh
authored
feat: масштабный рефакторинг и расширение функциональности (#1)
1 parent 0d60398 commit 09e650e

65 files changed

Lines changed: 4328 additions & 421 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.env.test‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,11 @@ TASKS_DB_HOST = db_tasks_app_test
1212
TASKS_DB_PORT = 5432
1313
TASKS_DB_NAME = tasks_app_test
1414

15+
# Kafka
16+
KAFKA_BOOTSTRAP = kafka_test:9092
17+
KAFKA_TOPIC = task_events_test
18+
KAFKA_CLIENT_ID = tasks_app_test
19+
1520
# Loki
1621
LOKI_PORT=3100
1722

‎.github/workflows/test.yml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,9 @@ name: CI
22

33
on:
44
push:
5-
branches: ["main"]
5+
branches: [main, experiment/workflows]
66
pull_request:
7-
branches: ["main"]
7+
branches: [main]
88

99
jobs:
1010
tests:
@@ -17,8 +17,8 @@ jobs:
1717
- name: Build and start test services
1818
run: docker compose -f docker-compose.test.yml --env-file ./tasks/.env.test up -d --build
1919

20-
- name: Run pytest inside tasks_app_test
21-
run: docker compose -f docker-compose.test.yml --env-file ./tasks/.env.test exec -T tasks_app_test pytest -v
20+
- name: Run tests
21+
run: docker compose -f docker-compose.test.yml --env-file ./tasks/.env.test exec -T tasks_app_test pytest
2222

2323
- name: Stop and remove test services
2424
if: always()

‎Makefile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,6 @@ build:
88
docker compose up --build -d
99

1010
test:
11-
docker compose -f docker-compose.test.yml up --build -d --remove-orphans
11+
docker compose -f docker-compose.test.yml up --build -d
1212
docker compose -f docker-compose.test.yml exec -it tasks_app_test bash -c "pytest -v"
13-
docker compose -f docker-compose.test.yml down
13+
docker compose -f docker-compose.test.yml down -v --remove-orphans

‎README.md‎

Lines changed: 34 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,9 @@
1616
- **Loki** - система логирования
1717
- **Grafana** - система мониторинга
1818
- **Promtail** - инструмент для сбора логов
19+
- **Kafka** - система обмена сообщениями
20+
- **Kafka-UI** - визуальный интерфейс для работы с Kafka
21+
- **Locust** - инструмент для нагрузочного тестирования
1922

2023
## 🚀 Запуск проекта
2124

@@ -35,9 +38,7 @@
3538
cd microservices-example
3639
```
3740

38-
2. Создайте файл `.env` на основе `.env.test`:
39-
40-
3. Запустите приложение с помощью Makefile:
41+
2. Запустите приложение с помощью Makefile:
4142

4243
```bash
4344
make build
@@ -51,29 +52,54 @@ make test
5152

5253
## 📚 Документация API
5354

54-
После запуска документация API будет доступна по адресу:
55+
После запуска документация API Gateway будет доступна по адресу:
5556

5657
- Swagger UI: http://localhost:5000/docs/
5758
![gateway/static/swagger-custom.png](gateway/static/swagger-custom.png)
5859

59-
## 🔧 Настройка окружения
60+
Также API Gateway предоставляет возможность работы с API через GraphQL:
61+
62+
- GraphQL UI: http://localhost:5000/api/v1/graphql
63+
![gateway/static/graphql-interface.png](gateway/static/graphql-interface.png)
64+
65+
## 🔧 Настройка окружения для разработки
6066

61-
Создайте файл `.env` в корне проекта и в каждом сервисе
67+
Создайте файл `.env` в корне сервиса
6268
`gateway/`, `tasks/`.
6369
Аналогично файлу `.env.test`.
6470

6571
```env
6672
MODE = DEVELOPMENT
6773
```
6874

69-
## 📊 Логирование
75+
## 📊 Логирование в Grafana
7076

7177
Логи приложения отправляются в Loki и доступны через Grafana.
7278
**Grafana**: `http://localhost:3010`
7379

80+
`Логин` - admin
81+
`Пароль` - admin (при первом входе)
82+
7483
### Подключение Loki к Grafana
7584

76-
**Loki-connection-url**: `http://loki:3100`
85+
#### Путь для подключения Loki к Grafana
86+
87+
`Connections` => `Loki` => `Add new data source`
88+
89+
- **Connection-url**: `http://loki:3100`
90+
- ⚙️ **Save & Test**
91+
92+
#### Путь для просмотра логов в Grafana
93+
94+
`Drilldown` => `Logs`
95+
96+
## 🎯 Подключение к Kafka-UI
97+
98+
**Kafka-UI**: `http://localhost:8080`
99+
100+
## 🧪 Запуск нагрузочного тестирования Locust
101+
102+
**Locust**: `http://localhost:8089`
77103

78104
## 📄 Лицензия
79105

‎docker-compose.test.yml‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,27 @@ services:
1818
depends_on:
1919
db_tasks_app_test:
2020
condition: service_healthy
21+
kafka_test:
22+
condition: service_healthy
23+
24+
tasks_app_worker_test:
25+
build:
26+
context: tasks/
27+
container_name: tasks_app_worker_test
28+
env_file:
29+
- ./tasks/.env.test
30+
environment:
31+
SERVICE_NAME: tasks_app_test
32+
KAFKA_BOOTSTRAP: kafka_test:9092
33+
entrypoint: ["sh", "-c"]
34+
command:
35+
- |
36+
python worker.py
37+
depends_on:
38+
db_tasks_app_test:
39+
condition: service_healthy
40+
kafka_test:
41+
condition: service_healthy
2142

2243
db_tasks_app_test:
2344
image: postgres:17
@@ -38,5 +59,30 @@ services:
3859
timeout: 5s
3960
retries: 5
4061

62+
kafka_test:
63+
image: bitnami/kafka:latest
64+
container_name: kafka_test
65+
environment:
66+
KAFKA_BROKER_ID: 1
67+
KAFKA_CFG_NODE_ID: 1
68+
KAFKA_CFG_PROCESS_ROLES: "controller,broker"
69+
KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: "1@kafka_test:9093"
70+
KAFKA_CFG_LISTENERS: "CONTROLLER://:9093,PLAINTEXT://:9092"
71+
KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: "CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT"
72+
KAFKA_CFG_ADVERTISED_LISTENERS: "PLAINTEXT://kafka_test:9092"
73+
KAFKA_CFG_CONTROLLER_LISTENER_NAMES: "CONTROLLER"
74+
KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
75+
KAFKA_CFG_INTER_BROKER_LISTENER_NAME: "PLAINTEXT"
76+
KAFKA_CFG_LOG_RETENTION_HOURS: "72" # 3 дня
77+
ports:
78+
- "9092:9092"
79+
- "9093:9093"
80+
healthcheck:
81+
test: ["CMD", "bash", "-c", "kafka-topics.sh --list --bootstrap-server localhost:9092"]
82+
interval: 10s
83+
timeout: 5s
84+
retries: 6
85+
start_period: 20s
86+
4187
volumes:
4288
db_tasks_app_test:

‎docker-compose.yml‎

Lines changed: 113 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,11 @@ services:
44
context: gateway/
55
container_name: gateway_app
66
env_file:
7-
- ./gateway/.env
7+
- ./gateway/.env.docker
88
ports:
99
- "${GATEWAY_PORT}:${GATEWAY_PORT}"
10+
networks:
11+
- appnet
1012
entrypoint: ["sh", "-c"]
1113
command:
1214
- |
@@ -16,13 +18,34 @@ services:
1618
depends_on:
1719
loki:
1820
condition: service_started
21+
kafka:
22+
condition: service_healthy
23+
24+
locust_gateway:
25+
build:
26+
context: gateway/
27+
container_name: locust_gateway
28+
env_file:
29+
- ./gateway/.env.docker
30+
ports:
31+
- "8089:8089"
32+
networks:
33+
- appnet
34+
entrypoint: ["sh", "-c"]
35+
command:
36+
- |
37+
poetry run locust -f locustfile.py \
38+
--host http://gateway_app:${GATEWAY_PORT}
39+
depends_on:
40+
gateway_app:
41+
condition: service_started
1942

2043
tasks_app:
2144
build:
2245
context: tasks/
2346
container_name: tasks_app
2447
env_file:
25-
- ./tasks/.env
48+
- ./tasks/.env.docker
2649
ports:
2750
- "${TASKS_APP_PORT}"
2851
entrypoint: ["sh", "-c"]
@@ -38,12 +61,41 @@ services:
3861
condition: service_healthy
3962
loki:
4063
condition: service_started
64+
kafka:
65+
condition: service_healthy
66+
networks:
67+
- appnet
68+
69+
tasks_app_worker:
70+
build:
71+
context: tasks/
72+
container_name: tasks_app_worker
73+
env_file:
74+
- ./tasks/.env.docker
75+
environment:
76+
SERVICE_NAME: tasks_app
77+
KAFKA_BOOTSTRAP: kafka:9092
78+
KAFKA_GROUP_ID: tasks_app_group
79+
entrypoint: ["sh", "-c"]
80+
command:
81+
- |
82+
python worker.py
83+
depends_on:
84+
db_tasks_app:
85+
condition: service_healthy
86+
kafka:
87+
condition: service_healthy
88+
networks:
89+
- appnet
90+
deploy:
91+
restart_policy:
92+
condition: on-failure
4193

4294
db_tasks_app:
4395
image: postgres:17
4496
container_name: db_tasks_app
4597
env_file:
46-
- ./tasks/.env
98+
- ./tasks/.env.docker
4799
environment:
48100
POSTGRES_USER: ${TASKS_DB_USER}
49101
POSTGRES_PASSWORD: ${TASKS_DB_PASS}
@@ -52,16 +104,64 @@ services:
52104
- "${TASKS_DB_PORT}"
53105
volumes:
54106
- db_tasks_app:/var/lib/postgresql/data
107+
networks:
108+
- appnet
55109
healthcheck:
56110
test: ["CMD-SHELL", "pg_isready -U ${TASKS_DB_USER}"]
57111
interval: 5s
58112
timeout: 5s
59113
retries: 5
60-
114+
115+
kafka:
116+
image: bitnami/kafka:latest
117+
container_name: kafka
118+
# volumes:
119+
# - kafka_data:/bitnami/kafka
120+
environment:
121+
KAFKA_BROKER_ID: 1
122+
KAFKA_CFG_NODE_ID: 1
123+
KAFKA_CFG_PROCESS_ROLES: "controller,broker"
124+
KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: "1@kafka:9093"
125+
KAFKA_CFG_LISTENERS: "CONTROLLER://:9093,PLAINTEXT://:9092"
126+
KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: "CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT"
127+
KAFKA_CFG_ADVERTISED_LISTENERS: "PLAINTEXT://kafka:9092"
128+
KAFKA_CFG_CONTROLLER_LISTENER_NAMES: "CONTROLLER"
129+
KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
130+
KAFKA_CFG_INTER_BROKER_LISTENER_NAME: "PLAINTEXT"
131+
KAFKA_CFG_LOG_RETENTION_HOURS: "72" # 3 дня
132+
ports:
133+
- "9092:9092"
134+
- "9093:9093"
135+
healthcheck:
136+
test: ["CMD", "bash", "-c", "kafka-topics.sh --list --bootstrap-server localhost:9092"]
137+
interval: 10s
138+
timeout: 5s
139+
retries: 6
140+
start_period: 20s
141+
networks:
142+
- appnet
143+
144+
kafka-ui:
145+
image: provectuslabs/kafka-ui:latest
146+
container_name: kafka-ui
147+
ports:
148+
- "8088:8080"
149+
environment:
150+
KAFKA_CLUSTERS_0_NAME: local
151+
KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka:9092
152+
KAFKA_CLUSTERS_0_READONLY: "false"
153+
depends_on:
154+
kafka:
155+
condition: service_healthy
156+
networks:
157+
- appnet
158+
61159
loki:
62160
image: grafana/loki:latest
63161
ports:
64162
- "${LOKI_PORT}"
163+
networks:
164+
- appnet
65165
volumes:
66166
- ./loki-config.yaml:/etc/loki/local-config.yaml
67167
- loki_data:/loki
@@ -73,6 +173,8 @@ services:
73173

74174
promtail:
75175
image: grafana/promtail:latest
176+
networks:
177+
- appnet
76178
volumes:
77179
- ./promtail-config.yaml:/etc/promtail/config.yaml
78180
- /var/run/docker.sock:/var/run/docker.sock
@@ -87,6 +189,8 @@ services:
87189

88190
grafana:
89191
image: grafana/grafana:latest
192+
networks:
193+
- appnet
90194
ports:
91195
- "${GRAFANA_PORT}:3000"
92196
volumes:
@@ -96,7 +200,12 @@ services:
96200
- GF_SECURITY_ADMIN_PASSWORD=admin
97201
- GF_USERS_ALLOW_SIGN_UP=false
98202

203+
networks:
204+
appnet:
205+
driver: bridge
206+
99207
volumes:
100208
db_tasks_app:
101209
loki_data:
102210
grafana_data:
211+
# kafka_data:

0 commit comments

Comments
 (0)