Skip to content

Commit 3e01b3e

Browse files
vladd-bitvladd-bit
authored
Cleanups (#55)
* NiFi: moved old dt4h templates to legacy folder. * NiFi: added new processors. * Deploy/docs: cleanup on services.yml. * Deploy: removed hacky access mode flag change in env loader script. * Deploy: removed nifi-registry entrypoint cmd. * Deploy: fixed nifi-registry startup issue (perms). * Deploy/docs: updated services defs, updated docs. * Deploy: fixed nifi registry perms.. * NiFi: .sh perms fix. * GHA: updated smoke test. * Smoke tests fix attempt. * Smoke test updates. * NiFi: updated base nifi processor def. * NiFi: updated processors to use new BaseNiFi processor impl. * Deploy: ES env file comments. * Deploy: added makefile utils for container deletion. * NiFi: added alternative cerner blob decompression script/processor. * NiFi: updated processors to not rely on override for process method from base class. * Deploy: removed nifi-registry container defs. * Services: removed nifi registry from nginx conf. * Deploy: removed nifi-registry env vars. * NiFi: removed nifi-registry dir. * GHA + Scripts: removed nifi-registry references. * Deploy: removed nifi-registry from Makefile targets.
1 parent ecb8092 commit 3e01b3e

39 files changed

Lines changed: 794 additions & 2239 deletions

‎.github/workflows/smoke-nifi-services.yml‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -33,18 +33,16 @@ jobs:
3333
- name: Start NiFi services (build)
3434
run: |
3535
set -euo pipefail
36-
source deploy/export_env_vars.sh
37-
set -euo pipefail
38-
docker compose -f deploy/services.dev.yml up -d --build nifi nifi-nginx nifi-registry-flow
36+
make -C deploy start-nifi-dev-build
3937
4038
- name: Smoke tests
4139
run: |
4240
set -euo pipefail
43-
echo "Running smoke checks against NiFi, NiFi Registry, and nginx."
41+
echo "Running smoke checks against NiFi and nginx."
4442
retries=30
4543
delay=15
4644
for attempt in $(seq 1 $retries); do
47-
if ./scripts/smoke_nifi_services.sh; then
45+
if ./scripts/tests/smoke_nifi_services.sh; then
4846
exit 0
4947
fi
5048
echo "Attempt ${attempt}/${retries} failed. Sleeping ${delay}s..."
@@ -60,7 +58,7 @@ jobs:
6058
source deploy/export_env_vars.sh
6159
set -euo pipefail
6260
docker compose -f deploy/services.dev.yml ps
63-
docker compose -f deploy/services.dev.yml logs --no-color nifi nifi-nginx nifi-registry-flow
61+
docker compose -f deploy/services.dev.yml logs --no-color nifi nifi-nginx
6462
6563
- name: Shutdown stack
6664
if: always()

‎deploy/Makefile‎

Lines changed: 35 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -25,17 +25,23 @@ load-env:
2525

2626
show-env:
2727
${WITH_ENV} >/dev/null 2>&1; printenv | sort
28+
29+
fix-nifi-registry-perms:
30+
$(WITH_ENV) SKIP_EXPORT_ENV=1 ../nifi/fix_nifi_registry_perms.sh $(COMPOSE_FILE)
2831

2932
# start services
3033

3134
start-nifi:
32-
$(WITH_ENV) docker compose -f services.yml $(DC_START_CMD) nifi nifi-nginx nifi-registry-flow
35+
$(WITH_ENV) SKIP_EXPORT_ENV=1 ../nifi/fix_nifi_registry_perms.sh services.yml; \
36+
docker compose -f services.yml $(DC_START_CMD) nifi nifi-nginx
3337

3438
start-nifi-dev:
35-
$(WITH_ENV) docker compose -f services.dev.yml $(DC_START_CMD) nifi nifi-nginx nifi-registry-flow
39+
$(WITH_ENV) SKIP_EXPORT_ENV=1 ../nifi/fix_nifi_registry_perms.sh services.dev.yml; \
40+
docker compose -f services.dev.yml $(DC_START_CMD) nifi nifi-nginx
3641

3742
start-nifi-dev-build:
38-
$(WITH_ENV) docker compose -f services.dev.yml up -d --build nifi nifi-nginx nifi-registry-flow
43+
$(WITH_ENV) SKIP_EXPORT_ENV=1 ../nifi/fix_nifi_registry_perms.sh services.dev.yml; \
44+
docker compose -f services.dev.yml up -d --build nifi nifi-nginx
3945

4046
start-elastic:
4147
$(WITH_ENV) docker compose -f services.yml $(DC_START_CMD) elasticsearch-1 elasticsearch-2 kibana
@@ -101,16 +107,38 @@ start-data-infra: start-nifi start-elastic start-samples
101107

102108
start-all: start-data-infra start-jupyter start-medcat-service start-ocr-services
103109

104-
.PHONY: start-all start-data-infra start-nifi start-nifi-dev start-nifi-dev-build start-elastic start-samples start-jupyter
110+
.PHONY: start-all start-data-infra start-nifi start-nifi-dev start-nifi-dev-build start-elastic start-samples start-jupyter fix-nifi-registry-perms
105111

106112

107113
# stop services
108114
#
109115
stop-nifi:
110-
$(WITH_ENV) docker compose -f services.yml $(DC_STOP_CMD) nifi nifi-nginx nifi-registry-flow
116+
$(WITH_ENV) docker compose -f services.yml $(DC_STOP_CMD) nifi nifi-nginx
111117

112118
stop-nifi-dev:
113-
$(WITH_ENV) docker compose -f services.dev.yml $(DC_STOP_CMD) nifi nifi-nginx nifi-registry-flow
119+
$(WITH_ENV) docker compose -f services.dev.yml $(DC_STOP_CMD) nifi nifi-nginx
120+
121+
delete-nifi-containers:
122+
$(WITH_ENV) docker compose -f services.yml rm -f -s nifi nifi-nginx
123+
124+
delete-nifi-dev-containers:
125+
$(WITH_ENV) docker compose -f services.dev.yml rm -f -s nifi nifi-nginx
126+
127+
delete-nifi-images:
128+
$(WITH_ENV) images="$$(docker compose -f services.yml config --images nifi nifi-nginx | sort -u)"; \
129+
if [ -n "$$images" ]; then \
130+
docker image rm -f $$images; \
131+
else \
132+
echo "No NiFi images found in services.yml"; \
133+
fi
134+
135+
delete-nifi-dev-images:
136+
$(WITH_ENV) images="$$(docker compose -f services.dev.yml config --images nifi nifi-nginx | sort -u)"; \
137+
if [ -n "$$images" ]; then \
138+
docker image rm -f $$images; \
139+
else \
140+
echo "No NiFi images found in services.dev.yml"; \
141+
fi
114142

115143
stop-elastic:
116144
$(WITH_ENV) docker compose -f services.yml $(DC_STOP_CMD) elasticsearch-1 elasticsearch-2 kibana
@@ -173,7 +201,7 @@ stop-data-infra: stop-nifi stop-elastic stop-samples
173201

174202
stop-all: stop-data-infra stop-jupyter stop-medcat-service stop-ocr-services
175203

176-
.PHONY: stop-data-infra stop-nifi stop-nifi-dev stop-elastic stop-samples stop-jupyter
204+
.PHONY: stop-data-infra stop-nifi stop-nifi-dev delete-nifi-containers delete-nifi-dev-containers delete-nifi-images delete-nifi-dev-images stop-elastic stop-samples stop-jupyter
177205

178206

179207
# cleanup

‎deploy/elasticsearch.env‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
ELASTICSEARCH_VERSION=opensearch
88

99
# possible values :
10-
# - elasticsearch : docker.elastic.co/elasticsearch/elasticsearch:8.18.2
10+
# - elasticsearch : docker.elastic.co/elasticsearch/elasticsearch:8.19.11
1111
# - elasticsearch (custom cogstack image) : cogstacksystems/cogstack-elasticsearch:latest
1212
# - opensearch : opensearchproject/opensearch:3.4.0
1313
# the custom cogstack image is always based on the last image of ES native
@@ -89,6 +89,8 @@ ELASTICSEARCH_SECURITY_DIR=../security/certificates/elastic/
8989

9090
# MEMORY CONFIG
9191
ELASTICSEARCH_JAVA_OPTS="-Xms512m -Xmx512m -Des.failure_store_feature_flag_enabled=true"
92+
ES_JAVA_OPTS=$ELASTICSEARCH_JAVA_OPTS
93+
OPENSEARCH_JAVA_OPTS=$ELASTICSEARCH_JAVA_OPTS
9294

9395
ELASTICSEARCH_DOCKER_CPU_MIN=1
9496
ELASTICSEARCH_DOCKER_CPU_MAX=1
@@ -163,7 +165,7 @@ KIBANA_VERSION=opensearch-dashboards
163165
KIBANA_CONFIG_FILE_VERSION=opensearch_dashboards
164166

165167
# possible values:
166-
# - elasticsearch : docker.elastic.co/kibana/kibana:8.18.2
168+
# - elasticsearch : docker.elastic.co/kibana/kibana:8.19.11
167169
# - elasticsearch (custom cogstack image) : cogstacksystems/cogstack-kibana:latest
168170
# - opensearch : opensearchproject/opensearch-dashboards:3.4.0
169171
# the custom cogstack image is always based on the last image of ES native
@@ -205,7 +207,7 @@ ELASTICSEARCH_XPACK_SECURITY_REPORTING_ENCRYPTION_KEY="e0Y1gTxHWOopIWMTtpjQsDS6K
205207

206208
######################################################################### METRICBEAT Env vars ##########################################################################
207209

208-
METRICBEAT_IMAGE="docker.elastic.co/beats/metricbeat:8.18.2"
210+
METRICBEAT_IMAGE="docker.elastic.co/beats/metricbeat:8.19.11"
209211

210212
METRICBEAT_DOCKER_SHM=512m
211213
METRICBEAT_DOCKER_CPU_MIN=1
@@ -222,7 +224,7 @@ FILEBEAT_STARTUP_COMMAND="-e --strict.perms=false"
222224

223225
FILEBEAT_HOST="https://elasticsearch-1:9200"
224226

225-
FILEBEAT_IMAGE="docker.elastic.co/beats/filebeat:8.18.2"
227+
FILEBEAT_IMAGE="docker.elastic.co/beats/filebeat:8.19.11"
226228

227229

228230
FILEBEAT_DOCKER_SHM=512m

‎deploy/export_env_vars.sh‎

Lines changed: 0 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -41,22 +41,6 @@ env_files=(
4141
"$SERVICES_DIR/cogstack-nlp/medcat-service/env/medcat.env"
4242
)
4343

44-
LINT_SCRIPT="$SCRIPT_DIR/../nifi/user_scripts/utils/lint_env.py"
45-
46-
if [ -e "$LINT_SCRIPT" ]; then
47-
chmod +x $LINT_SCRIPT
48-
fi
49-
50-
if [ -x "$LINT_SCRIPT" ]; then
51-
echo "🔍 Validating env files..."
52-
if ! python3 "$LINT_SCRIPT" "${env_files[@]}"; then
53-
echo "❌ Env validation failed. Fix the errors above before continuing."
54-
exit 1
55-
fi
56-
else
57-
echo "⚠️ Skipping env validation; $LINT_SCRIPT not found or not executable."
58-
fi
59-
6044
for env_file in "${env_files[@]}"; do
6145
if [ -f "$env_file" ]; then
6246
echo "✅ Sourcing $env_file"

‎deploy/network_settings.env‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,11 @@ ELASTICSEARCH_2_HOST_NAME="test-2:0.0.0.0"
1010
ELASTICSEARCH_3_HOST_NAME="test-3:0.0.0.0"
1111
KIBANA_HOST_NAME="test-4:0.0.0.0"
1212
NIFI_HOST_NAME="test-5:0.0.0.0"
13-
NIFI_REGISTRY_HOST_NAME="test-6:0.0.0.0"
1413

1514
# general network settings
1615
HTTPS_PROXY=""
1716
HTTP_PROXY=""
1817
NO_PROXY=""
1918
no_proxy=""
2019
http_proxy=""
21-
https_proxy=""
20+
https_proxy=""

‎deploy/nifi.env‎

Lines changed: 4 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -9,27 +9,20 @@ NIFI_JVM_HEAP_MAX=1g
99

1010

1111
NIFI_DOCKER_SHM_SIZE=1g
12-
NIFI_DOCKER_REGISTRY_SHM_SIZE=1g
1312

1413
NIFI_DOCKER_CPU_MIN=1
1514
NIFI_DOCKER_CPU_MAX=1
1615
NIFI_DOCKER_RAM=1g
1716

18-
NIFI_REGISTRY_DOCKER_CPU_MIN=1
19-
NIFI_REGISTRY_DOCKER_CPU_MAX=1
20-
NIFI_REGISTRY_DOCKER_RAM=1g
21-
2217
NIFI_DOCKER_LOG_SIZE_PER_FILE="250m"
2318
NIFI_DOCKER_LOG_NUM_FILES=10
2419

2520
##############################################################################################################################
2621

2722
NIFI_VERSION="2.7.2"
28-
NIFI_REGISTRY_VERSION=$NIFI_VERSION
2923

30-
# NiFi/NiFi Registry Docker image
24+
# NiFi Docker image
3125
NIFI_DOCKER_IMAGE="cogstacksystems/cogstack-nifi:latest"
32-
NIFI_REGISTRY_DOCKER_IMAGE="apache/nifi-registry:${NIFI_REGISTRY_VERSION:-2.7.2}"
3326

3427

3528
##############################################################################################################################
@@ -41,6 +34,9 @@ NIFI_DATA_PATH="../data/"
4134

4235
NIFI_TOOLKIT_VERSION=$NIFI_VERSION
4336

37+
# this is to mount medcat models (optional)
38+
NIFI_MEDCAT_SERVICE_MODEL_PRODUCTION_PATH="../services/cogstack-nlp/medcat-service/models/"
39+
4440
#### Port and network settings
4541
NIFI_WEB_PROXY_CONTEXT_PATH="/nifi"
4642

@@ -92,20 +88,3 @@ NIFI_TRUSTSTORE_TYPE=JKS
9288

9389
# this is from ./security/certificates_nifi.env, NIFI_SUBJ_LINE_CERTIFICATE_CN
9490
NIFI_INITIAL_ADMIN_IDENTITY="cogstack"
95-
96-
##############################################################################################################################
97-
# NIFI REGISTRY FLOW SECTION
98-
##############################################################################################################################
99-
NIFI_REGISTRY_DB_DIR=/opt/nifi-registry/nifi-registry-current/database
100-
NIFI_REGISTRY_FLOW_PROVIDER=file
101-
NIFI_REGISTRY_FLOW_STORAGE_DIR=/opt/nifi-registry/nifi-registry-current/flow_storage
102-
103-
NIFI_REGISTRY_FLOW_OUTPUT_PORT=8083
104-
NIFI_REGISTRY_FLOW_INPUT_PORT=18443
105-
106-
NIFI_REGISTRY_EXTERNAL_PORT_NGINX=18443
107-
NIFI_REGISTRY_INTERNAL_PORT_NGINX=18443
108-
109-
NIFI_REGISTRY_KEYSTORE_PATH="/security/certificates/nifi/nifi-keystore.jks"
110-
NIFI_REGISTRY_TRUSTSTORE_PATH="/security/certificates/nifi/nifi-truststore.jks"
111-

0 commit comments

Comments
 (0)