Nifi 2 8 0

deploy/Makefile

@@ -118,6 +118,8 @@ stop-nifi:
 stop-nifi-dev:
 	$(WITH_ENV) docker compose -f services.dev.yml $(DC_STOP_CMD) nifi nifi-nginx 
 
+delete-nifi: delete-nifi-containers
+
 delete-nifi-containers:
 	$(WITH_ENV) docker compose -f services.yml rm -f -s nifi nifi-nginx 
 
@@ -140,6 +142,27 @@ delete-nifi-dev-images:
 		echo "No NiFi images found in services.dev.yml"; \
 	fi
 
+delete-nifi-volumes:
+	$(WITH_ENV) docker compose -f services.yml $(DC_DOWN_CMD) -v nifi nifi-nginx
+
+delete-elastic:
+	$(WITH_ENV) docker compose -f services.yml rm -f -s elasticsearch-1 elasticsearch-2 elasticsearch-3 kibana
+
+delete-elastic-volumes:
+	$(WITH_ENV) docker compose -f services.yml $(DC_DOWN_CMD) -v elasticsearch-1 elasticsearch-2 elasticsearch-3 kibana
+
+delete-databank:
+	$(WITH_ENV) docker compose -f services.yml rm -f -s cogstack-databank-db cogstack-databank-db-mssql
+
+delete-databank-volumes:
+	$(WITH_ENV) docker compose -f services.yml $(DC_DOWN_CMD) -v cogstack-databank-db cogstack-databank-db-mssql
+
+delete-samples-db:
+	$(WITH_ENV) docker compose -f services.yml rm -f -s samples-db
+
+delete-samples-db-volumes:
+	$(WITH_ENV) docker compose -f services.yml $(DC_DOWN_CMD) -v samples-db
+
 stop-elastic:
 	$(WITH_ENV) docker compose -f services.yml $(DC_STOP_CMD) elasticsearch-1 elasticsearch-2 kibana
 
@@ -201,7 +224,7 @@ stop-data-infra: stop-nifi stop-elastic stop-samples
 
 stop-all: stop-data-infra stop-jupyter stop-medcat-service stop-ocr-services
 
-.PHONY: stop-data-infra stop-nifi stop-nifi-dev delete-nifi-containers delete-nifi-dev-containers delete-nifi-images delete-nifi-dev-images stop-elastic stop-samples stop-jupyter
+.PHONY: stop-data-infra stop-nifi stop-nifi-dev delete-nifi delete-nifi-containers delete-nifi-dev-containers delete-nifi-images delete-nifi-dev-images delete-nifi-volumes delete-elastic delete-elastic-volumes delete-databank delete-databank-volumes delete-samples-db delete-samples-db-volumes stop-elastic stop-samples stop-jupyter
 
 
 # cleanup

deploy/elasticsearch.env

@@ -9,10 +9,10 @@ ELASTICSEARCH_VERSION=opensearch
 # possible values :
 #   - elasticsearch                         : docker.elastic.co/elasticsearch/elasticsearch:8.19.11
 #   - elasticsearch (custom cogstack image) : cogstacksystems/cogstack-elasticsearch:latest 
-#   - opensearch                            : opensearchproject/opensearch:3.4.0
+#   - opensearch                            : opensearchproject/opensearch:3.5.0
 # the custom cogstack image is always based on the last image of ES native 
 
-ELASTICSEARCH_DOCKER_IMAGE=opensearchproject/opensearch:3.4.0
+ELASTICSEARCH_DOCKER_IMAGE=opensearchproject/opensearch:3.5.0
 
 ELASTICSEARCH_LOG_LEVEL=INFO
 
@@ -48,6 +48,11 @@ ELASTICSEARCH_NODE_1_LOG_VOL_NAME=elasticsearch-vol-log-1
 ELASTICSEARCH_NODE_2_LOG_VOL_NAME=elasticsearch-vol-log-2
 ELASTICSEARCH_NODE_3_LOG_VOL_NAME=elasticsearch-vol-log-3
 
+# Performance Analyzer volumes (OpenSearch)
+ELASTICSEARCH_NODE_1_PA_VOL_NAME=elasticsearch-vol-performance-analyzer-1
+ELASTICSEARCH_NODE_2_PA_VOL_NAME=elasticsearch-vol-performance-analyzer-2
+ELASTICSEARCH_NODE_3_PA_VOL_NAME=elasticsearch-vol-performance-analyzer-3
+
 # this is for native ES 
 # possible values:
 #   - trial  # 30 day trial
@@ -167,10 +172,10 @@ KIBANA_CONFIG_FILE_VERSION=opensearch_dashboards
 # possible values:
 #   - elasticsearch                         : docker.elastic.co/kibana/kibana:8.19.11
 #   - elasticsearch (custom cogstack image) : cogstacksystems/cogstack-kibana:latest
-#   - opensearch                            : opensearchproject/opensearch-dashboards:3.4.0
+#   - opensearch                            : opensearchproject/opensearch-dashboards:3.5.0
 # the custom cogstack image is always based on the last image of ES native 
 
-ELASTICSEARCH_KIBANA_DOCKER_IMAGE=opensearchproject/opensearch-dashboards:3.4.0
+ELASTICSEARCH_KIBANA_DOCKER_IMAGE=opensearchproject/opensearch-dashboards:3.5.0
 
 KIBANA_SERVER_NAME="cogstack-kibana"
 KIBANA_PUBLIC_BASE_URL="https://elasticsearch-1:5601"

deploy/nifi.env

@@ -19,7 +19,7 @@ NIFI_DOCKER_LOG_NUM_FILES=10
 
 ##############################################################################################################################
 
-NIFI_VERSION="2.7.2"
+NIFI_VERSION="2.8.0"
 
 # NiFi Docker image
 NIFI_DOCKER_IMAGE="cogstacksystems/cogstack-nifi:latest"

deploy/services.yml

@@ -311,6 +311,8 @@ services:
   elasticsearch-base:
     <<: *es-common
     profiles: ["_elasticsearch_base"]
+    security_opt:
+      - seccomp=unconfined
     volumes: *es-common-volumes
 
   elasticsearch-1:
@@ -327,6 +329,8 @@ services:
       - ${ELASTICSEARCH_NODE_1_DATA_VOL_NAME:-elasticsearch-vol-1}:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/data
       # ES logs
       - ${ELASTICSEARCH_NODE_1_LOG_VOL_NAME:-elasticsearch-vol-log-1}:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/logs
+      # OpenSearch performance analyzer
+      - ${ELASTICSEARCH_NODE_1_PA_VOL_NAME:-elasticsearch-vol-performance-analyzer-1}:/usr/share/opensearch/config/opensearch-performance-analyzer/plugin-stats-metadata:rw
     ports:
       - "${ELASTICSEARCH_NODE_1_OUTPUT_PORT:-9200}:9200"
       - "${ELASTICSEARCH_NODE_1_COMM_OUTPUT_PORT:-9300}:9300"
@@ -346,6 +350,8 @@ services:
       - ${ELASTICSEARCH_NODE_2_DATA_VOL_NAME:-elasticsearch-vol-2}:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/data
       # ES logs
       - ${ELASTICSEARCH_NODE_2_LOG_VOL_NAME:-elasticsearch-vol-log-2}:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/logs
+      # OpenSearch performance analyzer
+      - ${ELASTICSEARCH_NODE_2_PA_VOL_NAME:-elasticsearch-vol-performance-analyzer-2}:/usr/share/opensearch/config/opensearch-performance-analyzer/plugins-stats-metadata:rw
     ports:
       - "${ELASTICSEARCH_NODE_2_OUTPUT_PORT:-9201}:9200"
       - "${ELASTICSEARCH_NODE_2_COMM_OUTPUT_PORT:-9301}:9300"
@@ -365,6 +371,8 @@ services:
       - ${ELASTICSEARCH_NODE_3_DATA_VOL_NAME:-elasticsearch-vol-3}:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/data
       # ES logs
       - ${ELASTICSEARCH_NODE_3_LOG_VOL_NAME:-elasticsearch-vol-log-3}:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/logs
+      # OpenSearch performance analyzer
+      - ${ELASTICSEARCH_NODE_3_PA_VOL_NAME:-elasticsearch-vol-performance-analyzer-3}:/usr/share/opensearch/config/opensearch-performance-analyzer/plugins-stats-metadata:rw
     ports:
       - "${ELASTICSEARCH_NODE_3_OUTPUT_PORT:-9202}:9200"
       - "${ELASTICSEARCH_NODE_3_COMM_OUTPUT_PORT:-9302}:9300"
@@ -604,6 +612,13 @@ volumes:
   elasticsearch-vol-log-3:
     driver: local
 
+  elasticsearch-vol-performance-analyzer-1:
+    driver: local
+  elasticsearch-vol-performance-analyzer-2:
+    driver: local
+  elasticsearch-vol-performance-analyzer-3:
+    driver: local
+
   metricbeat-data-1:
     driver: local
   metricbeat-data-2:

docs/deploy/deployment.md

@@ -81,6 +81,8 @@ All commands automatically load environment variables via `export_env_vars.sh`.
 | Command                         | Description |
 |---------------------------------|-------------|
 | `make start-nifi`               | Start NiFi and NiFi-Nginx |
+| `make start-nifi-dev`           | Start NiFi dev services from `services.dev.yml` |
+| `make start-nifi-dev-build`     | Build and start NiFi dev services from `services.dev.yml` |
 | `make start-elastic`            | Start ES-1, ES-2, Kibana             |
 | `make start-elastic-cluster`    | Start ES-1, ES-2, ES-3               |
 | `make start-elastic-1/2/3`      | Start individual Elasticsearch nodes |
@@ -105,6 +107,7 @@ All commands automatically load environment variables via `export_env_vars.sh`.
 | Command                         | Description |
 |---------------------------------|-------------|
 | `make stop-nifi`                | Stop NiFi stack                     |
+| `make stop-nifi-dev`            | Stop NiFi dev services (`services.dev.yml`) |
 | `make stop-elastic`             | Stop ES-1, ES-2, Kibana             |
 | `make stop-elastic-cluster`     | Stop ES-1, ES-2                     |
 | `make stop-elastic-1/2/3`       | Stop individual ES nodes            |
@@ -124,6 +127,25 @@ All commands automatically load environment variables via `export_env_vars.sh`.
 
 ---
 
+### 🗑️ Delete Services
+
+| Command                         | Description |
+|---------------------------------|-------------|
+| `make delete-nifi`              | Delete NiFi and NiFi-Nginx containers |
+| `make delete-nifi-containers`   | Delete NiFi and NiFi-Nginx containers |
+| `make delete-nifi-dev-containers` | Delete NiFi dev containers (`services.dev.yml`) |
+| `make delete-nifi-images`       | Delete NiFi/NiFi-Nginx images from `services.yml` |
+| `make delete-nifi-dev-images`   | Delete NiFi/NiFi-Nginx images from `services.dev.yml` |
+| `make delete-nifi-volumes`      | Remove NiFi-related volumes (via compose down `-v`) |
+| `make delete-elastic`           | Delete Elasticsearch and Kibana containers |
+| `make delete-elastic-volumes`   | Remove Elasticsearch and Kibana volumes (via compose down `-v`) |
+| `make delete-databank`          | Delete Databank DB containers |
+| `make delete-databank-volumes`  | Remove Databank DB volumes (via compose down `-v`) |
+| `make delete-samples-db`        | Delete samples DB container |
+| `make delete-samples-db-volumes`| Remove samples DB volumes (via compose down `-v`) |
+
+---
+
 ### 🧹 Cleanup
 
 | Command           | Description                                |

docs/deploy/main.md

@@ -43,19 +43,19 @@ Execute the following commands in the root directory of the repo:
 
 1. `git-lfs pull`
 2. (OPTIONAL, if you already have the software in [this section installed](#-software-requirements-linuxmacos))`sudo bash ./scripts/installation_utils/install_docker_and_utils.sh` , and wait for it to finish, it may take a while to get all the packages..
-3. `bash ./scripts/git_update_submodules_in_repo.sh`
-4. check that docker works correctly : `docker pull hello-world`
-5. if no errors, run: `docker run --rm hello-world`, it should run without issues
-6. if there are any issues check the below warning section
-7. `cd deploy`
+3. `cd deploy`
+4. `make git-update-submodules`
+5. check that docker works correctly : `docker pull hello-world`
+6. if no errors, run: `docker run --rm hello-world`, it should run without issues
+7. if there are any issues check the below warning section
 
 :::{warning}
 IMPORTANT NOTE: Do a `git-lfs pull` so that you have everything downloaded from the repo (including bigger zipped files.).
 :::
 
 :::{warning}
 Ensure all Git submodules are initialized and updated:
-`cd deploy && git-update-submodules`
+`make git-update-submodules`
 :::
 
 :::{warning}

nifi/Dockerfile

@@ -1,11 +1,11 @@
-ARG NIFI_VERSION=2.7.2
+ARG NIFI_VERSION=2.8.0
 
 FROM apache/nifi:${NIFI_VERSION}
 
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
-ARG NIFI_VERSION=2.7.2
-ARG GROOVY_VERSION=5.0.2
+ARG NIFI_VERSION=2.8.0
+ARG GROOVY_VERSION=5.0.4
 
 ARG HTTP_PROXY=""
 ARG HTTPS_PROXY=""
@@ -142,7 +142,7 @@ RUN set -eux; \
     || curl -fsSL --retry 8 --retry-delay 3 --connect-timeout 20 -o groovy.zip "$url2"; \
   unzip groovy.zip; rm groovy.zip
 
-ENV GROOVY_BIN=/opt/nifi/groovy/groovy-5.0.2/bin
+ENV GROOVY_BIN=/opt/nifi/groovy/groovy-${GROOVY_VERSION}/bin
 RUN $GROOVY_BIN/grape -V install org.apache.avro avro 1.12.0
 
 ####################################### NAR INSTALL SECTION ##########################################

nifi/requirements.txt

@@ -2,7 +2,7 @@ wheel==0.46.2
 
 # data science pkgs
 nltk==3.9.1
-numpy==2.4.0
+numpy==2.4.2
 pandas==2.3.3
 dill>=0.4.0,<1.0.0
 bokeh==3.8.2

services/elasticsearch/config/opensearch.yml

@@ -3,6 +3,8 @@ network.publish_host: ${ELASTICSEARCH_NETWORK_PUBLISH_HOST}
 
 ###### CLUSTER CONFIGURATION ######
 
+bootstrap.system_call_filter: false
+
 # To allow multiple nodes add the node"s IP addresses to the discovery pool. 
 # ["<private IP of opensearch-d1>", "<private IP of opensearch-d2>", "<private IP of opensearch-c1>"]
 discovery.seed_hosts: ${ELASTICSEARCH_SEED_HOSTS}