diff --git a/deploy/Makefile b/deploy/Makefile index e6a57ac9..6d7d5973 100644 --- a/deploy/Makefile +++ b/deploy/Makefile @@ -118,6 +118,8 @@ stop-nifi: stop-nifi-dev: $(WITH_ENV) docker compose -f services.dev.yml $(DC_STOP_CMD) nifi nifi-nginx +delete-nifi: delete-nifi-containers + delete-nifi-containers: $(WITH_ENV) docker compose -f services.yml rm -f -s nifi nifi-nginx @@ -140,6 +142,27 @@ delete-nifi-dev-images: echo "No NiFi images found in services.dev.yml"; \ fi +delete-nifi-volumes: + $(WITH_ENV) docker compose -f services.yml $(DC_DOWN_CMD) -v nifi nifi-nginx + +delete-elastic: + $(WITH_ENV) docker compose -f services.yml rm -f -s elasticsearch-1 elasticsearch-2 elasticsearch-3 kibana + +delete-elastic-volumes: + $(WITH_ENV) docker compose -f services.yml $(DC_DOWN_CMD) -v elasticsearch-1 elasticsearch-2 elasticsearch-3 kibana + +delete-databank: + $(WITH_ENV) docker compose -f services.yml rm -f -s cogstack-databank-db cogstack-databank-db-mssql + +delete-databank-volumes: + $(WITH_ENV) docker compose -f services.yml $(DC_DOWN_CMD) -v cogstack-databank-db cogstack-databank-db-mssql + +delete-samples-db: + $(WITH_ENV) docker compose -f services.yml rm -f -s samples-db + +delete-samples-db-volumes: + $(WITH_ENV) docker compose -f services.yml $(DC_DOWN_CMD) -v samples-db + stop-elastic: $(WITH_ENV) docker compose -f services.yml $(DC_STOP_CMD) elasticsearch-1 elasticsearch-2 kibana @@ -201,7 +224,7 @@ stop-data-infra: stop-nifi stop-elastic stop-samples stop-all: stop-data-infra stop-jupyter stop-medcat-service stop-ocr-services -.PHONY: stop-data-infra stop-nifi stop-nifi-dev delete-nifi-containers delete-nifi-dev-containers delete-nifi-images delete-nifi-dev-images stop-elastic stop-samples stop-jupyter +.PHONY: stop-data-infra stop-nifi stop-nifi-dev delete-nifi delete-nifi-containers delete-nifi-dev-containers delete-nifi-images delete-nifi-dev-images delete-nifi-volumes delete-elastic delete-elastic-volumes delete-databank delete-databank-volumes delete-samples-db delete-samples-db-volumes stop-elastic stop-samples stop-jupyter # cleanup diff --git a/deploy/elasticsearch.env b/deploy/elasticsearch.env index 0c7519e0..720c1b28 100644 --- a/deploy/elasticsearch.env +++ b/deploy/elasticsearch.env @@ -9,10 +9,10 @@ ELASTICSEARCH_VERSION=opensearch # possible values : # - elasticsearch : docker.elastic.co/elasticsearch/elasticsearch:8.19.11 # - elasticsearch (custom cogstack image) : cogstacksystems/cogstack-elasticsearch:latest -# - opensearch : opensearchproject/opensearch:3.4.0 +# - opensearch : opensearchproject/opensearch:3.5.0 # the custom cogstack image is always based on the last image of ES native -ELASTICSEARCH_DOCKER_IMAGE=opensearchproject/opensearch:3.4.0 +ELASTICSEARCH_DOCKER_IMAGE=opensearchproject/opensearch:3.5.0 ELASTICSEARCH_LOG_LEVEL=INFO @@ -48,6 +48,11 @@ ELASTICSEARCH_NODE_1_LOG_VOL_NAME=elasticsearch-vol-log-1 ELASTICSEARCH_NODE_2_LOG_VOL_NAME=elasticsearch-vol-log-2 ELASTICSEARCH_NODE_3_LOG_VOL_NAME=elasticsearch-vol-log-3 +# Performance Analyzer volumes (OpenSearch) +ELASTICSEARCH_NODE_1_PA_VOL_NAME=elasticsearch-vol-performance-analyzer-1 +ELASTICSEARCH_NODE_2_PA_VOL_NAME=elasticsearch-vol-performance-analyzer-2 +ELASTICSEARCH_NODE_3_PA_VOL_NAME=elasticsearch-vol-performance-analyzer-3 + # this is for native ES # possible values: # - trial # 30 day trial @@ -167,10 +172,10 @@ KIBANA_CONFIG_FILE_VERSION=opensearch_dashboards # possible values: # - elasticsearch : docker.elastic.co/kibana/kibana:8.19.11 # - elasticsearch (custom cogstack image) : cogstacksystems/cogstack-kibana:latest -# - opensearch : opensearchproject/opensearch-dashboards:3.4.0 +# - opensearch : opensearchproject/opensearch-dashboards:3.5.0 # the custom cogstack image is always based on the last image of ES native -ELASTICSEARCH_KIBANA_DOCKER_IMAGE=opensearchproject/opensearch-dashboards:3.4.0 +ELASTICSEARCH_KIBANA_DOCKER_IMAGE=opensearchproject/opensearch-dashboards:3.5.0 KIBANA_SERVER_NAME="cogstack-kibana" KIBANA_PUBLIC_BASE_URL="https://elasticsearch-1:5601" diff --git a/deploy/nifi.env b/deploy/nifi.env index ddc5fa71..3369ba3c 100644 --- a/deploy/nifi.env +++ b/deploy/nifi.env @@ -19,7 +19,7 @@ NIFI_DOCKER_LOG_NUM_FILES=10 ############################################################################################################################## -NIFI_VERSION="2.7.2" +NIFI_VERSION="2.8.0" # NiFi Docker image NIFI_DOCKER_IMAGE="cogstacksystems/cogstack-nifi:latest" diff --git a/deploy/services.yml b/deploy/services.yml index d0b16037..c61be9b4 100644 --- a/deploy/services.yml +++ b/deploy/services.yml @@ -311,6 +311,8 @@ services: elasticsearch-base: <<: *es-common profiles: ["_elasticsearch_base"] + security_opt: + - seccomp=unconfined volumes: *es-common-volumes elasticsearch-1: @@ -327,6 +329,8 @@ services: - ${ELASTICSEARCH_NODE_1_DATA_VOL_NAME:-elasticsearch-vol-1}:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/data # ES logs - ${ELASTICSEARCH_NODE_1_LOG_VOL_NAME:-elasticsearch-vol-log-1}:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/logs + # OpenSearch performance analyzer + - ${ELASTICSEARCH_NODE_1_PA_VOL_NAME:-elasticsearch-vol-performance-analyzer-1}:/usr/share/opensearch/config/opensearch-performance-analyzer/plugin-stats-metadata:rw ports: - "${ELASTICSEARCH_NODE_1_OUTPUT_PORT:-9200}:9200" - "${ELASTICSEARCH_NODE_1_COMM_OUTPUT_PORT:-9300}:9300" @@ -346,6 +350,8 @@ services: - ${ELASTICSEARCH_NODE_2_DATA_VOL_NAME:-elasticsearch-vol-2}:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/data # ES logs - ${ELASTICSEARCH_NODE_2_LOG_VOL_NAME:-elasticsearch-vol-log-2}:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/logs + # OpenSearch performance analyzer + - ${ELASTICSEARCH_NODE_2_PA_VOL_NAME:-elasticsearch-vol-performance-analyzer-2}:/usr/share/opensearch/config/opensearch-performance-analyzer/plugins-stats-metadata:rw ports: - "${ELASTICSEARCH_NODE_2_OUTPUT_PORT:-9201}:9200" - "${ELASTICSEARCH_NODE_2_COMM_OUTPUT_PORT:-9301}:9300" @@ -365,6 +371,8 @@ services: - ${ELASTICSEARCH_NODE_3_DATA_VOL_NAME:-elasticsearch-vol-3}:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/data # ES logs - ${ELASTICSEARCH_NODE_3_LOG_VOL_NAME:-elasticsearch-vol-log-3}:/usr/share/${ELASTICSEARCH_VERSION:-opensearch}/logs + # OpenSearch performance analyzer + - ${ELASTICSEARCH_NODE_3_PA_VOL_NAME:-elasticsearch-vol-performance-analyzer-3}:/usr/share/opensearch/config/opensearch-performance-analyzer/plugins-stats-metadata:rw ports: - "${ELASTICSEARCH_NODE_3_OUTPUT_PORT:-9202}:9200" - "${ELASTICSEARCH_NODE_3_COMM_OUTPUT_PORT:-9302}:9300" @@ -604,6 +612,13 @@ volumes: elasticsearch-vol-log-3: driver: local + elasticsearch-vol-performance-analyzer-1: + driver: local + elasticsearch-vol-performance-analyzer-2: + driver: local + elasticsearch-vol-performance-analyzer-3: + driver: local + metricbeat-data-1: driver: local metricbeat-data-2: diff --git a/docs/deploy/deployment.md b/docs/deploy/deployment.md index 822fed0b..036386bb 100644 --- a/docs/deploy/deployment.md +++ b/docs/deploy/deployment.md @@ -81,6 +81,8 @@ All commands automatically load environment variables via `export_env_vars.sh`. | Command | Description | |---------------------------------|-------------| | `make start-nifi` | Start NiFi and NiFi-Nginx | +| `make start-nifi-dev` | Start NiFi dev services from `services.dev.yml` | +| `make start-nifi-dev-build` | Build and start NiFi dev services from `services.dev.yml` | | `make start-elastic` | Start ES-1, ES-2, Kibana | | `make start-elastic-cluster` | Start ES-1, ES-2, ES-3 | | `make start-elastic-1/2/3` | Start individual Elasticsearch nodes | @@ -105,6 +107,7 @@ All commands automatically load environment variables via `export_env_vars.sh`. | Command | Description | |---------------------------------|-------------| | `make stop-nifi` | Stop NiFi stack | +| `make stop-nifi-dev` | Stop NiFi dev services (`services.dev.yml`) | | `make stop-elastic` | Stop ES-1, ES-2, Kibana | | `make stop-elastic-cluster` | Stop ES-1, ES-2 | | `make stop-elastic-1/2/3` | Stop individual ES nodes | @@ -124,6 +127,25 @@ All commands automatically load environment variables via `export_env_vars.sh`. --- +### ๐Ÿ—‘๏ธ Delete Services + +| Command | Description | +|---------------------------------|-------------| +| `make delete-nifi` | Delete NiFi and NiFi-Nginx containers | +| `make delete-nifi-containers` | Delete NiFi and NiFi-Nginx containers | +| `make delete-nifi-dev-containers` | Delete NiFi dev containers (`services.dev.yml`) | +| `make delete-nifi-images` | Delete NiFi/NiFi-Nginx images from `services.yml` | +| `make delete-nifi-dev-images` | Delete NiFi/NiFi-Nginx images from `services.dev.yml` | +| `make delete-nifi-volumes` | Remove NiFi-related volumes (via compose down `-v`) | +| `make delete-elastic` | Delete Elasticsearch and Kibana containers | +| `make delete-elastic-volumes` | Remove Elasticsearch and Kibana volumes (via compose down `-v`) | +| `make delete-databank` | Delete Databank DB containers | +| `make delete-databank-volumes` | Remove Databank DB volumes (via compose down `-v`) | +| `make delete-samples-db` | Delete samples DB container | +| `make delete-samples-db-volumes`| Remove samples DB volumes (via compose down `-v`) | + +--- + ### ๐Ÿงน Cleanup | Command | Description | diff --git a/docs/deploy/main.md b/docs/deploy/main.md index 31416817..6d7fdeb5 100755 --- a/docs/deploy/main.md +++ b/docs/deploy/main.md @@ -43,11 +43,11 @@ Execute the following commands in the root directory of the repo: 1. `git-lfs pull` 2. (OPTIONAL, if you already have the software in [this section installed](#-software-requirements-linuxmacos))`sudo bash ./scripts/installation_utils/install_docker_and_utils.sh` , and wait for it to finish, it may take a while to get all the packages.. -3. `bash ./scripts/git_update_submodules_in_repo.sh` -4. check that docker works correctly : `docker pull hello-world` -5. if no errors, run: `docker run --rm hello-world`, it should run without issues -6. if there are any issues check the below warning section -7. `cd deploy` +3. `cd deploy` +4. `make git-update-submodules` +5. check that docker works correctly : `docker pull hello-world` +6. if no errors, run: `docker run --rm hello-world`, it should run without issues +7. if there are any issues check the below warning section :::{warning} IMPORTANT NOTE: Do a `git-lfs pull` so that you have everything downloaded from the repo (including bigger zipped files.). @@ -55,7 +55,7 @@ IMPORTANT NOTE: Do a `git-lfs pull` so that you have everything downloaded from :::{warning} Ensure all Git submodules are initialized and updated: -`cd deploy && git-update-submodules` +`make git-update-submodules` ::: :::{warning} diff --git a/nifi/Dockerfile b/nifi/Dockerfile index 45bd4d46..c3b93ee8 100644 --- a/nifi/Dockerfile +++ b/nifi/Dockerfile @@ -1,11 +1,11 @@ -ARG NIFI_VERSION=2.7.2 +ARG NIFI_VERSION=2.8.0 FROM apache/nifi:${NIFI_VERSION} SHELL ["/bin/bash", "-o", "pipefail", "-c"] -ARG NIFI_VERSION=2.7.2 -ARG GROOVY_VERSION=5.0.2 +ARG NIFI_VERSION=2.8.0 +ARG GROOVY_VERSION=5.0.4 ARG HTTP_PROXY="" ARG HTTPS_PROXY="" @@ -142,7 +142,7 @@ RUN set -eux; \ || curl -fsSL --retry 8 --retry-delay 3 --connect-timeout 20 -o groovy.zip "$url2"; \ unzip groovy.zip; rm groovy.zip -ENV GROOVY_BIN=/opt/nifi/groovy/groovy-5.0.2/bin +ENV GROOVY_BIN=/opt/nifi/groovy/groovy-${GROOVY_VERSION}/bin RUN $GROOVY_BIN/grape -V install org.apache.avro avro 1.12.0 ####################################### NAR INSTALL SECTION ########################################## diff --git a/nifi/requirements.txt b/nifi/requirements.txt index 7cbb706a..17b4ec5b 100644 --- a/nifi/requirements.txt +++ b/nifi/requirements.txt @@ -2,7 +2,7 @@ wheel==0.46.2 # data science pkgs nltk==3.9.1 -numpy==2.4.0 +numpy==2.4.2 pandas==2.3.3 dill>=0.4.0,<1.0.0 bokeh==3.8.2 diff --git a/services/elasticsearch/config/opensearch.yml b/services/elasticsearch/config/opensearch.yml index ad1809de..bbc9fd3c 100644 --- a/services/elasticsearch/config/opensearch.yml +++ b/services/elasticsearch/config/opensearch.yml @@ -3,6 +3,8 @@ network.publish_host: ${ELASTICSEARCH_NETWORK_PUBLISH_HOST} ###### CLUSTER CONFIGURATION ###### +bootstrap.system_call_filter: false + # To allow multiple nodes add the node"s IP addresses to the discovery pool. # ["", "", ""] discovery.seed_hosts: ${ELASTICSEARCH_SEED_HOSTS}