RANGER-5450:OpenSearch docker containerisation for Ranger

dev-support/ranger-docker/.env

@@ -76,6 +76,9 @@ OZONE_RUNNER_VERSION=20230615-1
 # Trino Configuration
 TRINO_VERSION=latest
 
+# Open Search
+OPENSEARCH_VERSION=1.3.19
+
 # Debug Configuration
 DEBUG_ADMIN=false
 DEBUG_USERSYNC=false

dev-support/ranger-docker/Dockerfile.ranger-opensearch

@@ -0,0 +1,55 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG RANGER_BASE_IMAGE
+ARG RANGER_BASE_VERSION
+FROM ${RANGER_BASE_IMAGE}:${RANGER_BASE_VERSION}
+
+ARG OPENSEARCH_VERSION
+
+VOLUME /etc/keytabs
+
+COPY ./dist/version                                                   /home/ranger/dist/
+COPY ./downloads/opensearch-${OPENSEARCH_VERSION}-linux-x64.tar.gz   /home/ranger/dist/
+
+COPY ./scripts/opensearch/ranger-opensearch-setup.sh      ${RANGER_SCRIPTS}/
+COPY ./scripts/opensearch/ranger-opensearch.sh            ${RANGER_SCRIPTS}/
+COPY ./scripts/opensearch/opensearch.yml                  ${RANGER_SCRIPTS}/
+COPY ./scripts/opensearch/opensearch-jaas.conf            ${RANGER_SCRIPTS}/
+COPY ./scripts/wait_for_keytab.sh                         ${RANGER_SCRIPTS}/
+COPY ./scripts/wait_for_testusers_keytab.sh               ${RANGER_SCRIPTS}/
+COPY ./scripts/kdc/krb5.conf                              /etc/krb5.conf
+
+# Create opensearch user and group
+RUN groupadd -g 3002 opensearch && \
+    useradd -u 3002 -g opensearch -G hadoop -s /bin/bash opensearch
+
+# Extract and setup OpenSearch
+RUN tar xvfz /home/ranger/dist/opensearch-${OPENSEARCH_VERSION}-linux-x64.tar.gz --directory=/opt/ && \
+    ln -s /opt/opensearch-${OPENSEARCH_VERSION} /opt/opensearch && \
+    rm -f /home/ranger/dist/opensearch-${OPENSEARCH_VERSION}-linux-x64.tar.gz && \
+    mkdir -p /opt/opensearch/data /opt/opensearch/logs && \
+    chown -R opensearch:hadoop /opt/opensearch* && \
+    chmod 755 ${RANGER_SCRIPTS}/wait_for_keytab.sh && \
+    chmod 755 ${RANGER_SCRIPTS}/wait_for_testusers_keytab.sh && \
+    chmod 755 ${RANGER_SCRIPTS}/ranger-opensearch-setup.sh && \
+    chmod 755 ${RANGER_SCRIPTS}/ranger-opensearch.sh
+
+ENV OPENSEARCH_HOME=/opt/opensearch
+ENV PATH=/usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/opensearch/bin
+
+ENTRYPOINT [ "/home/ranger/scripts/ranger-opensearch.sh" ]
+

dev-support/ranger-docker/README.md

@@ -30,11 +30,11 @@ Use Dockerfiles in this directory to create docker images and run them to build
 
 - Set ```dev-support/ranger-docker``` as your working directory.
 
-- Execute following command to download necessary archives to setup Ranger/HDFS/Hive/HBase/Kafka/Knox/Ozone services:
+- Execute following command to download necessary archives to setup Ranger/HDFS/Hive/HBase/Kafka/Knox/Ozone/OpenSearch services:
    ~~~
    chmod +x download-archives.sh
    # use a subset of the below to download specific services
-   ./download-archives.sh hadoop hive hbase kafka knox ozone
+   ./download-archives.sh hadoop hive hbase kafka knox ozone opensearch
    ~~~
 
 - Execute following commands to set environment variables to build Apache Ranger docker containers:
@@ -102,6 +102,10 @@ docker compose -f docker-compose.ranger.yml -f docker-compose.ranger-ozone.yml u
 ~~~
 docker compose -f docker-compose.ranger.yml -f docker-compose.ranger-trino.yml up -d
 ~~~
+#### Bring up opensearch container:
+~~~
+docker compose -f docker-compose.ranger.yml -f docker-compose.ranger-opensearch.yml up -d
+~~~
 Similarly, check the `depends` section of the `docker-compose.ranger-service.yaml` file and add docker-compose files for these services when trying to bring up the `service` container.
 
 #### Bring up all containers

dev-support/ranger-docker/docker-compose.ranger-opensearch.yml

@@ -0,0 +1,40 @@
+services:
+  ranger-opensearch:
+    build:
+      context: .
+      dockerfile: Dockerfile.ranger-opensearch
+      args:
+        - RANGER_BASE_IMAGE=${RANGER_BASE_IMAGE}
+        - RANGER_BASE_VERSION=${RANGER_BASE_VERSION}
+        - OPENSEARCH_VERSION=${OPENSEARCH_VERSION}
+        - KERBEROS_ENABLED=${KERBEROS_ENABLED}
+    image: ranger-opensearch
+    container_name: ranger-opensearch
+    hostname: ranger-opensearch.rangernw
+    volumes:
+      - ./dist/keytabs/ranger-opensearch:/etc/keytabs
+      - opensearch-data:/opt/opensearch/data
+      - opensearch-logs:/opt/opensearch/logs
+    stdin_open: true
+    tty: true
+    networks:
+      - ranger
+    ports:
+      - "9200:9200"
+      - "9300:9300"
+    depends_on:
+      ranger-kdc:
+        condition: service_started
+    environment:
+      - OPENSEARCH_VERSION
+      - KERBEROS_ENABLED=true
+      - OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m
+
+volumes:
+  opensearch-data:
+  opensearch-logs:
+
+networks:
+  ranger:
+    name: rangernw
+

dev-support/ranger-docker/download-archives.sh

@@ -87,6 +87,9 @@ else
       then
         tar xvfz downloads/ozone-${OZONE_VERSION}.tar.gz --directory=downloads/
       fi
+    elif [[ $arg == 'opensearch' ]]
+    then
+      downloadIfNotPresent opensearch-${OPENSEARCH_VERSION}-linux-x64.tar.gz https://artifacts.opensearch.org/releases/bundle/opensearch/${OPENSEARCH_VERSION}
     else
       echo "Passed argument $arg is invalid!"
     fi

dev-support/ranger-docker/scripts/kdc/entrypoint.sh

@@ -100,6 +100,9 @@ function create_keytabs() {
   create_principal_and_keytab knox ranger-knox
 
   create_principal_and_keytab HTTP ranger-solr
+
+  create_principal_and_keytab opensearch ranger-opensearch
+  create_principal_and_keytab HTTP ranger-opensearch
 }
 
 function create_testusers() {
@@ -126,7 +129,7 @@ if [ ! -f $DB_DIR/principal ]; then
   echo "Database initialized"
 
   create_keytabs
-  create_testusers ranger ranger-usersync ranger-tagsync ranger-audit ranger-hadoop ranger-hive ranger-hbase ranger-kafka ranger-solr ranger-knox ranger-kms ranger-ozone ranger-trino
+  create_testusers ranger ranger-usersync ranger-tagsync ranger-audit ranger-hadoop ranger-hive ranger-hbase ranger-kafka ranger-solr ranger-knox ranger-kms ranger-ozone ranger-trino ranger-opensearch
 else
   echo "KDC DB already exists; skipping create"
 fi

dev-support/ranger-docker/scripts/opensearch/opensearch-jaas.conf

@@ -0,0 +1,18 @@
+Client {
+  com.sun.security.auth.module.Krb5LoginModule required
+  useKeyTab=true
+  storeKey=true
+  useTicketCache=false
+  keyTab="/etc/keytabs/opensearch.keytab"
+  principal="opensearch/ranger-opensearch.rangernw@EXAMPLE.COM";
+};
+
+Server {
+  com.sun.security.auth.module.Krb5LoginModule required
+  useKeyTab=true
+  storeKey=true
+  useTicketCache=false
+  keyTab="/etc/keytabs/opensearch.keytab"
+  principal="HTTP/ranger-opensearch.rangernw@EXAMPLE.COM";
+};
+

dev-support/ranger-docker/scripts/opensearch/opensearch.yml

@@ -0,0 +1,49 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# OpenSearch Configuration
+cluster.name: ranger-opensearch-cluster
+node.name: ranger-opensearch.rangernw
+
+# Network settings
+network.host: 0.0.0.0
+http.port: 9200
+transport.port: 9300
+
+# Discovery settings
+discovery.type: single-node
+
+# Path settings
+path.data: /opt/opensearch/data
+path.logs: /opt/opensearch/logs
+
+# Memory settings
+bootstrap.memory_lock: false
+
+# Disable OpenSearch Security Plugin.
+# This can be enabled with Ranger Plugin.
+plugins.security.disabled: true
+
+# Allow HTTP methods
+http.cors.enabled: true
+http.cors.allow-origin: "*"
+http.cors.allow-methods: OPTIONS, HEAD, GET, POST, PUT, DELETE
+http.cors.allow-headers: "X-Requested-With, Content-Type, Content-Length, Authorization"
+
+# Kerberos authentication is enabled via JAAS configuration
+# See opensearch-jaas.conf for Kerberos principal and keytab settings
+# JVM is configured with: -Djava.security.auth.login.config and -Djava.security.krb5.conf
+

dev-support/ranger-docker/scripts/opensearch/ranger-opensearch-setup.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Wait for Kerberos keytabs (enabled by default)
+if [ "${KERBEROS_ENABLED}" != "false" ]
+then
+  echo "Kerberos is enabled, waiting for keytabs..."
+  ${RANGER_SCRIPTS}/wait_for_keytab.sh opensearch.keytab
+  ${RANGER_SCRIPTS}/wait_for_testusers_keytab.sh
+else
+  echo "Kerberos is disabled"
+fi
+
+# Copy configuration files
+cp ${RANGER_SCRIPTS}/opensearch.yml          ${OPENSEARCH_HOME}/config/
+cp ${RANGER_SCRIPTS}/opensearch-jaas.conf    ${OPENSEARCH_HOME}/config/
+
+# Set ownership
+chown -R opensearch:hadoop ${OPENSEARCH_HOME}/
+
+echo "OpenSearch setup completed successfully"
+

dev-support/ranger-docker/scripts/opensearch/ranger-opensearch.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ ! -e ${OPENSEARCH_HOME}/.setupDone ]
+then
+  if "${RANGER_SCRIPTS}"/ranger-opensearch-setup.sh;
+  then
+    touch "${OPENSEARCH_HOME}"/.setupDone
+  else
+    echo "OpenSearch Setup Script didn't complete proper execution."
+  fi
+fi
+
+# Start OpenSearch as opensearch user with Kerberos enabled by default
+if [ "${KERBEROS_ENABLED}" != "false" ]; then
+  echo "Starting OpenSearch with Kerberos authentication enabled..."
+  su -c "cd ${OPENSEARCH_HOME} && OPENSEARCH_JAVA_OPTS=\"${OPENSEARCH_JAVA_OPTS} -Djava.security.krb5.conf=/etc/krb5.conf -Djava.security.auth.login.config=/opt/opensearch/config/opensearch-jaas.conf\" ./bin/opensearch" opensearch
+else
+  echo "Starting OpenSearch without Kerberos..."
+  su -c "cd ${OPENSEARCH_HOME} && ./bin/opensearch" opensearch
+fi
+