Helpers/docker-compose.jupyter.yml at main · bluefate/Helpers · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
version: '3.8'

services:
  namenode:
    image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8
    container_name: namenode
    environment:
      - CLUSTER_NAME=test
    ports:
      - "9870:9870"
    volumes:
      - hadoop_namenode:/hadoop/dfs/name
    networks:
      - hadoop

  datanode:
    image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8
    container_name: datanode
    environment:
      - CORE_CONF_fs_defaultFS=hdfs://namenode:8020
    ports:
      - "9864:9864"
    volumes:
      - hadoop_datanode:/hadoop/dfs/data
    networks:
      - hadoop

  spark:
    image: bitnami/spark:latest
    container_name: spark
    environment:
      - SPARK_MODE=master
      - SPARK_RPC_AUTHENTICATION_ENABLED=no
      - SPARK_LOCAL_DIRS=/tmp
      - SPARK_MASTER_HOST=spark
      - SPARK_MASTER_PORT=7077
    ports:
      - "8080:8080"  # Spark Master Web UI
      - "7077:7077"
    networks:
      - hadoop

  jupyter:
    image: jupyter/pyspark-notebook
    container_name: jupyter
    ports:
      - "8888:8888"
    volumes:
      # - ./jupyter/notebooks:/home/jovyan/work
      - "/c/Users/usnername/somefolder:/home/jovyan/somefolder/"
    environment:
      - JUPYTER_ENABLE_LAB=yes
    networks:
      - hadoop
    depends_on:
      - spark
      - namenode
    user: root

    command: >
      bash -c "
        apt-get update &&
        apt-get install -y libnss3 libatk-bridge2.0-0 libcups2 libxcomposite1 libxdamage1 libxfixes3 libxrandr2 libgbm1 libxkbcommon0 libpango-1.0-0 libcairo2 libasound2 &&
        pip install --no-cache-dir beautifultable scikit-learn networkx plotly findspark scipy &&
        pip install --no-cache-dir --upgrade kaleido &&
        echo "Y" | plotly_get_chrome &&
        start-notebook.sh
      "

volumes:
  hadoop_namenode:
  hadoop_datanode:

networks:
  hadoop:
    driver: bridge


# HDFS UI	http://localhost:9870
# Spark Master UI	http://localhost:8080
# Jupyter Lab	http://localhost:8888