Unverified Commit 4f9d8cae authored by liwenhe1993's avatar liwenhe1993 Committed by GitHub
Browse files

Use docker-compose for deployment . Failed to execute python script (#2539) 

* Modify docker-compose and dockerfile

1. Add python3 in Dockerfile
2. Modify startup.sh file
3. Modify docker-compose.yml and add docker-stack.yml
4. Modify dolphinscheduler_env.sh

* Add logback-xx.xml and Modify Dockerfile
parent e6dec8d2

docker/docker-swarm/docker-compose.yml

+47 −75
Original line number Diff line number Diff line
@@ -16,36 +16,6 @@ 


version: "3.4"



networks:

  dolphinscheduler-postgresql:

    driver: bridge

  dolphinscheduler-zookeeper:

    driver: bridge

  dolphinscheduler-api:

    driver: bridge

  dolphinscheduler-frontend:

    driver: bridge

  dolphinscheduler-alert:

    driver: bridge

  dolphinscheduler-master:

    driver: bridge

  dolphinscheduler-worker:

    driver: bridge



volumes:

  dolphinscheduler-postgresql:

  dolphinscheduler-zookeeper:

  dolphinscheduler-api:

  dolphinscheduler-frontend:

  dolphinscheduler-alert:

  dolphinscheduler-master:

  dolphinscheduler-worker-data:

  dolphinscheduler-worker-logs:



configs:

  dolphinscheduler-worker-task-env:

    file: ./dolphinscheduler_env.sh



services:



  dolphinscheduler-postgresql:
@@ -58,16 +28,10 @@ services: 
      POSTGRESQL_USERNAME: root

      POSTGRESQL_PASSWORD: root

      POSTGRESQL_DATABASE: dolphinscheduler

    healthcheck:

      test: ["CMD", "pg_isready", "-U", "${POSTGRESQL_USERNAME}", "-d", "{POSTGRESQL_PASSWORD}", "-h", "localhost", "5432"]

      interval: 30s

      timeout: 5s

      retries: 3

      # start_period: 30s

    volumes:

    - dolphinscheduler-postgresql:/bitnami/postgresql

    networks:

    - dolphinscheduler-postgresql

    - dolphinscheduler



  dolphinscheduler-zookeeper:

    image: bitnami/zookeeper:latest
@@ -77,19 +41,13 @@ services: 
    environment:

      TZ: Asia/Shanghai

      ALLOW_ANONYMOUS_LOGIN: "yes"

    healthcheck:

      test: ["CMD-SHELL", "nc -z localhost 2181"]

      interval: 30s

      timeout: 5s

      retries: 3

      # start_period: 30s

    volumes:

    - dolphinscheduler-zookeeper:/bitnami/zookeeper

    networks:

    - dolphinscheduler-zookeeper

    - dolphinscheduler



  dolphinscheduler-api:

    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:1.2.1

    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev

    container_name: dolphinscheduler-api

    command: ["api-server"]

    ports:
@@ -103,23 +61,21 @@ services: 
      POSTGRESQL_DATABASE: dolphinscheduler

      ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181

    healthcheck:

      test: ["CMD-SHELL", "curl -f http://localhost:12345"]

      test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"]

      interval: 30s

      timeout: 5s

      retries: 3

      # start_period: 30s

      start_period: 30s

    depends_on:

    - dolphinscheduler-postgresql

    - dolphinscheduler-zookeeper

    volumes:

    - dolphinscheduler-api:/opt/dolphinscheduler/logs

    networks: 

    - dolphinscheduler-api

    - dolphinscheduler-postgresql

    - dolphinscheduler-zookeeper

    - dolphinscheduler



  dolphinscheduler-frontend:

    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:1.2.1

    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev

    container_name: dolphinscheduler-frontend

    command: ["frontend"]

    ports:
@@ -129,21 +85,20 @@ services: 
      FRONTEND_API_SERVER_HOST: dolphinscheduler-api

      FRONTEND_API_SERVER_PORT: 12345

    healthcheck:

      test: ["CMD-SHELL", "curl -f http://localhost:8888"]

      test: ["CMD", "nc", "-z", "localhost", "8888"]

      interval: 30s

      timeout: 5s

      retries: 3

      # start_period: 30s

      start_period: 30s

    depends_on:

    - dolphinscheduler-api

    volumes:

    - dolphinscheduler-frontend:/var/log/nginx

    networks:

    - dolphinscheduler-frontend

    - dolphinscheduler-api

    - dolphinscheduler



  dolphinscheduler-alert:

    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:1.2.1

    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev

    container_name: dolphinscheduler-alert

    command: ["alert-server"]

    environment:
@@ -172,17 +127,16 @@ services: 
      interval: 30s

      timeout: 5s

      retries: 3

      # start_period: 30s

      start_period: 30s

    depends_on:

    - dolphinscheduler-postgresql

    volumes: 

    - dolphinscheduler-alert:/opt/dolphinscheduler/logs

    networks: 

    - dolphinscheduler-alert

    - dolphinscheduler-postgresql

    - dolphinscheduler



  dolphinscheduler-master:

    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:1.2.1

    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev

    container_name: dolphinscheduler-master

    command: ["master-server"]

    ports:
@@ -207,19 +161,17 @@ services: 
      interval: 30s

      timeout: 5s

      retries: 3

      # start_period: 30s

      start_period: 30s

    depends_on: 

    - dolphinscheduler-postgresql

    - dolphinscheduler-zookeeper

    volumes: 

    - dolphinscheduler-master:/opt/dolphinscheduler/logs

    networks:

    - dolphinscheduler-master

    - dolphinscheduler-postgresql

    - dolphinscheduler-zookeeper

    - dolphinscheduler



  dolphinscheduler-worker:

    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:1.2.1

    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev

    container_name: dolphinscheduler-worker

    command: ["worker-server"]

    ports:
@@ -245,17 +197,37 @@ services: 
      interval: 30s

      timeout: 5s

      retries: 3

      # start_period: 30s

      start_period: 30s

    depends_on: 

    - dolphinscheduler-postgresql

    - dolphinscheduler-zookeeper

    volumes:

    - dolphinscheduler-worker-data:/tmp/dolphinscheduler

    - dolphinscheduler-worker-logs:/opt/dolphinscheduler/logs

    configs:

    - source: dolphinscheduler-worker-task-env

    - type: bind

      source: ./dolphinscheduler_env.sh

      target: /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh

    - type: volume

      source: dolphinscheduler-worker-data

      target: /tmp/dolphinscheduler

    - type: volume

      source: dolphinscheduler-worker-logs

      target: /opt/dolphinscheduler/logs

    networks:

    - dolphinscheduler-worker

    - dolphinscheduler-postgresql

    - dolphinscheduler-zookeeper
 
 No newline at end of file
 
    - dolphinscheduler



networks:

  dolphinscheduler:

    driver: bridge



volumes:

  dolphinscheduler-postgresql:

  dolphinscheduler-zookeeper:

  dolphinscheduler-api:

  dolphinscheduler-frontend:

  dolphinscheduler-alert:

  dolphinscheduler-master:

  dolphinscheduler-worker-data:

  dolphinscheduler-worker-logs:



configs:

  dolphinscheduler-worker-task-env:

    file: ./dolphinscheduler_env.sh
 
 No newline at end of file

docker/docker-swarm/docker-stack.yml

0 → 100644
+230 −0
Original line number Diff line number Diff line 
# Licensed to the Apache Software Foundation (ASF) under one

# or more contributor license agreements.  See the NOTICE file

# distributed with this work for additional information

# regarding copyright ownership.  The ASF licenses this file

# to you under the Apache License, Version 2.0 (the

# "License"); you may not use this file except in compliance

# with the License.  You may obtain a copy of the License at

#

#     http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.



version: "3.4"



services:



  dolphinscheduler-postgresql:

    image: bitnami/postgresql:latest

    ports:

    - 5432:5432

    environment:

      TZ: Asia/Shanghai

      POSTGRESQL_USERNAME: root

      POSTGRESQL_PASSWORD: root

      POSTGRESQL_DATABASE: dolphinscheduler

    volumes:

    - dolphinscheduler-postgresql:/bitnami/postgresql

    networks:

    - dolphinscheduler

    deploy:

      mode: replicated

      replicas: 1



  dolphinscheduler-zookeeper:

    image: bitnami/zookeeper:latest

    ports:

    - 2181:2181

    environment:

      TZ: Asia/Shanghai

      ALLOW_ANONYMOUS_LOGIN: "yes"

    volumes:

    - dolphinscheduler-zookeeper:/bitnami/zookeeper

    networks:

    - dolphinscheduler

    deploy:

      mode: replicated

      replicas: 1



  dolphinscheduler-api:

    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev

    command: ["api-server"]

    ports:

    - 12345:12345

    environment:

      TZ: Asia/Shanghai

      POSTGRESQL_HOST: dolphinscheduler-postgresql

      POSTGRESQL_PORT: 5432

      POSTGRESQL_USERNAME: root

      POSTGRESQL_PASSWORD: root

      POSTGRESQL_DATABASE: dolphinscheduler

      ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181

    healthcheck:

      test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"]

      interval: 30

      timeout: 5s

      retries: 3

      start_period: 30s

    volumes:

    - dolphinscheduler-api:/opt/dolphinscheduler/logs

    networks: 

    - dolphinscheduler

    deploy:

      mode: replicated

      replicas: 1



  dolphinscheduler-frontend:

    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev

    command: ["frontend"]

    ports:

    - 8888:8888

    environment:

      TZ: Asia/Shanghai

      FRONTEND_API_SERVER_HOST: dolphinscheduler-api

      FRONTEND_API_SERVER_PORT: 12345

    healthcheck:

      test: ["CMD", "nc", "-z", "localhost", "8888"]

      interval: 30

      timeout: 5s

      retries: 3

      start_period: 30s

    volumes:

    - dolphinscheduler-frontend:/var/log/nginx

    networks:

    - dolphinscheduler

    deploy:

      mode: replicated

      replicas: 1



  dolphinscheduler-alert:

    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev

    command: ["alert-server"]

    environment:

      TZ: Asia/Shanghai

      XLS_FILE_PATH: "/tmp/xls"

      MAIL_SERVER_HOST: ""

      MAIL_SERVER_PORT: ""

      MAIL_SENDER: ""

      MAIL_USER: ""

      MAIL_PASSWD: ""

      MAIL_SMTP_STARTTLS_ENABLE: "false"

      MAIL_SMTP_SSL_ENABLE: "false"

      MAIL_SMTP_SSL_TRUST: ""

      ENTERPRISE_WECHAT_ENABLE: "false"

      ENTERPRISE_WECHAT_CORP_ID: ""

      ENTERPRISE_WECHAT_SECRET: ""

      ENTERPRISE_WECHAT_AGENT_ID: ""

      ENTERPRISE_WECHAT_USERS: ""

      POSTGRESQL_HOST: dolphinscheduler-postgresql

      POSTGRESQL_PORT: 5432

      POSTGRESQL_USERNAME: root

      POSTGRESQL_PASSWORD: root

      POSTGRESQL_DATABASE: dolphinscheduler

    healthcheck:

      test: ["CMD", "/root/checkpoint.sh", "AlertServer"]

      interval: 30

      timeout: 5s

      retries: 3

      start_period: 30s

    volumes:

    - dolphinscheduler-alert:/opt/dolphinscheduler/logs

    networks: 

    - dolphinscheduler

    deploy:

      mode: replicated

      replicas: 1



  dolphinscheduler-master:

    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev

    command: ["master-server"]

    ports: 

    - 5678:5678

    environment: 

      TZ: Asia/Shanghai

      MASTER_EXEC_THREADS: "100"

      MASTER_EXEC_TASK_NUM: "20"

      MASTER_HEARTBEAT_INTERVAL: "10"

      MASTER_TASK_COMMIT_RETRYTIMES: "5"

      MASTER_TASK_COMMIT_INTERVAL: "1000"

      MASTER_MAX_CPULOAD_AVG: "100"

      MASTER_RESERVED_MEMORY: "0.1"

      POSTGRESQL_HOST: dolphinscheduler-postgresql

      POSTGRESQL_PORT: 5432

      POSTGRESQL_USERNAME: root

      POSTGRESQL_PASSWORD: root

      POSTGRESQL_DATABASE: dolphinscheduler

      ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181

    healthcheck:

      test: ["CMD", "/root/checkpoint.sh", "MasterServer"]

      interval: 30

      timeout: 5s

      retries: 3

      start_period: 30s

    volumes:

    - dolphinscheduler-master:/opt/dolphinscheduler/logs

    networks:

    - dolphinscheduler

    deploy:

      mode: replicated

      replicas: 1



  dolphinscheduler-worker:

    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev

    command: ["worker-server"]

    ports: 

    - 1234:1234

    - 50051:50051

    environment: 

      TZ: Asia/Shanghai

      WORKER_EXEC_THREADS: "100"

      WORKER_HEARTBEAT_INTERVAL: "10"

      WORKER_FETCH_TASK_NUM: "3"

      WORKER_MAX_CPULOAD_AVG: "100"

      WORKER_RESERVED_MEMORY: "0.1"

      WORKER_GROUP: "default"

      DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler"

      POSTGRESQL_HOST: dolphinscheduler-postgresql

      POSTGRESQL_PORT: 5432

      POSTGRESQL_USERNAME: root

      POSTGRESQL_PASSWORD: root

      POSTGRESQL_DATABASE: dolphinscheduler

      ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181

    healthcheck:

      test: ["CMD", "/root/checkpoint.sh", "WorkerServer"]

      interval: 30

      timeout: 5s

      retries: 3

      start_period: 30s

    volumes:

    - dolphinscheduler-worker-data:/tmp/dolphinscheduler

    - dolphinscheduler-worker-logs:/opt/dolphinscheduler/logs

    configs:

    - source: dolphinscheduler-worker-task-env

      target: /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh

    networks:

    - dolphinscheduler

    deploy:

      mode: replicated

      replicas: 1



networks:

  dolphinscheduler:

    driver: overlay



volumes:

  dolphinscheduler-postgresql:

  dolphinscheduler-zookeeper:

  dolphinscheduler-api:

  dolphinscheduler-frontend:

  dolphinscheduler-alert:

  dolphinscheduler-master:

  dolphinscheduler-worker-data:

  dolphinscheduler-worker-logs:



configs:

  dolphinscheduler-worker-task-env:

    file: ./dolphinscheduler_env.sh
 
 No newline at end of file

docker/docker-swarm/dolphinscheduler_env.sh

+3 −9
Original line number Diff line number Diff line
@@ -15,12 +15,6 @@ 
# limitations under the License.

#



export HADOOP_HOME=/opt/soft/hadoop

export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop

export SPARK_HOME1=/opt/soft/spark1

export SPARK_HOME2=/opt/soft/spark2

export PYTHON_HOME=/opt/soft/python

export JAVA_HOME=/opt/soft/java

export HIVE_HOME=/opt/soft/hive

export FLINK_HOME=/opt/soft/flink

export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$PATH
 
 No newline at end of file
 
export PYTHON_HOME=/usr/bin/python2

export JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk

export PATH=$PYTHON_HOME:$JAVA_HOME/bin:$PATH
 
 No newline at end of file

dockerfile/Dockerfile

+2 −1
Original line number Diff line number Diff line
@@ -27,7 +27,7 @@ ENV DEBIAN_FRONTEND noninteractive 
#If install slowly, you can replcae alpine's mirror with aliyun's mirror, Example:

#RUN sed -i "s/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g" /etc/apk/repositories

RUN apk update && \

    apk add dos2unix shadow bash openrc python sudo vim wget iputils net-tools openssh-server py2-pip tini && \

    apk add dos2unix shadow bash openrc python python3 sudo vim wget iputils net-tools openssh-server py2-pip tini && \

    apk add --update procps && \

    openrc boot && \

    pip install kazoo
@@ -67,6 +67,7 @@ ADD ./checkpoint.sh /root/checkpoint.sh 
ADD ./startup-init-conf.sh /root/startup-init-conf.sh

ADD ./startup.sh /root/startup.sh

ADD ./conf/dolphinscheduler/*.tpl /opt/dolphinscheduler/conf/

ADD ./conf/dolphinscheduler/logback/* /opt/dolphinscheduler/conf/

ADD conf/dolphinscheduler/env/dolphinscheduler_env.sh /opt/dolphinscheduler/conf/env/

RUN chmod +x /root/checkpoint.sh && \

    chmod +x /root/startup-init-conf.sh && \

dockerfile/conf/dolphinscheduler/env/dolphinscheduler_env.sh

+3 −9
Original line number Diff line number Diff line
@@ -15,12 +15,6 @@ 
# limitations under the License.

#



export HADOOP_HOME=/opt/soft/hadoop

export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop

export SPARK_HOME1=/opt/soft/spark1

export SPARK_HOME2=/opt/soft/spark2

export PYTHON_HOME=/opt/soft/python

export JAVA_HOME=/opt/soft/java

export HIVE_HOME=/opt/soft/hive

export FLINK_HOME=/opt/soft/flink

export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$PATH
 
export PYTHON_HOME=/usr/bin/python2

export JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk

export PATH=$PYTHON_HOME/bin:$JAVA_HOME/bin:$PATH