Commit c7f32e81 authored by qiaozhanwei's avatar qiaozhanwei Committed by lgcareer
Browse files

dockerfile modify (#1800) 

* fix #1775 bug，delete process definition when process instance is running

* revert CONTRIBUTING.md

* dockerfile modify

* dockerfile modify

* dockerfile modify
parent 8c724d1f

dockerfile/Dockerfile

+11 −36
Original line number Diff line number Diff line
@@ -61,28 +61,13 @@ ADD ./dockerfile/conf/maven/settings.xml /opt/maven/conf 
ENV MAVEN_HOME=/opt/maven

ENV PATH $PATH:$MAVEN_HOME/bin



#4,install node

RUN cd /opt && \

    wget https://nodejs.org/download/release/v8.9.4/node-v8.9.4-linux-x64.tar.gz && \

    tar -zxvf node-v8.9.4-linux-x64.tar.gz && \

    mv node-v8.9.4-linux-x64 node && \

    rm -rf ./node-v8.9.4-*tar.gz

ENV NODE_HOME=/opt/node

ENV PATH $PATH:$NODE_HOME/bin



#5,install postgresql

#4,install postgresql

RUN apt-get update && \

    apt-get install -y postgresql postgresql-contrib sudo && \

    sed -i 's/localhost/*/g' /etc/postgresql/10/main/postgresql.conf



#6,install nginx

RUN apt-get update && \

  apt-get install -y nginx && \

  rm -rf /var/lib/apt/lists/* && \

  echo "\ndaemon off;" >> /etc/nginx/nginx.conf && \

  chown -R www-data:www-data /var/lib/nginx



#7,install sudo,python,vim,ping and ssh command

#5,install sudo,python,vim,ping and ssh command

RUN apt-get update && \

  apt-get -y install sudo && \

  apt-get -y install python && \
@@ -90,40 +75,30 @@ RUN apt-get update && \ 
  apt-get -y install iputils-ping && \

  apt-get -y install net-tools && \

  apt-get -y install openssh-server && \

  apt-get -y install python-pip && \

  pip install kazoo

  apt-get -y install python-pip



#8,add dolphinscheduler source code to /opt/dolphinscheduler_source

#6,add dolphinscheduler source code to /opt/dolphinscheduler_source

ADD . /opt/dolphinscheduler_source





#9,backend compilation

#7,backend compilation

RUN cd /opt/dolphinscheduler_source && \

    mvn clean package -Prelease -Dmaven.test.skip=true



#10,frontend compilation

RUN chmod -R 777 /opt/dolphinscheduler_source/dolphinscheduler-ui && \

    cd /opt/dolphinscheduler_source/dolphinscheduler-ui && \

    rm -rf /opt/dolphinscheduler_source/dolphinscheduler-ui/node_modules && \

    npm install node-sass --unsafe-perm && \

    npm install && \

    npm run build



#11,modify dolphinscheduler configuration file

#8,modify dolphinscheduler configuration file

#backend configuration

RUN tar -zxvf /opt/dolphinscheduler_source/dolphinscheduler-dist/dolphinscheduler-backend/target/apache-dolphinscheduler-incubating-${tar_version}-dolphinscheduler-backend-bin.tar.gz -C /opt && \

    mv /opt/apache-dolphinscheduler-incubating-${tar_version}-dolphinscheduler-backend-bin /opt/dolphinscheduler && \

RUN tar -zxvf /opt/dolphinscheduler_source/dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${tar_version}-dolphinscheduler-bin.tar.gz -C /opt && \

    mv /opt/apache-dolphinscheduler-incubating-${tar_version}-dolphinscheduler-bin /opt/dolphinscheduler && \

    rm -rf /opt/dolphinscheduler/conf



ADD ./dockerfile/conf/dolphinscheduler/conf /opt/dolphinscheduler/conf

#frontend nginx configuration

ADD ./dockerfile/conf/nginx/dolphinscheduler.conf /etc/nginx/conf.d



#12,open port



#9,open port

EXPOSE 2181 2888 3888 3306 80 12345 8888



COPY ./dockerfile/startup.sh /root/startup.sh

#13,modify permissions and set soft links

#10,modify permissions and set soft links

RUN chmod +x /root/startup.sh && \

  chmod +x /opt/dolphinscheduler/script/create-dolphinscheduler.sh && \

  chmod +x /opt/zookeeper/bin/zkServer.sh && \

dockerfile/conf/dolphinscheduler/conf/application-dao.properties→dockerfile/conf/dolphinscheduler/conf/application.properties

+44 −2
Original line number Diff line number Diff line
@@ -17,9 +17,12 @@ 


# base spring data source configuration

spring.datasource.type=com.alibaba.druid.pool.DruidDataSource

# postgresql

# postgre

spring.datasource.driver-class-name=org.postgresql.Driver

spring.datasource.url=jdbc:postgresql://127.0.0.1:5432/dolphinscheduler

# mysql

#spring.datasource.driver-class-name=com.mysql.jdbc.Driver

#spring.datasource.url=jdbc:mysql://192.168.xx.xx:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8

spring.datasource.username=root

spring.datasource.password=root@123
@@ -92,8 +95,47 @@ mybatis-plus.configuration.cache-enabled=false 
mybatis-plus.configuration.call-setters-on-nulls=true

mybatis-plus.configuration.jdbc-type-for-null=null



# master settings

# master execute thread num

master.exec.threads=100



# master execute task number in parallel

master.exec.task.num=20



# master heartbeat interval

master.heartbeat.interval=10



# master commit task retry times

master.task.commit.retryTimes=5



# master commit task interval

master.task.commit.interval=1000





# only less than cpu avg load, master server can work. default value : the number of cpu cores * 2

master.max.cpuload.avg=100



# only larger than reserved memory, master server can work. default value : physical memory * 1/10, unit is G.

master.reserved.memory=0.1



# worker settings

# worker execute thread num

worker.exec.threads=100



# worker heartbeat interval

worker.heartbeat.interval=10



# submit the number of tasks at a time

worker.fetch.task.num = 3



# only less than cpu avg load, worker server can work. default value : the number of cpu cores * 2

worker.max.cpuload.avg=100



# only larger than reserved memory, worker server can work. default value : physical memory * 1/6, unit is G.

worker.reserved.memory=0.1



# data quality analysis is not currently in use. please ignore the following configuration

# task record flag

# task record

task.record.flag=false

task.record.datasource.url=jdbc:mysql://192.168.xx.xx:3306/etl?characterEncoding=UTF-8

task.record.datasource.username=xx

dockerfile/conf/dolphinscheduler/conf/common/common.properties→dockerfile/conf/dolphinscheduler/conf/common.properties

+44 −9
Original line number Diff line number Diff line
@@ -18,14 +18,21 @@ 
#task queue implementation, default "zookeeper"

dolphinscheduler.queue.impl=zookeeper



# user data directory path, self configuration, please make sure the directory exists and have read write permissions

data.basedir.path=/tmp/dolphinscheduler

#zookeeper cluster.  multiple are separated by commas.  eg. 192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181

zookeeper.quorum=127.0.0.1:2181



# directory path for user data download. self configuration, please make sure the directory exists and have read write permissions

data.download.basedir.path=/tmp/dolphinscheduler/download

#dolphinscheduler root directory

zookeeper.dolphinscheduler.root=/dolphinscheduler



# process execute directory. self configuration, please make sure the directory exists and have read write permissions

process.exec.basepath=/tmp/dolphinscheduler/exec

#dolphinscheduler failover directory

zookeeper.session.timeout=300

zookeeper.connection.timeout=300

zookeeper.retry.base.sleep=100

zookeeper.retry.max.sleep=30000

zookeeper.retry.maxtime=5



# resource upload startup type : HDFS,S3,NONE

res.upload.startup.type=NONE



# Users who have permission to create directories under the HDFS root path

hdfs.root.user=hdfs
@@ -33,8 +40,15 @@ hdfs.root.user=hdfs 
# data base dir, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended

data.store2hdfs.basepath=/dolphinscheduler



# resource upload startup type : HDFS,S3,NONE

res.upload.startup.type=NONE

# user data directory path, self configuration, please make sure the directory exists and have read write permissions

data.basedir.path=/tmp/dolphinscheduler



# directory path for user data download. self configuration, please make sure the directory exists and have read write permissions

data.download.basedir.path=/tmp/dolphinscheduler/download



# process execute directory. self configuration, please make sure the directory exists and have read write permissions

process.exec.basepath=/tmp/dolphinscheduler/exec





# whether kerberos starts

hadoop.security.authentication.startup.state=false
@@ -52,8 +66,29 @@ login.user.keytab.path=/opt/hdfs.headless.keytab 
dolphinscheduler.env.path=/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh



#resource.view.suffixs

resource.view.suffixs=txt,log,sh,conf,cfg,py,java,sql,hql,xml

resource.view.suffixs=txt,log,sh,conf,cfg,py,java,sql,hql,xml,properties



# is development state? default "false"

development.state=true





# ha or single namenode,If namenode ha needs to copy core-site.xml and hdfs-site.xml

# to the conf directory，support s3，for example : s3a://dolphinscheduler

fs.defaultFS=hdfs://mycluster:8020



# s3 need，s3 endpoint

fs.s3a.endpoint=http://192.168.199.91:9010



# s3 need，s3 access key

fs.s3a.access.key=A3DXS30FO22544RE



# s3 need，s3 secret key

fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK



#resourcemanager ha note this need ips , this empty if single

yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx



# If it is a single resourcemanager, you only need to configure one host name. If it is resourcemanager HA, the default configuration is fine

yarn.application.status.address=http://ark1:8088/ws/v1/cluster/apps/%s

dockerfile/conf/dolphinscheduler/conf/common/hadoop/hadoop.properties

deleted100644 → 0
+0 −35
Original line number Diff line number Diff line 
#

# Licensed to the Apache Software Foundation (ASF) under one or more

# contributor license agreements.  See the NOTICE file distributed with

# this work for additional information regarding copyright ownership.

# The ASF licenses this file to You under the Apache License, Version 2.0

# (the "License"); you may not use this file except in compliance with

# the License.  You may obtain a copy of the License at

#

#     http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

#



# ha or single namenode,If namenode ha needs to copy core-site.xml and hdfs-site.xml

# to the conf directory，support s3，for example : s3a://dolphinscheduler

fs.defaultFS=hdfs://mycluster:8020



# s3 need，s3 endpoint

fs.s3a.endpoint=http://192.168.199.91:9010



# s3 need，s3 access key

fs.s3a.access.key=A3DXS30FO22544RE



# s3 need，s3 secret key

fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK



#resourcemanager ha note this need ips , this empty if single

yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx



# If it is a single resourcemanager, you only need to configure one host name. If it is resourcemanager HA, the default configuration is fine

yarn.application.status.address=http://ark1:8088/ws/v1/cluster/apps/%s
 
 No newline at end of file

dockerfile/conf/dolphinscheduler/conf/config/install_config.conf

+5 −0
Original line number Diff line number Diff line
@@ -18,3 +18,8 @@ 
installPath=/data1_1T/dolphinscheduler

deployUser=dolphinscheduler

ips=ark0,ark1,ark2,ark3,ark4

sshPort=22

masters=ark0,ark1

workers=ark2,ark3,ark4

alertServer=ark3

apiServers=ark1