Commit d16500e8 authored by bao liang's avatar bao liang Committed by lgcareer
Browse files

[Feature] merge some configurations #1635 (#1636) 

* merge hadoop.properties into common.properties

* merge hadoop,zookeeper.properties into common.properties
remove combined.properties/master.properties/worker.properties

* change db user/pwd to test/test
parent 426c027d

dolphinscheduler-api/src/main/resources/application-combined.properties

deleted100644 → 0
+0 −40
Original line number Diff line number Diff line 
#

# Licensed to the Apache Software Foundation (ASF) under one or more

# contributor license agreements.  See the NOTICE file distributed with

# this work for additional information regarding copyright ownership.

# The ASF licenses this file to You under the Apache License, Version 2.0

# (the "License"); you may not use this file except in compliance with

# the License.  You may obtain a copy of the License at

#

#     http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

#



logging.config=classpath:combined_logback.xml



# server port

server.port=12345



# session config

server.servlet.session.timeout=7200



server.servlet.context-path=/dolphinscheduler/



# file size limit for upload

spring.servlet.multipart.max-file-size=1024MB

spring.servlet.multipart.max-request-size=1024MB



#post content

server.jetty.max-http-post-size=5000000



spring.messages.encoding=UTF-8



#i18n classpath folder , file prefix messages， if have many files, use "," seperator

spring.messages.basename=i18n/messages



server.is-combined-server=true

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java

+4 −17
Original line number Diff line number Diff line
@@ -26,25 +26,10 @@ import java.util.regex.Pattern; 
 */

public final class Constants {



    /**

     * zookeeper properties path

     */

    public static final String ZOOKEEPER_PROPERTIES_PATH = "zookeeper.properties";



    /**

     * hadoop properties path

     */

    public static final String HADOOP_PROPERTIES_PATH = "/common/hadoop/hadoop.properties";



    /**

     * common properties path

     */

    public static final String COMMON_PROPERTIES_PATH = "/common/common.properties";



    /**

     * dao properties path

     */

    public static final String DAO_PROPERTIES_PATH = "application.properties";

    public static final String COMMON_PROPERTIES_PATH = "/common.properties";



    /**

     * fs.defaultFS
@@ -227,6 +212,8 @@ public final class Constants { 
    public static final String ZOOKEEPER_CONNECTION_TIMEOUT = "zookeeper.connection.timeout";



    public static final String ZOOKEEPER_RETRY_SLEEP = "zookeeper.retry.sleep";

    public static final String ZOOKEEPER_RETRY_BASE_SLEEP = "zookeeper.retry.base.sleep";

    public static final String ZOOKEEPER_RETRY_MAX_SLEEP = "zookeeper.retry.max.sleep";



    public static final String ZOOKEEPER_RETRY_MAXTIME = "zookeeper.retry.maxtime";
@@ -469,7 +456,7 @@ public final class Constants { 
    /**

     * task record configuration path

     */

    public static final String APPLICATION_PROPERTIES = "application-dao.properties";

    public static final String APPLICATION_PROPERTIES = "application.properties";



    public static final String TASK_RECORD_URL = "task.record.datasource.url";

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java

+1 −2
Original line number Diff line number Diff line
@@ -29,7 +29,6 @@ import java.util.Map; 
import java.util.Properties;



import static org.apache.dolphinscheduler.common.Constants.COMMON_PROPERTIES_PATH;

import static org.apache.dolphinscheduler.common.Constants.HADOOP_PROPERTIES_PATH;



/**

 * property utils
@@ -51,7 +50,7 @@ public class PropertyUtils { 
    }



    private void init(){

        String[] propertyFiles = new String[]{HADOOP_PROPERTIES_PATH,COMMON_PROPERTIES_PATH};

        String[] propertyFiles = new String[]{COMMON_PROPERTIES_PATH};

        for (String fileName : propertyFiles) {

            InputStream fis = null;

            try {

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperConfig.java

+1 −1
Original line number Diff line number Diff line
@@ -24,7 +24,7 @@ import org.springframework.stereotype.Component; 
 * zookeeper conf

 */

@Component

@PropertySource("classpath:zookeeper.properties")

@PropertySource("classpath:common.properties")

public class ZookeeperConfig {



    //zk connect config

dolphinscheduler-common/src/main/resources/common/common.properties→dolphinscheduler-common/src/main/resources/common.properties

+43 −8
Original line number Diff line number Diff line
@@ -18,14 +18,21 @@ 
#task queue implementation, default "zookeeper"

dolphinscheduler.queue.impl=zookeeper



# user data directory path, self configuration, please make sure the directory exists and have read write permissions

data.basedir.path=/tmp/dolphinscheduler

#zookeeper cluster.  multiple are separated by commas.  eg. 192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181

zookeeper.quorum=localhost:2181



# directory path for user data download. self configuration, please make sure the directory exists and have read write permissions

data.download.basedir.path=/tmp/dolphinscheduler/download

#dolphinscheduler root directory

zookeeper.dolphinscheduler.root=/dolphinscheduler



# process execute directory. self configuration, please make sure the directory exists and have read write permissions

process.exec.basepath=/tmp/dolphinscheduler/exec

#dolphinscheduler failover directory

zookeeper.session.timeout=300

zookeeper.connection.timeout=300

zookeeper.retry.base.sleep=100

zookeeper.retry.max.sleep=30000

zookeeper.retry.maxtime=5



# resource upload startup type : HDFS,S3,NONE

res.upload.startup.type=NONE



# Users who have permission to create directories under the HDFS root path

hdfs.root.user=hdfs
@@ -33,8 +40,15 @@ hdfs.root.user=hdfs 
# data base dir, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended

data.store2hdfs.basepath=/dolphinscheduler



# resource upload startup type : HDFS,S3,NONE

res.upload.startup.type=NONE

# user data directory path, self configuration, please make sure the directory exists and have read write permissions

data.basedir.path=/tmp/dolphinscheduler



# directory path for user data download. self configuration, please make sure the directory exists and have read write permissions

data.download.basedir.path=/tmp/dolphinscheduler/download



# process execute directory. self configuration, please make sure the directory exists and have read write permissions

process.exec.basepath=/tmp/dolphinscheduler/exec





# whether kerberos starts

hadoop.security.authentication.startup.state=false
@@ -57,3 +71,24 @@ resource.view.suffixs=txt,log,sh,conf,cfg,py,java,sql,hql,xml,properties 
# is development state? default "false"

development.state=true





# ha or single namenode,If namenode ha needs to copy core-site.xml and hdfs-site.xml

# to the conf directory，support s3，for example : s3a://dolphinscheduler

fs.defaultFS=hdfs://mycluster:8020



# s3 need，s3 endpoint

fs.s3a.endpoint=http://192.168.199.91:9010



# s3 need，s3 access key

fs.s3a.access.key=A3DXS30FO22544RE



# s3 need，s3 secret key

fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK



#resourcemanager ha note this need ips , this empty if single

yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx



# If it is a single resourcemanager, you only need to configure one host name. If it is resourcemanager HA, the default configuration is fine

yarn.application.status.address=http://ark1:8088/ws/v1/cluster/apps/%s