Unverified Commit 620ecffa authored by dailidong's avatar dailidong Committed by GitHub
Browse files

fix singe resource manager will NPE (#3013) 

* fix worker group config no effect

* remove codehaus janino jar
the license about janino maybe not compatiable with Apache v2

* Merge remote-tracking branch 'upstream/dev-1.3.0' into dev-1.3.0

# Conflicts:
#	dolphinscheduler-server/src/main/resources/config/install_config.conf

* datasource config

* Update datasource.properties

* fix RunConfig bug

* remove param monitor server state

* fix table T_DS_ALERT

* update h2 database

* fix #2910 master server will show exception for some time when it restart

* fix oom when no master is active

* fix worker oom when master server restart

* fix oom

* fix

* add UT

* fix worker group config no effect

* fix singe resource manager will NPE

* Update HadoopUtilsTest.java
parent 08beae83

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java

+10 −10
Original line number Diff line number Diff line
@@ -186,7 +186,7 @@ public class HadoopUtils implements Closeable { 
     * @param applicationId application id

     * @return url of application

     */

    public String getApplicationUrl(String applicationId) {

    public String getApplicationUrl(String applicationId) throws Exception {

        /**

         * if rmHaIds contains xx, it signs not use resourcemanager

         * otherwise:
@@ -194,21 +194,21 @@ public class HadoopUtils implements Closeable { 
         *  if rmHaIds not empty: resourcemanager HA enabled

         */

        String appUrl = "";

        //not use resourcemanager

        if (rmHaIds.contains(Constants.YARN_RESOURCEMANAGER_HA_XX)){



            yarnEnabled = false;

            logger.warn("should not step here");

        } else if (!StringUtils.isEmpty(rmHaIds)) {

        if (StringUtils.isEmpty(rmHaIds)){

            //single resourcemanager enabled

            appUrl = appAddress;

            yarnEnabled = true;

        } else {

            //resourcemanager HA enabled

            appUrl = getAppAddress(appAddress, rmHaIds);

            yarnEnabled = true;

            logger.info("application url : {}", appUrl);

        } else {

            //single resourcemanager enabled

            yarnEnabled = true;

        }



        if(StringUtils.isBlank(appUrl)){

            throw new Exception("application url is blank");

        }

        return String.format(appUrl, applicationId);

    }
@@ -402,7 +402,7 @@ public class HadoopUtils implements Closeable { 
     * @return the return may be null or there may be other parse exceptions

     * @throws JSONException json exception

     */

    public ExecutionStatus getApplicationStatus(String applicationId) throws JSONException {

    public ExecutionStatus getApplicationStatus(String applicationId) throws Exception {

        if (StringUtils.isEmpty(applicationId)) {

            return null;

        }

dolphinscheduler-common/src/main/resources/common.properties

+1 −1
Original line number Diff line number Diff line
@@ -54,7 +54,7 @@ fs.s3a.access.key=A3DXS30FO22544RE 
# if resource.storage.type=S3，s3 secret key

fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK



# if not use hadoop resourcemanager, please keep default value; if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty

# if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty

yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx



# if resourcemanager HA enable or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname.

dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HadoopUtilsTest.java

+2 −2
Original line number Diff line number Diff line
@@ -142,7 +142,7 @@ public class HadoopUtilsTest { 
    @Test

    public void isYarnEnabled() {

        boolean result = hadoopUtils.isYarnEnabled();

        Assert.assertEquals(false, result);

        Assert.assertEquals(true, result);

    }



    @Test
@@ -185,7 +185,7 @@ public class HadoopUtilsTest { 
    }



    @Test

    public void getApplicationUrl(){

    public void getApplicationUrl() throws Exception {

        String application_1516778421218_0042 = hadoopUtils.getApplicationUrl("application_1529051418016_0167");

        logger.info(application_1516778421218_0042);

    }

dolphinscheduler-server/src/main/resources/config/install_config.conf

+1 −1
Original line number Diff line number Diff line
@@ -87,7 +87,7 @@ s3Endpoint="http://192.168.xx.xx:9010" 
s3AccessKey="xxxxxxxxxx"

s3SecretKey="xxxxxxxxxx"



# if not use hadoop resourcemanager, please keep default value; if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty

# if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty

yarnHaIps="192.168.xx.xx,192.168.xx.xx"



# if resourcemanager HA enable or not use resourcemanager, please skip this value setting; If resourcemanager is single, you only need to replace yarnIp1 to actual resourcemanager hostname.