Commit fd76f172 authored by lenboo's avatar lenboo
Browse files

Merge remote-tracking branch 'upstream/dev' into dev-merge 

# Conflicts:
#	dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java
parents cb6c7118 38e48537

.github/workflows/ci_ut.yml

+6 −2
Original line number Diff line number Diff line
@@ -67,10 +67,14 @@ jobs: 
      - name: Upload coverage report to codecov

        run: |

          CODECOV_TOKEN="09c2663f-b091-4258-8a47-c981827eb29a" bash <(curl -s https://codecov.io/bash)

      # Set up JDK 11 for SonarCloud.

      - name: Set up JDK 1.11

        uses: actions/setup-java@v1

        with:

          java-version: 1.11

      - name: Run SonarCloud Analysis

        run: >

          mvn verify --batch-mode

          org.sonarsource.scanner.maven:sonar-maven-plugin:3.6.1.1688:sonar

          mvn --batch-mode verify sonar:sonar

          -Dsonar.coverage.jacoco.xmlReportPaths=target/site/jacoco/jacoco.xml

          -Dmaven.test.skip=true

          -Dsonar.host.url=https://sonarcloud.io

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java

+1 −1
Original line number Diff line number Diff line
@@ -63,7 +63,7 @@ public class ResourcesController extends BaseController { 
    private UdfFuncService udfFuncService;



    /**

     * create resource

     * create directory

     *

     * @param loginUser login user

     * @param alias alias

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java

+1 −1
Original line number Diff line number Diff line
@@ -115,7 +115,7 @@ public class CheckUtils { 
   *

   * @param parameter parameter

   * @param taskType task type

   * @return true if taks node parameters are valid, otherwise return false

   * @return true if task node parameters are valid, otherwise return false

   */

  public static boolean checkTaskNodeParameters(String parameter, String taskType) {

    AbstractParameters abstractParameters = TaskParametersUtils.getParameters(taskType, parameter);

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/AbstractShell.java

+26 −14
Original line number Diff line number Diff line
@@ -16,9 +16,6 @@ 
 */

package org.apache.dolphinscheduler.common.shell;



import org.slf4j.Logger;

import org.slf4j.LoggerFactory;



import java.io.BufferedReader;

import java.io.File;

import java.io.IOException;
@@ -30,6 +27,9 @@ import java.util.TimerTask; 
import java.util.concurrent.ConcurrentHashMap;

import java.util.concurrent.atomic.AtomicBoolean;



import org.slf4j.Logger;

import org.slf4j.LoggerFactory;





/** 

 * A base class for running a Unix command.
@@ -153,11 +153,11 @@ public abstract class AbstractShell { 
      timeOutTimer.schedule(timeoutTimerTask, timeOutInterval);

    }

    final BufferedReader errReader = 

            new BufferedReader(new InputStreamReader(process

                                                     .getErrorStream()));

            new BufferedReader(

                    new InputStreamReader(process.getErrorStream()));

    BufferedReader inReader =

            new BufferedReader(new InputStreamReader(process

                                                     .getInputStream()));

            new BufferedReader(

                    new InputStreamReader(process.getInputStream()));

    final StringBuilder errMsg = new StringBuilder();

    

    // read error and input streams as this would free up the buffers
@@ -177,23 +177,35 @@ public abstract class AbstractShell { 
        }

      }

    };

    Thread inThread = new Thread() {

      @Override

      public void run() {

        try {

          parseExecResult(inReader);

        } catch (IOException ioe) {

          logger.warn("Error reading the in stream", ioe);

        }

        super.run();

      }

    };

    try {

      errThread.start();

      inThread.start();

    } catch (IllegalStateException ise) { }

    try {

      // parse the output

      parseExecResult(inReader);

      exitCode = process.waitFor();

      try {

        // make sure that the error thread exits

        // make sure that the error and in thread exits

        errThread.join();

        inThread.join();

      } catch (InterruptedException ie) {

        logger.warn("Interrupted while reading the error stream", ie);

        logger.warn("Interrupted while reading the error and in stream", ie);

      }

      completed.set(true);

      //the timeout thread handling

      //taken care in finally block

      if (exitCode != 0) {

      if (exitCode != 0 || errMsg.length() > 0) {

        throw new ExitCodeException(exitCode, errMsg.toString());

      }

    } catch (InterruptedException ie) {

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java

+46 −14
Original line number Diff line number Diff line
@@ -16,18 +16,32 @@ 
 */

package org.apache.dolphinscheduler.common.utils;



import static org.apache.dolphinscheduler.common.Constants.DATA_BASEDIR_PATH;

import static org.apache.dolphinscheduler.common.Constants.RESOURCE_VIEW_SUFFIXS;

import static org.apache.dolphinscheduler.common.Constants.RESOURCE_VIEW_SUFFIXS_DEFAULT_VALUE;

import static org.apache.dolphinscheduler.common.Constants.YYYYMMDDHHMMSS;



import java.io.BufferedReader;

import java.io.BufferedWriter;

import java.io.ByteArrayOutputStream;

import java.io.File;

import java.io.FileOutputStream;

import java.io.FileWriter;

import java.io.IOException;

import java.io.InputStream;

import java.io.OutputStream;

import java.io.StringReader;

import java.io.UnsupportedEncodingException;

import java.nio.charset.Charset;

import java.nio.charset.UnsupportedCharsetException;

import java.util.Optional;



import org.apache.commons.io.Charsets;

import org.apache.commons.io.IOUtils;

import org.apache.commons.lang.StringUtils;

import org.slf4j.Logger;

import org.slf4j.LoggerFactory;



import java.io.*;

import java.nio.charset.Charset;

import java.nio.charset.UnsupportedCharsetException;



import static org.apache.dolphinscheduler.common.Constants.*;



/**

 * file utils

 */
@@ -36,6 +50,8 @@ public class FileUtils { 


    public static final String DATA_BASEDIR = PropertyUtils.getString(DATA_BASEDIR_PATH,"/tmp/dolphinscheduler");



    public static final ThreadLocal<Logger> taskLoggerThreadLocal = new ThreadLocal<>();



    /**

     * get file suffix

     *
@@ -148,14 +164,30 @@ public class FileUtils { 


        //create work dir

        org.apache.commons.io.FileUtils.forceMkdir(execLocalPathFile);

        logger.info("create dir success {}" , execLocalPath);



        String mkdirLog = "create dir success " + execLocalPath;

        LoggerUtils.logInfo(Optional.ofNullable(logger), mkdirLog);

        LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), mkdirLog);



        //if not exists this user,then create

        OSUtils.taskLoggerThreadLocal.set(taskLoggerThreadLocal.get());

        try {

            if (!OSUtils.getUserList().contains(userName)) {

            OSUtils.createUser(userName);

                boolean isSuccessCreateUser = OSUtils.createUser(userName);



                String infoLog;

                if (isSuccessCreateUser) {

                    infoLog = String.format("create user name success %s", userName);

                } else {

                    infoLog = String.format("create user name fail %s", userName);

                }

                LoggerUtils.logInfo(Optional.ofNullable(logger), infoLog);

                LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), infoLog);

            }

        } catch (Throwable e) {

            LoggerUtils.logError(Optional.ofNullable(logger), e);

            LoggerUtils.logError(Optional.ofNullable(taskLoggerThreadLocal.get()), e);

        }

        logger.info("create user name success {}", userName);

        OSUtils.taskLoggerThreadLocal.remove();

    }