[Feature-2925][server] Init TaskLogger in TaskExecuteProcessor (#2925) (#2965) (38e48537) · Commits · SUMMER2020 / students / proj-2002008

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/AbstractShell.java

+26 −14

Original line number	Diff line number	Diff line
		@@ -16,9 +16,6 @@
		*/
		package org.apache.dolphinscheduler.common.shell;

		import org.slf4j.Logger;
		import org.slf4j.LoggerFactory;

		import java.io.BufferedReader;
		import java.io.File;
		import java.io.IOException;
		@@ -30,6 +27,9 @@ import java.util.TimerTask;
		import java.util.concurrent.ConcurrentHashMap;
		import java.util.concurrent.atomic.AtomicBoolean;

		import org.slf4j.Logger;
		import org.slf4j.LoggerFactory;


		/**
		* A base class for running a Unix command.
		@@ -153,11 +153,11 @@ public abstract class AbstractShell {
		timeOutTimer.schedule(timeoutTimerTask, timeOutInterval);
		}
		final BufferedReader errReader =
		new BufferedReader(new InputStreamReader(process
		.getErrorStream()));
		new BufferedReader(
		new InputStreamReader(process.getErrorStream()));
		BufferedReader inReader =
		new BufferedReader(new InputStreamReader(process
		.getInputStream()));
		new BufferedReader(
		new InputStreamReader(process.getInputStream()));
		final StringBuilder errMsg = new StringBuilder();

		// read error and input streams as this would free up the buffers
		@@ -177,23 +177,35 @@ public abstract class AbstractShell {
		}
		}
		};
		Thread inThread = new Thread() {
		@Override
		public void run() {
		try {
		parseExecResult(inReader);
		} catch (IOException ioe) {
		logger.warn("Error reading the in stream", ioe);
		}
		super.run();
		}
		};
		try {
		errThread.start();
		inThread.start();
		} catch (IllegalStateException ise) { }
		try {
		// parse the output
		parseExecResult(inReader);
		exitCode = process.waitFor();
		try {
		// make sure that the error thread exits
		// make sure that the error and in thread exits
		errThread.join();
		inThread.join();
		} catch (InterruptedException ie) {
		logger.warn("Interrupted while reading the error stream", ie);
		logger.warn("Interrupted while reading the error and in stream", ie);
		}
		completed.set(true);
		//the timeout thread handling
		//taken care in finally block
		if (exitCode != 0) {
		if (exitCode != 0 \|\| errMsg.length() > 0) {
		throw new ExitCodeException(exitCode, errMsg.toString());
		}
		} catch (InterruptedException ie) {

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java

+46 −14

Original line number	Diff line number	Diff line
		@@ -16,18 +16,32 @@
		*/
		package org.apache.dolphinscheduler.common.utils;

		import static org.apache.dolphinscheduler.common.Constants.DATA_BASEDIR_PATH;
		import static org.apache.dolphinscheduler.common.Constants.RESOURCE_VIEW_SUFFIXS;
		import static org.apache.dolphinscheduler.common.Constants.RESOURCE_VIEW_SUFFIXS_DEFAULT_VALUE;
		import static org.apache.dolphinscheduler.common.Constants.YYYYMMDDHHMMSS;

		import java.io.BufferedReader;
		import java.io.BufferedWriter;
		import java.io.ByteArrayOutputStream;
		import java.io.File;
		import java.io.FileOutputStream;
		import java.io.FileWriter;
		import java.io.IOException;
		import java.io.InputStream;
		import java.io.OutputStream;
		import java.io.StringReader;
		import java.io.UnsupportedEncodingException;
		import java.nio.charset.Charset;
		import java.nio.charset.UnsupportedCharsetException;
		import java.util.Optional;

		import org.apache.commons.io.Charsets;
		import org.apache.commons.io.IOUtils;
		import org.apache.commons.lang.StringUtils;
		import org.slf4j.Logger;
		import org.slf4j.LoggerFactory;

		import java.io.*;
		import java.nio.charset.Charset;
		import java.nio.charset.UnsupportedCharsetException;

		import static org.apache.dolphinscheduler.common.Constants.*;

		/**
		* file utils
		*/
		@@ -36,6 +50,8 @@ public class FileUtils {

		public static final String DATA_BASEDIR = PropertyUtils.getString(DATA_BASEDIR_PATH,"/tmp/dolphinscheduler");

		public static final ThreadLocal<Logger> taskLoggerThreadLocal = new ThreadLocal<>();

		/**
		* get file suffix
		*
		@@ -148,14 +164,30 @@ public class FileUtils {

		//create work dir
		org.apache.commons.io.FileUtils.forceMkdir(execLocalPathFile);
		logger.info("create dir success {}" , execLocalPath);

		String mkdirLog = "create dir success " + execLocalPath;
		LoggerUtils.logInfo(Optional.ofNullable(logger), mkdirLog);
		LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), mkdirLog);

		//if not exists this user,then create
		OSUtils.taskLoggerThreadLocal.set(taskLoggerThreadLocal.get());
		try {
		if (!OSUtils.getUserList().contains(userName)) {
		OSUtils.createUser(userName);
		boolean isSuccessCreateUser = OSUtils.createUser(userName);

		String infoLog;
		if (isSuccessCreateUser) {
		infoLog = String.format("create user name success %s", userName);
		} else {
		infoLog = String.format("create user name fail %s", userName);
		}
		LoggerUtils.logInfo(Optional.ofNullable(logger), infoLog);
		LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), infoLog);
		}
		} catch (Throwable e) {
		LoggerUtils.logError(Optional.ofNullable(logger), e);
		LoggerUtils.logError(Optional.ofNullable(taskLoggerThreadLocal.get()), e);
		}
		logger.info("create user name success {}", userName);
		OSUtils.taskLoggerThreadLocal.remove();
		}

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java

+24 −3

Original line number	Diff line number	Diff line
		@@ -16,14 +16,15 @@
		*/
		package org.apache.dolphinscheduler.common.utils;

		import org.apache.dolphinscheduler.common.Constants;
		import org.slf4j.Logger;

		import java.util.ArrayList;
		import java.util.List;
		import java.util.Optional;
		import java.util.regex.Matcher;
		import java.util.regex.Pattern;

		import org.apache.dolphinscheduler.common.Constants;
		import org.slf4j.Logger;

		/**
		* logger utils
		*/
		@@ -93,4 +94,24 @@ public class LoggerUtils {
		}
		return appIds;
		}

		public static void logError(Optional<Logger> optionalLogger
		, String error) {
		optionalLogger.ifPresent((Logger logger) -> logger.error(error));
		}

		public static void logError(Optional<Logger> optionalLogger
		, Throwable e) {
		optionalLogger.ifPresent((Logger logger) -> logger.error(e.getMessage(), e));
		}

		public static void logError(Optional<Logger> optionalLogger
		, String error, Throwable e) {
		optionalLogger.ifPresent((Logger logger) -> logger.error(error, e));
		}

		public static void logInfo(Optional<Logger> optionalLogger
		, String info) {
		optionalLogger.ifPresent((Logger logger) -> logger.info(info));
		}
		}
		No newline at end of file

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java

+61 −42

Original line number	Diff line number	Diff line
		@@ -16,16 +16,6 @@
		*/
		package org.apache.dolphinscheduler.common.utils;

		import org.apache.dolphinscheduler.common.Constants;
		import org.apache.dolphinscheduler.common.shell.ShellExecutor;
		import org.apache.commons.configuration.Configuration;
		import org.slf4j.Logger;
		import org.slf4j.LoggerFactory;
		import oshi.SystemInfo;
		import oshi.hardware.CentralProcessor;
		import oshi.hardware.GlobalMemory;
		import oshi.hardware.HardwareAbstractionLayer;

		import java.io.BufferedReader;
		import java.io.FileInputStream;
		import java.io.IOException;
		@@ -40,8 +30,21 @@ import java.util.ArrayList;
		import java.util.Arrays;
		import java.util.Collections;
		import java.util.List;
		import java.util.Optional;
		import java.util.StringTokenizer;
		import java.util.regex.Pattern;

		import org.apache.commons.configuration.Configuration;
		import org.apache.dolphinscheduler.common.Constants;
		import org.apache.dolphinscheduler.common.shell.ShellExecutor;
		import org.slf4j.Logger;
		import org.slf4j.LoggerFactory;

		import oshi.SystemInfo;
		import oshi.hardware.CentralProcessor;
		import oshi.hardware.GlobalMemory;
		import oshi.hardware.HardwareAbstractionLayer;

		/**
		* os utils
		*
		@@ -50,6 +53,8 @@ public class OSUtils {

		private static final Logger logger = LoggerFactory.getLogger(OSUtils.class);

		public static final ThreadLocal<Logger> taskLoggerThreadLocal = new ThreadLocal<>();

		private static final SystemInfo SI = new SystemInfo();
		public static final String TWO_DECIMAL = "0.00";

		@@ -251,7 +256,9 @@ public class OSUtils {
		try {
		String userGroup = OSUtils.getGroup();
		if (StringUtils.isEmpty(userGroup)) {
		logger.error("{} group does not exist for this operating system.", userGroup);
		String errorLog = String.format("%s group does not exist for this operating system.", userGroup);
		LoggerUtils.logError(Optional.ofNullable(logger), errorLog);
		LoggerUtils.logError(Optional.ofNullable(taskLoggerThreadLocal.get()), errorLog);
		return false;
		}
		if (isMacOS()) {
		@@ -263,7 +270,8 @@ public class OSUtils {
		}
		return true;
		} catch (Exception e) {
		logger.error(e.getMessage(), e);
		LoggerUtils.logError(Optional.ofNullable(logger), e);
		LoggerUtils.logError(Optional.ofNullable(taskLoggerThreadLocal.get()), e);
		}

		return false;
		@@ -276,10 +284,14 @@ public class OSUtils {
		* @throws IOException in case of an I/O error
		*/
		private static void createLinuxUser(String userName, String userGroup) throws IOException {
		logger.info("create linux os user : {}", userName);
		String cmd = String.format("sudo useradd -g %s %s", userGroup, userName);
		String infoLog1 = String.format("create linux os user : %s", userName);
		LoggerUtils.logInfo(Optional.ofNullable(logger), infoLog1);
		LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), infoLog1);

		logger.info("execute cmd : {}", cmd);
		String cmd = String.format("sudo useradd -g %s %s", userGroup, userName);
		String infoLog2 = String.format("execute cmd : %s", cmd);
		LoggerUtils.logInfo(Optional.ofNullable(logger), infoLog2);
		LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), infoLog2);
		OSUtils.exeCmd(cmd);
		}

		@@ -290,13 +302,24 @@ public class OSUtils {
		* @throws IOException in case of an I/O error
		*/
		private static void createMacUser(String userName, String userGroup) throws IOException {
		logger.info("create mac os user : {}", userName);
		String userCreateCmd = String.format("sudo sysadminctl -addUser %s -password %s", userName, userName);
		String appendGroupCmd = String.format("sudo dseditgroup -o edit -a %s -t user %s", userName, userGroup);

		logger.info("create user command : {}", userCreateCmd);
		OSUtils.exeCmd(userCreateCmd);
		logger.info("append user to group : {}", appendGroupCmd);
		Optional<Logger> optionalLogger = Optional.ofNullable(logger);
		Optional<Logger> optionalTaskLogger = Optional.ofNullable(taskLoggerThreadLocal.get());

		String infoLog1 = String.format("create mac os user : %s", userName);
		LoggerUtils.logInfo(optionalLogger, infoLog1);
		LoggerUtils.logInfo(optionalTaskLogger, infoLog1);

		String createUserCmd = String.format("sudo sysadminctl -addUser %s -password %s", userName, userName);
		String infoLog2 = String.format("create user command : %s", createUserCmd);
		LoggerUtils.logInfo(optionalLogger, infoLog2);
		LoggerUtils.logInfo(optionalTaskLogger, infoLog2);
		OSUtils.exeCmd(createUserCmd);

		String appendGroupCmd = String.format("sudo dseditgroup -o edit -a %s -t user %s", userName, userGroup);
		String infoLog3 = String.format("append user to group : %s", appendGroupCmd);
		LoggerUtils.logInfo(optionalLogger, infoLog3);
		LoggerUtils.logInfo(optionalTaskLogger, infoLog3);
		OSUtils.exeCmd(appendGroupCmd);
		}

		@@ -307,14 +330,20 @@ public class OSUtils {
		* @throws IOException in case of an I/O error
		*/
		private static void createWindowsUser(String userName, String userGroup) throws IOException {
		logger.info("create windows os user : {}", userName);
		String userCreateCmd = String.format("net user \"%s\" /add", userName);
		String appendGroupCmd = String.format("net localgroup \"%s\" \"%s\" /add", userGroup, userName);
		String infoLog1 = String.format("create windows os user : %s", userName);
		LoggerUtils.logInfo(Optional.ofNullable(logger), infoLog1);
		LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), infoLog1);

		logger.info("execute create user command : {}", userCreateCmd);
		String userCreateCmd = String.format("net user \"%s\" /add", userName);
		String infoLog2 = String.format("execute create user command : %s", userCreateCmd);
		LoggerUtils.logInfo(Optional.ofNullable(logger), infoLog2);
		LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), infoLog2);
		OSUtils.exeCmd(userCreateCmd);

		logger.info("execute append user to group : {}", appendGroupCmd);
		String appendGroupCmd = String.format("net localgroup \"%s\" \"%s\" /add", userGroup, userName);
		String infoLog3 = String.format("execute append user to group : %s", appendGroupCmd);
		LoggerUtils.logInfo(Optional.ofNullable(logger), infoLog3);
		LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), infoLog3);
		OSUtils.exeCmd(appendGroupCmd);
		}

		@@ -353,22 +382,12 @@ public class OSUtils {
		* @throws IOException errors
		*/
		public static String exeCmd(String command) throws IOException {
		BufferedReader br = null;

		try {
		Process p = Runtime.getRuntime().exec(command);
		br = new BufferedReader(new InputStreamReader(p.getInputStream()));
		String line;
		StringBuilder sb = new StringBuilder();

		while ((line = br.readLine()) != null) {
		sb.append(line + "\n");
		}

		return sb.toString();
		} finally {
		IOUtils.closeQuietly(br);
		StringTokenizer st = new StringTokenizer(command);
		String[] cmdArray = new String[st.countTokens()];
		for (int i = 0; st.hasMoreTokens(); i++) {
		cmdArray[i] = st.nextToken();
		}
		return exeShell(cmdArray);
		}

		/**
		@@ -377,7 +396,7 @@ public class OSUtils {
		* @return result of execute the shell
		* @throws IOException errors
		*/
		public static String exeShell(String command) throws IOException {
		public static String exeShell(String[] command) throws IOException {
		return ShellExecutor.execCommand(command);
		}

dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java

+5 −1

Original line number	Diff line number	Diff line
		@@ -68,7 +68,11 @@ public class OSUtilsTest {
		@Test
		public void createUser() {
		boolean result = OSUtils.createUser("test123");
		Assert.assertTrue(result);
		if (result) {
		Assert.assertTrue("create user test123 success", true);
		} else {
		Assert.assertTrue("create user test123 fail", true);
		}
		}

		@Test