[Fix-3262][common] When you request the URL through applicationID to get the... (5584f0cb) · Commits · SUMMER2020 / students / proj-2002251

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java

+6 −1

Original line number	Diff line number	Diff line
		@@ -417,7 +417,12 @@ public class HadoopUtils implements Closeable {
		String applicationUrl = getApplicationUrl(applicationId);
		logger.info("applicationUrl={}", applicationUrl);

		String responseContent = HttpUtils.get(applicationUrl);
		String responseContent ;
		if (PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false)) {
		responseContent = KerberosHttpClient.get(applicationUrl);
		} else {
		responseContent = HttpUtils.get(applicationUrl);
		}
		if (responseContent != null) {
		ObjectNode jsonObject = JSONUtils.parseObject(responseContent);
		result = jsonObject.path("app").path("finalStatus").asText();

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java

+43 −33

Original line number	Diff line number	Diff line
		@@ -128,11 +128,21 @@ public class HttpUtils {
		CloseableHttpClient httpclient = HttpUtils.getInstance();

		HttpGet httpget = new HttpGet(url);
		return getResponseContentString(httpget,httpclient);
		}

		/**
		* get http response content
		*
		* @param httpget httpget
		* @param httpClient httpClient
		* @return http get request response content
		*/
		public static String getResponseContentString(HttpGet httpget, CloseableHttpClient httpClient) {
		String responseContent = null;
		CloseableHttpResponse response = null;

		try {
		response = httpclient.execute(httpget);
		response = httpClient.execute(httpget);
		// check response status is 200
		if (response.getStatusLine().getStatusCode() == 200) {
		HttpEntity entity = response.getEntity();
		@@ -144,8 +154,8 @@ public class HttpUtils {
		} else {
		logger.error("http get:{} response status code is not 200!", response.getStatusLine().getStatusCode());
		}
		}catch (Exception e){
		logger.error(e.getMessage(),e);
		} catch (IOException ioe) {
		logger.error(ioe.getMessage(), ioe);
		} finally {
		try {
		if (response != null) {
		@@ -155,11 +165,11 @@ public class HttpUtils {
		} catch (IOException e) {
		logger.error(e.getMessage(), e);
		}

		if (!httpget.isAborted()) {
		httpget.releaseConnection();
		httpget.abort();
		}

		}
		return responseContent;
		}

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/KerberosHttpClient.java

0 → 100644

+156 −0

Original line number	Diff line number	Diff line
		/*
		* Licensed to the Apache Software Foundation (ASF) under one or more
		* contributor license agreements. See the NOTICE file distributed with
		* this work for additional information regarding copyright ownership.
		* The ASF licenses this file to You under the Apache License, Version 2.0
		* (the "License"); you may not use this file except in compliance with
		* the License. You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/
		package org.apache.dolphinscheduler.common.utils;

		import org.apache.dolphinscheduler.common.Constants;
		import org.apache.http.auth.AuthSchemeProvider;
		import org.apache.http.auth.AuthScope;
		import org.apache.http.auth.Credentials;
		import org.apache.http.client.config.AuthSchemes;
		import org.apache.http.client.methods.HttpGet;
		import org.apache.http.config.Lookup;
		import org.apache.http.config.RegistryBuilder;
		import org.apache.http.impl.auth.SPNegoSchemeFactory;
		import org.apache.http.impl.client.BasicCredentialsProvider;
		import org.apache.http.impl.client.CloseableHttpClient;
		import org.apache.http.impl.client.HttpClientBuilder;
		import org.slf4j.Logger;
		import org.slf4j.LoggerFactory;

		import javax.security.auth.Subject;
		import javax.security.auth.kerberos.KerberosPrincipal;
		import javax.security.auth.login.AppConfigurationEntry;
		import javax.security.auth.login.Configuration;
		import javax.security.auth.login.LoginContext;
		import javax.security.auth.login.LoginException;
		import java.security.Principal;
		import java.security.PrivilegedAction;
		import java.util.HashMap;
		import java.util.HashSet;
		import java.util.Map;
		import java.util.Set;

		/**
		* kerberos http client
		*/
		public class KerberosHttpClient {

		public static final Logger logger = LoggerFactory.getLogger(KerberosHttpClient.class);

		private String principal;
		private String keyTabLocation;

		public KerberosHttpClient(String principal, String keyTabLocation) {
		super();
		this.principal = principal;
		this.keyTabLocation = keyTabLocation;
		}

		public KerberosHttpClient(String principal, String keyTabLocation, boolean isDebug) {
		this(principal, keyTabLocation);
		if (isDebug) {
		System.setProperty("sun.security.spnego.debug", "true");
		System.setProperty("sun.security.krb5.debug", "true");
		}
		}

		public KerberosHttpClient(String principal, String keyTabLocation, String krb5Location, boolean isDebug) {
		this(principal, keyTabLocation, isDebug);
		System.setProperty("java.security.krb5.conf", krb5Location);
		}

		private static CloseableHttpClient buildSpengoHttpClient() {
		HttpClientBuilder builder = HttpClientBuilder.create();
		Lookup<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create()
		.register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory(true)).build();
		builder.setDefaultAuthSchemeRegistry(authSchemeRegistry);
		BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider();
		credentialsProvider.setCredentials(new AuthScope(null, -1, null), new Credentials() {
		@Override
		public Principal getUserPrincipal() {
		return null;
		}

		@Override
		public String getPassword() {
		return null;
		}
		});
		builder.setDefaultCredentialsProvider(credentialsProvider);
		return builder.build();
		}

		public String get(final String url, final String userId) {
		logger.info("Calling KerberosHttpClient {} {} {}", this.principal, this.keyTabLocation, url);
		Configuration config = new Configuration() {
		@SuppressWarnings("serial")
		@Override
		public AppConfigurationEntry[] getAppConfigurationEntry(String name) {
		Map<String, Object> options = new HashMap<>(9);
		options.put("useTicketCache", "false");
		options.put("useKeyTab", "true");
		options.put("keyTab", keyTabLocation);
		options.put("refreshKrb5Config", "true");
		options.put("principal", principal);
		options.put("storeKey", "true");
		options.put("doNotPrompt", "true");
		options.put("isInitiator", "true");
		options.put("debug", "true");
		return new AppConfigurationEntry[] {
		new AppConfigurationEntry("com.sun.security.auth.module.Krb5LoginModule",
		AppConfigurationEntry.LoginModuleControlFlag.REQUIRED, options) };
		}
		};
		Set<Principal> princ = new HashSet<>(1);
		princ.add(new KerberosPrincipal(userId));
		Subject sub = new Subject(false, princ, new HashSet<>(), new HashSet<>());

		LoginContext lc;
		try {
		lc = new LoginContext("", sub, null, config);
		lc.login();
		Subject serviceSubject = lc.getSubject();
		return Subject.doAs(serviceSubject, (PrivilegedAction<String>) () -> {
		CloseableHttpClient httpClient = buildSpengoHttpClient();
		HttpGet httpget = new HttpGet(url);
		return HttpUtils.getResponseContentString(httpget, httpClient);
		});
		} catch (LoginException le) {
		logger.error("Kerberos authentication failed ", le);
		}
		return null;
		}

		/**
		* get http request content by kerberosClient
		*
		* @param url url
		* @return http get request response content
		*/
		public static String get(String url) {

		String responseContent;
		KerberosHttpClient kerberosHttpClient = new KerberosHttpClient(
		PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME),
		PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH),
		PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH), true);
		responseContent = kerberosHttpClient.get(url, PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME));
		return responseContent;

		}

		}

dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java

+44 −11

Original line number	Diff line number	Diff line
		@@ -17,7 +17,13 @@
		package org.apache.dolphinscheduler.common.utils;

		import com.fasterxml.jackson.databind.node.ObjectNode;

		import org.apache.dolphinscheduler.common.Constants;
		import org.apache.http.client.config.RequestConfig;
		import org.apache.http.client.methods.HttpGet;
		import org.apache.http.impl.client.CloseableHttpClient;
		import org.apache.http.impl.client.HttpClients;

		import org.junit.Assert;
		import org.junit.Test;
		import org.slf4j.Logger;
		@@ -28,9 +34,8 @@ import org.slf4j.LoggerFactory;
		*/
		public class HttpUtilsTest {


		public static final Logger logger = LoggerFactory.getLogger(HttpUtilsTest.class);

		private HadoopUtils hadoopUtils = HadoopUtils.getInstance();

		@Test
		public void testGetTest() {
		@@ -39,15 +44,43 @@ public class HttpUtilsTest {
		Assert.assertNotNull(result);
		ObjectNode jsonObject = JSONUtils.parseObject(result);
		Assert.assertEquals("GitHub", jsonObject.path("name").asText());

		result = HttpUtils.get("https://123.333.111.33/ccc");
		Assert.assertNull(result);
		}

		@Test
		public void testGetByKerberos() {
		try {
		String applicationUrl = hadoopUtils.getApplicationUrl("application_1542010131334_0029");
		String responseContent;
		responseContent = HttpUtils.get(applicationUrl);
		Assert.assertNull(responseContent);

		} catch (Exception e) {
		logger.error(e.getMessage(), e);
		}

		}

		@Test
		public void testGetResponseContentString() {
		CloseableHttpClient httpclient = HttpClients.createDefault();
		HttpGet httpget = new HttpGet("https://github.com/manifest.json");
		/** set timeout、request time、socket timeout */
		RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(Constants.HTTP_CONNECT_TIMEOUT)
		.setConnectionRequestTimeout(Constants.HTTP_CONNECTION_REQUEST_TIMEOUT)
		.setSocketTimeout(Constants.SOCKET_TIMEOUT).setRedirectsEnabled(true).build();
		httpget.setConfig(requestConfig);
		String responseContent = HttpUtils.getResponseContentString(httpget, httpclient);
		Assert.assertNotNull(responseContent);
		}


		@Test
		public void testGetHttpClient() {
		CloseableHttpClient httpClient1 = HttpUtils.getInstance();
		CloseableHttpClient httpClient2 = HttpUtils.getInstance();
		Assert.assertEquals(httpClient1, httpClient2);
		}

		}

dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/KerberosHttpClientTest.java

0 → 100644

+46 −0

Original line number	Diff line number	Diff line
		/*
		* Licensed to the Apache Software Foundation (ASF) under one or more
		* contributor license agreements. See the NOTICE file distributed with
		* this work for additional information regarding copyright ownership.
		* The ASF licenses this file to You under the Apache License, Version 2.0
		* (the "License"); you may not use this file except in compliance with
		* the License. You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/
		package org.apache.dolphinscheduler.common.utils;

		import org.apache.dolphinscheduler.common.Constants;
		import org.junit.Assert;
		import org.junit.Test;
		import org.slf4j.Logger;
		import org.slf4j.LoggerFactory;

		/**
		* KerberosHttpClient test
		*/
		public class KerberosHttpClientTest {
		public static final Logger logger = LoggerFactory.getLogger(KerberosHttpClientTest.class);
		private HadoopUtils hadoopUtils = HadoopUtils.getInstance();

		@Test
		public void get() {
		try {
		String applicationUrl = hadoopUtils.getApplicationUrl("application_1542010131334_0029");
		String responseContent;
		KerberosHttpClient kerberosHttpClient = new KerberosHttpClient(PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME),
		PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH), PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH), true);
		responseContent = kerberosHttpClient.get(applicationUrl,
		PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME));
		Assert.assertNull(responseContent);
		} catch (Exception e) {
		logger.error(e.getMessage(), e);
		}
		}
		}
		No newline at end of file