[Improvement 11072][common] Added HTTPS request support in kerberos environment (#11127)

* [Improvement-11072][dolphinscheduler-common] Added HTTPS request support in kerberos environment

* [Improvement-11072][dolphinscheduler-common] supplement

Co-authored-by: liyangyang <liyangyang@bizseer.com>

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java

@@ -32,6 +32,7 @@ import org.apache.http.conn.socket.PlainConnectionSocketFactory;
 import org.apache.http.conn.ssl.NoopHostnameVerifier;
 import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
 import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.impl.client.HttpClients;
 import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
 import org.apache.http.util.EntityUtils;
@@ -66,9 +67,12 @@ public class HttpUtils {
     }
 
     private static class HttpClientInstance {
-        private static final CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(cm).setDefaultRequestConfig(requestConfig).build();
+        private static final CloseableHttpClient httpClient = getHttpClientBuilder().build();
     }
 
+    public static HttpClientBuilder getHttpClientBuilder() {
+        return HttpClients.custom().setConnectionManager(cm).setDefaultRequestConfig(requestConfig);
+    }
 
     private static PoolingHttpClientConnectionManager cm;
 
@@ -98,7 +102,7 @@ public class HttpUtils {
     static {
         try {
             ctx = SSLContext.getInstance(SSLConnectionSocketFactory.TLS);
-            ctx.init(null, new TrustManager[]{xtm}, null);
+            ctx.init(null, new TrustManager[] {xtm}, null);
         } catch (NoSuchAlgorithmException e) {
             logger.error("SSLContext init with NoSuchAlgorithmException", e);
         } catch (KeyManagementException e) {
@@ -107,14 +111,14 @@ public class HttpUtils {
         socketFactory = new SSLConnectionSocketFactory(ctx, NoopHostnameVerifier.INSTANCE);
         /** set timeout、request time、socket timeout */
         requestConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.IGNORE_COOKIES)
-                .setExpectContinueEnabled(Boolean.TRUE)
-                .setTargetPreferredAuthSchemes(Arrays.asList(AuthSchemes.NTLM, AuthSchemes.DIGEST))
-                .setProxyPreferredAuthSchemes(Arrays.asList(AuthSchemes.BASIC))
-                .setConnectTimeout(Constants.HTTP_CONNECT_TIMEOUT).setSocketTimeout(Constants.SOCKET_TIMEOUT)
-                .setConnectionRequestTimeout(Constants.HTTP_CONNECTION_REQUEST_TIMEOUT).setRedirectsEnabled(true)
-                .build();
+            .setExpectContinueEnabled(Boolean.TRUE)
+            .setTargetPreferredAuthSchemes(Arrays.asList(AuthSchemes.NTLM, AuthSchemes.DIGEST, AuthSchemes.SPNEGO))
+            .setProxyPreferredAuthSchemes(Arrays.asList(AuthSchemes.BASIC, AuthSchemes.SPNEGO))
+            .setConnectTimeout(Constants.HTTP_CONNECT_TIMEOUT).setSocketTimeout(Constants.SOCKET_TIMEOUT)
+            .setConnectionRequestTimeout(Constants.HTTP_CONNECTION_REQUEST_TIMEOUT).setRedirectsEnabled(true)
+            .build();
         socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
-                .register("http", PlainConnectionSocketFactory.INSTANCE).register("https", socketFactory).build();
+            .register("http", PlainConnectionSocketFactory.INSTANCE).register("https", socketFactory).build();
         cm = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
         cm.setDefaultMaxPerRoute(60);
         cm.setMaxTotal(100);
@@ -137,7 +141,7 @@ public class HttpUtils {
     /**
      * get http response content
      *
-     * @param httpget httpget
+     * @param httpget    httpget
      * @param httpClient httpClient
      * @return http get request response content
      */

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/KerberosHttpClient.java

@@ -19,14 +19,9 @@ package org.apache.dolphinscheduler.common.utils;
 
 import org.apache.dolphinscheduler.common.Constants;
 
-import org.apache.http.auth.AuthSchemeProvider;
 import org.apache.http.auth.AuthScope;
 import org.apache.http.auth.Credentials;
-import org.apache.http.client.config.AuthSchemes;
 import org.apache.http.client.methods.HttpGet;
-import org.apache.http.config.Lookup;
-import org.apache.http.config.RegistryBuilder;
-import org.apache.http.impl.auth.SPNegoSchemeFactory;
 import org.apache.http.impl.client.BasicCredentialsProvider;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClientBuilder;
@@ -56,6 +51,7 @@ public class KerberosHttpClient {
 
     private String principal;
     private String keyTabLocation;
+
     public KerberosHttpClient(String principal, String keyTabLocation) {
         super();
         this.principal = principal;
@@ -76,10 +72,7 @@ public class KerberosHttpClient {
     }
 
     private static CloseableHttpClient buildSpengoHttpClient() {
-        HttpClientBuilder builder = HttpClientBuilder.create();
-        Lookup<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create()
-                .register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory(true)).build();
-        builder.setDefaultAuthSchemeRegistry(authSchemeRegistry);
+        HttpClientBuilder builder = HttpUtils.getHttpClientBuilder();
         BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider();
         credentialsProvider.setCredentials(new AuthScope(null, -1, null), new Credentials() {
             @Override
@@ -114,9 +107,9 @@ public class KerberosHttpClient {
                 options.put("debug", "true");
                 return new AppConfigurationEntry[] {
                     new AppConfigurationEntry("com.sun.security.auth.module.Krb5LoginModule",
-                    AppConfigurationEntry.LoginModuleControlFlag.REQUIRED, options) };
-                }
-            };
+                        AppConfigurationEntry.LoginModuleControlFlag.REQUIRED, options)};
+            }
+        };
         Set<Principal> princ = new HashSet<>(1);
         princ.add(new KerberosPrincipal(userId));
         Subject sub = new Subject(false, princ, new HashSet<>(), new HashSet<>());

dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/LocalServerHttpUtilsTest.java

@@ -17,25 +17,30 @@
 
 package org.apache.dolphinscheduler.common.utils;
 
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import junit.framework.Test;
-import junit.framework.TestCase;
-import junit.framework.TestSuite;
 import org.apache.dolphinscheduler.common.Constants;
+
 import org.apache.http.client.config.RequestConfig;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClients;
+
 import org.junit.Assert;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-public class LocalServerHttpUtilsTest extends TestCase{
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+import junit.framework.Test;
+import junit.framework.TestCase;
+import junit.framework.TestSuite;
+
+public class LocalServerHttpUtilsTest extends TestCase {
 
     public static final Logger logger = LoggerFactory.getLogger(LocalServerHttpUtilsTest.class);
     private static LocalJettyHttpServer server = null;
-    public static Test suite(){
-        TestSuite suite=new TestSuite();
+
+    public static Test suite() {
+        TestSuite suite = new TestSuite();
         suite.addTestSuite(LocalServerHttpUtilsTest.class);
         server = new LocalJettyHttpServer(suite);
         return server;
@@ -44,21 +49,21 @@ public class LocalServerHttpUtilsTest extends TestCase{
     public void testGetTest() throws Exception {
         // success
         String result = null;
-        result = HttpUtils.get("http://localhost:" + server.getServerPort()+ "/test.json");
+        result = HttpUtils.get("http://localhost:" + server.getServerPort() + "/test.json");
         Assert.assertNotNull(result);
-		ObjectNode jsonObject = JSONUtils.parseObject(result);
-		Assert.assertEquals("Github",jsonObject.path("name").asText());
-		result = HttpUtils.get("http://123.333.111.33/ccc");
-		Assert.assertNull(result);
+        ObjectNode jsonObject = JSONUtils.parseObject(result);
+        Assert.assertEquals("Github", jsonObject.path("name").asText());
+        result = HttpUtils.get("http://123.333.111.33/ccc");
+        Assert.assertNull(result);
     }
 
     public void testGetResponseContentString() {
         CloseableHttpClient httpclient = HttpClients.createDefault();
-        HttpGet httpget = new HttpGet("http://localhost:" +server.getServerPort()+"/test.json");
+        HttpGet httpget = new HttpGet("http://localhost:" + server.getServerPort() + "/test.json");
         /** set timeout、request time、socket timeout */
         RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(Constants.HTTP_CONNECT_TIMEOUT)
-                .setConnectionRequestTimeout(Constants.HTTP_CONNECTION_REQUEST_TIMEOUT)
-                .setSocketTimeout(Constants.SOCKET_TIMEOUT).setRedirectsEnabled(true).build();
+            .setConnectionRequestTimeout(Constants.HTTP_CONNECTION_REQUEST_TIMEOUT)
+            .setSocketTimeout(Constants.SOCKET_TIMEOUT).setRedirectsEnabled(true).build();
         httpget.setConfig(requestConfig);
 
         String responseContent = null;
@@ -77,4 +82,26 @@ public class LocalServerHttpUtilsTest extends TestCase{
         CloseableHttpClient httpClient2 = HttpUtils.getInstance();
         Assert.assertEquals(httpClient1, httpClient2);
     }
+
+    public void testKerberosHttpsGet() {
+        logger.info(PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME));
+        logger.info(PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH));
+        logger.info(PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH));
+        String url = "https://www.apache.org/";
+        logger.info(KerberosHttpClient.get(url));
+        Assert.assertTrue(true);
+    }
+
+    public void testHttpsGet() {
+        String url = "https://www.apache.org/";
+        logger.info(HttpUtils.get(url));
+        Assert.assertTrue(true);
+    }
+
+    public void testHttpGet() {
+        String url = "http://www.apache.org/";
+        logger.info(HttpUtils.get(url));
+        Assert.assertTrue(true);
+    }
+
 }