[Fix][SqlTask] fix hive sqltask invalid connection parameters. (#4917)

* fix hive sqltask invalid connection parameters.

* add setConnParams test method.

* update base datasource test class code style.

* add BaseDataSourceTest in the pom xml.

* update hive conf list jdbc url stitching.

* update hive datasource test class.

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java

@@ -17,12 +17,16 @@
 
 package org.apache.dolphinscheduler.dao.datasource;
 
+import static org.apache.dolphinscheduler.common.Constants.PASSWORD;
+import static org.apache.dolphinscheduler.common.Constants.USER;
+
 import org.apache.dolphinscheduler.common.enums.DbType;
 import org.apache.dolphinscheduler.common.utils.CommonUtils;
 import org.apache.dolphinscheduler.common.utils.StringUtils;
 
 import java.sql.Connection;
 import java.sql.DriverManager;
+import java.util.Properties;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -157,6 +161,11 @@ public abstract class BaseDataSource {
                     separator = ":";
                     break;
                 case HIVE:
+                    if ("?".equals(otherParams.substring(0, 1))) {
+                        break;
+                    }
+                    separator = ";";
+                    break;
                 case SPARK:
                 case SQLSERVER:
                     separator = ";";
@@ -178,6 +187,19 @@ public abstract class BaseDataSource {
         return DriverManager.getConnection(getJdbcUrl(), getUser(), getPassword());
     }
 
+    /**
+     * the data source test connection
+     * @param info Properties
+     * @return Connection Connection
+     * @throws Exception Exception
+     */
+    public Connection getConnection(Properties info) throws Exception {
+        Class.forName(driverClassSelector());
+        info.setProperty(USER, getUser());
+        info.setProperty(PASSWORD, getPassword());
+        return DriverManager.getConnection(getJdbcUrl(), info);
+    }
+
     protected String filterOther(String otherParams) {
         return otherParams;
     }
@@ -226,6 +248,10 @@ public abstract class BaseDataSource {
         this.other = other;
     }
 
+    public void setConnParams(String connParams) {
+
+    }
+
     public String getJavaSecurityKrb5Conf() {
         return javaSecurityKrb5Conf;
     }

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java

@@ -17,13 +17,17 @@
 
 package org.apache.dolphinscheduler.dao.datasource;
 
+import static org.apache.dolphinscheduler.common.Constants.SEMICOLON;
+
 import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.enums.DbType;
+import org.apache.dolphinscheduler.common.utils.CollectionUtils;
 import org.apache.dolphinscheduler.common.utils.CommonUtils;
 import org.apache.dolphinscheduler.common.utils.HiveConfUtils;
 import org.apache.dolphinscheduler.common.utils.StringUtils;
 
 import java.sql.Connection;
+import java.util.Map;
 
 /**
  * data source of hive
@@ -100,4 +104,17 @@ public class HiveDataSource extends BaseDataSource {
         return super.getConnection();
     }
 
+    @Override
+    public void setConnParams(String connParams) {
+        // Verification parameters
+        Map<String, String> connParamMap = CollectionUtils.stringToMap(connParams, SEMICOLON);
+        if (connParamMap.isEmpty()) {
+            return;
+        }
+
+        StringBuilder otherSb = new StringBuilder();
+        connParamMap.forEach((k, v) -> otherSb.append(String.format("%s=%s%s", k, v, SEMICOLON)));
+        StringBuilder otherAppend = StringUtils.isNotBlank(getOther()) ? otherSb.append(getOther()) : otherSb.deleteCharAt(otherSb.length() - 1);
+        super.setOther(otherAppend.toString());
+    }
 }

dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSourceTest.java

@@ -158,4 +158,38 @@ public class BaseDataSourceTest {
 
     }
 
+    @Test
+    public void testSetConnParams() {
+
+        BaseDataSource hiveDataSource = new HiveDataSource();
+        hiveDataSource.setAddress("jdbc:hive2://127.0.0.1:10000");
+        hiveDataSource.setDatabase("test");
+        hiveDataSource.setPassword("123456");
+        hiveDataSource.setUser("test");
+        hiveDataSource.setConnParams("");
+        Assert.assertEquals("jdbc:hive2://127.0.0.1:10000/test", hiveDataSource.getJdbcUrl());
+
+        //set fake other
+        hiveDataSource.setConnParams("hive.tez.container.size=20000;");
+        Assert.assertEquals("jdbc:hive2://127.0.0.1:10000/test?hive.tez.container.size=20000", hiveDataSource.getJdbcUrl());
+
+        hiveDataSource.setOther(null);
+        hiveDataSource.setConnParams("hive.tez.container.size=20000");
+        Assert.assertEquals("jdbc:hive2://127.0.0.1:10000/test?hive.tez.container.size=20000", hiveDataSource.getJdbcUrl());
+
+        hiveDataSource.setOther(null);
+        hiveDataSource.setConnParams("hive.tez.container.size=20000;hive.zzz=100");
+        Assert.assertEquals("jdbc:hive2://127.0.0.1:10000/test;hive.zzz=100?hive.tez.container.size=20000", hiveDataSource.getJdbcUrl());
+
+        hiveDataSource.setOther("charset=UTF-8");
+        Assert.assertEquals("jdbc:hive2://127.0.0.1:10000/test;charset=UTF-8", hiveDataSource.getJdbcUrl());
+
+        hiveDataSource.setConnParams("hive.tez.container.size=20000;hive.zzz=100");
+        Assert.assertEquals("jdbc:hive2://127.0.0.1:10000/test;hive.zzz=100;charset=UTF-8?hive.tez.container.size=20000", hiveDataSource.getJdbcUrl());
+
+        hiveDataSource.setOther("charset=UTF-8;hive.exec.stagingdir=/tmp");
+        hiveDataSource.setConnParams("hive.tez.container.size=20000;hive.zzz=100");
+        Assert.assertEquals("jdbc:hive2://127.0.0.1:10000/test;hive.zzz=100;charset=UTF-8?hive.tez.container.size=20000;hive.exec.stagingdir=/tmp", hiveDataSource.getJdbcUrl());
+    }
+
 }

dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSourceTest.java

@@ -81,7 +81,7 @@ public class HiveDataSourceTest {
 
         hiveDataSource.setOther("hive.mapred.mode=strict;hive.server2.thrift.http.path=hs2");
         Assert.assertEquals(
-                "jdbc:hive2://127.0.0.1:10000/test;?hive.mapred.mode=strict;hive.server2.thrift.http.path=hs2",
+                "jdbc:hive2://127.0.0.1:10000/test?hive.mapred.mode=strict;hive.server2.thrift.http.path=hs2",
                 hiveDataSource.getJdbcUrl());
 
     }

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java

@@ -244,6 +244,9 @@ public class SqlTask extends AbstractTask {
         PreparedStatement stmt = null;
         ResultSet resultSet = null;
         try {
+
+            baseDataSource.setConnParams(sqlParameters.getConnParams());
+
             // create connection
             connection = baseDataSource.getConnection();
             // create temp function

pom.xml

@@ -990,6 +990,7 @@
                         <include>**/dao/AlertDaoTest.java</include>
                         <include>**/dao/datasource/OracleDataSourceTest.java</include>
                         <include>**/dao/datasource/HiveDataSourceTest.java</include>
+                        <include>**/dao/datasource/BaseDataSourceTest.java</include>
                         <include>**/dao/upgrade/ProcessDefinitionDaoTest.java</include>
                         <include>**/dao/upgrade/WokrerGrouopDaoTest.java</include>
                         <include>**/dao/upgrade/UpgradeDaoTest.java</include>