Browse Source

[Fix-13381][plugin] fix error while parsing hive load sql (#13378)

* [fix][task plugin][task sql] fix parsing error while using variables sometimes parsing exception happened if task sql contains a variable followed by \" or \' .

for example:
input sql: load inpath '/tmp/test_table/dt=${dt}'
into table test_table partition(dt=${dt})

after replace: preparing : load inpath '/tmp/test_table/dt=?
into table test_table partition(dt=?)
Alex Ting 1 year ago
parent
commit
606b56403e

+ 1 - 0
docs/docs/en/guide/upgrade/incompatible.md

@@ -4,6 +4,7 @@ This document records the incompatible updates between each version. You need to
 
 ## dev
 
+* Change regex matching sql params in SQL task plugin ([#13378](https://github.com/apache/dolphinscheduler/pull/13378))
 * Remove the spark version of spark task ([#11860](https://github.com/apache/dolphinscheduler/pull/11860)).
 * Change the default unix shell executor from sh to bash ([#12180](https://github.com/apache/dolphinscheduler/pull/12180)).
 

+ 1 - 0
docs/docs/zh/guide/upgrade/incompatible.md

@@ -4,6 +4,7 @@
 
 ## dev
 
+* 更新了SQL任务中用于匹配变量的正则表达式 ([#13378](https://github.com/apache/dolphinscheduler/pull/13378))
 * Remove the spark version of spark task ([#11860](https://github.com/apache/dolphinscheduler/pull/11860)).
 * Change the default unix shell executor from sh to bash ([#12180](https://github.com/apache/dolphinscheduler/pull/12180)).
 

+ 1 - 1
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractTask.java

@@ -38,7 +38,7 @@ public abstract class AbstractTask {
 
     protected final Logger log = LoggerFactory.getLogger(AbstractTask.class);
 
-    public String rgex = "['\"]*\\$\\{(.*?)\\}['\"]*";
+    public String rgex = "['\"]\\$\\{(.*?)}['\"]|\\$\\{(.*?)}";
 
     /**
      * varPool string

+ 96 - 0
dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java

@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.plugin.task.sql;
+
+import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
+import org.apache.dolphinscheduler.plugin.task.api.enums.ResourceType;
+import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.DataSourceParameters;
+import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper;
+import org.apache.dolphinscheduler.spi.enums.DbType;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+class SqlTaskTest {
+
+    private SqlTask sqlTask;
+
+    @BeforeEach
+    void setup() {
+        DataSourceParameters parameters = new DataSourceParameters();
+        parameters.setType(DbType.HIVE);
+        parameters.setResourceType(ResourceType.DATASOURCE.name());
+
+        ResourceParametersHelper resourceParametersHelper = new ResourceParametersHelper();
+        resourceParametersHelper.put(ResourceType.DATASOURCE, 1, parameters);
+
+        TaskExecutionContext ctx = new TaskExecutionContext();
+        ctx.setResourceParametersHelper(resourceParametersHelper);
+        ctx.setTaskParams("{\"type\":\"HIVE\",\"datasource\":1,\"sql\":\"select 1\"}");
+
+        sqlTask = new SqlTask(ctx);
+    }
+
+    @Test
+    void testReplacingSqlWithoutParams() {
+        String querySql = "select 1";
+        String expected = "select 1";
+        Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?"));
+    }
+
+    @Test
+    void testReplacingSqlWithDollarSymbol() {
+        String querySql = "select concat(amount, '$') as price from product";
+        String expected = "select concat(amount, '$') as price from product";
+        Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?"));
+    }
+
+    @Test
+    void testReplacingHiveLoadSql() {
+        String hiveLoadSql = "load inpath '/tmp/test_table/dt=${dt}' into table test_table partition(dt=${dt})";
+        String expected = "load inpath '/tmp/test_table/dt=?' into table test_table partition(dt=?)";
+        Assertions.assertEquals(expected, hiveLoadSql.replaceAll(sqlTask.rgex, "?"));
+    }
+
+    @Test
+    void testReplacingSelectSql() {
+        String querySql = "select id from student where dt='${dt}'";
+        String expected = "select id from student where dt=?";
+        Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?"));
+
+        querySql = "select id from student where dt=\"${dt}\"";
+        expected = "select id from student where dt=?";
+        Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?"));
+
+        querySql = "select id from student where dt=${dt}";
+        expected = "select id from student where dt=?";
+        Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?"));
+
+        querySql = "select id from student where dt=${dt} and gender=1";
+        expected = "select id from student where dt=? and gender=1";
+        Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?"));
+    }
+
+    @Test
+    void testReplacingSqlNonGreedy() {
+        String querySql = "select id from student where year=${year} and month=${month} and gender=1";
+        String expected = "select id from student where year=? and month=? and gender=1";
+        Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?"));
+    }
+}