test

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentExecute.java

@@ -0,0 +1,296 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.server.worker.task.dependent;
+
+import org.apache.dolphinscheduler.common.Constants;
+import org.apache.dolphinscheduler.common.enums.DependResult;
+import org.apache.dolphinscheduler.common.enums.DependentRelation;
+import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
+import org.apache.dolphinscheduler.common.model.DateInterval;
+import org.apache.dolphinscheduler.common.model.DependentItem;
+import org.apache.dolphinscheduler.common.model.TaskNode;
+import org.apache.dolphinscheduler.common.utils.DependentUtils;
+import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
+import org.apache.dolphinscheduler.dao.entity.TaskInstance;
+import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
+import org.apache.dolphinscheduler.service.process.ProcessService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+
+/**
+ * dependent item execute
+ */
+public class DependentExecute {
+    /**
+     * process service
+     */
+    private final ProcessService processService = SpringApplicationContext.getBean(ProcessService.class);
+
+    /**
+     * depend item list
+     */
+    private List<DependentItem> dependItemList;
+
+    /**
+     * dependent relation
+     */
+    private DependentRelation relation;
+
+    /**
+     * depend result
+     */
+    private DependResult modelDependResult = DependResult.WAITING;
+
+    /**
+     * depend result map
+     */
+    private Map<String, DependResult> dependResultMap = new HashMap<>();
+
+    /**
+     * logger
+     */
+    private Logger logger =  LoggerFactory.getLogger(DependentExecute.class);
+
+    /**
+     * constructor
+     * @param itemList  item list
+     * @param relation  relation
+     */
+    public DependentExecute(List<DependentItem> itemList, DependentRelation relation){
+        this.dependItemList = itemList;
+        this.relation = relation;
+    }
+
+    /**
+     * get dependent item for one dependent item
+     * @param dependentItem dependent item
+     * @param currentTime   current time
+     * @return DependResult
+     */
+    private DependResult getDependentResultForItem(DependentItem dependentItem, Date currentTime){
+        List<DateInterval> dateIntervals = DependentUtils.getDateIntervalList(currentTime, dependentItem.getDateValue());
+        return calculateResultForTasks(dependentItem, dateIntervals );
+    }
+
+    /**
+     * calculate dependent result for one dependent item.
+     * @param dependentItem dependent item
+     * @param dateIntervals date intervals
+     * @return dateIntervals
+     */
+    private DependResult calculateResultForTasks(DependentItem dependentItem,
+                                                 List<DateInterval> dateIntervals) {
+
+        DependResult result = DependResult.FAILED;
+        for(DateInterval dateInterval : dateIntervals){
+            ProcessInstance processInstance = findLastProcessInterval(dependentItem.getDefinitionId(),
+                                                    dateInterval);
+            if(processInstance == null){
+                logger.error("cannot find the right process instance: definition id:{}, start:{}, end:{}",
+                       dependentItem.getDefinitionId(), dateInterval.getStartTime(), dateInterval.getEndTime() );
+                return DependResult.FAILED;
+            }
+            // need to check workflow for updates, so get all task and check the task state
+            if(dependentItem.getDepTasks().equals(Constants.DEPENDENT_ALL)){
+                List<TaskNode> taskNodes =
+                        processService.getTaskNodeListByDefinitionId(dependentItem.getDefinitionId());
+
+                if(taskNodes != null && taskNodes.size() > 0){
+                    List<DependResult> results = new ArrayList<>();
+                    DependResult tmpResult =  DependResult.FAILED;
+                    for(TaskNode taskNode:taskNodes){
+                        tmpResult = getDependTaskResult(taskNode.getName(),processInstance);
+                        if(DependResult.FAILED == tmpResult){
+                            break;
+                        }else{
+                            results.add(getDependTaskResult(taskNode.getName(),processInstance));
+                        }
+                    }
+
+                    if(DependResult.FAILED == tmpResult){
+                        result = DependResult.FAILED;
+                    }else if(results.contains(DependResult.WAITING)){
+                        result = DependResult.WAITING;
+                    }else{
+                        result =  DependResult.SUCCESS;
+                    }
+                }else{
+                    result = DependResult.FAILED;
+                }
+            }else{
+                result = getDependTaskResult(dependentItem.getDepTasks(),processInstance);
+            }
+            if(result != DependResult.SUCCESS){
+                break;
+            }
+        }
+        return result;
+    }
+
+    /**
+     * get depend task result
+     * @param taskName
+     * @param processInstance
+     * @return
+     */
+    private DependResult getDependTaskResult(String taskName, ProcessInstance processInstance) {
+        DependResult result = DependResult.FAILED;
+        TaskInstance taskInstance = null;
+        List<TaskInstance> taskInstanceList = processService.findValidTaskListByProcessId(processInstance.getId());
+
+        for(TaskInstance task : taskInstanceList){
+            if(task.getName().equals(taskName)){
+                taskInstance = task;
+                break;
+            }
+        }
+
+        if(taskInstance == null){
+            // cannot find task in the process instance
+            // maybe because process instance is running or failed.
+            result = getDependResultByProcessStateWhenTaskNull(processInstance.getState());
+        }else{
+            result = getDependResultByState(taskInstance.getState());
+        }
+
+        return result;
+    }
+
+    /**
+     * find the last one process instance that :
+     * 1. manual run and finish between the interval
+     * 2. schedule run and schedule time between the interval
+     * @param definitionId  definition id
+     * @param dateInterval  date interval
+     * @return ProcessInstance
+     */
+    private ProcessInstance findLastProcessInterval(int definitionId, DateInterval dateInterval) {
+
+        ProcessInstance runningProcess = processService.findLastRunningProcess(definitionId, dateInterval);
+        if(runningProcess != null){
+            return runningProcess;
+        }
+
+        ProcessInstance lastSchedulerProcess = processService.findLastSchedulerProcessInterval(
+                definitionId, dateInterval
+        );
+
+        ProcessInstance lastManualProcess = processService.findLastManualProcessInterval(
+                definitionId, dateInterval
+        );
+
+        if(lastManualProcess ==null){
+            return lastSchedulerProcess;
+        }
+        if(lastSchedulerProcess == null){
+            return lastManualProcess;
+        }
+
+        return (lastManualProcess.getEndTime().after(lastSchedulerProcess.getEndTime()))?
+                lastManualProcess : lastSchedulerProcess;
+    }
+
+    /**
+     * get dependent result by task/process instance state
+     * @param state state
+     * @return DependResult
+     */
+    private DependResult getDependResultByState(ExecutionStatus state) {
+
+        if(state.typeIsRunning()
+                || state == ExecutionStatus.SUBMITTED_SUCCESS
+                || state == ExecutionStatus.WAITTING_THREAD){
+            return DependResult.WAITING;
+        }else if(state.typeIsSuccess()){
+            return DependResult.SUCCESS;
+        }else{
+            return DependResult.FAILED;
+        }
+    }
+
+    /**
+     * get dependent result by task instance state when task instance is null
+     * @param state state
+     * @return DependResult
+     */
+    private DependResult getDependResultByProcessStateWhenTaskNull(ExecutionStatus state) {
+
+        if(state.typeIsRunning()
+                || state == ExecutionStatus.SUBMITTED_SUCCESS
+                || state == ExecutionStatus.WAITTING_THREAD){
+            return DependResult.WAITING;
+        }else{
+            return DependResult.FAILED;
+        }
+    }
+
+    /**
+     * judge depend item finished
+     * @param currentTime current time
+     * @return boolean
+     */
+    public boolean finish(Date currentTime){
+        if(modelDependResult == DependResult.WAITING){
+            modelDependResult = getModelDependResult(currentTime);
+            return false;
+        }
+        return true;
+    }
+
+    /**
+     * get model depend result
+     * @param currentTime current time
+     * @return DependResult
+     */
+    public DependResult getModelDependResult(Date currentTime){
+
+        List<DependResult> dependResultList = new ArrayList<>();
+
+        for(DependentItem dependentItem : dependItemList){
+            DependResult dependResult = getDependResultForItem(dependentItem, currentTime);
+            if(dependResult != DependResult.WAITING){
+                dependResultMap.put(dependentItem.getKey(), dependResult);
+            }
+            dependResultList.add(dependResult);
+        }
+        modelDependResult = DependentUtils.getDependResultForRelation(
+                this.relation, dependResultList
+        );
+        return modelDependResult;
+    }
+
+    /**
+     * get dependent item result
+     * @param item          item
+     * @param currentTime   current time
+     * @return DependResult
+     */
+    private DependResult getDependResultForItem(DependentItem item, Date currentTime){
+        String key = item.getKey();
+        if(dependResultMap.containsKey(key)){
+            return dependResultMap.get(key);
+        }
+        return getDependentResultForItem(item, currentTime);
+    }
+
+    public Map<String, DependResult> getDependResultMap(){
+        return dependResultMap;
+    }
+
+}

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentTask.java

@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.server.worker.task.dependent;
+
+import org.apache.dolphinscheduler.common.Constants;
+import org.apache.dolphinscheduler.common.enums.DependResult;
+import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
+import org.apache.dolphinscheduler.common.model.DependentTaskModel;
+import org.apache.dolphinscheduler.common.task.AbstractParameters;
+import org.apache.dolphinscheduler.common.task.dependent.DependentParameters;
+import org.apache.dolphinscheduler.common.thread.Stopper;
+import org.apache.dolphinscheduler.common.utils.DependentUtils;
+import org.apache.dolphinscheduler.common.utils.JSONUtils;
+import org.apache.dolphinscheduler.dao.entity.TaskInstance;
+import org.apache.dolphinscheduler.server.worker.task.AbstractTask;
+import org.apache.dolphinscheduler.server.worker.task.TaskProps;
+import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
+import org.apache.dolphinscheduler.service.process.ProcessService;
+import org.slf4j.Logger;
+
+import java.util.*;
+
+import static org.apache.dolphinscheduler.common.Constants.DEPENDENT_SPLIT;
+
+/**
+ * Dependent Task
+ */
+public class DependentTask extends AbstractTask {
+
+    /**
+     * dependent task list
+     */
+    private List<DependentExecute> dependentTaskList = new ArrayList<>();
+
+    /**
+     * depend item result map
+     * save the result to log file
+     */
+    private Map<String, DependResult> dependResultMap = new HashMap<>();
+
+    /**
+     * dependent parameters
+     */
+    private DependentParameters dependentParameters;
+
+    /**
+     * dependent date
+     */
+    private Date dependentDate;
+
+    /**
+     * process service
+     */
+    private ProcessService processService;
+
+    /**
+     * constructor
+     * @param props     props
+     * @param logger    logger
+     */
+    public DependentTask(TaskProps props, Logger logger) {
+        super(props, logger);
+    }
+
+    @Override
+    public void init(){
+        logger.info("dependent task initialize");
+
+        this.dependentParameters = JSONUtils.parseObject(this.taskProps.getDependence(),
+                DependentParameters.class);
+        if(dependentParameters != null){
+            for(DependentTaskModel taskModel : dependentParameters.getDependTaskList()){
+                this.dependentTaskList.add(new DependentExecute(
+                        taskModel.getDependItemList(), taskModel.getRelation()));
+            }
+        }
+
+        this.processService = SpringApplicationContext.getBean(ProcessService.class);
+
+        if(taskProps.getScheduleTime() != null){
+            this.dependentDate = taskProps.getScheduleTime();
+        }else{
+            this.dependentDate = taskProps.getTaskStartTime();
+        }
+
+    }
+
+    @Override
+    public void handle() throws Exception {
+        // set the name of the current thread
+        String threadLoggerInfoName = String.format(Constants.TASK_LOG_INFO_FORMAT, taskProps.getTaskAppId());
+        Thread.currentThread().setName(threadLoggerInfoName);
+
+        try{
+            TaskInstance taskInstance = null;
+            while(Stopper.isRunning()){
+                taskInstance = processService.findTaskInstanceById(this.taskProps.getTaskInstId());
+
+                if(taskInstance == null){
+                    exitStatusCode = -1;
+                    break;
+                }
+
+                if(taskInstance.getState() == ExecutionStatus.KILL){
+                    this.cancel = true;
+                }
+
+                if(this.cancel || allDependentTaskFinish()){
+                    break;
+                }
+
+                Thread.sleep(Constants.SLEEP_TIME_MILLIS);
+            }
+
+            if(cancel){
+                exitStatusCode = Constants.EXIT_CODE_KILL;
+            }else{
+                DependResult result = getTaskDependResult();
+                exitStatusCode = (result == DependResult.SUCCESS) ?
+                        Constants.EXIT_CODE_SUCCESS : Constants.EXIT_CODE_FAILURE;
+            }
+        }catch (Exception e){
+            logger.error(e.getMessage(),e);
+            exitStatusCode = -1;
+            throw e;
+        }
+    }
+
+    /**
+     * get dependent result
+     * @return DependResult
+     */
+    private DependResult getTaskDependResult(){
+        List<DependResult> dependResultList = new ArrayList<>();
+        for(DependentExecute dependentExecute : dependentTaskList){
+            DependResult dependResult = dependentExecute.getModelDependResult(dependentDate);
+            dependResultList.add(dependResult);
+        }
+        DependResult result = DependentUtils.getDependResultForRelation(
+                this.dependentParameters.getRelation(), dependResultList
+        );
+        return result;
+    }
+
+    /**
+     * judge all dependent tasks finish
+     * @return whether all dependent tasks finish
+     */
+    private boolean allDependentTaskFinish(){
+        boolean finish = true;
+        for(DependentExecute dependentExecute : dependentTaskList){
+            for(Map.Entry<String, DependResult> entry: dependentExecute.getDependResultMap().entrySet()) {
+                if(!dependResultMap.containsKey(entry.getKey())){
+                    dependResultMap.put(entry.getKey(), entry.getValue());
+                    //save depend result to log
+                    logger.info("dependent item complete {} {},{}",
+                            DEPENDENT_SPLIT, entry.getKey(), entry.getValue().toString());
+                }
+            }
+            if(!dependentExecute.finish(dependentDate)){
+                finish = false;
+            }
+        }
+        return finish;
+    }
+
+
+    @Override
+    public void cancelApplication(boolean cancelApplication) throws Exception {
+        // cancel process
+        this.cancel = true;
+    }
+
+    @Override
+    public AbstractParameters getParameters() {
+        return null;
+    }
+}

dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentTaskTest.java

@@ -17,47 +17,106 @@
 package org.apache.dolphinscheduler.server.worker.task.dependent;
 
 import org.apache.dolphinscheduler.common.Constants;
+import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
+import org.apache.dolphinscheduler.common.model.DateInterval;
+import org.apache.dolphinscheduler.common.model.TaskNode;
+import org.apache.dolphinscheduler.common.utils.dependent.DependentDateUtils;
+import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
+import org.apache.dolphinscheduler.dao.entity.TaskInstance;
 import org.apache.dolphinscheduler.server.worker.task.TaskProps;
+import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
+import org.apache.dolphinscheduler.service.process.ProcessService;
 import org.junit.Assert;
+import org.junit.Before;
 import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mockito;
+import org.mockito.junit.MockitoJUnitRunner;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.context.ApplicationContext;
 
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+
+@RunWith(MockitoJUnitRunner.Silent.class)
 public class DependentTaskTest {
 
     private static final Logger logger = LoggerFactory.getLogger(DependentTaskTest.class);
 
+    private ProcessService processService;
+    private ApplicationContext applicationContext;
+
+
+    @Before
+    public void before() throws Exception{
+        processService = Mockito.mock(ProcessService.class);
+        Mockito.when(processService
+                .findLastRunningProcess(4,DependentDateUtils.getTodayInterval(new Date()).get(0)))
+                .thenReturn(findLastProcessInterval());
+        Mockito.when(processService
+                .getTaskNodeListByDefinitionId(4))
+                .thenReturn(getTaskNodes());
+        Mockito.when(processService
+                .findValidTaskListByProcessId(11))
+                .thenReturn(getTaskInstances());
+
+        Mockito.when(processService
+                .findTaskInstanceById(252612))
+                .thenReturn(getTaskInstance());
+        applicationContext = Mockito.mock(ApplicationContext.class);
+        SpringApplicationContext springApplicationContext = new SpringApplicationContext();
+        springApplicationContext.setApplicationContext(applicationContext);
+        Mockito.when(applicationContext.getBean(ProcessService.class)).thenReturn(processService);
+    }
 
     @Test
-    public void testDependInit() throws Exception{
+    public void test() throws Exception{
 
         TaskProps taskProps = new TaskProps();
-
-        String dependString = "{\n" +
-                "\"dependTaskList\":[\n" +
-                "    {\n" +
-                "        \"dependItemList\":[\n" +
-                "            {\n" +
-                "                    \"definitionId\": 101,\n" +
-                "                    \"depTasks\": \"ALL\",\n" +
-                "                    \"cycle\": \"day\",\n" +
-                "                    \"dateValue\": \"last1Day\"\n" +
-                "            }\n" +
-                "        ],\n" +
-                "        \"relation\": \"AND\"\n" +
-                "    }\n" +
-                "    ],\n" +
-                "\"relation\":\"OR\"\n" +
-                "}";
-
-        taskProps.setTaskInstanceId(252612);
+        String dependString = "{\"dependTaskList\":[{\"dependItemList\":[{\"dateValue\":\"today\",\"depTasks\":\"ALL\",\"projectId\":1,\"definitionList\":[{\"label\":\"C\",\"value\":4},{\"label\":\"B\",\"value\":3},{\"label\":\"A\",\"value\":2}],\"cycle\":\"day\",\"definitionId\":4}],\"relation\":\"AND\"}],\"relation\":\"AND\"}";
+        taskProps.setTaskInstId(252612);
         taskProps.setDependence(dependString);
-//        DependentTask dependentTask = new DependentTask(taskProps, logger);
-//        dependentTask.init();
-//        dependentTask.handle();
-//        Assert.assertEquals(dependentTask.getExitStatusCode(), Constants.EXIT_CODE_FAILURE );
+        taskProps.setTaskStartTime(new Date());
+        DependentTask dependentTask = new DependentTask(taskProps, logger);
+        dependentTask.init();
+        dependentTask.handle();
+        Assert.assertEquals(dependentTask.getExitStatusCode(), Constants.EXIT_CODE_SUCCESS );
     }
 
+    private ProcessInstance findLastProcessInterval(){
+        ProcessInstance processInstance = new ProcessInstance();
+        processInstance.setId(11);
+        processInstance.setState(ExecutionStatus.SUCCESS);
+        return  processInstance;
+    }
+
+    private List<TaskNode> getTaskNodes(){
+        List<TaskNode> list = new ArrayList<>();
+        TaskNode taskNode = new TaskNode();
+        taskNode.setName("C");
+        taskNode.setType("SQL");
+        list.add(taskNode);
+        return list;
+    }
 
+    private List<TaskInstance> getTaskInstances(){
+        List<TaskInstance> list = new ArrayList<>();
+        TaskInstance taskInstance = new TaskInstance();
+        taskInstance.setName("C");
+        taskInstance.setState(ExecutionStatus.SUCCESS);
+        taskInstance.setDependency("1231");
+        list.add(taskInstance);
+        return list;
+    }
+
+    private TaskInstance getTaskInstance(){
+        TaskInstance taskInstance = new TaskInstance();
+        taskInstance.setId(252612);
+        taskInstance.setName("C");
+        taskInstance.setState(ExecutionStatus.SUCCESS);
+        return taskInstance;
+    }
 
 }

dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java

@@ -230,6 +230,30 @@ public class ProcessService {
         return processInstanceMapper.queryDetailById(processId);
     }
 
+    /**
+     * get task node list by definitionId
+     * @param defineId
+     * @return
+     */
+    public  List<TaskNode> getTaskNodeListByDefinitionId(Integer defineId){
+        ProcessDefinition processDefinition = processDefineMapper.selectById(defineId);
+        if (processDefinition == null) {
+            logger.info("process define not exists");
+            return null;
+        }
+
+        String processDefinitionJson = processDefinition.getProcessDefinitionJson();
+        ProcessData processData = JSONUtils.parseObject(processDefinitionJson, ProcessData.class);
+
+        //process data check
+        if (null == processData) {
+            logger.error("process data is null");
+            return null;
+        }
+
+        return processData.getTasks();
+    }
+
     /**
      * find process instance by id
      * @param processId processId

dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/flink.vue

@@ -218,6 +218,19 @@
     },
     mixins: [disabledState],
     methods: {
+      /**
+       * getResourceId
+       */
+      marjarId(name) {
+        this.store.dispatch('dag/getResourceId',{
+          type: 'FILE',
+          fullName: '/'+name
+        }).then(res => {
+          this.mainJar = res.id
+        }).catch(e => {
+          this.$message.error(e.msg || '')
+        })
+      },
       /**
        * return localParams
        */
@@ -366,7 +379,13 @@
         // Non-null objects represent backfill
         if (!_.isEmpty(o)) {
           this.mainClass = o.params.mainClass || ''
-          this.mainJar = o.params.mainJar && o.params.mainJar.id ? o.params.mainJar.id : ''
+          if(o.params.mainJar.res) {
+            this.marjarId(o.params.mainJar.res)
+          } else if(o.params.mainJar.res=='') {
+            this.mainJar = ''
+          } else {
+            this.mainJar = o.params.mainJar.id || ''
+          }
           this.deployMode = o.params.deployMode || ''
           this.slot = o.params.slot || 1
           this.taskManager = o.params.taskManager || '2'
@@ -380,8 +399,19 @@
           // backfill resourceList
           let resourceList = o.params.resourceList || []
           if (resourceList.length) {
-            this.resourceList = _.map(resourceList, v => {
-              return v.id
+            _.map(resourceList, v => {
+              if(v.res) {
+                this.store.dispatch('dag/getResourceId',{
+                  type: 'FILE',
+                  fullName: '/'+v.res
+                }).then(res => {
+                  this.resourceList.push(res.id)
+                }).catch(e => {
+                  this.$message.error(e.msg || '')
+                })
+              } else {
+                this.resourceList.push(v.id)
+              }
             })
             this.cacheResourceList = resourceList
           }

dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue

@@ -142,6 +142,19 @@
     },
     mixins: [disabledState],
     methods: {
+      /**
+       * getResourceId
+       */
+      marjarId(name) {
+        this.store.dispatch('dag/getResourceId',{
+          type: 'FILE',
+          fullName: '/'+name
+        }).then(res => {
+          this.mainJar = res.id
+        }).catch(e => {
+          this.$message.error(e.msg || '')
+        })
+      },
       /**
        * return localParams
        */
@@ -245,7 +258,13 @@
         // Non-null objects represent backfill
         if (!_.isEmpty(o)) {
           this.mainClass = o.params.mainClass || ''
-          this.mainJar = o.params.mainJar.id || ''
+          if(o.params.mainJar.res) {
+            this.marjarId(o.params.mainJar.res)
+          } else if(o.params.mainJar.res=='') {
+            this.mainJar = ''
+          } else {
+            this.mainJar = o.params.mainJar.id || ''
+          }
           this.mainArgs = o.params.mainArgs || ''
           this.others = o.params.others
           this.programType = o.params.programType || 'JAVA'
@@ -253,8 +272,19 @@
           // backfill resourceList
           let resourceList = o.params.resourceList || []
           if (resourceList.length) {
-            this.resourceList = _.map(resourceList, v => {
-              return v.id
+            _.map(resourceList, v => {
+              if(v.res) {
+                this.store.dispatch('dag/getResourceId',{
+                  type: 'FILE',
+                  fullName: '/'+v.res
+                }).then(res => {
+                  this.resourceList.push(res.id)
+                }).catch(e => {
+                  this.$message.error(e.msg || '')
+                })
+              } else {
+                this.resourceList.push(v.id)
+              }
             })
             this.cacheResourceList = resourceList
           }

dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue

@@ -161,8 +161,7 @@
           resourceList: _.map(this.resourceList, v => {
             return {id: v}
           }),
-          localParams: this.localParams,
-          rawScript: editor ? editor.getValue() : ''
+          localParams: this.localParams
         }
       }
     },
@@ -176,8 +175,19 @@
         // backfill resourceList
         let resourceList = o.params.resourceList || []
         if (resourceList.length) {
-          this.resourceList = _.map(resourceList, v => {
-            return v.id
+          _.map(resourceList, v => {
+            if(v.res) {
+              this.store.dispatch('dag/getResourceId',{
+                type: 'FILE',
+                fullName: '/'+v.res
+              }).then(res => {
+                this.resourceList.push(res.id)
+              }).catch(e => {
+                this.$message.error(e.msg || '')
+              })
+            } else {
+              this.resourceList.push(v.id)
+            }
           })
           this.cacheResourceList = resourceList
         }

dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue

@@ -95,7 +95,7 @@
           return {
             label: node.name
           }
-        },
+        }
       }
     },
     mixins: [disabledState],
@@ -221,8 +221,7 @@
           resourceList: _.map(this.resourceList, v => {
             return {id: v}
           }),
-          localParams: this.localParams,
-          rawScript: editor ? editor.getValue() : ''
+          localParams: this.localParams
         }
       }
     },
@@ -238,12 +237,22 @@
         // backfill resourceList
         let resourceList = o.params.resourceList || []
         if (resourceList.length) {
-          this.resourceList = _.map(resourceList, v => {
-            return v.id
+           _.map(resourceList, v => {
+            if(v.res) {
+              this.store.dispatch('dag/getResourceId',{
+                type: 'FILE',
+                fullName: '/'+v.res
+              }).then(res => {
+                this.resourceList.push(res.id)
+              }).catch(e => {
+                this.$message.error(e.msg || '')
+              })
+            } else {
+              this.resourceList.push(v.id)
+            }
           })
           this.cacheResourceList = resourceList
         }
-
         // backfill localParams
         let localParams = o.params.localParams || []
         if (localParams.length) {

dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/spark.vue

@@ -261,6 +261,19 @@
     },
     mixins: [disabledState],
     methods: {
+      /**
+       * getResourceId
+       */
+      marjarId(name) {
+        this.store.dispatch('dag/getResourceId',{
+          type: 'FILE',
+          fullName: '/'+name
+        }).then(res => {
+          this.mainJar = res.id
+        }).catch(e => {
+          this.$message.error(e.msg || '')
+        })
+      },
       /**
        * return localParams
        */
@@ -414,7 +427,13 @@
         // Non-null objects represent backfill
         if (!_.isEmpty(o)) {
           this.mainClass = o.params.mainClass || ''
-          this.mainJar = o.params.mainJar && o.params.mainJar.id ? o.params.mainJar.id : ''
+          if(o.params.mainJar.res) {
+            this.marjarId(o.params.mainJar.res)
+          } else if(o.params.mainJar.res=='') {
+            this.mainJar = ''
+          } else {
+            this.mainJar = o.params.mainJar.id || ''
+          }
           this.deployMode = o.params.deployMode || ''
           this.driverCores = o.params.driverCores || 1
           this.driverMemory = o.params.driverMemory || '512M'
@@ -429,9 +448,20 @@
           // backfill resourceList
           let resourceList = o.params.resourceList || []
           if (resourceList.length) {
-            this.resourceList = _.map(resourceList, v => {
-            return v.id
-          })
+            _.map(resourceList, v => {
+              if(v.res) {
+                this.store.dispatch('dag/getResourceId',{
+                  type: 'FILE',
+                  fullName: '/'+v.res
+                }).then(res => {
+                  this.resourceList.push(res.id)
+                }).catch(e => {
+                  this.$message.error(e.msg || '')
+                })
+              } else {
+                this.resourceList.push(v.id)
+              }
+            })
             this.cacheResourceList = resourceList
           }
 

dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue

@@ -22,7 +22,7 @@
           <th scope="col" width="50">
             <x-checkbox @on-change="_topCheckBoxClick" v-model="checkAll"></x-checkbox>
           </th>
-          <th scope="col">
+          <th scope="col" width="30">
             <span>{{$t('#')}}</span>
           </th>
           <th scope="col" width="70">

dolphinscheduler-ui/src/js/conf/home/store/dag/actions.js

@@ -715,5 +715,14 @@ export default {
         reject(e)
       })
     })
-  }
-}
+  },
+  getResourceId ({ state }, payload) {
+    return new Promise((resolve, reject) => {
+      io.get(`resources/queryResource`, payload, res => {
+        resolve(res.data)
+      }).catch(e => {
+        reject(e)
+      })
+    })
+  },
+}

dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js

@@ -115,7 +115,7 @@ export default {
   'SQL Type': 'sql类型',
   'Title': '主题',
   'Please enter the title of email': '请输入邮件主题',
-  'Table': '表格',
+  'Table': '表名',
   'Attachment': '附件',
   'SQL Parameter': 'sql参数',
   'SQL Statement': 'sql语句',

pom.xml

@@ -683,183 +683,70 @@
                 <version>${maven-surefire-plugin.version}</version>
                 <configuration>
                     <includes>
-                        <include>**/alert/template/AlertTemplateFactoryTest.java</include>
-                        <include>**/alert/template/impl/DefaultHTMLTemplateTest.java</include>
-                        <include>**/alert/utils/EnterpriseWeChatUtilsTest.java</include>
-                        <include>**/alert/utils/ExcelUtilsTest.java</include>
-                        <include>**/alert/utils/FuncUtilsTest.java</include>
-                        <include>**/alert/utils/JSONUtilsTest.java</include>
-                        <include>**/alert/utils/MailUtilsTest.java</include>
-                        <include>**/alert/utils/PropertyUtilsTest.java</include>
-                        <!--<include>**/api/controller/AccessTokenControllerTest.java</include>-->
-                        <!--<include>**/api/controller/AlertGroupControllerTest.java</include>-->
-                        <!--<include>**/api/controller/DataAnalysisControllerTest.java</include>-->
-                        <!--<include>**/api/controller/DataSourceControllerTest.java</include>-->
-                        <!--<include>**/api/controller/ExecutorControllerTest.java</include>-->
-                        <!--<include>**/api/controller/LoggerControllerTest.java</include>-->
-                        <!--<include>**/api/controller/LoginControllerTest.java</include>-->
-                        <!--<include>**/api/controller/MonitorControllerTest.java</include>-->
-                        <!--<include>**/api/controller/ProcessDefinitionControllerTest.java</include>-->
-                        <!--<include>**/api/controller/ProcessInstanceControllerTest.java</include>-->
-                        <!--<include>**/api/controller/ProjectControllerTest.java</include>-->
-                        <!--<include>**/api/controller/QueueControllerTest.java</include>-->
-                        <!--<include>**/api/controller/ResourcesControllerTest.java</include>-->
-                        <!--<include>**/api/controller/SchedulerControllerTest.java</include>-->
-                        <!--<include>**/api/controller/TaskInstanceControllerTest.java</include>-->
-                        <!--<include>**/api/controller/TaskRecordControllerTest.java</include>-->
-                        <!--<include>**/api/controller/TenantControllerTest.java</include>-->
-                        <!--<include>**/api/controller/UsersControllerTest.java</include>-->
-                        <!--<include>**/api/controller/WorkerGroupControllerTest.java</include>-->
-                        <include>**/api/dto/resources/filter/ResourceFilterTest.java</include>
-                        <include>**/api/dto/resources/visitor/ResourceTreeVisitorTest.java</include>
-                        <include>**/api/enums/testGetEnum.java</include>
-                        <include>**/api/enums/StatusTest.java</include>
-                        <include>**/api/interceptor/LoginHandlerInterceptorTest.java</include>
-                        <include>**/api/security/PasswordAuthenticatorTest.java</include>
-                        <include>**/api/security/SecurityConfigTest.java</include>
+                        <include>**/common/utils/*.java</include>
+                        <include>**/common/utils/process/ProcessBuilderForWin32Test.java</include>
+                        <include>**/common/utils/process/ProcessEnvironmentForWin32Test.java</include>
+                        <include>**/common/utils/process/ProcessImplForWin32Test.java</include>
+                        <include>**/common/log/*.java</include>
+                        <include>**/common/threadutils/*.java</include>
+                        <include>**/common/graph/*.java</include>
+                        <include>**/common/queue/*.java</include>
+                        <include>**/common/task/FlinkParametersTest.java</include>
+                        <include>**/common/task/SqoopParameterEntityTest.java</include>
+                        <include>**/api/utils/CheckUtilsTest.java</include>
+                        <include>**/api/utils/FileUtilsTest.java</include>
+                        <include>**/api/utils/FourLetterWordTest.java</include>
+                        <include>**/api/utils/exportprocess/DataSourceParamTest.java</include>
+                        <include>**/api/utils/exportprocess/DependentParamTest.java</include>
+                        <include>**/api/enums/*.java</include>
+                        <include>**/api/controller/DataSourceControllerTest.java</include>
                         <include>**/api/service/AccessTokenServiceTest.java</include>
+                        <include>**/api/service/QueueServiceTest.java</include>
+                        <include>**/api/service/MonitorServiceTest.java</include>
+                        <include>**/api/service/SessionServiceTest.java</include>
+                        <include>**/api/service/UsersServiceTest.java</include>
+                        <include>**/api/service/TenantServiceTest.java</include>
+                        <include>**/api/service/WorkerGroupServiceTest.java</include>
                         <include>**/api/service/AlertGroupServiceTest.java</include>
-                        <include>**/api/service/BaseDAGServiceTest.java</include>
-                        <include>**/api/service/BaseServiceTest.java</include>
-                        <include>**/api/service/DataAnalysisServiceTest.java</include>
-                        <include>**/api/service/DataSourceServiceTest.java</include>
+                        <include>**/api/service/UserAlertGroupServiceTest.java</include>
+                        <include>**/api/service/ProjectServiceTest.java</include>
+                        <include>**/api/service/ProcessDefinitionServiceTest.java</include>
+                        <include>**/api/service/UdfFuncServiceTest.java</include>
+                        <include>**/api/service/ResourcesServiceTest.java</include>
                         <include>**/api/service/ExecutorService2Test.java</include>
-                        <include>**/api/service/ExecutorServiceTest.java</include>
+                        <include>**/api/service/BaseServiceTest.java</include>
+                        <include>**/api/service/BaseDAGServiceTest.java</include>
                         <include>**/api/service/LoggerServiceTest.java</include>
-                        <include>**/api/service/MonitorServiceTest.java</include>
-                        <include>**/api/service/ProcessDefinitionServiceTest.java</include>
+                        <include>**/api/service/DataAnalysisServiceTest.java</include>
                         <include>**/api/service/ProcessInstanceServiceTest.java</include>
-                        <include>**/api/service/ProjectServiceTest.java</include>
-                        <include>**/api/service/QueueServiceTest.java</include>
-                        <include>**/api/service/ResourcesServiceTest.java</include>
-                        <include>**/api/service/SchedulerServiceTest.java</include>
-                        <include>**/api/service/SessionServiceTest.java</include>
                         <include>**/api/service/TaskInstanceServiceTest.java</include>
-                        <include>**/api/service/TenantServiceTest.java</include>
-                        <include>**/api/service/UdfFuncServiceTest.java</include>
-                        <include>**/api/service/UserAlertGroupServiceTest.java</include>
-                        <include>**/api/service/UsersServiceTest.java</include>
-                        <include>**/api/service/WorkerGroupServiceTest.java</include>
-                        <include>**/api/utils/exportprocess/DataSourceParamTest.java</include>
-                        <include>**/api/utils/exportprocess/DependentParamTest.java</include>
-                        <include>**/api/utils/CheckUtilsTest.java</include>
-                        <include>**/api/utils/FileUtilsTest.java</include>
-                        <include>**/api/utils/FourLetterWordMainTest.java</include>
-                        <include>**/api/utils/ZookeeperMonitorUtilsTest.java</include>
-                        <include>**/api/utils/CheckUtilsTest.java</include>
-                        <include>**/api/utils/CheckUtilsTest.java</include>
-                        <include>**/api/HttpClientTest.java</include>
-                        <include>**/common/graph/DAGTest.java</include>
-                        <include>**/common/os/OshiTest.java</include>
-                        <include>**/common/os/OSUtilsTest.java</include>
-                        <include>**/common/shell/ShellExecutorTest.java</include>
-                        <include>**/common/task/EntityTestUtils.java</include>
-                        <include>**/common/task/FlinkParametersTest.java</include>
-                        <include>**/common/task/SqoopParameterEntityTest.java</include>
-                        <include>**/common/threadutils/ThreadPoolExecutorsTest.java</include>
-                        <include>**/common/threadutils/ThreadUtilsTest.java</include>
-                        <include>**/common/utils/placeholder/TimePlaceholderUtilsTest.java</include>
-                        <include>**/common/utils/process/ProcessBuilderForWin32Test.java</include>
-                        <include>**/common/utils/process/ProcessEnvironmentForWin32Test.java</include>
-                        <include>**/common/utils/process/ProcessImplForWin32Test.java</include>
-                        <include>**/common/utils/CollectionUtilsTest.java</include>
-                        <include>**/common/utils/CommonUtilsTest.java</include>
-                        <include>**/common/utils/DateUtilsTest.java</include>
-                        <include>**/common/utils/DependentUtilsTest.java</include>
-                        <include>**/common/utils/EncryptionUtilsTest.java</include>
-                        <include>**/common/utils/FileUtilsTest.java</include>
-                        <include>**/common/utils/HadoopUtilsTest.java</include>
-                        <include>**/common/utils/IpUtilsTest.java</include>
-                        <include>**/common/utils/JSONUtilsTest.java</include>
-                        <include>**/common/utils/LoggerUtilsTest.java</include>
-                        <include>**/common/utils/OSUtilsTest.java</include>
-                        <include>**/common/utils/ParameterUtilsTest.java</include>
-                        <include>**/common/utils/PreconditionsTest.java</include>
-                        <include>**/common/utils/PropertyUtilsTest.java</include>
-                        <include>**/common/utils/SchemaUtilsTest.java</include>
-                        <include>**/common/utils/ScriptRunnerTest.java</include>
-                        <include>**/common/utils/SensitiveLogUtilsTest.java</include>
-                        <include>**/common/utils/StringTest.java</include>
-                        <include>**/common/utils/StringUtilsTest.java</include>
-                        <include>**/common/utils/TaskParametersUtilsTest.java</include>
-                        <include>**/common/ConstantsTest.java</include>
+                        <include>**/alert/utils/ExcelUtilsTest.java</include>
+                        <include>**/alert/utils/FuncUtilsTest.java</include>
+                        <include>**/alert/utils/JSONUtilsTest.java</include>
+                        <include>**/alert/utils/PropertyUtilsTest.java</include>
+                        <include>**/server/utils/SparkArgsUtilsTest.java</include>
+                        <include>**/server/utils/FlinkArgsUtilsTest.java</include>
+                        <include>**/server/utils/ParamUtilsTest.java</include>
+                        <include>**/server/master/MasterExecThreadTest.java</include>
                         <include>**/dao/mapper/AccessTokenMapperTest.java</include>
                         <include>**/dao/mapper/AlertGroupMapperTest.java</include>
                         <include>**/dao/mapper/AlertMapperTest.java</include>
                         <include>**/dao/mapper/CommandMapperTest.java</include>
-                        <include>**/dao/mapper/ConnectionFactoryTest.java</include>
-                        <include>**/dao/mapper/DataSourceMapperTest.java</include>
-                        <include>**/dao/mapper/DataSourceUserMapperTest.java</include>
-                        <include>**/dao/mapper/ErrorCommandMapperTest.java</include>
-                        <include>**/dao/mapper/ProcessDefinitionMapperTest.java</include>
-                        <include>**/dao/mapper/ProcessInstanceMapMapperTest.java</include>
-                        <include>**/dao/mapper/ProcessInstanceMapperTest.java</include>
-                        <include>**/dao/mapper/ProjectMapperTest.java</include>
-                        <include>**/dao/mapper/ProjectUserMapperTest.java</include>
-                        <include>**/dao/mapper/QueueMapperTest.java</include>
-                        <include>**/dao/mapper/ResourceMapperTest.java</include>
-                        <include>**/dao/mapper/ResourceUserMapperTest.java</include>
-                        <include>**/dao/mapper/ScheduleMapperTest.java</include>
-                        <include>**/dao/mapper/SessionMapperTest.java</include>
-                        <include>**/dao/mapper/TaskInstanceMapperTest.java</include>
-                        <include>**/dao/mapper/TenantMapperTest.java</include>
-                        <include>**/dao/mapper/UdfFuncMapperTest.java</include>
-                        <include>**/dao/mapper/UDFUserMapperTest.java</include>
-                        <include>**/dao/mapper/UserAlertGroupMapperTest.java</include>
-                        <include>**/dao/mapper/UserMapperTest.java</include>
-                        <include>**/dao/mapper/WorkerGroupMapperTest.java</include>
+                        <include>**/dao/entity/TaskInstanceTest.java</include>
+                        <include>**/dao/cron/CronUtilsTest.java</include>
                         <include>**/dao/utils/DagHelperTest.java</include>
-                        <include>**/dao/AlertDaoTest.java</include>
-                        <include>**/remote/FastJsonSerializerTest.java</include>
-                        <include>**/remote/NettyRemotingClientTest.java</include>
-                        <include>**/remote/ResponseFutureTest.java</include>
-                        <include>**/server/log/MasterLogFilterTest.java</include>
-                        <include>**/server/log/SensitiveDataConverterTest.java</include>
-                        <include>**/server/log/TaskLogDiscriminatorTest.java</include>
-                        <include>**/server/log/TaskLogFilterTest.java</include>
-                        <include>**/server/log/WorkerLogFilterTest.java</include>
-                        <include>**/server/master/dispatch/executor/NettyExecutorManagerTest.java</include>
-                        <include>**/server/master/dispatch/host/assign/LowerWeightRoundRobinTest.java</include>
-                        <include>**/server/master/dispatch/host/assign/RandomSelectorTest.java</include>
-                        <include>**/server/master/dispatch/host/assign/RoundRobinSelectorTest.java</include>
-                        <include>**/server/master/dispatch/host/RoundRobinHostManagerTest.java</include>
-                        <include>**/server/master/dispatch/ExecutorDispatcherTest.java</include>
-                        <include>**/server/master/register/MasterRegistryTest.java</include>
-                        <include>**/server/master/AlertManagerTest.java</include>
-                        <include>**/server/master/MasterCommandTest.java</include>
-                        <include>**/server/master/MasterExecThreadTest.java</include>
-                        <include>**/server/master/ParamsTest.java</include>
-                        <include>**/server/register/ZookeeperNodeManagerTest.java</include>
-                        <include>**/server/utils/DataxUtilsTest.java</include>
-                        <include>**/server/utils/ExecutionContextTestUtils.java</include>
-                        <include>**/server/utils/FlinkArgsUtilsTest.java</include>
-                        <include>**/server/utils/ParamUtilsTest.java</include>
-                        <include>**/server/utils/ProcessUtilsTest.java</include>
-                        <include>**/server/utils/SparkArgsUtilsTest.java</include>
-                        <include>**/server/worker/processor/TaskCallbackServiceTest.java</include>
-                        <include>**/server/worker/registry/WorkerRegistryTest.java</include>
-                        <include>**/server/worker/shell/ShellCommandExecutorTest.java</include>
-                        <include>**/server/worker/sql/SqlExecutorTest.java</include>
-                        <include>**/server/worker/task/datax/DataxTaskTest.java</include>
-                        <include>**/server/worker/task/dependent/DependentTaskTest.java</include>
-                        <include>**/server/worker/task/spark/SparkTaskTest.java</include>
-                        <include>**/server/worker/task/EnvFileTest.java</include>
+                        <include>**/alert/template/AlertTemplateFactoryTest.java</include>
+                        <include>**/alert/template/impl/DefaultHTMLTemplateTest.java</include>
                         <include>**/server/worker/task/datax/DataxTaskTest.java</include>
-                        <include>**/server/worker/task/dependent/DependentTaskTest.java</include>
                         <include>**/server/worker/task/shell/ShellTaskTest.java</include>
-                        <include>**/server/worker/task/spark/SparkTaskTest.java</include>
                         <include>**/server/worker/task/sqoop/SqoopTaskTest.java</include>
-                        <include>**/server/worker/EnvFileTest.java</include>
-                        <include>**/service/quartz/cron/CronUtilsTest.java</include>
+                        <include>**/server/utils/DataxUtilsTest.java</include>
                         <include>**/service/zk/DefaultEnsembleProviderTest.java</include>
-                        <include>**/service/zk/ZKServerTest.java</include>
-                        <include>**/service/queue/TaskUpdateQueueTest.java</include>
+                        <include>**/dao/datasource/BaseDataSourceTest.java</include>
+                        <include>**/alert/utils/MailUtilsTest.java</include>
+                        <include>**/dao/AlertDaoTest.java</include>
                     </includes>
                     <!-- <skip>true</skip> -->
-                    <argLine>-Xmx2048m</argLine>
-                    <threadCount>3</threadCount>
-                    <forkCount>3</forkCount>
-                    <reuseForks>true</reuseForks>
                 </configuration>
             </plugin>