Merge pull request #1904 from lfyee/dev

Remove ScheduleUtil，use the existing CronUtils

dolphinscheduler-api/pom.xml

@@ -43,10 +43,6 @@
       <groupId>org.apache.dolphinscheduler</groupId>
       <artifactId>dolphinscheduler-dao</artifactId>
     </dependency>
-    <dependency>
-      <groupId>org.apache.dolphinscheduler</groupId>
-      <artifactId>dolphinscheduler-server</artifactId>
-    </dependency>
 
     <!--springboot-->
     <dependency>

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java

@@ -30,7 +30,7 @@ import org.apache.dolphinscheduler.dao.entity.*;
 import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper;
 import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
 import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
-import org.apache.dolphinscheduler.server.utils.ScheduleUtils;
+import org.apache.dolphinscheduler.dao.utils.cron.CronUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -512,8 +512,7 @@ public class ExecutorService extends BaseService{
                     List<Date> listDate = new LinkedList<>();
                     if(!CollectionUtils.isEmpty(schedules)){
                         for (Schedule item : schedules) {
-                            List<Date> list = ScheduleUtils.getRecentTriggerTime(item.getCrontab(), start, end);
-                            listDate.addAll(list);
+                            listDate.addAll(CronUtils.getSelfFireDateList(start, end, item.getCrontab()));
                         }
                     }
                     if(!CollectionUtils.isEmpty(listDate)){

dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ExecutorService2Test.java

@@ -203,7 +203,7 @@ public class ExecutorService2Test {
                     "", "", RunMode.RUN_MODE_PARALLEL,
                     Priority.LOW, 0, 110);
             Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
-            verify(processDao, times(16)).createCommand(any(Command.class));
+            verify(processDao, times(15)).createCommand(any(Command.class));
         }catch (Exception e){
             Assert.assertTrue(false);
         }

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/cron/CronUtils.java

@@ -156,6 +156,23 @@ public class CronUtils {
     return dateList;
   }
 
+  /**
+   * gets all scheduled times for a period of time based on self dependency
+   * @param startTime startTime
+   * @param endTime endTime
+   * @param cron cron
+   * @return date list
+   */
+  public static List<Date> getSelfFireDateList(Date startTime, Date endTime, String cron) {
+    CronExpression cronExpression = null;
+    try {
+      cronExpression = parse2CronExpression(cron);
+    }catch (ParseException e){
+      logger.error(e.getMessage(), e);
+      return Collections.EMPTY_LIST;
+    }
+    return getSelfFireDateList(startTime, endTime, cronExpression);
+  }
 
   /**
    * get expiration time

dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/cron/CronUtilsTest.java

@@ -17,6 +17,7 @@
 package org.apache.dolphinscheduler.dao.cron;
 
 import org.apache.dolphinscheduler.common.enums.CycleEnum;
+import org.apache.dolphinscheduler.common.utils.DateUtils;
 import org.apache.dolphinscheduler.dao.utils.cron.CronUtils;
 import com.cronutils.builder.CronBuilder;
 import com.cronutils.model.Cron;
@@ -31,10 +32,12 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.text.ParseException;
+import java.util.Date;
 
 import static com.cronutils.model.field.expression.FieldExpressionFactory.*;
 
 /**
+ * CronUtilsTest
  */
 public class CronUtilsTest {
 
@@ -55,8 +58,9 @@ public class CronUtilsTest {
                 .withSecond(on(0))
                 .instance();
         // Obtain the string expression
-        String cronAsString = cron.asString(); // 0 */5 * * * ? *  Every five minutes(once every 5 minutes)
+        String cronAsString = cron.asString();
 
+        // 0 */5 * * * ? *  Every five minutes(once every 5 minutes)
         Assert.assertEquals(cronAsString, "0 */5 * * * ? *");
     }
 
@@ -68,9 +72,6 @@ public class CronUtilsTest {
     @Test
     public void testCronParse() throws ParseException {
         String strCrontab = "0 1 2 3 * ? *";
-        strCrontab = "0/50 0/59 * * * ? *";
-        strCrontab = "3/5 * 0/5 * * ? *";
-        strCrontab = "1/5 3/5 1/5 3/30 * ? *";
 
         Cron depCron = CronUtils.parse2Cron(strCrontab);
         Assert.assertEquals(depCron.retrieve(CronFieldName.SECOND).getExpression().asString(), "0");
@@ -87,12 +88,14 @@ public class CronUtilsTest {
      */
     @Test
     public void testScheduleType() throws ParseException {
-
-        CycleEnum cycleEnum = CronUtils.getMaxCycle("0 */1 * * * ? *");
+        CycleEnum cycleEnum = CronUtils.getMaxCycle(CronUtils.parse2Cron("0 */1 * * * ? *"));
         Assert.assertEquals(cycleEnum.name(), "MINUTE");
 
         CycleEnum cycleEnum2 = CronUtils.getMaxCycle("0 * * * * ? *");
         Assert.assertEquals(cycleEnum2.name(), "MINUTE");
+
+        CycleEnum cycleEnum3 = CronUtils.getMiniCycle(CronUtils.parse2Cron("0 * * * * ? *"));
+        Assert.assertEquals(cycleEnum3.name(), "MINUTE");
     }
 
     /**
@@ -109,26 +112,9 @@ public class CronUtilsTest {
                 .withMinute(every(5))
                 .withSecond(on(0))
                 .instance();
-
-        String cronAsString = cron1.asString(); // 0 */5 * * * ? * once every 5 minutes
-        //logger.info(cronAsString);
-        // Obtain the string expression
-        //String minCrontab = "0 0 * * * ? *";
-        //String minCrontab = "0 0 10,14,16 * * ?";
-        //String minCrontab = "0 0-5 14 * * ? *";
-        //String minCrontab = "0 0 2 ? * SUN *";
-        //String minCrontab = "* 0,3 2 SUN * 1#1 *";
-        //String minCrontab = "* 0,3 * 1W * ? *";
-        //cron = CronUtils.parse2Cron("0 * * * * ? *");
-        // month cycle
-        /*String[] cronArayy = new String[]{"* 0,3 * 1W * ? *","* 0 0 1W * ? *",
-                "0 0 0 L 3/5 ? *","0 0 0 ? 3/5 2/2 *"};*/
         // minute cycle
         String[] cronArayy = new String[]{"* * * * * ? *","* 0 * * * ? *",
                 "* 5 * * 3/5 ? *","0 0 * * * ? *"};
-        // week cycle
-        /*String[] cronArayy = new String[]{"* * * ? * 2/1 *","0 *//*5 * ? * 2/1 *",
-                "* * *//*5 ? * 2/1 *"};*/
         for(String minCrontab:cronArayy){
             if (!org.quartz.CronExpression.isValidExpression(minCrontab)) {
                 throw new RuntimeException(minCrontab+" verify failure, cron expression not valid");
@@ -171,7 +157,6 @@ public class CronUtilsTest {
             logger.info("dayOfWeekField instanceof And:"+(dayOfWeekField.getExpression() instanceof And));
             logger.info("dayOfWeekField instanceof QuestionMark:"+(dayOfWeekField.getExpression() instanceof QuestionMark));
 
-
             CycleEnum cycleEnum = CronUtils.getMaxCycle(minCrontab);
             if(cycleEnum !=null){
                 logger.info(cycleEnum.name());
@@ -180,4 +165,34 @@ public class CronUtilsTest {
             }
         }
     }
-}
+
+    @Test
+    public void getSelfFireDateList() throws ParseException{
+        Date from = DateUtils.stringToDate("2020-01-01 00:00:00");
+        Date to = DateUtils.stringToDate("2020-01-31 00:00:00");
+        // test date
+        Assert.assertEquals(0, CronUtils.getSelfFireDateList(to, from, "0 0 0 * * ? ").size());
+        // test error cron
+        Assert.assertEquals(0, CronUtils.getSelfFireDateList(from, to, "0 0 0 * *").size());
+        // test cron
+        Assert.assertEquals(29, CronUtils.getSelfFireDateList(from, to, "0 0 0 * * ? ").size());
+        // test other
+        Assert.assertEquals(30, CronUtils.getFireDateList(from, to, CronUtils.parse2CronExpression("0 0 0 * * ? ")).size());
+        Assert.assertEquals(5, CronUtils.getSelfFireDateList(from, to, CronUtils.parse2CronExpression("0 0 0 * * ? "), 5).size());
+    }
+
+    @Test
+    public void getExpirationTime(){
+        Date startTime = DateUtils.stringToDate("2020-02-07 18:30:00");
+        Date expirationTime = CronUtils.getExpirationTime(startTime, CycleEnum.HOUR);
+        Assert.assertEquals("2020-02-07 19:30:00", DateUtils.dateToString(expirationTime));
+        expirationTime = CronUtils.getExpirationTime(startTime, CycleEnum.DAY);
+        Assert.assertEquals("2020-02-07 23:59:59", DateUtils.dateToString(expirationTime));
+        expirationTime = CronUtils.getExpirationTime(startTime, CycleEnum.WEEK);
+        Assert.assertEquals("2020-02-07 23:59:59", DateUtils.dateToString(expirationTime));
+        expirationTime = CronUtils.getExpirationTime(startTime, CycleEnum.MONTH);
+        Assert.assertEquals("2020-02-07 23:59:59", DateUtils.dateToString(expirationTime));
+        expirationTime = CronUtils.getExpirationTime(startTime, CycleEnum.YEAR);
+        Assert.assertEquals("2020-02-07 18:30:00", DateUtils.dateToString(expirationTime));
+    }
+}

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java

@@ -33,9 +33,9 @@ import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
 import org.apache.dolphinscheduler.dao.entity.Schedule;
 import org.apache.dolphinscheduler.dao.entity.TaskInstance;
 import org.apache.dolphinscheduler.dao.utils.DagHelper;
+import org.apache.dolphinscheduler.dao.utils.cron.CronUtils;
 import org.apache.dolphinscheduler.server.master.config.MasterConfig;
 import org.apache.dolphinscheduler.server.utils.AlertManager;
-import org.apache.dolphinscheduler.server.utils.ScheduleUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -213,8 +213,7 @@ public class MasterExecThread implements Runnable {
         List<Date> listDate = Lists.newLinkedList();
         if(!CollectionUtils.isEmpty(schedules)){
             for (Schedule schedule : schedules) {
-                List<Date> list = ScheduleUtils.getRecentTriggerTime(schedule.getCrontab(), startDate, endDate);
-                listDate.addAll(list);
+                listDate.addAll(CronUtils.getSelfFireDateList(startDate, endDate, schedule.getCrontab()));
             }
         }
         // get first fire date

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ScheduleUtils.java

@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.dolphinscheduler.server.utils;
-
-import org.quartz.impl.triggers.CronTriggerImpl;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.text.ParseException;
-import java.util.Date;
-import java.util.LinkedList;
-import java.util.List;
-
-/**
- * ScheduleUtils
- */
-public class ScheduleUtils {
-
-    private static final Logger logger = LoggerFactory.getLogger(ScheduleUtils.class);
-
-    /**
-     * Get the execution time of the time interval
-     * @param cron
-     * @param from
-     * @param to
-     * @return
-     */
-    public static List<Date> getRecentTriggerTime(String cron, Date from, Date to) {
-        return getRecentTriggerTime(cron, Integer.MAX_VALUE, from, to);
-    }
-
-    /**
-     * Get the execution time of the time interval
-     * @param cron
-     * @param size
-     * @param from
-     * @param to
-     * @return
-     */
-    public static List<Date> getRecentTriggerTime(String cron, int size, Date from, Date to) {
-        List list = new LinkedList<Date>();
-        if(to.before(from)){
-            logger.error("schedule date from:{} must before date to:{}!", from, to);
-            return list;
-        }
-        try {
-            CronTriggerImpl trigger = new CronTriggerImpl();
-            trigger.setCronExpression(cron);
-            trigger.setStartTime(from);
-            trigger.setEndTime(to);
-            trigger.computeFirstFireTime(null);
-            for (int i = 0; i < size; i++) {
-                Date schedule = trigger.getNextFireTime();
-                if(null == schedule){
-                    break;
-                }
-                list.add(schedule);
-                trigger.triggered(null);
-            }
-        } catch (ParseException e) {
-            logger.error("cron:{} error:{}", cron, e.getMessage());
-        }
-        return java.util.Collections.unmodifiableList(list);
-    }
-}

dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java

@@ -134,7 +134,7 @@ public class MasterExecThreadTest {
             method.setAccessible(true);
             method.invoke(masterExecThread);
             // one create save, and 15(1 to 31 step 2) for next save, and last day 31 no save
-            verify(processDao, times(16)).saveProcessInstance(processInstance);
+            verify(processDao, times(15)).saveProcessInstance(processInstance);
         }catch (Exception e){
             Assert.assertTrue(false);
         }

dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ScheduleUtilsTest.java

@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server.utils;
-
-import org.apache.dolphinscheduler.common.utils.DateUtils;
-import org.junit.Test;
-import java.util.Date;
-import static org.junit.Assert.assertEquals;
-
-/**
- * Test ScheduleUtils
- */
-public class ScheduleUtilsTest {
-
-    /**
-     * Test the getRecentTriggerTime method
-     */
-    @Test
-    public void testGetRecentTriggerTime() {
-        Date from = DateUtils.stringToDate("2020-01-01 00:00:00");
-        Date to = DateUtils.stringToDate("2020-01-31 01:00:00");
-        // test date
-        assertEquals(0, ScheduleUtils.getRecentTriggerTime("0 0 0 * * ? ", to, from).size());
-        // test error cron
-        assertEquals(0, ScheduleUtils.getRecentTriggerTime("0 0 0 * *", from, to).size());
-        // test cron
-        assertEquals(31, ScheduleUtils.getRecentTriggerTime("0 0 0 * * ? ", from, to).size());
-    }
-}

pom.xml

@@ -706,12 +706,12 @@
                         <include>**/server/utils/SparkArgsUtilsTest.java</include>
                         <include>**/server/utils/FlinkArgsUtilsTest.java</include>
                         <include>**/server/utils/ParamUtilsTest.java</include>
-                        <include>**/server/utils/ScheduleUtilsTest.java</include>
                         <include>**/server/master/MasterExecThreadTest.java</include>
                         <include>**/dao/mapper/AccessTokenMapperTest.java</include>
                         <include>**/dao/mapper/AlertGroupMapperTest.java</include>
                         <include>**/dao/mapper/AlertMapperTest.java</include>
                         <include>**/dao/mapper/CommandMapperTest.java</include>
+                        <include>**/dao/cron/CronUtilsTest.java</include>
                         <include>**/alert/template/AlertTemplateFactoryTest.java</include>
                         <include>**/alert/template/impl/DefaultHTMLTemplateTest.java</include>
                         <include>**/server/worker/task/datax/DataxTaskTest.java</include>