[Refactor worker] correct config error and remove unuse properties that may affect performance (#2337)

* update logback

* update log

* refactor worker registry (#2107)

* Refactor worker (#2115)

* refactor worker registry

* refactor master server

* Modify workgroupid parameter name (#2105)

* Delete worker group management page

* Modify workgroupid parameter name

* Refactor worker (#2121)

* refactor worker registry

* refactor master server

* refactor MasterSchedulerService

* cancelTaskInstance set TaskExecutionContext host,logPath,executePath (#2126)

* 1, master persistent task
2. extract  master and worker communication model

* 1, master persistent task
2. extract  master and worker communication model

* 1, master persistent task
2. extract  master and worker communication model

* add license

* modify javadoc error

* TaskExecutionContext create modify

* buildAckCommand taskInstanceId not set modify

* java doc error modify

* add comment

* ExecutorManager interface add generic type

* add TaskInstanceCacheManager receive Worker report result

* TaskInstance setExecutePath

* add TaskInstanceCacheManager to receive Worker Task result report

* TaskInstanceCacheManager add remove method

* add license

* add dispatcht task method

* AbstractCommandExecutor remove db access

* AbstractCommandExecutor remove db access

* AbstractCommandExecutor remove db access

* AbstractCommandExecutor remove db access

* AbstractCommandExecutor remove db access

* AbstractCommandExecutor remove db access

* AbstractCommandExecutor remove db access

* taskInstanceCache is null ,need load from db

* taskInstanceCache is null ,need load from db

* taskInstanceCache is null ,need load from db

* 1，worker TaskPros use TaskExecutionContext replase
2，Master kill Task , KillTaskProcessor modify

* worker remove db

* ShellTask modify

* master persistence processId and appIds

* master persistence processId and appIds

* master add kill task logic

* master add kill task logic

* master add kill task logic

* javadoc error modify

* remove chinese log

* executeDirectly method add Override

* remote module modify

* TaskKillResponseProcessor command type modify

* create buildKillCommand

* host add host:port format

* host add host:port format

* TaskAckProcessor modify

* TaskAckProcessor modify

* task prioriry refator

* remove ITaskQueue

* task prioriry refator

* remove ITaskQueue

* TaskPriority refactor

* remove logs

* WorkerServer refactor

* MasterSchedulerService modify

* WorkerConfig listen port modify

* modify master and worker listen port

* cancelTaskInstance set TaskExecutionContext host,logPath,executePath

* cancelTaskInstance set TaskExecutionContext host,logPath,executePath

Co-authored-by: qiaozhanwei <qiaozhanwei@analysys.com.cn>

* not exist in openjdk,just delete

* add master and worker properties

* add master and worker properties

* add master and worker properties

* fix cpu 100% bug

* simplify master、 worker、alert、dao properties

* add master and worker properties

* add master and worker properties

* add master and worker properties

* Optimize code , reduce cost time from 60ms to 0.5ms

* merge code form remote

* remove zookeeper as task queue relevant config

* correct config error and remove unuse properties that may affect performance

* correct config error and remove unuse properties that may affect performance

Co-authored-by: Tboy <guo.jiwei@immomo.com>
Co-authored-by: break60 <790061044@qq.com>
Co-authored-by: qiaozhanwei <qiaozhanwei@outlook.com>
Co-authored-by: qiaozhanwei <qiaozhanwei@analysys.com.cn>

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java

@@ -47,6 +47,7 @@ public final class Constants {
     public static final String ORG_QUARTZ_JOBSTORE_ISCLUSTERED = "org.quartz.jobStore.isClustered";
     public static final String ORG_QUARTZ_JOBSTORE_MISFIRETHRESHOLD = "org.quartz.jobStore.misfireThreshold";
     public static final String ORG_QUARTZ_JOBSTORE_CLUSTERCHECKININTERVAL = "org.quartz.jobStore.clusterCheckinInterval";
+    public static final String ORG_QUARTZ_JOBSTORE_ACQUIRETRIGGERSWITHINLOCK = "org.quartz.jobStore.acquireTriggersWithinLock";
     public static final String ORG_QUARTZ_JOBSTORE_DATASOURCE = "org.quartz.jobStore.dataSource";
     public static final String ORG_QUARTZ_DATASOURCE_MYDS_CONNECTIONPROVIDER_CLASS = "org.quartz.dataSource.myDs.connectionProvider.class";
 
@@ -61,6 +62,7 @@ public final class Constants {
     public static final String QUARTZ_THREADPRIORITY = "5";
     public static final String QUARTZ_INSTANCENAME = "DolphinScheduler";
     public static final String QUARTZ_INSTANCEID = "AUTO";
+    public static final String QUARTZ_ACQUIRETRIGGERSWITHINLOCK = "true";
 
     /**
      * common properties path
@@ -151,6 +153,11 @@ public final class Constants {
      */
     public static final String STRING_TRUE = "true";
 
+    /**
+     * string true
+     */
+    public static final String STRING_FALSE = "false";
+
     /**
      * resource storage type
      */

dolphinscheduler-common/src/main/resources/common.properties

@@ -46,7 +46,7 @@ resource.storage.type=HDFS
 hdfs.root.user=hdfs
 
 # if resource.storage.type=S3，the value like: s3a://dolphinscheduler ; if resource.storage.type=HDFS, When namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir
-fs.defaultFS=hdfs://l:8020
+fs.defaultFS=hdfs://mycluster:8020
 
 # if resource.storage.type=S3，s3 endpoint
 #fs.s3a.endpoint=http://192.168.199.91:9010

dolphinscheduler-dao/src/main/resources/datasource.properties

@@ -67,9 +67,4 @@ spring.datasource.password=root@123
 
 # open PSCache, specify count PSCache for every connection
 #spring.datasource.poolPreparedStatements=true
-#spring.datasource.maxPoolPreparedStatementPerConnectionSize=20
-
-#spring.datasource.filters=stat,wall,log4j
-#spring.datasource.connectionProperties=druid.stat.mergeSql=true;druid.stat.slowSqlMillis=5000
-
-
+#spring.datasource.maxPoolPreparedStatementPerConnectionSize=20

dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java

@@ -118,7 +118,7 @@ public class QuartzExecutors {
       properties.setProperty(ORG_QUARTZ_SCHEDULER_INSTANCENAME, conf.getString(ORG_QUARTZ_SCHEDULER_INSTANCENAME, QUARTZ_INSTANCENAME));
       properties.setProperty(ORG_QUARTZ_SCHEDULER_INSTANCEID, conf.getString(ORG_QUARTZ_SCHEDULER_INSTANCEID, QUARTZ_INSTANCEID));
       properties.setProperty(ORG_QUARTZ_SCHEDULER_MAKESCHEDULERTHREADDAEMON,conf.getString(ORG_QUARTZ_SCHEDULER_MAKESCHEDULERTHREADDAEMON,STRING_TRUE));
-      properties.setProperty(ORG_QUARTZ_JOBSTORE_USEPROPERTIES,conf.getString(ORG_QUARTZ_JOBSTORE_USEPROPERTIES,STRING_TRUE));
+      properties.setProperty(ORG_QUARTZ_JOBSTORE_USEPROPERTIES,conf.getString(ORG_QUARTZ_JOBSTORE_USEPROPERTIES,STRING_FALSE));
       properties.setProperty(ORG_QUARTZ_THREADPOOL_CLASS,conf.getString(ORG_QUARTZ_THREADPOOL_CLASS, SimpleThreadPool.class.getName()));
       properties.setProperty(ORG_QUARTZ_THREADPOOL_MAKETHREADSDAEMONS,conf.getString(ORG_QUARTZ_THREADPOOL_MAKETHREADSDAEMONS,STRING_TRUE));
       properties.setProperty(ORG_QUARTZ_THREADPOOL_THREADCOUNT,conf.getString(ORG_QUARTZ_THREADPOOL_THREADCOUNT, QUARTZ_THREADCOUNT));
@@ -128,6 +128,7 @@ public class QuartzExecutors {
       properties.setProperty(ORG_QUARTZ_JOBSTORE_ISCLUSTERED,conf.getString(ORG_QUARTZ_JOBSTORE_ISCLUSTERED,STRING_TRUE));
       properties.setProperty(ORG_QUARTZ_JOBSTORE_MISFIRETHRESHOLD,conf.getString(ORG_QUARTZ_JOBSTORE_MISFIRETHRESHOLD, QUARTZ_MISFIRETHRESHOLD));
       properties.setProperty(ORG_QUARTZ_JOBSTORE_CLUSTERCHECKININTERVAL,conf.getString(ORG_QUARTZ_JOBSTORE_CLUSTERCHECKININTERVAL, QUARTZ_CLUSTERCHECKININTERVAL));
+      properties.setProperty(ORG_QUARTZ_JOBSTORE_ACQUIRETRIGGERSWITHINLOCK,conf.getString(ORG_QUARTZ_JOBSTORE_ACQUIRETRIGGERSWITHINLOCK, QUARTZ_ACQUIRETRIGGERSWITHINLOCK));
       properties.setProperty(ORG_QUARTZ_JOBSTORE_DATASOURCE,conf.getString(ORG_QUARTZ_JOBSTORE_DATASOURCE, QUARTZ_DATASOURCE));
       properties.setProperty(ORG_QUARTZ_DATASOURCE_MYDS_CONNECTIONPROVIDER_CLASS,conf.getString(ORG_QUARTZ_DATASOURCE_MYDS_CONNECTIONPROVIDER_CLASS,DruidConnectionProvider.class.getName()));
 

dolphinscheduler-service/src/main/resources/quartz.properties

@@ -45,7 +45,7 @@
 #org.quartz.jobStore.isClustered = true
 #org.quartz.jobStore.misfireThreshold = 60000
 #org.quartz.jobStore.clusterCheckinInterval = 5000
-org.quartz.jobStore.acquireTriggersWithinLock=true
+#org.quartz.jobStore.acquireTriggersWithinLock=true
 #org.quartz.jobStore.dataSource = myDs
 
 #============================================================================