update application.yml to application.properties (#980)

* update english documents

* refactor zk client

* update documents

* update zkclient

* update zkclient

* update documents

* add architecture-design

* change i18n

* update i18n

* update english documents

* add architecture-design

* update english documents

* update en-US documents

* add architecture-design

* update demo site

* add mybatis plus model

* modify mybatisplus

* modify mybatisplus

* change interface by mybatisplus

* add unit test

* refactor dao interface.

* add unit test for dao...

* add unit test for dao...

* add unit test for dao...

* Merge remote-tracking branch 'upstream/dev-db' into dev-db

# Conflicts:
#	dolphinscheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProjectMapper.xml
#	dolphinscheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ScheduleMapper.xml
#	escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProcessInstanceMapper.xml
#	escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProjectUserMapper.xml
#	escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/QueueMapper.xml
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ProcessInstanceMapperTest.java
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ProjectUserMapperTest.java
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/QueueMapperTest.java
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ResourceUserMapperTest.java
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ScheduleMapperTest.java
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/SessionMapperTest.java
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/TenantMapperTest.java

* Merge remote-tracking branch 'upstream/dev-db' into dev-db

# Conflicts:
#	dolphinscheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProjectMapper.xml
#	dolphinscheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ScheduleMapper.xml
#	escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProcessInstanceMapper.xml
#	escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/ProjectUserMapper.xml
#	escheduler-dao/src/main/resources/cn.escheduler.dao.mapper/QueueMapper.xml
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ProcessInstanceMapperTest.java
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ProjectUserMapperTest.java
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/QueueMapperTest.java
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ResourceUserMapperTest.java
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/ScheduleMapperTest.java
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/SessionMapperTest.java
#	escheduler-dao/src/test/java/cn/escheduler/dao/mapper/TenantMapperTest.java

* Merge remote-tracking branch 'upstream/dev-db' into dev-db

# Conflicts:
#	dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.xml
#	dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.xml
#	dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml

* update some dao bugs

* update for some bugs

* update some bugs

* Merge remote-tracking branch 'upstream/dev-db' into dev-db

# Conflicts:
#	dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.xml
#	dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.xml
#	dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml

* update

* update

* add multiply settings for application.yml

* add multiply settings for application.yml

* revert

* update configuration settings in task record dao...

* change application_master to application-master

* change application_master to application-master

* update application.yml to application.properties

* revert

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/MonitorDBDao.java

@@ -17,7 +17,6 @@
 package org.apache.dolphinscheduler.dao;
 
 import org.apache.dolphinscheduler.common.Constants;
-import org.apache.dolphinscheduler.dao.config.YmlConfig;
 import org.apache.dolphinscheduler.dao.entity.MonitorRecord;
 import org.apache.commons.configuration.Configuration;
 import org.apache.commons.configuration.ConfigurationException;
@@ -42,16 +41,16 @@ public class MonitorDBDao {
     /**
      * 加载配置文件
      */
-//    private static Configuration conf;
+    private static Configuration conf;
 
-//    static {
-//        try {
-//            conf = new PropertiesConfiguration(Constants.DATA_SOURCE_PROPERTIES);
-//        }catch (ConfigurationException e){
-//            logger.error("load configuration excetpion",e);
-//            System.exit(1);
-//        }
-//    }
+    static {
+        try {
+            conf = new PropertiesConfiguration(Constants.DATA_SOURCE_PROPERTIES);
+        }catch (ConfigurationException e){
+            logger.error("load configuration excetpion",e);
+            System.exit(1);
+        }
+    }
 
 
     /**
@@ -59,9 +58,9 @@ public class MonitorDBDao {
      * @return
      */
     private static Connection getConn() {
-        String url =  YmlConfig.allMap.get(Constants.SPRING_DATASOURCE_URL);
-        String username = YmlConfig.allMap.get(Constants.SPRING_DATASOURCE_USERNAME);
-        String password = YmlConfig.allMap.get(Constants.SPRING_DATASOURCE_PASSWORD);
+        String url =  conf.getString(Constants.SPRING_DATASOURCE_URL);
+        String username = conf.getString(Constants.SPRING_DATASOURCE_USERNAME);
+        String password = conf.getString(Constants.SPRING_DATASOURCE_PASSWORD);
         Connection conn = null;
         try {
             //classloader,load driver

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/ProcessDao.java

@@ -231,7 +231,7 @@ public class ProcessDao extends AbstractBaseDao {
             JSONObject tempObj;
             int processInstanceId = cmdParamObj.getInteger(CMDPARAM_RECOVER_PROCESS_ID_STRING);
 
-            List<Command> commands = commandMapper.getAll(null);
+            List<Command> commands = commandMapper.selectList(null);
             //遍历所有命令
             for (Command tmpCommand:commands){
                 if(cmdTypeMap.containsKey(tmpCommand.getCommandType())){

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/TaskRecordDao.java

@@ -42,10 +42,8 @@ public class TaskRecordDao {
 
     private static Logger logger = LoggerFactory.getLogger(TaskRecordDao.class.getName());
 
-
-
     /**
-     * load conf file
+     * 加载配置文件
      */
     private static Configuration conf;
 
@@ -58,7 +56,6 @@ public class TaskRecordDao {
         }
     }
 
-
     /**
      *  get task record flag
      * @return
@@ -71,7 +68,7 @@ public class TaskRecordDao {
      * @return
      */
     private static Connection getConn() {
-        if(!conf.getBoolean(Constants.TASK_RECORD_FLAG)){
+        if(!getTaskRecordFlag()){
             return null;
         }
         String driver = "com.mysql.jdbc.Driver";

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/config/YmlConfig.java

@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.dolphinscheduler.dao.config;
-
-
-
-import org.yaml.snakeyaml.*;
-
-import java.io.InputStream;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.Map;
-
-public class YmlConfig {
-
-    public static Map<String,String> allMap=new HashMap<String,String>();
-    static {
-        Yaml yaml = new Yaml();
-        InputStream inputStream = YmlConfig.class.getResourceAsStream("/application.yml");
-        Iterator<Object> result = yaml.loadAll(inputStream).iterator();
-        while(result.hasNext()){
-            Map map=(Map)result.next();
-            iteratorYml( map,null);
-        }
-    }
-
-    public static void iteratorYml(Map map,String key){
-        Iterator iterator = map.entrySet().iterator();
-        while(iterator.hasNext()){
-            Map.Entry entry = (Map.Entry) iterator.next();
-            Object key2 = entry.getKey();
-            Object value = entry.getValue();
-            if(value instanceof LinkedHashMap){
-                if(key==null){
-                    iteratorYml((Map)value,key2.toString());
-                }else{
-                    iteratorYml((Map)value,key+"."+key2.toString());
-                }
-            }
-            if(value instanceof String){
-                if(key==null){
-                    allMap.put(key2.toString(), value.toString());
-                }
-                if(key!=null){
-                    allMap.put(key+"."+key2.toString(), value.toString());
-                }
-            }
-        }
-
-    }
-
-}

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/ConnectionFactory.java

@@ -17,7 +17,9 @@
 package org.apache.dolphinscheduler.dao.datasource;
 
 import com.alibaba.druid.pool.DruidDataSource;
-import org.apache.dolphinscheduler.dao.config.YmlConfig;
+import org.apache.commons.configuration.ConfigurationException;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.dolphinscheduler.common.Constants;
 import org.apache.ibatis.mapping.Environment;
 import org.apache.ibatis.session.Configuration;
 import org.apache.ibatis.session.SqlSession;
@@ -30,7 +32,6 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import javax.sql.DataSource;
-import java.util.Map;
 
 
 /**
@@ -41,16 +42,30 @@ public class ConnectionFactory {
 
   private static SqlSessionFactory sqlSessionFactory;
 
+  /**
+   * 加载配置文件
+   */
+  private static org.apache.commons.configuration.Configuration conf;
+
+  static {
+    try {
+      conf = new PropertiesConfiguration(Constants.DATA_SOURCE_PROPERTIES);
+    }catch (ConfigurationException e){
+      logger.error("load configuration excetpion",e);
+      System.exit(1);
+    }
+  }
+
   /**
    * get the data source
    */
   public static DruidDataSource getDataSource() {
     DruidDataSource druidDataSource = new DruidDataSource();
-    Map<String, String> allMap = YmlConfig.allMap;
-    druidDataSource.setDriverClassName(allMap.get("spring.datasource.driver-class-name"));
-    druidDataSource.setUrl(allMap.get("spring.datasource.url"));
-    druidDataSource.setUsername(allMap.get("spring.datasource.username"));
-    druidDataSource.setPassword(allMap.get("spring.datasource.password"));
+//    Map<String, String> allMap = YmlConfig.allMap;
+    druidDataSource.setDriverClassName(conf.getString("spring.datasource.driver-class-name"));
+    druidDataSource.setUrl(conf.getString("spring.datasource.url"));
+    druidDataSource.setUsername(conf.getString("spring.datasource.username"));
+    druidDataSource.setPassword(conf.getString("spring.datasource.password"));
     druidDataSource.setInitialSize(5);
     druidDataSource.setMinIdle(5);
     druidDataSource.setMaxActive(20);

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/CommandMapper.java

@@ -30,10 +30,6 @@ import java.util.List;
 public interface CommandMapper extends BaseMapper<Command> {
 
 
-
-    @Select("select * from t_ds_command ${ew.customSqlSegment}")
-    List<Command> getAll(@Param(Constants.WRAPPER) Wrapper wrapper);
-
     Command getOneToRun();
 
     List<CommandCount> countCommandState(

dolphinscheduler-dao/src/main/resources/application.yml

@@ -1,91 +0,0 @@
-# mysql
-spring:
-
-    datasource:
-        url: jdbc:mysql://192.168.220.188:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8
-        username: root
-        password: root@123
-        #
-        type: com.alibaba.druid.pool.DruidDataSource
-        driver-class-name: com.mysql.jdbc.Driver
-        initialSize: 5
-        minIdle: 5
-        maxActive: 20
-        maxWait: 60000
-        timeBetweenEvictionRunsMillis: 60000
-        minEvictableIdleTimeMillis: 300000
-        validationQuery: SELECT 1 FROM DUAL
-        testWhileIdle: true
-        testOnBorrow: false
-        testOnReturn: false
-        poolPreparedStatements: true
-        maxPoolPreparedStatementPerConnectionSize: 20
-        spring.datasource.filters: stat,wall,log4j
-        connectionProperties: druid.stat.mergeSql=true;druid.stat.slowSqlMillis=5000
-
-#    jpa:
-#      hibernate:
-#        ddl-auto : update
-#    datasource:
-#      platform:
-#      continue-on-error: true
-#      druid:
-#    driver-class-name: org.postgresql.Driver
-#    url: jdbc:postgresql://192.168.220.154:5432/escheduler
-
-
-
-#mybatis
-mybatis-plus:
-  mapper-locations: classpath*:/org.apache.dolphinscheduler.dao.mapper/*.xml
-  typeEnumsPackage: org.apache.dolphinscheduler.*.enums
-  #实体扫描，多个package用逗号或者分号分隔
-  typeAliasesPackage: org.apache.dolphinscheduler.dao.entity
-  global-config:
-    #数据库相关配置
-    db-config:
-      #主键类型  AUTO:"数据库ID自增", INPUT:"用户输入ID", ID_WORKER:"全局唯一ID (数字类型唯一ID)", UUID:"全局唯一ID UUID";
-      id-type: AUTO
-      #字段策略 IGNORED:"忽略判断",NOT_NULL:"非 NULL 判断"),NOT_EMPTY:"非空判断"
-      field-strategy: NOT_NULL
-      #驼峰下划线转换
-      column-underline: true
-      logic-delete-value: -1
-      logic-not-delete-value: 0
-    banner: false
-  #原生配置
-  configuration:
-    map-underscore-to-camel-case: true
-    cache-enabled: false
-    call-setters-on-nulls: true
-    jdbc-type-for-null: 'null'
-
-
-# Logger Config
-logging:
-  level:
-    org.apache.dolphinscheduler.dao: debug
-
----
-# master settings
-spring:
-  profiles: master
-
-server:
-    port: 5566
-
----
-# worker settings
-spring:
-  profiles: worker
-
-server:
-  port: 7788
-
----
-# alert settings
-spring:
-  profiles: alert
-
-server:
-  port: 7789

script/dolphinscheduler-daemon.sh

@@ -44,13 +44,13 @@ if [ "$command" = "api-server" ]; then
   LOG_FILE="-Dlogging.config=conf/apiserver_logback.xml"
   CLASS=org.apache.dolphinscheduler.api.ApiApplicationServer
 elif [ "$command" = "master-server" ]; then
-  LOG_FILE="-Dspring.profiles.active=master -Dlogging.config=conf/master_logback.xml -Ddruid.mysql.usePingMethod=false"
+  LOG_FILE="-Dspring.profiles.active=master -Ddruid.mysql.usePingMethod=false"
   CLASS=org.apache.dolphinscheduler.server.master.MasterServer
 elif [ "$command" = "worker-server" ]; then
-  LOG_FILE="-Dspring.profiles.active=worker -Dlogging.config=conf/worker_logback.xml -Ddruid.mysql.usePingMethod=false"
+  LOG_FILE="-Dspring.profiles.active=worker -Ddruid.mysql.usePingMethod=false"
   CLASS=org.apache.dolphinscheduler.server.worker.WorkerServer
 elif [ "$command" = "alert-server" ]; then
-  LOG_FILE="-Dspring.profiles.active=alert -Dlogging.config=conf/alert_logback.xml -Ddruid.mysql.usePingMethod=false"
+  LOG_FILE="-Dspring.profiles.active=alert -Ddruid.mysql.usePingMethod=false"
   CLASS=org.apache.dolphinscheduler.alert.AlertServer
 elif [ "$command" = "logger-server" ]; then
   CLASS=org.apache.dolphinscheduler.server.rpc.LoggerServer