add embedded database h2 and update embedded zookeeper (#2082)

add embedded database h2 and update embedded zookeeper

dolphinscheduler-dao/pom.xml

@@ -44,6 +44,12 @@
 			<groupId>com.baomidou</groupId>
 			<artifactId>mybatis-plus-boot-starter</artifactId>
 			<version>${mybatis-plus.version}</version>
+			<exclusions>
+				<exclusion>
+					<groupId>org.apache.logging.log4j</groupId>
+					<artifactId>log4j-to-slf4j</artifactId>
+				</exclusion>
+			</exclusions>
 		</dependency>
 		<dependency>
 			<groupId>org.postgresql</groupId>
@@ -71,6 +77,14 @@
 					<artifactId>log4j-api</artifactId>
 					<groupId>org.apache.logging.log4j</groupId>
 				</exclusion>
+				<exclusion>
+					<groupId>org.springframework.boot</groupId>
+					<artifactId>spring-boot-starter-tomcat</artifactId>
+				</exclusion>
+				<exclusion>
+					<groupId>org.apache.logging.log4j</groupId>
+					<artifactId>log4j-to-slf4j</artifactId>
+				</exclusion>
 			</exclusions>
 		</dependency>
 
@@ -78,7 +92,10 @@
 			<groupId>mysql</groupId>
 			<artifactId>mysql-connector-java</artifactId>
 		</dependency>
-
+		<dependency>
+			<groupId>com.h2database</groupId>
+			<artifactId>h2</artifactId>
+		</dependency>
 		<dependency>
 			<groupId>com.alibaba</groupId>
 			<artifactId>druid</artifactId>

dolphinscheduler-dao/src/main/resources/application.properties

@@ -23,6 +23,10 @@ spring.datasource.url=jdbc:postgresql://localhost:5432/dolphinscheduler
 # mysql
 #spring.datasource.driver-class-name=com.mysql.jdbc.Driver
 #spring.datasource.url=jdbc:mysql://192.168.xx.xx:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8
+# h2
+#spring.datasource.driver-class-name=org.h2.Driver
+#spring.datasource.url=jdbc:h2:file:../sql/h2;AUTO_SERVER=TRUE
+
 spring.datasource.username=test
 spring.datasource.password=test
 

dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/zk/StandaloneZKServerForTest.java

@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.dolphinscheduler.server.zk;
-
-import org.apache.dolphinscheduler.common.thread.ThreadPoolExecutors;
-import org.apache.zookeeper.server.ServerConfig;
-import org.apache.zookeeper.server.ZooKeeperServerMain;
-import org.apache.zookeeper.server.quorum.QuorumPeerConfig;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.util.Properties;
-
-
-/**
- * just for test
- */
-@Ignore
-public class StandaloneZKServerForTest {
-
-    private static final Logger logger = LoggerFactory.getLogger(StandaloneZKServerForTest.class);
-
-    private static volatile ZooKeeperServerMain zkServer = null;
-
-
-    @Before
-    public void before() {
-        logger.info("standalone zookeeper server for test service start ");
-
-        ThreadPoolExecutors.getInstance().execute(new Runnable() {
-            @Override
-            public void run() {
-
-                //delete zk data dir ?
-                File zkFile = new File(System.getProperty("java.io.tmpdir"), "zookeeper");
-
-                startStandaloneServer("2000", zkFile.getAbsolutePath(), "2181", "10", "5");
-            }
-        });
-
-    }
-
-
-    /**
-     * start zk server
-     * @param tickTime  zookeeper ticktime
-     * @param dataDir zookeeper data dir
-     * @param clientPort zookeeper client port
-     * @param initLimit zookeeper init limit
-     * @param syncLimit zookeeper sync limit
-     */
-    private void startStandaloneServer(String tickTime, String dataDir, String clientPort, String initLimit, String syncLimit) {
-        Properties props = new Properties();
-        props.setProperty("tickTime", tickTime);
-        props.setProperty("dataDir", dataDir);
-        props.setProperty("clientPort", clientPort);
-        props.setProperty("initLimit", initLimit);
-        props.setProperty("syncLimit", syncLimit);
-
-        QuorumPeerConfig quorumConfig = new QuorumPeerConfig();
-        try {
-            quorumConfig.parseProperties(props);
-
-            if(zkServer == null ){
-
-                synchronized (StandaloneZKServerForTest.class){
-                    if(zkServer == null ){
-                        zkServer = new ZooKeeperServerMain();
-                        final ServerConfig config = new ServerConfig();
-                        config.readFrom(quorumConfig);
-                        zkServer.runFromConfig(config);
-                    }
-                }
-
-            }
-
-        } catch (Exception e) {
-            logger.error("start standalone server fail！", e);
-        }
-    }
-
-
-}

dolphinscheduler-service/pom.xml

@@ -48,6 +48,11 @@
             </exclusions>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-core</artifactId>
+        </dependency>
+
         <dependency>
             <groupId>org.quartz-scheduler</groupId>
             <artifactId>quartz-jobs</artifactId>

dolphinscheduler-service/src/test/java/queue/ZKServer.java

@@ -14,11 +14,9 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package queue;
+package org.apache.dolphinscheduler.service.zk;
 
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.framework.CuratorFrameworkFactory;
-import org.apache.curator.retry.ExponentialBackoffRetry;
+import org.apache.zookeeper.server.ZooKeeperServer;
 import org.apache.zookeeper.server.ZooKeeperServerMain;
 import org.apache.zookeeper.server.quorum.QuorumPeerConfig;
 import org.slf4j.Logger;
@@ -26,27 +24,45 @@ import org.slf4j.LoggerFactory;
 
 import java.io.File;
 import java.io.IOException;
-import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 
 /**
- * just for test
+ * just speed experience version
+ * embedded zookeeper service
  */
 public class ZKServer {
-
     private static final Logger logger = LoggerFactory.getLogger(ZKServer.class);
 
     private static volatile PublicZooKeeperServerMain zkServer = null;
 
     public static final int DEFAULT_ZK_TEST_PORT = 2181;
 
-    public static final String DEFAULT_ZK_STR = "localhost:" + DEFAULT_ZK_TEST_PORT;
-
     private static String dataDir = null;
 
     private static final AtomicBoolean isStarted = new AtomicBoolean(false);
 
+    public static void main(String[] args) {
+        if(!isStarted()){
+            ZKServer.start();
+
+            /**
+             *  register hooks, which are called before the process exits
+             */
+            Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
+                @Override
+                public void run() {
+                    stop();
+                }
+            }));
+        }else{
+            logger.info("zk server aleady started");
+        }
+    }
+
+    /**
+     * start service
+     */
     public static void start() {
         try {
             startLocalZkServer(DEFAULT_ZK_TEST_PORT);
@@ -79,7 +95,8 @@ public class ZKServer {
      * @param port The port to listen on
      */
     public static void startLocalZkServer(final int port) {
-        startLocalZkServer(port, org.apache.commons.io.FileUtils.getTempDirectoryPath() + File.separator + "test-" + System.currentTimeMillis());
+
+        startLocalZkServer(port, System.getProperty("user.dir") +"/zookeeper_data", ZooKeeperServer.DEFAULT_TICK_TIME,"20");
     }
 
     /**
@@ -87,48 +104,28 @@ public class ZKServer {
      *
      * @param port        The port to listen on
      * @param dataDirPath The path for the Zk data directory
+     * @param tickTime    zk tick time
+     * @param maxClientCnxns    zk max client connections
      */
-    private static synchronized void startLocalZkServer(final int port, final String dataDirPath) {
+    private static synchronized void startLocalZkServer(final int port, final String dataDirPath,final int tickTime,String maxClientCnxns) {
         if (zkServer != null) {
             throw new RuntimeException("Zookeeper server is already started!");
         }
-        try {
-            zkServer = new PublicZooKeeperServerMain();
-            logger.info("Zookeeper data path : {} ", dataDirPath);
-            dataDir = dataDirPath;
-            final String[] args = new String[]{Integer.toString(port), dataDirPath};
-            Thread init = new Thread(new Runnable() {
-                @Override
-                public void run() {
-                    try {
-                        zkServer.initializeAndRun(args);
-                    } catch (QuorumPeerConfig.ConfigException e) {
-                        logger.warn("Caught exception while starting ZK", e);
-                    } catch (IOException e) {
-                        logger.warn("Caught exception while starting ZK", e);
-                    }
-                }
-            }, "init-zk-thread");
-            init.start();
-        } catch (Exception e) {
-            logger.warn("Caught exception while starting ZK", e);
-            throw new RuntimeException(e);
-        }
-
-        CuratorFramework zkClient = CuratorFrameworkFactory.builder()
-                .connectString(DEFAULT_ZK_STR)
-                .retryPolicy(new ExponentialBackoffRetry(10,100))
-                .sessionTimeoutMs(1000 * 30)
-                .connectionTimeoutMs(1000 * 30)
-                .build();
+        zkServer = new PublicZooKeeperServerMain();
+        logger.info("Zookeeper data path : {} ", dataDirPath);
+        dataDir = dataDirPath;
+        final String[] args = new String[]{Integer.toString(port), dataDirPath, Integer.toString(tickTime), maxClientCnxns};
 
         try {
-            zkClient.blockUntilConnected(10, TimeUnit.SECONDS);
-            zkClient.close();
-        } catch (InterruptedException ignore) {
+            logger.info("Zookeeper server started ");
+            isStarted.compareAndSet(false, true);
+
+            zkServer.initializeAndRun(args);
+        } catch (QuorumPeerConfig.ConfigException e) {
+            logger.warn("Caught exception while starting ZK", e);
+        } catch (IOException e) {
+            logger.warn("Caught exception while starting ZK", e);
         }
-        isStarted.compareAndSet(false, true);
-        logger.info("zk server started");
     }
 
     /**

dolphinscheduler-service/src/main/resources/quartz.properties

@@ -26,6 +26,10 @@ org.quartz.dataSource.myDs.URL = jdbc:postgresql://localhost:5432/dolphinschedul
 # mysql
 #org.quartz.dataSource.myDs.driver = com.mysql.jdbc.Driver
 #org.quartz.dataSource.myDs.URL = jdbc:mysql://localhost:3306/dolphinscheduler?characterEncoding=utf8
+#h2
+#org.quartz.dataSource.myDs.driver=org.h2.Driver
+#org.quartz.dataSource.myDs.URL=jdbc:h2:file:/Users/stone/work/myworkspace/incubator-dolphinscheduler/h2;AUTO_SERVER=TRUE
+
 org.quartz.dataSource.myDs.user = test
 org.quartz.dataSource.myDs.password = test
 

dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/zk/ZKServerTest.java

@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.service.zk;
+
+import org.junit.Ignore;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+@Ignore
+public class ZKServerTest {
+
+    @Test
+    public void start() {
+        //ZKServer is a process, can't unit test
+    }
+
+    @Test
+    public void isStarted() {
+
+    }
+
+    @Test
+    public void stop() {
+        ZKServer.stop();
+    }
+}

dolphinscheduler-service/src/test/java/queue/BaseTaskQueueTest.java

@@ -18,6 +18,7 @@ package queue;
 
 import org.apache.dolphinscheduler.service.queue.ITaskQueue;
 import org.apache.dolphinscheduler.service.queue.TaskQueueFactory;
+import org.apache.dolphinscheduler.service.zk.ZKServer;
 import org.junit.*;
 
 /**

pom.xml

@@ -72,7 +72,7 @@
         <cron.utils.version>5.0.5</cron.utils.version>
         <fastjson.version>1.2.61</fastjson.version>
         <druid.version>1.1.14</druid.version>
-        <h2.version>1.3.163</h2.version>
+        <h2.version>1.4.200</h2.version>
         <commons.codec.version>1.6</commons.codec.version>
         <commons.logging.version>1.1.1</commons.logging.version>
         <httpclient.version>4.4.1</httpclient.version>
@@ -200,13 +200,6 @@
                 <version>${spring.version}</version>
                 <scope>test</scope>
             </dependency>
-            <dependency>
-                <groupId>com.h2database</groupId>
-                <artifactId>h2</artifactId>
-                <version>${h2.version}</version>
-                <scope>test</scope>
-            </dependency>
-
             <dependency>
                 <groupId>org.apache.dolphinscheduler</groupId>
                 <artifactId>dolphinscheduler-server</artifactId>
@@ -349,6 +342,11 @@
                 <version>${mysql.connector.version}</version>
                 <scope>test</scope>
             </dependency>
+            <dependency>
+                <groupId>com.h2database</groupId>
+                <artifactId>h2</artifactId>
+                <version>${h2.version}</version>
+            </dependency>
             <dependency>
                 <groupId>org.slf4j</groupId>
                 <artifactId>slf4j-api</artifactId>