Reduce the size of distribution tar (#9019)

dolphinscheduler-dist/src/main/assembly/dolphinscheduler-bin.xml

@@ -58,6 +58,19 @@
         <fileSet>
             <directory>${basedir}/../dolphinscheduler-standalone-server/target/standalone-server</directory>
             <outputDirectory>standalone-server</outputDirectory>
+            <excludes>
+                <exclude>libs/</exclude>
+                <exclude>bin/</exclude>
+                <exclude>dist-bin/</exclude>
+            </excludes>
+        </fileSet>
+        <fileSet>
+            <directory>${basedir}/../dolphinscheduler-standalone-server/target/standalone-server/libs/standalone-server</directory>
+            <outputDirectory>standalone-server/libs/standalone-server</outputDirectory>
+        </fileSet>
+        <fileSet>
+            <directory>${basedir}/../dolphinscheduler-standalone-server/target/standalone-server/dist-bin</directory>
+            <outputDirectory>standalone-server/bin</outputDirectory>
         </fileSet>
 
         <fileSet>

dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-mysql/download-mysql.sh

@@ -17,7 +17,7 @@
 
 set -ex
 
-DS_HOME=/opt/dolphinscheduler/libs
+DS_HOME=/opt/dolphinscheduler/libs/standalone-server
 MYSQL_URL="https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar"
 MYSQL_DRIVER="mysql-connector-java-8.0.16.jar"
 

dolphinscheduler-standalone-server/src/main/assembly/dolphinscheduler-standalone-server.xml

@@ -26,46 +26,28 @@
     <baseDirectory>standalone-server</baseDirectory>
     <fileSets>
         <fileSet>
-            <directory>${basedir}/../dolphinscheduler-alert/dolphinscheduler-alert-server/target/alert-server</directory>
-            <outputDirectory>.</outputDirectory>
-            <includes>
-                <include>libs</include>
-            </includes>
+            <directory>${basedir}/../dolphinscheduler-alert/dolphinscheduler-alert-server/target/alert-server/libs</directory>
+            <outputDirectory>libs/alert-server</outputDirectory>
         </fileSet>
         <fileSet>
-            <directory>${basedir}/../dolphinscheduler-api/target/api-server</directory>
-            <includes>
-                <include>libs</include>
-            </includes>
-            <outputDirectory>.</outputDirectory>
+            <directory>${basedir}/../dolphinscheduler-api/target/api-server/libs</directory>
+            <outputDirectory>libs/api-server</outputDirectory>
         </fileSet>
         <fileSet>
-            <directory>${basedir}/../dolphinscheduler-master/target/master-server</directory>
-            <includes>
-                <include>libs</include>
-            </includes>
-            <outputDirectory>.</outputDirectory>
+            <directory>${basedir}/../dolphinscheduler-master/target/master-server/libs</directory>
+            <outputDirectory>libs/master-server</outputDirectory>
         </fileSet>
         <fileSet>
-            <directory>${basedir}/../dolphinscheduler-worker/target/worker-server</directory>
-            <includes>
-                <include>libs</include>
-            </includes>
-            <outputDirectory>.</outputDirectory>
+            <directory>${basedir}/../dolphinscheduler-worker/target/worker-server/libs</directory>
+            <outputDirectory>libs/worker-server</outputDirectory>
         </fileSet>
         <fileSet>
-            <directory>${basedir}/../dolphinscheduler-log-server/target/logger-server</directory>
-            <includes>
-                <include>libs</include>
-            </includes>
-            <outputDirectory>.</outputDirectory>
+            <directory>${basedir}/../dolphinscheduler-log-server/target/logger-server/libs</directory>
+            <outputDirectory>libs/logger-server</outputDirectory>
         </fileSet>
         <fileSet>
-            <directory>${basedir}/../dolphinscheduler-python-gateway/target/python-gateway</directory>
-            <includes>
-                <include>libs</include>
-            </includes>
-            <outputDirectory>.</outputDirectory>
+            <directory>${basedir}/../dolphinscheduler-python/target/python-gateway-server/libs</directory>
+            <outputDirectory>libs/python-gateway</outputDirectory>
         </fileSet>
 
         <fileSet>
@@ -83,6 +65,12 @@
             <fileMode>0755</fileMode>
             <directoryMode>0755</directoryMode>
         </fileSet>
+        <fileSet>
+            <directory>${basedir}/src/main/dist-bin</directory>
+            <outputDirectory>dist-bin</outputDirectory>
+            <fileMode>0755</fileMode>
+            <directoryMode>0755</directoryMode>
+        </fileSet>
         <fileSet>
             <directory>${basedir}/../script/env</directory>
             <outputDirectory>bin</outputDirectory>
@@ -92,6 +80,15 @@
             <fileMode>0755</fileMode>
             <directoryMode>0755</directoryMode>
         </fileSet>
+        <fileSet>
+            <directory>${basedir}/../script/env</directory>
+            <outputDirectory>dist-bin</outputDirectory>
+            <includes>
+                <include>dolphinscheduler_env.sh</include>
+            </includes>
+            <fileMode>0755</fileMode>
+            <directoryMode>0755</directoryMode>
+        </fileSet>
 
         <fileSet>
             <directory>${basedir}/../dolphinscheduler-dao/src/main/resources</directory>
@@ -108,7 +105,8 @@
 
     <dependencySets>
         <dependencySet>
-            <outputDirectory>libs</outputDirectory>
+            <useTransitiveDependencies>false</useTransitiveDependencies>
+            <outputDirectory>libs/standalone-server</outputDirectory>
             <excludes>
                 <exclude>com.amazonaws:aws-java-sdk-emr</exclude>
                 <exclude>com.amazonaws:aws-java-sdk-core</exclude>

dolphinscheduler-standalone-server/src/main/bin/start.sh

@@ -27,6 +27,13 @@ if [[ "$DOCKER" == "true" ]]; then
   JAVA_OPTS="${JAVA_OPTS} -XX:-UseContainerSupport"
 fi
 
+CP=""
+for d in $DOLPHINSCHEDULER_HOME/libs/*; do
+  for f in $d/*.jar; do
+    CP=$CP:$f
+  done
+done
+
 java $JAVA_OPTS \
-  -cp "$DOLPHINSCHEDULER_HOME/conf":"$DOLPHINSCHEDULER_HOME/libs/*" \
+  -cp "$DOLPHINSCHEDULER_HOME/conf":"$CP" \
   org.apache.dolphinscheduler.StandaloneServer

dolphinscheduler-standalone-server/src/main/dist-bin/start.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+BIN_DIR=$(dirname $0)
+DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/..; pwd)}
+
+source "$BIN_DIR/dolphinscheduler_env.sh"
+
+JAVA_OPTS=${JAVA_OPTS:-"-server -Xms1g -Xmx1g -Xmn512m -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"}
+
+if [[ "$DOCKER" == "true" ]]; then
+  JAVA_OPTS="${JAVA_OPTS} -XX:-UseContainerSupport"
+fi
+
+CP=$DOLPHINSCHEDULER_HOME/libs/standalone-server/*
+for d in alert-server api-server master-server python-gateway-server worker-server; do
+  for f in $DOLPHINSCHEDULER_HOME/../$d/libs/*.jar; do
+    CP=$CP:$f
+  done
+done
+
+java $JAVA_OPTS \
+  -cp "$DOLPHINSCHEDULER_HOME/conf":"$CP" \
+  org.apache.dolphinscheduler.StandaloneServer