[python] Add independent deployment for python gateway server (#7549)

* [python] Add independent deployment to python gateway server

* Correct some error

* Add missing deployment change

* Add missing expose port

* Add missing standalone config

* Keyword change from rpc to socket

* try to fix

* Add python gateway server to scp-hosts.sh

deploy/docker/docker-compose.yml

@@ -140,6 +140,29 @@ services:
     networks:
       - dolphinscheduler
 
+  dolphinscheduler-python-gateway:
+    image: ${HUB}/dolphinscheduler-python-gateway:${TAG}
+    ports:
+      - "54321:54321"
+      - "25333:25333"
+    env_file: .env
+    healthcheck:
+      test: [ "CMD", "curl", "http://localhost:54321/actuator/health" ]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+    depends_on:
+      dolphinscheduler-schema-initializer:
+        condition: service_completed_successfully
+      dolphinscheduler-zookeeper:
+        condition: service_healthy
+    volumes:
+      - dolphinscheduler-logs:/opt/dolphinscheduler/logs
+      - dolphinscheduler-shared-local:/opt/soft
+      - dolphinscheduler-resource-local:/dolphinscheduler
+    networks:
+      - dolphinscheduler
+
 networks:
   dolphinscheduler:
     driver: bridge

deploy/docker/docker-stack.yml

@@ -118,6 +118,27 @@ services:
       mode: replicated
       replicas: 1
 
+  dolphinscheduler-python-gateway:
+    image: apache/dolphinscheduler-python-gateway
+    ports:
+      - 54321:54321
+      - 25333:25333
+    env_file: .env
+    healthcheck:
+      test: [ "CMD", "curl", "http://localhost:54321/actuator/health" ]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+    volumes:
+      - dolphinscheduler-logs:/opt/dolphinscheduler/logs
+      - dolphinscheduler-shared-local:/opt/soft
+      - dolphinscheduler-resource-local:/dolphinscheduler
+    networks:
+      - dolphinscheduler
+    deploy:
+      mode: replicated
+      replicas: 1
+
 networks:
   dolphinscheduler:
     driver: overlay

deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl

@@ -44,6 +44,9 @@ Create default docker images' fullname.
 {{- define "dolphinscheduler.image.fullname.tools" -}}
 {{- .Values.image.registry }}/dolphinscheduler-tools:{{ .Values.image.tag | default .Chart.AppVersion -}}
 {{- end -}}
+{{- define "dolphinscheduler.image.fullname.python-gateway" -}}
+{{- .Values.image.registry }}/dolphinscheduler-python-gateway:{{ .Values.image.tag | default .Chart.AppVersion -}}
+{{- end -}}
 
 {{/*
 Create a default common labels.

dolphinscheduler-dist/src/main/assembly/dolphinscheduler-bin.xml

@@ -50,6 +50,11 @@
             <outputDirectory>logger-server</outputDirectory>
         </fileSet>
 
+        <fileSet>
+            <directory>${basedir}/../dolphinscheduler-python/target/python-gateway-server</directory>
+            <outputDirectory>python-gateway-server</outputDirectory>
+        </fileSet>
+
         <fileSet>
             <directory>${basedir}/../dolphinscheduler-standalone-server/target/standalone-server</directory>
             <outputDirectory>standalone-server</outputDirectory>

dolphinscheduler-python/pom.xml

@@ -57,4 +57,52 @@
         </dependency>
 
     </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-jar-plugin</artifactId>
+                <configuration>
+                    <excludes>
+                        <exclude>*.yaml</exclude>
+                        <exclude>*.xml</exclude>
+                    </excludes>
+                </configuration>
+            </plugin>
+            <plugin>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>dolphinscheduler-python-gateway-server</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                        <configuration>
+                            <finalName>python-gateway-server</finalName>
+                            <descriptors>
+                                <descriptor>src/main/assembly/dolphinscheduler-python-gateway-server.xml</descriptor>
+                            </descriptors>
+                            <appendAssemblyId>false</appendAssemblyId>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+    <profiles>
+        <profile>
+            <id>docker</id>
+            <build>
+                <plugins>
+                    <plugin>
+                        <groupId>org.codehaus.mojo</groupId>
+                        <artifactId>exec-maven-plugin</artifactId>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
+    </profiles>
 </project>

dolphinscheduler-python/src/main/assembly/dolphinscheduler-python-gateway-server.xml

@@ -0,0 +1,64 @@
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<assembly xmlns="http://maven.apache.org/ASSEMBLY/2.1.0"
+          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+          xsi:schemaLocation="http://maven.apache.org/ASSEMBLY/2.1.0 http://maven.apache.org/xsd/assembly-2.1.0.xsd">
+    <id>dolphinscheduler-python-gateway-server</id>
+    <formats>
+        <format>dir</format>
+    </formats>
+    <includeBaseDirectory>false</includeBaseDirectory>
+    <baseDirectory>python-gateway-server</baseDirectory>
+    <fileSets>
+        <fileSet>
+            <directory>${basedir}/src/main/resources</directory>
+            <includes>
+                <include>*.yaml</include>
+                <include>*.xml</include>
+            </includes>
+            <outputDirectory>conf</outputDirectory>
+        </fileSet>
+        <fileSet>
+            <directory>${basedir}/src/main/bin</directory>
+            <outputDirectory>bin</outputDirectory>
+            <fileMode>0755</fileMode>
+            <directoryMode>0755</directoryMode>
+        </fileSet>
+        <fileSet>
+            <directory>${basedir}/../script/env</directory>
+            <outputDirectory>bin</outputDirectory>
+            <includes>
+                <include>dolphinscheduler_env.sh</include>
+            </includes>
+            <fileMode>0755</fileMode>
+            <directoryMode>0755</directoryMode>
+        </fileSet>
+        <fileSet>
+            <directory>${basedir}/../dolphinscheduler-common/src/main/resources</directory>
+            <includes>
+                <include>**/*.properties</include>
+            </includes>
+            <outputDirectory>conf</outputDirectory>
+        </fileSet>
+    </fileSets>
+    <dependencySets>
+        <dependencySet>
+            <outputDirectory>libs</outputDirectory>
+        </dependencySet>
+    </dependencySets>
+</assembly>

dolphinscheduler-python/src/main/bin/start.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+BIN_DIR=$(dirname $0)
+DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/..; pwd)}
+
+source "$BIN_DIR/dolphinscheduler_env.sh"
+
+JAVA_OPTS=${JAVA_OPTS:-"-server -Xms1g -Xmx1g -Xmn512m -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"}
+
+if [[ "$DOCKER" == "true" ]]; then
+  JAVA_OPTS="${JAVA_OPTS} -XX:-UseContainerSupport"
+fi
+
+java $JAVA_OPTS \
+  -cp "$DOLPHINSCHEDULER_HOME/conf":"$DOLPHINSCHEDULER_HOME/libs/*" \
+  org.apache.dolphinscheduler.server.PythonGatewayServer

dolphinscheduler-python/src/main/docker/Dockerfile

@@ -0,0 +1,34 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+FROM openjdk:8-jre-slim-buster
+
+ENV DOCKER true
+ENV TZ Asia/Shanghai
+ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler
+
+RUN apt update ; \
+    apt install -y curl sudo ; \
+    rm -rf /var/lib/apt/lists/*
+
+WORKDIR $DOLPHINSCHEDULER_HOME
+
+ADD ./target/python-gateway-server $DOLPHINSCHEDULER_HOME
+
+EXPOSE 25333 54321
+
+CMD [ "/bin/bash", "./bin/start.sh" ]

dolphinscheduler-python/src/main/java/org/apache/dolphinscheduler/server/PythonGatewayServer.java

@@ -503,7 +503,7 @@ public class PythonGatewayServer extends SpringBootServletInitializer {
     public void run() {
         GatewayServer server = new GatewayServer(this);
         GatewayServer.turnLoggingOn();
-        // Start server to accept python client RPC
+        // Start server to accept python client socket
         server.start();
     }
 

dolphinscheduler-python/src/main/resources/application.yaml

@@ -0,0 +1,61 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+spring:
+  application:
+    name: python-gateway-server
+  main:
+    banner-mode: off
+  datasource:
+    driver-class-name: org.h2.Driver
+    url: jdbc:h2:mem:dolphinscheduler;MODE=MySQL;DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=true
+    username: sa
+    password: ""
+  jackson:
+    time-zone: GMT+8
+  servlet:
+    multipart:
+      max-file-size: 1024MB
+      max-request-size: 1024MB
+  messages:
+    basename: i18n/messages
+  jpa:
+    hibernate:
+      ddl-auto: none
+
+server:
+  port: 54321
+
+management:
+  endpoints:
+    web:
+      exposure:
+        include: '*'
+  metrics:
+    tags:
+      application: ${spring.application.name}
+
+# Override by profile
+---
+spring:
+  config:
+    activate:
+      on-profile: postgresql
+  quartz:
+    properties:
+      org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.PostgreSQLDelegate
+

dolphinscheduler-python/src/main/resources/logback-spring.xml

@@ -0,0 +1,57 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<configuration scan="true" scanPeriod="120 seconds">
+<property name="log.base" value="logs"/>
+
+<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+    <encoder>
+        <pattern>
+            [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
+        </pattern>
+        <charset>UTF-8</charset>
+    </encoder>
+</appender>
+
+<appender name="PYTHONGATEWAYLOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+    <file>${log.base}/dolphinscheduler-python-gateway.log</file>
+    <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+        <level>INFO</level>
+    </filter>
+    <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+        <fileNamePattern>${log.base}/dolphinscheduler-python-gateway.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
+        <maxHistory>168</maxHistory>
+        <maxFileSize>64MB</maxFileSize>
+    </rollingPolicy>
+    <encoder>
+        <pattern>
+            [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
+        </pattern>
+        <charset>UTF-8</charset>
+    </encoder>
+</appender>
+
+<root level="INFO">
+    <if condition="${DOCKER:-false}">
+        <then>
+            <appender-ref ref="STDOUT"/>
+        </then>
+    </if>
+    <appender-ref ref="PYTHONGATEWAYLOGFILE"/>
+</root>
+</configuration>

dolphinscheduler-standalone-server/pom.xml

@@ -50,6 +50,11 @@
             <artifactId>dolphinscheduler-log-server</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.dolphinscheduler</groupId>
+            <artifactId>dolphinscheduler-python</artifactId>
+        </dependency>
+
         <dependency>
             <groupId>org.apache.curator</groupId>
             <artifactId>curator-test</artifactId>
@@ -61,10 +66,6 @@
                 </exclusion>
             </exclusions>
         </dependency>
-        <dependency>
-            <groupId>org.apache.dolphinscheduler</groupId>
-            <artifactId>dolphinscheduler-python</artifactId>
-        </dependency>
     </dependencies>
 
     <build>

dolphinscheduler-standalone-server/src/main/assembly/dolphinscheduler-standalone-server.xml

@@ -60,6 +60,13 @@
             </includes>
             <outputDirectory>.</outputDirectory>
         </fileSet>
+        <fileSet>
+            <directory>${basedir}/../dolphinscheduler-python-gateway/target/python-gateway</directory>
+            <includes>
+                <include>libs</include>
+            </includes>
+            <outputDirectory>.</outputDirectory>
+        </fileSet>
 
         <fileSet>
             <directory>${basedir}/src/main/resources</directory>

script/dolphinscheduler-daemon.sh

@@ -16,7 +16,7 @@
 # limitations under the License.
 #
 
-usage="Usage: dolphinscheduler-daemon.sh (start|stop|status) <api-server|master-server|worker-server|alert-server|standalone-server> "
+usage="Usage: dolphinscheduler-daemon.sh (start|stop|status) <api-server|master-server|worker-server|alert-server|python-gateway-server|standalone-server> "
 
 # if no args specified, show usage
 if [ $# -le 1 ]; then
@@ -62,6 +62,8 @@ elif [ "$command" = "alert-server" ]; then
   :
 elif [ "$command" = "standalone-server" ]; then
   :
+elif [ "$command" = "python-gateway-server" ]; then
+  :
 else
   echo "Error: No command named '$command' was found."
   exit 1

script/env/install_env.sh

@@ -48,6 +48,11 @@ alertServer=${alertServer:-"ds3"}
 # Example for hostname: apiServers="ds1", Example for IP: apiServers="192.168.8.1"
 apiServers=${apiServers:-"ds1"}
 
+# A comma separated list of machine hostname or IP would be installed Python gateway server, it
+# must be a subset of configuration `ips`.
+# Example for hostname: pythonGatewayServers="ds1", Example for IP: pythonGatewayServers="192.168.8.1"
+pythonGatewayServers=${pythonGatewayServers:-"ds1"}
+
 # The directory to install DolphinScheduler for all machine we config above. It will automatically be created by `install.sh` script if not exists.
 # Do not set this configuration same as the current path (pwd)
 installPath=${installPath:-"/tmp/dolphinscheduler"}

script/scp-hosts.sh

@@ -49,7 +49,7 @@ do
   echo "scp dirs to $host/$installPath starting"
 	ssh -p $sshPort $host  "cd $installPath/; rm -rf bin/ conf/ lib/ script/ sql/ ui/"
 
-  for dsDir in bin master-server worker-server alert-server api-server ui
+  for dsDir in bin master-server worker-server alert-server api-server ui python-gateway-server
   do
     # if worker in workersGroupMap
     if [[ "${workersGroupMap[${host}]}" ]]; then

script/start-all.sh

@@ -56,6 +56,13 @@ do
   ssh -p $sshPort $apiServer  "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start api-server;"
 done
 
+pythonGatewayHost=(${pythonGatewayServers//,/ })
+for pythonGatewayServer in "${pythonGatewayHost[@]}"
+do
+  echo "$pythonGatewayServer python gateway server is starting"
+  ssh -p $sshPort $pythonGatewayServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start python-gateway-server;"
+done
+
 # query server status
 echo "query server status"
 cd $installPath/; sh bin/status-all.sh

script/status-all.sh

@@ -74,3 +74,11 @@ do
   apiState=`ssh -p $sshPort $apiServer  "cd $installPath/; sh bin/dolphinscheduler-daemon.sh status api-server;"`
   echo "$apiServer  $apiState"
 done
+
+# python gateway server check state
+pythonGatewayHost=(${pythonGatewayServers//,/ })
+for pythonGatewayServer in "${pythonGatewayHost[@]}"
+do
+  pythonGatewayState=`ssh -p $sshPort $pythonGatewayServer  "cd $installPath/; sh bin/dolphinscheduler-daemon.sh status python-gateway-server;"`
+  echo "$pythonGatewayServer  $pythonGatewayState"
+done

script/stop-all.sh

@@ -54,3 +54,10 @@ do
   echo "$apiServer api server is stopping"
   ssh -p $sshPort $apiServer  "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop api-server;"
 done
+
+pythonGatewayHost=(${pythonGatewayServers//,/ })
+for pythonGatewayServer in "${pythonGatewayHost[@]}"
+do
+  echo "$pythonGatewayServer python gateway server is stopping"
+  ssh -p $sshPort $pythonGatewayServer  "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop python-gateway-server;"
+done