Use docker-compose for deployment . Failed to execute python script (#2539)

* Modify docker-compose and dockerfile

1. Add python3 in Dockerfile
2. Modify startup.sh file
3. Modify docker-compose.yml and add docker-stack.yml
4. Modify dolphinscheduler_env.sh

* Add logback-xx.xml and Modify Dockerfile

docker/docker-swarm/docker-compose.yml

@@ -16,36 +16,6 @@
 
 version: "3.4"
 
-networks:
-  dolphinscheduler-postgresql:
-    driver: bridge
-  dolphinscheduler-zookeeper:
-    driver: bridge
-  dolphinscheduler-api:
-    driver: bridge
-  dolphinscheduler-frontend:
-    driver: bridge
-  dolphinscheduler-alert:
-    driver: bridge
-  dolphinscheduler-master:
-    driver: bridge
-  dolphinscheduler-worker:
-    driver: bridge
-
-volumes:
-  dolphinscheduler-postgresql:
-  dolphinscheduler-zookeeper:
-  dolphinscheduler-api:
-  dolphinscheduler-frontend:
-  dolphinscheduler-alert:
-  dolphinscheduler-master:
-  dolphinscheduler-worker-data:
-  dolphinscheduler-worker-logs:
-
-configs:
-  dolphinscheduler-worker-task-env:
-    file: ./dolphinscheduler_env.sh
-
 services:
 
   dolphinscheduler-postgresql:
@@ -58,16 +28,10 @@ services:
       POSTGRESQL_USERNAME: root
       POSTGRESQL_PASSWORD: root
       POSTGRESQL_DATABASE: dolphinscheduler
-    healthcheck:
-      test: ["CMD", "pg_isready", "-U", "${POSTGRESQL_USERNAME}", "-d", "{POSTGRESQL_PASSWORD}", "-h", "localhost", "5432"]
-      interval: 30s
-      timeout: 5s
-      retries: 3
-      # start_period: 30s
-    volumes: 
+    volumes:
     - dolphinscheduler-postgresql:/bitnami/postgresql
     networks:
-    - dolphinscheduler-postgresql
+    - dolphinscheduler
 
   dolphinscheduler-zookeeper:
     image: bitnami/zookeeper:latest
@@ -77,19 +41,13 @@ services:
     environment:
       TZ: Asia/Shanghai
       ALLOW_ANONYMOUS_LOGIN: "yes"
-    healthcheck:
-      test: ["CMD-SHELL", "nc -z localhost 2181"]
-      interval: 30s
-      timeout: 5s
-      retries: 3
-      # start_period: 30s
     volumes:
     - dolphinscheduler-zookeeper:/bitnami/zookeeper
     networks:
-    - dolphinscheduler-zookeeper
+    - dolphinscheduler
 
   dolphinscheduler-api:
-    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:1.2.1
+    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
     container_name: dolphinscheduler-api
     command: ["api-server"]
     ports:
@@ -103,23 +61,21 @@ services:
       POSTGRESQL_DATABASE: dolphinscheduler
       ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
     healthcheck:
-      test: ["CMD-SHELL", "curl -f http://localhost:12345"]
+      test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"]
       interval: 30s
       timeout: 5s
       retries: 3
-      # start_period: 30s
+      start_period: 30s
     depends_on:
     - dolphinscheduler-postgresql
     - dolphinscheduler-zookeeper
     volumes:
     - dolphinscheduler-api:/opt/dolphinscheduler/logs
     networks: 
-    - dolphinscheduler-api
-    - dolphinscheduler-postgresql
-    - dolphinscheduler-zookeeper
+    - dolphinscheduler
 
   dolphinscheduler-frontend:
-    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:1.2.1
+    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
     container_name: dolphinscheduler-frontend
     command: ["frontend"]
     ports:
@@ -129,21 +85,20 @@ services:
       FRONTEND_API_SERVER_HOST: dolphinscheduler-api
       FRONTEND_API_SERVER_PORT: 12345
     healthcheck:
-      test: ["CMD-SHELL", "curl -f http://localhost:8888"]
+      test: ["CMD", "nc", "-z", "localhost", "8888"]
       interval: 30s
       timeout: 5s
       retries: 3
-      # start_period: 30s
+      start_period: 30s
     depends_on:
     - dolphinscheduler-api
     volumes:
     - dolphinscheduler-frontend:/var/log/nginx
     networks:
-    - dolphinscheduler-frontend
-    - dolphinscheduler-api
+    - dolphinscheduler
 
   dolphinscheduler-alert:
-    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:1.2.1
+    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
     container_name: dolphinscheduler-alert
     command: ["alert-server"]
     environment:
@@ -172,17 +127,16 @@ services:
       interval: 30s
       timeout: 5s
       retries: 3
-      # start_period: 30s
+      start_period: 30s
     depends_on:
     - dolphinscheduler-postgresql
     volumes: 
     - dolphinscheduler-alert:/opt/dolphinscheduler/logs
     networks: 
-    - dolphinscheduler-alert
-    - dolphinscheduler-postgresql
+    - dolphinscheduler
 
   dolphinscheduler-master:
-    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:1.2.1
+    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
     container_name: dolphinscheduler-master
     command: ["master-server"]
     ports: 
@@ -207,19 +161,17 @@ services:
       interval: 30s
       timeout: 5s
       retries: 3
-      # start_period: 30s
+      start_period: 30s
     depends_on: 
     - dolphinscheduler-postgresql
     - dolphinscheduler-zookeeper
     volumes: 
     - dolphinscheduler-master:/opt/dolphinscheduler/logs
     networks:
-    - dolphinscheduler-master
-    - dolphinscheduler-postgresql
-    - dolphinscheduler-zookeeper
+    - dolphinscheduler
 
   dolphinscheduler-worker:
-    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:1.2.1
+    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
     container_name: dolphinscheduler-worker
     command: ["worker-server"]
     ports: 
@@ -245,17 +197,37 @@ services:
       interval: 30s
       timeout: 5s
       retries: 3
-      # start_period: 30s
+      start_period: 30s
     depends_on: 
     - dolphinscheduler-postgresql
     - dolphinscheduler-zookeeper
-    volumes: 
-    - dolphinscheduler-worker-data:/tmp/dolphinscheduler
-    - dolphinscheduler-worker-logs:/opt/dolphinscheduler/logs
-    configs:
-    - source: dolphinscheduler-worker-task-env
+    volumes:
+    - type: bind
+      source: ./dolphinscheduler_env.sh
       target: /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh
+    - type: volume
+      source: dolphinscheduler-worker-data
+      target: /tmp/dolphinscheduler
+    - type: volume
+      source: dolphinscheduler-worker-logs
+      target: /opt/dolphinscheduler/logs
     networks:
-    - dolphinscheduler-worker
-    - dolphinscheduler-postgresql
-    - dolphinscheduler-zookeeper
+    - dolphinscheduler
+
+networks:
+  dolphinscheduler:
+    driver: bridge
+
+volumes:
+  dolphinscheduler-postgresql:
+  dolphinscheduler-zookeeper:
+  dolphinscheduler-api:
+  dolphinscheduler-frontend:
+  dolphinscheduler-alert:
+  dolphinscheduler-master:
+  dolphinscheduler-worker-data:
+  dolphinscheduler-worker-logs:
+
+configs:
+  dolphinscheduler-worker-task-env:
+    file: ./dolphinscheduler_env.sh

docker/docker-swarm/docker-stack.yml

@@ -0,0 +1,230 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3.4"
+
+services:
+
+  dolphinscheduler-postgresql:
+    image: bitnami/postgresql:latest
+    ports:
+    - 5432:5432
+    environment:
+      TZ: Asia/Shanghai
+      POSTGRESQL_USERNAME: root
+      POSTGRESQL_PASSWORD: root
+      POSTGRESQL_DATABASE: dolphinscheduler
+    volumes:
+    - dolphinscheduler-postgresql:/bitnami/postgresql
+    networks:
+    - dolphinscheduler
+    deploy:
+      mode: replicated
+      replicas: 1
+
+  dolphinscheduler-zookeeper:
+    image: bitnami/zookeeper:latest
+    ports:
+    - 2181:2181
+    environment:
+      TZ: Asia/Shanghai
+      ALLOW_ANONYMOUS_LOGIN: "yes"
+    volumes:
+    - dolphinscheduler-zookeeper:/bitnami/zookeeper
+    networks:
+    - dolphinscheduler
+    deploy:
+      mode: replicated
+      replicas: 1
+
+  dolphinscheduler-api:
+    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
+    command: ["api-server"]
+    ports:
+    - 12345:12345
+    environment:
+      TZ: Asia/Shanghai
+      POSTGRESQL_HOST: dolphinscheduler-postgresql
+      POSTGRESQL_PORT: 5432
+      POSTGRESQL_USERNAME: root
+      POSTGRESQL_PASSWORD: root
+      POSTGRESQL_DATABASE: dolphinscheduler
+      ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
+    healthcheck:
+      test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"]
+      interval: 30
+      timeout: 5s
+      retries: 3
+      start_period: 30s
+    volumes:
+    - dolphinscheduler-api:/opt/dolphinscheduler/logs
+    networks: 
+    - dolphinscheduler
+    deploy:
+      mode: replicated
+      replicas: 1
+
+  dolphinscheduler-frontend:
+    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
+    command: ["frontend"]
+    ports:
+    - 8888:8888
+    environment:
+      TZ: Asia/Shanghai
+      FRONTEND_API_SERVER_HOST: dolphinscheduler-api
+      FRONTEND_API_SERVER_PORT: 12345
+    healthcheck:
+      test: ["CMD", "nc", "-z", "localhost", "8888"]
+      interval: 30
+      timeout: 5s
+      retries: 3
+      start_period: 30s
+    volumes:
+    - dolphinscheduler-frontend:/var/log/nginx
+    networks:
+    - dolphinscheduler
+    deploy:
+      mode: replicated
+      replicas: 1
+
+  dolphinscheduler-alert:
+    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
+    command: ["alert-server"]
+    environment:
+      TZ: Asia/Shanghai
+      XLS_FILE_PATH: "/tmp/xls"
+      MAIL_SERVER_HOST: ""
+      MAIL_SERVER_PORT: ""
+      MAIL_SENDER: ""
+      MAIL_USER: ""
+      MAIL_PASSWD: ""
+      MAIL_SMTP_STARTTLS_ENABLE: "false"
+      MAIL_SMTP_SSL_ENABLE: "false"
+      MAIL_SMTP_SSL_TRUST: ""
+      ENTERPRISE_WECHAT_ENABLE: "false"
+      ENTERPRISE_WECHAT_CORP_ID: ""
+      ENTERPRISE_WECHAT_SECRET: ""
+      ENTERPRISE_WECHAT_AGENT_ID: ""
+      ENTERPRISE_WECHAT_USERS: ""
+      POSTGRESQL_HOST: dolphinscheduler-postgresql
+      POSTGRESQL_PORT: 5432
+      POSTGRESQL_USERNAME: root
+      POSTGRESQL_PASSWORD: root
+      POSTGRESQL_DATABASE: dolphinscheduler
+    healthcheck:
+      test: ["CMD", "/root/checkpoint.sh", "AlertServer"]
+      interval: 30
+      timeout: 5s
+      retries: 3
+      start_period: 30s
+    volumes:
+    - dolphinscheduler-alert:/opt/dolphinscheduler/logs
+    networks: 
+    - dolphinscheduler
+    deploy:
+      mode: replicated
+      replicas: 1
+
+  dolphinscheduler-master:
+    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
+    command: ["master-server"]
+    ports: 
+    - 5678:5678
+    environment: 
+      TZ: Asia/Shanghai
+      MASTER_EXEC_THREADS: "100"
+      MASTER_EXEC_TASK_NUM: "20"
+      MASTER_HEARTBEAT_INTERVAL: "10"
+      MASTER_TASK_COMMIT_RETRYTIMES: "5"
+      MASTER_TASK_COMMIT_INTERVAL: "1000"
+      MASTER_MAX_CPULOAD_AVG: "100"
+      MASTER_RESERVED_MEMORY: "0.1"
+      POSTGRESQL_HOST: dolphinscheduler-postgresql
+      POSTGRESQL_PORT: 5432
+      POSTGRESQL_USERNAME: root
+      POSTGRESQL_PASSWORD: root
+      POSTGRESQL_DATABASE: dolphinscheduler
+      ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
+    healthcheck:
+      test: ["CMD", "/root/checkpoint.sh", "MasterServer"]
+      interval: 30
+      timeout: 5s
+      retries: 3
+      start_period: 30s
+    volumes:
+    - dolphinscheduler-master:/opt/dolphinscheduler/logs
+    networks:
+    - dolphinscheduler
+    deploy:
+      mode: replicated
+      replicas: 1
+
+  dolphinscheduler-worker:
+    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
+    command: ["worker-server"]
+    ports: 
+    - 1234:1234
+    - 50051:50051
+    environment: 
+      TZ: Asia/Shanghai
+      WORKER_EXEC_THREADS: "100"
+      WORKER_HEARTBEAT_INTERVAL: "10"
+      WORKER_FETCH_TASK_NUM: "3"
+      WORKER_MAX_CPULOAD_AVG: "100"
+      WORKER_RESERVED_MEMORY: "0.1"
+      WORKER_GROUP: "default"
+      DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler"
+      POSTGRESQL_HOST: dolphinscheduler-postgresql
+      POSTGRESQL_PORT: 5432
+      POSTGRESQL_USERNAME: root
+      POSTGRESQL_PASSWORD: root
+      POSTGRESQL_DATABASE: dolphinscheduler
+      ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
+    healthcheck:
+      test: ["CMD", "/root/checkpoint.sh", "WorkerServer"]
+      interval: 30
+      timeout: 5s
+      retries: 3
+      start_period: 30s
+    volumes:
+    - dolphinscheduler-worker-data:/tmp/dolphinscheduler
+    - dolphinscheduler-worker-logs:/opt/dolphinscheduler/logs
+    configs:
+    - source: dolphinscheduler-worker-task-env
+      target: /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh
+    networks:
+    - dolphinscheduler
+    deploy:
+      mode: replicated
+      replicas: 1
+
+networks:
+  dolphinscheduler:
+    driver: overlay
+
+volumes:
+  dolphinscheduler-postgresql:
+  dolphinscheduler-zookeeper:
+  dolphinscheduler-api:
+  dolphinscheduler-frontend:
+  dolphinscheduler-alert:
+  dolphinscheduler-master:
+  dolphinscheduler-worker-data:
+  dolphinscheduler-worker-logs:
+
+configs:
+  dolphinscheduler-worker-task-env:
+    file: ./dolphinscheduler_env.sh

docker/docker-swarm/dolphinscheduler_env.sh

@@ -15,12 +15,6 @@
 # limitations under the License.
 #
 
-export HADOOP_HOME=/opt/soft/hadoop
-export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop
-export SPARK_HOME1=/opt/soft/spark1
-export SPARK_HOME2=/opt/soft/spark2
-export PYTHON_HOME=/opt/soft/python
-export JAVA_HOME=/opt/soft/java
-export HIVE_HOME=/opt/soft/hive
-export FLINK_HOME=/opt/soft/flink
-export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$PATH
+export PYTHON_HOME=/usr/bin/python2
+export JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk
+export PATH=$PYTHON_HOME:$JAVA_HOME/bin:$PATH

dockerfile/Dockerfile

@@ -27,7 +27,7 @@ ENV DEBIAN_FRONTEND noninteractive
 #If install slowly, you can replcae alpine's mirror with aliyun's mirror, Example:
 #RUN sed -i "s/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g" /etc/apk/repositories
 RUN apk update && \
-    apk add dos2unix shadow bash openrc python sudo vim wget iputils net-tools openssh-server py2-pip tini && \
+    apk add dos2unix shadow bash openrc python python3 sudo vim wget iputils net-tools openssh-server py2-pip tini && \
     apk add --update procps && \
     openrc boot && \
     pip install kazoo
@@ -67,6 +67,7 @@ ADD ./checkpoint.sh /root/checkpoint.sh
 ADD ./startup-init-conf.sh /root/startup-init-conf.sh
 ADD ./startup.sh /root/startup.sh
 ADD ./conf/dolphinscheduler/*.tpl /opt/dolphinscheduler/conf/
+ADD ./conf/dolphinscheduler/logback/* /opt/dolphinscheduler/conf/
 ADD conf/dolphinscheduler/env/dolphinscheduler_env.sh /opt/dolphinscheduler/conf/env/
 RUN chmod +x /root/checkpoint.sh && \
     chmod +x /root/startup-init-conf.sh && \

dockerfile/conf/dolphinscheduler/env/dolphinscheduler_env.sh

@@ -15,12 +15,6 @@
 # limitations under the License.
 #
 
-export HADOOP_HOME=/opt/soft/hadoop
-export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop
-export SPARK_HOME1=/opt/soft/spark1
-export SPARK_HOME2=/opt/soft/spark2
-export PYTHON_HOME=/opt/soft/python
-export JAVA_HOME=/opt/soft/java
-export HIVE_HOME=/opt/soft/hive
-export FLINK_HOME=/opt/soft/flink
-export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$PATH
+export PYTHON_HOME=/usr/bin/python2
+export JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk
+export PATH=$PYTHON_HOME/bin:$JAVA_HOME/bin:$PATH

dockerfile/conf/dolphinscheduler/logback/logback-alert.xml

@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<!-- Logback configuration. See http://logback.qos.ch/manual/index.html -->
+<configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" -->
+
+    <property name="log.base" value="logs"/>
+    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder>
+            <pattern>
+                [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
+            </pattern>
+            <charset>UTF-8</charset>
+        </encoder>
+    </appender>
+
+    <appender name="ALERTLOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>${log.base}/dolphinscheduler-alert.log</file>
+        <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+            <fileNamePattern>${log.base}/dolphinscheduler-alert.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
+            <maxHistory>20</maxHistory>
+            <maxFileSize>64MB</maxFileSize>
+        </rollingPolicy>
+        <encoder>
+            <pattern>
+                [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
+            </pattern>
+            <charset>UTF-8</charset>
+        </encoder>
+    </appender>
+
+    <root level="INFO">
+        <appender-ref ref="STDOUT"/>
+        <appender-ref ref="APILOGFILE"/>
+    </root>
+
+</configuration>

dockerfile/conf/dolphinscheduler/logback/logback-api.xml

@@ -0,0 +1,62 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<!-- Logback configuration. See http://logback.qos.ch/manual/index.html -->
+<configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" -->
+
+    <property name="log.base" value="logs"/>
+    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder>
+            <pattern>
+                [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
+            </pattern>
+            <charset>UTF-8</charset>
+        </encoder>
+    </appender>
+
+    <!-- api server logback config start -->
+    <appender name="APILOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>${log.base}/dolphinscheduler-api-server.log</file>
+        <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+            <level>INFO</level>
+        </filter>
+        <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+            <fileNamePattern>${log.base}/dolphinscheduler-api-server.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
+            <maxHistory>168</maxHistory>
+            <maxFileSize>64MB</maxFileSize>
+        </rollingPolicy>
+        <encoder>
+            <pattern>
+                [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
+            </pattern>
+            <charset>UTF-8</charset>
+        </encoder>
+    </appender>
+    <!-- api server logback config end -->
+
+    <logger name="org.apache.zookeeper" level="WARN"/>
+    <logger name="org.apache.hbase" level="WARN"/>
+    <logger name="org.apache.hadoop" level="WARN"/>
+
+
+    <root level="INFO">
+        <appender-ref ref="STDOUT"/>
+        <appender-ref ref="APILOGFILE"/>
+    </root>
+
+</configuration>

dockerfile/conf/dolphinscheduler/logback/logback-master.xml

@@ -0,0 +1,82 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<!-- Logback configuration. See http://logback.qos.ch/manual/index.html -->
+<configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" -->
+
+    <property name="log.base" value="logs"/>
+    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder>
+            <pattern>
+                [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
+            </pattern>
+            <charset>UTF-8</charset>
+        </encoder>
+    </appender>
+
+    <conversionRule conversionWord="messsage"
+                    converterClass="org.apache.dolphinscheduler.server.log.SensitiveDataConverter"/>
+    <appender name="TASKLOGFILE" class="ch.qos.logback.classic.sift.SiftingAppender">
+        <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+            <level>INFO</level>
+        </filter>
+        <filter class="org.apache.dolphinscheduler.server.log.TaskLogFilter"/>
+        <Discriminator class="org.apache.dolphinscheduler.server.log.TaskLogDiscriminator">
+            <key>taskAppId</key>
+            <logBase>${log.base}</logBase>
+        </Discriminator>
+        <sift>
+            <appender name="FILE-${taskAppId}" class="ch.qos.logback.core.FileAppender">
+                <file>${log.base}/${taskAppId}.log</file>
+                <encoder>
+                    <pattern>
+                        [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %messsage%n
+                    </pattern>
+                    <charset>UTF-8</charset>
+                </encoder>
+                <append>true</append>
+            </appender>
+        </sift>
+    </appender>
+    <!-- master server logback config start -->
+    <appender name="MASTERLOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>${log.base}/dolphinscheduler-master.log</file>
+        <!--<filter class="org.apache.dolphinscheduler.server.log.MasterLogFilter">
+            <level>INFO</level>
+        </filter>-->
+        <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+            <fileNamePattern>${log.base}/dolphinscheduler-master.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
+            <maxHistory>168</maxHistory>
+            <maxFileSize>200MB</maxFileSize>
+        </rollingPolicy>
+        <encoder>
+            <pattern>
+                [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
+            </pattern>
+            <charset>UTF-8</charset>
+        </encoder>
+    </appender>
+    <!-- master server logback config end -->
+
+    <root level="INFO">
+        <appender-ref ref="STDOUT"/>
+        <appender-ref ref="TASKLOGFILE"/>
+        <appender-ref ref="MASTERLOGFILE"/>
+    </root>
+
+</configuration>

dockerfile/conf/dolphinscheduler/conf/worker_logback.xml

@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="UTF-8" ?>
+<?xml version="1.0" encoding="UTF-8"?>
 <!--
   ~ Licensed to the Apache Software Foundation (ASF) under one or more
   ~ contributor license agreements.  See the NOTICE file distributed with
@@ -17,7 +17,8 @@
   -->
 
 <!-- Logback configuration. See http://logback.qos.ch/manual/index.html -->
-<configuration scan="true" scanPeriod="120 seconds">
+<configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" -->
+
     <property name="log.base" value="logs"/>
     <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
         <encoder>
@@ -27,11 +28,15 @@
             <charset>UTF-8</charset>
         </encoder>
     </appender>
+
+    <!-- worker server logback config start -->
+    <conversionRule conversionWord="messsage"
+                    converterClass="org.apache.dolphinscheduler.server.log.SensitiveDataConverter"/>
     <appender name="TASKLOGFILE" class="ch.qos.logback.classic.sift.SiftingAppender">
         <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
             <level>INFO</level>
         </filter>
-        <filter class="org.apache.dolphinscheduler.server.log.TaskLogFilter"></filter>
+        <filter class="org.apache.dolphinscheduler.server.log.TaskLogFilter"/>
         <Discriminator class="org.apache.dolphinscheduler.server.log.TaskLogDiscriminator">
             <key>taskAppId</key>
             <logBase>${log.base}</logBase>
@@ -41,7 +46,7 @@
                 <file>${log.base}/${taskAppId}.log</file>
                 <encoder>
                     <pattern>
-                        [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
+                        [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %messsage%n
                     </pattern>
                     <charset>UTF-8</charset>
                 </encoder>
@@ -49,31 +54,30 @@
             </appender>
         </sift>
     </appender>
-
     <appender name="WORKERLOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
         <file>${log.base}/dolphinscheduler-worker.log</file>
-        <filter class="org.apache.dolphinscheduler.server.log.WorkerLogFilter">
+        <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
             <level>INFO</level>
         </filter>
-
+        <filter class="org.apache.dolphinscheduler.server.log.WorkerLogFilter"/>
         <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
             <fileNamePattern>${log.base}/dolphinscheduler-worker.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
             <maxHistory>168</maxHistory>
             <maxFileSize>200MB</maxFileSize>
         </rollingPolicy>
-        　　　　　
         <encoder>
             <pattern>
-                [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
+                [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %messsage%n
             </pattern>
             <charset>UTF-8</charset>
         </encoder>
-        　　
     </appender>
-
+    <!-- worker server logback config end -->
 
     <root level="INFO">
+        <appender-ref ref="STDOUT"/>
         <appender-ref ref="TASKLOGFILE"/>
         <appender-ref ref="WORKERLOGFILE"/>
     </root>
+
 </configuration>

dockerfile/startup.sh

@@ -25,7 +25,9 @@ DOLPHINSCHEDULER_LOGS=${DOLPHINSCHEDULER_HOME}/logs
 # start postgresql
 initPostgreSQL() {
     echo "checking postgresql"
-    if [ -n "$(ifconfig | grep ${POSTGRESQL_HOST})" ]; then
+    if [[ "${POSTGRESQL_HOST}" = "127.0.0.1" || "${POSTGRESQL_HOST}" = "localhost" ]]; then
+        export PGPORT=${POSTGRESQL_PORT}
+
         echo "start postgresql service"
         rc-service postgresql restart
 
@@ -47,10 +49,21 @@ initPostgreSQL() {
         sudo -u postgres psql -tAc "grant all privileges on database dolphinscheduler to ${POSTGRESQL_USERNAME}"
     fi
 
+    echo "test postgresql service"
+    while ! nc -z ${POSTGRESQL_HOST} ${POSTGRESQL_PORT}; do
+        counter=$((counter+1))
+        if [ $counter == 30 ]; then
+            echo "Error: Couldn't connect to postgresql."
+            exit 1
+        fi
+        echo "Trying to connect to postgresql at ${POSTGRESQL_HOST}:${POSTGRESQL_PORT}. Attempt $counter."
+        sleep 5
+    done
+
     echo "connect postgresql service"
-    v=$(sudo -u postgres PGPASSWORD=${POSTGRESQL_PASSWORD} psql -h ${POSTGRESQL_HOST} -U ${POSTGRESQL_USERNAME} -d dolphinscheduler -tAc "select 1")
+    v=$(sudo -u postgres PGPASSWORD=${POSTGRESQL_PASSWORD} psql -h ${POSTGRESQL_HOST} -p ${POSTGRESQL_PORT} -U ${POSTGRESQL_USERNAME} -d dolphinscheduler -tAc "select 1")
     if [ "$(echo '${v}' | grep 'FATAL' | wc -l)" -eq 1 ]; then
-        echo "Can't connect to database...${v}"
+        echo "Error: Can't connect to database...${v}"
         exit 1
     fi