Merge pull request #2522 from liwenhe1993/docker-compose-dev

Running the docker cluster of dolphinscheduler by docker-compse

docker/docker-swarm/docker-compose.yml

@@ -0,0 +1,261 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3.4"
+
+networks:
+  dolphinscheduler-postgresql:
+    driver: bridge
+  dolphinscheduler-zookeeper:
+    driver: bridge
+  dolphinscheduler-api:
+    driver: bridge
+  dolphinscheduler-frontend:
+    driver: bridge
+  dolphinscheduler-alert:
+    driver: bridge
+  dolphinscheduler-master:
+    driver: bridge
+  dolphinscheduler-worker:
+    driver: bridge
+
+volumes:
+  dolphinscheduler-postgresql:
+  dolphinscheduler-zookeeper:
+  dolphinscheduler-api:
+  dolphinscheduler-frontend:
+  dolphinscheduler-alert:
+  dolphinscheduler-master:
+  dolphinscheduler-worker-data:
+  dolphinscheduler-worker-logs:
+
+configs:
+  dolphinscheduler-worker-task-env:
+    file: ./dolphinscheduler_env.sh
+
+services:
+
+  dolphinscheduler-postgresql:
+    image: bitnami/postgresql:latest
+    container_name: dolphinscheduler-postgresql
+    ports:
+    - 5432:5432
+    environment:
+      TZ: Asia/Shanghai
+      POSTGRESQL_USERNAME: root
+      POSTGRESQL_PASSWORD: root
+      POSTGRESQL_DATABASE: dolphinscheduler
+    healthcheck:
+      test: ["CMD", "pg_isready", "-U", "${POSTGRESQL_USERNAME}", "-d", "{POSTGRESQL_PASSWORD}", "-h", "localhost", "5432"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      # start_period: 30s
+    volumes: 
+    - dolphinscheduler-postgresql:/bitnami/postgresql
+    networks:
+    - dolphinscheduler-postgresql
+
+  dolphinscheduler-zookeeper:
+    image: bitnami/zookeeper:latest
+    container_name: dolphinscheduler-zookeeper
+    ports:
+    - 2181:2181
+    environment:
+      TZ: Asia/Shanghai
+      ALLOW_ANONYMOUS_LOGIN: "yes"
+    healthcheck:
+      test: ["CMD-SHELL", "nc -z localhost 2181"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      # start_period: 30s
+    volumes:
+    - dolphinscheduler-zookeeper:/bitnami/zookeeper
+    networks:
+    - dolphinscheduler-zookeeper
+
+  dolphinscheduler-api:
+    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:1.2.1
+    container_name: dolphinscheduler-api
+    command: ["api-server"]
+    ports:
+    - 12345:12345
+    environment:
+      TZ: Asia/Shanghai
+      POSTGRESQL_HOST: dolphinscheduler-postgresql
+      POSTGRESQL_PORT: 5432
+      POSTGRESQL_USERNAME: root
+      POSTGRESQL_PASSWORD: root
+      POSTGRESQL_DATABASE: dolphinscheduler
+      ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
+    healthcheck:
+      test: ["CMD-SHELL", "curl -f http://localhost:12345"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      # start_period: 30s
+    depends_on:
+    - dolphinscheduler-postgresql
+    - dolphinscheduler-zookeeper
+    volumes:
+    - dolphinscheduler-api:/opt/dolphinscheduler/logs
+    networks: 
+    - dolphinscheduler-api
+    - dolphinscheduler-postgresql
+    - dolphinscheduler-zookeeper
+
+  dolphinscheduler-frontend:
+    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:1.2.1
+    container_name: dolphinscheduler-frontend
+    command: ["frontend"]
+    ports:
+    - 8888:8888
+    environment:
+      TZ: Asia/Shanghai
+      FRONTEND_API_SERVER_HOST: dolphinscheduler-api
+      FRONTEND_API_SERVER_PORT: 12345
+    healthcheck:
+      test: ["CMD-SHELL", "curl -f http://localhost:8888"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      # start_period: 30s
+    depends_on:
+    - dolphinscheduler-api
+    volumes:
+    - dolphinscheduler-frontend:/var/log/nginx
+    networks:
+    - dolphinscheduler-frontend
+    - dolphinscheduler-api
+
+  dolphinscheduler-alert:
+    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:1.2.1
+    container_name: dolphinscheduler-alert
+    command: ["alert-server"]
+    environment:
+      TZ: Asia/Shanghai
+      XLS_FILE_PATH: "/tmp/xls"
+      MAIL_SERVER_HOST: ""
+      MAIL_SERVER_PORT: ""
+      MAIL_SENDER: ""
+      MAIL_USER: ""
+      MAIL_PASSWD: ""
+      MAIL_SMTP_STARTTLS_ENABLE: "false"
+      MAIL_SMTP_SSL_ENABLE: "false"
+      MAIL_SMTP_SSL_TRUST: ""
+      ENTERPRISE_WECHAT_ENABLE: "false"
+      ENTERPRISE_WECHAT_CORP_ID: ""
+      ENTERPRISE_WECHAT_SECRET: ""
+      ENTERPRISE_WECHAT_AGENT_ID: ""
+      ENTERPRISE_WECHAT_USERS: ""
+      POSTGRESQL_HOST: dolphinscheduler-postgresql
+      POSTGRESQL_PORT: 5432
+      POSTGRESQL_USERNAME: root
+      POSTGRESQL_PASSWORD: root
+      POSTGRESQL_DATABASE: dolphinscheduler
+    healthcheck:
+      test: ["CMD", "/root/checkpoint.sh", "AlertServer"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      # start_period: 30s
+    depends_on:
+    - dolphinscheduler-postgresql
+    volumes: 
+    - dolphinscheduler-alert:/opt/dolphinscheduler/logs
+    networks: 
+    - dolphinscheduler-alert
+    - dolphinscheduler-postgresql
+
+  dolphinscheduler-master:
+    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:1.2.1
+    container_name: dolphinscheduler-master
+    command: ["master-server"]
+    ports: 
+    - 5678:5678
+    environment: 
+      TZ: Asia/Shanghai
+      MASTER_EXEC_THREADS: "100"
+      MASTER_EXEC_TASK_NUM: "20"
+      MASTER_HEARTBEAT_INTERVAL: "10"
+      MASTER_TASK_COMMIT_RETRYTIMES: "5"
+      MASTER_TASK_COMMIT_INTERVAL: "1000"
+      MASTER_MAX_CPULOAD_AVG: "100"
+      MASTER_RESERVED_MEMORY: "0.1"
+      POSTGRESQL_HOST: dolphinscheduler-postgresql
+      POSTGRESQL_PORT: 5432
+      POSTGRESQL_USERNAME: root
+      POSTGRESQL_PASSWORD: root
+      POSTGRESQL_DATABASE: dolphinscheduler
+      ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
+    healthcheck:
+      test: ["CMD", "/root/checkpoint.sh", "MasterServer"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      # start_period: 30s
+    depends_on: 
+    - dolphinscheduler-postgresql
+    - dolphinscheduler-zookeeper
+    volumes: 
+    - dolphinscheduler-master:/opt/dolphinscheduler/logs
+    networks:
+    - dolphinscheduler-master
+    - dolphinscheduler-postgresql
+    - dolphinscheduler-zookeeper
+
+  dolphinscheduler-worker:
+    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:1.2.1
+    container_name: dolphinscheduler-worker
+    command: ["worker-server"]
+    ports: 
+    - 1234:1234
+    - 50051:50051
+    environment: 
+      TZ: Asia/Shanghai
+      WORKER_EXEC_THREADS: "100"
+      WORKER_HEARTBEAT_INTERVAL: "10"
+      WORKER_FETCH_TASK_NUM: "3"
+      WORKER_MAX_CPULOAD_AVG: "100"
+      WORKER_RESERVED_MEMORY: "0.1"
+      WORKER_GROUP: "default"
+      DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler"
+      POSTGRESQL_HOST: dolphinscheduler-postgresql
+      POSTGRESQL_PORT: 5432
+      POSTGRESQL_USERNAME: root
+      POSTGRESQL_PASSWORD: root
+      POSTGRESQL_DATABASE: dolphinscheduler
+      ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
+    healthcheck:
+      test: ["CMD", "/root/checkpoint.sh", "WorkerServer"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      # start_period: 30s
+    depends_on: 
+    - dolphinscheduler-postgresql
+    - dolphinscheduler-zookeeper
+    volumes: 
+    - dolphinscheduler-worker-data:/tmp/dolphinscheduler
+    - dolphinscheduler-worker-logs:/opt/dolphinscheduler/logs
+    configs:
+    - source: dolphinscheduler-worker-task-env
+      target: /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh
+    networks:
+    - dolphinscheduler-worker
+    - dolphinscheduler-postgresql
+    - dolphinscheduler-zookeeper

docker/docker-swarm/dolphinscheduler_env.sh

@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+export HADOOP_HOME=/opt/soft/hadoop
+export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop
+export SPARK_HOME1=/opt/soft/spark1
+export SPARK_HOME2=/opt/soft/spark2
+export PYTHON_HOME=/opt/soft/python
+export JAVA_HOME=/opt/soft/java
+export HIVE_HOME=/opt/soft/hive
+export FLINK_HOME=/opt/soft/flink
+export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$PATH

kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml

@@ -166,19 +166,19 @@ spec:
               value: {{ template "dolphinscheduler.postgresql.fullname" . }}
               {{- else }}
               value: {{ .Values.externalDatabase.host | quote }}
-            {{- end }}
+              {{- end }}
             - name: POSTGRESQL_PORT
               {{- if .Values.postgresql.enabled }}
               value: "5432"
               {{- else }}
               value: {{ .Values.externalDatabase.port }}
-            {{- end }}
+              {{- end }}
             - name: POSTGRESQL_USERNAME
               {{- if .Values.postgresql.enabled }}
               value: {{ .Values.postgresql.postgresqlUsername }}
               {{- else }}
               value: {{ .Values.externalDatabase.username | quote }}
-            {{- end }}
+              {{- end }}
             - name: POSTGRESQL_PASSWORD
               valueFrom:
                 secretKeyRef:
@@ -188,7 +188,13 @@ spec:
                   {{- else }}
                   name: {{ printf "%s-%s" .Release.Name "externaldb"  }}
                   key: db-password
-          {{- end }}
+                  {{- end }}
+            - name: POSTGRESQL_DATABASE
+              {{- if .Values.postgresql.enabled }}
+              value: {{ .Values.postgresql.postgresqlDatabase }}
+              {{- else }}
+              value: {{ .Values.externalDatabase.database | quote }}
+              {{- end }}
           {{- if .Values.alert.livenessProbe.enabled }}
           livenessProbe:
             exec:

kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml

@@ -99,19 +99,19 @@ spec:
               value: {{ template "dolphinscheduler.postgresql.fullname" . }}
               {{- else }}
               value: {{ .Values.externalDatabase.host | quote }}
-            {{- end }}
+              {{- end }}
             - name: POSTGRESQL_PORT
               {{- if .Values.postgresql.enabled }}
               value: "5432"
               {{- else }}
               value: {{ .Values.externalDatabase.port }}
-            {{- end }}
+              {{- end }}
             - name: POSTGRESQL_USERNAME
               {{- if .Values.postgresql.enabled }}
               value: {{ .Values.postgresql.postgresqlUsername }}
               {{- else }}
               value: {{ .Values.externalDatabase.username | quote }}
-            {{- end }}
+              {{- end }}
             - name: POSTGRESQL_PASSWORD
               valueFrom:
                 secretKeyRef:
@@ -122,6 +122,12 @@ spec:
                   name: {{ printf "%s-%s" .Release.Name "externaldb"  }}
                   key: db-password
                   {{- end }}
+            - name: POSTGRESQL_DATABASE
+              {{- if .Values.postgresql.enabled }}
+              value: {{ .Values.postgresql.postgresqlDatabase }}
+              {{- else }}
+              value: {{ .Values.externalDatabase.database | quote }}
+              {{- end }}
             - name: ZOOKEEPER_QUORUM
               {{- if .Values.zookeeper.enabled }}
               value: "{{ template "dolphinscheduler.zookeeper.quorum" . }}"

kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml

@@ -188,6 +188,12 @@ spec:
                   name: {{ printf "%s-%s" .Release.Name "externaldb"  }}
                   key: db-password
                   {{- end }}
+            - name: POSTGRESQL_DATABASE
+              {{- if .Values.postgresql.enabled }}
+              value: {{ .Values.postgresql.postgresqlDatabase }}
+              {{- else }}
+              value: {{ .Values.externalDatabase.database | quote }}
+              {{- end }}
             - name: ZOOKEEPER_QUORUM
               {{- if .Values.zookeeper.enabled }}
               value: {{ template "dolphinscheduler.zookeeper.quorum" . }}

kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml

@@ -184,7 +184,13 @@ spec:
                   {{- else }}
                   name: {{ printf "%s-%s" .Release.Name "externaldb"  }}
                   key: db-password
-            {{- end }}
+                  {{- end }}
+            - name: POSTGRESQL_DATABASE
+              {{- if .Values.postgresql.enabled }}
+              value: {{ .Values.postgresql.postgresqlDatabase }}
+              {{- else }}
+              value: {{ .Values.externalDatabase.database | quote }}
+              {{- end }}
             - name: ZOOKEEPER_QUORUM
               {{- if .Values.zookeeper.enabled }}
               value: "{{ template "dolphinscheduler.zookeeper.quorum" . }}"

dockerfile/startup.sh

@@ -70,10 +70,10 @@ initZK() {
             while ! nc -z ${line%:*} ${line#*:}; do
                 counter=$((counter+1))
                 if [ $counter == 30 ]; then
-                    log "Error: Couldn't connect to zookeeper."
+                    echo "Error: Couldn't connect to zookeeper."
                     exit 1
                 fi
-                log "Trying to connect to zookeeper at ${line}. Attempt $counter."
+                echo "Trying to connect to zookeeper at ${line}. Attempt $counter."
                 sleep 5
             done
         done