Browse Source

[python] Integrate test run strategy change (#10825)

this patch change python integrate test strategy,
before that, only content change in dir
dolphinscheduler-python will run the test. And it
will fail our python api startter when some code
change in java gateway. So we should start the
integrate test except docs change.

and in the future, we may add python ci to
.asf.yaml to make sure it pass

close: #10488
Jiajie Zhong 2 years ago
parent
commit
598290487c
1 changed files with 48 additions and 35 deletions
  1. 48 35
      .github/workflows/py-ci.yml

+ 48 - 35
.github/workflows/py-ci.yml

@@ -22,8 +22,6 @@ on:
     paths:
     paths:
       - 'dolphinscheduler-python/**'
       - 'dolphinscheduler-python/**'
   pull_request:
   pull_request:
-    paths:
-      - 'dolphinscheduler-python/**'
 
 
 concurrency:
 concurrency:
   group: py-${{ github.event.pull_request.number || github.ref }}
   group: py-${{ github.event.pull_request.number || github.ref }}
@@ -38,9 +36,27 @@ env:
   DEPENDENCES: pip setuptools wheel tox
   DEPENDENCES: pip setuptools wheel tox
 
 
 jobs:
 jobs:
+  paths-filter:
+    name: Python-Path-Filter
+    runs-on: ubuntu-latest
+    outputs:
+      not-docs: ${{ steps.filter.outputs.not-docs }}
+      py-change: ${{ steps.filter.outputs.py-change }}
+    steps:
+      - uses: actions/checkout@v2
+      - uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721
+        id: filter
+        with:
+          filters: |
+            not-docs:
+              - '!(docs/**)'
+            py-change:
+              - 'dolphinscheduler-python/pydolphinscheduler'
   lint:
   lint:
     name: Lint
     name: Lint
+    if: ${{ (needs.paths-filter.outputs.py-change == 'true') || (github.event_name == 'push') }}
     timeout-minutes: 15
     timeout-minutes: 15
+    needs: paths-filter
     runs-on: ubuntu-latest
     runs-on: ubuntu-latest
     steps:
     steps:
       - uses: actions/checkout@v2
       - uses: actions/checkout@v2
@@ -112,14 +128,12 @@ jobs:
       - name: Run Tests Build Docs
       - name: Run Tests Build Docs
         run: |
         run: |
           python -m tox -vv -e local-ci
           python -m tox -vv -e local-ci
-  build-image:
+  integrate-test:
-    name: Build Image
+    name: Integrate Test
+    if: ${{ (needs.paths-filter.outputs.not-docs == 'true') || (github.event_name == 'push') }}
     runs-on: ubuntu-latest
     runs-on: ubuntu-latest
-    # Switch to project root directory to run mvnw command
+    needs: paths-filter
-    defaults:
+    timeout-minutes: 30
-      run:
-        working-directory: ./
-    timeout-minutes: 20
     steps:
     steps:
       - uses: actions/checkout@v2
       - uses: actions/checkout@v2
         with:
         with:
@@ -134,7 +148,9 @@ jobs:
           path: ~/.m2/repository
           path: ~/.m2/repository
           key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
           key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
           restore-keys: ${{ runner.os }}-maven-
           restore-keys: ${{ runner.os }}-maven-
+      # Switch to project root directory to run mvnw command
       - name: Build Image
       - name: Build Image
+        working-directory: ./
         run: |
         run: |
           ./mvnw -B clean install \
           ./mvnw -B clean install \
           -Dmaven.test.skip \
           -Dmaven.test.skip \
@@ -142,32 +158,6 @@ jobs:
           -Dcheckstyle.skip=true \
           -Dcheckstyle.skip=true \
           -Pdocker,release -Ddocker.tag=ci \
           -Pdocker,release -Ddocker.tag=ci \
           -pl dolphinscheduler-standalone-server -am
           -pl dolphinscheduler-standalone-server -am
-      - name: Export Docker Images
-        run: |
-          docker save apache/dolphinscheduler-standalone-server:ci -o /tmp/standalone-image.tar \
-          && du -sh /tmp/standalone-image.tar
-      - uses: actions/upload-artifact@v2
-        name: Upload Docker Images
-        with:
-          name: standalone-image
-          path: /tmp/standalone-image.tar
-          retention-days: 1
-  integrate-test:
-    name: Integrate Test
-    timeout-minutes: 20
-    needs:
-      - build-image
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/download-artifact@v2
-        name: Download Docker Images
-        with:
-          name: standalone-image
-          path: /tmp
-      - name: Load Docker Images
-        run: |
-          docker load -i /tmp/standalone-image.tar
       - name: Set up Python 3.7
       - name: Set up Python 3.7
         uses: actions/setup-python@v2
         uses: actions/setup-python@v2
         with:
         with:
@@ -178,3 +168,26 @@ jobs:
       - name: Run Tests Build Docs
       - name: Run Tests Build Docs
         run: |
         run: |
           python -m tox -vv -e integrate-test
           python -m tox -vv -e integrate-test
+  result:
+    name: Python
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    needs: [ paths-filter, local-ci, integrate-test ]
+    if: always()
+    steps:
+      - name: Status
+        # We need change CWD to current directory to avoid global default working directory not exists
+        working-directory: ./
+        run: |
+          if [[ ${{ needs.paths-filter.outputs.not-docs }} == 'false' && ${{ github.event_name }} == 'pull_request' ]]; then
+            echo "Only document change, skip both python unit and integrate test!"
+            exit 0
+          fi
+          if [[ ${{ needs.paths-filter.outputs.py-change }} == 'false' && ${{ needs.integrate-test.result }} == 'success' && ${{ github.event_name }} == 'pull_request' ]]; then
+            echo "No python code change, and integrate test pass!"
+            exit 0
+          fi
+          if [[ ${{ needs.integrate-test.result }} != 'success' || ${{ needs.local-ci.result }} != 'success' ]]; then
+            echo "py-ci Failed!"
+            exit -1
+          fi