[python] Add tox for out of box test (#8837)

* [python] Add tox for out of box test

* Add `tox` for easy and complete test
* Correct package data attribute
* Add more dirs in flake8 ignore

* Change GA from raw to tox

* Fix env setter

* Temp skip sanity

* Rm not exists job

* Add code test

* fix

* rm all need

* fix

* fix

* fix

* fix

* fix

* fix

* Migrate pip upgrede to tox setting

* Change install_commands to commands

* fix config of install command

* Add GA env to do that

* Fix env error

* Finial commit

* remove duplicate

* Change doc

* Change dependence of ci

* Change dependence of ci

.github/workflows/py-ci.yml

@@ -25,22 +25,21 @@ on:
     paths:
       - 'dolphinscheduler-python/**'
 
+concurrency:
+  group: py-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 defaults:
   run:
     working-directory: dolphinscheduler-python/pydolphinscheduler
 
+# We have to update setuptools wheel to package with package_data, LICENSE, NOTICE
+env:
+  DEPENDENCES: pip setuptools wheel tox
+
 jobs:
-  sanity:
-    name: Sanity
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: true
-      - name: Sanity Check
-        uses: ./.github/actions/sanity-check
   lint:
-    name: Code Style
+    name: Lint
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
@@ -48,19 +47,15 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: 3.7
-      - name: Install Development Dependences
-        run: pip install -e .[style]
-      - name: Run Isort Checking
-        run: isort --check .
-      - name: Run Black Checking
-        run: black --check .
-      - name: Run Flake8 Checking
-        run: flake8
+      - name: Install Dependences
+        run: |
+          python -m pip install --upgrade ${{ env.DEPENDENCES }}
+      - name: Run All Lint Check
+        run: |
+          python -m tox -vv -e lint
   pytest:
     name: Pytest
-    needs:
-      - lint
-      - sanity
+    needs: lint
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -73,16 +68,15 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Install dependencies & pydolphinscheduler
+      - name: Install Dependences
         run: |
-          pip install -e .[test]
-      - name: Run tests
+          python -m pip install --upgrade ${{ env.DEPENDENCES }}
+      - name: Run All Tests
         run: |
-          pytest
-  coverage:
-    name: Tests coverage
-    needs:
-      - pytest
+          python -m tox -vv -e code-test
+  doc-build:
+    name: Docs Build Test
+    needs: lint
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
@@ -90,14 +84,17 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: 3.7
-      - name: Install Development Dependences
+      - name: Install Dependences
         run: |
-          pip install -e .[test]
-      - name: Run Tests && Check coverage
-        run: coverage run && coverage report
-  doc-build:
-    name: Document Build Test
-    needs: pytest
+          python -m pip install --upgrade ${{ env.DEPENDENCES }}
+      - name: Run Tests Build Docs
+        run: |
+          python -m tox -vv -e doc-build-test
+  verify-local-ci:
+    name: Local CI
+    needs:
+      - pytest
+      - doc-build
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
@@ -105,9 +102,9 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: 3.7
-      - name: Install Development Dependences
+      - name: Install Dependences
+        run: |
+          python -m pip install --upgrade ${{ env.DEPENDENCES }}
+      - name: Run Tests Build Docs
         run: |
-          pip install -e .[doc]
-      - name: Test Build Document
-        working-directory: dolphinscheduler-python/pydolphinscheduler/docs
-        run: make clean && make html
+          python -m tox -vv -e local-ci

.gitignore

@@ -55,6 +55,7 @@ dolphinscheduler-worker/logs
 # ------------------
 # Cache
 __pycache__/
+.tox/
 
 # Build
 build/

dolphinscheduler-python/pydolphinscheduler/.coveragerc

@@ -20,8 +20,10 @@ command_line = -m pytest
 omit = 
     # Ignore all test cases in tests/
     tests/*
+    # Ignore examples directory
+    */pydolphinscheduler/examples/*
     # TODO. Temporary ignore java_gateway file, because we could not find good way to test it.
-    src/pydolphinscheduler/java_gateway.py
+    */pydolphinscheduler/java_gateway.py
 
 [report]
 # Don’t report files that are 100% covered

dolphinscheduler-python/pydolphinscheduler/.flake8

@@ -26,7 +26,9 @@ exclude =
     old,
     build,
     dist,
-    htmlcov
+    htmlcov,
+    .tox,
+    dist,
 ignore = 
     # It's clear and not need to add docstring
     D107,  # D107: Don't require docstrings on __init__

dolphinscheduler-python/pydolphinscheduler/DEVELOP.md

@@ -60,7 +60,53 @@ pydolphinscheduler tasks object, we use tasks to define exact job we want Dolphi
 we only support `shell` task to execute shell task. [This link][all-task] list all tasks support in DolphinScheduler
 and would be implemented in the further.
 
-## Code Style
+## Test Your Code
+
+Linting and tests is very important for open source project, so we pay more attention to it. We have continuous
+integration service run by GitHub Action to test whether the patch is good or not, which you could jump to
+section [With GitHub Action](#with-github-action) see more detail.
+
+And to make more convenience to local tests, we also have the way to run your [test automated with tox](#automated-testing-with-tox)
+locally. It is helpful when your try to find out the detail when continuous integration in GitHub Action failed,
+or you have a great patch and want to test local first.
+
+Besides [automated testing with tox](#automated-testing-with-tox) locally, we also have a [manual way](#manually)
+run tests. And it is scattered commands to reproduce each step of the integration test we told about.
+
+* Remote
+  * [With GitHub Action](#with-github-action)
+* Local
+  * [Automated Testing With tox](#automated-testing-with-tox)
+  * [Manually](#manually)
+
+### With GitHub Action
+
+GitHub Action test in various environment for pydolphinscheduler, including different python version in
+`3.6|3.7|3.8|3.9` and operating system `linux|macOS|windows`. It will trigger and run automatically when you
+submit pull requests to `apache/dolphinscheduler`. 
+
+### Automated Testing With tox
+
+[tox](https://tox.wiki) is a package aims to automate and standardize testing in Python, both our continuous
+integration and local test use it to run actual task. To use it, you should install it first
+
+```shell
+python -m pip install --upgrade tox
+```
+
+After installation, you could run a single command to run all the tests, it is almost like test in GitHub Action
+but not so much different environment.
+
+```shell
+tox -e local-ci
+```
+
+It will take a while when you run it the first time, because it has to install dependencies and make some prepare,
+and the next time you run it will be faster.
+
+### Manually
+
+#### Code Style
 
 We use [isort][isort] to automatically keep Python imports alphabetically, and use [Black][black] for code
 formatter and [Flake8][flake8] for pep8 checker. If you use [pycharm][pycharm]or [IntelliJ IDEA][idea],
@@ -74,33 +120,30 @@ GitHub, you could also run static check locally.
 # but Flake8 just hint when code style not match pep8
 
 # Run Isort
-isort .
+python -m isort .
 
 # Run Black
-black .
+python -m black .
 
 # Run Flake8
-flake8
+python -m flake8
 ```
 
-## Testing
+#### Testing
 
-pydolphinscheduler using [pytest][pytest] to test our codebase. GitHub Action will run our test when you create
-pull request or commit to dev branch, with python version `3.6|3.7|3.8|3.9` and operating system `linux|macOS|windows`.
-
-To test locally, you could directly run pytest after set `PYTHONPATH` 
+pydolphinscheduler using [pytest][pytest] to run all tests in directory `tests`. You could run tests by the commands
 
 ```shell
-PYTHONPATH=src/ pytest
+python -m pytest --cov=pydolphinscheduler --cov-config=.coveragerc tests/
 ```
 
-We try to keep pydolphinscheduler usable through unit test coverage. 90% test coverage is our target, but for
-now, we require test coverage up to 85%, and each pull request leas than 85% would fail our CI step
-`Tests coverage`. We use [coverage][coverage] to check our test coverage, and you could check it locally by
-run command.
+Besides run tests, it will also check the unit test [coverage][coverage] threshold, for now when test cover less than 90%
+will fail the coverage, as well as our GitHub Action.
+
+The command above will check test coverage automatically, and you could also test the coverage by command.
 
 ```shell
-coverage run && coverage report
+python -m coverage run && python -m  coverage report
 ```
 
 It would not only run unit test but also show each file coverage which cover rate less than 100%, and `TOTAL`

dolphinscheduler-python/pydolphinscheduler/setup.py

@@ -57,6 +57,7 @@ test = [
     "pytest>=6.2",
     "freezegun>=1.1",
     "coverage>=6.1",
+    "pytest-cov>=3.0",
 ]
 
 style = [
@@ -144,7 +145,7 @@ setup(
     package_dir={"": "src"},
     include_package_data=True,
     package_data={
-        "examples": ["examples.tutorial.py"],
+        "pydolphinscheduler": ["core/default_config.yaml"],
     },
     platforms=["any"],
     classifiers=[

dolphinscheduler-python/pydolphinscheduler/tox.ini

@@ -0,0 +1,48 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[tox]
+envlist = local-ci, lint, doc-build-test, code-test, py{36,37,38,39}
+
+[testenv]
+whitelist_externals = make
+
+[testenv:lint]
+extras = style
+commands =
+    python -m isort --check .
+    python -m black --check .
+    python -m flake8
+    
+[testenv:code-test]
+extras = test
+# Run both tests and coverage
+commands =
+    python -m pytest --cov=pydolphinscheduler --cov-config={toxinidir}/.coveragerc tests/
+
+[testenv:doc-build-test]
+extras = doc
+commands =
+    make -C {toxinidir}/docs clean
+    make -C {toxinidir}/docs html
+
+[testenv:local-ci]
+extras = dev
+commands =
+    {[testenv:lint]commands}
+    {[testenv:code-test]commands}
+    {[testenv:doc-build-test]commands}