Prechádzať zdrojové kódy

[python] Add autoflake to auto rm unused import (#8897)

* Add a new commit hook to pre-commit, to auto
  remove unused imports
* Remove all unused pass statements
* Add autoflake to tox and out CI

close: #8592
Jiajie Zhong 3 rokov pred
rodič
commit
e53ac4e304

+ 9 - 0
dolphinscheduler-python/pydolphinscheduler/.pre-commit-config.yaml

@@ -45,3 +45,12 @@ repos:
             --config, 
             --config, 
             dolphinscheduler-python/pydolphinscheduler/.flake8
             dolphinscheduler-python/pydolphinscheduler/.flake8
         ]
         ]
+  - repo: https://github.com/pycqa/autoflake
+    rev: v1.4
+    hooks:
+      - id: autoflake
+        args: [
+            --remove-all-unused-imports,
+            --ignore-init-module-imports,
+            --in-place
+        ]

+ 4 - 0
dolphinscheduler-python/pydolphinscheduler/DEVELOP.md

@@ -104,6 +104,10 @@ tox -e local-ci
 It will take a while when you run it the first time, because it has to install dependencies and make some prepare,
 It will take a while when you run it the first time, because it has to install dependencies and make some prepare,
 and the next time you run it will be faster.
 and the next time you run it will be faster.
 
 
+If you failed section `lint` when you run command `tox -e local-ci`, you could try to run command `tox -e auto-lint`
+which we provider fix as many lints as possible. When I finish, you could run command `tox -e local-ci` to see
+whether the linter pass or not, you have to fix it by yourself if linter still fail. 
+
 ### Manually
 ### Manually
 
 
 #### Code Style
 #### Code Style

+ 1 - 2
dolphinscheduler-python/pydolphinscheduler/setup.py

@@ -65,6 +65,7 @@ style = [
     "flake8-docstrings>=1.6",
     "flake8-docstrings>=1.6",
     "flake8-black>=0.2",
     "flake8-black>=0.2",
     "isort>=5.10",
     "isort>=5.10",
+    "autoflake>=1.4",
 ]
 ]
 
 
 dev = style + test + doc + build
 dev = style + test + doc + build
@@ -95,11 +96,9 @@ class CleanCommand(Command):
 
 
     def initialize_options(self) -> None:
     def initialize_options(self) -> None:
         """Set default values for options."""
         """Set default values for options."""
-        pass
 
 
     def finalize_options(self) -> None:
     def finalize_options(self) -> None:
         """Set final values for options."""
         """Set final values for options."""
-        pass
 
 
     def run(self) -> None:
     def run(self) -> None:
         """Run and remove temporary files."""
         """Run and remove temporary files."""

+ 0 - 10
dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/exceptions.py

@@ -21,26 +21,18 @@
 class PyDSBaseException(Exception):
 class PyDSBaseException(Exception):
     """Base exception for pydolphinscheduler."""
     """Base exception for pydolphinscheduler."""
 
 
-    pass
-
 
 
 class PyDSParamException(PyDSBaseException):
 class PyDSParamException(PyDSBaseException):
     """Exception for pydolphinscheduler parameter verify error."""
     """Exception for pydolphinscheduler parameter verify error."""
 
 
-    pass
-
 
 
 class PyDSTaskNoFoundException(PyDSBaseException):
 class PyDSTaskNoFoundException(PyDSBaseException):
     """Exception for pydolphinscheduler workflow task no found error."""
     """Exception for pydolphinscheduler workflow task no found error."""
 
 
-    pass
-
 
 
 class PyDSJavaGatewayException(PyDSBaseException):
 class PyDSJavaGatewayException(PyDSBaseException):
     """Exception for pydolphinscheduler Java gateway error."""
     """Exception for pydolphinscheduler Java gateway error."""
 
 
-    pass
-
 
 
 class PyDSProcessDefinitionNotAssignException(PyDSBaseException):
 class PyDSProcessDefinitionNotAssignException(PyDSBaseException):
     """Exception for pydolphinscheduler process definition not assign error."""
     """Exception for pydolphinscheduler process definition not assign error."""
@@ -48,5 +40,3 @@ class PyDSProcessDefinitionNotAssignException(PyDSBaseException):
 
 
 class PyDSConfException(PyDSBaseException):
 class PyDSConfException(PyDSBaseException):
     """Exception for pydolphinscheduler configuration error."""
     """Exception for pydolphinscheduler configuration error."""
-
-    pass

+ 9 - 1
dolphinscheduler-python/pydolphinscheduler/tox.ini

@@ -16,17 +16,25 @@
 # under the License.
 # under the License.
 
 
 [tox]
 [tox]
-envlist = local-ci, lint, doc-build-test, code-test, py{36,37,38,39}
+envlist = local-ci, auto-lint, lint, doc-build-test, code-test, py{36,37,38,39}
 
 
 [testenv]
 [testenv]
 whitelist_externals = make
 whitelist_externals = make
 
 
+[testenv:auto-lint]
+extras = style
+commands =
+    python -m isort .
+    python -m black .
+    python -m autoflake --in-place --remove-all-unused-imports --ignore-init-module-imports --recursive .
+
 [testenv:lint]
 [testenv:lint]
 extras = style
 extras = style
 commands =
 commands =
     python -m isort --check .
     python -m isort --check .
     python -m black --check .
     python -m black --check .
     python -m flake8
     python -m flake8
+    python -m autoflake --remove-all-unused-imports --ignore-init-module-imports --check --recursive .
     
     
 [testenv:code-test]
 [testenv:code-test]
 extras = test
 extras = test