Browse Source

[Feature-8222][python] move examples into the scope of source package (#8340)

Eric Gao 3 years ago
parent
commit
49a1270ef5
21 changed files with 41 additions and 18 deletions
  1. 1 1
      dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/condition.rst
  2. 1 1
      dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/datax.rst
  3. 1 1
      dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/dependent.rst
  4. 1 1
      dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/flink.rst
  5. 1 1
      dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/map_reduce.rst
  6. 1 1
      dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/shell.rst
  7. 1 1
      dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/spark.rst
  8. 1 1
      dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/switch.rst
  9. 7 7
      dolphinscheduler-python/pydolphinscheduler/docs/source/tutorial.rst
  10. 18 0
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/__init__.py
  11. 0 0
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/bulk_create_example.py
  12. 0 0
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_condition_example.py
  13. 0 0
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_datax_example.py
  14. 0 0
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_dependent_example.py
  15. 0 0
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_flink_example.py
  16. 0 0
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_map_reduce_example.py
  17. 0 0
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_spark_example.py
  18. 0 0
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_switch_example.py
  19. 0 0
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial.py
  20. 5 2
      dolphinscheduler-python/pydolphinscheduler/tests/example/test_example.py
  21. 3 1
      dolphinscheduler-python/pydolphinscheduler/tests/testing/path.py

+ 1 - 1
dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/condition.rst

@@ -23,7 +23,7 @@ A condition task type's example and dive into information of **PyDolphinSchedule
 Example
 -------
 
-.. literalinclude:: ../../../examples/task_condition_example.py
+.. literalinclude:: ../../../src/pydolphinscheduler/examples/task_condition_example.py
    :start-after: [start workflow_declare]
    :end-before: [end workflow_declare]
 

+ 1 - 1
dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/datax.rst

@@ -23,7 +23,7 @@ A DataX task type's example and dive into information of **PyDolphinScheduler**.
 Example
 -------
 
-.. literalinclude:: ../../../examples/task_datax_example.py
+.. literalinclude:: ../../../src/pydolphinscheduler/examples/task_datax_example.py
    :start-after: [start workflow_declare]
    :end-before: [end workflow_declare]
 

+ 1 - 1
dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/dependent.rst

@@ -23,7 +23,7 @@ A dependent task type's example and dive into information of **PyDolphinSchedule
 Example
 -------
 
-.. literalinclude:: ../../../examples/task_dependent_example.py
+.. literalinclude:: ../../../src/pydolphinscheduler/examples/task_dependent_example.py
    :start-after: [start workflow_declare]
    :end-before: [end workflow_declare]
 

+ 1 - 1
dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/flink.rst

@@ -23,7 +23,7 @@ A flink task type's example and dive into information of **PyDolphinScheduler**.
 Example
 -------
 
-.. literalinclude:: ../../../examples/task_flink_example.py
+.. literalinclude:: ../../../src/pydolphinscheduler/examples/task_flink_example.py
    :start-after: [start workflow_declare]
    :end-before: [end workflow_declare]
 

+ 1 - 1
dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/map_reduce.rst

@@ -24,7 +24,7 @@ A Map Reduce task type's example and dive into information of **PyDolphinSchedul
 Example
 -------
 
-.. literalinclude:: ../../../examples/task_map_reduce_example.py
+.. literalinclude:: ../../../src/pydolphinscheduler/examples/task_map_reduce_example.py
    :start-after: [start workflow_declare]
    :end-before: [end workflow_declare]
 

+ 1 - 1
dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/shell.rst

@@ -23,7 +23,7 @@ A shell task type's example and dive into information of **PyDolphinScheduler**.
 Example
 -------
 
-.. literalinclude:: ../../../examples/tutorial.py
+.. literalinclude:: ../../../src/pydolphinscheduler/examples/tutorial.py
    :start-after: [start workflow_declare]
    :end-before: [end task_relation_declare]
 

+ 1 - 1
dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/spark.rst

@@ -23,7 +23,7 @@ A spark task type's example and dive into information of **PyDolphinScheduler**.
 Example
 -------
 
-.. literalinclude:: ../../../examples/task_spark_example.py
+.. literalinclude:: ../../../src/pydolphinscheduler/examples/task_spark_example.py
    :start-after: [start workflow_declare]
    :end-before: [end workflow_declare]
 

+ 1 - 1
dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/switch.rst

@@ -23,7 +23,7 @@ A switch task type's example and dive into information of **PyDolphinScheduler**
 Example
 -------
 
-.. literalinclude:: ../../../examples/task_switch_example.py
+.. literalinclude:: ../../../src/pydolphinscheduler/examples/task_switch_example.py
    :start-after: [start workflow_declare]
    :end-before: [end workflow_declare]
 

+ 7 - 7
dolphinscheduler-python/pydolphinscheduler/docs/source/tutorial.rst

@@ -29,7 +29,7 @@ Overview of Tutorial
 Here have an overview of our tutorial, and it look a little complex but do not
 worry about that because we explain this example below as detailed as possible.
 
-.. literalinclude:: ../../examples/tutorial.py
+.. literalinclude:: ../../src/pydolphinscheduler/examples/tutorial.py
    :start-after: [start tutorial]
    :end-before: [end tutorial]
 
@@ -41,7 +41,7 @@ like other Python package. We just create a minimum demo here, so we just import
 :class:`pydolphinscheduler.core.process_definition` and
 :class:`pydolphinscheduler.tasks.shell`.
 
-.. literalinclude:: ../../examples/tutorial.py
+.. literalinclude:: ../../src/pydolphinscheduler/examples/tutorial.py
    :start-after: [start package_import]
    :end-before: [end package_import]
 
@@ -60,7 +60,7 @@ interval and schedule start_time, and argument `tenant` which changing workflow'
 task running user in the worker, :ref:`section tenant <concept:tenant>` in *PyDolphinScheduler*
 :doc:`concept` page have more detail information.
 
-.. literalinclude:: ../../examples/tutorial.py
+.. literalinclude:: ../../src/pydolphinscheduler/examples/tutorial.py
    :start-after: [start workflow_declare]
    :end-before: [end workflow_declare]
 
@@ -77,7 +77,7 @@ Here we declare four tasks, and bot of them are simple task of
 Beside the argument `command`, we also need setting argument `name` for each task *(not
 only shell task, `name` is required for each type of task)*.
 
-.. literalinclude:: ../../examples/tutorial.py
+.. literalinclude:: ../../src/pydolphinscheduler/examples/tutorial.py
    :dedent: 0
    :start-after: [start task_declare]
    :end-before: [end task_declare]
@@ -99,7 +99,7 @@ In this example, we set task `task_parent` is the upstream task of task
 `task_child_one` and `task_child_two`, and task `task_union` is the downstream
 task of both these two task.
 
-.. literalinclude:: ../../examples/tutorial.py
+.. literalinclude:: ../../src/pydolphinscheduler/examples/tutorial.py
    :dedent: 0
    :start-after: [start task_relation_declare]
    :end-before: [end task_relation_declare]
@@ -124,7 +124,7 @@ to the daemon, and set the schedule time we just declare in `process definition
 Now, we could run the Python code like other Python script, for the basic usage run
 :code:`python tutorial.py` to trigger and run it.
 
-.. literalinclude:: ../../examples/tutorial.py
+.. literalinclude:: ../../src/pydolphinscheduler/examples/tutorial.py
    :dedent: 0
    :start-after: [start submit_or_run]
    :end-before: [end submit_or_run]
@@ -142,7 +142,7 @@ After we run the tutorial code, you could login Apache DolphinScheduler web UI,
 go and see the `DolphinScheduler project page`_. they is a new process definition be
 created and named "Tutorial". It create by *PyDolphinScheduler* and the DAG graph as below
 
-.. literalinclude:: ../../examples/tutorial.py
+.. literalinclude:: ../../src/pydolphinscheduler/examples/tutorial.py
    :language: text
    :lines: 24-28
 

+ 18 - 0
dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/__init__.py

@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Init examples package which provides users with pydolphinscheduler examples."""

dolphinscheduler-python/pydolphinscheduler/examples/bulk_create_example.py → dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/bulk_create_example.py


dolphinscheduler-python/pydolphinscheduler/examples/task_condition_example.py → dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_condition_example.py


dolphinscheduler-python/pydolphinscheduler/examples/task_datax_example.py → dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_datax_example.py


dolphinscheduler-python/pydolphinscheduler/examples/task_dependent_example.py → dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_dependent_example.py



dolphinscheduler-python/pydolphinscheduler/examples/task_map_reduce_example.py → dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_map_reduce_example.py


dolphinscheduler-python/pydolphinscheduler/examples/task_spark_example.py → dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_spark_example.py


dolphinscheduler-python/pydolphinscheduler/examples/task_switch_example.py → dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_switch_example.py


dolphinscheduler-python/pydolphinscheduler/examples/tutorial.py → dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial.py


+ 5 - 2
dolphinscheduler-python/pydolphinscheduler/tests/example/test_example.py

@@ -96,8 +96,8 @@ def test_example_basic():
             ex.suffix == ".py"
         ), f"We expect all examples is python script, but get {ex.name}."
 
-        # All except tutorial is end with keyword "_example"
-        if ex.stem != "tutorial":
+        # All except tutorial and __init__ is end with keyword "_example"
+        if ex.stem != "tutorial" and ex.stem != "__init__":
             assert ex.stem.endswith(
                 "_example"
             ), f"We expect all examples script end with keyword '_example', but get {ex.stem}."
@@ -159,6 +159,9 @@ def test_file_name_in_process_definition(mock_code_version, mock_submit, mock_st
     """
     global process_definition_name
     for ex in get_all_examples():
+        # Skip __init__ file
+        if ex.stem == "__init__":
+            continue
         # Skip bulk_create_example check, cause it contain multiple workflow and
         # without one named bulk_create_example
         if ex.stem == "bulk_create_example":

+ 3 - 1
dolphinscheduler-python/pydolphinscheduler/tests/testing/path.py

@@ -23,7 +23,9 @@ from typing import Any, Generator
 path_code_tasks = Path(__file__).parent.parent.parent.joinpath(
     "src", "pydolphinscheduler", "tasks"
 )
-path_example = Path(__file__).parent.parent.parent.joinpath("examples")
+path_example = Path(__file__).parent.parent.parent.joinpath(
+    "src", "pydolphinscheduler", "examples"
+)
 path_doc_tasks = Path(__file__).parent.parent.parent.joinpath("docs", "source", "tasks")