Browse Source

[Improvement] The spark version is associated with the spark command (#7544)

* [Improvement] The spark version is associated with the spark command
J·Y 3 years ago
parent
commit
a4168ae24e

+ 2 - 14
dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTask.java

@@ -34,18 +34,6 @@ import java.util.Map;
 
 public class SparkTask extends AbstractYarnTask {
 
-    /**
-     * spark1 command
-     * usage: spark-submit [options] <app jar | python file> [app arguments]
-     */
-    private static final String SPARK1_COMMAND = "${SPARK_HOME1}/bin/spark-submit";
-
-    /**
-     * spark2 command
-     * usage: spark-submit [options] <app jar | python file> [app arguments]
-     */
-    private static final String SPARK2_COMMAND = "${SPARK_HOME2}/bin/spark-submit";
-
     /**
      * spark parameters
      */
@@ -90,10 +78,10 @@ public class SparkTask extends AbstractYarnTask {
         List<String> args = new ArrayList<>();
 
         // spark version
-        String sparkCommand = SPARK2_COMMAND;
+        String sparkCommand = SparkVersion.SPARK2.getCommand();
 
         if (SparkVersion.SPARK1.name().equals(sparkParameters.getSparkVersion())) {
-            sparkCommand = SPARK1_COMMAND;
+            sparkCommand = SparkVersion.SPARK1.getCommand();
         }
 
         args.add(sparkCommand);

+ 16 - 6
dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkVersion.java

@@ -23,17 +23,23 @@ public enum SparkVersion {
      * 0 SPARK1
      * 1 SPARK2
      */
-    SPARK1(0, "SPARK1"),
-    SPARK2(1, "SPARK2");
+    SPARK1(0, "SPARK1", "${SPARK_HOME1}/bin/spark-submit"),
+    SPARK2(1, "SPARK2", "${SPARK_HOME2}/bin/spark-submit"),
+    ;
 
-    SparkVersion(int code, String descp) {
+    private final int code;
+    private final String descp;
+    /**
+     * usage: spark-submit [options] <app jar | python file> [app arguments]
+     */
+    private final String command;
+
+    SparkVersion(int code, String descp, String command) {
         this.code = code;
         this.descp = descp;
+        this.command = command;
     }
 
-    private final int code;
-    private final String descp;
-
     public int getCode() {
         return code;
     }
@@ -41,4 +47,8 @@ public enum SparkVersion {
     public String getDescp() {
         return descp;
     }
+
+    public String getCommand() {
+        return command;
+    }
 }