SparkSubmit.main()方法提交外部引數,遠端提交standalone叢集任務
一、官方文件描述
二、引入pom【注意版本問題】
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.11</artifactId>
<version>2.2.0</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>2.2.0</version>
</dependency>
三、程式碼
import java.io.IOException;
import org.apache.spark.deploy.SparkSubmit;
public class Main {
public static void main(String[] args) throws IOException {
String appName = "test java web submit job to spark standalone";
String className="cfca.xfraud.mltrain.DataTrainning";
String [] arg0=new String[]{
"--master","spark://192.168.115.64:7077",//ip埠
"--deploy-mode","cluster",
"--name",appName,
"--class",className,//執行主類main
"--executor-memory","2G",
"--total-executor-cores","10",
"--executor-cores","2",
"/home/xfraud/ml_train/MachineLearningTraining-0.2.jar",//在linux上的包 可改為hdfs上面的路徑
"LR", "20180817111111", "66"//jar中的引數,注意這裡的引數寫法
};
SparkSubmit.main(arg0);
}
}
四、執行結果