spark 指定引數配置檔案
阿新 • • 發佈:2021-01-11
一般情況下,一個客戶端指定一個叢集,但是存在一個客戶端指定多個叢集的情況,這種情況下,需要動態調整配置檔案
//配置檔案可以通過傳參或者資料庫進行讀取
package com.cslc import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import scala.collection.JavaConversions._ import org.apache.spark.rdd.RDD import org.apache.spark.sql.SparkSession object Day01 { def main(args: Array[String]): Unit= { val sparkBuilder=SparkSession.builder val conf =new Configuration() val core=new Path("F:\\IdeaWorkspace\\lzm\\Resource\\core-site.xml") val hdfs=new Path("F:\\IdeaWorkspace\\lzm\\Resource\\hdfs-site.xml") val hive=new Path("F:\\IdeaWorkspace\\lzm\\Resource\\hive-site.xml") val yarn=new Path("F:\\IdeaWorkspace\\lzm\\Resource\\yarn-site.xml") val mapred=new Path("F:\\IdeaWorkspace\\lzm\\Resource\\mapred-site.xml") conf.addResource(hive) conf.addResource(core) conf.addResource(hdfs) conf.addResource(mapred) conf.addResource(yarn) for(c<-conf.iterator()){ sparkBuilder.config(c.getKey,c.getValue) } val spark:SparkSession=sparkBuilder.master("local[2]").getOrCreate() spark.sparkContext.setLogLevel("error") val data=spark.read.parquet("hdfs://cslcdip/home/dip/lzm/sparkdata/users.parquet") println(data.schema) data.show() } }