Spark中的各種action運算元操作(scala版)
這裡直接貼程式碼了,action的介紹都在java那裡。
package cn.spark.study.core
import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
object ActionOperation {
def main(args: Array[String]): Unit = {
//reduceTest()
// collectTest()
//takeTest()
//countTest()
countByKeyTest()
}
def reduceTest(){
val conf = new SparkConf()
.setAppName(“reduce”)
.setMaster(“local”)
val sc = new SparkContext(conf)
val list = Array(1,2,3,4,5,6,7,8,9,10)
val numbersRDD = sc.parallelize(list, 1)
val sum = numbersRDD.reduce(_+_)
println(sum)
}
def collectTest(){
val conf = new SparkConf()
.setAppName(“collect”)
.setMaster(“local”)
val sc = new SparkContext(conf)
val list = Array(1,2,3,4,5,6,7,8,9,10) val numbersRDD = sc.parallelize(list, 1) val doubleNumber = numbersRDD.map { num => num*2 } val doubleList = doubleNumber.collect() for(num <- doubleList){ println(num) }
}
def takeTest(){
val conf = new SparkConf()
.setAppName(“take”)
.setMaster(“local”)
val sc = new SparkContext(conf)
val list = Array(1,2,3,4,5,6,7,8,9,10)
val numbersRDD = sc.parallelize(list, 1) val top3 = numbersRDD.take(3) for(num <- top3){ println(num) }
}
def countTest(){
val conf = new SparkConf()
.setAppName(“count”)
.setMaster(“local”)
val sc = new SparkContext(conf)
val list = Array(1,2,3,4,5,6,7,8,9,10)
val numbersRDD = sc.parallelize(list, 1)
val count = numbersRDD.count()
println(count)
}
def saveAsTextFileTest(){
val conf = new SparkConf()
.setAppName(“saveAsTextFile”)
.setMaster(“local”)
val sc = new SparkContext(conf)
val linesRDD = sc.textFile(“”, 1)
linesRDD.saveAsTextFile("hdfs://spark1:9000/spark.txt")
}
def countByKeyTest(){
val conf = new SparkConf()
.setAppName(“countByKey”)
.setMaster(“local”)
val sc = new SparkContext(conf)
val studentList = Array(("class1","leo"),
("class2","jack"),
("class1","marry"),
("class2","ksc"),
("class2","my"))
val studentsRDD = sc.parallelize(studentList, 1)
val studentsCount = studentsRDD.countByKey()
for((k,v) <- studentsCount){
println(k+":"+v)
}
}
}
這裡不得不說scala的函數語言程式設計與各種特性使得scala的程式比java簡潔的多,但是對於我這裡菜鳥來說scala的特性學習起來真的很麻煩。