一、目的
本文主要用於記錄大資料學習過程中一些沉澱

from pyspark import SparkContext,SparkConf

spconf =SparkConf().setAppName("ji").setMaster("local[*]")
sc = SparkContext(conf=spconf)

#1.求平均數
df = [1,5,7,10,23,20,6,5,10,7,10]
rdd_data = sc.parallelize(df)
total = rdd_data.reduce(lambda x,y:x+y+0.0)
count = rdd_data.count()
ava = total/count
print(total,count,ava)

#2.求眾數，有多個眾數，求眾數平均值
data = [1,5,7,10,23,20,7,5,10,7,10]
rdd_data = sc.parallelize(data)
rdd_count = rdd_data.map(lambda x:(x,1)).reduceByKey(lambda x,y:x+y)
max_count = rdd_count.map(lambda x:x[1]).reduce(lambda x,y:x if x>y else y)
rdd_filter = rdd_count.filter(lambda x:x[1]==max_count).map(lambda x:x[0]) #
mode = rdd_filter.reduce(lambda x,y:x+y+0.0)/rdd_filter.count() #[10, 7]
print(mode)

#3.求topN
#有一批學生資訊表格，包括name,age,score, 找出score排名前3的學生, score相同可以任取
students = [("LiLei",18,87),("HanMeiMei",16,77),("DaChui",16,66),("Jim",18,77),("Ruanhua",18,50)]
n = 3
rdd_data = sc.parallelize(students)
rdd_sorted = rdd_data.sortBy(lambda x:x[2],ascending=False)
topN = rdd_sorted.take(n)
print(topN)

#4.排序並返回序號
#任務：按從小到大排序並返回序號, 大小相同的序號可以不同
import time
starttime = time.time()
data = [1,7,8,5,3,18,34,9,0,12,8]
rdd_data = sc.parallelize(data)
rdd_sort = rdd_data.map(lambda x:(x,1)).sortByKey().map(lambda x:x[0])
index_rdd = rdd_sort.zipWithIndex()
endtime = time.time()
sparktime = endtime-starttime
print("spakrtime",sparktime)

#5，二次排序
#任務：有一批學生資訊表格，包括name,age,score
#首先根據學生的score從大到小排序，如果score相同，根據age從大到小
students = [("LiLei",18,87),("HanMeiMei",16,77),("DaChui",16,66),("Jim",18,77),("Ruanhua",18,50)]
rdd_data = sc.parallelize(students)
from bigdata.example.exercise.student import STUDENT
rdd1 = rdd_data.map(lambda x:STUDENT(x[0],x[1],x[2])).sortBy(lambda x:x,ascending=False).map(lambda x:(x.name,x.age,x.score))
print(rdd1.collect())

#6.分組求眾數
#任務：有一批學生資訊表格，包括class和age。求每個班級學生年齡的眾數。
students = [("class1",15),("class1",15),("class2",16),("class2",16),("class1",17),("class2",19)]