Spark部分:Spark中取交集(intersection )和取差集(subtract )【Java版純程式碼】
阿新 • • 發佈:2019-02-01
package com.bjsxt.spark; import java.util.Arrays; import java.util.List; import org.apache.spark.SparkConf; import org.apache.spark.SparkContext; import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.api.java.function.Function; import org.apache.spark.api.java.function.Function2; import org.apache.spark.api.java.function.PairFunction; import org.apache.spark.api.java.function.VoidFunction; public class Inter { public static void main(String[] args) { SparkConf conf=new SparkConf().setAppName("test").setMaster("local"); JavaSparkContext sc=new JavaSparkContext(conf); List<String> list1 = Arrays.asList("a","b","c","d"); List<String> list2 = Arrays.asList("a","b","c","e"); JavaRDD<String> rdd1 = sc.parallelize(list1); JavaRDD<String> rdd2 = sc.parallelize(list2); /** * 取差集 */ JavaRDD<String> subtract = rdd1.subtract(rdd2); subtract.foreach(new VoidFunction<String>() { @Override public void call(String arg0) throws Exception { System.out.println(arg0); } }); /** * * 取交集 */ JavaRDD<String> intersection = rdd1.intersection(rdd2); intersection.foreach(new VoidFunction<String>() { @Override public void call(String arg0) throws Exception { System.out.println(arg0); } }); } }