1. 程式人生 > >spark 讀取orc檔案

spark 讀取orc檔案

        <dependency>
            <groupId>org.apache.orc</groupId>
            <artifactId>orc-mapreduce</artifactId>
            <version>1.1.0</version>
        </dependency>
    SparkConf sparkConf = new SparkConf();
    sparkConf.setAppName("spark-orc"
); sparkConf.set("fs.hdfs.impl",org.apache.hadoop.hdfs.DistributedFileSystem.class.getName()); JavaSparkContext sc = new JavaSparkContext(sparkConf); JavaPairRDD<LongWritable, OrcStruct> rdd =sc.hadoopFile(args[0],rcInputFormat.class,LongWritable.class, OrcStruct.class);