1. 程式人生 > 其它 >Mapreduce例項——求平均值———程式碼

Mapreduce例項——求平均值———程式碼

MyAverage.java:
package com.ya.mapreduce;

import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
public class MyAverage { static { try { System.load("D:\\install\\winutils-master\\winutils-master\\hadoop-3.0.0\\bin\\hadoop.dll");//建議採用絕對地址,bin目錄下的hadoop.dll檔案路徑 } catch (UnsatisfiedLinkError e) { System.err.println("Native code library failed to load.\n"+ e); System.exit(
1); } } public static class Map extends Mapper<Object, Text, Text, IntWritable> { private static Text newKey = new Text(); //實現map函式 public void map(Object key, Text value, Context context) throws IOException, InterruptedException { // 將輸入的純文字檔案的資料轉化成String String line = value.toString(); System.out.println(line); String arr[] = line.split("\t"); newKey.set(arr[0]); int click = Integer.parseInt(arr[1]); context.write(newKey, new IntWritable(click)); } } public static class Reduce extends Reducer<Text, IntWritable, Text, IntWritable> { //實現reduce函式 public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException { int num = 0; int count = 0; for (IntWritable val : values) { num += val.get();//每個元素求和num count++;//統計元素的次數count } int avg = num / count;//計算平均數 context.write(key, new IntWritable(avg)); } } public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException { Configuration conf = new Configuration(); System.out.println("start"); Job job = new Job(conf, "MyAverage"); job.setJarByClass(MyAverage.class); job.setMapperClass(Map.class); job.setReducerClass(Reduce.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); job.setInputFormatClass(TextInputFormat.class); job.setOutputFormatClass(TextOutputFormat.class); Path in = new Path("hdfs://node01:8020/kkb/data/mapreduce4/in/goods_click.txt"); Path out = new Path("hdfs://node01:8020/kkb/data/mapreduce4/out"); FileInputFormat.addInputPath(job, in); FileOutputFormat.setOutputPath(job, out); System.exit(job.waitForCompletion(true) ? 0 : 1); } }

結果: