hadoop中關於mapreduce的單元測試
阿新 • • 發佈:2019-02-07
這個功能好像是CDH那邊開發的。。。
1. 首先下載jar包:hadoop-0.21.0-mrunit.jar。(經測試,這個jar包是關聯舊API的。請自己根據情況決定)
2. 在eclipse的build path中加入此jar包。
3. 編寫測試程式:
package unittest; import java.io.IOException; import java.util.ArrayList; import java.util.Iterator; import java.util.List; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.OutputCollector; import org.apache.hadoop.mapred.Reducer; import org.apache.hadoop.mapred.Reporter; import org.apache.hadoop.mrunit.ReduceDriver; import org.junit.Before; import org.junit.Test; @SuppressWarnings("all") public class ReducerTest { public static class ReducerClass implements Reducer<Text, IntWritable, Text, IntWritable> { private IntWritable intvalue = new IntWritable(0); @Override public void configure(JobConf job) { job.setJobName("test"); System.err.println("in configure method"); System.out.println("job name is set to: ---------preset--------" + "------------------------" + job.getJobName()); // System.out.println("job.getJobEndNotificationURI() -----" + // "-----preset------------" + job.getJobEndNotificationURI()); // System.out.println("job.getJobLocalDir() -----" + // "------preset-----------" + job.getJobLocalDir()); // System.out.println("job.getJobPriority() -----" + // "------preset-----------" + job.getJobPriority()); } @Override public void close() throws IOException { System.err.println("in close method"); } @Override public void reduce(Text key, Iterator<IntWritable> values, OutputCollector<Text, IntWritable> output, Reporter reporter) throws IOException { int sum = 0; while (values.hasNext()) { sum += values.next().get(); } intvalue.set(sum); output.collect(key, intvalue); } } private Reducer reducer; private ReduceDriver reducerDriver; @Before public void init() { reducer = new ReducerClass(); reducerDriver = new ReduceDriver(reducer); } @Test public void test() { String key = "key"; List values = new ArrayList(); values.add(new IntWritable(5)); values.add(new IntWritable(3)); reducerDriver.withInput(new Text(key), values) .withOutput(new Text("key"), new IntWritable(8)) .runTest(); } }