hadoop劃分為多個輸出檔案
阿新 • • 發佈:2018-12-24
現在我們見到的MapReduce作業的輸出都是一組檔案,那如果我想輸出多組檔案怎麼辦,比如說我想統計每個國家的專利情況,想以國家名作為檔名來輸出。我們可以使用MultipleOutputFormat,它內部有一個方法generateFileNameForKeyValue,只要Override他,就可以根據自己的需要劃分檔案。他還有一些子類,像MultipleTextOutputFormat,MultipleSequenceFileOutputFormat
import java.io.IOException;
import java.util.Iterator;
import org.apache .hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobConf ;
import org.apache.hadoop.mapred.KeyValueTextInputFormat;
import org.apache.hadoop.mapred.MapReduceBase;
import org.apache.hadoop.mapred.Mapper;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop .mapred.TextInputFormat;
import org.apache.hadoop.mapred.TextOutputFormat;
import org.apache.hadoop.mapred.lib.IdentityReducer;
import org.apache.hadoop.mapred.lib.MultipleOutputFormat;
import org.apache.hadoop.mapred.lib.MultipleTextOutputFormat;
import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.FileOutputFormat;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import com.google.inject.Key;
import com.sun.tracing.dtrace.ArgsAttributes;
public class MultiFile extends Configured implements Tool {
public static class MapClass extends MapReduceBase implements Mapper<LongWritable, Text, NullWritable, Text>{
public void map(LongWritable key,Text value,OutputCollector<NullWritable, Text> output,Reporter reporter)throws IOException{
//System.out.println(value.toString());
output.collect(NullWritable.get(), value);
}
}
public static class PartitionByCountryMTOF extends MultipleTextOutputFormat<NullWritable, Text>{
private static int K=0;
@Override
protected String generateFileNameForKeyValue(NullWritable key, Text value, String name) {
// TODO Auto-generated method stub
if(K<10)System.out.println(name);
K++;
String fields []=value.toString().split(",",-1);
String country=fields[4].substring(1, 3);
return country+"/"+name;
}
}
@Override
public int run(String[] arg0) throws Exception {
// TODO Auto-generated method stub
Configuration configuration=getConf();
JobConf job=new JobConf(configuration,MultiFile.class);
FileInputFormat.setInputPaths(job, new Path(arg0[0]));
FileOutputFormat.setOutputPath(job, new Path(arg0[1]));
job.setJobName("MultiFile");
job.setMapperClass(MapClass.class);
job.setInputFormat(TextInputFormat.class);
job.setOutputFormat(PartitionByCountryMTOF.class);
job.setOutputKeyClass(NullWritable.class);
job.setOutputValueClass(Text.class);
job.setReducerClass(IdentityReducer.class);
job.setNumReduceTasks(0);
JobClient.runJob(job);
return 0;
}
public static void main(String[] args) throws Exception{
// TODO Auto-generated method stub
//ToolRunner.run(conf, tool, args)
int res=ToolRunner.run(new Configuration(), new MultiFile(), args);
System.exit(res);
}
}
這是橫向拆分資料,那我想縱向拆分怎麼辦?比如我想把專利中時間有關的項放到一個檔案,地理資訊相關的放入另一個檔案怎麼辦?Hadoop還提供了一個MultipleOutputs,它所採用的方法並不是給每條記錄請求一個檔名,而是建立多個OutputCollector
public class MultiOutput extends Configured implements Tool {
public static class MapClass extends MapReduceBase implements Mapper<LongWritable, Text, NullWritable, Text>{
private MultipleOutputs multipleOutputs;
private OutputCollector<NullWritable, Text> collector;
public void configure(JobConf job){
multipleOutputs=new MultipleOutputs(job);
}
public void map(LongWritable key,Text value,OutputCollector<NullWritable, Text> output,Reporter reporter)throws IOException{
//System.out.println(value.toString());
String arr []=value.toString().split(",",-1);
String chrono=arr[0]+","+arr[1]+","+arr[2];
String geo=arr[0]+","+arr[4]+","+arr[5];
collector=multipleOutputs.getCollector("chrono", reporter);
collector.collect(NullWritable.get(), new Text(chrono));
collector=multipleOutputs.getCollector("geo", reporter);
collector.collect(NullWritable.get(), new Text(geo));
}
@Override
public void close() throws IOException {
// TODO Auto-generated method stub
multipleOutputs.close();
}
}
@Override
public int run(String[] arg0) throws Exception {
// TODO Auto-generated method stub
Configuration configuration=getConf();
JobConf job=new JobConf(configuration,MultiOutput.class);
FileInputFormat.setInputPaths(job, new Path(arg0[0]));
FileOutputFormat.setOutputPath(job, new Path(arg0[1]));
job.setJobName("MultiFile");
job.setMapperClass(MapClass.class);
job.setInputFormat(TextInputFormat.class);
//job.setOutputFormat(PartitionByCountryMTOF.class);
job.setOutputKeyClass(NullWritable.class);
job.setOutputValueClass(Text.class);
job.setReducerClass(IdentityReducer.class);
job.setNumReduceTasks(0);
MultipleOutputs.addNamedOutput(job,"chrono", TextOutputFormat.class, NullWritable.class,Text.class);
MultipleOutputs.addNamedOutput(job, "geo", TextOutputFormat.class, NullWritable.class, Text.class);
JobClient.runJob(job);
return 0;
}
public static void main(String[] args) throws Exception{
// TODO Auto-generated method stub
//ToolRunner.run(conf, tool, args)
int res=ToolRunner.run(new Configuration(), new MultiOutput(), args);
System.exit(res);
}
}