學習mapreduce程式設計

分別寫三個類

Mapper類

package com.j.mapreduce.wordcount2;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;
/*
* KEYIN,map階段輸入的key的型別：Longwritable
*VALUEIN,map

階段輸入的value的型別：TEXT
*KEYOUT,map階段輸出的key的型別：TEXT
* VALUEOUT,map階段輸出的value的型別：IntWritable
* */

public class wordcountMapper extends Mapper<LongWritable, Text,Text, IntWritable> {
private Text outkey =new Text();
private IntWritable outvalue=new IntWritable(1);

@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
//獲取一行
String line = value.toString();//alue.toString().var 用.var可以自動生產變數
//切割
String[] words = line.split(" ");//分割標誌根據原檔案來寫
//迴圈寫出
for (String word : words) {
//封裝成outkey
outkey.set(word);
//寫出
context.write(outkey, outvalue);
}

}
}

reducer類

package com.j.mapreduce.wordcount2;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;

/*
* KEYIN,reduce

階段輸入的key的型別：Text
*VALUEIN,reduce階段輸入的value的型別：Intwritable
*KEYOUT,reduce階段輸出的key的型別：TEXT
* VALUEOUT,reduce階段輸出的value的型別：IntWritable
* */
public class wordcountReducer extends Reducer<Text, IntWritable,Text,IntWritable> {
@Override
protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
int sum=0;
//Iterable<IntWritable> values 一個集合 是迭代器的祖宗 可以通過迭代器方式訪問
//values.iterator().hasNext();
//累加
for (IntWritable value : values) {
sum+=value.get();
}
//轉換成intwritable型別
IntWritable outvalue = new IntWritable();
outvalue.set(sum);
//寫出
context.write(key,outvalue);
}
}

Driver類

package com.j.mapreduce.wordcount2;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;

public class wordcountDriver {
public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
//1.獲取job
Configuration conf = new Configuration();
Job job = Job.getInstance(conf);
//2.設定jar包路徑
job.setJarByClass(wordcountDriver.class);
//3.關聯mapper和reduce
job.setMapperClass(wordcountMapper.class);
job.setReducerClass(wordcountReducer.class);
//4.設定map輸出的kv型別
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);
//5.設定最終輸出的kv型別
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
//6.設定輸入路徑和輸出路徑
FileInputFormat.setInputPaths(job,new Path(args[0]));
FileOutputFormat.setOutputPath(job,new Path(args[1]));
//提交obj
boolean result = job.waitForCompletion(true);
System.exit(result? 0 : 1);
}
}

將寫好的程式通過maven打包成jar包，放到伺服器進行執行

hadoop jar wc.jar com.j.mapreduce.wordcount2.wordcountDriver /sanguo /output 注意末尾的輸出路徑要不存在

注意如果xshell報錯，我在查閱了一番資料之後，需要修改一個地方，

Container exited with a non-zero exit code 1. Error file: prelaunch.err.

Last 4096 bytes of prelaunch.err :

Last 4096 bytes of stderr :

Error: Could not find or load main class org.apache.hadoop.mapreduce.v2.app.MRAppMaster

Please check whether your etc/hadoop/mapred-site.xml contains the below configuration:

<property>

<name>yarn.app.mapreduce.am.env</name>

<value>HADOOP_MAPRED_HOME=${full path of your hadoop distribution directory}</value>

</property>

<property>

<name>mapreduce.map.env</name>

<value>HADOOP_MAPRED_HOME=${full path of your hadoop distribution directory}</value>

</property>

<property>

<name>mapreduce.reduce.env</name>

<value>HADOOP_MAPRED_HOME=${full path of your hadoop distribution directory}</value>

</property>

可以通過修改自己的mapred-site.xml

來修改

<property>

<name>mapreduce.application.classpath</name>

<value>

/opt/module/hadoop-3.1.3/etc/*,

/opt/module/hadoop-3.1.3/etc/hadoop/*,

/opt/module/hadoop-3.1.3/lib/*,

/opt/module/hadoop-3.1.3/share/hadoop/common/*,

/opt/module/hadoop-3.1.3/share/hadoop/common/lib/*,

/opt/module/hadoop-3.1.3/share/hadoop/mapreduce/*,

/opt/module/hadoop-3.1.3/share/hadoop/mapreduce/lib-examples/*,

/opt/module/hadoop-3.1.3/share/hadoop/hdfs/*,

/opt/module/hadoop-3.1.3/share/hadoop/hdfs/lib/*,

/opt/module/hadoop-3.1.3/share/hadoop/yarn/*,

/opt/module/hadoop-3.1.3/share/hadoop/yarn/lib/*,

</value>

</property>

注意裡面的路勁與自己Linux的系統中hadoop路徑相同

並分發給其他叢集，xsync mapred-site.xml 即可

學習時間：12：19到15：54