1. hdfs 和RDBMS區別
2. mr 和 網格計算，志願計算

1，資料儲存

2，分析計算

3，網格計算，志願計算

4，mr 對比linux:awk流處理

1，awk處理： 年度最高溫度統計

2，mapreduce處理：每年最高溫度統計

idea +maven: 新增依賴

		<dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>2.7.3</version>
        </dependency>
 

map方法

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import java.io.IOException;

public class Map1 extends Mapper<LongWritable, Text, IntWritable,IntWritable> {
    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        //整理的資料輸入：
        //1982,-8
        //1931,-4
        String str = value.toString();
        String[] arr = str.split(",");
        int year=0, tmp=Integer.MIN_VALUE;

        //資料轉換
        try {
             year= Integer.parseInt(arr[0]);
             tmp= Integer.parseInt(arr[1]);
        }catch (Exception e){
            e.printStackTrace();
        }
        //輸出：新資料
        context.write(new IntWritable(year),new IntWritable(tmp));
    }
}
 

reduce方法

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mapreduce.Reducer;
import java.io.IOException;
import java.util.Iterator;

public class Reduce1 extends Reducer<IntWritable,IntWritable,IntWritable,IntWritable> {
    @Override
    protected void reduce(IntWritable key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
        //輸入資料：1931,【-4,23,4,35,6】
        //聚合資料： 求每組資料中的max(tmp)
        int max=Integer.MIN_VALUE;
        Iterator<IntWritable> it = values.iterator();
        while (it.hasNext()){
            IntWritable next = it.next();
            int tmp = next.get();

            max= (max >tmp) ? max:tmp;
        }
        //輸出： 最高溫度
        context.write(key, new IntWritable(max));
    }
}

app類： 排程組織job

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import java.io.IOException;

public class App1 {
    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
        Configuration conf = new Configuration();
        Job job = Job.getInstance(conf);
        job.setJarByClass(App1.class);
        job.setJobName("maxTmp");

        //map,reduce
        job.setMapperClass(Map1.class);
        job.setReducerClass(Reduce1.class);

        job.setMapOutputKeyClass(IntWritable.class);
        job.setMapOutputValueClass(IntWritable.class);
        job.setOutputKeyClass(IntWritable.class);
        job.setOutputValueClass(IntWritable.class);

        job.setNumReduceTasks(3);
        //輸入輸出
        FileInputFormat.addInputPath(job,new Path("/home/wang/txt/tmp.txt"));
        FileOutputFormat.setOutputPath(job, new Path("/home/wang/tmp-out"));

        //提交等待
        job.waitForCompletion(true);
    }
}