1. 程式人生 > >IDEA+Maven開發第一個Hadoop程式WordCount

IDEA+Maven開發第一個Hadoop程式WordCount

IDEA+Maven開發第一個Hadoop程式WordCount

 

1. 新建一個maven專案選擇JDK版本。

 

2.設定GroupId和ArtifactId

 

 

 

3.設定專案名稱

 

 

4.來到setting的javaCompiler更改為自己對應的JDK版本。

 

 

5.複製叢集中hadoop-2.7.2.gz檔案到D盤,並且在bin目錄下新增hadoop.dll,winutils.exe,winutils.pdb檔案

 

 

 

6.設定環境變數,並且在path中加入%HADOOP_HOME\bin%;

 

 

7.新增依賴


<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.kevin.dt</groupId>
    <artifactId>DTWorker</artifactId>
    <version>1.0-SNAPSHOT</version>

    <properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <!--設定hadoop版本-->
        <hadoop.version>2.7.2</hadoop.version>
    </properties>

    <dependencies>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.12</version>
            <scope>test</scope>
        </dependency>

        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <version>${hadoop.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-hdfs</artifactId>
            <version>${hadoop.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-mapreduce-client-core</artifactId>
            <version>${hadoop.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-mapreduce-client-common</artifactId>
            <version>${hadoop.version}</version>
        </dependency>
        <dependency>
            <groupId>log4j</groupId>
            <artifactId>log4j</artifactId>
            <version>1.2.17</version>
        </dependency>

    </dependencies>


    <build>
        <plugins>
            <plugin>
                <artifactId>maven-dependency-plugin</artifactId>
                <configuration>
                    <excludeTransitive>false</excludeTransitive>
                    <stripVersion>true</stripVersion>
                    <outputDirectory>./lib</outputDirectory>
                </configuration>

            </plugin>
        </plugins>
    </build>

</project>

 

8.寫日誌可以看詳細的執行資訊或者異常


log4j.rootLogger=INFO, stdout

log4j.appender.stdout=org.apache.log4j.ConsoleAppender

log4j.appender.stdout.layout=org.apache.log4j.PatternLayout

log4j.appender.stdout.layout.ConversionPattern=%d %p [%c] - %m%n

log4j.appender.logfile=org.apache.log4j.FileAppender

log4j.appender.logfile.File=target/spring.log

log4j.appender.logfile.layout=org.apache.log4j.PatternLayout

log4j.appender.logfile.layout.ConversionPattern=%d %p [%c] - %m%n

 

9.WordCount


package com.kevin.hadoop;



import java.io.IOException;

import java.net.URI;

import java.util.StringTokenizer;



import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.FileSystem;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.IntWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Job;

import org.apache.hadoop.mapreduce.Mapper;

import org.apache.hadoop.mapreduce.Reducer;

import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;





/**

 * @author kevin

 * @version 1.0

 * @description     簡單的WordCount示例,單詞計數

 * @createDate 2018/12/17

 */

public class WordCount {



    public static class TokenizerMapper extends Mapper<Object, Text, Text, IntWritable> {



        private final static IntWritable one = new IntWritable(1);

        private Text word = new Text();



        public void map(Object key, Text value, Context context) throws IOException, InterruptedException {

            StringTokenizer itr = new StringTokenizer(value.toString());

            while (itr.hasMoreTokens()) {

                word.set(itr.nextToken());

                context.write(word, one);

            }

        }

    }



    public static class IntSumReducer extends Reducer<Text, IntWritable, Text, IntWritable> {

        private IntWritable result = new IntWritable();



        public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {

            int sum = 0;

            for (IntWritable val : values) {

                sum += val.get();

            }

            result.set(sum);

            context.write(key, result);

        }

    }



    public static void main(String[] args) throws Exception {



        Configuration conf = new Configuration(); // 讀取hadoop配置檔案



        Job job = Job.getInstance(conf, "word count"); // 新建一個Job類,傳入配置資訊

        job.setJarByClass(WordCount.class); // 設定主類

        job.setMapperClass(TokenizerMapper.class); // 設定map類

        job.setCombinerClass(IntSumReducer.class); // 設定combiner類

        job.setReducerClass(IntSumReducer.class); // 設定reduce類

        job.setOutputKeyClass(Text.class); // 設定輸出型別key

        job.setOutputValueClass(IntWritable.class); // 設定輸出型別value

        FileInputFormat.addInputPath(job, new Path("hdfs://192.168.171.100:9000/test/input_01/")); // 設定輸入檔案

        FileOutputFormat.setOutputPath(job, new Path("hdfs://192.168.171.100:9000/test/output_01")); // 設定輸出檔案

        System.exit(job.waitForCompletion(true) ? 0 : 1); // 等待完成退出

    }

}

 

10.輸入源有兩個檔案

file01:

Hello World Bye World

Hello Hadoop Bye Hadoop

Bye Hadoop Hello Hadoop

yes me

good yes

 

file02:

Hello World Bye World

Hello Hadoop Bye Hadoop

Bye Hadoop Hello Hadoop

yes me

catch ese

 

11.結果