MapReduce实现WordCount

最新推荐文章于 2023-05-05 22:14:43 发布

follweme888

最新推荐文章于 2023-05-05 22:14:43 发布

阅读量208

点赞数

分类专栏：大数据文章标签： hadoop

原文链接：https://blog.csdn.net/weixin_47263088/article/details/106549287

版权

大数据专栏收录该内容

18 篇文章 0 订阅

订阅专栏

MapReduce实现WordCount

在这里插入图片描述
1、spliting ：Documents会根据切割规则被切成若干块，
2、map阶段：然后进行Map过程，Map会并行读取文本，对读取的单词进行单词分割，并且每个词以键值对<key,value>形式生成。
例如：读取到”Hello World Hello Java“，分割单词形成Map.
<Hello,1> <World,1><Hello,1> <Java,1>

3、Combine阶段：接下来Combine(该阶段是可以选择的，Combine其实也是一种reduce),会对每个片相同的词进行统计。
4、shuffle阶段：将Map输出作为reduce的输入的过程就是shuffle,次阶段是最耗时间,也是重点需要优化的阶段。shuffle阶段会对数据进行拉取，对最后得到单词进行统计，每个单词的位置会根据Hash来确定所在的位置，
5、reduce阶段：对数据做最后的汇总，最后结果是存储在hdfs上。

项目结构：

在这里插入图片描述

java代码：

pom.xml

<?xml version="1.0" encoding="UTF-8"?>

<groupId>mymr</groupId>
<artifactId>mymr</artifactId>
<version>1.0-SNAPSHOT</version>

<name>mymr</name>
<!– FIXME change it to the project’s website –>
<url>http://www.example.com</url>

<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.11</version>
<scope>test</scope>
</dependency>
<!– https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-client –>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>2.6.0</version>
</dependency>
<!– https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-common –>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.6.0</version>
</dependency>
<!– https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-hdfs –>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>2.6.0</version>
</dependency>

</dependencies>
<build>
<plugins>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>2.3.2</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
<archive>
<manifest>
<mainClass>mymr.service.wc.MyDriver</mainClass>
</manifest>
</archive>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

MyMapper：

package mymr.service.wc;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;
//继承Mapper
public class MyMapper extends Mapper<LongWritable, Text,Text,LongWritable> {

<span class="token keyword">private</span> LongWritable one <span class="token operator">=</span> <span class="token keyword">new</span> <span class="token class-name">LongWritable</span><span class="token punctuation">(</span><span class="token number">1</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token keyword">private</span> Text word <span class="token operator">=</span> <span class="token keyword">new</span> <span class="token class-name">Text</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>

<span class="token annotation punctuation">@Override</span>
<span class="token keyword">protected</span> <span class="token keyword">void</span> <span class="token function">map</span><span class="token punctuation">(</span>LongWritable key<span class="token punctuation">,</span> Text value<span class="token punctuation">,</span> Context context<span class="token punctuation">)</span> <span class="token keyword">throws</span> IOException<span class="token punctuation">,</span> InterruptedException <span class="token punctuation">{</span>
    String <span class="token punctuation">[</span><span class="token punctuation">]</span> wds <span class="token operator">=</span> value<span class="token punctuation">.</span><span class="token function">toString</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span><span class="token function">split</span><span class="token punctuation">(</span><span class="token string">" "</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
    <span class="token keyword">for</span> <span class="token punctuation">(</span>String word <span class="token operator">:</span> wds<span class="token punctuation">)</span> <span class="token punctuation">{</span>
        Text wd <span class="token operator">=</span> <span class="token keyword">new</span> <span class="token class-name">Text</span><span class="token punctuation">(</span>word<span class="token punctuation">)</span><span class="token punctuation">;</span>
        context<span class="token punctuation">.</span><span class="token function">write</span><span class="token punctuation">(</span>wd<span class="token punctuation">,</span>one<span class="token punctuation">)</span><span class="token punctuation">;</span>
    <span class="token punctuation">}</span>
<span class="token punctuation">}</span>

}

MyReduce：

package mymr.service.wc;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;

public class MyReduce extends Reducer<Text, LongWritable,Text,LongWritable> {

<span class="token keyword">private</span> LongWritable res <span class="token operator">=</span> <span class="token keyword">new</span> <span class="token class-name">LongWritable</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>


<span class="token annotation punctuation">@Override</span>
<span class="token keyword">protected</span> <span class="token keyword">void</span> <span class="token function">reduce</span><span class="token punctuation">(</span>Text key<span class="token punctuation">,</span> Iterable<span class="token generics function"><span class="token punctuation">&lt;</span>LongWritable<span class="token punctuation">&gt;</span></span> values<span class="token punctuation">,</span> Context context<span class="token punctuation">)</span> <span class="token keyword">throws</span> IOException<span class="token punctuation">,</span> InterruptedException <span class="token punctuation">{</span>
    <span class="token keyword">long</span> ressum<span class="token operator">=</span><span class="token number">0</span><span class="token punctuation">;</span>
    <span class="token keyword">for</span> <span class="token punctuation">(</span>LongWritable one <span class="token operator">:</span> values<span class="token punctuation">)</span> <span class="token punctuation">{</span>
        ressum<span class="token operator">+=</span>one<span class="token punctuation">.</span><span class="token function">get</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
    <span class="token punctuation">}</span>
    res<span class="token punctuation">.</span><span class="token function">set</span><span class="token punctuation">(</span>ressum<span class="token punctuation">)</span><span class="token punctuation">;</span>
    context<span class="token punctuation">.</span><span class="token function">write</span><span class="token punctuation">(</span>key<span class="token punctuation">,</span>res<span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token punctuation">}</span>

}

MyDriver：

package mymr.service.wc;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;

public class MyDriver {
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
//准备一个空任务，并给名字
Job job = Job.getInstance(conf,“wc”);
//设置该任务主启动类
job.setJarByClass(MyDriver.class);
//设置任务的输入数据源
FileInputFormat.addInputPath(job,new Path(“f://abc.txt”));
//设置你的mapper任务类
job.setMapperClass(MyMapper.class);
//设置mapper任务类的输出数据类型
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(LongWritable.class);
//设置你的reducer任务类
job.setReducerClass(MyReduce.class);
//设置Reducer任务的输出的数据类
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(LongWritable.class);
//设置任务的输出数据目标
FileOutputFormat.setOutputPath(job,new Path(“f://abc1”));
job.waitForCompletion(true);
}
}

                                </div>
            <link href="https://csdnimg.cn/release/phoenix/mdeditor/markdown_views-60ecaf1f42.css" rel="stylesheet">
                            </div>