combiner - 本地统计,减少传输
partitioner - 分发规则自定义
jobhistory - 记录完整的mapreduce运行信息 - log
package wordcount;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import java.io.IOException;
public class WordCountApp{
/** Map : 读取输入文件 - 分裂文本
* mapper<行号, 一行一行文本数据, 单个单词, 计数1 >
* map 输出是 reduce 输入
*/
public static class MyMapper extends Mapper<LongWritable, Text, Text, LongWritable>{
LongWritable one = new LongWritable(1);
/**
* map 方法, 自动继承
* @param key - 是偏移量 - 行号
* @param value - 每一行字符串
* @param context
* @throws IOException
* @throws InterruptedException
*/
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
// 接收到每一行数据
String line = value.toString();
// 按照指定的方式正则分割
String[] wordList = line.split(" ");
// 循环打标记1
for (String word : wordList) {
context.write(new Text(word), one);
}
}
}
/**
* Text,
* LongWritable,
* Text,
* LongWritable
*/
public static class MyReducer extends Reducer<Text, LongWritable,Text, LongWritable>{
/**
*
* @param key - map输出的主键值,一个一个单词
* @param values - 是个
* @param context
* @throws IOException
* @throws InterruptedException
*/
@Override
protected void reduce(Text key, Iterable<LongWritable> values, Context context) throws IOException, InterruptedException {
// super.reduce(key, values, context);
// 循环统计 单词的个数
long sum = 0;
for (LongWritable value : values) {
// value.get(); 自动转化 longwrite类型
sum += value.get();
}
// 统计结果输出
context.write(key, new LongWritable(sum));
}
}
/**
* Driver 封装了mapreduce的所有信息
* @param args
*/
public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
// 1. 创建configuration
Configuration configuration = new Configuration();
// 删除已经存在的输出目录
Path ouputPath = new Path(args[1]);
FileSystem fileSystem = FileSystem.get(configuration);
if (fileSystem.exists(ouputPath)){
fileSystem.delete(ouputPath, true);
System.out.println("删除已经存在的wordcount输出目录");
}
// 2. 创建JOB
Job job = Job.getInstance(configuration);
// 3. 设置JOB的处理主类
job.setJarByClass(WordCountApp.class);
// 4. 设置作业处理的输入路径 - 控制台输入
FileInputFormat.setInputPaths(job,new Path(args[0]));
// 5. 设置map相关的参数
job.setMapperClass(MyMapper.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(LongWritable.class);
// 6. 设置reduce相关的参数
job.setReducerClass(MyReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(LongWritable.class);
// 7. 设置作业处理输出路径
FileOutputFormat.setOutputPath(job, new Path(args[1]));
// 8. 退出
System.exit(job.waitForCompletion(true) ? 0 : 1 );
}
}
// 编译 mvn clean package -DskipTests
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>youngpeng</groupId>
<artifactId>wordcount</artifactId>
<version>4.0</version>
<properties>
<hadoop.version>2.6.0</hadoop.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>${hadoop.version}</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
<plugin>
<!-- JAR Maven 管理-->
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<configuration>
<archive>
<manifest>
<!-- 配置主程序 java -jar 默认Class -->
<addClasspath>true</addClasspath>
<classpathPrefix>lib/</classpathPrefix>
<!-- 指定 mainclass,该配置将最终体现在jar包manifest文件中 -->
<mainClass>wordcount.WordCountApp</mainClass>
</manifest>
</archive>
</configuration>
</plugin>
<!-- maven 打包集成插件 -->
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
<configuration>
<descriptorRefs>
<!-- 将依赖一起打包到 JAR -->
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
<archive>
<manifest>
<!-- 配置主程序 java -jar 默认Class -->
<addClasspath>true</addClasspath>
<classpathPrefix>lib/</classpathPrefix>
<!-- 指定 mainclass,该配置将最终体现在jar包manifest文件中 -->
<mainClass>wordcount.WordCountApp</mainClass>
</manifest>
</archive>
</configuration>
</plugin>
</plugins>
</build>
</project>
// 删除已经存在的输出目录
Path ouputPath = new Path(args[1]);
FileSystem fileSystem = FileSystem.get(configuration);
if (fileSystem.exists(ouputPath)){
fileSystem.delete(ouputPath, true);
System.out.println("删除已经存在的wordcount输出目录");
}
mvn clean package
# 讲一个坑, 就是打包把依赖打进去的问题, 添加插件 - 变更打包命令