hadoop version 2.7.2(linux 完全集群模式)
jdk 1.8
maven 3.3.9
运行在win10
该例子来源于网络,需求:统计字母出现的次数
文本文件类似这样: a b c d 字母之间有一个空格
<properties>
<hadoop.version>2.7.2</hadoop.version>
</properties>
<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.11</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>${hadoop.version}</version>
</dependency>
</dependencies>
一、定义Map
package com.cisco.learn.hadoop.mr.local;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import java.io.IOException;
/**
* map function:输入一个key-value,然后经过计算,生产一个新的key-value
* LongWritable:key in 输入 key类型
* Text: value in 输入value类型
* Text: key out 输出key类型
* IntWritable:value out 输出value类型
*/
public class WordMapper extends Mapper<LongWritable,Text, Text, IntWritable> {
/**
*
* @param key 输入key
* @param value 输入value
* @param context
* @throws IOException
*/
@Override
protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, IntWritable>.Context context)
throws IOException, InterruptedException {
//接收到split中的输出结果作为map的输入
String line = value.toString();
String[] words = line.split(" ");
//放到list中,每个text统计加1,作为map的输出
//写到context上下文环境中,输出到下一个执行过程shuffle
for(String word : words) {
context.write(new Text(word), new IntWritable(1));
}
}
}
二、定义Reducer
package com.cisco.learn.hadoop.mr.local;
import java.io.IOException;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
public class WordReducer extends Reducer<Text, IntWritable, Text, LongWritable> {
/**
* 参数基本和map一样,只不过第二个参数values是个集合,他是等map结束后,然后对每个key汇总的结果
*/
@Override
protected void reduce(Text key, Iterable<IntWritable> values,
Reducer<Text, IntWritable, Text, LongWritable>.Context context) throws IOException, InterruptedException {
long count = 0;
//此时的输入是shuffle的输出
//输入的key是字符串,输入的value是shuffle派发过来的是一个个list
for(IntWritable v : values) {
count += v.get();
}
context.write(key, new LongWritable(count));
}
}
三、三种运行模式
代码部分
package com.cisco.learn.hadoop.mr.local;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import java.io.File;
public class Main {
static {
System.setProperty("hadoop.home.dir", "D:\\soft\\hadoop");
System.setProperty("HADOOP_USER_NAME", "root") ;
}
public static void main(String[] args) throws Exception {
if(args.length!=3){
System.err.println("param invalid");
System.exit(-1);
}
String inputPath = args[0];
String outPath = args[1];
String mode = args[2];
System.out.println("input:"+inputPath);
System.out.println("output:"+outPath);
System.out.println("mode:"+mode);
Configuration conf = new Configuration();
if("cluster".equals(mode)){
//core-site
conf.set("fs.defaultFS", "hdfs://s130:9000");
conf.set("mapreduce.job.jar","hadoop-demo\\target\\hadoop-demo-1.0.jar");
//mapred-site
conf.set("mapreduce.framework.name","yarn");
conf.set("mapreduce.app-submission.cross-platform", "true");
//yarn
conf.set("yarn.resourcemanager.hostname","s130");
conf.set("yarn.nodemanager.aux-services","mapreduce_shuffle");
deleteHdfs(conf,outPath);
}else{
File outPut = new File(args[1]);
if(outPut.isDirectory()){
deleteDirectory(args[1]);
}
}
Job job = Job.getInstance(conf);
job.setJarByClass(Main.class);
job.setMapperClass(WordMapper.class);
job.setReducerClass(WordReducer.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(LongWritable.class);
FileInputFormat.setInputPaths(job,new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
job.waitForCompletion(true);
System.out.println("success");
}
public static void deleteHdfs(Configuration configuration,String output) throws Exception{
FileSystem fs = FileSystem.get(configuration);
boolean b = fs.delete(new Path(output),true);
System.out.println("文件夹是否删除:" + b);
fs.close();
}
public static boolean deleteFile(String fileName){
File file = new File(fileName);
if(file.isFile() && file.exists()){
Boolean succeedDelete = file.delete();
if(succeedDelete){
System.out.println("删除单个文件"+fileName+"成功!");
return true;
}
else{
System.out.println("删除单个文件"+fileName+"失败!");
return true;
}
}else{
System.out.println("删除单个文件"+fileName+"失败!");
return false;
}
}
public static boolean deleteDirectory(String dir){
//如果dir不以文件分隔符结尾,自动添加文件分隔符
if(!dir.endsWith(File.separator)){
dir = dir+File.separator;
}
File dirFile = new File(dir);
//如果dir对应的文件不存在,或者不是一个目录,则退出
if(!dirFile.exists() || !dirFile.isDirectory()){
System.out.println("删除目录失败"+dir+"目录不存在!");
return false;
}
boolean flag = true;
//删除文件夹下的所有文件(包括子目录)
File[] files = dirFile.listFiles();
for(int i=0;i<files.length;i++){
//删除子文件
if(files[i].isFile()){
flag = deleteFile(files[i].getAbsolutePath());
if(!flag){
break;
}
}
//删除子目录
else{
flag = deleteDirectory(files[i].getAbsolutePath());
if(!flag){
break;
}
}
}
if(!flag){
System.out.println("删除目录失败");
return false;
}
//删除当前目录
if(dirFile.delete()){
System.out.println("删除目录"+dir+"成功!");
return true;
}else{
System.out.println("删除目录"+dir+"失败!");
return false;
}
}
}
3.1本地模式运行
传递参数
D:\soft\hadoop\data\input\input.txt D:\soft\hadoop\data\out local
在本地模拟个一个hadoop运行环境,执行LocalRunner,然后将结果输出接到D:\soft\hadoop\data\out中。
3.2远程提交
传递参数
/apps/icam/documents/hadoop/input.txt /apps/icam/documents/hadoop/output cluster
首先必须在hdfs上创建目录和上传好文件,最后一个参数是cluster,这是远程提交mr。
必须注意这个,我的项目名字是hadoop-demo,然后执行maven 的 clean和package后,生成的jar在target\hadoop-demo-1.0.jar
conf.set("mapreduce.job.jar","hadoop-demo\\target\\hadoop-demo-1.0.jar");
3.3打包jar在Hadoop集群上运行
将jar上传到hadoop所在的namenode的机器上。
hadoop jar hadoop-demo-1.0.jar com.cisco.learn.hadoop.mr.local.Main /apps/icam/documents/hadoop/input.txt /apps/icam/documents/hadoop/output cluster