pom文件
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.hadoop</groupId>
<artifactId>project</artifactId>
<version>1.0-SNAPSHOT</version>
<properties>
<hadoopVersion>2.7.2</hadoopVersion>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.7.2</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>2.7.2</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
<version>2.7.2</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>2.7.2</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.48</version>
</dependency>
</dependencies>
Driver端
package com.mr.serachFriends;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
public class MyDriverPro {
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
Job job = Job.getInstance(conf);
job.setJarByClass(MyDriverPro.class);
job.setMapperClass(MyMappr.class);
job.setReducerClass(MyReducerPro.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(Text.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
FileInputFormat.setInputPaths(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
job.waitForCompletion(true);
}
}
Map端
package com.mr.serachFriends;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import java.io.IOException;
public class MyMappr extends Mapper<LongWritable,Text,Text,Text> {
Text k =new Text();
Text v =new Text();
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
String str = value.toString();
String[] split = str.split(":");
String[] friends = split[1].split(",");
String people = split[0];
System.out.println(people);
for(String friend : friends){
k.set(friend);
v.set(people);
context.write(k,v);
}
}
}
Reduce端
package com.mr.serachFriends;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
import java.io.IOException;
import java.util.*;
public class MyReducerPro extends Reducer<Text,Text,Text,Text> {
Text k=new Text();
Text v=new Text();
Map<String,ArrayList<String>> map= new HashMap<String, ArrayList<String>>();
Map<String,String> map2=new HashMap<String, String>();
@Override
protected void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
ArrayList<String> list =new ArrayList<String >();
for (Text people: values) {
list.add(people.toString());
}
map.put(key.toString(),list);
}
@Override
protected void cleanup(Context context) throws IOException, InterruptedException {
Set<String> strings = map.keySet();
Iterator<String> iterator = strings.iterator();
while(iterator.hasNext()){
String next = iterator.next();
ArrayList<String> list = map.get(next);
Collections.sort(list);
for(int i=0;i<list.size()-1;i++){
for (int j=i+1;j<list.size();j++) {
String Peoples=list.get(i)+" "+list.get(j);
if(!map2.containsKey(Peoples)){
map2.put(Peoples,next);
}else{
map2.put(Peoples,map2.get(Peoples)+" "+next);
}
}
}
}
Set<String> mapSet = map2.keySet();
Iterator<String> iter = mapSet.iterator();
while(iter.hasNext()){
String ke = iter.next();
String va = map2.get(ke);
k.set(ke);
v.set(va);
context.write(k,v);
}
}
}
通过在Driver端输入参数,即需要处理的文件和输出文件夹,即可以查看结果!
compiled up by JiaMingcan
转载请署名:JiaMingcan