JoinMain:
package com.cys.tables;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.MultipleInputs;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
public class JoinMain {
public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
// if(null == args || args.length != 3){
// System.err.println("<Usage>: " +
// "JoinMain <in1> <in2> <out>");
// System.exit(-1);
// }
Job job = new Job(new Configuration(),"Join");
job.setJarByClass(JoinMain.class);
//whole
MultipleInputs.addInputPath(job,
new Path("hdfs://master:9000/hbase/data/default/ORDER_DETAIL_BAK/36941e450de2e45f16a2e0d731a5135b/DETAIL/048c74a6f5674ee0a0ca96f2474c051e"),
TextInputFormat.class,
WholeFileMapper.class);
//uuid
MultipleInputs.
addInputPath(job,
new Path("hdfs://master:9000/hbase/data/default/ORDER_INFO_BAK/0b169baaae48a7755e6dc56a2a59a9c0/INFO/b9868978fc8e4872b35d668732a0829f"),
TextInputFormat.class,
UuidMapper.class);
job.setReducerClass(JoinReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
FileOutputFormat.setOutputPath(job, new Path("hdfs://master:9000/out/con5"));
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
}
package com.cys.tables;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
/**
* whole sogou 500w
* @author THINK
*
*/
public class WholeFileMapper extends
Mapper<Object, Text, Text, Text> {
//label
public static final String LABEL = "W_";
private Text uidText = new Text();
private Text newValue = new Text();
/*
* key: offset
* value: 20111230000005 57375476989eea12893c0c3811607bcf 奇艺高清 1 1 http://www.qiyi.com/
* @see org.apache.hadoop.mapreduce.Mapper#map(KEYIN, VALUEIN, org.apache.hadoop.mapreduce.Mapper.Context)
*/
protected void map(Object key,
Text value,
Context context)
throws java.io.IOException,
InterruptedException {
String lineString = value.toString();
String[] arr = lineString.split("\t");
if(null != arr && arr.length !=6){
//label
String uidString = arr[1];
uidText.set(uidString);
newValue.set(LABEL + lineString);
/*
* uidText:57375476989eea12893c0c3811607bcf
* newValue: W_20111230000005 57375476989eea12893c0c3811607bcf 奇艺高清 1 1 http://www.qiyi.com/
*/
context.write(uidText, newValue);
}
};
}
UuidMapper:
package com.cys.tables;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
/**
* Uuid
* @author THINK
*
*/
public class UuidMapper extends
Mapper<Object, Text, Text, Text> {
//label
public static final String LABLE = "U_";
private Text newValue = new Text();
/*
* key: offset(1,100,1203)
* value: uid(57375476989eea12893c0c3811607bcf)
* @see org.apache.hadoop.mapreduce.Mapper#map(KEYIN, VALUEIN, org.apache.hadoop.mapreduce.Mapper.Context)
*/
protected void map(Object key, Text value,
Context context)
throws java.io.IOException ,InterruptedException {
String uidString = value.toString();
newValue.set(LABLE + uidString);
/*
* value: 57375476989eea12893c0c3811607bcf
* newValue: U_57375476989eea12893c0c3811607bcf
*/
context.write(value, newValue);
};
}
JoinReducer
package com.cys.tables;
import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
public class JoinReducer extends
Reducer<Text, Text, Text, Text> {
/*
* key:57375476989eea12893c0c3811607bcf
* values: {W_20111230000005 57375476989eea12893c0c3811607bcf 奇艺高清 1 1 http://www.qiyi.com/, U_57375476989eea12893c0c3811607bcf}
* @see org.apache.hadoop.mapreduce.Reducer#reduce(KEYIN, java.lang.Iterable, org.apache.hadoop.mapreduce.Reducer.Context)
*/
protected void reduce(Text key,
Iterable<Text> values,
Context context)
throws java.io.IOException,InterruptedException {
String uid = null;
String line = null;
List<String> list = new ArrayList<String>();
/*
* values: {W_20111230000005 57375476989eea12893c0c3811607bcf 奇艺高清 1 1 http://www.qiyi.com/,
* U_57375476989eea12893c0c3811607bcf}
* {W_20111230000005 57375476989eea12893c0c3811607bcf 奇艺高清 1 1 http://www.qiyi.com/}
*/
for(Text value : values){
if(value.toString().startsWith(UuidMapper.LABLE)){
//U_
uid = value.toString().substring(2);
}else if(value.toString().startsWith(WholeFileMapper.LABEL)){
//W_
line = value.toString().substring(2);//从value字符串的第二个位置开始截取
String[] arr = line.split("\t");
String keyword = arr[2];
list.add(keyword);
}
}
//judge
if(null != uid && list.size() > 0){
//write(uid,keyword)
for(String kw : list){
context.write(key, new Text(kw));
}
}
};
}