package com.ws.userprofile;
import com.google.gson.Gson;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs;
import java.io.IOException;
import java.util.Set;
public class userProfileMr {
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
Job job = Job.getInstance(conf);
job.setJarByClass(userProfileMr.class);
job.setMapperClass(M.class);
job.setNumReduceTasks(0);
job.setOutputKeyClass(NullWritable.class);
job.setOutputValueClass(Text.class);
FileInputFormat.setInputPaths(job, new Path("hdfs://dream1:9000/userprofile/input"));
FileOutputFormat.setOutputPath(job, new Path("hdfs://dream1:9000/userprofile/output"));
job.waitForCompletion(true);
}
public static class M extends Mapper<LongWritable, Text, NullWritable, Text> {
MultipleOutputs<NullWritable, Text> multipleOutputs;
Gson g = new Gson();
Text v = new Text();
NullWritable k = NullWritable.get();
@Override
protected void setup(Context context) {
//用多路输出器,输出gid,tags 格式数据到 目录
multipleOutputs = new MultipleOutputs<>(context);
}
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
UserTags userTags = g.fromJson(value.toString(), UserTags.class);
String gid = userTags.getGid();
v.set(gid + "@" + value.toString());
multipleOutputs.write(k, v, "hdfs://dream1:9000/userprofile/tags/");
Set<String> imei = userTags.getIds().keySet();
for (String id : imei) {
v.set(id + "@" + gid);
multipleOutputs.write(k, v, "hdfs://dream1:9000/userprofile/index/");
}
}
@Override
protected void cleanup(Context context) throws IOException, InterruptedException {
multipleOutputs.close();
}
}
}