MapReduce练习-----版本变动

题目要求：

在所有有版本变动的记录后面追加一条字段信息：该信息就是上一个版本的版本号，只限同用户
例如：
20170308,黄渤,光环斗地主,10,360手机助手,0.2版本,北京
20170308,黄渤,光环斗地主,13,360手机助手,0.3版本,北京,0.2版本
20170308,徐峥,光环斗地主,14,360手机助手,0.3版本,北京

20170308,徐峥,光环斗地主,15,360手机助手,0.4版本,北京,0.3版本

数据及字段解析：

20170308,黄渤,斗地主,8,360手机助手,0.1版本,北京
20170308,黄渤,斗地主,5,360手机助手,0.1版本,北京
20170308,黄渤,斗地主,7,360手机助手,0.1版本,北京
20170308,黄渤,斗地主,10,360手机助手,0.2版本,北京
20170308,黄渤,斗地主,9,360手机助手,0.2版本,北京
20170308,黄渤,斗地主,23,360手机助手,0.2版本,北京
20170308,黄渤,斗地主,22,360手机助手,0.2版本,北京
20170308,黄渤,斗地主,14,360手机助手,0.3版本,北京
20170308,黄渤,斗地主,13,360手机助手,0.3版本,北京
20170308,黄渤,斗地主,16,360手机助手,0.4版本,北京
20170308,黄渤,斗地主,18,360手机助手,0.4版本,北京
20170308,黄渤,斗地主,19,360手机助手,0.4版本,北京
20170308,黄渤,斗地主,15,360手机助手,0.4版本,北京
20170309,徐峥,斗地主,8,360手机助手,0.1版本,北京
20170309,徐峥,斗地主,5,360手机助手,0.1版本,北京
20170309,徐峥,斗地主,6,360手机助手,0.1版本,北京
20170309,徐峥,斗地主,10,360手机助手,0.2版本,北京
20170309,徐峥,斗地主,12,360手机助手,0.2版本,北京
20170309,徐峥,斗地主,11,360手机助手,0.3版本,北京
20170309,徐峥,斗地主,9,360手机助手,0.2版本,北京
20170309,徐峥,斗地主,23,360手机助手,0.2版本,北京
20170309,徐峥,斗地主,22,360手机助手,0.2版本,北京
20170309,徐峥,斗地主,14,360手机助手,0.3版本,北京
20170309,徐峥,斗地主,13,360手机助手,0.3版本,北京
20170309,徐峥,斗地主,16,360手机助手,0.4版本,北京
20170309,徐峥,斗地主,18,360手机助手,0.4版本,北京
20170309,徐峥,斗地主,19,360手机助手,0.5版本,北京
20170309,徐峥,斗地主,15,360手机助手,0.4版本,北京

字段信息：
用户ID，用户名，游戏名，小时，数据来源，游戏版本，用户所在地
id, name, game, hour, source, version, city

用户“黄渤”在10点钟是0.2版本，但是到了13点变成了0.3版本，那么就在13点钟这条记录的后面追加一个字段值0.2版本，也就是上个版本的版本号。当然，为什么从10点直接到了13点，因为11点和12点的数据没有收集到。

思路：分析数据得出，可以根据日期，用户分组，根据时间排序，然后判断版本，版本变动就添加标记。

使用自定义的数据类型：

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;

import org.apache.hadoop.io.WritableComparable;

public class VersionBean implements WritableComparable<VersionBean>{
	private String id; //日期
	private String name; //用户
	private String game; //游戏名称
	private int hour; //时间点
	private String source; //工具
	private String version;//游戏版本
	private String city; //城市
	private String lastVersion; //上一个版本
	
	public String getId() {
		return id;
	}
	public void setId(String id) {
		this.id = id;
	}
	public String getName() {
		return name;
	}
	public void setName(String name) {
		this.name = name;
	}
	public String getGame() {
		return game;
	}
	public void setGame(String game) {
		this.game = game;
	}
	public int getHour() {
		return hour;
	}
	public void setHour(int hour) {
		this.hour = hour;
	}
	public String getSource() {
		return source;
	}
	public void setSource(String source) {
		this.source = source;
	}
	public String getVersion() {
		return version;
	}
	public void setVersion(String version) {
		this.version = version;
	}
	public String getCity() {
		return city;
	}
	public void setCity(String city) {
		this.city = city;
	}
	public String getLastVersion() {
		return lastVersion;
	}
	public void setLastVersion(String lastVersion) {
		this.lastVersion = lastVersion;
	}
	
	public VersionBean(String id, String name, String game, int hour, String source, String version, String city) {
		super();
		this.id = id;
		this.name = name;
		this.game = game;
		this.hour = hour;
		this.source = source;
		this.version = version;
		this.city = city;
		this.lastVersion = "";
	}
	public VersionBean() {
		super();
	}
	
	@Override
	public String toString() {
		//没有变动
		if("".equals(lastVersion)){
			return  id + "," + name + "," + game + "," + hour + "," + source
					+ "," + version + "," + city;
		}else{
			//有变动
			return  id + "," + name + "," + game + "," + hour + "," + source
					+ "," + version + "," + city+","+lastVersion;
		}
		
	}
	

	@Override
	public void write(DataOutput out) throws IOException {
		out.writeUTF(id);
		out.writeUTF(name);
		out.writeUTF(game);
		out.writeInt(hour);
		out.writeUTF(source);
		out.writeUTF(version);
		out.writeUTF(city);
		out.writeUTF(lastVersion);
	}
	
	@Override
	public void readFields(DataInput in) throws IOException {
		id = in.readUTF();
		name = in.readUTF();
		game = in.readUTF();
		hour = in.readInt();
		source = in.readUTF();
		version = in.readUTF();
		city = in.readUTF();
		lastVersion = in.readUTF();
	} 
	
	@Override
	public int compareTo(VersionBean o) {
		
		//首先判断是不是一个用户，不同用户之间比较没有意义
		if(o.name.equals(this.name)){
			long index = Long.parseLong(this.id) - Long.parseLong(o.id);
			//然后根据日期进行排序
			if(index > 0){
				return 1;
			}else if(index < 0){
				return -1;
			}else{
				//然后根据时间点进行排序
				int flag = this.hour - o.hour;
				return flag > 0 ? 1:-1;
			}
		}else{
			return o.name.compareTo(this.name);
		}
		
		
	}
	
	
}

mapreduce程序：

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class VersionOne {
	//20170309,徐峥,光环斗地主,15,360手机助手,0.4版本,北京
	public static class MyMapper extends Mapper<LongWritable, Text, VersionBean, NullWritable>{
		@Override
		protected void map(LongWritable key, Text value,Context context)
				throws IOException, InterruptedException {
			String[] lines = value.toString().split(",");
			//直接切分字段，然后构建自定义输出对象 输出
			VersionBean vb = new VersionBean(lines[0],lines[1],lines[2],Integer.parseInt(lines[3]),lines[4],lines[5],lines[6]);
			context.write(vb, NullWritable.get());
		}
	}
	
	public static class MyReducer extends Reducer<VersionBean, NullWritable, VersionBean, NullWritable>{
		//使用这个字段来保存上一条记录的version
		String version = "";
		//使用这个字段来保存用户的名称
		String name = "";
		@Override
		protected void reduce(VersionBean key, Iterable<NullWritable> values,Context context)
				throws IOException, InterruptedException {
			
			for (NullWritable nullWritable : values) {
				//如果当前版本和上一条记录的版本相同
				if(version.equals(key.getVersion())){
					//将当前记录的版本赋值给version,用户名赋值给name
					version = key.getVersion();
					name = key.getName();
				}else{
					//如果当前版本和上一条记录的版本不相同
					//然后判断用户名是否相等，确定是同一个人在比较
					if(name.equals(key.getName())){
						//如果用户名也相同，那么就将version作为当前记录的上一版本字段的值
						key.setLastVersion(version);
					}else{
						//如果用户名不相同，那么就更新用户名称
						name = key.getName();
					}
					//最后将当前版本的值赋值给version
					version = key.getVersion();
				}
				
				
				context.write(key, nullWritable);
			}
		}
	}
	
	public static void main(String[] args) throws Exception {
		Configuration conf = new Configuration();
		Job job = Job.getInstance(conf);
		
		job.setJarByClass(VersionOne.class);
		job.setMapperClass(MyMapper.class);
		job.setReducerClass(MyReducer.class);
		
		job.setOutputKeyClass(VersionBean.class);
		job.setOutputValueClass(NullWritable.class);
		
		FileSystem fs = FileSystem.get(conf);
		if(fs.exists(new Path("G:/files/mr/day2/q4/output_1"))){
			fs.delete(new Path("G:/files/mr/day2/q4/output_1"),true);
		}
		
		FileInputFormat.setInputPaths(job, new Path("G:/files/mr/day2/q4/input"));
		FileOutputFormat.setOutputPath(job, new Path("G:/files/mr/day2/q4/output_1"));
		
		boolean isDone = job.waitForCompletion(true);
		System.exit(isDone?0:1);
		
	}
}