hadoop----mapreduce的案例（六）（实现定制的Writable集合）

案例六

二次排序：
就是先按照第一个字段排序，如果第一个字段相同，再按照第二个字段排序

账号(Account)          金额(price)
hadoop@apache          200
hive@apache            550
yarn@apache            580
hive@apache            159
hadoop@apache          300
hadoop@apache          300
hive@apache            258
hadoop@apache          300
yarn@apache            100
hadoop@apache          150
yarn@apache            560
yarn@apache            260

统计结果为：

hadoop@apache	150
hadoop@apache	200
hadoop@apache	300
hadoop@apache	300
hive@apache	159
hive@apache	258
hive@apache	550
yarn@apache	100
yarn@apache	260
yarn@apache	560
yarn@apache	580

思路
定义一个类型AccountBean：封装 账号(Account)和金额(price)

<k1,v1> ----<偏移量，hadoop@apache 150>
|
map函数
|
<k2,v2> ----<AccountBean,NullWritable>
|
reduce函数
|
<k3,v3> ----<AccountBean,NullWritable>
context.write(AccountBean,NullWritable.get())

AccountPrice.class

public class AccountPrice implements WritableComparable<AccountPrice>{
	private Text account;
	private IntWritable price;

	@Override
	public void readFields(DataInput in) throws IOException {
		// TODO Auto-generated method stub
		account.readFields(in);
		price.readFields(in);
		
	}

	@Override
	public void write(DataOutput out) throws IOException {
		// TODO Auto-generated method stub
		account.write(out);
		price.write(out);
		
	}

	@Override
	public int compareTo(AccountPrice o) {
		// TODO Auto-generated method stub
		int temp =this.account.compareTo(o.getAccount());
		if(temp==0) {
			return  this.price.compareTo(o.getPrice());
			//降序
			//return  -this.price.compareTo(o.getPrice());
		}
		return temp;
	}

	public Text getAccount() {
		return account;
	}

	public void setAccount(Text account) {
		this.account = account;
	}

	public IntWritable getPrice() {
		return price;
	}

	public void setPrice(IntWritable price) {
		this.price = price
	}

	@Override
	public int hashCode() {
		final int prime = 31;
		int result = 1;
		result = prime * result + ((account == null) ? 0 : account.hashCode());
		result = prime * result + ((price == null) ? 0 : price.hashCode());
		return result;
	}

	@Override
	public boolean equals(Object obj) {
		if (this == obj)
			return true;
		if (obj == null)
			return false;
		if (getClass() != obj.getClass())
			return false;
		AccountPrice other = (AccountPrice) obj;
		if (account == null) {
			if (other.account != null)
				return false;
		} else if (!account.equals(other.account))
			return false;
		if (price == null) {
			if (other.price != null)
				return false;
		} else if (!price.equals(other.price))
			return false;
		return true;
	}

	@Override
	public String toString() {
		return  account + "\t" + price ;
	}

	public AccountPrice(Text account, IntWritable price) {
		super();
		this.account = account;
		this.price = price;
	}

	public AccountPrice() {
		account = new Text();
		price = new IntWritable();
	}
	
	public AccountPrice(String account,int price){
		set(account,price);
	}
	
	public void set(String account,int price){
		this.account=new Text(account);
		this.price=new IntWritable(price);
	}
}

map函数

public class AccountPriceMapper extends Mapper<LongWritable, Text, AccountPrice, NullWritable>{
	AccountPrice a = new AccountPrice();
	
	@Override
	protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, AccountPrice, NullWritable>.Context context)
			throws IOException, InterruptedException {
		StringTokenizer st =  new StringTokenizer(value.toString());
		if(st.hasMoreTokens()){
			String account = st.nextToken();
			int price = Integer.parseInt(st.nextToken());
			a.set(account, price);
			context.write(a, NullWritable.get());
		}
	}
}

reduce函数

public class AccountPriceReducer extends Reducer<AccountPrice, NullWritable, AccountPrice, NullWritable>{

	@Override
	protected void reduce(AccountPrice key, Iterable<NullWritable> value,
			Reducer<AccountPrice, NullWritable, AccountPrice, NullWritable>.Context context)
			throws IOException, InterruptedException {
		for(NullWritable i : value){
			context.write(key, NullWritable.get());
		}
	}
}

驱动类

public class AccountPriceDriver {
	public static void main(String[] args) throws Exception {
		Configuration conf = new Configuration();
		FileSystem fs = FileSystem.get(conf);
		Path path = new Path("E:/data/price/output2");
		if(fs.exists(path)){
			fs.delete(path);
		}
		
		Job job = Job.getInstance();
		job.setJobName("AccountPrice");
		job.setJarByClass(AccountPriceDriver.class);
		job.setMapperClass(AccountPriceMapper.class);
		job.setReducerClass(AccountPriceReducer.class);
		job.setOutputKeyClass(AccountPrice.class);
		job.setOutputValueClass(NullWritable.class);
	
		FileInputFormat.addInputPath(job, new Path("E:/data/price/input/*"));
		FileOutputFormat.setOutputPath(job, new Path("E:/data/price/output2"));
		System.exit(job.waitForCompletion(true)?0:1);
	}
}

运行结果