package demo;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
/**
* 将自定义的map和reduce等类写一个文件中,创建静态内部类
*/
public class MTest {
/**
* 自定义MR序列化对象,将分析字符串封装到对象(java POJO)
* MR自定义序列化对象,实现org.apache.hadoop.io.Writable接口
*/
static class MyPo implements Writable{
private String date;//yyyy-MM-dd
private double temp;//气温
public String getDate() {
return date;
}
public void setDate(String date) {
this.date = date;
}
public double getTemp() {
return temp;
}
public void setTemp(double temp) {
this.temp = temp;
}
@Override
public void write(DataOutput out) throws IOException {
//将当前对象属性依次写入MR上下提供DataOutput参数中
out.writeUTF(date);//针对String类型的属性写入
out.writeDouble(temp);
}
@Override
public void readFields(DataInput in) throws IOException {
//将MR上下文环境中DataInput参数依次传入当前对象的每一个属性
this.date=in.readUTF();
this.temp=in.readDouble();
}
@Override
public String toString() {
// TODO Auto-generated method stub
return this.date;
}
}
static class MyMapper extends Mapper<LongWritable,Text,MyPo, DoubleWritable>{
private MyPo keypo=new MyPo();//输出的key
private DoubleWritable temp=new DoubleWritable();//输出的value
@Override
protected void map(LongWritable key, Text value,Context context)
throws IOException, InterruptedException {
//得到当前行
String line=value.toString();
//按空格分隔
String[] arr=line.split("\\s+");
if(arr==null||arr.length<1)
return;
//当前行分解的数据封装到自定义的序列化对象中
// 封装年-月
keypo.setDate(arr[0]+"-"+arr[1]);//2016-3
// 封装气温
temp.set(new Double(arr[3]));
//向MR上下文对象写入key-value对
context.write(keypo, temp);
}
}
static class MyReduce extends Reducer<MyPo, DoubleWritable, Text, Text>{
private Text mykey=new Text();
private Text myvalue=new Text();
@Override
protected void reduce(MyPo key, Iterable<DoubleWritable> iter,
Context ctx) throws IOException, InterruptedException {
// TODO Auto-generated method stub
StringBuilder buf=new StringBuilder("[");
for (DoubleWritable wea : iter) {
buf.append(wea.get()+",");
}
String temp=buf.substring(0,buf.lastIndexOf(","))+"]";
mykey.set(key.getDate());
myvalue.set(temp);
ctx.write(mykey, myvalue);
}
}
public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
Configuration cfg=new Configuration();
Job job = Job.getInstance(cfg);
job.setJobName("我是一个序列化");
job.setJarByClass(MTest.class);
job.setMapperClass(MyMapper.class);
job.setOutputKeyClass(MyPo.class);
job.setMapOutputValueClass(DoubleWritable.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
job.setReducerClass(MyReduce.class);
//格式化
FileInputFormat.addInputPath(job, new Path("hdfs://cai4:9000/minput/wea.txt"));
FileOutputFormat.setOutputPath(job, new Path("hdfs://cai4:9000/mouts"));
boolean flag = job.waitForCompletion(true);
System.exit(flag?0:1);
}
}
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
/**
* 将自定义的map和reduce等类写一个文件中,创建静态内部类
*/
public class MTest {
/**
* 自定义MR序列化对象,将分析字符串封装到对象(java POJO)
* MR自定义序列化对象,实现org.apache.hadoop.io.Writable接口
*/
static class MyPo implements Writable{
private String date;//yyyy-MM-dd
private double temp;//气温
public String getDate() {
return date;
}
public void setDate(String date) {
this.date = date;
}
public double getTemp() {
return temp;
}
public void setTemp(double temp) {
this.temp = temp;
}
@Override
public void write(DataOutput out) throws IOException {
//将当前对象属性依次写入MR上下提供DataOutput参数中
out.writeUTF(date);//针对String类型的属性写入
out.writeDouble(temp);
}
@Override
public void readFields(DataInput in) throws IOException {
//将MR上下文环境中DataInput参数依次传入当前对象的每一个属性
this.date=in.readUTF();
this.temp=in.readDouble();
}
@Override
public String toString() {
// TODO Auto-generated method stub
return this.date;
}
}
static class MyMapper extends Mapper<LongWritable,Text,MyPo, DoubleWritable>{
private MyPo keypo=new MyPo();//输出的key
private DoubleWritable temp=new DoubleWritable();//输出的value
@Override
protected void map(LongWritable key, Text value,Context context)
throws IOException, InterruptedException {
//得到当前行
String line=value.toString();
//按空格分隔
String[] arr=line.split("\\s+");
if(arr==null||arr.length<1)
return;
//当前行分解的数据封装到自定义的序列化对象中
// 封装年-月
keypo.setDate(arr[0]+"-"+arr[1]);//2016-3
// 封装气温
temp.set(new Double(arr[3]));
//向MR上下文对象写入key-value对
context.write(keypo, temp);
}
}
static class MyReduce extends Reducer<MyPo, DoubleWritable, Text, Text>{
private Text mykey=new Text();
private Text myvalue=new Text();
@Override
protected void reduce(MyPo key, Iterable<DoubleWritable> iter,
Context ctx) throws IOException, InterruptedException {
// TODO Auto-generated method stub
StringBuilder buf=new StringBuilder("[");
for (DoubleWritable wea : iter) {
buf.append(wea.get()+",");
}
String temp=buf.substring(0,buf.lastIndexOf(","))+"]";
mykey.set(key.getDate());
myvalue.set(temp);
ctx.write(mykey, myvalue);
}
}
public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
Configuration cfg=new Configuration();
Job job = Job.getInstance(cfg);
job.setJobName("我是一个序列化");
job.setJarByClass(MTest.class);
job.setMapperClass(MyMapper.class);
job.setOutputKeyClass(MyPo.class);
job.setMapOutputValueClass(DoubleWritable.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
job.setReducerClass(MyReduce.class);
//格式化
FileInputFormat.addInputPath(job, new Path("hdfs://cai4:9000/minput/wea.txt"));
FileOutputFormat.setOutputPath(job, new Path("hdfs://cai4:9000/mouts"));
boolean flag = job.waitForCompletion(true);
System.exit(flag?0:1);
}
}