【MapReduce】MapReduce计算酒店的平均消费值

数据即需求

点击下载数据

• 需求
  计算俩个城市的酒店平均消费值
  计算每个城市总共有多少个酒店参与计算

代码实现

自定义类

import org.apache.hadoop.io.WritableComparable;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;

public class HotelData implements WritableComparable<HotelData> {
    private String address;
    private String city_id;
    private String city_name;
    private String collectionTime;
    private String dpcount;
    private String dpscore;
    private String id;
    private String img;
    private String isSingleRec;
    private String lat;
    private String lon;
    private String name;
    private String pingyin;
    private String price;
    private String score;
    private String shortName;
    private String star;
    private String stardesc;
    private String url;

    public int compareTo(HotelData o) {
        return 0;
    }

    public void write(DataOutput dataOutput) throws IOException {
        dataOutput.writeUTF(address);
        dataOutput.writeUTF(city_id);
        dataOutput.writeUTF(city_name);
        dataOutput.writeUTF(collectionTime);
        dataOutput.writeUTF(dpcount);
        dataOutput.writeUTF(dpscore);
        dataOutput.writeUTF(id);
        dataOutput.writeUTF(img);
        dataOutput.writeUTF(isSingleRec);
        dataOutput.writeUTF(lat);
        dataOutput.writeUTF(lon);
        dataOutput.writeUTF(name);
        dataOutput.writeUTF(pingyin);
        dataOutput.writeUTF(price);
        dataOutput.writeUTF(score);
        dataOutput.writeUTF(shortName);
        dataOutput.writeUTF(star);
        dataOutput.writeUTF(stardesc);
        dataOutput.writeUTF(url);
    }

    public void readFields(DataInput dataInput) throws IOException {
        address = dataInput.readUTF();
        city_id = dataInput.readUTF();
        city_name = dataInput.readUTF();
        collectionTime = dataInput.readUTF();
        dpcount = dataInput.readUTF();
        dpscore = dataInput.readUTF();
        id = dataInput.readUTF();
        img = dataInput.readUTF();
        isSingleRec = dataInput.readUTF();
        lat = dataInput.readUTF();
        lon = dataInput.readUTF();
        name = dataInput.readUTF();
        pingyin = dataInput.readUTF();
        price = dataInput.readUTF();
        score = dataInput.readUTF();
        shortName = dataInput.readUTF();
        star = dataInput.readUTF();
        stardesc = dataInput.readUTF();
        url = dataInput.readUTF();
    }

    public String getAddress() {
        return address;
    }

    public void setAddress(String address) {
        this.address = address;
    }

    public String getCity_id() {
        return city_id;
    }

    public void setCity_id(String city_id) {
        this.city_id = city_id;
    }

    public String getCity_name() {
        return city_name;
    }

    public void setCity_name(String city_name) {
        this.city_name = city_name;
    }

    public String getCollectionTime() {
        return collectionTime;
    }

    public void setCollectionTime(String collectionTime) {
        this.collectionTime = collectionTime;
    }

    public String getDpcount() {
        return dpcount;
    }

    public void setDpcount(String dpcount) {
        this.dpcount = dpcount;
    }

    public String getDpscore() {
        return dpscore;
    }

    public void setDpscore(String dpscore) {
        this.dpscore = dpscore;
    }

    public String getId() {
        return id;
    }

    public void setId(String id) {
        this.id = id;
    }

    public String getImg() {
        return img;
    }

    public void setImg(String img) {
        this.img = img;
    }

    public String getIsSingleRec() {
        return isSingleRec;
    }

    public void setIsSingleRec(String isSingleRec) {
        this.isSingleRec = isSingleRec;
    }

    public String getLat() {
        return lat;
    }

    public void setLat(String lat) {
        this.lat = lat;
    }

    public String getLon() {
        return lon;
    }

    public void setLon(String lon) {
        this.lon = lon;
    }

    public String getName() {
        return name;
    }

    public void setName(String name) {
        this.name = name;
    }

    public String getPingyin() {
        return pingyin;
    }

    public void setPingyin(String pingyin) {
        this.pingyin = pingyin;
    }

    public String getPrice() {
        return price;
    }

    public void setPrice(String price) {
        this.price = price;
    }

    public String getScore() {
        return score;
    }

    public void setScore(String score) {
        this.score = score;
    }

    public String getShortName() {
        return shortName;
    }

    public void setShortName(String shortName) {
        this.shortName = shortName;
    }

    public String getStar() {
        return star;
    }

    public void setStar(String star) {
        this.star = star;
    }

    public String getStardesc() {
        return stardesc;
    }

    public void setStardesc(String stardesc) {
        this.stardesc = stardesc;
    }

    public String getUrl() {
        return url;
    }

    public void setUrl(String url) {
        this.url = url;
    }
}

Mapper阶段

import com.alibaba.fastjson.JSON;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;

public class MapTest extends Mapper<LongWritable, Text,Text, IntWritable> {
    Text k = new Text();
    IntWritable v = new IntWritable();
    HotelData hotelData = new HotelData();
    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        hotelData = JSON.parseObject(value.toString(),HotelData.class);
        k.set(hotelData.getCity_name());
        v.set(Integer.parseInt(hotelData.getPrice()));
        context.write(k,v);
    }
}

Reducer阶段

import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;

public class RedTest extends Reducer<Text, IntWritable,Text, DoubleWritable> {
    Text k = new Text();
    DoubleWritable v = new DoubleWritable();
    double count = 0;
    double sum;
    @Override
    protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
        for (IntWritable i:values){
            sum+=i.get();
            count++;
        }
        v.set(sum/count);
        k.set(key+"的酒店参与的有"+count+"所");
        context.write(k,v);
        sum=0;
        count=0;
    }
}

Driver阶段

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.File;

public class DriTest {
    public static void main(String[] args) throws Exception {
        File file = new File("D:\\MP\\宾馆的平均价\\output");
        if (file.exists()) {
            delFile(file);
            driver();
        } else {
            driver();
        }
    }

    public static void delFile(File file) {
        File[] files = file.listFiles();
        if (files != null && files.length != 0) {
            for (int i = 0; i < files.length; i++) {
                delFile(files[i]);
            }
        }
        file.delete();
    }

    public static void driver() throws Exception {
        Configuration conf = new Configuration();
        Job job = Job.getInstance(conf);

        job.setMapperClass(MapTest.class);
        job.setJarByClass(DriTest.class);
        job.setReducerClass(RedTest.class);

        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(IntWritable.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(DoubleWritable.class);

        FileInputFormat.setInputPaths(job, "D:\\MP\\宾馆的平均价\\input\\data");
        FileOutputFormat.setOutputPath(job, new Path("D:\\MP\\宾馆的平均价\\output"));
        boolean b = job.waitForCompletion(true);
        System.exit(b ? 0 : 1);
    }
}

结果