Mapreduce算法四、key值和value值分别是组合值

最新推荐文章于 2021-03-16 23:14:12 发布

NLP_Song

最新推荐文章于 2021-03-16 23:14:12 发布

阅读量1k

点赞数

分类专栏： Hdoop 文章标签： mapreduce 算法

本文链接：https://blog.csdn.net/qq_36273293/article/details/52915993

版权

Hdoop 专栏收录该内容

6 篇文章 0 订阅

订阅专栏

package CartsDemo;

import java.util.HashSet;
import java.util.Set;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;


public class TypeAndBrand {
public static void main(String[] args) throws Exception {
    if(args.length!=2){
        System.exit(0);
    }
    Job job = new Job(new Configuration(),"TypeAndBrand");
    job.setJarByClass(TypeAndBrand.class);
    FileInputFormat.setInputPaths(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));
    job.setMapperClass(TypeAndBrandMap.class);
    job.setReducerClass(TypeAndBrandReduce.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(Text.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);
    job.waitForCompletion(true);


}
public static class TypeAndBrandMap extends Mapper<LongWritable, Text, Text, Text>{
    protected void map(LongWritable key, Text value, org.apache.hadoop.mapreduce.Mapper<LongWritable,Text,Text,Text>.Context context) throws java.io.IOException ,InterruptedException {
        String[] lines = value.toString().split("\t");
        if(lines.length>7&&lines[7]!=null&&lines[8]!=null&&lines[7]!=""&&lines[8]!=""){
            String brand = lines[7].trim();
            String type = lines[8].trim();
            if (lines.length>15&&lines[12]!=null&&lines[15]!=null&&lines[12]!=""&&lines[15]!="") {
                String engine = lines[12].trim();
                String oil = lines[15].trim();
                context.write(new Text(type+"-"+brand), new Text(engine+"-"+oil));
            }
        }
    };
}
public static class TypeAndBrandReduce extends Reducer<Text, Text, Text, Text>{
    protected void reduce(Text key, java.lang.Iterable<Text> values, org.apache.hadoop.mapreduce.Reducer<Text,Text,Text,Text>.Context context) throws java.io.IOException ,InterruptedException {
        Set<String > set = new HashSet<>();
        for (Text value : values) {
            set.add(value.toString());

        }
        context.write(key, new Text(set.toString()));
    };
}
}