SortedPriceName排序代码(基于mapreduce处理逻辑)

原创 2016年06月01日 05:28:11

SortedPriceName排序代码(基于mapreduce处理逻辑)


mapper.java


package com.doggie.test;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;

import java.io.IOException;

/**
 * Created by root on 5/25/16.
 */
public class mapper extends Mapper<Object,Text,LongWritable,Text> {
    public void map(Object key,Text value,Context context)
        throws IOException,InterruptedException{
        String fileName = ((FileSplit)context.getInputSplit()).getPath().toString();
        String valueString= value.toString();
        String[] items=valueString.split(" ");

        LongWritable outputKey = null;
        Text outputValue=null;

        if(fileName.contains("price")){
            outputKey = new LongWritable(Long.valueOf(items[0]));
            outputValue = new Text(items[1]);
        }else{
            outputKey = new LongWritable(Long.valueOf(items[1]));
            outputValue = new Text("name" + items[0]);
        }
        context.write(outputKey,outputValue);
    }
}


---

reducer.java

---

package com.doggie.test;


import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;

import java.io.IOException;
import java.util.TreeSet;

/**
 * Created by root on 5/25/16.
 */
public class reducer extends Reducer<LongWritable,Text,Text,LongWritable> {
    public void reduce(LongWritable key, Iterable<Text>values, Context context)
            throws IOException, InterruptedException {

        Text itemName = null;
        TreeSet<LongWritable> queue = new TreeSet<LongWritable>();

        for (Text val : values){
            if(val.toString().startsWith("name")){
                String realName = val.toString().substring(4);
                itemName = new Text(realName);
            }else{
                LongWritable price = new LongWritable(Long.valueOf(val.toString()));
                queue.add(price);
            }
        }
        for (LongWritable val : queue) {
            context.write(itemName, val);
        }
    }
}



---

main

---
package com.doggie.mtest;


import com.doggie.test.mapper;
import com.doggie.test.reducer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;

/**
 * Created by root on 5/25/16.
 */
public class Homework {
    public static void main(String[] args) throws Exception {
        Configuration conf = new Configuration();
        String[] otherArgs = new GenericOptionsParser(conf, args)
                .getRemainingArgs();
        if (otherArgs.length != 2) {
            System.err.println("Usage: homework");
            System.exit(2);
        }
        //conf.setInt("mapred.task.timeout",100);
        Job job = new Job(conf, "homework");
        job.setInputFormatClass(TextInputFormat.class);
        job.setJarByClass(Homework.class);
        job.setMapperClass(mapper.class);
        job.setReducerClass(reducer.class);
        job.setMapOutputKeyClass(LongWritable.class);
        job.setMapOutputValueClass(Text.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(LongWritable.class);
        job.setNumReduceTasks(1);
        FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
        FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
        System.exit(job.waitForCompletion(true) ? 0 : 1);
    }
}

[两个输入文件样本](https://yunpan.cn/cSDYkqREN9N3H  访问密码 ff1b)
版权声明:本文为博主原创文章,未经博主允许不得转载。

相关文章推荐

MapReduce之二——收入支出数据处理与自定义排序

1.原始数据:trade_info.txtzhangsan@163.com 6000 0 2014-02-20 lisi@163.com 2000 0 2014-02-...

Hadoop二次排序及MapReduce处理流程实例详解

一、概述 MapReduce框架对处理结果的输出会根据key值进行默认的排序,这个默认排序可以满足一部分需求,但是也是十分有限的,在我们实际的需求当中,往往有要对reduce输出结果进行二次排序的需...

MapReduce处理数据去重与数据排序

一:MapReduce处理数据去重 Map的key具有数据去重的功能 /* * 去除数据中相同数据 * 数据去重问题 * 以整个数据作为key发送出去, value为null */ pub...

Hadoop二次排序及MapReduce处理流程实例详解

一、概述 MapReduce框架对处理结果的输出会根据key值进行默认的排序,这个默认排序可以满足一部分需求,但是也是十分有限的,在我们实际的需求当中,往往有要对reduce输出结果进行二次排序的需...

MapReduce模型--二次排序

  • 2013-05-28 15:39
  • 109KB
  • 下载

【JavaWeb-9】MVC设计模式、处理业务逻辑代码的分层思想、目录结构参照、实践案例

1、B/S也成为瘦客户端,C/S叫胖客户端。JavaWeb设计模式,第一种叫model1(JSP+useBean),基本已经废弃,教学的时候可以用用。第二种叫model2(JSP+servlet+ja...

mysql游标处理复杂逻辑,分支结构代码示例

mysql游标处理复杂分支逻辑代码示例

Hadoop - Map/Reduce 通过理解org.apache.hadoop.mapreduce.Job类来学习hadoop的执行逻辑

在我的上篇文章“Hadoop - Map/Reduce 通过WordCount例子的变化来了解新版hadoop接口的变化”中,已经跟大家提到,在Hadoop新的版本中,使用org.apache.had...
内容举报
返回顶部
收藏助手
不良信息举报
您举报文章:深度学习:神经网络中的前向传播和反向传播算法推导
举报原因:
原因补充:

(最多只允许输入30个字)