Hadoop MapReduce多路径输入与多个输入 例子

 package com.uabrand.search_task;

import java.io.IOException;
import java.util.HashSet;
import java.util.Set;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.MultipleInputs;
import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.LazyOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.ToolRunner;

import com.analyzer.SPAndroid;
import com.uabrand.search_task.Base;
import com.worm.util.RegexUtil;

public class SearchKeyWord  extends Base{   
    public static class ByteMapper  extends Mapper<LongWritable, Text, Text, Text> {    
        @Override   
        protected void map(LongWritable key, Text value, Context context)throws IOException, InterruptedException {     

             context.getCounter(CounterRecorder.TOTAL).increment(1);    

             if(value==null){                
                 return;
             }

             String valueText =value.toString();
             if(valueText==null || valueText.length()<5){                  
                 return;
             }

             String[] vals = valueText.split("\\|", -1);
             if(vals==null || vals.length<39){
                   return;
             }

             String line = vals[28];
             if(line!=null){                  
                 line = line.trim();
             }

             line = line.replaceAll("={2,}", "=");  
             line= line.trim(); 

             //统一转化为小谢  
             line = line.toLowerCase();                 

             //去除脏数据    
             if(line.contains("okhttp") || line.contains("httpclient") || line.contains("uuid") ){//||  line.contains("windows")
                 return;
             }

             //纯单行数据，不包含一些特殊字符               
             if(!RegexUtil.isSpecialChar(line)){


             }else if(line.contains("iphone") || line.contains("ios") || line.contains("cfnetwork")) {


             }else{

                 line = SPAndroid.filter_Data1(line);   
                 line = SPAndroid.filter_Data2(line);
                 line = SPAndroid.filter_Data3(line);                  
                 if(line==null){
                     return;

                 }

                 String standKey=SPAndroid.getStandKey(line);       

                 String[] strArray = line.split(" |,|\t");

                 if(strArray==null || strArray.length<1){                      
                     return;
                 }                 

                 for(String item : strArray){   
                     item = item.trim();
                     if(item!=null && item.length()>1){                                   
                        context.write(new Text(item), new Text(standKey+"@"+line));
                     }
                 }               
             }  
        }
    }   

    public static class TextMapper  extends Mapper<LongWritable, Text, Text, Text> {
        @Override   
        protected void map(LongWritable key, Text value, Context context)throws IOException, InterruptedException {

            if(value==null){    
                return;
            }           

            String valueText =value.toString();
            if(valueText==null || valueText.length()<5){    
                return;
            }

            String[] vals = valueText.split("\\|", -1);
            if(vals==null || vals.length<2){
                return;
            } 

            context.write(new Text(vals[0].trim()), new Text(vals[1]));

        }
    }

    public static class ActionReducer extends Reducer<Text, Text, NullWritable, Text> {      

        private MultipleOutputs<NullWritable,Text> mos;

        @Override
        protected void setup(Context context) throws IOException,InterruptedException {
            mos = new MultipleOutputs<NullWritable,Text>(context);
        }

        @Override
        protected void cleanup(Context context) throws IOException,InterruptedException {

            if(mos!=null){
                 mos.close();
                 mos =null;
            }           
        }

        @Override           
         protected void reduce(Text key, Iterable<Text> iter,Context context) throws IOException,InterruptedException {
             if(iter==null || iter.iterator()==null){
                 return;
             }

             Set<String>datas =new HashSet<String>();
             for(Text item : iter){
                 if(item!=null){
                     datas.add(item.toString());
                     item.clear();
                     item =null;
                 }
             }  

             //还需要调整如果没有关键字如何处理
             StringBuffer strBuf = new StringBuffer();           
             if(datas.size()>0){
                 for(String str:datas){
                     strBuf.append("|");
                     strBuf.append(str);
                 }
             }

             datas.clear();
             datas =null;

            //有数据且数据含有UA数据
             if(strBuf.indexOf("$")>0 && strBuf.indexOf("@")>0){
                 mos.write("UA",NullWritable.get(), new Text(key.toString()+strBuf.toString()));                    
                 context.getCounter(CounterRecorder.SUCCEED).increment(1);                   
             }
             strBuf =null;
         }
    }

    @Override
    public int run(String[] args) throws Exception {
        // TODO Auto-generated method stub

         String inPath_1 = args[0]; 
         String inPath_2 =args[1]; 
         String outPath =args[2];  


         Configuration conf =this.getConf();            
         Job job = Job.getInstance(conf);           
         job.setJobName("SearchKeyWordTask_T");         
         job.setJarByClass(SearchKeyWord.class);

         MultipleInputs.addInputPath(job, new Path(inPath_1), SequenceFileInputFormat.class, ByteMapper.class);
         MultipleInputs.addInputPath(job, new Path(inPath_2), TextInputFormat.class, TextMapper.class);

         job.setReducerClass(ActionReducer.class);   
         job.setMapOutputKeyClass(Text.class);          
         job.setMapOutputValueClass(Text.class);
         job.setOutputKeyClass(NullWritable.class);         
         job.setOutputValueClass(Text.class);

         MultipleOutputs.addNamedOutput(job,"UA",TextOutputFormat.class,NullWritable.class,Text.class);
         LazyOutputFormat.setOutputFormatClass(job,TextOutputFormat.class);

         FileSystem fs = FileSystem.get(conf);      
         Path outPath_1 = new Path(outPath);
         if(fs.exists(outPath_1)){           
             fs.deleteOnExit(outPath_1);
         }          

         FileOutputFormat.setOutputPath(job,outPath_1);

         return job.waitForCompletion(true) ? 0 : 1;
    }

    public static int startTask(Configuration con,String[] args) throws Exception{      

        return ToolRunner.run(con,new SearchKeyWord(),args);  
    }

    public static void main(String[]args) throws Exception{     

         Configuration con =new Configuration();

         String[] filePath = new String[]{
                 "/daas/20170428",//输入文件
                 "/user/_key",//输入的关键字文件
                 "/user/_temp"//输出文件
         };

         startTask(con,filePath);
    }       

}