hadoop之MR字节码转换

大家如果看过hadoop的文本文件输入字符集格式,就知道在TextOutputFormat源码中写死了输出字节码格式是UTF-8,源码如下

  
import java.io.DataOutputStream;  
import java.io.IOException;  
import java.io.UnsupportedEncodingException;  
  
import org.apache.hadoop.conf.Configuration;  
import org.apache.hadoop.fs.FileSystem;  
import org.apache.hadoop.fs.Path;  
import org.apache.hadoop.fs.FSDataOutputStream;  
  
import org.apache.hadoop.io.NullWritable;  
import org.apache.hadoop.io.Text;  
import org.apache.hadoop.io.compress.CompressionCodec;  
import org.apache.hadoop.io.compress.GzipCodec;  
import org.apache.hadoop.mapreduce.OutputFormat;  
import org.apache.hadoop.mapreduce.RecordWriter;  
import org.apache.hadoop.mapreduce.TaskAttemptContext;  
import org.apache.hadoop.util.*;  
  
/** An {@link OutputFormat} that writes plain text files. */  
public class TextOutputFormat<K, V> extends FileOutputFormat<K, V> {//TextInputFormat是默认的输出文件格式   
  protected static class LineRecordWriter<K, V>//默认   
    extends RecordWriter<K, V> {  
    private static final String utf8 = "UTF-8"; //这个地方写死了输出字符集是UTF-8 
    private static final byte[] newline;//行结束符?   
    static {  
      try {  
        newline = "\n".getBytes(utf8);  
      } catch (UnsupportedEncodingException uee) {  
        throw new IllegalArgumentException("can't find " + utf8 + " encoding");  
      }  
    }  
  
    protected DataOutputStream out;  
    private final byte[] keyValueSeparator;//key和value的分隔符,默认的好像是Tab   
  
    public LineRecordWriter(DataOutputStream out, String keyValueSeparator) {//构造函数,初始化输出流及分隔符    
      this.out = out;  
      try {  
        this.keyValueSeparator = keyValueSeparator.getBytes(utf8);  
      } catch (UnsupportedEncodingException uee) {  
        throw new IllegalArgumentException("can't find " + utf8 + " encoding");  
      }  
    }  
  
    public LineRecordWriter(DataOutputStream out) {//默认的分隔符   
      this(out, "\t");  
    }  
  
    /** 
     * Write the object to the byte stream, handling Text as a special输出流是byte格式的 
     * case. 
     * @param o the object to print是要输出的对象 
     * @throws IOException if the write throws, we pass it on 
     */  
    private void writeObject(Object o) throws IOException {//应该是一行一行的写 key keyValueSeparator value \n   
      if (o instanceof Text) {//如果o是Text的实例   
        Text to = (Text) o;  
        out.write(to.getBytes(), 0, to.getLength());//写出   
      } else {  
        out.write(o.toString().getBytes(utf8));  
      }  
    }  
  
    public synchronized void write(K key, V value)//给写线程加锁,写是互斥行为   
      throws IOException {  
<span style="white-space:pre">    </span>//下面是为了判断key和value是否为空值   
      boolean nullKey = key == null || key instanceof NullWritable;//这语句太牛了   
      boolean nullValue = value == null || value instanceof NullWritable;  
      if (nullKey && nullValue) {//   
        return;  
      }  
      if (!nullKey) {  
        writeObject(key);  
      }  
      if (!(nullKey || nullValue)) {  
        out.write(keyValueSeparator);  
      }  
      if (!nullValue) {  
        writeObject(value);  
      }  
      out.write(newline);  
    }  
  
    public synchronized   
    void close(TaskAttemptContext context) throws IOException {  
      out.close();  
    }  
  }  
  
  public RecordWriter<K, V>    getRecordWriter(TaskAttemptContext job//获得writer实例   
                         ) throws IOException, InterruptedException {  
    Configuration conf = job.getConfiguration();  
    boolean isCompressed = getCompressOutput(job);//   
    String keyValueSeparator= conf.get("mapred.textoutputformat.separator",  
                                       "\t");  
    CompressionCodec codec = null;//压缩格式 还是?   
    String extension = "";  
    if (isCompressed) {  
      Class<? extends CompressionCodec> codecClass =   
        getOutputCompressorClass(job, GzipCodec.class);  
      codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, conf);  
      extension = codec.getDefaultExtension();  
    }  
    Path file = getDefaultWorkFile(job, extension);//这个是获取缺省的文件路径及名称,在FileOutput中有对其的实现   
    FileSystem fs = file.getFileSystem(conf);  
    if (!isCompressed) {  
      FSDataOutputStream fileOut = fs.create(file, false);  
      return new LineRecordWriter<K, V>(fileOut, keyValueSeparator);  
    } else {  
      FSDataOutputStream fileOut = fs.create(file, false);  
      return new LineRecordWriter<K, V>(new DataOutputStream  
                                        (codec.createOutputStream(fileOut)),  
                                        keyValueSeparator);  
    }  
  }  
}  
但是在生产环境中,输入输出字符集格式总是不一定会是utf-8格式,有可能处理之后的文本要求输出格式是GBK、BIG5等之类,作为下一个程序的输入格式,尤其是银行业,日志格式一般都是GBK,指定输出格式是GBK替换上面的源代码中的UTF-8即可,但是字符集编码格式那么多,做一个大数据的平台产品,面向的就是全世界的客户,这样去指定输出格式没有那么自动化,受众也是极窄的。如果是我们能够在MR程序的设置指定字符集那就完美契合生产环境中复杂的需求。故而修改源码如下:

package com.huateng.hadoop.mapred.transcoding.format;

import java.io.DataOutputStream;
import java.io.IOException;
import java.io.UnsupportedEncodingException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.GzipCodec;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.ReflectionUtils;

public class EncodingOutputFormat<K, V> extends FileOutputFormat<K, V>{
	  
	  public static String SEPERATOR = "mapreduce.output.textoutputformat.separator";
	  protected static class LineRecordWriter<K, V>
	    extends RecordWriter<K, V>  {
	    private   String charset;//手动指定的参数,在这里我们可以任意去指定一个输出的字符集
	    private   byte[] newline;
	   
	    protected DataOutputStream out;
	    private final byte[] keyValueSeparator;
//传入的参数,在下面写进输出行记录,那么指定的字符集参数从哪里传进去了?必须在MR执行的时候能够起作用。所以我们就必须在构造器中赋予该参数
	    public LineRecordWriter(DataOutputStream out, String keyValueSeparator,String dsc_charset) {
	      this.out = out;
	      charset=dsc_charset; 
	      try {
	    	  	newline = "\n".getBytes(charset);
	    	  	this.keyValueSeparator = keyValueSeparator.getBytes(charset);
	      } catch (UnsupportedEncodingException uee) {
	    	  	throw new IllegalArgumentException("can't find " + charset + " encoding");
	      }
	    }
	    /**
	     * Write the object to the byte stream, handling Text as a special case.
	     * @param o the object to print
	     * @throws IOException if the write throws, we pass it on
	     */
	    private void writeObject(Object o) throws IOException {
	      if (o instanceof Text) {
//	        	Text to = (Text) o;
//	        	out.write(to.getBytes(), 0, to.getLength());
//	      } else {
	    	  	out.write(o.toString().getBytes(charset));
	      }
	    }

	    public synchronized void write(K key, V value)
	    		throws IOException {

	      boolean nullKey = key == null || key instanceof NullWritable;
	      boolean nullValue = value == null || value instanceof NullWritable;
	      if (nullKey && nullValue) {
	        return;
	      }
	      if (!nullKey) {
	        writeObject(key);
	      }
	      if (!(nullKey || nullValue)) {
	        out.write(keyValueSeparator);
	      }
	      if (!nullValue) {
	        writeObject(value);
	      }
	      out.write("\n".getBytes());
	    }

	    public synchronized 
	    		void close(TaskAttemptContext context) throws IOException {
	      out.close();
	    }
	  }

	  public RecordWriter<K, V> 
	         getRecordWriter(
	        		 TaskAttemptContext job
	                         ) throws IOException, InterruptedException {
	    Configuration conf = job.getConfiguration();
	    String dst_charset = job.getConfiguration().get("ark.dsccodec");
//	    compress压缩
//参数就是从这里传进来的,我们得到MR的job的参数,进行指定
	    boolean isCompressed = getCompressOutput(job);
	    String keyValueSeparator= conf.get(SEPERATOR, "\t");
	    CompressionCodec codec = null;
	    String extension = "";
	    if (isCompressed) {
	      Class<? extends CompressionCodec> codecClass = 
	        getOutputCompressorClass(job, GzipCodec.class);
	      codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, conf);
	      extension = codec.getDefaultExtension();
	    }
//	    setOutputName(job,"transform");
	    Path file = getDefaultWorkFile(job, extension);
	    FileSystem fs = file.getFileSystem(conf);
	    if (!isCompressed) {
	      FSDataOutputStream fileOut = fs.create(file, false);
	      return new LineRecordWriter<K, V>(fileOut, keyValueSeparator,dst_charset);
	    } else {
	      FSDataOutputStream fileOut = fs.create(file, false);
	      return new LineRecordWriter<K, V>(new DataOutputStream(codec.createOutputStream(fileOut)),
	                                				keyValueSeparator,dst_charset);
	    }
	  }
	}
 private   String charset;//手动指定的参数,在这里我们可以任意去指定一个输出的字符集
/传入的参数,在下面写进输出行记录,那么指定的字符集参数从哪里传进去了?必须在MR执行的时候能够起作用。所以我们就必须在构造器中赋予该参数
	    public LineRecordWriter(DataOutputStream out, String keyValueSeparator,String dsc_charset) {}
<pre name="code" class="html">protected static class LineRecordWriter<K, V>
	    extends RecordWriter<K, V>  
在LineRecordWriter的父类RecordWriter中得到指定的字符集

 
<pre name="code" class="html">public RecordWriter<K, V> 
	         getRecordWriter(
	        		 TaskAttemptContext job
	                         ) throws IOException, InterruptedException {
	    Configuration conf = job.getConfiguration();
	    String dst_charset = job.getConfiguration().get("ark.dsccodec");
//参数就是从这里传进来的,我们得到MR的job的参数,进行指定
}
 
<pre name="code" class="html">job.getConfiguration().get("ark.dsccodec")这个设定Configuration则来源于创建Job实例的进行指定,至此完成。本人生产环境(华为FI集群)亲测,JDK支持的字符集都可
转换。具体main方法设定参数如下:
<pre name="code" class="html">
import java.io.IOException;

import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;

import com.google.common.base.Preconditions;
import com.huateng.hadoop.mapred.MapRedAdapter;
import com.huateng.hadoop.mapred.transcoding.format.EncodingOutputFormat;
//import com.huateng.hadoop.mapred.transcoding.format.GB2312OutputFormat;
//import com.huateng.hadoop.mapred.transcoding.format.GBKOutputFormat;
import com.huateng.hdfs.common.HDFSClient;
import com.huateng.util.common.StringUtils;

/*
 * @author canMao
 */
public class TranscodingJob 
{
	String other_code=null;
	private Job internalJob;
	public TranscodingJob(String in_path,String src_charset,
			String out_path,String dst_charset)throws Exception{
		Preconditions.checkArgument(!StringUtils.hasNullOrEmpty(new String[]{src_charset, dst_charset})
				," source_encoding and destination_encoding is null at least one");
		Job job = MapRedAdapter.createJob();
		job.getConfiguration().set("ark.codec", src_charset);
		job.getConfiguration().set("ark.dsccodec", dst_charset);
		job.setJarByClass(TranscodingJob.class);
		job.setMapperClass(TranscodingMapper.class);
		job.setNumReduceTasks(0);
		job.setOutputKeyClass(Text.class);
		job.setOutputValueClass(NullWritable.class);
		if (dst_charset.equals("UTF-8")) {
			job.setOutputFormatClass(TextOutputFormat.class);
		}else{
			job.setOutputFormatClass(EncodingOutputFormat.class);
		}
		FileInputFormat.setInputPaths(job, new Path(in_path));
		if (HDFSClient.getFileSystem().exists(new Path(out_path))) {
			HDFSClient.getFileSystem().delete(new Path(out_path),true);
		}
		FileOutputFormat.setOutputPath(job, new Path(out_path));
		internalJob = job;
	}
	
	public boolean submit() throws ClassNotFoundException, IOException, InterruptedException
	{
//		float progress=0.0f;
//		internalJob.submit();
//		while(true){
//			internalJob.mapProgress();
//		}
		
		return internalJob.waitForCompletion(true);
//		internalJob.submit();
	}
	
	
}


 

 




  • 0
    点赞
  • 0
    收藏
    觉得还不错? 一键收藏
  • 0
    评论
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值