使用DBInputFormat作为输入数据源

该代码展示了如何在hadoop中与数据库连接，并将数据库中的数据作为数据源输入进行分析。

package com.ls.hadoopApp;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.db.DBConfiguration;
import org.apache.hadoop.mapreduce.lib.db.DBInputFormat;
import org.apache.hadoop.mapreduce.lib.db.DBWritable;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;


/***
 * 使用DBInputFormat作为输入，
 * 需要将驱动类放置到各个节点下，如报找不到驱动类错误则需要重启集群
 */
public class ConnectMysql extends Configured implements Tool{

	static String OUT_PATH = "hdfs://hadoop:9000/out";
	
	public static void main(String[] args) throws Exception{
		ToolRunner.run(new ConnectMysql(), args);
	}

	@Override
	public int run(String[] arg0) throws Exception {
		Configuration conf = new Configuration();
		//设置连接所需要的驱动，连接地址，数据库名，用户名及密码
		DBConfiguration.configureDB(conf, "com.mysql.jdbc.Driver", "jdbc:mysql://hadoop:3306/lee", "root", "xiao");
		
		Job job = new Job(conf,ConnectMysql.class.getName());
		job.setJarByClass(ConnectMysql.class);
		job.setInputFormatClass(DBInputFormat.class);
		//设置与数据库中的表对应的的映射类、数据表以及表中的字段信息
		DBInputFormat.setInput(job, StudentInfo.class, "student", null, null, "id","name");
		
		job.setNumReduceTasks(0);
		job.setOutputKeyClass(Text.class);
		job.setOutputKeyClass(Text.class);
		FileOutputFormat.setOutputPath(job, new Path(OUT_PATH));
		job.waitForCompletion(true);
		return 0;
	}

}

class ConnectMysqlMapper extends Mapper<Text,StudentInfo,Text,Text>{
	@Override
	protected void map(Text key, StudentInfo value,Context context)throws IOException, InterruptedException {
		String arr[] = value.toString().split("\t");
		for(String val: arr){
			System.out.println(val);
			context.write(new Text(val), new Text(val));
		}
	}
}

/***
 * 与数据库映射的类需要继承Writable和DBWritable接口
 */
class StudentInfo implements Writable,DBWritable{

	String id;
	String name;
//	int age;
	
	@Override
	public void readFields(ResultSet result) throws SQLException {
		this.id = result.getString(1);
		this.name = result.getString(2);
//		this.age = result.getInt(3);
	}

	@Override
	public void write(PreparedStatement prep) throws SQLException {
		prep.setString(1, id);
		prep.setString(2, name);
//		prep.setInt(3, age);
	}

	@Override
	public void readFields(DataInput in) throws IOException {
		this.id = Text.readString(in);
		this.name = Text.readString(in);
//		this.age = in.readInt();
	}

	@Override
	public void write(DataOutput out) throws IOException {
		Text.writeString(out, id);
		Text.writeString(out,name);
//		out.writeInt(age);
	}
	
	public String toString(){
		
		return id+"\t"+name;
	}
	
}