MapReduce的DBInputFormat

使用MapReduce直接从关系型数据库中取数据,需要将数据库驱动包放到hadoop的classpath下,

执行hadoop classpath可以查看路径信息,放到显示的任一个路径下即可。

package com.bigdata.hadoop.mapred;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.net.URI;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.filecache.DistributedCache;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.db.DBConfiguration;
import org.apache.hadoop.mapreduce.lib.db.DBInputFormat;
import org.apache.hadoop.mapreduce.lib.db.DBWritable;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class DBInputFormatApp {

	private static final String OUTPUT_PATH = "hdfs://hadoop1:9000/out";

	public static void main(String[] args) throws Exception {
		Configuration configuration = new Configuration();
		Job job = new Job(configuration, KpiApp.class.getSimpleName());
		DBConfiguration.configureDB(configuration, "com.mysql.jdbc.Driver",
				"jdbc:mysql://localhost:3306/test", "root", "hadoop");
				
		final FileSystem fileSystem = FileSystem.get(new URI(OUTPUT_PATH),
				configuration);
		fileSystem.delete(new Path(OUTPUT_PATH), true);

		job.setJarByClass(DBInputFormatApp.class);
		
		job.setInputFormatClass(DBInputFormat.class);
		//Job job, Class inputClass, String tableName, String conditions, String orderBy, String... fieldNames
		DBInputFormat.setInput(job, MyUser.class, "myuser", null,null,"id","name");

		// 当map输出类型和reduce输出类型一致时,可以不设置
		job.setMapperClass(MyMapper.class);
		job.setMapOutputKeyClass(Text.class);
		job.setMapOutputValueClass(LongWritable.class);
		
		//指定不需要使用reduce,直接把map任务的输出写入到HDFS中
		job.setNumReduceTasks(0);

		FileOutputFormat.setOutputPath(job, new Path(OUTPUT_PATH));
		
		DistributedCache.addFileToClassPath(new Path("/usr/local/mysql-connector-java-5.1.10.jar"), configuration,fileSystem);
		
		job.waitForCompletion(true);
	}

	public static class MyMapper extends
			Mapper<LongWritable, MyUser, Text, LongWritable> {
		final Text k2 = new Text();

		@Override
		protected void map(LongWritable key, MyUser value,
				Mapper<LongWritable, MyUser, Text, LongWritable>.Context context)
				throws IOException, InterruptedException {
			final String line = value.toString();
			final String[] splited = line.split("\t");
			
			for (String string : splited) {
				context.write(new Text(string), new LongWritable(1));
			}
		}
	}
	
	public static class MyUser implements Writable,DBWritable{
		int id;
		String name;
		
		@Override
		public void readFields(DataInput in) throws IOException {
			this.id = in.readInt();
			this.name = Text.readString(in);
		}
		@Override
		public void write(DataOutput out) throws IOException {
			out.writeInt(id);
			Text.writeString(out, name);
		}
		
		@Override
		public void readFields(ResultSet resultSet) throws SQLException {
			this.id = resultSet.getInt(1);
			this.name = resultSet.getString(2);
		}
		
		@Override
		public void write(PreparedStatement statement) throws SQLException {
			statement.setInt(1, id);
			statement.setString(2, name);
		}
		
		@Override
		public String toString() {
			return this.id + ":" + this.name;
		}
	}
}


评论 1
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值