mapreduce自定义输入为mysql_hadoop mapreduce v1接口实现自定义inputformat，mysql作为输入...

package com.demo7;

import java.io.DataInput;

import java.io.DataOutput;

import java.io.IOException;

import java.sql.Connection;

import java.sql.DriverManager;

import java.sql.ResultSet;

import java.sql.Statement;

import java.text.ParseException;

import java.text.SimpleDateFormat;

import java.util.ArrayList;

import java.util.Date;

import java.util.HashMap;

import java.util.Map;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapred.InputFormat;

import org.apache.hadoop.mapred.InputSplit;

import org.apache.hadoop.mapred.JobConf;

import org.apache.hadoop.mapred.RecordReader;

import org.apache.hadoop.mapred.Reporter;

import org.apache.log4j.Logger;

import com.google.common.base.Joiner;

public class MysqlInputformat implements InputFormat{

private static Logger logger = Logger.getLogger(MysqlInputformat.class);

private String beginTradeDay = Config.getConfig().getProperty("dealday.begin.mysqlinputformat");

private String endTradeDay = Config.getConfig().getProperty("dealday.end.mysqlinputformat");

private int oneTaskStocks = Integer.valueOf(Config.getConfig().getProperty("stocknumber_permap.mysqlinputformat"));

@Override

public RecordReader getRecordReader(InputSplit split,

JobConf conf, Reporter reporter) throws IOException {

MysqlInputSplit mysqlSplit = (MysqlInputSplit)split;

logger.info("---------------------ln------------------------");

logger.info(mysqlSplit.tradeDay);

logger.info(mysqlSplit.stockcodes);

return new MysqlRecordReader(mysqlSplit.tradeDay, mysqlSplit.stockcodes);

}

@Override

public InputSplit[] getSplits(JobConf arg0, int arg1) throws IOException {

ArrayList splits = new ArrayList();

logger.info(String.format("begin generate map task, from %s to %s", beginTradeDay, endTradeDay));

HashMap> dayStocks = new HashMap>(); //key为交易日，value为股票列表

Connection conn = null;

try {

conn = DriverManager.getConnection(MysqlProxy.getProxoolUrl(), MysqlProxy.getProxoolConf());

// 创建一个Statement对象

Statement stmt = conn.createStatement(); // 创建Statement对象

String sql = String.format(

"select date,stock_code from gushitong.s_kline_day_complexbeforeright_ln " +

" where date>='%s' and date<='%s'",

beginTradeDay, endTradeDay);

ResultSet rs = stmt.executeQuery(sql);// 创建数据对象

String date = null;

String stockcode = null;

while (rs.next()) {

date = rs.getString("date");

stockcode = rs.getString("stock_code");

if(dayStocks.containsKey(date) == false){

dayStocks.put(date, new ArrayList(3300));

}

dayStocks.get(date).add(stockcode);

}

rs.close();

stmt.close();

conn.close();

} catch (Exception e) {

logger.error(e);

}

Joiner joiner = Joiner.on(":").useForNull("");

SimpleDateFormat sdf_1 = new SimpleDateFormat("yyyyMMdd");

SimpleDateFormat sdf_2 = new SimpleDateFormat("yyyy-MM-dd");

for(Map.Entry> dayStockEntry : dayStocks.entrySet()){

String tradeDay = dayStockEntry.getKey();

for(int i=0; i

int endindex;

if(i+oneTaskStocks<=dayStockEntry.getValue().size()){

endindex = i+oneTaskStocks;

}else{

endindex = dayStockEntry.getValue().size();

}

String stocks = joiner.join(dayStockEntry.getValue().subList(i, endindex));

i = endindex;

try {

MysqlInputSplit split = new MysqlInputSplit();

split.tradeDay = sdf_2.format(sdf_1.parse(tradeDay));

split.stockcodes = stocks;

splits.add(split);

} catch (ParseException e) {

// TODO Auto-generated catch block

e.printStackTrace();

}

}

}

InputSplit[] rtn = splits.toArray(new InputSplit[splits.size()]);

return rtn;

}

public static class MysqlInputSplit implements InputSplit{

public String getTradeDay() {

return tradeDay;

}

public void setTradeDay(String tradeDay) {

this.tradeDay = tradeDay;

}

public String getStockcodes() {

return stockcodes;

}

public void setStockcodes(String stockcodes) {

this.stockcodes = stockcodes;

}

private String tradeDay = null;

private String stockcodes = null;

@Override

public void readFields(DataInput in) throws IOException {

this.tradeDay = Text.readString(in);

this.stockcodes = Text.readString(in);

}

@Override

public void write(DataOutput out) throws IOException {

Text.writeString(out, tradeDay);

Text.writeString(out, stockcodes);

}

@Override

public long getLength() throws IOException {

// TODO Auto-generated method stub

return 0;

}

@Override

public String[] getLocations() throws IOException {

String[] arr = {"aa"}; //必须有，因为不管有没有用，框架都要用。

return arr;

}

}

public static class MysqlRecordReader implements RecordReader{

public String tradeDay = null;

public String stockcodes = null;

private boolean isDeal = false;

private long begintimeLong = new Date().getTime();

public MysqlRecordReader(String tradeDay, String stockcodes){

this.tradeDay = tradeDay;

this.stockcodes = stockcodes;

}

@Override

public void close() throws IOException {

// TODO Auto-generated method stub

}

@Override

public Text createKey() {

return new Text();

}

@Override

public Text createValue() {

return new Text();

}

@Override

public long getPos() throws IOException {

// TODO Auto-generated method stub

return 0;

}

/**

* 预计15小时为100%

*/

@Override

public float getProgress() throws IOException {

logger.info(String.format("get process is %f", ((float)(new Date().getTime() - this.begintimeLong))/(float)(15*3600*1000)));

return Math.min(0.9f, ((float)(new Date().getTime() - this.begintimeLong))/(float)(15*3600*1000));

}

@Override

public synchronized boolean next(Text key, Text value) throws IOException {

if(this.isDeal == true){

return false;

}else{

this.isDeal = true;

}

key.set(this.tradeDay);

value.set(this.stockcodes);

return true;

}

}

}

分享到：

2016-03-24 14:17

浏览 659

评论