hadoop从oracle中到数据

最新推荐文章于 2021-04-10 08:11:47 发布

weixin_34351321

最新推荐文章于 2021-04-10 08:11:47 发布

阅读量217

点赞数

文章标签：数据库大数据 java

原文链接：https://my.oschina.net/caolinsheng/blog/279447

版权

2019独角兽企业重金招聘Python工程师标准>>>

你丫的调了两天，终于调通了。

MapperOracle.java

package com.uucun.hadooporacle;

import java.io.IOException;

import java.util.Iterator;
import java.util.List;
import java.util.Map;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapreduce.Mapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class MapperOracle extends Mapper<LongWritable, MyDbWriteble, Text, NullWritable>{
final Logger logger = LoggerFactory.getLogger(MapperOracle.class);

@Override
protected void map(LongWritable key, MyDbWriteble value,
   Context context)
   throws IOException, InterruptedException {


context.write(new Text(value.toString()), NullWritable.get());
}
}

MyDbWriteble.java

package com.uucun.hadooporacle;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Date;
import java.util.Iterator;
import java.util.List;

import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapreduce.lib.db.DBWritable;

public class MyDbWriteble implements DBWritable,WritableComparable {

private List list=new ArrayList();

class DownloadAd{

private int id;

private String session_id;

private int resource_id;

private Date visit_date;

public int getId() {
return id;
}

public void setId(int id) {
this.id = id;
}

public String getSession_id() {
return session_id;
}

public void setSession_id(String session_id) {
this.session_id = session_id;
}

public int getResource_id() {
return resource_id;
}

public void setResource_id(int resource_id) {
this.resource_id = resource_id;
}

public Date getVisit_date() {
return visit_date;
}

public void setVisit_date(Date visit_date) {
this.visit_date = visit_date;
}

}

@Override
public void write(PreparedStatement statement) throws SQLException {
//statement.execute();
}

@Override
public void readFields(ResultSet resultSet) throws SQLException {
  DownloadAd downloadAd;

  do {
    downloadAd=new DownloadAd();
   downloadAd.id=resultSet.getInt("id");
   downloadAd.session_id=resultSet.getString("session_id");
   downloadAd.resource_id=resultSet.getInt("resource_id");
   downloadAd.visit_date=resultSet.getDate("visit_date");
   list.add(downloadAd);
  } while (resultSet.next());

}

@Override
public String toString() {

  StringBuilder builder=new StringBuilder();

  for (Iterator iterator = list.iterator(); iterator.hasNext();) {
   DownloadAd downloadAd = (DownloadAd) iterator.next();
   builder.append(downloadAd.session_id);
   builder.append(",");
   builder.append(downloadAd.resource_id);
   builder.append("\n");
  }
  builder.deleteCharAt(builder.lastIndexOf("\n"));

  return builder.toString();
}

@Override
public void write(DataOutput out) throws IOException {
  // TODO Auto-generated method stub


}

@Override
public void readFields(DataInput in) throws IOException {

}

@Override
public int compareTo(Object o) {
// TODO Auto-generated method stub
return 0;
}

public List getList() {
return list;
}

public void setList(List list) {
this.list = list;
}

}

RunTool.java

package com.uucun.hadooporacle;

import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.db.DBConfiguration;
import org.apache.hadoop.mapreduce.lib.db.DataDrivenDBInputFormat;

import org.apache.hadoop.mapreduce.lib.db.OracleDataDrivenDBInputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;

import com.uucun.util.hadooputil.PluginUtil;

class RunTool extends PluginUtil{

public static void main(String[] args) throws Throwable{
     System.setProperty("HADOOP_USER_NAME","hadoop");

   final JobConf conf = new JobConf();

   System.out.println(args[0]);

  // System.exit(0);

       conf.set("fs.default.name", "hdfs://nn:9000");
     // conf.set("hadoop.job.user", "hadoop");
       conf.set("mapred.job.tracker", "nn:9001");

       conf.set("dfs.permissions","false");

//       conf.set("mapred.map.tasks", "1");

       conf.setInt("mapred.map.tasks", 4);

     //   DBConfiguration.configureDB(conf, "oracle.jdbc.driver.OracleDriver", "url");
        conf.set(DBConfiguration.INPUT_ORDER_BY_PROPERTY, "t.id");
      // conf.set(DBConfiguration.INPUT_CONDITIONS_PROPERTY, "visit_date>TO_TIMESTAMP('2013-05-23','YYYY-MM-DD HH24:MI:SS.FF') and visit_date<=TO_TIMESTAMP('2013-05-25','YYYY-MM-DD HH24:MI:SS.FF')");
        DBConfiguration.configureDB(conf, "oracle.jdbc.driver.OracleDriver", "jdbc:oracle:thin:@192.168.4.73:1521:XE", "oracle", "oracle");
       String jarName = runonhadoop().toString();
         conf.setJar(jarName);

          Job job = new Job( conf , "hadoopOracle" );

         job.setMapperClass( MapperOracle.class );
       // job.setReducerClass(Reducer.class);
       job.setInputFormatClass(OracleDataDrivenDBInputFormat.class);

         job.setMapOutputKeyClass(Text.class);
         job.setMapOutputValueClass(NullWritable.class);

         job.setOutputFormatClass(TextOutputFormat.class);
         job.setJarByClass(RunTool.class);

         String queryStr="select t.id,t.session_id,t.resource_id,t.visit_date from tbl_res_download_succeed_log t where t.visit_date>to_date('2013-05-23','yyyy-MM-dd HH24:mi:ss') and t.visit_date<=to_date('2013-05-25','yyyy-MM-dd HH24:mi:ss') and $CONDITIONS";
     // String queryStr="select id,session_id,resource_id,visit_date from tbl_res_download_succeed_log where visit_date>TO_TIMESTAMP('2013-05-23','YYYY-MM-DD HH24:MI:SS.FF') and visit_date<=TO_TIMESTAMP('2013-05-25','YYYY-MM-DD HH24:MI:SS.FF')";
      //   String queryStr="select id,session_id,resource_id,visit_date from tbl_res_download_succeed_log";

         String boundStr="select min(id),max(id) from tbl_res_download_succeed_log where visit_date>to_date('2013-05-23','yyyy-MM-dd HH24:mi:ss') and visit_date<=to_date('2013-05-25','yyyy-MM-dd HH24:mi:ss')";

         TextOutputFormat.setOutputPath(job, new Path(args[1]));

       OracleDataDrivenDBInputFormat.setInput(job, MyDbWriteble.class,queryStr ,boundStr);


       // OracleDataDrivenDBInputFormat.setInput(job, MyDbWriteble.class,"tbl_res_download_succeed_log","visit_date>to_date('2013-05-23','yyyy-MM-dd HH24:mi:ss') and visit_date<to_date('2013-05-24','yyyy-MM-dd HH24:mi:ss')","visit_date","id","session_id","resource_id","visit_date");
    // OracleDataDrivenDBInputFormat.setBoundingQuery(job.getConfiguration(), boundStr);

      FileSystem fs =FileSystem.get(conf);
        if (fs.exists(new Path(args[1]))) {
    fs.delete(new Path(args[1]),true);
    fs.close();
   }

         job.waitForCompletion(true);

}