hadoop读写mongodemo

最新推荐文章于 2024-07-25 09:48:00 发布

weixin_34200628

最新推荐文章于 2024-07-25 09:48:00 发布

阅读量75

点赞数

文章标签：大数据数据库 java

原文链接：https://my.oschina.net/caolinsheng/blog/281118

版权

2019独角兽企业重金招聘Python工程师标准>>>

MongoMapper.java

public class MongoMapper extends Mapper<Object,BSONObject, IntWritable, Text> {

@Override
protected void map(Object key, BSONObject value,
   org.apache.hadoop.mapreduce.Mapper.Context context)
   throws IOException, InterruptedException {

  double id=(Double)value.get("myid");
  String name=(String)value.get("name");

  context.write(new IntWritable((int)id),new Text(name));
}
}

MongoReducer.java

public class MongoReducer extends Reducer<IntWritable,DoubleWritable, IntWritable, BSONWritable> {
protected void reduce(IntWritable key, Iterable<DoubleWritable> values,
org.apache.hadoop.mapreduce.Reducer.Context context)
throws IOException, InterruptedException {

  BasicBSONObject output = new BasicBSONObject();
  int count=0;
        for (Iterator iterator = values.iterator(); iterator.hasNext();) {
   count++;

  }
        output.put("count", count);
        context.write( key, new BSONWritable( output ) );

}
}

RunTool.java

class RunTool extends PluginUtil{

public static void main(String[] args) throws Throwable{
     System.setProperty("HADOOP_USER_NAME","hadoop");

   final JobConf conf = new JobConf();
      // 定义MongoDB数据库的输入与输出表名，这里是调用本地的MongoDB，默认端口号为27017
         MongoConfigUtil.setInputURI( conf, "mongodb://192.168.4.73/foo.person" );
         MongoConfigUtil.setOutputURI( conf, "mongodb://192.168.4.73/foo.out" );
         MongoConfigUtil.setAuthURI(conf, "mongodb://mongo:mongo@192.168.4.73/admin");
         System.out.println( "Conf: " + conf );

         conf.set("fs.default.name", "hdfs://hadoop1:9000");
     // conf.set("hadoop.job.user", "hadoop");
       conf.set("mapred.job.tracker", "hadoop1:9001");
       conf.set("dfs.permissions","false");
       String jarName = runonhadoop().toString();
         conf.setJar(jarName);

         final Job job = new Job( conf , "mongohadoop" );
         job.setMapperClass( MongoMapper.class );
         job.setReducerClass( MongoReducer.class );

      // 定义Mapper与Reduce的输出key/value的类型
         job.setOutputKeyClass( IntWritable.class );
         job.setOutputValueClass( BSONWritable.class );

         job.setMapOutputKeyClass(IntWritable.class);
         job.setMapOutputValueClass(DoubleWritable.class);

      // 定义InputFormat与OutputFormat的类型
         job.setInputFormatClass( MongoInputFormat.class );
         job.setOutputFormatClass( MongoOutputFormat.class );
         System.exit( job.waitForCompletion( true ) ? 0 : 1 );
}