MongoDB相关API

pom依赖

<!-- https://mvnrepository.com/artifact/org.mongodb/mongo-java-driver -->
<dependency>
  <groupId>org.mongodb</groupId>
  <artifactId>mongo-java-driver</artifactId>
  <version>3.2.2</version>
</dependency>

1、简单的连接查询

MongoClient mongoClient = new MongoClient("192.168.221.205", 27017);
MongoDatabase dbMydemo = mongoClient.getDatabase("mydemo");
MongoCollection<Document> stus = dbMydemo.getCollection("students");
FindIterable<Document> iter = stus.find();

for (Document document : iter) {
    System.out.println(document);
}

2、过滤查询

MongoClient client = new MongoClient("192.168.221.205", 27017);
MongoDatabase db = client.getDatabase("mydemo");
MongoCollection<Document> stus = db.getCollection("students");
MongoCursor<Document> cursor = stus.find(Filters.and(
    Filters.lte("age", 33),
    Filters.gte("age", 14),
    Filters.regex("gender","^f")
)).iterator();

while (cursor.hasNext()) {
    System.out.println(cursor.next());
}

3、插入数据

MongoClient client = new MongoClient("192.168.221.205", 27017);
MongoDatabase db = client.getDatabase("mydemo");
MongoCollection<Document> stus = db.getCollection("students");
List<Document> docs = new ArrayList<>();
Document d1 = new Document("name", "hehe");
d1.append("record","collage").append("gender","female").append("age",21);
Document d2 = new Document("name", "xixi");
d1.append("record","collage").append("gender","female").append("age",21);
docs.add(d1);
docs.add(d2);
//插入数据
stus.insertMany(docs);
//查看插入的数据
FindIterable<Document> newdocs = stus.find(Filters.and(
    Filters.in("name", "hehe", "xixi")
));
for (Document newdoc : newdocs) {
    System.out.println(newdoc);
}

4、Kafka to MongoDB

//MongoDB配置
MongoClient client = new MongoClient("192.168.221.205", 27017);
MongoDatabase db = client.getDatabase("uf");
MongoCollection<Document> uf = db.getCollection("user_friends_md");

//kafka配置
Properties prop_consumer = new Properties();
prop_consumer.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,"192.168.221.221:9092");
prop_consumer.setProperty(ConsumerConfig.GROUP_ID_CONFIG,"xym");
prop_consumer.setProperty(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringDeserializer");
prop_consumer.setProperty(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringDeserializer");
prop_consumer.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG,"earliest");

//创建消费者和被订阅的对象
KafkaConsumer<String, String> consumer = new KafkaConsumer<>(prop_consumer);
TopicPartition tp = new TopicPartition("user_friends_ss", 0);
List<TopicPartition> list = new ArrayList<>();
list.add(tp);
consumer.assign(list);

//开启消费
while (true){
    ConsumerRecords<String, String> records = consumer.poll(Duration.ofMillis(1000));
    List<Document> docs = new ArrayList<>();
    records.forEach(x->{
        //创建mongo的消息对象
        Document doc = new Document("userid", x.value().split(",", -1)[0]);
        doc.append("friendid",x.value().split(",",-1)[1]);
        docs.add(doc);
    });
    //如果数据读完，数组里就是空，插入空就会报错，加个判断
    if (docs.size()!=0) {
        uf.insertMany(docs);
    }
}

5、使用java读取MongoDB的消费

//MongDb配置
MongoClient client = new MongoClient("192.168.221.205", 27017);
MongoDatabase ufdb = client.getDatabase("uf");
MongoCollection<Document> user_friends_md = ufdb.getCollection("user_friends_md");
MongoCursor<Document> iter = user_friends_md.find().iterator();

//连接hdfs，设置hdfs路径,config,登录用户
String path = "hdfs://single:9000";
Configuration conf = new Configuration();
conf.setBoolean("dfs.support.append", true);
conf.set("dfs.client.block.write.replace-datanode-on-failure.policy", "NEVER");
conf.set("dfs.client.block.write.replace-datanode-on-failure.enable", "true");

//用户可以不传，默认当前登录用户名
FileSystem fs = FileSystem.get(URI.create(path), conf, "root");
FSDataOutputStream fsdos = fs.create(new Path("/mongo/mongo.txt"));

//遍历MongoDB，使用正则对格式解析
while (iter.hasNext()) {
    String str = iter.next().toJson();
    String[] split = str.split("[,|\"]", -1);
    fsdos.write((split[10] + "," + split[15] + "\n").getBytes());//调用写入方法写进去，要转为bytes
}
//写完 要释放掉
fsdos.flush();