pom依赖
<!-- https://mvnrepository.com/artifact/org.mongodb/mongo-java-driver -->
<dependency>
<groupId>org.mongodb</groupId>
<artifactId>mongo-java-driver</artifactId>
<version>3.2.2</version>
</dependency>
1、简单的连接查询
MongoClient mongoClient = new MongoClient("192.168.221.205", 27017);
MongoDatabase dbMydemo = mongoClient.getDatabase("mydemo");
MongoCollection<Document> stus = dbMydemo.getCollection("students");
FindIterable<Document> iter = stus.find();
for (Document document : iter) {
System.out.println(document);
}
2、过滤查询
MongoClient client = new MongoClient("192.168.221.205", 27017);
MongoDatabase db = client.getDatabase("mydemo");
MongoCollection<Document> stus = db.getCollection("students");
MongoCursor<Document> cursor = stus.find(Filters.and(
Filters.lte("age", 33),
Filters.gte("age", 14),
Filters.regex("gender","^f")
)).iterator();
while (cursor.hasNext()) {
System.out.println(cursor.next());
}
3、插入数据
MongoClient client = new MongoClient("192.168.221.205", 27017);
MongoDatabase db = client.getDatabase("mydemo");
MongoCollection<Document> stus = db.getCollection("students");
List<Document> docs = new ArrayList<>();
Document d1 = new Document("name", "hehe");
d1.append("record","collage").append("gender","female").append("age",21);
Document d2 = new Document("name", "xixi");
d1.append("record","collage").append("gender","female").append("age",21);
docs.add(d1);
docs.add(d2);
//插入数据
stus.insertMany(docs);
//查看插入的数据
FindIterable<Document> newdocs = stus.find(Filters.and(
Filters.in("name", "hehe", "xixi")
));
for (Document newdoc : newdocs) {
System.out.println(newdoc);
}
4、Kafka to MongoDB
//MongoDB配置
MongoClient client = new MongoClient("192.168.221.205", 27017);
MongoDatabase db = client.getDatabase("uf");
MongoCollection<Document> uf = db.getCollection("user_friends_md");
//kafka配置
Properties prop_consumer = new Properties();
prop_consumer.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,"192.168.221.221:9092");
prop_consumer.setProperty(ConsumerConfig.GROUP_ID_CONFIG,"xym");
prop_consumer.setProperty(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringDeserializer");
prop_consumer.setProperty(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringDeserializer");
prop_consumer.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG,"earliest");
//创建消费者和被订阅的对象
KafkaConsumer<String, String> consumer = new KafkaConsumer<>(prop_consumer);
TopicPartition tp = new TopicPartition("user_friends_ss", 0);
List<TopicPartition> list = new ArrayList<>();
list.add(tp);
consumer.assign(list);
//开启消费
while (true){
ConsumerRecords<String, String> records = consumer.poll(Duration.ofMillis(1000));
List<Document> docs = new ArrayList<>();
records.forEach(x->{
//创建mongo的消息对象
Document doc = new Document("userid", x.value().split(",", -1)[0]);
doc.append("friendid",x.value().split(",",-1)[1]);
docs.add(doc);
});
//如果数据读完,数组里就是空,插入空就会报错,加个判断
if (docs.size()!=0) {
uf.insertMany(docs);
}
}
5、使用java读取MongoDB的消费
//MongDb配置
MongoClient client = new MongoClient("192.168.221.205", 27017);
MongoDatabase ufdb = client.getDatabase("uf");
MongoCollection<Document> user_friends_md = ufdb.getCollection("user_friends_md");
MongoCursor<Document> iter = user_friends_md.find().iterator();
//连接hdfs,设置hdfs路径,config,登录用户
String path = "hdfs://single:9000";
Configuration conf = new Configuration();
conf.setBoolean("dfs.support.append", true);
conf.set("dfs.client.block.write.replace-datanode-on-failure.policy", "NEVER");
conf.set("dfs.client.block.write.replace-datanode-on-failure.enable", "true");
//用户可以不传,默认当前登录用户名
FileSystem fs = FileSystem.get(URI.create(path), conf, "root");
FSDataOutputStream fsdos = fs.create(new Path("/mongo/mongo.txt"));
//遍历MongoDB,使用正则对格式解析
while (iter.hasNext()) {
String str = iter.next().toJson();
String[] split = str.split("[,|\"]", -1);
fsdos.write((split[10] + "," + split[15] + "\n").getBytes());//调用写入方法写进去,要转为bytes
}
//写完 要释放掉
fsdos.flush();