1.
Mongo的实例其实就是一个数据库连接池,这个连接池里默认有10个链接。我们没有必要重新实现这个链接池,但是我们可以更改这个连接池的配置。因为Mongo的实例就是一个连接池,所以,项目中最好只存在一个Mongo的实例。
常见的配置参数:
connectionsPerHost:每个主机的连接数
threadsAllowedToBlockForConnectionMultiplier:线程队列数,它以上面connectionsPerHost值相乘的结果就是线程队列最大值。如果连接线程排满了队列就会抛出“Out of semaphores to get db”错误。
maxWaitTime:最大等待连接的线程阻塞时间
connectTimeout:连接超时的毫秒。0是默认和无限
socketTimeout:socket超时。0是默认和无限
autoConnectRetry:这个控制是否在一个连接时,系统会自动重
2.初始化连接池
package cn.ac.iie;
import java.net.UnknownHostException;
import com.mongodb.DB;
import com.mongodb.Mongo;
import com.mongodb.MongoException;
import com.mongodb.MongoOptions;
public class MongoManager {
/*
connectionsPerHost:每个主机的连接数
threadsAllowedToBlockForConnectionMultiplier:线程队列数,它以上面connectionsPerHost值相乘的结果就是线程队列最大值。如果连接线程排满了队列就会抛出“Out of semaphores to get db”错误。
maxWaitTime:最大等待连接的线程阻塞时间
connectTimeout:连接超时的毫秒。0是默认和无限
socketTimeout:socket超时。0是默认和无限
autoConnectRetry:这个控制是否在一个连接时,系统会自动重试*/
private final static String HOST = "192.168.100.103";// 端口
private final static int PORT = 10111;// 端口
private final static int POOLSIZE = 100;// 连接数量
private final static int BLOCKSIZE = 100; // 等待队列长度
private static Mongo mongo = null;
private final static int MAXWAITTIME = 1000;
private final static int CONNECTTIMEOUT = 2000;
private final static int SOCKETTIMEOUT = 2000;
private final static boolean AUTOCONNECTRETRY = true;
private MongoManager() { }
static {
initDBPrompties();
}
public static DB getDB(String dbName) {
return mongo.getDB(dbName);
}
/**
* 初始化连接池
*/
@SuppressWarnings("deprecation")
private static void initDBPrompties() {
try {
mongo = new Mongo(HOST, PORT);
MongoOptions opt = mongo.getMongoOptions();
opt.connectionsPerHost = POOLSIZE;
opt.threadsAllowedToBlockForConnectionMultiplier = BLOCKSIZE;
opt.autoConnectRetry = AUTOCONNECTRETRY;
opt.connectTimeout = CONNECTTIMEOUT;
opt.socketTimeout = SOCKETTIMEOUT;
} catch (UnknownHostException e) {
} catch (MongoException e) {
}
}
}
3.Mongodb连接池的使用案例1
package cn.ac.iie;
import java.net.UnknownHostException;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Iterator;
import java.util.Map;
import org.bson.types.ObjectId;
import com.mongodb.BasicDBObject;
import com.mongodb.DB;
import com.mongodb.DBCollection;
import com.mongodb.DBCursor;
import com.mongodb.DBObject;
import com.mongodb.Mongo;
import com.mongodb.WriteResult;
public class MongoDB2 {
@SuppressWarnings("deprecation")
public static void main(String[] args) throws ParseException, InterruptedException {
try {
DB db = MongoManager.getDB("MyTestDB");
DBCollection myCollection = db.getCollection("MyCollection");
for(int i =0;i <10;i++){
DBObject myData= new BasicDBObject();
myData.put("_id","1");
myData.put("username","xuguokun");
myData.put("age","24");
DateFormat insertFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
Date date = null;
date = insertFormat.parse(insertFormat.format(new Date()));
myData.put("time",date.getTime()+"");
java.util.Date date1 = new Date(date.getTime());
String sDateTime1 = insertFormat.format(date1);
System.out.println("Insert before:"+sDateTime1);
//向数据库中插入数据
myCollection.insert(myData);
}
//Mongodb查询1,全表检索,没有查询条件
DBCursor cur = myCollection.find();
while(cur.hasNext()){
DBObject dbObject = cur.next();
Map map = dbObject.toMap();
System.out.println(map.size());
Iterator entries = map.entrySet().iterator();
while (entries.hasNext()) {
Map.Entry entry = (Map.Entry) entries.next();
String key = (String)entry.getKey();
String value = (String)entry.getValue();
//System.out.println("Key = " + key + ", Value = " + value);
if(key.equals("time")){
long time = Long.parseLong(value);
SimpleDateFormat selectFormat= new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
java.util.Date dt = new Date(time);
String sDateTime = selectFormat.format(dt); //得到精确到秒的表示:08/31/2006 21:08:00
System.out.println("Insert After:"+sDateTime);
}
}
}
//Mongodb查询2,根据ObjectId进行检索
DBObject dbObject1 = myCollection.findOne(new ObjectId("5608fe2f4cd5756a99003fa4"));
if(dbObject1!=null){
System.out.println("Mongodb查询2,根据ObjectId进行检索:"+dbObject1.toString());
}
//Mongodb查询2
DBObject dbObject2 = new BasicDBObject();
dbObject2.put("_id", "1");
if(myCollection.findOne(dbObject2)!=null){
System.out.println("Mongodb查询2:"+myCollection.findOne(dbObject2).toString());
}
//Mongodb查询3
DBObject dbObject3 = new BasicDBObject();
dbObject3.put("_id", "1");
dbObject3.put("username", "xuguokun");
if(myCollection.findOne(dbObject3)!=null){
System.out.println("Mongodb查询3:"+myCollection.findOne(dbObject3).toString());
}
//Mongodb删除1
DBObject delObject1 = new BasicDBObject();
delObject1.put("_id", "1");
delObject1.put("username", "xuguokun");
WriteResult delWriteResult = myCollection.remove(delObject1);
} catch (Exception e) {
e.printStackTrace();
}
}
}
4.Mongodb连接池使用案例2
package com.test.thread;
import java.net.UnknownHostException;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import com.mongodb.BasicDBObject;
import com.mongodb.DB;
import com.mongodb.DBCollection;
import com.mongodb.DBObject;
import com.mongodb.Mongo;
import com.test.model.NetFlow;
import com.test.utils.ESUtils;
import kafka.consumer.Consumer;
import kafka.consumer.ConsumerConfig;
import kafka.consumer.ConsumerIterator;
import kafka.consumer.KafkaStream;
import kafka.javaapi.consumer.ConsumerConnector;
public class ConsumerThread extends Thread {
//Kafka主题
private String topic;
//Mongodb相关配置
private String dbName;
private String collName;
//ES的主索引名字和索引类型
private String indexName;
private String indexType;
public ConsumerThread(String topic,String indexName,String indexType,String dbName,String collName) {
this.topic = topic;
this.indexName = indexName;
this.indexType = indexType;
this.dbName = dbName;
this.collName = collName;
}
@SuppressWarnings({ "deprecation", "resource" })
@Override
public void run() {
ConsumerConnector consumer = createConsumer();
Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
topicCountMap.put(this.topic, 1); // 一次从主题中获取一个数据
Map<String, List<KafkaStream<byte[], byte[]>>> messageStreams = consumer.createMessageStreams(topicCountMap);
KafkaStream<byte[], byte[]> stream = messageStreams.get(topic).get(0);// 获取每次接收到的这个数据
ConsumerIterator<byte[], byte[]> iterator = stream.iterator();
//Mongo mongo;
@SuppressWarnings("unused")
Client client;
try {
//mongo = new Mongo(this.mongodbIp, this.mongodbPort);
//DB db = mongo.getDB(this.dbName);
DB db = MongoManager.getDB("MyTestDB");
DBCollection dbColl = db.getCollection(this.collName);
client = new TransportClient().addTransportAddress(new InetSocketTransportAddress("10.10.16.253", 9300));
while (iterator.hasNext()) {
String message = new String(iterator.next().message());
System.out.println("write to mongodb:"+message);
String[] recMessage = message.split(",");
for (int j = 0;j < recMessage.length; j++) {
DBObject data4 = new BasicDBObject();
data4.put("_id", recMessage[0]);
data4.put("sourceIp", recMessage[1]);
data4.put("sourcePort", recMessage[2]);
data4.put("destIp", recMessage[3]);
data4.put("destPort", recMessage[4]);
DateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
Date date = null;
date = format.parse(format.format(new Date()));
Calendar cla = Calendar.getInstance();
cla.setTime(date);
cla.add(Calendar.HOUR_OF_DAY, 8);
data4.put("update_time", cla.getTime().getTime());
dbColl.insert(data4);
GetResponse getResponse = client.prepareGet().setIndex(this.indexName).setType(this.indexType).setId(recMessage[0]).execute().actionGet();
String searchResult = getResponse.getSourceAsString();
if(searchResult==null){
//向es中存入
NetFlow netFlow = new NetFlow();
netFlow.setSourceIp(recMessage[1]);
netFlow.setSourcePort(Integer.parseInt(recMessage[2].trim()));
netFlow.setDestIp(recMessage[3]);
netFlow.setDestPort(Integer.parseInt(recMessage[4].trim()));
String jsondata = ESUtils.toJson(netFlow);
IndexResponse indexResponse = client.prepareIndex().setIndex(this.indexName).setType(this.indexType).setId(recMessage[0]).setSource(jsondata).execute().actionGet();
}
if(searchResult!=null){
//什么操作都不做
}
}
}
} catch (Exception e) {
e.printStackTrace();
consumer.shutdown();
}
}
private ConsumerConnector createConsumer() {
Properties properties = new Properties();
properties.put("zookeeper.connect","10.10.16.252:2181,10.10.16.253:2181,10.10.16.249:2181");// 声明zk
properties.put("group.id", "test-consummer-group");
properties.put("serializer.class", "kafka.serializer.StringEncoder");
return Consumer.createJavaConsumerConnector(new ConsumerConfig(properties));
}
public static void main(String[] args) {
new ConsumerThread("flume1","myindex","netflows","mydb","netflows").start();// 使用kafka集群中创建好的主题 test
}
}