谷歌工具类maven:
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>18.0</version>
</dependency>
消费队列线程:
public class LogsConsumer implements Runnable{
/** 日志 **/
private static final Logger LOG = LoggerFactory.getLogger(LogsConsumer.class);
private volatile boolean isRunning = true;
private BlockingQueue queue;
public LogsConsumer(BlockingQueue queue) {
this.queue = queue;
}
/**
* 消费者
*/
public void run() {
while (isRunning) {
try {
//每次取出的数据存放到logsDatas里
List<String> logsDatas = new ArrayList<String>();
//每次到1000条数据才进行入库,等待1分钟,没达到1000条也继续入库
Queues.drain(queue, logsDatas, 1000, 1, TimeUnit.MINUTES);
this.insertLogs(logsDatas);
}catch (Exception e){
LOG.error("LogsConsumer drain queues error:",e);
}
}
}
/**
*批量添加方法
* @param logsDatas
*/
private void insertLogs(List<String> logsDatas) throws Exception {
if (!CollectionUtils.isEmpty(logsDatas)){
Client client = ElasticsearchClientUtil.getClient();
BulkRequestBuilder bulkRequest = client.prepareBulk();
for (String logsData:logsDatas){
bulkRequest.add(client.prepareIndex("statistics", "pvanduv").setSource(logsData));
}
bulkRequest.get();
}
}
}
启动线程:
BlockingQueue queue = BlockingQueueFactory.getQueue();
LogsConsumer consumer1 = new LogsConsumer(queue);
LogsConsumer consumer2 = new LogsConsumer(queue);
ExecutorService service = Executors.newCachedThreadPool();
service.execute(consumer1);
service.execute(consumer2);