一 .背景
使用java爬取所需的数据,使用spark streaming处理数据后,存入数据库(用的mysql,但不推荐,有很大的后遗症),使用web重新展示出来。
二 .代码
1.原先的想法与实现
- 数据库连接池的定义(时间自定义)
package org.com.wh;
import org.apache.commons.dbcp.BasicDataSource;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
/**
* Created by root on 8/2/16.
*/
public class ConnectionPoolTitle {
private static Log log = LogFactory.getLog(ConnectionPoolTitle.class);
private static BasicDataSource bs = null;
public static BasicDataSource getDataSource() throws Exception{
if(bs == null){
bs = new BasicDataSource();
bs.setDriverClassName("com.mysql.jdbc.Driver");
bs.setUrl("jdbc:mysql://master:3306/spark");
bs.setUsername("xxxxx");
bs.setPassword("xxxxx");
bs.setMaxActive(500);//设置数据库最大并发数
bs.setInitialSize(50);//数据库初始化时建立的连接数
bs.setMinIdle(50);//最小空闲连接数
bs.setMaxIdle(500);//数据库最大连接数
bs.setMaxWait(1000);
bs.setMinEvictableIdleTimeMillis(6);//空闲连接6ms后释放
bs.setTimeBetweenEvictionRunsMillis(60*1000);//检测是否有死掉的线程
bs.setTestOnBorrow(true);
}
return bs;
}
public static void shutDownDataSource() throws Exception{
if(bs!=null){
bs.close();
}
}
public static Connection getConnection(){
Connection connection = null;
try {
if (bs != null) {
connection = bs.getConnection();
} else {
connection = getDataSource().getConnection();
}
}catch(SQLException e){
log.error(e.getMessage(),e);
}catch(Exception e){
log.error(e.getMessage(),e);
}
return connection;
}
public static void closeConnection(ResultSet rs, Statement ps, Connection con){
if(rs!=null){
try {
rs.close();
}catch(SQLException e){
log.error("rs is Exception"+e.getMessage(),e);
}
}
if(ps!=null){
try {
ps.close();
}catch (SQLException e){
log.error("ps is Exception"+e.getMessage(),e);
}
}
if(con!=null){
try {
con.close();
}catch (SQLException e){
log.error("con is Exception"+e.getMessage(),e);
}
}
}
}
- 线程池的定义
package org.com.wh;
import scala.Tuple2;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
/**
* Created by root on 8/6/16.
*/
public class ThreadPool {
private static ExecutorService pool = Executors.newFixedThreadPool(20);//定义一个有界线程池,20个线程
private Tuple2<String,In