**
* 读取文件具体操作
* @author Thinkpad
*
*/
public class ReadFile
{
private static SimpleDateFormat logTimeSDF = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); //日志时间格式
private static String middle_file_path = ""; //中间文件路径
private static String read_process_path = ""; //读取进度文件路径
//初始化配置文件
public static void initConfig() throws Exception{
InputStream in = null;
try{
File file = new File("config/config.properties"); //打成jar包时配置
in = new FileInputStream(file);
Properties prop = new Properties();
prop.load(in);
middle_file_path = prop.getProperty("middle_file_path");
read_process_path = prop.getProperty("read_process_path");
} catch(Exception e) {
throw e;
}finally{
if(in != null){
in.close();
}
}
}
public static String readMiddleFile() throws Exception
{
System.out.println("开始读取【能见度中间文件】: " + logTimeSDF.format(System.currentTimeMillis()));
try{
if("".equals(middle_file_path) || "".equals(read_process_path)){
initConfig();
}
File fileDir = new File(middle_file_path); //中间文件目录
if(fileDir.exists() && fileDir.isDirectory()){ //中间文件目录是否存在
File[] fileList = fileDir.listFiles();
if(fileList.length > 0){ //有中间文件才启动读取流程
//第一步、读取进度文件
BufferedReader in = null;
File mfp = new File(read_process_path);
if(!mfp.exists()){
mfp.mkdir();
}
Map<String, String> readProcessParams = new LinkedHashMap<>(); //文件读取进度信息,保存读取的进度文件信息
File processFile = new File(read_process_path + "/ReadMiddleFileLog.txt");
if(processFile.exists()){ //文件存在,读取进度信息
try{
in = new BufferedReader(new InputStreamReader(new FileInputStream(processFile)));
String lineText = ""; //每行内容
while( (lineText = in.readLine()) != null ){ //读到结束
lineText = lineText.trim();
String[] processInfo = lineText.split("#PARAMS#"); //以"#PARAMS#"分割
String fileName = processInfo[0];
readProcessParams.put(fileName, processInfo[1]);
}
}catch(Exception e){
throw e;
}finally {
if(in != null){
in.close(); in = null;
}
}
}else{
processFile.createNewFile(); //创建文件
}
//第二步、循环读取目录下中间结果文件
for(int i = 0, cc = fileList.length; i < cc; i ++){
File file = fileList[i];
String fileName = file.getName();
if(file.isFile() && fileName.indexOf(".txt") != -1){ //只读取txt文件
try{
String processInfo = readProcessParams.get(fileName); //该文件读取进度信息
int readLineNo = 0; //上次读取行号,新文件从第一行开始读
if(processInfo != null){ //该文件已经读取
String[] params = processInfo.split("#PARAM#");
readLineNo = Integer.parseInt(params[0]); //上次读取行号
}
List<String> insertSqlList = new ArrayList<>(); //入库语句列表
StringBuilder insertSql = null; //入库语句
int validCount = 0; //符合格式的新数据数量
in = new BufferedReader(new InputStreamReader(new FileInputStream(file)));
int lineNo = 0; //读取第几行
String lineText = ""; //每行内容
while( (lineText = in.readLine()) != null ){
lineNo ++;
if(lineNo > readLineNo){
lineText = lineText.trim();
//保存内容,在这入库
String[] data = lineText.split(",");
if(data.length == 8){ //数量随文件格式调整
validCount ++; //符合格式的数据+1
if(insertSql == null){ //500条记录后重启一个insert
insertSql =
new StringBuilder("insert into middle_file_data(file_name, col1, col2, col3, col4, col5, col6, col7, col8) values ");
}
//组织数据
insertSql.append("('" + fileName + "', '" + data[0] + "', '" + data[1] + "', '" + data[2] + "', '"
+ data[3] + "'," + " '" + data[4] + "', '" + data[5] + "', '" + data[6] + "', '" + data[7] + "'),");
//每500条记录一个insert
if(validCount % 500 == 0){
insertSqlList.add(insertSql.substring(0, insertSql.length() -1)); //去除最后一个“,”
insertSql = null;
}
}
}
}
//第二步2、数据入库
boolean insertFlag = true; //入库成功标识
if(validCount > 0){
Connection conn = null;
Statement batchStat = null;
try{
if(insertSql != null) //处理最后500余的数据
{
insertSqlList.add(insertSql.substring(0, insertSql.length()-1));
}
conn = CreateConnection.getConn();
conn.setAutoCommit(false); //不自动提交
batchStat = conn.createStatement();
for(String sql : insertSqlList)
{
batchStat.addBatch(sql);
}
batchStat.executeBatch(); //批量添加,保证事物性
conn.commit(); //提交
}catch(Exception e){
insertFlag = false; //设置入库失败
System.out.println(fileName + " 最新数据入库失败");
}finally {
CreateConnection.closeConn(null, batchStat, conn);
}
}/*else{
insertFlag 还是true,没有符合格式的新数据,读取进度还要推进
}*/
if(lineNo > readLineNo && insertFlag){ //有读取新记录,并且入库成功
String newProcessInfo = lineNo + "#PARAM#"; //新的读取进度信息
readProcessParams.put(fileName, newProcessInfo); //放入进度集合,全部读完后存入进度文件
}/*else{
集合中原记录不变,和新进度一起保存到进度文件中
}*/
}catch(Exception e){
throw e;
}finally {
if(in != null){in.close(); in = null;}
}
}
}
//第三步、更新进度文件
OutputStream out = null;
try{
out = new FileOutputStream(processFile);
StringBuilder precessInfoStr = new StringBuilder("");
//遍历读取信息集合,集合包含本次读取修改过和未修改的文件读取信息
for(Entry<String, String> entry : readProcessParams.entrySet()){
String fileName = entry.getKey();
String processInf0 = entry.getValue();
precessInfoStr.append(fileName + "#PARAMS#" + processInf0 + /*换行*/"\r\n");
}
out.write(precessInfoStr.toString().getBytes()); //写入文件,覆盖原来记录
}catch(Exception e){
throw e;
}finally {
if(out != null){
out.close(); out = null;
}
}
}
}
}catch(Exception e){
throw e;
}
System.out.println("读取【能见度中间文件】结束: " + logTimeSDF.format(System.currentTimeMillis()));
return "success";
}
}
创立数据库连接
public class CreateConnection {
public static String dialect = "";
public static String driver = "";
public static String url = "";
public static String user = "";
public static String password = "";
public static Connection getConn() throws IOException {
Connection conn = null;
try {
if(driver == null || "".equals(driver))
{
readDBConfig();
}
Class.forName(driver);
conn = DriverManager.getConnection(url, user, password);// 获取连接
} catch (ClassNotFoundException e) {
e.printStackTrace();
} catch (SQLException e) {
e.printStackTrace();
}
return conn;
}
public static void readDBConfig() throws IOException
{
InputStream in = null;
try{
File file = new File("config/config.properties");
in = new FileInputStream(file);
Properties prop = new Properties();
prop.load(in);
driver = prop.getProperty("jdbc.driverClassName");
url = prop.getProperty("jdbc.url");
user = prop.getProperty("jdbc.username");
password = prop.getProperty("jdbc.password");
} catch(Exception e) {
System.out.println("读取数据库配置文件失败:");
e.printStackTrace();
} finally {
if(in != null){
in.close(); in = null;
}
}
}
public static void closeConn(ResultSet result, Statement statement, Connection conn)
{
try {
if(result != null && !result.isClosed()){
result.close();
}
} catch (SQLException e) {
e.printStackTrace();
}
try {
if(statement != null && !statement.isClosed()){
statement.close();
}
} catch (SQLException e) {
e.printStackTrace();
}
try {
if(conn != null && !conn.isClosed()){
conn.close();
}
} catch (SQLException e) {
e.printStackTrace();
}
}
}
/**
* 读取线程
* @author Thinkpad
*
*/
public class ReadRunnable implements Runnable
{
public void run()
{
try{
ReadFile.readMiddleFile();
}catch(Exception e){
System.out.println("【能见度中间文件】读取异常:" + e);
}catch(Error e){
System.out.println("【能见度中间文件】读取错误:" + e);
}
}
}
/**
* 读取能见度中间结果文件入口
* @author Thinkpad
*
*/
public class MainRead
{
public static void main(String[] args)
{
ReadRunnable readRun = new ReadRunnable();
ScheduledExecutorService schedule = Executors.newSingleThreadScheduledExecutor();
schedule.scheduleWithFixedDelay(readRun, 5, 15, TimeUnit.SECONDS); //10秒后开始执行,每10分钟执行一次
}
}