读取TXT入库并记录行数 下次从上次的行数开始读取

**
 * 读取文件具体操作
 * @author Thinkpad
 *
 */
public class ReadFile 
{
   private static SimpleDateFormat logTimeSDF = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); //日志时间格式
   
   private static String middle_file_path = ""; //中间文件路径
   private static String read_process_path = ""; //读取进度文件路径
   
   //初始化配置文件
   public static void initConfig() throws Exception{
      InputStream in = null;
      try{
           File file = new File("config/config.properties"); //打成jar包时配置
           in = new FileInputStream(file); 
           Properties prop = new Properties();
           prop.load(in);
           
           middle_file_path = prop.getProperty("middle_file_path");
           read_process_path = prop.getProperty("read_process_path");
        } catch(Exception e) {
           throw e;
        }finally{
           if(in != null){
              in.close();
           }
        }
   }
   
   public static String readMiddleFile() throws Exception
   {
      System.out.println("开始读取【能见度中间文件】： " + logTimeSDF.format(System.currentTimeMillis()));
      
      try{
         if("".equals(middle_file_path) || "".equals(read_process_path)){
            initConfig();
         }

         File fileDir = new File(middle_file_path); //中间文件目录
         if(fileDir.exists() && fileDir.isDirectory()){ //中间文件目录是否存在
            File[] fileList = fileDir.listFiles();
            if(fileList.length > 0){ //有中间文件才启动读取流程
               //第一步、读取进度文件
               BufferedReader in = null;
               
               File mfp = new File(read_process_path);
               if(!mfp.exists()){
                  mfp.mkdir();
               }
               
               Map<String, String> readProcessParams = new LinkedHashMap<>(); //文件读取进度信息，保存读取的进度文件信息
               File processFile = new File(read_process_path + "/ReadMiddleFileLog.txt");
               
               if(processFile.exists()){ //文件存在，读取进度信息
                  try{
                     in = new BufferedReader(new InputStreamReader(new FileInputStream(processFile)));
                     String lineText = ""; //每行内容
                     while( (lineText = in.readLine()) != null ){ //读到结束
                        lineText = lineText.trim();

                        String[] processInfo = lineText.split("#PARAMS#"); //以"#PARAMS#"分割
                        String fileName = processInfo[0];
                        readProcessParams.put(fileName, processInfo[1]);
                     }
                  }catch(Exception e){
                     throw e;
                  }finally {
                     if(in != null){
                        in.close(); in = null;
                     }
                  }
               }else{
                  processFile.createNewFile(); //创建文件
               }
      
               //第二步、循环读取目录下中间结果文件
               for(int i = 0, cc = fileList.length; i < cc; i ++){
                  File file = fileList[i];
                  String fileName = file.getName();
                  
                  if(file.isFile() && fileName.indexOf(".txt") != -1){ //只读取txt文件
                     try{
                        String processInfo = readProcessParams.get(fileName); //该文件读取进度信息
                        int readLineNo = 0; //上次读取行号，新文件从第一行开始读
                        if(processInfo != null){ //该文件已经读取
                           String[] params = processInfo.split("#PARAM#");
                           readLineNo = Integer.parseInt(params[0]); //上次读取行号
                        }
                        
                        List<String> insertSqlList = new ArrayList<>(); //入库语句列表
                        StringBuilder insertSql = null; //入库语句
                        int validCount = 0; //符合格式的新数据数量
                        
                        in = new BufferedReader(new InputStreamReader(new FileInputStream(file)));
                        int lineNo = 0; //读取第几行
                        String lineText = ""; //每行内容
                        while( (lineText = in.readLine()) != null ){
                           lineNo ++;
                           
                           if(lineNo > readLineNo){
                              lineText = lineText.trim();
                              //保存内容，在这入库
                              String[] data = lineText.split(",");
                              if(data.length == 8){ //数量随文件格式调整
                                 validCount ++; //符合格式的数据+1
                                 
                                 if(insertSql == null){ //500条记录后重启一个insert
                                    insertSql = 
                                       new StringBuilder("insert into middle_file_data(file_name, col1, col2, col3, col4, col5, col6, col7, col8) values "); 
                                 }
                                 //组织数据
                                 insertSql.append("('" + fileName + "', '" + data[0] + "', '" + data[1] + "', '" + data[2] + "', '" 
                                       + data[3] + "'," + " '" + data[4] + "', '" + data[5] + "', '" + data[6] + "', '" + data[7] + "'),");
                                 
                                 //每500条记录一个insert
                                 if(validCount % 500 == 0){ 
                                    
                                    insertSqlList.add(insertSql.substring(0, insertSql.length() -1)); //去除最后一个“,”
                                    insertSql = null;
                                 }
                              }
                           }
                        }
                        
                        //第二步2、数据入库
                        boolean insertFlag = true; //入库成功标识
                        
                        if(validCount > 0){ 
                           Connection conn = null;
                           Statement batchStat = null;
                           try{
                              if(insertSql != null) //处理最后500余的数据
                              {
                                 insertSqlList.add(insertSql.substring(0, insertSql.length()-1));
                              }
                              
                              conn = CreateConnection.getConn();
                              conn.setAutoCommit(false); //不自动提交
                              
                              batchStat = conn.createStatement();
                              for(String sql : insertSqlList)
                              {
                                 batchStat.addBatch(sql);
                              }
                              batchStat.executeBatch(); //批量添加，保证事物性
                              conn.commit(); //提交                      
                           }catch(Exception e){
                              insertFlag = false; //设置入库失败
                              System.out.println(fileName + " 最新数据入库失败");
                           }finally {
                              CreateConnection.closeConn(null, batchStat, conn);
                           }
                        }/*else{
                           insertFlag 还是true，没有符合格式的新数据，读取进度还要推进
                        }*/

                        if(lineNo > readLineNo && insertFlag){ //有读取新记录，并且入库成功
                           String newProcessInfo = lineNo + "#PARAM#"; //新的读取进度信息
                           readProcessParams.put(fileName, newProcessInfo); //放入进度集合，全部读完后存入进度文件
                        }/*else{
                           集合中原记录不变，和新进度一起保存到进度文件中
                        }*/
                     }catch(Exception e){
                        throw e;
                     }finally {
                        if(in != null){in.close(); in = null;}
                     }
                  }
               }

               //第三步、更新进度文件
               OutputStream out = null;
               try{
                  out = new FileOutputStream(processFile);
                  StringBuilder precessInfoStr = new StringBuilder("");
                  //遍历读取信息集合，集合包含本次读取修改过和未修改的文件读取信息
                  for(Entry<String, String> entry : readProcessParams.entrySet()){
                     String fileName = entry.getKey();
                     String processInf0 = entry.getValue();
                     
                     precessInfoStr.append(fileName + "#PARAMS#" + processInf0 + /*换行*/"\r\n");
                  }
                  out.write(precessInfoStr.toString().getBytes()); //写入文件，覆盖原来记录
               }catch(Exception e){
                  throw e;
               }finally {
                  if(out != null){
                     out.close(); out = null;
                  }
               }
            }
         }
      }catch(Exception e){
         throw e;
      }
      
      System.out.println("读取【能见度中间文件】结束： " + logTimeSDF.format(System.currentTimeMillis()));
      return "success";
   }
}

创立数据库连接

public class CreateConnection {
    
    public static String dialect = "";
    public static String driver = "";
    public static String url = "";
    public static String user = "";
    public static String password = "";

    public static Connection getConn() throws IOException {
        Connection conn = null;
        try {
           if(driver == null || "".equals(driver))
           {
              readDBConfig();
           }
           
            Class.forName(driver);
            conn = DriverManager.getConnection(url, user, password);// 获取连接
        } catch (ClassNotFoundException e) {
            e.printStackTrace();
        } catch (SQLException e) {
            e.printStackTrace();
        }
        return conn;
    }

    public static void readDBConfig() throws IOException
    {
       InputStream in = null;
       try{
           File file = new File("config/config.properties");
           in = new FileInputStream(file); 
           
           Properties prop = new Properties();
           prop.load(in);
           
           driver = prop.getProperty("jdbc.driverClassName");
           url = prop.getProperty("jdbc.url");
           user = prop.getProperty("jdbc.username");
           password = prop.getProperty("jdbc.password");
        } catch(Exception e) {
            System.out.println("读取数据库配置文件失败：");
            e.printStackTrace();
        } finally {
         if(in != null){
            in.close(); in = null;
         }
      }
    }
    
    public static void closeConn(ResultSet result, Statement statement, Connection conn)
    {
       try {
         if(result != null && !result.isClosed()){
            result.close();
         }
      } catch (SQLException e) {
         e.printStackTrace();
      }
      
      try {
         if(statement != null  && !statement.isClosed()){
            statement.close();
         }
      } catch (SQLException e) {
         e.printStackTrace();
      }
       
      try {
         if(conn != null && !conn.isClosed()){
            conn.close();
         }
      } catch (SQLException e) {
         e.printStackTrace();
      }
    }
    
    
}

/**
 * 读取线程
 * @author Thinkpad
 *
 */
public class ReadRunnable implements Runnable
{
   public void run() 
   {
      try{
         ReadFile.readMiddleFile();
      }catch(Exception e){
         System.out.println("【能见度中间文件】读取异常：" + e);
      }catch(Error e){
         System.out.println("【能见度中间文件】读取错误：" + e);
      }
   }
}

/**
 * 读取能见度中间结果文件入口
 * @author Thinkpad
 *
 */
public class MainRead 
{
   public static void main(String[] args) 
   {
      ReadRunnable readRun = new ReadRunnable();
      ScheduledExecutorService schedule = Executors.newSingleThreadScheduledExecutor();
      schedule.scheduleWithFixedDelay(readRun, 5, 15, TimeUnit.SECONDS); //10秒后开始执行，每10分钟执行一次
   }
}