java处理海量数据的初步解决思路

1. 在开发过程中经常遇到，海量数据处理的问题，最近，在工作中遇到给100W用户同事发送邮件，要求用JAVA去实现，由于数据量比较大，内存和服务器CUP都抗不住，为此必须解决这一些列的问题，于是我初步想出一个解决该问题的方法~
   
        废话少说，晒上代码：
   
      
   

2. package oracle;
   
   import java.sql.Connection;
   import java.sql.ResultSet;
   import java.sql.Statement;
   import java.util.ArrayList;
   import java.util.HashMap;
   import java.util.List;
   import java.util.Map;
   
   public class jdbcHelper {
   
   private static Connection con1, con2;
   private static Statement sta;
   private static ResultSet rs;
   /**
     * 分页结果集
     * @param sql
     * @param start
     * @param end
     * @param id
     * @return
     */
   public static List queryList(String sql, int start, int end, Map params) {
     List result=new ArrayList();
     try {
      con1 = JdbcUtils.getConnection();// 获得数据库连接
      StringBuffer buffer = new StringBuffer();
      buffer.append("select * from (select row_.*, rownum rownum_ from ( ");
      buffer.append(sql);
      buffer.append(" ) row_ where rownum ");
      buffer.append(start);
      if(params.size()>0){
       buffer.append(" and id=").append("'").append(params.get("id")).append("'");
      }
      sta = con1.createStatement();
      rs = sta.executeQuery(buffer.toString());
      while (rs.next()) {
       result.add(rs.getString(1));
      }
     } catch (Exception e) {
      e.printStackTrace();
     } finally {
      JdbcUtils.closeConnection(rs, sta, con1);
     }
     return result;
   }
   /**
     * 总数
     * @param sql
     * @param id
     * @return
     */
   public static int getCount(String sql,Map params){
    
     int count=0;
     try {
      con1 = JdbcUtils.getConnection();// 获得数据库连接
      StringBuffer buffer = new StringBuffer();
      buffer.append(sql);
      if(params.size()>0){
       buffer.append(" where id=").append("'").append(params.get("id")).append("'");
      }
      sta = con1.createStatement();
      rs = sta.executeQuery(buffer.toString());
      while (rs.next()) {
       count=rs.getInt(1);
      }
     } catch (Exception e) {
      e.printStackTrace();
     } finally {
      JdbcUtils.closeConnection(rs, sta, con1);
     }
     return count;
    
   }
   }
   
   上面这段代码主要实现ORCAL数据库分页，目的在于用多少取多少！
   
   package oracle;
   
   import java.util.List;
   import java.util.Map;
   
   /**
    * .
    *
    * @author dongwenhua
    *
    */
   public class PageList {
    private int curPage; // 当前是第几页
    private int maxPage; // 一共有多少页
    private int maxRowCount; // 一共有多少行
    private int rowsPerPage = 10;// 每页多少行
    private Object data;
    private int startPage;
    private int endPage;
    
    public int getStartPage() {
     return startPage;
    }
   
    public void setStartPage(int startPage) {
     this.startPage = startPage;
    }
   
    public int getEndPage() {
     return endPage;
    }
   
    public void setEndPage(int endPage) {
     this.endPage = endPage;
    }
   
    public void setCurPage(int c) {
     this.curPage = c ;
    }
   
    public void setMaxPage() { // 根据总行数计算总页数
     if (this.maxRowCount % this.rowsPerPage == 0) {
      this.maxPage = this.maxRowCount / this.rowsPerPage;
     } else {
      this.maxPage = this.maxRowCount / this.rowsPerPage + 1;
     }
    }
   
    public void setMaxRowCount(int m) {
     this.maxRowCount = m;
    }
   
    public int getCurPage() {
     return this.curPage ;
    }
   
    public int getMaxPage() {
     return this.maxPage;
    }
   
    public int getMaxRowCount() {
     return this.maxRowCount;
    }
   
    
    public int getRowsPerPage() {
     return this.rowsPerPage;
    }
   
    public void setMaxPage(int maxPage) {
     this.maxPage = maxPage;
    }
   
    public void setRowsPerPage(int rowsPerPage) {
     this.rowsPerPage = rowsPerPage;
    }
   
    public Object getData() {
     return data;
    }
   
    public void setData(Object data) {
     this.data = data;
    }
   
    
   }
   这个内处理分页和结果集的封装
   
   package oracle;
   
   import java.util.HashMap;
   import java.util.List;
   import java.util.Map;
   
   
   /**
    * 解决大数据量发邮件问题
    *
    * @author dongwenhua
    *
    */
   public class DataTest {
       private static int no=0;
     public static void main(String[] args) {
     String sql = "select * from test ";
     String count = "select count(1) from test";
     Map params=new HashMap();//存放参数
     int pageSize=10;
     int total= jdbcHelper.getCount(count, params);//总数
     int pageNo=1;
     while(true){
        if (no == total) {
    no=0;   break;
      }
      PageList list = getResultList(sql, count, params, pageNo, pageSize);
      sendMail(list.getData());
      try {
       Thread.sleep(1000);
      } catch (InterruptedException e) {
       e.printStackTrace();
      }
      pageNo++;
      
     }
         System.out.println("other programm is run contiue");
    }
    
    public static PageList getResultList(String querySql,String countSql,Map params, int pageNo,int pageSize){
     PageList page=new PageList();
     page.setCurPage(pageNo);//当前页
     page.setRowsPerPage(pageSize);//每页显示多少条
     // 计数
     int total = jdbcHelper.getCount(countSql, params);
     page.setMaxRowCount(total);
     page.setMaxPage();//计算出共有多少页
     if(pageNo>page.getMaxPage()){
      pageNo = page.getMaxPage();
     }
     int start = (pageNo - 1) * pageSize;
     int end = pageNo * pageSize;
     // 取单页结果集
     List data = jdbcHelper.queryList(querySql, start, end, params);
     page.setData(data);
     return page;
     
    }
   public static void sendMail(Object obj){
    if(obj instanceof List){
     List list=(List)obj;
     for(int i=0;i<list.size();i++){
      System.out.println("发送邮件--:"+list.get(i));
      no++;
     }
     
     System.out.println("--end--");
    }
   }
   }

3.  
4. 递归实现发送邮件