多线程从数据库中分页读,单线程将所有结果保存的数据库


public class OrderReadThread implements Runnable {

private List<Map<String, Object>> orderList;
private BaseDalClient dalClient;
private Map<String, Object> params;
private int totalRecords;
private CyclicBarrier cyclicBarrier;

@SuppressWarnings({ "rawtypes", "unchecked" })
@Override
public void run() {
// 每个线程分页查询,最终将结果放入公共的orderList
Page<?> pageInfo = dalClient.queryForListPage("eps.UserRegisterServiceImpl.selectOrderList", params,
totalRecords);
List tempList = pageInfo.getList();

orderList.addAll(tempList);
try {
// 子线程执行结束,等待其他未执行完的线程,如果所有线程都执行结束即执行同步线程
cyclicBarrier.await();
} catch (InterruptedException e) {
e.printStackTrace();
} catch (BrokenBarrierException e) {
e.printStackTrace();
}
}

public void setOrderList(List<Map<String, Object>> orderList) {
this.orderList = orderList;
}

public void setDalClient(BaseDalClient dalClient) {
this.dalClient = dalClient;
}

public void setParams(Map<String, Object> params) {
this.params = params;
}

public void setTotalRecords(int totalRecords) {
this.totalRecords = totalRecords;
}

public void setCyclicBarrier(CyclicBarrier cyclicBarrier) {
this.cyclicBarrier = cyclicBarrier;
}

}




public class FileWriteTaskThread implements Runnable {

private List<Map<String, Object>> orderList;

public FileWriteTaskThread(List<Map<String, Object>> orderList) {
this.orderList = orderList;
}

@Override
public void run() {
System.out.println("****************将所有子线程统计的最终的结果保存到文件中F*******************");

String outFileName = "order" + getCurrentTimeStr() + ".txt";
String outFilePath = "d:/order/" + outFileName;
BufferedWriter bw = null;
try {
bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(outFilePath), "UTF-8"));
for (Map<String, Object> order : orderList) {
StringBuilder line = new StringBuilder(200);

line.append(MapUtils.getString(order, "b2corderno", "")).append(",")
.append(MapUtils.getString(order, "phonenum", "")).append(",")
.append(MapUtils.getString(order, "customername", "")).append(",")
.append(MapUtils.getString(order, "customeraddress", "")).append(",")
.append(MapUtils.getString(order, "suppliercode", ""));

bw.write(line.toString());
bw.newLine();
}
} catch (Exception e) {
e.printStackTrace();
} finally {
if (bw != null) {
IOUtils.closeQuietly(bw);
}
}

}

private String getCurrentTimeStr() {
Date d = new Date();
SimpleDateFormat format = new SimpleDateFormat("yyyyMMddHHmmss");
return format.format(d);
}

}




/**
* 每次查询1000条
*/
private static final int PERCENT_DOWNLOAD_NUM = 1000;

public static void main(String[] args) {
// 启动Spring容器
ApplicationContext context = new FileSystemXmlApplicationContext(
"file:src/main/webapp/WEB-INF/applicationContext.xml");
// 获取数据库操作类
BaseDalClient dalClient = context.getBean("dalClient", BaseDalClient.class);
// 查询总记录数
int totalRecords = dalClient
.queryForObject("eps.UserRegisterServiceImpl.selectOrderCount", null, Integer.class);

int temp = totalRecords / PERCENT_DOWNLOAD_NUM;
int queryCount = (totalRecords % PERCENT_DOWNLOAD_NUM) != 0 ? temp + 1 : temp;

List<Map<String, Object>> orderList = Collections.synchronizedList(new ArrayList<Map<String, Object>>());// 线程安全的List
// 同步辅助类,当所有子线程全部执行结束后来执行不同辅助类指定的同步线程FileWriteTaskThread
CyclicBarrier cyclicBarrier = new CyclicBarrier(queryCount, new FileWriteTaskThread(orderList));
for (int i = 1; i <= queryCount; i++) {
Map<String, Object> params = new HashMap<String, Object>();
params.put("pageSize", PERCENT_DOWNLOAD_NUM);
params.put("page", i);

OrderReadThread orderReadThread = new OrderReadThread();
orderReadThread.setDalClient(dalClient);
orderReadThread.setOrderList(orderList);
orderReadThread.setParams(params);
orderReadThread.setTotalRecords(totalRecords);
orderReadThread.setCyclicBarrier(cyclicBarrier);

// 启动多线程来从数据库中分页读取数据
Thread thread = new Thread(orderReadThread);
thread.start();
}
}
  • 0
    点赞
  • 0
    收藏
    觉得还不错? 一键收藏
  • 0
    评论
您好!对于查询数据库百万条数据的情况,使用多线程和数据库分页查询是一种常见的优化策略。下面是一个基本的实现思路: 1. 多线程处理:将数据查询任务分配给多个线程并行执行,提高查询效率。可以使用Java的线程池来管理线程,例如使用`ExecutorService`和`Callable`接口。 2. 数据库分页查询:通过分页查询的方式减少单次查询的数据量,避免一次性查询大量数据导致性能问题。可以使用SQL语句的`LIMIT`子句来实现分页查询,例如`SELECT * FROM table_name LIMIT offset, limit`,其`offset`表示偏移量,`limit`表示每页查询的数据量。 下面是一个简单的示例代码: ```java import java.sql.*; import java.util.concurrent.*; public class DatabaseQuery { private static final int PAGE_SIZE = 100; // 每页查询的数据量 public static void main(String[] args) { ExecutorService threadPool = Executors.newFixedThreadPool(10); // 创建线程池 Connection connection = null; try { connection = DriverManager.getConnection("jdbc:mysql://localhost:3306/database_name", "username", "password"); int totalRows = getTotalRows(connection); // 获取数据总行数 int totalPages = (int) Math.ceil((double) totalRows / PAGE_SIZE); // 计算总页数 for (int page = 0; page < totalPages; page++) { int offset = page * PAGE_SIZE; threadPool.submit(new QueryTask(connection, offset)); } } catch (SQLException e) { e.printStackTrace(); } finally { if (connection != null) { try { connection.close(); } catch (SQLException e) { e.printStackTrace(); } } threadPool.shutdown(); } } private static int getTotalRows(Connection connection) throws SQLException { try (Statement statement = connection.createStatement(); ResultSet resultSet = statement.executeQuery("SELECT COUNT(*) FROM table_name")) { resultSet.next(); return resultSet.getInt(1); } } private static class QueryTask implements Callable<Void> { private Connection connection; private int offset; public QueryTask(Connection connection, int offset) { this.connection = connection; this.offset = offset; } @Override public Void call() throws Exception { try (PreparedStatement statement = connection.prepareStatement("SELECT * FROM table_name LIMIT ?, ?")) { statement.setInt(1, offset); statement.setInt(2, PAGE_SIZE); ResultSet resultSet = statement.executeQuery(); // 处理查询结果 while (resultSet.next()) { // 处理每条数据 // ... } resultSet.close(); } return null; } } } ``` 以上示例代码仅供参考,具体的实现需要根据实际情况进行调整和优化。同时,请确保在使用多线程和数据库查询时遵循相关的线程安全和数据库事务处理的规范。
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值