问题场景: 方法内部多线程调用内部局部变量集合,并发写入数据库,集合线程间隔离(线程调用完毕就需要清空集合),数据不重复写入问题
核心代码片段:
class ThreadSync implements Runnable {
private final String oldEsUrl = String.format("%s", BULK);
private Set<Relation> relation = new HashSet<>();
public ThreadSync(Set<Relation> relation) {
this.relation.addAll(relation);
}
@Override
public void run() {
try {
StringBuilder requestBody = new StringBuilder();
for (Relation real : relation) {
constrctBody(requestBody, real.getRowkey(), null, null, real.getPhone(), real.getMac(), real.getImei(), real.getImsi());
}
EsRestClient.getESDtats(requestBody.toString(), oldEsUrl, EsRestClient.Actions.POST);
logger.info("es 写入 sql:{} path:{}", requestBody.toString(), oldEsUrl);
} catch (Exception e) {
errorLog.error("保存数据库出错:" + relation, e);
}
}
}
业务逻辑调用代码分享
public void execute(String[] args) throws IOException {
// args[0] = "C:\\Users\\Administrator\\Desktop\\fmac.csv";
ExecutorService executorService = Executors.newFixedThreadPool(4);
String file = args[0];
AppContext.getApplicationContext();
long l = System.currentTimeMillis();
LineIterator it = FileUtils.lineIterator(new File(file), "UTF-8");
long skip = args.length == 2 ? Long.parseLong(args[1]) : 0;//跳过多少行
long total = 0, lineCount = 0;//总记录数
try {
//String sql = "INSERT INTO FCODE (AREA_CODE,IMEI_CODE,IMSI_CODE,MAC,PHONE) VALUES(?,?,?,?,?)";
Set<Relation> list = new HashSet<>();
while (it.hasNext()) {
lineCount++;
if (lineCount <= skip) {
continue;
}
String line = it.nextLine().replaceAll(" +|\\t+", " ");
logger.debug("{}: {}", lineCount, line);
String[] arr = StringUtils.split(line, " ");
if (arr.length < 5) {
errorLog.error(lineCount + "-" + "len error:" + line);
continue;
}
if (!arr[1].matches("[a-zA-Z0-9]{1,64}")) {
errorLog.error(lineCount + "-" + "imei error:" + line);
continue;
}
if (!arr[2].matches("\\d{1,64}")) {
arr[2] = "";
logger.warn(lineCount + "-" + "imsi error:" + line);
//continue;
}
if (!arr[3].matches("[a-zA-Z0-9\\-]{1,64}")) {
arr[3] = "";
logger.warn(lineCount + "-" + "mac error:" + line);
//continue;
}
if (!arr[arr.length - 1].matches("\\d{11}")) {
errorLog.error(lineCount + "-" + "phone error:" + line);
continue;
}
list.add(new Relation(arr[1], arr[2], arr[3], arr[arr.length - 1], RowKeyUtils.convertRowKeyToStr(2 + arr[3] + arr[2] + arr[1] + arr[arr.length - 1])));
if (list.size() >= 100000) {
executorService.execute(new ThreadSync(list));
list.clear();
}
}
if (!list.isEmpty()) {
executorService.execute(new ThreadSync(list));
}
} finally {
logger.fatal("处理结束,总共:{}行,用时:{}!", lineCount, total, Utils.prettyTime(System.currentTimeMillis() - l));
LineIterator.closeQuietly(it);
}
}