1.现状介绍
先来看下整个离线H5组件构建活动图：
由上图可知，jenkins构建离线H5组件时，是依次执行离线H5组件，直到没有离线H5组件为止。这样，不仅构建耗时长，还效率低。
那有什么办法可以提高效率？相比单线程，可以考虑多线程并发执行构建。
2.解决方案
我们再来看下并发下整个离线H5组件构建活动图：
由上图可知，通过高并发多线程执行离线H5组件构建，实现所有离线H5组件同时构建的场景。注意的是，当向文件追加数据的时候，需要对文件加锁，避免多线程读写该文件导致读写错误问题。
- 线程池
以前当我们每次执行一个任务时用new Thread，频繁创建对象会导致系统性能差，线程缺乏统一管理，可能无限制新建线程，相互之间竞争导致系统耗尽，并且缺乏定时任务，中断等功能。线程池可以有效的提高系统资源的使用率，同时避免过多资源竞争，重用存在的线程，减少对象创建。
ExecutorService提供了管理终止的方法，以及可为跟踪一个或多个异步任务执行状况而生成 Future 的方法。可以关闭 ExecutorService，这将导致其拒绝新任务。
线程池客户端类：NpmBuildClient.java
import hudson.Launcher;
import hudson.model.BuildListener;
import hudson.model.AbstractBuild;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
public class NpmBuildClient {
private ExecutorService executorService;
private List<Future<String>> futureResultList = new ArrayList<Future<String>>();
private List<NpmDTO> npmDTOList = new ArrayList<NpmDTO>();
private AbstractBuild<?, ?> build;
private Launcher launcher;
private BuildListener listener;
public NpmBuildClient(AbstractBuild<?, ?> build, Launcher launcher, BuildListener listener) {
super();
this.build = build;
this.launcher = launcher;
this.listener = listener;
this.executorService = Executors.newCachedThreadPool();
}
public static void main(String[] args) {
NpmDTO npmDTO = new NpmDTO();
npmDTO.setBizName("bizName");
npmDTO.setNamespace("namespace");
npmDTO.setNpm("E:\\app\\components.json");
npmDTO.setH5Version("E:\\app\\components1.json");
NpmDTO npmDTO2 = new NpmDTO();
npmDTO2.setBizName("bizName2");
npmDTO2.setNamespace("namespace2");
npmDTO2.setNpm("E:\\app\\components.json");
npmDTO2.setH5Version("E:\\app\\components1.json");
NpmDTO npmDTO3 = new NpmDTO();
npmDTO3.setBizName("h5111601");
npmDTO3.setNamespace("com.nd.sdp.component.h5111601");
npmDTO3.setNpm("E:\\app\\components.json");
npmDTO3.setH5Version("E:\\app\\components1.json");
NpmBuildClient npmBuildClient = new NpmBuildClient(null, null, null);
npmBuildClient.addData(npmDTO);
npmBuildClient.addData(npmDTO2);
npmBuildClient.addData(npmDTO3);
for (int i = 0; i < 3; i++) {
NpmDTO npmDTO4 = new NpmDTO();
npmDTO4.setBizName("h5111601" + i);
npmDTO4.setNamespace("com.nd.sdp.component.h5111601" + i);
npmDTO4.setNpm("E:\\app\\components.json");
npmDTO4.setH5Version("E:\\app\\components1.json");
npmBuildClient.addData(npmDTO4);
}
npmBuildClient.start();
}
public void addData(NpmDTO npmDTO) {
npmDTOList.add(npmDTO);
}
public void start() {
for (NpmDTO npmDTO : npmDTOList) {
// 使用ExecutorService执行Callable类型的任务，并将结果保存在future变量中
Future<String> future = executorService.submit(new NpmBuildCallable(build, launcher, listener, npmDTO));
// 将任务执行结果存储到List中
futureResultList.add(future);
}
//遍历任务的结果
for (Future<String> futureResult : futureResultList) {
try {
while (!futureResult.isDone());// Future返回如果没有完成，则一直循环等待，直到Future返回完成
System.out.println(futureResult.get());// 打印各个线程（任务）执行的结果
} catch (InterruptedException e){
e.printStackTrace();
} catch (ExecutionException e){
e.printStackTrace();
}
}
close();
}
private void close() {
executorService.shutdown();
}
}任务类：NpmBuildCallable.java
import hudson.Launcher;
import hudson.model.BuildListener;
import hudson.model.AbstractBuild;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.io.UnsupportedEncodingException;
import java.nio.channels.FileChannel;
import java.nio.channels.FileLock;
import java.util.concurrent.Callable;
import org.apache.commons.lang3.StringUtils;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.nd.sdp.portal.jenkins.factory.local.h5.model.NpmDTO;
import com.nd.sdp.portal.jenkins.factory.local.h5.service.LocalH5Service;
public class NpmBuildCallable implements Callable<String> {
private AbstractBuild<?, ?> build;
private Launcher launcher;
private BuildListener listener;
private NpmDTO npmDTO;
public NpmBuildCallable(AbstractBuild<?, ?> build, Launcher launcher, BuildListener listener, NpmDTO npmDTO) {
super();
this.build = build;
this.launcher = launcher;
this.listener = listener;
this.npmDTO = npmDTO;
}
public String call() throws IOException, InterruptedException {
test(npmDTO.getNamespace(), npmDTO.getBizName(), npmDTO.getNpm(), npmDTO.getH5Version());
return "SUCCESS";
}
private void test(String namespace, String bizName, String npm, String h5Version) throws UnsupportedEncodingException, IOException, InterruptedException {
appendComponentsFileData(npm, namespace, bizName);
appendFileData(h5Version, namespace, bizName);
}
private void appendFileData(String h5Version, String namespace, String bizName) throws UnsupportedEncodingException, IOException, InterruptedException {
String componentsContent = FileUtil.readFileToString(new File(h5Version),System.out);
JSONArray componentsJsonArr = JSONArray.parseArray(componentsContent);
for (Object componentsJson : componentsJsonArr) {
if (!(componentsJson instanceof JSONObject)) {
continue;
}
JSONObject components = (JSONObject) componentsJson;
JSONObject componentJson = components.getJSONObject("component");
String name_space = componentJson.getString("namespace");
String name = componentJson.getString("name");
if(StringUtils.equals(namespace, name_space) && StringUtils.equals(bizName, name)) {
JSONObject localH5Json = components.getJSONObject("local-h5");
localH5Json.put("host", namespace);
localH5Json.put("build_time", bizName);
break;
}
}
System.out.println(componentsJsonArr);
FileUtil.write(new File(h5Version), componentsJsonArr.toJSONString(), "UTF-8", false,System.out);
}
private void appendComponentsFileData(String npm, String namespace, String bizName) throws UnsupportedEncodingException, IOException, InterruptedException {
String componentsContent = FileUtil.readFileToString(new File(npm),System.out);
JSONArray componentsJsonArr = JSONArray.parseArray(componentsContent);
for (Object componentsJson : componentsJsonArr) {
if (!(componentsJson instanceof JSONObject)) {
continue;
}
JSONObject components = (JSONObject) componentsJson;
JSONObject componentJson = components.getJSONObject("component");
String name_space = componentJson.getString("namespace");
String name = componentJson.getString("name");
if(StringUtils.equals(namespace, name_space) && StringUtils.equals(bizName, name)) {
JSONObject localH5Json = components.getJSONObject("local-h5");
localH5Json.put("host", namespace);
localH5Json.put("build_time", bizName);
break;
}
}
System.out.println(componentsJsonArr);
FileUtil.write(new File(npm), componentsJsonArr.toJSONString(), "UTF-8", false,System.out);
}
}
文件操作类：FileUtil.java
import java.io.RandomAccessFile;
import java.io.UnsupportedEncodingException;
import java.nio.channels.FileChannel;
import java.nio.channels.FileLock;
public class FileUtil {
/**
* <p>Description: 高并发写文件</p>
* @param file
* @param data
* @param encoding
* @param append
* @throws UnsupportedEncodingException
* @throws IOException
* @throws InterruptedException
*/
public static void write(File file, String data, String encoding, boolean append, PrintStream printStream) throws UnsupportedEncodingException, IOException, InterruptedException {
if(!file.exists()) {
file.createNewFile();
}
//对该文件加锁
RandomAccessFile out = new RandomAccessFile(file, "rw");
FileChannel fcout = out.getChannel();
FileLock flout = null;
while(true) {
try {
flout = fcout.tryLock();// 独占锁， 其他线程不可读也不可写
printStream.println(Thread.currentThread().getName()+" get write lock: " + file.getName());
// Thread.sleep(1000);
// logger.info(Thread.currentThread().getName()+" get write lock: " + file.getName());
break;
} catch (Exception e) {
// logger.info(Thread.currentThread().getName()+" is block: " + file.getName());
printStream.println(Thread.currentThread().getName()+" is block: " + file.getName());
Thread.sleep(1000);
}
}
if(append) {
out.seek(out.length());// 将文件的读写指针定位到文件的末尾
} else {
fcout.truncate(0);// 清空文件
}
out.write(data.getBytes(encoding));
flout.release();
printStream.println(Thread.currentThread().getName()+" release write lock: " + file.getName());
out.close();
fcout.close();
}
/**
* <p>Description: 高并发读文件</p>
* @param file
* @param printStream
* @return
* @throws InterruptedException
* @throws UnsupportedEncodingException
* @throws IOException
*/
public static String readFileToString(File file, PrintStream printStream) throws InterruptedException, UnsupportedEncodingException, IOException {
//给该文件加锁
RandomAccessFile fis = new RandomAccessFile(file, "rw");
FileChannel fcin = fis.getChannel();
FileLock flin = null;
while(true){
try {
// flin = fcin.tryLock(0,Long.MAX_VALUE, true);// 不断的请求锁，如果请求不到，等1秒再请求 （共享锁：其他线程可读但不可写）
flin = fcin.tryLock();
printStream.println(Thread.currentThread().getName()+" get read lock: " + file.getName());
// logger.info(Thread.currentThread().getName()+"get read lock: " + file.getName());
// Thread.sleep(1000);
break;
} catch (Exception e) {
// 如果是同一进程的多线程，重复请求tryLock()会抛出异常
// printStream.println(Thread.currentThread().getName()+" is block: " + file.getName());
Thread.sleep(1000);
}
}
byte[] buf = new byte[(int) fis.length()];
StringBuffer sb = new StringBuffer();
while((fis.read(buf))!=-1){
sb.append(new String(buf,"utf-8"));
}
flin.release();// 释放锁
printStream.println(Thread.currentThread().getName()+" release read lock: " + file.getName());
fis.close();
fcin.close();
return sb.toString();
}
}
原来components.json文件内容：
[
{
"component": {
"name": "bizName",
"namespace": "namespace"
},
"local-h5": {
"build_time": 123,
"host": "",
"npm": "123"
},
"type": [
"local-h5"
]
},
{
"component": {
"name": "bizName2",
"namespace": "namespace2"
},
"local-h5": {
"build_time": 123,
"host": "",
"npm": "123"
},
"type": [
"local-h5"
]
},
{
"component": {
"name": "h5111601",
"namespace": "com.nd.sdp.component.h5111601"
},
"local-h5": {
"build_time": 123,
"host": "",
"npm": "123"
},
"type": [
"local-h5"
]
}
]原来components1.json文件内容：
[
{
"component": {
"name": "bizName",
"namespace": "namespace"
},
"local-h5": {
"build_time": 345,
"host": "",
"npm": "345"
},
"type": [
"local-h5"
]
},
{
"component": {
"name": "bizName2",
"namespace": "namespace2"
},
"local-h5": {
"build_time": 345,
"host": "",
"npm": "345"
},
"type": [
"local-h5"
]
},
{
"component": {
"name": "h5111601",
"namespace": "com.nd.sdp.component.h5111601"
},
"local-h5": {
"build_time": 345,
"host": "",
"npm": "345"
},
"type": [
"local-h5"
]
}
]测试：
public static void main(String[] args) {
NpmDTO npmDTO = new NpmDTO();
npmDTO.setBizName("bizName");
npmDTO.setNamespace("namespace");
npmDTO.setNpm("E:\\app\\components.json");
npmDTO.setH5Version("E:\\app\\components1.json");
NpmDTO npmDTO2 = new NpmDTO();
npmDTO2.setBizName("bizName2");
npmDTO2.setNamespace("namespace2");
npmDTO2.setNpm("E:\\app\\components.json");
npmDTO2.setH5Version("E:\\app\\components1.json");
NpmDTO npmDTO3 = new NpmDTO();
npmDTO3.setBizName("h5111601");
npmDTO3.setNamespace("com.nd.sdp.component.h5111601");
npmDTO3.setNpm("E:\\app\\components.json");
npmDTO3.setH5Version("E:\\app\\components1.json");
NpmBuildClient npmBuildClient = new NpmBuildClient(null, null, null);
npmBuildClient.addData(npmDTO);
npmBuildClient.addData(npmDTO2);
npmBuildClient.addData(npmDTO3);
for (int i = 0; i < 3; i++) {
NpmDTO npmDTO4 = new NpmDTO();
npmDTO4.setBizName("h5111601" + i);
npmDTO4.setNamespace("com.nd.sdp.component.h5111601" + i);
npmDTO4.setNpm("E:\\app\\components.json");
npmDTO4.setH5Version("E:\\app\\components1.json");
npmBuildClient.addData(npmDTO4);
}
npmBuildClient.start();
}测试结果：
pool-1-thread-2 get read lock: components.json
pool-1-thread-2 release read lock: components.json
pool-1-thread-2 get write lock: components.json
pool-1-thread-2 release write lock: components.json
pool-1-thread-2 get read lock: components1.json
pool-1-thread-2 release read lock: components1.json
pool-1-thread-2 get write lock: components1.json
pool-1-thread-2 release write lock: components1.json
pool-1-thread-4 get read lock: components.json
pool-1-thread-4 release read lock: components.json
pool-1-thread-4 get write lock: components.json
pool-1-thread-4 release write lock: components.json
pool-1-thread-4 get read lock: components1.json
pool-1-thread-4 release read lock: components1.json
pool-1-thread-4 get write lock: components1.json
pool-1-thread-4 release write lock: components1.json
pool-1-thread-5 get read lock: components.json
pool-1-thread-5 release read lock: components.json
pool-1-thread-5 get write lock: components.json
pool-1-thread-5 release write lock: components.json
pool-1-thread-5 get read lock: components1.json
pool-1-thread-5 release read lock: components1.json
pool-1-thread-5 get write lock: components1.json
pool-1-thread-5 release write lock: components1.json
pool-1-thread-1 get read lock: components.json
pool-1-thread-1 release read lock: components.json
pool-1-thread-1 get write lock: components.json
pool-1-thread-1 release write lock: components.json
pool-1-thread-1 get read lock: components1.json
pool-1-thread-1 release read lock: components1.json
pool-1-thread-1 get write lock: components1.json
pool-1-thread-1 release write lock: components1.json
SUCCESS
SUCCESS
pool-1-thread-6 get read lock: components.json
pool-1-thread-6 release read lock: components.json
pool-1-thread-6 get write lock: components.json
pool-1-thread-6 release write lock: components.json
pool-1-thread-6 get read lock: components1.json
pool-1-thread-6 release read lock: components1.json
pool-1-thread-6 get write lock: components1.json
pool-1-thread-6 release write lock: components1.json
pool-1-thread-3 get read lock: components.json
pool-1-thread-3 release read lock: components.json
pool-1-thread-3 get write lock: components.json
pool-1-thread-3 release write lock: components.json
pool-1-thread-3 get read lock: components1.json
pool-1-thread-3 release read lock: components1.json
pool-1-thread-3 get write lock: components1.json
pool-1-thread-3 release write lock: components1.json
SUCCESS
SUCCESS
SUCCESS
SUCCESS通过控制台日志，可以看到当线程获取到读锁时，其他线程会等待且不断请求该读锁；当该线程释放读锁，接着获取写锁，然后再释放锁时其他线程才能获取到该读写锁。好像没啥毛病这逻辑，后来我发现我错了。如果线程在释放读锁后获取写锁前，这个业务处理时间有点长，那么该线程在还没有获取写锁前，已经被其他线程获取到该写锁；那么其他线程读取到的文件内容就不是该线程准备写操作的文件内容了。这样会导致每个线程只会去更新它自己的内容，无法保证所有线程运行后，文件内容被全部更新。
怎么办？其实可以在每个线程释放读锁前，进行文件内容更新操作，然后再释放锁出来给其他线程进行读写操作。这样问题不就解决了。哈哈...
修改后的任务类：NpmBuildCallable.java
import hudson.Launcher;
import hudson.model.BuildListener;
import hudson.model.AbstractBuild;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.io.UnsupportedEncodingException;
import java.nio.channels.FileChannel;
import java.nio.channels.FileLock;
import java.util.concurrent.Callable;
import org.apache.commons.lang3.StringUtils;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.nd.sdp.portal.jenkins.factory.local.h5.model.NpmDTO;
public class NpmBuildCallable implements Callable<String> {
private AbstractBuild<?, ?> build;
private Launcher launcher;
private BuildListener listener;
private NpmDTO npmDTO;
public NpmBuildCallable(AbstractBuild<?, ?> build, Launcher launcher, BuildListener listener, NpmDTO npmDTO) {
super();
this.build = build;
this.launcher = launcher;
this.listener = listener;
this.npmDTO = npmDTO;
}
public String call() throws IOException, InterruptedException {
test(npmDTO.getNamespace(), npmDTO.getBizName(), npmDTO.getNpm(), npmDTO.getH5Version());
return "SUCCESS";
}
private void test(String namespace, String bizName, String npm, String h5Version) throws UnsupportedEncodingException, IOException, InterruptedException {
appendComponentsFileData(npm, namespace, bizName);
appendFileData(h5Version, namespace, bizName);
}
private void appendFileData(String h5Version, String namespace, String bizName) throws UnsupportedEncodingException, IOException, InterruptedException {
//给该文件加锁
RandomAccessFile randomAccessFile = new RandomAccessFile(new File(h5Version), "rw");
FileChannel fileChannel = randomAccessFile.getChannel();
FileLock fileLock = null;
while(true){
try {
fileLock = fileChannel.tryLock();
break;
} catch (Exception e) {
Thread.sleep(1000);
}
}
byte[] buf = new byte[(int) randomAccessFile.length()];
StringBuffer sb = new StringBuffer();
while((randomAccessFile.read(buf))!=-1){
sb.append(new String(buf,"utf-8"));
}
String componentsContent = sb.toString();
JSONArray componentsJsonArr = JSONArray.parseArray(componentsContent);
for (Object componentsJson : componentsJsonArr) {
if (!(componentsJson instanceof JSONObject)) {
continue;
}
JSONObject components = (JSONObject) componentsJson;
JSONObject componentJson = components.getJSONObject("component");
String name_space = componentJson.getString("namespace");
String name = componentJson.getString("name");
if(StringUtils.equals(namespace, name_space) && StringUtils.equals(bizName, name)) {
JSONObject localH5Json = components.getJSONObject("local-h5");
localH5Json.put("host", namespace);
localH5Json.put("build_time", bizName);
break;
}
}
fileChannel.truncate(0);// 清空文件
randomAccessFile.write(componentsJsonArr.toJSONString().getBytes("UTF-8"));
fileLock.release();
randomAccessFile.close();
fileChannel.close();
}
private void appendComponentsFileData(String npm, String namespace, String bizName) throws UnsupportedEncodingException, IOException, InterruptedException {
//给该文件加锁
RandomAccessFile randomAccessFile = new RandomAccessFile(new File(npm), "rw");
FileChannel fileChannel = randomAccessFile.getChannel();
FileLock fileLock = null;
while(true){
try {
fileLock = fileChannel.tryLock();
break;
} catch (Exception e) {
Thread.sleep(1000);
}
}
byte[] buf = new byte[(int) randomAccessFile.length()];
StringBuffer sb = new StringBuffer();
while((randomAccessFile.read(buf))!=-1){
sb.append(new String(buf,"utf-8"));
}
String componentsContent = sb.toString();
JSONArray componentsJsonArr = JSONArray.parseArray(componentsContent);
for (Object componentsJson : componentsJsonArr) {
if (!(componentsJson instanceof JSONObject)) {
continue;
}
JSONObject components = (JSONObject) componentsJson;
JSONObject componentJson = components.getJSONObject("component");
String name_space = componentJson.getString("namespace");
String name = componentJson.getString("name");
if(StringUtils.equals(namespace, name_space) && StringUtils.equals(bizName, name)) {
JSONObject localH5Json = components.getJSONObject("local-h5");
localH5Json.put("host", namespace);
localH5Json.put("build_time", bizName);
break;
}
}
fileChannel.truncate(0);// 清空文件
randomAccessFile.write(componentsJsonArr.toJSONString().getBytes("UTF-8"));
fileLock.release();
randomAccessFile.close();
fileChannel.close();
}
}
通过RandomAccessFile实现非阻塞式的读写，通过锁机制完成高并发下的文件的操作。
1137
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
