线程模型
- 生产者
Provider线程为一，主要进行深搜目录文件；、 - 消费者
Consumer线程多个， 因为RPC服务调用时延较长， 启用多个线程请求服务。 - 持久化线程
Persist将已经消费的消息存放在writeQueue， 启用一个线程从writeQueue取数据进行持久化到log.pic，这样每次启动压缩的时候，可以避免重复消费。 进而避免同一目录进行多次压缩 api_key.properties为申请的https://tinypng.com/ 的key ，每个key一个月可以压缩500张， 采用线程名的hashCode对key的个数取模运算，选择所要使用的key。
可扩展性
doCompress方法可以进行任意业务逻辑。只是我的实现是用来压缩图片了。
代码已push到github， 已经打好的jar也已上传。需要的可以clone一下。源码已打入jar包
github clone地址：https://github.com/cbamls/tinypngPicCompress
如何使用
1、下载Compress.jar包
2、maven手动安装jar
mvn install:install-file -DgroupId=com.hearglobal -DartifactId=multi -Dversion=2.0 -Dpackaging=jar -Dfile=C:/Users/cbam/Desktop/Compress.jar
3、公共接口说明：
compress.setApi_key_location("/api_key.properties"); // 指定api_key文件位置 key需要申请 可选设置 默认为项目根路径 如E:/work/{project}/api_key.properties或/work/{project}/api_key.properties
compress.setPic_log_location("/log.pic"); //指定log.pic路径 可选配置 默认为项目根路径 如/work/{project}/log.pic 或 /work/{project}/log.pic
compress.compress("E:/upan/test", "E:/upan/test_bak", 3);//压缩调用 第一个参数为要压缩的目录 第二个参数为 压缩输出目录 第三个参数 启动的线程数
3、程序调用jar示例一
建议指定绝对路径
public static void main(String[] args) throws Exception {
Starter compress = new Starter();
compress.setApi_key_location("/api_key.properties");
compress.setPic_log_location("/log.pic");
compress.compress("E:/upan/test", "E:/upan/test_bak", 3);
}
4、程序调用jar示例二
打开win控制台或 在Linux shell下jar包当前目录 运行如下命令
java -jar Compress.jar E:/upan/test E:/upan/test_bak 3 //压缩调用 第一个参数为要压缩的目录 第二个参数为 压缩输出目录 第三个参数 启动的线程数
运行截图：
注意：此方式无法设置配置文件位置， 故请将文件api_key.properties默认放置在 jar包所在目录
生产者Provider代码：
package com.hearglobal.multi;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.util.LinkedList;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Callable;
import java.util.concurrent.TimeUnit;
/**
* CopyRright (c)2014-2016 Haerbin Hearglobal Co.,Ltd
* Project: tinypngThread
* Comments:
* Author:cbam
* Create Date:2017/3/21
* Modified By:
* Modified Date:
* Modified Reason:
*/
public class Provider implements Callable{
private static final Logger LOGGER = LoggerFactory.getLogger(Consumer.class);
//消息队列
private BlockingQueue<File> messageQueue;
private static LinkedList<String> logList ;
//生产东西来源
private static File src;
/**
* Instantiates a new Provider.
*
* @param messageQueue the message queue
* @param src the src
*/
public Provider(BlockingQueue messageQueue, File src){
this.messageQueue = messageQueue;
this.src = src;
}
public Object call() throws Exception {
try {
load(src);
} catch (InterruptedException e) {
e.printStackTrace();
return false;
} finally {
return true;
}
}
private void load(File src) throws InterruptedException {
LOGGER.info("Provider load src:{}", src.getAbsolutePath());
// 当找到目录时，创建目录
if (src.isDirectory()) {
if(!logList.contains(src.getAbsolutePath())) {
if(!this.messageQueue.offer(src, 2, TimeUnit.SECONDS)) {
System.out.println("目录提交队列失败....");
};
}
File[] files = src.listFiles();
for(File file : files) {
load(file);
}
//当找到文件时
} else if (src.isFile()) {
if(validatePic(src) && !logList.contains(src.getAbsolutePath())) {
if(!this.messageQueue.offer(src, 2, TimeUnit.SECONDS)) {
System.out.println("文件提交队列失败....");
}
}
}
}
private boolean validatePic(File file) {
int loc = file.getAbsolutePath().lastIndexOf(".");
String suffix = file.getAbsolutePath().substring(++loc);
return suffix.equals("jpg") || suffix.equals("png");
}
/**
* Sets log list.
*
* @param logList the log list
*/
public static void setLogList(LinkedList<String> logList) {
Provider.logList = logList;
}
}
消费者Consumer代码：
package com.hearglobal.multi;
import com.tinify.Options;
import com.tinify.Source;
import com.tinify.Tinify;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.util.List;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
/**
* CopyRright (c)2014-2016 Haerbin Hearglobal Co.,Ltd
* Project: tinypngThread
* Comments:
* Author:cbam
* Create Date:2017/3/21
* Modified By:
* Modified Date:
* Modified Reason:
*/
public class Consumer implements Runnable{
private static final Logger LOGGER = LoggerFactory.getLogger(Consumer.class);
//多线程间是否启动变量，有强制从主内存中刷新的功能。即时返回线程的状态
private volatile boolean isRunning = true;
private static String srcBase;
private static String destBase;
//已消费队列
private BlockingQueue<String> writeQueue;
//消息队列
private BlockingQueue<File> messageQueue;
private static Lock lock = new ReentrantLock();
private static List<String> API_key = new CopyOnWriteArrayList<>();
private static File dest;
/**
* Instantiates a new Consumer.
*
* @param messageQueue the message queue
* @param writeQueue the write queue
* @param dest the dest
* @param srcBase the src base
* @param destBase the dest base
*/
public Consumer(BlockingQueue messageQueue,BlockingQueue writeQueue, File dest, String srcBase, String destBase){
this.messageQueue = messageQueue;
this.writeQueue = writeQueue;
this.dest = dest;
this.srcBase = srcBase;
this.destBase = destBase;
}
public void run() {
while(isRunning){
try {
lock.lock();
//获取数据
File file = this.messageQueue.poll(2, TimeUnit.SECONDS);
if(file == null) {
stop();
LOGGER.info("Current Consumer - {} - consume faild, messageQueue empty, thread is stopping...", Thread.currentThread().getName());
lock.unlock();
continue;
}
dest = new File(destBase + file.getAbsolutePath().substring(srcBase.length()));
lock.unlock();
//进行数据处理
doCompress(file, dest);
LOGGER.info("Current Consumer - {} - consume success, messageId : {} ", Thread.currentThread().getName(), file.getAbsolutePath());
writeQueue.add(file.getAbsolutePath());
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
private void doCompress(File src, File dest) {
LOGGER.info("Current Consumer - {} - Comsumer doCompress src:{}",Thread.currentThread().getName(), src.getAbsolutePath());
if(src.isDirectory()) {
dest.mkdirs();
} else {
Tinify.setKey(API_key.get(Thread.currentThread().getName().hashCode() % API_key.size()));
try {
Source source = Tinify.fromFile(src.getAbsolutePath());
BufferedImage bufferedImage = ImageIO.read(src);
if(bufferedImage.getWidth() > 800) {
Options options = new Options()
.with("method", "scale")
.with("width", 800);
Source resized = source.resize(options);
resized.toFile(dest.getAbsolutePath());
} else {
source.toFile(dest.getAbsolutePath());
}
} catch (Exception e) {
e.printStackTrace();
LOGGER.error("Current Consumer - {} - Consumer doCompress exception error:{}, src.path:{}", Thread.currentThread().getName(), e.getMessage(), src.getAbsolutePath());
copyFile(src, dest);
}
}
}
private void copyFile(File src, File dest) {
try {
LOGGER.info("Current Consumer - {} - src.path:{}, dest.path:{}", Thread.currentThread().getName(), src.getAbsolutePath(), dest.getAbsolutePath());
while(!dest.getParentFile().exists()) {
try {
Thread.sleep(500);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
// lock.lock();
Files.copy(src.toPath(), dest.toPath());
} catch (IOException e) {
e.printStackTrace();
LOGGER.error("Consumer copyFile cause error:{}, currentThread.getName:{}", e.getMessage(), Thread.currentThread().getName());
} finally {
// lock.unlock();
}
}
/**
* Stop.
*/
public void stop() {
isRunning = false;
}
/**
* Gets src base.
*
* @return the src base
*/
public static String getSrcBase() {
return srcBase;
}
/**
* Sets src base.
*
* @param srcBase the src base
*/
public static void setSrcBase(String srcBase) {
Consumer.srcBase = srcBase;
}
/**
* Gets dest base.
*
* @return the dest base
*/
public static String getDestBase() {
return destBase;
}
/**
* Sets dest base.
*
* @param destBase the dest base
*/
public static void setDestBase(String destBase) {
Consumer.destBase = destBase;
}
/**
* Gets api key.
*
* @return the api key
*/
public static List<String> getAPI_key() {
return API_key;
}
/**
* Sets api key.
*
* @param API_key the api key
*/
public static void setAPI_key(List<String> API_key) {
Consumer.API_key = API_key;
}
}持久化线程Persist代码：
package com.hearglobal.multi;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.TimeUnit;
/**
* CopyRright (c)2014-2016 Haerbin Hearglobal Co.,Ltd
* Project: tinypngThread
* Comments:
* Author:cbam
* Create Date:2017/3/21
* Modified By:
* Modified Date:
* Modified Reason:
*/
public class Persist extends Thread {
private static final Logger LOGGER = LoggerFactory.getLogger(Consumer.class);
private BlockingQueue<String> writeQueue;
private boolean isRunning = true;
private static String pic_log_location = "log.pic";
private long start;
/**
* Instantiates a new Persist.
*
* @param writeQueue the write queue
* @param start the start
*/
public Persist(BlockingQueue writeQueue, long start) {
this.writeQueue = writeQueue;
this.start = start;
}
@Override
public void run() {
try {
while(!shouldStop()) {
Thread.sleep(10 * 1000);
List<String> list = new LinkedList();
while(!writeQueue.isEmpty()) {
list.add(writeQueue.poll(5, TimeUnit.SECONDS));
}
doPersist(list);
}
LOGGER.info("Persist thread work finished");
LOGGER.info("耗时: {} ms",System.currentTimeMillis() - start);
} catch (InterruptedException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}
private void doPersist(List<String> messageIds) throws IOException {
FileWriter writer = new FileWriter(pic_log_location, true);
for(String str : messageIds) {
writer.write(str + "\n");
}
writer.close();
}
/**
* Is running boolean.
*
* @return the boolean
*/
public boolean isRunning() {
return isRunning;
}
/**
* Sets running.
*
* @param running the running
*/
public void setRunning(boolean running) {
isRunning = running;
}
//persist线程终止的条件是 所有消费线程已停止 && 当前 已消费消息队列为空
private boolean shouldStop() {
return isRunning == false && writeQueue.size() == 0;
}
public static void setPic_log_location(String pic_log_location) {
Persist.pic_log_location = pic_log_location;
}
}
启动类 Starter代码：
package com.hearglobal.multi;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.*;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.*;
/**
* CopyRright (c)2014-2016 Haerbin Hearglobal Co.,Ltd
* Project: tinypngThread
* Comments:
* Author:cbam
* Create Date:2017/3/21
* Modified By:
* Modified Date:
* Modified Reason:
*/
public class Starter {
private static CountDownLatch cdl = new CountDownLatch(1);
private static final Logger LOGGER = LoggerFactory.getLogger(Starter.class);
private static int coreNumber = Runtime.getRuntime().availableProcessors();
private static List<String> API_key = new CopyOnWriteArrayList<>();
private static LinkedList<String> logList = new LinkedList<>() ;
private static String api_key_location = "api_key.properties";
private static String pic_log_location = "log.pic";
private static String src;
private static String dest;
private static int threshold;
private static long start;
/**
* Main.
*
* @param args the args
*/
public static void main(String[] args){
String src = args[0];
String dest = args[1];
threshold = args.length < 3 || args[2] == null || args[2].equals("") ? coreNumber : Integer.parseInt(args[2]);
try {
new Starter().compress(src, dest, threshold);
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* Compress.
*
* @param src the src
* @param dest the dest
* @param threshold the threshold
* @throws Exception the exception
*/
public void compress(String src, String dest, int threshold) throws Exception {
start = System.currentTimeMillis();
if(!this.checkParam(src, dest)) {
throw new IllegalArgumentException("参数不正确！");
}
this.src = src;
this.dest = dest;
this.threshold = threshold <= 0 ? coreNumber : threshold;
LOGGER.info("运行参数：{}, {}, {}", src, dest, threshold);
try {
//消息队列
BlockingQueue<File> messageQueue = new LinkedBlockingQueue<>();
//已消费队列 用于已消费消息的持久化 使得单线程对文件读写
final BlockingQueue<String> writeQueue = new LinkedBlockingQueue<>();
ExecutorService cachePool = Executors.newCachedThreadPool();
//生产者提交 最多启动一个
Provider p = new Provider(messageQueue, new File(src));
initProvider();
p.setLogList(logList);
Future<Boolean> future = cachePool.submit(p);
//阻塞到provide 执行完成
while(!future.get()) {
future = cachePool.submit(p);
}
initConsumer();
//消费者提交
for(int i = 0; i < threshold; i++) {
Consumer c = new Consumer(messageQueue, writeQueue, new File(dest), src, dest);
c.setAPI_key(API_key);
cachePool.execute(c);
}
System.out.println(start + "start");
//启动持久化消息线程工作 消费writeQueue
Persist perssit = new Persist(writeQueue, start);
perssit.setPic_log_location(pic_log_location);
perssit.start();
cachePool.shutdown();
while(true) {
if(cachePool.isTerminated()) {
perssit.setRunning(false);
LOGGER.info("compress has finished !");
break;
}
}
} catch (Exception e) {
e.printStackTrace();
throw new Exception("压缩失败");
}
}
/**
* Check param boolean.
*
* @param src the src
* @param dest the dest
* @return the boolean
*/
private boolean checkParam(String src, String dest) {
if(src == null || dest == null || src.equals("") || dest.equals("")) {
LOGGER.error("参数为空！");
return false;
}
int loc_src = src.lastIndexOf(".");
String suffix_src = src.substring(++loc_src);
int loc_dest = dest.lastIndexOf(".");
String suffix_dest = src.substring(++loc_dest);
if(suffix_dest.equals("png") && suffix_src.equals("jpg")) {
LOGGER.error("错误图片类型转换！");
return false;
} else if(suffix_dest.equals("jpg") && suffix_src.equals("png")) {
LOGGER.error("错误图片类型转换！");
return false;
} else if(! new File(src).exists()) {
LOGGER.error("src 路径未找到！");
return false;
} else {
return true;
}
}
private void initConsumer() {
String line = "";
try {
//BufferedReader buff = new BufferedReader(new FileReader("E:\\workplace\\tinypngThread\\target\\api_key.properties"));
BufferedReader buff = new BufferedReader(new FileReader(api_key_location));
while ((line=buff.readLine()) != null) {
LOGGER.info("API_key => {} ",line);
API_key.add(line);
}
buff.close();
} catch (IOException e) {
e.printStackTrace();
LOGGER.error("Consumer initConsumer error:{}", e.getMessage());
}
}
/**
* Init provider.
*
* @throws IOException the io exception
*/
private static void initProvider() throws IOException {
LOGGER.info("Provider initProvider invoked");
File file = new File(pic_log_location);
if(!file.exists()) {
file.createNewFile();
}
FileReader reader = new FileReader(pic_log_location);
BufferedReader br = new BufferedReader(reader);
String str;
while((str = br.readLine()) != null) {
logList.add(str);
}
}
/**
* Sets api key location.
*
* @param api_key_location the api key location
*/
public static void setApi_key_location(String api_key_location) {
Starter.api_key_location = api_key_location;
}
/**
* Sets pic log location.
*
* @param pic_log_location the pic log location
*/
public static void setPic_log_location(String pic_log_location) {
Starter.pic_log_location = pic_log_location;
}
}
8472
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
