java异步下载图片_CMS项目实现异步图片下载

最新推荐文章于 2021-08-18 07:53:37 发布

唐永治

最新推荐文章于 2021-08-18 07:53:37 发布

阅读量164

点赞数

文章标签： java异步下载图片

本文链接：https://blog.csdn.net/weixin_36000045/article/details/114090671

版权

这是一个Java程序，实现了使用线程池下载远程图片的机制。当下载失败时，程序会等待一段时间后重新尝试。线程池配置了核心线程数、最大线程数和空闲时间，同时具有拒绝策略来处理任务队列满的情况。程序还包含了一个调度线程，每隔5秒检查一次数据库中的待处理图片并启动新的下载任务。

摘要由CSDN通过智能技术生成

create table CMS_IMAGES

(

id LONG PRIMARY KEY, --id

wsrc varchar2(256),--远程图片地扯

lsrc varchar2(256),--本地图片地扯

state NUMBER, --此图片的状态 1未处理 2下载中 3下载失败

last_modified LONG --上次状态修改时间

modified_count number--状态修改次数-缺省为0

)

先解释一下, 3下载失败了,等两个小时,重新通线程调用一次,因为有的网站,不能让你老是在他的网站上面下载图片的,只有过些时间再请求一次,就可以下载,实在不能下载的图片,比率很小,

/**

* 下载图片的后台线程池

* @author

* @version 1.0

public class ThreadPoolManager {

protected static Logger log = LoggerFactory

.getLogger(ThreadPoolManager.class);

private static ThreadPoolManager tpm = new ThreadPoolManager();

// 线程池维护线程的最少数量

private final static int CORE_POOL_SIZE = 4;

// 线程池维护线程的最大数量

private final static int MAX_POOL_SIZE = 10;

// 线程池维护线程所允许的空闲时间

private final static int KEEP_ALIVE_TIME = 0;

// 线程池所使用的缓冲队列大小

private final static int WORK_QUEUE_SIZE = 10;

// 图片缓冲队列

private Queue msgQueue = new LinkedList();

// 访问消息缓存的调度线程

final Runnable accessBufferThread = new Runnable() {

public void run() {

// 查看是否有待定请求，如果有，则创建一个新的Thread，并添加到线程池中

if (hasMoreAcquire()) {

ImageDO image = (ImageDO) msgQueue.poll();

Runnable task = new UploadThread(image);

threadPool.execute(task);

}

if (hasImagesInDB()) {

try {

BeanFactoryService beanFactory = (BeanFactoryService) BizServiceManager

.getInstance().getService(

BeanFactoryService.SERVICE_NAME);

ImageManager imageManager = (ImageManager) beanFactory

.getBean("imageManager");

List images = imageManager

.getImages(ImageDO.STATE_READY);

for (int i = 0; i < images.size(); i++) {

ImageDO img = images.get(i);

img.setState(ImageDO.STATE_RUNNING);

img.setModifiedCount(img.getModifiedCount() + 1);

img.setLastModified(CalendarUtil.getCurrentDate()

.getTime());

imageManager.updateImage(img);

Runnable task = new UploadThread(img);

threadPool.execute(task);

}

} catch (Exception e) {

log.error("获取下载图片队列失败", e);

}

};

// 用于被拒绝任务的处理程序

final RejectedExecutionHandler handler = new RejectedExecutionHandler() {

public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {

log.error(((UploadThread) r).getImg() + " 放入队列中重新等待执行");

msgQueue.offer(((UploadThread) r).getImg());

}

};

// 管理数据库访问的线程池

final ThreadPoolExecutor threadPool = new ThreadPoolExecutor(

CORE_POOL_SIZE, MAX_POOL_SIZE, KEEP_ALIVE_TIME, TimeUnit.SECONDS,

new ArrayBlockingQueue(WORK_QUEUE_SIZE), this.handler);

// 调度线程池

final ScheduledExecutorService scheduler = Executors

.newScheduledThreadPool(1);

final ScheduledFuture taskHandler = scheduler.scheduleAtFixedRate(

accessBufferThread, 0, 5, TimeUnit.SECONDS);

public static ThreadPoolManager newInstance() {

return tpm;

}

private ThreadPoolManager() {

}

private boolean hasMoreAcquire() {

return !msgQueue.isEmpty();

}

private boolean hasImagesInDB() {

boolean result = false;

try {

BeanFactoryService beanFactory = (BeanFactoryService) BizServiceManager

.getInstance().getService(BeanFactoryService.SERVICE_NAME);

if (beanFactory == null)

return false;

ImageManager imageManager = (ImageManager) beanFactory

.getBean("imageManager");

if (imageManager == null)

return false;

int imagesCount = imageManager.getImagesCount(ImageDO.STATE_READY);

result = (imagesCount != 0);

} catch (Exception e) {

log.error("获取下载图片队列失败", e);

}

return result;

}

public void processImg(ImageDO img) {

Runnable task = new UploadThread(img);

threadPool.execute(task);

}

public long getCompleteTasksCount() {

return this.threadPool.getCompletedTaskCount();

}

public long getActiveTasksCount() {

return this.threadPool.getActiveCount();

}

public long getAllTasksCount() {

return this.threadPool.getTaskCount();

}

public long getQuequeCount() {

return this.msgQueue.size();

}

public boolean getRunningStatus() {

return !this.threadPool.isShutdown();

}

public long getScheduledQuequeSize() {

return ((ScheduledThreadPoolExecutor)this.scheduler).getQueue().size();

}

public class UploadThread implements Runnable {

private ImageDO img;

protected Logger log = LoggerFactory.getLogger(UploadThread.class);

public ImageDO getImg() {

return img;

}

public void setImg(ImageDO img) {

this.img = img;

}

public UploadThread() {

super();

}

public UploadThread(ImageDO img) {

this.img = img;

}

public void run() {

BeanFactoryService beanFactory = (BeanFactoryService) BizServiceManager

.getInstance().getService(BeanFactoryService.SERVICE_NAME);

ImageManager imageManager = (ImageManager) beanFactory

.getBean("imageManager");

if (UploadUtil.upload(img)) {

try {

log.error("下載圖片成功:"+ img);

imageManager.deleteImageByid(img.getId());

} catch (ManagerException e) {

log.error("删除图片记录失败:"+ img.getId(), e);

}

} else {

img.setState(ImageDO.STATE_FAIL);

try {

imageManager.updateImage(img);

} catch (ManagerException e) {

log.error("设置图片下载状态失败:"+ img.getId(), e);

}

public class UploadUtil {

private static Logger logger = Logger.getLogger(UploadUtil.class);

private static FileStorage attachmentFileStorage;

private static String imgServer;

/**

* @param imgServer

* the imgServer to set

public void setImgServer(String imgServer) {

UploadUtil.imgServer = imgServer;

}

public static String getUrlPrefix() {

return imgServer;

}

public static boolean upload(ImageDO image) {

String url = image.getWsrc();

HttpMethod httpMethod = getHttpMethod(url);

ByteArrayDataSource ds = new ByteArrayDataSource(httpMethod

.getResponseBodyAsStream(), "image/jpeg");

ds.setName("upload.jpg");

boolean result = attachmentFileStorage.save(ds, image.getLsrc());

httpMethod.releaseConnection();

return result;

} catch (Exception e) {

logger.error("upload image fail:" + url, e);

}

return false;

}

private static HttpMethod getHttpMethod(String url) throws IOException {

HttpMethod httpMethod = new GetMethod(url);

httpMethod.setRequestHeader("Cache-Control", "no-cache");

httpMethod.setRequestHeader("Accept-Language", "zh-cn");

HttpClient clientTemp = new HttpClient(); // HttpClient创建

HttpClientParams clientParams = clientTemp.getParams();

clientParams.setParameter("http.socket.timeout", 5000); // 5秒socket等待数据

clientParams.setParameter("http.connection.timeout", 5000); // 5秒http

// connection建立超时

clientParams.setParameter("http.connection-manager.timeout", 5000L); // 5秒从http

// yahoo相册的图片需要设置Referer头

if (url.toLowerCase().indexOf("yahoo") != -1) {

httpMethod.addRequestHeader("Referer", "[url=http://forum.taobao.com/]http://forum.taobao.com/[/url]");

}

// connection

// manager获取可用的Http

// connection超时

clientParams.setParameter("http.method.retry-handler",

new DefaultHttpMethodRetryHandler()); // 如果Http出错，三次重试

clientTemp.executeMethod(httpMethod);

return httpMethod;

}

public void setAttachmentFileStorage(FileStorage attachmentFileStorage) {

UploadUtil.attachmentFileStorage = attachmentFileStorage;

}

public static void main(String[] args) throws Exception {

HttpMethod httpMethod = getHttpMethod("[url=http://cn.photos.yahoo.com/users/47df51c0z8a48b60a/255c/__sr_/dbce.jpg?wa86dTAyD7Rq05sklm3fMQ--&F18]http://cn.photos.yahoo.com/users/47df51c0z8a48b60a/255c/__sr_/dbce.jpg?wa86dTAyD7Rq05sklm3fMQ--&F18[/url]");

InputStream inputStream = httpMethod.getResponseBodyAsStream();

FileOutputStream os = new FileOutputStream("c:/test.jpg");

IOUtils.copy(inputStream, os);

os.close();

httpMethod.releaseConnection();

}

public class HtmlParseUtil {

protected static Logger log = LoggerFactory.getLogger(HtmlParseUtil.class);

public static String processImg(String imageURL) {

if (imageURL.toLowerCase().indexOf("taobao.com") == -1) {

BeanFactoryService beanFactory = (BeanFactoryService) BizServiceManager

.getInstance().getService(BeanFactoryService.SERVICE_NAME);