在互联网上采集数据做应用时,我们经常会遇到目标网站图片有防盗链,导致采集文章中的图片在应用中无法显示,解决这个问题有2个办法:
1、采集的时候将图片下载过来存储到自己的服务器上,将html的图片路径替换为自己的图片路径,缺点是存储资源太大,采集慢;
2、不下载图片,在显示时做处理,由服务端来解决防盗链,下面我们就来看实现方式:
大部分防盗链都是通过Referer来判断的,所以我们只需要在服务端加入Referer请求图片,将其转化为base64图片即可;
import java.awt.image.BufferedImage;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.net.URLConnection;
import java.util.zip.GZIPInputStream;
import javax.imageio.ImageIO;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.lang.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import net.coobird.thumbnailator.Thumbnails;
/**
* 将HTML内容链接替换为完整链接,支持将将图片替换为base64,防止图片因防盗链无法显示
* author:gq
*/
public class FixUrlHelper {
/**
* 基于baseUrl,补全content代码中的链接,并且将图片替换为base64,防止图片因防盗链无法显示
*
* @param baseUrl 原文链接或网站域名
* @param content HTML内容
* @param isFullImg 是否填充图片(将图片转换为base64)
* @param imgSizeK 图片保留的最大KB数
* @return
*/
public static String FixUrl(String baseUrl, String content,boolean isFullImg, int imgSizeK) {
Document document = Jsoup.parse(content);
document.setBaseUri(baseUrl);
Elements elements = document.select("img");
String base64 = "";
for (Element el : elements) {
//此处为特殊代码,修复部分网站使用data-src保存图片路径
if (StringUtils.isNotBlank(el.attr("data-src")) && StringUtils.isBlank( el.attr("src") )) {
el.attr("src", el.absUrl("data-src"));
el.removeAttr("data-src");
}
String imgUrl = el.attr("src");
if (!imgUrl.trim().startsWith("http")) {
el.attr("src", el.absUrl("src"));
}
try {
if(isFullImg) {
base64 = downLoadAndZoomToBase64(el.attr("src"), baseUrl, imgSizeK);
if(base64!=null) {
el.attr("src", base64);
}
}
} catch (Exception ex) {
}
}
elements = document.select("a[href]");
for (Element el : elements) {
String imgUrl = el.attr("href");
if (!imgUrl.trim().startsWith("http")) {
el.attr("href", el.absUrl("href"));
}
}
if (content.contains("<html") && content.contains("</html>")) {
return document.html();
}
return document.body().html();
}
/**
* 基于baseUrl,补全content代码中的链接
*
* @param baseUrl 原文链接或网站域名
* @param content HTML内容
* @return
*/
public static String FixUrl(String baseUrl, String content) {
return FixUrl( baseUrl, content,false, 0);
}
/**
* 重置图片大小并将图片转换成Base64编码 ,带头文件
*
* @param imgurl
* 图片地址
* @param referer
* 来源
* @param sizeK
* 缩小到指定大小(转化为50K大小的图片50000= 50K)
* @return
*/
private static String downLoadAndZoomToBase64(String imgurl, String referer, int sizeK) {
BufferedImage img = downloadImg(imgurl, referer);
if (img == null) {
return null;
}
String base64 = resizeImageTo(img, sizeK);
return "data:image/jpg;base64," + base64;
}
/**
* 重置图片大小
*
* @param src
* @param sizeK
* @return
*/
private static String resizeImageTo(BufferedImage src, int sizeK) {
try {
BufferedImage output = Thumbnails.of(src).size(src.getWidth(), src.getHeight()).asBufferedImage();
String base64 = imageToBase64(output);
while (base64.length() - base64.length() / 8 * 2 > sizeK) {
output = Thumbnails.of(output).scale(0.9f).asBufferedImage();
base64 = imageToBase64(output);
}
return imageToBase64(output);
} catch (Exception e) {
return imageToBase64(src);
}
}
/**
* 图片转为base64字符串
*
* @param bufferedImage
* @return
*/
private static String imageToBase64(BufferedImage bufferedImage) {
Base64 encoder = new Base64();
ByteArrayOutputStream baos = new ByteArrayOutputStream();
try {
ImageIO.write(bufferedImage, "jpg", baos);
} catch (IOException e) {
e.printStackTrace();
}
return new String(encoder.encode((baos.toByteArray())));
}
/**
* 下载图片
*
* @param imgurl 图片地址
* @param referer 来源,防止被防盗
* @return
*/
private static BufferedImage downloadImg(String imgurl, String referer) {
InputStream is = null;
try {
URL url = new URL(imgurl);
URLConnection con = url.openConnection();
con.setRequestProperty("Host", url.getHost());
con.setRequestProperty("Referer", referer);
is = con.getInputStream();
if (con.getContentEncoding() != null && con.getContentEncoding().equalsIgnoreCase("gzip")) {
is = new GZIPInputStream(con.getInputStream());
}
return ImageIO.read(is);
} catch (Exception ex) {
ex.printStackTrace();
return null;
} finally {
try {
if (is != null)
is.close();
} catch (Exception ex) {
}
}
}
public static void main(String[] args) {
String url = "https://mp.weixin.qq.com/s?__biz=MzUzNTk4OTMyNQ==&mid=2247484787&idx=2&sn=1b3f3e2e661de9112addbee7fb05e86b";
// 文章内容
String content = "<p style='text-align: center;'><img data-src='https://mmbiz.qpic.cn/mmbiz_jpg/DfBcAKibFd2o9pDgScS5pwDRfrRWeaO9Db56akuw4GCbBDSfagibl8kGZ1yNtGmhr4r4j6scpibrVO066NibxGysNw/640?wx_fmt=jpeg' ></p>";
// 最大图片大小kb
int maxImageSize = 20000;
//补全content代码中的链接,并且将图片替换为base64
String result= FixUrlHelper.FixUrl(url, content,true, maxImageSize);
System.out.println(result);
}
}
依赖包:
<!-- https://mvnrepository.com/artifact/org.jsoup/jsoup -->
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>1.13.1</version>
</dependency>
<!-- https://mvnrepository.com/artifact/net.coobird/thumbnailator -->
<dependency>
<groupId>net.coobird</groupId>
<artifactId>thumbnailator</artifactId>
<version>0.4.11</version>
</dependency>
上述示例运行结果:
<p style="text-align: center;"><img src="data:image/jpg;base64,/9j/4AAQSkZJRgABAgAAAQABAAD/2wBDAAgGBgcGBQgHBw*******901GKAFJ55pCQRSN0pooHYDSE8UHrRQIYTRTW6GimB//Z"></p>