有时候我们需要在爬虫的时候将网站的图片保存到本地,这就需要我们先获取到图片的url,然后利用url再去下载图片到本地。
下面介绍两种简单的方法:
1.利用java自带的URLConnection
此方法我还没有找到可以携带cookie或者其他信息去下载图片的方法。
package cn.qlq.craw.Jsoup; import java.io.FileOutputStream; import java.io.InputStream; import java.io.OutputStream; import java.net.MalformedURLException; import java.net.URL; import java.net.URLConnection; import org.jsoup.Connection; import org.jsoup.Jsoup; /** * url获取图片并且保存到本地 * * @author liqiang * */ public class UrlConnectionGetPicture { public static void main(String[] args) throws Exception { String url = "http://qiaoliqiang.cn/fileDown/zfb.bmp"; URL url1 = new URL(url); URLConnection conn = url1.openConnection(); InputStream inputStream = conn.getInputStream(); String path = "C:\\Users\\liqiang\\Desktop\\实习\\python\\javaCrawPicture\\test.bmp"; OutputStream out = new FileOutputStream(path); byte[] buff = new byte[1024]; int i = -1; while(( i = inputStream.read(buff))!= -1){ out.write(buff, 0 , i); } inputStream.close(); out.close(); } }
补充:org.apache.commons.io.IOUtils可以简单的将一个inputStream的文件读取到另一个outputStream,实现文件的拷贝,例如:
只用传两个参数,第一个传递InputStream,第二个传递OutputStream
package cn.qlq.craw.Jsoup; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.net.URL; import java.net.URLConnection; import org.apache.commons.io.IOUtils; public class IOutilsDownloadFile { public static void main(String[] args) throws IOException { String url = "http://qiaoliqiang.cn/fileDown/zfb.bmp"; URL url1 = new URL(url); URLConnection conn = url1.openConnection(); InputStream inputStream = conn.getInputStream(); String path = "C:\\Users\\liqiang\\Desktop\\test.bmp"; OutputStream outputStream = new FileOutputStream(path); // 利用IOutiks拷贝文件,简单快捷 IOUtils.copy(inputStream, outputStream); } }
2.利用Jsoup。此方法可以在下载图片的时候携带cookie或者携带一些额外的数据(重要)
下面是不携带cookie的方法:
package cn.qlq.craw.Jsoup; import java.io.BufferedOutputStream; import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import org.jsoup.Connection; import org.jsoup.Jsoup; /** * Jsoup下载图片并保存到本地 * @author liqiang * */ public class JsoupDoloadPicture { /** * @param args * @throws IOException */ public static void main(String[] args) throws IOException { // TODO Auto-generated method stub String imageSrc = "http://newjwc.tyust.edu.cn/CheckCode.aspx"; Connection.Response response = Jsoup.connect(imageSrc).ignoreContentType(true).execute(); byte[] img = response.bodyAsBytes(); System.out.println(img.length); savaImage(img, "C:\\Users\\liqiang\\Desktop\\实习\\python\\javaCrawJWXT", "test.png"); } public static void savaImage(byte[] img,String filePath,String fileName) { BufferedOutputStream bos = null; FileOutputStream fos = null; File file = null; File dir = new File(filePath); try { //判断文件目录是否存在 if(!dir.exists() && dir.isDirectory()){ dir.mkdir(); } file = new File(filePath+"\\"+fileName); fos = new FileOutputStream(file); bos = new BufferedOutputStream(fos); bos.write(img); } catch (FileNotFoundException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); }finally{ if(bos!=null){ try { bos.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } } if(fos!=null){ try { fos.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } } } } }
下面是携带cookie的方法,此方法可以在爬虫需要登录网站的时候带着cookie去获取验证码。
package cn.qlq.craw.JsoupCrawJWXT; import java.io.BufferedOutputStream; import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.util.Map; import org.jsoup.Connection; import org.jsoup.Jsoup; /** * Jsoup下载图片并保存到本地 * * @author liqiang * */ public class JsoupDoloadPicture { /** * 带着cookie下载验证码图片 * * @param url * @param cookies * @throws IOException */ public static void downloadImg(String url, Map<String, String> cookies) throws IOException { // TODO Auto-generated method stub Connection connect = Jsoup.connect(url); connect.cookies(cookies);// 携带cookies爬取图片 connect.timeout(5 * 10000); Connection.Response response = connect.ignoreContentType(true).execute(); byte[] img = response.bodyAsBytes(); System.out.println(img.length); // 读取文件存储位置 String directory = ResourcesUtil.getValue("path", "file"); savaImage(img, directory, "yzm.png"); } public static void savaImage(byte[] img, String filePath, String fileName) { BufferedOutputStream bos = null; FileOutputStream fos = null; File file = null; File dir = new File(filePath); try { // 判断文件目录是否存在 if (!dir.exists() && dir.isDirectory()) { dir.mkdir(); } file = new File(filePath + "\\" + fileName); fos = new FileOutputStream(file); bos = new BufferedOutputStream(fos); bos.write(img); System.out.println("验证码已经下载到:"+filePath); } catch (FileNotFoundException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } finally { if (bos != null) { try { bos.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } } if (fos != null) { try { fos.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } } } } }