Java爬虫实现图片下载

实现爬虫爬取页面的信息所需要的jar包
我是建立maven工程，pom添加的依赖信息

 - <dependency>
		<groupId>org.apache.httpcomponents</groupId>
		<artifactId>httpclient</artifactId>
		<version>4.5.3</version>
	</dependency>
 - <dependency>
		<groupId>org.jsoup</groupId>
		<artifactId>jsoup</artifactId>
		<version>1.10.3</version>
	</dependency>
    
	- <dependency>
		<groupId>org.apache.commons</groupId>
		<artifactId>commons-lang3</artifactId>
		<version>3.7</version>
	</dependency>
	- <dependency>
		<groupId>commons-io</groupId>
		<artifactId>commons-io</artifactId>
		<version>2.6</version>
	</dependency>

爬虫的代码如下，这里是爬取： http://news.4399.com/gonglue/wzlm/pifu/

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;

import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
//获取这个页面
public class Test {
	@SuppressWarnings("null")
	public static String gethtml(String html) throws IOException {
		// 创建http请求
		CloseableHttpClient createDefalt = HttpClients.createDefault();
		HttpGet get = new HttpGet(html);
		String resulthtml = "";
		CloseableHttpResponse response = null;
		try {
			response = createDefalt.execute(get);
			if (response.getStatusLine().getStatusCode() == 200) {
				resulthtml = EntityUtils.toString(response.getEntity(), "gb2312");
			}
		} catch (ClientProtocolException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		} finally {
			if (response == null) {
				try {
					response.close();
				} catch (IOException e) {
					e.printStackTrace();
				}
			}

			createDefalt.close();
		}
		return resulthtml;
	}

通过Document来解析html页面

//通过Document解析页面
	public static void parsHtml(String html) throws IOException {
		Document document = Jsoup.parse(html);
		Element list = document.getElementById("hreo_list");
		Elements select = list.select("img");
		for (Element element : select) {
			String url = element.attr("lz_src");
			String fileName = element.attr("alt");
			downImage(url, fileName);
			System.out.println(fileName);
			System.out.println("下载成功");
		}
	}

下面是通过io流来对页面的图片的下载

static String file = "D://pifu";//下载的目标路径
	public static void downImage(String imgurl, String fileName) {
		//判断目标文件夹是否存在
		File files = new File(file);
		if (!files.exists()) {
			files.mkdirs();
		}

		InputStream is;
		FileOutputStream out;
		try {
			URL url = new URL(imgurl);
			HttpURLConnection connection = (HttpURLConnection) url.openConnection();
			is = connection.getInputStream();
			// 创建文件
			File fileofImg = new File(file + "/" + fileName + ".gpg");
			out = new FileOutputStream(fileofImg);
			int i = 0;
			while ((i = is.read()) != -1) {
				out.write(i);
			}
			is.close();
			out.close();
		} catch (MalformedURLException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (FileNotFoundException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}

	}

运行代码

	public static void main(String[] args) throws IOException {
		String html = gethtml("http://news.4399.com/gonglue/wzlm/pifu/");
		parsHtml(html);
	}

运行结果