java爬虫爬取网站使用多线程(虎嗅网站)

图解虎嗅爬虫优化方案

pom 如下:

<dependencies>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.3</version>
</dependency>
<!-- jsoup HTML parser library @ https://jsoup.org/ -->
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>1.10.3</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.springframework/spring-jdbc -->
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-jdbc</artifactId>
<version>4.2.6.RELEASE</version>
</dependency>


<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.41</version>
</dependency>
<dependency>
<groupId>c3p0</groupId>
<artifactId>c3p0</artifactId>
<version>0.9.1.2</version>
</dependency>


<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.31</version>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>2.8.1</version>
</dependency>
<dependency>
<groupId>redis.clients</groupId>
<artifactId>jedis</artifactId>
<version>2.9.0</version>
</dependency>
</dependencies>

代码演示如下:

package cn.itcast.huxiu.query;


import java.util.ArrayList;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;


import org.apache.http.HttpEntity;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;


import com.google.gson.Gson;


import cn.itcast.huxiu.Article;
import cn.itcast.huxiu.ArticleDao;
import cn.itcast.huxiu.ResponseJson;


public class HuXiuTest {
public static final ArticleDao articleDao = new ArticleDao();
public static final ArrayBlockingQueue<String> blockingQueue = new ArrayBlockingQueue<String>(1000);
public static final ExecutorService threadPool = Executors.newFixedThreadPool(10);
public static final String prefix = "https://www.huxiu.com/article/";
public static final String end = ".html";
public static void main(String[] args) throws Exception {
//创建线程
for(int i=0;i<30;i++){
threadPool.execute(new ProcessPagingThreadQueue());
}
// 爬取首页的信息
String indexHtml = getIndex();
// 解析首页 得到首页里面的所有的id(根据id来查询每一个页面的信息) 存储到集合里面
parseIndexHtml(indexHtml);

/**
* 在首页的信息爬取了之后 就要准备爬取分页的信息 点击加载更多只时 就相当与是点击了下一页 点击之后 就会发送一个请求
* 这个请求就可以加载下一页的数据了 得到的下一页所有数据之后 就要解析每一页的数据
* 
*/
// 根据首页的信息来得到加载下一页数据按钮的数据值
String last_dateline = getValueAndIndexHtml(indexHtml);// 得到没加载一页数据的数值
// 点击 加载下一页的数据
for (int page = 2; page < 10; page++) {
// 获得请求的路径
String url = "https://www.huxiu.com/v2_action/article_list";
HttpPost httpPost = new HttpPost(url);
// 请求参数
ArrayList<BasicNameValuePair> list = new ArrayList<BasicNameValuePair>();
list.add(new BasicNameValuePair("huxiu_hash_code", "647893ceb60219effa36193702fd89a3"));
list.add(new BasicNameValuePair("page", page + ""));
list.add(new BasicNameValuePair("last_dateline", last_dateline));
// 参数设置
httpPost.setEntity(new UrlEncodedFormEntity(list));
// User-Agent
httpPost.setHeader("User-Agent",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0");
// 发起请求
CloseableHttpClient httpClient = HttpClients.createDefault();
CloseableHttpResponse execute = httpClient.execute(httpPost);
// 在发送请求之后 页面没有跳转 因为是在和首页的同一个页面下 所以不用判断 只有页面跳转才有状态码的判定
// 请求发送之后 就有返回值了 主要注意的是返回值是json数据的形式来进行返回的
String jsonDate = EntityUtils.toString(execute.getEntity());
// 得到json数据值 就要水对json的数据进行解析 解析json的数据使用到的是gson
Gson gson = new Gson();
// 将数据进行解析并且映射到实体类中 实体类中是根据返回的参数来进行设置的
// 得到分页数据的所有的信息 也就是分页数据的url
ResponseJson fromJson = gson.fromJson(jsonDate, ResponseJson.class);
// 得到的分页的每一个数据 每一个URL信息
String data = fromJson.getData();// 得到分页的信息
// 对分页的数据信息进行解析 也就要取得每一个详情信息页面的id值
getDate(data);// 得到所有id值的集合
System.out.println(page+"&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&");
}


}


private static void getDate(String data) {
if (data != null) {
Document document = Jsoup.parse(data);
Elements elements = document.select("div[data-aid]");
for (Element element : elements) {
try {
blockingQueue.put(element.attr("data-aid"));
System.out.println(element.attr("data-aid"));
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
}


// 得到加载下一页信息的数据值
private static String getValueAndIndexHtml(String indexHtml) {
if (indexHtml != null) {
Document document = Jsoup.parse(indexHtml);
Elements select = document.select("div[data-last_dateline]");
return select.get(0).attr("data-last_dateline");
}
return null;
}


public static Article parseXianQingYeMian(String html) {


if (html != null) {

Article article = new Article();
// 将详细页面的信息 转换为文档对象
Document document = Jsoup.parse(html);
// 获取文章的标题信息
String ownText = document.select(".t-h1").get(0).ownText();
article.setTitle(ownText);
// 获取作者
String author = document.select(".author-name").get(0).text();
article.setAuthor(author);
// 获取时间 根据页面上的信息可知时间有两种表示
Elements elements = document.select("span[class=article-time pull-left]");
if (elements.size() == 0) {
String createTime = document.select(".article-time").get(0).ownText();
article.setCreateTime(createTime);
} else {
String createTime = elements.get(0).ownText();
article.setCreateTime(createTime);
}
// 获取文章内容
String content = document.select(".article-content-wrap").get(0).text();
article.setContent(content);
// 获取点赞
article.setZan(document.select(".num").get(0).ownText());
// 获取评论
article.setPl(document.select(".article-pl").get(0).ownText());
System.out.println(article);

return article;
}
return null;
}




// 解析数据 得到url
private static void parseIndexHtml(String indexHtml) {
// TODO Auto-generated method stub
if (indexHtml != null) {
// 解析得到的页面的信息 将其变成文档对象
Document document = Jsoup.parse(indexHtml);
// 得到document对象后 就可以通过document对象来得到需要的东西
Elements elements = document.select(".mod-info-flow div[data-aid]");
for (Element element : elements) {
String aid = element.attr("data-aid");
try {
/**
* 在单线层转变多线程中   主要的几个是 队列 线程的创建 线程池的创建
* 这个主要是创建队列 并且把解析数据的详情的id传给了这个队列 
* 队列的主要的作用是 防止单个线程被同时访问造成拥堵引起并发的问题
* 队列就把这几个问题给解决了 
*/
blockingQueue.put(aid);
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
}


// 首页的获取
private static String getIndex() throws Exception {
String url = "https://www.huxiu.com";
// 发起一个get请求
HttpGet httpGet = new HttpGet(url);
// 设置请求头
httpGet.addHeader("User-Agent",
"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36");
// 返回页面的信息
return getHtml(httpGet);
}


// 执行发送请求的方法
public static String getHtml(HttpGet httpGet) throws Exception {
// TODO Auto-generated method stub
String html = null;
CloseableHttpClient httpClient = HttpClients.createDefault();
CloseableHttpResponse execute = httpClient.execute(httpGet);
// 判断响应码是否为200
if (execute.getStatusLine().getStatusCode() == 200) {
HttpEntity entity = execute.getEntity();
html = EntityUtils.toString(entity);
System.out.println(html);// 返回的的页面的所有信息
}
return html;
}


}

**********************************************************************************************************************

代码创建线程

package cn.itcast.huxiu.query;


import org.apache.http.client.methods.HttpGet;


import cn.itcast.huxiu.Article;


public class ProcessPagingThreadQueue extends Thread {


public void run() {
// TODO Auto-generated method stub
while (true) {
//得到每一给详情页面的id 这里面用到了while循环  因为存在队列里面的id值不知道是多少个 而且出来也是一个一个的出来的 所以就使用到了循环
try {
String parseInt = HuXiuTest.blockingQueue.take();//得到详情页的id
int id = Integer.parseInt(parseInt);
//创建发送请求
HttpGet httpGet = new HttpGet(HuXiuTest.prefix + id + HuXiuTest.end);
// 消息头
httpGet.addHeader("user-agent",
"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36");
String html = HuXiuTest.getHtml(httpGet);//得到页面的详情信息
Article article = HuXiuTest.parseXianQingYeMian(html);
if(article != null){
article.setId(id);
// article.setUrl(HuXiuTest.prefix + id + HuXiuTest.end);
HuXiuTest.articleDao.save(article);
}
} catch (Exception e) {
// TODO Auto-generated catch blockArticle
e.printStackTrace();
} 
}



}


}

**********************************************************************************************************************

实体类

package cn.itcast.huxiu;


public class Article {
private int id;
private String title;
private String author;
private String createTime;
private String sc;
private String zan;
private String pl;
private String content;
private String url;

public int getId() {
return id;
}
public void setId(int id) {
this.id = id;
}
public String getTitle() {
return title;
}
public void setTitle(String title) {
this.title = title;
}
public String getAuthor() {
return author;
}
public void setAuthor(String author) {
this.author = author;
}
public String getCreateTime() {
return createTime;
}
public void setCreateTime(String createTime) {
this.createTime = createTime;
}
public String getSc() {
return sc;
}
public void setSc(String sc) {
this.sc = sc;
}
public String getZan() {
return zan;
}
public void setZan(String zan) {
this.zan = zan;
}
public String getPl() {
return pl;
}
public void setPl(String pl) {
this.pl = pl;
}
public String getContent() {
return content;
}
public void setContent(String content) {
this.content = content;
}
public String getUrl() {
return url;
}
public void setUrl(String url) {
this.url = url;
}
@Override
public String toString() {
return "Article [id=" + id + ", title=" + title + ", author=" + author + ", createTime=" + createTime + ", sc="
+ sc + ", zan=" + zan + ", pl=" + pl + ", content=" + content + ", url=" + url + "]";
}

}

数据库连接

package cn.itcast.huxiu;


import org.springframework.jdbc.core.JdbcTemplate;


import com.mchange.v2.c3p0.ComboPooledDataSource;


public class ArticleDao extends JdbcTemplate{
public ArticleDao() {
// 创建C3P0的datasource 1.配置 2.代码
ComboPooledDataSource dataSource = new ComboPooledDataSource();
// 1.url
// 2.driver
// 3.username&password
dataSource.setUser("root");
dataSource.setPassword("123");
dataSource.setJdbcUrl("jdbc:mysql://localhost:3306/spider?characterEncoding=utf-8");
setDataSource(dataSource);
}
public void save(Article article) {
String sql = "INSERT INTO huxiu_article (id, title, author, createTime, zan, pl, sc, content, url ) VALUES( ?,?,?,?,?,?,?,?,?)";
update(sql, article.getId(),article.getTitle(),article.getAuthor(),article.getCreateTime(),article.getZan(),article.getPl(),article.getSc(),article.getContent(),article.getUrl());
}
}

实体类

package cn.itcast.huxiu;


public class ResponseJson {
private int result;
private String msg;
private String data;
private double total_page;
private double last_dateline;
public int getResult() {
return result;
}
public void setResult(int result) {
this.result = result;
}
public String getMsg() {
return msg;
}
public void setMsg(String msg) {
this.msg = msg;
}
public String getData() {
return data;
}
public void setData(String data) {
this.data = data;
}
public double getTotal_page() {
return total_page;
}
public void setTotal_page(double total_page) {
this.total_page = total_page;
}
public double getLast_dateline() {
return last_dateline;
}
public void setLast_dateline(double last_dateline) {
this.last_dateline = last_dateline;
}
@Override
public String toString() {
return "ResponseJson [result=" + result + ", msg=" + msg + ", data=" + data + ", total_page=" + total_page
+ ", last_dateline=" + last_dateline + "]";
}

}