java爬虫爬取虎嗅网站分页信息

java爬虫爬取首页信息并且爬取分页信息保存数据库

代码

package cn.itcast.huxiu;

import java.io.IOException;
import java.util.ArrayList;


import org.apache.http.HttpEntity;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;


import com.google.gson.Gson;


public class HuXiuTest {
public static void main(String[] args) throws Exception {


// 爬取首页的信息
String indexHtml = getIndex();
// 解析首页 得到首页里面的所有的id(根据id来查询每一个页面的信息) 存储到集合里面
ArrayList<String> ids = parseIndexHtml(indexHtml);
// 得到了所有详情文章的id 通过文章的id来查询每一篇文章的信息 并且把这些信息保存在自己的数据库里面
parseXianQingYeMian(ids);
/**
* 在首页的信息爬取了之后 就要准备爬取分页的信息 点击加载更多只时 就相当与是点击了下一页 点击之后 就会发送一个请求
* 这个请求就可以加载下一页的数据了 得到的下一页所有数据之后 就要解析每一页的数据
* 
*/
// 根据首页的信息来得到加载下一页数据按钮的数据值
String last_dateline = getValueAndIndexHtml(indexHtml);// 得到没加载一页数据的数值
// 点击 加载下一页的数据
for (int page = 2; page < 4; page++) {
// 获得请求的路径
String url = "https://www.huxiu.com/v2_action/article_list";
HttpPost httpPost = new HttpPost(url);
// 请求参数
ArrayList<BasicNameValuePair> list = new ArrayList<BasicNameValuePair>();
list.add(new BasicNameValuePair("huxiu_hash_code", "647893ceb60219effa36193702fd89a3"));
list.add(new BasicNameValuePair("page", page + ""));
list.add(new BasicNameValuePair("last_dateline", last_dateline));
// 参数设置
httpPost.setEntity(new UrlEncodedFormEntity(list));
// User-Agent
httpPost.setHeader("User-Agent",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0");
// 发起请求
CloseableHttpClient httpClient = HttpClients.createDefault();
CloseableHttpResponse execute = httpClient.execute(httpPost);
// 在发送请求之后 页面没有跳转 因为是在和首页的同一个页面下 所以不用判断 只有页面跳转才有状态码的判定
// 请求发送之后 就有返回值了 主要注意的是返回值是json数据的形式来进行返回的
String jsonDate = EntityUtils.toString(execute.getEntity());
// 得到json数据值 就要水对json的数据进行解析 解析json的数据使用到的是gson
Gson gson = new Gson();
// 将数据进行解析并且映射到实体类中 实体类中是根据返回的参数来进行设置的
// 得到分页数据的所有的信息 也就是分页数据的url
ResponseJson fromJson = gson.fromJson(jsonDate, ResponseJson.class);
// 得到的分页的每一个数据 每一个URL信息
String data = fromJson.getData();// 得到分页的信息
// 对分页的数据信息进行解析 也就要取得每一个详情信息页面的id值
ArrayList<String> date = getDate(data);// 得到所有id值的集合
// 遍历集合
for (String idss : date) {
ArticleDao articleDao = new ArticleDao();
// 根据id来查询详情页
int id = Integer.parseInt(idss);
// 创建发送请求
HttpGet httpGet = new HttpGet("https://www.huxiu.com/article/" + id + ".html");
// 消息头
httpGet.addHeader("user-agent",
"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36");
CloseableHttpClient httpClients = HttpClients.createDefault();
// 发送请求
CloseableHttpResponse executes = httpClients.execute(httpGet);
// 判断这个详细页面是否可以加载成功
if (executes.getStatusLine().getStatusCode() == 200) {// 200表示加载到了详情的页面的信息
HttpEntity entity = executes.getEntity();
String html = EntityUtils.toString(entity);
Article article = new Article();
article.setId(id);
// article.setUrl("https://www.huxiu.com/article/" + id + ".html");
// 将详细页面的信息 转换为文档对象
Document document = Jsoup.parse(html);
// 获取文章的标题信息
String ownText = document.select(".t-h1").get(0).ownText();
article.setTitle(ownText);
// 获取作者
String author = document.select(".author-name").get(0).text();
article.setAuthor(author);
// 获取时间 根据页面上的信息可知时间有两种表示
Elements elements = document.select("span[class=article-time pull-left]");
if (elements.size() == 0) {
String createTime = document.select(".article-time").get(0).ownText();
article.setCreateTime(createTime);
} else {
String createTime = elements.get(0).ownText();
article.setCreateTime(createTime);
}
// 获取文章内容
String content = document.select(".article-content-wrap").get(0).text();
article.setContent(content);
// 获取点赞
article.setZan(document.select(".num").get(0).ownText());
// 获取评论
article.setPl(document.select(".article-pl").get(0).ownText());
System.out.println("******************************************************************************************");
System.out.println(article);
System.out.println("******************************************************************************************");
articleDao.save(article);
}
}


}


}


private static ArrayList<String> getDate(String data) {
ArrayList<String> arrayList = new ArrayList<String>();
if (data != null) {
Document document = Jsoup.parse(data);
Elements elements = document.select("div[data-aid]");
for (Element element : elements) {
arrayList.add(element.attr("data-aid"));
}
return arrayList;
}
return null;
}


// 得到加载下一页信息的数据值
private static String getValueAndIndexHtml(String indexHtml) {
if (indexHtml != null) {
Document document = Jsoup.parse(indexHtml);
Elements select = document.select("div[data-last_dateline]");
return select.get(0).attr("data-last_dateline");
}
return null;
}


private static void parseXianQingYeMian(ArrayList<String> ids) throws IOException, ClientProtocolException {


if (ids.size() != 0) {
for (String pid : ids) {
// 遍历得到了 每一个页面的id
ArticleDao articleDao = new ArticleDao();
int id = Integer.parseInt(pid);
// 创建发送请求
HttpGet httpGet = new HttpGet("https://www.huxiu.com/article/" + id + ".html");
// 消息头
httpGet.addHeader("user-agent",
"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36");
CloseableHttpClient httpClient = HttpClients.createDefault();
// 发送请求
CloseableHttpResponse execute = httpClient.execute(httpGet);
// 判断这个详细页面是否可以加载成功
if (execute.getStatusLine().getStatusCode() == 200) {// 200表示加载到了详情的页面的信息
HttpEntity entity = execute.getEntity();
String html = EntityUtils.toString(entity);
Article article = new Article();
article.setId(id);
// article.setUrl("https://www.huxiu.com/article/" + id + ".html");
// 将详细页面的信息 转换为文档对象
Document document = Jsoup.parse(html);
// 获取文章的标题信息
String ownText = document.select(".t-h1").get(0).ownText();
article.setTitle(ownText);
// 获取作者
String author = document.select(".author-name").get(0).text();
article.setAuthor(author);
// 获取时间 根据页面上的信息可知时间有两种表示
Elements elements = document.select("span[class=article-time pull-left]");
if (elements.size() == 0) {
String createTime = document.select(".article-time").get(0).ownText();
article.setCreateTime(createTime);
} else {
String createTime = elements.get(0).ownText();
article.setCreateTime(createTime);
}
// 获取文章内容
String content = document.select(".article-content-wrap").get(0).text();
article.setContent(content);
// 获取点赞
article.setZan(document.select(".num").get(0).ownText());
// 获取评论
article.setPl(document.select(".article-pl").get(0).ownText());
System.out.println(article);
articleDao.save(article);
}
}
}
}


// 解析数据 得到url
private static ArrayList<String> parseIndexHtml(String indexHtml) {
// TODO Auto-generated method stub
if (indexHtml != null) {
ArrayList<String> urls = new ArrayList<String>();
// 解析得到的页面的信息 将其变成文档对象
Document document = Jsoup.parse(indexHtml);
// 得到document对象后 就可以通过document对象来得到需要的东西
Elements elements = document.select(".mod-info-flow div[data-aid]");
for (Element element : elements) {
String url = element.attr("data-aid");
urls.add(url);
System.out.println(url);
}
return urls;
}
return null;
}


// 首页的获取
private static String getIndex() throws Exception {
String url = "https://www.huxiu.com";
// 发起一个get请求
HttpGet httpGet = new HttpGet(url);
// 设置请求头
httpGet.addHeader("User-Agent",
"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36");
// 返回页面的信息
return getHtml(httpGet);
}


// 执行发送请求的方法
private static String getHtml(HttpGet httpGet) throws Exception {
// TODO Auto-generated method stub
String html = null;
CloseableHttpClient httpClient = HttpClients.createDefault();
CloseableHttpResponse execute = httpClient.execute(httpGet);
// 判断响应码是否为200
if (execute.getStatusLine().getStatusCode() == 200) {
HttpEntity entity = execute.getEntity();
html = EntityUtils.toString(entity);
System.out.println(html);// 返回的的页面的所有信息
}
return html;
}

}

package cn.itcast.huxiu;


public class Article {
private int id;
private String title;
private String author;
private String createTime;
private String sc;
private String zan;
private String pl;
private String content;
private String url;

public int getId() {
return id;
}
public void setId(int id) {
this.id = id;
}
public String getTitle() {
return title;
}
public void setTitle(String title) {
this.title = title;
}
public String getAuthor() {
return author;
}
public void setAuthor(String author) {
this.author = author;
}
public String getCreateTime() {
return createTime;
}
public void setCreateTime(String createTime) {
this.createTime = createTime;
}
public String getSc() {
return sc;
}
public void setSc(String sc) {
this.sc = sc;
}
public String getZan() {
return zan;
}
public void setZan(String zan) {
this.zan = zan;
}
public String getPl() {
return pl;
}
public void setPl(String pl) {
this.pl = pl;
}
public String getContent() {
return content;
}
public void setContent(String content) {
this.content = content;
}
public String getUrl() {
return url;
}
public void setUrl(String url) {
this.url = url;
}
@Override
public String toString() {
return "Article [id=" + id + ", title=" + title + ", author=" + author + ", createTime=" + createTime + ", sc="
+ sc + ", zan=" + zan + ", pl=" + pl + ", content=" + content + ", url=" + url + "]";
}

}

package cn.itcast.huxiu;


import org.springframework.jdbc.core.JdbcTemplate;


import com.mchange.v2.c3p0.ComboPooledDataSource;


public class ArticleDao extends JdbcTemplate{
public ArticleDao() {
// 创建C3P0的datasource 1.配置 2.代码
ComboPooledDataSource dataSource = new ComboPooledDataSource();
// 1.url
// 2.driver
// 3.username&password
dataSource.setUser("root");
dataSource.setPassword("123");
dataSource.setJdbcUrl("jdbc:mysql://localhost:3306/spider?characterEncoding=utf-8");
setDataSource(dataSource);
}
public void save(Article article) {
String sql = "INSERT INTO huxiu_article (id, title, author, createTime, zan, pl, sc, content, url ) VALUES( ?,?,?,?,?,?,?,?,?)";
update(sql, article.getId(),article.getTitle(),article.getAuthor(),article.getCreateTime(),article.getZan(),article.getPl(),article.getSc(),article.getContent(),article.getUrl());
}
}