该项目简单思路就是通过jsoup抓取到的数据先存入到ArrayList中,然后再通过jdbc存入到数据库。
创建的是maven项目,需要在pom.xml中添加两个依赖:
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>1.11.3</version>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.38</version>
</dependency>
创建实体类:
package dao;
/**
* 实体类
*/
public class Blog {
private String title;//标题
private String href;//链接
private String author;//作者
private String reads;//阅读量
public String getTitle() {
return title;
}
public void setTitle(String title) {
this.title = title;
}
public String getHref() {
return href;
}
public void setHref(String href) {
this.href = href;
}
public String getAuthor() {
return author;
}
public void setAuthor(String author) {
this.author = author;
}
public String getReads() {
return reads;
}
public void setReads(String reads) {
this.reads = reads;
}
public Blog(String title, String href, String author, String reads) {
super();
this.title = title;
this.href = href;
this.author = author;
this.reads = reads;
}
public Blog(){
super();
}
@Override
public String toString() {
return "Blog{" +
"title='" + title + '\'' +
", href='" + href + '\'' +
", author='" + author + '\'' +
", reads='" + reads + '\'' +
'}';
}
}
然后进行数据爬取操作和存数据库操作:
package control;
import dao.Blog;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList;
public class Grab {
private static Document doc;
public static void main(String[] args) {
try {
//获取的网页链接
doc = Jsoup.connect("https://www.cnblogs.com/").get();
} catch (IOException e) {
e.printStackTrace();
}
//调用Blog类
BlogZhua();
}
/**
* 抓取网页数据
*/
public static void BlogZhua() {
//获取网页中的元素
Elements eles = doc.select("div#post_list>div.post_item");
//将抓取的数据存入到ArrayList集合中
ArrayList<Blog> arrayList = new ArrayList();
//使用for循环遍历网页中的数据
for (Element ele : eles) {
//抓取页面中的文章标题
String txt = ele.select("div.post_item_body>h3>a.titlelnk").text();
//抓取页面中的文章链接
String href = ele.select("div.post_item_body>h3>a.titlelnk").attr("href");
//爬取文章作者
String author = ele.select("div.post_item_foot > a.lightblue").text();
//爬取文章发布时间
String reads = ele.select("div.post_item_foot > span.article_view > a.gray").text();
//将数据添加到集合中
Blog blog = new Blog();
blog.setTitle(txt);
blog.setHref(href);
blog.setAuthor(author);
blog.setReads(reads);
arrayList.add(blog);
}
//遍历ArrayList集合
for (Blog test : arrayList) {
// System.out.println("数据:" + test);
//此处调用AddBlog类,把集合中的数据添加到数据库中
AddBlog(test);
}
}
/**
* 数据库操作
*
* @param blog
*/
public static void AddBlog(Blog blog) {
try {
//加载mysql驱动
Class.forName("com.mysql.jdbc.Driver");
} catch (ClassNotFoundException e) {
e.printStackTrace();
}
Connection connection = null;
PreparedStatement pstmt = null;
try {
//连接mysql
connection = DriverManager.getConnection("jdbc:mysql://localhost:3306/jsoup?useSSL=true", "root", "1234");
//编写sql语句
String sql = "INSERT INTO `content`(`title`, `href`, `author`, `reads`) VALUES (?,?,?,?)";
pstmt = connection.prepareStatement(sql);
pstmt.setString(1, blog.getTitle());
pstmt.setString(2, blog.getHref());
pstmt.setString(3, blog.getAuthor());
pstmt.setString(4, blog.getReads());
int result = pstmt.executeUpdate();
//验证数据是否添加成功
if (result > 0) {
System.out.println("数据添加成功!");
}
} catch (SQLException throwables) {
throwables.printStackTrace();
System.out.println("数据库访问失败!");
}
}
}