一、controller层
package com.hd.history.controller;
import com.hd.history.dto.HistoryDto;
import com.hd.history.entity.History;
import com.hd.history.service.HistoryService;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@RestController
@RequestMapping("/findHistoryday")
public class HistoryController {
@Autowired
private HistoryService historyService;
@Scheduled(cron = "0 0 7 1 * ? ")//定时每个月1号7点执行一次
public void getHistoryMonthDay() throws IOException {
Integer[] dayNumber = {31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31};
//循环每一天
for (int i = 1; i <= 12; i++) {
for (int j = 1; j <= dayNumber[i - 1]; j++) {
getgetHistoday(i, j);
}
}
}
public void getgetHistoday(Integer month, Integer day) throws IOException {
//爬虫目标源网页的url路径要求不够10的0补齐
String monthDay = "";
String monthNumber = String.valueOf(month);
String dayNumber = String.valueOf(day);
if (month < 10) {
monthNumber = "0" + month;
}
if (day < 10) {
dayNumber = "0" + day;
}
monthDay = monthNumber + dayNumber;
Document document = Jsoup.connect("http://hao.360.com/histoday/" + monthDay + ".html").get();
Element body = document.body();
Elements lie12 = body.getElementsByClass("tih-item");
List<History> historyList = new ArrayList<>();
//这方面我们主要需要三个数据 标题图片和具体内容
for (Element element : lie12) {
//导入标题
Elements titleData = element.select("dt");
String title = titleData.text();
//导入图片
Elements imageData = element.getElementsByClass("item-img");
String image = imageData.attr("src");
String imageNumber = imageData.attr("data-src");
String imageUrl = "";
if (null!=imageNumber&&imageNumber!=""){
imageUrl = "http:" + imageNumber;
}else if (null != image && image != "") {
imageUrl = "http:" + image;
}
//具体内容
Elements contentData = element.getElementsByClass("desc");
String content = contentData.text();
History history = new History();
history.setTitle(title);
history.setImg(imageUrl);
history.setContent(content);
historyList.add(history);
}
String finalMonthDay = monthDay;
historyList.forEach(history -> history.setHistoryDate(finalMonthDay));
historyService.insertHistory(historyList);
}
//查找接口
@PostMapping("/getHistoryDay")
public List<History> getHistoryList(@RequestBody HistoryDto historyDto) {
return historyService.getHistoday(historyDto);
}
}
二、service层
package com.hd.history.service.impl;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.hd.history.dto.HistoryDto;
import com.hd.history.entity.History;
import com.hd.history.mapper.HistoryMapper;
import com.hd.history.service.HistoryService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.List;
@Service
public class HistoryServiceImpl implements HistoryService {
@Autowired
private HistoryMapper historyMapper;
@Override
public List<History> getHistoday(HistoryDto historyDto) {
QueryWrapper<History> queryWrapper = new QueryWrapper<>();
queryWrapper.lambda().eq(History::getHistoryDate, historyDto.getHistoryDate());
List<History> histories = historyMapper.selectList(queryWrapper);
return histories;
}
//采取的是先删除在插入,因为不好考虑唯一索引,而且整体删除也用不了多少时间
@Override
public void insertHistory(List<History> historyList) {
findHistory(historyList.get(0).getHistoryDate());
for (History history : historyList) {
historyMapper.insert(history);
}
}
public void findHistory(String historyDate) {
QueryWrapper<History> queryWrapper = new QueryWrapper<>();
queryWrapper.lambda().eq(History::getHistoryDate, historyDate);
List<History> histories = historyMapper.selectList(queryWrapper);
if (histories.size() > 0) {
historyMapper.delete(queryWrapper);
}
}
}
三、entity层
package com.hd.history.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.io.Serializable;
@Data
@TableName("hd_history")
public class History implements Serializable {
private static final long serialVersionUID = 1L;
@TableId(value = "id", type = IdType.AUTO)
private Long id;
private String historyDate;
private String title;
private String img;
private String content;
}
四、pom文件
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.hd.history</groupId>
<artifactId>maven-history</artifactId>
<version>1.0-SNAPSHOT</version>
<name>maven_history</name>
<description>Demo project for Spring Boot</description>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.2.1.RELEASE</version>
</parent>
<properties>
<java.version>1.8</java.version>
<cloud.version>Hoxton.RELEASE</cloud.version>
<cloud.alibaba.version>0.2.2.RELEASE</cloud.alibaba.version>
<gmall.version>1.0</gmall.version>
<mybatis-plus.version>3.0.5</mybatis-plus.version>
<mysql.version>5.1.46</mysql.version>
<swagger.version>2.7.0</swagger.version>
<fastdfs.version>1.27.0.0</fastdfs.version>
<lombok.version>1.18.10</lombok.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-dependencies</artifactId>
<version>Hoxton.RELEASE</version>
<type>pom</type>
<scope>import</scope>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-alibaba-dependencies</artifactId>
<version>0.2.2.RELEASE</version>
<type>pom</type>
<scope>import</scope>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<!--mybatis-plus 持久层-->
<dependency>
<groupId>com.baomidou</groupId>
<artifactId>mybatis-plus-boot-starter</artifactId>
<version>3.0.5</version>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.46</version>
</dependency>
<!-- jsoup HTML parser library @ http://jsoup.org/ -->
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>1.11.2</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-tomcat</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>
五、application.properties
# 服务端口
server.port=8088
# 服务名
spring.application.name=maven-history
# 环境设置:dev、test、prod
spring.profiles.active=dev
# mysql数据库连接
spring.datasource.driver-class-name=com.mysql.jdbc.Driver
spring.datasource.url=jdbc:mysql://localhost:3306/media?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=Asia/Shanghai&rewriteBatchedStatements=true&useServerPrepStmts=true&allowMultiQueries=true
spring.datasource.username=root
spring.datasource.password=root
#mybatis日志
mybatis-plus.configuration.log-impl=org.apache.ibatis.logging.stdout.StdOutImpl
六、表结构
DROP TABLE IF EXISTS `hd_history`;
CREATE TABLE `hd_history` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`history_date` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT '' COMMENT '当前日期',
`title` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT '' COMMENT '标题',
`img` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL COMMENT '图像',
`content` text CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL COMMENT '内容',
PRIMARY KEY (`id`) USING BTREE
)
七、爬取的网页HTML
<dl class="tih-item open">
<dt>
<em>1</em>. 1981年-我国正式实行学位制度
</dt>
<dd>
<img class="item-img" src="//p7.qhmsg.com/t01fd70f46cc423a8d7.jpg">
<div class="clearfix">
<div class="desc">
1981年1月1日,我国正式实行学位制度。我国从50年代开始,就试图建立学位制,因受极“左”思潮的影响和“文革”的干扰,一直未能实行。1976年,教育部和国务院科技干部局联合组织的学位小组,拟订了《中华人民共和国学位条例(草案)》,经审议修改,1980年2月15日由人大常委会第十三次会议讨论通过。学位制规定,我国设学士、硕士、博士3级学位。这些当时还看不出分量的名称后来却极大地规范了人才选拔制度。
</div>
<div class="right-btn-container">
<a href="http://baike.so.com/doc/6222143.html" target="_blank" class="read-btn">阅读全文</a>
</div>
</div>
</dd>
</dl>
<dl class="tih-item">
<dt>
<em>2</em>. 1814年-太平天国农民起义领袖洪秀全诞辰
</dt>
<dd>
<img class="item-img" data-src="//p8.qhmsg.com/t014369be2b3746e6d2.jpg" src="//hao3.qhimg.com/t0194aea54158184820.gif">
<div class="clearfix">
<div class="desc">
洪秀全(1814年1月1日-1864年6月1日),原名洪仁坤,小名火秀,客家人,是太平天国以宗教名义发动民变的领袖。清嘉庆十八年十二月初十生于广东花县(今广州市花都区)福源水村,后来移居到官禄布村。道光年间屡应科举不中,遂吸取早期基督教义中的平等思想,创立拜上帝会,撰《原道救世歌》以布教,主张建立远古“天下为公”盛世。道光三十年十二月初十(1851年1月11日)洪秀全发动金田起义,建国号太平天国,自称天王。1853年以南京作为首都,改名天京,1864年在天京病逝,太平天国在他死后不久灭亡。
</div>
<div class="right-btn-container">
<a href="http://baike.so.com/doc/4371050.html" target="_blank" class="read-btn">阅读全文</a>
</div>
</div>
</dd>
</dl>
总结
爬虫Jsoup这方面的知识以前从未接触过,通过实现(历史上的今天)学到了很多,解决问题的话需要静下心来,一个一个的去探索发掘。