1.准备工作(依赖包):redis和mysql请自行准备,此案例涉及到爬取数据的保存和去重(通过redis)
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.6</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.47</version>
</dependency>
<dependency>
<groupId>org.mybatis</groupId>
<artifactId>mybatis</artifactId>
<version>3.4.6</version>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.38</version>
</dependency>
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<version>1.2.12</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.25</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.25</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>redis.clients</groupId>
<artifactId>jedis</artifactId>
<version>3.0.1</version>
</dependency>
2.配置文件
2.1.db.properties
driver=com.mysql.jdbc.Driver
url=jdbc:mysql:///spider?useUnicode=true&characterEncoding=UTF-8
username=root
password=admin
2.2.log4j.properties
# Global logging configuration
log4j.rootLogger=ERROR, stdout
# MyBatis logging configuration...
log4j.logger.com.test.spider=TRACE
# Console output...
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%5p [%t] - %m%n
2.3.mybatis.xml
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE configuration
PUBLIC "-//mybatis.org//DTD Config 3.0//EN"
"http://mybatis.org/dtd/mybatis-3-config.dtd">
<configuration>
<!--属性配置-->
<properties resource="db.properties"/>
<environments default="development">
<environment id="development">
<!--事务管理器-->
<transactionManager type="JDBC"/>
<!--数据源-->
<dataSource type="POOLED">
<property name="driver" value="${driver}"/>
<property name="url" value="${url}"/>
<property name="username" value="${username}"/>
<property name="password" value="${password}"/>
</dataSource>
</environment>
</environments>
<mappers>
<mapper resource="com/test/spider/mapper/UserMapper.xml"/>
</mappers>
</configuration>
3.实体类News:com.test.spider.bean;
package com.test.spider.bean;
import java.util.Date;
public class News {
private Integer id;
private String title;
private String intro;
private String source;
private String vurl;
private Date publishTime;
@Override
public String toString() {
return "News{" +
"id=" + id +
", title='" + title + '\'' +
", intro='" + intro + '\'' +
", source='" + source + '\'' +
", vurl='" + vurl + '\'' +
", publishTime=" + publishTime +
'}';
}
public Integer getId() {
return id;
}
public void setId(Integer id) {
this.id = id;
}
public String getTitle() {
return title;
}
public void setTitle(String title) {
this.title = title;
}
public String getIntro() {
return intro;
}
public void setIntro(String intro) {
this.intro = intro;
}
public String getSource() {
return source;
}
public void setSource(String source) {
this.source = source;
}
public String getVurl() {
return vurl;
}
public void setVurl(String vurl) {
this.vurl = vurl;
}
public Date getPublishTime() {
return publishTime;
}
public void setPublishTime(Date publishTime) {
this.publishTime = publishTime;
}
}
4.NewsMapper接口:com.test.spider.mapper;
package com.test.spider.mapper;
import com.test.spider.bean.News;
import org.apache.ibatis.annotations.Param;
import java.util.List;
public interface NewsMapper {
Integer batchSave(@Param("news") List<News> list);
}
5.NewsMapper.xml配置文件:com.test.spider.mapper;
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE mapper
PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
"http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.test.spider.mapper.NewsMapper">
<insert id="batchSave">
insert into news(title,intro,source,vurl,publish_time)
VALUES
<foreach collection="news" separator="," item="item">
(#{item.title},#{item.intro},#{item.source},#{item.vurl},#{item.publishTime})
</foreach>
</insert>
</mapper>
6.MybatisUtil工具类:com.test.spider.util;
package com.test.spider.util;
import org.apache.ibatis.io.Resources;
import org.apache.ibatis.session.SqlSession;
import org.apache.ibatis.session.SqlSessionFactory;
import org.apache.ibatis.session.SqlSessionFactoryBuilder;
import java.io.IOException;
import java.io.InputStream;
public class MybatisUtil {
private static SqlSessionFactory factory = null;
static{
String resource = "mybatis.xml";
InputStream in = null;
try {
in = Resources.getResourceAsStream(resource);
} catch (IOException e) {
e.printStackTrace();
}
factory = new SqlSessionFactoryBuilder().build(in);
}
public static SqlSession getSession(){
return factory.openSession();
}
}
7.EntertainmentSpider主要功能:com.test.spider;
package com.test.spider;
import com.alibaba.fastjson.JSON;
import com.sun.org.apache.bcel.internal.generic.NEW;
import com.test.spider.bean.News;
import com.test.spider.mapper.NewsMapper;
import com.test.spider.util.MybatisUtil;
import jdk.nashorn.internal.scripts.JD;
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.apache.ibatis.session.SqlSession;
import org.junit.Test;
import redis.clients.jedis.Jedis;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* Created by thinkpad on 2019/9/18.
*/
public class EntertainmentSpider {
public void main() throws Exception {
Integer page = 0;
while(true){
//1.确定Url
String indexUrl = "https://pacaio.match.qq.com/irs/rcd?cid=146&token=49cbb2154853ef1a74ff4e53723372ce&ext=ent&page="+page+"&callback=__jp7";
//2.发送请求,获取数据
//获取httpclient对象
CloseableHttpClient httpClient = HttpClients.createDefault();
//创建请求方式对象
HttpGet get = new HttpGet(indexUrl);
//执行请求操作
CloseableHttpResponse response = httpClient.execute(get);
if(response.getStatusLine().getStatusCode() == 200){
HttpEntity entity = response.getEntity();
String res = EntityUtils.toString(entity, "utf-8");
//3.解析数据
res = toJsonString(res); //截取字符串
//转为map对象
Map<String, Object> map = jsonToMap(res);
//页面没有数据使,结束循环
if(Integer.parseInt(map.get("datanum").toString()) == 0){
break;
}
//转换为新闻对象
List<News> list = mapToBean(((List<Map>) map.get("data")));
//4.保存数据
//保存新闻数据
if(list.size()>0){
Integer count = saveNews(list);
}
System.out.println(page);
}
page++;
}
System.out.println("执行结束....");
}
//截取结果字符串
public String toJsonString(String src){
int start = src.indexOf("{");
int end = src.lastIndexOf("}")+1;
return src.substring(start,end);
}
//json字符串转换为java对象
public Map<String,Object> jsonToMap(String src){
return JSON.parseObject(src, HashMap.class);
}
//新闻数据转换为新闻对象
public List<News> mapToBean(List<Map> src) throws ParseException {
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
List<News> list = new ArrayList<News>();
for (Map map : src) {
//获取url的值
String url = map.get("vurl").toString();
if(!checkUrl(url)){
continue;
}
News news = new News();
news.setTitle(map.get("title").toString());
news.setIntro(map.get("intro").toString());
news.setSource(map.get("source").toString());
news.setVurl(url);
String publish_time = map.get("publish_time").toString();
news.setPublishTime(format.parse(publish_time));
list.add(news);
}
return list;
}
//保存新闻的数据
public int saveNews(List<News> list){
SqlSession session = MybatisUtil.getSession();
NewsMapper mapper = session.getMapper(NewsMapper.class);
Integer res = mapper.batchSave(list);
session.commit();
session.close();
return res;
}
//判断url是否已经在redis中保存
public Boolean checkUrl(String url){
Jedis jedis = new Jedis("localhost",6379);
Long count = jedis.sadd("bigdata:0701:spider:news:url", url);
jedis.close();
return count>0?true:false;
}
}
8.App(测试类):com.test.spider;
package com.test.spider;
import com.test.spider.EntertainmentSpider;
import java.util.Timer;
import java.util.TimerTask;
public class App {
public static void main(String[] args) {
final EntertainmentSpider spider = new EntertainmentSpider();
//创建定时器
Timer timer = new Timer();
timer.schedule(
new TimerTask() {
@Override
public void run() {
try {
spider.main();
} catch (Exception e) {
e.printStackTrace();
}
}
},
0,
1200000L
);
}
}