前言
因为做毕设,所以需要爬取食品相关数据,而且爬取后要做分页。
思路:
先用Java写爬虫爬取数据,然后用layui做分页。
涉及技术点:
(1)Java爬虫
需要用到httpclient,因为接口是post,所以需要带参数的post请求,又因为协议是https协议,所以需要绕过证书验证。
(2)分页
layui做分页(其实做过了就很简单)
因为这篇是实践篇,所以主要放代码
代码
- controller
@RequestMapping(value = "/goods/data/",method = RequestMethod.GET)
public Result<List<GoodsData>> get(HttpServletRequest request){
// 通过HttpServletRequest获取 limit 和 page
int limit = Integer.valueOf(request.getParameter("limit"));
int page = Integer.valueOf(request.getParameter("page"));
String url = "https://sac.nifdc.org.cn/api/goods/data";
JSONObject jsonStr = new JSONObject();
jsonStr.put("check_flag","uq");
jsonStr.put("order_by","time");
jsonStr.put("pageNumber",page);
jsonStr.put("pageSize",limit);
String respStr = HttpClientUtil.doPost(url, jsonStr, "utf-8");
//String httpOrgCreateTestRtn = HttpClientUtil.doPost(url, jsonStr, "utf-8");
//fastjson 将 String 转换为 jsonObject
JSONObject goodData = JSON.parseObject(respStr);
JSONObject data = (JSONObject) goodData.get("data");
String msg = (String) goodData.get("msg");
int code = (int) goodData.get("code");
int count = (int) data.get("total");
JSONArray rows = (JSONArray) data.get("rows");
List<GoodsData> list = new ArrayList<>();
for (int i = 0; i < rows.size(); i++) {
JSONObject obj = (JSONObject) rows.get(i);
GoodsData goodsData = new GoodsData();
goodsData.setId((Integer) obj.get("id"));
goodsData.setFoodName((String) obj.get("food_name"));
goodsData.setProductionName((String) obj.get("production_name"));
goodsData.setFoodMode((String) obj.get("food_mode"));
goodsData.setCheckNum((Integer) obj.get("check_num"));
goodsData.setQalifiedNum((Integer) obj.get("qualified_num"));
goodsData.setUnqualifiedNum((Integer) obj.get("unqualified_num"));
list.add(goodsData);
}
return Result.success(list,count);
}
- httpUtil(自己写的工具包,包含SSLClient.java和HttpClientUtil.java)
SSLClient.java
package com.neu.foodchain.common.httpUtil;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;
import org.apache.http.conn.ClientConnectionManager;
import org.apache.http.conn.scheme.Scheme;
import org.apache.http.conn.scheme.SchemeRegistry;
import org.apache.http.conn.ssl.SSLSocketFactory;
import org.apache.http.impl.client.DefaultHttpClient;
public class SSLClient extends DefaultHttpClient {
public SSLClient() throws Exception{
super();
SSLContext ctx = SSLContext.getInstance("TLS");
X509TrustManager tm = new X509TrustManager() {
@Override
public void checkClientTrusted(X509Certificate[] chain,
String authType) throws CertificateException {
}
@Override
public void checkServerTrusted(X509Certificate[] chain,
String authType) throws CertificateException {
}
@Override
public X509Certificate[] getAcceptedIssuers() {
return null;
}
};
ctx.init(null, new TrustManager[]{tm}, null);
SSLSocketFactory ssf = new SSLSocketFactory(ctx,SSLSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
ClientConnectionManager ccm = this.getConnectionManager();
SchemeRegistry sr = ccm.getSchemeRegistry();
sr.register(new Scheme("https", 443, ssf));
}
}
HttpClientUtil.java
package com.neu.foodchain.common.httpUtil;
import com.alibaba.fastjson.JSONObject;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.client.HttpClient;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
/**
* 利用HttpClient进行post请求的工具类
*/
public class HttpClientUtil {
public static String doPost(String url, JSONObject jsonstr, String charset){
HttpClient httpClient = null;
HttpPost httpPost = null;
String result = null;
try{
httpClient = new SSLClient();
httpPost = new HttpPost(url);
List<NameValuePair> paramList = new ArrayList<>();
if(jsonstr != null && jsonstr.size() > 0){
Set<String> keySet = jsonstr.keySet();
for(String key : keySet) {
paramList.add(new BasicNameValuePair(key, String.valueOf(jsonstr.get(key))));
}
}
UrlEncodedFormEntity formEntity = null;
try {
formEntity = new UrlEncodedFormEntity(paramList,"utf8");
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
}
//(5)设置表单的Entity对象到Post请求中
httpPost.setEntity(formEntity);
HttpResponse response = httpClient.execute(httpPost);
if(response != null){
HttpEntity resEntity = response.getEntity();
if(resEntity != null){
result = EntityUtils.toString(resEntity,charset);
}
}
}catch(Exception ex){
ex.printStackTrace();
}
return result;
}
}
- layui实现分页
<!DOCTYPE html>
<html class="x-admin-sm">
<head>
<meta charset="UTF-8">
<title>食品安全</title>
<meta name="renderer" content="webkit">
<meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
<meta name="viewport" content="width=device-width,user-scalable=yes, minimum-scale=0.4, initial-scale=0.8,target-densitydpi=low-dpi" />
<link rel="stylesheet" href="css/font.css">
<link rel="stylesheet" href="css/xadmin.css">
<script src="lib/layui/layui.js" charset="utf-8"></script>
<script type="text/javascript" src="js/xadmin.js"></script>
<!--[if lt IE 9]>
<script src="https://cdn.staticfile.org/html5shiv/r29/html5.min.js"></script>
<script src="https://cdn.staticfile.org/respond.js/1.4.2/respond.min.js"></script>
<![endif]-->
</head>
<body>
<div class="x-nav">
<span class="layui-breadcrumb">
<a href="">食品安全</a>
<a>
<cite>食品安全</cite></a>
</span>
<a class="layui-btn layui-btn-small" style="line-height:1.6em;margin-top:3px;float:right" onclick="location.reload()" title="刷新">
<i class="layui-icon layui-icon-refresh" style="line-height:30px"></i>
</a>
</div>
<div class="layui-fluid">
<div class="layui-row layui-col-space15">
<div class="layui-col-md12">
<div class="layui-card">
<div class="layui-card-body ">
<div class="layui-form layui-col-space5">
<div class="layui-inline layui-show-xs-block">
<input type="text" name="pro_code" id="pro_code" placeholder="请输入商品追溯码" autocomplete="off" class="layui-input"></div>
<div class="layui-inline layui-show-xs-block">
<input type="text" name="pro_bar_code" id="pro_bar_code" placeholder="请输入商品条码" autocomplete="off" class="layui-input"></div>
<div class="layui-inline layui-show-xs-block">
<input type="text" name="manu_code" id="manu_code" placeholder="请输入生产批号" autocomplete="off" class="layui-input"></div>
<div class="layui-inline layui-show-xs-block">
<button class="layui-btn" lay-submit="" lay-filter="sreach" id="search">
<i class="layui-icon"></i></button>
</div>
<a class="layui-btn layui-btn-small" style="line-height:1.6em;margin-top:3px;float:right" onclick="desc()" title="使用说明">
<i class="layui-icon layui-icon-tips" style="line-height:30px"></i>
</a>
</div>
</div>
<div class="layui-card-body ">
<table id="goodsData" lay-filter="demoEvent"></table>
</div>
</div>
</div>
</div>
</div>
</body>
<script>
layui.use(['table','form', 'layedit', 'laydate','element','jquery','layer'], function() {
var form = layui.form,
layer = layui.layer,
element = layui.element,
$ = layui.jquery,
laydate = layui.laydate,
table = layui.table;
var url = "/goods/data/";
renderProductTable(url);
function renderProductTable(url) {
table.render({
elem: '#goodsData'
//,height: 312
,url: url //数据接口
,page: true //开启分页
,cols: [
[ //表头
{field: 'id', title: 'ID', width:230, sort: true, fixed: 'left'}
,{field: 'foodName',width:150, title: '食品名称'}
,{field: 'productionName',width:150, title: '企业名称'}
,{field: 'foodMode', title: '包装方式', width:200, sort: true}
,{field: 'checkNum', title: '抽检数量', width:80}
,{field: 'qalifiedNum', title: '合格数量', width:80}
,{field: 'unqualifiedNum', title: '不合格数量', width:80}
]
]
});
};
});
</script>
<script>var _hmt = _hmt || []; (function() {
var hm = document.createElement("script");
hm.src = "https://hm.baidu.com/hm.js?b393d153aeb26b46e9431fabaf0f6190";
var s = document.getElementsByTagName("script")[0];
s.parentNode.insertBefore(hm, s);
})();</script>
</html>