mysql新闻类型数据库_分类爬取新闻并存入mysql数据库

packagecom.me.service;importcom.google.gson.Gson;importcom.me.dao.NewsDao;importcom.me.domain.News;importcom.me.domain.NewsListData;importcom.me.utils.HttpUtil;importcom.me.utils.JsoupNewsUtil;importjava.sql.SQLException;importjava.util.List;public classNewsList {

NewsDao dao= newNewsDao();public static void main(String[] args) throwsSQLException {

NewsList newsList= newNewsList();/*String url = "https://3g.163.com/touch/reconstruct/article/list/";

String [] typeArray = {"BBM54PGAwangning","BA10TA81wangning","BA8E6OEOwangning"

,"BA8EE5GMwangning","BAI67OGGwangning","BA8D4A3Rwangning","BAI6I0O5wangning"

,"BAI6JOD9wangning","BA8F6ICNwangning","BAI6RHDKwangning","BA8FF5PRwangning"

,"BDC4QSV3wangning","BEO4GINLwangning"};

for (int i = 0; i < typeArray.length; i++) {

}*/newsList.deleteAll();

newsList.addXW("https://3g.163.com/touch/reconstruct/article/list/BBM54PGAwangning/0-20.html");

newsList.addYL("https://3g.163.com/touch/reconstruct/article/list/BA10TA81wangning/0-20.html");

newsList.addTY("https://3g.163.com/touch/reconstruct/article/list/BA8E6OEOwangning/0-20.html");

newsList.addCJ("https://3g.163.com/touch/reconstruct/article/list/BA8EE5GMwangning/0-20.html");

newsList.addJS("https://3g.163.com/touch/reconstruct/article/list/BAI67OGGwangning/0-20.html");

newsList.addKJ("https://3g.163.com/touch/reconstruct/article/list/BA8D4A3Rwangning/0-20.html");

newsList.addSJ("https://3g.163.com/touch/reconstruct/article/list/BAI6I0O5wangning/0-20.html");

newsList.addSM("https://3g.163.com/touch/reconstruct/article/list/BAI6JOD9wangning/0-20.html");

newsList.addSS("https://3g.163.com/touch/reconstruct/article/list/BA8F6ICNwangning/0-20.html");

newsList.addYX("https://3g.163.com/touch/reconstruct/article/list/BAI6RHDKwangning/0-20.html");

newsList.addJY("https://3g.163.com/touch/reconstruct/article/list/BA8FF5PRwangning/0-20.html");

newsList.addJK("https://3g.163.com/touch/reconstruct/article/list/BDC4QSV3wangning/0-20.html");

newsList.addLY("https://3g.163.com/touch/reconstruct/article/list/BEO4GINLwangning/0-20.html");

newsList.zw();//newsList.test();

}/*** type : 军事

*@paramurl

*@throwsSQLException*/

public void addJS(String url) throwsSQLException {

String data=HttpUtil.setUrl(url);

Gson gson= newGson();

String ss= data.substring(9,data.length()-1);

NewsListData newsListData= gson.fromJson(ss, NewsListData.class);for (int i = 0; i < newsListData.getBAI67OGGwangning().size(); i++) {

News n=newsListData.getBAI67OGGwangning().get(i);

n.setType("军事");

dao.add(n);

}

}/*** type : 旅游

*@paramurl

*@throwsSQLException*/

public void addLY(String url) throwsSQLException {

String data=HttpUtil.setUrl(url);

Gson gson= newGson();

String ss= data.substring(9,data.length()-1);

NewsListData newsListData= gson.fromJson(ss, NewsListData.class);for (int i = 0; i < newsListData.getBEO4GINLwangning().size(); i++) {

News n=newsListData.getBEO4GINLwangning().get(i);

n.setType("旅游");

dao.add(n);

}

}/*** type : 健康

*@paramurl

*@throwsSQLException*/

public void addJK(String url) throwsSQLException {

String data=HttpUtil.setUrl(url);

Gson gson= newGson();

String ss= data.substring(9,data.length()-1);

NewsListData newsListData= gson.fromJson(ss, NewsListData.class);for (int i = 0; i < newsListData.getBDC4QSV3wangning().size(); i++) {

News n=newsListData.getBDC4QSV3wangning().get(i);

n.setType("健康");

dao.add(n);

}

}/*** type : 教育

*@paramurl

*@throwsSQLException*/

public void addJY(String url) throwsSQLException {

String data=HttpUtil.setUrl(url);

Gson gson= newGson();

String ss= data.substring(9,data.length()-1);

NewsListData newsListData= gson.fromJson(ss, NewsListData.class);for (int i = 0; i < newsListData.getBA8FF5PRwangning().size(); i++) {

News n=newsListData.getBA8FF5PRwangning().get(i);

n.setType("教育");

dao.add(n);

}

}/*** type : 游戏

*@paramurl

*@throwsSQLException*/

public void addYX(String url) throwsSQLException {

String data=HttpUtil.setUrl(url);

Gson gson= newGson();

String ss= data.substring(9,data.length()-1);

NewsListData newsListData= gson.fromJson(ss, NewsListData.class);for (int i = 0; i < newsListData.getBAI6RHDKwangning().size(); i++) {

News n=newsListData.getBAI6RHDKwangning().get(i);

n.setType("游戏");

dao.add(n);

}

}/*** type : 时尚

*@paramurl

*@throwsSQLException*/

public void addSS(String url) throwsSQLException {

String data=HttpUtil.setUrl(url);

Gson gson= newGson();

String ss= data.substring(9,data.length()-1);

NewsListData newsListData= gson.fromJson(ss, NewsListData.class);for (int i = 0; i < newsListData.getBA8F6ICNwangning().size(); i++) {

News n=newsListData.getBA8F6ICNwangning().get(i);

n.setType("时尚");

dao.add(n);

}

}/*** type : 数码

*@paramurl

*@throwsSQLException*/

public void addSM(String url) throwsSQLException {

String data=HttpUtil.setUrl(url);

Gson gson= newGson();

String ss= data.substring(9,data.length()-1);

NewsListData newsListData= gson.fromJson(ss, NewsListData.class);for (int i = 0; i < newsListData.getBAI6JOD9wangning().size(); i++) {

News n=newsListData.getBAI6JOD9wangning().get(i);

n.setType("数码");

dao.add(n);

}

}/*** type : 手机

*@paramurl

*@throwsSQLException*/

public void addSJ(String url) throwsSQLException {

String data=HttpUtil.setUrl(url);

Gson gson= newGson();

String ss= data.substring(9,data.length()-1);

NewsListData newsListData= gson.fromJson(ss, NewsListData.class);for (int i = 0; i < newsListData.getBAI6I0O5wangning().size(); i++) {

News n=newsListData.getBAI6I0O5wangning().get(i);

n.setType("手机");

dao.add(n);

}

}/*** type : 科技

*@paramurl

*@throwsSQLException*/

public void addKJ(String url) throwsSQLException {

String data=HttpUtil.setUrl(url);

Gson gson= newGson();

String ss= data.substring(9,data.length()-1);

NewsListData newsListData= gson.fromJson(ss, NewsListData.class);for (int i = 0; i < newsListData.getBA8D4A3Rwangning().size(); i++) {

News n=newsListData.getBA8D4A3Rwangning().get(i);

n.setType("科技");

dao.add(n);

}

}/*** type : 财经

*@paramurl

*@throwsSQLException*/

public void addCJ(String url) throwsSQLException {

String data=HttpUtil.setUrl(url);

Gson gson= newGson();

String ss= data.substring(9,data.length()-1);

NewsListData newsListData= gson.fromJson(ss, NewsListData.class);for (int i = 0; i < newsListData.getBA8EE5GMwangning().size(); i++) {

News n=newsListData.getBA8EE5GMwangning().get(i);

n.setType("财经");

dao.add(n);

}

}/*** type : 体育

*@paramurl

*@throwsSQLException*/

public void addTY(String url) throwsSQLException {

String data=HttpUtil.setUrl(url);

Gson gson= newGson();

String ss= data.substring(9,data.length()-1);

NewsListData newsListData= gson.fromJson(ss, NewsListData.class);for (int i = 0; i < newsListData.getBA8E6OEOwangning().size(); i++) {

News n=newsListData.getBA8E6OEOwangning().get(i);

n.setType("体育");

dao.add(n);

}

}/*** type : 娱乐

*@paramurl

*@throwsSQLException*/

public void addYL(String url) throwsSQLException {

String data=HttpUtil.setUrl(url);

Gson gson= newGson();

String ss= data.substring(9,data.length()-1);

NewsListData newsListData= gson.fromJson(ss, NewsListData.class);for (int i = 0; i < newsListData.getBA10TA81wangning().size(); i++) {

News n=newsListData.getBA10TA81wangning().get(i);

n.setType("娱乐");

dao.add(n);

}

}/*** 新闻

*@paramurl

*@throwsSQLException*/

public void addXW(String url) throwsSQLException {

String data=HttpUtil.setUrl(url);

Gson gson= newGson();

String ss= data.substring(9,data.length()-1);

NewsListData newsListData= gson.fromJson(ss, NewsListData.class);for (int i = 0; i < newsListData.getBBM54PGAwangning().size(); i++) {

News n=newsListData.getBBM54PGAwangning().get(i);

n.setType("新闻");

dao.add(n);

}

}/*** 正文

*@throwsSQLException*/

public void zw() throwsSQLException {

List news =dao.newsList();for (int i = 0; i < news.size(); i++) {

News n=news.get(i);

String s=HttpUtil.setUrl(n.getUrl());

String zw=JsoupNewsUtil.zw(s);

dao.zw(zw,n.getId());

}

}/*** 清空

*@throwsSQLException*/

public void deleteAll() throwsSQLException {

dao.deleteAll();

}/*** 测试*/

public voidtest (){

Gson gson= newGson();

News news= gson.fromJson("{\n" +

" \"imgextra\": [\n" +

" {\n" +

" \"imgsrc\": \"http://cms-bucket.ws.126.net/2020/0410/4ef9af5aj00q8jdsh00bpc000sg00sgc.jpg\"\n" +

" },\n" +

" {\n" +

" \"imgsrc\": \"http://cms-bucket.ws.126.net/2020/0410/6f7bd38bj00q8jdsh003xc000sg00dic.jpg\"\n" +

" }\n" +

" ],\n" +

" \"liveInfo\": null,\n" +

" \"docid\": \"17KK0006|2145433\",\n" +

" \"source\": \"极客鲜疯队\",\n" +

" \"title\": \"宅家赏美丽高原 一生中值得一看的美景\",\n" +

" \"priority\": 150,\n" +

" \"url\": \"17KK0006|2145433\",\n" +

" \"skipURL\": \"http://3g.163.com/touch/photoview.html?channelid=0006&setid=2145433\",\n" +

" \"commentCount\": 15,\n" +

" \"imgsrc3gtype\": \"2\",\n" +

" \"stitle\": \"17KK0006|2145433\",\n" +

" \"digest\": \"\",\n" +

" \"skipType\": \"photoset\",\n" +

" \"photosetID\": \"0006|2145433\",\n" +

" \"imgsrc\": \"http://cms-bucket.ws.126.net/2020/0410/1bd79be9j00q8jdsh008tc000sg00izc.jpg\",\n" +

" \"ptime\": \"2020-04-10 03:51:04\",\n" +

" \"modelmode\": \"u\"\n" +

" }", News.class);

System.out.println(news.toString());

}

}