mysql 自定义 存储_webmagic自定义存储（mysql、redis存储）

com.mdd.pip.pipeLine;importjava.util.List;importorg.springframework.beans.factory.annotation.Autowired;importorg.springframework.stereotype.Component;importcom.mdd.pip.model.ProxyIp;importcom.mdd.pip.service.ProxyIpService;importus.codecraft.webmagic.ResultItems;importus.codecraft.webmagic.Task;importus.codecraft.webmagic.pipeline.Pipeline;

@Componentpublic class DataPipeLine implementsPipeline {

@AutowiredprivateProxyIpService proxyIpService;/*** mysql 存储*/

/** public void process(ResultItems resultItems, Task task) {

* ListproxyIpList = resultItems.get("proxyIpList");

* if(proxyIpList!=null&&!proxyIpList.isEmpty()){

* proxyIpService.saveProxyIpList(proxyIpList); }

*

* }*/

/*** redis 存储*/

public voidprocess(ResultItems resultItems, Task task) {

List proxyIpList = resultItems.get("proxyIpList");if (proxyIpList != null && !proxyIpList.isEmpty()) {

proxyIpService.saveProxyListIpInRedis(proxyIpList);

}

}

}

ResultItems 对象本质是一个Map。因此要我们保存对象的时候，只需要在爬取时把爬取得数据封装成对象，保存在

ResultItems 里即可。如果有很多数据，则可以考虑用List保存。

packagecom.mdd.pip.crawler;importjava.util.ArrayList;importjava.util.List;importorg.apache.log4j.Logger;importorg.jsoup.nodes.Document;importorg.jsoup.nodes.Element;importorg.jsoup.select.Elements;importorg.springframework.stereotype.Component;importcom.mdd.pip.model.ProxyIp;importus.codecraft.webmagic.Page;importus.codecraft.webmagic.Site;importus.codecraft.webmagic.processor.PageProcessor;/*** 热刺代理网站ip抓取

*

*@authorxwl 2017.6.3*/@Componentpublic class XiCiProxyIpCrawler implementsPageProcessor {private Logger logger = Logger.getLogger(XiCiProxyIpCrawler.class);//部分一：抓取网站的相关配置，包括编码、抓取间隔、重试次数等

private Site site = Site.me().setCycleRetryTimes(3).setRetryTimes(3).setSleepTime(1000)

.setUserAgent("Mozilla/5.0 (Windows NT 6.1; WOW64; rv:53.0) Gecko/20100101 Firefox/53.0");publicSite getSite() {returnsite;

}public voidprocess(Page page) {

Document html=page.getHtml().getDocument();//结果集

List proxyIpList = new ArrayList();

Elements trElements= html.getElementById("ip_list").getElementsByTag("tr");for(Element trEle : trElements) {

Elements tdElements= trEle.getElementsByTag("td");if (tdElements == null||tdElements.size()<=0) {continue;

}try{

ProxyIp proxyIp= newProxyIp();

String ip= tdElements.get(1).text();

String proxyPort= tdElements.get(2).text();

String ipAddress= tdElements.get(3).text();

String anonymity= tdElements.get(4).text();

String proxyType= tdElements.get(5).text();

String aliveTime= tdElements.get(6).text();

proxyIp.setProxyIp(ip);

proxyIp.setProxyPort(Integer.parseInt(proxyPort));

proxyIp.setAliveTime(aliveTime);

proxyIp.setAnonymity(anonymity);

proxyIp.setIpAddress(ipAddress);

proxyIp.setProxyType(proxyType);

logger.info(proxyIp.getProxyIp()+":"+proxyIp.getProxyPort());

proxyIpList.add(proxyIp);

}catch(Exception e) {

logger.error("IP代理解析出错！", e);

}

}

page.putField("proxyIpList", proxyIpList);

}

}

page.putField("proxyIpList", proxyIpList);本质是设值到了ResultItems对象里了。

这样插件式、定制化很值的我们借鉴。希望下一步看源码。

参考：

http://webmagic.io/docs/zh/posts/ch1-overview/architecture.html

webmagic自定义存储(mysql、redis存储)

标签：nts   html   否则   自定义   com   文件   raw   控制台   processor

本条技术文章来源于互联网，如果无意侵犯您的权益请点击此处反馈版权投诉

本文系统来源：http://www.cnblogs.com/xwlhyy1072552712/p/7045930.html