1、解析器
package com.yihaodian.pis.crawler;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.htmlparser.Node;
import org.htmlparser.NodeFilter;
import org.htmlparser.Parser;
import org.htmlparser.filters.HasAttributeFilter;
import org.htmlparser.tags.Bullet;
import org.htmlparser.tags.BulletList;
import org.htmlparser.tags.Div;
import org.htmlparser.tags.ImageTag;
import org.htmlparser.tags.LinkTag;
import org.htmlparser.tags.ScriptTag;
import org.htmlparser.tags.Span;
import org.htmlparser.util.NodeList;
import org.htmlparser.util.ParserException;
import com.yihaodian.pis.dto.BestSellerDto;
public class SuningPageParser extends PageParser{
private static final Logger logger = Logger.getLogger(SuningPageParser.class);
public SuningPageParser(String html, String charset) {
super(html, charset);
// TODO Auto-generated constructor stub
}
@Override
public List<BestSellerDto> extractBestSeller(String bestSellerHtml)
throws ParserException {
List<BestSellerDto> sellers = new ArrayList<BestSellerDto>();
Parser parser = Parser.createParser(bestSellerHtml, charset);
NodeFilter filter = new HasAttributeFilter("class", "product_list02 profix02 clearfix");
NodeList nodeList = parser.extractAllNodesThatMatch(filter);
if (nodeList == null || nodeList.size() == 0) {
return null;
}
Node div = nodeList.elementAt(0);
NodeList divchildren = div.getChildren();
BulletList ul = (BulletList)divchildren.elementAt(1);
NodeList children = ul.getChildren();
BestSellerDto bestSeller = null;
for (int i = 0; i < children.size(); i++) {
bestSeller = new BestSellerDto();
Node child = children.elementAt(i);
if (child instanceof Bullet) {
Bullet li = (Bullet) child;
Span nameDiv = (Span) findTagByClassName(li, "pro_intro");
//寻找tagName是 LinkTag 的那个
LinkTag link = (LinkTag)findTagByName(nameDiv,"LinkTag");
bestSeller.setName(link.getLinkText());
bestSeller.setUrl("http://www.suning.cn"+link.getLink());
Span pricespan = (Span) findTagByClassName(li, "pro_price");
String priceteString =pricespan.getChildrenHTML().replace("<em>", "").replace("</em>", "").replace("¥", "");
bestSeller.setPrice(priceteString);
logger.info("畅销单品:" + bestSeller);
sellers.add(bestSeller);
} else {
continue;
}
}
return sellers;
}
@Override
public String extractNextPageUrlPattern(String bestSellerHtml)
throws ParserException {
String nextPageUrl="";
Parser parser = Parser.createParser(bestSellerHtml, "utf-8");
NodeFilter filter = new HasAttributeFilter("type", "text/javascript");;
NodeList children = parser.extractAllNodesThatMatch(filter);
if (children == null || children.size() == 0) {
System.out.println("没有值");
}else{
System.out.println("有值");
}
for (int i = 0; i < children.size(); i++) {
ScriptTag child = (ScriptTag) children.elementAt(i);
if(child.findPositionOf("¤tPage=")==0){
String putInCart1 = null;
String putInCart2 = null;
//Pattern pattern2 = Pattern.compile("(?<=currentPage[)] \\{)([^\\}]*?)(?=\\})");
Pattern pattern2 =
Pattern.compile("(?<=var[ \\s]{0,100}(jumpUrl)[\\s]{0,100}[=][\\s]{0,100}[\"])(.*?)(?=\"\\s{0,100}[+])");
Pattern pattern1 = Pattern.compile("(?<=var[ \\s]{0,100}dfy\\s{0,100}=\\s{0,100}[\"])(.*?)(?=[\"][\\s]{0,100})");
Matcher matcher1 = pattern1.matcher(child.getChildrenHTML());
if (matcher1.find()) {
putInCart1 = matcher1.group(0).trim();
}
Matcher matcher2 = pattern2.matcher(child.getChildrenHTML());
if (matcher2.find()) {
putInCart2 = matcher2.group(0).trim();
}
//System.out.println(putInCart2.substring(15, putInCart2.indexOf(" + dfy")-1));
nextPageUrl=putInCart2+putInCart1;
}
}
// parser = Parser.createParser(bestSellerHtml, "utf-8");
//得到当前页currentPage
// String currentPage="";
// filter = new HasAttributeFilter("class", "on");
// children = parser.extractAllNodesThatMatch(filter);
//LinkTag dLinkTag = (LinkTag) children.elementAt(0);
//System.out.println(dLinkTag.getLinkText());
// for (int i = 0; i < children.size(); i++) {
// Node node =children.elementAt(i);
// if (node.getChildren().size()<2) {
// LinkTag dLinkTag = (LinkTag)node;
// if(dLinkTag.getLink().equals("#"))
// currentPage= dLinkTag.getLinkText();
// }
// }
nextPageUrl+="&ip_sortBy=salevolumn0&sortType=4¤tPage=";// + currentPage;
logger.info("畅销榜下一页URL模式:" + nextPageUrl);
return nextPageUrl;
}
@Override
public String getNextPageUrl(String nextPageUrlPattern, int pageNum) {
StringBuilder sb = new StringBuilder();
sb.append("http://www.suning.cn/webapp/wcs/stores/servlet/");
sb.append(nextPageUrlPattern+(pageNum-1));
String nextPageUrl = sb.toString();
return nextPageUrl;
}
@Override
public String extractName(Map<String, String> params)
throws ParserException {
// TODO Auto-generated method stub
return null;
}
@Override
public String extractPrice(Map<String, String> params)
throws ParserException {
// TODO Auto-generated method stub
return null;
}
@Override
public String extractBrand(Map<String, String> params)
throws ParserException {
// TODO Auto-generated method stub
return null;
}
@Override
public String extractImageUrl(Map<String, String> params)
throws ParserException {
// TODO Auto-generated method stub
return null;
}
@Override
public boolean hasProduct(Map<String, String> params)
throws ParserException {
// TODO Auto-generated method stub
return false;
}
}
2、主程序DAO
public List<BestSellerDto> fetchBestSeller(Integer id, int amount) {
List<BestSellerDto> bestSellers = new ArrayList<BestSellerDto>();
if (amount <= 0) {
logger.warn("畅销品数目不能为负数!");
return null;
}
SiteCategoryDto siteCategory = getSiteCategoryById(id);
// 抓取参数
Map<String, String> params;
// 组织抓取参数与页面参数
params = pageParamItemDao.getPageConfigBySite(siteCategory.getSiteId());
params.putAll(crawlerParamItemDao.getCrawlConfigBySite(siteCategory.
getSiteId()));
// 构建抓取对象
String charset = params.get(PageParamNames.CONTENT_ENCODING);
int pageSize = Integer.parseInt(params.get(PageParamNames.BS_PAGE_SIZE));
int pages = amount / pageSize + 1;
logger.info("畅销榜页数:" + pages);
Crawler crawler = new Crawler(charset);
String bestSellerHtml = null;
PageParser pageParser = null;
try {
String categoryUrl = siteCategory.getCategoryUrl();
if (categoryUrl == null || categoryUrl.equals("")) {
logger.info("此分类不支持畅销榜!");
return null;
}
bestSellerHtml = crawler.crawl(siteCategory.getCategoryUrl());
pageParser = PageParserFactory.createPageParser(null, charset,
params);
String nextPageUrlPattern=pageParser.extractNextPageUrlPattern(bestSellerHtml);
if (siteCategory.getCategoryUrl().indexOf("suning")>0) {
String nextPageUrl0 = pageParser.getNextPageUrl(
nextPageUrlPattern, 1);
bestSellerHtml = crawler.crawl(nextPageUrl0);
}
bestSellers = pageParser.extractBestSeller(bestSellerHtml);
//if(amount<=bestSellers.size()) return bestSellers;
if (nextPageUrlPattern != null) {
for (int pageNum = 2; pageNum <= pages; pageNum++) {
logger.info("抓取畅销榜第 " + pageNum + " 页");
String nextPageUrl = pageParser.getNextPageUrl(
nextPageUrlPattern, pageNum);
bestSellerHtml = crawler.crawl(nextPageUrl);
List<BestSellerDto> moreBestSellers = pageParser.
extractBestSeller(bestSellerHtml);
if (moreBestSellers == null || moreBestSellers.isEmpty()) {
break;
}
if (bestSellers.get(bestSellers.size() - 1).getUrl().equals(
moreBestSellers.get(moreBestSellers.size() - 1).
getUrl())) {
break;
}
if (moreBestSellers != null && !moreBestSellers.isEmpty()) {
bestSellers.addAll(moreBestSellers);
}
}
}
} catch (IOException e) {
logger.error("抓取 " + siteCategory.getCategoryName() + " 畅销榜时出现异常!"
+ "URL为:" + siteCategory.getCategoryUrl(), e);
} catch (ParserException e) {
logger.error("解析畅销榜页面时出现异常!" + "URL为:"
+ siteCategory.getCategoryUrl(), e);
}
logger.info("++++++++++++++++++++++++++++++++++");
logger.info("畅销集合大小:" + (bestSellers == null ? 0 : bestSellers.size()));
if (bestSellers != null) {
if (bestSellers.size() <= amount) {
return bestSellers;
}
List<BestSellerDto> subList = bestSellers.subList(0, amount);
logger.info("----------------------------------");
logger.info("畅销集合大小:" + subList.size());
return subList;
} else {
return new ArrayList<BestSellerDto>();
}
}
不懂的联系QQ526151410