public class BaiduYuLeStar { /** * 保存出错 */ private static StringBuilder noResult = new StringBuilder(); /** * 当前页数 */ private static int pagePos = 1; public static void main(String[] args) { BaiduYuLeStar star = new BaiduYuLeStar(); star.work(Integer.MAX_VALUE); // star.workOnePage(); } /** * 影视娱乐资料库的基地址 */ private final String baseDataURL = "http://data.yule.baidu.com"; /** * 页面编码 */ private final String charset = "gbk"; /** * 捉取页面工具类 */ private HtmlCleaner cleaner; /** * star表的dao类实例 */ private final StarDao dao = StarDao.getInstance(); /** * 日志记录 */ private final Logger logger = LoggerFactory.getLogger(this.getClass()); /** * 之后定时运行捉取新增加的明星,以 一个阀值为准 */ private final int pageNum = 1; /** * 获取所有明星的URL地址 */ private final String starURL = "http://data.yule.baidu.com/index_star_s0_p#.html"; /** * 构造函数,先调用初始化的函数 */ public BaiduYuLeStar() { initCleaner(); } /** * 获取当前页面的结点 * * @return */ private TagNode getPage() { TagNode node = getPage(pagePos); pagePos++;// 下一页 return node; } /** * 获取指定某一页面的结点 * * @param pageNum * @return */ private TagNode getPage(int pageNum) { String pageURL = starURL.replace("#", String.valueOf(pageNum)); // 测试 // pageURL="http://data.yule.baidu.com/index_star_s0_p4.html"; try { TagNode pageNode = cleaner.clean(new URL(pageURL), charset); TagNode node = HtmlCleanerXpathUtil.xpathOne(pageNode, "//*[@id='container']"); return node; } catch (IOException e) { logger.info(e.getMessage(), e); } return null; } /** * 从详细的页面获取相关的详细信息并封装到类Star * * @param star * 相对地址 * @return */ public void getStarFullInfo(Star star) { try { URL url = new URL(star.getFromURL()); TagNode starInfoNode = cleaner.clean(url, charset); // //*[@id='photo']/div/div/h2 TagNode nameNode = HtmlCleanerXpathUtil.xpathOne(starInfoNode, "//*[@id='photo']//h2"); if (nameNode == null) { logger.info("没有获取到名称"); noResult.append(star.getFromURL() + " :没有获取到名称/n"); return; } // 保存名字 String name = nameNode.getText().toString().trim(); star.setName(StringUtils.unicodeTextDecode(name).toString()); // 保存图片 TagNode photoNode = HtmlCleanerXpathUtil.xpathOne(starInfoNode, "//*[@id='photo']/div/div/img"); if (photoNode == null) { photoNode = HtmlCleanerXpathUtil.xpathOne(starInfoNode, "//*[@id='photo']/img"); } String imgURL = photoNode.getAttributeByName("src"); star.setPoster(imgURL); // 保存id,放在获取图片之后,因为图片的名称跟id可能相关 TagNode idNode = HtmlCleanerXpathUtil.xpathOne(starInfoNode, "/head"); handleID(star, idNode.getText().toString().trim()); // 其它基本信息 handleBaseInfo(star, starInfoNode); // 获取并保存简介信息 TagNode introNode = HtmlCleanerXpathUtil.xpathOne(starInfoNode, "//div[@class='intro']"); String summary = introNode.getText().toString().trim(); star.setIntro(summary); } catch (IOException e) { logger.info(e.getMessage(), e); } } /** * 获取当前页面的所有明星的信息的链接 * * @return */ public List<Star> getStarInfoHref() { List<Star> starHrefList = new ArrayList<Star>(); TagNode pageNode = getPage(); if (pageNode == null) { return starHrefList; } TagNode starNode[] = HtmlCleanerXpathUtil.xpathArray(pageNode, "//li//a"); for (TagNode node : starNode) { String href = node.getAttributeByName("href"); if (href.startsWith("/star_details/")) {//找到详细资料的链接 Star star = new Star(); star.setFromURL(href); starHrefList.add(star); } } return starHrefList; } /** * 处理基本信息的节点 * * @param star * 封装明星的值对象 * @param starInfoNode */ private void handleBaseInfo(Star star, TagNode starInfoNode) { TagNode detailNode = HtmlCleanerXpathUtil.xpathOne(starInfoNode, "//div[@class='infolink infohome']"); if (detailNode == null) {// 多个版本的页面结构不一样,比如:吉赛贝-托纳多雷 detailNode = HtmlCleanerXpathUtil.xpathOne(starInfoNode, "//div[@class='infolink']"); } if (detailNode == null) { logger.info("detailNode为空,没有获取到基本信息的节点"); noResult.append(star.getName() + " 没有获取到基本信息的节点/n"); return; } String baseInfoStr[] = detailNode.getText().toString().trim().split( "(/n)|( )"); for (String map : baseInfoStr) { if (!map.trim().equals("")) { String temp[] = map.split(":"); if (temp.length != 2) { continue; } String mapName = temp[0].trim(); String mapValue = temp[1].trim(); if (mapName.equals("姓名")) { star.setAlias(mapValue); } else if (mapName.equals("性别")) { int sex = 0; if ("男".equals(star.getGender())) { sex = 1; } else if ("女".equals(star.getGender())) { sex = 2; } else if ("组合".equals(star.getGender())) { sex = 3; } star.setGender(sex); } else if (mapName.equals("身高")) { // 处理身高 int height = 0; if (mapValue != null) { String hStr = mapValue; String regex = "([0-9]+)"; Pattern pattern = Pattern.compile(regex); Matcher matcher = pattern.matcher(hStr); if (matcher.find()) { height = Integer.valueOf(matcher.group(1)); } } star.setHeight(height); } else if (mapName.equals("地区")) { star.setZone(mapValue); } else if (mapName.equals("星座")) { star.setConstellation(mapValue); } else if (mapName.equals("血型")) { star.setBloodType(mapValue); } else if (mapName.equals("职业")) { star.setJob(mapValue); } } } } /** * 从页面的某个内容获取id号 * * @param idText * @return */ private void handleID(Star star, String idText) { String regex = "ProductID//s*=//s*([0-9]+)"; Pattern p = Pattern.compile(regex); Matcher m = p.matcher(idText); if (m.find()) { star.setId(Integer.valueOf(m.group(1))); } else {// 从图片名称获取 if (star.getPoster() != null) { int nameIndex = star.getPoster().lastIndexOf("/"); String name = star.getPoster().substring(nameIndex + 1, star.getPoster().length()); String id = name.replace(".jpg", ""); star.setId(Integer.valueOf(id)); } } } /** * 处理获得的明星对象 * * @param star */ private void handleStar(Star star) { System.out.println(star); boolean isSucSave = dao.saveStar(star); logger.info("保存到数据库 " + isSucSave); } /** * 初始化HtmlCleaner实例,在构造函数里调用 */ private void initCleaner() { cleaner = new HtmlCleaner(); CleanerProperties props = cleaner.getProperties(); props.setUseEmptyElementTags(false); // props.setRecognizeUnicodeChars(true); // props.set } /** * 初次捉取全部页面 */ /** * * @param numPage * 要捉取的页面数量 */ public void work(int numPage) { int num = 1; int index = 0; do { List<Star> starList = getStarInfoHref(); if (starList == null || starList.size() <= 0) { logger.info("结束捉取。。"); break; } System.out.println(starList.size()); for (Star star : starList) { index = star.getFromURL().lastIndexOf("/"); String first = star.getFromURL().substring(0, index + 1); String last = star.getFromURL().substring(index + 1, star.getFromURL().length()); try { last = URLEncoder.encode(last, "gbk"); last = last.replaceAll("//+", "%20"); last = last.replaceAll("///", "%2"); String href = first + last; star.setFromURL(baseDataURL + href); } catch (UnsupportedEncodingException e) { logger.info(e.getMessage(), e); } getStarFullInfo(star); handleStar(star); // break; } num++; } while (num <= numPage); System.out.println(noResult.toString()); } /** * 捉取一个页面 */ public void workOnePage() { work(pageNum); } }