代码没整理 赶出来的代码 可优化空间巨大 (据了解半年使用一次,且有其他工作,就不优化了)。以美国站, Wallpaper前100为例吧。可执行更换产品。缺的jar包可以自己 下载 或者 留言 我给你。
// 自己建个方法体吧
Document document; Document document1; List<plsj> plsjList = new ArrayList<plsj>(); int num = 1; try { //因为 一页50 个产品 所以。。。。 for (int m = 1; m < 3; m++) { document = Jsoup.connect("https://www.amazon.com/Best-Sellers-Home-Improvement-Wallpaper/zgbs/hi/2242314011/ref=zg_bs_pg_" + m + "?_encoding=UTF8&pg=" + m).userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36").get(); Elements elementsByClass = document.getElementsByClass("a-list-item"); for (int i = 0; i < elementsByClass.size(); i++) { Document parse = Jsoup.parse(elementsByClass.get(i).toString()); plsj plsj = new plsj(); plsj.setPm(OtherUtil.getTextFromHTML(parse.getElementsByClass("zg-badge-text").toString().replace("#", ""))); plsj.setXj(OtherUtil.getTextFromHTML(parse.getElementsByClass("a-icon-alt").toString()).substring(0, 4)); plsj.setPls(OtherUtil.getTextFromHTML(parse.getElementsByClass("a-size-small a-link-normal").toString())); plsj.setJg(OtherUtil.getTextFromHTML(parse.getElementsByClass("a-size-base a-color-price").toString())); String str = parse.getElementsByClass("a-link-normal a-text-normal").toString(); str = OtherUtil.regularexpression("(?<=href=\").*?(?=\")", str).get(0); document1 = Jsoup.connect("https://www.amazon.com/" + str).userAgent("User-Agent:Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50").get(); try { Elements elementsByClass1 = document1.getElementsByClass("a-normal a-spacing-micro").select("table").select("tr"); for (int k = 0; k < elementsByClass1.size(); k++) { Document parse1 = Jsoup.parse(elementsByClass1.get(k).toString()); String temp = OtherUtil.getTextFromHTML(parse1.getElementsByClass("a-size-base a-text-bold").toString()); if ("Brand".equals(temp)) { plsj.setBrand(OtherUtil.getTextFromHTML(parse1.getElementsByClass("a-size-base").toString())); } } } catch (NullPointerException e) { } plsj.setDp(OtherUtil.getTextFromHTML(document1.getElementById("bylineInfo").toString())); try { Elements select1 = document1.getElementById("tabular-buybox-container").select("table").select("tr"); for (int l = 0; l < select1.size(); l++) { Document parse1 = Jsoup.parse(select1.get(l).toString()); String textFromHTML = OtherUtil.getTextFromHTML(parse1.getElementsByClass("a-color-tertiary tabular-buybox-label").toString()); if ("Ships from".equals(textFromHTML)) { plsj.setPsfs(OtherUtil.getTextFromHTML(parse1.getElementsByClass("tabular-buybox-text").toString())); } } } catch (NullPointerException e) { } Elements select = document1.getElementById("prodDetails").select("table").select("tr"); for (int j = 0; j < select.size(); j++) { String string = select.get(j).toString(); if (string.indexOf("Package Dimensions") != -1) { plsj.setCc(OtherUtil.getTextFromHTML(string.replace("Package Dimensions", ""))); } if (string.indexOf("ASIN") != -1) { plsj.setAsin(OtherUtil.getTextFromHTML(string.replace("ASIN", ""))); } if (string.indexOf("Item Weight") != -1) { plsj.setZl(OtherUtil.getTextFromHTML(string.replace("Item Weight", ""))); } } plsjList.add(plsj); System.out.println("第 " + num + " 个"); //别嫌慢 亚马逊检测很厉害 验证码 我没研究 不会过 try { Thread.sleep((int) (Math.random() * 5000)); } catch (InterruptedException e) { e.printStackTrace(); } num++; } } } catch (IOException e) { e.printStackTrace(); }
//去除标签的方法
public static String getTextFromHTML(String htmlStr) { Document doc = Jsoup.parse(htmlStr); String text = doc.text(); // remove extra white space StringBuilder builder = new StringBuilder(text); int index = 0; while (builder.length() > index) { char tmp = builder.charAt(index); if (Character.isSpaceChar(tmp) || Character.isWhitespace(tmp)) { builder.setCharAt(index, ' '); } index++; } text = builder.toString().replaceAll(" +", " ").trim(); return text; }
//封装的正则表达
public static List<String> regularexpression(String pattern, String str) { List<String> list = new ArrayList<String>(); Pattern r = Pattern.compile(pattern); Matcher matcher = r.matcher(str); if (matcher.find()) { //1000应该足够用了 for (int i = 0; i < 1000; i++) { try { list.add(matcher.group(i)); } catch (IndexOutOfBoundsException e) { return list; } } } return list; }
//实体类的
public class plsj { private String xj; private String sj; private String cc; private String ms; private String asin; /** * 品牌 */ private String brand; /** * 店铺 */ private String dp; /** * 配送方式 */ private String psfs; /** * 价格 */ private String jg; /** * 排名 */ private String pm; /** * 评论数 */ private String pls; /** *重量 */ private String zl; public String getCc() { return cc; } public void setCc(String cc) { this.cc = cc; } public String getMs() { return ms; } public void setMs(String ms) { this.ms = ms; } public String getXj() { return xj; } public void setXj(String xj) { this.xj = xj; } public String getSj() { return sj; } public void setSj(String sj) { this.sj = sj; } public String getAsin() { return asin; } public void setAsin(String asin) { this.asin = asin; } public String getBrand() { return brand; } public void setBrand(String brand) { this.brand = brand; } public String getDp() { return dp; } public void setDp(String dp) { this.dp = dp; } public String getPsfs() { return psfs; } public void setPsfs(String psfs) { this.psfs = psfs; } public String getJg() { return jg; } public void setJg(String jg) { this.jg = jg; } public String getPm() { return pm; } public void setPm(String pm) { this.pm = pm; } public String getPls() { return pls; } public void setPls(String pls) { this.pls = pls; } public String getZl() { return zl; } public void setZl(String zl) { this.zl = zl; } } //最后数据的部分展示