Jsoup 爬虫爬取网页内容
springboot导入maven依赖
<!--解析网页jsoup-->
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>1.10.2</version>
</dependency>
java操作
public List<Content> parseJD(String keywords) throws IOException {
//获取请求 https://search.jd.com/Search?keyword=java&enc=utf-8&pvid=5452d8c0790c4c6fb86b61a5c8e9b880
//前提,需要联网
String url = "https://search.jd.com/Search?keyword="+ keywords+"&enc=utf-8";
//解析网页(Jsoup 返回的就是Document 浏览器Doc对象)
Document document = Jsoup.parse(new URL(url), 30000);
Element element = document.getElementById("J_goodsList");
Elements elements = element.getElementsByTag("li");
ArrayList<Content> goodsList = new ArrayList<>();
// System.out.println(elements.html());
for (Element e1 : elements) {
String img = e1.getElementsByTag("img").eq(0).attr("data-lazy-img");
String price = e1.getElementsByClass("p-price").eq(0).text();
String title = e1.getElementsByClass("p-name").eq(0).text();
goodsList.add(new Content(title, img, price));
// System.out.println("==========================");
// System.out.println(img);
// System.out.println(price);
// System.out.println(title);
}
return goodsList;
}
结果: