爬取存入数据库
/**
* 京东爬虫依赖
*
* <dependency>
* <groupId>org.jsoup</groupId>
* <artifactId>jsoup</artifactId>
* <version>1.11.3</version>
* </dependency>
* <p>
* 爬取京东品牌和分类信息
*/
@RestController
public class DemoController {
@Autowired
CategoryService categoryService;
@Autowired
BrandService brandService;
@GetMapping("getCategoryFromJD")
public AxiosResult<Void> setData() throws IOException {
Document document = Jsoup.connect("https://www.jd.com/allSort.aspx").get();
Elements elementsByClass = document.getElementsByClass("category-items");
for (int i = 0; i < elementsByClass.size(); i++) {
Element element = elementsByClass.get(i);
Elements element1 = element.getElementsByClass("category-item");
for (int j = 0; j < element1.size(); j++) {
Element element2 = element1.get(j);
//一级分类名
String firstCategoryName = element2.getElementsByTag("span").text();
Category firstCategory = new Category();
firstCategory.setCatetoryName(firstCategoryName);
firstCategory.setCategoryLevel(1);
firstCategory.setParentId(0L);
categoryService.save(firstCategory);
//二级分类
Elements dt = element2.getElementsByTag("dl");
for (int k = 0; k < dt.size(); k++) {
Element element3 = dt.get(k);
String secondCategoryName = element3.getElementsByTag("dt").get(0).getElementsByTag("a").get(0).text();
Category sencondCategory = new Category();
sencondCategory.setParentId(firstCategory.getId());
sencondCategory.setCatetoryName(secondCategoryName);
sencondCategory.setCategoryLevel(2);
categoryService.save(sencondCategory);
Elements a = element3.getElementsByTag("dd").get(0).getElementsByTag("a");
for (int l = 0; l < a.size(); l++) {
Category threeCategory = new Category();
threeCategory.setParentId(sencondCategory.getId());
threeCategory.setCatetoryName(a.get(l).text());
threeCategory.setCategoryLevel(3);
categoryService.save(threeCategory);
}
}
}
}
return AxiosResult.success();
}
@GetMapping("getBrandFromJD")
public AxiosResult<Void> getBrandFromJd() throws Exception {
Document document = Jsoup.connect("https://www.jd.com/brand.aspx").get();
Elements brandslist = document.getElementsByClass("brandslist");
for (int i = 0; i < brandslist.size(); i++) {
Element element1 = brandslist.get(i);
Elements li = element1.getElementsByTag("li");
for (int j = 0; j < li.size(); j++) {
Element img = li.get(j).getElementsByTag("img").get(0);
System.out.println(img);
String src = img.attr("src");
String alt = img.attr("alt");
System.out.println(src);
Element span = li.get(j).getElementsByTag("span").get(1).getElementsByTag("a").get(0);
String text = span.text();
Brand brand = new Brand();
brand.setBrandName(text);
brand.setBrandDesc(alt);
brand.setBrandLogo(src);
brand.setBrandSite("http://www.baidu.com");
brandService.save(brand);
}
}
return AxiosResult.success();
}
}
爬取输出txt
public static void setData() throws IOException {
Document document = Jsoup.connect("https://www.jd.com/allSort.aspx").get();
Elements elementsByClass = document.getElementsByClass("category-items");
StringBuffer stringBuffer = new StringBuffer();
StringBuffer append = null;
for (int i = 0; i < elementsByClass.size(); i++) {
Element element = elementsByClass.get(i);
Elements element1 = element.getElementsByClass("category-item");
for (int j = 0; j < element1.size(); j++) {
Element element2 = element1.get(j);
//一级分类名
String firstCategoryName = element2.getElementsByTag("span").text();
//二级分类
Elements dt = element2.getElementsByTag("dl");
for (int k = 0; k < dt.size(); k++) {
Element element3 = dt.get(k);
String secondCategoryName = element3.getElementsByTag("dt").get(0).getElementsByTag("a").get(0).text();
Elements a = element3.getElementsByTag("dd").get(0).getElementsByTag("a");
for (int l = 0; l < a.size(); l++) {
System.out.print(a.get(l).text() + " ");
String text = a.get(l).text();
append = stringBuffer.append(text + " ");
}
System.out.println("\n");
append.append("\r\n");
}
}
}
FileOutputStream stream = new FileOutputStream("C://Users//Desktop//京东分类目录.txt");
byte[] bytes = append.toString().getBytes(StandardCharsets.UTF_8);
stream.write(bytes);
stream.close();
}