JAVA爬虫+jsoup+Kibana+Elasticsearch+vue爬取JD搜索页面

1.环境的搭建

2.相关业务层的代码

3.Vue页面的编写

4.测试

1.环境的搭建

1.1 创建一个springboot项目并添加相关依赖

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>
    <parent>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-parent</artifactId>
        <version>2.3.12.RELEASE</version>
        <relativePath/> <!-- lookup parent from repository -->
    </parent>
    <groupId>com.dd</groupId>
    <artifactId>qy151-JD</artifactId>
    <version>0.0.1-SNAPSHOT</version>
    <name>qy151-JD</name>
    <description>qy151-JD</description>
    <properties>
        <java.version>1.8</java.version>
    </properties>
    <dependencies>
        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>fastjson</artifactId>
            <version>1.2.83</version>
        </dependency>
        <dependency>
            <groupId>org.jsoup</groupId>
            <artifactId>jsoup</artifactId>
            <version>1.11.3</version>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-data-elasticsearch</artifactId>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
        </dependency>

        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
            <optional>true</optional>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-test</artifactId>
            <scope>test</scope>
        </dependency>
    </dependencies>

    <build>
        <plugins>
            <plugin>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-maven-plugin</artifactId>
                <configuration>
                    <excludes>
                        <exclude>
                            <groupId>org.projectlombok</groupId>
                            <artifactId>lombok</artifactId>
                        </exclude>
                    </excludes>
                </configuration>
            </plugin>
        </plugins>
    </build>

</project>

1.2 打开elasticsearch和Kibana黑窗口运行脚本文件

在这里我用的elasticsearch和Kibana版本号是
elasticsearch-7.6.1-windows-x86
kibana-7.6.1-windows-x86_64

1.3 打开Kibana图形化界面创建索引

2. 相关业务层的代码

2.1 创建utils工具包使用Jsoup对网页进行解析

这里爬取的是JD搜索界面的图片、价格、名字

public class HtmlParseUtil {
   
    //从京东爬虫----有可能从数据库中。
    public static List<Product> parseJd(String keyword) throws Exception {
   
        String path="https://search.jd.com/Search?keyword="+keyword;
        //Document整个网页对象
        Document document = Jsoup.parse(new URL(path), 30000);
        Element j_goodsList = document.getElementById("J_goodsList");
        Elements li = j_goodsList.getElementsByTag("li");
        List<Product> list=new ArrayList<>();
        for (Element element:li){
   
            String price = element.getElementsByClass("p-price").eq(0).text();
            String pname = element.getElementsByClass("p-name").eq(0).text();
            String img = element.getElementsByTag("img").eq(0).attr("data-lazy-img");
            list.add(new Product(pname,price,img));
        }
        return list;
    }
}

然后在工具包下创建CommonResult工具类返回给前端

@Data
@NoArgsConstructor
@AllArgsConstructor
public class CommonResult {
   
    private Integer code;
    private String msg;
    private Object result;
}

2.2 Controller层

@RestController
@RequestMapping("/product")
@CrossOrigin
public class ProductController {
   
    @Autowired
    private ProductImpl productService;
//用于添加爬取JD页面的搜索数据
    @GetMapping("/export/{keyword}")
    public CommonResult export(@PathVariable String keyword)throws Exception{
   
        return productService.export(keyword);
    }
//    用于在Vue界面使用Kibana进行高亮搜索和分页
    @PostMapping("/search/{keyword}/{currentPage}/{pageSize}")
    public CommonResult search(@PathVariable String keyword,@PathVariable Integer currentPage,@PathVariable Integer pageSize) throws Exception{
   
        return productService.search(keyword,currentPage,pageSize);
    }
}

2.3 entity类

@Data
@NoArgsConstructor
@AllArgsConstructor
public class Product {
   
    private String  title;
    private String  price;
    private String  imgUrl;
}

2.4 service层

接口

public interface ProductServe {
   
    CommonResult export(String keyword) throws Exception;
    CommonResult search(String keyword,Integer currentPage,Integer pageSize) throws Exception;
}

Impl实现类

package com.dd.service.impl;

import com.alibaba.fastjson.JSON;
import com.dd.entity.Product;
import com.dd.service.ProductServe;
import com.dd.utils.CommonResult;
import com.dd.utils.HtmlParseUtil;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.TermQueryBuilder;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.builder.SearchSourceBuilder