java-爬虫-es

---

文献：https://www.kuangstudy.com/bbs/1354069127022583809

1.数据来源：数据库、mq、爬虫

2.爬虫：获取想要的页面数据

1.导入依赖

jsoup：适合爬取解析网页信息等
tika：适合爬取电影、视频、音频等

<dependency>
            <groupId>org.jsoup</groupId>
            <artifactId>jsoup</artifactId>
            <version>1.10.2</version>
        </dependency>

2.爬取核心部分编码

public static void main(String[] args) throws Exception {
        new HtmlParseUtil().parseJD("vue").forEach(System.out::println);
    }

    public List<Content> parseJD(String keywords) throws Exception {
        // 获取请求：https://search.jd.com/Search?keyword=java
        String url = "https://search.jd.com/Search?keyword=" + keywords;
        //解析网页，document就是浏览器的document对象
        Document document = Jsoup.parse(new URL(url), 30000);
        //所有js操作都可以通过document对象进行操作

        // 获取商品列表
        Element jGoodsListElement = document.getElementById("J_goodsList");
        // 获取商品标签
        Elements li = jGoodsListElement.getElementsByTag("li");
        List<Content> goodsList = new ArrayList<>(li.size());
        for (Element el : li) {
            String img = el.getElementsByTag("img").eq(0).attr("source-data-lazy-img");
            String price = el.getElementsByClass("p-price").eq(0).text();
            String title = el.getElementsByClass("p-name").eq(0).text();
            goodsList.add(new Content().setImg(img).setTitle(title).setPrice(price));
        }
        return goodsList;
    }

3.测试解析成功

4.封装对象

package com.bie.pojo;

import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.experimental.Accessors;

/**
 * @author bjh
 * @date 2022/12/12
 */
@Data
@NoArgsConstructor
@AllArgsConstructor
@Accessors(chain = true)
public class Content {

    private String img;
    private String title;
    private String price;
}

5.引入es配置类

package com.bie.config;

import org.apache.http.HttpHost;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

/**
 * @author bjh
 * @date 2022/12/9
 */

@Configuration
public class ElasticSearchClientConfig {

    @Bean
    public RestHighLevelClient restHighLevelClient() {
        RestHighLevelClient client = new RestHighLevelClient(
                RestClient.builder(
                        new HttpHost("192.168.229.132", 9200, "http")));
        return client;
    }
}

6.将HtmlParseUtil注册到spring

package com.bie.utils;

import com.bie.pojo.Content;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.stereotype.Component;

import java.net.URL;
import java.util.ArrayList;
import java.util.List;

/**
 * @author bjh
 * @date 2022/12/12
 */
@Component
public class HtmlParseUtil {

    public static void main(String[] args) throws Exception {
        new HtmlParseUtil().parseJD("vue").forEach(System.out::println);
    }

    public List<Content> parseJD(String keywords) throws Exception {
        // 获取请求：https://search.jd.com/Search?keyword=java
        String url = "https://search.jd.com/Search?keyword=" + keywords;
        //解析网页，document就是浏览器的document对象
        Document document = Jsoup.parse(new URL(url), 30000);
        //所有js操作都可以通过document对象进行操作

        // 获取商品列表
        Element jGoodsListElement = document.getElementById("J_goodsList");
        // 获取商品标签
        Elements li = jGoodsListElement.getElementsByTag("li");
        List<Content> goodsList = new ArrayList<>(li.size());
        for (Element el : li) {
            String img = el.getElementsByTag("img").eq(0).attr("source-data-lazy-img");
            String price = el.getElementsByClass("p-price").eq(0).text();
            String title = el.getElementsByClass("p-name").eq(0).text();
            goodsList.add(new Content().setImg(img).setTitle(title).setPrice(price));
        }
        return goodsList;
    }
}

7.爬取的数据入es库

1.在es创建索引"jd_goods"

2.编写controller

package com.bie.controller;

import com.bie.service.ContentService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RestController;

/**
 * @author bjh
 * @date 2022/12/12
 */
@RestController
public class ContentController {

    @Autowired
    private ContentService contentService;

    @GetMapping("/parse/{keyword}")
    public boolean parse(@PathVariable("keyword") String keyword) throws Exception {
        return contentService.parseContent(keyword);
    }

}

3.编写service

package com.bie.service;

import com.alibaba.fastjson.JSON;
import com.bie.pojo.Content;
import com.bie.utils.HtmlParseUtil;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentType;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import java.util.List;

/**
 * @author bjh
 * @date 2022/12/12
 */
@Service
public class ContentService {

    @Autowired
    private RestHighLevelClient restHighLevelClient;

    public boolean parseContent(String keywords) throws Exception {
        // 解析
        List<Content> contents = new HtmlParseUtil().parseJD(keywords);
        // 放入es
        BulkRequest bulkRequest = new BulkRequest();
        bulkRequest.timeout(TimeValue.timeValueSeconds(1));

        for (int i = 0; i < contents.size(); i++) {
            bulkRequest.add(new IndexRequest("jd_goods")
                    .source(JSON.toJSONString(contents.get(i)), XContentType.JSON));
        }

        BulkResponse bulkResponse = this.restHighLevelClient.bulk(bulkRequest, RequestOptions.DEFAULT);
        return !bulkResponse.hasFailures();
    }

}

4.测试
爬取java相关：
访问：http://localhost:9090/parse/java

爬取vue相关：

8.空白文件初始化vue

初始化vue项目

#1.创建空文件夹
mkdir /home/test && cd /home/test
#2.初始化npm
npm init
#3.安装vue
npm install vue
#vue-min.js文件下载地址，将内容拷贝到vue-min.js文件中
https://cdn.staticfile.org/vue/2.6.11/vue.min.js

#4.安装axios 通信
npm install axios
#5.将初始化的vue目录，拷贝到boot项目中