京东搜索项目（elasticsearch）

环境搭建

新建模块es-jd
1.添加依赖

 <dependency>
   <groupId>com.alibaba</groupId>
     <artifactId>fastjson</artifactId>
     <version>1.2.62</version>
 </dependency>

2.导入资源

3.更改端口

server.port=9090

4.编写indexcontroller

@Controller
public class IndexController {
    @GetMapping({"/","/index"})
    public String getIndex(){
     return "index";
    }
}

5.访问

爬取数据

1.添加依赖

<!--  解析网页-->
        <dependency>
            <groupId>org.jsoup</groupId>
            <artifactId>jsoup</artifactId>
            <version>1.10.2</version>
        </dependency>

2.编写代码爬取京东的数据，并显示，这里固定了关键字查询，下一步我们将这个搜索封装为一个utils

package com.cyx.utils;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.stereotype.Component;
import java.net.URL;

@Component
public class HtmlParseUtil {
    public static void main(String[] args) throws Exception {
        //获取连接，前提需要联网
        String url="https://search.jd.com/Search?keyword=java";
        //解析网页  jsoup返回的document就是浏览器Document对象
        Document document = Jsoup.parse(new URL(url), 30000);
         //js中使用的方法，这里都可以使用
        Element element = document.getElementById("J_goodsList");
       // System.out.println(element.html());//获取完整的div信息
        //获取所有的li元素
        Elements elements = element.getElementsByTag("li");
        //获取元素的内容，el是每一个li标签
        for (Element el:elements){
            //获取第一个元素的document
            //source-data-lazy
            String img = el.getElementsByTag("img").eq(0).attr("src");
            String price = el.getElementsByClass("p-price").eq(0).text();
            String title = el.getElementsByClass( "p-name").eq(0).text();
            System.out.println("------------------------");
            System.out.println(img);
            System.out.println(price);
            System.out.println(title);
        }
    }
}

3.封装utils

@Component
public class HtmlParseUtil {
    public static void main(String[] args) throws Exception {
        new HtmlParseUtil().parseJD("java").forEach(System.out::println);
    }
    public List<Content> parseJD(String keywords) throws Exception {
        String url = "https://search.jd.com/Search?keyword="+keywords;
        Document document = Jsoup.parse(new URL(url), 30000);
        Element element = document.getElementById("J_goodsList");
        Elements elements = element.getElementsByTag("li");
        ArrayList<Content> goodsList=new ArrayList<Content>();
        for (Element el : elements) {
            String img = el.getElementsByTag("img").eq(0).attr("src");
            String price = el.getElementsByClass("p-price").eq(0).text();
            String title = el.getElementsByClass("p-name").eq(0).text();
            Content content=new Content();
            content.setImg(img);
            content.setPrice(price);
            content.setTitle(title);
            goodsList.add(content);
        }
        return goodsList;
    }
}

运行查看

业务编写

1.编写pojo

@Data
@AllArgsConstructor
@NoArgsConstructor
public class Content {
    private String title;
    private String img;
    private String price;
}

2.config

@Configuration
public class ElasticSearchClientConfig {
    @Bean
    public RestHighLevelClient restHighLevelClient(){
        RestHighLevelClient client = new RestHighLevelClient(
                RestClient.builder(
                        new HttpHost("127.0.0.1", 9200, "http")));
         return client;
    }
}

3.server

@Service
public class ContentService {
    @Autowired
   public RestHighLevelClient restHighLevelClient;
    //1将解析数据放到es索引中
    public Boolean parseContent(String keywords)throws Exception{
        List<Content> contents=new HtmlParseUtil().parseJD(keywords);
        //把查询的数据放到es中
        BulkRequest bulkRequest = new BulkRequest();
        bulkRequest.timeout("2m");
        for (int i = 0; i <contents.size() ; i++) {
            bulkRequest.add(new IndexRequest("jd_goods").source(JSON.toJSONString(contents.get(i)), XContentType.JSON));
        }
        BulkResponse bulk = restHighLevelClient.bulk(bulkRequest, RequestOptions.DEFAULT);
        return !bulk.hasFailures();
    }
}

4.controller

@RestController
public class ContentController {
    @Autowired
    private ContentService contentService;
    @GetMapping("/parse/{keyword}")
    public Boolean parase(@PathVariable("keyword") String keyword) throws Exception {
        return contentService.parseContent(keyword);
    }
}

5.启动访问
访问9090，我们看到
jd_goods的数据量为0
访问http://localhost:9090/parse/java
6.查询功能的编写
前面的部分将数据爬取存放到我们的es中，接下来就是从这个es中进行查询并返回查询结果，如果es中没有的数据，是查不出来的。
6.1server

 //获取数据实现搜索功能
    public List<Map<String,Object>> searchPage(String keyword, int pageNow, int pageSize) throws IOException {
        if (pageNow <= 1){
            pageNow = 1;
        }
        //条件搜索
        SearchRequest searchRequest = new SearchRequest("jd_goods");
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();

        //分页
        searchSourceBuilder.from(pageNow);
        searchSourceBuilder.size(pageSize);

        //精准匹配
        TermQueryBuilder termQueryBuilder = QueryBuilders.termQuery("title", keyword);
        searchSourceBuilder.query(termQueryBuilder);
        searchSourceBuilder.timeout(new TimeValue(60, TimeUnit.SECONDS));

        //执行搜索
        searchRequest.source(searchSourceBuilder);
        SearchResponse search = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);

        //解析结果
        ArrayList<Map<String,Object>> list = new ArrayList<>();
        for (SearchHit documentFields : search.getHits().getHits()) {
            list.add(documentFields.getSourceAsMap());
        }
        return list;
    }

6.2controller

 @GetMapping("/search/{keyword}/{pageNow}/{pageSize}")
    public List<Map<String,Object>> search(@PathVariable ("keyword")String keyword,
                                           @PathVariable ("pageNow")int pageNow,
                                           @PathVariable("pageSize") int pageSize) throws Exception {
        return contentService.searchPage(keyword,pageNow,pageSize);
    }

6.3查询

关键字高亮

service

//获取数据实现高亮功能
    public List<Map<String,Object>> searchPageHighlight(String keyword,int pageNow,int pageSize) throws IOException {
        if (pageNow <= 1){
            pageNow = 1;
        }
        keyword= URLDecoder.decode(keyword, "UTF-8");
        //条件搜索
        SearchRequest searchRequest = new SearchRequest("jd_goods");
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        //分页
        searchSourceBuilder.from(pageNow);
        searchSourceBuilder.size(pageSize);
        //精准匹配
        TermQueryBuilder termQueryBuilder = QueryBuilders.termQuery("title", keyword);
        searchSourceBuilder.query(termQueryBuilder);
        searchSourceBuilder.timeout(new TimeValue(60, TimeUnit.SECONDS));

        //高亮
        HighlightBuilder highlightBuilder = new HighlightBuilder();
        highlightBuilder.field("title");
        highlightBuilder.requireFieldMatch(true);//多个高亮显示
        highlightBuilder.preTags("<span style='color:red'>");
        highlightBuilder.postTags("</span>");
        searchSourceBuilder.highlighter(highlightBuilder);

        //执行搜索
        searchRequest.source(searchSourceBuilder);
        SearchResponse search = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);

        //解析结果
        ArrayList<Map<String,Object>> list = new ArrayList<>();
        for (SearchHit documentFields : search.getHits().getHits()) {

            //解析高亮的字段
            Map<String, HighlightField> highlightFields = documentFields.getHighlightFields();
            HighlightField title = highlightFields.get("title");
            Map<String, Object> sourceAsMap = documentFields.getSourceAsMap();
            if (title != null){
                Text[] fragments = title.fragments();
                String n_title = "";
                for (Text text : fragments) {
                    n_title += text;
                }
                sourceAsMap.put("title",n_title);
            }
            list.add(sourceAsMap);
        }
        return list;
    }

controller修改

 @GetMapping("/search/{keyword}/{pageNow}/{pageSize}")
    public List<Map<String,Object>> search(@PathVariable ("keyword")String keyword,
                                           @PathVariable ("pageNow")int pageNow,
                                           @PathVariable("pageSize") int pageSize) throws Exception {
        //return contentService.searchPage(keyword,pageNow,pageSize);
        contentService.parseContent(keyword);//先放到es中
        return contentService.searchPageHighlight(keyword,pageNow,pageSize);
    }

访问http://localhost:9090/

搜索的时候用英文，中文的还没有完成解析功能。
当查询一个es中没有的 例如mabatis，我们先输入之后，没有查出来显示，但是已经爬取数据放到es中了，在执行一遍便会显示出来。

学习视频https://www.bilibili.com/video/BV17a4y1x7zq?p=20