Nutch开发(七)

Nutch开发(七)

实现一个迷你的博客搜索引擎

网上有很多nutch制作搜索引擎的文章，但是他们使用的nutch都是十分老的版本，如：nutch1.2，我们也知道nutch发展到现在已经出来很多版本，就我现在使用的nutch1.18版本早就吧nutch的全文索引搜索的功能独立出去了。

所以我们现在使用nutch+slor做一个仿造必应的迷你的博客搜索引擎。

准备工作

• 下载一个solr8.11
• 下载一个nutch1.18
• 创建一个maven工程
• 创建一个vue3工程

使用solrJ

自动建议这功能，spring data solr是没封装的。
solrJ可定制性更高，建议用solrJ。关于solrJ的使用，请自己上网搜索资料。

创建SolrJUtil类

一个查询参数拼接的类

@Slf4j
public class SolrJUtil {
    public static String toQueryStr(Map<String,String> map){
        StringBuffer stringBuffer = new StringBuffer();
        map.forEach((field,q)->{
            stringBuffer.append(field);
            stringBuffer.append(":");
            stringBuffer.append(q);
            stringBuffer.append(" && ");
        });
        return stringBuffer.substring(0,stringBuffer.length()-4);
    }

    public static String toQueryStr(Map<String,String> map,Double similarity){
        StringBuffer stringBuffer = new StringBuffer();
        map.forEach((field,q)->{
            stringBuffer.append(field);
            stringBuffer.append(":");
            stringBuffer.append(q);
            if (CommonConstants.Q_DEC_NAME.equals(field))stringBuffer.append("~"+similarity.toString());
            stringBuffer.append(" || ");
        });
        log.info("stringBuffer == {}",stringBuffer.substring(0,stringBuffer.length()-4));
        return stringBuffer.substring(0,stringBuffer.length()-4);
    }

    public static String toQueryStr(String fieldName,String q){
        return fieldName+":"+q;
    }

    public static String toHighlightFieldStr(List highLights){
        StringBuffer stringBuffer = new StringBuffer();
        highLights.stream().forEach(v->{
            stringBuffer.append(v.toString());
            stringBuffer.append(",");
        });
        return stringBuffer.substring(0,stringBuffer.length()-1);
    }

    public static SolrQuery buildHighLightQuery(SolrQuery query,List highLights,String prefix, String postfix){
        //开启高亮
        query.setHighlight(true);
        //设置高亮域
        // 高亮的前后缀
        String highlightFieldStr = SolrJUtil.toHighlightFieldStr(highLights);
        query.addHighlightField(highlightFieldStr);
        query.setHighlightSimplePre(prefix);
        query.setHighlightSimplePost(postfix);
        return query;
    }


    public static SolrQuery buildQuery(Map<String,String> queryMap){
        SolrQuery query = new SolrQuery();
        String queryStr = SolrJUtil.toQueryStr(queryMap);
        query.setQuery(queryStr);
        return query;
    }

    public static SolrQuery buildQuery(Map<String,String> queryMap,Double similarity){
        SolrQuery query = new SolrQuery();
        String queryStr = SolrJUtil.toQueryStr(queryMap,similarity);
        query.setQuery(queryStr);
        return query;
    }

    public static SolrQuery buildSpellcheckQuery(SolrQuery query){
        query.set("spellcheck",true);
        query.set("spellcheck.extendedResults",true);
        query.set("spellcheck.count",6);
        query.set("spellcheck.collate",true);
        return query;
    }

}

创建SolrJQueryBuilder类

SolrQuery的builder类

@Slf4j
public class SolrJQueryBuilder {
    private SolrQuery solrQuery;

    public SolrJQueryBuilder(){
        solrQuery = new SolrQuery();
    }

    public static SolrJQueryBuilder builder(){
        return new SolrJQueryBuilder();
    }

    public SolrQuery build(){
        return solrQuery;
    }

    public SolrJQueryBuilder buildQuery(Map<String,String> queryMap){
        String queryStr = SolrJUtil.toQueryStr(queryMap);
        solrQuery.setQuery(queryStr);
        return this;
    }

    public SolrJQueryBuilder buildQuery(Map<String,String> queryMap,Double similarity){
        String queryStr = SolrJUtil.toQueryStr(queryMap,similarity);
        log.info("queryStr == {}",queryStr);
        solrQuery.setQuery(queryStr);
        return this;
    }

    public SolrJQueryBuilder buildDefaultOperator(String op){
        solrQuery.setParam("q.op", "OR");
        return this;
    }

    public SolrJQueryBuilder buildPageStart(String start){
        solrQuery.setStart(Integer.valueOf(start));
        return this;
    }

    public SolrJQueryBuilder buildHighLightsQuery(List highLights, String prefix, String postfix){
        //开启高亮
        solrQuery.setHighlight(true);
        //设置高亮域
        // 高亮的前后缀
        String highlightFieldStr = SolrJUtil.toHighlightFieldStr(highLights);
        solrQuery.addHighlightField(highlightFieldStr);
        solrQuery.setHighlightSimplePre(prefix);
        solrQuery.setHighlightSimplePost(postfix);
        return this;
    }

    public SolrJQueryBuilder buildSpellcheckQuery(){
        solrQuery.set("spellcheck",true);
        solrQuery.set("spellcheck.extendedResults",true);
        solrQuery.set("spellcheck.count",1);
        solrQuery.set("spellcheck.collate",true);
        return this;
    }

    public SolrJQueryBuilder buildSuggestQuery(String q){
        solrQuery.set("suggest",true);
        solrQuery.set("suggest.dictionary","suggester");
        solrQuery.set("suggest.q",q);
        solrQuery.set("suggest.count",5);
        return this;
    }

}

创建CommonConstants接口

这个用来装一些公共常量

public interface CommonConstants {

    String NUTCH_COLLECTION = "nutch";

    String Q_TITLE_NAME = "title";
    String Q_DEC_NAME = "description";
    String Q_CONTENT_NAME = "content";
    String Q_SELF = "self";

    String Q_HIGHLIGHT_MODEL = "hight_light";
    String Q_FULL_MODEL = "full_model";
    String Q_SUGGEST_MODEL = "suggest_model";
}

创建SolrJQueryDao类

spring日常开发的dao层

@Repository
public class SolrJQueryDao {

    @Autowired
    private HttpSolrClient httpSolrClient;

    public QueryResponse query(SolrQuery query) {

        QueryResponse response = null;
        try {
            response = httpSolrClient.query(query);
        } catch (SolrServerException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }
        return response;
    }

    public QueryResponse query(String query) throws IOException, SolrServerException {
        SolrQuery entries = new SolrQuery();
        entries.setQuery(query);
        //执行查询,获取结果
        return httpSolrClient.query(entries);
    }


    public QueryResponse query(String q,String fieldName) throws IOException, SolrServerException {
        SolrQuery entries = new SolrQuery();
        entries.setQuery(fieldName+":"+q);
        //执行查询,获取结果
        return httpSolrClient.query(entries);
    }

    public QueryResponse query(Map<String,String> map) throws IOException, SolrServerException {
        SolrQuery build = SolrJQueryBuilder
                .builder()
                .buildQuery(map)
                .build();
        return query(build);
    }

    public QueryResponse queryByHightLight(SolrQuery query, List highLights, String prefix, String postfix){
        // 高亮的前后缀 设置高亮域 开启高亮
        SolrQuery buildHighLightQ = SolrJUtil.buildHighLightQuery(query, highLights, prefix, postfix);
        return query(buildHighLightQ);
    }


    public QueryResponse queryByHightLight(Map<String,String> queryMap, List highLights, String prefix, String postfix,String start){
        SolrQuery query = SolrJQueryBuilder
                .builder()
                .buildPageStart(start)
                .buildQuery(queryMap, 0.7)
                .buildDefaultOperator("and")
                .buildHighLightsQuery(highLights, prefix, postfix)
                .build();
        return query(query);
    }

    public QueryResponse queryByFull(Map<String,String> queryMap, List highLights, String prefix,String postfix,String q){
        SolrQuery query= SolrJQueryBuilder
                                .builder()
                                .buildQuery(queryMap,0.7)
                                .buildDefaultOperator("and")
                                .buildSpellcheckQuery()
                                .buildHighLightsQuery(highLights, prefix, postfix)
                                .build();
        return query(query);
    }

    public QueryResponse queryBySuggest(String q){
        SolrQuery build = SolrJQueryBuilder
                .builder()
                .buildSuggestQuery(q)
                .build();
        return query(build);
    }
}

vue3工程

前端页面用vue3做，在引入组件依赖vue-element plus方便开发，前端页面代码不方便展示，这里给个展示图