Nutch开发(七)
实现一个迷你的博客搜索引擎
网上有很多nutch制作搜索引擎的文章,但是他们使用的nutch都是十分老的版本,如:nutch1.2,我们也知道nutch发展到现在已经出来很多版本,就我现在使用的nutch1.18版本早就吧nutch的全文索引搜索的功能独立出去了。
所以我们现在使用nutch+slor做一个仿造必应的迷你的博客搜索引擎。
准备工作
- 下载一个solr8.11
- 下载一个nutch1.18
- 创建一个maven工程
- 创建一个vue3工程
使用solrJ
自动建议这功能,spring data solr是没封装的。
solrJ可定制性更高,建议用solrJ。关于solrJ的使用,请自己上网搜索资料。
创建SolrJUtil类
一个查询参数拼接的类
@Slf4j
public class SolrJUtil {
public static String toQueryStr(Map<String,String> map){
StringBuffer stringBuffer = new StringBuffer();
map.forEach((field,q)->{
stringBuffer.append(field);
stringBuffer.append(":");
stringBuffer.append(q);
stringBuffer.append(" && ");
});
return stringBuffer.substring(0,stringBuffer.length()-4);
}
public static String toQueryStr(Map<String,String> map,Double similarity){
StringBuffer stringBuffer = new StringBuffer();
map.forEach((field,q)->{
stringBuffer.append(field);
stringBuffer.append(":");
stringBuffer.append(q);
if (CommonConstants.Q_DEC_NAME.equals(field))stringBuffer.append("~"+similarity.toString());
stringBuffer.append(" || ");
});
log.info("stringBuffer == {}",stringBuffer.substring(0,stringBuffer.length()-4));
return stringBuffer.substring(0,stringBuffer.length()-4);
}
public static String toQueryStr(String fieldName,String q){
return fieldName+":"+q;
}
public static String toHighlightFieldStr(List highLights){
StringBuffer stringBuffer = new StringBuffer();
highLights.stream().forEach(v->{
stringBuffer.append(v.toString());
stringBuffer.append(",");
});
return stringBuffer.substring(0,stringBuffer.length()-1);
}
public static SolrQuery buildHighLightQuery(SolrQuery query,List highLights,String prefix, String postfix){
//开启高亮
query.setHighlight(true);
//设置高亮域
// 高亮的前后缀
String highlightFieldStr = SolrJUtil.toHighlightFieldStr(highLights);
query.addHighlightField(highlightFieldStr);
query.setHighlightSimplePre(prefix);
query.setHighlightSimplePost(postfix);
return query;
}
public static SolrQuery buildQuery(Map<String,String> queryMap){
SolrQuery query = new SolrQuery();
String queryStr = SolrJUtil.toQueryStr(queryMap);
query.setQuery(queryStr);
return query;
}
public static SolrQuery buildQuery(Map<String,String> queryMap,Double similarity){
SolrQuery query = new SolrQuery();
String queryStr = SolrJUtil.toQueryStr(queryMap,similarity);
query.setQuery(queryStr);
return query;
}
public static SolrQuery buildSpellcheckQuery(SolrQuery query){
query.set("spellcheck",true);
query.set("spellcheck.extendedResults",true);
query.set("spellcheck.count",6);
query.set("spellcheck.collate",true);
return query;
}
}
创建SolrJQueryBuilder类
SolrQuery的builder类
@Slf4j
public class SolrJQueryBuilder {
private SolrQuery solrQuery;
public SolrJQueryBuilder(){
solrQuery = new SolrQuery();
}
public static SolrJQueryBuilder builder(){
return new SolrJQueryBuilder();
}
public SolrQuery build(){
return solrQuery;
}
public SolrJQueryBuilder buildQuery(Map<String,String> queryMap){
String queryStr = SolrJUtil.toQueryStr(queryMap);
solrQuery.setQuery(queryStr);
return this;
}
public SolrJQueryBuilder buildQuery(Map<String,String> queryMap,Double similarity){
String queryStr = SolrJUtil.toQueryStr(queryMap,similarity);
log.info("queryStr == {}",queryStr);
solrQuery.setQuery(queryStr);
return this;
}
public SolrJQueryBuilder buildDefaultOperator(String op){
solrQuery.setParam("q.op", "OR");
return this;
}
public SolrJQueryBuilder buildPageStart(String start){
solrQuery.setStart(Integer.valueOf(start));
return this;
}
public SolrJQueryBuilder buildHighLightsQuery(List highLights, String prefix, String postfix){
//开启高亮
solrQuery.setHighlight(true);
//设置高亮域
// 高亮的前后缀
String highlightFieldStr = SolrJUtil.toHighlightFieldStr(highLights);
solrQuery.addHighlightField(highlightFieldStr);
solrQuery.setHighlightSimplePre(prefix);
solrQuery.setHighlightSimplePost(postfix);
return this;
}
public SolrJQueryBuilder buildSpellcheckQuery(){
solrQuery.set("spellcheck",true);
solrQuery.set("spellcheck.extendedResults",true);
solrQuery.set("spellcheck.count",1);
solrQuery.set("spellcheck.collate",true);
return this;
}
public SolrJQueryBuilder buildSuggestQuery(String q){
solrQuery.set("suggest",true);
solrQuery.set("suggest.dictionary","suggester");
solrQuery.set("suggest.q",q);
solrQuery.set("suggest.count",5);
return this;
}
}
创建CommonConstants接口
这个用来装一些公共常量
public interface CommonConstants {
String NUTCH_COLLECTION = "nutch";
String Q_TITLE_NAME = "title";
String Q_DEC_NAME = "description";
String Q_CONTENT_NAME = "content";
String Q_SELF = "self";
String Q_HIGHLIGHT_MODEL = "hight_light";
String Q_FULL_MODEL = "full_model";
String Q_SUGGEST_MODEL = "suggest_model";
}
创建SolrJQueryDao类
spring日常开发的dao层
@Repository
public class SolrJQueryDao {
@Autowired
private HttpSolrClient httpSolrClient;
public QueryResponse query(SolrQuery query) {
QueryResponse response = null;
try {
response = httpSolrClient.query(query);
} catch (SolrServerException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
return response;
}
public QueryResponse query(String query) throws IOException, SolrServerException {
SolrQuery entries = new SolrQuery();
entries.setQuery(query);
//执行查询,获取结果
return httpSolrClient.query(entries);
}
public QueryResponse query(String q,String fieldName) throws IOException, SolrServerException {
SolrQuery entries = new SolrQuery();
entries.setQuery(fieldName+":"+q);
//执行查询,获取结果
return httpSolrClient.query(entries);
}
public QueryResponse query(Map<String,String> map) throws IOException, SolrServerException {
SolrQuery build = SolrJQueryBuilder
.builder()
.buildQuery(map)
.build();
return query(build);
}
public QueryResponse queryByHightLight(SolrQuery query, List highLights, String prefix, String postfix){
// 高亮的前后缀 设置高亮域 开启高亮
SolrQuery buildHighLightQ = SolrJUtil.buildHighLightQuery(query, highLights, prefix, postfix);
return query(buildHighLightQ);
}
public QueryResponse queryByHightLight(Map<String,String> queryMap, List highLights, String prefix, String postfix,String start){
SolrQuery query = SolrJQueryBuilder
.builder()
.buildPageStart(start)
.buildQuery(queryMap, 0.7)
.buildDefaultOperator("and")
.buildHighLightsQuery(highLights, prefix, postfix)
.build();
return query(query);
}
public QueryResponse queryByFull(Map<String,String> queryMap, List highLights, String prefix,String postfix,String q){
SolrQuery query= SolrJQueryBuilder
.builder()
.buildQuery(queryMap,0.7)
.buildDefaultOperator("and")
.buildSpellcheckQuery()
.buildHighLightsQuery(highLights, prefix, postfix)
.build();
return query(query);
}
public QueryResponse queryBySuggest(String q){
SolrQuery build = SolrJQueryBuilder
.builder()
.buildSuggestQuery(q)
.build();
return query(build);
}
}
vue3工程
前端页面用vue3做,在引入组件依赖vue-element plus方便开发,前端页面代码不方便展示,这里给个展示图