进阶-第26__深度探秘搜索技术_实战掌握四种常见的相关度分数优化方法

之前两节课，我觉得已经很了解整个es的相关度评分的算法了，算法思想，TF/IDF，vector model，boolean model; 实际的公式，query norm，query coordination，boost

 

对相关度评分进行调节和优化的常见的4种方法

 

1. query-time boost

不添加boost

GET /forum/article/_search {   "query": {     "bool": {       "should": [         {           "match": {             "title": {               "query": "java spark"             }           }         },         {           "match": {             "content": "java spark"           }         }       ]     }   } } 结果： {   "took": 2,   "timed_out": false,   "_shards": {     "total": 5,     "successful": 5,     "failed": 0   },   "hits": {     "total": 4,     "max_score": 0.970927,     "hits": [       {         "_index": "forum",         "_type": "article",         "_id": "5",         "_score": 0.970927,         "_source": {           "articleID": "DHJK-B-1395-#Ky5",           "userID": 3,           "hidden": false,           "postDate": "2017-03-01",           "tag": [             "elasticsearch"           ],           "tag_cnt": 1,           "view_cnt": 10,           "title": "this is spark blog",           "content": "spark is best big data solution based on scala ,an programming language similar to java spark",           "sub_title": "haha, hello world",           "author_first_name": "Tonny",           "author_last_name": "Peter Smith"         }       },       {         "_index": "forum",         "_type": "article",         "_id": "2",         "_score": 0.8849759,         "_source": {           "articleID": "KDKE-B-9947-#kL5",           "userID": 1,           "hidden": false,           "postDate": "2017-01-02",           "tag": [             "java"           ],           "tag_cnt": 1,           "view_cnt": 50,           "title": "this is java blog",           "content": "i think java is the best programming language",           "sub_title": "learned a lot of course",           "author_first_name": "Smith",           "author_last_name": "Williams"         }       },       {         "_index": "forum",         "_type": "article",         "_id": "1",         "_score": 0.26742277,         "_source": {           "articleID": "XHDK-A-1293-#fJ3",           "userID": 1,           "hidden": false,           "postDate": "2017-01-01",           "tag": [             "java",             "hadoop"           ],           "tag_cnt": 2,           "view_cnt": 30,           "title": "this is java and elasticsearch blog",           "content": "i like to write best elasticsearch article",           "sub_title": "learning more courses",           "author_first_name": "Peter",           "author_last_name": "Smith"         }       },       {         "_index": "forum",         "_type": "article",         "_id": "4",         "_score": 0.155468,         "_source": {           "articleID": "QQPX-R-3956-#aD8",           "userID": 2,           "hidden": true,           "postDate": "2017-01-02",           "tag": [             "java",             "elasticsearch"           ],           "tag_cnt": 2,           "view_cnt": 80,           "title": "this is java, elasticsearch, hadoop blog",           "content": "elasticsearch and hadoop are all very good solution, i am a beginner",           "sub_title": "both of them are good",           "author_first_name": "Robbin",           "author_last_name": "Li"         }       }     ]   } }

 

Boost 可以将对应的query 的权重增强哦

GET /forum/article/_search {   "query": {     "bool": {       "should": [         {           "match": {             "title": {               "query": "java spark",               "boost": 2             }           }         },         {           "match": {             "content": "java spark"           }         }       ]     }   } } 结果： {   "took": 2,   "timed_out": false,   "_shards": {     "total": 5,     "successful": 5,     "failed": 0   },   "hits": {     "total": 4,     "max_score": 1.258609,     "hits": [       {         "_index": "forum",         "_type": "article",         "_id": "5",         "_score": 1.258609,         "_source": {           "articleID": "DHJK-B-1395-#Ky5",           "userID": 3,           "hidden": false,           "postDate": "2017-03-01",           "tag": [             "elasticsearch"           ],           "tag_cnt": 1,           "view_cnt": 10,           "title": "this is spark blog",           "content": "spark is best big data solution based on scala ,an programming language similar to java spark",           "sub_title": "haha, hello world",           "author_first_name": "Tonny",           "author_last_name": "Peter Smith"         }       },       {         "_index": "forum",         "_type": "article",         "_id": "2",         "_score": 1.083544,         "_source": {           "articleID": "KDKE-B-9947-#kL5",           "userID": 1,           "hidden": false,           "postDate": "2017-01-02",           "tag": [             "java"           ],           "tag_cnt": 1,           "view_cnt": 50,           "title": "this is java blog",           "content": "i think java is the best programming language",           "sub_title": "learned a lot of course",           "author_first_name": "Smith",           "author_last_name": "Williams"         }       },       {         "_index": "forum",         "_type": "article",         "_id": "1",         "_score": 0.53484553,         "_source": {           "articleID": "XHDK-A-1293-#fJ3",           "userID": 1,           "hidden": false,           "postDate": "2017-01-01",           "tag": [             "java",             "hadoop"           ],           "tag_cnt": 2,           "view_cnt": 30,           "title": "this is java and elasticsearch blog",           "content": "i like to write best elasticsearch article",           "sub_title": "learning more courses",           "author_first_name": "Peter",           "author_last_name": "Smith"         }       },       {         "_index": "forum",         "_type": "article",         "_id": "4",         "_score": 0.310936,         "_source": {           "articleID": "QQPX-R-3956-#aD8",           "userID": 2,           "hidden": true,           "postDate": "2017-01-02",           "tag": [             "java",             "elasticsearch"           ],           "tag_cnt": 2,           "view_cnt": 80,           "title": "this is java, elasticsearch, hadoop blog",           "content": "elasticsearch and hadoop are all very good solution, i am a beginner",           "sub_title": "both of them are good",           "author_first_name": "Robbin",           "author_last_name": "Li"         }       }     ]   } }

 

2、重构查询结构

原本

GET /forum/article/_search {   "query": {     "bool": {       "should": [         {           "match": {             "content": "java"            }         },         {           "match": {             "content": "spark"           }         },          {                 "match": {                   "content": "solution"                 }               },               {                 "match": {                   "content": "beginner"                   }               }       ]     }   } } 结果： {   "took": 2,   "timed_out": false,   "_shards": {     "total": 5,     "successful": 5,     "failed": 0   },   "hits": {     "total": 4,     "max_score": 1.113083,     "hits": [       {         "_index": "forum",         "_type": "article",         "_id": "4",         "_score": 1.113083,         "_source": {           "articleID": "QQPX-R-3956-#aD8",           "userID": 2,           "hidden": true,           "postDate": "2017-01-02",           "tag": [             "java",             "elasticsearch"           ],           "tag_cnt": 2,           "view_cnt": 80,           "title": "this is java, elasticsearch, hadoop blog",           "content": "elasticsearch and hadoop are all very good solution, i am a beginner",           "sub_title": "both of them are good",           "author_first_name": "Robbin",           "author_last_name": "Li"         }       },       {         "_index": "forum",         "_type": "article",         "_id": "5",         "_score": 0.970927,         "_source": {           "articleID": "DHJK-B-1395-#Ky5",           "userID": 3,           "hidden": false,           "postDate": "2017-03-01",           "tag": [             "elasticsearch"           ],           "tag_cnt": 1,           "view_cnt": 10,           "title": "this is spark blog",           "content": "spark is best big data solution based on scala ,an programming language similar to java spark",           "sub_title": "haha, hello world",           "author_first_name": "Tonny",           "author_last_name": "Peter Smith"         }       },       {         "_index": "forum",         "_type": "article",         "_id": "2",         "_score": 0.68640786,         "_source": {           "articleID": "KDKE-B-9947-#kL5",           "userID": 1,           "hidden": false,           "postDate": "2017-01-02",           "tag": [             "java"           ],           "tag_cnt": 1,           "view_cnt": 50,           "title": "this is java blog",           "content": "i think java is the best programming language",           "sub_title": "learned a lot of course",           "author_first_name": "Smith",           "author_last_name": "Williams"         }       },       {         "_index": "forum",         "_type": "article",         "_id": "3",         "_score": 0.26742277,         "_source": {           "articleID": "JODL-X-1937-#pV7",           "userID": 2,           "hidden": false,           "postDate": "2017-01-01",           "tag": [             "hadoop"           ],           "tag_cnt": 1,           "view_cnt": 100,           "title": "this is elasticsearch blog",           "content": "i am only an elasticsearch beginner",           "sub_title": "we have a lot of fun",           "author_first_name": "Jack",           "author_last_name": "Ma"         }       }     ]   } }

 

重构后

GET /forum/article/_search {   "query": {     "bool": {       "should": [         {           "match": {             "content": "java"  //1/3           }         },         {           "match": {             "content": "spark"//1/3           }         },         {           "bool": {             "should": [               {                 "match": {                   "content": "solution"//1/6                 }               },               {                 "match": {                   "content": "beginner"//1/6                   }               }             ]           }         }       ]     }   } } 结果： {   "took": 3,   "timed_out": false,   "_shards": {     "total": 5,     "successful": 5,     "failed": 0   },   "hits": {     "total": 4,     "max_score": 1.113083,     "hits": [       {         "_index": "forum",         "_type": "article",         "_id": "4",         "_score": 1.113083,         "_source": {           "articleID": "QQPX-R-3956-#aD8",           "userID": 2,           "hidden": true,           "postDate": "2017-01-02",           "tag": [             "java",             "elasticsearch"           ],           "tag_cnt": 2,           "view_cnt": 80,           "title": "this is java, elasticsearch, hadoop blog",           "content": "elasticsearch and hadoop are all very good solution, i am a beginner",           "sub_title": "both of them are good",           "author_first_name": "Robbin",           "author_last_name": "Li"         }       },       {         "_index": "forum",         "_type": "article",         "_id": "5",         "_score": 0.970927,         "_source": {           "articleID": "DHJK-B-1395-#Ky5",           "userID": 3,           "hidden": false,           "postDate": "2017-03-01",           "tag": [             "elasticsearch"           ],           "tag_cnt": 1,           "view_cnt": 10,           "title": "this is spark blog",           "content": "spark is best big data solution based on scala ,an programming language similar to java spark",           "sub_title": "haha, hello world",           "author_first_name": "Tonny",           "author_last_name": "Peter Smith"         }       },       {         "_index": "forum",         "_type": "article",         "_id": "2",         "_score": 0.68640786,         "_source": {           "articleID": "KDKE-B-9947-#kL5",           "userID": 1,           "hidden": false,           "postDate": "2017-01-02",           "tag": [             "java"           ],           "tag_cnt": 1,           "view_cnt": 50,           "title": "this is java blog",           "content": "i think java is the best programming language",           "sub_title": "learned a lot of course",           "author_first_name": "Smith",           "author_last_name": "Williams"         }       },       {         "_index": "forum",         "_type": "article",         "_id": "3",         "_score": 0.26742277,         "_source": {           "articleID": "JODL-X-1937-#pV7",           "userID": 2,           "hidden": false,           "postDate": "2017-01-01",           "tag": [             "hadoop"           ],           "tag_cnt": 1,           "view_cnt": 100,           "title": "this is elasticsearch blog",           "content": "i am only an elasticsearch beginner",           "sub_title": "we have a lot of fun",           "author_first_name": "Jack",           "author_last_name": "Ma"         }       }     ]   } }

 

重构查询结果，达到调节不同query的比重,在es新版本中，影响越来越小了。一般情况下，没什么必要的话，大家不用也行。

3、negative boost

包含java 不包含spark

GET /forum/article/_search {   "query": {     "bool": {       "must": [         {           "match": {             "content": "java"           }         }       ],       "must_not": [         {           "match": {             "content": "spark"           }         }       ]     }   } } 结果： {   "took": 1,   "timed_out": false,   "_shards": {     "total": 5,     "successful": 5,     "failed": 0   },   "hits": {     "total": 1,     "max_score": 0.68640786,     "hits": [       {         "_index": "forum",         "_type": "article",         "_id": "2",         "_score": 0.68640786,         "_source": {           "articleID": "KDKE-B-9947-#kL5",           "userID": 1,           "hidden": false,           "postDate": "2017-01-02",           "tag": [             "java"           ],           "tag_cnt": 1,           "view_cnt": 50,           "title": "this is java blog",           "content": "i think java is the best programming language",           "sub_title": "learned a lot of course",           "author_first_name": "Smith",           "author_last_name": "Williams"         }       }     ]   } }

 

包含java,尽量不包含spark

搜索包含java，不包含spark的doc，但是这样子很死板（会导致spark 的doc的就没了）

搜索包含java，尽量不包含spark的doc，如果包含了spark，不会说排除掉这个doc，而是说将这个doc的分数降低

包含了negative(反搜索) term的doc，分数乘以negative boost，分数降低

GET /forum/article/_search {   "query": {     "boosting": {       "positive": {         "match": {           "content": "java"         }       },       "negative": {         "match": {           "content": "spark"         }       },       "negative_boost": 0.2     }   } } 结果： {   "took": 3,   "timed_out": false,   "_shards": {     "total": 5,     "successful": 5,     "failed": 0   },   "hits": {     "total": 2,     "max_score": 0.68640786,     "hits": [       {         "_index": "forum",         "_type": "article",         "_id": "2",         "_score": 0.68640786,         "_source": {           "articleID": "KDKE-B-9947-#kL5",           "userID": 1,           "hidden": false,           "postDate": "2017-01-02",           "tag": [             "java"           ],           "tag_cnt": 1,           "view_cnt": 50,           "title": "this is java blog",           "content": "i think java is the best programming language",           "sub_title": "learned a lot of course",           "author_first_name": "Smith",           "author_last_name": "Williams"         }       },       {         "_index": "forum",         "_type": "article",         "_id": "5",         "_score": 0.05753642,         "_source": {           "articleID": "DHJK-B-1395-#Ky5",           "userID": 3,           "hidden": false,           "postDate": "2017-03-01",           "tag": [             "elasticsearch"           ],           "tag_cnt": 1,           "view_cnt": 10,           "title": "this is spark blog",           "content": "spark is best big data solution based on scala ,an programming language similar to java spark",           "sub_title": "haha, hello world",           "author_first_name": "Tonny",           "author_last_name": "Peter Smith"         }       }     ]   } }

 

negative的doc，会乘以negative_boost，降低分数

4、constant_score

GET /forum/article/_search {   "query": {     "bool": {       "should": [         {           "constant_score": {             "query": {               "match": {                 "title": "java"               }             }           }         },         {           "constant_score": {             "query": {               "match": {                 "title": "spark"               }             }           }         }       ]     }   } } 结果： {   "took": 1,   "timed_out": false,   "_shards": {     "total": 5,     "successful": 5,     "failed": 0   },   "hits": {     "total": 4,     "max_score": 1,     "hits": [       {         "_index": "forum",         "_type": "article",         "_id": "5",         "_score": 1,         "_source": {           "articleID": "DHJK-B-1395-#Ky5",           "userID": 3,           "hidden": false,           "postDate": "2017-03-01",           "tag": [             "elasticsearch"           ],           "tag_cnt": 1,           "view_cnt": 10,           "title": "this is spark blog",           "content": "spark is best big data solution based on scala ,an programming language similar to java spark",           "sub_title": "haha, hello world",           "author_first_name": "Tonny",           "author_last_name": "Peter Smith"         }       },       {         "_index": "forum",         "_type": "article",         "_id": "2",         "_score": 1,         "_source": {           "articleID": "KDKE-B-9947-#kL5",           "userID": 1,           "hidden": false,           "postDate": "2017-01-02",           "tag": [             "java"           ],           "tag_cnt": 1,           "view_cnt": 50,           "title": "this is java blog",           "content": "i think java is the best programming language",           "sub_title": "learned a lot of course",           "author_first_name": "Smith",           "author_last_name": "Williams"         }       },       {         "_index": "forum",         "_type": "article",         "_id": "4",         "_score": 1,         "_source": {           "articleID": "QQPX-R-3956-#aD8",           "userID": 2,           "hidden": true,           "postDate": "2017-01-02",           "tag": [             "java",             "elasticsearch"           ],           "tag_cnt": 2,           "view_cnt": 80,           "title": "this is java, elasticsearch, hadoop blog",           "content": "elasticsearch and hadoop are all very good solution, i am a beginner",           "sub_title": "both of them are good",           "author_first_name": "Robbin",           "author_last_name": "Li"         }       },       {         "_index": "forum",         "_type": "article",         "_id": "1",         "_score": 1,         "_source": {           "articleID": "XHDK-A-1293-#fJ3",           "userID": 1,           "hidden": false,           "postDate": "2017-01-01",           "tag": [             "java",             "hadoop"           ],           "tag_cnt": 2,           "view_cnt": 30,           "title": "this is java and elasticsearch blog",           "content": "i like to write best elasticsearch article",           "sub_title": "learning more courses",           "author_first_name": "Peter",           "author_last_name": "Smith"         }       }     ]   } }

 

如果你压根儿不需要相关度评分，直接走constant_score加filter，所有的doc分数都是1，没有评分的概念了