前言
目前golang操作elasticsearch的第三方包中最流行的是:
https://github.com/olivere/elastic
版本说明
golang的elastic开发包和elasticsearch版本有一些对应关系,在开发前需要注意下,必须选择正确的版本,
下面是golang elastic开发包和elasticsearch版本关系表:
elasticsearch version go elastic version 7.x 7.0 6.x 6.0 5.x 5.0
1.go-elastic快速入门
1.1 创建客户端
在操作ES之前需要创建一个client,用于操作ES,在创建es client时需要提供es的连接参数
package main
import "fmt"
import "github.com/olivere/elastic/v7"
func main ( ) {
client, err := elastic. NewClient (
elastic. SetURL ( "http://127.0.0.1:9200" , "http://127.0.0.1:9201" ) ,
elastic. SetBasicAuth ( "user" , "secret" ) )
if err != nil {
fmt. Printf ( "连接失败: %v\n" , err)
} else {
fmt. Println ( "连接成功" )
}
}
1.2 创建索引
package main
import (
"context"
"fmt"
"github.com/olivere/elastic/v7"
)
const mapping = `
{
"mappings": {
"properties": {
"user": {
"type": "keyword"
},
"message": {
"type": "text"
},
"image": {
"type": "keyword"
},
"created": {
"type": "date"
},
"tags": {
"type": "keyword"
},
"location": {
"type": "geo_point"
},
"suggest_field": {
"type": "completion"
}
}
}
}`
func main ( ) {
client, err := elastic. NewClient (
elastic. SetURL ( "http://127.0.0.1:9200" , "http://127.0.0.1:9201" ) ,
elastic. SetBasicAuth ( "user" , "secret" ) )
if err != nil {
fmt. Printf ( "连接失败: %v\n" , err)
} else {
fmt. Println ( "连接成功" )
}
ctx := context. Background ( )
exists, err := client. IndexExists ( "weibo" ) . Do ( ctx)
if err != nil {
panic ( err)
}
if ! exists {
_ , err := client. CreateIndex ( "weibo" ) . BodyString ( mapping) . Do ( ctx)
if err != nil {
panic ( err)
}
}
}
1.3 插入数据
type Weibo struct {
User string `json:"user"`
Message string `json:"message"`
Retweets int `json:"retweets"`
Image string `json:"image,omitempty"`
Created time. Time `json:"created,omitempty"`
Tags [ ] string `json:"tags,omitempty"`
Location string `json:"location,omitempty"`
Suggest * elastic. SuggestField `json:"suggest_field,omitempty"`
}
上面struct定义的时候,都定义了json结构,因为ES请求使用的是json格式,在发送ES请求的时候,会自动
转换成json格式。使用struct结构插入一条ES文档数据,
msg1 := Weibo{ User: "olivere" , Message: "打酱油的一天" , Retweets: 0 }
put1, err := client. Index ( ) .
Index ( "weibo" ) .
Id ( "1" ) .
BodyJson ( msg1) .
Do ( ctx)
if err != nil {
panic ( err)
}
fmt. Printf ( "文档Id %s, 索引名 %s\n" , put1. Id, put1. Index)
1.4 查询数据
get1, err := client. Get ( ) .
Index ( "weibo" ) .
Id ( "1" ) .
Do ( ctx)
if err != nil {
panic ( err)
}
if get1. Found {
fmt. Printf ( "文档id=%s 版本号=%d 索引名=%s\n" , get1. Id, get1. Version, get1. Index)
}
# 手动将文档内容转换成go struct 对象
msg2 := Weibo{ }
data, _ := get1. Source. MarshalJSON ( )
json. Unmarshal ( data, & msg2)
fmt. Println ( msg2. Message)
1.5 更新数据
_ , err := client. Update ( ) .
Index ( "weibo" ) .
Id ( "1" ) .
Doc ( map [ string ] interface { } { "retweets" : 0 } ) .
Do ( ctx)
if err != nil {
panic ( err)
}
1.6 删除数据
_ , err := client. Delete ( ) .
Index ( "weibo" ) .
Id ( "1" ) .
Do ( ctx)
if err != nil {
panic ( err)
}
2.golang elasticsearch连接配置
创建客户端时,有时需要提供连接参数配置,其配置有如下几个:
elasticsearch 连接地址(必需) 日志输出(非必需) elasticsearch 账户/密码(非必需) 监控检测(非必需) 失败重试次数(非必需) gzip设置(非必需)
client, err := elastic. NewClient (
elastic. SetURL ( "http://10.0.1.1:9200" , "http://10.0.1.2:9200" ) ,
elastic. SetBasicAuth ( "user" , "secret" ) ,
elastic. SetGzip ( true ) ,
elastic. SetHealthcheckInterval ( 10 * time. Second) ,
elastic. SetMaxRetries ( 5 ) ,
elastic. SetErrorLog ( log. New ( os. Stderr, "ELASTIC " , log. LstdFlags) ) ,
elastic. SetInfoLog ( log. New ( os. Stdout, "" , log. LstdFlags) ) )
if err != nil {
panic ( err)
}
3 .golang elasticsearch文档操作(CRUD)
介绍go语言对elasticsearch文档的基础操作:增、删、改、查等
3.1 添加文档
package main
import (
"context"
"encoding/json"
"fmt"
"github.com/olivere/elastic/v7"
"log"
"os"
"time"
)
type Article struct {
Title string
Content string
Author string
Created time. Time
}
func main ( ) {
client, err := elastic. NewClient (
elastic. SetURL ( "http://127.0.0.1:9200" , "http://127.0.0.1:9201" ) ,
elastic. SetBasicAuth ( "user" , "secret" ) ,
elastic. SetGzip ( true ) ,
elastic. SetHealthcheckInterval ( 10 * time. Second) ,
elastic. SetMaxRetries ( 5 ) ,
elastic. SetErrorLog ( log. New ( os. Stderr, "ELASTIC " , log. LstdFlags) ) ,
elastic. SetInfoLog ( log. New ( os. Stdout, "" , log. LstdFlags) ) )
if err != nil {
fmt. Printf ( "连接失败: %v\n" , err)
} else {
fmt. Println ( "连接成功" )
}
ctx := context. Background ( )
blog := Article{ Title: "golang es教程" , Content: "go如何操作ES" , Author: "tizi" , Created: time. Now ( ) }
put1, err := client. Index ( ) .
Index ( "blogs" ) .
Id ( "1" ) .
BodyJson ( blog) .
Do ( ctx)
if err != nil {
panic ( err)
}
fmt. Printf ( "文档Id %s, 索引名 %s\n" , put1. Id, put1. Index)
}
3.2 查询文档
根据id查询文档:
get1, err := client. Get ( ) .
Index ( "blogs" ) .
Id ( "1" ) .
Do ( ctx)
if err != nil {
panic ( err)
}
if get1. Found {
fmt. Printf ( "文档id=%s 版本号=%d 索引名=%s\n" , get1. Id, get1. Version, get1. Index)
}
msg2 := Article{ }
data, _ := get1. Source. MarshalJSON ( )
json. Unmarshal ( data, & msg2)
fmt. Println ( msg2. Title)
批量查询文档:
result, err := client. MultiGet ( ) .
Add ( elastic. NewMultiGetItem ( ) .
Index ( "blogs" ) .
Id ( "1" ) ) .
Add ( elastic. NewMultiGetItem ( ) . Index ( "blogs" ) . Id ( "2" ) ) .
Add ( elastic. NewMultiGetItem ( ) . Index ( "blogs" ) . Id ( "3" ) ) .
Do ( ctx)
if err != nil {
panic ( err)
}
for _ , doc := range result. Docs {
var content Article
tmp, _ := doc. Source. MarshalJSON ( )
err := json. Unmarshal ( tmp, & content)
if err != nil {
panic ( err)
}
fmt. Println ( content. Title)
}
3.3 更新文档
根据id更新文档:
_ , err := client. Update ( ) .
Index ( "blogs" ) .
Id ( "1" ) .
Doc ( map [ string ] interface { } { "Title" : "新的文章标题" } ) .
Do ( ctx)
if err != nil {
panic ( err)
}
根据条件更新文档:
_ , err = client. UpdateByQuery ( "blogs" ) .
Query ( elastic. NewTermQuery ( "Author" , "tizi" ) ) .
Script ( elastic. NewScript ( "ctx._source['Title']='1111111'" ) ) .
ProceedOnVersionConflict ( ) .
Do ( ctx)
3.4 删除文档
根据id删除文档:
_ , err := client. Delete ( ) .
Index ( "blogs" ) .
Id ( "1" ) .
Do ( ctx)
if err != nil {
panic ( err)
}
根据条件删除文档:
_ , _ = client. DeleteByQuery ( "blogs" ) .
Query ( elastic. NewTermQuery ( "Author" , "tizi" ) ) .
ProceedOnVersionConflict ( ) .
Do ( ctx)
4. go-elasticsearch 查询语法
4.1 精确匹配单个字段
package main
import (
"context"
"fmt"
"github.com/olivere/elastic/v7"
"log"
"os"
"reflect"
"time"
)
type Article struct {
Title string
Content string
Author string
Created time. Time
}
func main ( ) {
client, err := elastic. NewClient (
elastic. SetURL ( "http://127.0.0.1:9200" , "http://127.0.0.1:9201" ) ,
elastic. SetBasicAuth ( "user" , "secret" ) ,
elastic. SetGzip ( true ) ,
elastic. SetHealthcheckInterval ( 10 * time. Second) ,
elastic. SetMaxRetries ( 5 ) ,
elastic. SetErrorLog ( log. New ( os. Stderr, "ELASTIC " , log. LstdFlags) ) ,
elastic. SetInfoLog ( log. New ( os. Stdout, "" , log. LstdFlags) ) )
if err != nil {
fmt. Printf ( "连接失败: %v\n" , err)
} else {
fmt. Println ( "连接成功" )
}
ctx := context. Background ( )
termQuery := elastic. NewTermQuery ( "Author" , "tizi" )
searchResult, err := client. Search ( ) .
Index ( "blogs" ) .
Query ( termQuery) .
Sort ( "Created" , true ) .
From ( 0 ) .
Size ( 10 ) .
Pretty ( true ) .
Do ( ctx)
if err != nil {
panic ( err)
}
fmt. Printf ( "查询消耗时间 %d ms, 结果总数: %d\n" , searchResult. TookInMillis, searchResult. TotalHits ( ) )
if searchResult. TotalHits ( ) > 0 {
var b1 Article
for _ , item := range searchResult. Each ( reflect. TypeOf ( b1) ) {
if t, ok := item. ( Article) ; ok {
fmt. Println ( t. Title)
}
}
}
}
4.2 通过terms实现SQL的in查询
通过terms语法可以实现多值查询效果。如:
termsQuery := elastic. NewTermsQuery ( "Author" , "tizi" , "tizi365" )
searchResult, err := client. Search ( ) .
Index ( "blogs" ) .
Query ( termsQuery) .
Sort ( "Created" , true ) .
From ( 0 ) .
Size ( 10 ) .
Do ( ctx)
4.3 匹配单个字段
某个字段使用全文搜索,也就是elasticsearch中的match语法。 如:
matchQuery := elastic. NewMatchQuery ( "Title" , "golang es教程" )
searchResult, err := client. Search ( ) .
Index ( "blogs" ) .
Query ( matchQuery) .
Sort ( "Created" , true ) .
From ( 0 ) .
Size ( 10 ) .
Do ( ctx)
4.4 范围查询
实现类似于Created > '2020-07-20' and Created < '2020-07-22'的范围查询条件。如:
rangeQuery := elastic. NewRangeQuery ( "Created" ) .
Gt ( "2020-07-20" ) .
Lt ( "2020-07-29" )
rangeQuery := elastic. NewRangeQuery ( "id" ) .
Gte ( 1 ) .
Lte ( 10 )
4.5 bool组合查询
must条件:类似于SQL的and条件,代表必需匹配。 如:
boolQuery := elastic. NewBoolQuery ( ) . Must ( )
termQuery := elastic. NewTermQuery ( "Author" , "tizi" )
matchQuery := elastic. NewMatchQuery ( "Title" , "golang es教程" )
boolQuery. Must ( termQuery, matchQuery)
searchResult, err := client. Search ( ) .
Index ( "blogs" ) .
Query ( boolQuery) .
Sort ( "Created" , true ) .
From ( 0 ) .
Size ( 10 ) .
Do ( ctx)
must_not条件:与must条件相反。
boolQuery := elastic. NewBoolQuery ( ) . Must ( )
termQuery := elastic. NewTermQuery ( "Author" , "tizi" )
boolQuery. MustNot ( termQuery)
should条件:类似于SQL中or,匹配其中一个即可。如:
boolQuery := elastic. NewBoolQuery ( ) . Must ( )
termQuery := elastic. NewTermQuery ( "Author" , "tizi" )
matchQuery := elastic. NewMatchQuery ( "Title" , "golang es教程" )
boolQuery. Should ( termQuery, matchQuery)
5. go-elasticsearch 聚合分析
elasticsearch聚合分析主要包括:
—指标聚合
—桶聚合
这两种聚合可以嵌套使用,桶聚合通常用于对数据分组,然后分组内的数据可以使用指标聚合汇总数据。
如下面这个例子:
client, err := elastic. NewClient ( )
if err != nil {
panic ( err)
}
timeline := elastic. NewTermsAggregation ( ) . Field ( "user" ) . Size ( 10 ) . OrderByCountDesc ( )
histogram := elastic. NewDateHistogramAggregation ( ) . Field ( "created" ) . CalendarInterval ( "year" )
timeline = timeline. SubAggregation ( "history" , histogram)
searchResult, err := client. Search ( ) .
Index ( "twitter" ) .
Query ( elastic. NewMatchAllQuery ( ) ) .
Aggregation ( "timeline" , timeline) .
Pretty ( true ) .
Do ( context. Background ( ) )
if err != nil {
panic ( err)
}
agg, found := searchResult. Aggregations. Terms ( "timeline" )
if ! found {
log. Fatalf ( "we should have a terms aggregation called %q" , "timeline" )
}
for _ , userBucket := range agg. Buckets {
user := userBucket. Key
histogram, found := userBucket. DateHistogram ( "history" )
if found {
for _ , year := range histogram. Buckets {
var key string
if s := year. KeyAsString; s != nil {
key = * s
}
fmt. Printf ( "user %q has %d tweets in %q\n" , user, year. DocCount, key)
}
}
}
5.1 指标聚合
值聚合:主要用于统计文档总数,类似SQL的count函数。
package main
import (
"context"
"fmt"
"github.com/olivere/elastic/v7"
"time"
)
func main ( ) {
client, err := elastic. NewClient ( )
if err != nil {
panic ( err)
}
ctx := context. Background ( )
aggs := elastic. NewValueCountAggregation ( ) .
Field ( "order_id" )
searchResult, err := client. Search ( ) .
Index ( "kibana_sample_data_flights" ) .
Query ( elastic. NewMatchAllQuery ( ) ) .
Aggregation ( "total" , aggs) .
Size ( 0 ) .
Do ( ctx)
if err != nil {
panic ( err)
}
agg, found := searchResult. Aggregations. ValueCount ( "total" )
if found {
fmt. Println ( * agg. Value)
}
}
基数聚合:也是用于统计文档的总数,跟Value Count的区别是,基数聚合会去重,不会统计重复的值
,类似SQL的count(DISTINCT 字段)用法。
aggs := elastic. NewCardinalityAggregation ( ) .
Field ( "order_id" )
searchResult, err := client. Search ( ) .
Index ( "kibana_sample_data_flights" ) .
Query ( elastic. NewMatchAllQuery ( ) ) .
Aggregation ( "total" , aggs) .
Size ( 0 ) .
Do ( ctx)
if err != nil {
panic ( err)
}
agg, found := searchResult. Aggregations. Cardinality ( "total" )
if found {
fmt. Println ( * agg. Value)
}
求平均值:
aggs := elastic. NewAvgAggregation ( ) .
Field ( "price" )
searchResult, err := client. Search ( ) .
Index ( "kibana_sample_data_flights" ) .
Query ( elastic. NewMatchAllQuery ( ) ) .
Aggregation ( "avg_price" , aggs) .
Size ( 0 ) .
Do ( ctx)
if err != nil {
panic ( err)
}
agg, found := searchResult. Aggregations. Avg ( "avg_price" )
if found {
fmt. Println ( * agg. Value)
}
求和计算:
aggs := elastic. NewSumAggregation ( ) .
Field ( "price" )
searchResult, err := client. Search ( ) .
Index ( "kibana_sample_data_flights" ) .
Query ( elastic. NewMatchAllQuery ( ) ) .
Aggregation ( "total_price" , aggs) .
Size ( 0 ) .
Do ( ctx)
if err != nil {
panic ( err)
}
agg, found := searchResult. Aggregations. Sum ( "total_price" )
if found {
fmt. Println ( * agg. Value)
}
求最大值:
aggs := elastic. NewMaxAggregation ( ) .
Field ( "price" )
searchResult, err := client. Search ( ) .
Index ( "kibana_sample_data_flights" ) .
Query ( elastic. NewMatchAllQuery ( ) ) .
Aggregation ( "max_price" , aggs) .
Size ( 0 ) .
Do ( ctx)
if err != nil {
panic ( err)
}
agg, found := searchResult. Aggregations. Max ( "max_price" )
if found {
fmt. Println ( * agg. Value)
}
求最小值:
aggs := elastic. NewMinAggregation ( ) .
Field ( "price" )
searchResult, err := client. Search ( ) .
Index ( "kibana_sample_data_flights" ) .
Query ( elastic. NewMatchAllQuery ( ) ) .
Aggregation ( "min_price" , aggs) .
Size ( 0 ) .
Do ( ctx)
if err != nil {
panic ( err)
}
agg, found := searchResult. Aggregations. Min ( "min_price" )
if found {
fmt. Println ( * agg. Value)
}
5.2 桶聚合
1.Terms聚合:
package main
import (
"context"
"fmt"
"github.com/olivere/elastic/v7"
"log"
)
func main ( ) {
client, err := elastic. NewClient ( )
if err != nil {
panic ( err)
}
ctx := context. Background ( )
aggs := elastic. NewTermsAggregation ( ) .
Field ( "shop_id" )
searchResult, err := client. Search ( ) .
Index ( "shops" ) .
Query ( elastic. NewMatchAllQuery ( ) ) .
Aggregation ( "shop" , aggs) .
Size ( 0 ) .
Do ( ctx)
if err != nil {
panic ( err)
}
agg, found := searchResult. Aggregations. Terms ( "shop" )
if ! found {
log. Fatal ( "没有找到聚合数据" )
}
for _ , bucket := range agg. Buckets {
bucketValue := bucket. Key
fmt. Printf ( "bucket = %q 文档总数 = %d\n" , bucketValue, bucket. DocCount)
}
}
2.Histogram聚合
aggs := elastic. NewHistogramAggregation ( ) .
Field ( "price" ) .
Interval ( 50 )
searchResult, err := client. Search ( ) .
Index ( "order" ) .
Query ( elastic. NewMatchAllQuery ( ) ) .
Aggregation ( "prices" , aggs) .
Size ( 0 ) .
Do ( ctx)
if err != nil {
panic ( err)
}
agg, found := searchResult. Aggregations. Histogram ( "prices" )
if ! found {
log. Fatal ( "没有找到聚合数据" )
}
for _ , bucket := range agg. Buckets {
bucketValue := bucket. Key
fmt. Printf ( "bucket = %q 文档总数 = %d\n" , bucketValue, bucket. DocCount)
}
3.Date histogram聚合
aggs := elastic. NewDateHistogramAggregation ( ) .
Field ( "date" ) .
CalendarInterval ( "month" ) .
Format ( "yyyy-MM-dd" )
searchResult, err := client. Search ( ) .
Index ( "order" ) .
Query ( elastic. NewMatchAllQuery ( ) ) .
Aggregation ( "sales_over_time" , aggs) .
Size ( 0 ) .
Do ( ctx)
if err != nil {
panic ( err)
}
agg, found := searchResult. Aggregations. DateHistogram ( "sales_over_time" )
if ! found {
log. Fatal ( "没有找到聚合数据" )
}
for _ , bucket := range agg. Buckets {
bucketValue := bucket. Key
fmt. Printf ( "bucket = %q 文档总数 = %d\n" , bucketValue, bucket. DocCount)
}
4.Range聚合
aggs := elastic. NewRangeAggregation ( ) .
Field ( "price" ) .
AddUnboundedFrom ( 100 ) .
AddRange ( 100.0 , 200.0 ) .
AddUnboundedTo ( 200.0 )
searchResult, err := client. Search ( ) .
Index ( "order" ) .
Query ( elastic. NewMatchAllQuery ( ) ) .
Aggregation ( "price_ranges" , aggs) .
Size ( 0 ) .
Do ( ctx)
if err != nil {
panic ( err)
}
agg, found := searchResult. Aggregations. Range ( "price_ranges" )
if ! found {
log. Fatal ( "没有找到聚合数据" )
}
for _ , bucket := range agg. Buckets {
bucketValue := bucket. Key
fmt. Printf ( "bucket = %q 文档总数 = %d\n" , bucketValue, bucket. DocCount)
}
5.嵌套聚合
aggs := elastic. NewTermsAggregation ( ) . Field ( "shop_id" )
sumAggs := elastic. NewSumAggregation ( ) . Field ( "price" )
aggs. SubAggregation ( "total_price" , sumAggs)