Elasticsearch ~ 从入门到入坑。
文章目录
Doug Cutting ~ Hadoop 之父。
下载。
https://www.elastic.co/downloads/past-releases#elasticsearch
geek@ubuntu:~/geek$ scp elasticsearch-7.6.1-linux-x86_64.tar.gz root@192.168.142.143:/root/geek/tools_my/
root@192.168.142.143's password:
elasticsearch-7.6.1-linux-x86_64.tar.gz 100% 283MB 92.6MB/s 00:03
配置。
- 内存配置。
默认是 1G 内存。如果服务器的内存小,可以自行设置。
elasticsearch-7.6.1/config/jvm.options。
- Xms1g ~ 初始默认内存 1g。
- Xmx1g ~ 最大内存 1g。
[root@192 config]# ll
total 36
-rw-rw----. 1 root root 2831 Feb 29 08:13 elasticsearch.yml
-rw-rw----. 1 root root 2301 Feb 29 08:13 jvm.options # Java 虚拟机配置。
-rw-rw----. 1 root root 17545 Feb 29 08:18 log4j2.properties # 日志配置。
-rw-rw----. 1 root root 473 Feb 29 08:18 role_mapping.yml
-rw-rw----. 1 root root 197 Feb 29 08:18 roles.yml
-rw-rw----. 1 root root 0 Feb 29 08:18 users
-rw-rw----. 1 root root 0 Feb 29 08:18 users_roles
bin/ 启动文件。
config/ 配置文件。
log4j2 日志配置文件。
jvm.properties Java 虚拟机配置。
wlasticsearch.yml elasticsearch 配置文件。默认端口 9200。(跨域问题)。
lib/ 相关 jar 包。
logs 日志。
modules 功能模块。
plugins 插件。ik...
启动。
geek@ubuntu:~/geek/elasticsearch-7.6.1/bin$ ./elasticsearch
Java HotSpot(TM) 64-Bit Server VM warning: Option UseConcMarkSweepGC was deprecated in version 9.0 and will likely be removed in a future release.
[2020-06-17T13:20:16,105][INFO ][o.e.e.NodeEnvironment ] [ubuntu] using [1] data paths, mounts [[/ (/dev/sda5)]], net usable_space [23.2gb], net total_space [48.4gb], types [ext4]
[2020-06-17T13:20:16,116][INFO ][o.e.e.NodeEnvironment ] [ubuntu] heap size [1015.6mb], compressed ordinary object pointers [true]
[2020-06-17T13:20:16,269][INFO ][o.e.n.Node ] [ubuntu] node name [ubuntu], node ID [uFxBfA6NS5qrZSrKaJ3W9g], cluster name [elasticsearch]
[2020-06-17T13:20:16,272][INFO ][o.e.n.Node ] [ubuntu] version[7.6.1], pid[32262], build[default/tar/aa751e09be0a5072e8570670309b1f12348f023b/2020-02-29T00:15:25.529771Z], OS[Linux/5.4.0-37-generic/amd64], JVM[Oracle Corporation/Java HotSpot(TM) 64-Bit Server VM/11.0.7/11.0.7+8-LTS]
[2020-06-17T13:20:16,275][INFO ][o.e.n.Node ] [ubuntu] JVM home [/home/geek/geek/tools_my/jdk-11.0.7]
[2020-06-17T13:20:16,276][INFO ][o.e.n.Node ] [ubuntu] JVM arguments [-Des.networkaddress.cache.ttl=60, -Des.networkaddress.cache.negative.ttl=10, -XX:+AlwaysPreTouch, -Xss1m, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djna.nosys=true, -XX:-OmitStackTraceInFastThrow, -Dio.netty.noUnsafe=true, -Dio.netty.noKeySetOptimization=true, -Dio.netty.recycler.maxCapacityPerThread=0, -Dio.netty.allocator.numDirectArenas=0, -Dlog4j.shutdownHookEnabled=false, -Dlog4j2.disable.jmx=true, -Djava.locale.providers=COMPAT, -Xms1g, -Xmx1g, -XX:+UseConcMarkSweepGC, -XX:CMSInitiatingOccupancyFraction=75, -XX:+UseCMSInitiatingOccupancyOnly, -Djava.io.tmpdir=/tmp/elasticsearch-3772212386124888272, -XX:+HeapDumpOnOutOfMemoryError, -XX:HeapDumpPath=data, -XX:ErrorFile=logs/hs_err_pid%p.log, -Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m, -XX:MaxDirectMemorySize=536870912, -Des.path.home=/home/geek/geek/elasticsearch-7.6.1, -Des.path.conf=/home/geek/geek/elasticsearch-7.6.1/config, -Des.distribution.flavor=default, -Des.distribution.type=tar, -Des.bundled_jdk=true]
[2020-06-17T13:20:20,094][INFO ][o.e.p.PluginsService ] [ubuntu] loaded module [aggs-matrix-stats]
[2020-06-17T13:20:20,095][INFO ][o.e.p.PluginsService ] [ubuntu] loaded module [analysis-common]
[2020-06-17T13:20:20,096][INFO ][o.e.p.PluginsService ] [ubuntu] loaded module [flattened]
[2020-06-17T13:20:20,097][INFO ][o.e.p.PluginsService ] [ubuntu] loaded module [frozen-indices]
[2020-06-17T13:20:20,097][INFO ][o.e.p.PluginsService ] [ubuntu] loaded module [ingest-common]
[2020-06-17T13:20:20,097][INFO ][o.e.p.PluginsService ] [ubuntu] loaded module [ingest-geoip]
[2020-06-17T13:20:20,098][INFO ][o.e.p.PluginsService ] [ubuntu] loaded module [ingest-user-agent]
[2020-06-17T13:20:20,099][INFO ][o.e.p.PluginsService ] [ubuntu] loaded module [lang-expression]
[2020-06-17T13:20:20,100][INFO ][o.e.p.PluginsService ] [ubuntu] loaded module [lang-mustache]
[2020-06-17T13:20:20,100][INFO ][o.e.p.PluginsService ] [ubuntu] loaded module [lang-painless]
[2020-06-17T13:20:20,101][INFO ][o.e.p.PluginsService ] [ubuntu] loaded module [mapper-extras]
[2020-06-17T13:20:20,101][INFO ][o.e.p.PluginsService ] [ubuntu] loaded module [parent-join]
[2020-06-17T13:20:20,102][INFO ][o.e.p.PluginsService ] [ubuntu] loaded module [percolator]
[2020-06-17T13:20:20,102][INFO ][o.e.p.PluginsService ] [ubuntu] loaded module [rank-eval]
[2020-06-17T13:20:20,104][INFO ][o.e.p.PluginsService ] [ubuntu] loaded module [reindex]
[2020-06-17T13:20:20,108][INFO ][o.e.p.PluginsService ] [ubuntu] loaded module [repository-url]
[2020-06-17T13:20:20,109][INFO ][o.e.p.PluginsService ] [ubuntu] loaded module [search-business-rules]
[2020-06-17T13:20:20,110][INFO ][o.e.p.PluginsService ] [ubuntu] loaded module [spatial]
[2020-06-17T13:20:20,110][INFO ][o.e.p.PluginsService ] [ubuntu] loaded module [transform]
[2020-06-17T13:20:20,111][INFO ][o.e.p.PluginsService ] [ubuntu] loaded module [transport-netty4]
[2020-06-17T13:20:20,111][INFO ][o.e.p.PluginsService ] [ubuntu] loaded module [vectors]
[2020-06-17T13:20:20,111][INFO ][o.e.p.PluginsService ] [ubuntu] loaded module [x-pack-analytics]
[2020-06-17T13:20:20,116][INFO ][o.e.p.PluginsService ] [ubuntu] loaded module [x-pack-ccr]
[2020-06-17T13:20:20,118][INFO ][o.e.p.PluginsService ] [ubuntu] loaded module [x-pack-core]
[2020-06-17T13:20:20,119][INFO ][o.e.p.PluginsService ] [ubuntu] loaded module [x-pack-deprecation]
[2020-06-17T13:20:20,119][INFO ][o.e.p.PluginsService ] [ubuntu] loaded module [x-pack-enrich]
[2020-06-17T13:20:20,119][INFO ][o.e.p.PluginsService ] [ubuntu] loaded module [x-pack-graph]
[2020-06-17T13:20:20,119][INFO ][o.e.p.PluginsService ] [ubuntu] loaded module [x-pack-ilm]
[2020-06-17T13:20:20,120][INFO ][o.e.p.PluginsService ] [ubuntu] loaded module [x-pack-logstash]
[2020-06-17T13:20:20,126][INFO ][o.e.p.PluginsService ] [ubuntu] loaded module [x-pack-ml]
[2020-06-17T13:20:20,128][INFO ][o.e.p.PluginsService ] [ubuntu] loaded module [x-pack-monitoring]
[2020-06-17T13:20:20,128][INFO ][o.e.p.PluginsService ] [ubuntu] loaded module [x-pack-rollup]
[2020-06-17T13:20:20,129][INFO ][o.e.p.PluginsService ] [ubuntu] loaded module [x-pack-security]
[2020-06-17T13:20:20,130][INFO ][o.e.p.PluginsService ] [ubuntu] loaded module [x-pack-sql]
[2020-06-17T13:20:20,133][INFO ][o.e.p.PluginsService ] [ubuntu] loaded module [x-pack-voting-only-node]
[2020-06-17T13:20:20,133][INFO ][o.e.p.PluginsService ] [ubuntu] loaded module [x-pack-watcher]
[2020-06-17T13:20:20,134][INFO ][o.e.p.PluginsService ] [ubuntu] no plugins loaded
[2020-06-17T13:20:27,212][INFO ][o.e.x.s.a.s.FileRolesStore] [ubuntu] parsed [0] roles from file [/home/geek/geek/elasticsearch-7.6.1/config/roles.yml]
[2020-06-17T13:20:28,302][INFO ][o.e.x.m.p.l.CppLogMessageHandler] [ubuntu] [controller/32338] [Main.cc@110] controller (64 bit): Version 7.6.1 (Build 6eb6e036390036) Copyright (c) 2020 Elasticsearch BV
[2020-06-17T13:20:29,582][DEBUG][o.e.a.ActionModule ] [ubuntu] Using REST wrapper from plugin org.elasticsearch.xpack.security.Security
[2020-06-17T13:20:29,902][INFO ][o.e.d.DiscoveryModule ] [ubuntu] using discovery type [zen] and seed hosts providers [settings]
[2020-06-17T13:20:32,097][INFO ][o.e.n.Node ] [ubuntu] initialized
[2020-06-17T13:20:32,108][INFO ][o.e.n.Node ] [ubuntu] starting ...
[2020-06-17T13:20:32,569][INFO ][o.e.t.TransportService ] [ubuntu] publish_address {127.0.0.1:9300}, bound_addresses {[::1]:9300}, {127.0.0.1:9300}
[2020-06-17T13:20:33,291][WARN ][o.e.b.BootstrapChecks ] [ubuntu] max virtual memory areas vm.max_map_count [65530] is too low, increase to at least [262144]
[2020-06-17T13:20:33,294][WARN ][o.e.b.BootstrapChecks ] [ubuntu] the default discovery settings are unsuitable for production use; at least one of [discovery.seed_hosts, discovery.seed_providers, cluster.initial_master_nodes] must be configured
[2020-06-17T13:20:33,345][INFO ][o.e.c.c.ClusterBootstrapService] [ubuntu] no discovery configuration found, will perform best-effort cluster bootstrapping after [3s] unless existing master is discovered
[2020-06-17T13:20:36,361][INFO ][o.e.c.c.Coordinator ] [ubuntu] setting initial configuration to VotingConfiguration{uFxBfA6NS5qrZSrKaJ3W9g}
[2020-06-17T13:20:36,660][INFO ][o.e.c.s.MasterService ] [ubuntu] elected-as-master ([1] nodes joined)[{ubuntu}{uFxBfA6NS5qrZSrKaJ3W9g}{AU_8ps0kTy6DdAOhlKwYOQ}{127.0.0.1}{127.0.0.1:9300}{dilm}{ml.machine_memory=4098269184, xpack.installed=true, ml.max_open_jobs=20} elect leader, _BECOME_MASTER_TASK_, _FINISH_ELECTION_], term: 1, version: 1, delta: master node changed {previous [], current [{ubuntu}{uFxBfA6NS5qrZSrKaJ3W9g}{AU_8ps0kTy6DdAOhlKwYOQ}{127.0.0.1}{127.0.0.1:9300}{dilm}{ml.machine_memory=4098269184, xpack.installed=true, ml.max_open_jobs=20}]}
[2020-06-17T13:20:36,739][INFO ][o.e.c.c.CoordinationState] [ubuntu] cluster UUID set to [-frBXq4mSDuSX7xmGpWLEw]
[2020-06-17T13:20:36,788][INFO ][o.e.c.s.ClusterApplierService] [ubuntu] master node changed {previous [], current [{ubuntu}{uFxBfA6NS5qrZSrKaJ3W9g}{AU_8ps0kTy6DdAOhlKwYOQ}{127.0.0.1}{127.0.0.1:9300}{dilm}{ml.machine_memory=4098269184, xpack.installed=true, ml.max_open_jobs=20}]}, term: 1, version: 1, reason: Publication{term=1, version=1}
[2020-06-17T13:20:36,957][INFO ][o.e.h.AbstractHttpServerTransport] [ubuntu] publish_address {127.0.0.1:9200}, bound_addresses {[::1]:9200}, {127.0.0.1:9200}
[2020-06-17T13:20:36,958][INFO ][o.e.n.Node ] [ubuntu] started
[2020-06-17T13:20:37,190][INFO ][o.e.g.GatewayService ] [ubuntu] recovered [0] indices into cluster_state
[2020-06-17T13:20:37,843][INFO ][o.e.c.m.MetaDataIndexTemplateService] [ubuntu] adding template [.watch-history-10] for index patterns [.watcher-history-10*]
[2020-06-17T13:20:37,971][INFO ][o.e.c.m.MetaDataIndexTemplateService] [ubuntu] adding template [.watches] for index patterns [.watches*]
[2020-06-17T13:20:38,056][INFO ][o.e.c.m.MetaDataIndexTemplateService] [ubuntu] adding template [.triggered_watches] for index patterns [.triggered_watches*]
[2020-06-17T13:20:38,146][INFO ][o.e.c.m.MetaDataIndexTemplateService] [ubuntu] adding template [ilm-history] for index patterns [ilm-history-1*]
[2020-06-17T13:20:38,230][INFO ][o.e.c.m.MetaDataIndexTemplateService] [ubuntu] adding template [.slm-history] for index patterns [.slm-history-1*]
[2020-06-17T13:20:38,331][INFO ][o.e.c.m.MetaDataIndexTemplateService] [ubuntu] adding template [.monitoring-logstash] for index patterns [.monitoring-logstash-7-*]
[2020-06-17T13:20:38,435][INFO ][o.e.c.m.MetaDataIndexTemplateService] [ubuntu] adding template [.monitoring-es] for index patterns [.monitoring-es-7-*]
[2020-06-17T13:20:38,537][INFO ][o.e.c.m.MetaDataIndexTemplateService] [ubuntu] adding template [.monitoring-beats] for index patterns [.monitoring-beats-7-*]
[2020-06-17T13:20:38,630][INFO ][o.e.c.m.MetaDataIndexTemplateService] [ubuntu] adding template [.monitoring-alerts-7] for index patterns [.monitoring-alerts-7]
[2020-06-17T13:20:38,750][INFO ][o.e.c.m.MetaDataIndexTemplateService] [ubuntu] adding template [.monitoring-kibana] for index patterns [.monitoring-kibana-7-*]
[2020-06-17T13:20:38,822][INFO ][o.e.x.i.a.TransportPutLifecycleAction] [ubuntu] adding index lifecycle policy [watch-history-ilm-policy]
[2020-06-17T13:20:38,906][INFO ][o.e.x.i.a.TransportPutLifecycleAction] [ubuntu] adding index lifecycle policy [ilm-history-ilm-policy]
[2020-06-17T13:20:38,988][INFO ][o.e.x.i.a.TransportPutLifecycleAction] [ubuntu] adding index lifecycle policy [slm-history-ilm-policy]
[2020-06-17T13:20:39,156][INFO ][o.e.l.LicenseService ] [ubuntu] license [1daea388-37af-40d8-b8ec-0d202c02ed07] mode [basic] + valid
[2020-06-17T13:20:39,157][INFO ][o.e.x.s.s.SecurityStatusChangeListener] [ubuntu] Active license is now [BASIC]; Security is disabled
[geek@192 bin]$ ./elasticsearch
OpenJDK 64-Bit Server VM warning: Option UseConcMarkSweepGC was deprecated in version 9.0 and will likely be removed in a future release.
[2020-05-06T14:18:28,583][INFO ][o.e.e.NodeEnvironment ] [192.168.142.143] using [1] data paths, mounts [[/ (rootfs)]], net usable_space [5.4gb], net total_space [16.9gb], types [rootfs]
[2020-05-06T14:18:28,597][INFO ][o.e.e.NodeEnvironment ] [192.168.142.143] heap size [1015.6mb], compressed ordinary object pointers [true]
[2020-05-06T14:18:28,695][INFO ][o.e.n.Node ] [192.168.142.143] node name [192.168.142.143], node ID [IVCFW76uRfWTmeK3fUxwgQ], cluster name [elasticsearch]
[2020-05-06T14:18:28,695][INFO ][o.e.n.Node ] [192.168.142.143] version[7.6.1], pid[83991], build[default/tar/aa751e09be0a5072e8570670309b1f12348f023b/2020-02-29T00:15:25.529771Z], OS[Linux/3.10.0-957.el7.x86_64/amd64], JVM[AdoptOpenJDK/OpenJDK 64-Bit Server VM/13.0.2/13.0.2+8]
[2020-05-06T14:18:28,696][INFO ][o.e.n.Node ] [192.168.142.143] JVM home [/home/geek/tools_my/elasticsearch-7.6.1/jdk]
[2020-05-06T14:18:28,696][INFO ][o.e.n.Node ] [192.168.142.143] JVM arguments [-Des.networkaddress.cache.ttl=60, -Des.networkaddress.cache.negative.ttl=10, -XX:+AlwaysPreTouch, -Xss1m, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djna.nosys=true, -XX:-OmitStackTraceInFastThrow, -Dio.netty.noUnsafe=true, -Dio.netty.noKeySetOptimization=true, -Dio.netty.recycler.maxCapacityPerThread=0, -Dio.netty.allocator.numDirectArenas=0, -Dlog4j.shutdownHookEnabled=false, -Dlog4j2.disable.jmx=true, -Djava.locale.providers=COMPAT, -Xms1g, -Xmx1g, -XX:+UseConcMarkSweepGC, -XX:CMSInitiatingOccupancyFraction=75, -XX:+UseCMSInitiatingOccupancyOnly, -Djava.io.tmpdir=/tmp/elasticsearch-6688252943057610121, -XX:+HeapDumpOnOutOfMemoryError, -XX:HeapDumpPath=data, -XX:ErrorFile=logs/hs_err_pid%p.log, -Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m, -XX:MaxDirectMemorySize=536870912, -Des.path.home=/home/geek/tools_my/elasticsearch-7.6.1, -Des.path.conf=/home/geek/tools_my/elasticsearch-7.6.1/config, -Des.distribution.flavor=default, -Des.distribution.type=tar, -Des.bundled_jdk=true]
[2020-05-06T14:18:34,221][INFO ][o.e.p.PluginsService ] [192.168.142.143] loaded module [aggs-matrix-stats]
[2020-05-06T14:18:34,222][INFO ][o.e.p.PluginsService ] [192.168.142.143] loaded module [analysis-common]
[2020-05-06T14:18:34,222][INFO ][o.e.p.PluginsService ] [192.168.142.143] loaded module [flattened]
[2020-05-06T14:18:34,222][INFO ][o.e.p.PluginsService ] [192.168.142.143] loaded module [frozen-indices]
[2020-05-06T14:18:34,222][INFO ][o.e.p.PluginsService ] [192.168.142.143] loaded module [ingest-common]
[2020-05-06T14:18:34,223][INFO ][o.e.p.PluginsService ] [192.168.142.143] loaded module [ingest-geoip]
[2020-05-06T14:18:34,223][INFO ][o.e.p.PluginsService ] [192.168.142.143] loaded module [ingest-user-agent]
[2020-05-06T14:18:34,223][INFO ][o.e.p.PluginsService ] [192.168.142.143] loaded module [lang-expression]
[2020-05-06T14:18:34,223][INFO ][o.e.p.PluginsService ] [192.168.142.143] loaded module [lang-mustache]
[2020-05-06T14:18:34,223][INFO ][o.e.p.PluginsService ] [192.168.142.143] loaded module [lang-painless]
[2020-05-06T14:18:34,223][INFO ][o.e.p.PluginsService ] [192.168.142.143] loaded module [mapper-extras]
[2020-05-06T14:18:34,224][INFO ][o.e.p.PluginsService ] [192.168.142.143] loaded module [parent-join]
[2020-05-06T14:18:34,224][INFO ][o.e.p.PluginsService ] [192.168.142.143] loaded module [percolator]
[2020-05-06T14:18:34,224][INFO ][o.e.p.PluginsService ] [192.168.142.143] loaded module [rank-eval]
[2020-05-06T14:18:34,224][INFO ][o.e.p.PluginsService ] [192.168.142.143] loaded module [reindex]
[2020-05-06T14:18:34,224][INFO ][o.e.p.PluginsService ] [192.168.142.143] loaded module [repository-url]
[2020-05-06T14:18:34,225][INFO ][o.e.p.PluginsService ] [192.168.142.143] loaded module [search-business-rules]
[2020-05-06T14:18:34,246][INFO ][o.e.p.PluginsService ] [192.168.142.143] loaded module [spatial]
[2020-05-06T14:18:34,247][INFO ][o.e.p.PluginsService ] [192.168.142.143] loaded module [transform]
[2020-05-06T14:18:34,247][INFO ][o.e.p.PluginsService ] [192.168.142.143] loaded module [transport-netty4]
[2020-05-06T14:18:34,247][INFO ][o.e.p.PluginsService ] [192.168.142.143] loaded module [vectors]
[2020-05-06T14:18:34,247][INFO ][o.e.p.PluginsService ] [192.168.142.143] loaded module [x-pack-analytics]
[2020-05-06T14:18:34,248][INFO ][o.e.p.PluginsService ] [192.168.142.143] loaded module [x-pack-ccr]
[2020-05-06T14:18:34,248][INFO ][o.e.p.PluginsService ] [192.168.142.143] loaded module [x-pack-core]
[2020-05-06T14:18:34,248][INFO ][o.e.p.PluginsService ] [192.168.142.143] loaded module [x-pack-deprecation]
[2020-05-06T14:18:34,248][INFO ][o.e.p.PluginsService ] [192.168.142.143] loaded module [x-pack-enrich]
[2020-05-06T14:18:34,248][INFO ][o.e.p.PluginsService ] [192.168.142.143] loaded module [x-pack-graph]
[2020-05-06T14:18:34,249][INFO ][o.e.p.PluginsService ] [192.168.142.143] loaded module [x-pack-ilm]
[2020-05-06T14:18:34,249][INFO ][o.e.p.PluginsService ] [192.168.142.143] loaded module [x-pack-logstash]
[2020-05-06T14:18:34,249][INFO ][o.e.p.PluginsService ] [192.168.142.143] loaded module [x-pack-ml]
[2020-05-06T14:18:34,249][INFO ][o.e.p.PluginsService ] [192.168.142.143] loaded module [x-pack-monitoring]
[2020-05-06T14:18:34,249][INFO ][o.e.p.PluginsService ] [192.168.142.143] loaded module [x-pack-rollup]
[2020-05-06T14:18:34,249][INFO ][o.e.p.PluginsService ] [192.168.142.143] loaded module [x-pack-security]
[2020-05-06T14:18:34,250][INFO ][o.e.p.PluginsService ] [192.168.142.143] loaded module [x-pack-sql]
[2020-05-06T14:18:34,250][INFO ][o.e.p.PluginsService ] [192.168.142.143] loaded module [x-pack-voting-only-node]
[2020-05-06T14:18:34,250][INFO ][o.e.p.PluginsService ] [192.168.142.143] loaded module [x-pack-watcher]
[2020-05-06T14:18:34,267][INFO ][o.e.p.PluginsService ] [192.168.142.143] no plugins loaded
[2020-05-06T14:18:45,044][INFO ][o.e.x.s.a.s.FileRolesStore] [192.168.142.143] parsed [0] roles from file [/home/geek/tools_my/elasticsearch-7.6.1/config/roles.yml]
[2020-05-06T14:18:46,648][INFO ][o.e.x.m.p.l.CppLogMessageHandler] [192.168.142.143] [controller/84069] [Main.cc@110] controller (64 bit): Version 7.6.1 (Build 6eb6e036390036) Copyright (c) 2020 Elasticsearch BV
[2020-05-06T14:18:48,113][DEBUG][o.e.a.ActionModule ] [192.168.142.143] Using REST wrapper from plugin org.elasticsearch.xpack.security.Security
[2020-05-06T14:18:48,498][INFO ][o.e.d.DiscoveryModule ] [192.168.142.143] using discovery type [zen] and seed hosts providers [settings]
[2020-05-06T14:18:50,484][INFO ][o.e.n.Node ] [192.168.142.143] initialized
[2020-05-06T14:18:50,490][INFO ][o.e.n.Node ] [192.168.142.143] starting ...
[2020-05-06T14:18:51,150][INFO ][o.e.t.TransportService ] [192.168.142.143] publish_address {127.0.0.1:9300}, bound_addresses {[::1]:9300}, {127.0.0.1:9300}
[2020-05-06T14:18:51,873][WARN ][o.e.b.BootstrapChecks ] [192.168.142.143] max file descriptors [4096] for elasticsearch process is too low, increase to at least [65535]
[2020-05-06T14:18:51,873][WARN ][o.e.b.BootstrapChecks ] [192.168.142.143] max number of threads [3795] for user [geek] is too low, increase to at least [4096]
[2020-05-06T14:18:51,873][WARN ][o.e.b.BootstrapChecks ] [192.168.142.143] max virtual memory areas vm.max_map_count [65530] is too low, increase to at least [262144]
[2020-05-06T14:18:51,873][WARN ][o.e.b.BootstrapChecks ] [192.168.142.143] the default discovery settings are unsuitable for production use; at least one of [discovery.seed_hosts, discovery.seed_providers, cluster.initial_master_nodes] must be configured
[2020-05-06T14:18:51,898][INFO ][o.e.c.c.ClusterBootstrapService] [192.168.142.143] no discovery configuration found, will perform best-effort cluster bootstrapping after [3s] unless existing master is discovered
[2020-05-06T14:18:54,916][INFO ][o.e.c.c.Coordinator ] [192.168.142.143] setting initial configuration to VotingConfiguration{IVCFW76uRfWTmeK3fUxwgQ}
[2020-05-06T14:18:55,457][INFO ][o.e.c.s.MasterService ] [192.168.142.143] elected-as-master ([1] nodes joined)[{192.168.142.143}{IVCFW76uRfWTmeK3fUxwgQ}{9AyallfnTtqfZhk4bfhR7g}{127.0.0.1}{127.0.0.1:9300}{dilm}{ml.machine_memory=1019797504, xpack.installed=true, ml.max_open_jobs=20} elect leader, _BECOME_MASTER_TASK_, _FINISH_ELECTION_], term: 1, version: 1, delta: master node changed {previous [], current [{192.168.142.143}{IVCFW76uRfWTmeK3fUxwgQ}{9AyallfnTtqfZhk4bfhR7g}{127.0.0.1}{127.0.0.1:9300}{dilm}{ml.machine_memory=1019797504, xpack.installed=true, ml.max_open_jobs=20}]}
[2020-05-06T14:18:55,712][INFO ][o.e.c.c.CoordinationState] [192.168.142.143] cluster UUID set to [B3iN4dRySzSsopg3Sxtvyg]
[2020-05-06T14:18:55,873][INFO ][o.e.c.s.ClusterApplierService] [192.168.142.143] master node changed {previous [], current [{192.168.142.143}{IVCFW76uRfWTmeK3fUxwgQ}{9AyallfnTtqfZhk4bfhR7g}{127.0.0.1}{127.0.0.1:9300}{dilm}{ml.machine_memory=1019797504, xpack.installed=true, ml.max_open_jobs=20}]}, term: 1, version: 1, reason: Publication{term=1, version=1}
[2020-05-06T14:18:56,105][INFO ][o.e.h.AbstractHttpServerTransport] [192.168.142.143] publish_address {127.0.0.1:9200}, bound_addresses {[::1]:9200}, {127.0.0.1:9200}
[2020-05-06T14:18:56,106][INFO ][o.e.n.Node ] [192.168.142.143] started
[2020-05-06T14:18:56,370][INFO ][o.e.g.GatewayService ] [192.168.142.143] recovered [0] indices into cluster_state
[2020-05-06T14:18:57,214][INFO ][o.e.c.m.MetaDataIndexTemplateService] [192.168.142.143] adding template [.watches] for index patterns [.watches*]
[2020-05-06T14:18:57,376][INFO ][o.e.c.m.MetaDataIndexTemplateService] [192.168.142.143] adding template [.triggered_watches] for index patterns [.triggered_watches*]
[2020-05-06T14:18:57,574][INFO ][o.e.c.m.MetaDataIndexTemplateService] [192.168.142.143] adding template [.watch-history-10] for index patterns [.watcher-history-10*]
[2020-05-06T14:18:57,679][INFO ][o.e.c.m.MetaDataIndexTemplateService] [192.168.142.143] adding template [ilm-history] for index patterns [ilm-history-1*]
[2020-05-06T14:18:57,850][INFO ][o.e.c.m.MetaDataIndexTemplateService] [192.168.142.143] adding template [.slm-history] for index patterns [.slm-history-1*]
[2020-05-06T14:18:57,967][INFO ][o.e.c.m.MetaDataIndexTemplateService] [192.168.142.143] adding template [.monitoring-logstash] for index patterns [.monitoring-logstash-7-*]
[2020-05-06T14:18:58,170][INFO ][o.e.c.m.MetaDataIndexTemplateService] [192.168.142.143] adding template [.monitoring-es] for index patterns [.monitoring-es-7-*]
[2020-05-06T14:18:58,336][INFO ][o.e.c.m.MetaDataIndexTemplateService] [192.168.142.143] adding template [.monitoring-beats] for index patterns [.monitoring-beats-7-*]
[2020-05-06T14:18:58,514][INFO ][o.e.c.m.MetaDataIndexTemplateService] [192.168.142.143] adding template [.monitoring-alerts-7] for index patterns [.monitoring-alerts-7]
[2020-05-06T14:18:58,614][INFO ][o.e.c.m.MetaDataIndexTemplateService] [192.168.142.143] adding template [.monitoring-kibana] for index patterns [.monitoring-kibana-7-*]
[2020-05-06T14:18:58,729][INFO ][o.e.x.i.a.TransportPutLifecycleAction] [192.168.142.143] adding index lifecycle policy [watch-history-ilm-policy]
[2020-05-06T14:18:58,849][INFO ][o.e.x.i.a.TransportPutLifecycleAction] [192.168.142.143] adding index lifecycle policy [ilm-history-ilm-policy]
[2020-05-06T14:18:58,950][INFO ][o.e.x.i.a.TransportPutLifecycleAction] [192.168.142.143] adding index lifecycle policy [slm-history-ilm-policy]
[2020-05-06T14:18:59,144][INFO ][o.e.l.LicenseService ] [192.168.142.143] license [9e94ce7a-4ea2-4dce-9c81-4a80aab83d8d] mode [basic] + valid
[2020-05-06T14:18:59,145][INFO ][o.e.x.s.s.SecurityStatusChangeListener] [192.168.142.143] Active license is now [BASIC]; Security is disabled
192.168.142.143:9200。
Elasticsearch 可视化界面。
http://mobz.github.io/elasticsearch-head/
需要 Node.js。
https://github.com/nodesource/distributions/blob/master/README.md
git clone git://github.com/mobz/elasticsearch-head.git
cd elasticsearch-head
npm install
npm run start
open http://localhost:9100/
你以为就能连上了?天真。
跨域问题。
Elasticsearch Head 端口是 9100,访问 Elasticsearch 9200 端口存在跨域问题。
Request URL: http://localhost:9200/
Referrer Policy: no-referrer-when-downgrade
[geek@192 config]$ sudo vim elasticsearch.yml
加入配置解决跨域问题。
http.cors.enabled: true
http.cors.allow-origin: "*"
重启。
[geek@192 elasticsearch-head]$ npm run start
> elasticsearch-head@0.0.0 start /home/geek/tools_my/elasticsearch-head
> grunt server
Running "connect:server" (connect) task
Waiting forever...
Started connect web server on http://localhost:9100
ok。
ELK ~ Kibana。
ELK 是 Elasticsearch、Logstash、Kibana 三大开源框架首字母大写简称。市面上也被称为 Elastic Stack。其中 Elasticsearch 是一个基于 Lucene 分布式、通过 Restful 方式进行交的近实时搜索平台框架。像类似百度、谷歌这种大数据全文搜索引擎的场景都可以使用Elasticsearch 作为底层支持框架,可见 Elasticsearch 提供的搜索能力确实强大,市面上很多时候我们简称 Elasticsearch 为 es 。
Logstash 是 ELK 的中央数据流引擎,用于从不同目标(文件 / 数据存储 / MQ)收集的不同格式数据,经过过滤后支持输出到不同目的地(文件 / MQ / redis / elasticsearch / kafka 等)。Kibana 可以将 elasticsearch 的数据通过友好的页面展示出来,提供实时分析的功能。
市面上很多开发发只要提到 ELK 能够一致说出它是一个日志分析架构技术栈总称,但实际上 ELK 不仅仅适用于日志分析,它还可以支持其他任何数据分析和收集的场景,日志分析和收集只是更具有代表性。并非唯一性。
[geek@192 kibana-7.6.1-linux-x86_64]$ tar -zxvf kibana-7.6.1-linux-x86_64.tar.gz
[geek@192 kibana-7.6.1-linux-x86_64]$ cd bin/
[geek@192 bin]$ ls
kibana kibana-keystore kibana-plugin
[geek@192 bin]$ ./kibana
firewall-cmd --permanent --add-port=5601/tcp
firewall-cmd --reload
汉化。
[geek@192 ~]$ vim tools_my/kibana-7.6.1-linux-x86_64/config/kibana.yml
# Specifies locale to be used for all localizable strings, dates and number formats.
# Supported languages are the following: English + en , by default , Chinese + zh-CN .
#i18n.locale: "en"
i18n.locale: "zh-CN"
ES 核心概念。
Elasticsearch 是面向文档的。一切都是 Json。
Rational DB | Elasticsearch |
---|---|
数据库(database) | 索引(index) |
表(table) | types |
行(row) | document |
字段(column) | field |
elastocsearch 集群中可以包含多个索引数据库,每个索引中可以包含多个类型(表),每个类型下又包含多个文档(行),每个文档中又包含多个字段(列)。
物理设计。
elasticsearch 在后台把每个索引划分成多个分片,每个分片可以在集群中的不同服务器间迁移。
一个人就是一个集群! 默认的集群名称就是 elasticsearch。
逻辑设计。
一个索引类型中,包含多个文档。比如说文档1,文档2。当我们索引一篇文档时,可以过这样的一个顺序找到它:索引 > 类型 > 文档 ID,通过这个组台我们就能索引到某个具体的文档。注意:ID 不必是整数,实际上它是个字符串。
文档。
之前说 Elasticsearch 是面向文档的,那么就意味着索引和搜索数据的最小单位是文档。Elasticsearch 中,文档有几个重要属性。
- 自我包含。一篇文档同时包含字段和对应的值,也就是同时包含key: value。
- 可以是层次型的。一个文档中包含自文档,复杂的逻辑实体就是这么来的!就是一个 json 对象。
- 灵活的结构,文档不依赖预先定义的模式,我们知道关系型数据库中,要提前定义字段才能使用,在 Elasticsearch 中,对于字段是非灵活的。有时候,我们可以忽路该字段,或者动态的添加一个新的字段。
尽管我们可以随意的新增或者忽略某个字段,但是,每个字段的类型非常重要,比如一个年龄字段类型的可以是字符串也可以是整形。因为 Elasticsearch 会保存字段和类型之间的映射及其他的设置。这种映射具体到每个映射的每种类型,这也是为什么在 Elasticsearch 中,类型有时候也称为映射类型。
类型。
类型是文档的逻辑容器。就像关系型数据库一样,表格是行的容器。类型中对于字段的定义称为映射,比如 name 映射为字符串类型。我们说文是无模式的,它们不需要拥有映射中所定义的所有字段,比如新增一个字段,那么 Elasticsearch 是怎么做的呢?
Elasticsearch 会自动的将新字段加入映射,但是这个字段不确定 ta 是什么类型,Elasticsearch 就开始猜,如果这个值是 18,那么 Elasticsearch 会认为它是整形。但是 Elasticsearch 也可能猜不对。所以最安全的方式就是提前定义好所需要的映射。这点跟关系型数据库殊途同归了。先定义好字段,然后再使用,别整什么幺蛾子。
索引。
索引是映射类型的容器。Elasticsearch 中的索引是一个非常大的文档集合。索引存储了映射类型的字段和其他设置。
分片。
一个集群至少有一个节点,而一个节点就是一个 Elasticsearch 进程,节点可以有多个索引。默认的,如果你创建索引,那么索引将会有个 5 个分片(primary shard,又称主分片)构成的,每一个主分片会有一个副本(replica shard,又称复制分片)。
上图是一个有 3 个节点的集群,可以看到主分片和对应的复制分片都不会在同一个节点内,这样有利于某个节点挂掉了,数据也不至于丢失。实际上,一个分片是一个 Lucene 索引,一个包含倒排索引的文件目录,倒排系引的结构使得 Elasticsearch 在不扫描全部文档的情况下,就能告诉你哪些文档包含定的关键字。不过,等等,倒排索引是什么鬼?
倒排索引。
Elasticsearch 使用的是一种称为倒排索引的结构,采用 Lucene 倒排索作为底层。这种结构适用于快速的全文搜索,一个索引由文档中所有不重复的列表构成,对于每一个词,都有一个包含它的文档列表。eg. 现在有两个文档,每个文档包含如下内容:
study every day, good good up to forever # 文档 1 包含的内容。
To forever, study every day, good good up # 文档 2 包含的内容。
为了创建倒排索引,我们首先要将每个文档拆分成独立的词(或称为词条或者 tokens),然后创建一个包含所有不重复的词条的排序列表,然后列出每个词条出现在哪个文档:
term | doc_1 | doc_2 |
---|---|---|
study | √ | × |
To | × | √ |
every | √ | √ |
forever | √ | √ |
day | √ | √ |
study | × | √ |
good | √ | × |
every | √ | √ |
to | √ | × |
up | √ | √ |
现在我们试图搜索 to forever,只需要查看包含每个词条的文档。
term | doc_1 | doc_2 |
---|---|---|
to | √ | × |
forever | √ | √ |
total | 2 | 1 |
权重(score)。
两个文档都匹配,但是第一个文档比第二个匹配程度更高。如果没有别的条件,现在,这两个包含关键字的文档都将返回。
再来看一个示例。
比如我们通过博客标签来搜索博客文章。那么倒排索引列表就是这样的一个结构。
博客文章(原始数据) | 索引列表(倒排索引) | ||
---|---|---|---|
博客文章 ID | 标签 | 标签 | 博客文章 ID |
1 | Python | Python | 1, 2, 3 |
2 | Python | Linux | 3, 4 |
3 | Linux, Python | ||
4 | Linux |
如果要搜索含有 Python 标签的文章,那相对于查找所有原始数据而言,查找倒排索引后的数据将会快的多,只需要查看标签这一栏,然后获取相关的文章 ID 即可。
- Elasticsearch 的索引和 Lucene 的索引对比。
在 Elasticsearch 中索引这个词被频繁用,这就是术语的使用。在 Elasticsearch 中,索引被分为多个分片,每份分片是一个 Lucene 的索引。所以一个 Elasticsearch 索引是由多个 Lucene 索引组成的 。别问为什么, 谁让 Elasticsearch 使用 Lucene 作为底层呢!
如无特指,说起索引都是 Elasticsearch 的索引。
接下来的一切作都在 kibana 中 Dev Tools 下的 consoles 里完成。基础操作!
IK 分词器。
https://github.com/medcl/elasticsearch-analysis-ik
https://github.com/medcl/elasticsearch-analysis-ik/releases
scp geek@192.168.142.161:/home/geek/tools_my/elasticsearch-7.6.1/plugins
[geek@192 plugins]$ sudo yum install unzip
[geek@192 plugins]$ mkdir ik
[geek@192 plugins]$ mv elasticsearch-analysis-ik-7.6.1.zip ik/
[geek@192 plugins]$ cd ik/
[geek@192 ik]$ unzip elasticsearch-analysis-ik-7.6.1.zip
# 重启 Elasticsearch。
[geek@192 bin]$ ./elasticsearch
[geek@192 bin]$ ./elasticsearch-plugin list
future versions of Elasticsearch will require Java 11; your Java version from [/home/geek/tools_my/jdk1.8.0_241/jre] does not meet this requirement
ik
使用 Kibana 测试。
GET _search
{
"query": {
"match_all": {}
}
}
{
"took" : 16,
"timed_out" : false,
"_shards" : {
"total" : 3,
"successful" : 3,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : {
"value" : 13,
"relation" : "eq"
},
"max_score" : 1.0,
"hits" : [
{
"_index" : ".kibana_1",
"_type" : "_doc",
"_id" : "space:default",
"_score" : 1.0,
"_source" : {
"space" : {
"name" : "Default",
"description" : "This is your default space!",
"color" : "#00bfb3",
"disabledFeatures" : [ ],
"_reserved" : true
},
"type" : "space",
"references" : [ ],
"migrationVersion" : {
"space" : "6.6.0"
},
"updated_at" : "2020-06-27T16:57:32.566Z"
}
},
{
"_index" : ".kibana_1",
"_type" : "_doc",
"_id" : "ui-metric:Kibana_home:sampleDataConfirm",
"_score" : 1.0,
"_source" : {
"ui-metric" : {
"count" : 1
},
"type" : "ui-metric",
"updated_at" : "2020-06-27T18:54:32.639Z"
}
},
{
"_index" : ".kibana_1",
"_type" : "_doc",
"_id" : "ui-metric:Kibana_home:welcomeScreenMount",
"_score" : 1.0,
"_source" : {
"ui-metric" : {
"count" : 3
},
"type" : "ui-metric",
"updated_at" : "2020-06-27T18:59:07.983Z"
}
},
{
"_index" : ".kibana_1",
"_type" : "_doc",
"_id" : "ui-metric:Kibana_home:sampleDataDecline",
"_score" : 1.0,
"_source" : {
"ui-metric" : {
"count" : 2
},
"type" : "ui-metric",
"updated_at" : "2020-06-27T18:59:07.983Z"
}
},
{
"_index" : ".kibana_1",
"_type" : "_doc",
"_id" : "config:7.6.1",
"_score" : 1.0,
"_source" : {
"config" : {
"buildNum" : 29118
},
"type" : "config",
"references" : [ ],
"updated_at" : "2020-06-27T16:57:52.163Z"
}
},
{
"_index" : ".kibana_1",
"_type" : "_doc",
"_id" : "ui-metric:index_management:app_load",
"_score" : 1.0,
"_source" : {
"ui-metric" : {
"count" : 1
},
"type" : "ui-metric",
"updated_at" : "2020-06-27T19:02:23.730Z"
}
},
{
"_index" : ".kibana_1",
"_type" : "_doc",
"_id" : "telemetry:telemetry",
"_score" : 1.0,
"_source" : {
"telemetry" : {
"userHasSeenNotice" : true
},
"type" : "telemetry",
"references" : [ ],
"updated_at" : "2020-06-27T18:51:21.445Z"
}
},
{
"_index" : ".kibana_1",
"_type" : "_doc",
"_id" : "maps-telemetry:maps-telemetry",
"_score" : 1.0,
"_source" : {
"maps-telemetry" : {
"settings" : {
"showMapVisualizationTypes" : false
},
"indexPatternsWithGeoFieldCount" : 0,
"mapsTotalCount" : 0,
"timeCaptured" : "2020-06-27T22:02:39.850Z",
"attributesPerMap" : {
"dataSourcesCount" : {
"min" : 0,
"max" : 0,
"avg" : 0
},
"layersCount" : {
"min" : 0,
"max" : 0,
"avg" : 0
},
"layerTypesCount" : { },
"emsVectorLayersCount" : { }
}
},
"type" : "maps-telemetry",
"references" : [ ],
"updated_at" : "2020-06-27T22:02:39.851Z"
}
},
{
"_index" : ".kibana_1",
"_type" : "_doc",
"_id" : "ui-metric:kibana-user_agent:Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:78.0) Gecko/20100101 Firefox/78.0",
"_score" : 1.0,
"_source" : {
"ui-metric" : {
"count" : 1
},
"type" : "ui-metric",
"references" : [ ],
"updated_at" : "2020-06-27T19:08:29.631Z"
}
},
{
"_index" : ".kibana_1",
"_type" : "_doc",
"_id" : "ui-metric:console:opened_app",
"_score" : 1.0,
"_source" : {
"ui-metric" : {
"count" : 3
},
"type" : "ui-metric",
"updated_at" : "2020-06-27T19:08:29.631Z"
}
}
]
}
}
ik。
- ik_smart。最少切分。
GET _analyze
{
"analyzer": "ik_smart",
"text": "中国共产党"
}
{
"tokens" : [
{
"token" : "中国共产党",
"start_offset" : 0,
"end_offset" : 5,
"type" : "CN_WORD",
"position" : 0
}
]
}
- ik_max_word。最细粒度。
GET _analyze
{
"analyzer": "ik_max_word",
"text": "中国共产党"
}
{
"tokens" : [
{
"token" : "中国共产党",
"start_offset" : 0,
"end_offset" : 5,
"type" : "CN_WORD",
"position" : 0
},
{
"token" : "中国",
"start_offset" : 0,
"end_offset" : 2,
"type" : "CN_WORD",
"position" : 1
},
{
"token" : "国共",
"start_offset" : 1,
"end_offset" : 3,
"type" : "CN_WORD",
"position" : 2
},
{
"token" : "共产党",
"start_offset" : 2,
"end_offset" : 5,
"type" : "CN_WORD",
"position" : 3
},
{
"token" : "共产",
"start_offset" : 2,
"end_offset" : 4,
"type" : "CN_WORD",
"position" : 4
},
{
"token" : "党",
"start_offset" : 4,
"end_offset" : 5,
"type" : "CN_CHAR",
"position" : 5
}
]
}
自定义词库。
IKAnalyzer.cfg.xml。
<properties>
<comment>IK Analyzer 扩展配置</comment>
<!-- 用户可以在这里配置自己的扩展字典 -->
<entry key="ext_dict">
my.dic
</entry>
<!-- 用户可以在这里配置自己的扩展停止词字典 -->
<entry key="ext_stopwords">
</entry>
<!-- 用户可以在这里配置远程扩展字典 -->
<!-- <entry key="remote_ext_dict">words_location</entry> -->
<!-- 用户可以在这里配置远程扩展停止词字典 -->
<!-- <entry key="remote_ext_stopwords">words_location</entry> -->
</properties>
Restful。
method | url | 描述 |
---|---|---|
PUT | localhost:9200/索引名称/类型名称/文档 id | 创建文档(指定文档 id) |
POST | localhost:9200/索引名称/类型名称 | 创建文档(随机文档 id) |
POST | localhost:9200/索引名称/类型名称/文档 id/update | 修改文档 |
DELETE | localhost:9200/索引名称/类型名称/文档 id | 删除文档 |
GET | localhost:9200/索引名称/类型名称/文档 id | 查询文档通过文档 id |
POST | localhost:9200/索引名称/类型名称/_search | 查询所有数据 |
字段数据类型。
https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-types.html
索引操作。
创建索引。
PUT /索引名/类型名/文档 id {
请求体
}
PUT /test1/type1/1
{
"name": "geek 李",
"age": 3
}
#! Deprecation: [types removal] Specifying types in document index requests is deprecated, use the typeless endpoints instead (/{index}/_doc/{id}, /{index}/_doc, or /{index}/_create/{id}).
{
"_index" : "test1",
"_type" : "type1",
"_id" : "1",
"_version" : 1,
"result" : "created",
"_shards" : {
"total" : 2,
"successful" : 1,
"failed" : 0
},
"_seq_no" : 0,
"_primary_term" : 1
}
- 创建规则,指定字段的类型。
PUT /test2
{
"mappings": {
"properties": {
"name": {
"type": "text"
},
"age": {
"type": "long"
},
"birthday": {
"type": "date"
}
}
}
}
{
"acknowledged" : true,
"shards_acknowledged" : true,
"index" : "test2"
}
- GET test2。
{
"test2" : {
"aliases" : { },
"mappings" : {
"properties" : {
"age" : {
"type" : "long"
},
"birthday" : {
"type" : "date"
},
"name" : {
"type" : "text"
}
}
},
"settings" : {
"index" : {
"creation_date" : "1593299392799",
"number_of_shards" : "1",
"number_of_replicas" : "1",
"uuid" : "knOsCiPzSamYDm3FYJPYHw",
"version" : {
"created" : "7060199"
},
"provided_name" : "test2"
}
}
}
}
- 查看默认信息。
PUT /test3/_doc/1
{
"name": "Geek 李",
"age": 13,
"birth": "1995-05-06"
}
{
"_index" : "test3",
"_type" : "_doc",
"_id" : "1",
"_version" : 1,
"result" : "created",
"_shards" : {
"total" : 2,
"successful" : 1,
"failed" : 0
},
"_seq_no" : 0,
"_primary_term" : 1
}
- GET test3。
{
"test3" : {
"aliases" : { },
"mappings" : {
"properties" : {
"age" : {
"type" : "long"
},
"birth" : {
"type" : "date"
},
"name" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"settings" : {
"index" : {
"creation_date" : "1593299645889",
"number_of_shards" : "1",
"number_of_replicas" : "1",
"uuid" : "rPcOsdbhSjCtRwWcvdxGTg",
"version" : {
"created" : "7060199"
},
"provided_name" : "test3"
}
}
}
}
修改索引。
PUT,覆盖。
PUT /test3/_doc/1
{
"name": "Geek 李123",
"age": 13,
"birth": "1995-05-06"
}
版本号增加了。
{
"_index" : "test3",
"_type" : "_doc",
"_id" : "1",
"_version" : 2,
"result" : "updated",
"_shards" : {
"total" : 2,
"successful" : 1,
"failed" : 0
},
"_seq_no" : 1,
"_primary_term" : 1
}
- 新办法。
POST /test3/_doc/1/_update
{
"doc": {
"name": "法外狂徒张三"
}
}
#! Deprecation: [types removal] Specifying types in document update requests is deprecated, use the endpoint /{index}/_update/{id} instead.
{
"_index" : "test3",
"_type" : "_doc",
"_id" : "1",
"_version" : 3,
"result" : "updated",
"_shards" : {
"total" : 2,
"successful" : 1,
"failed" : 0
},
"_seq_no" : 2,
"_primary_term" : 1
}
修改索引。
DELETE test1
{
"acknowledged" : true
}
文档操作。
添加数据。
PUT /geek/user/1
{
"name": "Geek 李",
"age": 25,
"desc": "一顿操作猛如虎,一看工资 2500",
"tags": ["技术宅", "温暖", "直男"]
}
#! Deprecation: [types removal] Specifying types in document index requests is deprecated, use the typeless endpoints instead (/{index}/_doc/{id}, /{index}/_doc, or /{index}/_create/{id}).
{
"_index" : "geek",
"_type" : "user",
"_id" : "1",
"_version" : 1,
"result" : "created",
"_shards" : {
"total" : 2,
"successful" : 1,
"failed" : 0
},
"_seq_no" : 0,
"_primary_term" : 1
}
PUT /geek/user/2
{
"name": "张三",
"age": 3,
"desc": "法外狂徒",
"tags": ["交友", "旅游", "渣男"]
}
PUT /geek/user/3
{
"name": "李四",
"age": 30,
"desc": "不知道",
"tags": ["靓女", "跳舞", "唱歌"]
}
查询数据。
GET geek/user/1
#! Deprecation: [types removal] Specifying types in document get requests is deprecated, use the /{index}/_doc/{id} endpoint instead.
{
"_index" : "geek",
"_type" : "user",
"_id" : "1",
"_version" : 1,
"_seq_no" : 0,
"_primary_term" : 1,
"found" : true,
"_source" : {
"name" : "Geek 李",
"age" : 25,
"desc" : "一顿操作猛如虎,一看工资 2500",
"tags" : [
"技术宅",
"温暖",
"直男"
]
}
}
更新数据 ~ PUT。
如果字段不写全,为改为空。
post。_update。(局部修改)。
POST geek/user/1/_update
{
"doc": {
"name": "狂神说 Java"
}
}
#! Deprecation: [types removal] Specifying types in document update requests is deprecated, use the endpoint /{index}/_update/{id} instead.
{
"_index" : "geek",
"_type" : "user",
"_id" : "1",
"_version" : 2,
"result" : "updated",
"_shards" : {
"total" : 2,
"successful" : 1,
"failed" : 0
},
"_seq_no" : 4,
"_primary_term" : 1
}
搜索。
GET geek/user/1
简单条件查询。
GET geek/user/_search?q=name:李
#! Deprecation: [types removal] Specifying types in search requests is deprecated.
{
"took" : 1,
"timed_out" : false,
"_shards" : {
"total" : 1,
"successful" : 1,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : {
"value" : 1,
"relation" : "eq"
},
"max_score" : 0.5070823,
"hits" : [
{
"_index" : "geek",
"_type" : "user",
"_id" : "3",
"_score" : 0.5070823,
"_source" : {
"name" : "李四222",
"age" : 30,
"desc" : "不知道",
"tags" : [
"靓女",
"跳舞",
"唱歌"
]
}
}
]
}
}
复杂搜索。
_score ~ 匹配度。
GET geek/user/_search
{
"query": {
"match": {
"name": "geek"
}
}
}
#! Deprecation: [types removal] Specifying types in search requests is deprecated.
{
"took" : 2,
"timed_out" : false,
"_shards" : {
"total" : 1,
"successful" : 1,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : {
"value" : 1,
"relation" : "eq"
},
"max_score" : 0.72615415,
"hits" : [
{
"_index" : "geek",
"_type" : "user",
"_id" : "1",
"_score" : 0.72615415,
"_source" : {
"name" : "Geek 李",
"age" : 25,
"desc" : "一顿操作猛如虎,一看工资 2500",
"tags" : [
"技术宅",
"温暖",
"直男"
]
}
}
]
}
}
GET geek/user/_search
{
"query": {
"match": {
"name": "狂神"
}
}
}
结果过滤。
GET geek/user/_search
{
"query": {
"match": {
"name": "狂神"
}
},
"_source": ["name", "desc"]
}
#! Deprecation: [types removal] Specifying types in search requests is deprecated.
{
"took" : 2,
"timed_out" : false,
"_shards" : {
"total" : 1,
"successful" : 1,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : {
"value" : 2,
"relation" : "eq"
},
"max_score" : 1.487407,
"hits" : [
{
"_index" : "geek",
"_type" : "user",
"_id" : "4",
"_score" : 1.487407,
"_source" : {
"name" : "狂神说 Java",
"desc" : "一顿操作猛如虎,一看工资 2500"
}
},
{
"_index" : "geek",
"_type" : "user",
"_id" : "5",
"_score" : 1.3314891,
"_source" : {
"name" : "狂神说前端",
"desc" : "一顿操作猛如虎,一看工资 2500"
}
}
]
}
}
排序。
GET geek/user/_search
{
"query": {
"match": {
"name": "狂神"
}
},
"sort": [
{
"age": {
"order": "desc"
}
}
]
}
分页。
GET geek/user/_search
{
"query": {
"match": {
"name": "狂神"
}
},
"sort": [
{
"age": {
"order": "desc"
}
}
],
"from": 0,
"size": 2
}
bool ~ 多条件查询。
- must(and) ~ 所有条件都要符合。
GET geek/user/_search
{
"query": {
"bool": {
"must": [
{
"match": {
"name": "狂神说"
}
},
{
"match": {
"age": 3
}
}
]
}
}
}
- should(or)。
GET geek/user/_search
{
"query": {
"bool": {
"should": [
{
"match": {
"name": "狂神说"
}
},
{
"match": {
"age": 3
}
}
]
}
}
}
- not。
GET geek/user/_search
{
"query": {
"bool": {
"must_not": [
{
"match": {
"age": 3
}
}
]
}
}
}
- 过滤条件。
GET geek/user/_search
{
"query": {
"bool": {
"must": [
{
"match": {
"name": "李"
}
}
],
"filter": {
"range": {
"age": {
"gte": 10,
"lte": 30
}
}
}
}
}
}
- 匹配多个条件。
空格隔开。
GET geek/user/_search
{
"query": {
"match": {
"tags": "男 技术"
}
}
}
精确查询。
term 查询是直接通过倒排索引指定的词条进行精确查找的。
关于分词。
- term。
直接精确查询。- match。
会使用分词器解析。(先分析文档,然后再通过分析的文档进行 查询)。
PUT testdb
{
"mappings": {
"properties": {
"name": {
"type": "text"
},
"desc": {
"type": "keyword"
}
}
}
}
PUT testdb/_doc/1
{
"name": "狂神说 Java name",
"desc": "狂神说 Java desc"
}
PUT testdb/_doc/2
{
"name": "狂神说 Java name",
"desc": "狂神说 Java desc2"
}
- 两种类型分词。
GET _analyze
{
"analyzer": "keyword",
"text": ["狂神说 Java name"]
}
{
"tokens" : [
{
"token" : "狂神说 Java name",
"start_offset" : 0,
"end_offset" : 13,
"type" : "word",
"position" : 0
}
]
}
GET _analyze
{
"analyzer": "standard",
"text": ["狂神说 Java name"]
}
{
"tokens" : [
{
"token" : "狂",
"start_offset" : 0,
"end_offset" : 1,
"type" : "<IDEOGRAPHIC>",
"position" : 0
},
{
"token" : "神",
"start_offset" : 1,
"end_offset" : 2,
"type" : "<IDEOGRAPHIC>",
"position" : 1
},
{
"token" : "说",
"start_offset" : 2,
"end_offset" : 3,
"type" : "<IDEOGRAPHIC>",
"position" : 2
},
{
"token" : "java",
"start_offset" : 4,
"end_offset" : 8,
"type" : "<ALPHANUM>",
"position" : 3
},
{
"token" : "name",
"start_offset" : 9,
"end_offset" : 13,
"type" : "<ALPHANUM>",
"position" : 4
}
]
}
GET testdb/_search
{
"query": {
"term": {
"name": "狂" // 全部。
}
}
}
GET testdb/_search
{
"query": {
"term": {
"desc": "狂神说 Java desc2" // 只有一个。(keyword 字段类型不会被分词器解析)。
}
}
}
- 精确查询多个值。
PUT testdb/_doc/3
{
"t1": "22",
"t2": "2020-2-2"
}
PUT testdb/_doc/4
{
"t1": "33",
"t2": "2020-3-3"
}
GET testdb/_search
{
"query": {
"bool": {
"should": [
{
"term": {
"t1": "22"
}
},
{
"term": {
"t1": "33"
}
}
]
}
}
}
高亮。
GET geek/user/_search
{
"query": {
"match": {
"name": "狂神"
}
},
"highlight": {
"fields": {
"name": {}
}
}
}
会在查询结果高亮的字加 <em></em>
。
#! Deprecation: [types removal] Specifying types in search requests is deprecated.
{
"took" : 199,
"timed_out" : false,
"_shards" : {
"total" : 1,
"successful" : 1,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : {
"value" : 2,
"relation" : "eq"
},
"max_score" : 1.6282711,
"hits" : [
{
"_index" : "geek",
"_type" : "user",
"_id" : "4",
"_score" : 1.6282711,
"_source" : {
"name" : "狂神说 Java",
"age" : 3,
"desc" : "一顿操作猛如虎,一看工资 2500",
"tags" : [
"技术宅",
"温暖",
"直男"
]
},
"highlight" : {
"name" : [
"<em>狂</em><em>神</em>说 Java"
]
}
},
{
"_index" : "geek",
"_type" : "user",
"_id" : "5",
"_score" : 1.4503717,
"_source" : {
"name" : "狂神说前端",
"age" : 3,
"desc" : "一顿操作猛如虎,一看工资 2500",
"tags" : [
"技术宅",
"温暖",
"直男"
]
},
"highlight" : {
"name" : [
"<em>狂</em><em>神</em>说前端"
]
}
}
]
}
}
- 自定义高亮条件。
GET geek/user/_search
{
"query": {
"match": {
"name": "狂神"
}
},
"highlight": {
"pre_tags": "<p class='key' style='color: red'>",
"post_tags": "</p>",
"fields": {
"name": {}
}
}
}
Spring Boot 集成 Elasticsearch。
https://www.elastic.co/guide/en/elasticsearch/client/java-rest/7.6/java-rest-high.html
官方文档是大爷。
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
<version>7.6.2</version>
</dependency>
- 确保 Spring Boot 的 Elasticsearch 版本。
<elasticsearch.version>7.6.2</elasticsearch.version>
改为 7.6.1。
<properties>
<java.version>1.8</java.version>
<elasticsearch.version>7.6.1</elasticsearch.version>
</properties>
- config 类。
RestClientBuilder。
RestHighLevelCient。
RestClient。
要用到。
package com.geek.esjdgeek.config;
import org.apache.http.HttpHost;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
/**
* @author geek
*/
@Configuration
public class ElasticsearchClientConfig {
@Bean
public RestHighLevelClient restHighLevelClient() {
RestHighLevelClient client = new RestHighLevelClient(
RestClient.builder(
new HttpHost("192.168.142.161", 9200, "http")));
return client;
}
// 集群。
// @Bean
// public RestHighLevelClient restHighLevelClient() {
// RestHighLevelClient client = new RestHighLevelClient(
// RestClient.builder(
// new HttpHost("localhost", 9200, "http"),
// new HttpHost("localhost", 9201, "http")));
// return client;
// }
}
- 测试。
package com.geek.esapigeek;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.indices.CreateIndexRequest;
import org.elasticsearch.client.indices.CreateIndexResponse;
import org.elasticsearch.client.indices.GetIndexRequest;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import java.io.IOException;
@SpringBootTest
class EsApiGeekApplicationTests {
// @Autowired
// @Qualifier("restHighLevelClient")
// private RestHighLevelClient client;
// or
@Autowired
private RestHighLevelClient restHighLevelClient;
// 测试索引的创建。
@Test
void testCreateIndex() throws IOException {
// 创建索引。
CreateIndexRequest request = new CreateIndexRequest("geek_index");
// 执行创建请求 IndicesClient。请求后获得相应。
CreateIndexResponse response = restHighLevelClient.indices().create(request, RequestOptions.DEFAULT);
System.out.println(response);// org.elasticsearch.client.indices.CreateIndexResponse@88c8e176
}
// 测试获取索引。判断是否存在。
@Test
void testExistIndex() throws IOException {
GetIndexRequest request = new GetIndexRequest("geek_index");
boolean exists = restHighLevelClient.indices().exists(request, RequestOptions.DEFAULT);
System.out.println("exists = " + exists);// exists = true
}
// 测试删除索引。
@Test
void testDeleteIndex() throws IOException {
DeleteIndexRequest request = new DeleteIndexRequest("geek_index");
AcknowledgedResponse delete = restHighLevelClient.indices().delete(request, RequestOptions.DEFAULT);
System.out.println(delete);// org.elasticsearch.action.support.master.AcknowledgedResponse@4ee
System.out.println(delete.isAcknowledged());// true。
}
@Test
void contextLoads() {
}
}
文档测试。
package com.geek.esapigeek.pojo;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.springframework.stereotype.Component;
/**
* @author geek
*/
@Component
@AllArgsConstructor
@NoArgsConstructor
@Data
public class User {
private String name;
private int age;
}
package com.geek.esapigeek;
import com.alibaba.fastjson.JSON;
import com.geek.esapigeek.pojo.User;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.delete.DeleteResponse;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.action.update.UpdateResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.indices.CreateIndexRequest;
import org.elasticsearch.client.indices.CreateIndexResponse;
import org.elasticsearch.client.indices.GetIndexRequest;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.TermQueryBuilder;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
@SpringBootTest
class EsApiGeekApplicationTests {
// @Autowired
// @Qualifier("restHighLevelClient")
// private RestHighLevelClient client;
// or
@Autowired
private RestHighLevelClient restHighLevelClient;
// ~ ~ ~ 文档。
/**
* 测试增加文档。
*
* @throws IOException
*/
@Test
void testAddDocument() throws IOException {
// 创建对象。
User user = new User("geek", 23);
// 创建请求。
IndexRequest request = new IndexRequest("geek_index");
// 规则。PUT /geek_index/_doc/1
request.id("3");
// request.timeout(TimeValue.timeValueSeconds(1));
request.timeout("1s");
// 将数据放入请求。
request.source(JSON.toJSONString(user), XContentType.JSON);
// 客户端发送请求。
IndexResponse response = restHighLevelClient.index(request, RequestOptions.DEFAULT);
System.out.println(response);
// IndexResponse[index=geek_index,type=_doc,id=1,version=1,result=created,seqNo=0,primaryTerm=1,shards={"total":2,"successful":1,"failed":0}]
System.out.println(response.status());
// CREATED
}
@Test
void contextLoads() {
}
}
package com.geek.esapigeek;
import com.alibaba.fastjson.JSON;
import com.geek.esapigeek.pojo.User;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.delete.DeleteResponse;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.action.update.UpdateResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.indices.CreateIndexRequest;
import org.elasticsearch.client.indices.CreateIndexResponse;
import org.elasticsearch.client.indices.GetIndexRequest;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.TermQueryBuilder;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
@SpringBootTest
class EsApiGeekApplicationTests {
// @Autowired
// @Qualifier("restHighLevelClient")
// private RestHighLevelClient client;
// or
@Autowired
private RestHighLevelClient restHighLevelClient;
/**
* 测试索引的创建。
*
* @throws IOException
*/
@Test
void testCreateIndex() throws IOException {
// 创建索引。
CreateIndexRequest request = new CreateIndexRequest("geek_index");
// 执行创建请求 IndicesClient。请求后获得相应。
CreateIndexResponse response = restHighLevelClient.indices().create(request, RequestOptions.DEFAULT);
System.out.println(response);// org.elasticsearch.client.indices.CreateIndexResponse@88c8e176
}
/**
* 测试获取索引。判断是否存在。
*
* @throws IOException
*/
@Test
void testExistIndex() throws IOException {
GetIndexRequest request = new GetIndexRequest("geek_index");
boolean exists = restHighLevelClient.indices().exists(request, RequestOptions.DEFAULT);
System.out.println("exists = " + exists);// exists = true
}
/**
* 测试删除索引。
*
* @throws IOException
*/
@Test
void testDeleteIndex() throws IOException {
DeleteIndexRequest request = new DeleteIndexRequest("geek_index");
AcknowledgedResponse delete = restHighLevelClient.indices().delete(request, RequestOptions.DEFAULT);
System.out.println(delete);// org.elasticsearch.action.support.master.AcknowledgedResponse@4ee
System.out.println(delete.isAcknowledged());// true。
}
// ~ ~ ~ 文档。
/**
* 测试增加文档。
*
* @throws IOException
*/
@Test
void testAddDocument() throws IOException {
// 创建对象。
User user = new User("geek", 23);
// 创建请求。
IndexRequest request = new IndexRequest("geek_index");
// 规则。PUT /geek_index/_doc/1
request.id("3");
// request.timeout(TimeValue.timeValueSeconds(1));
request.timeout("1s");
// 将数据放入请求。
request.source(JSON.toJSONString(user), XContentType.JSON);
// 客户端发送请求。
IndexResponse response = restHighLevelClient.index(request, RequestOptions.DEFAULT);
System.out.println(response);
// IndexResponse[index=geek_index,type=_doc,id=1,version=1,result=created,seqNo=0,primaryTerm=1,shards={"total":2,"successful":1,"failed":0}]
System.out.println(response.status());
// CREATED
}
/**
* 获取文档。判断是否存在。
*
* @throws IOException
*/
@Test
void testIsExist() throws IOException {
GetRequest getRequest = new GetRequest("geek_index", "1");
// 不获取返回的 _source 上下文。
getRequest.fetchSourceContext(new FetchSourceContext(false));
getRequest.storedFields("_none_");
boolean exists = restHighLevelClient.exists(getRequest, RequestOptions.DEFAULT);
System.out.println("exists = " + exists);// true。
}
/**
* 获取文档信息。
*
* @throws IOException
*/
@Test
void testGetDocument() throws IOException {
GetRequest getRequest = new GetRequest("geek_index", "1");
GetResponse getResponse = restHighLevelClient.get(getRequest, RequestOptions.DEFAULT);
System.out.println(getResponse.getSourceAsString());// 文档内容。
// {"age":3,"name":"geek"}
System.out.println(getResponse);
// {"_index":"geek_index","_type":"_doc","_id":"1","_version":1,"_seq_no":0,"_primary_term":1,"found":true,"_source":{"age":3,"name":"geek"}}
}
/**
* 更新文档信息。
*
* @throws IOException
*/
@Test
void testUpdateDocument() throws IOException {
UpdateRequest updateRequest = new UpdateRequest("geek_index", "1");
updateRequest.timeout("1s");
User user = new User("Geek", 3);
updateRequest.doc(JSON.toJSONString(user), XContentType.JSON);
UpdateResponse response = restHighLevelClient.update(updateRequest, RequestOptions.DEFAULT);
System.out.println(response);
// UpdateResponse[index=geek_index,type=_doc,id=1,version=2,seqNo=1,primaryTerm=1,result=updated,shards=ShardInfo{total=2, successful=1, failures=[]}]
System.out.println(response.status());// OK
}
/**
* 删除文档记录。
*
* @throws IOException
*/
@Test
void testDelete() throws IOException {
DeleteRequest request = new DeleteRequest("geek_index", "3");
request.timeout(TimeValue.timeValueSeconds(1));
DeleteResponse response = restHighLevelClient.delete(request, RequestOptions.DEFAULT);
System.out.println(response);
// DeleteResponse[index=geek_index,type=_doc,id=3,version=3,result=deleted,shards=ShardInfo{total=2, successful=1, failures=[]}]
// DeleteResponse[index=geek_index,type=_doc,id=3,version=4,result=not_found,shards=ShardInfo{total=2, successful=1, failures=[]}]
}
/**
* 批量。
*
* @throws IOException
*/
@Test
void testBulkRequest() throws IOException {
BulkRequest bulkRequest = new BulkRequest();
bulkRequest.timeout("10s");
List<User> userList = new ArrayList<>();
userList.add(new User("geek1", 3));
userList.add(new User("geek2", 3));
userList.add(new User("geek3", 3));
userList.add(new User("李1", 3));
userList.add(new User("李2", 3));
userList.add(new User("李3", 3));
for (int i = 0; i < userList.size(); i++) {
bulkRequest.add(
new IndexRequest("geek_index")
.id("" + (i + 1))// 如果不设置 id,为随机 id。
.source(JSON.toJSONString(userList.get(i)), XContentType.JSON));
}
BulkResponse bulkResponse = restHighLevelClient.bulk(bulkRequest, RequestOptions.DEFAULT);
System.out.println(bulkResponse.hasFailures());// false。代表成功。
}
/**
* 查询。
*
* @throws IOException
*/
@Test
void testSearch() throws IOException {
SearchRequest searchRequest = new SearchRequest("geek_index");
// 构建搜索条件。
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
// new QueryBuilder()
// 查询条件。可以使用 QueryBuilders 工具实现。
// 匹配所有。QueryBuilders.matchAllQuery()
// 精确匹配。QueryBuilders.termQuery("name", "geek1");
TermQueryBuilder termQueryBuilder = QueryBuilders.termQuery("name", "geek1");
searchSourceBuilder.query(termQueryBuilder);
// searchSourceBuilder.from();
// searchSourceBuilder.size()
searchSourceBuilder.timeout(new TimeValue(60, TimeUnit.SECONDS));
searchRequest.source(searchSourceBuilder);
SearchResponse response = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
System.out.println(JSON.toJSONString(response.getHits()));
// {"fragment":true,"hits":[{"fields":{},"fragment":false,"highlightFields":{},"id":"1","matchedQueries":[],"primaryTerm":0,"rawSortValues":[],"score":1.7836733,"seqNo":-2,"sortValues":[],"sourceAsMap":{"name":"geek1","age":3},"sourceAsString":"{\"age\":3,\"name\":\"geek1\"}","sourceRef":{"fragment":true},"type":"_doc","version":-1}],"maxScore":1.7836733,"totalHits":{"relation":"EQUAL_TO","value":1}}
System.out.println(" ~ ~ ~ ~ ~ ~ ~ ");
for (SearchHit documentFields : response.getHits().getHits()) {
System.out.println(documentFields.getSourceAsMap());// {name=geek1, age=3}
}
}
@Test
void contextLoads() {
}
}
实战 ~ 京东搜索。
爬虫工具类。
jsoup 解析网页。
<!-- https://mvnrepository.com/artifact/org.jsoup/jsoup -->
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>1.12.2</version>
</dependency>
package com.geek.esjdgeek.util;
import com.geek.esjdgeek.pojo.Content;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.stereotype.Component;
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
/**
* @author geek
*/
@Component
public class HtmlParseUtil {
public static void main(String[] args) throws IOException {
new HtmlParseUtil().parseJD("python").forEach(System.out::println);
}
// public static void main(String[] args) throws IOException {
// // 获取请求。
String url = "https://search.jd.com/Search?keyword=Java&enc=utf-8&wq=Java&pvid=c871636462c74583b7272bd6073a0a20";
// String url = "https://search.jd.com/Search?keyword=Java";
//
// // 解析网页。就是 Document 对象。
// Document document = Jsoup.parse(new URL(url), 30000);
// // js。
// Element element = document.getElementById("J_goodsList");
System.out.println(element);
// // 获取所有的 li 元素。
// Elements elements = element.getElementsByTag("li");
// // 获取元素中的内容。
// for (Element el : elements) {
// // <img width="220" height="220" data-img="1" src="//img10.360buyimg.com/n7/jfs/t1/26339/8/10661/124305/5c8af829E4470835f/99742c91174d3d7a.jpg" data-lazy-img="done" source-data-lazy-img="">
// // 懒加载。
// String img = el.getElementsByTag("img").eq(0).attr("src");
// String price = el.getElementsByClass("p-price").eq(0).text();
// String title = el.getElementsByClass("p-name").eq(0).text();
//
// System.out.println(" ~ ~ ~ ~ ~ ~ ~ ");
//
// System.out.println(img);
// System.out.println(price);
// System.out.println(title);
// }
// }
public List<Content> parseJD(String keywords) throws IOException {
// 获取请求。
String url = "https://search.jd.com/Search?keyword=" + keywords;
// 解析网页。就是 Document 对象。
Document document = Jsoup.parse(new URL(url), 30000);
// js。
Element element = document.getElementById("J_goodsList");
// 获取所有的 li 元素。
Elements elements = element.getElementsByTag("li");
List<Content> goodsList = new ArrayList<>();
// 获取元素中的内容。
for (Element el : elements) {
String img = el.getElementsByTag("img").eq(0).attr("src");
String price = el.getElementsByClass("p-price").eq(0).text();
String title = el.getElementsByClass("p-name").eq(0).text();
Content content = new Content();
content.setImg(img);
content.setPrice(price);
content.setTitle(title);
goodsList.add(content);
}
return goodsList;
}
}
业务逻辑。
- service。
package com.geek.esjdgeek.service;
import com.alibaba.fastjson.JSON;
import com.geek.esjdgeek.pojo.Content;
import com.geek.esjdgeek.util.HtmlParseUtil;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.TermQueryBuilder;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
/**
* @author geek
*/
@Service
public class ContentService {
@Autowired
private RestHighLevelClient restHighLevelClient;
@Autowired
private HtmlParseUtil htmlParseUtil;
/**
* 解析数据放入 es 索引中。
*
* @param keywords
* @return
* @throws IOException
*/
public Boolean parseContent(String keywords) throws IOException {
List<Content> contents = htmlParseUtil.parseJD(keywords);
// 把查询的数据放入 es 中。
BulkRequest bulkRequest = new BulkRequest();
bulkRequest.timeout("2m");
for (Content content : contents) {
bulkRequest.add(
new IndexRequest("jd_goods")
.source(JSON.toJSONString(content), XContentType.JSON)
);
}
BulkResponse bulkResponse = restHighLevelClient.bulk(bulkRequest, RequestOptions.DEFAULT);
return !bulkResponse.hasFailures();
}
/**
* 获取这些数据实现搜索功能。
*
* @param keyword
* @param pageNo
* @param pageSize
* @return
* @throws IOException
*/
public List<Map<String, Object>> searchPage(String keyword, int pageNo, int pageSize) throws IOException {
if (pageNo <= 1) {
pageNo = 1;
}
// 条件搜索。
SearchRequest searchRequest = new SearchRequest("jd_goods");
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
// 分页。
searchSourceBuilder.from(pageNo);
searchSourceBuilder.size(pageSize);
// 精准匹配。
TermQueryBuilder termQueryBuilder = QueryBuilders.termQuery("title", keyword);
searchSourceBuilder.query(termQueryBuilder);
searchSourceBuilder.timeout(new TimeValue(60, TimeUnit.SECONDS));
// 执行搜索。
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
// 解析结果。
ArrayList<Map<String, Object>> list = new ArrayList<>();
for (SearchHit documentFields : searchResponse.getHits().getHits()) {
list.add(documentFields.getSourceAsMap());
}
return list;
}
/**
* 获取这些数据实现搜索功能 ~ 高亮。
*
* @param keyword
* @param pageNo
* @param pageSize
* @return
* @throws IOException
*/
public List<Map<String, Object>> searchPageHighlight(String keyword, int pageNo, int pageSize) throws IOException {
if (pageNo <= 1) {
pageNo = 1;
}
// 条件搜索。
SearchRequest searchRequest = new SearchRequest("jd_goods");
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
// 分页。
searchSourceBuilder.from(pageNo);
searchSourceBuilder.size(pageSize);
// 精准匹配。
TermQueryBuilder termQueryBuilder = QueryBuilders.termQuery("title", keyword);
searchSourceBuilder.query(termQueryBuilder);
searchSourceBuilder.timeout(new TimeValue(60, TimeUnit.SECONDS));
// 高亮。
HighlightBuilder highlightBuilder = new HighlightBuilder();
highlightBuilder.field("title");
// highlightBuilder.requireFieldMatch(false);// 关闭多个字段高亮。(只高亮第一个)。
highlightBuilder.preTags("<span style='color: red'>");
highlightBuilder.postTags("</span>");
searchSourceBuilder.highlighter(highlightBuilder);
// 执行搜索。
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
// 解析结果。
ArrayList<Map<String, Object>> list = new ArrayList<>();
for (SearchHit hit : searchResponse.getHits().getHits()) {
// 解析高亮字段。
Map<String, HighlightField> highlightFields = hit.getHighlightFields();
HighlightField title = highlightFields.get("title");
Map<String, Object> sourceAsMap = hit.getSourceAsMap();// 原来的结果。
if (title != null) {
Text[] fragments = title.fragments();
// 高亮字段替换原来没有高亮的字段。
String newTitle = "";
for (Text fragment : fragments) {
newTitle += fragment;
}
sourceAsMap.put("title", newTitle);
}
list.add(hit.getSourceAsMap());
}
return list;
}
}
- controller。
package com.geek.esjdgeek.controller;
import com.geek.esjdgeek.service.ContentService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RestController;
import java.io.IOException;
import java.util.List;
import java.util.Map;
/**
* @author geek
*/
@RestController
public class ContentController {
@Autowired
private ContentService contentService;
@GetMapping("/parse/{keyword}")
public Boolean parse(@PathVariable("keyword") String keyword) throws IOException {
Boolean aBoolean = contentService.parseContent(keyword);
System.out.println("aBoolean = " + aBoolean);
return aBoolean;
}
@GetMapping("/search/{keyword}/{pageNo}/{pageSize}")
public List<Map<String, Object>> search(@PathVariable("keyword") String keyword,
@PathVariable("pageNo") int pageNo,
@PathVariable("pageSize") int pageSize
) throws IOException {
// if (pageNo==null)
return contentService.searchPageHighlight(keyword, pageNo, pageSize);
}
}
前后端交互 ~ Vue.js。
<!DOCTYPE html>
<html xmlns:th="http://www.thymeleaf.org">
<head>
<meta charset="utf-8"/>
<title>狂神说Java-ES仿京东实战</title>
<link rel="stylesheet" th:href="@{/css/style.css}"/>
<script th:src="@{/js/jquery.min.js}"></script>
</head>
<body class="pg">
<div class="page" id="app">
<div class="mallist tmall- page-not-market" id="mallPage">
<!-- 头部搜索。-->
<div class="header-list-app" id="header">
<div class="headerLayout">
<div class="headerCon ">
<!-- Logo。-->
<h1 id="mallLogo">
<img alt="" th:src="@{/images/jdlogo.png}">
</h1>
<div class="header-extra">
<!-- 搜索。-->
<div class="mall-search" id="mallSearch">
<form class="mallSearch-form clearfix" name="searchTop">
<fieldset>
<legend>天猫搜索</legend>
<div class="mallSearch-input clearfix">
<div class="s-combobox" id="s-combobox-685">
<div class="s-combobox-input-wrap">
<input aria-haspopup="true" autocomplete="off" class="s-combobox-input"
id="mq"
type="text"
v-model="keyword" value="dd">
</div>
</div>
<button @click.prevent="searchKey" id="searchbtn" type="submit">搜索</button>
</div>
</fieldset>
</form>
<ul class="relKeyTop">
<li><a>狂神说 Java</a></li>
<li><a>狂神说前端</a></li>
<li><a>狂神说 Linux</a></li>
<li><a>狂神说大数据</a></li>
<li><a>狂神聊理财</a></li>
</ul>
</div>
</div>
</div>
</div>
</div>
<!-- 商品详情页面。-->
<div id="content">
<div class="main">
<!-- 品牌分类。-->
<form class="navAttrsForm">
<div class="attrs j_NavAttrs" style="display:block">
<div class="brandAttr j_nav_brand">
<div class="j_Brand attr">
<div class="attrKey">
品牌
</div>
<div class="attrValues">
<ul class="av-collapse row-2">
<li><a href="#"> 狂神说 </a></li>
<li><a href="#"> Java </a></li>
</ul>
</div>
</div>
</div>
</div>
</form>
<!-- 排序规则。-->
<div class="filter clearfix">
<a class="fSort fSort-cur">综合<i class="f-ico-arrow-d"></i></a>
<a class="fSort">人气<i class="f-ico-arrow-d"></i></a>
<a class="fSort">新品<i class="f-ico-arrow-d"></i></a>
<a class="fSort">销量<i class="f-ico-arrow-d"></i></a>
<a class="fSort">价格<i class="f-ico-triangle-mt"></i><i class="f-ico-triangle-mb"></i></a>
</div>
<!-- 商品详情。-->
<div class="view grid-nosku">
<div class="product" v-for="result in results">
<div class="product-iWrap">
<!-- 商品封面。-->
<div class="productImg-wrap">
<a class="productImg">
<img :src="result.img">
</a>
</div>
<!-- 价格。-->
<p class="productPrice">
<em><b>¥</b>{{result.price}}</em>
</p>
<!-- 标题。-->
<!--<p class="productTitle">-->
<!--<a> {{result.title}} </a>-->
<!--</p>-->
<p class="productTitle">
<a v-html="result.title"></a>
</p>
<!-- 店铺名。-->
<div class="productShop">
<span>店铺:Geek</span>
</div>
<!-- 成交信息。-->
<p class="productStatus">
<span>月成交 <em>999笔</em></span>
<span>评价 <a>3</a></span>
</p>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
<script th:src="@{/js/axios.min.js}"></script>
<script th:src="@{/js/vue.min.js}"></script>
<script>
new Vue({
el: "#app",
data: {
keyword: '',
results: ''
},
methods: {
searchKey() {
var keyword = this.keyword;
// console.log(keyword);
// 对接后端的接口。
axios.get('search/' + keyword + '/1/10').then(response => {
console.log(response);
// 绑定数据。
this.results = response.data;
})
}
}
})
</script>
</body>
</html>
高亮。
package com.geek.esjdgeek.service;
import com.alibaba.fastjson.JSON;
import com.geek.esjdgeek.pojo.Content;
import com.geek.esjdgeek.util.HtmlParseUtil;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.TermQueryBuilder;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
/**
* @author geek
*/
@Service
public class ContentService {
@Autowired
private RestHighLevelClient restHighLevelClient;
@Autowired
private HtmlParseUtil htmlParseUtil;
/**
* 获取这些数据实现搜索功能 ~ 高亮。
*
* @param keyword
* @param pageNo
* @param pageSize
* @return
* @throws IOException
*/
public List<Map<String, Object>> searchPageHighlight(String keyword, int pageNo, int pageSize) throws IOException {
if (pageNo <= 1) {
pageNo = 1;
}
// 条件搜索。
SearchRequest searchRequest = new SearchRequest("jd_goods");
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
// 分页。
searchSourceBuilder.from(pageNo);
searchSourceBuilder.size(pageSize);
// 精准匹配。
TermQueryBuilder termQueryBuilder = QueryBuilders.termQuery("title", keyword);
searchSourceBuilder.query(termQueryBuilder);
searchSourceBuilder.timeout(new TimeValue(60, TimeUnit.SECONDS));
// 高亮。
HighlightBuilder highlightBuilder = new HighlightBuilder();
highlightBuilder.field("title");
// highlightBuilder.requireFieldMatch(false);// 关闭多个字段高亮。(只高亮第一个)。
highlightBuilder.preTags("<span style='color: red'>");
highlightBuilder.postTags("</span>");
searchSourceBuilder.highlighter(highlightBuilder);
// 执行搜索。
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
// 解析结果。
ArrayList<Map<String, Object>> list = new ArrayList<>();
for (SearchHit hit : searchResponse.getHits().getHits()) {
// 解析高亮字段。
Map<String, HighlightField> highlightFields = hit.getHighlightFields();
HighlightField title = highlightFields.get("title");
Map<String, Object> sourceAsMap = hit.getSourceAsMap();// 原来的结果。
if (title != null) {
Text[] fragments = title.fragments();
// 高亮字段替换原来没有高亮的字段。
String newTitle = "";
for (Text fragment : fragments) {
newTitle += fragment;
}
sourceAsMap.put("title", newTitle);
}
list.add(hit.getSourceAsMap());
}
return list;
}
}
- Vue.js 中。
<!-- 标题。-->
<!--<p class="productTitle">-->
<!--<a> {{result.title}} </a>-->
<!--</p>-->
<p class="productTitle">
<a v-html="result.title"></a>
</p>