1.解决虚拟机占满宿主机磁盘的问题
本篇博客是在ES集群搭建的基础上而来
首先解决一个问题,就是虚拟机把宿主机磁盘占满的问题,导致虚拟机无法正常运行,首先楼主的虚拟机安装在了D盘,每台虚拟机分配20G的磁盘空间,结果跑着跑着随机虚拟机占用磁盘的增大,沾满了D盘,导致无法运行虚拟机,
下面将解决办法:现在需要对其他盘的空闲空间合并一部分到D盘
首先打开计算机管理:
右击C盘,点击压缩卷,输入需要分出来的磁盘的大小
压缩成功,成功分配了空闲的磁盘出来
接下来下载安装分区助手
就是这个玩意儿,也可以看到分区的情况,右击合并分区选择合并到D盘再提交
ok,这样就为D盘扩容了
然后在此路径中找到.vmx文件,在文件中查找(Ctrl+F快速查找)vmci0.present,此时会看到“vmci0.present = "TRUE"”
修改为FALSE即可。
如果还是有问题,可以执行下面的操作:
删除以.lck为后缀名的文件,然后重新打开虚拟机
2.ES的java api操作
新建maven工程,添加依赖
<properties>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
<encoding>UTF-8</encoding>
</properties>
<dependencies>
<!-- es的客户端-->
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>transport</artifactId>
<version>5.4.3</version>
</dependency>
<!-- 依赖2.x的log4j -->
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<version>2.8.2</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>2.8.2</version>
</dependency>
<!-- 单元测试 -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
</dependency>
</dependencies>
log4j2
<?xml version="1.0" encoding="UTF-8"?>
<Configuration status="WARN">
<Appenders>
<Console name="Console" target="SYSTEM_OUT">
<PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
</Console>
</Appenders>
<Loggers>
<Root level="error">
<AppenderRef ref="Console"/>
</Root>
</Loggers>
</Configuration>
第一个简单操作:
package com.wx.es1;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.transport.client.PreBuiltTransportClient;
import org.junit.Test;
import java.net.InetAddress;
public class HelloEs {
/*
1.连接es集群,通过id查询数据
*/
@Test
public void helloES()
{
TransportClient client=null;
try {
//1.设置集群的配置信息
Settings settings=Settings.builder()
.put("cluster.name","my-es")
.build();
//2.连接集群,创建客户端
client=new PreBuiltTransportClient(settings).addTransportAddresses(
new InetSocketTransportAddress(InetAddress.getByName("192.168.203.128"),9300),
new InetSocketTransportAddress(InetAddress.getByName("192.168.203.129"),9300),
new InetSocketTransportAddress(InetAddress.getByName("192.168.203.130"),9300));
//3.使用客户端操作写操作语句,操作集群 .actionGet()方法是同步的,没有返回就等待
GetResponse response = client.prepareGet("news", "fulltext", "2").execute().actionGet();
//4.打印执行结果
System.out.print(response);
}
catch (Exception e)
{
e.printStackTrace();
}
finally {
client.close();
}
}
}
CRUD操作:
package com.wx.es1;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.bulk.byscroll.BulkByScrollResponse;
import org.elasticsearch.action.delete.DeleteResponse;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.get.MultiGetItemResponse;
import org.elasticsearch.action.get.MultiGetResponse;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.reindex.DeleteByQueryAction;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
import org.elasticsearch.transport.client.PreBuiltTransportClient;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
import java.net.InetAddress;
import java.util.Date;
import java.util.Iterator;
import java.util.Map;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.index.query.QueryBuilders.rangeQuery;
public class CRUDEs {
private TransportClient client=null;
@Before
public void init()
{
try {
Settings settings=Settings.builder()
.put("cluster.name","my-es")
.build();
client=new PreBuiltTransportClient(settings).addTransportAddresses(
new InetSocketTransportAddress(InetAddress.getByName("192.168.203.128"),9300),
new InetSocketTransportAddress(InetAddress.getByName("192.168.203.129"),9300),
new InetSocketTransportAddress(InetAddress.getByName("192.168.203.130"),9300)
);
}catch (Exception e)
{
e.printStackTrace();
}
}
/*
关闭客户端
*/
@After
public void closeClient()
{
if (client!=null)
{
client.close();
}
}
//插入一条数据
@Test
public void createIndex () throws Exception
{
IndexResponse response = client.prepareIndex("gamelog", "users", "2")
.setSource(
jsonBuilder()
.startObject()
.field("username", "李四")
.field("gender", "male")
.field("birthday", new Date())
.field("fv", 9999)
.field("message", "trying out Elasticsearch")
.endObject()
).get();
}
/*
查询一条数据
*/
@Test
public void getOne()
{
GetResponse response = client.prepareGet("gamelog", "users", "1").get();
System.out.print(response);
}
/*
查找多条数据
*/
@Test
public void getMulti() throws IOException
{
MultiGetResponse multiGetItemResponses = client.prepareMultiGet()
.add("gamelog", "users", "1")
.add("news", "fulltext", "2","3")
.get();
for(MultiGetItemResponse responses: multiGetItemResponses) {
GetResponse response = responses.getResponse();
System.out.print(response);
}
}
/*
修改
*/
@Test
public void testUpdate() throws Exception {
UpdateRequest updateRequest = new UpdateRequest();
updateRequest.index("gamelog");
updateRequest.type("users");
updateRequest.id("1");
updateRequest.doc(
jsonBuilder()
.startObject()
.field("fv", 999.9)
.endObject());
client.update(updateRequest).get();
}
/*
删除
*/
@Test
public void testDelete() {
DeleteResponse response = client.prepareDelete("gamelog", "users", "2").get();
System.out.println(response);
}
/*
指定条件删除
*/
@Test
public void testDeleteByQuery() {
BulkByScrollResponse response =
DeleteByQueryAction.INSTANCE.newRequestBuilder(client)
//指定查询条件
.filter(QueryBuilders.matchQuery("username", "李四"))
//指定索引名称
.source("gamelog")
.get();
long deleted = response.getDeleted();
System.out.println(deleted);
}
/*
异步删除
*/
//异步删除
@Test
public void testDeleteByQueryAsync() {
DeleteByQueryAction.INSTANCE.newRequestBuilder(client)
.filter(QueryBuilders.matchQuery("gender", "male"))
.source("gamelog")
.execute(new ActionListener<BulkByScrollResponse>() {
@Override
public void onResponse(BulkByScrollResponse response) {
long deleted = response.getDeleted();
System.out.println("数据删除了");
System.out.println(deleted);
}
@Override
public void onFailure(Exception e) {
e.printStackTrace();
}
});
try {
System.out.println("异步删除");
Thread.sleep(10000);
} catch (Exception e) {
e.printStackTrace();
}
}
/*
范围内查询
*/
@Test
public void testRange() {
QueryBuilder qb = rangeQuery("fv")
// [88.99, 10000)
.from(88.99)
.to(10000)
.includeLower(true)
.includeUpper(false);
SearchResponse response = client.prepareSearch("gamelog").setQuery(qb).get();
System.out.println(response);
}
/*
进行聚合查询 select team, count(*) as player_count from player group by team;
*/
@Test
public void testAgg1() {
//指定索引和type
SearchRequestBuilder builder = client.prepareSearch("player_info").setTypes("player");
//按team分组然后聚合,但是并没有指定聚合函数
TermsAggregationBuilder teamAgg = AggregationBuilders.terms("player_count").field("team");
//添加聚合器
builder.addAggregation(teamAgg);
//触发
SearchResponse response = builder.execute().actionGet();
//System.out.println(response);
//将返回的结果放入到一个map中
Map<String, Aggregation> aggMap = response.getAggregations().getAsMap();
// Set<String> keys = aggMap.keySet();
//
// for (String key: keys) {
// System.out.println(key);
// }
// //取出聚合属性
StringTerms terms = (StringTerms) aggMap.get("player_count");
//
//依次迭代出分组聚合数据
// for (Terms.Bucket bucket : terms.getBuckets()) {
// //分组的名字
// String team = (String) bucket.getKey();
// //count,分组后一个组有多少数据
// long count = bucket.getDocCount();
// System.out.println(team + " " + count);
// }
Iterator<Terms.Bucket> teamBucketIt = terms.getBuckets().iterator();
while (teamBucketIt .hasNext()) {
Terms.Bucket bucket = teamBucketIt.next();
String team = (String) bucket.getKey();
long count = bucket.getDocCount();
System.out.println(team + " " + count);
}
}
//select team, position, count(*) as pos_count from player group by team, position;
/*
查找相同球队相同位置球员的数量
*/
@Test
public void testAgg2() {
SearchRequestBuilder builder = client.prepareSearch("player_info").setTypes("player");
//指定别名和分组的字段
TermsAggregationBuilder teamAgg = AggregationBuilders.terms("team_name").field("team");
TermsAggregationBuilder posAgg= AggregationBuilders.terms("pos_count").field("position");
//添加两个聚合构建器
builder.addAggregation(teamAgg.subAggregation(posAgg));
//执行查询
SearchResponse response = builder.execute().actionGet();
//将查询结果放入map中
Map<String, Aggregation> aggMap = response.getAggregations().getAsMap();
//根据属性名到map中查找
StringTerms teams = (StringTerms) aggMap.get("team_name");
//循环查找结果
for (Terms.Bucket teamBucket : teams.getBuckets()) {
//先按球队进行分组
String team = (String) teamBucket.getKey();
Map<String, Aggregation> subAggMap = teamBucket.getAggregations().getAsMap();
StringTerms positions = (StringTerms) subAggMap.get("pos_count");
//因为一个球队有很多位置,那么还要依次拿出位置信息
for (Terms.Bucket posBucket : positions.getBuckets()) {
//拿到位置的名字
String pos = (String) posBucket.getKey();
//拿出该位置的数量
long docCount = posBucket.getDocCount();
//打印球队,位置,人数
System.out.println(team + " " + pos + " " + docCount);
}
}
}
}
AdminAPI:
package com.wx.es1;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.AdminClient;
import org.elasticsearch.client.IndicesAdminClient;
import org.elasticsearch.client.Requests;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.avg.AvgAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.avg.InternalAvg;
import org.elasticsearch.search.aggregations.metrics.max.InternalMax;
import org.elasticsearch.search.aggregations.metrics.max.MaxAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.sum.InternalSum;
import org.elasticsearch.search.aggregations.metrics.sum.SumAggregationBuilder;
import org.elasticsearch.transport.client.PreBuiltTransportClient;
import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
import java.net.InetAddress;
import java.util.HashMap;
import java.util.Map;
public class AdminAPI {
/*
AdminApi主要就是用来操作创建索引分片之类的admin操作
*/
//初始化集群设置
private TransportClient client;
@Before
public void init()throws Exception{
Settings settings=Settings.builder().put("cluster.name","my-es").build();
client=new PreBuiltTransportClient(settings).addTransportAddresses(
new InetSocketTransportAddress(InetAddress.getByName("192.168.203.128"),9300),
new InetSocketTransportAddress(InetAddress.getByName("192.168.203.129"),9300),
new InetSocketTransportAddress(InetAddress.getByName("192.168.203.130"),9300)
);
}
//创建索引,并配置一些参数
@Test
public void createIndexWithSettings() {
//获取Admin的API
AdminClient admin = client.admin();
//使用Admin API对索引进行操作
IndicesAdminClient indices = admin.indices();
//准备创建索引
indices.prepareCreate("gamelog")
//配置索引参数
.setSettings(
//参数配置器
Settings.builder()//指定索引分区的数量
.put("index.number_of_shards", 4)
//指定索引副本的数量(注意:不包括本身,如果设置数据存储副本为2,实际上数据存储了3份)
.put("index.number_of_replicas", 2)
)
//真正执行
.get();
}
/**
* 你可以通过dynamic设置来控制这一行为,它能够接受以下的选项:
* true:默认值。动态添加字段
* false:忽略新字段
* strict:如果碰到陌生字段,抛出异常
* @throws
*/
//创建索引和mapping映射
@Test
public void createSettingsMappings() throws Exception {
//1.设置Settings
HashMap<String, Object> settings_map = new HashMap<String, Object>();
//设置分片的数量
settings_map.put("number_of_shards", 3);
//设置副本的数量,如果设置两个副本就代表会存三份数据
settings_map.put("number_of_replicas", 2);
//2.这是mapping映射,这相当于配置一个域信息
XContentBuilder contentBuilder = XContentFactory.jsonBuilder()
.startObject()
.field("dynamic", true)
//设置type中的属性
.startObject("properties")
//设置num属性的信息
.startObject("num")
//类型是Integer
.field("type", "integer")
//不分词,但是建索引
.field("index", "not_analyzed")
//在文档中存储
.field("store", "yes")
.endObject()
//设置name属性的信息
.startObject("name")
//类型为string
.field("type", "string")
//在文档中存储
.field("store", "yes")
//建立索引,分词
.field("index", "analyzed")
//分词器使用ik
.field("analyzer", "ik_max_word")
.endObject()
.endObject()
.endObject();
//创建索引,名字叫user_info
CreateIndexRequestBuilder builder = client.admin().indices().prepareCreate("user_info");
//管理索引(user_info)然后关联type(user)
builder.setSettings(settings_map).addMapping( "user",contentBuilder).get();
}
/**
* index这个属性,no代表不建索引
* not_analyzed,建索引不分词
* analyzed 即分词,又建立索引
* expected [no], [not_analyzed] or [analyzed]
* @throws IOException
*/
@Test
public void testSettingsPlayerMappings() throws IOException {
//1:settings
HashMap<String, Object> settings_map = new HashMap<String, Object>(2);
settings_map.put("number_of_shards", 3);
settings_map.put("number_of_replicas", 1);
//2:mappings
XContentBuilder builder = XContentFactory.jsonBuilder()
.startObject()//
.field("dynamic", "true")
.startObject("properties")
.startObject("id")
.field("type", "integer")
.field("store", "yes")
.endObject()
.startObject("name")
.field("type", "string")
.field("index", "not_analyzed")
.endObject()
.startObject("age")
.field("type", "integer")
.endObject()
.startObject("salary")
.field("type", "integer")
.endObject()
.startObject("team")
.field("type", "string")
.field("index", "not_analyzed")
.endObject()
.startObject("position")
.field("type", "string")
.field("index", "not_analyzed")
.endObject()
.startObject("description")
.field("type", "string")
.field("store", "no")
.field("index", "analyzed")
.field("analyzer", "ik_smart")
.endObject()
.startObject("addr")
.field("type", "string")
.field("store", "yes")
.field("index", "analyzed")
.field("analyzer", "ik_smart")
.endObject()
.endObject()
.endObject();
CreateIndexRequestBuilder prepareCreate = client.admin().indices().prepareCreate("player_info");
prepareCreate.setSettings(settings_map).addMapping("player", builder).get();
}
//select team, max(age) as max_age from player group by team;查询每个队年龄最大的球员
@Test
public void testAgg3() {
SearchRequestBuilder builder = client.prepareSearch("player_info").setTypes("player");
//指定安球队进行分组
TermsAggregationBuilder teamAgg = AggregationBuilders.terms("team_name").field("team");
//指定分组求最大值
MaxAggregationBuilder maxAgg = AggregationBuilders.max("max_age").field("age");
//分组后求最大值
builder.addAggregation(teamAgg.subAggregation(maxAgg));
//查询
SearchResponse response = builder.execute().actionGet();
Map<String, Aggregation> aggMap = response.getAggregations().getAsMap();
//根据team属性,获取map中的内容
StringTerms teams = (StringTerms) aggMap.get("team_name");
for (Terms.Bucket teamBucket : teams.getBuckets()) {
//分组的属性名
String team = (String) teamBucket.getKey();
//在将聚合后取最大值的内容取出来放到map中
Map<String, Aggregation> subAggMap = teamBucket.getAggregations().getAsMap();
//取分组后的最大值
InternalMax ages = (InternalMax)subAggMap.get("max_age");
double max = ages.getValue();
System.out.println(team + " " + max);
}
}
//select team, avg(age) as avg_age, sum(salary) as total_salary from player group by team;
//查询每个队的平均年龄和平均工资
@Test
public void testAgg4() {
SearchRequestBuilder builder = client.prepareSearch("player_info").setTypes("player");
//指定分组字段
TermsAggregationBuilder termsAgg = AggregationBuilders.terms("team_name").field("team");
//指定聚合函数是求平均数据
AvgAggregationBuilder avgAgg = AggregationBuilders.avg("avg_age").field("age");
//指定另外一个聚合函数是求和
SumAggregationBuilder sumAgg = AggregationBuilders.sum("total_salary").field("salary");
//分组的聚合器关联了两个聚合函数
builder.addAggregation(termsAgg.subAggregation(avgAgg).subAggregation(sumAgg));
SearchResponse response = builder.execute().actionGet();
Map<String, Aggregation> aggMap = response.getAggregations().getAsMap();
//按分组的名字取出数据
StringTerms teams = (StringTerms) aggMap.get("team_name");
for (Terms.Bucket teamBucket : teams.getBuckets()) {
//获取球队名字
String team = (String) teamBucket.getKey();
Map<String, Aggregation> subAggMap = teamBucket.getAggregations().getAsMap();
//根据别名取出平均年龄
InternalAvg avgAge = (InternalAvg)subAggMap.get("avg_age");
//根据别名取出薪水总和
InternalSum totalSalary = (InternalSum)subAggMap.get("total_salary");
double avgAgeValue = avgAge.getValue();
double totalSalaryValue = totalSalary.getValue();
System.out.println(team + " " + avgAgeValue + " " + totalSalaryValue);
}
}
//select team, sum(salary) as total_salary from player group by team order by total_salary desc;
//查询每个队的队员的总工资并按升序排列
@Test
public void testAgg5() {
SearchRequestBuilder builder = client.prepareSearch("player_info").setTypes("player");
//按team进行分组,然后指定排序规则
TermsAggregationBuilder termsAgg = AggregationBuilders.terms("team_name").field("team").order(Terms.Order.aggregation("total_salary ", true));
SumAggregationBuilder sumAgg = AggregationBuilders.sum("total_salary").field("salary");
builder.addAggregation(termsAgg.subAggregation(sumAgg));
SearchResponse response = builder.execute().actionGet();
Map<String, Aggregation> aggMap = response.getAggregations().getAsMap();
StringTerms teams = (StringTerms) aggMap.get("team_name");
for (Terms.Bucket teamBucket : teams.getBuckets()) {
String team = (String) teamBucket.getKey();
Map<String, Aggregation> subAggMap = teamBucket.getAggregations().getAsMap();
InternalSum totalSalary = (InternalSum)subAggMap.get("total_salary");
double totalSalaryValue = totalSalary.getValue();
System.out.println(team + " " + totalSalaryValue);
}
}
}
ES的sql插件安装:所有的机器都要安装,在安装目录下执行
./bin/elasticsearch-plugin install https://github.com/NLPchina/elasticsearch-sql/releases/download/5.4.3.0/elasticsearch-sql-5.4.3.0.zip
重启ES:
简单查询:http://192.168.203.128:9200/_sql?sql=select * from player_info limit 10
聚合查询:http://192.168.203.128:9200/_sql?sql=select team,avg(salary) avg_sal from player_info group by team
安装sql的ui插件:
修改配置:
步骤命令:
用npm编译安装
unzip es-sql-site-standalone.zip
cd site-server/
npm install express --save
修改SQL的Server的端口
vi site_configuration.json
启动服务
node node-server.js &
Select team,avg(salary) from player_info group by team
接下解决ES分片的问题