此代码示例展示了如何使用NEST客户端在.NET7中进行Elasticsearch的操作，包括创建和删除索引、创建索引模板、添加和更新索引对象、局部更新、批量操作以及分页和滚动查询。还包含了处理异常和日志记录的方法。
目录
NEST是elasticsearch为.net 提供的高级客户端依赖组件。这里为了方便在NET7中使用NEST客户端，我们对NEST的相关方法做了封装，以便在NET7中调用。
创建索引
搜索我们创建索引的时候，需要先判断索引是否存在，如果存在索引，则返回，不存在，则添加。
GetIndexName()方法是通过实体配置属性来获取索引名称，如下：
private string GetIndexName()
{
var indexName = typeof(TEntity).GetCustomAttribute<ElasticsearchTypeAttribute>()?.RelationName ?? string.Empty;
if (indexName.IsBlank())
{
throw new ArgumentNullException($"{nameof(TEntity)}没有配置MetaIndex属性。");
}
return indexName;
}
/// <summary>
/// 创建索引
/// </summary>
/// <returns></returns>
/// <exception cref="BusinessException"></exception>
public async Task CreateIndexAsync()
{
var indexName = GetIndexName();
var existResponse = await _elasticClient.Indices.ExistsAsync(indexName);
if (existResponse.Exists)
{
return;
}
//创建索引，自动转换类型，order为索引名称
var response = await _elasticClient.Indices.CreateAsync(indexName, c => c.Map<TEntity>(m => m.AutoMap()));
if (!response.IsValid)
{
throw new BusinessException($"{indexName}索引创建失败:{response.DebugInformation}。");
}
}创建索引模板
关于elasticsearch的模板索引的定义，以及使用方法，这里就不做过多的解释，如果需要了解的，请移步：ElasticSearch索引模板 - 简书
/// <summary>
/// 创建索引模板
/// </summary>
/// <param name="templateName">索引模板编号名称</param>
/// <param name="templateAlias">别名</param>
/// <param name="templatePattern">模板通配符表达式</param>
/// <returns></returns>
public async Task CreateTemplateAsync(string templateName, string templateAlias, string templatePattern)
{
var existResponse = await _elasticClient.Indices.TemplateV2ExistsAsync(templateName);
if (existResponse.Exists)
{
return;
}
ITemplate selectorTemplate(TemplateDescriptor x) => x.Aliases(a => a.Alias(templateAlias)).Mappings<TEntity>(m => m.AutoMap());
IPutIndexTemplateV2Request selector(PutIndexTemplateV2Descriptor x) => x.IndexPatterns(templatePattern).Template(selectorTemplate);
var templateResponse = await _elasticClient.Indices.PutTemplateV2Async(templateName, selector);
if (!templateResponse.IsValid)
{
throw new BusinessException($"{templateName}索引模板添加失败：{templateResponse.DebugInformation}。");
}
}
/// <summary>
/// 删除索引模板
/// </summary>
/// <param name="templateName">模板名称编码</param>
/// <returns></returns>
/// <exception cref="BusinessException"></exception>
public async Task DeleteTemplateAsync(string templateName)
{
var existResponse = await _elasticClient.Indices.TemplateV2ExistsAsync(templateName);
if (!existResponse.Exists)
{
return;
}
var response = await _elasticClient.Indices.DeleteTemplateV2Async(templateName);
if (!response.IsValid)
{
throw new BusinessException($"{templateName}索引模板删除失败：{response.DebugInformation}。");
}
}添加索引对象
添加单个索引对象
/// <summary>
/// 单个对象添加
/// </summary>
/// <param name="data">数据对象</param>
/// <returns></returns>
/// <exception cref="BusinessException"></exception>
public async Task AddAsync(TEntity data)
{
var indexName = GetIndexName();
await AddAsync(indexName, data);
}
/// <summary>
/// 添加单个对象到指定索引
/// </summary>
/// <param name="indexName">索引名称</param>
/// <param name="data">数据对象</param>
/// <returns></returns>
/// <exception cref="BusinessException"></exception>
public async Task AddAsync(string indexName, TEntity data)
{
var response = await _elasticClient.IndexAsync(data, x => x.Index(indexName));
if (!response.IsValid)
{
throw new BusinessException($"{indexName}添加失败。");
}
}批量添加索引对象
/// <summary>
/// 批量添加
/// </summary>
/// <param name="datas"></param>
/// <returns></returns>
/// <exception cref="BusinessException"></exception>
public async Task AddsAsync(IList<TEntity> datas)
{
var indexName = GetIndexName();
await AddsAsync(indexName, datas);
}
/// <summary>
/// 批量添加对象集合到指定索引
/// </summary>
/// <param name="indexName">索引名称</param>
/// <param name="datas"></param>
/// <returns></returns>
/// <exception cref="BusinessException"></exception>
public async Task AddsAsync(string indexName, IList<TEntity> datas)
{
if (datas.IsNullOrEmpty())
{
return;
}
//分页推送-防止数据量过大接口超时
int page = 1;
int pageSize = 1000;
int totalPage = (datas.Count + pageSize - 1) / pageSize;
while (page <= totalPage)
{
try
{
//本次推送的数据
var currentPushDatas = datas.Skip((page - 1) * pageSize).Take(pageSize).ToList();
var response = await _elasticClient.BulkAsync(b => b.Index(indexName).IndexMany(currentPushDatas));
if (!response.IsValid)
{
_logger?.LogInformation($"{indexName}批量添加失败：{JsonSerializer.Serialize(response)}");
}
}
catch (Exception ex)
{
//记录日志
_logger?.LogError(ex, $"{indexName}批量添加异常page={page}：{ex}");
}
//下一页
page++;
}
}
public void BulkAll(IList<TEntity> datas)
{
var index = GetIndexName();
BulkAll(index, datas);
}
public void BulkAll(string indexName, IList<TEntity> datas)
{
if (datas.IsNullOrEmpty())
{
return;
}
var bulkAllObserver = _elasticClient.BulkAll(datas, b => b.Index(indexName)
.BackOffTime("30s")//集群繁忙，报429错误码的时候，等待多久进行重试
.BackOffRetries(2) //重试次数
.RefreshOnCompleted() //
.MaxDegreeOfParallelism(Environment.ProcessorCount)
.Size(1000) //每次 bulk 请求时的文档数量。
).Wait(TimeSpan.FromSeconds(30), next =>
{
//执行索引并等待 1min，BulkAll 请求虽然时异步的，但是是一个阻塞操作。
});
}更新索引对象
单个更新索引对象
/// <summary>
/// 单个更新
/// </summary>
/// <param name="data"></param>
/// <returns></returns>
/// <exception cref="BusinessException"></exception>
public async Task UpdateAsync(TEntity data)
{
var indexName = GetIndexName();
await UpdateAsync(indexName, data);
}
/// <summary>
/// 更新指定索引的对象
/// </summary>
/// <param name="indexName">索引名称</param>
/// <param name="data"></param>
/// <returns></returns>
/// <exception cref="BusinessException"></exception>
public async Task UpdateAsync(string indexName, TEntity data)
{
DocumentPath<TEntity> documentPath = new(data.Id);
var response = await _elasticClient.UpdateAsync(documentPath, (p) => p.Doc(data).Index(indexName));
if (!response.IsValid)
{
throw new BusinessException($"{indexName}更新失败。");
}
}批量更新索引对象
/// <summary>
/// 批量更新
/// </summary>
/// <param name="datas"></param>
/// <returns></returns>
/// <exception cref="BusinessException"></exception>
public async Task UpdatesAsnyc(IList<TEntity> datas)
{
var indexName = GetIndexName();
await UpdatesAsnyc(indexName, datas);
}
/// <summary>
/// 批量更新指定索引的对象
/// </summary>
/// <param name="indexName">索引名称</param>
/// <param name="datas"></param>
/// <returns></returns>
/// <exception cref="BusinessException"></exception>
public async Task UpdatesAsnyc(string indexName, IList<TEntity> datas)
{
var updateDescriptor = new BulkDescriptor();
foreach (var data in datas)
{
updateDescriptor.Update<TEntity>(i => i
.Index(indexName)
.Id(data.Id)
.Doc(data)
.DocAsUpsert(true));
}
var response = await _elasticClient.BulkAsync(updateDescriptor);
if (!response.IsValid)
{
throw new BusinessException($"{indexName}批量更新失败。");
}
}
/// <summary>
/// 批量局部更新
/// </summary>
/// <param name="datas"></param>
/// <param name="express"></param>
/// <returns></returns>
/// <exception cref="BusinessException"></exception>
public async Task UpdatesPartitialAsnyc(IList<TEntity> datas, Expression<Func<TEntity, dynamic>> express)
{
var indexName = GetIndexName();
await UpdatesPartitialAsnyc(indexName, datas, express);
}
/// <summary>
/// 批量局部更新指定索引
/// </summary>
/// <param name="indexName">索引名称</param>
/// <param name="datas"></param>
/// <param name="express"></param>
/// <returns></returns>
/// <exception cref="BusinessException"></exception>
public async Task UpdatesPartitialAsnyc(string indexName, IList<TEntity> datas, Expression<Func<TEntity, dynamic>> express)
{
var updateDescriptor = new BulkDescriptor();
var columns = Metas(express);
foreach (var data in datas)
{
var updateData = ToPaititailUpdateData(data, columns);
updateDescriptor.Update<IDictionary<string, object>>(i => i
.Index(indexName)
.Id(data.Id)
.Doc(updateData)
.DocAsUpsert(true));
}
var response = await _elasticClient.BulkAsync(updateDescriptor);
if (!response.IsValid)
{
throw new BusinessException($"{indexName}批量局部更新失败。");
}
}局部更新
/// <summary>
/// 批量局部更新
/// </summary>
/// <param name="datas"></param>
/// <param name="express"></param>
/// <returns></returns>
/// <exception cref="BusinessException"></exception>
public async Task UpdatesPartitialAsnyc(IList<TEntity> datas, Expression<Func<TEntity, dynamic>> express)
{
var indexName = GetIndexName();
await UpdatesPartitialAsnyc(indexName, datas, express);
}
/// <summary>
/// 批量局部更新指定索引
/// </summary>
/// <param name="indexName">索引名称</param>
/// <param name="datas"></param>
/// <param name="express"></param>
/// <returns></returns>
/// <exception cref="BusinessException"></exception>
public async Task UpdatesPartitialAsnyc(string indexName, IList<TEntity> datas, Expression<Func<TEntity, dynamic>> express)
{
var updateDescriptor = new BulkDescriptor();
var columns = Metas(express);
foreach (var data in datas)
{
var updateData = ToPaititailUpdateData(data, columns);
updateDescriptor.Update<IDictionary<string, object>>(i => i
.Index(indexName)
.Id(data.Id)
.Doc(updateData)
.DocAsUpsert(true));
}
var response = await _elasticClient.BulkAsync(updateDescriptor);
if (!response.IsValid)
{
throw new BusinessException($"{indexName}批量局部更新失败。");
}
}删除索引对象
单个删除索引对象
/// <summary>
/// 单个删除
/// </summary>
/// <param name="id"></param>
/// <returns></returns>
/// <exception cref="BusinessException"></exception>
public async Task DeleteAsync(string id)
{
var indexName = GetIndexName();
await DeleteAsync(indexName, id);
}
/// <summary>
/// 单个删除
/// </summary>
/// <param name="indexName">索引名称</param>
/// <param name="id">数据id</param>
/// <returns></returns>
/// <exception cref="BusinessException"></exception>
public async Task DeleteAsync(string indexName, string id)
{
if (string.IsNullOrWhiteSpace(id))
{
return;
}
var response = await _elasticClient.DeleteAsync<TEntity>(id, i => i.Index(indexName));
if (!response.IsValid)
{
throw new BusinessException($"{indexName}删除失败。");
}
}批量删除索引对象
/// <summary>
/// 批量删除
/// </summary>
/// <param name="ids"></param>
/// <returns></returns>
/// <exception cref="BusinessException"></exception>
public async Task DeletesAsync(IList<string> ids)
{
var indexName = GetIndexName();
await DeletesAsync(indexName, ids);
}
/// <summary>
/// 批量删除
/// </summary>
/// <param name="indexName">索引名称</param>
/// <param name="ids"></param>
/// <returns></returns>
/// <exception cref="BusinessException"></exception>
public async Task DeletesAsync(string indexName, IList<string> ids)
{
if (ids.IsNullOrEmpty())
{
return;
}
var bulkQuest = new BulkRequest()
{
Operations = new List<IBulkOperation>()
};
foreach (var id in ids)
{
bulkQuest.Operations.Add(new BulkDeleteOperation<TEntity>(id));
}
var response = await _elasticClient.BulkAsync(bulkQuest);
if (!response.IsValid)
{
throw new BusinessException($"{indexName}批量删除失败。");
}
}根据Id获取索引对象
单个Id获取索引对象
/// <summary>
/// 根据Id获取对象
/// </summary>
/// <param name="id"></param>
/// <returns></returns>
public async Task<TEntity> GetByIdAsync(string id)
{
var indexName = GetIndexName();
return await GetByIdAsync(indexName, id);
}
/// <summary>
/// 根据Id获取对象
/// </summary>
/// <param name="indexName">索引名称</param>
/// <param name="id"></param>
/// <returns></returns>
public async Task<TEntity> GetByIdAsync(string indexName, string id)
{
if (string.IsNullOrWhiteSpace(id))
{
return null;
}
var response = await _elasticClient.GetAsync<TEntity>(id, x => x.Index(indexName));
return response.Source;
}根据Id批量获取索引对象
/// <summary>
/// 根据Id集合批量获取数据集合
/// </summary>
/// <param name="ids"></param>
/// <returns></returns>
public async Task<IList<TEntity>> GetByIdsAsync(IList<string> ids)
{
var indexName = GetIndexName();
return await GetByIdsAsync(indexName, ids);
}
/// <summary>
/// 根据Id集合批量获取数据集合
/// </summary>
/// <param name="indexName">索引名称</param>
/// <param name="ids"></param>
/// <returns></returns>
public async Task<IList<TEntity>> GetByIdsAsync(string indexName, IList<string> ids)
{
if (ids.IsNullOrEmpty())
{
return new List<TEntity>();
}
var response = await _elasticClient.SearchAsync<TEntity>(x => x.Query(o => o.Ids(m => m.Values(ids))).Index(indexName));
return response.Documents.ToList();
}普通分页
/// <summary>
/// 普通分页搜索
/// </summary>
/// <param name="condition"></param>
/// <returns></returns>
public async Task<PagedResult<TEntity>> SearchPageAsync(BaseCondition condition)
{
var indexName = GetIndexName();
return await SearchPageAsync(indexName, condition);
}
/// <summary>
/// 普通分页搜索
/// </summary>
/// <param name="indexName">索引名称</param>
/// <param name="condition"></param>
/// <returns></returns>
public async Task<PagedResult<TEntity>> SearchPageAsync(string indexName, BaseCondition condition)
{
var query = new QueryContainerDescriptor<TEntity>();
foreach (var data in condition.SearchFields)
{
query.Term(data.Key, data.Value);
}
ISearchRequest searchSelector(SearchDescriptor<TEntity> s) => s.Index(indexName)
.Query(x => query)
.From((condition.Page - 1) * condition.PageSize)
.Size(condition.PageSize);
var response = await _elasticClient.SearchAsync<TEntity>(searchSelector);
ICountRequest countQuery(CountDescriptor<TEntity> s) => s.Index(indexName)
.Query(x => query);
var countResponse = await _elasticClient.CountAsync<TEntity>(countQuery);
return new PagedResult<TEntity>
{
Page = condition.Page,
PageSize = condition.PageSize,
Total = countResponse.Count,
Rows = response.Documents.ToList()
};
}滚动查询
/// <summary>
/// 滚动查询
/// </summary>
/// <param name="condition"></param>
/// <returns></returns>
public async Task<List<TEntity>> ScrollSearchAsync(BaseCondition condition)
{
var indexName = GetIndexName();
return await ScrollSearchAsync(indexName, condition);
}
/// <summary>
/// 滚动查询
/// </summary>
/// <param name="indexName">索引名称</param>
/// <param name="condition"></param>
/// <returns></returns>
public async Task<List<TEntity>> ScrollSearchAsync(string indexName, BaseCondition condition)
{
var query = new QueryContainerDescriptor<TEntity>();
foreach (var data in condition.SearchFields)
{
query.Term(data.Key, data.Value);
}
ISearchRequest searchSelector(SearchDescriptor<TEntity> s) => s.Index(indexName)
.Query(x => query)
.Size(condition.PageSize)
.Scroll(Time.MinusOne);
var response = await _elasticClient.SearchAsync<TEntity>(searchSelector);
var returnDatas = new List<TEntity>();
returnDatas.AddRange(response.Documents);
string scrollId = response.ScrollId;
while (true)
{
var scrollResponse = await _elasticClient.ScrollAsync<TEntity>("1m", scrollId);
if (scrollResponse.Documents.IsNullOrEmpty())
{
break;
}
returnDatas.AddRange(scrollResponse.Documents);
}
return returnDatas;
}其它使用到的私有方法
/// <summary>
/// 获取MetaName
/// </summary>
/// <param name="express"></param>
/// <param name="containPre"></param>
/// <returns></returns>
/// <exception cref="ArgumentNullException"></exception>
/// <exception cref="ArgumentException"></exception>
private string[] Metas(Expression<Func<TEntity, dynamic>> express)
{
if (express == null)
{
throw new ArgumentNullException(nameof(express));
}
if (express.Body.NodeType == ExpressionType.MemberAccess)
{
return new[] { (express.Body as MemberExpression).Member.GetMetaName<TEntity>() };
}
if (express.Body.NodeType == ExpressionType.New)
{
NewExpression newExpression = express.Body as NewExpression;
if (newExpression == null || newExpression.Arguments.Count == 0)
{
throw new ArgumentException($"请为类型 {typeof(TEntity).FullName} 的指定一个字段（Field）或属性（Property）的集合作为 Lambda 的主体（Body）。");
}
string[] array = new string[newExpression.Arguments.Count];
for (int i = 0; i < array.Length; i++)
{
array[i] = (newExpression.Arguments[i] as MemberExpression).Member.GetMetaName<TEntity>();
}
return array;
}
if (express.Body.NodeType == ExpressionType.Convert)
{
MemberExpression memberExpression2;
if ((memberExpression2 = (express.Body as UnaryExpression)?.Operand as MemberExpression) == null)
{
throw new ArgumentException($"请为类型 {typeof(TEntity).FullName} 的指定一个字段（Field）或属性（Property）的集合作为 Lambda 的主体（Body）。");
}
return new[] { memberExpression2.Member.GetMetaName<TEntity>() };
}
throw new ArgumentException($"请为类型{typeof(TEntity).FullName}的指定一个字段（Field）或属性（Property）的集合作为 Lambda 的主体（Body）。");
}
/// <summary>
/// 转换成局部更新对象
/// </summary>
/// <param name="entity"></param>
/// <param name="updateColumns"></param>
/// <returns></returns>
private Dictionary<string, object> ToPaititailUpdateData(TEntity entity, string[] updateColumns)
{
var properties = entity.GetType().GetProperties();
var updateDatas = new Dictionary<string, object>();
foreach (var property in properties)
{
if (updateColumns.Any(col => col.ToLower() == property.Name.ToLower()))
{
var newName = property.Name.ToLowerCamel();
updateDatas[newName] = property.GetValue(entity);
}
}
return updateDatas;
}全部代码
using JuCheap.Core.Exceptions;
using JuCheap.Core.Extentions;
using JuCheap.Entity;
using Microsoft.Extensions.Logging;
using Nest;
using System.Linq.Expressions;
using System.Reflection;
using System.Security.Cryptography;
using System.Text.Json;
namespace JuCheap.Core.Elasticsearch
{
public class ElasticsearchBaseProvider<TEntity> where TEntity : class, IBaseEntity, new()
{
private readonly IElasticClient _elasticClient;
private readonly ILogger<ElasticsearchBaseProvider<TEntity>> _logger;
public ElasticsearchBaseProvider(IElasticClient elasticClient
, ILogger<ElasticsearchBaseProvider<TEntity>> logger)
{
_elasticClient = elasticClient;
_logger = logger;
}
private string GetIndexName()
{
var indexName = typeof(TEntity).GetCustomAttribute<ElasticsearchTypeAttribute>()?.RelationName ?? string.Empty;
if (indexName.IsBlank())
{
throw new ArgumentNullException($"{nameof(TEntity)}没有配置MetaIndex属性。");
}
return indexName;
}
/// <summary>
/// 创建索引
/// </summary>
/// <returns></returns>
/// <exception cref="BusinessException"></exception>
public async Task CreateIndexAsync()
{
var indexName = GetIndexName();
var existResponse = await _elasticClient.Indices.ExistsAsync(indexName);
if (existResponse.Exists)
{
return;
}
//创建索引，自动转换类型，order为索引名称
var response = await _elasticClient.Indices.CreateAsync(indexName, c => c.Map<TEntity>(m => m.AutoMap()));
if (!response.IsValid)
{
throw new BusinessException($"{indexName}索引创建失败:{response.DebugInformation}。");
}
}
/// <summary>
/// 创建索引模板
/// </summary>
/// <param name="templateName">索引模板编号名称</param>
/// <param name="templateAlias">别名</param>
/// <param name="templatePattern">模板通配符表达式</param>
/// <returns></returns>
public async Task CreateTemplateAsync(string templateName, string templateAlias, string templatePattern)
{
var existResponse = await _elasticClient.Indices.TemplateV2ExistsAsync(templateName);
if (existResponse.Exists)
{
return;
}
ITemplate selectorTemplate(TemplateDescriptor x) => x.Aliases(a => a.Alias(templateAlias)).Mappings<TEntity>(m => m.AutoMap());
IPutIndexTemplateV2Request selector(PutIndexTemplateV2Descriptor x) => x.IndexPatterns(templatePattern).Template(selectorTemplate);
var templateResponse = await _elasticClient.Indices.PutTemplateV2Async(templateName, selector);
if (!templateResponse.IsValid)
{
throw new BusinessException($"{templateName}索引模板添加失败：{templateResponse.DebugInformation}。");
}
}
/// <summary>
/// 删除索引模板
/// </summary>
/// <param name="templateName">模板名称编码</param>
/// <returns></returns>
/// <exception cref="BusinessException"></exception>
public async Task DeleteTemplateAsync(string templateName)
{
var existResponse = await _elasticClient.Indices.TemplateV2ExistsAsync(templateName);
if (!existResponse.Exists)
{
return;
}
var response = await _elasticClient.Indices.DeleteTemplateV2Async(templateName);
if (!response.IsValid)
{
throw new BusinessException($"{templateName}索引模板删除失败：{response.DebugInformation}。");
}
}
/// <summary>
/// 单个对象添加
/// </summary>
/// <param name="data">数据对象</param>
/// <returns></returns>
/// <exception cref="BusinessException"></exception>
public async Task AddAsync(TEntity data)
{
var indexName = GetIndexName();
await AddAsync(indexName, data);
}
/// <summary>
/// 添加单个对象到指定索引
/// </summary>
/// <param name="indexName">索引名称</param>
/// <param name="data">数据对象</param>
/// <returns></returns>
/// <exception cref="BusinessException"></exception>
public async Task AddAsync(string indexName, TEntity data)
{
var response = await _elasticClient.IndexAsync(data, x => x.Index(indexName));
if (!response.IsValid)
{
throw new BusinessException($"{indexName}添加失败。");
}
}
/// <summary>
/// 批量添加
/// </summary>
/// <param name="datas"></param>
/// <returns></returns>
/// <exception cref="BusinessException"></exception>
public async Task AddsAsync(IList<TEntity> datas)
{
var indexName = GetIndexName();
await AddsAsync(indexName, datas);
}
/// <summary>
/// 批量添加对象集合到指定索引
/// </summary>
/// <param name="indexName">索引名称</param>
/// <param name="datas"></param>
/// <returns></returns>
/// <exception cref="BusinessException"></exception>
public async Task AddsAsync(string indexName, IList<TEntity> datas)
{
if (datas.IsNullOrEmpty())
{
return;
}
//分页推送-防止数据量过大接口超时
int page = 1;
int pageSize = 1000;
int totalPage = (datas.Count + pageSize - 1) / pageSize;
while (page <= totalPage)
{
try
{
//本次推送的数据
var currentPushDatas = datas.Skip((page - 1) * pageSize).Take(pageSize).ToList();
var response = await _elasticClient.BulkAsync(b => b.Index(indexName).IndexMany(currentPushDatas));
if (!response.IsValid)
{
_logger?.LogInformation($"{indexName}批量添加失败：{JsonSerializer.Serialize(response)}");
}
}
catch (Exception ex)
{
//记录日志
_logger?.LogError(ex, $"{indexName}批量添加异常page={page}：{ex}");
}
//下一页
page++;
}
}
public void BulkAll(IList<TEntity> datas)
{
var index = GetIndexName();
BulkAll(index, datas);
}
public void BulkAll(string indexName, IList<TEntity> datas)
{
if (datas.IsNullOrEmpty())
{
return;
}
var bulkAllObserver = _elasticClient.BulkAll(datas, b => b.Index(indexName)
.BackOffTime("30s")//集群繁忙，报429错误码的时候，等待多久进行重试
.BackOffRetries(2) //重试次数
.RefreshOnCompleted() //
.MaxDegreeOfParallelism(Environment.ProcessorCount)
.Size(1000) //每次 bulk 请求时的文档数量。
).Wait(TimeSpan.FromSeconds(30), next =>
{
//执行索引并等待 1min，BulkAll 请求虽然时异步的，但是是一个阻塞操作。
});
}
/// <summary>
/// 单个更新
/// </summary>
/// <param name="data"></param>
/// <returns></returns>
/// <exception cref="BusinessException"></exception>
public async Task UpdateAsync(TEntity data)
{
var indexName = GetIndexName();
await UpdateAsync(indexName, data);
}
/// <summary>
/// 更新指定索引的对象
/// </summary>
/// <param name="indexName">索引名称</param>
/// <param name="data"></param>
/// <returns></returns>
/// <exception cref="BusinessException"></exception>
public async Task UpdateAsync(string indexName, TEntity data)
{
DocumentPath<TEntity> documentPath = new(data.Id);
var response = await _elasticClient.UpdateAsync(documentPath, (p) => p.Doc(data).Index(indexName));
if (!response.IsValid)
{
throw new BusinessException($"{indexName}更新失败。");
}
}
/// <summary>
/// 批量更新
/// </summary>
/// <param name="datas"></param>
/// <returns></returns>
/// <exception cref="BusinessException"></exception>
public async Task UpdatesAsnyc(IList<TEntity> datas)
{
var indexName = GetIndexName();
await UpdatesAsnyc(indexName, datas);
}
/// <summary>
/// 批量更新指定索引的对象
/// </summary>
/// <param name="indexName">索引名称</param>
/// <param name="datas"></param>
/// <returns></returns>
/// <exception cref="BusinessException"></exception>
public async Task UpdatesAsnyc(string indexName, IList<TEntity> datas)
{
var updateDescriptor = new BulkDescriptor();
foreach (var data in datas)
{
updateDescriptor.Update<TEntity>(i => i
.Index(indexName)
.Id(data.Id)
.Doc(data)
.DocAsUpsert(true));
}
var response = await _elasticClient.BulkAsync(updateDescriptor);
if (!response.IsValid)
{
throw new BusinessException($"{indexName}批量更新失败。");
}
}
/// <summary>
/// 批量局部更新
/// </summary>
/// <param name="datas"></param>
/// <param name="express"></param>
/// <returns></returns>
/// <exception cref="BusinessException"></exception>
public async Task UpdatesPartitialAsnyc(IList<TEntity> datas, Expression<Func<TEntity, dynamic>> express)
{
var indexName = GetIndexName();
await UpdatesPartitialAsnyc(indexName, datas, express);
}
/// <summary>
/// 批量局部更新指定索引
/// </summary>
/// <param name="indexName">索引名称</param>
/// <param name="datas"></param>
/// <param name="express"></param>
/// <returns></returns>
/// <exception cref="BusinessException"></exception>
public async Task UpdatesPartitialAsnyc(string indexName, IList<TEntity> datas, Expression<Func<TEntity, dynamic>> express)
{
var updateDescriptor = new BulkDescriptor();
var columns = Metas(express);
foreach (var data in datas)
{
var updateData = ToPaititailUpdateData(data, columns);
updateDescriptor.Update<IDictionary<string, object>>(i => i
.Index(indexName)
.Id(data.Id)
.Doc(updateData)
.DocAsUpsert(true));
}
var response = await _elasticClient.BulkAsync(updateDescriptor);
if (!response.IsValid)
{
throw new BusinessException($"{indexName}批量局部更新失败。");
}
}
/// <summary>
/// 单个删除
/// </summary>
/// <param name="id"></param>
/// <returns></returns>
/// <exception cref="BusinessException"></exception>
public async Task DeleteAsync(string id)
{
var indexName = GetIndexName();
await DeleteAsync(indexName, id);
}
/// <summary>
/// 单个删除
/// </summary>
/// <param name="indexName">索引名称</param>
/// <param name="id">数据id</param>
/// <returns></returns>
/// <exception cref="BusinessException"></exception>
public async Task DeleteAsync(string indexName, string id)
{
if (string.IsNullOrWhiteSpace(id))
{
return;
}
var response = await _elasticClient.DeleteAsync<TEntity>(id, i => i.Index(indexName));
if (!response.IsValid)
{
throw new BusinessException($"{indexName}删除失败。");
}
}
/// <summary>
/// 批量删除
/// </summary>
/// <param name="ids"></param>
/// <returns></returns>
/// <exception cref="BusinessException"></exception>
public async Task DeletesAsync(IList<string> ids)
{
var indexName = GetIndexName();
await DeletesAsync(indexName, ids);
}
/// <summary>
/// 批量删除
/// </summary>
/// <param name="indexName">索引名称</param>
/// <param name="ids"></param>
/// <returns></returns>
/// <exception cref="BusinessException"></exception>
public async Task DeletesAsync(string indexName, IList<string> ids)
{
if (ids.IsNullOrEmpty())
{
return;
}
var bulkQuest = new BulkRequest()
{
Operations = new List<IBulkOperation>()
};
foreach (var id in ids)
{
bulkQuest.Operations.Add(new BulkDeleteOperation<TEntity>(id));
}
var response = await _elasticClient.BulkAsync(bulkQuest);
if (!response.IsValid)
{
throw new BusinessException($"{indexName}批量删除失败。");
}
}
/// <summary>
/// 根据Id获取对象
/// </summary>
/// <param name="id"></param>
/// <returns></returns>
public async Task<TEntity> GetByIdAsync(string id)
{
var indexName = GetIndexName();
return await GetByIdAsync(indexName, id);
}
/// <summary>
/// 根据Id获取对象
/// </summary>
/// <param name="indexName">索引名称</param>
/// <param name="id"></param>
/// <returns></returns>
public async Task<TEntity> GetByIdAsync(string indexName, string id)
{
if (string.IsNullOrWhiteSpace(id))
{
return null;
}
var response = await _elasticClient.GetAsync<TEntity>(id, x => x.Index(indexName));
return response.Source;
}
/// <summary>
/// 根据Id集合批量获取数据集合
/// </summary>
/// <param name="ids"></param>
/// <returns></returns>
public async Task<IList<TEntity>> GetByIdsAsync(IList<string> ids)
{
var indexName = GetIndexName();
return await GetByIdsAsync(indexName, ids);
}
/// <summary>
/// 根据Id集合批量获取数据集合
/// </summary>
/// <param name="indexName">索引名称</param>
/// <param name="ids"></param>
/// <returns></returns>
public async Task<IList<TEntity>> GetByIdsAsync(string indexName, IList<string> ids)
{
if (ids.IsNullOrEmpty())
{
return new List<TEntity>();
}
var response = await _elasticClient.SearchAsync<TEntity>(x => x.Query(o => o.Ids(m => m.Values(ids))).Index(indexName));
return response.Documents.ToList();
}
/// <summary>
/// 普通分页搜索
/// </summary>
/// <param name="condition"></param>
/// <returns></returns>
public async Task<PagedResult<TEntity>> SearchPageAsync(BaseCondition condition)
{
var indexName = GetIndexName();
return await SearchPageAsync(indexName, condition);
}
/// <summary>
/// 普通分页搜索
/// </summary>
/// <param name="indexName">索引名称</param>
/// <param name="condition"></param>
/// <returns></returns>
public async Task<PagedResult<TEntity>> SearchPageAsync(string indexName, BaseCondition condition)
{
var query = new QueryContainerDescriptor<TEntity>();
foreach (var data in condition.SearchFields)
{
query.Term(data.Key, data.Value);
}
ISearchRequest searchSelector(SearchDescriptor<TEntity> s) => s.Index(indexName)
.Query(x => query)
.From((condition.Page - 1) * condition.PageSize)
.Size(condition.PageSize);
var response = await _elasticClient.SearchAsync<TEntity>(searchSelector);
ICountRequest countQuery(CountDescriptor<TEntity> s) => s.Index(indexName)
.Query(x => query);
var countResponse = await _elasticClient.CountAsync<TEntity>(countQuery);
return new PagedResult<TEntity>
{
Page = condition.Page,
PageSize = condition.PageSize,
Total = countResponse.Count,
Rows = response.Documents.ToList()
};
}
/// <summary>
/// 滚动查询
/// </summary>
/// <param name="condition"></param>
/// <returns></returns>
public async Task<List<TEntity>> ScrollSearchAsync(BaseCondition condition)
{
var indexName = GetIndexName();
return await ScrollSearchAsync(indexName, condition);
}
/// <summary>
/// 滚动查询
/// </summary>
/// <param name="indexName">索引名称</param>
/// <param name="condition"></param>
/// <returns></returns>
public async Task<List<TEntity>> ScrollSearchAsync(string indexName, BaseCondition condition)
{
var query = new QueryContainerDescriptor<TEntity>();
foreach (var data in condition.SearchFields)
{
query.Term(data.Key, data.Value);
}
ISearchRequest searchSelector(SearchDescriptor<TEntity> s) => s.Index(indexName)
.Query(x => query)
.Size(condition.PageSize)
.Scroll(Time.MinusOne);
var response = await _elasticClient.SearchAsync<TEntity>(searchSelector);
var returnDatas = new List<TEntity>();
returnDatas.AddRange(response.Documents);
string scrollId = response.ScrollId;
while (true)
{
var scrollResponse = await _elasticClient.ScrollAsync<TEntity>("1m", scrollId);
if (scrollResponse.Documents.IsNullOrEmpty())
{
break;
}
returnDatas.AddRange(scrollResponse.Documents);
}
return returnDatas;
}
/// <summary>
/// 获取MetaName
/// </summary>
/// <param name="express"></param>
/// <param name="containPre"></param>
/// <returns></returns>
/// <exception cref="ArgumentNullException"></exception>
/// <exception cref="ArgumentException"></exception>
private string[] Metas(Expression<Func<TEntity, dynamic>> express)
{
if (express == null)
{
throw new ArgumentNullException(nameof(express));
}
if (express.Body.NodeType == ExpressionType.MemberAccess)
{
return new[] { (express.Body as MemberExpression).Member.GetMetaName<TEntity>() };
}
if (express.Body.NodeType == ExpressionType.New)
{
NewExpression newExpression = express.Body as NewExpression;
if (newExpression == null || newExpression.Arguments.Count == 0)
{
throw new ArgumentException($"请为类型 {typeof(TEntity).FullName} 的指定一个字段（Field）或属性（Property）的集合作为 Lambda 的主体（Body）。");
}
string[] array = new string[newExpression.Arguments.Count];
for (int i = 0; i < array.Length; i++)
{
array[i] = (newExpression.Arguments[i] as MemberExpression).Member.GetMetaName<TEntity>();
}
return array;
}
if (express.Body.NodeType == ExpressionType.Convert)
{
MemberExpression memberExpression2;
if ((memberExpression2 = (express.Body as UnaryExpression)?.Operand as MemberExpression) == null)
{
throw new ArgumentException($"请为类型 {typeof(TEntity).FullName} 的指定一个字段（Field）或属性（Property）的集合作为 Lambda 的主体（Body）。");
}
return new[] { memberExpression2.Member.GetMetaName<TEntity>() };
}
throw new ArgumentException($"请为类型{typeof(TEntity).FullName}的指定一个字段（Field）或属性（Property）的集合作为 Lambda 的主体（Body）。");
}
/// <summary>
/// 转换成局部更新对象
/// </summary>
/// <param name="entity"></param>
/// <param name="updateColumns"></param>
/// <returns></returns>
private Dictionary<string, object> ToPaititailUpdateData(TEntity entity, string[] updateColumns)
{
var properties = entity.GetType().GetProperties();
var updateDatas = new Dictionary<string, object>();
foreach (var property in properties)
{
if (updateColumns.Any(col => col.ToLower() == property.Name.ToLower()))
{
var newName = property.Name.ToLowerCamel();
updateDatas[newName] = property.GetValue(entity);
}
}
return updateDatas;
}
}
}BaseCondition.cs
using JuCheap.Core.Exceptions;
using System.Text.Json.Serialization;
namespace JuCheap.Core
{
/// <summary>
/// 分页查询条件基础类
/// </summary>
public class BaseCondition
{
public BaseCondition()
{
SearchFields = new Dictionary<string, ConditionItem>();
}
/// <summary>
/// 升序排列常量
/// </summary>
public const string Asc = "ascending";
/// <summary>
/// 降序排列常量
/// </summary>
public const string Desc = "descending";
/// <summary>
/// 当前页码
/// </summary>
public int Page { get; set; }
/// <summary>
/// 每页显示的数据条数
/// </summary>
public int PageSize { get; set; }
/// <summary>
/// 排序字段
/// </summary>
[JsonPropertyName("prop")]
public string? SortField { get; set; }
/// <summary>
/// 排序方式
/// </summary>
[JsonPropertyName("order")]
public string? SortType { get; set; }
/// <summary>
/// 公司Id
/// </summary>
public string? CompanyId { get; set; }
/// <summary>
/// 搜索关键字
/// </summary>
public string? Keyword { get; set; }
/// <summary>
/// 高级搜索条件
/// </summary>
public IDictionary<string, ConditionItem> SearchFields { get; set; }
/// <summary>
/// 转换成数据库排序规则
/// </summary>
/// <returns></returns>
public virtual string ToSort()
{
string direction = SortType == Asc ? "asc" : "desc";
return $" {SortField} {direction} ";
}
/// <summary>
/// 自定义参数校验
/// </summary>
protected virtual void CustomValidate() { }
/// <summary>
/// 参数校验
/// </summary>
/// <exception cref="BusinessException">校验不通过抛出BusinessException异常</exception>
public void Validate()
{
if (Page <= 0)
throw new BusinessException("当前页码必须大于0 !");
if (PageSize < 1)
throw new BusinessException("每页显示条数必须大于0 !");
if (PageSize > 1000)
throw new BusinessException("每页显示数据最大条数不能超过1千条");
CustomValidate();
}
}
/// <summary>
/// 搜索条件模型
/// </summary>
public class ConditionItem
{
public ConditionItem() { }
public ConditionItem(string value)
{
Value = value;
}
/// <summary>
/// 搜索值
/// </summary>
public string Value { get; set; }
/// <summary>
/// 匹配类型(比如：like,eq,gt,gte,lt,lte等)
/// </summary>
public string Type { get; set; }
/// <summary>
/// 支持区间查询
/// </summary>
public string EndValue { get; set; }
/// <summary>
/// 控件类型
/// </summary>
public byte? ControlType { get; set; }
/// <summary>
/// 显示名称
/// </summary>
public string DisplayName { get; set; }
}
}
GetMetaName的扩展方法
public static class MetaNameExtention
{
public static string GetMetaName<TEntity>(this MemberInfo member)
{
return typeof(TEntity).GetCustomAttribute<ElasticsearchPropertyAttributeBase>()?.Name ?? member.Name;
}
}
扫一扫
1918
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
