目的:
目前有一家供应商数据有110w条数据,需要每天更新全部数据的Createtime创建时间(创建时间到秒不能全部都是一样的),如果使用es游标或者使用分页查询出来再更新,方案是行不通的,这时想到使用Script脚本来批量更新(半小时内全部更新完成),代码如下所示:
/// <summary>
/// 定时作业调用,设置xx电子物料创建时间每日更新
/// </summary>
/// <returns></returns>
private void SetCreateTimeBySiTime()
{
int record = 0;
short flag = (short)QuoteEnum.SupplierFlag.SiTime;
//short flag = (short)QuoteEnum.SupplierFlag.Mobius;
//当月的索引中取数据
var count = _supplierQuoteMonthEsClient.Count(c => c.Term(t => t.SupplierFlag, flag));
if (count.Count == 0)
{
LogUtil.NLog.Error("未查询到xx电子当月物料数据,更新时间失败!");
return ;
}
while (true)
{
//昨天的最大时间
var maxDT = DateTime.Parse(DateTime.Now.AddDays(-1).ToString("yyyy-MM-dd") + " 23:59:59");
var maxDTStrmp = TimeStampUtil.GenerateTimeStampBySeconds(maxDT);
var count2 = _supplierQuoteMonthEsClient
.Count(c => c.Bool(b => b.Must(m =>
m.Term(t => t.SupplierFlag, flag) &&
m.LongRange(t => t.Field(f => f.CreateTimeStamp).LessThanOrEquals(maxDTStrmp)))));
if (count2.Count == 0)
{
break;
}
//当前时间
var currentDT = DateTime.Now;
var currentDTStrmp = TimeStampUtil.GenerateTimeStampBySeconds(currentDT);
var indexName = EsIndexExtend.Quote.SUPPLIER_QUOTE_MONTH;
var updateByQuery = new UpdateByQueryRequest<SupplierQuoteEsModel>(indexName);
//查询昨天的物料数据
updateByQuery.Query = new QueryContainer(new BoolQuery()
{
Must = new List<QueryContainer>()
{
new TermQuery() { Field = new Field("SupplierFlag"),Value = flag },
new LongRangeQuery() { Field = new Field("CreateTimeStamp"), LessThan = maxDTStrmp },
}
});
updateByQuery.Script = new ScriptDescriptor()
.Source($"ctx._source.CreateTimeStamp = params.CreateTimeStamp;ctx._source.CreateTime = params.CreateTime;")
.Params(new Dictionary<string, object>() {
{ "CreateTimeStamp",currentDTStrmp },
{ "CreateTime",currentDT}
});
updateByQuery.WaitForCompletion = true;
//(可选,整数)支持操作的滚动请求的大小。默认为 1000。
updateByQuery.ScrollSize = 3000;
//(可选,整数)要处理的最大文档数。默认为所有文档。当设置为小于或等于scroll_size 的值时,将不会使用滚动来检索操作的结果。
updateByQuery.MaximumDocuments = 3000;
//版本冲突时继续,不返回409
updateByQuery.Conflicts = Conflicts.Proceed;
var res = _supplierQuoteMonthEsClient.Client.UpdateByQuery(updateByQuery);
if(!res.IsValid)
LogUtil.NLog.Error("今天xx电子物料数据更新Es失败,"+res.DebugInformation);
Thread.Sleep(5000);
record++;
}
LogUtil.NLog.Info($"今天xx电子物料更新时间共:{record}批次!");
}
}
对应dsl语句写法
get /supplier_quote-2024.02/_update_by_query?wait_for_completion=false
{
"query": {
"term": {
"SupplierFlag": {
"value": 49
}
}
},
"script": {
"source": "ctx._source['CreateTimeStamp']=1708920000;ctx._source['CreateTime']='2024-02-26T12:00:00'"
}
}