下载地址:https://download.csdn.net/download/yunfengfengfeng/12875501
安装时选择C:\python C盘新建python文件夹
cmd执行:
path=%path%;C:\python
idea里面配置python参考:
我的第一个python程序(es插入数据):
#
#
# 每日执行job
# version: 1.0.0
#
import time
import uuid
import yaml
from elasticsearch import Elasticsearch
from elasticsearch import helpers
import logging
LOGGER = logging.getLogger()
timeStamp = int(time.time())
timeStamp = timeStamp + 8 * 3600
timeArray = time.localtime(timeStamp)
otherStyleTime = time.strftime("%Y-%m-%d", timeArray)
dateStyle = time.strptime(otherStyleTime, "%Y-%m-%d")
endTimeDate = int(time.mktime(dateStyle)) - 8 * 3600 - 3600 * 24
print(endTimeDate)
config = {};
yamlPath = 'config.yaml'
with open(yamlPath, 'rb') as f:
# yaml文件通过---分节,多个节组合成一个列表
date = yaml.safe_load_all(f)
# salf_load_all方法得到的是一个迭代器,需要使用list()方法转换为列表
config = list(date)[0]
print(config)
es = Elasticsearch([config['es']['dbUrl']], http_auth=(config['es']['dbUser'], config['es']['dbPassword']),
port=int(config['es']['port']));
dataIndex = 0;
def batchInsert(vinCommitList, z, lastFlag=False):
z = z + 1;
if lastFlag == True:
if len(vinCommitList) > 0:
print(z * 5000);
helpers.bulk(es, vinCommitList);
return True;
if len(vinCommitList) >= 5000:
print(z * 5000);
helpers.bulk(es, vinCommitList);
return True;
return False;
indexDate = 0
insertDate = 1600162985539
i = 0;
j = 0;
vinCommitList=[];
while i < 0:
uuidStr =str(uuid.uuid4()).replace('-', '');
dayData = {
"_index": "app_crash_log",
"_type": "app_crash_log",
"_id": uuidStr,
"_source": {
"crashName": "NSInvalidArgumentException",
"crashReason": "*** -[__NSPlaceholderDictionary initWithObjects:forKeys:count:]: attempt to insert nil object from objects[0]",
"platform": {
"source": "Android",
"network": "",
"osModel": "HUAWEI P20",
"appVersion": "4.29.4",
"sysVersion": "10.0.2"
},
"crashStack": "0 CoreFoundation 0x00000001ba445668 E2D6A76B-6879-31A3-8168-DF49F94E17CD + 1222248\n1 libobjc.A.dylib 0x00000001ba167bcc objc_exception_throw + 56\n2 CoreFoundation 0x00000001ba49b280 E2D6A76B-6879-31A3-8168-DF49F94E17CD + 1573504\n3 CoreFoundation 0x00000001ba4a46e8 E2D6A76B-6879-31A3-8168-DF49F94E17CD + 1611496\n4 CoreFoundation 0x00000001ba3330bc E2D6A76B-6879-31A3-8168-DF49F94E17CD + 98492\n5 CoreFoundation 0x00000001ba3250c4 E2D6A76B-6879-31A3-8168-DF49F94E17CD + 41156\n6 KoraApp 0x000000010291c45c KoraApp + 115804\n7 CoreFoundation 0x00000001ba3a1504 E2D6A76B-6879-31A3-8168-DF49F94E17CD + 550148\n8 CoreFoundation 0x00000001ba3a154c E2D6A76B-6879-31A3-8168-DF49F94E17CD + 550220\n9 CoreFoundation 0x00000001ba3a08a4 E2D6A76B-6879-31A3-8168-DF49F94E17CD + 546980\n10 CoreFoundation 0x00000001ba3a057c E2D6A76B-6879-31A3-8168-DF49F94E17CD + 546172\n11 CoreFoundation 0x00000001ba31d2d4 E2D6A76B-6879-31A3-8168-DF49F94E17CD + 8916\n12 CoreFoundation 0x00000001ba39fedc _CFXNotificationPost + 1236\n13 Foundation 0x00000001ba70d974 EEBCBA18-269A-351C-BCE0-D23E156923D4 + 26996\n14 KoraApp 0x000000010291fa78 KoraApp + 129656\n15 KoraApp 0x0000000102ecc164 KoraApp + 6078820\n16 KoraApp 0x0000000102ec1130 KoraApp + 6033712\n17 UIKitCore 0x00000001be0d1ca4 F486088D-724E-3713-B9A5-7C9FE0A817D8 + 6319268\n18 UIKitCore 0x00000001be0db264 F486088D-724E-3713-B9A5-7C9FE0A817D8 + 6357604\n19 UIKitCore 0x00000001be0d7bf0 F486088D-724E-3713-B9A5-7C9FE0A817D8 + 6343664\n20 UIKitCore 0x00000001be0d7168 F486088D-724E-3713-B9A5-7C9FE0A817D8 + 6340968\n21 UIKitCore 0x00000001be0cabd8 F486088D-724E-3713-B9A5-7C9FE0A817D8 + 6290392\n22 UIKitCore 0x00000001be0ca328 F486088D-724E-3713-B9A5-7C9FE0A817D8 + 6288168\n23 UIKitCore 0x00000001be0ca0f4 F486088D-724E-3713-B9A5-7C9FE0A817D8 + 6287604\n24 UIKitCore 0x00000001be535c48 F486088D-724E-3713-B9A5-7C9FE0A817D8 + 10923080\n25 UIKitCore 0x00000001be5118ec F486088D-724E-3713-B9A5-7C9FE0A817D8 + 10774764\n26 KoraApp 0x00000001035a28f8 KoraApp + 13248760\n27 UIKitCore 0x00000001be592970 F486088D-724E-3713-B9A5-7C9FE0A817D8 + 11303280\n28 UIKitCore 0x00000001be5954ec F486088D-724E-3713-B9A5-7C9FE0A817D8 + 11314412\n29 UIKitCore 0x00000001be58d168 F486088D-724E-3713-B9A5-7C9FE0A817D8 + 11280744\n30 CoreFoundation 0x00000001ba3c3ad8 E2D6A76B-6879-31A3-8168-DF49F94E17CD + 690904\n31 CoreFoundation 0x00000001ba3c3a30 E2D6A76B-6879-31A3-8168-DF49F94E17CD + 690736\n32 CoreFoundation 0x00000001ba3c31b8 E2D6A76B-6879-31A3-8168-DF49F94E17CD + 688568\n33 CoreFoundation 0x00000001ba3be1e8 E2D6A76B-6879-31A3-8168-DF49F94E17CD + 668136\n34 CoreFoundation 0x00000001ba3bdba8 CFRunLoopRunSpecific + 424\n35 GraphicsServices 0x00000001c452d344 GSEventRunModal + 160\n36 UIKitCore 0x00000001be4f93e4 UIApplicationMain + 1932\n37 KoraApp 0x0000000102a3bb40 KoraApp + 1293120\n38 libdyld.dylib 0x00000001ba2458f0 0CB11832-70AD-3EC6-A191-AFB85E3BF348 + 6384",
"path": "/mobilelog/2020-9-14-b4d6257a-faea-481d-abc7-545754c68044.crash",
"phoneNo": "13211223344",
"time": "1600072957219",
"timestamp": int(time.time()) * 1000-(600000*i) -86400*1000*dataIndex,
"date":int(time.time()) * 1000-(61185*i) -86400*1000*dataIndex,
"calSum":1
}
}
vinCommitList.append(dayData);
if batchInsert(vinCommitList, j, False):
j = j + 1
vinCommitList = [];
i = i + 1
batchInsert(vinCommitList, j, True)
indexDate = 0
insertDate = 1600162985539
i = 0;
j = 0;
# Success:12,13,14
# NetError: 13,14,15,16,17
# ServerError: 12,13,14
# RequestError: 12,13,14,18 int(time.time()) * 1000-(600000*i) -86400*1000*6,
vinCommitList=[];
while i < 20:
uuidStr = str(uuid.uuid4()).replace('-', '');
dayData = {
"_index": "app_api_request_log",
"_type": "app_api_request_log",
"_id": uuidStr,
"_source": {
"request.url": "http://65.12.18.17/fordcloud-user-service/api/helloWord",
"request.requestTime": int(time.time()) * 1000 - 236-(600000*i),
"request.responseTime": 300,
"request.requestMethod": "POST",
"request.requestHeader": '{"Content - Type": "application / json"}',
"request.requestBody": '{"userName":"13118941894","password":"123456"}',
"request.responseHeader": '{"Content-Type":"application/json","token":"12345678916dsfa13sdf15asd1f"}',
"request.responseBody": '{"userName":"13118941894","realName":"李四","sex":1}',
"request.httpCode": 200,
"request.requestResult": "RequestError",
"request.responseType": "3s",
"platform.appVersion": "1.0.2",
"platform.network": "ChinaNet",
"platform.osModel": "HUAWEI P20",
"platform.source": "ANDROID",
"platform.sysVersion": "9.0.2",
"phoneNum": "13012341234",
"timestamp": 1599894708000+86400000*6,
"calSum": 1
}
}
vinCommitList.append(dayData);
if batchInsert(vinCommitList, j, False):
j = j + 1
vinCommitList = [];
i = i + 1
batchInsert(vinCommitList, j, True)
# # 存在则更新,不存在则插入,已验证
# def handler(event, context):
# return
# "requestTime": 1600162985539,
# "param": '{ "id": 123456, "name": "asdfasdfasdf"}',
# "url": "https://es-cn-v641lkyr0000ds4ny.kibana.elasticsearch.aliyuncs.com:5601/app/kibana#/management/elasticsearch/index_management/indices?_g=()",
# "host": "es-cn-v641lkyr0000ds4ny.kibana.elasticsearch.aliyuncs.com",
# "method": "GET",
# "responseTime": 1600162985539,
# "responseBody": '{ "id": 123456, "name": "asdfasdfasdf"}',
# "description": "success",
# "platform.appVersion": "1.0.1",
# "platform.network": "net",
# "platform.osModel": "IOS",
# "platform.source": "IOS",
# "platform.sysVersion": "1.2.3",
# "phoneNo": "13211223344",
# "timestamp": 1600162985539