2020-10-23

一些朋友
1、python处理es数据

import sys, os, shutil
import traceback
import json
import datetime
import requests

reload(sys)  # reload 才能调用 setdefaultencoding 方法
sys.setdefaultencoding('utf-8')  # 设置 'utf-8'

headers = {'Content-Type': 'application/json'}
data = "SELECT hour from test_pv_di where dt = '2020-09-21' and hour = '99' "
result = requests.get('http://es_lianjie_info/_sql', headers=headers, data=data)
print result.url
print '接口返回code：%s' % result.status_code
print '接口返回结果：%s' % result
print result.content

ret = json.loads(result.content) #获取es上的数据内容
print(ret.get('hits').get('hits')[0].get('_source').get('req_pv')) #获取具体内容

2、python处理kafka消息

import datetime
from kafka import KafkaConsumer
consumer = KafkaConsumer('topic',
                         bootstrap_servers=['10.1.1.1:9092','10.1.1.2:9092','10.1.1.3:9092'],
                         client_id='clientid',
                         group_id='groupid',
                         auto_offset_reset='earliest'
                         )
for message in consumer:
    if not message.offset:
        offset = str(message.offset)
    else:
        offset = "0"
    if not message.key:
        key = str(message.key)
    else:
        key = "NoneKey"
    print(str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')) + '---' + offset + '---' + key + '---' + message.value)

3、python处理MySQL数据

import os
import sys, getopt
import commands
import subprocess
import traceback
from datetime import timedelta,datetime
import urllib2
import os.path
import MySQLdb
import json

host = "mysql_host"
user = "username"
passwd = "passwd"
db = "mysql_dbname"

sql = ("select data_1 from test_table")
con = MySQLdb.connect(host=host,user=user,passwd=passwd,db=db) #连接mysql
cur=con.cursor() 
cur.execute(sql)
rows = cur.fetchall()
cur.close()
con.close()
if len(rows) > 0:
    print (rows[0][0])

4、python处理redis数据

# *** coding:utf8
import sys, os, shutil
import traceback
import json
import requests
import redis
import datetime
reload(sys)  # reload 才能调用 setdefaultencoding 方法
sys.setdefaultencoding('utf-8')  # 设置 'utf-8'
dtm = datetime.datetime.now().strftime("%m%d")
print dtm
now_hour = datetime.datetime.now().strftime("%H")
print now_hour
pv_key = 'pv%s%s00_0' % (dtm, now_hour)  #pv20231400_0
print pv_key
r = redis.StrictRedis(host='redis_host', port=19116, password = 'passwd')
today_pv = r.get(pv_key)
print '%s ,当前pv is : %s' % (pv_key, today_pv)
r.set(pv_key,999999)
r.expire(pv_key, 604800)  #失效时间 一周
r.delete(pv_key)