#coding=utf-8
import urllib,urllib2,cookielib,time,json
import psycopg2,re
print time.strftime("%Y-%m-%d %X")
print '模拟登陆xx平台'
url = "http://xxx.xxx.xxx.xxx"
#设置参数
data = {"__EVENTTARGET":"",
"__EVENTARGUMENT":"",
"__VIEWSTATE":"xxxxx",
"__VIEWSTATEGENERATOR":"xxxx",
"__EVENTVALIDATION":"xxxxx",
"username":"xxxxx",
"password":"xxxxx",
"btnSubmit":""
}
#设置headers请求消息头
headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36',
'Referer': 'http://xxx.xxx.xxx.xxx:xxx'
}
#将post参数消息化成可以让服务器编码的方式
parms_data = urllib.urlencode(data)
#初始化一个CookieJar来处理Cookie
cj=cookielib.CookieJar()
#实例化一个全局opener(包含cookie)
opener=urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
#生成请求对象
request = urllib2.Request(url,parms_data,headers)
#发送请求,获取到cookie并自动存入到cj中
respond = opener.open(request)
#返回fileObject类型,
# html = respond.read()
#返回请求html消息头
# h = respond.info()
print '模拟登陆成功'
print '模拟拉取五分钟数据'
url2 = "http://xxx.xxx.xxx.xxx:xxxx/Channels/Air/AirData/AirRt.aspx"
data2 = {"action": "GetData",
"pointCode": "",
"items": "'PM25','PM10','SO2','NO2','NO','NOX','CO','O3','AP','TP','WS','HU','WD'",
"pointLevel": "",
"pointType": ""
}
# resp = urllib2.urlopen(req) //urllib2的urlopen方法不带验证、cookie
params = urllib.urlencode(data2)
req = urllib2.Request(url2,params,headers)
#带着cookie的请求
resp = opener.open(req)
html = resp.read() #读取fileObject类型
if len(html) == 0:
print '再次拉取五分钟数据'
resp = opener.open(req)
html = resp.read()
#使用json模块的loads方法把json格式字符串转成python对象
list = json.loads(html)
for i in range(1,len(list)):
#使用psycopg2连接postgreSQL数据库(psycopg2要先安装)
conn = psycopg2.connect(database='xxxx',user='xxxx',password='xxxx',host='xxxxx',port='xxxx')
cur = conn.cursor()
li = list[i] #列表[{},{},{}]
#使用psycopg2连接postgreSQL数据库(psycopg2要先安装)
conn = psycopg2.connect(database='ZZAirQueen_development',user='postgres',password='12345678',host='172.168.10.7',port='5432')
cur = conn.cursor()
li = list[i] #列表[{},{},{}]
station_id = (li[1]['value'].split('|',1))[0]
cur.execute("SELECT d_station_id FROM s_station_mns where station_mn = %s;", (station_id,))
d_station_id = (cur.fetchone())[0] #获取到查询到的一个元组
# status = li[2]['value']
# station_name = (li[3]['value'].split('|',1))[0]
data_time = li[4]['value']
#pm2.5数据
pm25 = float((li[5]['value'].split('|',1))[0]) if (li[5]['value'].split('|',1))[0] else 0
#pm10
pm10 = float((li[6]['value'].split('|',1))[0]) if (li[6]['value'].split('|',1))[0] else 0
#so2
so2 = float((li[7]['value'].split('|',1))[0]) if (li[7]['value'].split('|',1))[0] else 0
#no2
no2 = float((li[8]['value'].split('|',1))[0]) if (li[8]['value'].split('|',1))[0] else 0
#no
no = float((li[9]['value'].split('|',1))[0]) if (li[9]['value'].split('|',1))[0] else 0
#nox
nox = float((li[10]['value'].split('|',1))[0]) if (li[10]['value'].split('|',1))[0] else 0
#co
co = float((li[11]['value'].split('|',1))[0]) if (li[11]['value'].split('|',1))[0] else 0
#o3
o3 = float((li[12]['value'].split('|',1))[0]) if (li[12]['value'].split('|',1))[0] else 0
#气压
barometer = float((li[13]['value'].split('|',1))[0]) if (li[13]['value'].split('|',1))[0] else 0 #气压
#气温
temperature = float((li[14]['value'].split('|',1))[0]) if (li[14]['value'].split('|',1))[0] else 0 #温度
#风速
windspeed = float((li[15]['value'].split('|',1))[0]) if (li[15]['value'].split('|',1))[0] else 0 #风速
#湿度
humidity = float((li[16]['value'].split('|',1))[0]) if (li[16]['value'].split('|',1))[0] else 0#湿度
#风向
direction = li[17]['value'].split('|',1)[0]
dir = re.search(r"\d*\.\d*",direction).span()
winddirection = float(direction[dir[0]:dir[1]])
#获取监测因子
monitor = ['PM2.5','PM10','SO2','NO2','NO','NOX','CO','O3','气压','气温','风速','湿度','风向']
#获取监测因子的value值
item_value = [pm25,pm10,so2,no2,no,nox,co,o3,barometer,temperature,windspeed,humidity,winddirection]
for i in range(0,len(monitor)):
cur.execute("SELECT item_code FROM s_data_items where item_name = %s;",(monitor[i],))
item_code = (cur.fetchone())[0]
cur.execute('''INSERT INTO d_data_yyyymms (station_id,item_code,data_cycle,data_attr,data_time,data_vale,created_at,updated_at)
VALUES (%s,%s,%s,%s,%s,%s,%s,%s);''',
(d_station_id,item_code,300,'T',data_time,item_value[i],time.strftime("%Y-%m-%d %X"),time.strftime("%Y-%m-%d %X")))
conn.commit()
conn.close()
print time.strftime("%Y-%m-%d %X")
print '-------------------'