Python定期爬取数据

#coding: utf-8
from bs4 import BeautifulSoup
import requests
from requests.exceptions import RequestException
import csv
import pandas as pd
import datetime
import time
import matplotlib.pyplot as plt
import random

#设置中文显示
plt.rcParams[‘font.sans-serif’]=[‘SimHei’] #用来正常显示中文标签
plt.rcParams[‘axes.unicode_minus’]=False #用来正常显示负号

#由于只爬取两个网页的内容，就直接将该两个网页放入列表中
url_sell = ‘https://otcbtc.com/sell_offers?currency=eth&fiat_currency=cny&payment_type=all’
url_buy = ‘https://otcbtc.com/buy_offers?currency=eth&fiat_currency=cny&payment_type=all’
urls = [url_sell, url_buy]

#更换User-Agent
def getHeaders():
user_agent_list = [
‘Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36’,
‘Mozilla/5.0(Windows;U;WindowsNT6.1;en-us)AppleWebKit/534.50(KHTML,likeGecko)Version/5.1Safari/534.50’,
‘Mozilla/5.0(WindowsNT6.1;rv:2.0.1)Gecko/20100101Firefox/4.0.1’,
‘Mozilla/4.0(compatible;MSIE7.0;WindowsNT5.1;Trident/4.0;SE2.XMetaSr1.0;SE2.XMetaSr1.0;.NETCLR2.0.50727;SE2.XMetaSr1.0)’
]
index = random.randrange(0,len(user_agent_list))
headers = {
‘User-Agent’: user_agent_list[index]
}
return headers

#获取页面内容
def getHtml(url):
try:
response = requests.get(url, headers=getHeaders())
if response.status_code == 200:
return response.text
except RequestException:
print(’=request exception=’)
return None

#解析网页 获取价格
def parse_html(html):
try:
soup = BeautifulSoup(html, ‘lxml’)
price = soup.find(“div”, class_=‘can-buy-count’).contents[0]
price = float(str(price).strip().strip(’\n’).replace(’,’, ‘’))
return price
except Exception:
print(’=parseHtml exception=’)
return None

#保存到csv表中
def save2csv(prices, sched_timer):
with open(‘CNY-ETH.csv’, ‘a+’, newline=’’, encoding=‘utf-8’) as csvfile:
writer = csv.writer(csvfile)
# writer.writerow([‘购买价格最低’, ‘出售价格最低’])
writer.writerow([sched_timer, prices[0], prices[1]])
df = pd.read_csv(‘CNY-ETH.c
sv’)
print(df)

#绘图
def drawPic():
eth_file = pd.read_csv(‘CNY-ETH.csv’)
eth_file[‘DATE’] = pd.to_datetime(eth_file[‘DATE’])
# print(eth_file.head(80))
xlen = eth_file[0:120]
plt.plot(eth_file[‘DATE’], xlen[‘购买价格最低’], c=‘g’, ls=’-’)
plt.plot(eth_file[‘DATE’], xlen[‘出售价格最低’], c=‘r’, ls=’-’)
plt.xticks(rotation=120)
# plt.yticks(rotation=range(3400,3800))
plt.xlabel(‘时间’)
plt.ylabel(‘价格’)
plt.title(‘CNY-ETH价格波动’)
# 由于plt.show()程序会停留不运行 ，采用以下方法，绘图后，停留15s，再关闭
plt.ion()
plt.pause(15)
plt.close()

def main(sched_timer):
prices = []
for url in urls:
html = getHtml(url)
price = parse_html(html)
if price == None:
prices.append(None)
else:
prices.append(price)
# print(‘购买价格最低：’, prices[0],’\n出售价格最低：’, prices[1])
if prices[0] == None or prices[1] == None:
print(’=price has None=’)
pass
else:
save2csv(prices, sched_timer)
drawPic()

if name == ‘main’:
# 定时任务
# 设定一个标签 确保是运行完定时任务后 再修改时间
flag = 0
# 获取当前时间
now = datetime.datetime.now()
# 启动时间
# 启动时间为当前时间 加5秒
sched_timer = datetime.datetime(now.year, now.month, now.day, now.hour, now.minute,
now.second) + datetime.timedelta(seconds=5)
# 启动时间也可自行手动设置
# sched_timer = datetime.datetime(2017,12,13,9,30,10)
while (True):
# 当前时间
now = datetime.datetime.now()
# 本想用当前时间 == 启动时间作为判断标准，但是测试的时候 毫秒级的时间相等成功率很低 而且存在启动时间秒级与当前时间毫秒级比较的问题，后来换成了以下方式，允许1秒之差
if sched_timer < now < sched_timer + datetime.timedelta(seconds=1):
time.sleep(1)
print(now)
# 运行程序
main(sched_timer)
# 将标签设为 1
flag = 1
else:
# 标签控制 表示主程序已运行，才修改定时任务时间
if flag == 1:
# 修改定时任务时间 时间间隔为2分钟
sched_timer = sched_timer + datetime.timedelta(minutes=2)
flag = 0