Python定时爬取某网页内容

最新推荐文章于 2024-08-06 09:49:30 发布

iccool-cc

最新推荐文章于 2024-08-06 09:49:30 发布

阅读量2w

点赞数 12

分类专栏： Python 文章标签： python

本文链接：https://blog.csdn.net/qq807237096/article/details/78794039

版权

Python 专栏收录该内容

8 篇文章 0 订阅

订阅专栏

前言

昨天小伙伴说他最近在投资一个类似比特币的叫ETH币(以太币)，他想定时获取某网站以太币的“购买价格最低”和“出售价格最低”，他自己没学过Python，也找到一个定时获取股票的例子(没看懂)，看我平时有在玩Python，就问我能不能做到；平时我对Python挺感兴趣的，就想试试看。

思路

定时任务
主程序
- 爬取网页
- 解析网页获取所要内容
- 存入表格
- 绘图
- 异常处理

如何实现定时爬取

# 定时任务
# 设定一个标签 确保是运行完定时任务后 再修改时间
flag = 0
# 获取当前时间
now = datetime.datetime.now()
# 启动时间
# 启动时间为当前时间 加5秒
sched_timer = datetime.datetime(now.year, now.month, now.day, now.hour, now.minute,
                                now.second) + datetime.timedelta(seconds=5)
# 启动时间也可自行手动设置
# sched_timer = datetime.datetime(2017,12,13,9,30,10)
while (True):
    # 当前时间
    now = datetime.datetime.now()
    # print(type(now))
    # 本想用当前时间 == 启动时间作为判断标准，但是测试的时候 毫秒级的时间相等成功率很低 而且存在启动时间秒级与当前时间毫秒级比较的问题
    # 后来换成了以下方式，允许1秒之差
    if sched_timer < now < sched_timer + datetime.timedelta(seconds=1):
        time.sleep(1)
        
        
        # 主程序
        main()
        
        
        
        flag = 1
    else:
        # 标签控制 表示主程序已运行，才修改定时任务时间
        if flag == 1:
            # 修改定时任务时间 时间间隔为2分钟
            sched_timer = sched_timer + datetime.timedelta(minutes=2)
            flag = 0

代码(内带注释)

# -*- coding: utf-8 -*-
__author__ = 'iccool'

from bs4 import BeautifulSoup
import requests
from requests.exceptions import RequestException
import csv
import pandas as pd
import datetime
import time
import matplotlib.pyplot as plt
import random

# 设置中文显示
plt.rcParams['font.sans-serif']=['SimHei'] #用来正常显示中文标签
plt.rcParams['axes.unicode_minus']=False #用来正常显示负号



# 由于只爬取两个网页的内容，就直接将该两个网页放入列表中
url_sell = 'https://otcbtc.com/sell_offers?currency=eth&fiat_currency=cny&payment_type=all'

url_buy = 'https://otcbtc.com/buy_offers?currency=eth&fiat_currency=cny&payment_type=all'
urls = [url_sell, url_buy]


# 更换User-Agent
def getHeaders():
    user_agent_list = [
        'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36',
        'Mozilla/5.0(Windows;U;WindowsNT6.1;en-us)AppleWebKit/534.50(KHTML,likeGecko)Version/5.1Safari/534.50',
        'Mozilla/5.0(WindowsNT6.1;rv:2.0.1)Gecko/20100101Firefox/4.0.1',      'Mozilla/4.0(compatible;MSIE7.0;WindowsNT5.1;Trident/4.0;SE2.XMetaSr1.0;SE2.XMetaSr1.0;.NETCLR2.0.50727;SE2.XMetaSr1.0)'
    ]
    index = random.randrange(0,len(user_agent_list))
    headers = {
        'User-Agent': user_agent_list[index]
    }
    return headers

# 获取页面内容
def getHtml(url):
    try:
        response = requests.get(url, headers=getHeaders())
        if response.status_code == 200:
            return response.text
    except RequestException:
        print('===request exception===')
        return None


# 解析网页 获取价格
def parse_html(html):
    try:
        soup = BeautifulSoup(html, 'lxml')
        price = soup.find("div", class_='can-buy-count').contents[0]
        price = float(str(price).strip().strip('\n').replace(',', ''))
        return price
    except Exception:
        print('===parseHtml exception===')
        return None


# 保存到csv表中
def save2csv(prices, sched_timer):
    with open('CNY-ETH.csv', 'a+', newline='', encoding='utf-8') as csvfile:
        writer = csv.writer(csvfile)
        # writer.writerow(['购买价格最低', '出售价格最低'])
        writer.writerow([sched_timer, prices[0], prices[1]])
    df = pd.read_csv('CNY-ETH.csv')
    print(df)


# 绘图
def drawPic():
    eth_file = pd.read_csv('CNY-ETH.csv')
    eth_file['DATE'] = pd.to_datetime(eth_file['DATE'])
    # print(eth_file.head(80))
    xlen = eth_file[0:120]
    plt.plot(eth_file['DATE'], xlen['购买价格最低'], c='g', ls='-')
    plt.plot(eth_file['DATE'], xlen['出售价格最低'], c='r', ls='-')
    plt.xticks(rotation=120)
    plt.xlabel('时间')
    plt.ylabel('价格')
    plt.title('CNY-ETH价格波动')
    # 由于plt.show()程序会停留不运行 ，采用以下方法，绘图后，停留15s，再关闭
    plt.ion()
    plt.pause(15)
    plt.close()

def main(sched_timer):
    prices = []
    for url in urls:
        html = getHtml(url)
        price = parse_html(html)
        if price == None:
            prices.append(None)
        else:
            prices.append(price)
    # print('购买价格最低：', prices[0],'\n出售价格最低：', prices[1])
    if prices[0] == None or prices[1] == None:
        print('===price has None===')
        pass
    else:
        save2csv(prices, sched_timer)
        drawPic()


if __name__ == '__main__':
    # 定时任务
    # 设定一个标签 确保是运行完定时任务后 再修改时间
    flag = 0
    # 获取当前时间
    now = datetime.datetime.now()
    # 启动时间
    # 启动时间为当前时间 加5秒
    sched_timer = datetime.datetime(now.year, now.month, now.day, now.hour, now.minute,
                                    now.second) + datetime.timedelta(seconds=5)
    # 启动时间也可自行手动设置
    # sched_timer = datetime.datetime(2017,12,13,9,30,10)
    while (True):
        # 当前时间
        now = datetime.datetime.now()
        # print(type(now))
        # 本想用当前时间 == 启动时间作为判断标准，但是测试的时候 毫秒级的时间相等成功率很低 而且存在启动时间秒级与当前时间毫秒级比较的问题
        # 后来换成了以下方式，允许1秒之差
        if sched_timer < now < sched_timer + datetime.timedelta(seconds=1):
            time.sleep(1)
            print(now)
            # 运行程序
            main(sched_timer)
            # 将标签设为 1
            flag = 1
        else:
            # 标签控制 表示主程序已运行，才修改定时任务时间
            if flag == 1:
                # 修改定时任务时间 时间间隔为2分钟
                sched_timer = sched_timer + datetime.timedelta(minutes=2)
                flag = 0

效果

能够成功获取信息，并插入表格，根据表格绘制图形。

DATE,购买价格最低,出售价格最低
2017-12-13 08:52:47,4515.93,4353.02
2017-12-13 08:53:47,4515.93,4353.02
2017-12-13 08:58:53,4516.0,4285.3
2017-12-13 08:59:53,4471.71,4267.73
2017-12-13 09:00:53,4471.71,4161.03
2017-12-13 09:01:53,4471.71,4267.76
2017-12-13 09:05:03,4439.7,4261.13
2017-12-13 09:06:03,4439.7,4289.99
2017-12-13 09:09:32,4416.94,4287.58
2017-12-13 09:11:32,4416.52,4287.58
2017-12-13 09:13:32,4401.11,4287.58
2017-12-13 09:15:32,4388.56,4269.03
2017-12-13 09:19:32,4375.76,4269.03
2017-12-13 09:25:33,4293.11,4244.36
2017-12-13 09:27:33,4293.1,4244.36
2017-12-13 09:29:33,4284.38,4202.34
2017-12-13 09:37:33,4284.56,4156.24
2017-12-13 09:39:33,4284.5,4156.18
2017-12-13 09:41:33,4284.5,4202.13
2017-12-13 09:47:33,4300.42,4151.83
2017-12-13 09:49:33,4291.85,4252.71
2017-12-13 09:53:33,4291.85,4232.38
2017-12-13 09:55:33,4320.55,4261.77
2017-12-13 09:57:33,4320.55,4261.77
2017-12-13 09:59:33,4320.55,4265.97
2017-12-13 10:01:33,4332.18,4277.45
2017-12-13 10:05:33,4331.0,4295.9
2017-12-13 10:17:18,4356.05,4322.61
2017-12-13 10:25:16,4272.99,4279.22
2017-12-13 10:41:45,4196.1,4163.88
2017-12-13 10:43:45,4200.18,4163.88
2017-12-13 10:45:45,4200.0,4174.56
2017-12-13 10:47:45,4200.0,4104.58
2017-12-13 10:49:45,4200.0,4104.61
2017-12-13 10:51:45,4190.0,4104.61
2017-12-13 10:59:45,4063.06,3984.8
2017-12-13 11:15:54,4200.0,4134.65
2017-12-13 11:17:54,4200.0,4134.65
2017-12-13 11:19:54,4200.0,4118.04
2017-12-13 11:21:54,4200.0,4118.04
2017-12-13 11:23:54,4197.94,4118.04
2017-12-13 11:27:54,4186.77,4107.08
2017-12-13 11:29:54,4186.52,4106.83
2017-12-13 11:31:54,4162.25,4107.24
2017-12-13 11:33:54,4162.25,4107.24
2017-12-13 11:35:54,4200.0,4131.92
2017-12-13 11:41:43,4200.2,4141.52
2017-12-13 11:43:43,4200.2,4141.52
2017-12-13 12:52:19,4198.0,4140.7
2017-12-13 13:33:01,4258.28,4216.2
2017-12-13 13:35:01,4259.0,4230.96
2017-12-13 13:47:02,4259.0,4218.04
2017-12-13 13:49:02,4258.98,4201.4
2017-12-13 13:51:02,4371.48,4205.84
2017-12-13 13:53:02,4259.0,4205.84
2017-12-13 13:55:02,4328.08,4203.23
2017-12-13 13:59:02,4288.99,4203.23
2017-12-13 14:01:02,4278.99,4215.75
2017-12-13 14:03:02,4253.35,4200.29
2017-12-13 14:03:30,4253.27,4200.29
2017-12-13 14:05:03,4236.82,4200.48
2017-12-13 14:07:06,4228.48,4183.35
2017-12-13 14:14:11,4189.99,4100.0
2017-12-13 14:19:13,4200.0,4101.37
2017-12-13 14:25:42,4238.39,4143.44
2017-12-13 14:27:42,4237.57,4143.44
2017-12-13 14:33:42,4259.31,4165.12
2017-12-13 14:36:50,4282.96,4206.91
2017-12-13 14:38:50,4270.65,4206.91
2017-12-13 14:42:50,4266.46,4206.97