steamdb免费游戏信息爬取（不是爬虫教学，日常记录，贼不工整，不喜勿看）

日常记录而已，不是特别工整，不喜勿喷，不喜勿看。

1. 数据来源于steamdb， 目标网址：https://steamdb.info/upcoming/free/

2. 由于网址存在反爬措施，在没有cookie的情况下，网站会由js进行跳转，跳转过程中post表单的数据由js计算而来，详情请看另一篇文章：stemadb反扒机制分析。为了简化工作量，使用selenium进行访问网页，进行跳转然后获取cookie, cookie有效期为一天。

3. cookie有效时，网站直接采用requests进行访问，爬取数据。当访问失败时，使用selenium访问网站刷新cookie。

4. 上疗效，原先准备作为网站小功能的一部分，每日定时更新并提供订阅定时发送邮件功能，但是由于太耗资源，因而放弃了。

5. 然后直接上代码，这里只贴有关steamdb的代码，涉及到网站的部分就不贴了

5.1 本代码负责手动更新cookie. 也用于selenium测试。

# -*- coding: utf-8 -*-
# @Author  : LG

from selenium import webdriver

import argparse
from selenium.webdriver.support.ui import WebDriverWait

def update_cookie(url, file_name, delayed):
    # 无头模式，不打开浏览器窗口
    option = webdriver.FirefoxOptions()
    option.add_argument('--headless')

    driver = webdriver.Firefox(options=option)
    driver.get(url)
    # 这里加了延时，直到页面找到特定元素 或超时。
    WebDriverWait(driver, delayed).until(lambda aaa: driver.find_element_by_id('live-promotions'))

    # 将cookie保存在特定文件中，平时更新时由其他程序读取
    with open(file_name, 'w')as f:
        for cookie in driver.get_cookies():
            print(cookie)
            f.write(cookie['name']+','+cookie['value']+'\n')
    driver.close()
    return True

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='update cookie')
    parser.add_argument('-u', '--url', type=str, default='https://steamdb.info/upcoming/free/')
    parser.add_argument('-f', '--filename', type=str, default='cookie.txt')
    parser.add_argument('-d', '--delayed', type=int, default=15)
    args = parser.parse_args()

    update_cookie(url=args.url, file_name=args.filename, delayed=args.delayed)

5.2 自动更新以及自动发送邮件等功能的实现。

# 爬取的时间 转UTC 转本地
def time_analysis(time_str):
    months = {'January': '01', 'February': '02', 'March': '03', 'April': '04', 'May': '05', 'June': '06',
              'July': '07', 'August': '08', 'September': '09', 'October': '10', 'November': '11', 'December': '12'}

    time = time_str.split('')
    time = '{}-{}-{} {}'.format(time[2], time[1], time[0], time[4])
    for month_en, month_num in months.items():
        time = time.replace(month_en, month_num)
    utc_dt = datetime.strptime(time, "%Y-%m-%d %H:%M:%S")
    local_tz = pytz.timezone('Asia/Chongqing')
    local_dt = utc_dt.replace(tzinfo=pytz.utc).astimezone(local_tz)
    local_dt = str(local_dt).split('+')[0]
    return local_dt

# 更新cookie 
def update_cookie():
    url = 'https://steamdb.info/upcoming/free/'

    option = webdriver.FirefoxOptions()
    option.add_argument('--headless')

    driver = webdriver.Firefox(options=option)
    driver.get(url)

    WebDriverWait(driver, delayed).until(lambda aaa: driver.find_element_by_id('live-promotions'))

    # 这只是简单调试时的日志，不要在意
    if len(driver.get_cookies()) ==1:
        with open('XXX/steamfree_log.txt', 'a') as f:
            f.write("{} {}\n".format(datetime.now(), 'cookie更新失败'))
        return False

    with open('cookie.txt', 'w')as f:
        for cookie in driver.get_cookies():
            f.write(cookie['name']+','+cookie['value']+'\n')

    with open('XXX/steamfree_log.txt', 'a') as f:
        f.write("{} {}\n".format(datetime.now(), 'cookie更新失败'))

    return True

# 读取cookie,并访问网站
def get_html(url='https://steamdb.info/upcoming/free/'):
    session = requests.session()

    headers = {"User-Agent":"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:70.0) Gecko/20100101 Firefox/70.0"}
    cookie_str = ''
    try:
        f = open('cookie.txt','r')
        lines = f.readlines()

        for line in lines:
            name, value = line.rstrip('\n').split(',')
            cookie_str+=name+"="+value+";"
    except:
        pass
    headers["Cookie"]=cookie_str
    html = session.get(url, headers=headers)
    return html

# 从网站页面抽取需要爬取的数据，这里俩个表格格式略有不同，后面会分开处理
def get_tables():
    html = get_html()
    if html.status_code != 200:
        with open('XXX/steamfree_log.txt', 'a') as f:
            f.write("{} {} {}\n".format(datetime.now(), '请求失败刷新cookie', html.status_code))
        # 请求失败，更新cookies
        if update_cookie():
            # 重新读取网页
            html = get_html()
        else:
            # 更新失败
            return None
        if html.status_code != 200:
            return None

    # 请求成功
    soup = BeautifulSoup(html.text, 'lxml')
    tables = soup.find_all('table')
    return tables


def currently(currently_table):
    # 第一个表格
    trs = currently_table.find('tbody').find_all('tr')
    for tr in trs:
        tds = tr.find_all('td')
        link = tds[0].a['href']
        pic = tds[0].img['src']
        name = tds[1].a.b.string
        type = tds[3].string or tds[3].b.string
        starts = time_analysis(tds[4].string)
        ends = time_analysis(tds[5].string)
        # 这里是网站专门为这个功能添加一个数据库
        steamfree = Steamfree(link=link, pic=pic, name=name, type=type, start=starts, end=ends)
        db.session.add(steamfree)
    return True


def upcoming(upcoming_table):
    # 第二个表格
    trs = upcoming_table.find('tbody').find_all('tr')
    for tr in trs:
        tds = tr.find_all('td')
        link = tds[0].a['href']
        pic = tds[0].img['src']
        name = tds[1].a.b.string
        type = tds[2].string or tds[2].b.string
        starts = time_analysis(tds[3].string)
        ends = time_analysis(tds[4].string)
        steamfree = Steamfree(link=link, pic=pic,name=name,type=type,start=starts,end=ends)
        db.session.add(steamfree)
    return True


def steamfree_delete_all():
    # 更新数据库前，需要全部删除
    steamfrees = Steamfree.query.all()
    for steamfree in steamfrees:
        db.session.delete(steamfree)
    db.session.commit()


def steamdb_free_update():
    # 这里添加了一个手动刷新数据的函数，在前端给了接口调用，不用在意
    tables = get_tables()
    if not isinstance(tables, int):
        steamfree_delete_all()
        currently(tables[0])
        upcoming(tables[1])
        with open('XXX/steamfree_log.txt', 'a') as f:
            f.write("{} {}\n".format(datetime.now(), 'steamfree手动更新成功'))
    else:
        with open('XXX/steamfree_log.txt', 'a') as f:
            f.write("{} {}\n".format(datetime.now(), 'steamfree手动更新失败'))

# 自动发送邮件
def send_steamfree_auto():
    # 这里定时，用到了 flask_apscheduler，在操作数据库时，需要一个app
    with scheduler.app.app_context():
        steamfrees = Steamfree.query.all()
        # 订阅用户
        subers = Steamfree_sub.query.all()    
        for sub in subers:
            send_email(to=sub.email,
                       subject='steam免费信息',
                       template='laboratory/email/steam_free_send',
                       steamfrees=steamfrees,
                       link='')
            with open('XXX/steamfree_log.txt', 'a') as f:
                f.write("{} {} {}\n".format(datetime.now(), 'steamfree自动发送邮件成功', sub.email))

# 自动更新
def steamdb_free_update_auto():
    with scheduler.app.app_context():
        tables = get_tables()
        if not isinstance(tables, int):
            steamfree_delete_all()
            currently(tables[0])
            upcoming(tables[1])
            with open('XXX/steamfree_log.txt', 'a') as f:
                f.write("{} {}\n".format(datetime.now(), 'steamfree自动更新成功'))
        else:
            with open('XXX/steamfree_log.txt', 'a') as f:
                f.write("{} {}\n".format(datetime.now(), 'steamfree自动更新失败'))


# 每6小时自动更新一次数据库
scheduler.add_job(func=steamdb_free_update_auto,
                  id='1',
                  trigger='interval',   # trigger='interval' 表示是一个循环任务，每隔多久执行一次
                  hours=6)

# 每天早上10点自动发送邮件
scheduler.add_job(func=send_steamfree_auto,
                  id='2',
                  trigger='cron',       # 定时任务
                  day_of_week='0-6',
                  hour=10,
                  minute=0,
                  second=0)

5.3 数据库模型也贴一下

# steam 免费游戏数据库
class Steamfree(db.Model):
    __bind_key__ = 'laboratory'    # 这里由于网站有多个数据库，所以需要指定bind
    __name__ = 'steamfrees'
    id = db.Column(db.Integer, primary_key=True)    
    link = db.Column(db.Text)    # steam游戏页面链接
    pic = db.Column(db.Text)    # 缩略图
    name = db.Column(db.String)    # 游戏名
    type = db.Column(db.String)    # 免费类型
    start = db.Column(db.String)    # 开始时间
    end = db.Column(db.String)    # 结束时间


# steam免费游戏服务订阅数据库
class Steamfree_sub(db.Model):
    __bind_key__ = 'laboratory'
    __name__ = 'steamfree_subs'
    id = db.Column(db.Integer, primary_key=True)
    email = db.Column(db.Text)    # 这里只保存订阅者的邮箱

5.4 网站前端也发出来吧（但是基础页面不发）

{% extends "base.html" %}

{% block contents %}

<div class="container">
    <div  style="margin-bottom: 20px">
        <h2>Steam免费游戏资讯</h2>
        <div class="pull-right">
            <a class="btn btn-success" href="{{ url_for('laboratory.subscribe') }}">订阅</a>
            <a class="btn btn-danger" href="{{ url_for('laboratory.unsubscribe') }}">退订</a>
        </div>
        <p>数据来源于
            <a href="https://steamdb.info/upcoming/free/" class="text-dark">steamdb</a>,
            本站每天6:00刷新一次数据，并10:00发送一封邮件到订阅的用户邮箱中。
            您可以免费订阅，并可随时退订。
        </p>
    </div>
    <div style="background-color: white">
        <table class="table table-striped table-hover">
            <thead>
            <tr>
                <th></th>
                <th>游戏名</th>
                <th>免费类型</th>
                <th>开始时间</th>
                <th>结束时间</th>
            </tr>
            </thead>
            <tbody>
            {% for steamfree in steamfrees %}
                <tr>
                    <td><img src="{{ steamfree.pic }}"></td>
                    <td><a href="{{ steamfree.link }}" target="_blank">{{ steamfree.name }}</a></td>
                    <td>
                        {% if steamfree.type == 'Weekend' %}
                        <span class="text-dark">限时游玩</span>
                        {% elif steamfree.type == 'Keep' %}
                        <span class="text-success">限时领取</span>
                        {% endif %}
                    </td>
                    <td>{{ steamfree.start }}</td>
                    <td>{{ steamfree.end }}</td>
                </tr>
            {% endfor %}
            </tbody>

        </table>

    </div>
</div>

{% endblock %}

5.5 视图，这里只贴部分代码

# 免费资讯主页面
@laboratory_blueprint.route('/laboratory/steamfree')
def steamfree():
    steamfrees = Steamfree.query.all()
    return render_template('laboratory/steamfree.html', steamfrees=steamfrees, current_user=current_user)

# 手动更新
@laboratory_blueprint.route('/laboratory/steamfree/update')
@login_required
def steamfree_update():

    steamdb_free_update()
    return redirect(url_for('laboratory.steamfree'))

# 订阅
@laboratory_blueprint.route('/laboratory/steamfree/subscribe')
@login_required
def subscribe():
    sub = Steamfree_sub.query.filter(Steamfree_sub.email == current_user.email).first()
    if sub is None:
        sub = Steamfree_sub(email=current_user.email)
        flash('您已订阅steam免费游戏资讯。')
        db.session.add(sub)
    else:
        flash('您已订阅steam免费游戏资讯，无需重复订阅。')


    return redirect(url_for('laboratory.steamfree'))

# 退订
@laboratory_blueprint.route('/laboratory/steamfree/unsubscribe')
@login_required
def unsubscribe():
    sub = Steamfree_sub.query.filter(Steamfree_sub.email==current_user.email).first()
    if sub is None:
        flash('您还未订阅steam免费游戏资讯，无需退订。')
    else:
        db.session.delete(sub)
        db.session.commit()
        flash('您已退订steam免费游戏资讯。')
    return redirect(url_for('laboratory.steamfree'))