python爬虫爬取steam,epic,origin平台游戏数据

这是我们课程实训的一个功能模块,实现将steam,epic,origin游戏价格信息爬取出来,由于三个网站的构造不一样,加载数据的方式也不一样所以我们需要采用不同的方法来爬取这三个平台的游戏数据

用到的工具包

BeautifulSoup包 提取爬取网页标签的属性值(游戏的价格信息等)
selenium的webdriver  利用脚本实现动态加载数据
requests 爬取网页数据用

mysql 数据表

在这里插入图片描述

具体实现步骤

1. 提取steam数据
首页url
https://store.steampowered.com/search/?specials=1&page=1

steam网站的游戏数据是分页的,我们可以通过url拼接进行爬取

获取页面html信息

# 获取页面信息
def getPage(pagenum):
    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) '
                             'Chrome/51.0.2704.63 Safari/537.36'}
    #https://store.steampowered.com/search/?specials=1&page=1
    urlh = "https://store.steampowered.com/search/?specials=1&page="
    url = urlh+str(pagenum);
    print(url)
    reponse = requests.get(url, headers=headers)

    reponse.encoding = 'utf-8'
    return reponse.text

将html信息存入txt文件中
进行这一步是为了方便测试,也是为了如果爬取的页面发生变动或更新,相当于留了一个备份

def saveHtmlCode(html,path):
    file = open(path, "wb")
    file.write(html)

提取页面中的游戏数据
这里面要进行数据清理,去除游戏价格中多余的特殊字符(空格,换行等)

# 获取游戏信息
def getGameInfo1(html,game_list):
    global count
    soup = BeautifulSoup(html, 'html.parser')

    # 游戏列表
    games_Info = soup.find(id='search_resultsRows')
    games_a = games_Info.find_all('a');
    for i in range(0,len(games_a)):
        #print(games_a[i])
        #商品是否打折
        is_free = games_a[i].find('div',class_="col search_price responsive_secondrow");
        if(is_free!=None): continue
        #获取商品src
        game_src = games_a[i].find('img')['src']
        #print("src = "+game_src)
        #获取商品名字
        game_name = games_a[i].find('span',class_='title').get_text()
        #print("name = "+game_name)
        #获取商品折扣
        game_discount = stripAndreplace(games_a[i].find('div',class_="col search_discount responsive_secondrow").get_text())
        #print(game_discount)
        #获取折扣信息
        priceText = stripAndreplace(games_a[i].find('div',class_="col search_price discounted responsive_secondrow").get_text())
        priceText = split(priceText)
        if (len(priceText) < 2): continue
        #获取商品原来价格
        game_original_price = priceText[0].replace(' ','');
        #获取商品折扣价格
        game_final_price = priceText[1].replace(' ','');
        #print("op = "+game_original_price+" fp = "+game_final_price)
        #来源
        game_source = 'steam'

        game_now_time = datetime.datetime.now().strftime('%Y-%m-%d-%H:%M')
        # name, original_price,final_price,discount,source,src
        game = EpicGamePrice.Game(game_name,game_original_price,game_final_price,game_discount,game_source,game_src,game_now_time)
        print(game.printProperty())
        game_list.append(game)

将爬取的游戏信息存入csv文件中

# game_list中代表的是游戏信息list,path是存入的路径
def write_to_excel(game_list,path):
    file = open(path,'w',encoding='utf-8-sig')
    csv_writer = csv.writer(file)
    csv_writer.writerow(["game_name", "game_original_price", "game_final_price","game_discount","game_source",'game_src','game_now_date'])

    for i in range(0,len(game_list)):
        game = game_list[i];
        csv_writer.writerow([game.get_name, game.get_original_price, game.get_final_price,game.get_discount,game.get_source,game.get_src,game.get_now_date])

    file.close()

csv文件展示
我是爬了200页数据
在这里插入图片描述

2.提取epic数据
首页url
url = "https://www.epicgames.com/store/zh-CN/browse?sortBy=releaseDate&sortDir=DESC&pageSize=30"

因为epic数据是需要点击下面这个加载更多来显示更多游戏的
在这里插入图片描述
所以我们第一步需要写一个脚本来点击这个加载按钮,这就用到了我们的webdriver
1. 编写脚本来点击加载更多按钮实现加载更多数据并获取网页源代码

#打开浏览器
def openFireFoxDiver():
    url = "https://www.epicgames.com/store/zh-CN/browse?sortBy=releaseDate&sortDir=DESC&pageSize=30"
    driver = webdriver.Firefox()
    driver.get(url)
    time.sleep(10)
    button = driver.find_element_by_id('browse-pagination')
    button.click()
    #等待15秒让网页加载完全数据
    time.sleep(15)

    html_page = driver.page_source.encode('utf-8')  # 取得网页的源代码
    return html_page;

获取页面游戏信息

#获取页面信息
def getGameInfo(game_list,path):
    html =getHtmlCode(path)
    html_page_soup = BeautifulSoup(html, 'html.parser')
    gameInfo = html_page_soup.find_all('li', class_='css-1adx3p4-BrowseGrid-styles__card')
    print(len(gameInfo))
    #print(gameInfo)

    for i in range(0,len(gameInfo)):
        if(i==2):
            print(gameInfo[i])
        #游戏图片
        game_img = gameInfo[i].find('img')
        #游戏图片链接
        game_src = ''
        #游戏名称
        game_name =''
        #游戏现价
        game_original_price = ''
        if game_img.has_attr('data-image'):
            game_src = game_img['data-image'];
            #print(game_src +"       "+str(i))
        else: continue
        if game_img.has_attr('alt'):
            game_name = game_img['alt']
            #print(game_name + "       " + str(i))
        else: continue
        #游戏折扣
        game_discount = gameInfo[i].find('span',attrs={'data-component':'DiscountAmount'})
        if(game_discount==None): game_discount = 'no discount';
        else: game_discount = game_discount.get_text();
        print(game_discount)
        #游戏现价
        game_final_price = gameInfo[i].find('span',attrs = {'data-component':'Price'})
        if(game_final_price==None): game_final_price = '0'
        else: game_final_price = game_final_price.get_text()
        #print(game_final_price)
        #游戏原价
        game_original_price = gameInfo[i].find('s',attrs ={'data-component':'Price'})
        if(game_original_price==None): game_original_price = game_final_price
        else: game_original_price = game_original_price.get_text()
        #print(game_original_price)
        game_final_price = stripAndreplace(game_final_price)
        game_original_price = stripAndreplace(game_original_price)
        game_final_price = str(round(float(game_final_price)*6.53,2))
        game_original_price = str(round(float(game_original_price)*6.53,2))
        print("op =  " + str(game_original_price) + "  fp = " + str(game_final_price))
        game_source = 'epic'
        game_now_time = datetime.datetime.now().strftime('%Y-%m-%d-%H:%M')
        game = Game(game_name, game_original_price, game_final_price, game_discount, game_source, game_src,game_now_time)

        print(game.printProperty())
        game_list.append(game)

写入csv文件

def write_to_excel(game_list,path):
    file = open(path,'w',encoding='utf-8-sig')
    csv_writer = csv.writer(file)
    csv_writer.writerow(["game_name", "game_original_price", "game_final_price","game_discount","game_source",'game_src','game_now_date'])

    for i in range(0,len(game_list)):
        game = game_list[i];
        csv_writer.writerow([game.get_name, game.get_original_price, game.get_final_price,game.get_discount,game.get_source,game.get_src,game.get_now_date])

    file.close()

爬取页面展示

def write_to_excel(game_list,path):
    file = open(path,'w',encoding='utf-8-sig')
    csv_writer = csv.writer(file)
    csv_writer.writerow(["game_name", "game_original_price", "game_final_price","game_discount","game_source",'game_src','game_now_date'])

    for i in range(0,len(game_list)):
        game = game_list[i];
        csv_writer.writerow([game.get_name, game.get_original_price, game.get_final_price,game.get_discount,game.get_source,game.get_src,game.get_now_date])

    file.close()
3. 提取origin平台数据
首页url
url = "https://www.origin.com/hkg/en-us/store/deals/holidaysale"

origin平台的游戏数据是通过下拉滑动框来进行动态加载的,因此我们需要写一个脚本来实现定时将滑动框移到最低端,这个操作重复20次就够了,因为origin平台就这么些游戏
编写脚本实现自动下拉滑动框并获取网页源代码

#打开浏览器
def openFireFoxDiver():
    #url = 'https://www.origin.com/hkg/en-us/store/browse?fq=platform:pc-download'
    url = "https://www.origin.com/hkg/en-us/store/deals/holidaysale"
    driver = webdriver.Firefox()
    driver.get(url)
    time.sleep(15)
    for i in range(0,30):
        driver.execute_script('window.scrollTo(0, document.body.scrollHeight)')
        time.sleep(5)
        print(i)
    html_page = driver.page_source.encode('utf-8')  # 取得网页的源代码
    return html_page;

获取页面游戏信息

#获取页面信息
def getGameInfo(game_list,path):
    html = EpicGamePrice.getHtmlCode(path)
    html_page_soup = BeautifulSoup(html, 'html.parser')
    gameInfo = html_page_soup.find_all('origin-store-bundle-offer')
    # print(len(gameInfo))
    # print(gameInfo[0])
    for i in range(0,len(gameInfo)):
        #游戏图片src
        game_src = gameInfo[i].find('img')['src']
        # print(game_src +"    "+str(i))

        #游戏名称
        game_name = gameInfo[i].find('h2',attrs={'class':'otktitle-4 origin-storebundleoffer-title'}).get_text()
        # print(game_name)

        #游戏现价
        game_final_price = gameInfo[i].find('p', attrs={'class': 'origin-store-offerprice-price otkprice'})
        if(game_final_price==None):
            game_final_price='-1'
        else:
            game_final_price = game_final_price.get_text();
        game_final_price = stripAndreplace(game_final_price)
        # print(game_final_price)

        #游戏折扣

        game_discount = gameInfo[i].find('span', attrs={'class': 'otkprice-sale'});
        if(game_discount==None):
            game_discount = 'Internert not connect'
        else:
            game_discount = gameInfo[i].find('span',attrs={'class':'otkprice-sale'}).get_text();
        game_discount = dealSaveUpto(game_discount)
        # print(game_discount)
        #游戏原价
        [game_original_price,game_final_price] = getOriginPriceAndFinalPrice(game_final_price,game_discount)
        #print(game_original_price)

        #游戏来源
        game_source = 'origin'

        game_now_time = datetime.datetime.now().strftime('%Y-%m-%d-%H:%M')

        game = EpicGamePrice.Game(game_name, game_original_price, game_final_price, game_discount, game_source, game_src,game_now_time)
        print(game.printProperty())
        game_list.append(game)

将数据写入csv文件中

def write_to_excel(game_list,path):
    file = open(path,'w',encoding='utf-8-sig')
    csv_writer = csv.writer(file)
    csv_writer.writerow(["game_name", "game_original_price", "game_final_price","game_discount","game_source",'game_src','game_now_date'])

    for i in range(0,len(game_list)):
        game = game_list[i];
        csv_writer.writerow([game.get_name, game.get_original_price, game.get_final_price,game.get_discount,game.get_source,game.get_src,game.get_now_date])

    file.close()

爬取效果展示
在这里插入图片描述

  • 3
    点赞
  • 64
    收藏
    觉得还不错? 一键收藏
  • 3
    评论

“相关推荐”对你有帮助么?

  • 非常没帮助
  • 没帮助
  • 一般
  • 有帮助
  • 非常有帮助
提交
评论 3
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值