3每天Python小例-爬取淘宝网页商品

代码是从https://github.com/gxcuizy/Python/tree/master/%E4%BB%8E%E9%9B%B6%E5%AD%A6Python-%E6%8E%98%E9%87%91%E6%B4%BB%E5%8A%A8/day21上找的

#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
selenium模拟浏览器抓取淘宝商品信息
author: gxcuizy
date: 2018-11-13
"""

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from time import sleep


class GetTaobaoGoods(object):
    """模拟浏览器，抓取淘宝商品"""

    def __init__(self, goods):
        """初始化变量"""
        self.driver = webdriver.Chrome()
        self.taobao_url = 'https://www.taobao.com'
        self.goods = goods

    def search_goods(self):
        """进入浏览器，搜索商品"""
        # 全屏
        self.driver.maximize_window()
        # 打开淘宝首页
        self.driver.get(self.taobao_url)
        # 输入商品，回车搜索
        search = self.driver.find_element_by_name('q')
        search.send_keys(self.goods)
        search.send_keys(Keys.RETURN)
        # 登陆淘宝
        self.login()

    def login(self):
        """登陆淘宝"""
        if 'login' in self.driver.current_url:
            print('请扫码登陆……')
            while True:
                if 'login' in self.driver.current_url:
                    sleep(1.5)
                else:
                    print('恭喜，登陆成功！')
                    break

    def scroll_to_button(self):
        # 滑动至底部
        for i in range(0, 4):
            # 每次滑动1000像素
            height = 1000 * i
            js_code = "window.scrollBy(0," + str(height) + ")"
            self.driver.execute_script(js_code)
            sleep(2)

    def get_goods_info(self):
        # 获取商品列表
        goods_list = self.driver.find_elements_by_class_name('J_MouserOnverReq')
        for goods_info in goods_list:
            goods = {}
            # 商品图片
            img_element = goods_info.find_element_by_class_name('img')
            goods_img = 'https:' + img_element.get_attribute('data-src')
            goods.update({'img': goods_img})
            # 商品价格
            price_element = goods_info.find_element_by_css_selector('.g_price strong')
            goods_price = price_element.text
            goods.update({'price': goods_price})
            # 购买人数
            count_element = goods_info.find_element_by_class_name('deal-cnt')
            goods_sale = count_element.text
            goods.update({'sale': goods_sale})
            # 商品名称
            title_element = goods_info.find_element_by_class_name('title')
            goods_title = title_element.text
            goods.update({'title': goods_title})
            # 店铺
            shop_element = goods_info.find_element_by_class_name('shop')
            goods_shop = title_element.text
            goods.update({'shop': goods_shop})
            # 所在地
            location_element = goods_info.find_element_by_class_name('location')
            goods_location = location_element.text
            goods.update({'location': goods_location})
            # 链接
            href_element = goods_info.find_element_by_css_selector('.title a')
            goods_href = href_element.get_attribute('href')
            goods.update({'href': goods_href})
            print(goods)

    def run(self):
        """执行脚本"""
        # 打开网页搜索商品
        self.search_goods()
        page_num = 1
        sleep(30)
        while True:
            print('正在获取第%s页的商品……' % page_num)
            # 滑动底部
            self.scroll_to_button()
            # 抓取商品
            self.get_goods_info()
            print('第%s页的商品抓取结束！' % page_num)
            # 查找下一页
            next_element = self.driver.find_element_by_class_name('next')
            next_page = next_element.find_element_by_tag_name('a')
            if next_page:
                next_page.click()
                page_num += 1
                sleep(2)
            else:
                break
        # 退出浏览器
        self.driver.quit()


# 程序主入口
if __name__ == '__main__':
    search_goods = '三只松鼠'
    tao_bao = GetTaobaoGoods(search_goods)
    tao_bao.run()

1.它的用的是谷歌浏览器，如果要用火狐请把webdriver.Chrome()改为webdriver.Firefox()
2.我一直试结果一直卡在next_element = self.driver.find_element_by_class_name(‘next’)这，然后研究了一下，如果class的值中间有空格，只能用其中一个，他这里的网页中class=item next，于是我又换了一种表达方法，next_element = self.driver.find_element_by_css_selector("[class=‘item next’]")，结果还是不行。后来才发现用手机扫二维码登陆，还要过一段时间才能确认，而该程序还没等我确认完就开始处理页面了，也就是页面还没出来就开始爬取，于是我在while True:前加了sleep(30)，等我确认完才开始爬取，这样就得到结果了。
每天一个小例，加油！