selenium爬取亚马逊商品评论

亚马逊商品评论有反爬虫，所以就用selenium爬了。网速一定要好，不然爬的真的是天昏地暗。配合多线程就会快很多，这个不写了，爬的时候手动复制了N个代码去爬。还有一个点，中文和英文的设置，可以在评论里面加进去

from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.common.exceptions import NoSuchElementException, TimeoutException, ElementClickInterceptedException
import csv


class AsinSpider():
    options = webdriver.ChromeOptions()
    # options.add_argument("--proxy-sever = 218.60.8.99:3129")
    # options.add_argument("disable-web-security")
    options.add_argument("--lang=en-US")
    options.add_argument(
        'user-agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36 Edg/87.0.664.75"')
    # 为加快网页加载速度，禁止加载图片和CSS样式
    prefs = {"profile.managed_default_content_settings.images": 2, 'permissions.default.stylesheet': 2}
    options.add_experimental_option("prefs", prefs)
    driver = webdriver.Chrome(executable_path=r"C:\Users\zxzy\AppData\Local\Google\Chrome\Application\chromedriver.exe",
                              options=options)

    def __init__(self, re_url, re_asin):
        self.review_url = re_url
        self.review_asin = re_asin
        self.item = None
        self.reviews_content = []
        self.review_stars = []
        self.review_customer = []
        self.review_img = []
        self.next_page = None

    def close_pop(self):
        self.driver.get(self.review_url)
        # 关闭弹窗
        try:
            WebDriverWait(self.driver, 100).until(
                EC.presence_of_element_located((By.CLASS_NAME, "a-button-input"))
            )
            pop = self.driver.find_element_by_class_name("a-button-input")
            WebDriverWait(self.driver, 100).until(
                EC.presence_of_element_located((By.CLASS_NAME, "a-button-input"))
            )
            pop.click()

        except:
            pass

    # 设置网页最大加载时长，超过刷新
    def get_url(self):
        self.driver.set_page_load_timeout(20)
        try:
            self.driver.get(self.next_page)
        except TimeoutException:
            self.driver.refresh()

    def get_main_information(self):
        # 等待页面加载完毕
        while True:
            try:
                WebDriverWait(self.driver, 30).until(
                    EC.presence_of_element_located((By.ID, 'cm_cr-review_list'))
                )
                break
            except Exception as e:
                print(e)
                self.driver.refresh()
        us = self.driver.find_element_by_id("cm_cr-review_list")

        # 爬取评论信息
        self.reviews_content = []
        self.review_stars = []
        self.review_customer = []
        self.review_img = []

        # 获取每页的全部评论信息
        reviews = self.driver.find_elements_by_xpath('//div[@class = "a-section celwidget"]')
        # 先以列表形式存储每条评论的文字信息，后期再处理
        self.reviews_content = [review.text for review in reviews]
        # 遍历每条评论，获取星级，客户链接，评论图片链接
        for review in reviews:
            # 获取评论星级
            try:
                self.review_stars.append(
                    review.find_element_by_xpath('.//i[@data-hook = "review-star-rating"]').get_attribute("innerHTML"))
            except:
                self.review_stars.append(
                    review.find_element_by_xpath('.//i[@data-hook = "cmps-review-star-rating"]').get_attribute(
                        "innerHTML"))
            # 获取客户链接
            try:
                WebDriverWait(review, 5).until(
                    EC.presence_of_element_located((By.XPATH, ".//a[@class = 'a-profile']"))
                )
                self.review_customer.append(
                    review.find_element_by_xpath(".//a[@class = 'a-profile']").get_attribute("href"))
            except:
                self.review_customer.append(None)

            # 获取评论中包含的图片链接
            try:
                WebDriverWait(review, 1).until(
                    EC.presence_of_element_located((By.XPATH, ".//*[@class='a-declarative']/a/img"))
                )
                imgs = review.find_elements_by_xpath(".//*[@class='a-declarative']/a/img")
                self.review_img.append([img.get_attribute("src") for img in imgs])
            except:
                self.review_img.append(None)

        # 判断是否还有下一页next_page
        try:
            WebDriverWait(self.driver, 5).until(
                EC.element_to_be_clickable((By.XPATH, '//li[@class = "a-last"]/a'))
            )
            self.next_page = us.find_element_by_xpath('.//li[@class = "a-last"]/a').get_attribute("href")

        except NoSuchElementException:
            self.driver.find_elements_by_xpath('//li[@class = "a-disabled a-last"]')
            self.next_page = None
            print("未有下一页")

        except TimeoutException:
            self.next_page = None
            print("时间超了")
            self.driver.refresh()
        print("提取完毕")

    # 先简单分割下content内容，后面发现其实没必要。后续又写了一段代码专门用来解析
    def parse(self):
        for cc, cu, sa, img in zip(self.reviews_content, self.review_customer, self.review_stars, self.review_img):
            cc = "".join(cc).split("\n")
            yield {
                "review_names": cc[0],
                "review_customer": cu,
                "review_stars": sa,
                "review_titles": cc[1],
                "review_date": cc[2],
                "review_sizes": cc[3] if len(cc) > 4 else None,
                "review_revs": cc[4] if len(cc) > 5 else None,
                "review_others1": cc[5] if len(cc) > 6 else None,
                "review_others2": cc[6] if len(cc) > 7 else None,
                "asin": self.review_asin,
                'review_images': img
            }

    # 保存数据
    def save(self):
        with open(r'D:\python\amazon_review\test.csv', 'a', newline='', encoding='gb18030') as f:
            fieldnames = ["review_names", "review_customer", "review_stars", "review_titles", "review_date",
                          "review_sizes", "review_revs", "review_others1", "review_others2", "asin", "review_images"]
            writer = csv.DictWriter(f, fieldnames=fieldnames)
            writer.writeheader()
            for item in self.parse():
                writer.writerow(item)

    # 运行
    def run(self):
        # 先关闭修改地址的弹窗，这个函数其实也不用些，没必要
        self.close_pop()
        # 获取评论信息
        self.get_main_information()
        # 保存数据
        self.save()
        # 如果存在下一页，则继续爬取
        while self.next_page:
            # 设置网页最长加载时间，默认第一页一定会加载
            self.get_url()
            self.get_main_information()
            self.save()


def main():
    re_url = 'https://www.amazon.com/product-reviews/B07FMQKBSP'
    re_asin = 'B07FMQKBSP'
    spider = AsinSpider(re_url, re_asin)
    spider.run()
    print("%s爬取完毕" % re_asin)


if __name__ == '__main__':
    main()

存下来的数据再进行处理，这里主要用正则来处理

import pandas as pd
import re

# 读取数据
data = pd.read_csv(r'D:\python\amazon_review\review.csv',encoding = "gb18030")


# 提取评论星级
review_stars = [re.sub("<.+?>",'',i) for i in data['review_stars']]
data['review_stars'] = review_stars

# 处理review_titles错位的数据，先取出处理，再删除原来数据，最好再concat
findex = data.loc[data['review_titles'].str.contains("前|VINE|vine|制造",na=False)].index
f_data = data.loc[findex,:]
del f_data['review_titles']
f_data.columns = ['review_names', 'review_customer', 'review_stars', 'review_titles','review_date', 'review_sizes', 'review_revs', 
                  'review_others1','asin']
data.drop(findex,inplace = True)
data = pd.concat([data,f_data],sort = False)



# 处理review_date错位数据，根据开头不是20判断，错位数据只有1条，直接删了
data['review_date'] = data['review_date'].str.strip()
dindex = data.loc[True^ data['review_date'].str.startswith("20",na=False)].index
print(len(dindex))#只有1条
data.drop(dindex,inplace = True)



# 处理尺寸颜色错位的数据
data['review_sizes'] = data['review_sizes'].str.strip()
strs = ('Color','COLOR','color','SIZE','Size','size','已确认','早期','免费')
sindex = data.loc[True ^data['review_sizes'].str.startswith(strs,na=False)].index
sindex_data = data.loc[sindex,:]

# 更改列名
del sindex_data['review_others2']
sindex_data.rename(columns = {'review_others1':'review_others2',
                              'review_revs':'review_others1',
                              'review_sizes':'review_revs'},inplace = True)

# 删除再拼接
data.drop(sindex,inplace = True)
data = pd.concat([data,sindex_data],sort = False)



# 处理颜色和尺码
color = ('Color','COLOR','color')
size = ('SIZE','Size','size')
data['review_sizes'] = data['review_sizes'].str.replace("：",":")
c_data = data.loc[data['review_sizes'].str.startswith(color,na=False)]
s_data = data.loc[data['review_sizes'].str.startswith(size,na=False)]
data.drop(c_data.index,inplace = True)
data.drop(s_data.index,inplace = True)

# 由于size和color不固定哪儿开始，所以得弄四个dataframe
c_data['review_colors'] = c_data['review_sizes'].str.split(":").str[1].str.replace("Size","",re.I)
c_data['review_sizes1'] = c_data['review_sizes'].str.split(":").str[2].str.replace("color","",re.I)
s_data['review_colors'] = s_data['review_sizes'].str.split(":").str[2].str.replace("color","",re.I)
s_data['review_sizes1'] = s_data['review_sizes'].str.split(":").str[1].str.replace("color","",re.I)
data = pd.concat([data,c_data,s_data],sort = False)

# 保存数据
data.to_csv(r'D:\python\amazon_review\review.csv',index = False)