Python - 获取亚马逊 自拍杆美国市场排名

 通过在亚马逊美国站获取数据，再将英文数据转换中文，最后写入excle表格中

# -*- coding: utf-8 -*-
from urllib import request, parse
import requests, time, re, xlrd, json, pymssql, datetime
from bs4 import BeautifulSoup
from lxml import etree
from xlutils.copy import copy
from datetime import timedelta
from gevent import monkey



class Amazon_books(object):
    def __init__(self):
        self.url = "https://www.amazon.com/gp/bestsellers/boost/12035215011/ref=pd_zg_hrsr_boost_1_3_last"
        self.base_url = "http://fy.iciba.com/ajax.php?a=fy"  # 翻译url
        self.index = 1  # 记录图书序号
        self.headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36",
        }
        self.workbook = xlrd.open_workbook(r'D:\PycharmProjects\reptile\亚马逊\自拍杆美国市场分析表 .xls')  # 文件路径
        self.xlsx = copy(self.workbook)
        self.shtc = self.xlsx.get_sheet(0)
        self.num = 1

    def get_page(self):
        """获取前20个商品信息"""
        reqeons = requests.get(self.url, headers=self.headers).text
        soup = BeautifulSoup(reqeons, "lxml")
        li_list = soup.select('li[class="zg-item-immersion"]')
        for li in li_list[:20]:
            commodity_url = li.select('a')[0].attrs['href']  # 商品url
            commodity_url = "https://www.amazon.com" + commodity_url
            self.shtc.write(self.num, 2, commodity_url)
            self.xlsx.save('自拍杆美国市场分析表 .xls')
            self.handle_page(commodity_url)
            img_url = li.select('img')[0].attrs['src']  # 图片url
            self.shtc.write(self.num, 1, img_url)
            self.xlsx.save('自拍杆美国市场分析表 .xls')
            price = li.select('span[class="p13n-sc-price"]')[0].get_text()  # 商品价格
            self.shtc.write(self.num, 5, price)
            self.xlsx.save('自拍杆美国市场分析表 .xls')
            try:
                grade = li.select('span[class="a-icon-alt"]')[0].get_text()  # 评分
                grade = "".join(re.findall(r"\d.\d", grade))
                self.shtc.write(self.num, 8, grade)
                self.xlsx.save('自拍杆美国市场分析表 .xls')
            except Exception as e:
                grade = 0
                self.shtc.write(self.num, 8, grade)
                self.xlsx.save('自拍杆美国市场分析表 .xls')
            try:
                comment_amounts = li.select('a[class="a-size-small a-link-normal"]')[0].get_text()  # 总评论数
                self.shtc.write(self.num, 6, comment_amounts)
                self.xlsx.save('自拍杆美国市场分析表 .xls')
            except Exception as e:
                comment_amounts = 0
                self.shtc.write(self.num, 6, comment_amounts)
                self.xlsx.save('自拍杆美国市场分析表 .xls')
            try:
                comment_url1 = li.select('a[class="a-size-small a-link-normal"]')[0].attrs['href']  # 评论url
                comment_url = "https://www.amazon.com" + comment_url1 + "&sortBy=recent&pageNumber=1&reviewerType=avp_only_reviews&filterByStar=one_star"  # 一星评论url
                self.one_star_comment(comment_url)
                comment_amount_url = "https://www.amazon.com" + comment_url1 + "&sortBy=recent&pageNumber=1&reviewerType=avp_only_reviews&filterByStar=positive"  # 实际评论数
                response = requests.get(comment_amount_url, headers=self.headers).text
                soup = etree.HTML(response)
                time.sleep(1)
                comment_amount = "".join(soup.xpath('//*[@id="filter-info-section"]/span[1]/text()'))
                comment_amount1 = re.findall(r"of (\d,\d+)", comment_amount)
                if not comment_amount1:
                    comment_amount = re.findall(r"of (\d+)", comment_amount)
                    self.shtc.write(self.num, 7, comment_amount)
                    self.xlsx.save('自拍杆美国市场分析表 .xls')
                    self.num += 1
                else:
                    print(comment_amount1)
                    self.shtc.write(self.num, 7, comment_amount1)
                    self.xlsx.save('自拍杆美国市场分析表 .xls')
                    self.num += 1
            except Exception as e:
                comment_amount = 0
                self.shtc.write(self.num, 7, comment_amount)
                self.xlsx.save('自拍杆美国市场分析表 .xls')
                self.num += 1

    def handle_page(self, commodity_url):
        """解析商品详情"""
        response = requests.get(commodity_url, headers=self.headers).text
        soup = etree.HTML(response)
        trademark = "".join(soup.xpath('//*[@id="bylineInfo"]/text()'))  # 品牌
        self.shtc.write(self.num, 3, trademark)
        self.xlsx.save('自拍杆美国市场分析表 .xls')
        trait = soup.xpath('//*[@id="feature-bullets"]/ul/li[position()>1]//span/text()')  # 卖点
        num = 9
        for i in trait:
            i = i.replace("\t", "")
            i = i.replace("\n", "")
            self.translate(i, num)
            num += 1
        soup1 = BeautifulSoup(response, "lxml")
        li_list = soup1.find(attrs={"id": "productDetails_detailBullets_sections1"})
        try:
            li_list = li_list.find_all("tr")
            data = {}
            for i in li_list:
                key = i.find(attrs={"class": "a-color-secondary a-size-base prodDetSectionEntry"})
                key = key.text
                try:
                    key = key.replace("\n", "")
                    key = key.replace("\r", "")
                    key = key.replace(" ", "")
                    key = key.replace("\t", "")
                except Exception as e:
                    pass
                try:
                    value = i.find('td', class_="a-size-base").get_text()
                except Exception as e:
                    value = None
                try:
                    value = value.replace("\n", "")
                    value = value.replace("\r", "")
                    value = value.replace("\t", "")
                    value = value.replace(" ", "")
                except Exception as e:
                    pass
                data.update({key: value})
            items = data.get('DatefirstlistedonAmazon')
            if items is None:
                items = data.get('DateFirstAvailable')
                num = 4
                self.translate(items, num)
            else:
                num = 4
                self.translate(items, num)
        except Exception as e:
            num = 4
            current_content = "2008/08/08"
            self.shtc.write(self.num, num, current_content)
            self.xlsx.save('自拍杆美国市场分析表 .xls')

    def page_img(self, img_url):
        """下载图片"""
        img_url1 = requests.get(img_url, headers=self.headers).content
        with open(r'./图片/' + str(self.index) + ".jpg", 'wb') as file:
            file.write(img_url1)
            self.index += 1
        # with open('图片.txt', 'a', encoding='utf8') as f:
        #     # for data in img_url:
        #     json.dump(img_url, f, ensure_ascii=False)
        #     f.write('\n')

    def one_star_comment(self, comment_url):
        """解析一星评论"""
        response = requests.get(comment_url, headers=self.headers).text
        soup = etree.HTML(response)
        one_comment = soup.xpath('//*[@data-hook="review"]//span[@data-hook="review-body"]//span/text()')
        num = 14
        for i in one_comment[:5]:
            try:
                self.translate(i, num)
                num += 1
            except Exception as e:
                pass

    def translate(self, i, num):
        """翻译"""
        data = {
            "f": "auto",
            "t": "auto",
            "w": i
        }
        data_str = parse.urlencode(data)
        # 封装
        response = request.Request(url=self.base_url, headers=self.headers, data=bytes(data_str, encoding="utf-8"))
        req = json.loads(request.urlopen(response).read().decode("utf-8"))
        current_state = req["status"]
        # 在这里判断中译英
        if current_state == 1:
            current_content = req["content"]
            current_content = current_content["out"]
        # 否则则汉译英
        else:
            current_content = req["content"]
            current_content = current_content["word_mean"]
        try:
            current_content = current_content.replace("年", "/")
            current_content = current_content.replace("月", "/")
            current_content = current_content.replace("日", "")
            current_content = current_content.replace("。", "")
        except Exception as e:
            pass
        self.shtc.write(self.num, num, current_content)
        self.xlsx.save('自拍杆美国市场分析表 .xls')

    def run(self):
        self.get_page()
        time.sleep(50)


if __name__ == '__main__':
    while True:
        now = datetime.datetime.now()
        print(now.hour, now.minute)
        amazon = Amazon_books()
        amazon.run()
        # 每隔60秒检测一次
        time.sleep(50)