用requests方式爬取亚马逊Best sellers商品数据

 亚马逊Best Sellers没有什么反爬机制，但有限制，就是爬着爬着就爬不动了，爬的类目也不多，就clothing这个类目，所以就将就用着，在里面加了个是否爬取，如果之前爬了就不爬了。

 直接下代码，也没有什么难度。就是爬不动的时候要重新爬，而且也没有设置再自动重新爬。太懒了吧....可以加个queue和开启多线程，应该会快很多。之后再改改

import requests
import csv
from bs4 import BeautifulSoup
import pandas as pd


class AmazonSpider():
    def __init__(self):
        '''
        url:爬取的类目链接
        category：爬取的类目
        img_dict：图片和asin字典
        items:每页的asin列表
        '''
        self.headers = {'user-agent': 'Mozilla/5.0'}
        self.url = None
        self.catelogy = None
        self.img_dict = {}
        self.items = None

    def response(self):

        # 加入代理，因为代理是免费的，所以实际爬取的时候根本就不用，太慢了
        # proxies = {
        #     'http': 'http://116.17.102.20:3128',
        #     'http': 'http://103.247.162.218:3128',
        #     'http': 'http://3.36.65.133:80',
        # }
        try:
            r = requests.get(self.url, headers=self.headers)
            r.raise_for_status()
            # r.encoding是根据header得来的，r.apparent_encoding是分析内容得来的。此处用哪个其实都可以
            r.encoding = r.apparent_encoding
            soup = BeautifulSoup(r.text, 'lxml')
            self.items = soup.select("#zg-ordered-list > li.zg-item-immersion")
            if len(self.items) < 1:
                print(self.url)
        except Exception as e:
            print("%s爬取失败" % self.url,e)

    def parse(self):
        for item in self.items:
            try:
                asin = item.select("a.a-link-normal")[0].get("href").split("/")[3]
            except Exception as e:
                asin = None
                print(e)
                
            # title的class样式会改变，目前发现的是这两个
            try:
                title = item.select("div.p13n-sc-truncate")[0].text.strip()
            except IndexError:
                title = item.select("div.p13n-sc-truncated")[0].text.strip()
            except Exception as e:
                print("title_error",e)

            yield {
                'catelogy': self.catelogy,
                'url': self.url,
                "asin": asin,
                "item_img": item.select("div.a-section > img")[0].get("src") if len(
                    item.select("div.a-section > img")) else None,
                "item_rank": item.select("span.zg-badge-text")[0].text if len(
                    item.select("span.zg-badge-text")) else None,
                "item_title": title,
                "item_url": item.select("a.a-link-normal")[0].get("href") if len(
                    item.select("a.a-link-normal")) else None,
                "item_star": item.select("span.a-icon-alt")[0].text if len(item.select("span.a-icon-alt")) else None,
                "item_number": item.select("a.a-size-small")[0].text if len(item.select("a.a-size-small")) else None,
                "item_review_url": "https://www.amazon.com" + item.select("a.a-size-small")[0].get("href") if len(
                    item.select("a.a-size-small")) else None,
                "price": item.select("span.a-size-base")[0].text if len(item.select("span.a-size-base")) else None,
            }

    def save(self):
        with open('asin.csv', 'a', newline='', encoding='gb18030') as f:
            fieldnames = ['catelogy', 'url', 'asin', 'item_img', 'item_rank', 'item_title', 'item_url', 'item_star',
                          'item_number', 'item_review_url', "price"]
            writer = csv.DictWriter(f, fieldnames=fieldnames)
            writer.writeheader()
            for item in self.parse():
                writer.writerow(item)
                self.img_dict[item['asin']] = item['item_img']

    def request_img(self):
        for img_url in self.img_dict.keys():
            try:
                img = requests.get(self.img_dict[img_url], headers=self.headers)
                with open(r"D:\python\amazon_asin\asin_img\{}.jpg".format(img_url), "wb") as fp:
                    fp.write(img.content)
            except:
                print(img_url)

    def run(self):
        #打开需要爬取的类目链接，类目链接之前已经爬好了
        urls_df = pd.read_excel(r'亚马逊类目链接.xlsx')
        urls_df = urls_df[urls_df['爬取'] == '是']
        urls = urls_df['爬取链接']
        catelogys = urls_df['目录']
        for ur, ca in zip(urls, catelogys):
            self.url = ur
            self.catelogy = ca
            self.response()
            self.save()
            self.request_img()


def main():
    spider = AmazonSpider()
    spider.run()


if __name__ == "__main__":
    main()