亚马逊Best Sellers没有什么反爬机制,但有限制,就是爬着爬着就爬不动了,爬的类目也不多,就clothing这个类目,所以就将就用着,在里面加了个是否爬取,如果之前爬了就不爬了。
直接下代码,也没有什么难度。就是爬不动的时候要重新爬,而且也没有设置再自动重新爬。太懒了吧....可以加个queue和开启多线程,应该会快很多。之后再改改
import requests
import csv
from bs4 import BeautifulSoup
import pandas as pd
class AmazonSpider():
def __init__(self):
'''
url:爬取的类目链接
category:爬取的类目
img_dict:图片和asin字典
items:每页的asin列表
'''
self.headers = {'user-agent': 'Mozilla/5.0'}
self.url = None
self.catelogy = None
self.img_dict = {}
self.items = None
def response(self):
# 加入代理,因为代理是免费的,所以实际爬取的时候根本就不用,太慢了
# proxies = {
# 'http': 'http://116.17.102.20:3128',
# 'http': 'http://103.247.162.218:3128',
# 'http': 'http://3.36.65.133:80',
# }
try:
r = requests.get(self.url, headers=self.headers)
r.raise_for_status()
# r.encoding是根据header得来的,r.apparent_encoding是分析内容得来的。此处用哪个其实都可以
r.encoding = r.apparent_encoding
soup = BeautifulSoup(r.text, 'lxml')
self.items = soup.select("#zg-ordered-list > li.zg-item-immersion")
if len(self.items) < 1:
print(self.url)
except Exception as e:
print("%s爬取失败" % self.url,e)
def parse(self):
for item in self.items:
try:
asin = item.select("a.a-link-normal")[0].get("href").split("/")[3]
except Exception as e:
asin = None
print(e)
# title的class样式会改变,目前发现的是这两个
try:
title = item.select("div.p13n-sc-truncate")[0].text.strip()
except IndexError:
title = item.select("div.p13n-sc-truncated")[0].text.strip()
except Exception as e:
print("title_error",e)
yield {
'catelogy': self.catelogy,
'url': self.url,
"asin": asin,
"item_img": item.select("div.a-section > img")[0].get("src") if len(
item.select("div.a-section > img")) else None,
"item_rank": item.select("span.zg-badge-text")[0].text if len(
item.select("span.zg-badge-text")) else None,
"item_title": title,
"item_url": item.select("a.a-link-normal")[0].get("href") if len(
item.select("a.a-link-normal")) else None,
"item_star": item.select("span.a-icon-alt")[0].text if len(item.select("span.a-icon-alt")) else None,
"item_number": item.select("a.a-size-small")[0].text if len(item.select("a.a-size-small")) else None,
"item_review_url": "https://www.amazon.com" + item.select("a.a-size-small")[0].get("href") if len(
item.select("a.a-size-small")) else None,
"price": item.select("span.a-size-base")[0].text if len(item.select("span.a-size-base")) else None,
}
def save(self):
with open('asin.csv', 'a', newline='', encoding='gb18030') as f:
fieldnames = ['catelogy', 'url', 'asin', 'item_img', 'item_rank', 'item_title', 'item_url', 'item_star',
'item_number', 'item_review_url', "price"]
writer = csv.DictWriter(f, fieldnames=fieldnames)
writer.writeheader()
for item in self.parse():
writer.writerow(item)
self.img_dict[item['asin']] = item['item_img']
def request_img(self):
for img_url in self.img_dict.keys():
try:
img = requests.get(self.img_dict[img_url], headers=self.headers)
with open(r"D:\python\amazon_asin\asin_img\{}.jpg".format(img_url), "wb") as fp:
fp.write(img.content)
except:
print(img_url)
def run(self):
#打开需要爬取的类目链接,类目链接之前已经爬好了
urls_df = pd.read_excel(r'亚马逊类目链接.xlsx')
urls_df = urls_df[urls_df['爬取'] == '是']
urls = urls_df['爬取链接']
catelogys = urls_df['目录']
for ur, ca in zip(urls, catelogys):
self.url = ur
self.catelogy = ca
self.response()
self.save()
self.request_img()
def main():
spider = AmazonSpider()
spider.run()
if __name__ == "__main__":
main()