简单的爬虫代码（Amazon）

最新推荐文章于 2024-01-19 14:06:52 发布

CYB-dengBer

最新推荐文章于 2024-01-19 14:06:52 发布

阅读量850

点赞数

文章标签：爬虫

本文链接：https://blog.csdn.net/m0_62885442/article/details/130755836

版权

爬取亚马逊商品评论

import requests
import re
from bs4 import BeautifulSoup
from wca import create
import json
import openpyxl


headers = {
    'cookie': 'session-id=141-1894534-1178019; session-id-time=2082787201l; i18n-prefs=USD; ubid-main=135-4702725-2873831; lc-main=zh_CN; sp-cdn="L5Z9:CN"; session-token=eNLeoBT2mh3t9TXlsMz30p8eZLtdgMl4JGw3Ww0FtJUuey87Yhnd6MctLDawM81u0gSTSK1MRQ1SGFSaI1IX90G4GI9XQBCdyT5U6YWspKMI3T35uuykbgLp/L5eKfssk4KmEb/pbRi5ASQkT2878ksz/RPCfw320g01wlLoLt8ufDpM92frrZilc6++UusZn5GINn5pwoKgqp1jv3vkE8nfSA6fgx9x/mNb3Sw4ieA=; csm-hit=adb:adblk_no&t:1683439336293&tb:7A42W6J79HAD3WT8J0TD+sa-BXPK16W2AH53A6A0TC4S-A19JVG1Z46DH3DT28F7M|1683439336293',
    'referer': 'https://www.amazon.com/-/zh/dp/B08BHJ5XH9/ref=sr_1_7?crid=3PR2HOFK5T4AB&keywords=Mobile%2Bphone&qid=1680827474&refinements=p_89%3AApple&rnid=2528832011&s=wireless&sprefix=%E6%89%8B%E6%9C%BA%2Caps%2C445&sr=1-7&th=1',
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36 Edg/112.0.1722.68',
}
def main(url):
    response = requests.get(url=url,headers=headers)
    soup = BeautifulSoup(response.text,'lxml')
    comment_list = []
    comment_list = soup.find_all('div', attrs={'data-hook':'review'})


    for comment in comment_list:
        user = comment.find('span', attrs={'class':'a-profile-name'}).text
        content = comment.find('span', attrs={'data-hook': 'review-body'}).text
        score = comment.find('span', attrs={'class': 'a-icon-alt'}).text
        useful = comment.find('span', attrs={'data-hook': 'review-voting-widget'}).text
        status = comment.find('span', attrs={'data-hook':'avp-badge'}).text
        date = comment.find('span', attrs={'data-hook': 'review-date'}).text


        with open('a.txt','a',encoding='utf-8') as f0:
            data = {'user':user,'content':content,'score':score,'useful':useful,'status':status,'data':date}
            f0.write(json.dumps(data))
            f0.write('\n')
            f0.close()

        print('user',user)
        print('content',content)
        print('score',score)
        print('thought that useful-',useful)
        print('status',status)
        print('date',date)
        print('------------------------------------------------------------------------------')

        workbook = openpyxl.load_workbook('a.xlsx')
        worksheet = workbook.active
        data1 = {'user':user,'content':content,'score':score,'useful':useful,'status':status,'data':date}
        worksheet.append(list(data1.keys()))  
        worksheet.append(list(data1.values()))
        workbook.save('a.xlsx')

for i in range(30):
    print('page', i + 1)
    main('https://www.amazon.com/-/zh/product-reviews/B08BHJ5XH9/ref=cm_cr_getr_d_paging_btm_prev_1?ie=UTF8&reviewerType=all_reviews&pageNumber=' + str(i + 1))

f = open('c.txt', 'r', encoding='utf-8')
text = f.read()
create(text)