jingdong

前言

京东是电商平台，最好获取数据的平台的吧。基本没有什么反爬措施。

本文章只是实现了 京东 关键词和单个商品信息获取的功能。

如果要稳定的话，还需修改代码。

本文只做学习用，请知悉。如有侵权，请联系作者。

import requests,re,json
from lxml import etree
from urllib import parse

proxies = ''

class JingDong(object):
    def __init__(self,input_str,input_type):
        if input_type == 1:
            print('单个商品链接爬取')
            self.get_goods(input_str)
        elif input_type == 2:
            print('关键词爬取，目前限制最大1000')
            self.get_keywords(input_str)
        else:
            pass

    def get_keywords(self,keywords):
        words = parse.quote(keywords)
        page = 1
        s = 0
        total = 0
        list1 = []
        while page <= 30:
            url = 'https://search.jd.com/s_new.php?keyword={}&psort=3&suggest=4.def.0.V04--38s0&wq={}&psort=3&page={}&s={}&click=0'.format(words,words,str(page),str(s))
            headers = {
                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36',
                'Accept-Language': 'zh-CN,zh;q=0.9',
                'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
                'Accept-Encoding': 'gzip, deflate, br',
                'Host': 'search.jd.com'
            }
            try:
                key_resp = requests.get(url,headers=headers,proxies=proxies,verify=False)
                resp_html = etree.HTML(key_resp.text)
                goods_list = resp_html.xpath('*//div[@id="J_goodsList"]//li')
                print(len(goods_list))
                total += len(goods_list)
                list = []
                for j in goods_list:

                    href = j.xpath('.//a/@href')[0]
                    list.append(href)
                    list1.append(href)
                print(list)
            except:
                pass
            page += 1
            s += 30
        print('total goods num....',total)
        list2 = set(list1)
        print('去重后goods num.....',len(list2))
        # for j in list2:
        #     j = 'https:' + j
        #     self.get_goods(j)

    def get_goods(self,url):
        headers = {
            'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36'}
        resp = requests.get(url,headers=headers,proxies=proxies,verify=False)
        selector = etree.HTML(resp.text)
        # 商品原始链接
        commodity_url = selector.xpath('//link[@rel="canonical"]/@href')[0]
        # 商品ID
        goods_id = re.findall("(\d+)",commodity_url)[0]
        # 商品名称
        goods_name = selector.xpath('*//div[@class="item ellipsis"]/@title')[0]
        # 店铺名称
        shop_name = selector.xpath('*//div[@class="name"]/a/text()')[0]
        # 店铺链接
        shop_link = selector.xpath('*//div[@class="name"]/a/@href')[0]
        # 商品详情
        # 1.文本信息
        detail_list = selector.xpath('//ul[@class="parameter2 p-parameter-list"]/li/text()')
        detail_content = ""
        for i in detail_list:
            i = "".join(i.split()) + ","
            detail_content += i

        # 2.详情图片
        detail_link = re.findall("desc: '(.*?)',",resp.text)[0]
        detail_img_list = self.get_detail_link(detail_link,commodity_url)
        # 轮播图
        picture_list = selector.xpath('//ul[@class="lh"]/li/img/@src')
        for j in range(len(picture_list)):
            picture_list[j] = 'https:' + picture_list[j].replace('/n5/','/n0/')

        # 商品属性
        property = []
        coloeSize = re.findall("colorSize: \[(.*?)\]",resp.text)
        if len(coloeSize) > 0:
            str_size = "[" + coloeSize[0] + "]"
            list = eval(str_size)
            # price
            url = 'https://p.3.cn/prices/mgets?skuIds='
            for m in list:
                skuId = str(m['skuId'])
                url += skuId + ','
            resp_price = requests.get(url,headers=headers,verify=False)
            json_price = json.loads(resp_price.text)
            for h in json_price:
                price_skuid = h['id']
                p_skuid = re.findall(('\d+'),price_skuid)[0]
                for q in list:
                    if str(q['skuId']) == p_skuid:
                        q['price'] = h['p']
                    else:
                        pass

            for i in list:
                pro_dict = {}
                pro_dict['skuId'] = str(i['skuId'])
                pro_dict['pro'] = i
                if 'price' in i:
                    pro_dict['price'] = i['price']
                else:
                    pro_dict['price'] = ''
                property.append(pro_dict)
                #  skuid  img
            img = selector.xpath('//div[@id="choose-attr-1"]/div[@class="dd"]/div')
            for j in img:
                sku_id = j.xpath('./@data-sku')[0]
                img = j.xpath('./a/img/@src')[0]
                for k in property:
                    if sku_id == k['skuId']:
                        k['image'] = img
                    else:
                        pass
        else:
            property = []
            skuId = goods_id
            url = 'https://p.3.cn/prices/mgets?skuIds='+ skuId
            resp_price = requests.get(url, headers=headers, verify=False)
            json_price = json.loads(resp_price.text)
            # print(json_price)
            price = json_price[0]['p']
            pro_dict = {}
            pro_dict['skuId'] = skuId
            pro_dict['price'] = price
            pro_dict['image'] = ''
            pro_dict['pro'] = []
            property.append(pro_dict)


        comments_list = self.get_comments(goods_id,commodity_url)
        goods = {}
        goods['goods_name'] = goods_name
        goods['shop_name'] = shop_name
        goods['shop_link'] = shop_link
        goods['detail_content'] = detail_content
        goods['detail_img_list'] = detail_img_list
        goods['round_pictures'] = picture_list
        goods['property'] = property
        goods['comments'] = comments_list
        print(goods)





    def get_detail_link(self,url,referer_url):
        # https://cd.jd.com/description/channel?skuId=33406234360&mainSkuId=11427847914&charset=utf-8&cdn=2&callback=showdesc
        real_url = "https:" + url
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36',
            'Referer': 'https:' + referer_url,
            'Accept-Encoding': 'gzip, deflate, br',
            'Accept-Language': 'zh-CN,zh;q=0.9',
            'Host': 'cd.jd.com',
        }
        resp = requests.get(real_url,headers=headers,proxies=proxies,verify=False)
        # print(resp.text)
        json_data = json.loads(resp.text)
        content = json_data['content']
        img_list = re.findall('data-lazyload="(.*?)"',content)
        return img_list


    def get_comments(self,id,referer_url):
        page = 0
        comment_url = 'https://club.jd.com/comment/productPageComments.action?callback=fetchJSON_comment98&productId={}&score=3&sortType=5&page={}&pageSize=10&isShadowSku=0&rid=0&fold=1'.format(id,str(page))
        com_headers = {
            'Referer': 'https:' + referer_url,
            'Host': 'club.jd.com',
            'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36'
        }
        com_resp = requests.get(comment_url,headers=com_headers,proxies=proxies,verify=False)
        # print(com_resp.text)
        comment_str = re.findall('fetchJSON_comment98\((.*)\);',com_resp.text)[0]
        # print(comment_str)
        comment_json = json.loads(comment_str)
        print(len(comment_json['comments']))
        comment_list = []
        for j in comment_json['comments']:
            comment_dict = {}
            comment_dict['id'] = j['id']
            comment_dict['content'] = j['content']
            comment_dict['images'] = [k['imgUrl'] for k in j['images']]
            comment_list.append(comment_dict)
        return comment_list


JingDong('https://item.jd.com/69422164809.html',1)