小红书PC端文章页面信息提取

# -*- coding:utf-8 -*-

import json, re, time
import requests
import uuid

def str_to_int(str_obj):
    if str_obj:
        try:
            return int(str_obj)
        except Exception:
            return 0
    return 0


def gotXhsData(url, cookie):
    headers = {
        'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3',
        'accept-encoding': 'gzip, deflate, br',
        'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8',
        'cache-control': 'max-age=0',
        'upgrade-insecure-requests': '1',
        'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36',
        'cookie': cookie
    }

    response = requests.get(url=url, headers=headers, verify=False)

    if response:
        res = response.content.decode('utf-8')

        # re获取json 数据
        user_info_list = re.findall(r"<script>window.__INITIAL_SSR_STATE__[\s\S]*?{([\s\S]+?)}</script>", res)

        # 完善 json 格式
        if user_info_list:
            user_info_str = user_info_list[0].strip()
            user_info_str = "{" + "{}".format(user_info_str) + "}"

            user_info_json = json.loads(user_info_str.replace('undefined', 'null'))
            note_info = user_info_json.get('NoteView').get('noteInfo')

            # 文章信息
            if note_info:
                details_dict = dict()
                # 所有的图片的链接
                image_url_list = ['https:' + i['url'].replace('\u002F', '/') for i in note_info['imageList'] if
                                  i.get('url').strip()]
                if image_url_list:
                    details_dict['note