csdn博主文章存入本地

import re
import requests
from lxml import etree
import os

headers = {
    'User-Agent': 'Mozilla/5.0 (Linux; Android 8.0.0; Pixel 2 XL Build/OPD1.170816.004) AppleWebKit/537.36 (KHTML, '
                  'like Gecko) Chrome/91.0.4472.114 Mobile Safari/537.36',
}

for i in range(100):
    url_list = f'https://blog.csdn.net/community/home-api/v1/get-business-list?page={i}&size=20&businessType=blog' \
               f'&orderby=&noMore=false&year=&month=&username=m0_50546016 '

    rep_list = requests.get(url_list, headers=headers).text
    
    url_l = re.findall('"url":"(.*?)"', rep_list)
    if url_l:
        for url in url_l:

            lj_name = url.split('/')[-1]

            list_res = requests.get(url, headers=headers)

            r = etree.HTML(list_res.text)

            title = ''.join(r.xpath('//title/text()'))

            content_text = r.xpath('//div[@id="content_views"]')  # 资讯正文1

            content_text = str(etree.tostring(content_text[0], encoding='utf-8'), encoding='utf-8')

            filename = f'./MD/{lj_name}/'

            if not os.path.exists(filename):  # 判断创建文件夹
                os.makedirs(filename)

            with open(filename + title + '.text', 'w', encoding='utf-8') as f:

                f.write(content_text.replace('\U0001f631', ''))

            print('okok')


    else:
        break