python | 妈妈网（mama.cn）爬虫采集代码

# 创建时间 : 2021/8/19
# 作者 : 小章哥儿
import requests
import random
from lxml import etree


def header():
    head = [
        {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:87.0) Gecko/20100101 Firefox/87.0"},
        {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:84.0) Gecko/20100101 Firefox/84.0"},
        {"User-Agent": "Mozilla/5.0 (Windows NT 5.1; rv:21.0) Gecko/20130331 Firefox/21.0"},
        {"User-Agent": "Opera/9.80 (Windows NT 6.1; U; en-GB) Presto/2.7.62 Version/11.00"},
        {"User-Agent": "Opera/9.80 (Windows NT 6.1 x64; U; en) Presto/2.7.62 Version/11.00"},
        {'User-Agent': "Opera/9.80 (Windows NT 6.0; U; en) Presto/2.7.39 Version/11.00"}]
    return random.choice(head)


# 构造列表URL
def get_html(num):
    headers = header()
    for i in range(10, num):
        url = f'http://so.mama.cn/search?token=1f7227860ec434a9f1274c3a1c1dacc8&csite=all&source=all&q=%E6%99%BA%E5%8A%9B&x=0&y=0&size=15&sortMode=1&page={i}'
        html = requests.get(url, headers=headers)
        html = etree.HTML(html.text)
        lis = html.xpath('//div[@class="main"]/div[@class="quanzi-com"]/h1/a/@href')
        content(lis)


# 内容页URL 数据提取内容
def content(lis):
    for i in lis:
        url = f'{i}'
        print('采集链接成功：', url)
        headers = header()
        html = requests.get(url, headers=headers)
        html.encoding = 'UTF-8'
        html = etree.HTML(html.text)
        txt = html.xpath('//div[@class="topic-item__content topic-detail-content"]//text()')
        for i in txt:
            s = i.replace('。', '。\n')
            st = s.strip()
            with open('C:\\Users\\语料库\\AAA.json', 'a+', encoding='utf-8') as f:
                f.write(st)
            print('写入成功', st)

get_html(51)