# 创建时间 : 2021/8/19
# 作者 : 小章哥儿
import requests
import random
from lxml import etree
def header():
head = [
{"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:87.0) Gecko/20100101 Firefox/87.0"},
{"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:84.0) Gecko/20100101 Firefox/84.0"},
{"User-Agent": "Mozilla/5.0 (Windows NT 5.1; rv:21.0) Gecko/20130331 Firefox/21.0"},
{"User-Agent": "Opera/9.80 (Windows NT 6.1; U; en-GB) Presto/2.7.62 Version/11.00"},
{"User-Agent": "Opera/9.80 (Windows NT 6.1 x64; U; en) Presto/2.7.62 Version/11.00"},
{'User-Agent': "Opera/9.80 (Windows NT 6.0; U; en) Presto/2.7.39 Version/11.00"}]
return random.choice(head)
# 构造列表URL
def get_html(num):
headers = header()
for i in range(10, num):
url = f'http://so.mama.cn/search?token=1f7227860ec434a9f1274c3a1c1dacc8&csite=all&source=all&q=%E6%99%BA%E5%8A%9B&x=0&y=0&size=15&sortMode=1&page={i}'
html = requests.get(url, headers=headers)
html = etree.HTML(html.text)
lis = html.xpath('//div[@class="main"]/div[@class="quanzi-com"]/h1/a/@href')
content(lis)
# 内容页URL 数据提取内容
def content(lis):
for i in lis:
url = f'{i}'
print('采集链接成功:', url)
headers = header()
html = requests.get(url, headers=headers)
html.encoding = 'UTF-8'
html = etree.HTML(html.text)
txt = html.xpath('//div[@class="topic-item__content topic-detail-content"]//text()')
for i in txt:
s = i.replace('。', '。\n')
st = s.strip()
with open('C:\\Users\\语料库\\AAA.json', 'a+', encoding='utf-8') as f:
f.write(st)
print('写入成功', st)
get_html(51)
06-07
3558
![](https://csdnimg.cn/release/blogv2/dist/pc/img/readCountWhite.png)
08-23
120
![](https://csdnimg.cn/release/blogv2/dist/pc/img/readCountWhite.png)