import os
import requests
from bs4 import BeautifulSoup
# learning example:
# astr = '''aaaaa何时when 杖尔看see南雪snow,我me与梅花plum blossom两白头'''
# res = re.findall('[\u4e00-\u9fa5]', astr)
# print(res)
# a = []
# for i in range(0x4E00, 0x9FA6):
# a.append(str(hex(i)).lstrip('0x'))
# print(a, len(a))
# unicode = b"\u" + chinese[0].encode("unicode-escape")
# print(unicode, unicode.decode("unicode-escape"))
#
# 处理生成对应的每个汉字 以及 每个汉字字典网址的请求url
urls = [] # 所有汉字的字典网址
chinese = [] # 所有汉字 (20902个)
for i in range(0x4E00, 0x9FA6):
chinese.append((b"\u" + str(hex(i)).lstrip("0x").encode("unicode-escape")).decode("unicode-escape"))
urls.append("http://www.guoxuedashi.net/zidian/" + str(hex(i)).lstrip("0x") + ".html")
print("共有{}个所要请求的url 以及 对应的{}个汉字".format(len(urls), len(chinese)))
# 请求头 伪装浏览器 python自带浏览器会被反爬
request_header = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/96.0.4664.93 Safari/537.36"
}
# 汉字所要保存的路径
path = "E:\汉字数据集"
# 遍历请求每个汉字字典url
for (i, url), character in zip(enumerate(urls), chinese):
response = requests.get(url, request_header, verify=False)
print("******************第 {} 个 url --- {}".format(i+1, url), response, "******************")
response.encoding = response.apparent_encoding
response = response.text
soup = BeautifulSoup(response, "html.parser")
all_a = soup.find_all("a")
href = "http://www.guoxuedashi.net" # 得到这个汉字的演变url
for a in all_a:
if ' 详情\r\n ' in list(a.children):
# print(a) 测试用
href += a["href"]
r = requests.get(href, request_header, verify=False)
print("******************", href, r, "******************")
r.encoding = r.apparent_encoding
r = r.text
s = BeautifulSoup(r, "html.parser")
imgs = s.find_all("img", {"width": "80"})
print("****当前汉字\"{}\"文件夹***共有{}张汉字图片***正在生成...****".format(character, len(imgs)))
# 如果没有这个汉字的图片 跳过创建这个汉字文件夹 执行下一个汉字
if(len(imgs) == 0):
continue
for j, img in enumerate(imgs):
src = img["src"]
rp = requests.get(src, request_header, verify=False)
chinese_path = os.path.join(path, character)
if not os.path.exists(chinese_path):
os.makedirs(chinese_path)
img_path = os.path.join(chinese_path, str(j+1) + '.png')
with open(img_path, 'wb+') as pic:
pic.write(rp.content)
print("****当前汉字\"{}\"文件夹***共有{}张汉字图片***生成完毕!!!****".format(character, len(imgs)))
执行结果