必须安装的第三方库
pip install urllib
pip install pypingin
# -*- coding:utf-8 -*-
import re
import urllib.request
import urllib
import xlwt
import time
import pypinyin
import requests
from requests.exceptions import ReadTimeout,HTTPError,RequestException
# 网址
buwei_url1 = "https://jbk.99.com.cn/"
buwei_url2 = ".html"
# 爬取部位信息https://jbk.99.com.cn/部位/
buwei_url = buwei_url1 + 'buwei/'
try:
buwei_urlop = urllib.request.urlopen(buwei_url, timeout=100)
# buwei_urlop = requests.get(buwei_url)
# print(buwei_urlop)
except Exception as e:
print("超时")
# HTTP异常
except HTTPError:
print('httperror')
# 请求异常
except RequestException:
print('reqerror')
data_buwei = buwei_urlop.read().decode('GBK') # !!!!!原网页编码方式为GBK
# 得到部位名称,保存在列表ls_buwei中
link_buwei = "<div class=\"part-txt\">\\s*<h3>\\s*<span>\\s*(.*?)</span>"
ls_buwei = []
patten = re.compile(link_buwei)
ls_buwei = patten.findall(data_buwei)
print(ls_buwei)
# 新建表格文本
book = xlwt.Workbook(encoding='utf-8')
heads = [u'疾病名称', u'链接', u'发病部位', u'就诊科室', u'体状特征', u'症状']
def hp(word):
s = ''
for i in pypinyin.pinyin(word, style=pypinyin.NORMAL):
s += ''.join(i)
return s
# 得到各部位疾病名称
for i, bw in enumerate(ls_buwei):
catch = 1 # 更换部位开关
ii = 31
# 同一部位疾病放置在一个sheet中
sheet = book.add_sheet(bw, cell_overwrite_ok=True)
for j, head in enumerate(heads):
sheet.write(0, j, head)
while catch == 1:
ii += 1
if ii >60:
break
# 具体部位网址
buwei_url_s = buwei_url + hp(bw) + '-' + str(ii) + buwei_url2
# print(buwei_url_s)
try:
# 抓网页
buwei_urlop_s = urllib.request.urlopen(buwei_url_s, timeout=100)
except Exception:
print('超时')
continue
try:
data_buwei = buwei_urlop_s.read().decode('GBK')
# 网页源代码抓取
except:
continue
# 判断是否是最后一页,对比最后一页源代码可得
link_is_msg = "<div class=\"part-cont3\">\\s*(.*?)<div class=\"digg\">"
patten_is_msg = re.compile(link_is_msg)
is_msg = patten_is_msg.findall(data_buwei)
# 如果是最后一页切换到下一个部位
if is_msg == ['']:
catch = 0
continue
# 获取本页所有疾病名称及链接,结果保存在列表ls_des中
link_buwei = "<dd>\\s*<h3>\\s*<span>\\s*</span>\\s*<a href=\"(.*?)\"\\s*title=\".+\" target=\"_blank\">(.*?)</a></h3>"
patten_buwei = re.compile(link_buwei)
ls_des = patten_buwei.findall(data_buwei)
# 抓取ls_des中各疾病的具体信息
iii = 1
for x in ls_des:
t1 = time.clock()
row = (ii - 1) * 10 + iii
print(row)
# 将疾病名称和链接写入表格
sheet.write(row, 0, x[1])
sheet.write(row, 1, x[0])
# 疾病链接
jibing_url = 'https:' + x[0] + 'zhengzhuang.html'
print(jibing_url)
try:
jibing_urlop = urllib.request.urlopen(jibing_url, timeout=100)
except Exception:
print('超时')
continue
try:
# 源代码获取
data_jibing = jibing_urlop.read().decode('GBK')
except:
continue
# 发病部位信息
link_fbbw = "<li><font>发病部位:(.*?)</a></li>"
patten_fbbw = re.compile(link_fbbw)
st_abbw = patten_fbbw.findall(data_jibing)
if st_abbw == []:
hz_fbbw = re.findall('[\u4e00-\u9fa5]+','')
else:
hz_fbbw = re.findall('[\u4e00-\u9fa5]+', st_abbw[0]) # 中文提取
fbbw = ','.join(ff for ff in hz_fbbw)
sheet.write(row, 2, fbbw)
# 抓写就诊科室信息
link_jzks = "<li><font>就诊科室:(.*?)</a></li>"
patten_jzks = re.compile(link_jzks)
st_jzks = patten_jzks.findall(data_jibing)
if st_jzks == []:
jzks = ','.join(ff for ff in re.findall('[\u4e00-\u9fa5]+','')) # 中文提取
else:
jzks = ','.join(ff for ff in re.findall('[\u4e00-\u9fa5]+', st_jzks[0])) # 中文提取
sheet.write(row, 3, jzks)
# 抓写体状特征
link_tztz = "<dt><a>症状体征:</a></dt>\\s*<dd>\\s*<ul>(.*?)</a></li></ul>"
patten_tztz = re.compile(link_tztz)
st_tztz = patten_tztz.findall(data_jibing)
if st_tztz == []:
tztz = ','.join(ff for ff in re.findall('[\u4e00-\u9fa5]+','')) if st_tztz != [] else 0
else:
tztz = ','.join(ff for ff in re.findall('[\u4e00-\u9fa5]+', st_tztz[0])) if st_tztz != [] else 0
sheet.write(row, 4, tztz)
iii += 1
t2 = time.clock()
print(t2 - t1)
book.save("F:/li结果/jibing11.xls") # 保存表格