爬取好大夫网站医生数据

一、主要过程：
1.爬取医生主页url
2.爬取医生个人信息

二、具体过程
1.爬取医生主页url
观察url链接结构，可以发现同一种病的网址，在不同页面切换时，只有.htm前的数字回变，且页码与链接对应。而对于不同种类的病，如类风湿，改变的也仅是网址种jibing/后的部分。因此，在爬取医生个人主页url时，可以通过设置两个循环来设置读取不同病不同页码。

from requests_html import HTMLSession
from lxml import etree
import pandas as pd
from requests_html import HTMLSession
import time
session = HTMLSession()
sel=' div.oh.zoom.lh180 > p:nth-child(1) > a.blue_a3'
#常见五种疾病：糖尿病、腰椎盘突出、类风湿关节炎、高血压、颈椎病
bings=['tangniaobing','yaozhuijianpantuchu','leifengshixingguanjieyan','gaoxueya','jingzhuibing']
def get_text_link_from_sel(sel):
    mylist = []
    try:
        for bing in bings:
            for i in range(2):
                time.sleep(1)
                url = 'https://www.haodf.com/jibing/'+ str(bing)+'/daifu_all_all_all_all_all_all_'+str(i)+'.htm'
                r = session.get(url)
                results = r.html.find(sel)
                for result in results:
                    mytext = result.text
                    mylink = list(result.absolute_links)[0]
                    mylist.append((mytext, mylink))
        return mylist
    except:
        return None
df = pd.DataFrame(get_text_link_from_sel(sel))
df.to_csv('doctorurl.csv', encoding='gbk', index=False)

2.爬取医生个人信息
与爬取url过程类似，这里根据需要分析的问题共爬取了10个指标，包括姓名、职位、单位、所属科室、综合推荐热度、擅长、简介（包括社会任职、获奖荣誉、科研成果）。
如下图所示，可以自由选择想爬取的内容，复制路径，在下面代码替换一下就ok啦！

from requests_html import HTMLSession
from lxml import etree
import time
import csv
from retrying import retry
df = pd.read_csv('doctorurl_7.csv',header=None)
data = []    #空列表存储团队名称
for j in range(1,500):    
    try:
        url = df.iloc[[j]].values[0][1]
        session = HTMLSession()
        r = session.get(url)
        retry(stop_max_attempt_number=10, wait_fixed=2000)
        #姓名
        sel0='div.banner-wrap > div > div.profile-container.clearfix > div.profile-txt > h1'
        name = r.html.find(sel0)[0].text
        #职位
        sel0_1='div.banner-wrap > div > div.profile-container.clearfix > div.profile-txt > span.doctor-title'
        career = r.html.find(sel0_1)[0].text
        
        sel0_3='div.banner-wrap > div > div.profile-container.clearfix > div.profile-txt > ul > li > a:nth-child(1)'
        position = r.html.find(sel0_3)
        
        try:
            position=position[0].text
        except:
            position=None
        
        sel0_4='div.banner-wrap > div > div.profile-container.clearfix > div.profile-txt > ul > li > a:nth-child(2)'
        department = r.html.find(sel0_4)
        try:
            department=department[0].text
        except:
            department=None
        
        #综合推荐热度
        sel0_5='div.banner-wrap > div > div.profile-container.clearfix > ul > li:nth-child(1) > span.value'
        score = r.html.find(sel0_5)
        try:
            score=score[0].text
        except:
            score=None
        
        sel1='div.wrap-container.clearfix > main > section.container.brief-outer > div > div:nth-child(1) > div > p'
        shanchang = r.html.find(sel1)[0].text
        
        sel1_1='div.wrap-container.clearfix > main > section.container.brief-outer > div > div:nth-child(2) > div > div > p'
        total_intro = r.html.find(sel1_1)
        try:
            total_intro=total_intro[0].text
        except:
            total_intro=None
        
        sel2='div.wrap-container.clearfix > main > section.container.brief-outer > div > div:nth-child(2) > div > div > div:nth-child(2)>p'
        social_job = r.html.find(sel2)
        try:
            social_job=social_job[0].text
        except:
            social_job=None
        
        sel3='div.wrap-container.clearfix > main > section.container.brief-outer > div > div:nth-child(2) > div > div > div:nth-child(3)>p'
        honor = r.html.find(sel3)
        try:   
            honor = honor[0].text
        except:
            honor = None
            
        sel4='div.wrap-container.clearfix > main > section.container.brief-outer > div > div:nth-child(2) > div > div > div:nth-child(4)>p'
        sci_achi = r.html.find(sel4)
        try:
            sci_achi=sci_achi[0].text
        except:
            sci_achi=None
            
        data.append([name,career,position,department,score,total_intro,shanchang,social_job,honor,sci_achi])
        print("第" + str(j) + "次成功")
        if j%10 == 0:
            time.sleep(10)
            
    except IndexError:
        #pass
        print("第" + str(j) + "次失败:")