selenium 爬取春野医生会话记录

使用selenium爬取春野医生会话记录

学习完web自动化后，尝试去爬取一些东西，这个是本人学习总结的一些记录，分为两部分。

这里本人是通过 白月黑羽 所学习的selenium，推荐新手学习，讲的很好！

爬取信息存入到CSV表格中

import selenium
from selenium import webdriver
from selenium.webdriver.common.by import By
import time, csv

driver = webdriver.Chrome()
driver.implicitly_wait(5)

driver.get("https://m.chunyuyisheng.com/qalist/?page=1&high_quality=0")
f = open('会诊信息.csv', 'w', newline='', encoding='utf-8-sig')
f_csv = csv.DictWriter(f,
                       ['医生信息',
                        "聊天记录",
                        ])
f_csv.writeheader()
for a in range(1, 5):
    for i in range(1, len(driver.find_elements(By.CSS_SELECTOR, ".hot-qa-list")) + 1):
        driver.find_element(By.XPATH, f'/html/body/main/section/a[{i}]').click()
        driver.execute_script('window.scrollBy(0,800)')
        # 判断当会话长度不够没有点开会话操作时
        try:
            driver.find_element(By.CSS_SELECTOR, ".showmore-btn").click()
        except selenium.common.exceptions.ElementNotInteractableException:
            pass

        # 处理每页数据
        sessions = driver.find_elements(By.CSS_SELECTOR, ".problem-detail-inner")
        for session in sessions:
            txt = session.find_elements(By.CSS_SELECTOR, '.block-line')
            name = session.find_elements(By.CSS_SELECTOR, '.doctor-name')
            name_txt = [b.text for b in name]
            rows = []
            stringtxt = [i.text for i in txt]
            price = [x.strip() for x in stringtxt if x.strip() != '']
            price = [x.replace('\n', ':') for x in price if x.split('\n') != '']
            data = {
                "医生信息": name_txt[1],
                "聊天记录": price,
            }
            rows.append(data)
            f_csv.writerows(rows)
        driver.back()
    driver.find_element(By.XPATH, "/html/body/main/section/div/a[3]").click()
f.close()

爬取信息存入到mysql中

注意： 因为爬取信息中可能存在表情，所以数据库使用的是utf8mb4格式
比较麻烦的就是对爬取数据的处理，踩过很多坑

# coding=utf-8
import json

import pymysql
import selenium
from selenium import webdriver
from selenium.webdriver.common.by import By

conn = pymysql.connect(user='root', password='123456', db='db1', charset='utf8mb4')
# 创建数据库对象
cursor = conn.cursor()
sql01 = 'create table if not exists 会诊信息 (doctor varchar(10), session varchar(10000)) '
cursor.execute(sql01)

driver = webdriver.Chrome()
driver.implicitly_wait(5)
driver.get("https://m.chunyuyisheng.com/qalist/?page=1&high_quality=0")
for a in range(1, 5):
    for i in range(1, len(driver.find_elements(By.CSS_SELECTOR, ".hot-qa-list")) + 1):
        driver.find_element(By.XPATH, f'/html/body/main/section/a[{i}]').click()
        driver.execute_script('window.scrollBy(0,800)')
        # 判断当会话长度不够没有点开会话操作时
        try:
            driver.find_element(By.CSS_SELECTOR, ".showmore-btn").click()
        except selenium.common.exceptions.ElementNotInteractableException:
            pass

        # 处理每页数据
        sessions = driver.find_elements(By.CSS_SELECTOR, ".problem-detail-inner")
        for session in sessions:
            txt = session.find_elements(By.CSS_SELECTOR, '.block-line')
            name = session.find_elements(By.CSS_SELECTOR, '.doctor-name')
            name_txt = [b.text for b in name]
            # 对数据进行处理， 清理掉多余的空格， 并且将 \n 转化为 ：
            stringtxt = [i.text for i in txt]
            price = [x.strip() for x in stringtxt if x.strip() != '']
            price = [x.replace('\n', ':') for x in price if x.split('\n') != '']
            data = {
                "医生信息": name_txt[1],
                "聊天记录": price,
            }
            # 将爬取的信息转换为字符型
            doctor_name = data['医生信息']
            session_info = data['聊天记录']
            doctor_name = ''.join([str(x) for x in doctor_name])
            session_info = ''.join([str(x) for x in session_info])
            sql02 = 'insert into 会诊信息 (doctor, session) value(%s, %s);'
            cursor.execute(sql02, (doctor_name, session_info))
        driver.back()
    driver.find_element(By.XPATH, "/html/body/main/section/div/a[3]").click()
conn.commit()
conn.close()