Python 简单爬虫农药英雄技能-CSDN博客

本文链接：https://blog.csdn.net/mxxxkuku/article/details/96184189

简单试一下爬取信息，更新以后有记录到数据库

过程遇到的问题：

1、中文乱码

解决：在代码中指定编码： responce.encoding = 'GBK'

2、想要根据href链接中的关键字筛选信息

解决：导入 re， href = re.compile('herodetail')

未导入数据库：

import requests
import re

from bs4 import BeautifulSoup

url = 'https://pvp.qq.com/web201605/herolist.shtml'
#解析
def get_link(url):
    responce = requests.get(url)
    responce.encoding = 'GBK'
    soup = BeautifulSoup(responce.text,'lxml')
    return soup

soup = get_link(url)

#英雄详情的链接
link_ = soup.find_all('a',href = re.compile('herodetail'))


for i in range(len(link_)):
    #将链接补充完整
    link = 'https://pvp.qq.com/web201605/'+ link_[i].get('href')
    print(link)

    #解析英雄技能详情网页
    soup = get_link(link)
    #查找技能
    skill = soup.find_all('div',class_ = 'show-list')
    #输出技能（text类型）
    for i in range(len(skill)-1):
        print(skill[i].text)

-------------------------------------------------------------------------------------------------------------------------------------------------------------------------

更新：记录到数据库

因为英雄技能太长数据库设计的时候类型选用text

import requests
import re
from bs4 import BeautifulSoup
#解析链接
def get_link(url):
    responce = requests.get(url)
    responce.encoding = 'GBK'
    soup = BeautifulSoup(responce.text,'lxml')
    return soup

#将获取的链接放到list中返回
def get_links(url):
    soup = get_link(url)
    link_ = soup.find_all('a',href = re.compile('herodetail'))
    links = []
    for i in range(len(link_)):
        links.append ('https://pvp.qq.com/web201605/'+ link_[i].get('href'))
    return links

#将获取的信息放到list里返回
def get_skills(link):
    soup = get_link(link)
    skill = soup.find_all('div',class_ = 'show-list')
    skill_ = []
    for i in range(len(skill)):
        skill_.append(skill[i].text)
    return skill_
            
#连接数据库
import pymysql
def connDB():
    conn = pymysql.connect(host='localhost',user='root',passwd='root',db='python_pachong',charset='utf8')
    cur = conn.cursor()
    return (conn,cur)

def connClose(conn,cur):
    cur.close()
    conn.close()


connDB1 = connDB()

sql_1 = 'insert into moba_skills(link, passive, skill_1, skill_2, skill_3, skill_4)values '

#循环链接
for link in get_links('https://pvp.qq.com/web201605/herolist.shtml'):
    skill = get_skills(link)

        sql = sql_1 + ' ("' + link + '","'+ skill[0] + '","'+ skill[1] + '","'+ skill[2] + '","'+ skill[3] + '","'+ skill[4] + '")'

    
    connDB1[1].execute(sql)
connDB1[0].commit()

connClose(connDB1[0],connDB1[1])

网格查看可能看不出技能，需要表单查看