使用urllib爬取数据，lxml、bs4、正则解析数据合集，pymysql存储数据

根据输入的类别和页数来爬取糗事百科,基于练习，分别使用xpath匹配段子模块、正则匹配视频模块、bs4 匹配热图模块。

# 连接数据库类
class SQL_connect:
    conn = ""
    cur = ""
    def __init__(self):
        self.conn = pymysql.connect(
            host='localhost',
            port=3306,
            user='root',
            password='123456',
            db='pachong'
        )
        self.cur = self.conn.cursor()

    # 添加段子模块元素到数据库table1表中
    def ins_text(self, a, b):
        s = 'insert into table1 values(%s,%s)'
        self.cur.execute(s, (a, b))
        self.conn.commit()

    # 添加视频模块元素到数据库vid表中
    def ins_vid(self, a):
        s = 'insert into video values(%s)'
        self.cur.execute(s, a)
        self.conn.commit()

    # 添加热图模块元素到数据库img表中
    def ins_img(self, a):
        s = 'insert into image values(%s)'
        self.cur.execute(s, a)
        self.conn.commit()

# 请求类
class URL_Request():
    def req(self,a,b):
        for i in range(1,b+1):
            url = "https://www.qiushibaike.com/{type}/page/{pge}/".format(type= a, pge= i)

            header = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:82.0) Gecko/20100101 Firefox/82.0"}
            res = request.Request(url=url, headers=header)
            # 获取响应
            response = request.urlopen(res)
            # 读取响应内容
            yya = response.read().decode("utf-8")
        return yya

#解析HTML类
class Pipei():

    # xpath匹配段子模块
    def Text_type(self,a,b):
        u = URL_Request()
        yya = u.req(a, b)
        yyb = etree.HTML(yya)
        # 作者
        yyc = yyb.xpath('//div[@class="article block untagged mb15 typs_hot"]/div/a/h2/text()')
        # 内容
        yye = yyb.xpath('//div[@class="article block untagged mb15 typs_hot"]/a/div/span/text()')
        list1 = []
        list2 = []
        for i in yyc:
            a = i.strip()
            list1.append(a)
        for i in yye:
            a = i.strip()
            list2.append(a)
        print("成功抓取到作者信息：",list1)
        print("成功抓取到发布内容：",list2)
        sql = SQL_connect()
        for i in range(len(list1)):
            sql.ins_text(list1[i], list2[i])
        print("------------------------------数据已成功上传数据库！------------------------------")

    # 正则匹配视频模块
    def Video_type(self, a, b):
        u = URL_Request()
        yya = u.req(a, b)
        yyb = re.findall(r'<source src="(.*?hd.mp4)', yya)
        list1 = []
        for i in yyb:
            j = "http:"+i
            list1.append(j)
            for z in list1:
                print("成功抓取到视频路径：",z)
        sql = SQL_connect()
        for i in range(len(list1)):
            sql.ins_vid(list1[i])
        print("------------------------------数据已成功上传数据库！------------------------------")

    # bs4 匹配热图模块
    def Imgrank_type(self, a, b):
        u = URL_Request()
        yya = u.req(a, b)
        soup = BeautifulSoup(yya,'lxml')
        yyb = soup.findAll('img')
        list1 = []
        for i in yyb:
            j = "http:"+i.get('src')
            list1.append(j)
            for z in list1:
                print("成功抓取到图片路径：",z)
        sql = SQL_connect()
        for i in range(len(list1)):
            sql.ins_img(list1[i])
        print("------------------------------数据已成功上传数据库！------------------------------")

if __name__ == '__main__':


    type  = input("请选择要爬取的分类（text/video/imgrank）:")
    pge = int(input("请输入爬取页数："))

    if type == 'text':
        Pipei().Text_type(type, pge)

    elif type == 'video':

        Pipei().Video_type(type,pge)
    elif type == 'imgrank':

        Pipei().Imgrank_type(type,pge)
    else:
        print("输入错误！！！")