Python爬虫抓取【糗事百科视频】

学习爬虫时写的Demo！修改数据库连接就可以直接运行！

import requests
from lxml import etree
from urllib.request import *
import pymysql

class QiubaiSpider:

    def __init__(self):
        self.url_temp = "https://www.qiushibaike.com/video/page/{}/"
        self.headers = {
            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36"
        }

    def get_url_list(self):  # 根据url地址的规律,构造url list
        url_list = [self.url_temp.format(i) for i in range(1, 5)]
        return url_list
    def parse_url(self,url):
        print("now parseing :",url)
        response = requests.get(url,headers=self.headers)
        return response.content.decode()

    def get_content_list(self, html_str):  # 3.提取所有视频播放地址数据
        html = etree.HTML(html_str)
        source_list = html.xpath("//div[@class='col1 old-style-col1']/div")
        #print(source_list)
        # for source in source_list:
        #     #截取/后面的字符串
        #     name = source.rindex('/')
        #     name = source[name+1:]
        #     urlretrieve("http:" + source, r'C:\Users\Administrator\Desktop\video/' + str(name))#保存到本地
        content_list = []
        save_list = []
        for div in source_list:#提取数据
            item = {}
            item["title"] = div.xpath(".//div[@class='content']/span/text()")[0].strip() if len(div.xpath(".//div[@class='content']/span/text()")) > 0 else None
            item["source"] = "http:" + div.xpath(".//source/@src")[0] if len(div.xpath(".//source/@src")) > 0 else None
            #print(item)
            content_list.append(item)
        for d in content_list: #将字典列表转换成元组列表
            save_list.append(tuple([(v) for k, v in d.items()]))
        print(save_list)
        return save_list

    def save_content(self,conn,content_list): #保存到数据库
        # 创建游标
        cursor = conn.cursor()
        sql = "insert into scandal_encyclopedia(title,source_url) values (%s,%s)"
        count = cursor.executemany(sql,content_list)
        conn.commit()
        conn.close()
        return count
    def opMysql(self): #连接mysql
        # 创建连接
        conn = pymysql.connect(host='127.0.0.1', port=3306, user='root', passwd='123456', db='db0')
        return conn

    def run(self):
        url_list = self.get_url_list()#获取url列表
        for url in url_list:
            html_str = self.parse_url(url)
            save_list = self.get_content_list(html_str)
            conn = self.opMysql()
            count = self.save_content(conn,save_list)
            print("影响行数:",count)
if __name__ == '__main__':
    qiushi = QiubaiSpider()
    qiushi.run()