爬取小说网小说

爬取小说网小说

coding=utf-8
import re
import requests
from bs4 import BeautifulSoup
import pymysql
db = pymysql.connect("localhost","root","zgx675050748","RUNOOB",charset='utf8')
cursor = db.cursor()
url ='https://www.qu.la/dushixiaoshuo/'#爬取都市小说区
response = requests.get(url)
response = response.content
response = str(response,'utf-8')
soup = BeautifulSoup(response,'html.parser')
book = soup.select('a[target="_blank"]')
for a in range(0,len(book),2):
    url1 = book[a]['href']#找到每一本小说，爬取当然小说的第一章网址
    url2 = url+str(url1)
    response = requests.get(url2)
    response = response.content
    response = str(response, 'utf-8')
    soup1 = BeautifulSoup(response, 'html.parser')
    book1 = soup1.find_all(href=re.compile(str(url1)+"(.*?)"))
    name = soup1.select('h1')
    textname = name[0].text
    sql = """CREATE TABLE """ +str(textname)+ """(章节 CHAR(25),内容 text )"""
    cursor.execute(sql)
    url3 = url+book1[0]['href']
    for b in book1:
        response = requests.get(url3)#利用小说每章网页中的下一章按钮爬取接下来一章的地址，循环往复
        response = response.content
        response = str(response, 'utf-8')
        soup1 = BeautifulSoup(response, 'html.parser')
        content1 = soup1.select('#pager_next')
        bookname = soup1.select('h1')
        content = soup1.select('#content')
        print(bookname[0].text,content[0].text)
        sql = """INSERT INTO """ +textname+"""(章节,内容)VALUES ("%s","%s")""" %(bookname[0].text,content[0].text)
        cursor.execute(sql)
        db.commit()

        url3 = url2+content1[0]['href']
db.close()