python爬取小说（四）代码优化

最新推荐文章于 2024-09-04 16:11:59 发布

MTbaby

最新推荐文章于 2024-09-04 16:11:59 发布

阅读量1.3k

点赞数 5

分类专栏：爬虫实战 Python 文章标签： python 爬虫 python爬虫实践小说爬取 beautifulbeautifulSoup4

本文链接：https://blog.csdn.net/MTbaby/article/details/82557442

版权

为方便后续做前端展示，本次主要优化章节字段根据爬取顺序入库，各功能函数模块化。

# -*- coding: utf-8 -*-
import urllib.request
import bs4
import re
import sqlite3
import time
print ('连接数据库……')
cx = sqlite3.connect('PaChong.db')
# 在该数据库下创建表

# cx.execute('''CREATE TABLE book_info(
#        id INTEGER PRIMARY KEY   AUTOINCREMENT,
#        title             verchar(128)     not null,
#        img             verchar(512)      null,
#        auther         verchar(64)      null,
#        type             verchar(128)      null,
#        status             verchar(64)      null,
#        num             int      null,
#        updatatime             datetime      null,
#        newchapter             verchar(512)      null,
#        authsummery             verchar(1024)      null,
#        summery            verchar(1024)      null,
#        notipurl             verchar(512)      null);
#        ''')
# cx.execute('''CREATE TABLE book_chapter(
#        id INTEGER PRIMARY KEY   AUTOINCREMENT,
#        book_id         int      null ,
#        chapter_no         int      null ,
#        chapter_name             verchar(128)      null,
#        chapter_url             verchar(512)      null,
#        chapter_content         text      null);
#        ''')

print("Table created successfully")
print("数据库连接完成")

def getHtml(url):
    user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36"
    headers = {"User-Agent":user_agent}
    request = urllib.request.Request(url,headers=headers)
    response = urllib.request.urlopen(request)
    html = response.read()
    return html


# 爬取整个网页
def parse(url):
    html_doc = getHtml(url)
    sp = bs4.BeautifulSoup(html_doc, 'html.parser', from_encoding="utf-8")
    return sp
# 爬取书籍基本信息
def get_book_baseinfo(url):
    # class =