Python爬取小说

Python爬取小说

这里主要爬取笔趣阁的链接
因为笔趣阁对段时间的爬取次数做了限制，所以每次我们只能爬取十章

# coding:utf-8
import re

import soup as soup
from bs4 import BeautifulSoup
import requests
import chardet

i=0

# #限定每行字数
def cut_text(text,lenth):
    textArr = re.findall('.{'+str(lenth)+'}', text)
    textArr.append(text[(len(textArr)*lenth):])
    return textArr


def read(url, i):
    i+=1
    if i>10:
        return
    r = requests.get(url)
    encode=chardet.detect(r.content)
    r.encoding=encode['encoding']
    doc = r.text  # 服务器返回响应
    soup = BeautifulSoup(doc, "html.parser")
    for title in soup.find_all('h1'):
        print(" ")
        print(title.string)  # 输出响应的html对象
    for txt in soup.find_all(id='content'):
    	#  #每行字数限定
        #  #笔趣阁1
        # for line in txt.text.split('　　'):
        #     if len(line)>40:
        #         for cutline in cut_text(line,40):
        #             print(cutline)
        #     else:print(line)
        #笔趣阁2
        for line in txt.text.split():
            if len(line)>40:
                for cutline in cut_text(line,40):
                    print('   '+cutline)
            else:print('   '+line)

    # #笔趣阁1
    # for c in soup.find_all(class_='page_chapter'):
    #     d= c.find_all('a')
    # href=d[2]['href']
    #笔趣阁2-1
    for c in soup.find_all(id='pager_next'):
        href=c['href']

    # #笔趣阁2-2
    # for c in soup.find_all(class_='bottem2'):
    #     d= c.find_all('a')
    #     href=d[3]['href']
    if len(href)>2:
            # #笔趣阁1
            # href='https://www.biqukan.com'+href
            # 笔趣阁2-1
            href='https://www.xinxs.la/489_489289/'+href
            # # 笔趣阁2-2
            print(href)
            # href='https://www.guibuyu.org/'+href
            read(href,i)

# #这里需要持续更新链接，
# #笔趣阁1 最初进化
# url='https://www.biqukan.com/75_75537/521520773.html'
# 笔趣阁2-1
url = 'https://www.xinxs.la/489_489289/2647622.html'
# # 笔趣阁2-2
# url = 'https://www.guibuyu.org/96_96741/33481914.html'
read(url,i)