起点中文网（主要是在目录下创建文件）

# - *- coding:utf-8-*-
import urllib2
import re
import os
import sys
reload(sys)
sys.setdefaultencoding('utf-8') #编码
from bs4 import BeautifulSoup


for page in range(1,700):
    print '*'
    User_Agent= 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:49.0) Gecko/20100101 Firefox/49.0'
    url="http://f.qidian.com/all?size=-1&sign=-1&tag=-1&chanId=-1&subCateId=-1&orderId=&update=-1&page=%s&month=-1&style=1&action=1"%page
    headers={'User-Agent':User_Agent}
    request=urllib2.Request(url,headers=headers)
    html=urllib2.urlopen(request).read()
    soup = BeautifulSoup(html, 'html.parser')
    l = soup.find_all('div', class_ = 'book-mid-info')
    print #

    for htmltile in l:
        name = htmltile.find('h4').encode('utf-8')
        reg=r'<h4><a data-bid=".*?" data-eid=".*?" href="(.*?)" target="_blank">(.*?)</a></h4>'
        text=re.finditer(reg,name)

        for i in text:
            curl=i.group(1)
            title=i.group(2)

os.mkdir(title.encode('gbk'))  #创建目录
            #os.chdir(title.encode('gbk'))   #在当前目录下操作
            html1 = urllib2.urlopen('http:'+curl+'#Catalog').read()
            reg=re.compile(r'<li data-rid=".*?"><a href="(.*?)" target="_blank" data-eid="qd_G55" data-cid=".*?" title=".*?">(.*?)</a>')
            titles=re.finditer(reg,html1)

            for i in titles:
                curl_=i.group(1)
                names=i.group(2)


fd=open(title.encode('gbk')+'/'+names.encode('gbk')+'.txt','wb') #在指定目录下创建文件
                #fd=open(names.encode('gbk')+'.txt','wb')
                print "正在爬取%s本"%names
                htmlll=urllib2.urlopen('http:'+curl_).read()
                regs=re.compile(r'<div class="read-content j_readContent">\s*([\s\S]*?)\s*</div>') #正则多行时注意用\s*
                content=re.findall(regs,htmlll)
                for m in content:
                    contents=m.replace('<p>','\r\n')
                    fd.write(names+'\r\n'+contents)
                    print "已完成%s"%names
                fd.close()

转载于:https://www.cnblogs.com/ZHANG576433951/p/6109075.html