爬取新笔趣阁排行并保存到mysql_Python爬虫爬取新笔趣阁小说——转载

from lxml importetreeimportrequests

header={"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8","Content-Type": "application/x-www-form-urlencoded;charset=UTF-8","Accept-Encoding": "gzip, deflate","Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8","Cache-Control": "no-cache","Connection": "keep-alive","User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36"}deftrim(text_array):return "".join("".join(text_array).split())classxiaoshuo():def getText(self, url, charset="utf-8"):return str(requests.get(url, headers=header).content, charset)def __init__(self, page):

r=self.getText(page)

html=etree.HTML(r)

self.book_name= trim(html.xpath('//*[@id="info"]/h1/text()'))

self.book_author= trim(html.xpath('//*[@id="info"]/p[1]/text()'))

self.book_desc= trim(html.xpath('//*[@id="intro"]/p[position()>1]/text()'))

a= html.xpath('//*[@id="list"]/dl/dd/a')

self.chapters=[]for a1 ina:

chapter= {'name': a1.xpath('./text()')[0], 'link': 'http://www.xbiquge.la' + a1.xpath('./@href')[0]}

self.chapters.append(chapter)def __writeChapter(self, lines):

i=0for line inlines:

lines[i]= "".join(line.split()) + "\n"i+= 1with open('D:\\%s.txt' % self.book_name, 'a', encoding='utf-8') as f:

f.writelines(lines)def __findContent(self, chapter):

content= etree.HTML(self.getText(chapter['link'])).xpath('//*[@id="content"]/text()')

content.insert(0, chapter['name'] + '\n')

self.__writeChapter(content)print(chapter['name'] + '下载完成！')defdown(self):for chapter inself.chapters:

self.__findContent(chapter)if __name__ == '__main__':print("\n\t\t欢迎使用《新笔趣阁》 http://www.xbiquge.la 小说下载小工具\n\n\t\t作者:mbh \t时间:2017-05-06\n")print("*************************************************************************")#小说地址

#target_url = str(input("请输入小说目录下载地址:\n"))

page_link= 'http://www.xbiquge.la/9/9419/'xs=xiaoshuo(page_link)

xs.down()