Python小爬虫 : 网上爬小说,图片,视频等
以爬取某网站小说为试例:
#coding=utf-8
from urllib.request import urlopen,Request
import ssl(导入验签的包)
from lxml import etree(需要下载阿里云的插件 http://pypi.python.org/simple)
‘’’
下载数据
‘’’
def getData(path,headers,charset):
req = Request(path,headers=headers)
conn = urlopen(req)
if conn.code == 200:
data = conn.read()
data = data.decode(encoding=charset)
return data
else:
return “no 200”
‘’’
解析
‘’’
def jxHTML(data,tag):
html = etree.HTML(data)
datas = html.xpath(tag)
return datas
ssl._create_default_https_context = ssl._create_unverified_context(验签)
path = “网站地址”
headers = {
“user-agent”: “Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/。。。 Safari/。。。”
}
#下载章节列表页的数据
data = getData(path,headers,“UTF-8”)
myas = jxHTML(data,"//div[@id=‘list’]/dl/dd/a")
for a in myas:
hrefs = a.xpath("./@href")
texts= a.xpath("./text()")
if len(texts) > 0:
print(texts[0])
if len(hrefs) > 0:
#下载章节数据
data = getData(path+hrefs[0],headers,“UTF-8”)
#解析每一章的数据
infos = jxHTML(data,"//div[@id=‘content’]/text()")
info = “\n”.join(infos)
info = info.replace("\xa0", “”)
print(info)