Python爬虫

Python小爬虫 ： 网上爬小说，图片，视频等

以爬取某网站小说为试例：
#coding=utf-8
from urllib.request import urlopen,Request
import ssl(导入验签的包)
from lxml import etree（需要下载阿里云的插件 http://pypi.python.org/simple）
‘’’
下载数据
‘’’
def getData(path,headers,charset):
req = Request(path,headers=headers)
conn = urlopen(req)
if conn.code == 200:
data = conn.read()
data = data.decode(encoding=charset)
return data
else:
return “no 200”

‘’’
解析
‘’’
def jxHTML(data,tag):
html = etree.HTML(data)
datas = html.xpath(tag)
return datas

ssl._create_default_https_context = ssl._create_unverified_context（验签）
path = “网站地址”
headers = {
“user-agent”: “Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/。。。 Safari/。。。”
}
#下载章节列表页的数据
data = getData(path,headers,“UTF-8”)

myas = jxHTML(data,"//div[@id=‘list’]/dl/dd/a")

for a in myas:
hrefs = a.xpath("./@href")
texts= a.xpath("./text()")
if len(texts) > 0:
print(texts[0])
if len(hrefs) > 0:
#下载章节数据
data = getData(path+hrefs[0],headers,“UTF-8”)
#解析每一章的数据
infos = jxHTML(data,"//div[@id=‘content’]/text()")
info = “\n”.join(infos)
info = info.replace("\xa0", “”)
print(info)