搜狗-微信爬虫实例
import re
import urllib.request
import urllib.error
import time
#自定义代理服务器爬虫函数:
def proxy(proxy_addr,url):
try:
#添加报头伪装成浏览器
req=urllib.request.Request(url)
req.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 Safari/537.36 SE 2.X MetaSr 1.0')
#创建代理服务器
proxy= urllib.request.ProxyHandler({'http':proxy_addr})
opener = urllib.request.build_opener(proxy, urllib.request.HTTPHandler)
#opener安装为全局模式
urllib.request.install_opener(opener)
#获取数据
data = urllib.request.urlopen(req).read()
return data
#异常处理
except urllib.error.URLError as e:
if hasattr(e,"code"):
print(e.code)
if hasattr(e,"reason"):
print(e.reason)
#若为URLError异常,延时10秒执行
time.sleep(10)
except Exception as e:
print("exception:"+str(e))
#若为Exception异常,延时1秒执行
time.sleep(1)
key="Python"
#设置代理服务器
proxy="61.234.77.139:80"
for i in range(1,10):
key=urllib.request.quote(key)
thispageurl="http://weixin.sogou.com/weixin?query="+key+"&_sug_type_=&sut=1553&lkt=1%2C1520818514033%2C1520818514033&s_from=input&_sug_=y&type=2&sst0=1520818514135&page="+str(i)
thispagedata=use_proxy(proxy,thispageurl)
print(len(str(thispagedata)))
pat1='<a href="(.*?)"'
rs1=re.compile(pat1,re.S).findall(str(thispagedata)) #一页的数据
if(len(rs1)==0):
print("此次("+str(i)+"页)没成功")
continue
for j in range(0,len(rs1)):
#爬取每一页的所有文章
thisurl=rs1[j]
# thisurl=thisurl.replace("amp;","")
file="weixin/"+str(i)+"页第"+str(j)+"篇文章.html"
thisdata=use_proxy(proxy,thisurl)
#得到文章数据
try:
fh=open(file,"wb")
fh.write(thisdata)
fh.close()
print("第"+str(i)+"页第"+str(j)+"篇文章成功")
except Exception as e:
print(e)
print("第"+str(i)+"页第"+str(j)+"篇文章失败")