python爬虫-day3

搜狗-微信爬虫实例

import re
import urllib.request
import urllib.error
import time
#自定义代理服务器爬虫函数：
def proxy(proxy_addr,url):
  try:
  #添加报头伪装成浏览器
  req=urllib.request.Request(url)
  req.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 Safari/537.36 SE 2.X MetaSr 1.0')
  #创建代理服务器
  proxy= urllib.request.ProxyHandler({'http':proxy_addr})
  opener = urllib.request.build_opener(proxy, urllib.request.HTTPHandler)
  #opener安装为全局模式
  urllib.request.install_opener(opener)  
  #获取数据
  data = urllib.request.urlopen(req).read()
  return data
#异常处理
  except urllib.error.URLError as e:
        if hasattr(e,"code"):
            print(e.code)
        if hasattr(e,"reason"):
            print(e.reason)
  #若为URLError异常，延时10秒执行
        time.sleep(10)
  except Exception as e:
        print("exception:"+str(e))
        #若为Exception异常，延时1秒执行
        time.sleep(1)

key="Python"
#设置代理服务器
proxy="61.234.77.139:80"     

for i in range(1,10):
    key=urllib.request.quote(key)
    thispageurl="http://weixin.sogou.com/weixin?query="+key+"&_sug_type_=&sut=1553&lkt=1%2C1520818514033%2C1520818514033&s_from=input&_sug_=y&type=2&sst0=1520818514135&page="+str(i)

    thispagedata=use_proxy(proxy,thispageurl)
    print(len(str(thispagedata)))
    pat1='<a href="(.*?)"'
    rs1=re.compile(pat1,re.S).findall(str(thispagedata))  #一页的数据
    if(len(rs1)==0):
        print("此次（"+str(i)+"页）没成功")
        continue
    for  j in range(0,len(rs1)):
        #爬取每一页的所有文章
        thisurl=rs1[j]
      #  thisurl=thisurl.replace("amp;","")
        file="weixin/"+str(i)+"页第"+str(j)+"篇文章.html"
        thisdata=use_proxy(proxy,thisurl) 
        #得到文章数据
        try:
            fh=open(file,"wb")
            fh.write(thisdata)
            fh.close()
            print("第"+str(i)+"页第"+str(j)+"篇文章成功")
        except Exception as e:
            print(e)
            print("第"+str(i)+"页第"+str(j)+"篇文章失败")