urllib和urllib2的区别,urllib只能根据url爬取,不能设置头信息,但可以对url进行编码;urllib2可以设置request请求头信息
# -*- coding: utf-8 -*-
import urllib
f = urllib.urlopen('http://www.cnblogs.com/')
result = f.read() #读取html页面的第一行
print result
# -*- coding: utf-8 -*-
import urllib2
import sys
def clawer(url):
send_headers={
'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Connection':'keep-alive',
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0'
}
req = urllib2.Request(url,headers=send_headers)
r = urllib2.urlopen(req)
html = r.read() #返回网页内容
receive_header = r.info() #返回的报头信息
print sys.getfilesystemencoding()
#html = html.decode('utf-8','replace').encode(sys.getfilesystemencoding()) #转码:避免输出出现乱码
#print receive_header
print html.decode('gbk')
url= "http://www.jb51.net"
url = "http://uvs.youdao.com/search?site=photogalaxy.163.com&sort=time&channelid=&q=%B7%C9%BB%FA&length=10&start=0" #网页地址
clawer(url)
import urllib
import urllib2
values = {"username":"1016903103@qq.com","password":"XXXX"}
data = urllib.urlencode(values)
url = "https://passport.csdn.net/account/login?from=http://my.csdn.net/my/mycsdn"
request = urllib2.Request(url,data)
response = urllib2.urlopen(request)
print response.read()