#coding utf-8
import cookielib
import urllib2,re,urllib
turl='http://www.test.saofang.cn/service/agent/agentHouseRent.do'
purl='http://www.test.saofang.cn/pub/doAgentLogin.do' #处理post数据的url
data={'userName':13316750525,'userPwd':123456789} #post数据,根据实际情况。
post_data=urllib.urlencode(data) # post数据编码
headers={'Host':'www.test.saofang.cn','Referer':'http://www.test.saofang.cn/pub/agentLogin.do'}
cookieJar=cookielib.CookieJar() #初始化cookie对象
cookie_support=urllib2.HTTPCookieProcessor(cookieJar) #处理cookie的程序 用于下载 上传cookie
open=urllib2.build_opener(cookie_support,urllib2.HTTPHandler)
urllib2.install_opener(open)
request=urllib2.Request(purl,post_data,headers)
response=urllib2.urlopen(request) # get cookie
#tn=urllib2.Request(turl)
#tnl=urllib2.urlopen(tn)
tnl=urllib2.urlopen(turl)
text=tnl.read()
req='</strong>/</em><span>.+?(\d+)</b>' #login success?difference,确认cookie登录成功
lcc=re.search(req,text)
print lcc
tc=re.findall(req,text)
print tc
for list in tc:
print list
if lcc:
print 'login success!'
else:
print 'login fail'
#print lcc.group()
surl='http://www.test.saofang.cn/service/agent/agentHouseRent.do'
type=['Rent','Sold']
tp=['zf','sp','xzl']
http='http://www.test.saofang.cn/service/agent/agentHouse'
zs='.do?houseUsage='
for ty in type:
for tss in tp:
# 二手房时参与变化
if ty=='Sold' and tss=='zf':
tss='esf'
https=http+ty+zs+tss
hss=http+ty+'.do'
print https
print hss
#page=1
for page in range(1,44):
data1={'pageIndex':page,'houseState':'PUB','houseUsage':tss}
print data1
data=urllib.urlencode(data1)
headers={'Host':'www.test.saofang.cn','Referer':https}
reque=urllib2.Request(hss,data,headers)
#print reque
t2=urllib2.urlopen(reque)
rt2=t2.read()
#print rt2
reqq="src='(.+?min\.jpg)"
reqq_list=re.findall(reqq,rt2)
#print reqq_list
m=0
for list in reqq_list:
s='%d'%m
p='%d'%page
#print s, p, list
print 'e:\\img\\'+ty+tss+p+s+'.jpg'
sre='http://.+\\.+\\'
m+=1
print m
print list
#检查链接是否可用,不可用时跳过下载图片。
try:
urllib2.urlopen(list).read()
except:
continue
#保存图片的命名规则
filename='e:\\img\\'+ty+'_'+tss+'_'+p+'_'+s+'.jpg'
urllib.urlretrieve(list,filename)
1.下载图片问题1 地址打不开时跳过处理过程
2.Url中有反斜杠,暂时没处理这类情况。