读入网站
import requests
from bs4 import BeautifulSoup
r=requests.get("http://www.pyclass.com/real-estate/rock-springs-wy/LCWYROCKSPRINGS/",headers={"User-agent":'Mozilla/5.0(X11;Ubuntu;Linux x86_64;rv:61.0)Gecko/20100101 Firefox/61.0'})
soup=BeautifulSoup(r.content)
soup保存网站数据
提取div——class;
提取h4——class;
删除多余的\n和空格;
import requests
from bs4 import BeautifulSoup
r=requests.get("http://www.pyclass.com/real-estate/rock-springs-wy/LCWYROCKSPRINGS/",headers={"User-agent":'Mozilla/5.0(X11;Ubuntu;Linux x86_64;rv:61.0)Gecko/20100101 Firefox/61.0'})
soup=BeautifulSoup(r.content)
all = soup.find_all("div",{"class":"propertyRow"}) #得到所有class为propertyrow的div 类似list格式
#len(all) #10个结果
price=all[0].find_all("h4",{"class":"propPrice"})
price #得到一个list
all[0].find("h4",{"class":"propPrice"}).text #得到text string类型
all[0].find("h4",{"class":"propPrice"}).text.replace("\n","").replace(" ",'') #replace去除\n 以及" "空格
Try_except:
得到每个房子的info;
bed、area、bath等;
从span+class中提取;
try——except防止无数据报错;
for i in all: #遍历得到所有price
print(i.find("h4",{"class":"propPrice"}).text.replace("\n","").replace(" ",''))
print(i.find_all("span",{"class":"propAddressCollapse"})[0].text)
print(i.find_all("span",{"class":"propAddressCollapse"})[1].text)
#找到bed个数
try: #防止error
print(i.find("span",{"class":"infoBed"}).find("b").text) #text error 因为有none try解决
except:
print(None)
#找到area个数
try: #防止error
print(i.find("span",{"class":"infoSqFt"}).find("b").text) #text error 因为有none try解决
except:
print(None)
try: #防止error
print(i.find("span",{"class":"infoValueFullBath"}).find("b").text) #text error 因为有none try解决
except:
print(None)
try: #防止error
print(i.find("span",{"class":"infoValueHalfBath"}).find("b").text) #text error 因为有none try解决
except:
print(None)
print(" ")
#print(i.find_all("span",{"class":"infoBed"}))
添加代码,zip遍历两个list,得到feature信息
若增加条件判断,可得到特定条件的feature信息;
#得到feature 信息
for column_group in i.find_all("div",{"class":"columnGroup"}):
#print(column_group)
#继续iterator
for feature_group, feature_name in zip(column_group.find_all("span",{"class":"featureGroup"}),column_group.find_all("span",{"class":"featureName"})): #zip遍历两个list
#print(feature_group.text,feature_name.text)
if "Lot Size" in feature_group.text: #若 feature_group中存在Lot size属性,输出该属性的name
print(feature_name.text)
将数据放入DataFrame中
建立数组,存放字典;
字典为 feature:value
#加入dictionary
#dictionary保存在list中
l=[]
for i in all: #遍历得到所有price
d={}
d["Address"]=i.find_all("span",{"class":"propAddressCollapse"})[0].text
d["Locality"]=i.find_all("span",{"class":"propAddressCollapse"})[1].text
d["Price"]=i.find("h4",{"class":"propPrice"}).text.replace("\n","").replace(" ",'')
#找到bed个数
try: #防止error
d["Beds"]=i.find("span",{"class":"infoBed"}).find("b").text #text error 因为有none try解决
except:
d["Beds"]=None
#找到area个数
try: #防止error
d["Area"]=i.find("span",{"class":"infoSqFt"}).find("b").text #text error 因为有none try解决
except:
d["Area"]=None
try: #防止error
d["Full Baths"]=i.find("span",{"class":"infoValueFullBath"}).find("b").text #text error 因为有none try解决
except:
d["Full Baths"]=None
try: #防止error
d["Half Baths"]=i.find("span",{"class":"infoValueHalfBath"}).find("b").text #text error 因为有none try解决
except:
d["Half Baths"]=None
#得到feature 信息
for column_group in i.find_all("div",{"class":"columnGroup"}):
#print(column_group)
#继续iterator
for feature_group, feature_name in zip(column_group.find_all("span",{"class":"featureGroup"}),column_group.find_all("span",{"class":"featureName"})): #zip遍历两个list
#print(feature_group.text,feature_name.text)
if "Lot Size" in feature_group.text: #若 feature_group中存在Lot size属性,输出该属性的name
d["Lot Size"]=feature_name.text
#得到dictionary
l.append(d)
#print(i.find_all("span",{"class":"infoBed"}))
将字典转为DataFrame:
to_csv保存为csv格式的文件;
import pandas
df=pandas.DataFrame(l)
df.to_csv("out_put.csv") #转为csv格式 可用excel打开
进行翻页抓取
requests.get得到网页html代码;
soup将代码格式化;
find_all得到代码中的指定标签(类似list类型);
page_nr为页码数;
import requests
from bs4 import BeautifulSoup
r=requests.get("http://www.pyclass.com/real-estate/rock-springs-wy/LCWYROCKSPRINGS/",headers={"User-agent":'Mozilla/5.0(X11;Ubuntu;Linux x86_64;rv:61.0)Gecko/20100101 Firefox/61.0'})
soup=BeautifulSoup(r.content)
all = soup.find_all("div",{"class":"propertyRow"}) #得到所有class为propertyrow的div 类似list格式
#price=all[0].find_all("h4",{"class":"propPrice"})
#得到一个list
#all[0].find("h4",{"class":"propPrice"}).text #得到text string类型
#all[0].find("h4",{"class":"propPrice"}).text.replace("\n","").replace(" ",'') #replace去除\n 以及" "空格
page_nr=soup.find_all("a",{"class":"Page"})[-1].text
翻页抓取:
通过for循环得到每次的url;
每次传入url得到all;
对all进行遍历抓取;
#加入dictionary
#dictionary保存在list中
l=[]
base_url="http://www.pyclass.com/real-estate/rock-springs-wy/LCWYROCKSPRINGS/t=0&s="
for page in range(0,int(page_nr)*10,10): #每次+10
print(base_url+str(page)+".html")
r=requests.get(base_url+str(page)+".html",headers={"User-agent":'Mozilla/5.0(X11;Ubuntu;Linux x86_64;rv:61.0)Gecko/20100101 Firefox/61.0'})
c=r.content
soup=BeautifulSoup(c,"html.parser")
#print(soup.prettify())
all = soup.find_all("div",{"class":"propertyRow"}) #得到所有class为propertyrow的div 类似list格式
for i in all: #遍历得到所有price
d={}
d["Address"]=i.find_all("span",{"class":"propAddressCollapse"})[0].text
try:
d["Locality"]=i.find_all("span",{"class":"propAddressCollapse"})[1].text
except:
d["Locality"]=None
d["Price"]=i.find("h4",{"class":"propPrice"}).text.replace("\n","").replace(" ",'')
#找到bed个数
try: #防止error
d["Beds"]=i.find("span",{"class":"infoBed"}).find("b").text #text error 因为有none try解决
except:
d["Beds"]=None
#找到area个数
try: #防止error
d["Area"]=i.find("span",{"class":"infoSqFt"}).find("b").text #text error 因为有none try解决
except:
d["Area"]=None
try: #防止error
d["Full Baths"]=i.find("span",{"class":"infoValueFullBath"}).find("b").text #text error 因为有none try解决
except:
d["Full Baths"]=None
try: #防止error
d["Half Baths"]=i.find("span",{"class":"infoValueHalfBath"}).find("b").text #text error 因为有none try解决
except:
d["Half Baths"]=None
#得到feature 信息
for column_group in i.find_all("div",{"class":"columnGroup"}):
#print(column_group)
#继续iterator
for feature_group, feature_name in zip(column_group.find_all("span",{"class":"featureGroup"}),column_group.find_all("span",{"class":"featureName"})): #zip遍历两个list
#print(feature_group.text,feature_name.text)
if "Lot Size" in feature_group.text: #若 feature_group中存在Lot size属性,输出该属性的name
d["Lot Size"]=feature_name.text
#得到dictionary
l.append(d)
保存至csv文件
#加入 numpy array
#将l加入dataframe
import pandas
df=pandas.DataFrame(l)
df.to_csv("out_put.csv") #转为csv格式 可用excel打开
完整代码
import requests
from bs4 import BeautifulSoup
import pandas
#加入dictionary
#dictionary保存在list中
l=[]
base_url="http://www.pyclass.com/real-estate/rock-springs-wy/LCWYROCKSPRINGS/t=0&s="
for page in range(0,int(page_nr)*10,10): #每次+10
print(base_url+str(page)+".html")
r=requests.get(base_url+str(page)+".html",headers={"User-agent":'Mozilla/5.0(X11;Ubuntu;Linux x86_64;rv:61.0)Gecko/20100101 Firefox/61.0'})
c=r.content
soup=BeautifulSoup(c,"html.parser")
#print(soup.prettify())
all = soup.find_all("div",{"class":"propertyRow"}) #得到所有class为propertyrow的div 类似list格式
for i in all: #遍历得到所有price
d={}
d["Address"]=i.find_all("span",{"class":"propAddressCollapse"})[0].text
try:
d["Locality"]=i.find_all("span",{"class":"propAddressCollapse"})[1].text
except:
d["Locality"]=None
d["Price"]=i.find("h4",{"class":"propPrice"}).text.replace("\n","").replace(" ",'')
#找到bed个数
try: #防止error
d["Beds"]=i.find("span",{"class":"infoBed"}).find("b").text #text error 因为有none try解决
except:
d["Beds"]=None
#找到area个数
try: #防止error
d["Area"]=i.find("span",{"class":"infoSqFt"}).find("b").text #text error 因为有none try解决
except:
d["Area"]=None
try: #防止error
d["Full Baths"]=i.find("span",{"class":"infoValueFullBath"}).find("b").text #text error 因为有none try解决
except:
d["Full Baths"]=None
try: #防止error
d["Half Baths"]=i.find("span",{"class":"infoValueHalfBath"}).find("b").text #text error 因为有none try解决
except:
d["Half Baths"]=None
#得到feature 信息
for column_group in i.find_all("div",{"class":"columnGroup"}):
#print(column_group)
#继续iterator
for feature_group, feature_name in zip(column_group.find_all("span",{"class":"featureGroup"}),column_group.find_all("span",{"class":"featureName"})): #zip遍历两个list
#print(feature_group.text,feature_name.text)
if "Lot Size" in feature_group.text: #若 feature_group中存在Lot size属性,输出该属性的name
d["Lot Size"]=feature_name.text
#得到dictionary
l.append(d)
#l
df=pandas.DataFrame(l)
df.to_csv("out_put.csv") #转为csv格式 可用excel打开