第一次写 有很多毛病,没防封ip。。。没多线程 不过数据量不大凑活着用吧,代码如下:
from bs4 import BeautifulSoup
import requests
import time
import re
urls = ["http://sjz.58.com/hezu/pn{}/?PGTID=0d100000-000f-12db-512a-5598dd1c3104&ClickID=1".format(str(i)) for i in range(1,20)]
for url in urls:
headers = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537."}
response = requests.get(url,headers = headers)
web_data = BeautifulSoup(response.text,"lxml")
allurl = web_data.select("body > div.conwrap.clearfix > div.content.clearfix > div.listcon > div > dl > dd > h3 > a")
for i in allurl:
x = []
y = []
z = []
v = []
url = i.get("href")
#headers = {
# "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537."}
response1 = requests.get(url, headers=headers)
web_data1 = BeautifulSoup(response1.text, "lxml")
prices = web_data1.select("body > div.main > div.house-info.white-block.clearfix > div.house-primary-content-wrap.fr > ul > li.house-primary-content-li.house-primary-content-fir.clearfix > div > i > em")
payformants = web_data1.select("body > div.main > div.house-info.white-block.clearfix > div.house-primary-content-wrap.fr > ul > li.house-primary-content-li.house-primary-content-fir.clearfix > div > span")
rooms = web_data1.select("body > div.main > div.house-info.white-block.clearfix > div.house-primary-content-wrap.fr > ul > li:nth-of-type(2) > div")
address = web_data1.select("body > div.main > div.house-info.white-block.clearfix > div.house-primary-content-wrap.fr > ul > li:nth-of-type(4) > div")
peizhis = web_data1.select("body > div.main > div.house-info.white-block.clearfix > div.house-primary-content-wrap.fr > ul > li.house-primary-content-li.clearfix.broker-config > div > span")
pat = re.compile("\s")
for peizhi in peizhis:
x.append(re.sub(pat, "", peizhi.get_text()))
l = " ".join(x)
for price in prices:
y.append(re.sub(pat, "", price.get_text()))
for room in rooms:
z.append(re.sub(pat, "", room.get_text()))
for addres in address:
v.append(re.sub(pat, "", addres.get_text()))
p = " ".join(v)
data = {"配置": l, "价格": y[0], "房型": z[0], "地址": p}
print(data)