✨作者主页:IT研究室✨
个人简介:曾从事计算机专业培训教学,擅长Java、Python、微信小程序、Golang、安卓Android等。接项目定制开发、代码讲解、答辩教学、文档编写、降重等。
☑文末获取源码☑
精彩专栏推荐⬇⬇⬇
Java项目
Python项目
安卓项目
微信小程序项目
一、开发环境
- 开发语言:Python
- 数据库:Hive
- 系统架构:Hadoop
- 后端:Spark
- 前端:Vue
二、系统功能模块
- 角色:用户、管理员
- 功能:
用户:
个人中心、房价数据管理;
管理员:
地区管理、用户管理、房价数据爬虫、可视化看板。
亮点:大数据+爬虫+可视化
三 、系统界面展示
四、部分代码设计
def pdShow(area_name):
print(area_name)
if area_name=="全部地区平均":
# if 1==1:
data = pd.read_csv("RegionalHousePrice/全部地区平均.csv", encoding="gbk", header=None)
data = data.rename(columns={0: "name", 1: "value"})
else:
data = pd.read_csv(f"RegionalHousePrice/{area_name}.csv", encoding="gbk", header=None)
data = data.loc[:, [1, 2]]
data = data.rename(columns={1: "name", 2: "value"})
data = data.to_dict(orient="records")
print(data)
return data
@app.route("/")
def index():
return "index"
class PricesApi(MethodView):
def get(self,area_name):
show_data = pdShow(area_name)
return show_data
prices_view =PricesApi.as_view("prices_api")
app.add_url_rule("/prices",defaults={"area_name":"全部地区平均"}
,view_func=prices_view,methods=['GET',])
app.add_url_rule("/prices/<area_name>",view_func=prices_view,methods=['GET',])
if __name__ == '__main__':
app.run(debug=True)
async def area_house_prices(area_name, area_url):
with open("RegionalHousePrice/全部地区平均.csv", mode="a") as fall:
with open(f"RegionalHousePrice/{area_name}.csv", mode="a") as f:
async with aiohttp.ClientSession() as session:
async with session.get(area_url, headers=headers) as resp:
next_html = await resp.text()
nex_html_page = BeautifulSoup(next_html, "html.parser")
next_search_search_areas_trs = nex_html_page.find_all("tr")
prices = 0
for tr in next_search_search_areas_trs:
index = 0
lst = tr.find_all("td")
if len(lst) != 0:
for td in lst:
tdtext = td.text
for i in tdtext:
if i == ',':
tdtext = tdtext.replace(i, '')
if (index == 2):
prices += int(tdtext)
index += 1
f.write(tdtext)
f.write(",")
f.write("\n")
prices = round(prices / (len(next_search_search_areas_trs)-1),2)
fall.write(area_name)
fall.write(",")
fall.write(str(prices))
fall.write("\n")
print(area_name,"over!")
def del_files(dir_path):
if os.path.isfile(dir_path):
try:
os.remove(dir_path)
except BaseException as e:
print(e)
elif os.path.isdir(dir_path):
file_lis = os.listdir(dir_path)
for file_name in file_lis:
tf = os.path.join(dir_path, file_name)
del_files(tf)
async def main(top_url):
del_files("RegionalHousePrice")
top_html_resp = requests.get(url=top_url, headers=headers)
top_html_resp.close()
top_html_tree = etree.HTML(top_html_resp.text) # 获取到地区
top_html_areas_names = top_html_tree.xpath(top_url_search_areas_names)
top_html_areas_urls = top_html_tree.xpath(top_url_search_areas_urls)
index = 0
tasks = []
for area in top_html_areas_names:
tasks.append(asyncio.create_task(area_house_prices(area + "", top_html_areas_urls[index] + "")))
index += 1
await asyncio.wait(tasks)
if __name__ == '__main__':
loop = asyncio.get_event_loop()
loop.run_until_complete(main(top_url))
# asyncio.run(main(top_url))
五、论文参考
结语
大家可以帮忙点赞、收藏、关注、评论啦~
源码获取:私信我