python找思路_python 爬取贝壳的一些思路和方法设计(用地址找到小区名字)

最新推荐文章于 2023-04-27 13:18:34 发布

weixin_39807954

最新推荐文章于 2023-04-27 13:18:34 发布

阅读量302

点赞数

文章标签： python找思路

1.查mysql 获取 title 或者 address

2.用title或者address参数,request.get方法查百度地图的地址api:

http://api.map.baidu.com/place/v2/search?query=成都市青羊区青羊上街336号2栋2单元21楼2102号房屋及室内装修&tag=房地产&region=成都

&output=json&ak=CAG9eY150GAfXkGCLPL69N5CvCpGYjm5&extensions_adcode=true&coord_type=gcj02ll&scope=2

注意里面城市和query的参数

title和查询到的结果,相似匹配度代码:

# -*- coding: utf-8 -*-

"""

@Time : 2020/11/24 17:15

@Athor : LinXiao

@功能 :

"""

import os

import sys

sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from difflib import SequenceMatcher

import requests

import ast

import operator

from pprint import pprint

# 搜索百度接口

def search_baidu_map(baidu_param):

if "号" in baidu_param:

addr_param=baidu_param.split('号')[0] + '号'

pprint(addr_param)

else:

addr_param=baidu_param

city="成都"

api=f'http://api.map.baidu.com/place/v2/search?query="{addr_param}' \

f'&tag=房地产&region={city}&output=json&ak=CAG9eY150GAfXkGCLPL69N5CvCpGYjm5&extensions_adcode=true&coord_type=gcj02ll&scope=2'

# 注意 addr_param 和 city 的对应关系

res=requests.request("GET", api).text # str

response=ast.literal_eval(res)

pprint(response) # dict

baidu_map_detail=response

results=baidu_map_detail["results"] # list

return results

# 字符串相似度比较

def similarity(a, b):

return SequenceMatcher(None, a, b).ratio()

# 将搜索到的相似度最高的字典倒序,返回相似度最高的地址

def find_max_ratio(results):

add_list=[]

for addr in results:

ratio_address={}

address=addr["address"]

ratio=similarity(baidu_param, address)

ratio_address["address"]=address

ratio_address["ratio"]=ratio

add_list.append(ratio_address)

sorted_x=sorted(add_list, key=operator.itemgetter('ratio'), reverse=True) # list

max_ratio=sorted_x[0] # dict

key=max_ratio['address']

return key

# 获取相似度最高的地址:name

def get_max_same_addr(results, key):

for addr in results:

if addr["address"] == key:

# pprint(addr)

name=addr["name"]

pprint(name)

return name

if __name__ == '__main__':

baidu_param = '四川省成都市新都区大丰街道方元路222号6栋1单元701号'

# baidu_param='成都市锦江区茂业豪园1-810'

# baidu_param = '成都市高新区西区蓝岸街555号11栋2单元1楼1号住房'

# baidu_param='成都市新津区花源镇白云路419号1981栋(-1)-2层住宅'

print(baidu_param)

results = search_baidu_map(baidu_param)

key=find_max_ratio(results)

get_max_same_addr(results, key)

3.用相似度最高的地址去查贝壳地图找房:

当输入地址时,会自动提示,然后解析这个提示下最接近的小区

思路二:

利用百度地图和selenium选择定位小区:

# -*- coding: utf-8 -*-

"""

@Time : 2020/10/20 9:59

@Athor : LinXiao

@功能 :

"""

# ------------------------------

import os

import sys

import requests

from lxml import etree

sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

import time

from pprint import pprint

from fake_useragent import UserAgent

from loguru import logger

from selenium import webdriver

from selenium.webdriver import ChromeOptions

import random

from bs4 import BeautifulSoup

from urllib.request import quote, unquote

from proxypool.ip_queue import ip_queue_redis

def new_broswer():

# ua列表

user_agent_list=[

'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)',

'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1)',

'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; AcooBrowser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)',

'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)',

'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E; LBBROWSER)',

'Mozilla/4.0 (compatible; MSIE 7.0; AOL 9.5; AOLBuild 4337.35; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)',

'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)',

'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; 360SE)Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 UBrowser/6.2.4094.1 Safari/537.36',

'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Avant Browser)',

'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)',

'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; TencentTraveler 4.0)',

'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; The World)',

'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET CLR 2.0.50727; SE 2.X MetaSr 1.0)',

'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; 360SE)',

'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)',

'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)',

'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E)',

'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.3; .NET4.0C; .NET4.0E)',

'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.3; .NET4.0C; .NET4.0E) QQBrowser/6.9.11079.201',

'Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)',

'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; GTB7.0)',

'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)',

'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; InfoPath.3)',

'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv,2.0.1) Gecko/20100101 Firefox/4.0.1',

'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11',

'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20',

'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36',

'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1',

'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.41 Safari/535.1 QQBrowser/6.9.11079.201',

'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',

'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.163 Safari/535.1',

'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11',

'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 LBBROWSER',

'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.11 TaoBrowser/2.0 Safari/536.11',

'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER',

'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1',

'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',

'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:6.0) Gecko/20100101 Firefox/6.0',

'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:2.0b13pre) Gecko/20110307 Firefox/4.0b13pre',

'Mozilla/5.0 (Windows NT 6.1; rv,2.0.1) Gecko/20100101 Firefox/4.0.1',

'Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)',

'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1',

'Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)',

'Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0',

'Mozilla/5.0 (Windows; U; Windows NT 6.1; ) AppleWebKit/534.12 (KHTML, like Gecko) Maxthon/3.0 Safari/534.12',

'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.472.33 Safari/534.3 SE 2.X MetaSr 1.0',

'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',

'Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5',

'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6',

'Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10',

'Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6',

'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:16.0) Gecko/20100101 Firefox/16.0',

'Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)',

'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)',

'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)',

'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; LBBROWSER)',

'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)',

'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.3; .NET4.0C; .NET4.0E)',

'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 2.0.50727; SLCC2; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.3; .NET4.0C; Tablet PC 2.0; .NET4.0E)',

'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)',

'Mozilla/5.0 (iPad; U; CPU OS 4_2_1 like Mac OS X; zh-cn) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8C148 Safari/6533.18.5',

'Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52',

'Opera/9.80 (Windows NT 6.1; U; zh-cn) Presto/2.9.168 Version/11.50']

user_agent=random.choice(user_agent_list)

options=ChromeOptions()

# 指定浏览器分辨率

# options.add_argument('window-size=800x720')

# 无界面

# options.add_argument('--headless')

# 开发者模式

options.add_experimental_option('excludeSwitches', ['enable-automation'])

# 不加载图片

options.add_argument('blink-settings=imagesEnabled=false')

# 禁用浏览器正在被自动化程序控制的提示

options.add_argument('--disable-infobars')

# 谷歌文档提到需要加上这个属性来规避bug

options.add_argument('--disable-gpu')

# ua 代理

options.add_argument('user-agent=%s' % user_agent)

# ip代理

proxy_ip=ip_queue_redis()

# options.add_argument(f"--proxy-server={proxy_ip}")

logger.info(f'proxy_ip is : {proxy_ip}')

# options.add_argument("--proxy-server=183.56.107.248:4545")

browser=webdriver.Chrome(executable_path=r"D:\software\chromedriver_win32/chromedriver.exe", options=options)

time.sleep(random.random())

return proxy_ip, browser

# 汉字转url参数

def url_quote(str):

url_quote=quote(str)

return url_quote

def main(city, param):

ip, broswer=new_broswer()

url_qianzui=f"https://{city}.ke.com/xiaoqu/rs"

houzui=str(url_quote(param)) + '/'

url=url_qianzui + houzui

print(url)

broswer.get(url)

# sleep(5)

# 解决编码问题

# pprint(browser.page_source.encode('utf-8').decode('utf-8'))

html=broswer.page_source

pprint(html)

# broswer.quit()

def open_baidu_map(address):

if "号" in address:

addr_param=address.split('号')[0] + '号'

pprint(addr_param)

else:

addr_param=address

url="https://map.baidu.com"

ip, broswer=new_broswer()

broswer.get(url)

# html=broswer.page_source

time.sleep(random.randint(3, 8))

# print(html)

broswer.find_element_by_id("sole-input").send_keys(addr_param)

time.sleep(random.randint(1, 3))

broswer.find_element_by_id("search-button").click()

time.sleep(2)

broswer.find_element_by_xpath('//*[@id="tblPreciseAddr"]/tbody/tr/th/div').click()

time.sleep(1)

broswer.find_element_by_id("nav_tab2").click()

time.sleep(1)

broswer.find_element_by_id("rangekw").send_keys('小区')

time.sleep(random.random())

broswer.find_element_by_id("iw_ns_btn").click()

time.sleep(1.5)

broswer.find_element_by_xpath('//*[@class="poilist"]/li').click()

time.sleep(1.2)

html=broswer.page_source

# pprint(html)

tree_html=etree.HTML(html)

title=tree_html.xpath('//*[@id="generalheader"]/div[1]/div[1]/span/text()')[0].strip().strip('\n')

# soup=BeautifulSoup(html, "lxml")

# soup.a.attrs["属性名"]

pprint(title)

time.sleep(1.5)

# broswer.quit()

if __name__ == '__main__':

city="cd"

# param="嘉祥瑞庭南城"

# main(city, param)

# address="成都市郫都区犀浦镇兴业南街199号25栋2单元3楼5号房屋、装修"

address="成都市龙泉驿区同安街道办事处同达街31号2栋3单元5楼10号房屋"

open_baidu_map(address)

运行的结果:

验证:

weixin_39807954

关注

0
点赞
踩
0

收藏

觉得还不错? 一键收藏
0
评论
python找思路_python 爬取贝壳的一些思路和方法设计(用地址找到小区名字)

1.查mysql 获取 title 或者 address2.用title或者address参数,request.get方法查百度地图的地址api:http://api.map.baidu.com/place/v2/search?query=成都市青羊区青羊上街336号2栋2单元21楼2102号房屋及室内装修&tag=房地产&region=成都&output=json&am...
复制链接

扫一扫