高德找房爬取房源

最新推荐文章于 2024-09-23 14:24:34 发布

月光不温柔q

最新推荐文章于 2024-09-23 14:24:34 发布

阅读量485

点赞数 18

文章标签： python

本文链接：https://blog.csdn.net/weixin_73238667/article/details/141254521

版权

爬取房源

这里爬的是咕咕找房的房源，无需登录，页面结构简单。

#爬虫程序  位于scrap.py 第6行
import requests
from bs4 import BeautifulSoup
import csv

def get_housing_info(url):
    response = requests.get(url)
    soup = BeautifulSoup(response.text, 'html.parser')

    # 找到所有的房源
    house_list = soup.find_all('dl', class_='list')
    housing_info = []

    for house in house_list:
        # 提取房源的标题
        title = house.find('p', class_='title').get_text(strip=True)
        ls =title.split(' ')
        title = ls[1]
        # 提取房源的地址
        address = house.find('p', class_='gray6').get_text(strip=True)
        ls = address.split('-')
        address = ''.join(ls)
        housing_info.append([title, address])

    return housing_info

def save_to_csv(data, filename,mode):
    with open(filename, mode, newline='', encoding='utf-8') as f:
        writer = csv.writer(f)
        writer.writerows(data)

def main():
    location = input('请输入要搜索的区域：')
    base_url = 'https://www.gu-gu.com/index/index/index?title='+location
    max= eval(input('请输入要爬取的页数(每页20个房源):'))
    url = base_url
    i=1
    mode = 'w'
    while i<=max:
        housing_info= get_housing_info(url)
        i = i + 1
        url = '&page=' + str(i)
        url = base_url + url
        save_to_csv(housing_info, 'housing_info.csv',mode)
        print("第{}页已爬取完成".format(i-1))
        mode = 'a'
    print("所有房源已爬取完成")
if __name__ == '__main__':
    main()

地图显示

需要在securityJsCode处写自己的高德地图的安全密钥，在https://webapi.amap.com/maps?v=1.4.15&key处写自己的api

<!-- 地图程序在index.html 第124行 -->
<html>
<head>
    <meta charset="utf-8">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta http-equiv="Content-Security-Policy" content="upgrade-insecure-requests">
    <meta name="viewport" content="initial-scale=1.0, user-scalable=no, width=device-width">

    <title>工作地点租房</title>
    <link rel="stylesheet" href="https://cache.amap.com/lbs/static/main1119.css" />
    <link rel="stylesheet" href="https://cache.amap.com/lbs/static/jquery.range.css" />

    <!--加载AMap的CSS样式、jQuery库、AMap的JavaScript库  -->
    <script type="text/javascript">
        window._AMapSecurityConfig = {
            securityJsCode: 'api',
        }
    </script>
    <script
        src="https://webapi.amap.com/maps?v=1.4.15&key= 
 
&plugin=AMap.ArrivalRange,AMap.Scale,AMap.Geocoder,AMap.Transfer,AMap.Driving,AMap.Autocomplete"></script>
    <script src="https://cache.amap.com/lbs/static/jquery-1.9.1.js"></script>
    <script src="https://cache.amap.com/lbs/static/es5.min.js"></script>
    <script src="https://cdn.bootcdn.net/ajax/libs/jquery/3.6.0/jquery.min.js"></script>
    <script src="https://cdn.bootcdn.net/ajax/libs/jquery-csv/1.0.11/jquery.csv.min.js"></script>
    <script src="https://cache.amap.com/lbs/static/jquery.range.js"></script>

    <!-- 定义CSS样式，主要是控制面板和信息窗口的样式 -->
    <style>
        .control-panel {
            position: absolute;
            top: 30px;
            right: 20px;
        }

        .control-entry {
            width: 280px;
            background-color: rgba(119, 136, 153, 0.8);
            font-family: fantasy, sans-serif;
            text-align: left;
            color: white;
            overflow: auto;
            padding: 10px;
            margin-bottom: 10px;
        }

        .control-input {
            margin-left: 120px;
        }

        .control-input input[type="text"] {
            width: 160px;
        }

        .control-panel label {
            float: left;
            width: 120px;
        }

        #transfer-panel {
            position: absolute;
            background-color: white;
            max-height: 80%;
            overflow-y: auto;
            top: 30px;
            left: 20px;
            width: 250px;
        }

        .center-button {
            display: block;
            margin: auto;
        }
    </style>
</head>

<body>
    <!-- 地图 -->
    <div id="container"></div>
    <!-- 控制面板 -->
    <div class="control-panel">
        <!-- 工作地点输入框 -->
        <div class="control-entry">
            <label>选择工作地点：</label>
            <div class="control-input">
                <input id="work-location" type="text">
            </div>
        </div>
        <!-- 通勤方式选择 -->
        <div class="control-entry">
            <label>选择通勤方式：</label>
            <div class="control-input">
                <div>
                    <input type="radio" class="vehicle-option" name="vehicle" value="SUBWAY,BUS" onClick="takeBus(this)"
                        checked />
                    <span>公交+地铁</span>
                </div>
                <div>
                    <input type="radio" class="vehicle-option" name="vehicle" value="SUBWAY"
                        onClick="takeSubway(this)" />
                    <span>地铁</span>
                </div>
                <div>
                    <input type="radio" class="vehicle-option" name="vehicle" value="DRIVING"
                        onClick="takeDriving(this)" />
                    <span>驾车</span>
                </div>
            </div>
        </div>
        <!-- 导入房源文件 -->
        <div class="control-entry">
            <label>导入房源文件：</label>
            <div class="control-input">
                <input type="file" id="file" accept=".csv" onChange="importRentInfo(this)">
            </div>
        </div>
        <!-- 重置 -->
        <div class="control-entry">
            <button id="reset-button" class="center-button">重置</button>
        </div>
    </div>
    <!-- 显示通勤路线的面板 -->
    <div id="transfer-panel"></div>

    <script>
        //创建AMap地图对象
        var map = new AMap.Map('container', {
            resizeEnable: true,
            zoomEnable: true,
            center: [116.397428, 39.90923],
            zoom: 11
        });
        //定义全局变量
        var scale = new AMap.Scale();   //到达范围
        map.addControl(scale);

        var arrivalRange = new AMap.ArrivalRange();     //到达范围
        var x, y, t, vehicle = "SUBWAY,BUS";    //工作地点坐标和通勤方式
        var workAddress, workMarker;     //工作地址和标记
        var rentMarkerArray = [];       //房源标记数组
        var polygonArray = [];      //通勤范围多边形数组
        var amapTransfer;       //通勤对象
        //创建信息窗口对象
        var infoWindow = new AMap.InfoWindow({
            offset: new AMap.Pixel(0, -30)
        });
        //自动完成输入框对象
        var auto = new AMap.Autocomplete({
            input: "work-location"
        });

        AMap.event.addListener(auto, "select", workLocationSelected);

        //在用户选择不同的通勤方式时被调用，它们会更新全局变量 vehicle 的值，并重新加载工作地点。
        function takeBus(radio) {
            vehicle = radio.value;
            loadWorkLocation()
        }
        function takeDriving(radio) {
            vehicle = radio.value;
            loadWorkLocation()
        }

        function takeSubway(radio) {
            vehicle = radio.value;
            loadWorkLocation()
        }
        //在用户导入房源文件时被调用，它会调用 loadHouses() 函数来加载房源信息。
        function importRentInfo(fileInfo) {
            loadHouses()
        }
        //在用户在自动完成输入框中选择一个工作地点时被调用，它会更新全局变量 workAddress 的值，并重新加载工作地点。
        function workLocationSelected(e) {
            workAddress = e.poi.name;
            loadWorkLocation();
        }
        //在地图上添加一个表示工作地点的标记。
        function loadWorkMarker(x, y, locationName) {
            workMarker = new AMap.Marker({
                map: map,
                title: locationName,
                icon: 'https://webapi.amap.com/theme/v1.3/markers/n/mark_r.png',
                position: [x, y]
            });
        }
        //在地图上画出一个表示通勤范围的多边形。
        function loadWorkRange(x, y, t, color, v) {
            arrivalRange.search([x, y], t, function (status, result) {
                if (result.bounds) {
                    for (var i = 0; i < result.bounds.length; i++) {
                        var polygon = new AMap.Polygon({
                            map: map,
                            fillColor: color,
                            fillOpacity: "0.4",
                            strokeColor: color,
                            strokeOpacity: "0.8",
                            strokeWeight: 1
                        });
                        polygon.setPath(result.bounds[i]);
                        polygonArray.push(polygon);
                    }
                }
            }, {
                policy: v
            });
        }
        //根据给定的地址在地图上添加一个表示房源的标记，并设置点击事件，当用户点击这个标记时，会显示一个信息窗口，并在地图上画出从工作地点到这个房源的通勤路线。
        function addMarkerByAddress(address) {
            var geocoder = new AMap.Geocoder({
                city: "",
                radius: 1000
            });
            geocoder.getLocation(address.toString().split(',')[1], function (status, result) {
                if (status === "complete" && result.info === 'OK') {
                    var geocode = result.geocodes[0];
                    var rentMarker = new AMap.Marker({
                        map: map,
                        title: address.toString().split(',')[0],
                        icon: 'https://webapi.amap.com/theme/v1.3/markers/n/mark_b.png',
                        position: [geocode.location.getLng(), geocode.location.getLat()]
                    });
                    rentMarkerArray.push(rentMarker);

                    rentMarker.content = "<div>房源：" + address.toString().split(',')[0] +
                        "<br>地址：" + address.toString().split(',')[1] + "</div>";
                    rentMarker.on('click', function (e) {
                        infoWindow.setContent(e.target.content);
                        infoWindow.open(map, e.target.getPosition());
                        if (amapTransfer) amapTransfer.clear();

                        if (vehicle === 'DRIVING') {
                            amapTransfer = new AMap.Driving({
                                map: map,
                                policy: AMap.DrivingPolicy.LEAST_TIME,
                                city: "全国",
                                panel: 'transfer-panel'
                            });
                        } else {
                            amapTransfer = new AMap.Transfer({
                                map: map,
                                policy: AMap.TransferPolicy.LEAST_TIME,
                                city: "全国",
                                panel: 'transfer-panel'
                            });
                        }

                        amapTransfer.search([{
                            keyword: workAddress,
                        }, {
                            keyword: address.toString().split(',')[1],
                        }], function (status, result) {
                            console.log(result);
                        })
                    });
                }
            })
        }
        //删除地图上的工作地点标记和通勤范围多边形。

        function delWorkLocation() {
            if (polygonArray) map.remove(polygonArray);
            if (workMarker) map.remove(workMarker);
            polygonArray = [];
        }
        //删除地图上的所有房源标记。

        function delRentLocation() {
            if (rentMarkerArray) map.remove(rentMarkerArray);
            rentMarkerArray = [];
        }
        //根据全局变量 workAddress 的值加载工作地点，它会先删除地图上的工作地点标记和通勤范围多边形，然后添加新的工作地点标记和通勤范围多边形。
        function loadWorkLocation() {
            delWorkLocation();
            var geocoder = new AMap.Geocoder({
                city: "",
                radius: 1000
            });

            geocoder.getLocation(workAddress, function (status, result) {
                if (status === "complete" && result.info === 'OK') {
                    var geocode = result.geocodes[0];
                    x = geocode.location.getLng();
                    y = geocode.location.getLat();
                    loadWorkMarker(x, y);
                    loadWorkRange(x, y, 60, "#3f67a5", vehicle);
                    map.setZoomAndCenter(12, [x, y]);
                }
            })
        }
        //加载用户导入的房源文件，它会先读取文件内容，然后解析CSV数据，对于每一行数据，它会在地图上添加一个表示房源的标记，并计算这个房源到工作地点的距离，最后，它会找出距离最近的房源，并将这个房源的标记改为另一种标识。

        function loadHouses() {
            var file = document.getElementById('file').files[0];
            var vehicle = document.querySelector('input[name="vehicle"]:checked').value;
            var reader = new FileReader();
            reader.onload = function (e) {
                var houses = $.csv.toArrays(e.target.result);
                console.log(houses); // Debug: log the parsed CSV data

                var nearestHouse = null;
                var minDistance = Infinity;

                for (var i = 1; i < houses.length; i++) {
                    var title = houses[i][0];
                    var address = houses[i][1];
                    // 将地址和标题组合成一个字符串，然后调用addMarkerByAddress函数
                    var house = addMarkerByAddress(address + ',' + title);
                }

                
            };

            reader.onerror = function (e) {
                console.error("File could not be read! Code " + e.target.error.code); // Debug: log any file reading errors
            };

            reader.readAsText(file, 'utf-8');
        }
        //重置
        document.getElementById('reset-button').addEventListener('click', function () {
            // 清除地图上的所有标记和多边形
            delWorkLocation();
            delRentLocation();

            // 清除信息窗口和通勤路线面板的内容
            infoWindow.close();
            document.getElementById('transfer-panel').innerHTML = '';

            // 重置所有全局变量的值
            x = y = t = workAddress = null;
            vehicle = "SUBWAY,BUS";
            rentMarkerArray = [];
            polygonArray = [];

            // 清除输入框和文件输入框的内容
            document.getElementById('work-location').value = '';
            document.getElementById('file').value = '';

            // 恢复单选按钮的默认选项
            document.querySelector('input[value="SUBWAY,BUS"]').checked = true;

            amapTransfer.clear();
        });
    </script>
</body>

</html>