详解如何通过Python的BeautifulSoup爬虫+NLP标签提取+Dijkstra规划路径和KMeans聚类分析帮助用户规划旅行路线

本文链接：https://blog.csdn.net/mosquito_lover1/article/details/146408923

系统模块：

数据采集模块（爬虫）：负责从目标网站抓取地点数据（如名称、经纬度、描述等）
数据预处理模块（标签算法）：对抓取到的地点数据进行清洗和分类。根据地点特征（如经纬度、描述文本）打上标签（如“适合家庭”、“适合冒险”）。
地理数据处理模块（地图API）：使用地图API获取地点的详细信息（如地址、距离、路径等）。计算地点之间的距离或路径。
路径规划模块：根据用户输入的起点和终点，规划最优路径。支持多种规划策略（如最短路径、最快路径）。
聚类分析模块（K-means）：对地点进行聚类，找出热点区域或相似区域。帮助用户更好地理解地点分布。
可视化模块：将聚类结果和路径规划结果可视化。使用工具：Matplotlib、Folium（地图可视化）。
用户交互模块：提供用户界面（如命令行或Web界面），允许用户输入起点、终点和偏好。
返回规划路径和聚类结果。

系统架构图：

+-------------------+       +-------------------+       +-------------------+
|  数据采集模块      |       |  数据预处理模块    |       |  地理数据处理模块  |
|  (爬虫)           | ----> |  (标签算法)       | ----> |  (地图API)        |
+-------------------+       +-------------------+       +-------------------+
                                                                 |
                                                                 v
+-------------------+       +-------------------+       +-------------------+
|  路径规划模块      | <---- |  聚类分析模块      |       |  可视化模块        |
|  (Dijkstra算法)   |       |  (K-means)        | ----> |  (Matplotlib)     |
+-------------------+       +-------------------+       +-------------------+
                                                                 |
                                                                 v
+-------------------+
|  用户交互模块      |
|  (命令行/Web界面)  |
+-------------------+

核心代码：

import requests
from bs4 import BeautifulSoup
from sklearn.cluster import KMeans
import numpy as np
import matplotlib.pyplot as plt
import networkx as nx

# 1. 爬虫：从网页抓取地点数据
def crawl_locations(url):
    response = requests.get(url)
    soup = BeautifulSoup(response.text, 'html.parser')
    
    # 假设网页中有地点的名称和经纬度
    locations = []
    for item in soup.find_all('div', class_='location'):
        name = item.find('h2').text
        lat = float(item.find('span', class_='lat').text)
        lon = float(item.find('span', class_='lon').text)
        locations.append({'name': name, 'lat': lat, 'lon': lon})
    
    return locations

# 2. 标签算法：对地点进行分类
def tag_locations(locations):
    for loc in locations:
        # 假设根据经纬度判断地点类型（这里简单示例）
        if loc['lat'] > 30:
            loc['tag'] = 'North'
        else:
            loc['tag'] = 'South'
    return locations

# 3. 地图API：计算地点之间的距离（这里用欧几里得距离模拟）
def calculate_distances(locations):
    n = len(locations)
    distance_matrix = np.zeros((n, n))
    
    for i in range(n):
        for j in range(n):
            lat1, lon1 = locations[i]['lat'], locations[i]['lon']
            lat2, lon2 = locations[j]['lat'], locations[j]['lon']
            # 欧几里得距离（实际应用中可以使用地图API的路径距离）
            distance_matrix[i][j] = np.sqrt((lat1 - lat2)**2 + (lon1 - lon2)**2)
    
    return distance_matrix

# 4. 路径规划：使用Dijkstra算法规划最短路径
def plan_route(distance_matrix, start_index, end_index):
    G = nx.Graph()
    n = distance_matrix.shape[0]
    
    # 构建图
    for i in range(n):
        for j in range(n):
            if i != j:
                G.add_edge(i, j, weight=distance_matrix[i][j])
    
    # 使用Dijkstra算法计算最短路径
    shortest_path = nx.dijkstra_path(G, start_index, end_index, weight='weight')
    shortest_distance = nx.dijkstra_path_length(G, start_index, end_index, weight='weight')
    
    return shortest_path, shortest_distance

# 5. K-means聚类：对地点进行聚类
def cluster_locations(locations, n_clusters=3):
    coords = np.array([[loc['lat'], loc['lon']] for loc in locations])
    kmeans = KMeans(n_clusters=n_clusters, random_state=0).fit(coords)
    labels = kmeans.labels_
    centers = kmeans.cluster_centers_
    
    # 将聚类结果添加到地点数据中
    for i, loc in enumerate(locations):
        loc['cluster'] = labels[i]
    
    return locations, centers

# 6. 可视化聚类结果和路径
def visualize(locations, centers, shortest_path):
    coords = np.array([[loc['lat'], loc['lon']] for loc in locations])
    labels = [loc['cluster'] for loc in locations]
    
    plt.figure(figsize=(10, 6))
    
    # 绘制聚类结果
    plt.scatter(coords[:, 1], coords[:, 0], c=labels, cmap='viridis', label='Locations')
    plt.scatter(centers[:, 1], centers[:, 0], c='red', marker='x', s=100, label='Cluster Centers')
    
    # 绘制路径
    path_coords = coords[shortest_path]
    plt.plot(path_coords[:, 1], path_coords[:, 0], 'r--', label='Shortest Path')
    
    plt.title("Location Clustering and Route Planning")
    plt.xlabel("Longitude")
    plt.ylabel("Latitude")
    plt.legend()
    plt.show()

# 主函数
def main():
    # 1. 爬虫：抓取地点数据
    url = 'https://example.com/locations'  # 替换为实际URL
    locations = crawl_locations(url)
    
    # 2. 标签算法：对地点进行分类
    locations = tag_locations(locations)
    
    # 3. 地图API：计算地点之间的距离
    distance_matrix = calculate_distances(locations)
    
    # 4. 路径规划：规划最短路径
    start_index = 0  # 起点索引
    end_index = len(locations) - 1  # 终点索引
    shortest_path, shortest_distance = plan_route(distance_matrix, start_index, end_index)
    
    print(f"Shortest Path: {[locations[i]['name'] for i in shortest_path]}")
    print(f"Shortest Distance: {shortest_distance}")
    
    # 5. K-means聚类：对地点进行聚类
    locations, centers = cluster_locations(locations, n_clusters=3)
    
    # 6. 可视化聚类结果和路径
    visualize(locations, centers, shortest_path)

if __name__ == '__main__':
    main()