公交信息爬取

参考文章：https://blog.csdn.net/WenWu_Both/article/details/70168760?depth_1-utm_source（中间部分有改动）

import requests ##导入requests
from bs4 import BeautifulSoup ##导入bs4中的BeautifulSoup
import os
import bs4

headers =  {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.221 Safari/537.36 SE 2.X MetaSr 1.0'}
all_url = 'https://jiaxing.8684.cn/'  ##开始的URL地址
start_html = requests.get(all_url, headers=headers) 
Soup = BeautifulSoup(start_html.text, 'lxml') # 以lxml的方式解析html文档

all_a = Soup.find('div',class_='bus-layer depth w120').find_all('a')

Network_list = []
for i in all_a:
    href = i['href'] #取出a标签的href 属性
    if 'x_' not in href:
        html = all_url + href
    #print(html)
    second_html = requests.get(html,headers=headers)
    Soup2 = BeautifulSoup(second_html.text, 'lxml') 
    all_a2 = Soup2.find('div',class_='list clearfix').find_all('a')
    for a2 in all_a2:
        title1 = a2.get_text() #取出a1标签的文本
        href1 = a2['href'] #取出a标签的href 属性
        html_bus = all_url + href1
        #print(html_bus)
        thrid_html = requests.get(html_bus,headers=headers)
        Soup3 = BeautifulSoup(thrid_html.text, 'lxml') 
        all_line = Soup3.find_all('div',class_='bus-excerpt mb15')#线路简介
        all_site = Soup3.find_all('div',class_='bus-lzlist mb15')#公交站点
        line_x = all_line[0].find('div',class_='excerpt fl').get_text()+all_line[0].find('div',class_='other fr').get_text()
        sites_x = all_site[0].find_all('a')
        sites_x_list = []
        for site_x in sites_x:
            sites_x_list.append(site_x.get_text())
        line_num = len(all_line)
        if line_num==2:  # 如果存在环线，也返回两个list，只是其中一个为空
            line_y = all_line[1].find('div',class_='excerpt fl').get_text()+all_line[1].find('div',class_='other fr').get_text()
            sites_y = all_site[1].find_all('a')
            sites_y_list = []
            for site_y in sites_y:
                sites_y_list.append(site_y.get_text())
        else:
            line_y,sites_y_list=[],[]


        information = [line_x,sites_x_list,line_y,sites_y_list]
        Network_list.append(information)
        
        
# 定义保存函数，将运算结果保存为txt文件
def text_save(content,filename,mode='a'):
    # Try to save a list variable in txt file.
    file = open(filename,mode)
    for i in range(len(content)):
        file.write(str(content[i])+'\n')
    file.close()
# 输出处理后的数据     
text_save(Network_list,'D://test//嘉兴_bus.txt')

#保存为excel
import pandas as pd
import numpy as np
data=pd.DataFrame(Network_list,columns=['公交线路','站点','公交线路（返程）','公交站（返程）'])
data.to_excel('D://test//嘉兴_bus.xlsx',index=False)