selenium等动态爬取北京市公共自行车网站

获取北京市公共自行车网点经纬度等数据

#coding=utf-8

from selenium import webdriver
from time import sleep
import re
import pandas as pd


# 2.构造网址
url = r'http://bjggzxc.jtw.beijing.gov.cn/bdmap.html'

# 3.打开浏览器
browser = webdriver.Chrome(r'C:\Program Files\Google\Chrome\Application\chromedriver.exe')
browser.get(url)
sleep(10)       #视网速而定
html = str(browser.page_source.encode('utf8'))
# print(html)

# 4.寻找按钮多节点
# 末页数字
page_final = browser.find_elements_by_xpath('''//span[@style]''')
final_num = int(page_final[1].text[1:-1])

station_all = []
for j in range(final_num):

    buttons_st = browser.find_elements_by_xpath('''//tr[@id]''')
    st_many = len(buttons_st)

    for i in range(st_many):
        station_num_1 = buttons_st[i].text.split('\n')  #第一个站点

        station_lon = re.findall('class="datagrid-cell datagrid-cell-c1-lon">(.*?)</div></td>',html)
        browser.implicitly_wait(30)
        # print(station_lon)
        station_lon_1 = station_lon[i]      #第一个站经度

        station_lat = re.findall('class="datagrid-cell datagrid-cell-c1-lat">(.*?)</div></td>',html)
        station_lat_1 = station_lat[i]      #第一个站纬度

        station_1 = [station_num_1[0],station_num_1[1],station_lon_1,station_lat_1]
        station_all.append(station_1)

    #下一页节点
    buttons_page = browser.find_elements_by_xpath('''//div[@class='datagrid-pager pagination']//span[@class="l-btn-text"]''')
    next_page_bt = buttons_page[2]  #下一页
    browser.execute_script('arguments[0].click();',next_page_bt)      #点击按钮

    html = str(browser.page_source.encode('utf8'))
    browser.implicitly_wait(30)

print(station_all)
print(len(station_all),len(station_all[-1]))

path=r'.\beijing_public_bike2.csv'
table=pd.DataFrame(station_all)
table.to_csv(path, encoding='gbk')


# browser.close()