from urllib.request import urlopen
from bs4 import BeautifulSoup
import re
import numpy
import csv
import time
def getdatawithtablehead(url):
""" 该函数用于获取带表头的数据 """
html = urlopen(url)
bsobj = BeautifulSoup(html, "lxml", from_encoding="gb18030") # 获取BeautifulSoup对象
tablelist = bsobj.findAll("tr") # 获取所有的表格
Dataset = []
tablehead = tablelist[0].get_text().strip("\n").split("\n\n")
Dataset.append(tablehead) # 获取表头
dataset = []
for datalist in tablelist[1:]:
data = datalist.get_text().replace(" ", "").replace("\r\n", "").strip("\n").split("\n")
dataset.append(data) # 获取当月每一天的数据
Dataset = numpy.row_stack((Dataset, dataset))
return Dataset
def getdata(url):
""" 该函数用于获取不带表头的数据 """
html = urlopen(url)
bsobj = BeautifulSoup(html, "lxml", from_encoding="gb18030")
tablelist = bsobj.findAll("tr")
dataset = []
for datalist in tablelist[1:]:
data = datalist.get_text().replace(" ", "").replace("\r\n", "").strip("\n").split("\n")
dataset.append(data)
return dataset
start =time.clock()
# 长沙空气质量指数(AQI)-PM2.5查询地址:
starturl = "http://www.tianqihoubao.com/aqi/shijiazhuang.html"
html = urlopen(starturl)
bsobj = BeautifulSoup(html, "lxml") # 获取BeautifulSoup对象
# 找到所有存放月度数据的网页链接,并以列表的形式按月份先后顺序保存这些链接
Sites = []
for link in bsobj.findAll(href=re.compile("^(/aqi/shijiazhuang-)")):
site = "http://www.tianqihoubao.com" + link.attrs['href']
Sites.append(site)
# 数组反转
Sites.reverse()
print(Sites)
Dataset = getdatawithtablehead(Sites[0]) # 获取表头和第一个月度数据
for url in Sites[1:]:
dataset = getdata(url)
Dataset = numpy.row_stack((Dataset, dataset)) # 获取所有月度数据
csvfile = open("shijiazhuang.csv", "w") # 创建csv文件用于保存数据
try:
writer = csv.writer(csvfile)
for i in range(numpy.shape(Dataset)[0]):
writer.writerow((Dataset[i, :])) # 将数据逐行写入csv文件
finally:
csvfile.close() # 关闭csv文件
end = time.clock()
print('Running time: %s Seconds' % (end - start))
原文地址:
https://blog.csdn.net/qq_36185831/article/details/79123144
https://blog.csdn.net/u013337691/article/details/51894453#commentsedit