一个简易爬虫,主要功能是爬取全国各地空气质量数据,输出到一个csv文件,后续加入相关数据处理得出实时空气质量排名前10和后10的城市及其空气质量(AQI)。
使用的是BeautifulSoup库,具体用法参照http://wiki.jikexueyuan.com/project/python-crawler-guide/beautiful-soup.html
import requests
from bs4 import BeautifulSoup
import csv
def get_city_aqi(city_pinyin):
url = 'http://www.pm25.in/' + city_pinyin
r = requests.get(url, timeout=30)
soup = BeautifulSoup(r.text, 'lxml')
div_list = soup.find_all('div', {'class', 'span1'})
city_aqi = []
for i in range(8):
div_content = div_list[i]
caption = div_content.find('div', {'class', 'caption'}).text.strip()
value = div_content.find('div', {'class', 'value'}).text.strip()
# city_aqi.append((caption, value))
city_aqi.append(value)
return city_aqi
def get_all_cities():
url = 'http://www.pm25.in'
city_list = []
r = requests.get(url, timeout=30)
soup = BeautifulSoup(r.text, 'lxml')
city_div = soup.find_all('div', {'class', 'bottom'})[1]
city_link_list = city_div.find_all('a')
for city_link in city_link_list:
city_name = city_link.text
city_pinyin = city_link['href'][1:]
city_list.append((city_name, city_pinyin))
return city_list
def main():
city_list = get_all_cities()
header = ['City', 'AQI', 'PM2.5/h', 'PM10/h', 'CO/h', 'NO2/h', 'O3/h', 'O3/8h', 'SO2/h']
with open('china_cities_aqi.csv', 'w', encoding='utf-8', newline='') as f:
writer = csv.writer(f)
writer.writerow(header)
for i, city in enumerate(city_list):
if(i + 1) % 10 == 0:
print('已处理{}条记录(共{}条)'.format(i + 1, len(city_list)))
city_name = city[0]
city_pinyin = city[1]
city_aqi = get_city_aqi(city_pinyin)
row = [city_name] + city_aqi
writer.writerow(row)
if __name__ == '__main__':
main()