第一个爬虫程序，基于requests和BeautifulSoup

断断续续学了1年多python，最近总算感觉自己入门了，记录下这几天用requests和BeautifulSoup写的爬虫。
python的环境是anaconda+pycharm。
直接上代码

@requires_authorization
"""
    作者：西瓜不是我的
    日期：2017年12月26日
    功能：爬取全国每个城市各个监测点的AQI
    版本：v11.0
"""
import requests
from bs4 import BeautifulSoup
import csv

def get_city_area_aqi(url):
    '''
        获取城市监测点aqi
    '''
    r = requests.get(url, timeout=20)
    soup = BeautifulSoup(r.text, 'lxml')
    area_name = soup.find_all('thead')  # len = 1
    area_final_name = area_name[0].find_all('th')  # len = 13
    area_aqi = soup.find_all('tbody')  # len = 1
    area_final_aqi = area_aqi[0].find_all('tr')  # len = 13
    # 各监测点名称
    area_name_list = []
    # 最终目录
    final_list = []
    # 为各监测点名称列表赋初值
    for i in range(len(area_final_name)):
        area_name_list.append(area_final_name[i].text)
    # 将信息写入到最终目录中
    for i in range(len(area_final_aqi)):
        final_aqi = area_final_aqi[i].text.strip()
        aqi = final_aqi.split('\n')
        for j in range(len(area_name_list)):
            final_list.append((area_name_list[j], aqi[j]))
    return final_list

def write_to_csv(final_list,city_name):
    '''
        将获取到的city_aqi列表写到csv文件中
    '''
    with open('aqi.csv', 'a', encoding='utf-8', newline='') as f:
        writer = csv.writer(f)
        row = [city_name]
        # for循环用来写csv文件的数据行
        for i, name in enumerate(final_list):
            row.append(name[1])
            if (i + 1) % 11 == 0:
                writer.writerow(row)
                row = [city_name]

def get_all_city_name(url):
    '''
        获取所有城市的中文名和英文名
    '''
    r = requests.get(url,timeout = 30)
    soup = BeautifulSoup(r.text, 'lxml')
    city_div = soup.find_all('div',{'class':'bottom'})[1]
    city_name_list = city_div.find_all('a')

    city_name = []
    for name in city_name_list:
        name_text = name.text
        name_pinyin = name['href'][1:]
        city_name.append((name_text,name_pinyin))
    return city_name



def main():
    url = 'http://www.pm25.in'
    all_city_list = get_all_city_name(url)
    #写csv文件的第一行，即标题行
    write_row = ['city','监测点','AQI','空气质量','首要污染物','PM2.5','PM10','CO','NO2','O3-1','O3-8','SO2']
    with open('aqi.csv', 'w', encoding='utf-8', newline='') as f:
        writer = csv.writer(f)
        writer.writerow(write_row)
    #先获取所以城市的英文名，根据英文名找到对应的url，再获得每个城市各监测点的aqi值，最后写入到scv文件中
    for i,city in enumerate(all_city_list):
        city_name = city[0]
        city_pinyin = city[1]
        url = 'http://www.pm25.in/' + city_pinyin
        city_list = get_city_area_aqi(url)
        write_to_csv(city_list,city_name)
        #只爬取10个城市
        if i == 10:
            break

if __name__=='__main__':
    main()

第一次用markdown，就写这么多。