保定市2018-2019天气爬取

本文通过运用python网络爬虫爬取了24个网页并对其进行解析，获取每日天气情况并将其写入csv文件，并进行后续的数据分析及其可视化得出文章结论。

例如，2018年1月的网址是http://www.tianqihoubao.com/lishi/baoding/month/201801.html

注意问题：
1）目标网页禁止访问；电脑IP访问过多
2）获得网页HTML内容有乱码，是编码格式原因，将utf-8改为gbk。
3）用标签获取目标内容时遇到问题
4）正则表达式运用

写入csv文件后

所得词云图、柱状图、折线图

在这里插入图片描述

下面是全部代码

import requests
from bs4 import BeautifulSoup as BS
import re
import csv
import jieba
import wordcloud
import imageio
import matplotlib
import matplotlib.pyplot as plt
import os


# 打开网页
def getHtml(year, month):
    url = 'http://www.tianqihoubao.com/lishi/baoding/month/'+year + month + '.html'
    rs = requests.get(url, timeout=30)
    statusCode = rs.status_code
    if statusCode == 200:
        rs.encoding = 'GB2312'
        return rs.text
    else:
        return "Failed to get url html"


# 获取天气情况
def getComments(html, month):
    soup = BS(html, 'html.parser')   #解析网页
    span_s = soup.find_all('tr')   # 获取'tr'标签内容
    comment_s = []
    for span in span_s:
        comment = span.find_all('td')   # 获取'td'标签内容
        comment_s.append(comment)
    comments = [[]]
    for p in comment_s[0]:
        comments[0].append(p.b.string)
    for p in comment_s[1:]:   #每个网页所获列表均有题头标签，为后续方便，只要第一页的题头
        comments.append(p)
    if month == '01':
        comments.pop(1)
    comments[0].append('最高气温')
    comments[0].append('最低气温')
    for p in comments[1:]:
        p[0] = p[0].a
        for i in range(4):
            p[i] = p[i].string
            p[i] = re.sub('\s+', ' ', p[i])
        pp = p[2].split("/")   # 以'/'分开最高和最低气温并加到后两列
        p.append(pp[0])
        p.append(pp[1])
    if month != '01':
        comments.pop(0)
    return comments


# 将天气数据写入csv
def getData(file,year):
    month = ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
    for i in month:
        html = getHtml(year,i)
        comments = getComments(html, i)
        with open(file, 'a', newline='') as fp:   # 将comments[]追加到file文件中
            writeCsv = csv.writer(fp)
            for s in comments:
                writeCsv.writerow(s)


# 分词处理
def cutWord(txt):
    if isinstance(txt, list):
        txt = " ".join(txt)    # 用空格把词连起来
    word_s = jieba.lcut(txt)
    newWord_s = []
    for word in word_s:
        if word != ' ' and word != '局部' and word != '/':
            newWord_s.append(word)
    return newWord_s


# 天气状况处理
def coWord(file):
    with open(file,'r') as fp:
        reader = csv.DictReader(fp)
        status = [item['天气状况'] for item in reader]
    txt = ''
    for i in status:
        txt += i
    newwords = cutWord(txt)
    for i in range(len(newwords)):
        if newwords[i]=='晴':
            newwords[i]='晴天'
        if newwords[i]=='阴':
            newwords[i]='阴天'
    words = " ".join(newwords)
    return words


# 生成词云
def drawWordCloud(txt, maskPic, shieldWords=['局部'], filePic='./tianqi.png'):
    txt = txt
    maskImage = imageio.imread(maskPic)
    width, height, ichannel= maskImage.shape
    wc = wordcloud.WordCloud(font_path='c:\windows\fonts\simhei.ttf',
                            mask = maskImage,
                            width = width, height = height,
                            #width=1000,height=700,
                            background_color='white',
                            stopwords=shieldWords
                            )
    wc.generate(txt)
    wc.to_file(filePic)
    wc.to_image().show()
    print('Draw wordCloudChart done! check picture {}'.format(filePic))


# 统计词频:对wordList列表中词进行词频统计，并按照降序依次写入csv文件中，并返回排序后的词频列表
def count2csv(file, fileCSV):
    with open(file,'r') as fp:
        reader = csv.DictReader(fp)
        status = [item['天气状况'] for item in reader]
    txt = ''
    for i in status:
        txt += i
    wordList = cutWord(txt)
    from collections import Counter
    wordsCount = dict(Counter(wordList)) #K:word,V:count
    wordsCountList = list(wordsCount.items())
    wordsCountList.sort(key=lambda item:item[1],reverse=True) #按照count，降序
    with open(fileCSV, 'w', newline='') as fp:
        writer = csv.writer(fp)
        writer.writerows(wordsCountList)
    print("Write wordsCountList to CSV done! check %s" % fileCSV)
    return wordsCountList


# 绘制天气状况柱形图
def drawBar(x, y, series, title='柱形图', picName='picBar1.jpg'):
    matplotlib.rcParams['font.family'] = 'SimHei'
    matplotlib.rcParams['font.sans-serif'] = ['SimHei']
    series = series   # 系列名称：series = []
    dataX = x   # x轴数据
    dataY = y   # y轴数据
    n = len(series)
    if n == 1:
        ind = list(range(len(dataY)))
    else:
        ind = list(range(len(dataY[0])))
    barWidth = 0.3   # 设置柱形图宽度
    if n == 1:
        ind1 = ind
        rects1 = plt.bar(ind1, dataY, barWidth, label=series)
    else:
        for i in range(n):
            ind2 = list(map(lambda x: x+barWidth*(i+1), ind))
            plt.bar(ind2, dataY[i], barWidth, label=series[i])
    if n == 1:
        plt.xticks(ind, dataX)   # ind:标签位置 dataX：标签内容
    else:
        ind3 = list(map(lambda x: x+(barWidth/2)*(n+1), ind))
        plt.xticks(ind3, dataX)   # ind3:标签位置 dataX：标签内容
    plt.title(title)   # 图表标题
    plt.legend(series)  # 图例
    plt.savefig(picName)   # 保存
    plt.show()   # 显示
    print('Draw barChart done! check picture {}'.format(picName))


# 处理最高气温数据
def Maximum(file):
    with open(file, 'r') as fp:
        reader = csv.DictReader(fp)
        zuigao = [item['最高气温'] for item in reader]
    for i in range(len(zuigao)):   # 去掉气温单位摄氏度
        zuigao[i] = zuigao[i][-3::-1]
        zuigao[i] = eval(zuigao[i][::-1])
    return zuigao



# 处理最低气温数据
def Minimum(file):
    with open(file, 'r') as fp:
        reader = csv.DictReader(fp)
        zuidi = [item['最低气温'] for item in reader]
    for i in range(len(zuidi)):
        zuidi[i] = zuidi[i][-3::-1]
        zuidi[i] = eval(zuidi[i][::-1])
    return zuidi



# 绘制一年气温变化折线图
def lineChart(file,y1, y2, title='折线图', picName='picBar1.jpg'):
    a = open(file, 'r')
    x = range(1, len(a.readlines()))   # 设置横坐标天数
    plt.plot(x, y1, label='Zuigao line', color='r')
    plt.plot(x, y2, label='Zuidi line')
    plt.xlabel('Day Number')
    plt.ylabel('Qiwen')
    plt.title(title)
    plt.legend()
    plt.savefig(picName)   # 保存
    plt.show()   # 显示
    print('Draw lineChart done! check picture {}'.format(picName))


# 绘制两年气温变化折线图
def linesChart(y1, y2, y3, y4, title='折线图', picName='picChart.jpg'):
    x = range(1,335)
    plt.plot(x, y1, label='Zuigao2018 line', color='green')
    plt.plot(x, y1, label='Zuigao2019 line', color='red')
    plt.plot(x, y2, label='Zuidi2018 line', color='purple')
    plt.plot(x, y2, label='Zuidi2019 line', color='blue')
    plt.xlabel('Day Number')
    plt.ylabel('Qiwen')
    plt.title(title)
    plt.legend()
    plt.savefig(picName)
    plt.show()
    print('Draw lineChart done! check picture {}'.format(picName))


if __name__ == '__main__':
    catalog = os.getcwd()   # 获取当前文件所在路径
    catalogue = catalog.replace('codes','others')
    
    file = catalogue+'\\tianqi2018.csv'
    getData(file,'2018')   # 获取2018年天气情况

    word2018 = coWord(file)
    drawWordCloud(word2018, maskPic=catalogue+'\\chinaMap.png',filePic=catalogue+'\\tianqi2018.jpg')

    gao2018 = Maximum(file)
    di2018 = Minimum(file)
    max2018 = max(gao2018)
    min2018 = min(di2018)

    file1 = catalogue+'\\tianqi2019.csv'
    getData(file1,'2019')   # 获取2019年天气情况

    word2019 = coWord(file1)
    drawWordCloud(word2019,maskPic=catalogue+'\\chinaMap.png',filePic=catalogue+'\\tianqi2019.jpg')

    gao2019 = Maximum(file1)
    di2019 = Minimum(file1)
    max2019 = max(gao2019)
    min2019 = min(di2019)

    print("2018年最高气温是",max2018,"摄氏度")
    print("2018年最低气温是",min2018,"摄氏度")
    print("2019年最高气温是",max2019,"摄氏度")
    print("2019年最低气温是",min2019,"摄氏度")
   
    fileCSV = catalogue+'\\Status2018.csv'
    wordFrequency = count2csv(file, fileCSV)
    x = [item[0] for item in wordFrequency]
    y = [item[1] for item in wordFrequency]
    series = ['wordFrequency']
    drawBar(x, y, series, title='2018年保定市全年天气状况天数统计', picName=catalogue + '\\status2018.png')
    lineChart(file,gao2018, di2018, title='2018年保定市气温变化', picName=catalogue+'\\qiwen2018.png')

    fileCSV1 = catalogue+'\\Status2019.csv'
    wordFrequency1 = count2csv(file1, fileCSV1)
    x1 = [item[0] for item in wordFrequency1]
    y1 = [item[1] for item in wordFrequency1]
    series1 = ['wordFrequency']
    drawBar(x1, y1, series1, title='2019年保定市全年天气状况天数统计', picName=catalogue + '\\status2019.png')
    lineChart(file1,gao2019, di2019, title='2019年保定市气温变化', picName=catalogue+'\\qiwen2019.png')

    linesChart(gao2018[:334], gao2019[:334], di2018[:334], di2019[:334], title='2018-2019年保定市气温变化',picName=catalogue+'\\qiwen.png')

codes运行过程说明：
1.获取当前文件所在路径

2.调用getdata()函数获取2018年天气情况并追加写入’\tianqi2018.csv’文件：
getdata()中调用getHtml()函数打开网页，再调用getComments()函数获取每月天气状况
3.调用coWord()函数读取第2步生成的’tianqi2018.csv’文件，对文件中"天气状况"一列数据取出并处理；
再调用drawWordCloud()函数对“天气状况”以’chinaMap.png’为背景做出词云图’tianqi2018.jpg’
4.分别调用Maximun(),Minimum()获取’tianqi2018.csv’文件中最高温度最低温度的列表，再求出2018年最高最低气温

5.调用getdata()函数获取2019年天气情况并追加写入’\tianqi2019.csv’文件：
getdata()中调用getHtml()函数打开网页，再调用getComments()函数获取每月天气状况
6.调用coWord()函数读取第5步生成的’tianqi2019.csv’文件，对文件中"天气状况"一列数据取出并处理；
再调用drawWordCloud()函数对“天气状况”以’chinaMap.png’为背景做出词云图’tianqi2019.jpg’
7.分别调用Maximun(),Minimum()获取’tianqi2019.csv’文件中最高温度最低温度的列表，再求出2019年最高最低气温

8.调用count2csv()函数，对’tianqi2018.csv’文件中“天气状况”一列进行词频统计，并将统计结果写入’Status2018.csv’文件中；
再调用drawBar()函数，将’Status2018.csv’文件中内容做成柱状图显示，保存为图片’status2018.png’；
再调用lineChart()函数将’tianqi2018.csv’文件中最高气温最低气温两列数据做成折线图显示，保存为图片’qiwen2018.png’

9.调用count2csv()函数，对’tianqi2019.csv’文件中“天气状况”一列进行词频统计，并将统计结果写入’Status2019.csv’文件中；
再调用drawBar()函数，将’Status2019.csv’文件中内容做成柱状图显示，保存为图片’status2019.png’；
再调用lineChart()函数将’tianqi2019.csv’文件中最高气温最低气温两列数据做成折线图显示，保存为图片’qiwen2019.png’

10.调用linesChart()函数以第4、7步中所获气温列表为参数做折线图显示，并保存为’qiwen.png’