AERONET站点数据处理思路

1、从官网下载中国地区站点对应的压缩包文件

2、筛选符合条件（这里是2018年及以后的文件）的压缩包文件并解压缩到指定文件夹extracted

## AERONET_ExtractToFiles 用于解压缩AERONET 的压缩包同时删选给定年份之后的所有数据文件
import os
import zipfile
import os
import glob
import pandas as pd
import datetime as dt
import math
##------------------------------------------------------------------------------------------
# #main函数用于从AERONET官网下载的所有压缩包文件提取出2018年4月到2019年4月逐月分的10min分辨率的13个csv文件

# check_and_delete_file用于筛选和解压缩
def check_and_delete_file(filepath, filename):
    # 提取文件名中的年份
    year_str = filename[9:13]
    year = int(year_str)

    # 判断年份是否大于2018
    if year > 2017:
        print(f'{filename} 中的年份大于2018，保留该文件')
        # 指定解压目标路径
        extract_path = os.path.join(folder, 'extracted')
        os.makedirs(extract_path, exist_ok=True)
        # 解压缩文件到指定路径
        with zipfile.ZipFile(filepath, 'r') as zip_ref:
            zip_ref.extractall(extract_path)

    else:
        print(f'{filename} 中的年份不大于2018，删除该文件')
        # 删除文件
        os.remove(filepath)




# 1 将压缩包文件按照年份进行筛选和解压缩，得到满足要求的lev15和level20数据

folder = './Station_Process'  # 指定文件夹路径
# 遍历文件夹中的每个文件
for filename in os.listdir(folder):
    filepath = os.path.join(folder, filename)
    if os.path.isfile(filepath) and filepath.endswith('.zip'):
        check_and_delete_file(filepath, filename)

# 2 对得到的lev15和level20文件修改文件后缀变成txt
extract_path = os.path.join(folder, 'extracted')

3、把extracted文件夹下面的所有文件合并起来成为一个表格

#!/usr/bin/python
# -*- coding: utf-8 -*-
from typing import List, Any
## 将站点做初始合并
import pandas as pd
import numpy as np
import os
import glob
from scipy.optimize import curve_fit
import datetime as dt
import math
# AERONET_SelectToExcel
# 批量更改后缀名
InputPath=r'.\Station_Process\extracted'
File=glob.glob(os.path.join(InputPath,'*'))

# 合并所需要的站点数据
# InputPath=r'G:\AERONET\AOD_Level20_All_Points_V3\AOD\AOD20\ALL_POINTS'
File = glob.glob(os.path.join(InputPath, '*.txt'))
# df_=pd.read_csv(File[0],header=6,index_col=False,na_values=['N/A'])#index_col=False为去掉第一列作为索引值
dateparse = lambda x: dt.datetime.strptime(x, "%d:%m:%Y %H:%M:%S")  # 日期的转换
df = pd.DataFrame(
    columns=['AERONET_Site_Name', 'Site_Latitude(Degrees)', 'Site_Longitude(Degrees)', 'Site_Elevation(m)',
             'Dates', "Day_of_Year", "Precipitable_Water(cm)", 'AOD_440nm', "AOD_500nm", 'AOD_675nm',
             '440-675_Angstrom_Exponent', 'Interpolation_AOD_550nm'])  # 所需要的字段

for i in File:
    aeronet = pd.read_csv(i, header=6, na_values=['N/A'], index_col=False,
                          parse_dates={'Dates': [0, 1]},
                          date_parser=dateparse)

    aeronet = aeronet.loc[:,
              ['AERONET_Site_Name', 'Site_Latitude(Degrees)', 'Site_Longitude(Degrees)', 'Site_Elevation(m)',
               'Dates', "Day_of_Year", "Precipitable_Water(cm)", 'AOD_440nm', "AOD_500nm", 'AOD_675nm',
               '440-675_Angstrom_Exponent', ]]
    # 去除无效值
    aeronet = aeronet.loc[aeronet['AOD_440nm'].map(int) != -999]
    aeronet = aeronet.loc[aeronet['AOD_500nm'].map(int) != -999]
    aeronet = aeronet.loc[aeronet['AOD_675nm'].map(int) != -999]
    aeronet = aeronet.loc[aeronet['440-675_Angstrom_Exponent'].map(int) != -999]
    aeronet = aeronet.loc[aeronet['Precipitable_Water(cm)'].map(int) != -999]
    df = pd.concat([aeronet, df], axis=0)


#df.to_csv(r'F:\36-AERONET_Validation\aeronet\DATA_merge\aod_merge.csv',index=False)
#使用波长指数（440-675_Angstrom）拟合插值550nmAOD

df['AOD_440nm']=df['AOD_440nm'].astype(float)
df['AOD_440nm']=df['AOD_440nm'].astype(float)
df['AOD_675nm']=df['AOD_675nm'].astype(float)
df['440-675_Angstrom_Exponent']=df['440-675_Angstrom_Exponent'].astype(float)
df['Interpolation_AOD_550nm']=df['Interpolation_AOD_550nm'].astype(float)
df['Interpolation_AOD_550nm']=df.apply(lambda x: x['AOD_440nm']*math.pow((550/440), -x['440-675_Angstrom_Exponent']) , axis=1)

df.to_csv(r'.\Result\aod_convert_china.csv',index=False)