Python3爬取UN(联合国数据) api调用

# -*- coding=UTF-8 -*-
import time
import requests
import re
import os
from bs4 import BeautifulSoup
from requests.exceptions import ConnectionError, ReadTimeout
import csv
import json
import MySQLdb

headers = {
           'Accept': 'application/json',
           'Accept-Encoding': 'gzip, deflate, br',
           'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
           'Cache-Control': 'max-age=0',
           'Cookie':'_ga=GA1.2.656705372.1574068541; _gid=GA1.2.753921981.1579349838; modal_dismiss=null; ASP.NET_SessionId=qfvgigymys0wx4ehzakn0c0f',
           'Host': 'comtrade.un.org',
           'If-None-Match': '7c28fb33-3fa3-4587-9045-773f2bb8d9cb',
           'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1',
           }

#地区
area=[4,8,10,12,16,12,24,28,32,36,40,44,48,50,52,56,60,64,68,72,74,76,84,86,90,92,96,100,104,108,112,116,120,124,128,132,136,140,144,148,152,156,162,166,170,174,178,180,184,188,192,196,200,201,204,208,212,214,216,218,222,226,230,234,238,242,246,250,254,258,262,266,270,280,288,292,296,300,304,308,312,316,320,324,328,332,334,336,340,344,348,352,356,360,364,368,372,374,376,380,384,388,392,396,400,404,408,410,414,418,462,466,470,474,478,480,484,488,492,496,500,504,508,512,516,520,524,528,532,536,540,548,554,558,562,566,570,574,578,582,422,426,430,434,438,442,446,450,454,458,586,590,598,600,604,608,612,616,620,624,626,630,634,638,642,646,654,658,662,666,670,674,678,682,686,690,694,702,704,706,710,716,720,724,732,736,740,744,748,752,756,760,764,768,772,776,780,784,788,792,796,798,800,804,810,818,826,834,840,849,850,854,858,862,872,876,882,886,890,894]
year=['2018','2017','2016','2015','2014','2013','2012','2011','2010','2009','2008','2007','2006','2005','2004','2003','2002','2001','2000','1999','1998']

def getURL(url): #发送网络请求
    a = requests.get(url)
    #a.encoding = 'utf-8'  # 改变乱码问题
    html = a.text
    #print(html)
    return html

def doDown():
    #拼接请求URL
    # 打开数据库连接
    #db = MySQLdb.connect("1.1.1.1", "un", "123456", "un", charset='utf8')
    # 使用cursor()方法获取操作游标
    #cursor = db.cursor()
    for ye in year:
        # print(ar)
        for ar in area:
            url = 'https://comtrade.un.org/api/get?r=' + str(ar) + '&px=HS&ps=' + str(ye) + '&p=all&rg=all&cc=740400&max=500&type=C&freq=A&uitoken=99fb563dfb83da075b8de9dc782f6698'
            print(url)
            soup = BeautifulSoup(getURL(url), 'html.parser')  # 解析爬取网址
            json_str=json.loads(str(soup))
            datas=json_str['dataset']
            for da in datas:
                with open('123.csv', 'a', encoding='utf-8-sig', newline='') as f:
                    csv_writer = csv.writer(f, delimiter=',')
                    csv_writer.writerow(
                        [str(da['pfCode']), da['yr'], da['period'], da['periodDesc'], da['aggrLevel'], da['IsLeaf'], da['rgCode'],
                         da['rgDesc'], da['rtCode'], da['rtTitle'], da['rt3ISO'],
                         da['ptCode'], da['ptTitle'], da['pt3ISO'], da['ptCode2'],
                         da['ptTitle2'], da['pt3ISO2'], da['cstCode'], da['cstDesc'],
                         da['motCode'], da['motDesc'], da['cmdCode'], da['cmdDescE'],
                         da['qtCode'], da['qtDesc'], da['qtAltCode'], da['qtDesc'],
                         da['TradeQuantity'], da['AltQuantity'], da['NetWeight'], da['GrossWeight'],
                         da['TradeValue'], da['CIFValue'], da['FOBValue'], da['estCode']])

if __name__ == '__main__':
    doDown() # 执行下载方法
    print('结束！！！！！！！！！！！！！！！！！！！！！！！！！！！！！')