世界各国1960年到2020年gdp数据抓取

文末附有打包的数据

不让我放网站图片和网址，说是涉及版权，网站名叫做快易数据，那就感兴趣的小伙伴去看看吧，文末也有爬好汇总好的数据

这个网站设置一下请求头就能爬，几乎没有什么反爬措施，没啥好讲的，所有的网站链接也都写在了html里，清洗一下就能拿到，就分享一下源代码和爬取到的数据吧，希望能减轻一点服务器的压力

代码：

import requests,time,random
from bs4 import BeautifulSoup
from multiprocessing import Pool

# 定义一个函数请求网页，返回网页的text
def get_html(url):
    try:
        headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:100.0) Gecko/20100101 Firefox/100.0', }

        r = requests.get(url, headers=headers,timeout=10)
        text = r.text
        print(r.status_code)
    except Exception as e:
        text = ''
        print('请求网页{}出错！！！,请重试'.format(url))
        with open('error.txt','a+',encoding='utf-8') as f:
            f.write(url + '\n')
        # print(r.status_code)
        print(e)
        pass
    time.sleep(2)

    return text

# 定义一个清洗和保存函数
def clean_and_save(name,html):
    if len(html) != '':
        try:
            soup = BeautifulSoup(html, 'lxml')
            tbody = soup.select('.table')
            with open('./各国gdp/' + name + '.txt', 'w', encoding='utf-8') as f:
                f.write('年份,' + name + '\n')
                for t in tbody:
                    text = t.text.split('\n\n\n')
                    for i in text[3:]:
                        st = i.split('\n')
                        if len(st) == 3:
                            year = st[0]
                            gdp_num = st[1].split(' ')[1].replace(',', '').strip('(').strip(')')
                        elif len(st) == 5:
                            year = st[2]
                            gdp_num = st[3].split(' ')[1].replace(',', '').strip('(').strip(')')
                        f.write(year + ',' + gdp_num + '\n')
            print('数据写入成功')
        except:
            print('数据写入失败！！！请重试')
            pass
# 功能和上一个一样，接受的传参不一样，用于多线程版
def clean_and_save1(d):
    name,url = d[0],d[1]
    html = get_html(url)
    if len(html) != '':
        try:
            soup = BeautifulSoup(html, 'lxml')
            tbody = soup.select('.table')
            with open('./各国gdp/' + name + '.txt', 'w', encoding='utf-8') as f:
                f.write('年份,' + name + '\n')
                for t in tbody:
                    text = t.text.split('\n\n\n')
                    for i in text[3:]:
                        st = i.split('\n')
                        if len(st) == 3:
                            year = st[0]
                            gdp_num = st[1].split(' ')[1].replace(',', '').strip('(').strip(')')
                            baifenbi = st[2]
                        elif len(st) == 5:
                            year = st[2]
                            gdp_num = st[3].split(' ')[1].replace(',', '').strip('(').strip(')')
                            baifenbi = st[4]
                        f.write(year + ',' + gdp_num + '\n')
            print('数据写入成功')
        except:
            print('数据写入失败！！！请重试')
            pass
# 读取url内的信息，处理之后返回一个列表
def get_tulpe():
    f = open('各国url.txt', 'r', encoding='utf-8')
    data = []
    ls = f.read().split('\n')
    for l in ls[:-1]:
        d1,d2 = l.split(',')
        data.append((d1,d2))
    return data
# 定义单线程版主函数
def main():
    print('开始执行...')
    f = open('各国url.txt','r',encoding='utf-8')
    data = f.read().split('\n')
    for d in data[:-1]:
        d = d.split(',')
        html = get_html(d[1])
        clean_and_save(d[0],html)
        time.sleep(3)
    print('执行结束')
    # print(data)
# 定义多线程版主函数
def run():
    data = get_tulpe()
    # data = [('泰国','https://www.kylc.com/stats/global/yearly_per_country/g_gdp/tha.html'),('瑞典','https://www.kylc.com/stats/global/yearly_per_country/g_gdp/swe.html'),('圭亚那','https://www.kylc.com/stats/global/yearly_per_country/g_gdp/guy.html'),('冈比亚','https://www.kylc.com/stats/global/yearly_per_country/g_gdp/gmb.html'),('马拉维','https://www.kylc.com/stats/global/yearly_per_country/g_gdp/mwi.html')]
    pool = Pool(processes=4)
    pool.map(clean_and_save1,data)
if __name__ == '__main__':
    # 单个请求版
    main()
    # 多进程版
    # run()

效果展示：

里面大概是211个国家的数据吧！通过源代码下载下来的数据是一个国家一个文件，整合的方法也特地写了一篇博客：数据文件整合方法传送门

世界各国1960-2020年gdp数据蓝奏云链接