联合国数据库爬虫

最新推荐文章于 2024-10-04 21:25:15 发布

San Chang

最新推荐文章于 2024-10-04 21:25:15 发布

阅读量738

点赞数

文章标签：大数据

本文链接：https://blog.csdn.net/qq_26657001/article/details/104330465

版权

本文主要介绍了如何使用爬虫技术抓取联合国数据库中的表格数据，包括数据的预处理和爬虫程序的主要实现步骤。

摘要由CSDN通过智能技术生成

表格处理

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import xlrd
import xlwt
import time
import numpy as np
import os

def xls_reader():
    path="C:/Users/Administrator/Desktop/lz/对照表.xlsx"
    data=xlrd.open_workbook(path)
    sheet_classfication=data.sheet_by_name(u"对比盛斌版")
    #print(sheet_classfication.ncols)
    gb_classfication=[]
    for i in range(239-2):
        if sheet_classfication.cell_value(i+3,2)!="":
            gb_classfication.append(sheet_classfication.cell_value(i+3,2))
    print(gb_classfication)
    SITC_codes = []
    SITC_code=[]
    for i in range(sheet_classfication.nrows-3-2):
        if sheet_classfication.cell_value(i + 3, 3) == "":
            SITC_codes.append(SITC_code)
            SITC_code=[]
        else:
            SITC_code.append(sheet_classfication.cell_value(i + 3, 3))
    print(len(gb_classfication),len(SITC_codes))

    path2="C:/Users/Administrator/Desktop/lz/OFDI计算版.xlsx"
    data=xlrd.open_workbook(path2)

    country = ["United Arab Emirates", "Philippines", "Rep.of Korea", "Cambodia",
               "Lao People's Dem. Rep.",
               "Malaysia", "Japan", "Turkey", "Thailand",
               "Viet Nam",
               "Indonesia", "Egypt", "Cameroon", "Kenya", "Madagascar",
               "Mauritius", "South Africa", "Nigeria", "Zambia",
               "Russian Federation", "France", "Sweden", "Italy", "United Kingdom",
               "Argentina", "Brazil", "Ecuador", "Peru", "Mexico",
               "Venezuela", "Chile", "Canada", "USA",
               "Australia", "New Zealand"]
    country = ["Kazakhstan", "Bangladesh","Uruguay"]
    country=["Germany"]
    #year=np.arange(2003,2019,1)

    year=[2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017,2018]#
    year=[2003,2004,2005,2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017,2018]
    print(year)
    return country,year,gb_classfication,SITC_codes
country,year,gb_classfication,SITC_codes=xls_reader()

def filter_fun(x):
    if x in ["0","1","2","3","4","5","6","7","8","9"]:
        return True

def excle_done(country,year,gb_classfication,SITC_codes):
    codes = []
    for j in range(len(SITC_codes)):
        for k in range(len(SITC_codes[j])):
            codes.append(SITC_codes[j][k])
    #print(len(codes))

    path2 = "C:/Users/Administrator/Desktop/lz/03-06/OFDI计算版2"
    write_data=xlwt.Workbook(encoding="utf-8")
    wt_sheet_sitc=write_data.add_sheet("SITC",cell_overwrite_ok=True)
    wt_sheet_gb=write_data.add_sheet("GB",cel