搭建自己的金融数据源和量化分析平台（六）：下载并存储沪深两市上市公司财报

KogRow(接毕设和大作业版)

已于 2024-08-06 19:11:49 修改

阅读量371

点赞数 2

分类专栏：杂项文章标签：金融

于 2024-08-02 12:43:07 首次发布

本文链接：https://blog.csdn.net/shuaicenglou3032/article/details/140869527

版权

杂项专栏收录该内容

26 篇文章 3 订阅

订阅专栏

基于不依赖wind、某花顺等第三方平台数据的考虑，尝试直接从财报中解析三大报表进而计算ROE等财务指标，因此需要下载沪深两市的上市公司财报数据，便于后续从pdf中解析三大报表。
深市爬虫好做，先放深市爬虫：

'''
根据时间段下载深交所上市公司财报
path str 指定财报存储路径
time str 财报年度 如[2023,2024]
stock_list list 下载财报的股票代码列表 例如['000001','000002']
financial_statements_type list 财报的类别 例如['annual','semi-annual','quarterly_1','quarterly_3'] 分别为年报、半年报、一季报、三季报
'''
def get_financial_statements(path, time, stock_list, financial_statements_type):
    url = "https://www.szse.cn/api/disc/announcement/annList"
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3',
        'Content-Type': 'application/json',
        'Connection': 'close'
    }
    headers_download = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3',
        'Connection': 'close'
    }
    download_url = "https://disc.static.szse.cn/download"
    # 逐只股票读取相应pdf报表
    for stock in stock_list:
        # 逐年循环
        for year in time:
            # 根据财报类型逐个读取pdf
            for fs_type in financial_statements_type:
                if fs_type == 'annual':
                    title = "年报"
                    bigCategoryId = '010301'   # 年报查询代码
                    timestart = str(year)+"-12-31"
                    timeend = str(year+1)+"-09-01"  # 防止出现财报更正之后时间节点覆盖不到，统一往后推三个月
                elif fs_type == 'semi-annual':
                    title = "中报"
                    bigCategoryId = '010303'   # 中报查询代码
                    timestart = str(year) + "-07-01"
                    timeend = str(year) + "-12-31"
                elif fs_type == 'quarterly_1':
                    title = "一季报"
                    bigCategoryId = '010305'   # 一季报查询代码
                    timestart = str(year) + "-04-01"
                    timeend = str(year) + "-07-31"
                else:
                    title = "三季报"
                    bigCategoryId = '010307'   # 三季报查询代码
                    timestart = str(year) + "-10-01"
                    timeend = str(year) + "-12-31"
                data = {
                    "seDate": [timestart, timeend],
                    "stock": [stock],
                    "channelCode": ["listedNotice_disc"],
                    "bigCategoryId": [bigCategoryId],
                    "pageSize": 50,
                    "pageNum": 1
                }
                response = requests.post(url=url, data=json.dumps(data), headers=headers)
                data = json.loads(response.text)["data"]
                response.close()
                if len(data) == 0 or data is None:
                    print("警告:股票代码:"+stock+" "+str(year)+title+"不存在!")
                else:
                    for entry in data:
                        # 对摘要栏目做特殊处理，去除摘要
                        if entry['title'].find("摘要") < 0:
                            # 检查path路径下stock代码文件夹、年份文件夹是否存在，不存在则创建
                            file_path = path+stock+"/"+str(year)
                            if Tools.check_folder_exists(path+stock) == False:
                                os.mkdir(path+stock)
                            if Tools.check_folder_exists(file_path) == False:
                                os.mkdir(file_path)
                            file = file_path + "/" + str(year) + title + "##" + entry['title'].replace("*", "") + ".pdf"
                            # 检查文件是否已存在，不存在再下载
                            if os.path.exists(file):
                                print("警告:股票代码:" + stock + " " + str(year) + title + "已存在!")
                            else:
                                filecontent = requests.get(url = download_url + entry["attachPath"],headers=headers_download)
                                with open(file, "wb") as pdf:
                                    pdf.write(filecontent.content)
                                print("股票代码:" + stock + " " + str(year) + title + "写入成功。")
                                filecontent.close()

沪市爬虫：

'''
根据时间段下载上交所上市公司财报
time str 财报年度 如2024、2023
stock_list list 下载财报的股票代码列表 例如['000001','000002']
financial_statements_type list 财报的类别 例如['annual','semi-annual','quarterly_1','quarterly_3'] 分别为年报、半年报、一季报、三季报
'''
def get_financial_statements(path, time, stock_list, financial_statements_type):
    s = requests.session()
    s.keep_alive = False
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3',
        'Referer': 'https://www.sse.com.cn/',
        'Connection': 'close'
    }
    download_url = "https://www.sse.com.cn"
    # 逐只股票读取相应pdf报表
    for stock in stock_list:
        # 逐年循环
        for year in time:
            # 根据财报类型逐个读取pdf
            for fs_type in financial_statements_type:
                if fs_type == 'annual':
                    title = "年报"
                    bigCategoryId = 'YEARLY'  # 年报查询代码
                    timestart = str(year) + "-12-31"
                    timeend = str(year + 1) + "-09-01"  # 防止出现财报更正之后时间节点覆盖不到，统一往后推三个月
                elif fs_type == 'semi-annual':
                    title = "中报"
                    bigCategoryId = 'QUATER2'  # 中报查询代码
                    timestart = str(year) + "-07-01"
                    timeend = str(year) + "-12-31"
                elif fs_type == 'quarterly_1':
                    title = "一季报"
                    bigCategoryId = 'QUATER1'  # 一季报查询代码
                    timestart = str(year) + "-04-01"
                    timeend = str(year) + "-07-31"
                else:
                    title = "三季报"
                    bigCategoryId = 'QUATER3'  # 三季报查询代码
                    timestart = str(year) + "-10-01"
                    timeend = str(year) + "-12-31"
                url = "https://query.sse.com.cn/security/stock/queryCompanyBulletin.do?jsonCallBack=jsonpCallback"+str(random.randint(10000, 999999))+"&isPagination=true&pageHelp.pageSize=50&pageHelp.pageNo=1&pageHelp.beginPage=1&pageHelp.cacheSize=1&pageHelp.endPage=1&productId="+stock+"&securityType=0101%2C120100%2C020100%2C020200%2C120200&reportType2=DQBG&reportType="+bigCategoryId+"&beginDate="+timestart+"&endDate="+timeend
                response = requests.get(url=url, headers=headers)
                datas = json.loads(response.text.split('"keyWord":null,"pageHelp":')[1].split(',"productId":')[0])['data']
                response.close()
                if len(datas) == 0 or datas is None:
                    print("警告:股票代码:" + stock + " " + str(year) + title + "不存在!")
                else:
                    for entry in datas:
                        # 对摘要栏目做特殊处理，去除摘要
                        if entry['TITLE'].find("摘要") < 0:
                            # 检查path路径下stock代码文件夹、年份文件夹是否存在，不存在则创建
                            file_path = path + stock + "/" + str(year)
                            if Tools.check_folder_exists(path + stock) == False:
                                os.mkdir(path + stock)
                            if Tools.check_folder_exists(file_path) == False:
                                os.mkdir(file_path)
                            file = file_path + "/" + str(year) + title + "##" + entry['TITLE'].replace("*", "").replace("\t","") + ".pdf"
                            # 检查文件是否已存在，不存在再下载
                            if os.path.exists(file):
                                print("警告:股票代码:" + stock + " " + str(year) + title + "已存在!")
                            else:
                                filecontent = requests.get(url = download_url + entry["URL"],headers=headers)
                                with open(file, "wb") as pdf:
                                    pdf.write(filecontent.content)
                                print("股票代码:" + stock + " " + str(year) + title + "写入成功。")
                                filecontent.close()
# timestart = [2023]
# stock_list = ['600011']
# financial_statements_type = ['annual', 'semi-annual', 'quarterly_1', 'quarterly_3']
# SZ_financial_statement_path = "F:/data/SH/"
# get_financial_statements(SZ_financial_statement_path, timestart,stock_list,financial_statements_type)

控制模块代码：

#更新A股股票财报数据
def update_A_financial_data(SZ=False,SH=False,BJ=False,time=[],financial_statements_type=[]):
    database = "stock_a"
    if SZ == True:
        # 读取深交所上市处于上市状态的公司股票代码
        select_sql = "select stock_code from stock_list where exchange = '"+SZSE+"' and list_status = '"+LIST+"'"
        select_result = ExecSelect(database, select_sql)  # 读取查询结果
        stocks = []
        for stock in select_result:
            stocks.append(stock[0])
        A_SZ_basic.get_financial_statements(SZ_financial_statement_path, time, stocks, financial_statements_type)
    if SH == True:
        # 读取上交所上市处于上市状态的公司股票代码
        select_sql = "select stock_code from stock_list where exchange = '" + SSE + "' and list_status = '" + LIST + "'"
        select_result = ExecSelect(database, select_sql)  # 读取查询结果
        stocks = []
        for stock in select_result:
            stocks.append(stock[0])
        A_SH_basic.get_financial_statements(SH_financial_statement_path, time, stocks, financial_statements_type)
    if BJ == True:
        pass