获取东方财富A股所有股票代码

最新推荐文章于 2025-03-24 13:02:59 发布

qh0526wy

最新推荐文章于 2025-03-24 13:02:59 发布

阅读量2.5k

点赞数

分类专栏： Python 文章标签：其他 python visual studio

本文链接：https://blog.csdn.net/qh0526wy/article/details/121455383

版权

Python 专栏收录该内容

68 篇文章

订阅专栏

本文介绍了一个针对东方财富网的爬虫程序实现，包括参数设置、网络数据抓取及存储过程。通过Python实现了从网站获取股票数据，并将其保存到SQLite数据库中。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

# encoding:utf-8
import sys
import os
import datetime
import logging
import time
import csv
import json
# 得到当前根目录  先找到应用的顶级根目录
qh_o_path = os.path.abspath(os.path.join(os.getcwd(), "..")) # 返回当前工作目录   先到达 QH_网络配置 的父文件夹路径
sys.path.append(qh_o_path)        # 添加自己指定的搜索路径
import QH_网络配置.QH_爬虫网络配置 as qh_wl_set
import QH_Sqlite数据库.qh_sqlite_ORM as qh_db_orm
import QH_Sqlite数据库.QH_Get_DB_Set as qh_db_set
import QH_爬虫对象.QH_东方财富网 as qh_PaChong
import QH_爬虫对象.QH_搜狐首页对象 as qh_souhu
import QH_Tool.QH_TOOL as qh_tool
import QH_Log_日志.QH_Log as qh_log

#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%参数部分，阙辉%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
#爬虫ID
qh_chuan_can = sys.argv
try:
    qh_spider_id = qh_chuan_can[1]
except:
    qh_spider_id = "QH_Spider_00058"
qh_cunchu_id00 = "QH_Spider_00058"
qh_cunchu_id01 = "QH_Spider_00058_1"
qh_cunchu_id02 = "QH_Spider_00058_2"
# 设置log 日志
qh_log_write = qh_log.Qh_log_write(qh_o_path,qh_spider_id)
qh_log_write.info("QH_该应用的顶级目录//：{}".format(qh_o_path))
# 获取搜狐交易日
qh_now_date = qh_souhu.QH_Get_Dangri_Date()  # 参数无意义，只是占位，否则报错
#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%网络爬取数据，阙辉%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
#获取网络参数
qh_log_write.info('获取网络参数！')
qh_wangluo = qh_wl_set.QH_Get_WangLuo_CanShu(qh_o_path, qh_spider_id)
#out
qh_url = qh_wangluo["QH_URL"]                        # 获取爬虫的URL
qh_header = qh_wangluo["QH_Header"]                  # 获取爬虫的请求头信息
qh_Params = qh_wangluo["QH_Params"]                  # 获取爬虫的GET请求的url参数
qh_Payload = qh_wangluo["QH_Payload"]                # 获取爬虫的POST请求的form参数
qh_filed_name = qh_wangluo["qh_Filed_EN"]            # 获取爬虫的解析字段的中文说明
qh_id_type = qh_wangluo["QH_Id_Type"]                # 获取数据库表字段的ID的拼接方法类型（旧）
qh_id_type_filed = qh_wangluo["QH_Id_Type_Filed"]    # 获取数据库表字段的ID的拼接方法具体值（旧）
#实例化爬虫对象
qh_PaChong_ject = qh_PaChong.QH_DongFangCaiFu_Spider(qh_url, qh_header, qh_Params, qh_filed_name)
qh_JieGuo_List = qh_PaChong_ject.QH_DongFang_Main()
qh_JieGuo_List_stock = []

for qh_i,qh_row in enumerate(qh_JieGuo_List):
    # 添加交易日期
    qh_row.append(qh_now_date)
    # 截取需要的字段存储数据中心
    qh_JieGuo_List_stock_row = []
    qh_zjs = qh_row[12]
    if str(qh_zjs) == "1":
        qh_zjs = "上交所"
    elif str(qh_zjs) == "0":
        qh_zjs = "深交所"
    else:
        qh_zjs = "它交所"
    qh_JieGuo_List_stock_row.append(qh_zjs)           #证交所名称
    qh_JieGuo_List_stock_row.append(qh_row[12])       #证交所 0深交所 1上交所
    qh_JieGuo_List_stock_row.append(qh_row[11])       #股票代码
    qh_JieGuo_List_stock_row.append(qh_row[13])       #股票名称
    qh_JieGuo_List_stock_row.append(qh_row[18])       #总市值
    qh_JieGuo_List_stock_row.append(qh_row[19])       #流通市值
    qh_JieGuo_List_stock_row.append(qh_row[1])        #最新价
    qh_JieGuo_List_stock_row.append(qh_row[25])       #市盈率

    qh_JieGuo_List_stock.append(qh_JieGuo_List_stock_row)

#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%数据存储，阙辉%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
def QH_CunChu_Data(qh_o_path,Qh_ShuJu_Data,qh_cunchu_id,qh_spider_id):
    """
    存储数据到数据库，作者：阙辉
    :param qh_o_path:        相对路径
    :param Qh_ShuJu_Data:    需要存储的数据，列表
    :param qh_cunchu_id:     存储方案
    :param qh_spider_id:     爬虫id
    :return:
    """
    if qh_cunchu_id == qh_spider_id:
        qh_log_write.info('存储方案和爬虫id一致_{}！'.format(qh_cunchu_id))
    else:
        qh_log_write.info('存储方案和爬虫id不一致_{}！'.format(qh_cunchu_id))
    #获取数据库参数
    qh_log_write.info('获取数据库参数！')
    qh_db_lujing = qh_db_set.QH_Get_DB_CanShu(qh_o_path, qh_cunchu_id)
    #out
    qh_db_path = qh_db_lujing["QH_DB_Path"]                  # 获取数据库路径
    qh_db_name = qh_db_lujing["QH_DB_Name"]                  # 获取数据库名称
    qh_table_name = qh_db_lujing["QH_Table_Name"]            # 获取数据库表名称
    qh_filed_set = qh_db_lujing["qh_Filed_Set"]              # 获取数据库表字段的设置列表
    qh_filed_value = qh_db_lujing["qh_Filed_Value"]          # 获取数据库表字段的名称
    qh_id_type = qh_db_lujing["QH_Id_Type"]                  # 获取数据库表字段的ID的拼接方法类型（新）
    qh_id_type_filed = qh_db_lujing["QH_Id_Type_Filed"]      # 获取数据库表字段的ID的拼接方法具体值（新）
    #存储数据到数据库   数据中心数据库
    #对象实例化
    qh_cunshu_db = qh_db_orm.QH_SQLite_DB_CunChu(qh_db_path,
                                                 qh_table_name,
                                                 qh_filed_set,
                                                 qh_filed_value,
                                                 qh_cunchu_id,
                                                 Qh_ShuJu_Data,
                                                 Qh_id_lie_c=qh_id_type_filed,
                                                 Qh_ID_Type=qh_id_type)
    qh_db_star = qh_cunshu_db.QH_CunChu_Data_Main()
    if qh_db_star:
        qh_log_write.info('对象数据库存储成功！')
    else:
        qh_log_write.info('对象数据库存储失败！')

#1、存入东方财富数据>>>一只股只存一条
QH_CunChu_Data(qh_o_path,qh_JieGuo_List,qh_cunchu_id00,qh_spider_id)
#2、存入东方财富数据>>>一只股一个交易日只存一条
QH_CunChu_Data(qh_o_path,qh_JieGuo_List,qh_cunchu_id01,qh_spider_id)
#3、存入数据中心>>>一只股只存一条
QH_CunChu_Data(qh_o_path,qh_JieGuo_List_stock,qh_cunchu_id02,qh_spider_id)