获取所有股票历史行情数据

第一段 所有股票 周期:天  复权:不复权

# encoding:utf-8
import os
import sys
import pandas as qh_pd
import datetime
import time
import random
if __name__ == "__main__":
    print("本文件引用")
    qh_path = os.path.abspath(os.path.join(os.getcwd(), ".."))  # 返回当前工作目录   先到达 QH_网络配置 的父文件夹路径
    sys.path.append(qh_path)  # 添加自己指定的搜索路径
else:
    print("外文件引用")
    qh_path = os.path.abspath(os.path.join(os.getcwd(), ".."))  # 返回当前工作目录   先到达 QH_网络配置 的父文件夹路径
    sys.path.append(qh_path)  # 添加自己指定的搜索路径
#引入自己的文件模块
import QH001_Spider_Object.QH_DongFang_Object as Qh_Req_Obj           # 东方财富
import QH001_Spider_Object.QH_SouHu_Home_Object as Qh_SHH_Obj         # 搜狐交易日
import QH_Stock_Treasure_Box.qh_sqlite_orm_json as qh_db              # 数据库
import QH_Stock_Treasure_Box.qh_Log as qh_log                         # 日志
import QH_Stock_Treasure_Box.qh_parameter_chuanru as qh_chcan         # 传入参数
import QH_DF_Interface as qh_df_interface
import QH_IP_KDL as qh_ip_m                                           #ip

if __name__ == "__main__":

    qh_Log = qh_log.Qh_log_write(qh_path, "Qh_DF_His_D_Log")
    qh_Trading_day = Qh_SHH_Obj.QH_Get_Dangri_Date()  # 同一个文件的函数  日期会自动传入
    qh_Log.qh_Log("获取交易日期-搜狐,交易日期:{}".format(qh_Trading_day))
    qh_all_stock_list_sql = qh_chcan.Qh_CX_Sql_DB(qh_path, qh_chaxun_id="QH_ChaXun_00005")
    qh_Log.qh_Log("获取所有股票代码,代码列表:{}".format(qh_all_stock_list_sql))
    qh_ip_list = qh_ip_m.Qh_Get_Ip(qh_path, qh_flag="B")
    qh_Log.qh_Log("获取所有HTTP代理IP,HTTP代理IP列表:{}".format(qh_ip_list))
    qh_wangzhan0 = "DF"  # 网站代码
    qh_Log.qh_Log("网站代码:{}".format(qh_wangzhan0))
    qh_klt_dic = {"天": "D", "周": "W", "月": "M"}
    qh_Log.qh_Log("周期字典:{}".format(qh_klt_dic))
    qh_fqt_dic = {"不复权": "0", "前复权": "1", "后复权": "2"}
    qh_Log.qh_Log("是否复权字典:{}".format(qh_fqt_dic))
    qh_db_canshu = qh_chcan.Qh_InF_To_DB(qh_path, [], qh_spider_id="QH_Spider_00008", qh_canshu=True)
    qh_Log.qh_Log("获取所有数据库参数:{}".format(qh_db_canshu))
    qh_table_00 = qh_db_canshu['QH_Table']  # 需补充参数的组合表名
    qh_Log.qh_Log("需补充参数的组合表名:{}".format(qh_table_00))
    qh_sql0 = qh_chcan.Qh_CX_Sql_DB(qh_path, qh_chaxun_id="QH_ChaXun_00015", qh_sql0="", qh_ret_sql=True)
    qh_Log.qh_Log("获取需要补充参数的股票流记录SQL:{}".format(qh_sql0))
    #循环所有股票列表
    for qh_i, qh_row in enumerate(qh_all_stock_list_sql[1][1]):
        qh_code = qh_row[15]   # 股票代码
        qh_zjs = qh_row[16]    # 证交所
        qh_name = qh_row[17]   # 股票名称
        # 组合股票代码
        if str(qh_zjs) == "0":
            qh_code = "SZ{}".format(qh_code)
        else:
            qh_code = "SH{}".format(qh_code)
        qh_Log.qh_Log("正在获取第{}支股票,股票代码:{},证交所:{},股票名称:{}".format(qh_i, qh_code, qh_zjs, qh_name))
        qh_klt = "天"
        qh_fqt = "不复权"
        qh_Log.qh_Log("周期:{},复权:{}".format(qh_klt, qh_fqt))
        qh_df_interface.Qh_Get_His_Flow(qh_code, qh_name, qh_Trading_day, qh_klt, qh_fqt,
                                        qh_wangzhan0, qh_klt_dic, qh_fqt_dic,
                                        qh_table_00, qh_sql0, qh_Log, qh_ip_list)

第二段 所有股票 所有周期:天/周/月  所有复权:不复权/前复权/后复权

# encoding:utf-8
import os
import sys
import pandas as qh_pd
import datetime
import time
import random
if __name__ == "__main__":
    print("本文件引用")
    qh_path = os.path.abspath(os.path.join(os.getcwd(), ".."))  # 返回当前工作目录   先到达 QH_网络配置 的父文件夹路径
    sys.path.append(qh_path)  # 添加自己指定的搜索路径
else:
    print("外文件引用")
    qh_path = os.path.abspath(os.path.join(os.getcwd(), ".."))  # 返回当前工作目录   先到达 QH_网络配置 的父文件夹路径
    sys.path.append(qh_path)  # 添加自己指定的搜索路径
#引入自己的文件模块
import QH001_Spider_Object.QH_DongFang_Object as Qh_Req_Obj           # 东方财富
import QH001_Spider_Object.QH_SouHu_Home_Object as Qh_SHH_Obj         # 搜狐交易日
import QH_Stock_Treasure_Box.qh_sqlite_orm_json as qh_db              # 数据库
import QH_Stock_Treasure_Box.qh_Log as qh_log                         # 日志
import QH_Stock_Treasure_Box.qh_parameter_chuanru as qh_chcan         # 传入参数
import QH_DF_Interface as qh_df_interface
import QH_IP_KDL as qh_ip_m                                           #ip


if __name__ == "__main__":

    qh_Log = qh_log.Qh_log_write(qh_path, "Qh_DF_His_DWM_Log")
    qh_Trading_day = Qh_SHH_Obj.QH_Get_Dangri_Date()                  # 同一个文件的函数  日期会自动传入
    qh_Log.qh_Log("获取交易日期-搜狐,交易日期:{}".format(qh_Trading_day))
    qh_all_stock_list_sql = qh_chcan.Qh_CX_Sql_DB(qh_path, qh_chaxun_id="QH_ChaXun_00005")
    qh_Log.qh_Log("获取所有股票代码,代码列表:{}".format(qh_all_stock_list_sql))
    qh_ip_list = qh_ip_m.Qh_Get_Ip(qh_path, qh_flag="B")
    qh_Log.qh_Log("获取所有HTTP代理IP,HTTP代理IP列表:{}".format(qh_ip_list))
    qh_wangzhan0 = "DF"  # 网站代码
    qh_Log.qh_Log("网站代码:{}".format(qh_wangzhan0))
    qh_klt_dic = {"天": "D", "周": "W", "月": "M"}
    qh_Log.qh_Log("周期字典:{}".format(qh_klt_dic))
    qh_fqt_dic = {"不复权": "0", "前复权": "1", "后复权": "2"}
    qh_Log.qh_Log("是否复权字典:{}".format(qh_fqt_dic))
    qh_db_canshu = qh_chcan.Qh_InF_To_DB(qh_path, [], qh_spider_id="QH_Spider_00008", qh_canshu=True)
    qh_Log.qh_Log("获取所有数据库参数:{}".format(qh_db_canshu))
    qh_table_00 = qh_db_canshu['QH_Table']  # 需补充参数的组合表名
    qh_Log.qh_Log("需补充参数的组合表名:{}".format(qh_table_00))
    qh_sql0 = qh_chcan.Qh_CX_Sql_DB(qh_path, qh_chaxun_id="QH_ChaXun_00015", qh_sql0="", qh_ret_sql=True)
    qh_Log.qh_Log("获取需要补充参数的股票流记录SQL:{}".format(qh_sql0))
    # 循环所有股票列表
    for qh_i,qh_row in enumerate(qh_all_stock_list_sql[1][1]):
        qh_code = qh_row[15]  # 股票代码
        qh_zjs = qh_row[16]  # 证交所
        qh_name = qh_row[17]  # 股票名称
        # 组合股票代码
        if str(qh_zjs) == "0":
            qh_code = "SZ{}".format(qh_code)
        else:
            qh_code = "SH{}".format(qh_code)
        qh_Log.qh_Log("正在获取第{}支股票,股票代码:{},证交所:{},股票名称:{}".format(qh_i,qh_code,qh_zjs,qh_name))
        for qh_key in qh_klt_dic:
            qh_klt = qh_key          #周期
            for qh_key01 in qh_fqt_dic:
                qh_fqt = qh_key01    #复权
                qh_Log.qh_Log("周期:{},复权:{}".format(qh_klt, qh_fqt))
                qh_df_interface.Qh_Get_His_Flow(qh_code, qh_name, qh_Trading_day, qh_klt, qh_fqt,
                                                qh_wangzhan0, qh_klt_dic, qh_fqt_dic,
                                                qh_table_00, qh_sql0, qh_Log,qh_ip_list)

第三段:调用函数

def Qh_Get_His_Flow(qh_code,qh_name,qh_Trading_day, qh_klt, qh_fqt,qh_wangzhan0,qh_klt_dic,qh_fqt_dic,qh_table_00,qh_sql0,qh_Log,qh_ip_list=[]):
    """
    获取所有历史交易数据函数         作者:阙辉
    :param qh_code:         股票代码  带证交所 SZ000001
    :param qh_name:         股票名称
    :param qh_Trading_day:  最近交易日期
    :param qh_klt:          周期
    :param qh_fqt:          复权
    :param qh_wangzhan0:    网站
    :param qh_klt_dic:      周期字典
    :param qh_fqt_dic:      复权字典
    :param qh_table_00:     需传参数的表名
    :param qh_sql0:         需传参数的SQL
    :param qh_ip_list:      代理IP
    :return:
    """
    qh_flow_jilu = []      #初始化记录列表
    qh_wangzhan = qh_wangzhan0 + "_" + str(qh_klt_dic[qh_klt]) + "_" + str(qh_fqt_dic[qh_fqt])   #拼接网站id  关键代码,隔离了周期和复权
    qh_flow_jilu.append(qh_wangzhan)     # qh_flow_jilu 0 网站
    qh_flow_jilu.append(qh_code)         # qh_flow_jilu 1 股票代码
    qh_flow_jilu.append(qh_name)         # qh_flow_jilu 2 股票名称
    qh_flow_jilu.append(qh_Trading_day)  # qh_flow_jilu 3 交易日期
    # 爬取历史股票
    qh_type_gg = "QH_JY_HIS"         # 获取历史数据类型  历史交易数据
    qh_flow_jilu.append(qh_type_gg)  # #qh_flow_jilu 4 交易类型   历史交易
    qh_end_date = qh_Trading_day     # 最近一个交易日期
    # 判断最近交易日期的历史数据是否爬取
    try:
        qh_sql = qh_sql0.format(qh_wangzhan, qh_code, qh_type_gg)
        qh_list_qq = qh_chcan.Qh_CX_Sql_DB(qh_path, qh_chaxun_id="QH_ChaXun_00015", qh_sql0=qh_sql, qh_ret_sql=False,qh_filed=False)
        qh_list_qq = qh_list_qq[1][1]
        if len(qh_list_qq) > 0:
            qh_list_qq = qh_list_qq[0]      # 取第一行
            qh_pq_yy = qh_list_qq[8]        # 爬取状态
            qh_zong_count = qh_list_qq[10]  # 上一次总数量
            qh_stare_date = qh_list_qq[6]   # 上一次爬取交易日期
            qh_Log.qh_Log("股票流有记录,记录为:爬取状态:{},上一次总数量:{},上一次爬取交易日期:{}".format(qh_pq_yy, qh_zong_count, qh_stare_date))
        else:
            qh_pq_yy = "N"  # 爬取状态
            qh_zong_count = 0  # 总数量
            qh_stare_date = "1989-01-01"    # 默认起始日期
            qh_Log.qh_Log("股票流无记录,默认记录为:爬取状态:{},上一次总数量:{},上一次爬取交易日期:{}".format(qh_pq_yy, qh_zong_count, qh_stare_date))
    except:
        qh_pq_yy = "N"  # 爬取状态
        qh_zong_count = 0  # 总数量
        qh_stare_date = "1989-01-01"  # 默认起始日期
        qh_Log.qh_Log("股票流表未创建,默认记录为:爬取状态:{},上一次总数量:{},上一次爬取交易日期:{}".format(qh_pq_yy, qh_zong_count, qh_stare_date))

    # 判断是否要请求数据
    if qh_pq_yy == "Y" and str(qh_end_date) == str(qh_stare_date):
        qh_Log.qh_Log("股票代码({})已爬取,最后交易日期{},阙辉!".format(qh_code, qh_end_date))
    else:
        # 随机从IP列表中获取一个代理IP
        if len(qh_ip_list) > 0:
            qh_ip_kk = random.choice(qh_ip_list)
            qh_htt = qh_ip_kk[1]
            qh_htt = "https"
            qh_ip = qh_ip_kk[0]
            qh_htt_ip = "{}\\:{}".format(qh_htt, qh_ip)
            qh_proxy = {
                qh_htt: qh_htt_ip
            }
            qh_Log.qh_Log("随机获取代理IP({}),阙辉!".format(qh_proxy))
        else:
            qh_proxy = {}
            qh_Log.qh_Log("不使用代理IP,阙辉!")
        # 请求数据
        qh_Log.qh_Log("开始请求({}),阙辉!".format(qh_code))
        qh_his_jiaoyi_list = Qh_DF_His_Flow_Interface01(qh_Trading_day, qh_code,
                                                                        qh_stare_date=qh_stare_date,
                                                                        qh_end_date=qh_end_date, qh_klt=qh_klt,
                                                                        qh_fqt=qh_fqt,
                                                                        qh_spider_id="QH_Spider_00008",
                                                                        qh_out="qh_list", qh_day_t=False,
                                                                        qh_proxy=qh_proxy)
        qh_his_jiaoyi_list_len = len(qh_his_jiaoyi_list)  # 统计本次的数据量
        qh_new_table = qh_table_00.format(qh_code)  # 生成表格名
        qh_Log.qh_Log("组合存储表格名({}),阙辉!".format(qh_new_table))
        # 存储数据 返回存储状态
        if qh_his_jiaoyi_list_len > 0:
            qh_Log.qh_Log("请求({})数据成功,本次共{}条,阙辉!".format(qh_code,qh_his_jiaoyi_list_len))
            qh_cc_star = qh_chcan.Qh_InF_To_DB(qh_path, qh_his_jiaoyi_list, qh_spider_id="QH_Spider_00008_D",
                                               qh_canshu=False, qh_new_table=qh_new_table)
        else:
            qh_cc_star = False
        #判断数据是否更新成功,更新成功则
        if qh_cc_star:
            qh_Log.qh_Log("存储({})数据成功,本次共{}条,阙辉!".format(qh_code, qh_his_jiaoyi_list_len))
            qh_flow_jilu.append("Y")  # #qh_flow_jilu 5 是否成功 爬取状态
            qh_flow_jilu.append(qh_his_jiaoyi_list_len)  # #qh_flow_jilu 6 本次数量
            qh_flow_jilu.append(qh_his_jiaoyi_list_len + int(qh_zong_count))  # #qh_flow_jilu 7 总数量
            qh_flow_jilu = qh_chcan.Qh_TC_List(qh_flow_jilu, 18)  # 补充不满18的字段为空  #qh_flow_jilu 8以后为空的字段
            if qh_chcan.Qh_InF_To_DB(qh_path, [qh_flow_jilu], qh_spider_id="QH_Spider_99999"):  # 更新数据存储状态
                qh_Log.qh_Log("更新股票记录流成功,股票记录流为:{},阙辉!".format(qh_flow_jilu))

  • 0
    点赞
  • 7
    收藏
    觉得还不错? 一键收藏
  • 0
    评论

“相关推荐”对你有帮助么?

  • 非常没帮助
  • 没帮助
  • 一般
  • 有帮助
  • 非常有帮助
提交
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值