第一段 所有股票 周期:天 复权:不复权
# encoding:utf-8
import os
import sys
import pandas as qh_pd
import datetime
import time
import random
if __name__ == "__main__":
print("本文件引用")
qh_path = os.path.abspath(os.path.join(os.getcwd(), "..")) # 返回当前工作目录 先到达 QH_网络配置 的父文件夹路径
sys.path.append(qh_path) # 添加自己指定的搜索路径
else:
print("外文件引用")
qh_path = os.path.abspath(os.path.join(os.getcwd(), "..")) # 返回当前工作目录 先到达 QH_网络配置 的父文件夹路径
sys.path.append(qh_path) # 添加自己指定的搜索路径
#引入自己的文件模块
import QH001_Spider_Object.QH_DongFang_Object as Qh_Req_Obj # 东方财富
import QH001_Spider_Object.QH_SouHu_Home_Object as Qh_SHH_Obj # 搜狐交易日
import QH_Stock_Treasure_Box.qh_sqlite_orm_json as qh_db # 数据库
import QH_Stock_Treasure_Box.qh_Log as qh_log # 日志
import QH_Stock_Treasure_Box.qh_parameter_chuanru as qh_chcan # 传入参数
import QH_DF_Interface as qh_df_interface
import QH_IP_KDL as qh_ip_m #ip
if __name__ == "__main__":
qh_Log = qh_log.Qh_log_write(qh_path, "Qh_DF_His_D_Log")
qh_Trading_day = Qh_SHH_Obj.QH_Get_Dangri_Date() # 同一个文件的函数 日期会自动传入
qh_Log.qh_Log("获取交易日期-搜狐,交易日期:{}".format(qh_Trading_day))
qh_all_stock_list_sql = qh_chcan.Qh_CX_Sql_DB(qh_path, qh_chaxun_id="QH_ChaXun_00005")
qh_Log.qh_Log("获取所有股票代码,代码列表:{}".format(qh_all_stock_list_sql))
qh_ip_list = qh_ip_m.Qh_Get_Ip(qh_path, qh_flag="B")
qh_Log.qh_Log("获取所有HTTP代理IP,HTTP代理IP列表:{}".format(qh_ip_list))
qh_wangzhan0 = "DF" # 网站代码
qh_Log.qh_Log("网站代码:{}".format(qh_wangzhan0))
qh_klt_dic = {"天": "D", "周": "W", "月": "M"}
qh_Log.qh_Log("周期字典:{}".format(qh_klt_dic))
qh_fqt_dic = {"不复权": "0", "前复权": "1", "后复权": "2"}
qh_Log.qh_Log("是否复权字典:{}".format(qh_fqt_dic))
qh_db_canshu = qh_chcan.Qh_InF_To_DB(qh_path, [], qh_spider_id="QH_Spider_00008", qh_canshu=True)
qh_Log.qh_Log("获取所有数据库参数:{}".format(qh_db_canshu))
qh_table_00 = qh_db_canshu['QH_Table'] # 需补充参数的组合表名
qh_Log.qh_Log("需补充参数的组合表名:{}".format(qh_table_00))
qh_sql0 = qh_chcan.Qh_CX_Sql_DB(qh_path, qh_chaxun_id="QH_ChaXun_00015", qh_sql0="", qh_ret_sql=True)
qh_Log.qh_Log("获取需要补充参数的股票流记录SQL:{}".format(qh_sql0))
#循环所有股票列表
for qh_i, qh_row in enumerate(qh_all_stock_list_sql[1][1]):
qh_code = qh_row[15] # 股票代码
qh_zjs = qh_row[16] # 证交所
qh_name = qh_row[17] # 股票名称
# 组合股票代码
if str(qh_zjs) == "0":
qh_code = "SZ{}".format(qh_code)
else:
qh_code = "SH{}".format(qh_code)
qh_Log.qh_Log("正在获取第{}支股票,股票代码:{},证交所:{},股票名称:{}".format(qh_i, qh_code, qh_zjs, qh_name))
qh_klt = "天"
qh_fqt = "不复权"
qh_Log.qh_Log("周期:{},复权:{}".format(qh_klt, qh_fqt))
qh_df_interface.Qh_Get_His_Flow(qh_code, qh_name, qh_Trading_day, qh_klt, qh_fqt,
qh_wangzhan0, qh_klt_dic, qh_fqt_dic,
qh_table_00, qh_sql0, qh_Log, qh_ip_list)
第二段 所有股票 所有周期:天/周/月 所有复权:不复权/前复权/后复权
# encoding:utf-8
import os
import sys
import pandas as qh_pd
import datetime
import time
import random
if __name__ == "__main__":
print("本文件引用")
qh_path = os.path.abspath(os.path.join(os.getcwd(), "..")) # 返回当前工作目录 先到达 QH_网络配置 的父文件夹路径
sys.path.append(qh_path) # 添加自己指定的搜索路径
else:
print("外文件引用")
qh_path = os.path.abspath(os.path.join(os.getcwd(), "..")) # 返回当前工作目录 先到达 QH_网络配置 的父文件夹路径
sys.path.append(qh_path) # 添加自己指定的搜索路径
#引入自己的文件模块
import QH001_Spider_Object.QH_DongFang_Object as Qh_Req_Obj # 东方财富
import QH001_Spider_Object.QH_SouHu_Home_Object as Qh_SHH_Obj # 搜狐交易日
import QH_Stock_Treasure_Box.qh_sqlite_orm_json as qh_db # 数据库
import QH_Stock_Treasure_Box.qh_Log as qh_log # 日志
import QH_Stock_Treasure_Box.qh_parameter_chuanru as qh_chcan # 传入参数
import QH_DF_Interface as qh_df_interface
import QH_IP_KDL as qh_ip_m #ip
if __name__ == "__main__":
qh_Log = qh_log.Qh_log_write(qh_path, "Qh_DF_His_DWM_Log")
qh_Trading_day = Qh_SHH_Obj.QH_Get_Dangri_Date() # 同一个文件的函数 日期会自动传入
qh_Log.qh_Log("获取交易日期-搜狐,交易日期:{}".format(qh_Trading_day))
qh_all_stock_list_sql = qh_chcan.Qh_CX_Sql_DB(qh_path, qh_chaxun_id="QH_ChaXun_00005")
qh_Log.qh_Log("获取所有股票代码,代码列表:{}".format(qh_all_stock_list_sql))
qh_ip_list = qh_ip_m.Qh_Get_Ip(qh_path, qh_flag="B")
qh_Log.qh_Log("获取所有HTTP代理IP,HTTP代理IP列表:{}".format(qh_ip_list))
qh_wangzhan0 = "DF" # 网站代码
qh_Log.qh_Log("网站代码:{}".format(qh_wangzhan0))
qh_klt_dic = {"天": "D", "周": "W", "月": "M"}
qh_Log.qh_Log("周期字典:{}".format(qh_klt_dic))
qh_fqt_dic = {"不复权": "0", "前复权": "1", "后复权": "2"}
qh_Log.qh_Log("是否复权字典:{}".format(qh_fqt_dic))
qh_db_canshu = qh_chcan.Qh_InF_To_DB(qh_path, [], qh_spider_id="QH_Spider_00008", qh_canshu=True)
qh_Log.qh_Log("获取所有数据库参数:{}".format(qh_db_canshu))
qh_table_00 = qh_db_canshu['QH_Table'] # 需补充参数的组合表名
qh_Log.qh_Log("需补充参数的组合表名:{}".format(qh_table_00))
qh_sql0 = qh_chcan.Qh_CX_Sql_DB(qh_path, qh_chaxun_id="QH_ChaXun_00015", qh_sql0="", qh_ret_sql=True)
qh_Log.qh_Log("获取需要补充参数的股票流记录SQL:{}".format(qh_sql0))
# 循环所有股票列表
for qh_i,qh_row in enumerate(qh_all_stock_list_sql[1][1]):
qh_code = qh_row[15] # 股票代码
qh_zjs = qh_row[16] # 证交所
qh_name = qh_row[17] # 股票名称
# 组合股票代码
if str(qh_zjs) == "0":
qh_code = "SZ{}".format(qh_code)
else:
qh_code = "SH{}".format(qh_code)
qh_Log.qh_Log("正在获取第{}支股票,股票代码:{},证交所:{},股票名称:{}".format(qh_i,qh_code,qh_zjs,qh_name))
for qh_key in qh_klt_dic:
qh_klt = qh_key #周期
for qh_key01 in qh_fqt_dic:
qh_fqt = qh_key01 #复权
qh_Log.qh_Log("周期:{},复权:{}".format(qh_klt, qh_fqt))
qh_df_interface.Qh_Get_His_Flow(qh_code, qh_name, qh_Trading_day, qh_klt, qh_fqt,
qh_wangzhan0, qh_klt_dic, qh_fqt_dic,
qh_table_00, qh_sql0, qh_Log,qh_ip_list)
第三段:调用函数
def Qh_Get_His_Flow(qh_code,qh_name,qh_Trading_day, qh_klt, qh_fqt,qh_wangzhan0,qh_klt_dic,qh_fqt_dic,qh_table_00,qh_sql0,qh_Log,qh_ip_list=[]):
"""
获取所有历史交易数据函数 作者:阙辉
:param qh_code: 股票代码 带证交所 SZ000001
:param qh_name: 股票名称
:param qh_Trading_day: 最近交易日期
:param qh_klt: 周期
:param qh_fqt: 复权
:param qh_wangzhan0: 网站
:param qh_klt_dic: 周期字典
:param qh_fqt_dic: 复权字典
:param qh_table_00: 需传参数的表名
:param qh_sql0: 需传参数的SQL
:param qh_ip_list: 代理IP
:return:
"""
qh_flow_jilu = [] #初始化记录列表
qh_wangzhan = qh_wangzhan0 + "_" + str(qh_klt_dic[qh_klt]) + "_" + str(qh_fqt_dic[qh_fqt]) #拼接网站id 关键代码,隔离了周期和复权
qh_flow_jilu.append(qh_wangzhan) # qh_flow_jilu 0 网站
qh_flow_jilu.append(qh_code) # qh_flow_jilu 1 股票代码
qh_flow_jilu.append(qh_name) # qh_flow_jilu 2 股票名称
qh_flow_jilu.append(qh_Trading_day) # qh_flow_jilu 3 交易日期
# 爬取历史股票
qh_type_gg = "QH_JY_HIS" # 获取历史数据类型 历史交易数据
qh_flow_jilu.append(qh_type_gg) # #qh_flow_jilu 4 交易类型 历史交易
qh_end_date = qh_Trading_day # 最近一个交易日期
# 判断最近交易日期的历史数据是否爬取
try:
qh_sql = qh_sql0.format(qh_wangzhan, qh_code, qh_type_gg)
qh_list_qq = qh_chcan.Qh_CX_Sql_DB(qh_path, qh_chaxun_id="QH_ChaXun_00015", qh_sql0=qh_sql, qh_ret_sql=False,qh_filed=False)
qh_list_qq = qh_list_qq[1][1]
if len(qh_list_qq) > 0:
qh_list_qq = qh_list_qq[0] # 取第一行
qh_pq_yy = qh_list_qq[8] # 爬取状态
qh_zong_count = qh_list_qq[10] # 上一次总数量
qh_stare_date = qh_list_qq[6] # 上一次爬取交易日期
qh_Log.qh_Log("股票流有记录,记录为:爬取状态:{},上一次总数量:{},上一次爬取交易日期:{}".format(qh_pq_yy, qh_zong_count, qh_stare_date))
else:
qh_pq_yy = "N" # 爬取状态
qh_zong_count = 0 # 总数量
qh_stare_date = "1989-01-01" # 默认起始日期
qh_Log.qh_Log("股票流无记录,默认记录为:爬取状态:{},上一次总数量:{},上一次爬取交易日期:{}".format(qh_pq_yy, qh_zong_count, qh_stare_date))
except:
qh_pq_yy = "N" # 爬取状态
qh_zong_count = 0 # 总数量
qh_stare_date = "1989-01-01" # 默认起始日期
qh_Log.qh_Log("股票流表未创建,默认记录为:爬取状态:{},上一次总数量:{},上一次爬取交易日期:{}".format(qh_pq_yy, qh_zong_count, qh_stare_date))
# 判断是否要请求数据
if qh_pq_yy == "Y" and str(qh_end_date) == str(qh_stare_date):
qh_Log.qh_Log("股票代码({})已爬取,最后交易日期{},阙辉!".format(qh_code, qh_end_date))
else:
# 随机从IP列表中获取一个代理IP
if len(qh_ip_list) > 0:
qh_ip_kk = random.choice(qh_ip_list)
qh_htt = qh_ip_kk[1]
qh_htt = "https"
qh_ip = qh_ip_kk[0]
qh_htt_ip = "{}\\:{}".format(qh_htt, qh_ip)
qh_proxy = {
qh_htt: qh_htt_ip
}
qh_Log.qh_Log("随机获取代理IP({}),阙辉!".format(qh_proxy))
else:
qh_proxy = {}
qh_Log.qh_Log("不使用代理IP,阙辉!")
# 请求数据
qh_Log.qh_Log("开始请求({}),阙辉!".format(qh_code))
qh_his_jiaoyi_list = Qh_DF_His_Flow_Interface01(qh_Trading_day, qh_code,
qh_stare_date=qh_stare_date,
qh_end_date=qh_end_date, qh_klt=qh_klt,
qh_fqt=qh_fqt,
qh_spider_id="QH_Spider_00008",
qh_out="qh_list", qh_day_t=False,
qh_proxy=qh_proxy)
qh_his_jiaoyi_list_len = len(qh_his_jiaoyi_list) # 统计本次的数据量
qh_new_table = qh_table_00.format(qh_code) # 生成表格名
qh_Log.qh_Log("组合存储表格名({}),阙辉!".format(qh_new_table))
# 存储数据 返回存储状态
if qh_his_jiaoyi_list_len > 0:
qh_Log.qh_Log("请求({})数据成功,本次共{}条,阙辉!".format(qh_code,qh_his_jiaoyi_list_len))
qh_cc_star = qh_chcan.Qh_InF_To_DB(qh_path, qh_his_jiaoyi_list, qh_spider_id="QH_Spider_00008_D",
qh_canshu=False, qh_new_table=qh_new_table)
else:
qh_cc_star = False
#判断数据是否更新成功,更新成功则
if qh_cc_star:
qh_Log.qh_Log("存储({})数据成功,本次共{}条,阙辉!".format(qh_code, qh_his_jiaoyi_list_len))
qh_flow_jilu.append("Y") # #qh_flow_jilu 5 是否成功 爬取状态
qh_flow_jilu.append(qh_his_jiaoyi_list_len) # #qh_flow_jilu 6 本次数量
qh_flow_jilu.append(qh_his_jiaoyi_list_len + int(qh_zong_count)) # #qh_flow_jilu 7 总数量
qh_flow_jilu = qh_chcan.Qh_TC_List(qh_flow_jilu, 18) # 补充不满18的字段为空 #qh_flow_jilu 8以后为空的字段
if qh_chcan.Qh_InF_To_DB(qh_path, [qh_flow_jilu], qh_spider_id="QH_Spider_99999"): # 更新数据存储状态
qh_Log.qh_Log("更新股票记录流成功,股票记录流为:{},阙辉!".format(qh_flow_jilu))