用Python从新浪下载A股复权因子信息

最近在对日线进行分析回测时，需要用到股票的复权因子，因为TuShare的复权因子并没有提供公开调用API，这里考虑从Sina下载并解析。

主要参考了这篇文章：用Python从sina下载复权因子

 

新浪财经的复权因子数据例子，例如600000股票2017年第一季度复权因子在这里：http://vip.stock.finance.sina.com.cn/corp/go.php/vMS_FuQuanMarketHistory/stockid/600000.phtml?year=2017&jidu=1

print '\xc0\xfa\xca\xb7\xbd\xbb\xd2\xd7'.decode('GB2312') #将GB2312编码打印出汉字

import numpy as np  
import pandas as pd  
import tushare as ts  
import datetime  
import time  
import tushare as ts  
import os  
import urllib2 #urllib2是Python的一个获取URLs的组件
import re #python的正则表达式模块

#global variables
data_dir = 'D:\\python_study\\stock_hist_data\\'  #下载数据的存放路径  
sleep_time=2 #default 2s

#找到特定str所在的那一行
def find_str(str_list, str):
    idx = -1
    row = 0
    for l in str_list:        
        idx = l.find(str)
        if idx != -1:
            break
        row += 1
    return row

#第二类解析
def deco_url2(url_list, str):
    s='\xc0\xfa\xca\xb7\xbd\xbb\xd2\xd7begin-->'  #开始标志：历史交易begin-->
    e='\xc0\xfa\xca\xb7\xbd\xbb\xd2\xd7end-->'    #结束标志：历史交易end-->
    s_idx = find_str(url_list, s)
    e_idx = find_str(url_list, e)
    factor=[]
    if e_idx - s_idx == 966-899:  #这一行没看懂，返回空的因子list
        return factor

    else:
        x='\xbc\xbe\xb6\xc8\xb8\xb4\xc8\xa8\xc0\xfa\xca\xb7\xbd\xbb\xd2\xd7'  #查找'季度复权历史交易'
        x_idx=find_str(url_list,x)
        time_row=x_idx+17 #加17找到有日期的那一行， 这些加多少完全是根据网页源码显示的来手工数出来的，写的此时我也不知道更高级点的做法该怎么做
        factor_row=x_idx+26 #加26找到有因子的那一行
        delta = 14   #加14找到下一天因子行
        while factor_row <= e_idx-7:
            t = url_list[time_row][-14:-4]
            #print t, url_list[factor_row-1]
            f_re = re.search('\d+(\.\d*)?',url_list[factor_row])
            temp = t+','+f_re.group()+','+str
            factor.append(temp.strip())
            time_row += delta
            factor_row += delta
        return factor

#第一类解析
def deco_url1(url_list, str):
    s='\xc0\xfa\xca\xb7\xbd\xbb\xd2\xd7begin-->'
    e='\xc0\xfa\xca\xb7\xbd\xbb\xd2\xd7end-->'
    s_idx = find_str(url_list, s)
    e_idx = find_str(url_list, e)
    factor=[]
    if e_idx - s_idx == 966-899:
        return factor

    else:
        x='\xbc\xbe\xb6\xc8\xb8\xb4\xc8\xa8\xc0\xfa\xca\xb7\xbd\xbb\xd2\xd7'
        x_idx=find_str(url_list,x)
        time_row=x_idx+18
        factor_row=x_idx+26
        delta = 14
        while factor_row <= e_idx-7:
            if delta == 14:
                t = url_list[time_row-1][5:15]
            if delta == 11:
                t = url_list[time_row-1][6:16]
            #print t, url_list[factor_row-1]
            f_re = re.search('\d+(\.\d*)?',url_list[factor_row-1])
            if f_re == None:
                time_row -= 2
                factor_row -= 3
                delta = 11
                ti = url_list[time_row-1].find('2')
                t = url_list[time_row-1][ti:(10+ti)]
                #print url_list[factor_row-1]
                f_re = re.search('\d+(\.\d*)?',url_list[factor_row-1])
            temp = t+','+f_re.group()+','+str
            factor.append(temp.strip())
            time_row += delta
            factor_row += delta
        return factor
#获取感兴趣的所有股票信息，这里只获取沪深300股票
def get_all_stock_id():
    stock_info=ts.get_hs300s()
    return stock_info['code'].values

#sample: get_former_right_factor('000001', 2017, '1')
def get_former_right_factor(sym, year, jidu):
    global sleep_time,data_dir  
    l1="http://vip.stock.finance.sina.com.cn/corp/go.php/vMS_FuQuanMarketHistory/stockid/"
    l2=".phtml?year="
    l3="&jidu="
    str_year = str(year)
    
    file_path = data_dir + sym + '\\' + str(year) + '\\'
    file_name = str(year) + '_' + jidu + '_factor.txt'
    if not os.path.exists(file_path):  
       os.makedirs(file_path)  
    if not os.path.exists(file_path+file_name):         
        link=l1 + sym + l2 + str(year) + l3 + jidu
        try:
            url_test = urllib2.urlopen(link)
        except IOError, msg:  
            print str(msg).decode('UTF-8')  
            sleep_time=min(sleep_time*2, 512)#每次下载失败后sleep_time翻倍，但是最大512s (新浪为防止恶意查询设置的封禁间隔为6min=360s)  
            print 'Get factor data error: symbol: '+ sym + ', year: ' + str_year + ', jidu: ' + jidu + ', sleep time is: '+str(sleep_time)  
            return True  
        else:  
            url_content = url_test.readlines()
            time = str(year)+jidu
            #if time >= '20062':
            #    s = deco_url1(url_content,sym)
            #else:
            s = deco_url2(url_content,sym) 
            #save to files

            fobj= open(file_path + file_name, 'w')
            for ss in s:
                fobj.write(ss+'\n')
            fobj.close()
            sleep_time=max(sleep_time/2, 2) #每次成功下载后sleep_time变为一半，但是至少2s  
            return True

years=range(2010,2017,1)
stocks=get_all_stock_id()
for stock in stocks:
    for year in years:
       for jd in ['4','3','2','1']:  #一年四个季度
           get_former_right_factor(stock, year, jd)
           time.sleep(sleep_time)