爬取同花顺HS300成分股的预测收益及增长率

import requests
from bs4 import BeautifulSoup
import re
import json
import numpy as np
import time
import tushare as ts
import pandas as pd

df1 = ts.get_hs300s()
#print(df)
ss=df1.head(10)
code = ss.code
#print(code)

data = {}

for stkCode in code:
    url = "http://stockpage.10jqka.com.cn/" + stkCode[:6] + "/worth/#forecast"
    headers = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"}
    html = requests.get(url, headers = headers).text
    soup = BeautifulSoup(html, "lxml")
    s = soup.find_all(class_="tip clearfix")[0]




    pattern = re.compile("预测\d+.*?(\d+.\d+).*?(\d+.\d+%)")
    result = re.findall(pattern, s.text)
    data[stkCode] = result
    t = np.random.randint(10)
    time.sleep(t+5)
with open('data.json', 'w+') as f:
    json.dump(data, f)
    f.close()

    with open("data.json", 'r') as f:
        d = json.load(f)
        f.close()

    new_dict = {}
    for key, values in d.items():
        if len(values) == 0:
            values = [[None, None], [None, None]]
        elif len(values) == 1:
            values.append([None, None])
        t = (values[0][0], values[0][1], values[1][0], values[1][1])
        new_dict[key] = t
    #print (values)
    index = ['预测2018年每股收益/元', '每股收益较去年同比增长', '预测2018年净利润/亿元', '净利润较去年同比增长']
    df = pd.DataFrame(index=index, data=new_dict)
    df = df.T
    print(df)

抓取结果如下图