Python处理爬取数据

# -*- codeing = utf-8 -*-
# @Time : 2021/10/24 22:37
# @Author : fangqm
# @File : shares2.py
# @Software : PyCharm
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import xlwt
import re
import urllib.request
from bs4 import BeautifulSoup
import random
import json
import time

def read_excel():

    df = pd.read_excel("白酒.xls")
    # print(df)
    plt.figure(figsize=(100, 20),dpi=10)
    plt.plot(df["date"], df["rate"], label='rate', linewidth=3, color='y', marker='o',
             markerfacecolor='blue', markersize=0)
    plt.plot(df["date"], df["end"], label='end', linewidth=3, color='r', marker='o',
             markerfacecolor='blue', markersize=0)
    plt.xlabel("date")
    plt.ylabel('rate')
    plt.title("399975")
    plt.legend()
    plt.grid()
    plt.show()


def getdata():
    url = "https://stock.xueqiu.com/v5/stock/chart/kline.json?symbol=BK0478&begin=1635258229866&period=day&type=before&count=-284&indicator=kline,pe,pb,ps,pcf,market_capital,agt,ggt,balance"
    html = askURL(url)
    getshares(html)

proxy_list = [
    "101.200.49.180",
    "120.236.130.132"
]

# 收集到的常用Header
my_headers = {
    'User-Agent':'"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36"',
    'cookie':'device_id=fe89a71f05b0a0abff77904b2153e1a9; s=e312337kcl; bid=6bdee62c3998c330d889745f81f1d638_kotewic4; Hm_lvt_1db88642e346389874251b5a1eded6e3=1634959247,1635003724,1635086541,1635168631; xq_a_token=fbba234e827a98a7d88dd6786421316228317645; xqat=fbba234e827a98a7d88dd6786421316228317645; xq_r_token=94d1a99345252da87629b47a05f5852695cd9f10; xq_id_token=eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJ1aWQiOjY3ODQ3ODE2NTUsImlzcyI6InVjIiwiZXhwIjoxNjM3NzM1MjM0LCJjdG0iOjE2MzUxNjg2NDQyNjgsImNpZCI6ImQ5ZDBuNEFadXAifQ.nE-eCOQevkjN-fvevzAmN4-OfUBu9_qYQEzMbqbuwj4DyxeU47jg0DgzX3nq-8vfMnpjXjKlKBIyxgSYl4igDB0TU2z73le6rRz7nme8rIZ3bbAkX74EVQR516JLPPIUrEhyR-q_dy4tPzRHohly5B67n-9sXdMNsTJl-WGBL2uc_43S6wDbjcq1cj19_uxNGXOOEOvC_KcfHPhGdRuYZN9Zh-Fr35NK1M7cvH-7M0zQ5oxW9B-HqONA5i5djjQ8TAEQX_H1waR7Tn_lIeeDsiBVvJLoiArPIw083Xxh1WQGTjk6DUqyC5LZEs9oI0J8QCDdcOXNDcFVNIdgcQDK5A; xq_is_login=1; u=6784781655; is_overseas=0; Hm_lpvt_1db88642e346389874251b5a1eded6e3=1635171822'
}
# 获得单个页面的数据
def askURL(baseUrl):

    proxy = random.choice(proxy_list)
    head=my_headers

    urlhandle = urllib.request.ProxyHandler({'http': proxy})
    opener = urllib.request.build_opener(urlhandle)
    urllib.request.install_opener(opener)


    request = urllib.request.Request(url=baseUrl,headers=head)
    try:
        response = urllib.request.urlopen(request)
        html = response.read().decode("utf-8")
    except urllib.error.URLError as e:
        if hasattr(e, "code"):
            print(e.code)
        if hasattr(e, "reason"):
            print(e.reason)
    return html
#处理爬取雪球数据
def getshares(html):

    zhengquan = 500 #证券公司
    baijiu = 10000  # 白酒
    xinnengyuan = 2000 #新能源
    guanfu = 600 # 光伏

    j = json.loads(html)
    list1 = [] #时间戳
    list2 = [] #收盘价
    list3 = [] #换手率
    list4 = [] # 最高价
    list5 = [] # 最低价

    data = j['data']

    items = data['item']

    for item in items:
        list1.append(item[0])
        list4.append(item[3])
        list5.append(item[4])
        list2.append(item[5])
        list3.append(item[8])

    work = xlwt.Workbook(encoding="utf-8")
    sheet = work.add_sheet("光伏.xlsx",cell_overwrite_ok=True)
    col =("date","high","low","end","rate")
    for i in range(0,5):
        sheet.write(0,i,col[i])

    #处理时间戳
    for f in range(0,len(list1)):
        times = list1[f] / 1000
        timeArray = time.localtime(times)
        otherStyleTime = time.strftime("%Y/%m/%d", timeArray)
        sheet.write(f+1, 0,otherStyleTime)

    #处理最高价
    for j in range(0,len(list4)):
        sheet.write(j+1,1,(list4[j]/guanfu))

    # 处理最低价
    for j in range(0, len(list5)):
        sheet.write(j + 1, 2, (list5[j] / guanfu))

    # 处理收盘价
    for j in range(0, len(list2)):
        sheet.write(j + 1, 3, (list2[j] / guanfu))

    #处理换手率
    for k in range(0,len(list3)):
        sheet.write(k+1,4,list3[k])

    #high


    work.save(".\\光伏.xlsx")

if __name__ == '__main__':
    getdata()

kdj数据计算

# -*- codeing = utf-8 -*-
# @Time : 2021/10/26 22:06
# @Author : fangqm
# @File : ThreeData.py
# @Software : PyCharm
import pandas as pd



def kdjdata():
    df = pd.read_excel("光伏.xlsx")
    low_list = df['low'].rolling(9, min_periods=9).min()
    low_list.fillna(value=df['low'].expanding().min(), inplace=True)
    high_list = df['high'].rolling(9, min_periods=9).max()
    high_list.fillna(value=df['high'].expanding().max(), inplace=True)
    rsv = (df['end'] - low_list) / (high_list - low_list) * 100

    datas = pd.DataFrame(rsv).ewm(com=2).mean()

    datas.to_excel("光伏.xlsx",startrow=1,startcol=7,)
    # df['D'] = df['K'].ewm(com=2).mean()
    # df['J'] = 3 * df['K'] - 2 * df['D']

if __name__ == '__main__':
    kdjdata()