量化交易学习笔记（13）交易数据获取（优化版本）

小小蜗牛139

已于 2022-08-11 18:51:33 修改

阅读量582

点赞数 1

分类专栏：量化交易文章标签：学习 python 大数据金融

于 2022-08-11 18:48:13 首次发布

本文链接：https://blog.csdn.net/weixin_43688870/article/details/126291036

版权

量化交易专栏收录该内容

10 篇文章 6 订阅

订阅专栏

这里写目录标题

前言
优化
效果
代码
虚拟货币

前言

在上一篇文章中简单介绍数据的获取，这篇对上一篇文章的内容优化

优化

1.提供下载数据的进度调
2.提供批量下载
4.优化数据的保存逻辑

效果

参数
symbol 货币类型
interval 时间级别

下载ETH数据（down_load_data）

在这里插入图片描述

批量下载数据（bath_down_load_data）

在这里插入图片描述

保存到本地的数据

在这里插入图片描述

代码

# -*- coding: UTF-8 -*-
import datetime

import logging
import time
import backtrader
import setting
import numpy
import pandas
import os
from util import file_util, date_util
import constant
from binance.spot import Spot
import tqdm





def get_local_file_path(symbol: str, interval: str, coin="USDT"):
    """
    获取数据的本地路径
    :param symbol:
    :param interval:
    :param coin:
    :return:
    """
    # 文件名称
    file_name = symbol + coin + "_" + interval
    # 文件位置
    path = setting.date_root_path + "\\" + file_name + ".csv"
    return path


def get_k_lines_asc(symbol: str, interval: str, coin='USDT', start_time: int = None,
                    end_time: int = datetime.datetime.now().timestamp() * 1000):
    """
    获取k线数据 从结束时间 倒序获取
    :param coin:
    :param symbol: 虚拟货币类型
    :param interval: 时间基本
    :param start_time:
    :param end_time:
    :return:
    """

    proxies = {'https': 'https://127.0.0.1:7890'}
    client = Spot(proxies=proxies)
    stop_time = end_time
    if start_time is None:
        start_time = get_earliest_start_time(symbol, coin)
        end_time = datetime.datetime.now().timestamp() * 1000
        stop_time = datetime.datetime.now().timestamp() * 1000
    end_time = start_time + constant.interval_second[interval] * 1000 * 300

    if stop_time > end_time:
        stop_time = datetime.datetime.now().timestamp() * 1000
        end_time = start_time + constant.interval_second[interval] * 1000 * 399
    init_time = start_time
    query_symbol = symbol + coin
    data = []
    size = (stop_time - start_time) / 1000 / constant.interval_second[interval] / 300 + 1
    down_number = 0
    pbar = tqdm.trange(0, int(size), 1)
    for idx, element in enumerate(pbar):
        response = client.klines(query_symbol, interval, startTime=int(start_time), endTime=int(end_time))

        # 解析数据
        k_lines = parse_k_lines(response)
        down_number = down_number + len(k_lines)

        if start_time > stop_time or stop_time < end_time:
            # 没数据返回 || 到要求的数据
            pbar.set_description(f"No.{idx}: [down-{symbol}-{interval}]")
            logging.info(
                f"【下载数据】 【{symbol}-{interval}】 "
                f"【{date_util.timestamp2str(init_time / 1000)}  {date_util.timestamp2str(stop_time / 1000)}】")
            print(date_util.timestamp2str(end_time / 1000), date_util.timestamp2str(start_time / 1000))
            break
        pbar.set_description(
            f"No.{idx}: [down-{symbol}-{interval}]")
        if len(k_lines) == 0:
            logging.info(f"数据缺失 [{date_util.timestamp2str(start_time / 1000)}]")
            start_time = end_time + constant.interval_second[interval] * 1000
        else:
            start_time = response[-1][0] + constant.interval_second[interval] * 1000
        end_time = start_time + constant.interval_second[interval] * 1000 * 300
        if len(k_lines) > 0:
            data.extend(k_lines)
        if end_time > stop_time:
            end_time = stop_time
    return data


def get_k_lines_desc(symbol: str, interval: str, coin='USDT', start_time: int = None,
                     end_time: int = datetime.datetime.now().timestamp() * 1000):
    """
    获取k线数据 从结束时间 倒序获取
    :param symbol: 虚拟货币类型
    :param interval: 时间基本
    :param start_time:
    :param end_time:
    :return:
    """

    proxies = {'https': 'https://127.0.0.1:7890'}
    client = Spot(proxies=proxies)
    # 结束查询时间
    query_symbol = symbol + coin
    stop_time = start_time
    if stop_time is None:
        stop_time = 0
    if start_time is None or end_time is None:
        # 获取数据
        response = client.klines(query_symbol, interval)
        start_time = response[0][0]
        end_time = response[-1][0]

    data = []
    pbar = tqdm.trange(0, 10000, 1)
    for idx, element in enumerate(pbar):
        response = client.klines(query_symbol, interval, startTime=int(start_time), endTime=int(end_time))
        # 解析数据

        k_lines = parse_k_lines(response)
        k_lines.reverse()
        data.extend(k_lines)
        if len(k_lines) == 0 or len(k_lines) == 1 or stop_time > start_time:
            # 没数据返回 || 到要求的数据

            pbar.set_description(f"No.{idx}: [down_data] [total][{len(data)}]")
            break
        pbar.set_description(
            f"No.{idx}: [down_data]-[{symbol}-{interval}]")
        end_time = response[0][0] - constant.interval_second[interval] * 1000
        start_time = response[0][0] - constant.interval_second[interval] * 1000 * 400

    data.reverse()
    return data


def parse_k_lines(response):
    """
    解析返回的k线数据
    :param response:
    :return:
    """
    k_lines = []
    for item in response:
        k_line = []
        # 开盘时间
        time_array = time.localtime(item[0] / 1000)
        k_line.append(time.strftime('%Y-%m-%d %H:%M:%S', time_array))
        # 开盘价格
        k_line.append(item[1])
        # 收盘价格
        k_line.append(item[4])
        # 最高价格
        k_line.append(item[2])
        # 最低价格
        k_line.append(item[3])
        # 交易量
        k_line.append(item[5])
        k_lines.append(k_line)
    return k_lines


def load_generic_csv_data(data_path, start=None, end=None):
    return backtrader.feeds.GenericCSVData(
        dataname=data_path,
        nullvalue=0.0,
        fromdate=start,
        todate=end,
        dtformat="%Y-%m-%d %H:%M:%S",
        timeframe=backtrader.TimeFrame.Minutes,
        datetime=0,
        high=1,
        low=2,
        open=3,
        close=4,
        volume=5,
        openinterest=-1
    )


def get_earliest_start_time(symbol, coin='USDT'):
    """
    获取数据测开始时间
    通过获取日线级别的时间
    :param symbol:
    :param coin:
    :return:
    """
    path = get_local_file_path(symbol, '1d', coin=coin)

    if not os.path.exists(path):
        # 获取数据
        data = get_k_lines_desc(symbol, '1d', coin=coin)
        file_util.date2csv(data, path, constant.k_lines_csv_head, write_type='a')
        start_time = date_util.str2datetime(data[0][0]).timestamp() * 1000

    else:
        data = numpy.array(pandas.read_csv(path))
        latest_record_time = date_util.str2datetime(data[0][0])
        start_time = latest_record_time.timestamp() * 1000
    return int(start_time)


def down_load_data(symbol, interval: str, coin='USDT'):
    """
    下载数据
    :param symbol:
    :param interval:
    :param coin:
    :return:
    """
    path = get_local_file_path(symbol, interval, coin)

    if os.access(path, os.X_OK):
        # 文件存在
        data = numpy.array(pandas.read_csv(path))
        latest_record_time = date_util.str2datetime(data[-1][0])
        start_time = date_util.add_time(latest_record_time, minutes=int(constant.interval_second[interval] / 60))
        # 获取数据
        data = get_k_lines_asc(symbol, interval, coin, start_time=start_time.timestamp() * 1000)
    else:
        # 本地没有数据
        data = get_k_lines_asc(symbol, interval, coin, )
    # 保存数据
    if data:
        file_util.date2csv(data, path, constant.k_lines_csv_head, 'a')


def bath_down_load_data(symbols, intervals, coin='USDT'):
    """
    批量下载数据
    :param symbols:
    :param intervals:
    :param coin:
    :return:
    """
    for symbol in symbols:
        for interval in intervals:
            logging.info(f"【下载数据】 【{symbol}-{interval}】")
            down_load_data(symbol, interval)


if __name__ == '__main__':
    down_load_data('BTC', '5m')

虚拟货币

coin_symbol = [
    "1INCH",
    "AAVE",
    "ACA",
    "ACH",
    "ADA",
    "AGLD",
    "AION",
    "AKRO",
    "ALCX",
    "ALGO",
    "ALICE",
    "ALPACA",
    "ALPHA",
    "AMP",
    "ANC",
    "ANKR",
    "ANT",
    "ANY",
    "APE",
    "API3",
    "ARDR",
    "ARPA",
    "AR",
    "ASTR",
    "ATA",
    "ATOM",
    "AUDIO",
    "AUD",
    "AUTO",
    "AVAX",
    "AXS",
    "BADGER",
    "BAKE",
    "BAL",
    "BAND",
    "BAT",
    "BCH",
    "BEAM",
    "BEL",
    "BETA",
    "BICO",
    "BLZ",
    "BNB",
    "BNT",
    "BNX",
    "BOND",
    "BSW",
    "BTCST",
    "BTC",
    "BTG",
    "BTS",
    "BTTC",
    "BTT",
    "BURGER",
    "BUSD",
    "BZRX",
    "C98",
    "CAKE",
    "CELO",
    "CELR",
    "CFX",
    "CHESS",
    "CHR",
    "CHZ",
    "CKB",
    "CLV",
    "COCOS",
    "COMP",
    "COS",
    "COTI",
    "CRV",
    "CTK",
    "CTSI",
    "CTXC",
    "CVC",
    "CVP",
    "CVX",
    "DAR",
    "DASH",
    "DATA",
    "DCR",
    "DEGO",
    "DENT",
    "DEXE",
    "DGB",
    "DIA",
    "DNT",
    "DOCK",
    "DODO",
    "DOGE",
    "DOT",
    "DREP",
    "DUSK",
    "DYDX",
    "EGLD",
    "ELF",
    "ENJ",
    "ENS",
    "EOS",
    "EPS",
    "EPX",
    "ERN",
    "ETC",
    "ETH",
    "EUR",
    "FARM",
    "FET",
    "FIDA",
    "FIL",
    "FIO",
    "FIRO",
    "FIS",
    "FLM",
    "FLOW",
    "FLUX",
    "FORTH",
    "FOR",
    "FTM",
    "FTT",
    "FUN",
    "FXS",
    "GALA",
    "GAL",
    "GBP",
    "GLMR",
    "GMT",
    "GNO",
    "GRT",
    "GTC",
    "GTO",
    "GXS",
    "HARD",
    "HBAR",
    "HIGH",
    "HIVE",
    "HNT",
    "HOT",
    "ICP",
    "ICX",
    "IDEX",
    "ILV",
    "IMX",
    "INJ",
    "IOST",
    "IOTA",
    "IOTX",
    "IRIS",
    "JASMY",
    "JOE",
    "JST",
    "KAVA",
    "KDA",
    "KEEP",
    "KEY",
    "KLAY",
    "KMD",
    "KNC",
    "KP3R",
    "KSM",
    "LDO",
    "LEND",
    "LEVER",
    "LINA",
    "LINK",
    "LIT",
    "LOKA",
    "LPT",
    "LRC",
    "LSK",
    "LTC",
    "LTO",
    "LUNA",
    "MANA",
    "MASK",
    "MATIC",
    "MBL",
    "MBOX",
    "MC",
    "MDT",
    "MDX",
    "MFT",
    "MINA",
    "MIR",
    "MITH",
    "MKR",
    "MLN",
    "MOB",
    "MOVR",
    "MTL",
    "NANO",
    "NBS",
    "NEAR",
    "NEO",
    "NEXO",
    "NKN",
    "NMR",
    "NPXS",
    "NULS",
    "NU",
    "OCEAN",
    "OGN",
    "OG",
    "OMG",
    "OM",
    "ONE",
    "ONG",
    "ONT",
    "OOKI",
    "OP",
    "ORN",
    "OXT",
    "PAXG",
    "PEOPLE",
    "PERL",
    "PERP",
    "PHA",
    "PLA",
    "PNT",
    "POLS",
    "POLY",
    "POND",
    "POWR",
    "PUNDIX",
    "PYR",
    "QI",
    "QNT",
    "QTUM",
    "QUICK",
    "RAD",
    "RAMP",
    "RARE",
    "RAY",
    "REEF",
    "REI",
    "REN",
    "REP",
    "REQ",
    "RGT",
    "RLC",
    "RNDR",
    "ROSE",
    "RSR",
    "RUNE",
    "RVN",
    "SAND",
    "SCRT",
    "SC",
    "SFP",
    "SHIB",
    "SKL",
    "SLP",
    "SNX",
    "SOL",
    "SPELL",
    "SRM",
    "STMX",
    "STORJ",
    "STPT",
    "STRAX",
    "STX",
    "SUN",
    "SUPER",
    "SUSHI",
    "SXP",
    "SYS",
    "TCT",
    "TFUEL",
    "THETA",
    "TKO",
    "TLM",
    "TOMO",
    "TORN",
    "TRB",
    "TRIBE",
    "TROY",
    "TRU",
    "TRX",
    "T",
    "TUSD",
    "TVK",
    "TWT",
    "UMA",
    "UNFI",
    "UNI",
    "USDC",
    "UST",
    "UTK",
    "VET",
    "VGX",
    "VIDT",
    "VITE",
    "VOXEL",
    "VTHO",
    "WAN",
    "WAVES",
    "WAXP",
    "WING",
    "WIN",
    "WNXM",
    "WOO",
    "WRX",
    "WTC",
    "XEC",
    "XEM",
    "XLM",
    "XMR",
    "XRP",
    "XTZ",
    "XVS",
    "XZC",
    "YFII",
    "YFI",
    "YGG",
    "ZEC",
    "ZEN",
    "ZIL",
    "ZRX"
]