从零开始量化交易，数据篇

CCTX介绍

cctx是一款轻量级的加密交易所API库，是针对多个加密交易所设计的通用API封装。cctx的目标是为了成为一个易于使用、可扩展的库，它允许互动式并发访问多个交易所的市场数据和交易功能。

cctx支持多个加密货币交易所，包括Binance、BitMEX、Huobi、OKEx等等。使用cctx，用户可以简单地编写代码来访问不同交易所的市场数据，获取深度、K线数据、交易信息等。同时，cctx还允许用户通过同一套API来执行交易操作，例如，购买、卖出、取消订单等。

使用cctx，用户可以减少自己与多个加密交易所API的交互，当交易所API更改时，只需要更新cctx库而不是所有交易所的API代码。此外，cctx还对异步I/O进行了支持，可以让您更好地管理和调度您的交易API请求。

cctx有文档详细介绍了如何使用他们的API和示例代码帮助用户开始使用。总的来说，cctx是一个非常实用的工具，简化了多个交易所API的使用，降低了用户学习和开发成本。同时，还提供了灵活的选项，方便用户根据自己的需要进行自定义和扩展。

• 读取本地数据
• 数据不满足条件，更新本地数据

代码实现

效果

cctx封装

import time
from typing import List

import ccxt
from loguru import logger

from config.config import CcxtConfig
from core.model.ccxt import SymbolCurrentPrice, OHLCV, Account, Trade, BalanceResponse, OrderResponse
from core.utils.date_util import DateUtil

config = {
    'apiKey': CcxtConfig.api_key,
    'secret': CcxtConfig.secret,
    'enableRateLimit': True,
    'nonce': lambda: str(int(time.time() * 1000)),
}

client = ccxt.binance(config)
# 沙盒模拟
# exchange.set_sandbox_mode(True)
# 限制下载速度
client.enableRateLimit = True



class ExchangeFactory:
    @staticmethod
    def get_exchange():
        return client

    @staticmethod
    def get_config() -> dict:
        return config


class OrderBookStructure:
    pass


class CCtxAdapter:

    @staticmethod
    def fetch_order_book(symbol: str, limit=None, params=None) -> OrderBookStructure:
        if params is None:
            params = {''}
        orderbook = ExchangeFactory.get_exchange().fetch_order_book(symbol, limit, params=params)
        response = OrderBookStructure(orderbook)
        response.exchange_id = ExchangeFactory.get_exchange().id
        return response

    @staticmethod
    def fetch_order_book_l2(symbol: str, limit=None, params=None) -> OrderBookStructure:
        if params is None:
            params = {}
        orderbook = ExchangeFactory.get_exchange().fetch_l2_order_book(symbol, limit, params=params)
        response = OrderBookStructure(orderbook)
        response.exchange_id = ExchangeFactory.get_exchange().id
        return response

    @staticmethod
    def fetch_order_book_l1(symbol: str, limit=None, params=None) -> OrderBookStructure:
        if params is None:
            params = {}
        orderbook = ExchangeFactory.get_exchange().order
        response = OrderBookStructure(orderbook)
        response.exchange_id = ExchangeFactory.get_exchange().id
        return response

    @staticmethod
    def fetch_current_price(symbol: str) -> SymbolCurrentPrice:
        exchange = ExchangeFactory.get_exchange()
        orderbook = exchange.fetch_order_book(symbol)
        bid = orderbook['bids'][0][0] if len(orderbook['bids']) > 0 else None
        ask = orderbook['asks'][0][0] if len(orderbook['asks']) > 0 else None
        spread = (ask - bid) if (bid and ask) else None
        return SymbolCurrentPrice(ask, bid, spread, exchange.id, symbol)

    @staticmethod
    def fetch_ohlcv(symbol: str, timeframe='1m', since=None, limit=None, params=None) -> List[OHLCV]:
        """

        :param symbol:
        :param timeframe:
        :param since:
        :param limit:
        :param params: 'mark'（标记） 'index'(指标) 'premiumIndex'（溢价指数）
        :return:
        """
        if params is None:
            params = {}
        resp = []
        ohlcv = ExchangeFactory.get_exchange().fetch_mark_ohlcv(symbol, timeframe, since, limit, params)

        for oh in ohlcv:
            resp.append(OHLCV(oh[0], oh[1], oh[2], oh[3], oh[4], oh[5], ExchangeFactory.get_exchange().id, symbol, oh))
        return resp

    @staticmethod
    def fetch_trades(symbol, since=None, limit=None, params=None) -> List[Trade]:
        resp = []
        if params is None:
            params = {}
        trades = ExchangeFactory.get_exchange().fetch_trades(symbol, since, limit, params)
        for trade in trades:
            t = Trade.decode(trade)
            t.exchange_id = ExchangeFactory.get_exchange().id
            t.resp = trade
            resp.append(t)
        return resp

    @staticmethod
    def query_user_account() -> List[Account]:
        resp = []
        accounts = ExchangeFactory.get_exchange().account()
        time.sleep(2)
        for account in accounts:
            account_decode = Account.decode(account)
            account_decode.exchange_id = ExchangeFactory.get_exchange().id
            account_decode.resp = account
            resp.append(account_decode)

        return resp

    @staticmethod
    def fetch_balance() -> BalanceResponse:
        balance_resp = ExchangeFactory.get_exchange().fetch_balance()
        return BalanceResponse.decode(balance_resp)

    @staticmethod
    def fetch_orders(symbol: str = None, since: int = None, limit: int = None, params: dict = None) -> OrderResponse:
        if params is None:
            params = {}
        order_resp = ExchangeFactory.get_exchange().fetch_orders(symbol, since, limit, params)
        return OrderResponse.decode(order_resp)

    @staticmethod
    def create_market_buy_order(symbol: str, amount: float):

        order_info = ExchangeFactory.get_exchange().create_market_buy_order(symbol, amount)
        pass

    @staticmethod
    def create_market_sell_order(symbol: str, amount, price):

        order_info = ExchangeFactory.get_exchange().create_market_sell_order(symbol, amount)
        pass

    @staticmethod
    def query_begin_timestamp(symbol: str) -> int:
        start_time = '2010-01-01 08:00:00'
        fmt = '%Y-%m-%d %H:%M:%S'
        time_array = time.strptime(start_time, fmt)
        timestamp = int(time.mktime(time_array)) * 1000
        params = {
            "startTime": timestamp
        }
        ohlcv = CCtxAdapter.fetch_ohlcv(symbol=symbol, timeframe='1d', params=params, limit=1)

        start_time = DateUtil.timestamp_to_format(ohlcv[0].timestamp / 1000)
        logger.debug("symbol {} start_time {}".format(symbol, start_time))

        return ohlcv[0].timestamp

Response Model

from typing import List

from munch import DefaultMunch


class Order:
    """
    订单
    """
    price = 0
    amount = 0

    def __init__(self, price, amount):
        self.price = price
        self.amount = amount


class SymbolCurrentPrice:
    """
    当前价格
    """

    def __init__(self, buy_price, sell_price, spread, exchange_id, symbol):
        self.buy_price = buy_price
        self.sell_price = sell_price
        self.spread = spread
        self.exchange_id = exchange_id
        self.symbol = symbol


class OHLCV:
    timestamp: int = 0
    openPrice: int = 0
    highestPrice: int = 0
    lowestPrice: int = 0
    closePrice: int = 0
    volume: int = 0
    exchangeId: str = ""
    symbol = ""
    resp: dict = {}

    def __init__(self, timestamp, open_price, highest_price, lowest_price, close_price, volume, symbol, exchange_id,
                 resp: dict):
        self.symbol = symbol
        self.exchange_id = exchange_id
        self.volume = volume
        self.closePrice = close_price
        self.openPrice = open_price
        self.highestPrice = highest_price
        self.lowestPrice = lowest_price
        self.timestamp = timestamp
        self.resp = resp


class OrderBookStructure:
    """
    订单簿结构
    """
    symbol: str = None
    timestamp: int = 0
    datetime: int = 0
    nonce: int = 0
    bids: List[Order] = []
    asks: List[Order] = []
    resp: dict = {}
    exchange_id: str = ""

    def __init__(self, orderbook: dict):
        self.resp = orderbook
        self.decoder(orderbook)

    def decoder(self, orderbook: dict):
        self.symbol = orderbook['symbol']
        self.timestamp = orderbook['timestamp']
        self.nonce = orderbook['nonce']
        for ask in orderbook['asks']:
            order = Order(ask[0], ask[1])
            self.asks.append(order)
        for bid in orderbook['bids']:
            order = Order(bid[0], bid[1])
            self.bids.append(order)

    def get_bids(self) -> List:
        return self.bids


class TradFree:
    cost: float = 0
    currency: str = ""
    rate: float = 0


class Trade:
    resp = []
    exchange_id: str = ""
    info: str = ""
    id: str = ""
    timestamp: int = 0
    datetime: str = 0
    symbol: str = ""
    # 'market', 'limit'
    type: str = ""
    # 'buy' or 'sell'
    side: str = ""
    #  'taker' or 'maker'
    takerOrMaker: str
    price: float = 0
    amount: float = 0
    cost: float = 0
    fee: TradFree = {}
    fees: List[TradFree] = []

    def __init__(self, o: object, exchange_id=None, resp=None):
        self.symbol = o.symbol
        self.exchange_id = exchange_id
        self.resp = resp
        self.fee = o.fee
        self.fees = o.fees
        self.cost = o.cost
        self.amount = o.amount
        self.type = o.type
        self.side = o.side
        self.datetime = o.datetime
        self.timestamp = o.timestamp
        self.takerOrMaker = o.takerOrMaker
        self.id = o.id
        self.info = o.info
        self.price = o.price

    @staticmethod
    def decode(trade: dict):
        munch_object = DefaultMunch.fromDict(trade)
        return Trade(munch_object)


class Account:
    id: str = ''
    type: str = ''
    name: str = ''
    code: str = ''
    info: dict = {}
    resp: dict = {}
    exchange_id: str = ''

    def __init__(self, o: object):
        self.info = o.info
        self.code = o.code
        self.name = o.name
        self.type = o.type
        self.id = o.id

    @staticmethod
    def decode(account: dict):
        munch_object = DefaultMunch.fromDict(account)
        return Trade(munch_object)


class Balance:
    # 可以使用的资产数量
    free: float
    #
    asset: str
    # 已经使用的资产数量，
    locked: float


class BalanceResponse:
    makerCommission: float = 0
    takerCommission: float = 0
    buyerCommission: float = 0
    sellerCommission: float = 0
    commissionRates: dict = {}
    canTrade: bool
    canWithdraw: bool
    canDeposit: bool
    brokered: bool
    requireSelfTradePrevention: bool
    updateTime: int
    accountType: str
    balances: List[Balance]
    permissions: str = []
    free: float
    # 已经使用的资产数量
    used: float
    # 总资产数量
    total: float

    def __init__(self, o: object):
        self.permissions = o.permissions
        self.balances = o.balances
        self.accountType = o.accountType
        self.updateTime = o.updateTime
        self.requireSelfTradePrevention = o.requireSelfTradePrevention
        self.canDeposit = o.canDeposit
        self.canWithdraw = o.canWithdraw
        self.canTrade = o.canTrade
        self.commissionRates = o.commissionRates
        self.makerCommission = o.makerCommission
        self.takerCommission = o.takerCommission
        self.buyerCommission = o.buyerCommission

    @staticmethod
    def decode(balance: dict):
        munch_object = DefaultMunch.fromDict(balance['info'])
        resp = BalanceResponse(munch_object)
        resp.free = balance['free']
        resp.total = balance['total']
        resp.used = balance['used']
        return resp


class OrderResponse:
    resp: dict
    id: int
    clientOrderId: int
    price: float
    status: str

    type: str
    side: str

    amount: float
    cost: float
    average: float
    datetime: str
    timestamp: int
    resp: dict

    def __init__(self, o: object):
        self.id = o.id
        self.clientOrderId = o.clientOrderId
        self.price = o.price
        self.status = o.status
        self.side = o.side
        self.amount = o.amount
        self.cost = o.cost
        self.average = o.average
        self.datetime = o.datetime
        self.timestamp = o.timestamp

    @staticmethod
    def decode(order: dict):
        munch_object = DefaultMunch.fromDict(order)
        resp = OrderResponse(munch_object)
        resp.resp = order
        return resp

数据下载

import csv
import datetime as dt
import os

import pandas as pd
import tqdm
from loguru import logger

from config.config import SystemConfig
from core.rpc.ccxt_adapter import CCtxAdapter
from core.utils.date_util import DateUtil

interval_second = {
    "1m": 60,
    "5m": 300,
    "15m": 900,
    "30m": 1800,
    "1h": 3600,
    "2h": 72000,
    "4h": 14400,
    "1d": 86400,
}
_ohlv_head = ["Time", "Open", "Close", "High", "Low", "Volume"]


class OhlvUtil:
    @staticmethod
    def load_ohlv_as_pd(symbol: str, timeframe: str, start: dt.datetime = None,
                        end: dt.datetime = None) -> pd.DataFrame:
        logger.debug("加载数据 {}_{}".format(symbol, timeframe))
        filepath = OhlvUtil.symbol_local_path(symbol, timeframe)
        if not os.access(filepath, os.X_OK):
            # 更新本地数据
            filepath = OhlvUtil.download_ohlv(symbol=symbol, timeframe=timeframe, )
        # 读取数据
        klines = pd.read_csv(filepath, parse_dates=True, skiprows=0, header=0)
        last_row = klines.tail(1)
        if end is None:
            end = dt.datetime.now()
        if start is None:
            return klines
        start_str = DateUtil.datetime_to_format(start)
        end_str = DateUtil.datetime_to_format(end)
        # 更新本地数据

        if end_str > last_row['Time'].iloc[0]:
            OhlvUtil.download_ohlv(symbol=symbol, timeframe=timeframe,
                                   since=DateUtil.datetime_to_timestamp(start) * 1000)
            klines = pd.read_csv(filepath, parse_dates=True, skiprows=0, header=0)

        # 筛选数据
        klines = klines[(klines['Time'] <= end_str) & (klines['Time'] >= start_str)]
        klines.set_index('Time')
        return klines

    @staticmethod
    def symbol_local_path(symbol: str, timeframe: str):
        resource = SystemConfig.resource_path
        filepath = resource + '\\ohlv\\' + symbol.replace("/", '-')
        if not os.path.exists(filepath):
            os.makedirs(filepath)
        return filepath + "\\" + symbol.replace("/", '-') + '_' + timeframe + '.csv'

    @staticmethod
    def download_ohlv(symbol: str, timeframe: str, since: int = None, ):
        """
        下载数据

        :return:
        """
        filepath = OhlvUtil.symbol_local_path(symbol, timeframe)
        logger.info("下载数据 {}_{}".format(symbol, timeframe))

        if os.path.exists(filepath):
            # 读取最新时间
            with open(filepath, 'r') as file:
                reader = csv.reader(file)
                rows = list(reader)
                last_row = rows[-1]
                since = DateUtil.format_to_timestamp(last_row[0]) * 1000 + interval_second[timeframe] * 1000
                logger.info(
                    "下载数据 本地最新数据 {}".format(DateUtil.timestamp_to_format(since / 1000)))
        if since is None:
            since = CCtxAdapter.query_begin_timestamp(symbol)
        limit = 300
        from datetime import datetime
        stop_tmp = datetime.now().timestamp() * 1000
        size = (stop_tmp - since) / 1000 / interval_second[timeframe] / limit + 1
        pbar = tqdm.trange(0, int(size), 1)
        data = []
        for idx, element in enumerate(pbar):
            fetch_ohlcv = CCtxAdapter.fetch_ohlcv(symbol=symbol, timeframe=timeframe, since=since, limit=limit)
            if fetch_ohlcv is None or len(fetch_ohlcv) == 0:
                break
            # 更新查询时间
            for index in range(len(fetch_ohlcv)):
                ohlv = fetch_ohlcv[index]
                datetime = DateUtil.timestamp_to_format(ohlv.timestamp / 1000)
                data.append(
                    [datetime, ohlv.openPrice, ohlv.closePrice, ohlv.highestPrice, ohlv.lowestPrice, ohlv.volume])

            since = fetch_ohlcv[len(fetch_ohlcv) - 1].timestamp + interval_second[timeframe] * 1000
            pbar.set_description(
                f"No.{idx}: [{symbol}-{timeframe}]")
            if len(data) > 0:
                OhlvUtil.__ohlv_to_csv(data, filepath, _ohlv_head)
                data.clear()
        logger.info("{}-{} 位置:{}".format(symbol, timeframe, filepath))

        return filepath

    @staticmethod
    def __ohlv_to_csv(data: [], save_path: str, head):
        """
        数据保存为csv

        """

        if not os.path.exists(save_path):
            with open(save_path, 'w') as csvfile:
                writer = csv.writer(csvfile, lineterminator='\n')
                writer.writerow(head)
        with open(save_path, 'a') as csvfile:
            writer = csv.writer(csvfile, lineterminator='\n')
            writer.writerows(data)

技术细节

• 提示443 在request模块添加代理设置

小结

` 通过对cctx 封装获取数据，用来后面策略回测数据

源码获取

点赞关注收藏,私信作者，秒关秒回