From ab8b43a53367e00985519e13c42beb68d9aa3def Mon Sep 17 00:00:00 2001 From: Nick Ochiel Date: Thu, 14 Apr 2022 07:01:55 +0300 Subject: [PATCH] Migration to FastApi. --- app.py | 679 ++++++++++++++++++++++++++++----------------------------- 1 file changed, 332 insertions(+), 347 deletions(-) diff --git a/app.py b/app.py index e4c7f71..ab06f5d 100644 --- a/app.py +++ b/app.py @@ -1,33 +1,24 @@ -# TODO(nochiel) Add tests / data validation. -# FIXME(nochiel) Fix crash when you make an API call that throws an exception. -# TODO(nochiel) Migrate to FastAPI -# TODO(nochiel) Give the client structured errors. -# TODO(nochiel) - Create a new exception type so that client receives structured errors as a response. -# TODO(nochiel) - If there is no data available for a response, provide a good error. +# TODO(nochiel) Add tests # TODO(nochiel) - Standard error page/response for non-existent routes? -# TODO(nochiel) TEST that all endpoints return consistent types. import asyncio from datetime import datetime, timedelta -import time -import sys +from http import HTTPStatus +import logging import os import pathlib -import statistics -from http import HTTPStatus +import sys +import time import ccxt -import flask -from flask import request - - +from fastapi import FastAPI, HTTPException from pydantic import BaseModel, BaseSettings, validator class ServerErrors: # TODO(nochiel) Replace these with HTTPException NO_DATA = 'Spotbit did not find any data.' EXCHANGE_NOT_SUPPORTED = 'Spotbit is not configured to support the exchange.' - + BAD_DATE_FORMAT = 'Please use dates in YYYY-MM-DDTHH:mm:ss ISO8601 format or unix timestamps.' class Error(BaseModel): code : int @@ -38,11 +29,10 @@ class Error(BaseModel): class Settings(BaseSettings): exchanges: list[str] = [] - currencies: list[str] = [] - - averaging_time: int = 1 + currencies: list[str] onion: str | None = None + debug: bool = False @validator('currencies') def uppercase_currency_names(cls, v): @@ -56,47 +46,68 @@ class Config: # we can determine which ones shouldn't be supported i.e. populate # this list with exchanges that fail tests. _unsupported_exchanges = [] -_supported_exchanges: dict[str, ccxt.Exchange] = {} # TODO(nochiel) Load these when self.exchanges is loaded. - -app = flask.Flask(__name__) - -# TODO(nochiel) TEST Ensure logging can tell us when and why the app crashes. -import logging.handlers -handler = logging.handlers.RotatingFileHandler( - filename = 'spotbit.log', - maxBytes = 1 << 20, - backupCount = 2, - ) -handler.setLevel(logging.DEBUG) -formatter = logging.Formatter( - '[%(asctime)s] %(levelname)s (thread %(thread)d):\t%(module)s.%(funcName)s: %(message)s') -handler.setFormatter(formatter) - -from flask.logging import default_handler -default_handler.setFormatter(formatter) -app.logger.addHandler(handler) +_supported_exchanges: dict[str, ccxt.Exchange] = {} + +def get_logger(): + + import logging + logger = logging.getLogger(__name__) + if _settings.debug: logger.setLevel(logging.DEBUG) + + formatter = logging.Formatter( + '[%(asctime)s] %(levelname)s (thread %(thread)d):\t%(module)s.%(funcName)s: %(message)s') + + handler = logging.StreamHandler() + handler.setFormatter(formatter) + logger.addHandler(handler) + + import logging.handlers + handler = logging.handlers.RotatingFileHandler( + filename = 'spotbit.log', + maxBytes = 1 << 20, + backupCount = 2, + ) + handler.setFormatter(formatter) + logger.addHandler(handler) + + return logger _settings = Settings() -app.logger.debug(f'using currencies: {_settings.currencies}') -if len(_settings.exchanges) == 0: - app.logger.info('using all exchanges.') + +# TODO(nochiel) Move this to the settings class. +from enum import Enum +CurrencyName = Enum('CurrencyName', [(currency, currency) for currency in _settings.currencies]) + +logger = get_logger() +assert logger + +app = FastAPI(debug = _settings.debug) + +logger.debug(f'Using currencies: {_settings.currencies}') +if not _settings.exchanges: + logger.info('using all exchanges.') _settings.exchanges = list(ccxt.exchanges) assert _settings.exchanges -app.logger.info('Initialising supported exchanges.') +logger.info('Initialising supported exchanges.') for e in _settings.exchanges: if e in ccxt.exchanges and e not in _unsupported_exchanges: _supported_exchanges[e] = ccxt.__dict__[e]() - if not app.debug: _supported_exchanges[e].load_markets() + try: + if app.debug is False: _supported_exchanges[e].load_markets() + except Exception as e: + logger.debug(f'Error loading markets for {e}.') assert _supported_exchanges +ExchangeName = Enum('ExchangeName', [(id.upper(), id) for id in _supported_exchanges]) + # Exchange data is sometimes returned as epoch milliseconds. def is_ms(timestamp): return timestamp % 1e3 == 0 class Candle(BaseModel): - timestamp : int + timestamp : datetime open : float high : float low : float @@ -106,7 +117,10 @@ class Candle(BaseModel): @validator('timestamp') def time_in_seconds(cls, v): result = v - if is_ms(v): result = int(v * 1e-3) + + if type(v) is int and is_ms(v): + result = int(v * 1e-3) + return result class Config: @@ -114,34 +128,36 @@ class Config: datetime: lambda v: v.isoformat() } -def request_single(exchange: ccxt.Exchange, currency: str) -> Candle | None: +# FIXME(nochiel) Redundancy: Merge this with get_history. +# TODO(nochiel) TEST Do we really need to check if fetchOHLCV exists in the exchange api? +# TEST ccxt abstracts internally using fetch_trades so we don't have to use fetch_ticker ourselves. +def request_single(exchange: ccxt.Exchange, currency: CurrencyName) -> Candle | None: ''' Make a single request, without having to loop through all exchanges and currency pairs. ''' assert exchange and isinstance(exchange, ccxt.Exchange) - assert str + assert currency - # TODO(nochiel) Add a type for result. result = None latest_candle = None - ticker = f'BTC/{currency}' + ticker = f'BTC/{currency.value}' dt = None + exchange.load_markets() if ticker not in exchange.markets: return None - # TODO(nochiel) Check that exchange supports currency. if exchange.has['fetchOHLCV']: - app.logger.debug('fetchOHLCV') + logger.debug('fetchOHLCV') timeframe = '1m' match exchange.id: - case 'bleutrade' | 'btcalpha' | 'rightbtc' | 'hollaex': + case 'btcalpha' | 'hollaex': timeframe = '1h' case 'poloniex': timeframe = '5m' - # some exchanges have explicit limits on how many candles you can get at once + # Some exchanges have explicit limits on how many candles you can get at once # TODO(nochiel) Simplify this by checking for 2 canonical limits to use. limit = 1000 match exchange.id: @@ -158,7 +174,7 @@ def request_single(exchange: ccxt.Exchange, currency: str) -> Candle | None: since = round((datetime.now() - timedelta(hours=1)).timestamp() * 1e3) - # TODO(nochiel) TEST other exchanges requiring special conditions: bitstamp, bitmart + # TODO(nochiel) TEST other exchanges requiring special conditions: bitstamp, bitmart? params = [] if exchange.id == 'bitfinex': params = { @@ -178,16 +194,16 @@ def request_single(exchange: ccxt.Exchange, currency: str) -> Candle | None: latest_candle = candles[-1] except Exception as e: - app.logger.error(f'error requesting candle from {exchange.name}: {e}') + logger.error(f'error requesting candle from {exchange.name}: {e}') else: # TODO(nochiel) TEST - app.logger.debug(f'fetch_ticker: {ticker}') + logger.debug(f'fetch_ticker: {ticker}') candle = None try: candle = exchange.fetch_ticker(ticker) except Exception as e: - app.logger.error(f'error on {exchange} fetch_ticker: {e}') + logger.error(f'error on {exchange} fetch_ticker: {e}') latest_candle = candle @@ -203,49 +219,39 @@ def request_single(exchange: ccxt.Exchange, currency: str) -> Candle | None: return result + # Routes # TODO(nochiel) Add tests for routes. -# TODO(nochiel) Add new route for the Spotbit UI. # TODO(nochiel) Put the api behind an /api/v1 path. +# TODO(nochiel) Make this the Spotbit frontend. @app.get('/') def index(): - # FIXME(nochiel) - - date_start = (datetime.now() - timedelta(days=5)).timestamp() * 1e3 - date_end = (datetime.now()).timestamp() * 1e3 - onion = _settings.onion - f0 = f"{onion}/now/USD/coinbasepro" - f1 = f"{onion}/now/USD" - f2 = f"{onion}/hist/USD/coinbasepro/{date_start}/{date_end}" - f3 = f"{onion}/configure" - - return flask.render_template('index.html', - fetch_0 = f0, - fetch_1 = f1, - fetch_2 = f2, - fetch_3 = f3, - date_start = date_start, - date_end = date_end) - -@app.get('/status') + + raise HTTPException( + detail = 'Not implemented.', + status_code = 404 + ) + + +@app.get('/api/status') def status(): return "server is running" # TODO(nochiel) FINDOUT Do we need to enable clients to change configuration? # If clients should be able to change configuration, use sessions. -@app.get('/configure') +@app.get('/api/configure') def get_configuration(): return { - 'currencies': _settings.currencies, - 'exchanges': _settings.exchanges, + 'currencies': _settings.currencies, + 'exchanges': _settings.exchanges, } def calculate_average_price(candles: list[Candle]) -> Candle: - assert candles and len(candles) + assert candles - mean = statistics.mean + from statistics import mean average_open = mean(candle.open for candle in candles) average_high = mean(candle.high for candle in candles) average_low = mean(candle.low for candle in candles) @@ -262,37 +268,29 @@ def calculate_average_price(candles: list[Candle]) -> Candle: ) return candle -class ExchangeDetails(BaseModel): - id: str - name: str - url : str - countries: list[str] - currencies: list[str] - +# TODO(nochiel) Verify: Do we need this? Will it be correct in all cases? def get_supported_currencies(exchange: ccxt.Exchange) -> list[str] : required = set(_settings.currencies) - given = set([c for c in exchange.currencies]) + given = set(exchange.currencies.keys()) return list(required & given) +class ExchangeDetails(BaseModel): + id: str + name: str + url : str + countries: list[str] + currencies: list[str] -@app.get('/exchanges') +@app.get('/api/exchanges', response_model = list[ExchangeDetails]) async def get_exchanges(): # Ref. https://github.com/BlockchainCommons/spotbit/issues/54 - # Expected: output that looks like: ''' - [ - {"id": "kraken", "name": "Kraken", "url": "https://www.kraken.com/", "country": "US", "currencies": ["USD"]}, - {"id": "ascendex", "name": "AscendEX", "url": "https://ascendex.com/", "country": "SG", "currencies": ["USD", "JPY", "GBP"]} - ] + Get a list of exchanges that this instance of Spotbit has been configured to use. ''' - ... - class ExchangeResult(BaseModel): - exchanges: list[ExchangeDetails] - - result: ExchangeResult | None = None + result: list[ExchangeDetails] = [] assert _supported_exchanges @@ -313,7 +311,7 @@ def get_exchange_details(exchange: ccxt.Exchange) -> ExchangeDetails: name = exchange.name, url = exchange.urls['www'], countries = exchange.countries, - currencies = get_supported_currencies(exchange)) + currencies = get_supported_currencies(exchange)) # TODO(nochiel) TEST result = details return result @@ -321,50 +319,47 @@ def get_exchange_details(exchange: ccxt.Exchange) -> ExchangeDetails: tasks = [asyncio.to_thread(get_exchange_details, exchange) for exchange in _supported_exchanges.values()] details = await asyncio.gather(*tasks) - result = ExchangeResult(exchanges = list(details)) - return result.dict() + result = list(details) -@app.get('/now/') -async def now_average(currency: str): + return result + +class PriceResponse(BaseModel): + candle : Candle + exchanges_used : list[str] + failed_exchanges : list[str] + +@app.get('/api/now/{currency}', response_model = PriceResponse) +async def now_average(currency: CurrencyName): ''' - Return an average of the 5 curated exchanges for that currency. + Return an average price from the exchanges configured for the given currency. ''' - class PriceResponse(BaseModel): - candle : Candle - exchanges_used : list[str] - failed_exchanges : list[str] - result = None - currency = currency.upper() - if currency not in _settings.currencies: - flask.abort(flask.Response(response = f'Spotbit is not configured to use {currency}.', - status = HTTPStatus.INTERNAL_SERVER_ERROR)) - averaging_time = _settings.averaging_time + logger.debug(f'currency: {currency}') - def get_candle(exchange: ccxt.Exchange, currency: str) -> tuple[ccxt.Exchange, Candle | None]: + def get_candle(exchange: ccxt.Exchange, currency: CurrencyName) -> tuple[ccxt.Exchange, Candle | None]: assert exchange assert currency result = (exchange, None) exchange.load_markets() - if currency in exchange.currencies: + if currency.value in exchange.currencies: try: candle = None - candle = request_single(exchange, currency) + candle = request_single(exchange, currency.value) if candle: result = exchange, candle except Exception as e: - app.logger.error(f'error requesting data from exchange: {e}') + logger.error(f'error requesting data from exchange: {e}') return result tasks = [asyncio.to_thread(get_candle, exchange, currency) for exchange in _supported_exchanges.values()] task_results = await asyncio.gather(*tasks) - app.logger.debug(f'task results: {task_results}') + logger.debug(f'task results: {task_results}') candles = [] failed_exchanges = [] @@ -374,12 +369,14 @@ def get_candle(exchange: ccxt.Exchange, currency: str) -> tuple[ccxt.Exchange, C else: failed_exchanges.append(exchange.name) - app.logger.debug(f'candles: {candles}') - if len(candles) > 0 : + logger.debug(f'candles: {candles}') + average_price_candle = None + if len(candles): average_price_candle = calculate_average_price(candles) else: - flask.abort(flask.Response(response = 'SpotBit could get any candle data from the configured exchanges.', - status = HTTPStatus.INTERNAL_SERVER_ERROR)) + raise HTTPException( + status_code = HTTPStatus.INTERNAL_SERVER_ERROR, + detail = 'Spotbit could get any candle data from the configured exchanges.') exchanges_used = [exchange.name for exchange in _supported_exchanges.values() if exchange.name not in failed_exchanges] @@ -390,44 +387,43 @@ def get_candle(exchange: ccxt.Exchange, currency: str) -> tuple[ccxt.Exchange, C failed_exchanges = failed_exchanges, ) - return result.dict() + return result -@app.get('/now//') -def now(currency, exchange): +@app.get('/api/now/{currency}/{exchange}', response_model = Candle) +def now(currency: CurrencyName, exchange: ExchangeName): ''' parameters: - exchange(required): an exchange to use. - currency(required): the symbol for the base currency to use e.g. USD, GBP, UST. + exchange: an exchange to use. + currency: the symbol for the base currency to use e.g. USD, GBP, UST. ''' - if exchange not in _supported_exchanges: - flask.abort(flask.Response(response = f'SpotBit is not configured to use {exchange} exchange.', - status = HTTPStatus.INTERNAL_SERVER_ERROR)) + if exchange.value not in _supported_exchanges: + raise HTTPException( + status_code = HTTPStatus.INTERNAL_SERVER_ERROR, + detail = f'Spotbit is not configured to use {exchange.value} exchange.') result = None - exchange = _supported_exchanges[exchange] - exchange.load_markets() - currency = currency.upper() - if currency not in _settings.currencies: - flask.abort(flask.Response(response = f'Spotbit is not configured to use {currency}.', - status = HTTPStatus.INTERNAL_SERVER_ERROR)) - if currency not in exchange.currencies: - flask.abort(flask.Response(response = f'Spotbit does not support the {currency} on {exchange}.', - status = HTTPStatus.INTERNAL_SERVER_ERROR)) - assert exchange - assert currency + ccxt_exchange = _supported_exchanges[exchange.value] + assert ccxt_exchange + ccxt_exchange.load_markets() - # TODO(nochiel) Handle exception. - result = request_single(exchange, currency) + if currency.value not in ccxt_exchange.currencies: + raise HTTPException( + status_code = HTTPStatus.INTERNAL_SERVER_ERROR, + detail = f'Spotbit does not support {currency.value} on {ccxt_exchange}.' ) + result = request_single(ccxt_exchange, currency) if not result: - flask.abort(flask.Response(response = ServerErrors.NO_DATA, - status = HTTPStatus.INTERNAL_SERVER_ERROR)) + raise HTTPException( + status_code = HTTPStatus.INTERNAL_SERVER_ERROR, + detail = ServerErrors.NO_DATA + ) - return result.dict() + return result -class OHLCV: +from enum import IntEnum +class OHLCV(IntEnum): ''' Indices for components ina candle list. ''' @@ -438,75 +434,115 @@ class OHLCV: close = 4 volume = 5 -# FIXME(nochiel) Use query parameters. -# - exchange, currency, date_end are optional. -# FIXME(nochiel) Add parameter validation. -# TODO(nochiel) Write tests -@app.get('/history////') -async def get_candles_in_range(currency, exchange, date_start, date_end): +def get_history(*, + exchange: ccxt.Exchange, + since: datetime, + limit: int, + timeframe: str, + pair: str) -> list[Candle] | None: + + assert exchange + logger.debug(f'{exchange} {pair} {since}') + + result = None + + rateLimit = exchange.rateLimit + _since = int(since.timestamp() * 1e3) + + params = {} + if exchange == "bitfinex": + params = { 'end' : round(end.timestamp() * 1e3) } + + candles = None + try: + wait = 1 + while wait > 0: + try: + candles = exchange.fetchOHLCV( + symbol = pair, + limit = limit, + timeframe = timeframe, + since = _since, + params = params) + + wait = 0 + + except ccxt.errors.RateLimitExceeded as e: + logger.debug(f'{e}. Rate limit for {exchange} is {rateLimit}') + time.sleep(wait) + wait *= 2 + + except Exception as e: + logger.error(f'{exchange} candle request error: {e}') + + if candles: + result = [Candle( + timestamp = candle[OHLCV.timestamp], + open = candle[OHLCV.open], + high = candle[OHLCV.high], + low = candle[OHLCV.low], + close = candle[OHLCV.close], + volume = candle[OHLCV.volume] + ) + + for candle in candles] + + return result + +@app.get('/api/history/{currency}/{exchange}', response_model = list[Candle]) +async def get_candles_in_range( + currency: CurrencyName, + exchange: ExchangeName, + start: datetime, + end: datetime = datetime.now()): ''' parameters: exchange(required): an exchange to use. currency(required): the symbol for the base currency to use e.g. USD, GBP, UST. - date_start, date_end(required): datetime formatted as ISO8601 "YYYY-MM-DDTHH:mm:SS". + start, end(required): datetime formatted as ISO8601 "YYYY-MM-DDTHH:mm:SS" or unix timestamp. ''' - if exchange not in _supported_exchanges: - flask.abort(flask.Response(response = f'SpotBit is not configured to use {exchange} exchange.', - status = HTTPStatus.INTERNAL_SERVER_ERROR)) + ccxt_exchange = _supported_exchanges[exchange.value] + ccxt_exchange.load_markets() + assert ccxt_exchange.currencies + assert ccxt_exchange.markets - exchange = _supported_exchanges[exchange] - exchange.load_markets() - assert exchange.currencies - assert exchange.markets - currency = currency.upper() - if currency not in exchange.currencies: - flask.abort(flask.Response(response = f'Spotbit does not support the {currency} on {exchange}', - status = HTTPStatus.INTERNAL_SERVER_ERROR)) - - pair = f'BTC/{currency}' - if exchange.id == 'bitmex': + pair = f'BTC/{currency.value}' + if ccxt_exchange.id == 'bitmex': pair = f'BTC/{currency}:{currency}' - if pair not in exchange.markets: - flask.abort(flask.Response(response = f'Spotbit does not support the {pair} pair on {exchange}', - status = HTTPStatus.INTERNAL_SERVER_ERROR)) + if pair not in ccxt_exchange.markets: + raise HTTPException( + detail = f'Spotbit does not support the {pair} pair on {ccxt_exchange}', + status_code = HTTPStatus.INTERNAL_SERVER_ERROR) result = None - try: - # start = round((datetime.fromisoformat(date_start)).timestamp() * 1e3) - start = datetime.fromisoformat(date_start) - end = datetime.fromisoformat(date_end) - (start, end) = (end, start) if end < start else (start, end) - except ValueError as e: - flask.abort(flask.Response(response = f'Error: {e}. Please use dates in YYYY-MM-DDTHH:mm:ss ISO8601 format.', - status = HTTPStatus.BAD_REQUEST)) - limit = 100 + start = start.astimezone(start.tzinfo) + end = end.astimezone(end.tzinfo) + + (start, end) = (end, start) if end < start else (start, end) + logger.debug(f'start: {start}, end: {end}') + limit = 100 candles = None periods = [] - # Consider enabling a backup historical candle feed e.g. coinmarketcap, coingecko, cryptowa.ch, graph or a dex? - # Especially if there's a feed that allows you to filter by an exchange. - if exchange.has['fetchOHLCV'] is not True: - flask.abort(flask.Response( - response = f'{exchange} does not support pagination of historical candles. Please try to use a different exchange.', - status = HTTPStatus.BAD_REQUEST)) - - dt = timedelta(0) + dt = timedelta(hours = 1) params = None - timeframe = '' - if '1h' in exchange.timeframes: - timeframe = '1h' - dt = timedelta(hours = 1) + timeframe = '1h' + + if ccxt_exchange.timeframes: + if '1h' in ccxt_exchange.timeframes: + timeframe = '1h' + dt = timedelta(hours = 1) - elif '30m' in exchange.timeframes: - timeframe = '30m' - dt = timedelta(minutes = 30) + elif '30m' in ccxt_exchange.timeframes: + timeframe = '30m' + dt = timedelta(minutes = 30) n_periods, remaining_frames_duration = divmod(end - start, dt * 100) remaining_frames = remaining_frames_duration // dt - app.logger.debug(f'requesting #{n_periods + remaining_frames} periods') + logger.debug(f'requesting #{n_periods + remaining_frames} periods') if n_periods == 0: n_periods = 1 @@ -514,197 +550,146 @@ async def get_candles_in_range(currency, exchange, date_start, date_end): for i in range(n_periods): periods.append(start + i * (dt * 100)) - app.logger.debug(f'requesting periods with {limit} limit: {periods}') - - def get_history(*, - exchange: ccxt.Exchange = exchange, - since: datetime, - limit: int = limit, - timeframe: str = timeframe) -> list: - assert exchange - - app.logger.debug(f'{exchange} {pair} {since}') - - rateLimit = exchange.rateLimit - _since = int(since.timestamp() * 1e3) - - params = {} - if exchange == "bitfinex": - params = { 'end' : round(end.timestamp() * 1e3) } - - candles = None - try: - wait = 1 - while wait > 0: - try: - candles = exchange.fetchOHLCV( - symbol = pair, - limit = limit, - timeframe = timeframe, - since = _since, - params = params) - - wait = 0 - - except ccxt.base.errors.RateLimitExceeded as e: - app.logger.debug(f'{e}. Rate limit for {exchange} is {rateLimit}') - time.sleep(wait) - wait *= 2 - - except Exception as e: - app.logger.error(f'{exchange} candle request error: {e}') - - return candles + logger.debug(f'requesting periods with {limit} limit: {periods}') tasks = [] - for p in periods: + args = dict( exchange = ccxt_exchange, + limit = limit, + timeframe = timeframe, + pair = pair) + + for period in periods: + args['since'] = period task = asyncio.to_thread(get_history, - exchange = exchange, - since = p) + **args) tasks.append(task) if remaining_frames > 0: last_candle_time = periods[-1] + (dt * 100) assert last_candle_time < end - app.logger.debug(f'remaining_frames: {remaining_frames}') + logger.debug(f'remaining_frames: {remaining_frames}') + + args['since'] = last_candle_time + args['limit'] = remaining_frames task = asyncio.to_thread(get_history, - since = last_candle_time, - limit = remaining_frames) + **args) tasks.append(task) new_last_candle_time = last_candle_time + (dt * remaining_frames ) - app.logger.debug(f'new_last_candle_time: {new_last_candle_time}') + logger.debug(f'new_last_candle_time: {new_last_candle_time}') task_results = await asyncio.gather(*tasks) candles = [] for result in task_results: if result: candles.extend(result) - candles = [Candle( - timestamp = candle[OHLCV.timestamp], - open = candle[OHLCV.open], - high = candle[OHLCV.high], - low = candle[OHLCV.low], - close = candle[OHLCV.close], - volume = candle[OHLCV.volume] - ) - - for candle in candles] - expected_number_of_candles = (n_periods * limit) + remaining_frames received_number_of_candles = len(candles) if received_number_of_candles < expected_number_of_candles: - logger.info(f'{exchange} does not have data for the entire period. Expected {expected_number_of_candles} candles. Got {received_number_of_candles} candles') + logger.info(f'{ccxt_exchange} does not have data for the entire period. Expected {expected_number_of_candles} candles. Got {received_number_of_candles} candles') if candles is None or len(candles) == 0: - flask.abort(flask.Response(response = f'Spotbit did not receive any candle history for the period {start} - {end} from {exchange}', - status = HTTPStatus.INTERNAL_SERVER_ERROR)) + raise HTTPException( + detail = f'Spotbit did not receive any candle history for the period {start} - {end} from {ccxt_exchange}', + status_code = HTTPStatus.INTERNAL_SERVER_ERROR) - app.logger.debug(f'got: {len(candles)} candles') - app.logger.debug(f'candles: {candles[:10]} ... {candles[-10:]}') + logger.debug(f'got: {len(candles)} candles') + logger.debug(f'candles: {candles[:10]} ... {candles[-10:]}') + result = candles - result = flask.jsonify([candle.dict() for candle in candles]) return result -tests_for_get_candles_at_dates = [ - '''curl http://localhost:5000/history/usdt/binance --header 'Content-Type:application/json' --data '[\"2022-01-01T00:00 \", \"2022-02-01T00:00\", \"2021-12-01T00:00\"]''', - ] # Return all database rows within `tolerance` for each of the supplied dates -@app.post('/history//') -async def get_candles_at_dates(currency, exchange): +@app.post('/api/history/{currency}/{exchange}') +async def get_candles_at_dates( + currency: CurrencyName, + exchange: ExchangeName, + dates: list[datetime]): ''' - Parameters: - Dates should be provided in the body of the request as a json array of dates formatted as ISO8601 "YYYY-MM-DDTHH:mm:SS". + Dates should be provided in the body of the request as a json array of dates formatted as ISO8601 "YYYY-MM-DDTHH:mm:SS". ''' - app.logger.debug(f'{request.get_data()}') - - if exchange not in _supported_exchanges: - flask.abort(flask.Response( - response = ServerErrors.EXCHANGE_NOT_SUPPORTED, - status = HTTPStatus.INTERNAL_SERVER_ERROR - )) - exchange = _supported_exchanges[exchange] - - if exchange.has['fetchOHLCV'] is not True: - flask.abort(flask.Response( - response = f'{exchange} does not support pagination of historical candles. Please try to use a different exchange.', - status = HTTPStatus.BAD_REQUEST)) - - try: - dates = request.get_json() - # TODO(nochiel) Check if isoformat or timestamps. Raise HTTPException if dates don't parse. - dates = [datetime.fromisoformat(date) for date in dates] if dates else [] - if len(dates) == 0: - raise Exception(f'received: ${dates}') - except Exception as e: - flask.abort(flask.Response( - response = f'error processing dates: {e}.\n' + - 'Dates should be provided in the body of the request as a json array of dates formatted as ISO8601 "YYYY-MM-DDTHH:mm:SS".', - status = HTTPStatus.INTERNAL_SERVER_ERROR)) + if exchange.value not in _supported_exchanges: + raise HTTPException( + detail = ServerErrors.EXCHANGE_NOT_SUPPORTED, + status_code = HTTPStatus.INTERNAL_SERVER_ERROR) - results = None - exchange.load_markets() - pair = f'BTC/{currency.upper()}' - if pair not in exchange.markets: - flask.abort(flask.Response(response = f'Spotbit does not support the {pair} pair on {exchange}', - status = HTTPStatus.INTERNAL_SERVER_ERROR)) + ccxt_exchange = _supported_exchanges[exchange.value] + ccxt_exchange.load_markets() - limit = 1 + pair = f'BTC/{currency.value}' + if pair not in ccxt_exchange.markets: + raise HTTPException( + detail = f'Spotbit does not support the {pair} pair on {exchange.value}', + status_code = HTTPStatus.INTERNAL_SERVER_ERROR) + + limit = 100 timeframe = '1h' - if '1h' in exchange.timeframes: - timeframe = '1h' - elif '30m' in exchange.timeframes: - timeframe = '30m' + if ccxt_exchange.timeframes: + if '1h' in ccxt_exchange.timeframes: + timeframe = '1h' + + elif '30m' in ccxt_exchange.timeframes: + timeframe = '30m' candles = {} - params = {} - def get_candle(*, - exchange: ccxt.Exchange = exchange, - since: datetime, - limit: int = limit, - timeframe: str = timeframe, - pair: str = pair, - ) -> Candle | None: + args = [dict(exchange = ccxt_exchange, + limit = limit, + timeframe = timeframe, + pair = pair, - assert exchange + since = date) + for date in dates] + tasks = [asyncio.to_thread(get_history, **arg) + for arg in args] + candles = await asyncio.gather(*tasks) - result = None - candles = None - try: - candles = exchange.fetchOHLCV( - symbol = pair, - timeframe = timeframe, - limit = limit, - since = int(since.timestamp() * 1e3), - params = params) + result = candles - except Exception as e: - app.logger.error(e) + return result - if candles and len(candles[0]) > 0: - app.logger.debug(f'candles: {candles}') - candle = candles[0] +def tests(): + # Placeholder + # Expected: validation errors or server errors or valid responses. - result = Candle( - timestamp = candle[OHLCV.timestamp], - open = candle[OHLCV.open], - high = candle[OHLCV.high], - low = candle[OHLCV.low], - close = candle[OHLCV.close], - volume = candle[OHLCV.volume] - ) + import requests + response = requests.get('http://[::1]:5000/api/now/FOOBAR') + response = requests.get('http://[::1]:5000/api/now/usd') + response = requests.get('http://[::1]:5000/api/now/USD') + response = requests.get('http://[::1]:5000/api/now/JPY') - return result + response = requests.get('http://[::1]:5000/api/now/USD/Bitstamp') + response = requests.get('http://[::1]:5000/api/now/USD/bitstamp') + response = requests.get('http://[::1]:5000/api/now/usdt/bitstamp') - assert dates - tasks = [asyncio.to_thread(get_candle, since = date) - for date in dates] - candles = await asyncio.gather(*tasks) - candles = [candle.dict() for candle in candles if candle is not None] + response = requests.get( + 'http://[::1]:5000/api/history/USD/bitstamp?start=2019-01-01T0000&end=1522641600' + ) - result = flask.jsonify(candles) + response = requests.get( + "http://[::1]:5000/api/history/USD/liquid?start=2022-01-01T00:00&end=2022-02-01T00:00" + ) - return result + response = requests.post('http://[::1]:5000/history/USDT/binance', + json = ['2022-01-01T00:00', '2022-02-01T00:00', '2021-12-01T00:00'] + ) + + response = requests.post( + "http://[::1]:5000/api/history/JPY/liquid", + json=["2022-01-01T00:00", "2022-02-01T00:00", "2021-12-01T00:00"], + ) + +if __name__ == '__main__': + import uvicorn + + assert logger + logger.debug('Running in debug mode') + logger.debug(f'app.debug: {app.debug}') + uvicorn.run('app:app', + host ='::', + port = 5000, + debug = True, + log_level = 'debug', + reload = True)