208 lines
6.9 KiB
Python
208 lines
6.9 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
import requests, logging, json, time, sys
|
|
from datetime import datetime
|
|
from db import DB
|
|
|
|
from math import log10, floor
|
|
|
|
class Collector:
|
|
_auth = {}
|
|
_heaeder = {}
|
|
_mathod = ''
|
|
_name = 'target'
|
|
_temp_api_url = ''
|
|
_api_url = ''
|
|
|
|
allow_items = None
|
|
|
|
db = None
|
|
|
|
def _set_allow_items(self):
|
|
if self.is_db_connection():
|
|
self.allow_items = [_['item_code'] for _ in self.db.select_allow_items_all()]
|
|
|
|
def _get(self, url, headers=None, data=None, params=None):
|
|
resp = requests.get(url, headers=headers, data=data, params=params)
|
|
|
|
if resp.status_code == 429 : # 업비트 요청 수 제한으로 인한 1분 지연
|
|
time.sleep(60)
|
|
return self._get(url, headers, data, params)
|
|
|
|
if resp.status_code not in [200, 201]:
|
|
logging.error('get(%s) failed(%d)' % (url, resp.status_code))
|
|
if resp.text is not None:
|
|
logging.error('resp: %s' % resp.text)
|
|
raise Exception('request.get() failed(%s)' % resp.text)
|
|
raise Exception(
|
|
'request.get() failed(status_code:%d)' % resp.status_code)
|
|
self._update_remaining_req(resp)
|
|
|
|
return json.loads(resp.text)
|
|
|
|
def data_columns_init(self, df):
|
|
t_col = []
|
|
for c in df.columns:
|
|
t_col.append(c.lower().capitalize())
|
|
|
|
df.columns = t_col
|
|
|
|
def cal_ceil(self, x, sig=4):
|
|
return float(str(x)[:sig])
|
|
|
|
def packaging_data_per_time_unit(self, df, t_time):
|
|
time_type = {
|
|
'hour': '60min',
|
|
'hour4':'240min',
|
|
'hour6': '360min',
|
|
'hour12': '720min',
|
|
}
|
|
ohlc_dict = {
|
|
'Open': 'first',
|
|
'High': 'max',
|
|
'Low': 'min',
|
|
'Close': 'last',
|
|
'Volume': 'sum'
|
|
}
|
|
|
|
df = df.resample(time_type[t_time], how=ohlc_dict, label='left', base=540) # UTC (트레이딩뷰)
|
|
|
|
# return df[:-1]
|
|
return df
|
|
|
|
def _update_remaining_req(self, resp):
|
|
if 'Remaining-Req' not in resp.headers.keys():
|
|
return None
|
|
keyvals = resp.headers['Remaining-Req'].split('; ')
|
|
group = None
|
|
keyval = dict()
|
|
for _keyval in keyvals:
|
|
kv = _keyval.split('=')
|
|
if kv[0] == 'group':
|
|
group = kv[1]
|
|
else:
|
|
keyval[kv[0]] = kv[1]
|
|
if group is None:
|
|
return
|
|
keyval['update_time'] = datetime.now()
|
|
self.remaining_req[group] = keyval
|
|
|
|
def is_db_connection(self):
|
|
if self.db is None :
|
|
self.db = DB()
|
|
|
|
return True
|
|
|
|
# 테이블 체크 후 없을 시 기존 데이터를 추가(거래소별 max의 개수)
|
|
def is_table_in_db(self, finance, exchange, item, time_type = None):
|
|
if self.is_db_connection() :
|
|
return self.db.is_item_table_in_db(finance, exchange, item, time_type)
|
|
|
|
# 테이블 확인이 안될 시 없는 것으로 간주
|
|
return False
|
|
|
|
def _save_to_db_from_collectors_data(self, finance = 'crypto', exchange = 'upbit', item='KRW-BTC', data = None, time_type = None):
|
|
if self.is_db_connection() :
|
|
if data is not None and time_type is not None :
|
|
self.db.insertItemData(finance, exchange, item, data, time_type)
|
|
|
|
def _save_to_db_from_collectors_dataframe(self, finance = 'crypto', exchange = 'upbit', item='KRW-BTC', data = None, time_type = None):
|
|
if self.is_db_connection() :
|
|
if data is not None and time_type is not None :
|
|
self.db.insertItemDataframe(finance, exchange, item, data, time_type)
|
|
|
|
|
|
def _get_api_url(self):
|
|
if self.is_db_connection() :
|
|
url = self.db.select_api_url_from_tb_exchange(self._name)
|
|
if url is () :
|
|
return self._temp_api_url
|
|
|
|
return str(url['api_url'])
|
|
|
|
# 거래소 24시간 기준금액 업데이트
|
|
def update_exchange_standard_options(self):
|
|
markets = self._get_data_all_markets()
|
|
|
|
self._standard_price = self._get_standard_trade_price(markets)
|
|
self.db.update_standard_options_from_exchange(self._name, self._standard_price)
|
|
|
|
def get_exchange_info_from_db(self):
|
|
return self.db.select_row_data_from_exchange(self._name)
|
|
|
|
def _get_item_list_tb_exchange_and_item(self): # dev
|
|
e_info = self.get_exchange_info_from_db()
|
|
|
|
if e_info is () :
|
|
return None
|
|
|
|
return self.db.select_items_data_from_tb_items_by_exchange_id(e_info['id'])
|
|
|
|
# 거래소 기본정보 데이터 저장
|
|
def _add_exchange_info_to_db(self, finance, exchange, s_price, url):
|
|
if self.is_db_connection() :
|
|
self.db.insert_finance_and_exchange_data(finance, exchange, s_price, url)
|
|
|
|
# 아이템 종목 리스트 저장(종목 행 데이터 생성)
|
|
def _save_item_data_to_item_table(self, finance, exchange, item, total_trade_price_24h, total_trade_volume_24h):
|
|
if self.is_db_connection() :
|
|
self.db.insert_item_main_data(finance, exchange, item, total_trade_price_24h, total_trade_volume_24h)
|
|
|
|
# 거래종목 리스트 로드
|
|
def _load_markets(self):
|
|
print('not defined function : %s' % sys._getframe().f_code.co_name)
|
|
|
|
# 거래금액으로 필터링 된 거래종목 리스트 반환
|
|
def _return_list_filted_markets(self, markets):
|
|
print('not defined function : %s' % sys._getframe().f_code.co_name)
|
|
|
|
# 거래 금액 기준 반환 (달러 or 원화)
|
|
def _get_trade_price(self, markets):
|
|
print('not defined function : %s' % sys._getframe().f_code.co_name)
|
|
|
|
def _get_currency_type(self, item):
|
|
'''
|
|
0 : doller
|
|
1 : krw
|
|
|
|
:param item: target item
|
|
:return: currency_type
|
|
'''
|
|
|
|
if 'KRW' in str(item).upper() :
|
|
return 1
|
|
else :
|
|
return 0
|
|
|
|
def get_based_time_from_data(self, data):
|
|
if 'KRW' in data['target']:
|
|
return '9'
|
|
|
|
return '0'
|
|
|
|
# need override
|
|
def save_current_data(self):
|
|
print('not defined function : %s' % sys._getframe().f_code.co_name)
|
|
|
|
def _return_list_filted_markets(self, markets):
|
|
print('not defined function : %s' % sys._getframe().f_code.co_name)
|
|
|
|
def save_current_min_data(self):
|
|
print('not defined function : %s' % sys._getframe().f_code.co_name)
|
|
|
|
def save_current_hour_data(self):
|
|
print('not defined function : %s' % sys._getframe().f_code.co_name)
|
|
|
|
def save_current_day_data(self):
|
|
print('not defined function : %s' % sys._getframe().f_code.co_name)
|
|
|
|
def _set_standard_price(self):
|
|
print('not defined function : %s' % sys._getframe().f_code.co_name)
|
|
|
|
def get_history_data(self):
|
|
print('not defined function : %s' % sys._getframe().f_code.co_name)
|
|
|
|
def get_last_price_from_orderbook(self):
|
|
print('not defined function : %s' % sys._getframe().f_code.co_name)
|
|
|