첫 번째 커밋
This commit is contained in:
javamon
207
collectors/collector.py
Normal file
207
collectors/collector.py
Normal file
@@ -0,0 +1,207 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import requests, logging, json, time, sys
|
||||
from datetime import datetime
|
||||
from db import DB
|
||||
|
||||
from math import log10, floor
|
||||
|
||||
class Collector:
|
||||
_auth = {}
|
||||
_heaeder = {}
|
||||
_mathod = ''
|
||||
_name = 'target'
|
||||
_temp_api_url = ''
|
||||
_api_url = ''
|
||||
|
||||
allow_items = None
|
||||
|
||||
db = None
|
||||
|
||||
def _set_allow_items(self):
|
||||
if self.is_db_connection():
|
||||
self.allow_items = [_['item_code'] for _ in self.db.select_allow_items_all()]
|
||||
|
||||
def _get(self, url, headers=None, data=None, params=None):
|
||||
resp = requests.get(url, headers=headers, data=data, params=params)
|
||||
|
||||
if resp.status_code == 429 : # 업비트 요청 수 제한으로 인한 1분 지연
|
||||
time.sleep(60)
|
||||
return self._get(url, headers, data, params)
|
||||
|
||||
if resp.status_code not in [200, 201]:
|
||||
logging.error('get(%s) failed(%d)' % (url, resp.status_code))
|
||||
if resp.text is not None:
|
||||
logging.error('resp: %s' % resp.text)
|
||||
raise Exception('request.get() failed(%s)' % resp.text)
|
||||
raise Exception(
|
||||
'request.get() failed(status_code:%d)' % resp.status_code)
|
||||
self._update_remaining_req(resp)
|
||||
|
||||
return json.loads(resp.text)
|
||||
|
||||
def data_columns_init(self, df):
|
||||
t_col = []
|
||||
for c in df.columns:
|
||||
t_col.append(c.lower().capitalize())
|
||||
|
||||
df.columns = t_col
|
||||
|
||||
def cal_ceil(self, x, sig=4):
|
||||
return float(str(x)[:sig])
|
||||
|
||||
def packaging_data_per_time_unit(self, df, t_time):
|
||||
time_type = {
|
||||
'hour': '60min',
|
||||
'hour4':'240min',
|
||||
'hour6': '360min',
|
||||
'hour12': '720min',
|
||||
}
|
||||
ohlc_dict = {
|
||||
'Open': 'first',
|
||||
'High': 'max',
|
||||
'Low': 'min',
|
||||
'Close': 'last',
|
||||
'Volume': 'sum'
|
||||
}
|
||||
|
||||
df = df.resample(time_type[t_time], how=ohlc_dict, label='left', base=540) # UTC (트레이딩뷰)
|
||||
|
||||
# return df[:-1]
|
||||
return df
|
||||
|
||||
def _update_remaining_req(self, resp):
|
||||
if 'Remaining-Req' not in resp.headers.keys():
|
||||
return None
|
||||
keyvals = resp.headers['Remaining-Req'].split('; ')
|
||||
group = None
|
||||
keyval = dict()
|
||||
for _keyval in keyvals:
|
||||
kv = _keyval.split('=')
|
||||
if kv[0] == 'group':
|
||||
group = kv[1]
|
||||
else:
|
||||
keyval[kv[0]] = kv[1]
|
||||
if group is None:
|
||||
return
|
||||
keyval['update_time'] = datetime.now()
|
||||
self.remaining_req[group] = keyval
|
||||
|
||||
def is_db_connection(self):
|
||||
if self.db is None :
|
||||
self.db = DB()
|
||||
|
||||
return True
|
||||
|
||||
# 테이블 체크 후 없을 시 기존 데이터를 추가(거래소별 max의 개수)
|
||||
def is_table_in_db(self, finance, exchange, item, time_type = None):
|
||||
if self.is_db_connection() :
|
||||
return self.db.is_item_table_in_db(finance, exchange, item, time_type)
|
||||
|
||||
# 테이블 확인이 안될 시 없는 것으로 간주
|
||||
return False
|
||||
|
||||
def _save_to_db_from_collectors_data(self, finance = 'crypto', exchange = 'upbit', item='KRW-BTC', data = None, time_type = None):
|
||||
if self.is_db_connection() :
|
||||
if data is not None and time_type is not None :
|
||||
self.db.insertItemData(finance, exchange, item, data, time_type)
|
||||
|
||||
def _save_to_db_from_collectors_dataframe(self, finance = 'crypto', exchange = 'upbit', item='KRW-BTC', data = None, time_type = None):
|
||||
if self.is_db_connection() :
|
||||
if data is not None and time_type is not None :
|
||||
self.db.insertItemDataframe(finance, exchange, item, data, time_type)
|
||||
|
||||
|
||||
def _get_api_url(self):
|
||||
if self.is_db_connection() :
|
||||
url = self.db.select_api_url_from_tb_exchange(self._name)
|
||||
if url is () :
|
||||
return self._temp_api_url
|
||||
|
||||
return str(url['api_url'])
|
||||
|
||||
# 거래소 24시간 기준금액 업데이트
|
||||
def update_exchange_standard_options(self):
|
||||
markets = self._get_data_all_markets()
|
||||
|
||||
self._standard_price = self._get_standard_trade_price(markets)
|
||||
self.db.update_standard_options_from_exchange(self._name, self._standard_price)
|
||||
|
||||
def get_exchange_info_from_db(self):
|
||||
return self.db.select_row_data_from_exchange(self._name)
|
||||
|
||||
def _get_item_list_tb_exchange_and_item(self): # dev
|
||||
e_info = self.get_exchange_info_from_db()
|
||||
|
||||
if e_info is () :
|
||||
return None
|
||||
|
||||
return self.db.select_items_data_from_tb_items_by_exchange_id(e_info['id'])
|
||||
|
||||
# 거래소 기본정보 데이터 저장
|
||||
def _add_exchange_info_to_db(self, finance, exchange, s_price, url):
|
||||
if self.is_db_connection() :
|
||||
self.db.insert_finance_and_exchange_data(finance, exchange, s_price, url)
|
||||
|
||||
# 아이템 종목 리스트 저장(종목 행 데이터 생성)
|
||||
def _save_item_data_to_item_table(self, finance, exchange, item, total_trade_price_24h, total_trade_volume_24h):
|
||||
if self.is_db_connection() :
|
||||
self.db.insert_item_main_data(finance, exchange, item, total_trade_price_24h, total_trade_volume_24h)
|
||||
|
||||
# 거래종목 리스트 로드
|
||||
def _load_markets(self):
|
||||
print('not defined function : %s' % sys._getframe().f_code.co_name)
|
||||
|
||||
# 거래금액으로 필터링 된 거래종목 리스트 반환
|
||||
def _return_list_filted_markets(self, markets):
|
||||
print('not defined function : %s' % sys._getframe().f_code.co_name)
|
||||
|
||||
# 거래 금액 기준 반환 (달러 or 원화)
|
||||
def _get_trade_price(self, markets):
|
||||
print('not defined function : %s' % sys._getframe().f_code.co_name)
|
||||
|
||||
def _get_currency_type(self, item):
|
||||
'''
|
||||
0 : doller
|
||||
1 : krw
|
||||
|
||||
:param item: target item
|
||||
:return: currency_type
|
||||
'''
|
||||
|
||||
if 'KRW' in str(item).upper() :
|
||||
return 1
|
||||
else :
|
||||
return 0
|
||||
|
||||
def get_based_time_from_data(self, data):
|
||||
if 'KRW' in data['target']:
|
||||
return '9'
|
||||
|
||||
return '0'
|
||||
|
||||
# need override
|
||||
def save_current_data(self):
|
||||
print('not defined function : %s' % sys._getframe().f_code.co_name)
|
||||
|
||||
def _return_list_filted_markets(self, markets):
|
||||
print('not defined function : %s' % sys._getframe().f_code.co_name)
|
||||
|
||||
def save_current_min_data(self):
|
||||
print('not defined function : %s' % sys._getframe().f_code.co_name)
|
||||
|
||||
def save_current_hour_data(self):
|
||||
print('not defined function : %s' % sys._getframe().f_code.co_name)
|
||||
|
||||
def save_current_day_data(self):
|
||||
print('not defined function : %s' % sys._getframe().f_code.co_name)
|
||||
|
||||
def _set_standard_price(self):
|
||||
print('not defined function : %s' % sys._getframe().f_code.co_name)
|
||||
|
||||
def get_history_data(self):
|
||||
print('not defined function : %s' % sys._getframe().f_code.co_name)
|
||||
|
||||
def get_last_price_from_orderbook(self):
|
||||
print('not defined function : %s' % sys._getframe().f_code.co_name)
|
||||
|
||||
Reference in New Issue
Block a user