fintic-tracker/database_caller.py

692 lines
24 KiB
Python
Raw Normal View History

# Copyright (C) 2020 Fintic, finticofficial@gmail.com
#
# This file is part of Fintic project, developed by Neythen Treloar and Justin Dunn
#
# This code can not be copied and/or distributed without the express
# permission of Fintic
import finnhub
import time
import csv
import pytz
from datetime import datetime, timedelta
import json
import datetime as dt
import sys, os, base64, hashlib, hmac, select
import requests
from pycoingecko import CoinGeckoAPI
from newsapi import NewsApiClient
import traceback
from geopy import geocoders
def getInput(Block=False):
if Block or select.select([sys.stdin], [], [], 0) == ([sys.stdin], [], []):
msg = sys.stdin.read(1)
#sys.stdin.flush()
else:
msg = ''
return msg
def emptyInfo(symbols, stock_info):
update = False
for symbol in symbols:
if stock_info[symbol] == -1: # stock with no info
update = True
return update
def updateUpdate(NY_time):
NY_str = NY_time.strftime("%d/%m/%Y %H:%M:%S")
f = open('csv/last_update.csv', 'w+')
f.write(NY_str + '\n')
f.close()
2022-01-31 19:11:01 +00:00
def updateStocks(api_key):
try:
max_stocks = 200
f = open('csv/stocks_settings.json', 'r')
all_stocks_settings = json.load(f)
f.close()
stock_info = all_stocks_settings['symbols']
symbols = list(stock_info.keys())
print(symbols)
url = 'https://bm7p954xoh.execute-api.us-east-2.amazonaws.com/default/ScriptsAPI/stocks?symbols='
for symbol in symbols:
url += symbol + ','
2022-01-31 19:11:01 +00:00
url += '&apiKey=' + api_key
response = requests.get(url)
data = response.json()
print(response)
print(data)
stock_info = {}
for stock in data:
stock_info[stock['symbol']] = {'current': stock['price'], 'opening': float(stock['price']) - float(stock['change_since'])}
2022-01-31 19:11:01 +00:00
print(stock_info)
all_stocks_settings['symbols'] = stock_info
json.dump(all_stocks_settings, open('csv/stocks_settings.json', 'w+'))
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
logf.write(str(e))
logf.write('. file: ' + fname)
logf.write('. line: ' + str(exc_tb.tb_lineno))
logf.write('. type: ' + str(exc_type))
logf.write('\n ' + "".join(traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])))
2022-01-31 19:11:01 +00:00
def updateCrypto(api_key):
#cypto_info['symbol, base'].keys() = ['current','24hr change']
try:
f = open('csv/crypto_settings.json', 'r')
all_crypto_settings = json.load(f)
f.close()
coin_info = all_crypto_settings['symbols']
symbol_base = list(coin_info.keys())
symbols = [sb.split(',')[0] for sb in symbol_base]
bases = [sb.split(',')[1] for sb in symbol_base]
unique_bases = list(set(bases))
2022-01-29 09:26:55 +00:00
url = 'https://bm7p954xoh.execute-api.us-east-2.amazonaws.com/default/ScriptsAPI/crypto?symbols='
2022-01-29 09:26:55 +00:00
for i,s in enumerate(symbols):
url += bases[i] + '-' + s + ','
url = url[:-1] #remove last comma
2022-01-31 19:11:01 +00:00
url += '&apiKey=' + api_key
2022-01-29 09:26:55 +00:00
print(url)
response = requests.get(url)
data = response.json()
print(data)
coin_info = {}
2022-01-29 09:26:55 +00:00
for i,d in enumerate(data): #TODO get base from the server
symbol = d['symbol']
base = d['currency']
coin_info[symbol.upper() + ',' + base.upper()] = {'current': d['price'], '24hr_change': d['percent_over_24hr']}
2022-01-29 09:26:55 +00:00
all_crypto_settings['symbols'] = coin_info
json.dump(all_crypto_settings, open('csv/crypto_settings.json', 'w+'))
except Exception as e:
2022-01-31 19:11:01 +00:00
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
logf.write(str(e))
logf.write('. file: ' + fname)
logf.write('. line: ' + str(exc_tb.tb_lineno))
logf.write('. type: ' + str(exc_type))
logf.write('\n ' + "".join(traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])))
2022-01-31 19:11:01 +00:00
def updateForex(api_key):
try:
f = open('csv/forex_settings.json', 'r')
all_forex_settings = json.load(f)
f.close()
forex_info = all_forex_settings['symbols']
symbol_base = list(forex_info.keys())
symbols = [sb.split(',')[0] for sb in symbol_base]
bases = [sb.split(',')[1] for sb in symbol_base]
unique_bases = list(set(bases))
targets = ','.join(symbols)
2022-01-29 09:26:55 +00:00
url = 'https://bm7p954xoh.execute-api.us-east-2.amazonaws.com/default/ScriptsAPI/forex?symbols='
2022-01-29 09:26:55 +00:00
for i,s in enumerate(symbols):
url += s + '-' + bases[i] + ','
url = url[:-1] #remove last comma
2022-01-31 19:11:01 +00:00
url += '&apiKey=' + api_key
2022-01-29 09:26:55 +00:00
response = requests.get(url)
data = response.json()
2022-01-29 09:26:55 +00:00
c_dict = {}
2022-01-29 09:26:55 +00:00
for d in data:
c_dict[d['uid'].replace('/',',')] = {'current': d['rate'], '24hr_change': d['rate_over_24hr']}
all_forex_settings['symbols'] = c_dict
json.dump(all_forex_settings, open( "csv/forex_settings.json", 'w+' ))
except Exception as e:
2022-01-29 09:26:55 +00:00
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
logf.write(str(e))
logf.write('. file: ' + fname)
logf.write('. line: ' + str(exc_tb.tb_lineno))
logf.write('. type: ' + str(exc_type))
logf.write('\n ' + "".join(traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])))
2022-01-31 19:11:01 +00:00
def updateNews(api_key):
#'https://bm7p954xoh.execute-api.us-east-2.amazonaws.com/default/ScriptsAPI/news?category=technology'
#'https://bm7p954xoh.execute-api.us-east-2.amazonaws.com/default/ScriptsAPI/news?country=GB'
#'https://bm7p954xoh.execute-api.us-east-2.amazonaws.com/default/ScriptsAPI/news?lang=en'
max_per_cat = 10
try:
all_settings = json.load(open('csv/news_settings.json', 'r'))
2022-01-31 19:11:01 +00:00
2022-01-24 18:40:06 +00:00
2022-01-31 19:11:01 +00:00
url = 'https://bm7p954xoh.execute-api.us-east-2.amazonaws.com/default/ScriptsAPI/news?country={}'.format(all_settings['country'])
url += '&apiKey=' + api_key
response = requests.get(url)
data = response.json()
#load user settings
headlines = data
headline_sources = [headline['source'] for headline in headlines]
2022-01-24 18:40:06 +00:00
headline_titles = [headline['title'] for headline in headlines]
headline_times = [headline['publishedAt'] for headline in headlines]
headlines = list(zip(headline_titles, headline_sources, headline_times))
2022-01-24 18:40:06 +00:00
print(headlines)
all_settings['headlines'] = headlines
json.dump(all_settings, open('csv/news_settings.json', 'w+'))
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
logf.write(str(e))
logf.write('. file: ' + fname)
logf.write('. line: ' + str(exc_tb.tb_lineno))
logf.write('. type: ' + str(exc_type))
logf.write('\n ' + "".join(traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])))
2022-01-31 19:11:01 +00:00
def updateWeather(api_key):
max_cities = 30
2022-01-31 19:11:01 +00:00
try:
gn = geocoders.GeoNames(username='fintic')
f = open('csv/daily_weather.json', 'r')
all_daily_settings = json.load(f)
f.close()
f = open('csv/current_weather.json', 'r')
all_current_settings = json.load(f)
f.close()
current_locations = list(all_current_settings['locations'].keys())
daily_locations = list(all_daily_settings['locations'].keys())
all_locations = list(set(current_locations + daily_locations))
current_list = []
daily_list = []
for location in all_locations:
loc = gn.geocode(location)
current_weather = {}
lat = loc.latitude
lon = loc.longitude
url = 'https://api.openweathermap.org/data/2.5/onecall?lat={}&units=metric&lon={}&appid={}'.format(lat, lon, api_key)
r = requests.get(url)
weather = r.json()['current']
current_weather['main_weather'] = weather['weather'][0]['main']
current_weather['description'] = weather['weather'][0]['description']
current_weather['temp'] = weather['temp']
current_weather['min_temp'] = r.json()['daily'][0]['temp']['min']
current_weather['max_temp'] = r.json()['daily'][0]['temp']['max']
current_weather['feels_like'] = weather['feels_like']
current_weather['humidity'] = weather['humidity']
current_weather['clouds'] = weather['clouds']
current_weather['wind_speed'] = weather['wind_speed']
current_weather['wind_direction'] = weather['wind_deg']
current_weather['visibility'] = weather['visibility']
current_weather['uv'] = weather['uvi']
current_weather['rain_chance'] = r.json()['hourly'][0]['pop']
if location in current_locations:
current_list.append(current_weather)
daily_weather = []
daily = r.json()['daily']
for day in daily:
dct = {}
dct['main_weather'] = day['weather'][0]['main']
dct['description'] = day['weather'][0]['description']
dct['min_temp'] = day['temp']['min']
dct['max_temp'] = day['temp']['max']
daily_weather.append(dct)
#add relevant urrent information to first day in daily
daily_weather[0]['temp'] = weather['temp']
daily_weather[0]['rain_chance'] = current_weather['rain_chance']
daily_weather[0]['humidity'] = current_weather['humidity']
daily_weather[0]['wind_speed'] = current_weather['wind_speed']
daily_weather[0]['uv'] = current_weather['uv']
daily_weather[0]['clouds'] = current_weather['clouds']
daily_weather[0]['wind_speed'] = current_weather['wind_speed']
daily_weather[0]['wind_direction'] = current_weather['wind_direction']
daily_weather[0]['visibility'] = current_weather['visibility']
if location in daily_locations:
daily_list.append(daily_weather)
current_weathers = {}
daily_weathers = {}
for i,loc in enumerate(current_locations):
current_weathers[loc] = current_list[i]
for i,loc in enumerate(daily_locations):
daily_weathers[loc] = daily_list[i]
all_current_settings['locations'] = current_weathers
all_daily_settings['locations'] = daily_weathers
json.dump( all_current_settings, open( "csv/current_weather.json", 'w+' ))
json.dump( all_daily_settings, open( "csv/daily_weather.json", 'w+' ))
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
logf.write(str(e))
logf.write('. file: ' + fname)
logf.write('. line: ' + str(exc_tb.tb_lineno))
logf.write('. type: ' + str(exc_type))
logf.write('\n ' + "".join(traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])))
2022-01-31 19:11:01 +00:00
def updateLeagueTables(api_key):
2022-01-29 09:26:55 +00:00
url = 'https://bm7p954xoh.execute-api.us-east-2.amazonaws.com/default/ScriptsAPI/sports?stats='
try:
f = open('csv/league_tables.json', 'r')
all_settings = json.load(f)
f.close()
leagues = all_settings['leagues'].keys()
leagues_info = {}
2022-01-29 09:26:55 +00:00
for league in leagues:
2022-01-29 09:26:55 +00:00
url += league + ','
url = url[:-1] # remove last comma
2022-01-31 19:11:01 +00:00
url += '&apiKey=' + api_key
2022-01-29 09:26:55 +00:00
r = requests.get(url)
all_data = r.json()
for league in all_data[0].keys():
2022-01-29 09:26:55 +00:00
teams = []
2022-01-29 09:26:55 +00:00
for d in all_data[0][league]:
team = {}
2022-01-29 09:26:55 +00:00
team['name'] = d['strTeam']
team['wins'] = d['intWin']
team['loss'] = d['intLoss']
team['draw'] = d['intDraw']
#team['played'] = d['intPlayed']
team['standing'] = d['intRank']
#team['points'] = d['intPoints']
teams.append(team)
2022-01-29 09:26:55 +00:00
leagues_info[league.upper()] = teams
2022-01-31 19:11:01 +00:00
all_settings['leagues'] = leagues_info
json.dump(all_settings, open( "csv/league_tables.json".format(league), 'w+' ))
except Exception as e:
2022-01-31 19:11:01 +00:00
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
logf.write(str(e))
logf.write('. file: ' + fname)
logf.write('. line: ' + str(exc_tb.tb_lineno))
logf.write('. type: ' + str(exc_type))
logf.write('\n ' + "".join(traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])))
2022-01-31 19:11:01 +00:00
def updateLeagueEvents(api_key, time):
2022-01-29 09:26:55 +00:00
url = 'https://bm7p954xoh.execute-api.us-east-2.amazonaws.com/default/ScriptsAPI/sports?{}='.format(time)
if time == 'past':
2022-01-29 09:26:55 +00:00
f = open('csv/past_games.json')
elif time == 'upcoming':
2022-01-29 09:26:55 +00:00
f = open('csv/upcoming_games.json')
2022-01-29 09:26:55 +00:00
elif time == 'livescore':
f = open('csv/live_games.json')
2022-01-29 09:26:55 +00:00
2022-01-31 19:11:01 +00:00
try:
all_settings = json.load(f)
f.close()
leagues = all_settings['leagues'].keys()
leagues_info = {}
2022-01-29 09:26:55 +00:00
for league in leagues:
2022-01-29 09:26:55 +00:00
url += league + ','
url = url[:-1] # remove last comma
2022-01-31 19:11:01 +00:00
url += '&apiKey=' + api_key
2022-01-29 09:26:55 +00:00
r = requests.get(url)
all_data = r.json()
2022-01-31 19:11:01 +00:00
print(all_data)
2022-01-29 09:26:55 +00:00
events = []
for league in all_data.keys():
for d in all_data[league]:
event = {}
event['date'] = d['dateEvent']
2022-01-29 09:26:55 +00:00
if time == 'live':
event['progess'] = d['strProgress']
event['status'] = d['strStatus']
else:
2022-01-29 09:26:55 +00:00
event['round'] = d['intRound']
event['time'] = d['strTime']
event['home_team'] = d['strHomeTeam']
event['away_team'] = d['strAwayTeam']
if time != 'upcoming':
event['away_score'] = d['intAwayScore']
event['home_score'] = d['intHomeScore']
events.append(event)
leagues_info[league] = events
all_settings['leagues'] = leagues_info
json.dump(all_settings, open( "csv/{}_games.json".format(time), 'w+' ))
except Exception as e:
2022-01-29 09:26:55 +00:00
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
logf.write(str(e))
logf.write('. file: ' + fname)
logf.write('. line: ' + str(exc_tb.tb_lineno))
logf.write('. type: ' + str(exc_type))
logf.write('\n ' + "".join(traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])))
2022-01-31 19:11:01 +00:00
def updateSports(api_key):
#read user settings to decide which sprots to update
2022-01-29 09:26:55 +00:00
2022-01-31 19:11:01 +00:00
updateLeagueTables(api_key)
2022-01-31 19:11:01 +00:00
updateLeagueEvents(api_key,'livescore')
updateLeagueEvents(api_key,'past')
updateLeagueEvents(api_key,'upcoming')
def checkStocks(last_update, update_frequency):
NY_time = datetime.now(NY_zone).replace(tzinfo=None)
opening = NY_time.replace(hour=9, minute=30, second=0, microsecond=0).replace(tzinfo=None)
closing = NY_time.replace(hour=16, minute=0, second=0, microsecond=0).replace(tzinfo=None)
f = open('csv/stocks_settings.json', 'r')
all_stocks_settings = json.load(f)
f.close()
stock_info = all_stocks_settings['symbols']
symbols = list(stock_info.keys())
updated = False
diff = (NY_time - last_update).total_seconds()/60 #minutes
if opening < NY_time < closing and datetime.today().weekday() < 5: # we need to do real time updating
if diff >= update_frequency:
updated = True
2022-01-31 19:11:01 +00:00
elif emptyInfo(symbols, stock_info): # if theres any empty stocks
updated = True
2022-01-31 19:11:01 +00:00
else:
# update if last update was before the previous days closing
yday_closing = closing - dt.timedelta(days=1)
yday_str = yday_closing.strftime("%d/%m/%Y %H:%M:%S")
yday_closing = datetime.strptime(yday_str, "%d/%m/%Y %H:%M:%S")
if last_update < yday_closing:
updated = True
2022-01-31 19:11:01 +00:00
return updated
def updateAll(api_key, weather_key):
2022-01-31 19:11:01 +00:00
updateStocks(api_key)
updateCrypto(api_key)
updateForex(api_key)
updateNews(api_key)
updateSports(api_key)
2022-02-21 17:18:45 +00:00
if weather_key:
updateWeather(weather_key)
if __name__ == '__main__':
logf = open("log.txt", "a")
t = time.time()
2022-01-31 19:11:01 +00:00
update_frequencies = {'stocks':1, 'crypto':5, 'forex':60, 'news':120, 'weather': 120, 'sports': 1440} #minutes
NY_zone = pytz.timezone('America/New_York')
CET_zone = pytz.timezone('Europe/Berlin')
NY_time = datetime.now(NY_zone)
CET_time = datetime.now(CET_zone)
NY_str = NY_time.strftime("%d/%m/%Y %H:%M:%S")
CET_str = NY_time.strftime("%d/%m/%Y %H:%M:%S")
#f = open('csv/last_updates.json', 'w+')
#update_times = {'stocks':NY_str, 'crypto':NY_str, 'news':NY_str, 'weather': NY_str, 'forex': CET_str} # all in NY time apart from forex in CET
#json.dump(update_times, f)
#f.close()
2022-01-31 19:11:01 +00:00
f = open('api_keys.txt')
api_keys = f.readlines()
api_key = api_keys[0].strip()
try:
weather_key = api_keys[1].strip()
except Exception as e:
2022-02-21 17:18:45 +00:00
weather_key = False
2022-01-31 19:11:01 +00:00
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
logf.write(str(e))
logf.write('. file: ' + fname)
logf.write('. line: ' + str(exc_tb.tb_lineno))
logf.write('. type: ' + str(exc_type))
logf.write('\n ' + "".join(traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])))
try:
f = open('csv/last_updates.json', 'r')
last_updates = json.load(f)
f.close()
except:
last_updates = {"stocks": "27/06/2021 07:05:39", "crypto": "27/06/2021 07:05:39", "news": "27/06/2021 07:05:39", "weather": "27/06/2021 07:05:39", "forex": "27/06/2021 07:05:39", "sports": "27/06/2021 07:05:39"}
t = time.time()
try:
while True:
NY_time = datetime.now(NY_zone).replace(tzinfo=None)
msg = getInput()
if msg == 'A':
updateAll(api_key, weather_key)
#stocks
stock_time = datetime.strptime(last_updates['stocks'], "%d/%m/%Y %H:%M:%S")
stock_frequency = update_frequencies['stocks']
if checkStocks(stock_time, stock_frequency) or msg == 's':
stock_time = NY_time.strftime("%d/%m/%Y %H:%M:%S")
last_updates['stocks'] = stock_time
2022-01-31 19:11:01 +00:00
updateStocks(api_key)
# crypto
crypto_time = datetime.strptime(last_updates['crypto'], "%d/%m/%Y %H:%M:%S")
NY_time = datetime.now(NY_zone).replace(tzinfo=None)
diff = (NY_time - crypto_time).total_seconds()/60 #minutes
if diff >= update_frequencies['crypto'] or msg == 'c':
crypto_time = NY_time.strftime("%d/%m/%Y %H:%M:%S")
2022-01-31 19:11:01 +00:00
updateCrypto(api_key)
last_updates['crypto'] = crypto_time
# weather
weather_time = datetime.strptime(last_updates['weather'], "%d/%m/%Y %H:%M:%S")
NY_time = datetime.now(NY_zone).replace(tzinfo=None)
diff = (NY_time - weather_time).total_seconds()/60 #minutes
if diff >= update_frequencies['weather'] or msg == 'w':
weather_time = NY_time.strftime("%d/%m/%Y %H:%M:%S")
2022-01-31 19:11:01 +00:00
updateWeather(weather_key)
last_updates['weather'] = weather_time
# news
news_time = datetime.strptime(last_updates['news'], "%d/%m/%Y %H:%M:%S")
NY_time = datetime.now(NY_zone).replace(tzinfo=None)
diff = (NY_time - news_time).total_seconds()/60 #minutes
if diff >= update_frequencies['news'] or msg == 'n':
news_time = NY_time.strftime("%d/%m/%Y %H:%M:%S")
2022-01-31 19:11:01 +00:00
updateNews(api_key)
last_updates['news'] = news_time
# sports
sports_time = datetime.strptime(last_updates['sports'], "%d/%m/%Y %H:%M:%S")
NY_time = datetime.now(NY_zone).replace(tzinfo=None)
diff = (NY_time - sports_time).total_seconds()/60 #minutes
if diff >= update_frequencies['sports'] or msg == 'S':
sports_time = NY_time.strftime("%d/%m/%Y %H:%M:%S")
2022-01-31 19:11:01 +00:00
updateSports(api_key)
last_updates['sports'] = sports_time
#forex updates once every 24hours at 1700 CET
# update if last update was before the previous days closing
forex_time = datetime.strptime(last_updates['forex'], "%d/%m/%Y %H:%M:%S")
CET_time = datetime.now(CET_zone)
yday_update = (CET_time.replace(hour=17, minute=00, second=0, microsecond=0) - dt.timedelta(days=1)).replace(tzinfo=None)
diff = (CET_time.replace(tzinfo=None) - forex_time).total_seconds()/60
opening = CET_time.replace(hour=17, minute=0, second=0, microsecond=0).replace(tzinfo=None)
#forex updates between 5pm sunday and 5pm friday every hour
forex_open = datetime.today().weekday() < 4 or (datetime.today().weekday() == 6 and CET_time > opening) or (datetime.today().weekday() == 4 and CET_time < opening)
if forex_time < yday_update or msg == 'f' or (diff >= update_frequencies['forex'] and forex_open):
forex_time = CET_time.strftime("%d/%m/%Y %H:%M:%S")
last_updates['forex'] = forex_time
2022-01-31 19:11:01 +00:00
updateForex(api_key)
json.dump(last_updates, open('csv/last_updates.json', 'w+'))
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
logf.write(str(e))
logf.write('. file: ' + fname)
logf.write('. line: ' + str(exc_tb.tb_lineno))
logf.write('. type: ' + str(exc_type))
logf.write('\n ' + "".join(traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])))