database caller new method

This commit is contained in:
Neythen Treloar 2022-03-06 09:29:42 +00:00
parent b6646f60cb
commit 37f6c097f3
5 changed files with 130 additions and 84 deletions

View File

@ -701,7 +701,8 @@ if __name__ == '__main__':
#stocks
stock_time = datetime.strptime(last_updates['stocks'], "%d/%m/%Y %H:%M:%S")
stock_frequency = update_frequencies['stocks']
if checkStocks(stock_time, stock_frequency) or msg == 's':
diff = (NY_time - stock_time).total_seconds()/60 #minutes
if diff >= update_frequencies['stocks'] or msg == 's':
stock_time = NY_time.strftime("%d/%m/%Y %H:%M:%S")
last_updates['stocks'] = stock_time
updateStocks()

View File

@ -45,7 +45,7 @@ def updateUpdate(NY_time):
def updateStocks(api_key, logf):
print('STOCKS')
try:
@ -56,7 +56,7 @@ def updateStocks(api_key, logf):
f.close()
stock_info = all_stocks_settings['symbols']
symbols = list(stock_info.keys())
print(symbols)
#print(symbols)
url = 'https://bm7p954xoh.execute-api.us-east-2.amazonaws.com/default/ScriptsAPI/stocks?symbols='
@ -66,9 +66,9 @@ def updateStocks(api_key, logf):
url += '&apiKey=' + api_key
response = requests.get(url)
data = response.json()
print(response)
print(dir(response))
print(len(data))
#print(response)
#print(dir(response))
#print(len(data))
stock_info = {}
if len(data) > 0:
@ -97,7 +97,7 @@ def updateStocks(api_key, logf):
def updateCrypto(api_key, logf):
print('CRYPTO')
#cypto_info['symbol, base'].keys() = ['current','24hr change']
try:
@ -119,11 +119,11 @@ def updateCrypto(api_key, logf):
url += bases[i] + '-' + s + ','
url = url[:-1] #remove last comma
url += '&apiKey=' + api_key
print(url)
#print(url)
response = requests.get(url)
data = response.json()
print(url)
print(data)
#print(url)
#print(data)
coin_info = {}
@ -155,7 +155,7 @@ def updateCrypto(api_key, logf):
def updateForex(api_key, logf):
print('FOREX')
try:
f = open('csv/forex_settings.json', 'r')
@ -185,7 +185,7 @@ def updateForex(api_key, logf):
if len(data) > 0:
print(data)
#print(data)
c_dict = {}
for sb in symbol_base:
for d in data:
@ -214,7 +214,7 @@ def updateNews(api_key, logf):
#'https://bm7p954xoh.execute-api.us-east-2.amazonaws.com/default/ScriptsAPI/news?country=GB'
#'https://bm7p954xoh.execute-api.us-east-2.amazonaws.com/default/ScriptsAPI/news?lang=en'
print('NEWS')
try:
all_settings = json.load(open('csv/news_settings.json', 'r'))
@ -234,7 +234,7 @@ def updateNews(api_key, logf):
url += '&apiKey=' + api_key
response = requests.get(url)
data = response.json()
print(data)
#print(data)
if len(data) > 0:
max_headlines = int(all_settings['num_headlines'])
#load user settings
@ -247,7 +247,7 @@ def updateNews(api_key, logf):
headline_times = [headline['publishedAt'] for headline in headlines]
headlines = list(zip(headline_titles, headline_sources, headline_times))
print(headlines)
#print(headlines)
all_settings['headlines'] = headlines
json.dump(all_settings, open('csv/news_settings.json', 'w+'))
@ -266,6 +266,7 @@ def updateNews(api_key, logf):
def updateWeather(api_key, logf):
print('WEATHER')
max_cities = 30
try:
@ -351,7 +352,7 @@ def updateWeather(api_key, logf):
all_current_settings['locations'] = current_weathers
all_daily_settings['locations'] = daily_weathers
print(all_current_settings, all_daily_settings)
#print(all_current_settings, all_daily_settings)
json.dump( all_current_settings, open( "csv/current_weather.json", 'w+' ))
json.dump( all_daily_settings, open( "csv/daily_weather.json", 'w+' ))
@ -366,12 +367,12 @@ def updateWeather(api_key, logf):
def updateLeagueTables(api_key, logf):
print('LEAGUE TABLES')
url = 'https://bm7p954xoh.execute-api.us-east-2.amazonaws.com/default/ScriptsAPI/sports?stats='
try:
f = open('csv/league_tables.json', 'r')
all_settings = json.load(f)
print(all_settings['leagues'].keys())
#print(all_settings['leagues'].keys())
f.close()
leagues = all_settings['leagues'].keys()
@ -390,12 +391,12 @@ def updateLeagueTables(api_key, logf):
all_data = r.json()
print('key', [all_data[i].keys() for i in range(len(all_data))])
#print('key', [all_data[i].keys() for i in range(len(all_data))])
#print('key', all_data.keys())
for i,l in enumerate(all_data):
print(l)
#print(l)
league = list(l.keys())[0]
print('league', league, list(l.keys()))
#print('league', league, list(l.keys()))
teams = []
@ -417,7 +418,7 @@ def updateLeagueTables(api_key, logf):
leagues_info[league.upper()] = teams
all_settings['leagues'] = leagues_info
print(all_settings['leagues'].keys())
#print(all_settings['leagues'].keys())
json.dump(all_settings, open( "csv/league_tables.json".format(league), 'w+' ))
except Exception as e:
@ -431,7 +432,7 @@ def updateLeagueTables(api_key, logf):
def updateLeagueEvents(api_key, time, logf):
print('LEAGUE EVENTS')
url = 'https://bm7p954xoh.execute-api.us-east-2.amazonaws.com/default/ScriptsAPI/sports?{}='.format(time)
if time == 'past':
@ -448,7 +449,7 @@ def updateLeagueEvents(api_key, time, logf):
try:
all_settings = json.load(f)
print(all_settings['leagues'].keys())
#print(all_settings['leagues'].keys())
f.close()
leagues = all_settings['leagues'].keys()
leagues_info = {}
@ -495,7 +496,7 @@ def updateLeagueEvents(api_key, time, logf):
events.append(event)
leagues_info[league.upper()] = events
all_settings['leagues'] = leagues_info
print(all_settings['leagues'].keys())
#print(all_settings['leagues'].keys())
json.dump(all_settings, open( "csv/{}_games.json".format(time), 'w+' ))
except Exception as e:
@ -567,7 +568,7 @@ def updateAll(api_key, weather_key, logf):
updateCrypto(api_key, logf)
#updateForex(api_key, logf)
updateForex(api_key, logf)
updateNews(api_key, logf)
@ -586,7 +587,7 @@ if __name__ == '__main__':
update_frequencies = {'stocks':1, 'crypto':1, 'forex':1, 'news':1, 'weather': 1, 'sports': 1} #minutes
update_frequencies = {'stocks':1, 'crypto':5, 'forex':60, 'news':60, 'weather': 120, 'sports': 1440} #minutes
NY_zone = pytz.timezone('America/New_York')
CET_zone = pytz.timezone('EST')
@ -621,107 +622,112 @@ if __name__ == '__main__':
logf.write('. type: ' + str(exc_type))
logf.write('\n ' + "".join(traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])))
try:
f = open('csv/last_updates.json', 'r')
last_updates = json.load(f)
f.close()
except:
last_updates = {"stocks": "27/06/2021 07:05:39", "crypto": "27/06/2021 07:05:39", "news": "27/06/2021 07:05:39", "weather": "27/06/2021 07:05:39", "forex": "27/06/2021 07:05:39", "sports": "27/06/2021 07:05:39"}
t = time.time()
update_processes = []
try:
while True:
try:
f = open('csv/last_updates.json', 'r')
last_updates = json.load(f)
f.close()
except:
last_updates = {"stocks": {True, "27/06/2021 07:05:39"}, "crypto": {True, "27/06/2021 07:05:39"}, "news": {True, "27/06/2021 07:05:39"}, "weather": {True, "27/06/2021 07:05:39"},"forex": {True, "27/06/2021 07:05:39"}, "sports": {True, "27/06/2021 07:05:39"}}
NY_time = datetime.now(NY_zone).replace(tzinfo=None)
msg = getInput()
if msg == 'A':
update_process = Process(target = updateAll, args = (api_key,weather_key, logf))
update_process.start()
update_processes.append(update_process)
stock_time = NY_time.strftime("%d/%m/%Y %H:%M:%S")
last_updates['stocks'] = stock_time
last_updates['crypto'] = stock_time
last_updates['weather'] = stock_time
last_updates['news'] = stock_time
last_updates['sports'] = stock_time
CET_time = datetime.now(CET_zone)
forex_time = CET_time.strftime("%d/%m/%Y %H:%M:%S")
last_updates['forex'] = forex_time
#msg = getInput()
NY_time = datetime.now(NY_zone).replace(tzinfo=None)
#stocks
stock_time = datetime.strptime(last_updates['stocks'], "%d/%m/%Y %H:%M:%S")
stock_time = datetime.strptime(last_updates['stocks']['time'], "%d/%m/%Y %H:%M:%S")
stock_frequency = update_frequencies['stocks']
diff = (NY_time - stock_time).total_seconds()/60 #minutes
if diff >= update_frequencies['stocks'] or msg == 's':
if last_updates['stocks']['force'] or diff >= update_frequencies['stocks']:# or msg == 's':
stock_time = NY_time.strftime("%d/%m/%Y %H:%M:%S")
last_updates['stocks'] = stock_time
last_updates['stocks']['time'] = stock_time
last_updates['stocks']['force'] = False
#updateStocks(api_key)
update_process = Process(target = updateStocks, args = (api_key,logf))
update_process.start()
update_processes.append(update_process)
# crypto
crypto_time = datetime.strptime(last_updates['crypto'], "%d/%m/%Y %H:%M:%S")
crypto_time = datetime.strptime(last_updates['crypto']['time'], "%d/%m/%Y %H:%M:%S")
NY_time = datetime.now(NY_zone).replace(tzinfo=None)
diff = (NY_time - crypto_time).total_seconds()/60 #minutes
if diff >= update_frequencies['crypto'] or msg == 'c':
if last_updates['crypto']['force'] or diff >= update_frequencies['crypto']:# or msg == 'c':
crypto_time = NY_time.strftime("%d/%m/%Y %H:%M:%S")
updateCrypto(api_key, logf)
print('UPDATING CRYPTO')
#updateCrypto(api_key, logf)
last_updates['crypto'] = crypto_time
#update_process = Process(target = updateCrypto, args = (api_key,logf))
#update_process.start()
#update_processes.append(update_process)
last_updates['crypto']['time'] = crypto_time
last_updates['crypto']['force'] = False
update_process = Process(target = updateCrypto, args = (api_key,logf))
update_process.start()
update_processes.append(update_process)
# weather
weather_time = datetime.strptime(last_updates['weather'], "%d/%m/%Y %H:%M:%S")
weather_time = datetime.strptime(last_updates['weather']['time'], "%d/%m/%Y %H:%M:%S")
NY_time = datetime.now(NY_zone).replace(tzinfo=None)
diff = (NY_time - weather_time).total_seconds()/60 #minutes
if diff >= update_frequencies['weather'] or msg == 'w':
if last_updates['weather']['force'] or diff >= update_frequencies['weather']:# or msg == 'w':
weather_time = NY_time.strftime("%d/%m/%Y %H:%M:%S")
#updateWeather(weather_key)
last_updates['weather'] = weather_time
last_updates['weather']['time'] = weather_time
last_updates['weather']['force'] = False
update_process = Process(target = updateWeather, args = (weather_key,logf))
update_process.start()
update_processes.append(update_process)
# news
news_time = datetime.strptime(last_updates['news'], "%d/%m/%Y %H:%M:%S")
news_time = datetime.strptime(last_updates['news']['time'], "%d/%m/%Y %H:%M:%S")
NY_time = datetime.now(NY_zone).replace(tzinfo=None)
diff = (NY_time - news_time).total_seconds()/60 #minutes
if diff >= update_frequencies['news'] or msg == 'n':
if last_updates['news']['force'] or diff >= update_frequencies['news']:# or msg == 'n':
news_time = NY_time.strftime("%d/%m/%Y %H:%M:%S")
#updateNews(api_key)
last_updates['news'] = news_time
last_updates['news']['time'] = news_time
last_updates['news']['force'] = False
update_process = Process(target = updateNews, args = (api_key,logf))
update_process.start()
update_processes.append(update_process)
# sports
sports_time = datetime.strptime(last_updates['sports'], "%d/%m/%Y %H:%M:%S")
sports_time = datetime.strptime(last_updates['sports']['time'], "%d/%m/%Y %H:%M:%S")
NY_time = datetime.now(NY_zone).replace(tzinfo=None)
diff = (NY_time - sports_time).total_seconds()/60 #minutes
if diff >= update_frequencies['sports'] or msg == 'S':
if last_updates['sports']['force'] or diff >= update_frequencies['sports']:# or msg == 'S':
sports_time = NY_time.strftime("%d/%m/%Y %H:%M:%S")
#updateSports(api_key)
last_updates['sports'] = sports_time
last_updates['sports']['time'] = sports_time
last_updates['sports']['force'] = False
update_process = Process(target = updateSports, args = (api_key,logf))
update_process.start()
update_processes.append(update_process)
@ -729,7 +735,7 @@ if __name__ == '__main__':
#forex updates once every 24hours at 1700 CET
# update if last update was before the previous days closing
forex_time = datetime.strptime(last_updates['forex'], "%d/%m/%Y %H:%M:%S")
forex_time = datetime.strptime(last_updates['forex']['time'], "%d/%m/%Y %H:%M:%S")
CET_time = datetime.now(CET_zone).replace(tzinfo=None)
yday_update = (CET_time.replace(hour=17, minute=00, second=0, microsecond=0) - dt.timedelta(days=1)).replace(tzinfo=None)
diff = (CET_time.replace(tzinfo=None) - forex_time).total_seconds()/60
@ -739,25 +745,29 @@ if __name__ == '__main__':
#forex updates between 5pm sunday and 5pm friday every hour
forex_open = datetime.today().weekday() < 4 or (datetime.today().weekday() == 6 and CET_time > opening) or (datetime.today().weekday() == 4 and CET_time < opening)
if forex_time < yday_update or msg == 'f' or (diff >= update_frequencies['forex'] and forex_open):
if last_updates['forex']['force'] or (diff >= update_frequencies['forex'] and forex_open):# or msg == 'f':
forex_time = CET_time.strftime("%d/%m/%Y %H:%M:%S")
last_updates['forex'] = forex_time
last_updates['forex']['time'] = forex_time
last_updates['forex']['force'] = False
#updateForex(api_key)
update_process = Process(target = updateForex, args = (api_key,logf))
update_process.start()
update_processes.append(update_process)
json.dump(last_updates, open('csv/last_updates.json', 'w+'))
f = open('csv/last_updates.json', 'w+')
json.dump(last_updates, f)
f.close()
for process in update_processes:
if not process.is_alive():
process.join()
process.terminate()
update_processes.remove(process)
print('Number active processes', len(update_processes))
time.sleep(10)
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
logf.write(str(e))

View File

@ -648,7 +648,8 @@ if __name__ == '__main__':
#stocks
stock_time = datetime.strptime(last_updates['stocks'], "%d/%m/%Y %H:%M:%S")
stock_frequency = update_frequencies['stocks']
if checkStocks(stock_time, stock_frequency) or msg == 's':
diff = (NY_time - stock_time).total_seconds()/60 #minutes
if diff >= update_frequencies['stocks'] or msg == 's':
stock_time = NY_time.strftime("%d/%m/%Y %H:%M:%S")
last_updates['stocks'] = stock_time
updateStocks(api_key, logf)

View File

@ -29,7 +29,7 @@ import sys
#api_caller = pexpect.spawn("sudo -E taskset -c 3 python3 api_caller.py")
api_caller = pexpect.spawn("sudo -E taskset -c 3 python3 database_caller2.py")
api_caller = pexpect.spawn("sudo -E taskset -c 3 python3 database_caller.py")
time.sleep(3)
@ -573,14 +573,21 @@ def save_trade_settings(input_settings):
f = open('csv/' + filename, 'w+')
json.dump(current_settings, f)
f.close()
f = open('csv/last_updates.json', 'r')
last_updates = json.load(f)
f.close()
if input_settings['feature'].lower() == 'stocks':
api_caller.sendline('s')
last_updates['stocks']['force'] = True
elif input_settings['feature'].lower() == 'crypto':
api_caller.sendline('c')
last_updates['crypto']['force'] = True
elif input_settings['feature'].lower() == 'forex':
api_caller.sendline('f')
last_updates['forex']['force'] = True
f = open('csv/last_updates.json', 'w+')
json.dump(last_updates, f)
f.close()
def save_weather_settings(input_settings):
@ -613,7 +620,16 @@ def save_weather_settings(input_settings):
current_settings = combine_dict(current_settings, input_settings['locations'], 'locations')
json.dump(current_settings, open('csv/' + filename, 'w+'))
api_caller.sendline('w')
#api_caller.sendline('w')
f = open('csv/last_updates.json', 'r')
last_updates = json.load(f)
f.close()
last_updates['weather']['force'] = True
f = open('csv/last_updates.json', 'w+')
json.dump(last_updates, f)
f.close()
def save_news_settings(input_settings):
filename = 'news_settings.json'
@ -636,7 +652,16 @@ def save_news_settings(input_settings):
print(current_settings)
json.dump(current_settings, open('csv/' + filename, 'w+'))
api_caller.sendline('n')
#api_caller.sendline('n')
f = open('csv/last_updates.json', 'r')
last_updates = json.load(f)
f.close()
last_updates['news']['force'] = True
f = open('csv/last_updates.json', 'w+')
json.dump(last_updates, f)
f.close()
def save_sports_settings(input_settings):
@ -662,7 +687,16 @@ def save_sports_settings(input_settings):
json.dump(current_settings, open('csv/' + filename, 'w+'))
api_caller.sendline('S')
#api_caller.sendline('S')
f = open('csv/last_updates.json', 'r')
last_updates = json.load(f)
f.close()
last_updates['sports']['force'] = True
f = open('csv/last_updates.json', 'w+')
json.dump(last_updates, f)
f.close()
# for images and GIFs
def save_image_settings(input_settings):

View File

@ -10,7 +10,7 @@ mkdir user_uploads
mkdir csv
cd csv
rm *.json
touch coin_list.json crypto_settings.json last_updates.json system_info.json
touch crypto_settings.json last_updates.json system_info.json last_updates.json
touch league_tables.json
mkdir sports
touch current_weather.json stocks_settings.json daily_weather.json live_games.json
@ -21,7 +21,7 @@ filenames="crypto_settings.json last_updates.json league_tables.json current_wea
echo '{"update_available": false, "first_boot": true}' >> system_info.json
echo [\"Standard\", [[\"Stocks\", \"Crypto\", \"Forex\"]]] >> display_settings.json
echo '{"stocks": {"time": "06/03/2022 04:12:09", "force": true}, "crypto": {"time": "06/03/2022 04:10:39", "force": true}, "news": {"time": "06/03/2022 04:07:09", "force": true}, "weather": {"time": "06/03/2022 04:08:20", "force": true}, "forex": {"time": "06/03/2022 03:54:02", "force": true}, "sports": {"time": "06/03/2022 04:10:09", "force": true}}' >> last_updates.json
echo '{"feature": "Stocks", "speed": "medium", "animation": "down", "percent": false, "point": true, "logos": true, "chart": false, "title": true, "symbols": {"ETH,USD": {"current": "2629.32", "24hr_change": "-27.6432", "percent_change": "-1.04"}, "BTC,USD": {"current": "38161.00", "24hr_change": "-50.8386", "percent_change": "-0.13"}, "BNB,USD": {"current": "372.57", "24hr_change": "0.4140", "percent_change": "0.11"}, "ADA,BTC": {"current": "0.0000", "24hr_change": "-0.0000", "percent_change": "-3.74"}}}' >> crypto_settings.json