From 37f6c097f3f9d3e475cfd2112ed4ea627c4b3930 Mon Sep 17 00:00:00 2001 From: Neythen Treloar Date: Sun, 6 Mar 2022 09:29:42 +0000 Subject: [PATCH] database caller new method --- api_caller.py | 3 +- database_caller.py | 156 ++++++++++++++++++++++-------------------- database_caller2.py | 3 +- server.py | 48 +++++++++++-- setup_config_files.sh | 4 +- 5 files changed, 130 insertions(+), 84 deletions(-) diff --git a/api_caller.py b/api_caller.py index a3bc50c..f165f7c 100755 --- a/api_caller.py +++ b/api_caller.py @@ -701,7 +701,8 @@ if __name__ == '__main__': #stocks stock_time = datetime.strptime(last_updates['stocks'], "%d/%m/%Y %H:%M:%S") stock_frequency = update_frequencies['stocks'] - if checkStocks(stock_time, stock_frequency) or msg == 's': + diff = (NY_time - stock_time).total_seconds()/60 #minutes + if diff >= update_frequencies['stocks'] or msg == 's': stock_time = NY_time.strftime("%d/%m/%Y %H:%M:%S") last_updates['stocks'] = stock_time updateStocks() diff --git a/database_caller.py b/database_caller.py index 0411e00..1baf394 100755 --- a/database_caller.py +++ b/database_caller.py @@ -45,7 +45,7 @@ def updateUpdate(NY_time): def updateStocks(api_key, logf): - + print('STOCKS') try: @@ -56,7 +56,7 @@ def updateStocks(api_key, logf): f.close() stock_info = all_stocks_settings['symbols'] symbols = list(stock_info.keys()) - print(symbols) + #print(symbols) url = 'https://bm7p954xoh.execute-api.us-east-2.amazonaws.com/default/ScriptsAPI/stocks?symbols=' @@ -66,9 +66,9 @@ def updateStocks(api_key, logf): url += '&apiKey=' + api_key response = requests.get(url) data = response.json() - print(response) - print(dir(response)) - print(len(data)) + #print(response) + #print(dir(response)) + #print(len(data)) stock_info = {} if len(data) > 0: @@ -97,7 +97,7 @@ def updateStocks(api_key, logf): def updateCrypto(api_key, logf): - + print('CRYPTO') #cypto_info['symbol, base'].keys() = ['current','24hr change'] try: @@ -119,11 +119,11 @@ def updateCrypto(api_key, logf): url += bases[i] + '-' + s + ',' url = url[:-1] #remove last comma url += '&apiKey=' + api_key - print(url) + #print(url) response = requests.get(url) data = response.json() - print(url) - print(data) + #print(url) + #print(data) coin_info = {} @@ -155,7 +155,7 @@ def updateCrypto(api_key, logf): def updateForex(api_key, logf): - + print('FOREX') try: f = open('csv/forex_settings.json', 'r') @@ -185,7 +185,7 @@ def updateForex(api_key, logf): if len(data) > 0: - print(data) + #print(data) c_dict = {} for sb in symbol_base: for d in data: @@ -214,7 +214,7 @@ def updateNews(api_key, logf): #'https://bm7p954xoh.execute-api.us-east-2.amazonaws.com/default/ScriptsAPI/news?country=GB' #'https://bm7p954xoh.execute-api.us-east-2.amazonaws.com/default/ScriptsAPI/news?lang=en' - + print('NEWS') try: all_settings = json.load(open('csv/news_settings.json', 'r')) @@ -234,7 +234,7 @@ def updateNews(api_key, logf): url += '&apiKey=' + api_key response = requests.get(url) data = response.json() - print(data) + #print(data) if len(data) > 0: max_headlines = int(all_settings['num_headlines']) #load user settings @@ -247,7 +247,7 @@ def updateNews(api_key, logf): headline_times = [headline['publishedAt'] for headline in headlines] headlines = list(zip(headline_titles, headline_sources, headline_times)) - print(headlines) + #print(headlines) all_settings['headlines'] = headlines json.dump(all_settings, open('csv/news_settings.json', 'w+')) @@ -266,6 +266,7 @@ def updateNews(api_key, logf): def updateWeather(api_key, logf): + print('WEATHER') max_cities = 30 try: @@ -351,7 +352,7 @@ def updateWeather(api_key, logf): all_current_settings['locations'] = current_weathers all_daily_settings['locations'] = daily_weathers - print(all_current_settings, all_daily_settings) + #print(all_current_settings, all_daily_settings) json.dump( all_current_settings, open( "csv/current_weather.json", 'w+' )) json.dump( all_daily_settings, open( "csv/daily_weather.json", 'w+' )) @@ -366,12 +367,12 @@ def updateWeather(api_key, logf): def updateLeagueTables(api_key, logf): - + print('LEAGUE TABLES') url = 'https://bm7p954xoh.execute-api.us-east-2.amazonaws.com/default/ScriptsAPI/sports?stats=' try: f = open('csv/league_tables.json', 'r') all_settings = json.load(f) - print(all_settings['leagues'].keys()) + #print(all_settings['leagues'].keys()) f.close() leagues = all_settings['leagues'].keys() @@ -390,12 +391,12 @@ def updateLeagueTables(api_key, logf): all_data = r.json() - print('key', [all_data[i].keys() for i in range(len(all_data))]) + #print('key', [all_data[i].keys() for i in range(len(all_data))]) #print('key', all_data.keys()) for i,l in enumerate(all_data): - print(l) + #print(l) league = list(l.keys())[0] - print('league', league, list(l.keys())) + #print('league', league, list(l.keys())) teams = [] @@ -417,7 +418,7 @@ def updateLeagueTables(api_key, logf): leagues_info[league.upper()] = teams all_settings['leagues'] = leagues_info - print(all_settings['leagues'].keys()) + #print(all_settings['leagues'].keys()) json.dump(all_settings, open( "csv/league_tables.json".format(league), 'w+' )) except Exception as e: @@ -431,7 +432,7 @@ def updateLeagueTables(api_key, logf): def updateLeagueEvents(api_key, time, logf): - + print('LEAGUE EVENTS') url = 'https://bm7p954xoh.execute-api.us-east-2.amazonaws.com/default/ScriptsAPI/sports?{}='.format(time) if time == 'past': @@ -448,7 +449,7 @@ def updateLeagueEvents(api_key, time, logf): try: all_settings = json.load(f) - print(all_settings['leagues'].keys()) + #print(all_settings['leagues'].keys()) f.close() leagues = all_settings['leagues'].keys() leagues_info = {} @@ -495,7 +496,7 @@ def updateLeagueEvents(api_key, time, logf): events.append(event) leagues_info[league.upper()] = events all_settings['leagues'] = leagues_info - print(all_settings['leagues'].keys()) + #print(all_settings['leagues'].keys()) json.dump(all_settings, open( "csv/{}_games.json".format(time), 'w+' )) except Exception as e: @@ -567,7 +568,7 @@ def updateAll(api_key, weather_key, logf): updateCrypto(api_key, logf) - #updateForex(api_key, logf) + updateForex(api_key, logf) updateNews(api_key, logf) @@ -586,7 +587,7 @@ if __name__ == '__main__': - update_frequencies = {'stocks':1, 'crypto':1, 'forex':1, 'news':1, 'weather': 1, 'sports': 1} #minutes + update_frequencies = {'stocks':1, 'crypto':5, 'forex':60, 'news':60, 'weather': 120, 'sports': 1440} #minutes NY_zone = pytz.timezone('America/New_York') CET_zone = pytz.timezone('EST') @@ -621,107 +622,112 @@ if __name__ == '__main__': logf.write('. type: ' + str(exc_type)) logf.write('\n ' + "".join(traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]))) - try: - f = open('csv/last_updates.json', 'r') - last_updates = json.load(f) - f.close() - - except: - last_updates = {"stocks": "27/06/2021 07:05:39", "crypto": "27/06/2021 07:05:39", "news": "27/06/2021 07:05:39", "weather": "27/06/2021 07:05:39", "forex": "27/06/2021 07:05:39", "sports": "27/06/2021 07:05:39"} + + t = time.time() update_processes = [] + + + + + try: while True: + try: + f = open('csv/last_updates.json', 'r') + last_updates = json.load(f) + f.close() + + except: + last_updates = {"stocks": {True, "27/06/2021 07:05:39"}, "crypto": {True, "27/06/2021 07:05:39"}, "news": {True, "27/06/2021 07:05:39"}, "weather": {True, "27/06/2021 07:05:39"},"forex": {True, "27/06/2021 07:05:39"}, "sports": {True, "27/06/2021 07:05:39"}} + + + NY_time = datetime.now(NY_zone).replace(tzinfo=None) - msg = getInput() - if msg == 'A': - update_process = Process(target = updateAll, args = (api_key,weather_key, logf)) - update_process.start() - update_processes.append(update_process) - stock_time = NY_time.strftime("%d/%m/%Y %H:%M:%S") - last_updates['stocks'] = stock_time - last_updates['crypto'] = stock_time - last_updates['weather'] = stock_time - last_updates['news'] = stock_time - last_updates['sports'] = stock_time - CET_time = datetime.now(CET_zone) - forex_time = CET_time.strftime("%d/%m/%Y %H:%M:%S") - last_updates['forex'] = forex_time + #msg = getInput() + + NY_time = datetime.now(NY_zone).replace(tzinfo=None) #stocks - stock_time = datetime.strptime(last_updates['stocks'], "%d/%m/%Y %H:%M:%S") + + stock_time = datetime.strptime(last_updates['stocks']['time'], "%d/%m/%Y %H:%M:%S") stock_frequency = update_frequencies['stocks'] diff = (NY_time - stock_time).total_seconds()/60 #minutes - if diff >= update_frequencies['stocks'] or msg == 's': + if last_updates['stocks']['force'] or diff >= update_frequencies['stocks']:# or msg == 's': stock_time = NY_time.strftime("%d/%m/%Y %H:%M:%S") - last_updates['stocks'] = stock_time + last_updates['stocks']['time'] = stock_time + last_updates['stocks']['force'] = False #updateStocks(api_key) update_process = Process(target = updateStocks, args = (api_key,logf)) update_process.start() update_processes.append(update_process) # crypto - crypto_time = datetime.strptime(last_updates['crypto'], "%d/%m/%Y %H:%M:%S") + crypto_time = datetime.strptime(last_updates['crypto']['time'], "%d/%m/%Y %H:%M:%S") NY_time = datetime.now(NY_zone).replace(tzinfo=None) diff = (NY_time - crypto_time).total_seconds()/60 #minutes - if diff >= update_frequencies['crypto'] or msg == 'c': + if last_updates['crypto']['force'] or diff >= update_frequencies['crypto']:# or msg == 'c': crypto_time = NY_time.strftime("%d/%m/%Y %H:%M:%S") - updateCrypto(api_key, logf) - print('UPDATING CRYPTO') + #updateCrypto(api_key, logf) + - last_updates['crypto'] = crypto_time - #update_process = Process(target = updateCrypto, args = (api_key,logf)) - #update_process.start() - #update_processes.append(update_process) + last_updates['crypto']['time'] = crypto_time + last_updates['crypto']['force'] = False + update_process = Process(target = updateCrypto, args = (api_key,logf)) + update_process.start() + update_processes.append(update_process) # weather - weather_time = datetime.strptime(last_updates['weather'], "%d/%m/%Y %H:%M:%S") + weather_time = datetime.strptime(last_updates['weather']['time'], "%d/%m/%Y %H:%M:%S") NY_time = datetime.now(NY_zone).replace(tzinfo=None) diff = (NY_time - weather_time).total_seconds()/60 #minutes - if diff >= update_frequencies['weather'] or msg == 'w': + if last_updates['weather']['force'] or diff >= update_frequencies['weather']:# or msg == 'w': weather_time = NY_time.strftime("%d/%m/%Y %H:%M:%S") #updateWeather(weather_key) - last_updates['weather'] = weather_time + last_updates['weather']['time'] = weather_time + last_updates['weather']['force'] = False update_process = Process(target = updateWeather, args = (weather_key,logf)) update_process.start() update_processes.append(update_process) # news - news_time = datetime.strptime(last_updates['news'], "%d/%m/%Y %H:%M:%S") + news_time = datetime.strptime(last_updates['news']['time'], "%d/%m/%Y %H:%M:%S") NY_time = datetime.now(NY_zone).replace(tzinfo=None) diff = (NY_time - news_time).total_seconds()/60 #minutes - if diff >= update_frequencies['news'] or msg == 'n': + if last_updates['news']['force'] or diff >= update_frequencies['news']:# or msg == 'n': news_time = NY_time.strftime("%d/%m/%Y %H:%M:%S") #updateNews(api_key) - last_updates['news'] = news_time + last_updates['news']['time'] = news_time + last_updates['news']['force'] = False update_process = Process(target = updateNews, args = (api_key,logf)) update_process.start() update_processes.append(update_process) # sports - sports_time = datetime.strptime(last_updates['sports'], "%d/%m/%Y %H:%M:%S") + sports_time = datetime.strptime(last_updates['sports']['time'], "%d/%m/%Y %H:%M:%S") NY_time = datetime.now(NY_zone).replace(tzinfo=None) diff = (NY_time - sports_time).total_seconds()/60 #minutes - if diff >= update_frequencies['sports'] or msg == 'S': + if last_updates['sports']['force'] or diff >= update_frequencies['sports']:# or msg == 'S': sports_time = NY_time.strftime("%d/%m/%Y %H:%M:%S") #updateSports(api_key) - last_updates['sports'] = sports_time + last_updates['sports']['time'] = sports_time + last_updates['sports']['force'] = False update_process = Process(target = updateSports, args = (api_key,logf)) update_process.start() update_processes.append(update_process) @@ -729,7 +735,7 @@ if __name__ == '__main__': #forex updates once every 24hours at 1700 CET # update if last update was before the previous days closing - forex_time = datetime.strptime(last_updates['forex'], "%d/%m/%Y %H:%M:%S") + forex_time = datetime.strptime(last_updates['forex']['time'], "%d/%m/%Y %H:%M:%S") CET_time = datetime.now(CET_zone).replace(tzinfo=None) yday_update = (CET_time.replace(hour=17, minute=00, second=0, microsecond=0) - dt.timedelta(days=1)).replace(tzinfo=None) diff = (CET_time.replace(tzinfo=None) - forex_time).total_seconds()/60 @@ -739,25 +745,29 @@ if __name__ == '__main__': #forex updates between 5pm sunday and 5pm friday every hour forex_open = datetime.today().weekday() < 4 or (datetime.today().weekday() == 6 and CET_time > opening) or (datetime.today().weekday() == 4 and CET_time < opening) - if forex_time < yday_update or msg == 'f' or (diff >= update_frequencies['forex'] and forex_open): + if last_updates['forex']['force'] or (diff >= update_frequencies['forex'] and forex_open):# or msg == 'f': forex_time = CET_time.strftime("%d/%m/%Y %H:%M:%S") - last_updates['forex'] = forex_time + last_updates['forex']['time'] = forex_time + last_updates['forex']['force'] = False #updateForex(api_key) update_process = Process(target = updateForex, args = (api_key,logf)) update_process.start() update_processes.append(update_process) - - json.dump(last_updates, open('csv/last_updates.json', 'w+')) + f = open('csv/last_updates.json', 'w+') + json.dump(last_updates, f) + f.close() for process in update_processes: if not process.is_alive(): process.join() process.terminate() update_processes.remove(process) + print('Number active processes', len(update_processes)) + time.sleep(10) except Exception as e: - + exc_type, exc_obj, exc_tb = sys.exc_info() fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] logf.write(str(e)) diff --git a/database_caller2.py b/database_caller2.py index 27657bf..4d3c64a 100755 --- a/database_caller2.py +++ b/database_caller2.py @@ -648,7 +648,8 @@ if __name__ == '__main__': #stocks stock_time = datetime.strptime(last_updates['stocks'], "%d/%m/%Y %H:%M:%S") stock_frequency = update_frequencies['stocks'] - if checkStocks(stock_time, stock_frequency) or msg == 's': + diff = (NY_time - stock_time).total_seconds()/60 #minutes + if diff >= update_frequencies['stocks'] or msg == 's': stock_time = NY_time.strftime("%d/%m/%Y %H:%M:%S") last_updates['stocks'] = stock_time updateStocks(api_key, logf) diff --git a/server.py b/server.py index 0dff16d..d1d21d9 100755 --- a/server.py +++ b/server.py @@ -29,7 +29,7 @@ import sys #api_caller = pexpect.spawn("sudo -E taskset -c 3 python3 api_caller.py") -api_caller = pexpect.spawn("sudo -E taskset -c 3 python3 database_caller2.py") +api_caller = pexpect.spawn("sudo -E taskset -c 3 python3 database_caller.py") time.sleep(3) @@ -573,14 +573,21 @@ def save_trade_settings(input_settings): f = open('csv/' + filename, 'w+') json.dump(current_settings, f) f.close() + f = open('csv/last_updates.json', 'r') + last_updates = json.load(f) + f.close() if input_settings['feature'].lower() == 'stocks': - api_caller.sendline('s') + last_updates['stocks']['force'] = True elif input_settings['feature'].lower() == 'crypto': - api_caller.sendline('c') + last_updates['crypto']['force'] = True elif input_settings['feature'].lower() == 'forex': - api_caller.sendline('f') + last_updates['forex']['force'] = True + + f = open('csv/last_updates.json', 'w+') + json.dump(last_updates, f) + f.close() def save_weather_settings(input_settings): @@ -613,7 +620,16 @@ def save_weather_settings(input_settings): current_settings = combine_dict(current_settings, input_settings['locations'], 'locations') json.dump(current_settings, open('csv/' + filename, 'w+')) - api_caller.sendline('w') + #api_caller.sendline('w') + f = open('csv/last_updates.json', 'r') + last_updates = json.load(f) + f.close() + + last_updates['weather']['force'] = True + + f = open('csv/last_updates.json', 'w+') + json.dump(last_updates, f) + f.close() def save_news_settings(input_settings): filename = 'news_settings.json' @@ -636,7 +652,16 @@ def save_news_settings(input_settings): print(current_settings) json.dump(current_settings, open('csv/' + filename, 'w+')) - api_caller.sendline('n') + #api_caller.sendline('n') + f = open('csv/last_updates.json', 'r') + last_updates = json.load(f) + f.close() + + last_updates['news']['force'] = True + + f = open('csv/last_updates.json', 'w+') + json.dump(last_updates, f) + f.close() def save_sports_settings(input_settings): @@ -662,7 +687,16 @@ def save_sports_settings(input_settings): json.dump(current_settings, open('csv/' + filename, 'w+')) - api_caller.sendline('S') + #api_caller.sendline('S') + f = open('csv/last_updates.json', 'r') + last_updates = json.load(f) + f.close() + + last_updates['sports']['force'] = True + + f = open('csv/last_updates.json', 'w+') + json.dump(last_updates, f) + f.close() # for images and GIFs def save_image_settings(input_settings): diff --git a/setup_config_files.sh b/setup_config_files.sh index 708fdb7..2d4fb83 100755 --- a/setup_config_files.sh +++ b/setup_config_files.sh @@ -10,7 +10,7 @@ mkdir user_uploads mkdir csv cd csv rm *.json -touch coin_list.json crypto_settings.json last_updates.json system_info.json +touch crypto_settings.json last_updates.json system_info.json last_updates.json touch league_tables.json mkdir sports touch current_weather.json stocks_settings.json daily_weather.json live_games.json @@ -21,7 +21,7 @@ filenames="crypto_settings.json last_updates.json league_tables.json current_wea echo '{"update_available": false, "first_boot": true}' >> system_info.json echo [\"Standard\", [[\"Stocks\", \"Crypto\", \"Forex\"]]] >> display_settings.json - +echo '{"stocks": {"time": "06/03/2022 04:12:09", "force": true}, "crypto": {"time": "06/03/2022 04:10:39", "force": true}, "news": {"time": "06/03/2022 04:07:09", "force": true}, "weather": {"time": "06/03/2022 04:08:20", "force": true}, "forex": {"time": "06/03/2022 03:54:02", "force": true}, "sports": {"time": "06/03/2022 04:10:09", "force": true}}' >> last_updates.json echo '{"feature": "Stocks", "speed": "medium", "animation": "down", "percent": false, "point": true, "logos": true, "chart": false, "title": true, "symbols": {"ETH,USD": {"current": "2629.32", "24hr_change": "-27.6432", "percent_change": "-1.04"}, "BTC,USD": {"current": "38161.00", "24hr_change": "-50.8386", "percent_change": "-0.13"}, "BNB,USD": {"current": "372.57", "24hr_change": "0.4140", "percent_change": "0.11"}, "ADA,BTC": {"current": "0.0000", "24hr_change": "-0.0000", "percent_change": "-3.74"}}}' >> crypto_settings.json