diff --git a/checker.py b/checker.py index 74a135e..4a27d69 100644 --- a/checker.py +++ b/checker.py @@ -94,25 +94,13 @@ def check_logs(logger): sync_count = 0 total_challenges = 0 - # Get current time and 24 hours ago - current_time = datetime.now() - logger.info(f"Current time: {current_time}") - day_ago = current_time - timedelta(days=1) - logger.info(f"Max logs timestamp: {day_ago}") - try: - result = subprocess.run(['docker', 'compose', 'logs'], cwd='/root/node/', capture_output=True, text=True) + result = subprocess.run(['docker', 'compose', 'logs', '--since', '24h'], cwd='/root/node/', capture_output=True, text=True) log_content = result.stdout except subprocess.CalledProcessError as e: raise Exception(f"Error running docker compose logs: {e}") for line in log_content.split('\n'): - timestamp_match = re.search(r'(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})', line) - if timestamp_match: timestamp = datetime.strptime(timestamp_match.group(1), '%Y-%m-%dT%H:%M:%S') - else: timestamp = None - - if not timestamp: continue - if timestamp < day_ago: continue if "Error from tendermint rpc" in line: error_count += 1 logger.error(f"RPC error: {line}") @@ -123,12 +111,10 @@ def check_logs(logger): sync_count += 1 logger.info(f"Synced with network: {line}") - challenge_match = re.search(r'made (\d+) secret challenges', line) if challenge_match: total_challenges += int(challenge_match.group(1)) logger.info(f"Made {total_challenges} secret challenges: {line}") - result = { "errors": error_count, "sync_events": sync_count, @@ -158,12 +144,16 @@ if __name__ == "__main__": current_vm = grist.find_record(name=GRIST_ROW_NAME, table=NODES_TABLE)[0] def grist_callback(msg): grist.update(current_vm.id, msg, NODES_TABLE) - try: - result = check_logs(logger) - data = f"{result['sync_events']}/{result['total_challenges']}/{result['errors']}" # Syncs/Challenges/Errors - grist_callback({ "Health": data }) - print(result) - except Exception as e: - logger.error(f"Error: {e}") - grist_callback({ "Health": f"Error: {e}" }) - \ No newline at end of file + for attempt in range(3): + try: + result = check_logs(logger) + data = f"{result['sync_events']}/{result['total_challenges']}/{result['errors']}" # Syncs/Challenges/Errors + grist_callback({ "Health": data }) + print(result) + break + except Exception as e: + logger.error(f"Error on attempt {attempt+1}/3: {e}") + if attempt == 2: + grist_callback({ "Health": f"Error: {e}" }) + if attempt < 2: + time.sleep(5) \ No newline at end of file