vvzvlad/nillion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

log 24h update logic

Browse Source
This commit is contained in:
vvzvlad 2024-11-24 02:45:59 +03:00
parent e6f24496a8
commit 90aa23cf63
1 changed files with 14 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
checker.py
View File

@ -94,25 +94,13 @@ def check_logs(logger):
sync_count = 0 sync_count = 0
total_challenges = 0 total_challenges = 0
# Get current time and 24 hours ago
current_time = datetime.now()
logger.info(f"Current time: {current_time}")
day_ago = current_time - timedelta(days=1)
logger.info(f"Max logs timestamp: {day_ago}")
try: try:
result = subprocess.run(['docker', 'compose', 'logs'], cwd='/root/node/', capture_output=True, text=True) result = subprocess.run(['docker', 'compose', 'logs', '--since', '24h'], cwd='/root/node/', capture_output=True, text=True)
log_content = result.stdout log_content = result.stdout
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
raise Exception(f"Error running docker compose logs: {e}") raise Exception(f"Error running docker compose logs: {e}")
for line in log_content.split('\n'): for line in log_content.split('\n'):
timestamp_match = re.search(r'(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})', line)
if timestamp_match: timestamp = datetime.strptime(timestamp_match.group(1), '%Y-%m-%dT%H:%M:%S')
else: timestamp = None
if not timestamp: continue
if timestamp < day_ago: continue
if "Error from tendermint rpc" in line: if "Error from tendermint rpc" in line:
error_count += 1 error_count += 1
logger.error(f"RPC error: {line}") logger.error(f"RPC error: {line}")
@ -123,12 +111,10 @@ def check_logs(logger):
sync_count += 1 sync_count += 1
logger.info(f"Synced with network: {line}") logger.info(f"Synced with network: {line}")
challenge_match = re.search(r'made (\d+) secret challenges', line) challenge_match = re.search(r'made (\d+) secret challenges', line)
if challenge_match: if challenge_match:
total_challenges += int(challenge_match.group(1)) total_challenges += int(challenge_match.group(1))
logger.info(f"Made {total_challenges} secret challenges: {line}") logger.info(f"Made {total_challenges} secret challenges: {line}")
result = { result = {
"errors": error_count, "errors": error_count,
"sync_events": sync_count, "sync_events": sync_count,
@ -158,12 +144,16 @@ if __name__ == "__main__":
current_vm = grist.find_record(name=GRIST_ROW_NAME, table=NODES_TABLE)[0] current_vm = grist.find_record(name=GRIST_ROW_NAME, table=NODES_TABLE)[0]
def grist_callback(msg): grist.update(current_vm.id, msg, NODES_TABLE) def grist_callback(msg): grist.update(current_vm.id, msg, NODES_TABLE)
try: for attempt in range(3):
result = check_logs(logger) try:
data = f"{result['sync_events']}/{result['total_challenges']}/{result['errors']}" # Syncs/Challenges/Errors result = check_logs(logger)
grist_callback({ "Health": data }) data = f"{result['sync_events']}/{result['total_challenges']}/{result['errors']}" # Syncs/Challenges/Errors
print(result) grist_callback({ "Health": data })
except Exception as e: print(result)
logger.error(f"Error: {e}") break
grist_callback({ "Health": f"Error: {e}" }) except Exception as e:
logger.error(f"Error on attempt {attempt+1}/3: {e}")
if attempt == 2:
grist_callback({ "Health": f"Error: {e}" })
if attempt < 2:
time.sleep(5)