Compare commits
No commits in common. "90aa23cf638ab9a207168b77843339416cf1817f" and "4de16544a3a6ea772903a1a7486e2ffe732c6f10" have entirely different histories.
90aa23cf63
...
4de16544a3
42
checker.py
42
checker.py
@ -94,19 +94,28 @@ def check_logs(logger):
|
|||||||
sync_count = 0
|
sync_count = 0
|
||||||
total_challenges = 0
|
total_challenges = 0
|
||||||
|
|
||||||
|
# Get current time and 24 hours ago
|
||||||
|
current_time = datetime.now()
|
||||||
|
logger.info(f"Current time: {current_time}")
|
||||||
|
day_ago = current_time - timedelta(days=1)
|
||||||
|
logger.info(f"Max logs timestamp: {day_ago}")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
result = subprocess.run(['docker', 'compose', 'logs', '--since', '24h'], cwd='/root/node/', capture_output=True, text=True)
|
result = subprocess.run(['docker', 'compose', 'logs'], cwd='/root/node/', capture_output=True, text=True)
|
||||||
log_content = result.stdout
|
log_content = result.stdout
|
||||||
except subprocess.CalledProcessError as e:
|
except subprocess.CalledProcessError as e:
|
||||||
raise Exception(f"Error running docker compose logs: {e}")
|
raise Exception(f"Error running docker compose logs: {e}")
|
||||||
|
|
||||||
for line in log_content.split('\n'):
|
for line in log_content.split('\n'):
|
||||||
|
timestamp_match = re.search(r'(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})', line)
|
||||||
|
if timestamp_match: timestamp = datetime.strptime(timestamp_match.group(1), '%Y-%m-%dT%H:%M:%S')
|
||||||
|
else: timestamp = None
|
||||||
|
|
||||||
|
if not timestamp: continue
|
||||||
|
if timestamp < day_ago: continue
|
||||||
if "Error from tendermint rpc" in line:
|
if "Error from tendermint rpc" in line:
|
||||||
error_count += 1
|
error_count += 1
|
||||||
logger.error(f"RPC error: {line}")
|
logger.error(f"RPC error: {line}")
|
||||||
if "Is your verifier's account funded" in line:
|
|
||||||
logger.error(f"Verifier account not funded: {line}")
|
|
||||||
error_count += 1
|
|
||||||
if "Synced with network" in line:
|
if "Synced with network" in line:
|
||||||
sync_count += 1
|
sync_count += 1
|
||||||
logger.info(f"Synced with network: {line}")
|
logger.info(f"Synced with network: {line}")
|
||||||
@ -115,8 +124,9 @@ def check_logs(logger):
|
|||||||
if challenge_match:
|
if challenge_match:
|
||||||
total_challenges += int(challenge_match.group(1))
|
total_challenges += int(challenge_match.group(1))
|
||||||
logger.info(f"Made {total_challenges} secret challenges: {line}")
|
logger.info(f"Made {total_challenges} secret challenges: {line}")
|
||||||
|
|
||||||
result = {
|
result = {
|
||||||
"errors": error_count,
|
"rpc_errors": error_count,
|
||||||
"sync_events": sync_count,
|
"sync_events": sync_count,
|
||||||
"total_challenges": total_challenges
|
"total_challenges": total_challenges
|
||||||
}
|
}
|
||||||
@ -144,16 +154,12 @@ if __name__ == "__main__":
|
|||||||
current_vm = grist.find_record(name=GRIST_ROW_NAME, table=NODES_TABLE)[0]
|
current_vm = grist.find_record(name=GRIST_ROW_NAME, table=NODES_TABLE)[0]
|
||||||
def grist_callback(msg): grist.update(current_vm.id, msg, NODES_TABLE)
|
def grist_callback(msg): grist.update(current_vm.id, msg, NODES_TABLE)
|
||||||
|
|
||||||
for attempt in range(3):
|
try:
|
||||||
try:
|
result = check_logs(logger)
|
||||||
result = check_logs(logger)
|
data = f"{result['sync_events']}/{result['total_challenges']}/{result['rpc_errors']}" # Syncs/Challenges/RPC errors
|
||||||
data = f"{result['sync_events']}/{result['total_challenges']}/{result['errors']}" # Syncs/Challenges/Errors
|
grist_callback({ "Health": data })
|
||||||
grist_callback({ "Health": data })
|
print(result)
|
||||||
print(result)
|
except Exception as e:
|
||||||
break
|
logger.error(f"Error: {e}")
|
||||||
except Exception as e:
|
grist_callback({ "Health": f"Error: {e}" })
|
||||||
logger.error(f"Error on attempt {attempt+1}/3: {e}")
|
|
||||||
if attempt == 2:
|
|
||||||
grist_callback({ "Health": f"Error: {e}" })
|
|
||||||
if attempt < 2:
|
|
||||||
time.sleep(5)
|
|
Loading…
Reference in New Issue
Block a user