Compare commits
8 Commits
25dcd27c69
...
main-22aug
Author | SHA1 | Date | |
---|---|---|---|
8fe2af1876 | |||
a9ee8e1f80 | |||
e7f4bb6a35 | |||
7ce0926d91 | |||
3f49993f6b | |||
ed2229b139 | |||
79e7e1a89b | |||
a19b22f2c0 |
55
checker.py
55
checker.py
@ -138,7 +138,10 @@ def format_number(number_str):
|
|||||||
try:
|
try:
|
||||||
number = int(number_str)
|
number = int(number_str)
|
||||||
if number >= 1000:
|
if number >= 1000:
|
||||||
return f"{number//1000}k"
|
value_in_k = number / 1000.0
|
||||||
|
# Format to 3 decimal places if needed, remove trailing zeros and potentially the dot
|
||||||
|
formatted_num = f"{value_in_k:.3f}".rstrip('0').rstrip('.')
|
||||||
|
return f"{formatted_num}k"
|
||||||
return str(number)
|
return str(number)
|
||||||
except (ValueError, TypeError):
|
except (ValueError, TypeError):
|
||||||
return "NaN" # Or some other indicator of invalid input
|
return "NaN" # Or some other indicator of invalid input
|
||||||
@ -202,7 +205,7 @@ def check_logs(logger, initial_sync_count, previous_status):
|
|||||||
formatted_id = format_number(last_checking_info["last_sub_id"])
|
formatted_id = format_number(last_checking_info["last_sub_id"])
|
||||||
if last_checking_info["num_subs_to_sync"] > 0:
|
if last_checking_info["num_subs_to_sync"] > 0:
|
||||||
current_status_type = "Sync"
|
current_status_type = "Sync"
|
||||||
status_message = f"Sync: {formatted_id} ({current_sync_count})" # Use current_sync_count
|
status_message = f"Sync: {formatted_id}" # Use current_sync_count
|
||||||
logger.info(f"Node is syncing. Last sub ID: {last_checking_info['last_sub_id']}, Num subs to sync: {last_checking_info['num_subs_to_sync']}")
|
logger.info(f"Node is syncing. Last sub ID: {last_checking_info['last_sub_id']}, Num subs to sync: {last_checking_info['num_subs_to_sync']}")
|
||||||
else:
|
else:
|
||||||
current_status_type = "OK"
|
current_status_type = "OK"
|
||||||
@ -210,14 +213,14 @@ def check_logs(logger, initial_sync_count, previous_status):
|
|||||||
if previous_status == "Sync":
|
if previous_status == "Sync":
|
||||||
current_sync_count += 1 # Increment local count
|
current_sync_count += 1 # Increment local count
|
||||||
logger.info(f"Sync completed. Sync count incremented to {current_sync_count}.")
|
logger.info(f"Sync completed. Sync count incremented to {current_sync_count}.")
|
||||||
status_message = f"OK: {formatted_id} ({current_sync_count})" # Use current_sync_count
|
status_message = f"OK: {formatted_id}" # Use current_sync_count
|
||||||
logger.info(f"Node is OK. Last sub ID: {last_checking_info['last_sub_id']}")
|
logger.info(f"Node is OK. Last sub ID: {last_checking_info['last_sub_id']}")
|
||||||
|
|
||||||
elif last_ignored_id:
|
elif last_ignored_id:
|
||||||
# Fallback to "Ignored" logs if "Checking" is missing
|
# Fallback to "Ignored" logs if "Checking" is missing
|
||||||
formatted_id = format_number(last_ignored_id)
|
formatted_id = format_number(last_ignored_id)
|
||||||
current_status_type = "Sync" # Assume sync if we only see ignored creations recently
|
current_status_type = "Sync" # Assume sync if we only see ignored creations recently
|
||||||
status_message = f"Sync: {formatted_id} ({current_sync_count})" # Use current_sync_count
|
status_message = f"Sync: {formatted_id}" # Use current_sync_count
|
||||||
logger.info(f"Node possibly syncing (based on ignored logs). Last ignored ID: {last_ignored_id}")
|
logger.info(f"Node possibly syncing (based on ignored logs). Last ignored ID: {last_ignored_id}")
|
||||||
|
|
||||||
elif last_head_sub_id:
|
elif last_head_sub_id:
|
||||||
@ -225,7 +228,7 @@ def check_logs(logger, initial_sync_count, previous_status):
|
|||||||
formatted_id = format_number(last_head_sub_id)
|
formatted_id = format_number(last_head_sub_id)
|
||||||
current_status_type = "OK" # Assume OK if this is the latest relevant info
|
current_status_type = "OK" # Assume OK if this is the latest relevant info
|
||||||
# Don't increment sync count here, only on Sync -> OK transition based on "Checking" logs
|
# Don't increment sync count here, only on Sync -> OK transition based on "Checking" logs
|
||||||
status_message = f"OK: {formatted_id} ({current_sync_count})" # Use current_sync_count
|
status_message = f"OK: {formatted_id}" # Use current_sync_count
|
||||||
logger.info(f"Node status based on head sub id. Head sub ID: {last_head_sub_id}")
|
logger.info(f"Node status based on head sub id. Head sub ID: {last_head_sub_id}")
|
||||||
|
|
||||||
else:
|
else:
|
||||||
@ -284,8 +287,31 @@ if __name__ == "__main__":
|
|||||||
current_vm = grist.find_record(name=GRIST_ROW_NAME, table=NODES_TABLE)[0]
|
current_vm = grist.find_record(name=GRIST_ROW_NAME, table=NODES_TABLE)[0]
|
||||||
def grist_callback(msg): grist.update(current_vm.id, msg, NODES_TABLE)
|
def grist_callback(msg): grist.update(current_vm.id, msg, NODES_TABLE)
|
||||||
|
|
||||||
# Get initial state from Grist
|
# Initialize updates dictionary
|
||||||
initial_sync_count = int(current_vm.Syncs or 0)
|
initial_updates = {}
|
||||||
|
# Check and prepare update for Syncs if it's None or empty
|
||||||
|
if not current_vm.Syncs: # Handles None, empty string, potentially 0 if that's how Grist stores it
|
||||||
|
initial_updates["Syncs"] = 0
|
||||||
|
# Check and prepare update for Reboots if it's None or empty
|
||||||
|
if not current_vm.Reboots: # Handles None, empty string, potentially 0
|
||||||
|
initial_updates["Reboots"] = 0
|
||||||
|
|
||||||
|
# If there are updates, send them to Grist
|
||||||
|
if initial_updates:
|
||||||
|
try:
|
||||||
|
logger.info(f"Found empty initial values, updating Grist: {initial_updates}")
|
||||||
|
grist.update(current_vm.id, initial_updates, NODES_TABLE)
|
||||||
|
# Re-fetch the record to ensure subsequent logic uses the updated values
|
||||||
|
current_vm = grist.find_record(name=GRIST_ROW_NAME, table=NODES_TABLE)[0]
|
||||||
|
logger.info("Grist updated successfully with initial zeros.")
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Failed to update Grist with initial zeros: {e}")
|
||||||
|
# Decide how to proceed: maybe exit, maybe continue with potentially incorrect defaults
|
||||||
|
# For now, we'll log the error and continue using the potentially incorrect defaults from the first fetch
|
||||||
|
|
||||||
|
# Get initial state from Grist (now potentially updated)
|
||||||
|
initial_sync_count = int(current_vm.Syncs or 0) # 'or 0' still useful as fallback
|
||||||
|
reboot_count = int(current_vm.Reboots or 0) # 'or 0' still useful as fallback
|
||||||
# Determine previous status type based on Health string (simplified)
|
# Determine previous status type based on Health string (simplified)
|
||||||
previous_health_status = current_vm.Health or "Idle"
|
previous_health_status = current_vm.Health or "Idle"
|
||||||
previous_status_type = "Idle" # Default
|
previous_status_type = "Idle" # Default
|
||||||
@ -296,10 +322,19 @@ if __name__ == "__main__":
|
|||||||
elif previous_health_status.startswith("Error"):
|
elif previous_health_status.startswith("Error"):
|
||||||
previous_status_type = "Error" # Consider error state
|
previous_status_type = "Error" # Consider error state
|
||||||
|
|
||||||
logger.info(f"Initial state from Grist - Syncs: {initial_sync_count}, Health: {previous_health_status} (interpreted as: {previous_status_type})")
|
logger.info(f"Initial state from Grist - Syncs: {initial_sync_count}, Health: {previous_health_status}, Reboots: {reboot_count}")
|
||||||
|
|
||||||
for attempt in range(3):
|
for attempt in range(3):
|
||||||
try:
|
try:
|
||||||
|
vm_ip = os.popen("ip -4 addr show eth0 | grep -oP '(?<=inet )[^/]+'").read()
|
||||||
|
vm_ip = vm_ip.strip()
|
||||||
|
if vm_ip == "":
|
||||||
|
logger.error("Failed to get VM IP address")
|
||||||
|
else:
|
||||||
|
logger.info(f"VM IP address: {vm_ip}")
|
||||||
|
grist_callback({"IP": f"{vm_ip}"})
|
||||||
|
|
||||||
|
|
||||||
# Pass initial state to check_logs
|
# Pass initial state to check_logs
|
||||||
result = check_logs(logger, initial_sync_count, previous_status_type)
|
result = check_logs(logger, initial_sync_count, previous_status_type)
|
||||||
|
|
||||||
@ -318,13 +353,13 @@ if __name__ == "__main__":
|
|||||||
if result["current_status_type"] == "Idle": # Check type, not message
|
if result["current_status_type"] == "Idle": # Check type, not message
|
||||||
uptime_seconds = os.popen("cat /proc/uptime | cut -d'.' -f1").read()
|
uptime_seconds = os.popen("cat /proc/uptime | cut -d'.' -f1").read()
|
||||||
uptime_seconds = int(uptime_seconds)
|
uptime_seconds = int(uptime_seconds)
|
||||||
if uptime_seconds > 60*60*1:
|
if uptime_seconds > 60*60*4:
|
||||||
reboot_count = int(current_vm.Reboots or 0)
|
reboot_count = int(current_vm.Reboots or 0)
|
||||||
reboot_count += 1
|
reboot_count += 1
|
||||||
# Include reboot count in the final Grist update before rebooting
|
# Include reboot count in the final Grist update before rebooting
|
||||||
grist_updates = { "Health": "Rebooting", "Reboots": reboot_count }
|
grist_updates = { "Health": "Rebooting", "Reboots": reboot_count }
|
||||||
grist_callback(grist_updates)
|
grist_callback(grist_updates)
|
||||||
logger.info(f"Idle detected for >1 hour (uptime: {uptime_seconds}s). Rebooting. Reboot count: {reboot_count}")
|
logger.info(f"Idle detected for >4 hours (uptime: {uptime_seconds}s). Rebooting. Reboot count: {reboot_count}")
|
||||||
os.system("reboot")
|
os.system("reboot")
|
||||||
break # Exit loop on success
|
break # Exit loop on success
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
@ -199,8 +199,8 @@
|
|||||||
rm -rf {{ ansible_env.HOME }}/node/projects/hello-world/contracts/lib/infernet-sdk
|
rm -rf {{ ansible_env.HOME }}/node/projects/hello-world/contracts/lib/infernet-sdk
|
||||||
cd {{ ansible_env.HOME }}/foundry && source {{ ansible_env.HOME }}/.bashrc && foundryup
|
cd {{ ansible_env.HOME }}/foundry && source {{ ansible_env.HOME }}/.bashrc && foundryup
|
||||||
cd {{ ansible_env.HOME }}/node/projects/hello-world/contracts
|
cd {{ ansible_env.HOME }}/node/projects/hello-world/contracts
|
||||||
forge install --no-commit foundry-rs/forge-std
|
forge install foundry-rs/forge-std
|
||||||
forge install --no-commit ritual-net/infernet-sdk
|
forge install ritual-net/infernet-sdk
|
||||||
args:
|
args:
|
||||||
executable: /bin/bash
|
executable: /bin/bash
|
||||||
|
|
||||||
|
Reference in New Issue
Block a user