Compare commits

..

8 Commits

Author SHA1 Message Date
8fe2af1876 Update playbook.yml 2025-05-04 13:07:59 +03:00
a9ee8e1f80 add ip update 2025-04-20 18:52:55 +03:00
e7f4bb6a35 Fix status message formatting in check_logs function 2025-04-12 17:02:08 +03:00
7ce0926d91 Enhance number formatting 2025-04-12 00:22:25 +03:00
3f49993f6b Update idle detection threshold to 4 hours 2025-04-12 00:11:13 +03:00
ed2229b139 Fix status message 2025-04-12 00:09:45 +03:00
79e7e1a89b Refactor status message 2025-04-12 00:09:03 +03:00
a19b22f2c0 Log enhancement 2025-04-12 00:05:23 +03:00
2 changed files with 47 additions and 12 deletions

View File

@ -138,7 +138,10 @@ def format_number(number_str):
try: try:
number = int(number_str) number = int(number_str)
if number >= 1000: if number >= 1000:
return f"{number//1000}k" value_in_k = number / 1000.0
# Format to 3 decimal places if needed, remove trailing zeros and potentially the dot
formatted_num = f"{value_in_k:.3f}".rstrip('0').rstrip('.')
return f"{formatted_num}k"
return str(number) return str(number)
except (ValueError, TypeError): except (ValueError, TypeError):
return "NaN" # Or some other indicator of invalid input return "NaN" # Or some other indicator of invalid input
@ -202,7 +205,7 @@ def check_logs(logger, initial_sync_count, previous_status):
formatted_id = format_number(last_checking_info["last_sub_id"]) formatted_id = format_number(last_checking_info["last_sub_id"])
if last_checking_info["num_subs_to_sync"] > 0: if last_checking_info["num_subs_to_sync"] > 0:
current_status_type = "Sync" current_status_type = "Sync"
status_message = f"Sync: {formatted_id} ({current_sync_count})" # Use current_sync_count status_message = f"Sync: {formatted_id}" # Use current_sync_count
logger.info(f"Node is syncing. Last sub ID: {last_checking_info['last_sub_id']}, Num subs to sync: {last_checking_info['num_subs_to_sync']}") logger.info(f"Node is syncing. Last sub ID: {last_checking_info['last_sub_id']}, Num subs to sync: {last_checking_info['num_subs_to_sync']}")
else: else:
current_status_type = "OK" current_status_type = "OK"
@ -210,14 +213,14 @@ def check_logs(logger, initial_sync_count, previous_status):
if previous_status == "Sync": if previous_status == "Sync":
current_sync_count += 1 # Increment local count current_sync_count += 1 # Increment local count
logger.info(f"Sync completed. Sync count incremented to {current_sync_count}.") logger.info(f"Sync completed. Sync count incremented to {current_sync_count}.")
status_message = f"OK: {formatted_id} ({current_sync_count})" # Use current_sync_count status_message = f"OK: {formatted_id}" # Use current_sync_count
logger.info(f"Node is OK. Last sub ID: {last_checking_info['last_sub_id']}") logger.info(f"Node is OK. Last sub ID: {last_checking_info['last_sub_id']}")
elif last_ignored_id: elif last_ignored_id:
# Fallback to "Ignored" logs if "Checking" is missing # Fallback to "Ignored" logs if "Checking" is missing
formatted_id = format_number(last_ignored_id) formatted_id = format_number(last_ignored_id)
current_status_type = "Sync" # Assume sync if we only see ignored creations recently current_status_type = "Sync" # Assume sync if we only see ignored creations recently
status_message = f"Sync: {formatted_id} ({current_sync_count})" # Use current_sync_count status_message = f"Sync: {formatted_id}" # Use current_sync_count
logger.info(f"Node possibly syncing (based on ignored logs). Last ignored ID: {last_ignored_id}") logger.info(f"Node possibly syncing (based on ignored logs). Last ignored ID: {last_ignored_id}")
elif last_head_sub_id: elif last_head_sub_id:
@ -225,7 +228,7 @@ def check_logs(logger, initial_sync_count, previous_status):
formatted_id = format_number(last_head_sub_id) formatted_id = format_number(last_head_sub_id)
current_status_type = "OK" # Assume OK if this is the latest relevant info current_status_type = "OK" # Assume OK if this is the latest relevant info
# Don't increment sync count here, only on Sync -> OK transition based on "Checking" logs # Don't increment sync count here, only on Sync -> OK transition based on "Checking" logs
status_message = f"OK: {formatted_id} ({current_sync_count})" # Use current_sync_count status_message = f"OK: {formatted_id}" # Use current_sync_count
logger.info(f"Node status based on head sub id. Head sub ID: {last_head_sub_id}") logger.info(f"Node status based on head sub id. Head sub ID: {last_head_sub_id}")
else: else:
@ -284,8 +287,31 @@ if __name__ == "__main__":
current_vm = grist.find_record(name=GRIST_ROW_NAME, table=NODES_TABLE)[0] current_vm = grist.find_record(name=GRIST_ROW_NAME, table=NODES_TABLE)[0]
def grist_callback(msg): grist.update(current_vm.id, msg, NODES_TABLE) def grist_callback(msg): grist.update(current_vm.id, msg, NODES_TABLE)
# Get initial state from Grist # Initialize updates dictionary
initial_sync_count = int(current_vm.Syncs or 0) initial_updates = {}
# Check and prepare update for Syncs if it's None or empty
if not current_vm.Syncs: # Handles None, empty string, potentially 0 if that's how Grist stores it
initial_updates["Syncs"] = 0
# Check and prepare update for Reboots if it's None or empty
if not current_vm.Reboots: # Handles None, empty string, potentially 0
initial_updates["Reboots"] = 0
# If there are updates, send them to Grist
if initial_updates:
try:
logger.info(f"Found empty initial values, updating Grist: {initial_updates}")
grist.update(current_vm.id, initial_updates, NODES_TABLE)
# Re-fetch the record to ensure subsequent logic uses the updated values
current_vm = grist.find_record(name=GRIST_ROW_NAME, table=NODES_TABLE)[0]
logger.info("Grist updated successfully with initial zeros.")
except Exception as e:
logger.error(f"Failed to update Grist with initial zeros: {e}")
# Decide how to proceed: maybe exit, maybe continue with potentially incorrect defaults
# For now, we'll log the error and continue using the potentially incorrect defaults from the first fetch
# Get initial state from Grist (now potentially updated)
initial_sync_count = int(current_vm.Syncs or 0) # 'or 0' still useful as fallback
reboot_count = int(current_vm.Reboots or 0) # 'or 0' still useful as fallback
# Determine previous status type based on Health string (simplified) # Determine previous status type based on Health string (simplified)
previous_health_status = current_vm.Health or "Idle" previous_health_status = current_vm.Health or "Idle"
previous_status_type = "Idle" # Default previous_status_type = "Idle" # Default
@ -296,10 +322,19 @@ if __name__ == "__main__":
elif previous_health_status.startswith("Error"): elif previous_health_status.startswith("Error"):
previous_status_type = "Error" # Consider error state previous_status_type = "Error" # Consider error state
logger.info(f"Initial state from Grist - Syncs: {initial_sync_count}, Health: {previous_health_status} (interpreted as: {previous_status_type})") logger.info(f"Initial state from Grist - Syncs: {initial_sync_count}, Health: {previous_health_status}, Reboots: {reboot_count}")
for attempt in range(3): for attempt in range(3):
try: try:
vm_ip = os.popen("ip -4 addr show eth0 | grep -oP '(?<=inet )[^/]+'").read()
vm_ip = vm_ip.strip()
if vm_ip == "":
logger.error("Failed to get VM IP address")
else:
logger.info(f"VM IP address: {vm_ip}")
grist_callback({"IP": f"{vm_ip}"})
# Pass initial state to check_logs # Pass initial state to check_logs
result = check_logs(logger, initial_sync_count, previous_status_type) result = check_logs(logger, initial_sync_count, previous_status_type)
@ -318,13 +353,13 @@ if __name__ == "__main__":
if result["current_status_type"] == "Idle": # Check type, not message if result["current_status_type"] == "Idle": # Check type, not message
uptime_seconds = os.popen("cat /proc/uptime | cut -d'.' -f1").read() uptime_seconds = os.popen("cat /proc/uptime | cut -d'.' -f1").read()
uptime_seconds = int(uptime_seconds) uptime_seconds = int(uptime_seconds)
if uptime_seconds > 60*60*1: if uptime_seconds > 60*60*4:
reboot_count = int(current_vm.Reboots or 0) reboot_count = int(current_vm.Reboots or 0)
reboot_count += 1 reboot_count += 1
# Include reboot count in the final Grist update before rebooting # Include reboot count in the final Grist update before rebooting
grist_updates = { "Health": "Rebooting", "Reboots": reboot_count } grist_updates = { "Health": "Rebooting", "Reboots": reboot_count }
grist_callback(grist_updates) grist_callback(grist_updates)
logger.info(f"Idle detected for >1 hour (uptime: {uptime_seconds}s). Rebooting. Reboot count: {reboot_count}") logger.info(f"Idle detected for >4 hours (uptime: {uptime_seconds}s). Rebooting. Reboot count: {reboot_count}")
os.system("reboot") os.system("reboot")
break # Exit loop on success break # Exit loop on success
except Exception as e: except Exception as e:

View File

@ -199,8 +199,8 @@
rm -rf {{ ansible_env.HOME }}/node/projects/hello-world/contracts/lib/infernet-sdk rm -rf {{ ansible_env.HOME }}/node/projects/hello-world/contracts/lib/infernet-sdk
cd {{ ansible_env.HOME }}/foundry && source {{ ansible_env.HOME }}/.bashrc && foundryup cd {{ ansible_env.HOME }}/foundry && source {{ ansible_env.HOME }}/.bashrc && foundryup
cd {{ ansible_env.HOME }}/node/projects/hello-world/contracts cd {{ ansible_env.HOME }}/node/projects/hello-world/contracts
forge install --no-commit foundry-rs/forge-std forge install foundry-rs/forge-std
forge install --no-commit ritual-net/infernet-sdk forge install ritual-net/infernet-sdk
args: args:
executable: /bin/bash executable: /bin/bash