From 0f9d20a90599aaf7be12feb7f85000590bd7cf10 Mon Sep 17 00:00:00 2001 From: vvzvlad Date: Sat, 23 Nov 2024 04:32:31 +0300 Subject: [PATCH] add checker --- checker.py | 152 +++++++++++++++++++++++++++++++++++++++++++++++++++ playbook.yml | 33 +++++++++++ update.sh | 1 + 3 files changed, 186 insertions(+) create mode 100644 checker.py diff --git a/checker.py b/checker.py new file mode 100644 index 0000000..e2bcc0f --- /dev/null +++ b/checker.py @@ -0,0 +1,152 @@ +import re +from datetime import datetime, timedelta +import subprocess +import os +import sys +import pkg_resources + +required_packages = ['grist-api', 'colorama'] +installed_packages = [pkg.key for pkg in pkg_resources.working_set] + +for package in required_packages: + if package not in installed_packages: + subprocess.check_call([sys.executable, '-m', 'pip', 'install', package, '--break-system-packages']) + +from grist_api import GristDocAPI +import colorama +import requests +import logging +import socket + + +class GRIST: + def __init__(self, server, doc_id, api_key, logger): + self.server = server + self.doc_id = doc_id + self.api_key = api_key + self.logger = logger + self.grist = GristDocAPI(doc_id, server=server, api_key=api_key) + + def table_name_convert(self, table_name): + return table_name.replace(" ", "_") + + def to_timestamp(self, dtime: datetime) -> int: + if dtime.tzinfo is None: + dtime = dtime.replace(tzinfo=timezone(timedelta(hours=3))) + return int(dtime.timestamp()) + + def insert_row(self, data, table): + data = {key.replace(" ", "_"): value for key, value in data.items()} + row_id = self.grist.add_records(self.table_name_convert(table), [data]) + return row_id + + def update_column(self, row_id, column_name, value, table): + if isinstance(value, datetime): + value = self.to_timestamp(value) + column_name = column_name.replace(" ", "_") + self.grist.update_records(self.table_name_convert(table), [{ "id": row_id, column_name: value }]) + + def delete_row(self, row_id, table): + self.grist.delete_records(self.table_name_convert(table), [row_id]) + + def update(self, row_id, updates, table): + for column_name, value in updates.items(): + if isinstance(value, datetime): + updates[column_name] = self.to_timestamp(value) + updates = {column_name.replace(" ", "_"): value for column_name, value in updates.items()} + self.grist.update_records(self.table_name_convert(table), [{"id": row_id, **updates}]) + + def fetch_table(self, table): + return self.grist.fetch_table(self.table_name_convert(table)) + + def find_record(self, id=None, state=None, name=None, table=None): + if table is None: + raise ValueError("Table is not specified") + table_data = self.grist.fetch_table(self.table_name_convert(table)) + if id is not None: + record = [row for row in table_data if row.id == id] + return record + if state is not None and name is not None: + record = [row for row in table_data if row.State == state and row.name == name] + return record + if state is not None: + record = [row for row in table_data if row.State == state] + return record + if name is not None: + record = [row for row in table_data if row.Name == name] + return record + + def find_settings(self, key, table): + table = self.fetch_table(self.table_name_convert(table)) + for record in table: + if record.Setting == key: + if record.Value is None or record.Value == "": + raise ValueError(f"Setting {key} blank") + return record.Value + raise ValueError(f"Setting {key} not found") + + +def check_logs(): + # Initialize counters + error_count = 0 + sync_count = 0 + total_challenges = 0 + + # Get current time and 24 hours ago + current_time = datetime.now() + day_ago = current_time - timedelta(days=1) + + try: + result = subprocess.run(['docker', 'compose', 'logs'], cwd='/root/node/', capture_output=True, text=True) + log_content = result.stdout + except subprocess.CalledProcessError as e: + raise Exception(f"Error running docker compose logs: {e}") + + for line in log_content.split('\n'): + timestamp_match = re.search(r'(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})', line) + if timestamp_match: timestamp = datetime.strptime(timestamp_match.group(1), '%Y-%m-%dT%H:%M:%S') + else: timestamp = None + + if not timestamp: continue + if timestamp < day_ago: continue + if "Error from tendermint rpc" in line: error_count += 1 + if "Synced with network" in line: sync_count += 1 + + challenge_match = re.search(r'made (\d+) secret challenges', line) + if challenge_match: total_challenges += int(challenge_match.group(1)) + + return { + "rpc_errors": error_count, + "sync_events": sync_count, + "total_challenges": total_challenges + } + +if __name__ == "__main__": + colorama.init(autoreset=True) + logger = logging.getLogger("Impact updater") + logger.setLevel(logging.INFO) + formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") + ch = logging.StreamHandler() + ch.setFormatter(formatter) + logger.addHandler(ch) + + grist_server = "###GRIST_SERVER###" + grist_doc_id = "###GRIST_DOC_ID###" + grist_api_key = "###GRIST_API_KEY###" + grist_row_name = socket.gethostname() + nodes_table = "Nodes" + grist = GRIST(grist_server, grist_doc_id, grist_api_key, logger) + vms = grist.find_record(name=grist_row_name, table=nodes_table) + current_vm = vms[0] + def grist_callback(msg): grist.update(current_vm.id, msg, nodes_table) + #register_url = grist.find_settings("Nillion checker url", "Settings") + + try: + result = check_logs() + data = f"{result['sync_events']}/{result['total_challenges']}/{result['rpc_errors']}" # Syncs/Challenges/RPC errors + grist_callback({ "Health": data }) + print(result) + except Exception as e: + logger.error(f"Error: {e}") + grist_callback({ "Health": f"Error: {e}" }) + \ No newline at end of file diff --git a/playbook.yml b/playbook.yml index 0c60476..c1478a6 100644 --- a/playbook.yml +++ b/playbook.yml @@ -174,6 +174,9 @@ ./update.sh PRIVATE "{{ private_key }}" ./update.sh PUBLIC "{{ public_key }}" ./update.sh RPC "{{ rpc_url }}" + ./update.sh GRIST_SERVER "{{ grist_server }}" + ./update.sh GRIST_DOC_ID "{{ grist_doc_id }}" + ./update.sh GRIST_API_KEY "{{ grist_api_key }}" args: chdir: "{{ ansible_env.HOME }}/node" changed_when: false @@ -316,6 +319,36 @@ async: "{{ 60 * 80 }}" poll: "{{ 60 }}" + - name: Copy checker service file + ansible.builtin.copy: + dest: /etc/systemd/system/nillion-checker.service + content: | + [Unit] + Description=Nillion Checker Service + After=network.target + + [Service] + Type=simple + User=root + WorkingDirectory={{ ansible_env.HOME }}/node + ExecStart=/usr/bin/python3 {{ ansible_env.HOME }}/node/checker.py + Restart=always + RestartSec=1800 + + [Install] + WantedBy=multi-user.target + mode: '0644' + + - name: Reload systemd + ansible.builtin.systemd: + daemon_reload: yes + + - name: Enable and start nillion-checker service + ansible.builtin.systemd: + name: nillion-checker + enabled: yes + state: started + # - name: Check "not have enough balance" # ansible.builtin.command: docker logs {{ item }} 2>&1 diff --git a/update.sh b/update.sh index 03532b9..a059944 100644 --- a/update.sh +++ b/update.sh @@ -12,6 +12,7 @@ NEW_VALUE=$2 FILES=( "credentials.json" "docker-compose.yml" + "checker.py" ) for FILE in "${FILES[@]}"; do