From 244ea79d0a1e00c280785966c0e690804250553c Mon Sep 17 00:00:00 2001
From: conache <C.conache@gmail.com>
Date: Wed, 29 May 2024 13:10:08 +0300
Subject: [PATCH 1/3] Add inference service healthcheck to ensure the updater
 service properly starts up

---
 docker-compose.yml | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index ea8eb16..c4777a6 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -13,15 +13,27 @@ services:
         aliases:
           - inference
         ipv4_address: 172.22.0.4
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8000/inference/ETH"]
+      interval: 10s
+      timeout: 5s
+      retries: 12
     
   updater:
     container_name: updater-basic-eth-pred
     build: .
     environment:
       - INFERENCE_API_ADDRESS=http://inference:8000
-    command: python -u /app/update_app.py
+    command: >
+      sh -c "
+      while true; do
+        python -u /app/update_app.py;
+        sleep 10;
+      done
+      "
     depends_on:
-      - inference
+      inference:
+        condition: service_healthy
     networks:
       eth-model-local:
         aliases:

From 87acbc110ae61be65d23f9b152c74da16012903e Mon Sep 17 00:00:00 2001
From: conache <C.conache@gmail.com>
Date: Wed, 29 May 2024 13:13:02 +0300
Subject: [PATCH 2/3] Run update command every 60s

---
 docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index c4777a6..adefc66 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -28,7 +28,7 @@ services:
       sh -c "
       while true; do
         python -u /app/update_app.py;
-        sleep 10;
+        sleep 60;
       done
       "
     depends_on:

From 8b93b94ab239343ec84bc0a4cdda2c92148184a4 Mon Sep 17 00:00:00 2001
From: conache <C.conache@gmail.com>
Date: Wed, 29 May 2024 13:41:09 +0300
Subject: [PATCH 3/3] Run updater once a day, downloading only the new files

---
 .gitignore         |  1 +
 docker-compose.yml |  7 +++++--
 update_app.py      |  2 ++
 updater.py         | 12 ++++++++----
 4 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/.gitignore b/.gitignore
index e865921..0bb4c44 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,6 +6,7 @@ logs/*
 .env
 keys
 data
+inference-data
 worker-data
 head-data
 lib
\ No newline at end of file
diff --git a/docker-compose.yml b/docker-compose.yml
index adefc66..3a3daea 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -18,7 +18,9 @@ services:
       interval: 10s
       timeout: 5s
       retries: 12
-    
+    volumes:
+      - ./inference-data:/app/data
+  
   updater:
     container_name: updater-basic-eth-pred
     build: .
@@ -28,7 +30,7 @@ services:
       sh -c "
       while true; do
         python -u /app/update_app.py;
-        sleep 60;
+        sleep 24h;
       done
       "
     depends_on:
@@ -114,5 +116,6 @@ networks:
         - subnet: 172.22.0.0/24
 
 volumes:
+  inference-data:
   worker-data:
   head-data:
diff --git a/update_app.py b/update_app.py
index 0b3efd6..60a3d83 100644
--- a/update_app.py
+++ b/update_app.py
@@ -4,6 +4,8 @@ import requests
 inference_address = os.environ["INFERENCE_API_ADDRESS"]
 url = f"{inference_address}/update"
 
+print("UPDATING INFERENCE WORKER DATA")
+
 response = requests.get(url)
 if response.status_code == 200:
     # Request was successful
diff --git a/updater.py b/updater.py
index 2eb4332..65381f1 100644
--- a/updater.py
+++ b/updater.py
@@ -5,18 +5,22 @@ from concurrent.futures import ThreadPoolExecutor
 
 # Function to download the URL, called asynchronously by several child processes
 def download_url(url, download_path):
+    target_file_path = os.path.join(download_path, os.path.basename(url)) 
+    if os.path.exists(target_file_path):
+        print(f"File already exists: {url}")
+        return
+    
     response = requests.get(url)
     if response.status_code == 404:
         print(f"File not exist: {url}")
     else:
-        file_name = os.path.join(download_path, os.path.basename(url))
 
         # create the entire path if it doesn't exist
-        os.makedirs(os.path.dirname(file_name), exist_ok=True)
+        os.makedirs(os.path.dirname(target_file_path), exist_ok=True)
 
-        with open(file_name, "wb") as f:
+        with open(target_file_path, "wb") as f:
             f.write(response.content)
-        print(f"Downloaded: {url} to {file_name}")
+        print(f"Downloaded: {url} to {target_file_path}")
 
 
 def download_binance_monthly_data(