.env

@@ -0,0 +1,7 @@
+TOKEN=###TOKEN###
+TRAINING_DAYS=###TRAINING_DAYS###
+TIMEFRAME=###TIMEFRAME###
+MODEL=###MODEL###
+REGION=EU
+DATA_PROVIDER=###DATA_PROVIDER###
+CG_API_KEY=###CG_API_KEY###

.gitignore

@@ -0,0 +1,19 @@
+.DS_Store
+__pycache__
+*.pyc
+logs/*
+
+.allorad
+.cache
+inference-data
+worker-data
+
+/data
+
+**/*.venv*
+**/.cache
+**/env_file
+**/.gitkeep*
+**/*.csv
+**/*.pkl
+**/*.zip

CONTRIBUTING.md

@@ -0,0 +1,18 @@
+Any contribution that you make to this repository will
+be under the Apache 2 License, as dictated by that
+[license](http://www.apache.org/licenses/LICENSE-2.0.html):
+
+~~~
+5. Submission of Contributions. Unless You explicitly state otherwise,
+   any Contribution intentionally submitted for inclusion in the Work
+   by You to the Licensor shall be under the terms and conditions of
+   this License, without any additional terms or conditions.
+   Notwithstanding the above, nothing herein shall supersede or modify
+   the terms of any separate license agreement you may have executed
+   with Licensor regarding such Contributions.
+~~~
+
+Contributors must sign-off each commit by adding a `Signed-off-by: ...`
+line to commit messages to certify that they have the right to submit
+the code they are contributing to the project according to the
+[Developer Certificate of Origin (DCO)](https://developercertificate.org/).

Dockerfile

@@ -1,5 +1,7 @@
-# Use an official Python runtime as the base image
-FROM amd64/python:3.9-buster as project_env
+FROM python:3.11-slim AS project_env
+
+# Install curl
+RUN apt-get update && apt-get install -y curl
 
 # Set the working directory in the container
 WORKDIR /app

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -0,0 +1,112 @@
+# Basic Price Prediction Node
+
+This repository provides an example [Allora network](https://docs.allora.network/) worker node, designed to offer price predictions. The primary objective is to demonstrate the use of a basic inference model running within a dedicated container, showcasing its integration with the Allora network infrastructure to contribute valuable inferences.
+
+## Components
+
+- **Worker**: The node that publishes inferences to the Allora chain.
+- **Inference**: A container that conducts inferences, maintains the model state, and responds to internal inference requests via a Flask application. This node operates with a basic linear regression model for price predictions.
+- **Updater**: A cron-like container designed to update the inference node's data by daily fetching the latest market information from the data provider, ensuring the model stays current with new market trends.
+
+Check the `docker-compose.yml` file for the detailed setup of each component.
+
+## Docker-Compose Setup
+
+A complete working example is provided in the `docker-compose.yml` file.
+
+### Steps to Setup
+
+1. **Clone the Repository**
+2. **Copy and Populate Model Configuration environment file**
+    
+    Copy the example .env.example file and populate it with your variables:
+    ```sh
+    cp .env.example .env
+    ```
+
+    Here are the currently accepted configurations:
+    - TOKEN
+    Must be one in ('ETH','SOL','BTC','BNB','ARB'). 
+    Note: if you are using `Binance` as the data provider, any token could be used.
+    If you are using Coingecko, you should add its `coin_id` in the [token_map here](https://github.com/allora-network/basic-coin-prediction-node/blob/main/updater.py#L107). Find [more info here](https://docs.coingecko.com/reference/simple-price) and the [list here](https://docs.google.com/spreadsheets/d/1wTTuxXt8n9q7C4NDXqQpI3wpKu1_5bGVmP9Xz0XGSyU/edit?usp=sharing).
+    - TRAINING_DAYS
+    Must be an `int` >= 1. 
+    Represents how many days of historical data to use. 
+    - TIMEFRAME
+    This should be in this form: `10min`, `1h`, `1d`, `1m`, etc.
+    Note: For Coingecko, Data granularity (candle's body) is automatic - [see here](https://docs.coingecko.com/reference/coins-id-ohlc). To avoid downsampling, it is recommanded to use with Coingecko:
+        - TIMEFRAME >= 30m if TRAINING_DAYS <= 2
+        - TIMEFRAME >= 4h if TRAINING_DAYS <= 30
+        - TIMEFRAME >= 4d if TRAINING_DAYS >= 31
+    - MODEL
+    Must be one in ('LinearRegression','SVR','KernelRidge','BayesianRidge'). 
+    You can easily add support for any other models by [adding it here](https://github.com/allora-network/basic-coin-prediction-node/blob/main/model.py#L133).
+    - REGION
+    Used for the Binance API. This should be in this form: `US`, `EU`, etc.
+    - DATA_PROVIDER
+    Must be `binance` or `coingecko`. Feel free to add support for other data providers to personalize your model!
+    - CG_API_KEY
+    This is your `Coingecko` API key, if you've set `DATA_PROVIDER=coingecko`.
+
+3. **Copy and Populate Worker Configuration**
+
+    Copy the example configuration file and populate it with your variables:
+    ```sh
+    cp config.example.json config.json
+    ```
+
+4. **Initialize Worker**
+    
+    Run the following commands from the project's root directory to initialize the worker:
+    ```sh
+    chmod +x init.config
+    ./init.config
+    ```
+    These commands will:
+    - Automatically create Allora keys for your worker.
+    - Export the needed variables from the created account to be used by the worker node, bundle them with your provided `config.json`, and pass them to the node as environment variables.
+
+5. **Faucet Your Worker Node**
+    
+    You can find the offchain worker node's address in `./worker-data/env_file` under `ALLORA_OFFCHAIN_ACCOUNT_ADDRESS`. [Add faucet funds](https://docs.allora.network/devs/get-started/setup-wallet#add-faucet-funds) to your worker's wallet before starting it.
+
+6. **Start the Services**
+    
+    Run the following command to start the worker node, inference, and updater nodes:
+    ```sh
+    docker compose up --build
+    ```
+    To confirm that the worker successfully sends the inferences to the chain, look for the following log:
+    ```
+    {"level":"debug","msg":"Send Worker Data to chain","txHash":<tx-hash>,"time":<timestamp>,"message":"Success"}
+    ```
+
+## Testing Inference Only
+
+This setup allows you to develop your model without the need to bring up the offchain worker or the updater. To test the inference model only:
+
+1. Run the following command to start the inference node:
+    ```sh
+    docker compose up --build inference
+    ```
+    Wait for the initial data load.
+
+2. Send requests to the inference model. For example, request ETH price inferences:
+    
+    ```sh
+    curl http://127.0.0.1:8000/inference/ETH
+    ```
+    Expected response:
+    ```json
+    {"value":"2564.021586281073"}
+    ```
+
+3. Update the node's internal state (download pricing data, train, and update the model):
+    
+    ```sh
+    curl http://127.0.0.1:8000/update
+    ```
+    Expected response:
+    ```sh
+    0
+    ```

app.py

@@ -1,43 +1,28 @@
 import json
-import pickle
-import pandas as pd
-import numpy as np
-from datetime import datetime
-from flask import Flask, jsonify, Response
-from model import download_data, format_data, train_model
-from config import model_file_path
+from flask import Flask, Response
+from model import download_data, format_data, train_model, get_inference
+from config import model_file_path, TOKEN, TIMEFRAME, TRAINING_DAYS, REGION, DATA_PROVIDER
+
 
 app = Flask(__name__)
 
 
 def update_data():
     """Download price data, format data and train model."""
-    download_data()
-    format_data()
-    train_model()
-
-
-def get_eth_inference():
-    """Load model and predict current price."""
-    with open(model_file_path, "rb") as f:
-        loaded_model = pickle.load(f)
-
-    now_timestamp = pd.Timestamp(datetime.now()).timestamp()
-    X_new = np.array([now_timestamp]).reshape(-1, 1)
-    current_price_pred = loaded_model.predict(X_new)
-
-    return current_price_pred[0]
+    files = download_data(TOKEN, TRAINING_DAYS, REGION, DATA_PROVIDER)
+    format_data(files, DATA_PROVIDER)
+    train_model(TIMEFRAME)
 
 
 @app.route("/inference/<string:token>")
 def generate_inference(token):
     """Generate inference for given token."""
-    if not token or token != "ETH":
+    if not token or token.upper() != TOKEN:
         error_msg = "Token is required" if not token else "Token not supported"
         return Response(json.dumps({"error": error_msg}), status=400, mimetype='application/json')
 
     try:
-        inference = get_eth_inference()
+        inference = get_inference(token.upper(), TIMEFRAME, REGION, DATA_PROVIDER)
         return Response(str(inference), status=200)
     except Exception as e:
         return Response(json.dumps({"error": str(e)}), status=500, mimetype='application/json')
@@ -55,4 +40,4 @@ def update():
 
 if __name__ == "__main__":
     update_data()
-    app.run(host="0.0.0.0", port=8000)
+    app.run(host="0.0.0.0", port=8000)

config.json

@@ -3,39 +3,21 @@
         "addressKeyName": "###WALLET###",
         "addressRestoreMnemonic": "###MNEMONIC###",
         "alloraHomeDir": "",
-        "gas": "1000000",
-        "gasAdjustment": 1.0,
+        "gas": "auto",
+        "gasAdjustment": 1.5,
         "nodeRpc": "###RPC_URL###",
         "maxRetries": 10,
-        "delay": 30,
-        "submitTx": false
+        "delay": 20,
+        "submitTx": true
     },
     "worker": [
         {
-            "topicId": 1,
+            "topicId": ###TOPIC###,
             "inferenceEntrypointName": "api-worker-reputer",
             "loopSeconds": 5,
             "parameters": {
                 "InferenceEndpoint": "http://inference:8000/inference/{Token}",
-                "Token": "ETH"
-            }
-        },
-        {
-            "topicId": 2,
-            "inferenceEntrypointName": "api-worker-reputer",
-            "loopSeconds": 5,
-            "parameters": {
-                "InferenceEndpoint": "http://inference:8000/inference/{Token}",
-                "Token": "ETH"
-            }
-        },
-        {
-            "topicId": 7,
-            "inferenceEntrypointName": "api-worker-reputer",
-            "loopSeconds": 5,
-            "parameters": {
-                "InferenceEndpoint": "http://inference:8000/inference/{Token}",
-                "Token": "ETH"
+                "Token": "###TOKEN###"
             }
         }
     ]

config.py

@@ -1,5 +1,21 @@
 import os
+from dotenv import load_dotenv
+
+# Load environment variables from .env file
+load_dotenv()
 
 app_base_path = os.getenv("APP_BASE_PATH", default=os.getcwd())
 data_base_path = os.path.join(app_base_path, "data")
 model_file_path = os.path.join(data_base_path, "model.pkl")
+
+TOKEN = os.getenv("TOKEN").upper()
+TRAINING_DAYS = os.getenv("TRAINING_DAYS")
+TIMEFRAME = os.getenv("TIMEFRAME")
+MODEL = os.getenv("MODEL")
+REGION = os.getenv("REGION").lower()
+if REGION in ["us", "com", "usa"]:
+    REGION = "us"
+else:
+    REGION = "com"
+DATA_PROVIDER = os.getenv("DATA_PROVIDER").lower()
+CG_API_KEY = os.getenv("CG_API_KEY", default=None)

docker-compose.yml

@@ -1,21 +1,22 @@
 services:
   inference:
-    container_name: inference-basic-eth-pred
+    container_name: inference
+    env_file:
+      - .env
     build: .
     command: python -u /app/app.py
     ports:
       - "8000:8000"
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:8000/inference/ETH"]
+      test: ["CMD", "curl", "-f", "http://inference:8000/inference/${TOKEN}"]
       interval: 10s
       timeout: 5s
       retries: 12
     volumes:
       - ./inference-data:/app/data
-    restart: always
-
+  
   updater:
-    container_name: updater-basic-eth-pred
+    container_name: updater
     build: .
     environment:
       - INFERENCE_API_ADDRESS=http://inference:8000
@@ -29,11 +30,10 @@ services:
     depends_on:
       inference:
         condition: service_healthy
-    restart: always
 
   worker:
     container_name: worker
-    image: alloranetwork/allora-offchain-node:latest
+    image: alloranetwork/allora-offchain-node:v0.3.0
     volumes:
       - ./worker-data:/data
     depends_on:
@@ -41,7 +41,6 @@ services:
         condition: service_healthy
     env_file:
       - ./worker-data/env_file
-    restart: always
 
 volumes:
   inference-data:

init.config

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 
 set -e
 
@@ -25,7 +25,7 @@ if [ -n "$mnemonic" ]; then
     echo "NAME=$nodeName" >> ./worker-data/env_file
     echo "ENV_LOADED=true" >> ./worker-data/env_file
     echo "wallet mnemonic already provided by you, loading config.json . Please proceed to run docker compose"
-    exit 0
+    exit 1
 fi
 
 if [ ! -f ./worker-data/env_file ]; then
@@ -36,7 +36,7 @@ ENV_LOADED=$(grep '^ENV_LOADED=' ./worker-data/env_file | cut -d '=' -f 2)
 if [ "$ENV_LOADED" = "false" ]; then
     json_content=$(cat ./config.json)
     stringified_json=$(echo "$json_content" | jq -c .)
-    docker run -it --entrypoint=bash -v $(pwd)/worker-data:/data -v $(pwd)/scripts:/scripts -e NAME="${nodeName}" -e ALLORA_OFFCHAIN_NODE_CONFIG_JSON="${stringified_json}" alloranetwork/allora-chain:latest -c "bash /scripts/init.sh"
+    docker run -it --entrypoint=bash -v $(pwd)/worker-data:/data -v $(pwd)/scripts:/scripts -e NAME="${nodeName}" -e ALLORA_OFFCHAIN_NODE_CONFIG_JSON="${stringified_json}" alloranetwork/allora-chain:v0.4.0 -c "bash /scripts/init.sh"
     echo "config.json saved to ./worker-data/env_file"
 else
     echo "config.json is already loaded, skipping the operation. You can set ENV_LOADED variable to false in ./worker-data/env_file to reload the config.json"

logs_parser.py

@@ -47,6 +47,9 @@ def parse_logs(timeout):
                         if current_retry == max_retry:
                             print(f"Max Retry Reached: {data}", flush=True)
                             return False, "Max Retry Reached"
+                    elif data.get("message") == "Error getting latest open worker nonce on topic":
+                        print(f"Error: {data}", flush=True)
+                        return False, "Error getting latest open worker nonce on topic"
         except Exception as e:
             print(f"Exception occurred: {e}", flush=True)
 

model.py

@@ -1,101 +1,148 @@
+import json
 import os
 import pickle
 from zipfile import ZipFile
-from datetime import datetime
 import pandas as pd
-import numpy as np
-from sklearn.model_selection import train_test_split
-from sklearn import linear_model
-from updater import download_binance_monthly_data, download_binance_daily_data
-from config import data_base_path, model_file_path
+from sklearn.kernel_ridge import KernelRidge
+from sklearn.linear_model import BayesianRidge, LinearRegression
+from sklearn.svm import SVR
+from updater import download_binance_daily_data, download_binance_current_day_data, download_coingecko_data, download_coingecko_current_day_data
+from config import data_base_path, model_file_path, TOKEN, MODEL, CG_API_KEY
 
 
-binance_data_path = os.path.join(data_base_path, "binance/futures-klines")
-training_price_data_path = os.path.join(data_base_path, "eth_price_data.csv")
+binance_data_path = os.path.join(data_base_path, "binance")
+coingecko_data_path = os.path.join(data_base_path, "coingecko")
+training_price_data_path = os.path.join(data_base_path, "price_data.csv")
 
 
-def download_data():
-    cm_or_um = "um"
-    symbols = ["ETHUSDT"]
-    intervals = ["1d"]
-    years = ["2020", "2021", "2022", "2023", "2024"]
-    months = ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12"]
-    download_path = binance_data_path
-    download_binance_monthly_data(
-        cm_or_um, symbols, intervals, years, months, download_path
-    )
-    print(f"Downloaded monthly data to {download_path}.")
-    current_datetime = datetime.now()
-    current_year = current_datetime.year
-    current_month = current_datetime.month
-    download_binance_daily_data(
-        cm_or_um, symbols, intervals, current_year, current_month, download_path
-    )
-    print(f"Downloaded daily data to {download_path}.")
+def download_data_binance(token, training_days, region):
+    files = download_binance_daily_data(f"{token}USDT", training_days, region, binance_data_path)
+    print(f"Downloaded {len(files)} new files")
+    return files
+
+def download_data_coingecko(token, training_days):
+    files = download_coingecko_data(token, training_days, coingecko_data_path, CG_API_KEY)
+    print(f"Downloaded {len(files)} new files")
+    return files
 
 
-def format_data():
-    files = sorted([x for x in os.listdir(binance_data_path)])
+def download_data(token, training_days, region, data_provider):
+    if data_provider == "coingecko":
+        return download_data_coingecko(token, int(training_days))
+    elif data_provider == "binance":
+        return download_data_binance(token, training_days, region)
+    else:
+        raise ValueError("Unsupported data provider")
+    
+def format_data(files, data_provider):
+    if not files:
+        print("Already up to date")
+        return
+    
+    if data_provider == "binance":
+        files = sorted([x for x in os.listdir(binance_data_path) if x.startswith(f"{TOKEN}USDT")])
+    elif data_provider == "coingecko":
+        files = sorted([x for x in os.listdir(coingecko_data_path) if x.endswith(".json")])
 
     # No files to process
     if len(files) == 0:
         return
 
     price_df = pd.DataFrame()
-    for file in files:
-        zip_file_path = os.path.join(binance_data_path, file)
+    if data_provider == "binance":
+        for file in files:
+            zip_file_path = os.path.join(binance_data_path, file)
 
-        if not zip_file_path.endswith(".zip"):
-            continue
+            if not zip_file_path.endswith(".zip"):
+                continue
 
-        myzip = ZipFile(zip_file_path)
-        with myzip.open(myzip.filelist[0]) as f:
-            line = f.readline()
-            header = 0 if line.decode("utf-8").startswith("open_time") else None
-        df = pd.read_csv(myzip.open(myzip.filelist[0]), header=header).iloc[:, :11]
-        df.columns = [
-            "start_time",
-            "open",
-            "high",
-            "low",
-            "close",
-            "volume",
-            "end_time",
-            "volume_usd",
-            "n_trades",
-            "taker_volume",
-            "taker_volume_usd",
-        ]
-        df.index = [pd.Timestamp(x + 1, unit="ms") for x in df["end_time"]]
-        df.index.name = "date"
-        price_df = pd.concat([price_df, df])
+            myzip = ZipFile(zip_file_path)
+            with myzip.open(myzip.filelist[0]) as f:
+                line = f.readline()
+                header = 0 if line.decode("utf-8").startswith("open_time") else None
+            df = pd.read_csv(myzip.open(myzip.filelist[0]), header=header).iloc[:, :11]
+            df.columns = [
+                "start_time",
+                "open",
+                "high",
+                "low",
+                "close",
+                "volume",
+                "end_time",
+                "volume_usd",
+                "n_trades",
+                "taker_volume",
+                "taker_volume_usd",
+            ]
+            df.index = [pd.Timestamp(x + 1, unit="ms").to_datetime64() for x in df["end_time"]]
+            df.index.name = "date"
+            price_df = pd.concat([price_df, df])
 
-    price_df.sort_index().to_csv(training_price_data_path)
+            price_df.sort_index().to_csv(training_price_data_path)
+    elif data_provider == "coingecko":
+        for file in files:
+            with open(os.path.join(coingecko_data_path, file), "r") as f:
+                data = json.load(f)
+                df = pd.DataFrame(data)
+                df.columns = [
+                    "timestamp",
+                    "open",
+                    "high",
+                    "low",
+                    "close"
+                ]
+                df["date"] = pd.to_datetime(df["timestamp"], unit="ms")
+                df.drop(columns=["timestamp"], inplace=True)
+                df.set_index("date", inplace=True)
+                price_df = pd.concat([price_df, df])
+
+            price_df.sort_index().to_csv(training_price_data_path)
 
 
-def train_model():
-    # Load the eth price data
+def load_frame(frame, timeframe):
+    print(f"Loading data...")
+    df = frame.loc[:,['open','high','low','close']].dropna()
+    df[['open','high','low','close']] = df[['open','high','low','close']].apply(pd.to_numeric)
+    df['date'] = frame['date'].apply(pd.to_datetime)
+    df.set_index('date', inplace=True)
+    df.sort_index(inplace=True)
+
+    return df.resample(f'{timeframe}', label='right', closed='right', origin='end').mean()
+
+def train_model(timeframe):
+    # Load the price data
     price_data = pd.read_csv(training_price_data_path)
-    df = pd.DataFrame()
+    df = load_frame(price_data, timeframe)
 
-    # Convert 'date' to a numerical value (timestamp) we can use for regression
-    df["date"] = pd.to_datetime(price_data["date"])
-    df["date"] = df["date"].map(pd.Timestamp.timestamp)
 
-    df["price"] = price_data[["open", "close", "high", "low"]].mean(axis=1)
+    if df.empty:
+        raise ValueError("No data available after loading and formatting. Check the data source or timeframe.")
 
-    # Reshape the data to the shape expected by sklearn
-    x = df["date"].values.reshape(-1, 1)
-    y = df["price"].values.reshape(-1, 1)
+    print(df.tail())
 
-    # Split the data into training set and test set
-    x_train, _, y_train, _ = train_test_split(x, y, test_size=0.2, random_state=0)
+    y_train = df['close'].shift(-1).dropna().values
+    X_train = df[:-1]
 
+    if X_train.empty or len(y_train) == 0:
+        raise ValueError("Training data is empty. Ensure there is enough data for training.")
+
+    print(f"Training data shape: {X_train.shape}, {y_train.shape}")
+
+    # Define the model
+    if MODEL == "LinearRegression":
+        model = LinearRegression()
+    elif MODEL == "SVR":
+        model = SVR()
+    elif MODEL == "KernelRidge":
+        model = KernelRidge()
+    elif MODEL == "BayesianRidge":
+        model = BayesianRidge()
+    # Add more models here
+    else:
+        raise ValueError("Unsupported model")
+    
     # Train the model
-    print("Training model...")
-    model = linear_model.Lasso(alpha=0.1)
-    model.fit(x_train, y_train)
-    print("Model trained.")
+    model.fit(X_train, y_train)
 
     # create the model's parent directory if it doesn't exist
     os.makedirs(os.path.dirname(model_file_path), exist_ok=True)
@@ -104,4 +151,23 @@ def train_model():
     with open(model_file_path, "wb") as f:
         pickle.dump(model, f)
 
-    print(f"Trained model saved to {model_file_path}")
+    print(f"Trained model saved to {model_file_path}")
+
+
+def get_inference(token, timeframe, region, data_provider):
+    """Load model and predict current price."""
+    with open(model_file_path, "rb") as f:
+        loaded_model = pickle.load(f)
+
+    # Get current price
+    if data_provider == "coingecko":
+        X_new = load_frame(download_coingecko_current_day_data(token, CG_API_KEY), timeframe)
+    else:
+        X_new = load_frame(download_binance_current_day_data(f"{TOKEN}USDT", region), timeframe)
+    
+    print(X_new.tail())
+    print(X_new.shape)
+
+    current_price_pred = loaded_model.predict(X_new)
+
+    return current_price_pred[0]

playbook.yml

@@ -0,0 +1,470 @@
+- name: Allora deployment playbook
+  hosts: all
+  become: true
+  vars:
+    ansible_python_interpreter: /usr/bin/python3.11
+    ipfs_url: https://bafybeigpiwl3o73zvvl6dxdqu7zqcub5mhg65jiky2xqb4rdhfmikswzqm.ipfs.w3s.link/manifest.json
+
+  tasks:
+    - name: Append command to .bash_history
+      ansible.builtin.blockinfile:
+        path: "~/.bash_history"
+        create: true
+        block: |
+          #1724983098
+          cd basic-coin-prediction-node/ ; docker compose logs -f
+          #1724983099
+          docker logs worker -f
+          cd basic-coin-prediction-node/ ; docker compose up 
+        marker: ""
+        mode: '0644'
+
+    - name: Set locale to C.UTF-8
+      ansible.builtin.command:
+        cmd: localectl set-locale LANG=C.UTF-8
+      changed_when: false
+
+    - name: Create APT configuration file to assume yes
+      ansible.builtin.copy:
+        dest: /etc/apt/apt.conf.d/90forceyes
+        content: |
+          APT::Get::Assume-Yes "true";
+        mode: '0644'
+
+    - name: Update /etc/bash.bashrc
+      ansible.builtin.blockinfile:
+        path: /etc/bash.bashrc
+        block: |
+          export HISTTIMEFORMAT='%F, %T '
+          export HISTSIZE=10000
+          export HISTFILESIZE=10000
+          shopt -s histappend
+          export PROMPT_COMMAND='history -a'
+          export HISTCONTROL=ignoredups
+          export LANG=C.UTF-8
+          export LC_ALL=C.UTF-8
+          alias ls='ls --color=auto'
+          shopt -s cmdhist
+
+    - name: Ensure ~/.inputrc exists
+      ansible.builtin.file:
+        path: /root/.inputrc
+        state: touch
+        mode: '0644'
+
+    - name: Update ~/.inputrc
+      ansible.builtin.blockinfile:
+        path: ~/.inputrc
+        block: |
+          "\e[A": history-search-backward
+          "\e[B": history-search-forward
+
+    - name: Ensure ~/.nanorc exists
+      ansible.builtin.file:
+        path: /root/.nanorc
+        state: touch
+        mode: '0644'
+
+    - name: Update ~/.nanorc
+      ansible.builtin.blockinfile:
+        path: ~/.nanorc
+        block: |
+          set nohelp
+          set tabsize 4
+          set tabstospaces
+          set autoindent
+          set positionlog
+          set backup
+          set backupdir /tmp/
+          set locking
+          include /usr/share/nano/*.nanorc
+
+    - name: Set hostname
+      ansible.builtin.shell: |
+        hostnamectl set-hostname {{ serverid }}
+        echo "127.0.1.1 {{ serverid }}" >> /etc/hosts
+      changed_when: false
+
+    - name: Update apt cache
+      ansible.builtin.apt:
+        update_cache: true
+      register: apt_update_result
+      retries: 5
+      delay: 50
+      until: apt_update_result is succeeded
+
+    - name: Upgrade packages
+      ansible.builtin.apt:
+        upgrade: dist
+        force_apt_get: true
+        autoremove: true
+      register: apt_upgrade_result
+      retries: 5
+      delay: 50
+      until: apt_upgrade_result is succeeded
+
+#    - name: Install packages
+#      ansible.builtin.apt:
+#        name:
+#          - ca-certificates
+#          - zlib1g-dev
+#          - libncurses5-dev
+#          - libgdbm-dev
+#          - libnss3-dev
+#          - curl
+#          - jq
+#          - git
+#          - zip
+#          - wget
+#          - make
+#          - python3
+#          - python3-pip
+#          - iftop
+#        state: present
+#        update_cache: true
+#      async: "{{ 60 * 20 }}"
+#      poll: 30
+
+#    - name: Check no-proxy ipfs access
+#      ansible.builtin.shell: |
+#        curl -s -w "%{http_code}" -o response.json {{ ipfs_url }}
+#      register: noproxy_check
+#      changed_when: false
+#      failed_when: noproxy_check.stdout != "200"
+#
+#    - name: Check proxy ipfs access
+#      ansible.builtin.shell: |
+#        curl -s -w "%{http_code}" -o response.json -x {{ proxy }} {{ ipfs_url }}
+#      register: proxy_check
+#      changed_when: false
+#      failed_when: proxy_check.stdout != "200"
+
+#    - name: Install Docker
+#      ansible.builtin.shell: curl -fsSL https://get.docker.com | bash
+#      changed_when: false
+#      async: "{{ 60 * 5 }}"
+#      poll: 30
+
+#    - name: Update Docker daemon journald logging
+#      ansible.builtin.copy:
+#        dest: /etc/docker/daemon.json
+#        content: |
+#          {
+#            "log-driver": "journald"
+#          }
+#        mode: '0644'
+#
+#    - name: Restart Docker
+#      ansible.builtin.service:
+#        name: docker
+#        state: restarted
+#
+#    - name: Update journald log SystemMaxUse=2G configuration
+#      ansible.builtin.lineinfile:
+#        path: /etc/systemd/journald.conf
+#        line: 'SystemMaxUse=2G'
+#        insertafter: EOF
+#        create: true
+#        mode: '0644'
+#
+#    - name: Restart journald
+#      ansible.builtin.service:
+#        name: systemd-journald
+#        state: restarted
+
+    - name: Docker login
+      ansible.builtin.shell: docker login -u "{{ docker_username }}" -p "{{ docker_password }}"
+      register: docker_login_result
+      changed_when: false
+      failed_when: "'Login Succeeded' not in docker_login_result.stdout"
+
+    - name: Clone repository
+      ansible.builtin.git:
+        repo: https://gitea.vvzvlad.xyz/vvzvlad/allora
+        dest: "{{ ansible_env.HOME }}/basic-coin-prediction-node"
+        version: "{{ git_version }}"
+        force: true
+      async: "{{ 60 * 15 }}"
+      poll: 30
+
+    - name: Update environment variables
+      ansible.builtin.shell: |
+        ./update.sh WALLET "{{ wallet }}"
+        ./update.sh MNEMONIC "{{ mnemonic }}"
+        ./update.sh RPC_URL "{{ rpc_url }}"
+        ./update.sh TOKEN "{{ token }}"
+        ./update.sh TOPIC "{{ topic }}"
+        ./update.sh TRAINING_DAYS "{{ training_days }}"
+        ./update.sh TIMEFRAME "{{ timeframe }}"
+        ./update.sh MODEL "{{ model }}"
+        ./update.sh DATA_PROVIDER "{{ data_provider }}"
+        ./update.sh CG_API_KEY "{{ cg_api_key }}"
+      args:
+        chdir: "{{ ansible_env.HOME }}/basic-coin-prediction-node"
+      changed_when: false
+
+    - name: Init config
+      ansible.builtin.shell: ./init.config ; true
+      args:
+        chdir: "{{ ansible_env.HOME }}/basic-coin-prediction-node"
+      changed_when: false
+
+    - name: Build docker compose
+      ansible.builtin.command: docker compose build -q
+      args:
+        chdir: "{{ ansible_env.HOME }}/basic-coin-prediction-node"
+      environment:
+        COMPOSE_INTERACTIVE_NO_CLI: 'true'
+      changed_when: false
+      async: "{{ 60 * 45 }}"
+      poll: "{{ 60 * 5 }}"
+
+#    - name: Docker pre-up
+#      ansible.builtin.command: docker compose up -d
+#      args:
+#        chdir: "{{ ansible_env.HOME }}/basic-coin-prediction-node"
+#      environment:
+#        COMPOSE_INTERACTIVE_NO_CLI: 'true'
+#      changed_when: false
+#      async: "{{ 60 * 80 }}"
+#      poll: "{{ 60 * 5 }}"
+
+#    - name: Check Docker container status
+#      ansible.builtin.shell: >
+#        if [ $(docker ps -q | wc -l) -eq $(docker ps -a -q | wc -l) ]; then
+#          echo "all_running";
+#        else
+#          echo "not_all_running";
+#        fi
+#      register: container_status
+#      retries: 10
+#      delay: 30
+#      until: container_status.stdout.find("all_running") != -1
+#
+#    - name: Docker stop (pre-up)
+#      ansible.builtin.command: docker compose stop
+#      args:
+#        chdir: "{{ ansible_env.HOME }}/basic-coin-prediction-node"
+#      environment:
+#        COMPOSE_INTERACTIVE_NO_CLI: 'true'
+#      changed_when: false
+#
+#    - name: Check external IP before
+#      ansible.builtin.command: curl https://ifconfig.me
+#      register: ip_before
+#      changed_when: false
+#
+#    - name: Validate IP address
+#      ansible.builtin.assert:
+#        that:
+#          - ip_before.stdout | ansible.utils.ipaddr
+#        fail_msg: "The returned value is not a valid IP address."
+#        success_msg: "The returned value is a valid IP address."
+
+#    - name: Download tun2socks
+#      ansible.builtin.get_url:
+#        url: https://github.com/xjasonlyu/tun2socks/releases/download/v2.5.2/tun2socks-linux-amd64.zip
+#        dest: /tmp/tun2socks-linux-amd64.zip
+#        mode: '0644'
+#      async: "{{ 60 * 5 }}"
+#      poll: 30
+#
+#    - name: Unzip tun2socks
+#      ansible.builtin.unarchive:
+#        src: /tmp/tun2socks-linux-amd64.zip
+#        dest: /usr/local/sbin/
+#        remote_src: true
+#        mode: '0755'
+#
+#    - name: Create proxy file
+#      ansible.builtin.copy:
+#        content: "{{ proxy }}"
+#        dest: /root/proxy
+#        mode: '0644'
+#
+#    - name: Create tun2socks systemd service
+#      ansible.builtin.copy:
+#        dest: /etc/systemd/system/tun2socks.service
+#        content: |
+#          [Unit]
+#          Description=Tun2Socks gateway
+#          After=network.target
+#          Wants=network.target
+#
+#          [Service]
+#          User=root
+#          Type=simple
+#          RemainAfterExit=true
+#          ExecStartPre=/bin/sh -c 'ip route add $(cat /root/proxy | grep -oP "(?<=@)[0-9.]+(?=:)" )/32 via $(ip route | grep -oP "(?<=default via )[0-9.]+")'
+#          ExecStart=/bin/sh -c '/usr/local/sbin/tun2socks-linux-amd64 --device tun0 --proxy $(cat /root/proxy)'
+#          ExecStopPost=/bin/sh -c 'ip route del $(cat /root/proxy | grep -oP "(?<=@)[0-9.]+(?=:)" )/32 via $(ip route | grep -oP "(?<=default via )[0-9.]+")'
+#          Restart=always
+#
+#          [Install]
+#          WantedBy=multi-user.target
+#        mode: '0644'
+#
+#    - name: Create network configuration for tun0
+#      ansible.builtin.copy:
+#        dest: /etc/systemd/network/10-proxy.network
+#        content: |
+#          [Match]
+#          Name=tun0
+#
+#          [Network]
+#          Address=10.20.30.1/24
+#
+#          [Route]
+#          Gateway=0.0.0.0
+#        mode: '0644'
+#
+#    - name: Enable and start tun2socks service
+#      ansible.builtin.systemd:
+#        name: tun2socks
+#        enabled: true
+#        state: started
+#
+#    - name: Reload network configuration
+#      ansible.builtin.command: networkctl reload
+#      changed_when: false
+#
+#    - name: Restart tun2socks service
+#      ansible.builtin.systemd:
+#        name: tun2socks
+#        state: restarted
+
+    - name: Check RPC availability
+      ansible.builtin.uri:
+        url: "{{ rpc_url }}/health?"
+        method: GET
+        return_content: true
+        timeout: 30
+      register: rpc_url_response
+      retries: 3
+      delay: 120
+      failed_when:
+        - rpc_url_response.status != 200
+        - rpc_url_response.json is not none and rpc_url_response.json is not defined
+
+    - name: Check Binance URL availability
+      ansible.builtin.uri:
+        url: "https://api.binance.com/api/v3/klines?symbol=BTCUSDT&interval=1M&limit=1"
+        method: GET
+        return_content: true
+      register: binance_url_response
+      retries: 3
+      delay: 60
+      failed_when:
+        - binance_url_response.status != 200
+        - binance_url_response.json is not none and binance_url_response.json is not defined
+
+#    - name: Get balance for the wallet
+#      retries: 3
+#      delay: 30
+#      ansible.builtin.shell: |
+#        response=$(curl --silent --location --request GET "https://allora-api.testnet.allora.network/cosmos/bank/v1beta1/balances/{{ wallet }}") && \
+#        echo "$response" && \
+#        uallo_balance=$(echo "$response" | jq -r '.balances[] | select(.denom == "uallo") | .amount // 0') && \
+#        echo "uallo_balance: $uallo_balance" && \
+#        if [ "$uallo_balance" -gt 100000 ]; then
+#          echo "Balance {{ wallet }} > 100000"
+#        else
+#          echo "Balance {{ wallet }} <  100000"
+#          exit 1
+#        fi
+#      register: wallet_balance_check
+#      failed_when: wallet_balance_check.rc != 0
+
+#    - name: Check external IP after
+#      ansible.builtin.command: curl https://ifconfig.me
+#      register: ip_after
+#      changed_when: false
+#
+#    - name: Validate IP address
+#      ansible.builtin.assert:
+#        that:
+#          - ip_after.stdout | ansible.utils.ipaddr
+#        fail_msg: "The returned value is not a valid IP address."
+#        success_msg: "The returned value is a valid IP address."
+#
+#    - name: Show IPs
+#      ansible.builtin.debug:
+#        msg: "External IP before: {{ ip_before.stdout }}, External IP after: {{ ip_after.stdout }}"
+#
+#    - name: Compare external IPs
+#      ansible.builtin.fail:
+#        msg: "External IP before and after should not be the same"
+#      when: ip_before.stdout == ip_after.stdout
+
+    - name: Docker up
+      ansible.builtin.command: docker compose up -d
+      args:
+        chdir: "{{ ansible_env.HOME }}/basic-coin-prediction-node"
+      environment:
+        COMPOSE_INTERACTIVE_NO_CLI: 'true'
+      changed_when: false
+      async: "{{ 60 * 80 }}"
+      poll: "{{ 60 * 5 }}"
+
+    - name: Check Docker containers status
+      ansible.builtin.shell: >
+        if [ $(docker ps -q | wc -l) -eq $(docker ps -a -q | wc -l) ]; then
+          echo "all_running";
+        else
+          echo "not_all_running";
+        fi
+      register: container_status
+      retries: 10
+      delay: 30
+      until: container_status.stdout.find("all_running") != -1
+
+    - name: Check "not have enough balance"
+      ansible.builtin.command: docker logs {{ item }} 2>&1
+      register: docker_logs_check
+      changed_when: false
+      failed_when: '"not have enough balance" in docker_logs_check.stdout'
+      with_items:
+        - worker
+        - worker-1
+        - worker-2
+
+    - name: Check updater endpoint
+      ansible.builtin.shell: |
+        response=$(curl --silent --location --request GET http://localhost:8000/update) && \
+        if [ "$response" != "0" ]; then
+          echo "Updater endpoint check failed: $response != 0"
+          exit 1
+        fi
+      register: updater_shell_response
+      retries: 2
+      delay: 60
+      until: updater_shell_response.rc == 0
+      changed_when: false
+
+    - name: Check inference endpoint
+      ansible.builtin.shell: |
+        response=$(curl --silent --location --request GET http://localhost:8000/inference/{{ token }}) && \
+        status=$(curl -o /dev/null -s -w "%{http_code}\\n" http://localhost:8000/inference/{{ token }}) && \
+        if [ "$status" -ne 200 ] || ! echo "$response" | grep -qE '^[0-9]+(\.[0-9]+)?$'; then
+          echo "Inference endpoint check failed: status $status, response $response"
+          exit 1
+        fi
+      register: inference_shell_response
+      retries: 2
+      delay: 60
+      failed_when: inference_shell_response.rc != 0
+      changed_when: false
+
+#    - name: Wait success send
+#      ansible.builtin.shell: |
+#        python3 logs_parser.py 80
+#      args:
+#        chdir: "{{ ansible_env.HOME }}/basic-coin-prediction-node"
+#      register: docker_logs_check
+#      changed_when: false
+#      failed_when: docker_logs_check.rc != 0
+
+    - name: Remove docker login credentials
+      ansible.builtin.file:
+        path: /root/.docker/config.json
+        state: absent

requirements.txt

@@ -1,7 +1,9 @@
 flask[async]
 gunicorn[gthread]
-numpy==1.26.2
-pandas==2.1.3
-Requests==2.32.0
-scikit_learn==1.3.2
-werkzeug>=3.0.3 # not directly required, pinned by Snyk to avoid a vulnerability
+numpy
+pandas
+Requests
+aiohttp
+multiprocess
+scikit_learn
+python-dotenv

update.sh

@@ -1,22 +1,20 @@
 #!/usr/bin/env bash
 
-if [ "$#" -ne 3 ]; then
-    echo "Usage: $0 <mnemonic> <wallet> <rpc_url>"
+if [ "$#" -ne 2 ]; then
+    echo "Usage: $0 <PARAMETER> <NEW_VALUE>"
     exit 1
 fi
 
-MNEMONIC=$1
-WALLET=$2
-RPC_URL=$3
+PARAMETER=$1
+NEW_VALUE=$2
 
-# List of files
+# Список файлов
 FILES=(
     "./config.json"
+    ".env"
 )
 
 for FILE in "${FILES[@]}"; do
     EXPANDED_FILE=$(eval echo "$FILE")
-    sed -i "s|###MNEMONIC###|$MNEMONIC|g" "$EXPANDED_FILE"
-    sed -i "s|###WALLET###|$WALLET|g" "$EXPANDED_FILE"
-    sed -i "s|###RPC_URL###|$RPC_URL|g" "$EXPANDED_FILE"
-done
+    sed -i "s|###$PARAMETER###|$NEW_VALUE|g" "$EXPANDED_FILE"
+done

updater.py

@@ -1,59 +1,175 @@
 import os
+from datetime import date, timedelta
+import pathlib
+import time
 import requests
+from requests.adapters import HTTPAdapter
+from urllib3.util import Retry
 from concurrent.futures import ThreadPoolExecutor
+import pandas as pd
+import json
+
+
+# Define the retry strategy
+retry_strategy = Retry(
+    total=4,  # Maximum number of retries
+    backoff_factor=2, # Exponential backoff factor (e.g., 2 means 1, 2, 4, 8 seconds, ...)
+    status_forcelist=[429, 500, 502, 503, 504],  # HTTP status codes to retry on
+)
+
+# Create an HTTP adapter with the retry strategy and mount it to session
+adapter = HTTPAdapter(max_retries=retry_strategy)
+
+# Create a new session object
+session = requests.Session()
+session.mount('http://', adapter)
+session.mount('https://', adapter)
+
+
+files = []
 
 
 # Function to download the URL, called asynchronously by several child processes
-def download_url(url, download_path):
-    target_file_path = os.path.join(download_path, os.path.basename(url)) 
-    if os.path.exists(target_file_path):
-        # print(f"File already exists: {url}")
-        return
+def download_url(url, download_path, name=None):
+    try:
+        global files
+        if name:
+            file_name = os.path.join(download_path, name)
+        else:
+            file_name = os.path.join(download_path, os.path.basename(url))
+        dir_path = os.path.dirname(file_name)
+        pathlib.Path(dir_path).mkdir(parents=True, exist_ok=True) 
+        if os.path.isfile(file_name):
+            # print(f"{file_name} already exists")
+            return
+        # Make a request using the session object
+        response = session.get(url)
+        if response.status_code == 404:
+            print(f"File does not exist: {url}")
+        elif response.status_code == 200:
+            with open(file_name, 'wb') as f:
+                f.write(response.content)
+            # print(f"Downloaded: {url} to {file_name}")
+            files.append(file_name)
+            return
+        else:
+            print(f"Failed to download {url}")
+            return
+    except Exception as e:
+        print(str(e))
+
+
+# Function to generate a range of dates
+def daterange(start_date, end_date):
+    for n in range(int((end_date - start_date).days)):
+        yield start_date + timedelta(n)
+
+
+# Function to download daily data from Binance
+def download_binance_daily_data(pair, training_days, region, download_path):
+    base_url = f"https://data.binance.vision/data/spot/daily/klines"
+
+    end_date = date.today()
+    start_date = end_date - timedelta(days=int(training_days))
     
-    response = requests.get(url)
-    if response.status_code == 404:
-        # print(f"File not exist: {url}")
-        pass
+    global files
+    files = []
+
+    with ThreadPoolExecutor() as executor:
+        print(f"Downloading data for {pair}")
+        for single_date in daterange(start_date, end_date):
+            url = f"{base_url}/{pair}/1m/{pair}-1m-{single_date}.zip"
+            executor.submit(download_url, url, download_path)
+    
+    return files
+
+
+def download_binance_current_day_data(pair, region):
+    limit = 1000
+    base_url = f'https://api.binance.{region}/api/v3/klines?symbol={pair}&interval=1m&limit={limit}'
+
+    # Make a request using the session object
+    response = session.get(base_url)
+    response.raise_for_status()
+    resp = str(response.content, 'utf-8').rstrip()
+
+    columns = ['start_time','open','high','low','close','volume','end_time','volume_usd','n_trades','taker_volume','taker_volume_usd','ignore']
+    
+    df = pd.DataFrame(json.loads(resp),columns=columns)
+    df['date'] = [pd.to_datetime(x+1,unit='ms') for x in df['end_time']]
+    df['date'] = df['date'].apply(pd.to_datetime)
+    df[["volume", "taker_volume", "open", "high", "low", "close"]] = df[["volume", "taker_volume", "open", "high", "low", "close"]].apply(pd.to_numeric)
+
+    return df.sort_index()
+
+
+def get_coingecko_coin_id(token):
+    token_map = {
+        'ETH': 'ethereum',
+        'SOL': 'solana',
+        'BTC': 'bitcoin',
+        'BNB': 'binancecoin',
+        'ARB': 'arbitrum',
+        # Add more tokens here
+    }
+    
+    token = token.upper()
+    if token in token_map:
+        return token_map[token]
     else:
-
-        # create the entire path if it doesn't exist
-        os.makedirs(os.path.dirname(target_file_path), exist_ok=True)
-
-        with open(target_file_path, "wb") as f:
-            f.write(response.content)
-        # print(f"Downloaded: {url} to {target_file_path}")
+        raise ValueError("Unsupported token")
 
 
-def download_binance_monthly_data(
-    cm_or_um, symbols, intervals, years, months, download_path
-):
-    # Verify if CM_OR_UM is correct, if not, exit
-    if cm_or_um not in ["cm", "um"]:
-        print("CM_OR_UM can be only cm or um")
-        return
-    base_url = f"https://data.binance.vision/data/futures/{cm_or_um}/monthly/klines"
+def download_coingecko_data(token, training_days, download_path, CG_API_KEY):
+    if training_days <= 7:
+        days = 7
+    elif training_days <= 14:
+        days = 14
+    elif training_days <= 30:
+        days = 30
+    elif training_days <= 90:
+        days = 90
+    elif training_days <= 180:
+        days = 180
+    elif training_days <= 365:
+        days = 365
+    else:
+        days = "max"
+    print(f"Days: {days}")
 
-    # Main loop to iterate over all the arrays and launch child processes
-    with ThreadPoolExecutor() as executor:
-        for symbol in symbols:
-            for interval in intervals:
-                for year in years:
-                    for month in months:
-                        url = f"{base_url}/{symbol}/{interval}/{symbol}-{interval}-{year}-{month}.zip"
-                        executor.submit(download_url, url, download_path)
+    coin_id = get_coingecko_coin_id(token)
+    print(f"Coin ID: {coin_id}")
 
+    # Get OHLC data from Coingecko
+    url = f'https://api.coingecko.com/api/v3/coins/{coin_id}/ohlc?vs_currency=usd&days={days}&api_key={CG_API_KEY}'
 
-def download_binance_daily_data(
-    cm_or_um, symbols, intervals, year, month, download_path
-):
-    if cm_or_um not in ["cm", "um"]:
-        print("CM_OR_UM can be only cm or um")
-        return
-    base_url = f"https://data.binance.vision/data/futures/{cm_or_um}/daily/klines"
+    global files
+    files = []
 
     with ThreadPoolExecutor() as executor:
-        for symbol in symbols:
-            for interval in intervals:
-                for day in range(1, 32):  # Assuming days range from 1 to 31
-                    url = f"{base_url}/{symbol}/{interval}/{symbol}-{interval}-{year}-{month:02d}-{day:02d}.zip"
-                    executor.submit(download_url, url, download_path)
+        print(f"Downloading data for {coin_id}")
+        name = os.path.basename(url).split("?")[0].replace("/", "_") + ".json"
+        executor.submit(download_url, url, download_path, name)
+    
+    return files
+
+
+def download_coingecko_current_day_data(token, CG_API_KEY):
+    coin_id = get_coingecko_coin_id(token)
+    print(f"Coin ID: {coin_id}")
+
+    url = f'https://api.coingecko.com/api/v3/coins/{coin_id}/ohlc?vs_currency=usd&days=1&api_key={CG_API_KEY}'
+
+    # Make a request using the session object
+    response = session.get(url)
+    response.raise_for_status()
+    resp = str(response.content, 'utf-8').rstrip()
+
+    columns = ['timestamp','open','high','low','close']
+    
+    df = pd.DataFrame(json.loads(resp), columns=columns)
+    df['date'] = [pd.to_datetime(x,unit='ms') for x in df['timestamp']]
+    df['date'] = df['date'].apply(pd.to_datetime)
+    df[["open", "high", "low", "close"]] = df[["open", "high", "low", "close"]].apply(pd.to_numeric)
+
+    return df.sort_index()