.env

@@ -1,7 +0,0 @@
-TOKEN=###TOKEN###
-TRAINING_DAYS=###TRAINING_DAYS###
-TIMEFRAME=###TIMEFRAME###
-MODEL=###MODEL###
-REGION=EU
-DATA_PROVIDER=###DATA_PROVIDER###
-CG_API_KEY=###CG_API_KEY###

.gitignore

@@ -1,19 +0,0 @@
-.DS_Store
-__pycache__
-*.pyc
-logs/*
-
-.allorad
-.cache
-inference-data
-worker-data
-
-/data
-
-**/*.venv*
-**/.cache
-**/env_file
-**/.gitkeep*
-**/*.csv
-**/*.pkl
-**/*.zip

CONTRIBUTING.md

@@ -1,18 +0,0 @@
-Any contribution that you make to this repository will
-be under the Apache 2 License, as dictated by that
-[license](http://www.apache.org/licenses/LICENSE-2.0.html):
-
-~~~
-5. Submission of Contributions. Unless You explicitly state otherwise,
-   any Contribution intentionally submitted for inclusion in the Work
-   by You to the Licensor shall be under the terms and conditions of
-   this License, without any additional terms or conditions.
-   Notwithstanding the above, nothing herein shall supersede or modify
-   the terms of any separate license agreement you may have executed
-   with Licensor regarding such Contributions.
-~~~
-
-Contributors must sign-off each commit by adding a `Signed-off-by: ...`
-line to commit messages to certify that they have the right to submit
-the code they are contributing to the project according to the
-[Developer Certificate of Origin (DCO)](https://developercertificate.org/).

Dockerfile

@@ -1,7 +1,5 @@
-FROM python:3.11-slim AS project_env
+# Use an official Python runtime as the base image
-
+FROM amd64/python:3.9-buster as project_env
-# Install curl
-RUN apt-get update && apt-get install -y curl
 
 # Set the working directory in the container
 WORKDIR /app

LICENSE

@@ -1,201 +0,0 @@
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.

README.md

@@ -1,112 +0,0 @@
-# Basic Price Prediction Node
-
-This repository provides an example [Allora network](https://docs.allora.network/) worker node, designed to offer price predictions. The primary objective is to demonstrate the use of a basic inference model running within a dedicated container, showcasing its integration with the Allora network infrastructure to contribute valuable inferences.
-
-## Components
-
-- **Worker**: The node that publishes inferences to the Allora chain.
-- **Inference**: A container that conducts inferences, maintains the model state, and responds to internal inference requests via a Flask application. This node operates with a basic linear regression model for price predictions.
-- **Updater**: A cron-like container designed to update the inference node's data by daily fetching the latest market information from the data provider, ensuring the model stays current with new market trends.
-
-Check the `docker-compose.yml` file for the detailed setup of each component.
-
-## Docker-Compose Setup
-
-A complete working example is provided in the `docker-compose.yml` file.
-
-### Steps to Setup
-
-1. **Clone the Repository**
-2. **Copy and Populate Model Configuration environment file**
-    
-    Copy the example .env.example file and populate it with your variables:
-    ```sh
-    cp .env.example .env
-    ```
-
-    Here are the currently accepted configurations:
-    - TOKEN
-    Must be one in ('ETH','SOL','BTC','BNB','ARB'). 
-    Note: if you are using `Binance` as the data provider, any token could be used.
-    If you are using Coingecko, you should add its `coin_id` in the [token_map here](https://github.com/allora-network/basic-coin-prediction-node/blob/main/updater.py#L107). Find [more info here](https://docs.coingecko.com/reference/simple-price) and the [list here](https://docs.google.com/spreadsheets/d/1wTTuxXt8n9q7C4NDXqQpI3wpKu1_5bGVmP9Xz0XGSyU/edit?usp=sharing).
-    - TRAINING_DAYS
-    Must be an `int` >= 1. 
-    Represents how many days of historical data to use. 
-    - TIMEFRAME
-    This should be in this form: `10min`, `1h`, `1d`, `1m`, etc.
-    Note: For Coingecko, Data granularity (candle's body) is automatic - [see here](https://docs.coingecko.com/reference/coins-id-ohlc). To avoid downsampling, it is recommanded to use with Coingecko:
-        - TIMEFRAME >= 30m if TRAINING_DAYS <= 2
-        - TIMEFRAME >= 4h if TRAINING_DAYS <= 30
-        - TIMEFRAME >= 4d if TRAINING_DAYS >= 31
-    - MODEL
-    Must be one in ('LinearRegression','SVR','KernelRidge','BayesianRidge'). 
-    You can easily add support for any other models by [adding it here](https://github.com/allora-network/basic-coin-prediction-node/blob/main/model.py#L133).
-    - REGION
-    Used for the Binance API. This should be in this form: `US`, `EU`, etc.
-    - DATA_PROVIDER
-    Must be `binance` or `coingecko`. Feel free to add support for other data providers to personalize your model!
-    - CG_API_KEY
-    This is your `Coingecko` API key, if you've set `DATA_PROVIDER=coingecko`.
-
-3. **Copy and Populate Worker Configuration**
-
-    Copy the example configuration file and populate it with your variables:
-    ```sh
-    cp config.example.json config.json
-    ```
-
-4. **Initialize Worker**
-    
-    Run the following commands from the project's root directory to initialize the worker:
-    ```sh
-    chmod +x init.config
-    ./init.config
-    ```
-    These commands will:
-    - Automatically create Allora keys for your worker.
-    - Export the needed variables from the created account to be used by the worker node, bundle them with your provided `config.json`, and pass them to the node as environment variables.
-
-5. **Faucet Your Worker Node**
-    
-    You can find the offchain worker node's address in `./worker-data/env_file` under `ALLORA_OFFCHAIN_ACCOUNT_ADDRESS`. [Add faucet funds](https://docs.allora.network/devs/get-started/setup-wallet#add-faucet-funds) to your worker's wallet before starting it.
-
-6. **Start the Services**
-    
-    Run the following command to start the worker node, inference, and updater nodes:
-    ```sh
-    docker compose up --build
-    ```
-    To confirm that the worker successfully sends the inferences to the chain, look for the following log:
-    ```
-    {"level":"debug","msg":"Send Worker Data to chain","txHash":<tx-hash>,"time":<timestamp>,"message":"Success"}
-    ```
-
-## Testing Inference Only
-
-This setup allows you to develop your model without the need to bring up the offchain worker or the updater. To test the inference model only:
-
-1. Run the following command to start the inference node:
-    ```sh
-    docker compose up --build inference
-    ```
-    Wait for the initial data load.
-
-2. Send requests to the inference model. For example, request ETH price inferences:
-    
-    ```sh
-    curl http://127.0.0.1:8000/inference/ETH
-    ```
-    Expected response:
-    ```json
-    {"value":"2564.021586281073"}
-    ```
-
-3. Update the node's internal state (download pricing data, train, and update the model):
-    
-    ```sh
-    curl http://127.0.0.1:8000/update
-    ```
-    Expected response:
-    ```sh
-    0
-    ```

app.py

@@ -1,31 +1,72 @@
 import json
-from flask import Flask, Response
+import pickle
-from model import download_data, format_data, train_model, get_inference
+import pandas as pd
-from config import model_file_path, TOKEN, TIMEFRAME, TRAINING_DAYS, REGION, DATA_PROVIDER
+import numpy as np
-
+from datetime import datetime
+from flask import Flask, jsonify, Response
+from model import download_data, format_data, train_model, training_price_data_path
+from config import model_file_path
 
 app = Flask(__name__)
 
 
 def update_data():
     """Download price data, format data and train model."""
-    files = download_data(TOKEN, TRAINING_DAYS, REGION, DATA_PROVIDER)
+    download_data()
-    format_data(files, DATA_PROVIDER)
+    format_data()
-    train_model(TIMEFRAME)
+    train_model()
+
+
+def get_eth_inference():
+    """Load model and predict current price."""
+    try:
+        with open(model_file_path, "rb") as f:
+            loaded_model = pickle.load(f)
+
+        # Загружаем последние данные из файла
+        price_data = pd.read_csv(training_price_data_path)
+
+        # Используем последние значения признаков для предсказания
+        X_new = (
+            price_data[
+                [
+                    "timestamp",
+                    "price_diff",
+                    "volatility",
+                    "volume",
+                    "moving_avg_7",
+                    "moving_avg_30",
+                ]
+            ]
+            .iloc[-1]
+            .values.reshape(1, -1)
+        )
+
+        # Делаем предсказание
+        current_price_pred = loaded_model.predict(X_new)
+
+        return current_price_pred[0]
+    except Exception as e:
+        print(f"Error during inference: {str(e)}")
+        raise
 
 
 @app.route("/inference/<string:token>")
 def generate_inference(token):
     """Generate inference for given token."""
-    if not token or token.upper() != TOKEN:
+    if not token or token != "ETH":
         error_msg = "Token is required" if not token else "Token not supported"
-        return Response(json.dumps({"error": error_msg}), status=400, mimetype='application/json')
+        return Response(
+            json.dumps({"error": error_msg}), status=400, mimetype="application/json"
+        )
 
     try:
-        inference = get_inference(token.upper(), TIMEFRAME, REGION, DATA_PROVIDER)
+        inference = get_eth_inference()
         return Response(str(inference), status=200)
     except Exception as e:
-        return Response(json.dumps({"error": str(e)}), status=500, mimetype='application/json')
+        return Response(
+            json.dumps({"error": str(e)}), status=500, mimetype="application/json"
+        )
 
 
 @app.route("/update")

config.json

@@ -3,21 +3,39 @@
         "addressKeyName": "###WALLET###",
         "addressRestoreMnemonic": "###MNEMONIC###",
         "alloraHomeDir": "",
-        "gas": "auto",
+        "gas": "1000000",
-        "gasAdjustment": 1.5,
+        "gasAdjustment": 1.0,
         "nodeRpc": "###RPC_URL###",
         "maxRetries": 10,
-        "delay": 20,
+        "delay": 30,
-        "submitTx": true
+        "submitTx": false
     },
     "worker": [
         {
-            "topicId": ###TOPIC###,
+            "topicId": 1,
             "inferenceEntrypointName": "api-worker-reputer",
             "loopSeconds": 5,
             "parameters": {
                 "InferenceEndpoint": "http://inference:8000/inference/{Token}",
-                "Token": "###TOKEN###"
+                "Token": "ETH"
+            }
+        },
+        {
+            "topicId": 2,
+            "inferenceEntrypointName": "api-worker-reputer",
+            "loopSeconds": 5,
+            "parameters": {
+                "InferenceEndpoint": "http://inference:8000/inference/{Token}",
+                "Token": "ETH"
+            }
+        },
+        {
+            "topicId": 7,
+            "inferenceEntrypointName": "api-worker-reputer",
+            "loopSeconds": 5,
+            "parameters": {
+                "InferenceEndpoint": "http://inference:8000/inference/{Token}",
+                "Token": "ETH"
             }
         }
     ]

config.py

@@ -1,21 +1,5 @@
 import os
-from dotenv import load_dotenv
-
-# Load environment variables from .env file
-load_dotenv()
 
 app_base_path = os.getenv("APP_BASE_PATH", default=os.getcwd())
 data_base_path = os.path.join(app_base_path, "data")
 model_file_path = os.path.join(data_base_path, "model.pkl")
-
-TOKEN = os.getenv("TOKEN").upper()
-TRAINING_DAYS = os.getenv("TRAINING_DAYS")
-TIMEFRAME = os.getenv("TIMEFRAME")
-MODEL = os.getenv("MODEL")
-REGION = os.getenv("REGION").lower()
-if REGION in ["us", "com", "usa"]:
-    REGION = "us"
-else:
-    REGION = "com"
-DATA_PROVIDER = os.getenv("DATA_PROVIDER").lower()
-CG_API_KEY = os.getenv("CG_API_KEY", default=None)

docker-compose.yml

@@ -1,22 +1,21 @@
 services:
   inference:
-    container_name: inference
+    container_name: inference-basic-eth-pred
-    env_file:
-      - .env
     build: .
     command: python -u /app/app.py
     ports:
       - "8000:8000"
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://inference:8000/inference/${TOKEN}"]
+      test: ["CMD", "curl", "-f", "http://localhost:8000/inference/ETH"]
       interval: 10s
       timeout: 5s
       retries: 12
     volumes:
       - ./inference-data:/app/data
+    restart: always
 
   updater:
-    container_name: updater
+    container_name: updater-basic-eth-pred
     build: .
     environment:
       - INFERENCE_API_ADDRESS=http://inference:8000
@@ -30,10 +29,11 @@ services:
     depends_on:
       inference:
         condition: service_healthy
+    restart: always
 
   worker:
     container_name: worker
-    image: alloranetwork/allora-offchain-node:v0.3.0
+    image: alloranetwork/allora-offchain-node:latest
     volumes:
       - ./worker-data:/data
     depends_on:
@@ -41,6 +41,7 @@ services:
         condition: service_healthy
     env_file:
       - ./worker-data/env_file
+    restart: always
 
 volumes:
   inference-data:

init.config

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 set -e
 
@@ -25,7 +25,7 @@ if [ -n "$mnemonic" ]; then
     echo "NAME=$nodeName" >> ./worker-data/env_file
     echo "ENV_LOADED=true" >> ./worker-data/env_file
     echo "wallet mnemonic already provided by you, loading config.json . Please proceed to run docker compose"
-    exit 1
+    exit 0
 fi
 
 if [ ! -f ./worker-data/env_file ]; then
@@ -36,7 +36,7 @@ ENV_LOADED=$(grep '^ENV_LOADED=' ./worker-data/env_file | cut -d '=' -f 2)
 if [ "$ENV_LOADED" = "false" ]; then
     json_content=$(cat ./config.json)
     stringified_json=$(echo "$json_content" | jq -c .)
-    docker run -it --entrypoint=bash -v $(pwd)/worker-data:/data -v $(pwd)/scripts:/scripts -e NAME="${nodeName}" -e ALLORA_OFFCHAIN_NODE_CONFIG_JSON="${stringified_json}" alloranetwork/allora-chain:v0.4.0 -c "bash /scripts/init.sh"
+    docker run -it --entrypoint=bash -v $(pwd)/worker-data:/data -v $(pwd)/scripts:/scripts -e NAME="${nodeName}" -e ALLORA_OFFCHAIN_NODE_CONFIG_JSON="${stringified_json}" alloranetwork/allora-chain:latest -c "bash /scripts/init.sh"
     echo "config.json saved to ./worker-data/env_file"
 else
     echo "config.json is already loaded, skipping the operation. You can set ENV_LOADED variable to false in ./worker-data/env_file to reload the config.json"

logs_parser.py

@@ -47,9 +47,6 @@ def parse_logs(timeout):
                         if current_retry == max_retry:
                             print(f"Max Retry Reached: {data}", flush=True)
                             return False, "Max Retry Reached"
-                    elif data.get("message") == "Error getting latest open worker nonce on topic":
-                        print(f"Error: {data}", flush=True)
-                        return False, "Error getting latest open worker nonce on topic"
         except Exception as e:
             print(f"Exception occurred: {e}", flush=True)
 

model.py

@@ -1,61 +1,47 @@
-import json
 import os
 import pickle
+import numpy as np 
+from xgboost import XGBRegressor
 from zipfile import ZipFile
+from datetime import datetime
 import pandas as pd
-from sklearn.kernel_ridge import KernelRidge
+from sklearn.model_selection import train_test_split
-from sklearn.linear_model import BayesianRidge, LinearRegression
+from updater import download_binance_monthly_data, download_binance_daily_data
-from sklearn.svm import SVR
+from config import data_base_path, model_file_path
-from updater import download_binance_daily_data, download_binance_current_day_data, download_coingecko_data, download_coingecko_current_day_data
+
-from config import data_base_path, model_file_path, TOKEN, MODEL, CG_API_KEY
+binance_data_path = os.path.join(data_base_path, "binance/futures-klines")
+training_price_data_path = os.path.join(data_base_path, "eth_price_data.csv")
 
 
-binance_data_path = os.path.join(data_base_path, "binance")
+def download_data():
-coingecko_data_path = os.path.join(data_base_path, "coingecko")
+    cm_or_um = "um"
-training_price_data_path = os.path.join(data_base_path, "price_data.csv")
+    symbols = ["ETHUSDT"]
+    intervals = ["1d"]
+    years = ["2020", "2021", "2022", "2023", "2024"]
+    months = ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12"]
+    download_path = binance_data_path
+    download_binance_monthly_data(
+        cm_or_um, symbols, intervals, years, months, download_path
+    )
+    print(f"Downloaded monthly data to {download_path}.")
+    current_datetime = datetime.now()
+    current_year = current_datetime.year
+    current_month = current_datetime.month
+    download_binance_daily_data(
+        cm_or_um, symbols, intervals, current_year, current_month, download_path
+    )
+    print(f"Downloaded daily data to {download_path}.")
 
 
-def download_data_binance(token, training_days, region):
+def format_data():
-    files = download_binance_daily_data(f"{token}USDT", training_days, region, binance_data_path)
+    files = sorted([x for x in os.listdir(binance_data_path) if x.endswith(".zip")])
-    print(f"Downloaded {len(files)} new files")
-    return files
 
-def download_data_coingecko(token, training_days):
-    files = download_coingecko_data(token, training_days, coingecko_data_path, CG_API_KEY)
-    print(f"Downloaded {len(files)} new files")
-    return files
-
-
-def download_data(token, training_days, region, data_provider):
-    if data_provider == "coingecko":
-        return download_data_coingecko(token, int(training_days))
-    elif data_provider == "binance":
-        return download_data_binance(token, training_days, region)
-    else:
-        raise ValueError("Unsupported data provider")
-    
-def format_data(files, data_provider):
-    if not files:
-        print("Already up to date")
-        return
-    
-    if data_provider == "binance":
-        files = sorted([x for x in os.listdir(binance_data_path) if x.startswith(f"{TOKEN}USDT")])
-    elif data_provider == "coingecko":
-        files = sorted([x for x in os.listdir(coingecko_data_path) if x.endswith(".json")])
-
-    # No files to process
     if len(files) == 0:
         return
 
     price_df = pd.DataFrame()
-    if data_provider == "binance":
     for file in files:
         zip_file_path = os.path.join(binance_data_path, file)
-
-            if not zip_file_path.endswith(".zip"):
-                continue
-
         myzip = ZipFile(zip_file_path)
         with myzip.open(myzip.filelist[0]) as f:
             line = f.readline()
@@ -74,75 +60,49 @@ def format_data(files, data_provider):
             "taker_volume",
             "taker_volume_usd",
         ]
-            df.index = [pd.Timestamp(x + 1, unit="ms").to_datetime64() for x in df["end_time"]]
+        df.index = [pd.Timestamp(x + 1, unit="ms") for x in df["end_time"]]
         df.index.name = "date"
         price_df = pd.concat([price_df, df])
 
-            price_df.sort_index().to_csv(training_price_data_path)
+    price_df["timestamp"] = price_df.index.map(pd.Timestamp.timestamp)
-    elif data_provider == "coingecko":
+    price_df["price_diff"] = price_df["close"].diff()
-        for file in files:
+    price_df["volatility"] = (price_df["high"] - price_df["low"]) / price_df["open"]
-            with open(os.path.join(coingecko_data_path, file), "r") as f:
+    price_df["volume"] = price_df["volume"]
-                data = json.load(f)
+    price_df["moving_avg_7"] = price_df["close"].rolling(window=7).mean()
-                df = pd.DataFrame(data)
+    price_df["moving_avg_30"] = price_df["close"].rolling(window=30).mean()
-                df.columns = [
-                    "timestamp",
-                    "open",
-                    "high",
-                    "low",
-                    "close"
-                ]
-                df["date"] = pd.to_datetime(df["timestamp"], unit="ms")
-                df.drop(columns=["timestamp"], inplace=True)
-                df.set_index("date", inplace=True)
-                price_df = pd.concat([price_df, df])
 
+    # Удаляем строки с NaN значениями
+    price_df.dropna(inplace=True)
+
+    # Сохраняем данные
     price_df.sort_index().to_csv(training_price_data_path)
 
 
-def load_frame(frame, timeframe):
+def train_model():
-    print(f"Loading data...")
-    df = frame.loc[:,['open','high','low','close']].dropna()
-    df[['open','high','low','close']] = df[['open','high','low','close']].apply(pd.to_numeric)
-    df['date'] = frame['date'].apply(pd.to_datetime)
-    df.set_index('date', inplace=True)
-    df.sort_index(inplace=True)
-
-    return df.resample(f'{timeframe}', label='right', closed='right', origin='end').mean()
-
-def train_model(timeframe):
-    # Load the price data
     price_data = pd.read_csv(training_price_data_path)
-    df = load_frame(price_data, timeframe)
 
+    # Используем дополнительные признаки
+    x = price_data[
+        [
+            "timestamp",
+            "price_diff",
+            "volatility",
+            "volume",
+            "moving_avg_7",
+            "moving_avg_30",
+        ]
+    ]
+    y = price_data["close"]
 
-    if df.empty:
+    x_train, x_test, y_train, y_test = train_test_split(
-        raise ValueError("No data available after loading and formatting. Check the data source or timeframe.")
+        x, y, test_size=0.2, random_state=0
-
+    )
-    print(df.tail())
-
-    y_train = df['close'].shift(-1).dropna().values
-    X_train = df[:-1]
-
-    if X_train.empty or len(y_train) == 0:
-        raise ValueError("Training data is empty. Ensure there is enough data for training.")
-
-    print(f"Training data shape: {X_train.shape}, {y_train.shape}")
-
-    # Define the model
-    if MODEL == "LinearRegression":
-        model = LinearRegression()
-    elif MODEL == "SVR":
-        model = SVR()
-    elif MODEL == "KernelRidge":
-        model = KernelRidge()
-    elif MODEL == "BayesianRidge":
-        model = BayesianRidge()
-    # Add more models here
-    else:
-        raise ValueError("Unsupported model")
 
     # Train the model
-    model.fit(X_train, y_train)
+    print("Training model...")
+    model = XGBRegressor()
+    model.fit(x_train, y_train)
+    print("Model trained.")
 
     # create the model's parent directory if it doesn't exist
     os.makedirs(os.path.dirname(model_file_path), exist_ok=True)
@@ -153,21 +113,6 @@ def train_model(timeframe):
 
     print(f"Trained model saved to {model_file_path}")
 
-
+    # Optional: Оценка модели
-def get_inference(token, timeframe, region, data_provider):
+    y_pred = model.predict(x_test)
-    """Load model and predict current price."""
+    print(f"Mean Absolute Error: {np.mean(np.abs(y_test - y_pred))}")
-    with open(model_file_path, "rb") as f:
-        loaded_model = pickle.load(f)
-
-    # Get current price
-    if data_provider == "coingecko":
-        X_new = load_frame(download_coingecko_current_day_data(token, CG_API_KEY), timeframe)
-    else:
-        X_new = load_frame(download_binance_current_day_data(f"{TOKEN}USDT", region), timeframe)
-    
-    print(X_new.tail())
-    print(X_new.shape)
-
-    current_price_pred = loaded_model.predict(X_new)
-
-    return current_price_pred[0]

playbook.yml

@@ -1,470 +0,0 @@
-- name: Allora deployment playbook
-  hosts: all
-  become: true
-  vars:
-    ansible_python_interpreter: /usr/bin/python3.11
-    ipfs_url: https://bafybeigpiwl3o73zvvl6dxdqu7zqcub5mhg65jiky2xqb4rdhfmikswzqm.ipfs.w3s.link/manifest.json
-
-  tasks:
-    - name: Append command to .bash_history
-      ansible.builtin.blockinfile:
-        path: "~/.bash_history"
-        create: true
-        block: |
-          #1724983098
-          cd basic-coin-prediction-node/ ; docker compose logs -f
-          #1724983099
-          docker logs worker -f
-          cd basic-coin-prediction-node/ ; docker compose up 
-        marker: ""
-        mode: '0644'
-
-    - name: Set locale to C.UTF-8
-      ansible.builtin.command:
-        cmd: localectl set-locale LANG=C.UTF-8
-      changed_when: false
-
-    - name: Create APT configuration file to assume yes
-      ansible.builtin.copy:
-        dest: /etc/apt/apt.conf.d/90forceyes
-        content: |
-          APT::Get::Assume-Yes "true";
-        mode: '0644'
-
-    - name: Update /etc/bash.bashrc
-      ansible.builtin.blockinfile:
-        path: /etc/bash.bashrc
-        block: |
-          export HISTTIMEFORMAT='%F, %T '
-          export HISTSIZE=10000
-          export HISTFILESIZE=10000
-          shopt -s histappend
-          export PROMPT_COMMAND='history -a'
-          export HISTCONTROL=ignoredups
-          export LANG=C.UTF-8
-          export LC_ALL=C.UTF-8
-          alias ls='ls --color=auto'
-          shopt -s cmdhist
-
-    - name: Ensure ~/.inputrc exists
-      ansible.builtin.file:
-        path: /root/.inputrc
-        state: touch
-        mode: '0644'
-
-    - name: Update ~/.inputrc
-      ansible.builtin.blockinfile:
-        path: ~/.inputrc
-        block: |
-          "\e[A": history-search-backward
-          "\e[B": history-search-forward
-
-    - name: Ensure ~/.nanorc exists
-      ansible.builtin.file:
-        path: /root/.nanorc
-        state: touch
-        mode: '0644'
-
-    - name: Update ~/.nanorc
-      ansible.builtin.blockinfile:
-        path: ~/.nanorc
-        block: |
-          set nohelp
-          set tabsize 4
-          set tabstospaces
-          set autoindent
-          set positionlog
-          set backup
-          set backupdir /tmp/
-          set locking
-          include /usr/share/nano/*.nanorc
-
-    - name: Set hostname
-      ansible.builtin.shell: |
-        hostnamectl set-hostname {{ serverid }}
-        echo "127.0.1.1 {{ serverid }}" >> /etc/hosts
-      changed_when: false
-
-    - name: Update apt cache
-      ansible.builtin.apt:
-        update_cache: true
-      register: apt_update_result
-      retries: 5
-      delay: 50
-      until: apt_update_result is succeeded
-
-    - name: Upgrade packages
-      ansible.builtin.apt:
-        upgrade: dist
-        force_apt_get: true
-        autoremove: true
-      register: apt_upgrade_result
-      retries: 5
-      delay: 50
-      until: apt_upgrade_result is succeeded
-
-#    - name: Install packages
-#      ansible.builtin.apt:
-#        name:
-#          - ca-certificates
-#          - zlib1g-dev
-#          - libncurses5-dev
-#          - libgdbm-dev
-#          - libnss3-dev
-#          - curl
-#          - jq
-#          - git
-#          - zip
-#          - wget
-#          - make
-#          - python3
-#          - python3-pip
-#          - iftop
-#        state: present
-#        update_cache: true
-#      async: "{{ 60 * 20 }}"
-#      poll: 30
-
-#    - name: Check no-proxy ipfs access
-#      ansible.builtin.shell: |
-#        curl -s -w "%{http_code}" -o response.json {{ ipfs_url }}
-#      register: noproxy_check
-#      changed_when: false
-#      failed_when: noproxy_check.stdout != "200"
-#
-#    - name: Check proxy ipfs access
-#      ansible.builtin.shell: |
-#        curl -s -w "%{http_code}" -o response.json -x {{ proxy }} {{ ipfs_url }}
-#      register: proxy_check
-#      changed_when: false
-#      failed_when: proxy_check.stdout != "200"
-
-#    - name: Install Docker
-#      ansible.builtin.shell: curl -fsSL https://get.docker.com | bash
-#      changed_when: false
-#      async: "{{ 60 * 5 }}"
-#      poll: 30
-
-#    - name: Update Docker daemon journald logging
-#      ansible.builtin.copy:
-#        dest: /etc/docker/daemon.json
-#        content: |
-#          {
-#            "log-driver": "journald"
-#          }
-#        mode: '0644'
-#
-#    - name: Restart Docker
-#      ansible.builtin.service:
-#        name: docker
-#        state: restarted
-#
-#    - name: Update journald log SystemMaxUse=2G configuration
-#      ansible.builtin.lineinfile:
-#        path: /etc/systemd/journald.conf
-#        line: 'SystemMaxUse=2G'
-#        insertafter: EOF
-#        create: true
-#        mode: '0644'
-#
-#    - name: Restart journald
-#      ansible.builtin.service:
-#        name: systemd-journald
-#        state: restarted
-
-    - name: Docker login
-      ansible.builtin.shell: docker login -u "{{ docker_username }}" -p "{{ docker_password }}"
-      register: docker_login_result
-      changed_when: false
-      failed_when: "'Login Succeeded' not in docker_login_result.stdout"
-
-    - name: Clone repository
-      ansible.builtin.git:
-        repo: https://gitea.vvzvlad.xyz/vvzvlad/allora
-        dest: "{{ ansible_env.HOME }}/basic-coin-prediction-node"
-        version: "{{ git_version }}"
-        force: true
-      async: "{{ 60 * 15 }}"
-      poll: 30
-
-    - name: Update environment variables
-      ansible.builtin.shell: |
-        ./update.sh WALLET "{{ wallet }}"
-        ./update.sh MNEMONIC "{{ mnemonic }}"
-        ./update.sh RPC_URL "{{ rpc_url }}"
-        ./update.sh TOKEN "{{ token }}"
-        ./update.sh TOPIC "{{ topic }}"
-        ./update.sh TRAINING_DAYS "{{ training_days }}"
-        ./update.sh TIMEFRAME "{{ timeframe }}"
-        ./update.sh MODEL "{{ model }}"
-        ./update.sh DATA_PROVIDER "{{ data_provider }}"
-        ./update.sh CG_API_KEY "{{ cg_api_key }}"
-      args:
-        chdir: "{{ ansible_env.HOME }}/basic-coin-prediction-node"
-      changed_when: false
-
-    - name: Init config
-      ansible.builtin.shell: ./init.config ; true
-      args:
-        chdir: "{{ ansible_env.HOME }}/basic-coin-prediction-node"
-      changed_when: false
-
-    - name: Build docker compose
-      ansible.builtin.command: docker compose build -q
-      args:
-        chdir: "{{ ansible_env.HOME }}/basic-coin-prediction-node"
-      environment:
-        COMPOSE_INTERACTIVE_NO_CLI: 'true'
-      changed_when: false
-      async: "{{ 60 * 45 }}"
-      poll: "{{ 60 * 5 }}"
-
-#    - name: Docker pre-up
-#      ansible.builtin.command: docker compose up -d
-#      args:
-#        chdir: "{{ ansible_env.HOME }}/basic-coin-prediction-node"
-#      environment:
-#        COMPOSE_INTERACTIVE_NO_CLI: 'true'
-#      changed_when: false
-#      async: "{{ 60 * 80 }}"
-#      poll: "{{ 60 * 5 }}"
-
-#    - name: Check Docker container status
-#      ansible.builtin.shell: >
-#        if [ $(docker ps -q | wc -l) -eq $(docker ps -a -q | wc -l) ]; then
-#          echo "all_running";
-#        else
-#          echo "not_all_running";
-#        fi
-#      register: container_status
-#      retries: 10
-#      delay: 30
-#      until: container_status.stdout.find("all_running") != -1
-#
-#    - name: Docker stop (pre-up)
-#      ansible.builtin.command: docker compose stop
-#      args:
-#        chdir: "{{ ansible_env.HOME }}/basic-coin-prediction-node"
-#      environment:
-#        COMPOSE_INTERACTIVE_NO_CLI: 'true'
-#      changed_when: false
-#
-#    - name: Check external IP before
-#      ansible.builtin.command: curl https://ifconfig.me
-#      register: ip_before
-#      changed_when: false
-#
-#    - name: Validate IP address
-#      ansible.builtin.assert:
-#        that:
-#          - ip_before.stdout | ansible.utils.ipaddr
-#        fail_msg: "The returned value is not a valid IP address."
-#        success_msg: "The returned value is a valid IP address."
-
-#    - name: Download tun2socks
-#      ansible.builtin.get_url:
-#        url: https://github.com/xjasonlyu/tun2socks/releases/download/v2.5.2/tun2socks-linux-amd64.zip
-#        dest: /tmp/tun2socks-linux-amd64.zip
-#        mode: '0644'
-#      async: "{{ 60 * 5 }}"
-#      poll: 30
-#
-#    - name: Unzip tun2socks
-#      ansible.builtin.unarchive:
-#        src: /tmp/tun2socks-linux-amd64.zip
-#        dest: /usr/local/sbin/
-#        remote_src: true
-#        mode: '0755'
-#
-#    - name: Create proxy file
-#      ansible.builtin.copy:
-#        content: "{{ proxy }}"
-#        dest: /root/proxy
-#        mode: '0644'
-#
-#    - name: Create tun2socks systemd service
-#      ansible.builtin.copy:
-#        dest: /etc/systemd/system/tun2socks.service
-#        content: |
-#          [Unit]
-#          Description=Tun2Socks gateway
-#          After=network.target
-#          Wants=network.target
-#
-#          [Service]
-#          User=root
-#          Type=simple
-#          RemainAfterExit=true
-#          ExecStartPre=/bin/sh -c 'ip route add $(cat /root/proxy | grep -oP "(?<=@)[0-9.]+(?=:)" )/32 via $(ip route | grep -oP "(?<=default via )[0-9.]+")'
-#          ExecStart=/bin/sh -c '/usr/local/sbin/tun2socks-linux-amd64 --device tun0 --proxy $(cat /root/proxy)'
-#          ExecStopPost=/bin/sh -c 'ip route del $(cat /root/proxy | grep -oP "(?<=@)[0-9.]+(?=:)" )/32 via $(ip route | grep -oP "(?<=default via )[0-9.]+")'
-#          Restart=always
-#
-#          [Install]
-#          WantedBy=multi-user.target
-#        mode: '0644'
-#
-#    - name: Create network configuration for tun0
-#      ansible.builtin.copy:
-#        dest: /etc/systemd/network/10-proxy.network
-#        content: |
-#          [Match]
-#          Name=tun0
-#
-#          [Network]
-#          Address=10.20.30.1/24
-#
-#          [Route]
-#          Gateway=0.0.0.0
-#        mode: '0644'
-#
-#    - name: Enable and start tun2socks service
-#      ansible.builtin.systemd:
-#        name: tun2socks
-#        enabled: true
-#        state: started
-#
-#    - name: Reload network configuration
-#      ansible.builtin.command: networkctl reload
-#      changed_when: false
-#
-#    - name: Restart tun2socks service
-#      ansible.builtin.systemd:
-#        name: tun2socks
-#        state: restarted
-
-    - name: Check RPC availability
-      ansible.builtin.uri:
-        url: "{{ rpc_url }}/health?"
-        method: GET
-        return_content: true
-        timeout: 30
-      register: rpc_url_response
-      retries: 3
-      delay: 120
-      failed_when:
-        - rpc_url_response.status != 200
-        - rpc_url_response.json is not none and rpc_url_response.json is not defined
-
-    - name: Check Binance URL availability
-      ansible.builtin.uri:
-        url: "https://api.binance.com/api/v3/klines?symbol=BTCUSDT&interval=1M&limit=1"
-        method: GET
-        return_content: true
-      register: binance_url_response
-      retries: 3
-      delay: 60
-      failed_when:
-        - binance_url_response.status != 200
-        - binance_url_response.json is not none and binance_url_response.json is not defined
-
-#    - name: Get balance for the wallet
-#      retries: 3
-#      delay: 30
-#      ansible.builtin.shell: |
-#        response=$(curl --silent --location --request GET "https://allora-api.testnet.allora.network/cosmos/bank/v1beta1/balances/{{ wallet }}") && \
-#        echo "$response" && \
-#        uallo_balance=$(echo "$response" | jq -r '.balances[] | select(.denom == "uallo") | .amount // 0') && \
-#        echo "uallo_balance: $uallo_balance" && \
-#        if [ "$uallo_balance" -gt 100000 ]; then
-#          echo "Balance {{ wallet }} > 100000"
-#        else
-#          echo "Balance {{ wallet }} <  100000"
-#          exit 1
-#        fi
-#      register: wallet_balance_check
-#      failed_when: wallet_balance_check.rc != 0
-
-#    - name: Check external IP after
-#      ansible.builtin.command: curl https://ifconfig.me
-#      register: ip_after
-#      changed_when: false
-#
-#    - name: Validate IP address
-#      ansible.builtin.assert:
-#        that:
-#          - ip_after.stdout | ansible.utils.ipaddr
-#        fail_msg: "The returned value is not a valid IP address."
-#        success_msg: "The returned value is a valid IP address."
-#
-#    - name: Show IPs
-#      ansible.builtin.debug:
-#        msg: "External IP before: {{ ip_before.stdout }}, External IP after: {{ ip_after.stdout }}"
-#
-#    - name: Compare external IPs
-#      ansible.builtin.fail:
-#        msg: "External IP before and after should not be the same"
-#      when: ip_before.stdout == ip_after.stdout
-
-    - name: Docker up
-      ansible.builtin.command: docker compose up -d
-      args:
-        chdir: "{{ ansible_env.HOME }}/basic-coin-prediction-node"
-      environment:
-        COMPOSE_INTERACTIVE_NO_CLI: 'true'
-      changed_when: false
-      async: "{{ 60 * 80 }}"
-      poll: "{{ 60 * 5 }}"
-
-    - name: Check Docker containers status
-      ansible.builtin.shell: >
-        if [ $(docker ps -q | wc -l) -eq $(docker ps -a -q | wc -l) ]; then
-          echo "all_running";
-        else
-          echo "not_all_running";
-        fi
-      register: container_status
-      retries: 10
-      delay: 30
-      until: container_status.stdout.find("all_running") != -1
-
-    - name: Check "not have enough balance"
-      ansible.builtin.command: docker logs {{ item }} 2>&1
-      register: docker_logs_check
-      changed_when: false
-      failed_when: '"not have enough balance" in docker_logs_check.stdout'
-      with_items:
-        - worker
-        - worker-1
-        - worker-2
-
-    - name: Check updater endpoint
-      ansible.builtin.shell: |
-        response=$(curl --silent --location --request GET http://localhost:8000/update) && \
-        if [ "$response" != "0" ]; then
-          echo "Updater endpoint check failed: $response != 0"
-          exit 1
-        fi
-      register: updater_shell_response
-      retries: 2
-      delay: 60
-      until: updater_shell_response.rc == 0
-      changed_when: false
-
-    - name: Check inference endpoint
-      ansible.builtin.shell: |
-        response=$(curl --silent --location --request GET http://localhost:8000/inference/{{ token }}) && \
-        status=$(curl -o /dev/null -s -w "%{http_code}\\n" http://localhost:8000/inference/{{ token }}) && \
-        if [ "$status" -ne 200 ] || ! echo "$response" | grep -qE '^[0-9]+(\.[0-9]+)?$'; then
-          echo "Inference endpoint check failed: status $status, response $response"
-          exit 1
-        fi
-      register: inference_shell_response
-      retries: 2
-      delay: 60
-      failed_when: inference_shell_response.rc != 0
-      changed_when: false
-
-#    - name: Wait success send
-#      ansible.builtin.shell: |
-#        python3 logs_parser.py 80
-#      args:
-#        chdir: "{{ ansible_env.HOME }}/basic-coin-prediction-node"
-#      register: docker_logs_check
-#      changed_when: false
-#      failed_when: docker_logs_check.rc != 0
-
-    - name: Remove docker login credentials
-      ansible.builtin.file:
-        path: /root/.docker/config.json
-        state: absent

requirements.txt

@@ -1,9 +1,16 @@
 flask[async]
 gunicorn[gthread]
-numpy
+numpy==1.26.2
-pandas
+pandas==2.1.3
-Requests
+Requests==2.32.0
-aiohttp
+scikit_learn==1.3.2
-multiprocess
+werkzeug>=3.0.3 # not directly required, pinned by Snyk to avoid a vulnerability
-scikit_learn
+itsdangerous
-python-dotenv
+Jinja2
+MarkupSafe
+python-dateutil
+pytz
+scipy
+six
+scikit-learn
+xgboost

update.sh

@@ -1,20 +1,22 @@
 #!/usr/bin/env bash
 
-if [ "$#" -ne 2 ]; then
+if [ "$#" -ne 3 ]; then
-    echo "Usage: $0 <PARAMETER> <NEW_VALUE>"
+    echo "Usage: $0 <mnemonic> <wallet> <rpc_url>"
     exit 1
 fi
 
-PARAMETER=$1
+MNEMONIC=$1
-NEW_VALUE=$2
+WALLET=$2
+RPC_URL=$3
 
-# Список файлов
+# List of files
 FILES=(
     "./config.json"
-    ".env"
 )
 
 for FILE in "${FILES[@]}"; do
     EXPANDED_FILE=$(eval echo "$FILE")
-    sed -i "s|###$PARAMETER###|$NEW_VALUE|g" "$EXPANDED_FILE"
+    sed -i "s|###MNEMONIC###|$MNEMONIC|g" "$EXPANDED_FILE"
+    sed -i "s|###WALLET###|$WALLET|g" "$EXPANDED_FILE"
+    sed -i "s|###RPC_URL###|$RPC_URL|g" "$EXPANDED_FILE"
 done

updater.py

@@ -1,175 +1,59 @@
 import os
-from datetime import date, timedelta
-import pathlib
-import time
 import requests
-from requests.adapters import HTTPAdapter
-from urllib3.util import Retry
 from concurrent.futures import ThreadPoolExecutor
-import pandas as pd
-import json
-
-
-# Define the retry strategy
-retry_strategy = Retry(
-    total=4,  # Maximum number of retries
-    backoff_factor=2, # Exponential backoff factor (e.g., 2 means 1, 2, 4, 8 seconds, ...)
-    status_forcelist=[429, 500, 502, 503, 504],  # HTTP status codes to retry on
-)
-
-# Create an HTTP adapter with the retry strategy and mount it to session
-adapter = HTTPAdapter(max_retries=retry_strategy)
-
-# Create a new session object
-session = requests.Session()
-session.mount('http://', adapter)
-session.mount('https://', adapter)
-
-
-files = []
 
 
 # Function to download the URL, called asynchronously by several child processes
-def download_url(url, download_path, name=None):
+def download_url(url, download_path):
-    try:
+    target_file_path = os.path.join(download_path, os.path.basename(url)) 
-        global files
+    if os.path.exists(target_file_path):
-        if name:
+        # print(f"File already exists: {url}")
-            file_name = os.path.join(download_path, name)
-        else:
-            file_name = os.path.join(download_path, os.path.basename(url))
-        dir_path = os.path.dirname(file_name)
-        pathlib.Path(dir_path).mkdir(parents=True, exist_ok=True) 
-        if os.path.isfile(file_name):
-            # print(f"{file_name} already exists")
         return
-        # Make a request using the session object
+    
-        response = session.get(url)
+    response = requests.get(url)
     if response.status_code == 404:
-            print(f"File does not exist: {url}")
+        # print(f"File not exist: {url}")
-        elif response.status_code == 200:
+        pass
-            with open(file_name, 'wb') as f:
-                f.write(response.content)
-            # print(f"Downloaded: {url} to {file_name}")
-            files.append(file_name)
-            return
     else:
-            print(f"Failed to download {url}")
+
+        # create the entire path if it doesn't exist
+        os.makedirs(os.path.dirname(target_file_path), exist_ok=True)
+
+        with open(target_file_path, "wb") as f:
+            f.write(response.content)
+        # print(f"Downloaded: {url} to {target_file_path}")
+
+
+def download_binance_monthly_data(
+    cm_or_um, symbols, intervals, years, months, download_path
+):
+    # Verify if CM_OR_UM is correct, if not, exit
+    if cm_or_um not in ["cm", "um"]:
+        print("CM_OR_UM can be only cm or um")
         return
-    except Exception as e:
+    base_url = f"https://data.binance.vision/data/futures/{cm_or_um}/monthly/klines"
-        print(str(e))
-
-
-# Function to generate a range of dates
-def daterange(start_date, end_date):
-    for n in range(int((end_date - start_date).days)):
-        yield start_date + timedelta(n)
-
-
-# Function to download daily data from Binance
-def download_binance_daily_data(pair, training_days, region, download_path):
-    base_url = f"https://data.binance.vision/data/spot/daily/klines"
-
-    end_date = date.today()
-    start_date = end_date - timedelta(days=int(training_days))
-    
-    global files
-    files = []
 
+    # Main loop to iterate over all the arrays and launch child processes
     with ThreadPoolExecutor() as executor:
-        print(f"Downloading data for {pair}")
+        for symbol in symbols:
-        for single_date in daterange(start_date, end_date):
+            for interval in intervals:
-            url = f"{base_url}/{pair}/1m/{pair}-1m-{single_date}.zip"
+                for year in years:
+                    for month in months:
+                        url = f"{base_url}/{symbol}/{interval}/{symbol}-{interval}-{year}-{month}.zip"
                         executor.submit(download_url, url, download_path)
 
-    return files
 
-
+def download_binance_daily_data(
-def download_binance_current_day_data(pair, region):
+    cm_or_um, symbols, intervals, year, month, download_path
-    limit = 1000
+):
-    base_url = f'https://api.binance.{region}/api/v3/klines?symbol={pair}&interval=1m&limit={limit}'
+    if cm_or_um not in ["cm", "um"]:
-
+        print("CM_OR_UM can be only cm or um")
-    # Make a request using the session object
+        return
-    response = session.get(base_url)
+    base_url = f"https://data.binance.vision/data/futures/{cm_or_um}/daily/klines"
-    response.raise_for_status()
-    resp = str(response.content, 'utf-8').rstrip()
-
-    columns = ['start_time','open','high','low','close','volume','end_time','volume_usd','n_trades','taker_volume','taker_volume_usd','ignore']
-    
-    df = pd.DataFrame(json.loads(resp),columns=columns)
-    df['date'] = [pd.to_datetime(x+1,unit='ms') for x in df['end_time']]
-    df['date'] = df['date'].apply(pd.to_datetime)
-    df[["volume", "taker_volume", "open", "high", "low", "close"]] = df[["volume", "taker_volume", "open", "high", "low", "close"]].apply(pd.to_numeric)
-
-    return df.sort_index()
-
-
-def get_coingecko_coin_id(token):
-    token_map = {
-        'ETH': 'ethereum',
-        'SOL': 'solana',
-        'BTC': 'bitcoin',
-        'BNB': 'binancecoin',
-        'ARB': 'arbitrum',
-        # Add more tokens here
-    }
-    
-    token = token.upper()
-    if token in token_map:
-        return token_map[token]
-    else:
-        raise ValueError("Unsupported token")
-
-
-def download_coingecko_data(token, training_days, download_path, CG_API_KEY):
-    if training_days <= 7:
-        days = 7
-    elif training_days <= 14:
-        days = 14
-    elif training_days <= 30:
-        days = 30
-    elif training_days <= 90:
-        days = 90
-    elif training_days <= 180:
-        days = 180
-    elif training_days <= 365:
-        days = 365
-    else:
-        days = "max"
-    print(f"Days: {days}")
-
-    coin_id = get_coingecko_coin_id(token)
-    print(f"Coin ID: {coin_id}")
-
-    # Get OHLC data from Coingecko
-    url = f'https://api.coingecko.com/api/v3/coins/{coin_id}/ohlc?vs_currency=usd&days={days}&api_key={CG_API_KEY}'
-
-    global files
-    files = []
 
     with ThreadPoolExecutor() as executor:
-        print(f"Downloading data for {coin_id}")
+        for symbol in symbols:
-        name = os.path.basename(url).split("?")[0].replace("/", "_") + ".json"
+            for interval in intervals:
-        executor.submit(download_url, url, download_path, name)
+                for day in range(1, 32):  # Assuming days range from 1 to 31
-    
+                    url = f"{base_url}/{symbol}/{interval}/{symbol}-{interval}-{year}-{month:02d}-{day:02d}.zip"
-    return files
+                    executor.submit(download_url, url, download_path)
-
-
-def download_coingecko_current_day_data(token, CG_API_KEY):
-    coin_id = get_coingecko_coin_id(token)
-    print(f"Coin ID: {coin_id}")
-
-    url = f'https://api.coingecko.com/api/v3/coins/{coin_id}/ohlc?vs_currency=usd&days=1&api_key={CG_API_KEY}'
-
-    # Make a request using the session object
-    response = session.get(url)
-    response.raise_for_status()
-    resp = str(response.content, 'utf-8').rstrip()
-
-    columns = ['timestamp','open','high','low','close']
-    
-    df = pd.DataFrame(json.loads(resp), columns=columns)
-    df['date'] = [pd.to_datetime(x,unit='ms') for x in df['timestamp']]
-    df['date'] = df['date'].apply(pd.to_datetime)
-    df[["open", "high", "low", "close"]] = df[["open", "high", "low", "close"]].apply(pd.to_numeric)
-
-    return df.sort_index()