feat: publishing infernet-container-starter v0.2.0

projects/tgi-llm/container/.gitignore

@@ -0,0 +1 @@
+config.json

projects/tgi-llm/container/Dockerfile

@@ -0,0 +1,25 @@
+FROM python:3.11-slim as builder
+
+WORKDIR /app
+
+ENV PYTHONUNBUFFERED 1
+ENV PYTHONDONTWRITEBYTECODE 1
+ENV PIP_NO_CACHE_DIR 1
+ENV RUNTIME docker
+ENV PYTHONPATH src
+
+RUN apt-get update
+RUN apt-get install -y git curl
+
+# install uv
+ADD --chmod=755 https://astral.sh/uv/install.sh /install.sh
+RUN /install.sh && rm /install.sh
+
+COPY src/requirements.txt .
+
+RUN /root/.cargo/bin/uv pip install --system --no-cache -r requirements.txt
+
+COPY src src
+
+ENTRYPOINT ["hypercorn", "app:create_app()"]
+CMD ["-b", "0.0.0.0:3000"]

projects/tgi-llm/container/Makefile

@@ -0,0 +1,17 @@
+DOCKER_ORG := ritualnetwork
+EXAMPLE_NAME := tgi-llm
+TAG := $(DOCKER_ORG)/example-$(EXAMPLE_NAME)-infernet:latest
+
+.phony: build run build-multiplatform
+
+build:
+	@docker build -t $(TAG) .
+
+run:
+	docker run -p 3000:3000 --env-file tgi-llm.env $(TAG)
+
+# You may need to set up a docker builder, to do so run:
+# docker buildx create --name mybuilder --bootstrap --use
+# refer to https://docs.docker.com/build/building/multi-platform/#building-multi-platform-images for more info
+build-multiplatform:
+	docker buildx build --platform linux/amd64,linux/arm64 -t $(TAG) --push .

projects/tgi-llm/container/README.md

@@ -0,0 +1,88 @@
+# TGI LLM
+
+In this example, we're running an infernet node along with a TGI service.
+
+## Deploying TGI Service
+
+If you have your own TGI service running, feel free to skip this part. Otherwise,
+you can deploy the TGI service using the following command.
+
+Make sure you have a machine with proper GPU support. Clone this repository &
+run the following command:
+
+```bash
+make run-service project=tgi-llm service=tgi
+```
+
+## Deploying Infernet Node Locally
+
+Running an infernet node involves a simple configuration step & running step.
+
+### Configuration
+
+Copy our [sample config file](./config.sample.json) into a new file
+called `config.json`.
+
+```bash
+cp config.sample.json config.json
+```
+
+Then provide the `"env"` field of the `"containers"` section of the file to point to the
+TGI Service you just deployed.
+
+```json
+{
+  // etc.
+  "containers": [
+    {
+      "id": "tgi-llm",
+      "image": "ritualnetwork/llm_inference_service:latest",
+      "external": true,
+      "port": "3000",
+      "allowed_delegate_addresses": [],
+      "allowed_addresses": [],
+      "allowed_ips": [],
+      "command": "--bind=0.0.0.0:3000 --workers=2",
+      "env": {
+        "TGI_SERVICE_URL": "http://{your-service-ip}:{your-service-port}" // <- Change this to the TGI service you deployed
+      }
+    }
+  ]
+}
+```
+
+### Running the Infernet Node Locally
+
+With that out of the way, you can now run the infernet node using the following command
+at the top-level directory of this repo:
+
+```
+make deploy-container project=tgi-llm
+```
+
+## Testing the Infernet Node
+
+You can test the infernet node by posting a job in the node's REST api.
+
+```bash
+curl -X POST "http://127.0.0.1:4000/api/jobs" \
+   -H "Content-Type: application/json" \
+   -d '{"containers":["tgi-llm"], "data": {"prompt": "can shrimp actually fry rice?"}}'
+```
+
+You can expect a response similar to the following:
+
+```json
+{
+  "id": "f026c7c2-7027-4c2d-b662-2b48c9433a12"
+}
+```
+
+You can then check the status of the job using the following command:
+
+```bash
+curl -X GET http://127.0.0.1:4000/api/jobs\?id\=f026c7c2-7027-4c2d-b662-2b48c9433a12
+[{"id":"f026c7c2-7027-4c2d-b662-2b48c9433a12","result":{"container":"tgi-llm","output":{"output":"\n\nI\u2019m not sure if this is a real question or not, but I\u2019m"}},"status":"success"}]
+```
+
+Congratulations! You've successfully ran an infernet node with a TGI service.

projects/tgi-llm/container/config.sample.json

@@ -0,0 +1,52 @@
+{
+  "log_path": "infernet_node.log",
+  "server": {
+    "port": 4000
+  },
+  "chain": {
+    "enabled": true,
+    "trail_head_blocks": 0,
+    "rpc_url": "http://host.docker.internal:8545",
+    "coordinator_address": "0x5FbDB2315678afecb367f032d93F642f64180aa3",
+    "wallet": {
+      "max_gas_limit": 4000000,
+      "private_key": "0x59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d"
+    }
+  },
+  "startup_wait": 1.0,
+  "docker": {
+    "username": "your-username",
+    "password": ""
+  },
+  "redis": {
+    "host": "redis",
+    "port": 6379
+  },
+  "forward_stats": true,
+  "containers": [
+    {
+      "id": "tgi-llm",
+      "image": "ritualnetwork/example-tgi-llm-infernet:latest",
+      "external": true,
+      "port": "3000",
+      "allowed_delegate_addresses": [],
+      "allowed_addresses": [],
+      "allowed_ips": [],
+      "command": "--bind=0.0.0.0:3000 --workers=2",
+      "env": {
+        "TGI_SERVICE_URL": "http://{your_service_ip}:{your_service_port}"
+      }
+    },
+    {
+      "id": "anvil-node",
+      "image": "ritualnetwork/infernet-anvil:0.0.0",
+      "external": true,
+      "port": "8545",
+      "allowed_delegate_addresses": [],
+      "allowed_addresses": [],
+      "allowed_ips": [],
+      "command": "",
+      "env": {}
+    }
+  ]
+}

projects/tgi-llm/container/src/app.py

@@ -0,0 +1,85 @@
+import logging
+import os
+from typing import Any, cast
+
+from eth_abi import decode, encode  # type: ignore
+from infernet_ml.utils.service_models import InfernetInput, InfernetInputSource
+from infernet_ml.workflows.inference.tgi_client_inference_workflow import (
+    TGIClientInferenceWorkflow,
+)
+from quart import Quart, request
+
+log = logging.getLogger(__name__)
+
+
+def create_app() -> Quart:
+    app = Quart(__name__)
+
+    workflow = TGIClientInferenceWorkflow(
+        server_url=cast(str, os.environ.get("TGI_SERVICE_URL"))
+    )
+
+    workflow.setup()
+
+    @app.route("/")
+    def index() -> str:
+        """
+        Utility endpoint to check if the service is running.
+        """
+        return "LLM Inference Service is running."
+
+    @app.route("/service_output", methods=["POST"])
+    async def inference() -> dict[str, Any]:
+        req_data = await request.get_json()
+        """
+        InfernetInput has the format:
+            source: (0 on-chain, 1 off-chain)
+            data: dict[str, Any]
+        """
+        infernet_input: InfernetInput = InfernetInput(**req_data)
+
+        if infernet_input.source == InfernetInputSource.OFFCHAIN:
+            prompt = cast(dict[str, Any], infernet_input.data).get("prompt")
+        else:
+            # On-chain requests are sent as a generalized hex-string which we will
+            # decode to the appropriate format.
+            (prompt,) = decode(
+                ["string"], bytes.fromhex(cast(str, infernet_input.data))
+            )
+
+        result: dict[str, Any] = workflow.inference({"text": prompt})
+
+        if infernet_input.source == InfernetInputSource.OFFCHAIN:
+            """
+            In case of an off-chain request, the result is returned as a dict. The
+            infernet node expects a dict format.
+            """
+            return {"data": result}
+        else:
+            """
+            In case of an on-chain request, the result is returned in the format:
+            {
+                "raw_input": str,
+                "processed_input": str,
+                "raw_output": str,
+                "processed_output": str,
+                "proof": str,
+            }
+            refer to: https://docs.ritual.net/infernet/node/containers for more info.
+            """
+            return {
+                "raw_input": "",
+                "processed_input": "",
+                "raw_output": encode(["string"], [result]).hex(),
+                "processed_output": "",
+                "proof": "",
+            }
+
+    return app
+
+
+if __name__ == "__main__":
+    """
+    Utility to run the app locally. For development purposes only.
+    """
+    create_app().run(port=3000)

projects/tgi-llm/container/src/requirements.txt

@@ -0,0 +1,6 @@
+quart==0.19.4
+infernet_ml==0.1.0
+PyArweave @ git+https://github.com/ritual-net/pyarweave.git
+web3==6.15.0
+retry2==0.9.5
+text-generation==0.6.1

projects/tgi-llm/container/tgi-llm.env.sample

@@ -0,0 +1 @@
+TGI_SERVICE_URL=http://{your-service-ip}:{your-service-port}