vvzvlad/ritual
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

infernet-1.0.0 update

Browse Source
This commit is contained in:
arshan-ritual
2024-06-06 13:18:48 -04:00
parent 2a11fd3953
commit 40a6c590da
98 changed files with 879 additions and 506 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
projects/tgi-llm/container/Dockerfile
View File

@ -7,12 +7,15 @@ ENV PYTHONDONTWRITEBYTECODE 1
ENV PIP_NO_CACHE_DIR 1
ENV RUNTIME docker
ENV PYTHONPATH src
ARG index_url
ENV UV_EXTRA_INDEX_URL ${index_url}
RUN apt-get update
RUN apt-get install -y git curl
# install uv
ADD --chmod=755 https://astral.sh/uv/install.sh /install.sh
ADD https://astral.sh/uv/install.sh /install.sh
RUN chmod 755 /install.sh
RUN /install.sh && rm /install.sh
COPY src/requirements.txt .

2
projects/tgi-llm/container/Makefile
View File

@ -5,7 +5,7 @@ TAG := $(DOCKER_ORG)/example-$(EXAMPLE_NAME)-infernet:latest
.phony: build run build-multiplatform
build:
@docker build -t $(TAG) .
@docker build -t $(TAG) --build-arg index_url=$(index_url) .
run:
docker run -p 3000:3000 --env-file tgi-llm.env $(TAG)

92
projects/tgi-llm/container/config.sample.json
View File

@ -1,52 +1,46 @@
{
"log_path": "infernet_node.log",
"server": {
"port": 4000
},
"chain": {
"enabled": true,
"trail_head_blocks": 0,
"rpc_url": "http://host.docker.internal:8545",
"coordinator_address": "0x5FbDB2315678afecb367f032d93F642f64180aa3",
"wallet": {
"max_gas_limit": 4000000,
"private_key": "0x59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d"
}
},
"startup_wait": 1.0,
"docker": {
"username": "your-username",
"password": ""
},
"redis": {
"host": "redis",
"port": 6379
},
"forward_stats": true,
"containers": [
{
"id": "tgi-llm",
"image": "ritualnetwork/example-tgi-llm-infernet:latest",
"external": true,
"port": "3000",
"allowed_delegate_addresses": [],
"allowed_addresses": [],
"allowed_ips": [],
"command": "--bind=0.0.0.0:3000 --workers=2",
"env": {
"TGI_SERVICE_URL": "http://{your_service_ip}:{your_service_port}"
}
"log_path": "infernet_node.log",
"server": {
"port": 4000
},
{
"id": "anvil-node",
"image": "ritualnetwork/infernet-anvil:0.0.0",
"external": true,
"port": "8545",
"allowed_delegate_addresses": [],
"allowed_addresses": [],
"allowed_ips": [],
"command": "",
"env": {}
}
]
"chain": {
"enabled": true,
"trail_head_blocks": 0,
"rpc_url": "http://host.docker.internal:8545",
"registry_address": "0x663F3ad617193148711d28f5334eE4Ed07016602",
"wallet": {
"max_gas_limit": 4000000,
"private_key": "0x59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d"
}
},
"startup_wait": 1.0,
"docker": {
"username": "your-username",
"password": ""
},
"redis": {
"host": "redis",
"port": 6379
},
"forward_stats": true,
"snapshot_sync": {
"sleep": 3,
"batch_size": 100
},
"containers": [
{
"id": "tgi-llm",
"image": "ritualnetwork/example-tgi-llm-infernet:latest",
"external": true,
"port": "3000",
"allowed_delegate_addresses": [],
"allowed_addresses": [],
"allowed_ips": [],
"command": "--bind=0.0.0.0:3000 --workers=2",
"env": {
"TGI_SERVICE_URL": "http://{your_service_ip}:{your_service_port}"
},
"accepted_payments": {}
}
]
}

84
projects/tgi-llm/container/src/app.py
View File

@ -2,10 +2,11 @@ import logging
import os
from typing import Any, cast
from eth_abi import decode, encode # type: ignore
from infernet_ml.utils.service_models import InfernetInput, InfernetInputSource
from eth_abi.abi import decode, encode
from infernet_ml.utils.service_models import InfernetInput, JobLocation
from infernet_ml.workflows.inference.tgi_client_inference_workflow import (
TGIClientInferenceWorkflow,
TgiInferenceRequest,
)
from quart import Quart, request
@ -16,7 +17,7 @@ def create_app() -> Quart:
app = Quart(__name__)
workflow = TGIClientInferenceWorkflow(
server_url=cast(str, os.environ.get("TGI_SERVICE_URL"))
server_url=os.environ["TGI_SERVICE_URL"],
)
workflow.setup()
@ -38,42 +39,51 @@ def create_app() -> Quart:
"""
infernet_input: InfernetInput = InfernetInput(**req_data)
if infernet_input.source == InfernetInputSource.OFFCHAIN:
prompt = cast(dict[str, Any], infernet_input.data).get("prompt")
else:
# On-chain requests are sent as a generalized hex-string which we will
# decode to the appropriate format.
(prompt,) = decode(
["string"], bytes.fromhex(cast(str, infernet_input.data))
)
match infernet_input:
case InfernetInput(source=JobLocation.OFFCHAIN):
prompt = cast(dict[str, Any], infernet_input.data).get("prompt")
case InfernetInput(source=JobLocation.ONCHAIN):
# On-chain requests are sent as a generalized hex-string which we will
# decode to the appropriate format.
(prompt,) = decode(
["string"], bytes.fromhex(cast(str, infernet_input.data))
)
case _:
raise ValueError("Invalid source")
result: dict[str, Any] = workflow.inference({"text": prompt})
result: dict[str, Any] = workflow.inference(
TgiInferenceRequest(text=cast(str, prompt))
)
if infernet_input.source == InfernetInputSource.OFFCHAIN:
"""
In case of an off-chain request, the result is returned as a dict. The
infernet node expects a dict format.
"""
return {"data": result}
else:
"""
In case of an on-chain request, the result is returned in the format:
{
"raw_input": str,
"processed_input": str,
"raw_output": str,
"processed_output": str,
"proof": str,
}
refer to: https://docs.ritual.net/infernet/node/containers for more info.
"""
return {
"raw_input": "",
"processed_input": "",
"raw_output": encode(["string"], [result]).hex(),
"processed_output": "",
"proof": "",
}
match infernet_input:
case InfernetInput(destination=JobLocation.OFFCHAIN):
"""
In case of an off-chain request, the result is returned as a dict. The
infernet node expects a dict format.
"""
return {"data": result}
case InfernetInput(destination=JobLocation.ONCHAIN):
"""
In case of an on-chain request, the result is returned in the format:
{
"raw_input": str,
"processed_input": str,
"raw_output": str,
"processed_output": str,
"proof": str,
}
refer to: https://docs.ritual.net/infernet/node/containers for more
info.
"""
return {
"raw_input": "",
"processed_input": "",
"raw_output": encode(["string"], [result]).hex(),
"processed_output": "",
"proof": "",
}
case _:
raise ValueError("Invalid destination")
return app

5
projects/tgi-llm/container/src/requirements.txt
View File

@ -1,6 +1,5 @@
quart==0.19.4
infernet_ml==0.1.0
PyArweave @ git+https://github.com/ritual-net/pyarweave.git
infernet-ml==1.0.0
infernet-ml[tgi_inference]==1.0.0
web3==6.15.0
retry2==0.9.5
text-generation==0.6.1

2
projects/tgi-llm/contracts/lib/forge-std

Submodule projects/tgi-llm/contracts/lib/forge-std updated: e4aef94c17...52715a217d

2
projects/tgi-llm/contracts/lib/infernet-sdk

Submodule projects/tgi-llm/contracts/lib/infernet-sdk updated: 2d04a7f5ed...8e6cd6f5cb

2
projects/tgi-llm/contracts/script/CallContract.s.sol
View File

@ -10,7 +10,7 @@ contract CallContract is Script {
uint256 deployerPrivateKey = vm.envUint("PRIVATE_KEY");
vm.startBroadcast(deployerPrivateKey);
Prompter prompter = Prompter(0x663F3ad617193148711d28f5334eE4Ed07016602);
Prompter prompter = Prompter(0x13D69Cf7d6CE4218F646B759Dcf334D82c023d8e);
prompter.promptLLM(vm.envString("prompt"));

4
projects/tgi-llm/contracts/script/Deploy.s.sol
View File

@ -14,9 +14,9 @@ contract Deploy is Script {
address deployerAddress = vm.addr(deployerPrivateKey);
console2.log("Loaded deployer: ", deployerAddress);
address coordinator = 0x5FbDB2315678afecb367f032d93F642f64180aa3;
address registry = 0x663F3ad617193148711d28f5334eE4Ed07016602;
// Create consumer
Prompter prompter = new Prompter(coordinator);
Prompter prompter = new Prompter(registry);
console2.log("Deployed Prompter: ", address(prompter));
// Execute

14
projects/tgi-llm/contracts/src/Prompter.sol
View File

@ -13,15 +13,17 @@ contract Prompter is CallbackConsumer {
"| | \\ \\ _| |_ | | | |__| / ____ \\| |____ \n"
"|_| \\_\\_____| |_| \\____/_/ \\_\\______| \n\n";
constructor(address coordinator) CallbackConsumer(coordinator) {}
constructor(address registry) CallbackConsumer(registry) {}
function promptLLM(string calldata prompt) public {
_requestCompute(
"tgi-llm",
abi.encode(prompt),
20 gwei,
1_000_000,
1
1, // redundancy
address(0), // paymentToken
0, // paymentAmount
address(0), // wallet
address(0) // prover
);
}
@ -32,7 +34,9 @@ contract Prompter is CallbackConsumer {
address node,
bytes calldata input,
bytes calldata output,
bytes calldata proof
bytes calldata proof,
bytes32 containerId,
uint256 index
) internal override {
console2.log(EXTREMELY_COOL_BANNER);
(bytes memory raw_output, bytes memory processed_output) = abi.decode(output, (bytes, bytes));

6
projects/tgi-llm/tgi-llm.md
View File

@ -334,7 +334,7 @@ Notice that in [the step above](#check-the-running-containers) we have an Anvil
By default, the [`anvil-node`](https://hub.docker.com/r/ritualnetwork/infernet-anvil) image used deploys the
[Infernet SDK](https://docs.ritual.net/infernet/sdk/introduction) and other relevant contracts for you:
- Coordinator: `0x5FbDB2315678afecb367f032d93F642f64180aa3`
- Coordinator: `0x663F3ad617193148711d28f5334eE4Ed07016602`
- Primary node: `0x70997970C51812dc3A010C7d01b50e0d17dc79C8`
### Deploy our `Prompter` smart contract
@ -367,7 +367,7 @@ You should expect to see similar Anvil logs:
eth_getTransactionReceipt
Transaction: 0x17a9d17cc515d39eef26b6a9427e04ed6f7ce6572d9756c07305c2df78d93ffe
Contract created: 0x663f3ad617193148711d28f5334ee4ed07016602
Contract created: 0x13D69Cf7d6CE4218F646B759Dcf334D82c023d8e
Gas used: 731312
Block Number: 1
@ -378,7 +378,7 @@ eth_getTransactionByHash
```
From our logs, we can see that the `Prompter` contract has been deployed to address
`0x663f3ad617193148711d28f5334ee4ed07016602`.
`0x13D69Cf7d6CE4218F646B759Dcf334D82c023d8e`.
### Call the contract

2
projects/tgi-llm/ui/Dockerfile
View File

@ -5,6 +5,8 @@ WORKDIR /app
ENV PYTHONUNBUFFERED 1
ENV PYTHONDONTWRITEBYTECODE 1
ENV PYTHONPATH src
ARG index_url
ENV UV_EXTRA_INDEX_URL ${index_url}
WORKDIR /app

2
projects/tgi-llm/ui/Makefile
View File

@ -5,7 +5,7 @@ TAG := $(DOCKER_ORG)/example-$(EXAMPLE_NAME)-infernet:latest
.phony: build run publish
build:
@docker build -t $(TAG) .
@docker build -t $(TAG) --build-arg index_url=$(index_url) .
run: build
docker run --env-file ./gradio_ui.env -p 3001:7860 $(TAG)

2
projects/tgi-llm/ui/src/requirements.txt
View File

@ -1,4 +1,4 @@
python-dotenv==1.0.0
gradio==3.47.1
gradio==4.19.2
huggingface-hub==0.17.3
text-generation==0.6.1