mirror of https://github.com/skygpu/skynet.git
Compare commits
No commits in common. "master" and "v0.1a10" have entirely different histories.
|
|
@ -7,4 +7,3 @@ outputs
|
|||
*.egg-info
|
||||
**/*.key
|
||||
**/*.cert
|
||||
.venv
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
skynet.toml
|
||||
skynet.ini
|
||||
.python-version
|
||||
hf_home
|
||||
outputs
|
||||
|
|
@ -9,5 +9,5 @@ secrets
|
|||
**/*.cert
|
||||
docs
|
||||
ipfs-docker-data
|
||||
ipfs-staging
|
||||
ipfs-docker-staging
|
||||
weights
|
||||
|
|
|
|||
104
README.md
104
README.md
|
|
@ -1,104 +1,30 @@
|
|||
# skynet
|
||||
<div align="center">
|
||||
<img src="https://explorer.skygpu.net/v2/explore/assets/logo.png" width=512 height=512>
|
||||
</div>
|
||||
|
||||
## decentralized compute platform
|
||||
|
||||
### native install
|
||||
|
||||
system dependencies:
|
||||
- `cuda` 11.8
|
||||
- `llvm` 10
|
||||
- `python` 3.10+
|
||||
- `docker` (for ipfs node)
|
||||
### decentralized compute platform
|
||||
|
||||
To launch a worker:
|
||||
```
|
||||
# create and edit config from template
|
||||
cp skynet.toml.example skynet.toml
|
||||
cp skynet.ini.example skynet.ini
|
||||
|
||||
# install poetry package manager
|
||||
curl -sSL https://install.python-poetry.org | python3 -
|
||||
# create python virtual envoirment 3.10+
|
||||
python3 -m venv venv
|
||||
|
||||
# install
|
||||
poetry install
|
||||
# enable envoirment
|
||||
source venv/bin/activate
|
||||
|
||||
# enable environment
|
||||
poetry shell
|
||||
# install requirements
|
||||
pip install -r requirements.txt
|
||||
pip install -r requirements.cuda.0.txt
|
||||
pip install -r requirements.cuda.1.txt
|
||||
pip install -r requirements.cuda.2.txt
|
||||
|
||||
# install skynet
|
||||
pip install -e .
|
||||
|
||||
# test you can run this command
|
||||
skynet --help
|
||||
|
||||
# launch ipfs node
|
||||
skynet run ipfs
|
||||
|
||||
# to launch worker
|
||||
skynet run dgpu
|
||||
|
||||
```
|
||||
|
||||
### dockerized install
|
||||
|
||||
## frontend
|
||||
|
||||
system dependencies:
|
||||
- `docker`
|
||||
|
||||
```
|
||||
# create and edit config from template
|
||||
cp skynet.toml.example skynet.toml
|
||||
|
||||
# pull runtime container
|
||||
docker pull guilledk/skynet:runtime-frontend
|
||||
|
||||
# run telegram bot
|
||||
docker run \
|
||||
-it \
|
||||
--rm \
|
||||
--network host \
|
||||
--name skynet-telegram \
|
||||
--mount type=bind,source="$(pwd)",target=/root/target \
|
||||
guilledk/skynet:runtime-frontend \
|
||||
skynet run telegram --db-pass PASSWORD --db-user USER --db-host HOST
|
||||
```
|
||||
|
||||
## worker
|
||||
|
||||
system dependencies:
|
||||
- `docker` with gpu enabled
|
||||
|
||||
```
|
||||
# create and edit config from template
|
||||
cp skynet.toml.example skynet.toml
|
||||
|
||||
# pull runtime container
|
||||
docker pull guilledk/skynet:runtime-cuda
|
||||
|
||||
# or build it (takes a bit of time)
|
||||
./build_docker.sh
|
||||
|
||||
# launch simple ipfs node
|
||||
./launch_ipfs.sh
|
||||
|
||||
# run worker with all gpus
|
||||
docker run \
|
||||
-it \
|
||||
--rm \
|
||||
--gpus all \
|
||||
--network host \
|
||||
--name skynet-worker \
|
||||
--mount type=bind,source="$(pwd)",target=/root/target \
|
||||
guilledk/skynet:runtime-cuda \
|
||||
skynet run dgpu
|
||||
|
||||
# run worker with specific gpu
|
||||
docker run \
|
||||
-it \
|
||||
--rm \
|
||||
--gpus '"device=1"' \
|
||||
--network host \
|
||||
--name skynet-worker-1 \
|
||||
--mount type=bind,source="$(pwd)",target=/root/target \
|
||||
guilledk/skynet:runtime-cuda \
|
||||
skynet run dgpu
|
||||
```
|
||||
|
|
|
|||
|
|
@ -0,0 +1,7 @@
|
|||
docker build \
|
||||
-t skynet:runtime-cuda \
|
||||
-f docker/Dockerfile.runtime+cuda .
|
||||
|
||||
docker build \
|
||||
-t skynet:runtime \
|
||||
-f docker/Dockerfile.runtime .
|
||||
|
|
@ -1,25 +1,16 @@
|
|||
from python:3.11
|
||||
from python:3.10.0
|
||||
|
||||
env DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
run apt-get update && apt-get install -y \
|
||||
git
|
||||
|
||||
run curl -sSL https://install.python-poetry.org | python3 -
|
||||
|
||||
env PATH "/root/.local/bin:$PATH"
|
||||
|
||||
copy . /skynet
|
||||
|
||||
workdir /skynet
|
||||
|
||||
env POETRY_VIRTUALENVS_PATH /skynet/.venv
|
||||
copy requirements.txt requirements.txt
|
||||
copy pytest.ini ./
|
||||
copy setup.py ./
|
||||
copy skynet ./skynet
|
||||
|
||||
run poetry install
|
||||
run pip install \
|
||||
-e . \
|
||||
-r requirements.txt
|
||||
|
||||
workdir /root/target
|
||||
|
||||
copy docker/entrypoint.sh /entrypoint.sh
|
||||
entrypoint ["/entrypoint.sh"]
|
||||
|
||||
cmd ["skynet", "--help"]
|
||||
copy tests ./
|
||||
|
|
|
|||
|
|
@ -0,0 +1,29 @@
|
|||
from nvidia/cuda:11.7.0-devel-ubuntu20.04
|
||||
from python:3.11
|
||||
|
||||
env DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
run apt-get update && \
|
||||
apt-get install -y ffmpeg libsm6 libxext6
|
||||
|
||||
workdir /skynet
|
||||
|
||||
copy requirements.cuda* ./
|
||||
|
||||
run pip install -U pip ninja
|
||||
run pip install -v -r requirements.cuda.0.txt
|
||||
run pip install -v -r requirements.cuda.1.txt
|
||||
run pip install -v -r requirements.cuda.2.txt
|
||||
|
||||
copy requirements.txt requirements.txt
|
||||
copy pytest.ini pytest.ini
|
||||
copy setup.py setup.py
|
||||
copy skynet skynet
|
||||
|
||||
run pip install -e . -r requirements.txt
|
||||
|
||||
env PYTORCH_CUDA_ALLOC_CONF max_split_size_mb:128
|
||||
env NVIDIA_VISIBLE_DEVICES=all
|
||||
env HF_HOME /hf_home
|
||||
|
||||
copy tests tests
|
||||
|
|
@ -1,46 +0,0 @@
|
|||
from nvidia/cuda:11.8.0-devel-ubuntu20.04
|
||||
from python:3.10
|
||||
|
||||
env DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
run apt-get update && apt-get install -y \
|
||||
git \
|
||||
clang \
|
||||
cmake \
|
||||
ffmpeg \
|
||||
libsm6 \
|
||||
libxext6 \
|
||||
ninja-build
|
||||
|
||||
env CC /usr/bin/clang
|
||||
env CXX /usr/bin/clang++
|
||||
|
||||
# install llvm10 as required by llvm-lite
|
||||
run git clone https://github.com/llvm/llvm-project.git -b llvmorg-10.0.1
|
||||
workdir /llvm-project
|
||||
# this adds a commit from 12.0.0 that fixes build on newer compilers
|
||||
run git cherry-pick -n b498303066a63a203d24f739b2d2e0e56dca70d1
|
||||
run cmake -S llvm -B build -G Ninja -DCMAKE_BUILD_TYPE=Release
|
||||
run ninja -C build install # -j8
|
||||
|
||||
run curl -sSL https://install.python-poetry.org | python3 -
|
||||
|
||||
env PATH "/root/.local/bin:$PATH"
|
||||
|
||||
copy . /skynet
|
||||
|
||||
workdir /skynet
|
||||
|
||||
env POETRY_VIRTUALENVS_PATH /skynet/.venv
|
||||
|
||||
run poetry install --with=cuda -v
|
||||
|
||||
workdir /root/target
|
||||
|
||||
env PYTORCH_CUDA_ALLOC_CONF max_split_size_mb:128
|
||||
env NVIDIA_VISIBLE_DEVICES=all
|
||||
|
||||
copy docker/entrypoint.sh /entrypoint.sh
|
||||
entrypoint ["/entrypoint.sh"]
|
||||
|
||||
cmd ["skynet", "--help"]
|
||||
|
|
@ -1,46 +0,0 @@
|
|||
from nvidia/cuda:11.8.0-devel-ubuntu20.04
|
||||
from python:3.11
|
||||
|
||||
env DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
run apt-get update && apt-get install -y \
|
||||
git \
|
||||
clang \
|
||||
cmake \
|
||||
ffmpeg \
|
||||
libsm6 \
|
||||
libxext6 \
|
||||
ninja-build
|
||||
|
||||
env CC /usr/bin/clang
|
||||
env CXX /usr/bin/clang++
|
||||
|
||||
# install llvm10 as required by llvm-lite
|
||||
run git clone https://github.com/llvm/llvm-project.git -b llvmorg-10.0.1
|
||||
workdir /llvm-project
|
||||
# this adds a commit from 12.0.0 that fixes build on newer compilers
|
||||
run git cherry-pick -n b498303066a63a203d24f739b2d2e0e56dca70d1
|
||||
run cmake -S llvm -B build -G Ninja -DCMAKE_BUILD_TYPE=Release
|
||||
run ninja -C build install # -j8
|
||||
|
||||
run curl -sSL https://install.python-poetry.org | python3 -
|
||||
|
||||
env PATH "/root/.local/bin:$PATH"
|
||||
|
||||
copy . /skynet
|
||||
|
||||
workdir /skynet
|
||||
|
||||
env POETRY_VIRTUALENVS_PATH /skynet/.venv
|
||||
|
||||
run poetry install --with=cuda -v
|
||||
|
||||
workdir /root/target
|
||||
|
||||
env PYTORCH_CUDA_ALLOC_CONF max_split_size_mb:128
|
||||
env NVIDIA_VISIBLE_DEVICES=all
|
||||
|
||||
copy docker/entrypoint.sh /entrypoint.sh
|
||||
entrypoint ["/entrypoint.sh"]
|
||||
|
||||
cmd ["skynet", "--help"]
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
from python:3.11
|
||||
|
||||
env DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
run apt-get update && apt-get install -y \
|
||||
git
|
||||
|
||||
run curl -sSL https://install.python-poetry.org | python3 -
|
||||
|
||||
env PATH "/root/.local/bin:$PATH"
|
||||
|
||||
copy . /skynet
|
||||
|
||||
workdir /skynet
|
||||
|
||||
env POETRY_VIRTUALENVS_PATH /skynet/.venv
|
||||
|
||||
run poetry install --with=frontend -v
|
||||
|
||||
workdir /root/target
|
||||
|
||||
copy docker/entrypoint.sh /entrypoint.sh
|
||||
entrypoint ["/entrypoint.sh"]
|
||||
|
||||
cmd ["skynet", "--help"]
|
||||
|
|
@ -1,20 +0,0 @@
|
|||
|
||||
docker build \
|
||||
-t guilledk/skynet:runtime \
|
||||
-f docker/Dockerfile.runtime .
|
||||
|
||||
docker build \
|
||||
-t guilledk/skynet:runtime-frontend \
|
||||
-f docker/Dockerfile.runtime+frontend .
|
||||
|
||||
docker build \
|
||||
-t guilledk/skynet:runtime-cuda-py311 \
|
||||
-f docker/Dockerfile.runtime+cuda-py311 .
|
||||
|
||||
docker build \
|
||||
-t guilledk/skynet:runtime-cuda \
|
||||
-f docker/Dockerfile.runtime+cuda-py311 .
|
||||
|
||||
docker build \
|
||||
-t guilledk/skynet:runtime-cuda-py310 \
|
||||
-f docker/Dockerfile.runtime+cuda-py310 .
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
#!/bin/sh
|
||||
|
||||
export VIRTUAL_ENV='/skynet/.venv'
|
||||
poetry env use $VIRTUAL_ENV/bin/python
|
||||
|
||||
poetry install
|
||||
|
||||
exec poetry run "$@"
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
FROM ubuntu:22.04
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
RUN apt-get update && apt-get install -y wget
|
||||
|
||||
# install eosio tools
|
||||
RUN wget https://github.com/AntelopeIO/leap/releases/download/v4.0.1/leap_4.0.1-ubuntu22.04_amd64.deb
|
||||
|
||||
RUN apt-get install -y ./leap_4.0.1-ubuntu22.04_amd64.deb
|
||||
|
||||
RUN mkdir -p /root/nodeos
|
||||
WORKDIR /root/nodeos
|
||||
COPY config.ini config.ini
|
||||
COPY contracts contracts
|
||||
COPY genesis genesis
|
||||
|
||||
EXPOSE 42000
|
||||
EXPOSE 29876
|
||||
EXPOSE 39999
|
||||
|
||||
CMD sleep 9999999999
|
||||
|
|
@ -0,0 +1,52 @@
|
|||
agent-name = Telos Skynet Testnet
|
||||
|
||||
wasm-runtime = eos-vm-jit
|
||||
eos-vm-oc-compile-threads = 4
|
||||
eos-vm-oc-enable = true
|
||||
|
||||
chain-state-db-size-mb = 65536
|
||||
enable-account-queries = true
|
||||
|
||||
http-server-address = 0.0.0.0:42000
|
||||
access-control-allow-origin = *
|
||||
contracts-console = true
|
||||
http-validate-host = false
|
||||
p2p-listen-endpoint = 0.0.0.0:29876
|
||||
p2p-server-address = 0.0.0.0:29876
|
||||
verbose-http-errors = true
|
||||
|
||||
state-history-endpoint = 0.0.0.0:39999
|
||||
trace-history = true
|
||||
chain-state-history = true
|
||||
trace-history-debug-mode = true
|
||||
state-history-dir = state-history
|
||||
|
||||
sync-fetch-span = 1600
|
||||
max-clients = 250
|
||||
|
||||
signature-provider = EOS5fLreY5Zq5owBhmNJTgQaLqQ4ufzXSTpStQakEyfxNFuUEgNs1=KEY:5JnvSc6pewpHHuUHwvbJopsew6AKwiGnexwDRc2Pj2tbdw6iML9
|
||||
|
||||
disable-subjective-billing = true
|
||||
max-transaction-time = 500
|
||||
read-only-read-window-time-us = 600000
|
||||
|
||||
abi-serializer-max-time-ms = 2000000
|
||||
|
||||
p2p-max-nodes-per-host = 1
|
||||
|
||||
connection-cleanup-period = 30
|
||||
allowed-connection = any
|
||||
http-max-response-time-ms = 100000
|
||||
max-body-size = 10000000
|
||||
|
||||
enable-stale-production = true
|
||||
|
||||
|
||||
plugin = eosio::http_plugin
|
||||
plugin = eosio::chain_plugin
|
||||
plugin = eosio::chain_api_plugin
|
||||
plugin = eosio::net_api_plugin
|
||||
plugin = eosio::net_plugin
|
||||
plugin = eosio::producer_plugin
|
||||
plugin = eosio::producer_api_plugin
|
||||
plugin = eosio::state_history_plugin
|
||||
|
|
@ -0,0 +1,360 @@
|
|||
{
|
||||
"____comment": "This file was generated with eosio-abigen. DO NOT EDIT Thu Apr 14 07:49:43 2022",
|
||||
"version": "eosio::abi/1.1",
|
||||
"structs": [
|
||||
{
|
||||
"name": "action",
|
||||
"base": "",
|
||||
"fields": [
|
||||
{
|
||||
"name": "account",
|
||||
"type": "name"
|
||||
},
|
||||
{
|
||||
"name": "name",
|
||||
"type": "name"
|
||||
},
|
||||
{
|
||||
"name": "authorization",
|
||||
"type": "permission_level[]"
|
||||
},
|
||||
{
|
||||
"name": "data",
|
||||
"type": "bytes"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "approval",
|
||||
"base": "",
|
||||
"fields": [
|
||||
{
|
||||
"name": "level",
|
||||
"type": "permission_level"
|
||||
},
|
||||
{
|
||||
"name": "time",
|
||||
"type": "time_point"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "approvals_info",
|
||||
"base": "",
|
||||
"fields": [
|
||||
{
|
||||
"name": "version",
|
||||
"type": "uint8"
|
||||
},
|
||||
{
|
||||
"name": "proposal_name",
|
||||
"type": "name"
|
||||
},
|
||||
{
|
||||
"name": "requested_approvals",
|
||||
"type": "approval[]"
|
||||
},
|
||||
{
|
||||
"name": "provided_approvals",
|
||||
"type": "approval[]"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "approve",
|
||||
"base": "",
|
||||
"fields": [
|
||||
{
|
||||
"name": "proposer",
|
||||
"type": "name"
|
||||
},
|
||||
{
|
||||
"name": "proposal_name",
|
||||
"type": "name"
|
||||
},
|
||||
{
|
||||
"name": "level",
|
||||
"type": "permission_level"
|
||||
},
|
||||
{
|
||||
"name": "proposal_hash",
|
||||
"type": "checksum256$"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "cancel",
|
||||
"base": "",
|
||||
"fields": [
|
||||
{
|
||||
"name": "proposer",
|
||||
"type": "name"
|
||||
},
|
||||
{
|
||||
"name": "proposal_name",
|
||||
"type": "name"
|
||||
},
|
||||
{
|
||||
"name": "canceler",
|
||||
"type": "name"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "exec",
|
||||
"base": "",
|
||||
"fields": [
|
||||
{
|
||||
"name": "proposer",
|
||||
"type": "name"
|
||||
},
|
||||
{
|
||||
"name": "proposal_name",
|
||||
"type": "name"
|
||||
},
|
||||
{
|
||||
"name": "executer",
|
||||
"type": "name"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "extension",
|
||||
"base": "",
|
||||
"fields": [
|
||||
{
|
||||
"name": "type",
|
||||
"type": "uint16"
|
||||
},
|
||||
{
|
||||
"name": "data",
|
||||
"type": "bytes"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "invalidate",
|
||||
"base": "",
|
||||
"fields": [
|
||||
{
|
||||
"name": "account",
|
||||
"type": "name"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "invalidation",
|
||||
"base": "",
|
||||
"fields": [
|
||||
{
|
||||
"name": "account",
|
||||
"type": "name"
|
||||
},
|
||||
{
|
||||
"name": "last_invalidation_time",
|
||||
"type": "time_point"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "old_approvals_info",
|
||||
"base": "",
|
||||
"fields": [
|
||||
{
|
||||
"name": "proposal_name",
|
||||
"type": "name"
|
||||
},
|
||||
{
|
||||
"name": "requested_approvals",
|
||||
"type": "permission_level[]"
|
||||
},
|
||||
{
|
||||
"name": "provided_approvals",
|
||||
"type": "permission_level[]"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "permission_level",
|
||||
"base": "",
|
||||
"fields": [
|
||||
{
|
||||
"name": "actor",
|
||||
"type": "name"
|
||||
},
|
||||
{
|
||||
"name": "permission",
|
||||
"type": "name"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "proposal",
|
||||
"base": "",
|
||||
"fields": [
|
||||
{
|
||||
"name": "proposal_name",
|
||||
"type": "name"
|
||||
},
|
||||
{
|
||||
"name": "packed_transaction",
|
||||
"type": "bytes"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "propose",
|
||||
"base": "",
|
||||
"fields": [
|
||||
{
|
||||
"name": "proposer",
|
||||
"type": "name"
|
||||
},
|
||||
{
|
||||
"name": "proposal_name",
|
||||
"type": "name"
|
||||
},
|
||||
{
|
||||
"name": "requested",
|
||||
"type": "permission_level[]"
|
||||
},
|
||||
{
|
||||
"name": "trx",
|
||||
"type": "transaction"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "transaction",
|
||||
"base": "transaction_header",
|
||||
"fields": [
|
||||
{
|
||||
"name": "context_free_actions",
|
||||
"type": "action[]"
|
||||
},
|
||||
{
|
||||
"name": "actions",
|
||||
"type": "action[]"
|
||||
},
|
||||
{
|
||||
"name": "transaction_extensions",
|
||||
"type": "extension[]"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "transaction_header",
|
||||
"base": "",
|
||||
"fields": [
|
||||
{
|
||||
"name": "expiration",
|
||||
"type": "time_point_sec"
|
||||
},
|
||||
{
|
||||
"name": "ref_block_num",
|
||||
"type": "uint16"
|
||||
},
|
||||
{
|
||||
"name": "ref_block_prefix",
|
||||
"type": "uint32"
|
||||
},
|
||||
{
|
||||
"name": "max_net_usage_words",
|
||||
"type": "varuint32"
|
||||
},
|
||||
{
|
||||
"name": "max_cpu_usage_ms",
|
||||
"type": "uint8"
|
||||
},
|
||||
{
|
||||
"name": "delay_sec",
|
||||
"type": "varuint32"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "unapprove",
|
||||
"base": "",
|
||||
"fields": [
|
||||
{
|
||||
"name": "proposer",
|
||||
"type": "name"
|
||||
},
|
||||
{
|
||||
"name": "proposal_name",
|
||||
"type": "name"
|
||||
},
|
||||
{
|
||||
"name": "level",
|
||||
"type": "permission_level"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"types": [],
|
||||
"actions": [
|
||||
{
|
||||
"name": "approve",
|
||||
"type": "approve",
|
||||
"ricardian_contract": ""
|
||||
},
|
||||
{
|
||||
"name": "cancel",
|
||||
"type": "cancel",
|
||||
"ricardian_contract": ""
|
||||
},
|
||||
{
|
||||
"name": "exec",
|
||||
"type": "exec",
|
||||
"ricardian_contract": ""
|
||||
},
|
||||
{
|
||||
"name": "invalidate",
|
||||
"type": "invalidate",
|
||||
"ricardian_contract": ""
|
||||
},
|
||||
{
|
||||
"name": "propose",
|
||||
"type": "propose",
|
||||
"ricardian_contract": ""
|
||||
},
|
||||
{
|
||||
"name": "unapprove",
|
||||
"type": "unapprove",
|
||||
"ricardian_contract": ""
|
||||
}
|
||||
],
|
||||
"tables": [
|
||||
{
|
||||
"name": "approvals",
|
||||
"type": "old_approvals_info",
|
||||
"index_type": "i64",
|
||||
"key_names": [],
|
||||
"key_types": []
|
||||
},
|
||||
{
|
||||
"name": "approvals2",
|
||||
"type": "approvals_info",
|
||||
"index_type": "i64",
|
||||
"key_names": [],
|
||||
"key_types": []
|
||||
},
|
||||
{
|
||||
"name": "invals",
|
||||
"type": "invalidation",
|
||||
"index_type": "i64",
|
||||
"key_names": [],
|
||||
"key_types": []
|
||||
},
|
||||
{
|
||||
"name": "proposal",
|
||||
"type": "proposal",
|
||||
"index_type": "i64",
|
||||
"key_names": [],
|
||||
"key_types": []
|
||||
}
|
||||
],
|
||||
"ricardian_clauses": [],
|
||||
"variants": [],
|
||||
"abi_extensions": []
|
||||
}
|
||||
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
|
|
@ -0,0 +1,185 @@
|
|||
{
|
||||
"____comment": "This file was generated with eosio-abigen. DO NOT EDIT ",
|
||||
"version": "eosio::abi/1.1",
|
||||
"types": [],
|
||||
"structs": [
|
||||
{
|
||||
"name": "account",
|
||||
"base": "",
|
||||
"fields": [
|
||||
{
|
||||
"name": "balance",
|
||||
"type": "asset"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "close",
|
||||
"base": "",
|
||||
"fields": [
|
||||
{
|
||||
"name": "owner",
|
||||
"type": "name"
|
||||
},
|
||||
{
|
||||
"name": "symbol",
|
||||
"type": "symbol"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "create",
|
||||
"base": "",
|
||||
"fields": [
|
||||
{
|
||||
"name": "issuer",
|
||||
"type": "name"
|
||||
},
|
||||
{
|
||||
"name": "maximum_supply",
|
||||
"type": "asset"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "currency_stats",
|
||||
"base": "",
|
||||
"fields": [
|
||||
{
|
||||
"name": "supply",
|
||||
"type": "asset"
|
||||
},
|
||||
{
|
||||
"name": "max_supply",
|
||||
"type": "asset"
|
||||
},
|
||||
{
|
||||
"name": "issuer",
|
||||
"type": "name"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "issue",
|
||||
"base": "",
|
||||
"fields": [
|
||||
{
|
||||
"name": "to",
|
||||
"type": "name"
|
||||
},
|
||||
{
|
||||
"name": "quantity",
|
||||
"type": "asset"
|
||||
},
|
||||
{
|
||||
"name": "memo",
|
||||
"type": "string"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "open",
|
||||
"base": "",
|
||||
"fields": [
|
||||
{
|
||||
"name": "owner",
|
||||
"type": "name"
|
||||
},
|
||||
{
|
||||
"name": "symbol",
|
||||
"type": "symbol"
|
||||
},
|
||||
{
|
||||
"name": "ram_payer",
|
||||
"type": "name"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "retire",
|
||||
"base": "",
|
||||
"fields": [
|
||||
{
|
||||
"name": "quantity",
|
||||
"type": "asset"
|
||||
},
|
||||
{
|
||||
"name": "memo",
|
||||
"type": "string"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "transfer",
|
||||
"base": "",
|
||||
"fields": [
|
||||
{
|
||||
"name": "from",
|
||||
"type": "name"
|
||||
},
|
||||
{
|
||||
"name": "to",
|
||||
"type": "name"
|
||||
},
|
||||
{
|
||||
"name": "quantity",
|
||||
"type": "asset"
|
||||
},
|
||||
{
|
||||
"name": "memo",
|
||||
"type": "string"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"actions": [
|
||||
{
|
||||
"name": "close",
|
||||
"type": "close",
|
||||
"ricardian_contract": "---\nspec_version: \"0.2.0\"\ntitle: Close Token Balance\nsummary: 'Close {{nowrap owner}}’s zero quantity balance'\nicon: http://127.0.0.1/ricardian_assets/eosio.contracts/icons/token.png#207ff68b0406eaa56618b08bda81d6a0954543f36adc328ab3065f31a5c5d654\n---\n\n{{owner}} agrees to close their zero quantity balance for the {{symbol_to_symbol_code symbol}} token.\n\nRAM will be refunded to the RAM payer of the {{symbol_to_symbol_code symbol}} token balance for {{owner}}."
|
||||
},
|
||||
{
|
||||
"name": "create",
|
||||
"type": "create",
|
||||
"ricardian_contract": "---\nspec_version: \"0.2.0\"\ntitle: Create New Token\nsummary: 'Create a new token'\nicon: http://127.0.0.1/ricardian_assets/eosio.contracts/icons/token.png#207ff68b0406eaa56618b08bda81d6a0954543f36adc328ab3065f31a5c5d654\n---\n\n{{$action.account}} agrees to create a new token with symbol {{asset_to_symbol_code maximum_supply}} to be managed by {{issuer}}.\n\nThis action will not result any any tokens being issued into circulation.\n\n{{issuer}} will be allowed to issue tokens into circulation, up to a maximum supply of {{maximum_supply}}.\n\nRAM will deducted from {{$action.account}}’s resources to create the necessary records."
|
||||
},
|
||||
{
|
||||
"name": "issue",
|
||||
"type": "issue",
|
||||
"ricardian_contract": "---\nspec_version: \"0.2.0\"\ntitle: Issue Tokens into Circulation\nsummary: 'Issue {{nowrap quantity}} into circulation and transfer into {{nowrap to}}’s account'\nicon: http://127.0.0.1/ricardian_assets/eosio.contracts/icons/token.png#207ff68b0406eaa56618b08bda81d6a0954543f36adc328ab3065f31a5c5d654\n---\n\nThe token manager agrees to issue {{quantity}} into circulation, and transfer it into {{to}}’s account.\n\n{{#if memo}}There is a memo attached to the transfer stating:\n{{memo}}\n{{/if}}\n\nIf {{to}} does not have a balance for {{asset_to_symbol_code quantity}}, or the token manager does not have a balance for {{asset_to_symbol_code quantity}}, the token manager will be designated as the RAM payer of the {{asset_to_symbol_code quantity}} token balance for {{to}}. As a result, RAM will be deducted from the token manager’s resources to create the necessary records.\n\nThis action does not allow the total quantity to exceed the max allowed supply of the token."
|
||||
},
|
||||
{
|
||||
"name": "open",
|
||||
"type": "open",
|
||||
"ricardian_contract": "---\nspec_version: \"0.2.0\"\ntitle: Open Token Balance\nsummary: 'Open a zero quantity balance for {{nowrap owner}}'\nicon: http://127.0.0.1/ricardian_assets/eosio.contracts/icons/token.png#207ff68b0406eaa56618b08bda81d6a0954543f36adc328ab3065f31a5c5d654\n---\n\n{{ram_payer}} agrees to establish a zero quantity balance for {{owner}} for the {{symbol_to_symbol_code symbol}} token.\n\nIf {{owner}} does not have a balance for {{symbol_to_symbol_code symbol}}, {{ram_payer}} will be designated as the RAM payer of the {{symbol_to_symbol_code symbol}} token balance for {{owner}}. As a result, RAM will be deducted from {{ram_payer}}’s resources to create the necessary records."
|
||||
},
|
||||
{
|
||||
"name": "retire",
|
||||
"type": "retire",
|
||||
"ricardian_contract": "---\nspec_version: \"0.2.0\"\ntitle: Remove Tokens from Circulation\nsummary: 'Remove {{nowrap quantity}} from circulation'\nicon: http://127.0.0.1/ricardian_assets/eosio.contracts/icons/token.png#207ff68b0406eaa56618b08bda81d6a0954543f36adc328ab3065f31a5c5d654\n---\n\nThe token manager agrees to remove {{quantity}} from circulation, taken from their own account.\n\n{{#if memo}} There is a memo attached to the action stating:\n{{memo}}\n{{/if}}"
|
||||
},
|
||||
{
|
||||
"name": "transfer",
|
||||
"type": "transfer",
|
||||
"ricardian_contract": "---\nspec_version: \"0.2.0\"\ntitle: Transfer Tokens\nsummary: 'Send {{nowrap quantity}} from {{nowrap from}} to {{nowrap to}}'\nicon: http://127.0.0.1/ricardian_assets/eosio.contracts/icons/transfer.png#5dfad0df72772ee1ccc155e670c1d124f5c5122f1d5027565df38b418042d1dd\n---\n\n{{from}} agrees to send {{quantity}} to {{to}}.\n\n{{#if memo}}There is a memo attached to the transfer stating:\n{{memo}}\n{{/if}}\n\nIf {{from}} is not already the RAM payer of their {{asset_to_symbol_code quantity}} token balance, {{from}} will be designated as such. As a result, RAM will be deducted from {{from}}’s resources to refund the original RAM payer.\n\nIf {{to}} does not have a balance for {{asset_to_symbol_code quantity}}, {{from}} will be designated as the RAM payer of the {{asset_to_symbol_code quantity}} token balance for {{to}}. As a result, RAM will be deducted from {{from}}’s resources to create the necessary records."
|
||||
}
|
||||
],
|
||||
"tables": [
|
||||
{
|
||||
"name": "accounts",
|
||||
"type": "account",
|
||||
"index_type": "i64",
|
||||
"key_names": [],
|
||||
"key_types": []
|
||||
},
|
||||
{
|
||||
"name": "stat",
|
||||
"type": "currency_stats",
|
||||
"index_type": "i64",
|
||||
"key_names": [],
|
||||
"key_types": []
|
||||
}
|
||||
],
|
||||
"ricardian_clauses": [],
|
||||
"variants": []
|
||||
}
|
||||
Binary file not shown.
|
|
@ -0,0 +1,130 @@
|
|||
{
|
||||
"____comment": "This file was generated with eosio-abigen. DO NOT EDIT Thu Apr 14 07:49:40 2022",
|
||||
"version": "eosio::abi/1.1",
|
||||
"structs": [
|
||||
{
|
||||
"name": "action",
|
||||
"base": "",
|
||||
"fields": [
|
||||
{
|
||||
"name": "account",
|
||||
"type": "name"
|
||||
},
|
||||
{
|
||||
"name": "name",
|
||||
"type": "name"
|
||||
},
|
||||
{
|
||||
"name": "authorization",
|
||||
"type": "permission_level[]"
|
||||
},
|
||||
{
|
||||
"name": "data",
|
||||
"type": "bytes"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "exec",
|
||||
"base": "",
|
||||
"fields": [
|
||||
{
|
||||
"name": "executer",
|
||||
"type": "name"
|
||||
},
|
||||
{
|
||||
"name": "trx",
|
||||
"type": "transaction"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "extension",
|
||||
"base": "",
|
||||
"fields": [
|
||||
{
|
||||
"name": "type",
|
||||
"type": "uint16"
|
||||
},
|
||||
{
|
||||
"name": "data",
|
||||
"type": "bytes"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "permission_level",
|
||||
"base": "",
|
||||
"fields": [
|
||||
{
|
||||
"name": "actor",
|
||||
"type": "name"
|
||||
},
|
||||
{
|
||||
"name": "permission",
|
||||
"type": "name"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "transaction",
|
||||
"base": "transaction_header",
|
||||
"fields": [
|
||||
{
|
||||
"name": "context_free_actions",
|
||||
"type": "action[]"
|
||||
},
|
||||
{
|
||||
"name": "actions",
|
||||
"type": "action[]"
|
||||
},
|
||||
{
|
||||
"name": "transaction_extensions",
|
||||
"type": "extension[]"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "transaction_header",
|
||||
"base": "",
|
||||
"fields": [
|
||||
{
|
||||
"name": "expiration",
|
||||
"type": "time_point_sec"
|
||||
},
|
||||
{
|
||||
"name": "ref_block_num",
|
||||
"type": "uint16"
|
||||
},
|
||||
{
|
||||
"name": "ref_block_prefix",
|
||||
"type": "uint32"
|
||||
},
|
||||
{
|
||||
"name": "max_net_usage_words",
|
||||
"type": "varuint32"
|
||||
},
|
||||
{
|
||||
"name": "max_cpu_usage_ms",
|
||||
"type": "uint8"
|
||||
},
|
||||
{
|
||||
"name": "delay_sec",
|
||||
"type": "varuint32"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"types": [],
|
||||
"actions": [
|
||||
{
|
||||
"name": "exec",
|
||||
"type": "exec",
|
||||
"ricardian_contract": ""
|
||||
}
|
||||
],
|
||||
"tables": [],
|
||||
"ricardian_clauses": [],
|
||||
"variants": [],
|
||||
"abi_extensions": []
|
||||
}
|
||||
Binary file not shown.
File diff suppressed because it is too large
Load Diff
Binary file not shown.
|
|
@ -0,0 +1,25 @@
|
|||
{
|
||||
"initial_timestamp": "2023-05-22T00:00:00.000",
|
||||
"initial_key": "EOS5fLreY5Zq5owBhmNJTgQaLqQ4ufzXSTpStQakEyfxNFuUEgNs1",
|
||||
"initial_configuration": {
|
||||
"max_block_net_usage": 1048576,
|
||||
"target_block_net_usage_pct": 1000,
|
||||
"max_transaction_net_usage": 1048575,
|
||||
"base_per_transaction_net_usage": 12,
|
||||
"net_usage_leeway": 500,
|
||||
"context_free_discount_net_usage_num": 20,
|
||||
"context_free_discount_net_usage_den": 100,
|
||||
"max_block_cpu_usage": 200000,
|
||||
"target_block_cpu_usage_pct": 1000,
|
||||
"max_transaction_cpu_usage": 150000,
|
||||
"min_transaction_cpu_usage": 100,
|
||||
"max_transaction_lifetime": 3600,
|
||||
"deferred_trx_expiration_window": 600,
|
||||
"max_transaction_delay": 3888000,
|
||||
"max_inline_action_size": 4096,
|
||||
"max_inline_action_depth": 4,
|
||||
"max_authority_depth": 6
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
docker push guilledk/skynet:runtime
|
||||
docker push guilledk/skynet:runtime-frontend
|
||||
docker push guilledk/skynet:runtime-cuda
|
||||
docker push guilledk/skynet:runtime-cuda-py311
|
||||
docker push guilledk/skynet:runtime-cuda-py310
|
||||
|
|
@ -1,36 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
name='skynet-ipfs'
|
||||
peers=("$@")
|
||||
|
||||
data_dir="$(pwd)/ipfs-docker-data"
|
||||
data_target='/data/ipfs'
|
||||
|
||||
# Create data directory if it doesn't exist
|
||||
mkdir -p "$data_dir"
|
||||
|
||||
# Run the container
|
||||
docker run -d \
|
||||
--name "$name" \
|
||||
-p 8080:8080/tcp \
|
||||
-p 4001:4001/tcp \
|
||||
-p 127.0.0.1:5001:5001/tcp \
|
||||
--mount type=bind,source="$data_dir",target="$data_target" \
|
||||
--rm \
|
||||
ipfs/go-ipfs:latest
|
||||
|
||||
# Change ownership
|
||||
docker exec "$name" chown 1000:1000 -R "$data_target"
|
||||
|
||||
# Wait for Daemon to be ready
|
||||
while read -r log; do
|
||||
echo "$log"
|
||||
if [[ "$log" == *"Daemon is ready"* ]]; then
|
||||
break
|
||||
fi
|
||||
done < <(docker logs -f "$name")
|
||||
|
||||
# Connect to peers
|
||||
for peer in "${peers[@]}"; do
|
||||
docker exec "$name" ipfs swarm connect "$peer" || echo "Error connecting to peer: $peer"
|
||||
done
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -1,2 +0,0 @@
|
|||
[virtualenvs]
|
||||
in-project = true
|
||||
|
|
@ -1,67 +0,0 @@
|
|||
[tool.poetry]
|
||||
name = 'skynet'
|
||||
version = '0.1a12'
|
||||
description = 'Decentralized compute platform'
|
||||
authors = ['Guillermo Rodriguez <guillermo@telos.net>']
|
||||
license = 'AGPL'
|
||||
readme = 'README.md'
|
||||
|
||||
[tool.poetry.dependencies]
|
||||
python = '>=3.10,<3.12'
|
||||
pytz = '^2023.3.post1'
|
||||
trio = '^0.22.2'
|
||||
asks = '^3.0.0'
|
||||
Pillow = '^10.0.1'
|
||||
docker = '^6.1.3'
|
||||
py-leap = {git = 'https://github.com/guilledk/py-leap.git', rev = 'v0.1a14'}
|
||||
toml = '^0.10.2'
|
||||
|
||||
[tool.poetry.group.frontend]
|
||||
optional = true
|
||||
|
||||
[tool.poetry.group.frontend.dependencies]
|
||||
triopg = {version = '^0.6.0'}
|
||||
aiohttp = {version = '^3.8.5'}
|
||||
psycopg2-binary = {version = '^2.9.7'}
|
||||
pyTelegramBotAPI = {version = '^4.14.0'}
|
||||
'discord.py' = {version = '^2.3.2'}
|
||||
|
||||
[tool.poetry.group.dev]
|
||||
optional = true
|
||||
|
||||
[tool.poetry.group.dev.dependencies]
|
||||
pdbpp = {version = '^0.10.3'}
|
||||
pytest = {version = '^7.4.2'}
|
||||
|
||||
[tool.poetry.group.cuda]
|
||||
optional = true
|
||||
|
||||
[tool.poetry.group.cuda.dependencies]
|
||||
torch = {version = '2.0.1+cu118', source = 'torch'}
|
||||
scipy = {version = '^1.11.2'}
|
||||
numba = {version = '0.57.0'}
|
||||
quart = {version = '^0.19.3'}
|
||||
triton = {version = '2.0.0', source = 'torch'}
|
||||
basicsr = {version = '^1.4.2'}
|
||||
xformers = {version = '^0.0.22'}
|
||||
hypercorn = {version = '^0.14.4'}
|
||||
diffusers = {version = '^0.21.2'}
|
||||
realesrgan = {version = '^0.3.0'}
|
||||
quart-trio = {version = '^0.11.0'}
|
||||
torchvision = {version = '0.15.2+cu118', source = 'torch'}
|
||||
accelerate = {version = '^0.23.0'}
|
||||
transformers = {version = '^4.33.2'}
|
||||
huggingface-hub = {version = '^0.17.3'}
|
||||
invisible-watermark = {version = '^0.2.0'}
|
||||
|
||||
[[tool.poetry.source]]
|
||||
name = 'torch'
|
||||
url = 'https://download.pytorch.org/whl/cu118'
|
||||
priority = 'explicit'
|
||||
|
||||
[build-system]
|
||||
requires = ['poetry-core', 'cython']
|
||||
build-backend = 'poetry.core.masonry.api'
|
||||
|
||||
[tool.poetry.scripts]
|
||||
skynet = 'skynet.cli:skynet'
|
||||
|
|
@ -1,4 +1,3 @@
|
|||
[pytest]
|
||||
log_cli = True
|
||||
log_level = info
|
||||
trio_mode = True
|
||||
|
|
|
|||
|
|
@ -0,0 +1,9 @@
|
|||
scipy
|
||||
triton
|
||||
accelerate
|
||||
transformers
|
||||
huggingface_hub
|
||||
diffusers[torch]>=0.18.0
|
||||
invisible-watermark
|
||||
torch==1.13.0+cu117
|
||||
--extra-index-url https://download.pytorch.org/whl/cu117
|
||||
|
|
@ -0,0 +1 @@
|
|||
git+https://github.com/facebookresearch/xformers.git@main#egg=xformers
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
basicsr
|
||||
realesrgan
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
pytz
|
||||
trio
|
||||
asks
|
||||
numpy
|
||||
pdbpp
|
||||
Pillow
|
||||
triopg
|
||||
pytest
|
||||
docker
|
||||
aiohttp
|
||||
psycopg2-binary
|
||||
pyTelegramBotAPI
|
||||
discord.py
|
||||
|
||||
py-leap@git+https://github.com/guilledk/py-leap.git@v0.1a14
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
from setuptools import setup, find_packages
|
||||
|
||||
from skynet.constants import VERSION
|
||||
|
||||
setup(
|
||||
name='skynet',
|
||||
version=VERSION,
|
||||
description='Decentralized compute platform',
|
||||
author='Guillermo Rodriguez',
|
||||
author_email='guillermo@telos.net',
|
||||
packages=find_packages(),
|
||||
entry_points={
|
||||
'console_scripts': [
|
||||
'skynet = skynet.cli:skynet',
|
||||
'txt2img = skynet.cli:txt2img',
|
||||
'img2img = skynet.cli:img2img',
|
||||
'upscale = skynet.cli:upscale'
|
||||
]
|
||||
},
|
||||
install_requires=['click']
|
||||
)
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
[skynet.dgpu]
|
||||
account = testworkerX
|
||||
permission = active
|
||||
key = 5Xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
|
||||
|
||||
node_url = https://skynet.ancap.tech
|
||||
hyperion_url = https://skynet.ancap.tech
|
||||
ipfs_url = /ip4/169.197.140.154/tcp/4001/p2p/12D3KooWKWogLFNEcNNMKnzU7Snrnuj84RZdMBg3sLiQSQc51oEv
|
||||
|
||||
hf_home = hf_home
|
||||
hf_token = hf_XxXxXxXxXxXxXxXxXxXxXxXxXxXxXxXxXx
|
||||
|
||||
auto_withdraw = True
|
||||
|
||||
[skynet.telegram]
|
||||
account = telegram
|
||||
permission = active
|
||||
key = 5Xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
|
||||
|
||||
node_url = https://skynet.ancap.tech
|
||||
hyperion_url = https://skynet.ancap.tech
|
||||
ipfs_url = /ip4/169.197.140.154/tcp/4001/p2p/12D3KooWKWogLFNEcNNMKnzU7Snrnuj84RZdMBg3sLiQSQc51oEv
|
||||
|
||||
token = XXXXXXXXXX:xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
|
||||
|
||||
[skynet.discord]
|
||||
account = discord
|
||||
permission = active
|
||||
key = 5Xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
|
||||
|
||||
node_url = https://skynet.ancap.tech
|
||||
hyperion_url = https://skynet.ancap.tech
|
||||
ipfs_url = /ip4/169.197.140.154/tcp/4001/p2p/12D3KooWKWogLFNEcNNMKnzU7Snrnuj84RZdMBg3sLiQSQc51oEv
|
||||
|
||||
token = XXXXXXXXXX:xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
|
||||
|
|
@ -1,47 +0,0 @@
|
|||
# config sections are optional, depending on which services
|
||||
# you wish to run
|
||||
|
||||
[skynet.dgpu]
|
||||
account = 'testworkerX'
|
||||
permission = 'active'
|
||||
key = '5Xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'
|
||||
node_url = 'https://testnet.skygpu.net'
|
||||
hyperion_url = 'https://testnet.skygpu.net'
|
||||
ipfs_gateway_url = '/ip4/169.197.140.154/tcp/4001/p2p/12D3KooWKWogLFNEcNNMKnzU7Snrnuj84RZdMBg3sLiQSQc51oEv'
|
||||
ipfs_url = 'http://127.0.0.1:5001'
|
||||
hf_home = 'hf_home'
|
||||
hf_token = 'hf_XxXxXxXxXxXxXxXxXxXxXxXxXxXxXxXxXx'
|
||||
auto_withdraw = true
|
||||
non_compete = []
|
||||
api_bind = '127.0.0.1:42690'
|
||||
|
||||
[skynet.telegram]
|
||||
account = 'telegram'
|
||||
permission = 'active'
|
||||
key = '5Xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'
|
||||
node_url = 'https://testnet.skygpu.net'
|
||||
hyperion_url = 'https://testnet.skygpu.net'
|
||||
ipfs_gateway_url = '/ip4/169.197.140.154/tcp/4001/p2p/12D3KooWKWogLFNEcNNMKnzU7Snrnuj84RZdMBg3sLiQSQc51oEv'
|
||||
ipfs_url = 'http://127.0.0.1:5001'
|
||||
token = 'XXXXXXXXXX:xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'
|
||||
|
||||
[skynet.discord]
|
||||
account = 'discord'
|
||||
permission = 'active'
|
||||
key = '5Xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'
|
||||
node_url = 'https://testnet.skygpu.net'
|
||||
hyperion_url = 'https://testnet.skygpu.net'
|
||||
ipfs_gateway_url = '/ip4/169.197.140.154/tcp/4001/p2p/12D3KooWKWogLFNEcNNMKnzU7Snrnuj84RZdMBg3sLiQSQc51oEv'
|
||||
ipfs_url = 'http://127.0.0.1:5001'
|
||||
token = 'XXXXXXXXXX:xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'
|
||||
|
||||
[skynet.pinner]
|
||||
hyperion_url = 'https://testnet.skygpu.net'
|
||||
ipfs_url = 'http://127.0.0.1:5001'
|
||||
|
||||
[skynet.user]
|
||||
account = 'testuser'
|
||||
permission = 'active'
|
||||
key = '5Xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'
|
||||
node_url = 'https://testnet.skygpu.net'
|
||||
|
||||
379
skynet/cli.py
379
skynet/cli.py
|
|
@ -6,12 +6,25 @@ import random
|
|||
|
||||
from functools import partial
|
||||
|
||||
import trio
|
||||
import asks
|
||||
import click
|
||||
import asyncio
|
||||
import requests
|
||||
|
||||
from leap.sugar import Name, asset_from_str
|
||||
from leap.cleos import CLEOS
|
||||
from leap.sugar import collect_stdout, Name, asset_from_str
|
||||
from leap.hyperion import HyperionAPI
|
||||
|
||||
from skynet.ipfs import IPFSHTTP
|
||||
|
||||
|
||||
from .db import open_new_database
|
||||
from .config import *
|
||||
from .nodeos import open_cleos, open_nodeos
|
||||
from .constants import *
|
||||
from .frontend.telegram import SkynetTelegramFrontend
|
||||
from .frontend.discord import SkynetDiscordFrontend
|
||||
|
||||
|
||||
@click.group()
|
||||
|
|
@ -31,11 +44,7 @@ def skynet(*args, **kwargs):
|
|||
@click.option('--seed', '-S', default=None)
|
||||
def txt2img(*args, **kwargs):
|
||||
from . import utils
|
||||
|
||||
config = load_skynet_toml()
|
||||
hf_token = load_key(config, 'skynet.dgpu.hf_token')
|
||||
hf_home = load_key(config, 'skynet.dgpu.hf_home')
|
||||
set_hf_vars(hf_token, hf_home)
|
||||
_, hf_token, _, _ = init_env_from_config()
|
||||
utils.txt2img(hf_token, **kwargs)
|
||||
|
||||
@click.command()
|
||||
|
|
@ -50,10 +59,7 @@ def txt2img(*args, **kwargs):
|
|||
@click.option('--seed', '-S', default=None)
|
||||
def img2img(model, prompt, input, output, strength, guidance, steps, seed):
|
||||
from . import utils
|
||||
config = load_skynet_toml()
|
||||
hf_token = load_key(config, 'skynet.dgpu.hf_token')
|
||||
hf_home = load_key(config, 'skynet.dgpu.hf_home')
|
||||
set_hf_vars(hf_token, hf_home)
|
||||
_, hf_token, _, _ = init_env_from_config()
|
||||
utils.img2img(
|
||||
hf_token,
|
||||
model=model,
|
||||
|
|
@ -81,55 +87,48 @@ def upscale(input, output, model):
|
|||
@skynet.command()
|
||||
def download():
|
||||
from . import utils
|
||||
config = load_skynet_toml()
|
||||
hf_token = load_key(config, 'skynet.dgpu.hf_token')
|
||||
hf_home = load_key(config, 'skynet.dgpu.hf_home')
|
||||
set_hf_vars(hf_token, hf_home)
|
||||
utils.download_all_models(hf_token, hf_home)
|
||||
_, hf_token, _, _ = init_env_from_config()
|
||||
utils.download_all_models(hf_token)
|
||||
|
||||
@skynet.command()
|
||||
@click.option(
|
||||
'--account', '-A', default=None)
|
||||
@click.option(
|
||||
'--permission', '-P', default=None)
|
||||
@click.option(
|
||||
'--key', '-k', default=None)
|
||||
@click.option(
|
||||
'--node-url', '-n', default='https://skynet.ancap.tech')
|
||||
@click.option(
|
||||
'--reward', '-r', default='20.0000 GPU')
|
||||
@click.option('--jobs', '-j', default=1)
|
||||
@click.option('--model', '-m', default='stabilityai/stable-diffusion-xl-base-1.0')
|
||||
@click.option('--model', '-m', default='prompthero/openjourney')
|
||||
@click.option(
|
||||
'--prompt', '-p', default='a red old tractor in a sunny wheat field')
|
||||
@click.option('--output', '-o', default='output.png')
|
||||
@click.option('--width', '-w', default=1024)
|
||||
@click.option('--height', '-h', default=1024)
|
||||
@click.option('--width', '-w', default=512)
|
||||
@click.option('--height', '-h', default=512)
|
||||
@click.option('--guidance', '-g', default=10)
|
||||
@click.option('--step', '-s', default=26)
|
||||
@click.option('--seed', '-S', default=None)
|
||||
@click.option('--upscaler', '-U', default='x4')
|
||||
@click.option('--binary_data', '-b', default='')
|
||||
@click.option('--strength', '-Z', default=None)
|
||||
def enqueue(
|
||||
account: str,
|
||||
permission: str,
|
||||
key: str | None,
|
||||
node_url: str,
|
||||
reward: str,
|
||||
jobs: int,
|
||||
**kwargs
|
||||
):
|
||||
import trio
|
||||
from leap.cleos import CLEOS
|
||||
key, account, permission = load_account_info(
|
||||
'user', key, account, permission)
|
||||
|
||||
config = load_skynet_toml()
|
||||
|
||||
key = load_key(config, 'skynet.user.key')
|
||||
account = load_key(config, 'skynet.user.account')
|
||||
permission = load_key(config, 'skynet.user.permission')
|
||||
node_url = load_key(config, 'skynet.user.node_url')
|
||||
|
||||
cleos = CLEOS(None, None, url=node_url, remote=node_url)
|
||||
|
||||
binary = kwargs['binary_data']
|
||||
if not kwargs['strength']:
|
||||
if binary:
|
||||
raise ValueError('strength -Z param required if binary data passed')
|
||||
|
||||
del kwargs['strength']
|
||||
|
||||
else:
|
||||
kwargs['strength'] = float(kwargs['strength'])
|
||||
node_url, _, _ = load_endpoint_info(
|
||||
'user', node_url, None, None)
|
||||
|
||||
with open_cleos(node_url, key=key) as cleos:
|
||||
async def enqueue_n_jobs():
|
||||
for i in range(jobs):
|
||||
if not kwargs['seed']:
|
||||
|
|
@ -139,6 +138,7 @@ def enqueue(
|
|||
'method': 'diffuse',
|
||||
'params': kwargs
|
||||
})
|
||||
binary = kwargs['binary_data']
|
||||
|
||||
res = await cleos.a_push_action(
|
||||
'telos.gpu',
|
||||
|
|
@ -153,23 +153,31 @@ def enqueue(
|
|||
account, key, permission,
|
||||
)
|
||||
print(res)
|
||||
|
||||
trio.run(enqueue_n_jobs)
|
||||
|
||||
|
||||
@skynet.command()
|
||||
@click.option('--loglevel', '-l', default='INFO', help='Logging level')
|
||||
@click.option(
|
||||
'--account', '-A', default='telos.gpu')
|
||||
@click.option(
|
||||
'--permission', '-P', default='active')
|
||||
@click.option(
|
||||
'--key', '-k', default=None)
|
||||
@click.option(
|
||||
'--node-url', '-n', default='https://skynet.ancap.tech')
|
||||
def clean(
|
||||
loglevel: str,
|
||||
account: str,
|
||||
permission: str,
|
||||
key: str | None,
|
||||
node_url: str,
|
||||
):
|
||||
import trio
|
||||
from leap.cleos import CLEOS
|
||||
key, account, permission = load_account_info(
|
||||
'user', key, account, permission)
|
||||
|
||||
config = load_skynet_toml()
|
||||
key = load_key(config, 'skynet.user.key')
|
||||
account = load_key(config, 'skynet.user.account')
|
||||
permission = load_key(config, 'skynet.user.permission')
|
||||
node_url = load_key(config, 'skynet.user.node_url')
|
||||
node_url, _, _ = load_endpoint_info(
|
||||
'user', node_url, None, None)
|
||||
|
||||
logging.basicConfig(level=loglevel)
|
||||
cleos = CLEOS(None, None, url=node_url, remote=node_url)
|
||||
|
|
@ -184,10 +192,11 @@ def clean(
|
|||
)
|
||||
|
||||
@skynet.command()
|
||||
def queue():
|
||||
import requests
|
||||
config = load_skynet_toml()
|
||||
node_url = load_key(config, 'skynet.user.node_url')
|
||||
@click.option(
|
||||
'--node-url', '-n', default='https://skynet.ancap.tech')
|
||||
def queue(node_url: str):
|
||||
node_url, _, _ = load_endpoint_info(
|
||||
'user', node_url, None, None)
|
||||
resp = requests.post(
|
||||
f'{node_url}/v1/chain/get_table_rows',
|
||||
json={
|
||||
|
|
@ -200,11 +209,12 @@ def queue():
|
|||
print(json.dumps(resp.json(), indent=4))
|
||||
|
||||
@skynet.command()
|
||||
@click.option(
|
||||
'--node-url', '-n', default='https://skynet.ancap.tech')
|
||||
@click.argument('request-id')
|
||||
def status(request_id: int):
|
||||
import requests
|
||||
config = load_skynet_toml()
|
||||
node_url = load_key(config, 'skynet.user.node_url')
|
||||
def status(node_url: str, request_id: int):
|
||||
node_url, _, _ = load_endpoint_info(
|
||||
'user', node_url, None, None)
|
||||
resp = requests.post(
|
||||
f'{node_url}/v1/chain/get_table_rows',
|
||||
json={
|
||||
|
|
@ -217,86 +227,106 @@ def status(request_id: int):
|
|||
print(json.dumps(resp.json(), indent=4))
|
||||
|
||||
@skynet.command()
|
||||
@click.option(
|
||||
'--account', '-a', default='telegram')
|
||||
@click.option(
|
||||
'--permission', '-p', default='active')
|
||||
@click.option(
|
||||
'--key', '-k', default=None)
|
||||
@click.option(
|
||||
'--node-url', '-n', default='https://skynet.ancap.tech')
|
||||
@click.argument('request-id')
|
||||
def dequeue(request_id: int):
|
||||
import trio
|
||||
from leap.cleos import CLEOS
|
||||
def dequeue(
|
||||
account: str,
|
||||
permission: str,
|
||||
key: str | None,
|
||||
node_url: str,
|
||||
request_id: int
|
||||
):
|
||||
key, account, permission = load_account_info(
|
||||
'user', key, account, permission)
|
||||
|
||||
config = load_skynet_toml()
|
||||
key = load_key(config, 'skynet.user.key')
|
||||
account = load_key(config, 'skynet.user.account')
|
||||
permission = load_key(config, 'skynet.user.permission')
|
||||
node_url = load_key(config, 'skynet.user.node_url')
|
||||
node_url, _, _ = load_endpoint_info(
|
||||
'user', node_url, None, None)
|
||||
|
||||
cleos = CLEOS(None, None, url=node_url, remote=node_url)
|
||||
res = trio.run(
|
||||
partial(
|
||||
cleos.a_push_action,
|
||||
with open_cleos(node_url, key=key) as cleos:
|
||||
res = trio.run(cleos.a_push_action,
|
||||
'telos.gpu',
|
||||
'dequeue',
|
||||
{
|
||||
'user': Name(account),
|
||||
'request_id': int(request_id),
|
||||
},
|
||||
account, key, permission=permission
|
||||
)
|
||||
account, key, permission,
|
||||
)
|
||||
print(res)
|
||||
|
||||
|
||||
@skynet.command()
|
||||
@click.option(
|
||||
'--account', '-a', default='telos.gpu')
|
||||
@click.option(
|
||||
'--permission', '-p', default='active')
|
||||
@click.option(
|
||||
'--key', '-k', default=None)
|
||||
@click.option(
|
||||
'--node-url', '-n', default='https://skynet.ancap.tech')
|
||||
@click.option(
|
||||
'--token-contract', '-c', default='eosio.token')
|
||||
@click.option(
|
||||
'--token-symbol', '-S', default='4,GPU')
|
||||
def config(
|
||||
account: str,
|
||||
permission: str,
|
||||
key: str | None,
|
||||
node_url: str,
|
||||
token_contract: str,
|
||||
token_symbol: str
|
||||
):
|
||||
import trio
|
||||
from leap.cleos import CLEOS
|
||||
key, account, permission = load_account_info(
|
||||
'user', key, account, permission)
|
||||
|
||||
config = load_skynet_toml()
|
||||
|
||||
key = load_key(config, 'skynet.user.key')
|
||||
account = load_key(config, 'skynet.user.account')
|
||||
permission = load_key(config, 'skynet.user.permission')
|
||||
node_url = load_key(config, 'skynet.user.node_url')
|
||||
|
||||
cleos = CLEOS(None, None, url=node_url, remote=node_url)
|
||||
res = trio.run(
|
||||
partial(
|
||||
cleos.a_push_action,
|
||||
node_url, _, _ = load_endpoint_info(
|
||||
'user', node_url, None, None)
|
||||
with open_cleos(node_url, key=key) as cleos:
|
||||
res = trio.run(cleos.a_push_action,
|
||||
'telos.gpu',
|
||||
'config',
|
||||
{
|
||||
'token_contract': token_contract,
|
||||
'token_symbol': token_symbol,
|
||||
},
|
||||
account, key, permission=permission
|
||||
)
|
||||
account, key, permission,
|
||||
)
|
||||
print(res)
|
||||
|
||||
|
||||
@skynet.command()
|
||||
@click.option(
|
||||
'--account', '-a', default='telegram')
|
||||
@click.option(
|
||||
'--permission', '-p', default='active')
|
||||
@click.option(
|
||||
'--key', '-k', default=None)
|
||||
@click.option(
|
||||
'--node-url', '-n', default='https://skynet.ancap.tech')
|
||||
@click.argument('quantity')
|
||||
def deposit(quantity: str):
|
||||
import trio
|
||||
from leap.cleos import CLEOS
|
||||
def deposit(
|
||||
account: str,
|
||||
permission: str,
|
||||
key: str | None,
|
||||
node_url: str,
|
||||
quantity: str
|
||||
):
|
||||
key, account, permission = load_account_info(
|
||||
'user', key, account, permission)
|
||||
|
||||
config = load_skynet_toml()
|
||||
node_url, _, _ = load_endpoint_info(
|
||||
'user', node_url, None, None)
|
||||
|
||||
key = load_key(config, 'skynet.user.key')
|
||||
account = load_key(config, 'skynet.user.account')
|
||||
permission = load_key(config, 'skynet.user.permission')
|
||||
node_url = load_key(config, 'skynet.user.node_url')
|
||||
cleos = CLEOS(None, None, url=node_url, remote=node_url)
|
||||
|
||||
res = trio.run(
|
||||
partial(
|
||||
cleos.a_push_action,
|
||||
'telos.gpu',
|
||||
with open_cleos(node_url, key=key) as cleos:
|
||||
res = trio.run(cleos.a_push_action,
|
||||
'eosio.token',
|
||||
'transfer',
|
||||
{
|
||||
'sender': Name(account),
|
||||
|
|
@ -304,8 +334,7 @@ def deposit(quantity: str):
|
|||
'amount': asset_from_str(quantity),
|
||||
'memo': f'{account} transferred {quantity} to telos.gpu'
|
||||
},
|
||||
account, key, permission=permission
|
||||
)
|
||||
account, key, permission,
|
||||
)
|
||||
print(res)
|
||||
|
||||
|
|
@ -316,8 +345,6 @@ def run(*args, **kwargs):
|
|||
|
||||
@run.command()
|
||||
def db():
|
||||
from .db import open_new_database
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
with open_new_database(cleanup=False) as db_params:
|
||||
container, passwd, host = db_params
|
||||
|
|
@ -325,8 +352,6 @@ def db():
|
|||
|
||||
@run.command()
|
||||
def nodeos():
|
||||
from .nodeos import open_nodeos
|
||||
|
||||
logging.basicConfig(filename='skynet-nodeos.log', level=logging.INFO)
|
||||
with open_nodeos(cleanup=False):
|
||||
...
|
||||
|
|
@ -334,29 +359,36 @@ def nodeos():
|
|||
@run.command()
|
||||
@click.option('--loglevel', '-l', default='INFO', help='Logging level')
|
||||
@click.option(
|
||||
'--config-path', '-c', default=DEFAULT_CONFIG_PATH)
|
||||
'--config-path', '-c', default='skynet.ini')
|
||||
def dgpu(
|
||||
loglevel: str,
|
||||
config_path: str
|
||||
):
|
||||
import trio
|
||||
from .dgpu import open_dgpu_node
|
||||
|
||||
logging.basicConfig(level=loglevel)
|
||||
|
||||
config = load_skynet_toml(file_path=config_path)
|
||||
hf_token = load_key(config, 'skynet.dgpu.hf_token')
|
||||
hf_home = load_key(config, 'skynet.dgpu.hf_home')
|
||||
set_hf_vars(hf_token, hf_home)
|
||||
config = load_skynet_ini(file_path=config_path)
|
||||
|
||||
assert 'skynet' in config
|
||||
assert 'dgpu' in config['skynet']
|
||||
assert 'skynet.dgpu' in config
|
||||
|
||||
trio.run(open_dgpu_node, config['skynet']['dgpu'])
|
||||
trio.run(open_dgpu_node, config['skynet.dgpu'])
|
||||
|
||||
|
||||
@run.command()
|
||||
@click.option('--loglevel', '-l', default='INFO', help='logging level')
|
||||
@click.option(
|
||||
'--account', '-a', default='telegram')
|
||||
@click.option(
|
||||
'--permission', '-p', default='active')
|
||||
@click.option(
|
||||
'--key', '-k', default=None)
|
||||
@click.option(
|
||||
'--hyperion-url', '-y', default=f'https://{DEFAULT_DOMAIN}')
|
||||
@click.option(
|
||||
'--node-url', '-n', default=f'https://{DEFAULT_DOMAIN}')
|
||||
@click.option(
|
||||
'--ipfs-url', '-i', default=DEFAULT_IPFS_REMOTE)
|
||||
@click.option(
|
||||
'--db-host', '-h', default='localhost:5432')
|
||||
@click.option(
|
||||
|
|
@ -365,43 +397,25 @@ def dgpu(
|
|||
'--db-pass', '-u', default='password')
|
||||
def telegram(
|
||||
loglevel: str,
|
||||
account: str,
|
||||
permission: str,
|
||||
key: str | None,
|
||||
hyperion_url: str,
|
||||
ipfs_url: str,
|
||||
node_url: str,
|
||||
db_host: str,
|
||||
db_user: str,
|
||||
db_pass: str
|
||||
):
|
||||
import asyncio
|
||||
from .frontend.telegram import SkynetTelegramFrontend
|
||||
|
||||
logging.basicConfig(level=loglevel)
|
||||
|
||||
config = load_skynet_toml()
|
||||
tg_token = load_key(config, 'skynet.telegram.tg_token')
|
||||
_, _, tg_token, _ = init_env_from_config()
|
||||
|
||||
key = load_key(config, 'skynet.telegram.key')
|
||||
account = load_key(config, 'skynet.telegram.account')
|
||||
permission = load_key(config, 'skynet.telegram.permission')
|
||||
node_url = load_key(config, 'skynet.telegram.node_url')
|
||||
hyperion_url = load_key(config, 'skynet.telegram.hyperion_url')
|
||||
key, account, permission = load_account_info(
|
||||
'telegram', key, account, permission)
|
||||
|
||||
try:
|
||||
ipfs_gateway_url = load_key(config, 'skynet.telegram.ipfs_gateway_url')
|
||||
|
||||
except ConfigParsingError:
|
||||
ipfs_gateway_url = None
|
||||
|
||||
ipfs_url = load_key(config, 'skynet.telegram.ipfs_url')
|
||||
|
||||
try:
|
||||
explorer_domain = load_key(config, 'skynet.telegram.explorer_domain')
|
||||
|
||||
except ConfigParsingError:
|
||||
explorer_domain = DEFAULT_EXPLORER_DOMAIN
|
||||
|
||||
try:
|
||||
ipfs_domain = load_key(config, 'skynet.telegram.ipfs_domain')
|
||||
|
||||
except ConfigParsingError:
|
||||
ipfs_domain = DEFAULT_IPFS_DOMAIN
|
||||
node_url, _, ipfs_url = load_endpoint_info(
|
||||
'telegram', node_url, None, None)
|
||||
|
||||
async def _async_main():
|
||||
frontend = SkynetTelegramFrontend(
|
||||
|
|
@ -411,11 +425,8 @@ def telegram(
|
|||
node_url,
|
||||
hyperion_url,
|
||||
db_host, db_user, db_pass,
|
||||
ipfs_url,
|
||||
remote_ipfs_node=ipfs_gateway_url,
|
||||
key=key,
|
||||
explorer_domain=explorer_domain,
|
||||
ipfs_domain=ipfs_domain
|
||||
remote_ipfs_node=ipfs_url,
|
||||
key=key
|
||||
)
|
||||
|
||||
async with frontend.open():
|
||||
|
|
@ -427,6 +438,18 @@ def telegram(
|
|||
|
||||
@run.command()
|
||||
@click.option('--loglevel', '-l', default='INFO', help='logging level')
|
||||
@click.option(
|
||||
'--account', '-a', default='discord')
|
||||
@click.option(
|
||||
'--permission', '-p', default='active')
|
||||
@click.option(
|
||||
'--key', '-k', default=None)
|
||||
@click.option(
|
||||
'--hyperion-url', '-y', default=f'https://{DEFAULT_DOMAIN}')
|
||||
@click.option(
|
||||
'--node-url', '-n', default=f'https://{DEFAULT_DOMAIN}')
|
||||
@click.option(
|
||||
'--ipfs-url', '-i', default=DEFAULT_IPFS_REMOTE)
|
||||
@click.option(
|
||||
'--db-host', '-h', default='localhost:5432')
|
||||
@click.option(
|
||||
|
|
@ -435,38 +458,25 @@ def telegram(
|
|||
'--db-pass', '-u', default='password')
|
||||
def discord(
|
||||
loglevel: str,
|
||||
account: str,
|
||||
permission: str,
|
||||
key: str | None,
|
||||
hyperion_url: str,
|
||||
ipfs_url: str,
|
||||
node_url: str,
|
||||
db_host: str,
|
||||
db_user: str,
|
||||
db_pass: str
|
||||
):
|
||||
import asyncio
|
||||
from .frontend.discord import SkynetDiscordFrontend
|
||||
|
||||
logging.basicConfig(level=loglevel)
|
||||
|
||||
config = load_skynet_toml()
|
||||
dc_token = load_key(config, 'skynet.discord.dc_token')
|
||||
_, _, _, dc_token = init_env_from_config()
|
||||
|
||||
key = load_key(config, 'skynet.discord.key')
|
||||
account = load_key(config, 'skynet.discord.account')
|
||||
permission = load_key(config, 'skynet.discord.permission')
|
||||
node_url = load_key(config, 'skynet.discord.node_url')
|
||||
hyperion_url = load_key(config, 'skynet.discord.hyperion_url')
|
||||
key, account, permission = load_account_info(
|
||||
'discord', key, account, permission)
|
||||
|
||||
ipfs_gateway_url = load_key(config, 'skynet.discord.ipfs_gateway_url')
|
||||
ipfs_url = load_key(config, 'skynet.discord.ipfs_url')
|
||||
|
||||
try:
|
||||
explorer_domain = load_key(config, 'skynet.discord.explorer_domain')
|
||||
|
||||
except ConfigParsingError:
|
||||
explorer_domain = DEFAULT_EXPLORER_DOMAIN
|
||||
|
||||
try:
|
||||
ipfs_domain = load_key(config, 'skynet.discord.ipfs_domain')
|
||||
|
||||
except ConfigParsingError:
|
||||
ipfs_domain = DEFAULT_IPFS_DOMAIN
|
||||
node_url, _, ipfs_url = load_endpoint_info(
|
||||
'discord', node_url, None, None)
|
||||
|
||||
async def _async_main():
|
||||
frontend = SkynetDiscordFrontend(
|
||||
|
|
@ -476,11 +486,8 @@ def discord(
|
|||
node_url,
|
||||
hyperion_url,
|
||||
db_host, db_user, db_pass,
|
||||
ipfs_url,
|
||||
remote_ipfs_node=ipfs_gateway_url,
|
||||
key=key,
|
||||
explorer_domain=explorer_domain,
|
||||
ipfs_domain=ipfs_domain
|
||||
remote_ipfs_node=ipfs_url,
|
||||
key=key
|
||||
)
|
||||
|
||||
async with frontend.open():
|
||||
|
|
@ -492,28 +499,24 @@ def discord(
|
|||
@run.command()
|
||||
@click.option('--loglevel', '-l', default='INFO', help='logging level')
|
||||
@click.option('--name', '-n', default='skynet-ipfs', help='container name')
|
||||
@click.option('--peer', '-p', default=(), help='connect to peer', multiple=True, type=str)
|
||||
def ipfs(loglevel, name, peer):
|
||||
def ipfs(loglevel, name):
|
||||
from skynet.ipfs.docker import open_ipfs_node
|
||||
|
||||
logging.basicConfig(level=loglevel)
|
||||
with open_ipfs_node(name=name, peers=peer):
|
||||
with open_ipfs_node(name=name):
|
||||
...
|
||||
|
||||
@run.command()
|
||||
@click.option('--loglevel', '-l', default='INFO', help='logging level')
|
||||
def pinner(loglevel):
|
||||
import trio
|
||||
from leap.hyperion import HyperionAPI
|
||||
from .ipfs import AsyncIPFSHTTP
|
||||
@click.option(
|
||||
'--ipfs-rpc', '-i', default='http://127.0.0.1:5001')
|
||||
@click.option(
|
||||
'--hyperion-url', '-y', default='http://127.0.0.1:42001')
|
||||
def pinner(loglevel, ipfs_rpc, hyperion_url):
|
||||
from .ipfs.pinner import SkynetPinner
|
||||
|
||||
config = load_skynet_toml()
|
||||
hyperion_url = load_key(config, 'skynet.pinner.hyperion_url')
|
||||
ipfs_url = load_key(config, 'skynet.pinner.ipfs_url')
|
||||
|
||||
logging.basicConfig(level=loglevel)
|
||||
ipfs_node = AsyncIPFSHTTP(ipfs_url)
|
||||
ipfs_node = IPFSHTTP(ipfs_rpc)
|
||||
hyperion = HyperionAPI(hyperion_url)
|
||||
|
||||
pinner = SkynetPinner(hyperion, ipfs_node)
|
||||
|
|
|
|||
116
skynet/config.py
116
skynet/config.py
|
|
@ -1,33 +1,113 @@
|
|||
#!/usr/bin/python
|
||||
|
||||
import os
|
||||
import toml
|
||||
import json
|
||||
|
||||
from pathlib import Path
|
||||
from configparser import ConfigParser
|
||||
from re import sub
|
||||
|
||||
from .constants import DEFAULT_CONFIG_PATH
|
||||
|
||||
|
||||
class ConfigParsingError(BaseException):
|
||||
...
|
||||
|
||||
|
||||
def load_skynet_toml(file_path=DEFAULT_CONFIG_PATH) -> dict:
|
||||
config = toml.load(file_path)
|
||||
return config
|
||||
|
||||
|
||||
def load_key(config: dict, key: str) -> str:
|
||||
for skey in key.split('.'):
|
||||
if skey not in config:
|
||||
conf_keys = [k for k in config]
|
||||
raise ConfigParsingError(f'key \"{skey}\" not in {conf_keys}')
|
||||
|
||||
config = config[skey]
|
||||
def load_skynet_ini(
|
||||
file_path=DEFAULT_CONFIG_PATH
|
||||
):
|
||||
config = ConfigParser()
|
||||
config.read(file_path)
|
||||
|
||||
return config
|
||||
|
||||
|
||||
def set_hf_vars(hf_token: str, hf_home: str):
|
||||
def init_env_from_config(
|
||||
hf_token: str | None = None,
|
||||
hf_home: str | None = None,
|
||||
tg_token: str | None = None,
|
||||
dc_token: str | None = None,
|
||||
file_path=DEFAULT_CONFIG_PATH
|
||||
):
|
||||
config = load_skynet_ini(file_path=file_path)
|
||||
|
||||
if 'HF_TOKEN' in os.environ:
|
||||
hf_token = os.environ['HF_TOKEN']
|
||||
|
||||
elif 'skynet.dgpu' in config:
|
||||
sub_config = config['skynet.dgpu']
|
||||
if 'hf_token' in sub_config:
|
||||
hf_token = sub_config['hf_token']
|
||||
os.environ['HF_TOKEN'] = hf_token
|
||||
|
||||
if 'HF_HOME' in os.environ:
|
||||
hf_home = os.environ['HF_HOME']
|
||||
|
||||
elif 'skynet.dgpu' in config:
|
||||
sub_config = config['skynet.dgpu']
|
||||
if 'hf_home' in sub_config:
|
||||
hf_home = sub_config['hf_home']
|
||||
os.environ['HF_HOME'] = hf_home
|
||||
|
||||
if 'TG_TOKEN' in os.environ:
|
||||
tg_token = os.environ['TG_TOKEN']
|
||||
elif 'skynet.telegram' in config:
|
||||
sub_config = config['skynet.telegram']
|
||||
if 'token' in sub_config:
|
||||
tg_token = sub_config['token']
|
||||
|
||||
if 'DC_TOKEN' in os.environ:
|
||||
dc_token = os.environ['DC_TOKEN']
|
||||
elif 'skynet.discord' in config:
|
||||
sub_config = config['skynet.discord']
|
||||
if 'token' in sub_config:
|
||||
dc_token = sub_config['token']
|
||||
|
||||
return hf_home, hf_token, tg_token, dc_token
|
||||
|
||||
|
||||
def load_account_info(
|
||||
_type: str,
|
||||
key: str | None = None,
|
||||
account: str | None = None,
|
||||
permission: str | None = None,
|
||||
file_path=DEFAULT_CONFIG_PATH
|
||||
):
|
||||
config = load_skynet_ini(file_path=file_path)
|
||||
|
||||
type_key = f'skynet.{_type}'
|
||||
|
||||
if type_key in config:
|
||||
sub_config = config[type_key]
|
||||
if not key and 'key' in sub_config:
|
||||
key = sub_config['key']
|
||||
|
||||
if not account and 'account' in sub_config:
|
||||
account = sub_config['account']
|
||||
|
||||
if not permission and 'permission' in sub_config:
|
||||
permission = sub_config['permission']
|
||||
|
||||
return key, account, permission
|
||||
|
||||
|
||||
def load_endpoint_info(
|
||||
_type: str,
|
||||
node_url: str | None = None,
|
||||
hyperion_url: str | None = None,
|
||||
ipfs_url: str | None = None,
|
||||
file_path=DEFAULT_CONFIG_PATH
|
||||
):
|
||||
config = load_skynet_ini(file_path=file_path)
|
||||
|
||||
type_key = f'skynet.{_type}'
|
||||
|
||||
if type_key in config:
|
||||
sub_config = config[type_key]
|
||||
if not node_url and 'node_url' in sub_config:
|
||||
node_url = sub_config['node_url']
|
||||
|
||||
if not hyperion_url and 'hyperion_url' in sub_config:
|
||||
hyperion_url = sub_config['hyperion_url']
|
||||
|
||||
if not ipfs_url and 'ipfs_url' in sub_config:
|
||||
ipfs_url = sub_config['ipfs_url']
|
||||
|
||||
return node_url, hyperion_url, ipfs_url
|
||||
|
|
|
|||
|
|
@ -1,24 +1,21 @@
|
|||
#!/usr/bin/python
|
||||
|
||||
VERSION = '0.1a12'
|
||||
VERSION = '0.1a10'
|
||||
|
||||
DOCKER_RUNTIME_CUDA = 'skynet:runtime-cuda'
|
||||
|
||||
MODELS = {
|
||||
'prompthero/openjourney': {'short': 'midj', 'mem': 6},
|
||||
'runwayml/stable-diffusion-v1-5': {'short': 'stable', 'mem': 6},
|
||||
'stabilityai/stable-diffusion-2-1-base': {'short': 'stable2', 'mem': 6},
|
||||
'snowkidy/stable-diffusion-xl-base-0.9': {'short': 'stablexl0.9', 'mem': 8.3},
|
||||
'Linaqruf/anything-v3.0': {'short': 'hdanime', 'mem': 6},
|
||||
'hakurei/waifu-diffusion': {'short': 'waifu', 'mem': 6},
|
||||
'nitrosocke/Ghibli-Diffusion': {'short': 'ghibli', 'mem': 6},
|
||||
'dallinmackay/Van-Gogh-diffusion': {'short': 'van-gogh', 'mem': 6},
|
||||
'lambdalabs/sd-pokemon-diffusers': {'short': 'pokemon', 'mem': 6},
|
||||
'Envvi/Inkpunk-Diffusion': {'short': 'ink', 'mem': 6},
|
||||
'nousr/robo-diffusion': {'short': 'robot', 'mem': 6},
|
||||
|
||||
# default is always last
|
||||
'stabilityai/stable-diffusion-xl-base-1.0': {'short': 'stablexl', 'mem': 8.3},
|
||||
'prompthero/openjourney': { 'short': 'midj'},
|
||||
'runwayml/stable-diffusion-v1-5': { 'short': 'stable'},
|
||||
'stabilityai/stable-diffusion-2-1-base': { 'short': 'stable2'},
|
||||
'snowkidy/stable-diffusion-xl-base-0.9': { 'short': 'stablexl'},
|
||||
'Linaqruf/anything-v3.0': { 'short': 'hdanime'},
|
||||
'hakurei/waifu-diffusion': { 'short': 'waifu'},
|
||||
'nitrosocke/Ghibli-Diffusion': { 'short': 'ghibli'},
|
||||
'dallinmackay/Van-Gogh-diffusion': { 'short': 'van-gogh'},
|
||||
'lambdalabs/sd-pokemon-diffusers': { 'short': 'pokemon'},
|
||||
'Envvi/Inkpunk-Diffusion': { 'short': 'ink'},
|
||||
'nousr/robo-diffusion': { 'short': 'robot'}
|
||||
}
|
||||
|
||||
SHORT_NAMES = [
|
||||
|
|
@ -39,7 +36,7 @@ commands work on a user per user basis!
|
|||
config is individual to each user!
|
||||
|
||||
/txt2img TEXT - request an image based on a prompt
|
||||
/img2img <attach_image> TEXT - request an image base on an image and a prompt
|
||||
/img2img <attach_image> TEXT - request an image base on an image and a promtp
|
||||
|
||||
/redo - redo last command (only works for txt2img for now!)
|
||||
|
||||
|
|
@ -56,17 +53,14 @@ config is individual to each user!
|
|||
{N.join(SHORT_NAMES)}
|
||||
|
||||
/config step NUMBER - set amount of iterations
|
||||
/config seed [auto|NUMBER] - set the seed, deterministic results!
|
||||
/config width NUMBER - set horizontal size in pixels
|
||||
/config height NUMBER - set vertical size in pixels
|
||||
/config upscaler [off/x4] - enable/disable x4 size upscaler
|
||||
/config seed NUMBER - set the seed, deterministic results!
|
||||
/config size WIDTH HEIGHT - set size in pixels
|
||||
/config guidance NUMBER - prompt text importance
|
||||
/config strength NUMBER - importance of the input image for img2img
|
||||
'''
|
||||
|
||||
UNKNOWN_CMD_TEXT = 'Unknown command! Try sending \"/help\"'
|
||||
|
||||
DONATION_INFO = '0xf95335682DF281FFaB7E104EB87B69625d9622B6\ngoal: 0.0465/1.0000 ETH'
|
||||
DONATION_INFO = '0xf95335682DF281FFaB7E104EB87B69625d9622B6\ngoal: 25/650usd'
|
||||
|
||||
COOL_WORDS = [
|
||||
'cyberpunk',
|
||||
|
|
@ -125,19 +119,10 @@ ing more may produce a slightly different picture, but not necessarily better \
|
|||
quality.
|
||||
''',
|
||||
|
||||
'guidance': '''
|
||||
'guidance': '''
|
||||
The guidance scale is a parameter that controls how much the image generation\
|
||||
process follows the text prompt. The higher the value, the more image sticks\
|
||||
to a given text input. Value range 0 to 20. Recomended range: 4.5-7.5.
|
||||
''',
|
||||
|
||||
'strength': '''
|
||||
Noise is added to the image you use as an init image for img2img, and then the\
|
||||
diffusion process continues according to the prompt. The amount of noise added\
|
||||
depends on the \"Strength of img2img\"” parameter, which ranges from 0 to 1,\
|
||||
where 0 adds no noise at all and you will get the exact image you added, and\
|
||||
1 completely replaces the image with noise and almost acts as if you used\
|
||||
normal txt2img instead of img2img.
|
||||
to a given text input.
|
||||
'''
|
||||
}
|
||||
|
||||
|
|
@ -154,20 +139,21 @@ MAX_HEIGHT = 1024
|
|||
MAX_GUIDANCE = 20
|
||||
|
||||
DEFAULT_SEED = None
|
||||
DEFAULT_WIDTH = 1024
|
||||
DEFAULT_HEIGHT = 1024
|
||||
DEFAULT_WIDTH = 512
|
||||
DEFAULT_HEIGHT = 512
|
||||
DEFAULT_GUIDANCE = 7.5
|
||||
DEFAULT_STRENGTH = 0.5
|
||||
DEFAULT_STEP = 28
|
||||
DEFAULT_STEP = 35
|
||||
DEFAULT_CREDITS = 10
|
||||
DEFAULT_MODEL = list(MODELS.keys())[-1]
|
||||
DEFAULT_MODEL = list(MODELS.keys())[0]
|
||||
DEFAULT_ROLE = 'pleb'
|
||||
DEFAULT_UPSCALER = None
|
||||
|
||||
DEFAULT_CONFIG_PATH = 'skynet.toml'
|
||||
DEFAULT_CONFIG_PATH = 'skynet.ini'
|
||||
|
||||
DEFAULT_INITAL_MODELS = [
|
||||
'stabilityai/stable-diffusion-xl-base-1.0'
|
||||
'prompthero/openjourney',
|
||||
'runwayml/stable-diffusion-v1-5'
|
||||
]
|
||||
|
||||
DATE_FORMAT = '%B the %dth %Y, %H:%M:%S'
|
||||
|
|
@ -183,13 +169,6 @@ CONFIG_ATTRS = [
|
|||
'upscaler'
|
||||
]
|
||||
|
||||
DEFAULT_EXPLORER_DOMAIN = 'explorer.skygpu.net'
|
||||
DEFAULT_IPFS_DOMAIN = 'ipfs.skygpu.net'
|
||||
DEFAULT_DOMAIN = 'skygpu.net'
|
||||
|
||||
DEFAULT_IPFS_REMOTE = '/ip4/169.197.140.154/tcp/4001/p2p/12D3KooWKWogLFNEcNNMKnzU7Snrnuj84RZdMBg3sLiQSQc51oEv'
|
||||
DEFAULT_IPFS_LOCAL = 'http://127.0.0.1:5001'
|
||||
|
||||
TG_MAX_WIDTH = 1280
|
||||
TG_MAX_HEIGHT = 1280
|
||||
|
||||
DEFAULT_SINGLE_CARD_MAP = 'cuda:0'
|
||||
|
|
|
|||
|
|
@ -43,7 +43,6 @@ CREATE TABLE IF NOT EXISTS skynet.user_config(
|
|||
guidance DECIMAL NOT NULL,
|
||||
strength DECIMAL NOT NULL,
|
||||
upscaler VARCHAR(128),
|
||||
autoconf BOOLEAN DEFAULT TRUE,
|
||||
CONSTRAINT fk_config
|
||||
FOREIGN KEY(id)
|
||||
REFERENCES skynet.user(id)
|
||||
|
|
@ -166,15 +165,6 @@ async def open_database_connection(
|
|||
else:
|
||||
await conn.execute(DB_INIT_SQL)
|
||||
|
||||
col_check = await conn.fetch(f'''
|
||||
select column_name
|
||||
from information_schema.columns
|
||||
where table_name = 'user_config' and column_name = 'autoconf';
|
||||
''')
|
||||
|
||||
if not col_check:
|
||||
await conn.execute('alter table skynet.user_config add column autoconf boolean default true;')
|
||||
|
||||
async def _db_call(method: str, *args, **kwargs):
|
||||
method = getattr(db, method)
|
||||
|
||||
|
|
|
|||
|
|
@ -2,9 +2,6 @@
|
|||
|
||||
import trio
|
||||
|
||||
from hypercorn.config import Config
|
||||
from hypercorn.trio import serve
|
||||
|
||||
from skynet.dgpu.compute import SkynetMM
|
||||
from skynet.dgpu.daemon import SkynetDGPUDaemon
|
||||
from skynet.dgpu.network import SkynetGPUConnector
|
||||
|
|
@ -13,18 +10,7 @@ from skynet.dgpu.network import SkynetGPUConnector
|
|||
async def open_dgpu_node(config: dict):
|
||||
conn = SkynetGPUConnector(config)
|
||||
mm = SkynetMM(config)
|
||||
daemon = SkynetDGPUDaemon(mm, conn, config)
|
||||
|
||||
api = None
|
||||
if 'api_bind' in config:
|
||||
api_conf = Config()
|
||||
api_conf.bind = [config['api_bind']]
|
||||
api = await daemon.generate_api()
|
||||
|
||||
async with trio.open_nursery() as n:
|
||||
n.start_soon(daemon.snap_updater_task)
|
||||
|
||||
if api:
|
||||
n.start_soon(serve, api, api_conf)
|
||||
|
||||
await daemon.serve_forever()
|
||||
async with conn.open() as conn:
|
||||
await (SkynetDGPUDaemon(mm, conn, config)
|
||||
.serve_forever())
|
||||
|
|
|
|||
|
|
@ -3,43 +3,31 @@
|
|||
# Skynet Memory Manager
|
||||
|
||||
import gc
|
||||
import logging
|
||||
|
||||
from hashlib import sha256
|
||||
import zipfile
|
||||
from PIL import Image
|
||||
import json
|
||||
import logging
|
||||
from diffusers import DiffusionPipeline
|
||||
|
||||
import trio
|
||||
import torch
|
||||
|
||||
from skynet.constants import DEFAULT_INITAL_MODELS, MODELS
|
||||
from skynet.dgpu.errors import DGPUComputeError, DGPUInferenceCancelled
|
||||
from skynet.dgpu.errors import DGPUComputeError
|
||||
|
||||
from skynet.utils import crop_image, convert_from_cv2_to_image, convert_from_image_to_cv2, convert_from_img_to_bytes, init_upscaler, pipeline_for
|
||||
from skynet.utils import convert_from_bytes_and_crop, convert_from_cv2_to_image, convert_from_image_to_cv2, convert_from_img_to_bytes, init_upscaler, pipeline_for
|
||||
|
||||
|
||||
def prepare_params_for_diffuse(
|
||||
params: dict,
|
||||
input_type: str,
|
||||
binary = None
|
||||
binary: bytes | None = None
|
||||
):
|
||||
_params = {}
|
||||
if binary != None:
|
||||
match input_type:
|
||||
case 'png':
|
||||
image = crop_image(
|
||||
binary, params['width'], params['height'])
|
||||
image = None
|
||||
if binary:
|
||||
image = convert_from_bytes_and_crop(binary, 512, 512)
|
||||
|
||||
_params = {}
|
||||
if image:
|
||||
_params['image'] = image
|
||||
_params['strength'] = float(params['strength'])
|
||||
|
||||
case 'none':
|
||||
...
|
||||
|
||||
case _:
|
||||
raise DGPUComputeError(f'Unknown input_type {input_type}')
|
||||
|
||||
else:
|
||||
_params['width'] = int(params['width'])
|
||||
_params['height'] = int(params['height'])
|
||||
|
|
@ -63,10 +51,6 @@ class SkynetMM:
|
|||
if 'initial_models' in config else DEFAULT_INITAL_MODELS
|
||||
)
|
||||
|
||||
self.cache_dir = None
|
||||
if 'hf_home' in config:
|
||||
self.cache_dir = config['hf_home']
|
||||
|
||||
self._models = {}
|
||||
for model in self.initial_models:
|
||||
self.load_model(model, False, force=True)
|
||||
|
|
@ -91,9 +75,7 @@ class SkynetMM:
|
|||
):
|
||||
logging.info(f'loading model {model_name}...')
|
||||
if force or len(self._models.keys()) == 0:
|
||||
pipe = pipeline_for(
|
||||
model_name, image=image, cache_dir=self.cache_dir)
|
||||
|
||||
pipe = pipeline_for(model_name, image=image)
|
||||
self._models[model_name] = {
|
||||
'pipe': pipe,
|
||||
'generated': 0,
|
||||
|
|
@ -115,8 +97,7 @@ class SkynetMM:
|
|||
gc.collect()
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
pipe = pipeline_for(
|
||||
model_name, image=image, cache_dir=self.cache_dir)
|
||||
pipe = pipeline_for(model_name, image=image)
|
||||
|
||||
self._models[model_name] = {
|
||||
'pipe': pipe,
|
||||
|
|
@ -141,61 +122,38 @@ class SkynetMM:
|
|||
|
||||
def compute_one(
|
||||
self,
|
||||
request_id: int,
|
||||
method: str,
|
||||
params: dict,
|
||||
input_type: str = 'png',
|
||||
binary: bytes | None = None
|
||||
):
|
||||
def maybe_cancel_work(step, *args, **kwargs):
|
||||
if self._should_cancel:
|
||||
should_raise = trio.from_thread.run(self._should_cancel, request_id)
|
||||
if should_raise:
|
||||
logging.warn(f'cancelling work at step {step}')
|
||||
raise DGPUInferenceCancelled()
|
||||
|
||||
maybe_cancel_work(0)
|
||||
|
||||
output_type = 'png'
|
||||
if 'output_type' in params:
|
||||
output_type = params['output_type']
|
||||
|
||||
output = None
|
||||
output_hash = None
|
||||
try:
|
||||
match method:
|
||||
case 'diffuse':
|
||||
arguments = prepare_params_for_diffuse(
|
||||
params, input_type, binary=binary)
|
||||
image = None
|
||||
|
||||
arguments = prepare_params_for_diffuse(params, binary)
|
||||
prompt, guidance, step, seed, upscaler, extra_params = arguments
|
||||
model = self.get_model(params['model'], 'image' in extra_params)
|
||||
|
||||
output = model(
|
||||
image = model(
|
||||
prompt,
|
||||
guidance_scale=guidance,
|
||||
num_inference_steps=step,
|
||||
generator=seed,
|
||||
callback=maybe_cancel_work,
|
||||
callback_steps=1,
|
||||
**extra_params
|
||||
).images[0]
|
||||
|
||||
output_binary = b''
|
||||
match output_type:
|
||||
case 'png':
|
||||
if upscaler == 'x4':
|
||||
input_img = output.convert('RGB')
|
||||
input_img = image.convert('RGB')
|
||||
up_img, _ = self.upscaler.enhance(
|
||||
convert_from_image_to_cv2(input_img), outscale=4)
|
||||
|
||||
output = convert_from_cv2_to_image(up_img)
|
||||
image = convert_from_cv2_to_image(up_img)
|
||||
|
||||
output_binary = convert_from_img_to_bytes(output)
|
||||
img_raw = convert_from_img_to_bytes(image)
|
||||
img_sha = sha256(img_raw).hexdigest()
|
||||
|
||||
case _:
|
||||
raise DGPUComputeError(f'Unsupported output type: {output_type}')
|
||||
|
||||
output_hash = sha256(output_binary).hexdigest()
|
||||
return img_sha, img_raw
|
||||
|
||||
case _:
|
||||
raise DGPUComputeError('Unsupported compute method')
|
||||
|
|
@ -206,5 +164,3 @@ class SkynetMM:
|
|||
|
||||
finally:
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
return output_hash, output
|
||||
|
|
|
|||
|
|
@ -1,35 +1,17 @@
|
|||
#!/usr/bin/python
|
||||
|
||||
import json
|
||||
import random
|
||||
import logging
|
||||
import time
|
||||
import traceback
|
||||
|
||||
from hashlib import sha256
|
||||
from datetime import datetime
|
||||
from functools import partial
|
||||
|
||||
import trio
|
||||
|
||||
from quart import jsonify
|
||||
from quart_trio import QuartTrio as Quart
|
||||
|
||||
from skynet.constants import MODELS, VERSION
|
||||
|
||||
from skynet.dgpu.errors import *
|
||||
from skynet.dgpu.compute import SkynetMM
|
||||
from skynet.dgpu.network import SkynetGPUConnector
|
||||
|
||||
|
||||
def convert_reward_to_int(reward_str):
|
||||
int_part, decimal_part = (
|
||||
reward_str.split('.')[0],
|
||||
reward_str.split('.')[1].split(' ')[0]
|
||||
)
|
||||
return int(int_part + decimal_part)
|
||||
|
||||
|
||||
class SkynetDGPUDaemon:
|
||||
|
||||
def __init__(
|
||||
|
|
@ -45,120 +27,27 @@ class SkynetDGPUDaemon:
|
|||
if 'auto_withdraw' in config else False
|
||||
)
|
||||
|
||||
self.account = config['account']
|
||||
|
||||
self.non_compete = set()
|
||||
if 'non_compete' in config:
|
||||
self.non_compete = set(config['non_compete'])
|
||||
|
||||
self.model_whitelist = set()
|
||||
if 'model_whitelist' in config:
|
||||
self.model_whitelist = set(config['model_whitelist'])
|
||||
|
||||
self.model_blacklist = set()
|
||||
if 'model_blacklist' in config:
|
||||
self.model_blacklist = set(config['model_blacklist'])
|
||||
|
||||
self.backend = 'sync-on-thread'
|
||||
if 'backend' in config:
|
||||
self.backend = config['backend']
|
||||
|
||||
self._snap = {
|
||||
'queue': [],
|
||||
'requests': {},
|
||||
'my_results': []
|
||||
}
|
||||
|
||||
self._benchmark = []
|
||||
self._last_benchmark = None
|
||||
self._last_generation_ts = None
|
||||
|
||||
def _get_benchmark_speed(self) -> float:
|
||||
if not self._last_benchmark:
|
||||
return 0
|
||||
|
||||
start = self._last_benchmark[0]
|
||||
end = self._last_benchmark[-1]
|
||||
|
||||
elapsed = end - start
|
||||
its = len(self._last_benchmark)
|
||||
speed = its / elapsed
|
||||
|
||||
logging.info(f'{elapsed} s total its: {its}, at {speed} it/s ')
|
||||
|
||||
return speed
|
||||
|
||||
async def should_cancel_work(self, request_id: int):
|
||||
self._benchmark.append(time.time())
|
||||
competitors = set([
|
||||
status['worker']
|
||||
for status in self._snap['requests'][request_id]
|
||||
if status['worker'] != self.account
|
||||
])
|
||||
return bool(self.non_compete & competitors)
|
||||
|
||||
|
||||
async def snap_updater_task(self):
|
||||
while True:
|
||||
self._snap = await self.conn.get_full_queue_snapshot()
|
||||
await trio.sleep(1)
|
||||
|
||||
async def generate_api(self):
|
||||
app = Quart(__name__)
|
||||
|
||||
@app.route('/')
|
||||
async def health():
|
||||
return jsonify(
|
||||
account=self.account,
|
||||
version=VERSION,
|
||||
last_generation_ts=self._last_generation_ts,
|
||||
last_generation_speed=self._get_benchmark_speed()
|
||||
)
|
||||
|
||||
return app
|
||||
|
||||
async def serve_forever(self):
|
||||
try:
|
||||
while True:
|
||||
if self.auto_withdraw:
|
||||
await self.conn.maybe_withdraw_all()
|
||||
|
||||
queue = self._snap['queue']
|
||||
|
||||
random.shuffle(queue)
|
||||
queue = sorted(
|
||||
queue,
|
||||
key=lambda req: convert_reward_to_int(req['reward']),
|
||||
reverse=True
|
||||
)
|
||||
queue = await self.conn.get_work_requests_last_hour()
|
||||
|
||||
for req in queue:
|
||||
rid = req['id']
|
||||
|
||||
# parse request
|
||||
body = json.loads(req['body'])
|
||||
model = body['params']['model']
|
||||
|
||||
# if model not known
|
||||
if model not in MODELS:
|
||||
logging.warning(f'Unknown model {model}')
|
||||
continue
|
||||
|
||||
# if whitelist enabled and model not in it continue
|
||||
if (len(self.model_whitelist) > 0 and
|
||||
not model in self.model_whitelist):
|
||||
continue
|
||||
|
||||
# if blacklist contains model skip
|
||||
if model in self.model_blacklist:
|
||||
continue
|
||||
|
||||
my_results = [res['id'] for res in self._snap['my_results']]
|
||||
if rid not in my_results and rid in self._snap['requests']:
|
||||
statuses = self._snap['requests'][rid]
|
||||
my_results = [res['id'] for res in (await self.conn.find_my_results())]
|
||||
if rid not in my_results:
|
||||
statuses = await self.conn.get_status_by_request_id(rid)
|
||||
|
||||
if len(statuses) == 0:
|
||||
binary, input_type = await self.conn.get_input_data(req['binary_data'])
|
||||
|
||||
# parse request
|
||||
body = json.loads(req['body'])
|
||||
|
||||
binary = await self.conn.get_input_data(req['binary_data'])
|
||||
|
||||
hash_str = (
|
||||
str(req['nonce'])
|
||||
|
|
@ -181,40 +70,17 @@ class SkynetDGPUDaemon:
|
|||
|
||||
else:
|
||||
try:
|
||||
output_type = 'png'
|
||||
if 'output_type' in body['params']:
|
||||
output_type = body['params']['output_type']
|
||||
img_sha, img_raw = self.mm.compute_one(
|
||||
body['method'], body['params'], binary=binary)
|
||||
|
||||
output = None
|
||||
output_hash = None
|
||||
match self.backend:
|
||||
case 'sync-on-thread':
|
||||
self.mm._should_cancel = self.should_cancel_work
|
||||
output_hash, output = await trio.to_thread.run_sync(
|
||||
partial(
|
||||
self.mm.compute_one,
|
||||
rid,
|
||||
body['method'], body['params'],
|
||||
input_type=input_type,
|
||||
binary=binary
|
||||
)
|
||||
)
|
||||
ipfs_hash = self.conn.publish_on_ipfs( img_raw)
|
||||
|
||||
case _:
|
||||
raise DGPUComputeError(f'Unsupported backend {self.backend}')
|
||||
self._last_generation_ts = datetime.now().isoformat()
|
||||
self._last_benchmark = self._benchmark
|
||||
self._benchmark = []
|
||||
|
||||
ipfs_hash = await self.conn.publish_on_ipfs(output, typ=output_type)
|
||||
|
||||
await self.conn.submit_work(rid, request_hash, output_hash, ipfs_hash)
|
||||
await self.conn.submit_work(rid, request_hash, img_sha, ipfs_hash)
|
||||
break
|
||||
|
||||
except BaseException as e:
|
||||
traceback.print_exc()
|
||||
await self.conn.cancel_work(rid, str(e))
|
||||
|
||||
finally:
|
||||
break
|
||||
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -3,6 +3,3 @@
|
|||
|
||||
class DGPUComputeError(BaseException):
|
||||
...
|
||||
|
||||
class DGPUInferenceCancelled(BaseException):
|
||||
...
|
||||
|
|
|
|||
|
|
@ -1,28 +1,24 @@
|
|||
#!/usr/bin/python
|
||||
|
||||
from functools import partial
|
||||
import io
|
||||
import json
|
||||
import time
|
||||
import logging
|
||||
|
||||
from pathlib import Path
|
||||
from functools import partial
|
||||
|
||||
import asks
|
||||
import trio
|
||||
import anyio
|
||||
from PIL import Image
|
||||
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
from contextlib import ExitStack
|
||||
from contextlib import asynccontextmanager as acm
|
||||
|
||||
from leap.cleos import CLEOS
|
||||
from leap.sugar import Checksum256, Name, asset_from_str
|
||||
from skynet.constants import DEFAULT_IPFS_DOMAIN
|
||||
from skynet.constants import DEFAULT_DOMAIN
|
||||
|
||||
from skynet.ipfs import AsyncIPFSHTTP, get_ipfs_file
|
||||
from skynet.dgpu.errors import DGPUComputeError
|
||||
|
||||
|
||||
REQUEST_UPDATE_TIME = 3
|
||||
from skynet.ipfs import get_ipfs_file
|
||||
from skynet.ipfs.docker import open_ipfs_node
|
||||
|
||||
|
||||
async def failable(fn: partial, ret_fail=None):
|
||||
|
|
@ -30,11 +26,8 @@ async def failable(fn: partial, ret_fail=None):
|
|||
return await fn()
|
||||
|
||||
except (
|
||||
OSError,
|
||||
json.JSONDecodeError,
|
||||
asks.errors.RequestTimeout,
|
||||
asks.errors.BadHttpResponse,
|
||||
anyio.BrokenResourceError
|
||||
json.JSONDecodeError
|
||||
):
|
||||
return ret_fail
|
||||
|
||||
|
|
@ -45,25 +38,28 @@ class SkynetGPUConnector:
|
|||
self.account = Name(config['account'])
|
||||
self.permission = config['permission']
|
||||
self.key = config['key']
|
||||
|
||||
self.node_url = config['node_url']
|
||||
self.hyperion_url = config['hyperion_url']
|
||||
self.ipfs_url = config['ipfs_url']
|
||||
|
||||
self.cleos = CLEOS(
|
||||
None, None, self.node_url, remote=self.node_url)
|
||||
|
||||
self.ipfs_gateway_url = None
|
||||
if 'ipfs_gateway_url' in config:
|
||||
self.ipfs_gateway_url = config['ipfs_gateway_url']
|
||||
self.ipfs_url = config['ipfs_url']
|
||||
self._exit_stack = ExitStack()
|
||||
|
||||
self.ipfs_client = AsyncIPFSHTTP(self.ipfs_url)
|
||||
def connect(self):
|
||||
self.ipfs_node = self._exit_stack.enter_context(
|
||||
open_ipfs_node())
|
||||
|
||||
self.ipfs_domain = DEFAULT_IPFS_DOMAIN
|
||||
if 'ipfs_domain' in config:
|
||||
self.ipfs_domain = config['ipfs_domain']
|
||||
def disconnect(self):
|
||||
self._exit_stack.close()
|
||||
|
||||
@acm
|
||||
async def open(self):
|
||||
self.connect()
|
||||
yield self
|
||||
self.disconnect()
|
||||
|
||||
self._wip_requests = {}
|
||||
|
||||
# blockchain helpers
|
||||
|
||||
|
|
@ -114,36 +110,6 @@ class SkynetGPUConnector:
|
|||
else:
|
||||
return None
|
||||
|
||||
async def get_competitors_for_req(self, request_id: int) -> set:
|
||||
competitors = [
|
||||
status['worker']
|
||||
for status in
|
||||
(await self.get_status_by_request_id(request_id))
|
||||
if status['worker'] != self.account
|
||||
]
|
||||
logging.info(f'competitors: {competitors}')
|
||||
return set(competitors)
|
||||
|
||||
|
||||
async def get_full_queue_snapshot(self):
|
||||
snap = {
|
||||
'requests': {},
|
||||
'my_results': []
|
||||
}
|
||||
|
||||
snap['queue'] = await self.get_work_requests_last_hour()
|
||||
|
||||
async def _run_and_save(d, key: str, fn, *args, **kwargs):
|
||||
d[key] = await fn(*args, **kwargs)
|
||||
|
||||
async with trio.open_nursery() as n:
|
||||
n.start_soon(_run_and_save, snap, 'my_results', self.find_my_results)
|
||||
for req in snap['queue']:
|
||||
n.start_soon(
|
||||
_run_and_save, snap['requests'], req['id'], self.get_status_by_request_id, req['id'])
|
||||
|
||||
return snap
|
||||
|
||||
async def begin_work(self, request_id: int):
|
||||
logging.info('begin_work')
|
||||
return await failable(
|
||||
|
|
@ -239,74 +205,29 @@ class SkynetGPUConnector:
|
|||
)
|
||||
|
||||
# IPFS helpers
|
||||
async def publish_on_ipfs(self, raw, typ: str = 'png'):
|
||||
Path('ipfs-staging').mkdir(exist_ok=True)
|
||||
|
||||
def publish_on_ipfs(self, raw_img: bytes):
|
||||
logging.info('publish_on_ipfs')
|
||||
img = Image.open(io.BytesIO(raw_img))
|
||||
img.save(f'ipfs-docker-staging/image.png')
|
||||
|
||||
target_file = ''
|
||||
match typ:
|
||||
case 'png':
|
||||
raw: Image
|
||||
target_file = 'ipfs-staging/image.png'
|
||||
raw.save(target_file)
|
||||
|
||||
case _:
|
||||
raise ValueError(f'Unsupported output type: {typ}')
|
||||
|
||||
if self.ipfs_gateway_url:
|
||||
# check peer connections, reconnect to skynet gateway if not
|
||||
gateway_id = Path(self.ipfs_gateway_url).name
|
||||
peers = await self.ipfs_client.peers()
|
||||
if gateway_id not in [p['Peer'] for p in peers]:
|
||||
await self.ipfs_client.connect(self.ipfs_gateway_url)
|
||||
peers = self.ipfs_node.check_connect()
|
||||
if self.ipfs_url not in peers:
|
||||
self.ipfs_node.connect(self.ipfs_url)
|
||||
|
||||
file_info = await self.ipfs_client.add(Path(target_file))
|
||||
file_cid = file_info['Hash']
|
||||
ipfs_hash = self.ipfs_node.add('image.png')
|
||||
|
||||
await self.ipfs_client.pin(file_cid)
|
||||
self.ipfs_node.pin(ipfs_hash)
|
||||
|
||||
return file_cid
|
||||
|
||||
async def get_input_data(self, ipfs_hash: str) -> tuple[bytes, str]:
|
||||
input_type = 'none'
|
||||
return ipfs_hash
|
||||
|
||||
async def get_input_data(self, ipfs_hash: str) -> bytes:
|
||||
if ipfs_hash == '':
|
||||
return b'', input_type
|
||||
return b''
|
||||
|
||||
results = {}
|
||||
ipfs_link = f'https://{self.ipfs_domain}/ipfs/{ipfs_hash}'
|
||||
ipfs_link_legacy = ipfs_link + '/image.png'
|
||||
|
||||
async with trio.open_nursery() as n:
|
||||
async def get_and_set_results(link: str):
|
||||
res = await get_ipfs_file(link, timeout=1)
|
||||
logging.info(f'got response from {link}')
|
||||
if not res or res.status_code != 200:
|
||||
logging.warning(f'couldn\'t get ipfs binary data at {link}!')
|
||||
|
||||
else:
|
||||
try:
|
||||
# attempt to decode as image
|
||||
results[link] = Image.open(io.BytesIO(res.raw))
|
||||
input_type = 'png'
|
||||
n.cancel_scope.cancel()
|
||||
|
||||
except UnidentifiedImageError:
|
||||
logging.warning(f'couldn\'t get ipfs binary data at {link}!')
|
||||
|
||||
n.start_soon(
|
||||
get_and_set_results, ipfs_link)
|
||||
n.start_soon(
|
||||
get_and_set_results, ipfs_link_legacy)
|
||||
|
||||
input_data = None
|
||||
if ipfs_link_legacy in results:
|
||||
input_data = results[ipfs_link_legacy]
|
||||
|
||||
if ipfs_link in results:
|
||||
input_data = results[ipfs_link]
|
||||
|
||||
if input_data == None:
|
||||
resp = await get_ipfs_file(f'https://ipfs.{DEFAULT_DOMAIN}/ipfs/{ipfs_hash}/image.png')
|
||||
if not resp:
|
||||
raise DGPUComputeError('Couldn\'t gather input data from ipfs')
|
||||
|
||||
return input_data, input_type
|
||||
return resp.raw
|
||||
|
|
|
|||
|
|
@ -1,7 +1,5 @@
|
|||
#!/usr/bin/python
|
||||
|
||||
import random
|
||||
|
||||
from ..constants import *
|
||||
|
||||
|
||||
|
|
@ -17,14 +15,10 @@ class ConfigUnknownAlgorithm(BaseException):
|
|||
class ConfigUnknownUpscaler(BaseException):
|
||||
...
|
||||
|
||||
class ConfigUnknownAutoConfSetting(BaseException):
|
||||
...
|
||||
|
||||
class ConfigSizeDivisionByEight(BaseException):
|
||||
...
|
||||
|
||||
|
||||
|
||||
def validate_user_config_request(req: str):
|
||||
params = req.split(' ')
|
||||
|
||||
|
|
@ -84,18 +78,6 @@ def validate_user_config_request(req: str):
|
|||
raise ConfigUnknownUpscaler(
|
||||
f'\"{val}\" is not a valid upscaler')
|
||||
|
||||
case 'autoconf':
|
||||
val = params[2]
|
||||
if val == 'on':
|
||||
val = True
|
||||
|
||||
elif val == 'off':
|
||||
val = False
|
||||
|
||||
else:
|
||||
raise ConfigUnknownAutoConfSetting(
|
||||
f'\"{val}\" not a valid setting for autoconf')
|
||||
|
||||
case _:
|
||||
raise ConfigUnknownAttribute(
|
||||
f'\"{attr}\" not a configurable parameter')
|
||||
|
|
@ -110,22 +92,3 @@ def validate_user_config_request(req: str):
|
|||
except ValueError:
|
||||
raise ValueError(f'\"{val}\" is not a number silly')
|
||||
|
||||
|
||||
def perform_auto_conf(config: dict) -> dict:
|
||||
model = config['model']
|
||||
prefered_size_w = 512
|
||||
prefered_size_h = 512
|
||||
|
||||
if 'xl' in model:
|
||||
prefered_size_w = 1024
|
||||
prefered_size_h = 1024
|
||||
|
||||
else:
|
||||
prefered_size_w = 512
|
||||
prefered_size_h = 512
|
||||
|
||||
config['step'] = random.randint(20, 35)
|
||||
config['width'] = prefered_size_w
|
||||
config['height'] = prefered_size_h
|
||||
|
||||
return config
|
||||
|
|
|
|||
|
|
@ -19,8 +19,9 @@ from leap.hyperion import HyperionAPI
|
|||
import discord
|
||||
import io
|
||||
|
||||
from skynet.db import open_database_connection
|
||||
from skynet.ipfs import get_ipfs_file, AsyncIPFSHTTP
|
||||
from skynet.db import open_new_database, open_database_connection
|
||||
from skynet.ipfs import get_ipfs_file
|
||||
from skynet.ipfs.docker import open_ipfs_node
|
||||
from skynet.constants import *
|
||||
|
||||
from . import *
|
||||
|
|
@ -43,11 +44,8 @@ class SkynetDiscordFrontend:
|
|||
db_host: str,
|
||||
db_user: str,
|
||||
db_pass: str,
|
||||
ipfs_url: str,
|
||||
remote_ipfs_node: str,
|
||||
key: str,
|
||||
explorer_domain: str,
|
||||
ipfs_domain: str
|
||||
key: str
|
||||
):
|
||||
# self.token = token
|
||||
self.account = account
|
||||
|
|
@ -57,23 +55,23 @@ class SkynetDiscordFrontend:
|
|||
self.db_host = db_host
|
||||
self.db_user = db_user
|
||||
self.db_pass = db_pass
|
||||
self.ipfs_url = ipfs_url
|
||||
self.remote_ipfs_node = remote_ipfs_node
|
||||
self.key = key
|
||||
self.explorer_domain = explorer_domain
|
||||
self.ipfs_domain = ipfs_domain
|
||||
|
||||
self.bot = DiscordBot(self)
|
||||
self.cleos = CLEOS(None, None, url=node_url, remote=node_url)
|
||||
self.hyperion = HyperionAPI(hyperion_url)
|
||||
self.ipfs_node = AsyncIPFSHTTP(ipfs_node)
|
||||
|
||||
self._exit_stack = ExitStack()
|
||||
self._async_exit_stack = AsyncExitStack()
|
||||
|
||||
async def start(self):
|
||||
if self.remote_ipfs_node:
|
||||
await self.ipfs_node.connect(self.remote_ipfs_node)
|
||||
self.ipfs_node = self._exit_stack.enter_context(
|
||||
open_ipfs_node())
|
||||
|
||||
self.ipfs_node.connect(self.remote_ipfs_node)
|
||||
logging.info(
|
||||
f'connected to remote ipfs node: {self.remote_ipfs_node}')
|
||||
|
||||
self.db_call = await self._async_exit_stack.enter_async_context(
|
||||
open_database_connection(
|
||||
|
|
@ -123,7 +121,7 @@ class SkynetDiscordFrontend:
|
|||
ctx: discord.ext.commands.context.Context | discord.Message,
|
||||
file_id: str | None = None,
|
||||
binary_data: str = ''
|
||||
) -> bool:
|
||||
):
|
||||
send = ctx.channel.send
|
||||
|
||||
if params['seed'] == None:
|
||||
|
|
@ -170,10 +168,10 @@ class SkynetDiscordFrontend:
|
|||
await self.bot.channel.send(
|
||||
status_msg,
|
||||
'skynet has suffered an internal error trying to fill this request')
|
||||
return False
|
||||
return
|
||||
|
||||
enqueue_tx_id = res['transaction_id']
|
||||
enqueue_tx_link = f'[**Your request on Skynet Explorer**](https://{self.explorer_domain}/v2/explore/transaction/{enqueue_tx_id})'
|
||||
enqueue_tx_link = f'[**Your request on Skynet Explorer**](https://explorer.{DEFAULT_DOMAIN}/v2/explore/transaction/{enqueue_tx_id})'
|
||||
|
||||
msg_text += f'**broadcasted!** \n{enqueue_tx_link}\n[{timestamp_pretty()}] *workers are processing request...* '
|
||||
embed = discord.Embed(
|
||||
|
|
@ -232,7 +230,7 @@ class SkynetDiscordFrontend:
|
|||
color=discord.Color.blue())
|
||||
|
||||
await message.edit(embed=embed)
|
||||
return False
|
||||
return
|
||||
|
||||
tx_link = f'[**Your result on Skynet Explorer**](https://explorer.{DEFAULT_DOMAIN}/v2/explore/transaction/{tx_hash})'
|
||||
|
||||
|
|
@ -245,48 +243,8 @@ class SkynetDiscordFrontend:
|
|||
await message.edit(embed=embed)
|
||||
|
||||
# attempt to get the image and send it
|
||||
results = {}
|
||||
ipfs_link = f'https://{self.ipfs_domain}/ipfs/{ipfs_hash}'
|
||||
ipfs_link_legacy = ipfs_link + '/image.png'
|
||||
|
||||
async def get_and_set_results(link: str):
|
||||
res = await get_ipfs_file(link)
|
||||
logging.info(f'got response from {link}')
|
||||
if not res or res.status_code != 200:
|
||||
logging.warning(f'couldn\'t get ipfs binary data at {link}!')
|
||||
|
||||
else:
|
||||
try:
|
||||
with Image.open(io.BytesIO(res.raw)) as image:
|
||||
tmp_buf = io.BytesIO()
|
||||
image.save(tmp_buf, format='PNG')
|
||||
png_img = tmp_buf.getvalue()
|
||||
results[link] = png_img
|
||||
|
||||
except UnidentifiedImageError:
|
||||
logging.warning(f'couldn\'t get ipfs binary data at {link}!')
|
||||
|
||||
tasks = [
|
||||
get_and_set_results(ipfs_link),
|
||||
get_and_set_results(ipfs_link_legacy)
|
||||
]
|
||||
await asyncio.gather(*tasks)
|
||||
|
||||
png_img = None
|
||||
if ipfs_link_legacy in results:
|
||||
png_img = results[ipfs_link_legacy]
|
||||
|
||||
if ipfs_link in results:
|
||||
png_img = results[ipfs_link]
|
||||
|
||||
if not png_img:
|
||||
await self.update_status_message(
|
||||
status_msg,
|
||||
caption,
|
||||
reply_markup=build_redo_menu(),
|
||||
parse_mode='HTML'
|
||||
)
|
||||
return True
|
||||
ipfs_link = f'https://ipfs.{DEFAULT_DOMAIN}/ipfs/{ipfs_hash}/image.png'
|
||||
resp = await get_ipfs_file(ipfs_link)
|
||||
|
||||
# reword this function, may not need caption
|
||||
caption, embed = generate_reply_caption(
|
||||
|
|
@ -307,5 +265,3 @@ class SkynetDiscordFrontend:
|
|||
else: # txt2img
|
||||
embed.set_image(url=ipfs_link)
|
||||
await send(embed=embed, view=SkynetView(self))
|
||||
|
||||
return True
|
||||
|
|
|
|||
|
|
@ -115,9 +115,9 @@ def create_handler_context(frontend: 'SkynetDiscordFrontend'):
|
|||
await db_call(
|
||||
'update_user_stats', user.id, 'txt2img', last_prompt=prompt)
|
||||
|
||||
success = await work_request(user, status_msg, 'txt2img', params, ctx)
|
||||
ec = await work_request(user, status_msg, 'txt2img', params, ctx)
|
||||
|
||||
if success:
|
||||
if ec == None:
|
||||
await db_call('increment_generated', user.id)
|
||||
|
||||
@bot.command(name='redo', help='Redo last request')
|
||||
|
|
@ -153,13 +153,13 @@ def create_handler_context(frontend: 'SkynetDiscordFrontend'):
|
|||
**user_config
|
||||
}
|
||||
|
||||
success = await work_request(
|
||||
ec = await work_request(
|
||||
user, status_msg, 'redo', params, ctx,
|
||||
file_id=file_id,
|
||||
binary_data=binary
|
||||
)
|
||||
|
||||
if success:
|
||||
if ec == None:
|
||||
await db_call('increment_generated', user.id)
|
||||
|
||||
@bot.command(name='img2img', help='Responds with an image')
|
||||
|
|
@ -217,12 +217,10 @@ def create_handler_context(frontend: 'SkynetDiscordFrontend'):
|
|||
image.thumbnail((512, 512))
|
||||
logging.warning(f'resized it to {image.size}')
|
||||
|
||||
image_loc = 'ipfs-staging/image.png'
|
||||
image.save(image_loc, format='PNG')
|
||||
image.save(f'ipfs-docker-staging/image.png', format='PNG')
|
||||
|
||||
ipfs_info = await ipfs_node.add(image_loc)
|
||||
ipfs_hash = ipfs_info['Hash']
|
||||
await ipfs_node.pin(ipfs_hash)
|
||||
ipfs_hash = ipfs_node.add('image.png')
|
||||
ipfs_node.pin(ipfs_hash)
|
||||
|
||||
logging.info(f'published input image {ipfs_hash} on ipfs')
|
||||
|
||||
|
|
@ -245,13 +243,13 @@ def create_handler_context(frontend: 'SkynetDiscordFrontend'):
|
|||
last_binary=ipfs_hash
|
||||
)
|
||||
|
||||
success = await work_request(
|
||||
ec = await work_request(
|
||||
user, status_msg, 'img2img', params, ctx,
|
||||
file_id=file_id,
|
||||
binary_data=ipfs_hash
|
||||
)
|
||||
|
||||
if success:
|
||||
if ec == None:
|
||||
await db_call('increment_generated', user.id)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -63,9 +63,9 @@ class Txt2ImgButton(discord.ui.Button):
|
|||
await db_call(
|
||||
'update_user_stats', user.id, 'txt2img', last_prompt=prompt)
|
||||
|
||||
success = await work_request(user, status_msg, 'txt2img', params, msg)
|
||||
ec = await work_request(user, status_msg, 'txt2img', params, msg)
|
||||
|
||||
if success:
|
||||
if ec == None:
|
||||
await db_call('increment_generated', user.id)
|
||||
|
||||
|
||||
|
|
@ -145,13 +145,13 @@ class Img2ImgButton(discord.ui.Button):
|
|||
last_binary=ipfs_hash
|
||||
)
|
||||
|
||||
success = await work_request(
|
||||
ec = await work_request(
|
||||
user, status_msg, 'img2img', params, msg,
|
||||
file_id=file_id,
|
||||
binary_data=ipfs_hash
|
||||
)
|
||||
|
||||
if success:
|
||||
if ec == None:
|
||||
await db_call('increment_generated', user.id)
|
||||
|
||||
|
||||
|
|
@ -195,13 +195,13 @@ class RedoButton(discord.ui.Button):
|
|||
'prompt': prompt,
|
||||
**user_config
|
||||
}
|
||||
success = await work_request(
|
||||
ec = await work_request(
|
||||
user, status_msg, 'redo', params, interaction,
|
||||
file_id=file_id,
|
||||
binary_data=binary
|
||||
)
|
||||
|
||||
if success:
|
||||
if ec == None:
|
||||
await db_call('increment_generated', user.id)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -81,12 +81,11 @@ def generate_reply_caption(
|
|||
params: dict,
|
||||
tx_hash: str,
|
||||
worker: str,
|
||||
reward: str,
|
||||
explorer_domain: str
|
||||
reward: str
|
||||
):
|
||||
explorer_link = discord.Embed(
|
||||
title='[SKYNET Transaction Explorer]',
|
||||
url=f'https://{explorer_domain}/v2/explore/transaction/{tx_hash}',
|
||||
url=f'https://explorer.{DEFAULT_DOMAIN}/v2/explore/transaction/{tx_hash}',
|
||||
color=discord.Color.blue())
|
||||
|
||||
meta_info = prepare_metainfo_caption(user, worker, reward, params, explorer_link)
|
||||
|
|
|
|||
|
|
@ -1,27 +1,26 @@
|
|||
#!/usr/bin/python
|
||||
|
||||
import io
|
||||
from json import JSONDecodeError
|
||||
import random
|
||||
import logging
|
||||
import asyncio
|
||||
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
from json import JSONDecodeError
|
||||
from decimal import Decimal
|
||||
from hashlib import sha256
|
||||
from datetime import datetime
|
||||
from contextlib import AsyncExitStack
|
||||
from contextlib import ExitStack, AsyncExitStack
|
||||
from contextlib import asynccontextmanager as acm
|
||||
|
||||
from leap.cleos import CLEOS
|
||||
from leap.sugar import Name, asset_from_str, collect_stdout
|
||||
from leap.hyperion import HyperionAPI
|
||||
|
||||
from telebot.types import InputMediaPhoto
|
||||
|
||||
from telebot.async_telebot import AsyncTeleBot
|
||||
|
||||
from skynet.db import open_database_connection
|
||||
from skynet.ipfs import get_ipfs_file, AsyncIPFSHTTP
|
||||
from skynet.db import open_new_database, open_database_connection
|
||||
from skynet.ipfs import get_ipfs_file
|
||||
from skynet.ipfs.docker import open_ipfs_node
|
||||
from skynet.constants import *
|
||||
|
||||
from . import *
|
||||
|
|
@ -42,11 +41,8 @@ class SkynetTelegramFrontend:
|
|||
db_host: str,
|
||||
db_user: str,
|
||||
db_pass: str,
|
||||
ipfs_node: str,
|
||||
remote_ipfs_node: str | None,
|
||||
key: str,
|
||||
explorer_domain: str,
|
||||
ipfs_domain: str
|
||||
remote_ipfs_node: str,
|
||||
key: str
|
||||
):
|
||||
self.token = token
|
||||
self.account = account
|
||||
|
|
@ -58,19 +54,21 @@ class SkynetTelegramFrontend:
|
|||
self.db_pass = db_pass
|
||||
self.remote_ipfs_node = remote_ipfs_node
|
||||
self.key = key
|
||||
self.explorer_domain = explorer_domain
|
||||
self.ipfs_domain = ipfs_domain
|
||||
|
||||
self.bot = AsyncTeleBot(token, exception_handler=SKYExceptionHandler)
|
||||
self.cleos = CLEOS(None, None, url=node_url, remote=node_url)
|
||||
self.hyperion = HyperionAPI(hyperion_url)
|
||||
self.ipfs_node = AsyncIPFSHTTP(ipfs_node)
|
||||
|
||||
self._exit_stack = ExitStack()
|
||||
self._async_exit_stack = AsyncExitStack()
|
||||
|
||||
async def start(self):
|
||||
if self.remote_ipfs_node:
|
||||
await self.ipfs_node.connect(self.remote_ipfs_node)
|
||||
self.ipfs_node = self._exit_stack.enter_context(
|
||||
open_ipfs_node())
|
||||
|
||||
self.ipfs_node.connect(self.remote_ipfs_node)
|
||||
logging.info(
|
||||
f'connected to remote ipfs node: {self.remote_ipfs_node}')
|
||||
|
||||
self.db_call = await self._async_exit_stack.enter_async_context(
|
||||
open_database_connection(
|
||||
|
|
@ -80,6 +78,7 @@ class SkynetTelegramFrontend:
|
|||
|
||||
async def stop(self):
|
||||
await self._async_exit_stack.aclose()
|
||||
self._exit_stack.close()
|
||||
|
||||
@acm
|
||||
async def open(self):
|
||||
|
|
@ -117,7 +116,7 @@ class SkynetTelegramFrontend:
|
|||
params: dict,
|
||||
file_id: str | None = None,
|
||||
binary_data: str = ''
|
||||
) -> bool:
|
||||
):
|
||||
if params['seed'] == None:
|
||||
params['seed'] = random.randint(0, 0xFFFFFFFF)
|
||||
|
||||
|
|
@ -160,12 +159,12 @@ class SkynetTelegramFrontend:
|
|||
await self.update_status_message(
|
||||
status_msg,
|
||||
'skynet has suffered an internal error trying to fill this request')
|
||||
return False
|
||||
return
|
||||
|
||||
enqueue_tx_id = res['transaction_id']
|
||||
enqueue_tx_link = hlink(
|
||||
'Your request on Skynet Explorer',
|
||||
f'https://{self.explorer_domain}/v2/explore/transaction/{enqueue_tx_id}'
|
||||
f'https://explorer.{DEFAULT_DOMAIN}/v2/explore/transaction/{enqueue_tx_id}'
|
||||
)
|
||||
|
||||
await self.append_status_message(
|
||||
|
|
@ -222,11 +221,11 @@ class SkynetTelegramFrontend:
|
|||
f'\n[{timestamp_pretty()}] <b>timeout processing request</b>',
|
||||
parse_mode='HTML'
|
||||
)
|
||||
return False
|
||||
return
|
||||
|
||||
tx_link = hlink(
|
||||
'Your result on Skynet Explorer',
|
||||
f'https://{self.explorer_domain}/v2/explore/transaction/{tx_hash}'
|
||||
f'https://explorer.{DEFAULT_DOMAIN}/v2/explore/transaction/{tx_hash}'
|
||||
)
|
||||
|
||||
await self.append_status_message(
|
||||
|
|
@ -237,60 +236,23 @@ class SkynetTelegramFrontend:
|
|||
parse_mode='HTML'
|
||||
)
|
||||
|
||||
caption = generate_reply_caption(
|
||||
user, params, tx_hash, worker, reward, self.explorer_domain)
|
||||
|
||||
# attempt to get the image and send it
|
||||
results = {}
|
||||
ipfs_link = f'https://{self.ipfs_domain}/ipfs/{ipfs_hash}'
|
||||
ipfs_link_legacy = ipfs_link + '/image.png'
|
||||
ipfs_link = f'https://ipfs.{DEFAULT_DOMAIN}/ipfs/{ipfs_hash}/image.png'
|
||||
resp = await get_ipfs_file(ipfs_link)
|
||||
|
||||
async def get_and_set_results(link: str):
|
||||
res = await get_ipfs_file(link)
|
||||
logging.info(f'got response from {link}')
|
||||
if not res or res.status_code != 200:
|
||||
logging.warning(f'couldn\'t get ipfs binary data at {link}!')
|
||||
caption = generate_reply_caption(
|
||||
user, params, tx_hash, worker, reward)
|
||||
|
||||
else:
|
||||
try:
|
||||
with Image.open(io.BytesIO(res.raw)) as image:
|
||||
w, h = image.size
|
||||
|
||||
if w > TG_MAX_WIDTH or h > TG_MAX_HEIGHT:
|
||||
logging.warning(f'result is of size {image.size}')
|
||||
image.thumbnail((TG_MAX_WIDTH, TG_MAX_HEIGHT))
|
||||
|
||||
tmp_buf = io.BytesIO()
|
||||
image.save(tmp_buf, format='PNG')
|
||||
png_img = tmp_buf.getvalue()
|
||||
|
||||
results[link] = png_img
|
||||
|
||||
except UnidentifiedImageError:
|
||||
logging.warning(f'couldn\'t get ipfs binary data at {link}!')
|
||||
|
||||
tasks = [
|
||||
get_and_set_results(ipfs_link),
|
||||
get_and_set_results(ipfs_link_legacy)
|
||||
]
|
||||
await asyncio.gather(*tasks)
|
||||
|
||||
png_img = None
|
||||
if ipfs_link_legacy in results:
|
||||
png_img = results[ipfs_link_legacy]
|
||||
|
||||
if ipfs_link in results:
|
||||
png_img = results[ipfs_link]
|
||||
|
||||
if not png_img:
|
||||
if not resp or resp.status_code != 200:
|
||||
logging.error(f'couldn\'t get ipfs hosted image at {ipfs_link}!')
|
||||
await self.update_status_message(
|
||||
status_msg,
|
||||
caption,
|
||||
reply_markup=build_redo_menu(),
|
||||
parse_mode='HTML'
|
||||
)
|
||||
return True
|
||||
|
||||
else:
|
||||
logging.info(f'success! sending generated image')
|
||||
await self.bot.delete_message(
|
||||
chat_id=status_msg.chat.id, message_id=status_msg.id)
|
||||
|
|
@ -300,7 +262,7 @@ class SkynetTelegramFrontend:
|
|||
media=[
|
||||
InputMediaPhoto(file_id),
|
||||
InputMediaPhoto(
|
||||
png_img,
|
||||
resp.raw,
|
||||
caption=caption,
|
||||
parse_mode='HTML'
|
||||
)
|
||||
|
|
@ -311,9 +273,7 @@ class SkynetTelegramFrontend:
|
|||
await self.bot.send_photo(
|
||||
status_msg.chat.id,
|
||||
caption=caption,
|
||||
photo=png_img,
|
||||
photo=resp.raw,
|
||||
reply_markup=build_redo_menu(),
|
||||
parse_mode='HTML'
|
||||
)
|
||||
|
||||
return True
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ from datetime import datetime, timedelta
|
|||
from PIL import Image
|
||||
from telebot.types import CallbackQuery, Message
|
||||
|
||||
from skynet.frontend import validate_user_config_request, perform_auto_conf
|
||||
from skynet.frontend import validate_user_config_request
|
||||
from skynet.constants import *
|
||||
|
||||
|
||||
|
|
@ -118,9 +118,6 @@ def create_handler_context(frontend: 'SkynetTelegramFrontend'):
|
|||
user = message.from_user
|
||||
chat = message.chat
|
||||
|
||||
if chat.type == 'private':
|
||||
return
|
||||
|
||||
reply_id = None
|
||||
if chat.type == 'group' and chat.id == GROUP_ID:
|
||||
reply_id = message.message_id
|
||||
|
|
@ -149,9 +146,6 @@ def create_handler_context(frontend: 'SkynetTelegramFrontend'):
|
|||
user_config = {**user_row}
|
||||
del user_config['id']
|
||||
|
||||
if user_config['autoconf']:
|
||||
user_config = perform_auto_conf(user_config)
|
||||
|
||||
params = {
|
||||
'prompt': prompt,
|
||||
**user_config
|
||||
|
|
@ -160,9 +154,9 @@ def create_handler_context(frontend: 'SkynetTelegramFrontend'):
|
|||
await db_call(
|
||||
'update_user_stats', user.id, 'txt2img', last_prompt=prompt)
|
||||
|
||||
success = await work_request(user, status_msg, 'txt2img', params)
|
||||
ec = await work_request(user, status_msg, 'txt2img', params)
|
||||
|
||||
if success:
|
||||
if ec == 0:
|
||||
await db_call('increment_generated', user.id)
|
||||
|
||||
|
||||
|
|
@ -180,9 +174,6 @@ def create_handler_context(frontend: 'SkynetTelegramFrontend'):
|
|||
user = message.from_user
|
||||
chat = message.chat
|
||||
|
||||
if chat.type == 'private':
|
||||
return
|
||||
|
||||
reply_id = None
|
||||
if chat.type == 'group' and chat.id == GROUP_ID:
|
||||
reply_id = message.message_id
|
||||
|
|
@ -212,31 +203,26 @@ def create_handler_context(frontend: 'SkynetTelegramFrontend'):
|
|||
file_path = (await bot.get_file(file_id)).file_path
|
||||
image_raw = await bot.download_file(file_path)
|
||||
|
||||
user_config = {**user_row}
|
||||
del user_config['id']
|
||||
if user_config['autoconf']:
|
||||
user_config = perform_auto_conf(user_config)
|
||||
|
||||
with Image.open(io.BytesIO(image_raw)) as image:
|
||||
w, h = image.size
|
||||
|
||||
if w > user_config['width'] or h > user_config['height']:
|
||||
if w > 512 or h > 512:
|
||||
logging.warning(f'user sent img of size {image.size}')
|
||||
image.thumbnail(
|
||||
(user_config['width'], user_config['height']))
|
||||
image.thumbnail((512, 512))
|
||||
logging.warning(f'resized it to {image.size}')
|
||||
|
||||
image_loc = 'ipfs-staging/image.png'
|
||||
image.save(image_loc, format='PNG')
|
||||
image.save(f'ipfs-docker-staging/image.png', format='PNG')
|
||||
|
||||
ipfs_info = await ipfs_node.add(image_loc)
|
||||
ipfs_hash = ipfs_info['Hash']
|
||||
await ipfs_node.pin(ipfs_hash)
|
||||
ipfs_hash = ipfs_node.add('image.png')
|
||||
ipfs_node.pin(ipfs_hash)
|
||||
|
||||
logging.info(f'published input image {ipfs_hash} on ipfs')
|
||||
|
||||
logging.info(f'mid: {message.id}')
|
||||
|
||||
user_config = {**user_row}
|
||||
del user_config['id']
|
||||
|
||||
params = {
|
||||
'prompt': prompt,
|
||||
**user_config
|
||||
|
|
@ -251,13 +237,13 @@ def create_handler_context(frontend: 'SkynetTelegramFrontend'):
|
|||
last_binary=ipfs_hash
|
||||
)
|
||||
|
||||
success = await work_request(
|
||||
ec = await work_request(
|
||||
user, status_msg, 'img2img', params,
|
||||
file_id=file_id,
|
||||
binary_data=ipfs_hash
|
||||
)
|
||||
|
||||
if success:
|
||||
if ec == 0:
|
||||
await db_call('increment_generated', user.id)
|
||||
|
||||
|
||||
|
|
@ -277,9 +263,6 @@ def create_handler_context(frontend: 'SkynetTelegramFrontend'):
|
|||
user = message.from_user
|
||||
chat = message.chat
|
||||
|
||||
if chat.type == 'private':
|
||||
return
|
||||
|
||||
init_msg = 'started processing redo request...'
|
||||
if is_query:
|
||||
status_msg = await bot.send_message(chat.id, init_msg)
|
||||
|
|
@ -309,23 +292,18 @@ def create_handler_context(frontend: 'SkynetTelegramFrontend'):
|
|||
'new_user_request', user.id, message.id, status_msg.id, status=init_msg)
|
||||
user_config = {**user_row}
|
||||
del user_config['id']
|
||||
if user_config['autoconf']:
|
||||
user_config = perform_auto_conf(user_config)
|
||||
|
||||
params = {
|
||||
'prompt': prompt,
|
||||
**user_config
|
||||
}
|
||||
|
||||
success = await work_request(
|
||||
await work_request(
|
||||
user, status_msg, 'redo', params,
|
||||
file_id=file_id,
|
||||
binary_data=binary
|
||||
)
|
||||
|
||||
if success:
|
||||
await db_call('increment_generated', user.id)
|
||||
|
||||
|
||||
# "proxy" handlers just request routers
|
||||
|
||||
|
|
|
|||
|
|
@ -67,12 +67,11 @@ def generate_reply_caption(
|
|||
params: dict,
|
||||
tx_hash: str,
|
||||
worker: str,
|
||||
reward: str,
|
||||
explorer_domain: str
|
||||
reward: str
|
||||
):
|
||||
explorer_link = hlink(
|
||||
'SKYNET Transaction Explorer',
|
||||
f'https://explorer.{explorer_domain}/v2/explore/transaction/{tx_hash}'
|
||||
f'https://explorer.{DEFAULT_DOMAIN}/v2/explore/transaction/{tx_hash}'
|
||||
)
|
||||
|
||||
meta_info = prepare_metainfo_caption(tguser, worker, reward, params)
|
||||
|
|
|
|||
|
|
@ -1,64 +1,33 @@
|
|||
#!/usr/bin/python
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import asks
|
||||
import requests
|
||||
|
||||
|
||||
class IPFSClientException(BaseException):
|
||||
...
|
||||
|
||||
|
||||
class AsyncIPFSHTTP:
|
||||
class IPFSHTTP:
|
||||
|
||||
def __init__(self, endpoint: str):
|
||||
self.endpoint = endpoint
|
||||
|
||||
async def _post(self, sub_url: str, *args, **kwargs):
|
||||
resp = await asks.post(
|
||||
self.endpoint + sub_url,
|
||||
*args, **kwargs
|
||||
)
|
||||
|
||||
if resp.status_code != 200:
|
||||
raise IPFSClientException(resp.text)
|
||||
|
||||
return resp.json()
|
||||
|
||||
async def add(self, file_path: Path, **kwargs):
|
||||
files = {
|
||||
'file': file_path
|
||||
}
|
||||
return await self._post(
|
||||
'/api/v0/add',
|
||||
files=files,
|
||||
params=kwargs
|
||||
)
|
||||
|
||||
async def pin(self, cid: str):
|
||||
return (await self._post(
|
||||
'/api/v0/pin/add',
|
||||
def pin(self, cid: str):
|
||||
return requests.post(
|
||||
f'{self.endpoint}/api/v0/pin/add',
|
||||
params={'arg': cid}
|
||||
))['Pins']
|
||||
|
||||
async def connect(self, multi_addr: str):
|
||||
return await self._post(
|
||||
'/api/v0/swarm/connect',
|
||||
params={'arg': multi_addr}
|
||||
)
|
||||
|
||||
async def peers(self, **kwargs):
|
||||
return (await self._post(
|
||||
'/api/v0/swarm/peers',
|
||||
params=kwargs
|
||||
))['Peers']
|
||||
async def a_pin(self, cid: str):
|
||||
return await asks.post(
|
||||
f'{self.endpoint}/api/v0/pin/add',
|
||||
params={'arg': cid}
|
||||
)
|
||||
|
||||
|
||||
async def get_ipfs_file(ipfs_link: str, timeout: int = 60):
|
||||
async def get_ipfs_file(ipfs_link: str):
|
||||
logging.info(f'attempting to get image at {ipfs_link}')
|
||||
resp = None
|
||||
for i in range(timeout):
|
||||
for i in range(20):
|
||||
try:
|
||||
resp = await asks.get(ipfs_link, timeout=3)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
#!/usr/bin/python
|
||||
|
||||
import os
|
||||
import sys
|
||||
import logging
|
||||
|
||||
|
|
@ -9,24 +10,61 @@ from contextlib import contextmanager as cm
|
|||
import docker
|
||||
|
||||
from docker.types import Mount
|
||||
from docker.models.containers import Container
|
||||
|
||||
|
||||
class IPFSDocker:
|
||||
|
||||
def __init__(self, container: Container):
|
||||
self._container = container
|
||||
|
||||
def add(self, file: str) -> str:
|
||||
ec, out = self._container.exec_run(
|
||||
['ipfs', 'add', '-w', f'/export/{file}', '-Q'])
|
||||
if ec != 0:
|
||||
logging.error(out)
|
||||
assert ec == 0
|
||||
|
||||
return out.decode().rstrip()
|
||||
|
||||
def pin(self, ipfs_hash: str):
|
||||
ec, _ = self._container.exec_run(
|
||||
['ipfs', 'pin', 'add', ipfs_hash])
|
||||
assert ec == 0
|
||||
|
||||
def connect(self, remote_node: str):
|
||||
ec, out = self._container.exec_run(
|
||||
['ipfs', 'swarm', 'connect', remote_node])
|
||||
if ec != 0:
|
||||
logging.error(out)
|
||||
|
||||
assert ec == 0
|
||||
|
||||
def check_connect(self):
|
||||
ec, out = self._container.exec_run(
|
||||
['ipfs', 'swarm', 'peers'])
|
||||
if ec != 0:
|
||||
logging.error(out)
|
||||
assert ec == 0
|
||||
|
||||
return out.splitlines()
|
||||
|
||||
|
||||
@cm
|
||||
def open_ipfs_node(
|
||||
name: str = 'skynet-ipfs',
|
||||
teardown: bool = False,
|
||||
peers: list[str] = []
|
||||
):
|
||||
def open_ipfs_node(name='skynet-ipfs'):
|
||||
dclient = docker.from_env()
|
||||
|
||||
container = None
|
||||
try:
|
||||
container = dclient.containers.get(name)
|
||||
|
||||
except docker.errors.NotFound:
|
||||
staging_dir = Path().resolve() / 'ipfs-docker-staging'
|
||||
staging_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
data_dir = Path().resolve() / 'ipfs-docker-data'
|
||||
data_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
export_target = '/export'
|
||||
data_target = '/data/ipfs'
|
||||
|
||||
container = dclient.containers.run(
|
||||
|
|
@ -38,15 +76,19 @@ def open_ipfs_node(
|
|||
'5001/tcp': ('127.0.0.1', 5001)
|
||||
},
|
||||
mounts=[
|
||||
Mount(export_target, str(staging_dir), 'bind'),
|
||||
Mount(data_target, str(data_dir), 'bind')
|
||||
],
|
||||
detach=True,
|
||||
remove=True
|
||||
)
|
||||
|
||||
uid, gid = 1000, 1000
|
||||
|
||||
if sys.platform != 'win32':
|
||||
uid = os.getuid()
|
||||
gid = os.getgid()
|
||||
ec, out = container.exec_run(['chown', f'{uid}:{gid}', '-R', export_target])
|
||||
logging.info(out)
|
||||
assert ec == 0
|
||||
ec, out = container.exec_run(['chown', f'{uid}:{gid}', '-R', data_target])
|
||||
logging.info(out)
|
||||
assert ec == 0
|
||||
|
|
@ -57,13 +99,4 @@ def open_ipfs_node(
|
|||
if 'Daemon is ready' in log:
|
||||
break
|
||||
|
||||
for peer in peers:
|
||||
ec, out = container.exec_run(
|
||||
['ipfs', 'swarm', 'connect', peer])
|
||||
if ec != 0:
|
||||
logging.error(out)
|
||||
|
||||
yield
|
||||
|
||||
if teardown and container:
|
||||
container.stop()
|
||||
yield IPFSDocker(container)
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import trio
|
|||
|
||||
from leap.hyperion import HyperionAPI
|
||||
|
||||
from . import AsyncIPFSHTTP
|
||||
from . import IPFSHTTP
|
||||
|
||||
|
||||
MAX_TIME = timedelta(seconds=20)
|
||||
|
|
@ -20,7 +20,7 @@ class SkynetPinner:
|
|||
def __init__(
|
||||
self,
|
||||
hyperion: HyperionAPI,
|
||||
ipfs_http: AsyncIPFSHTTP
|
||||
ipfs_http: IPFSHTTP
|
||||
):
|
||||
self.hyperion = hyperion
|
||||
self.ipfs_http = ipfs_http
|
||||
|
|
@ -85,9 +85,9 @@ class SkynetPinner:
|
|||
for _ in range(6):
|
||||
try:
|
||||
with trio.move_on_after(5):
|
||||
pins = await self.ipfs_http.pin(cid)
|
||||
if cid not in pins:
|
||||
logging.error(f'error pinning {cid}')
|
||||
resp = await self.ipfs_http.a_pin(cid)
|
||||
if resp.status_code != 200:
|
||||
logging.error(f'error pinning {cid}:\n{resp.text}')
|
||||
del self._pinned[cid]
|
||||
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -4,12 +4,44 @@ import json
|
|||
import time
|
||||
import logging
|
||||
|
||||
from datetime import datetime
|
||||
from contextlib import contextmanager as cm
|
||||
|
||||
import docker
|
||||
|
||||
from leap.cleos import CLEOS
|
||||
from leap.sugar import get_container, Symbol
|
||||
from pytz import timezone
|
||||
from leap.cleos import CLEOS, default_nodeos_image
|
||||
from leap.sugar import get_container, Symbol, random_string
|
||||
|
||||
|
||||
@cm
|
||||
def open_cleos(
|
||||
node_url: str,
|
||||
key: str | None
|
||||
):
|
||||
vtestnet = None
|
||||
try:
|
||||
dclient = docker.from_env()
|
||||
vtestnet = get_container(
|
||||
dclient,
|
||||
default_nodeos_image(),
|
||||
name=f'skynet-wallet-{random_string(size=8)}',
|
||||
force_unique=True,
|
||||
detach=True,
|
||||
network='host',
|
||||
remove=True)
|
||||
|
||||
cleos = CLEOS(dclient, vtestnet, url=node_url, remote=node_url)
|
||||
|
||||
if key:
|
||||
cleos.setup_wallet(key)
|
||||
|
||||
yield cleos
|
||||
|
||||
finally:
|
||||
if vtestnet:
|
||||
vtestnet.stop()
|
||||
|
||||
|
||||
|
||||
@cm
|
||||
|
|
|
|||
102
skynet/utils.py
102
skynet/utils.py
|
|
@ -2,14 +2,11 @@
|
|||
|
||||
import io
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import random
|
||||
import logging
|
||||
|
||||
from typing import Optional
|
||||
from pathlib import Path
|
||||
import asks
|
||||
|
||||
import torch
|
||||
import numpy as np
|
||||
|
|
@ -18,11 +15,12 @@ from PIL import Image
|
|||
from basicsr.archs.rrdbnet_arch import RRDBNet
|
||||
from diffusers import (
|
||||
DiffusionPipeline,
|
||||
StableDiffusionPipeline,
|
||||
StableDiffusionImg2ImgPipeline,
|
||||
EulerAncestralDiscreteScheduler
|
||||
)
|
||||
from realesrgan import RealESRGANer
|
||||
from huggingface_hub import login
|
||||
import trio
|
||||
|
||||
from .constants import MODELS
|
||||
|
||||
|
|
@ -51,23 +49,19 @@ def convert_from_img_to_bytes(image: Image, fmt='PNG') -> bytes:
|
|||
return byte_arr.getvalue()
|
||||
|
||||
|
||||
def crop_image(image: Image, max_w: int, max_h: int) -> Image:
|
||||
def convert_from_bytes_and_crop(raw: bytes, max_w: int, max_h: int) -> Image:
|
||||
image = convert_from_bytes_to_img(raw)
|
||||
w, h = image.size
|
||||
if w > max_w or h > max_h:
|
||||
image.thumbnail((max_w, max_h))
|
||||
image.thumbnail((512, 512))
|
||||
|
||||
return image.convert('RGB')
|
||||
|
||||
|
||||
def pipeline_for(
|
||||
model: str,
|
||||
mem_fraction: float = 1.0,
|
||||
image: bool = False,
|
||||
cache_dir: str | None = None
|
||||
) -> DiffusionPipeline:
|
||||
|
||||
def pipeline_for(model: str, mem_fraction: float = 1.0, image=False) -> DiffusionPipeline:
|
||||
assert torch.cuda.is_available()
|
||||
torch.cuda.empty_cache()
|
||||
torch.cuda.set_per_process_memory_fraction(mem_fraction)
|
||||
torch.backends.cuda.matmul.allow_tf32 = True
|
||||
torch.backends.cudnn.allow_tf32 = True
|
||||
|
||||
|
|
@ -78,54 +72,31 @@ def pipeline_for(
|
|||
torch.backends.cudnn.benchmark = False
|
||||
torch.use_deterministic_algorithms(True)
|
||||
|
||||
model_info = MODELS[model]
|
||||
|
||||
req_mem = model_info['mem']
|
||||
mem_gb = torch.cuda.mem_get_info()[1] / (10**9)
|
||||
mem_gb *= mem_fraction
|
||||
over_mem = mem_gb < req_mem
|
||||
if over_mem:
|
||||
logging.warn(f'model requires {req_mem} but card has {mem_gb}, model will run slower..')
|
||||
|
||||
shortname = model_info['short']
|
||||
|
||||
params = {
|
||||
'safety_checker': None,
|
||||
'torch_dtype': torch.float16,
|
||||
'cache_dir': cache_dir,
|
||||
'variant': 'fp16'
|
||||
'safety_checker': None
|
||||
}
|
||||
|
||||
match shortname:
|
||||
case 'stable':
|
||||
if model == 'runwayml/stable-diffusion-v1-5':
|
||||
params['revision'] = 'fp16'
|
||||
|
||||
torch.cuda.set_per_process_memory_fraction(mem_fraction)
|
||||
if image:
|
||||
pipe_class = StableDiffusionImg2ImgPipeline
|
||||
elif model == 'snowkidy/stable-diffusion-xl-base-0.9':
|
||||
pipe_class = DiffusionPipeline
|
||||
else:
|
||||
pipe_class = StableDiffusionPipeline
|
||||
|
||||
pipe = DiffusionPipeline.from_pretrained(
|
||||
pipe = pipe_class.from_pretrained(
|
||||
model, **params)
|
||||
|
||||
pipe.scheduler = EulerAncestralDiscreteScheduler.from_config(
|
||||
pipe.scheduler.config)
|
||||
|
||||
pipe.enable_xformers_memory_efficient_attention()
|
||||
|
||||
if over_mem:
|
||||
if not image:
|
||||
pipe.enable_vae_slicing()
|
||||
pipe.enable_vae_tiling()
|
||||
|
||||
pipe.enable_model_cpu_offload()
|
||||
|
||||
else:
|
||||
if sys.version_info[1] < 11:
|
||||
# torch.compile only supported on python < 3.11
|
||||
pipe.unet = torch.compile(
|
||||
pipe.unet, mode='reduce-overhead', fullgraph=True)
|
||||
|
||||
pipe = pipe.to('cuda')
|
||||
|
||||
return pipe
|
||||
return pipe.to('cuda')
|
||||
|
||||
|
||||
def txt2img(
|
||||
|
|
@ -138,6 +109,12 @@ def txt2img(
|
|||
steps: int = 28,
|
||||
seed: Optional[int] = None
|
||||
):
|
||||
assert torch.cuda.is_available()
|
||||
torch.cuda.empty_cache()
|
||||
torch.cuda.set_per_process_memory_fraction(1.0)
|
||||
torch.backends.cuda.matmul.allow_tf32 = True
|
||||
torch.backends.cudnn.allow_tf32 = True
|
||||
|
||||
login(token=hf_token)
|
||||
pipe = pipeline_for(model)
|
||||
|
||||
|
|
@ -165,6 +142,12 @@ def img2img(
|
|||
steps: int = 28,
|
||||
seed: Optional[int] = None
|
||||
):
|
||||
assert torch.cuda.is_available()
|
||||
torch.cuda.empty_cache()
|
||||
torch.cuda.set_per_process_memory_fraction(1.0)
|
||||
torch.backends.cuda.matmul.allow_tf32 = True
|
||||
torch.backends.cudnn.allow_tf32 = True
|
||||
|
||||
login(token=hf_token)
|
||||
pipe = pipeline_for(model, image=True)
|
||||
|
||||
|
|
@ -205,6 +188,12 @@ def upscale(
|
|||
output: str = 'output.png',
|
||||
model_path: str = 'weights/RealESRGAN_x4plus.pth'
|
||||
):
|
||||
assert torch.cuda.is_available()
|
||||
torch.cuda.empty_cache()
|
||||
torch.cuda.set_per_process_memory_fraction(1.0)
|
||||
torch.backends.cuda.matmul.allow_tf32 = True
|
||||
torch.backends.cudnn.allow_tf32 = True
|
||||
|
||||
input_img = Image.open(img_path).convert('RGB')
|
||||
|
||||
upscaler = init_upscaler(model_path=model_path)
|
||||
|
|
@ -213,26 +202,17 @@ def upscale(
|
|||
convert_from_image_to_cv2(input_img), outscale=4)
|
||||
|
||||
image = convert_from_cv2_to_image(up_img)
|
||||
|
||||
|
||||
image.save(output)
|
||||
|
||||
|
||||
async def download_upscaler():
|
||||
print('downloading upscaler...')
|
||||
weights_path = Path('weights')
|
||||
weights_path.mkdir(exist_ok=True)
|
||||
upscaler_url = 'https://github.com/xinntao/Real-ESRGAN/releases/download/v0.1.0/RealESRGAN_x4plus.pth'
|
||||
save_path = weights_path / 'RealESRGAN_x4plus.pth'
|
||||
response = await asks.get(upscaler_url)
|
||||
with open(save_path, 'wb') as f:
|
||||
f.write(response.content)
|
||||
print('done')
|
||||
|
||||
def download_all_models(hf_token: str, hf_home: str):
|
||||
def download_all_models(hf_token: str):
|
||||
assert torch.cuda.is_available()
|
||||
|
||||
trio.run(download_upscaler)
|
||||
|
||||
login(token=hf_token)
|
||||
for model in MODELS:
|
||||
print(f'DOWNLOADING {model.upper()}')
|
||||
pipeline_for(model, cache_dir=hf_home)
|
||||
pipeline_for(model)
|
||||
print(f'DOWNLOADING IMAGE {model.upper()}')
|
||||
pipeline_for(model, image=True)
|
||||
|
|
|
|||
|
|
@ -1,18 +1,15 @@
|
|||
#!/usr/bin/python
|
||||
|
||||
import logging
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from skynet.db import open_new_database
|
||||
from skynet.ipfs import AsyncIPFSHTTP
|
||||
from skynet.ipfs.docker import open_ipfs_node
|
||||
from skynet.nodeos import open_nodeos
|
||||
|
||||
|
||||
@pytest.fixture(scope='session')
|
||||
def ipfs_client():
|
||||
with open_ipfs_node(teardown=True):
|
||||
yield AsyncIPFSHTTP('http://127.0.0.1:5001')
|
||||
|
||||
@pytest.fixture(scope='session')
|
||||
def postgres_db():
|
||||
with open_new_database() as db_params:
|
||||
|
|
|
|||
|
|
@ -1,26 +0,0 @@
|
|||
#!/usr/bin/python
|
||||
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
async def test_connection(ipfs_client):
|
||||
await ipfs_client.connect(
|
||||
'/ip4/169.197.140.154/tcp/4001/p2p/12D3KooWKWogLFNEcNNMKnzU7Snrnuj84RZdMBg3sLiQSQc51oEv')
|
||||
peers = await ipfs_client.peers()
|
||||
assert '12D3KooWKWogLFNEcNNMKnzU7Snrnuj84RZdMBg3sLiQSQc51oEv' in [p['Peer'] for p in peers]
|
||||
|
||||
|
||||
async def test_add_and_pin_file(ipfs_client):
|
||||
test_file = Path('hello_world.txt')
|
||||
with open(test_file, 'w+') as file:
|
||||
file.write('Hello Skynet!')
|
||||
|
||||
file_info = await ipfs_client.add(test_file)
|
||||
file_cid = file_info['Hash']
|
||||
|
||||
pin_resp = await ipfs_client.pin(file_cid)
|
||||
|
||||
assert file_cid in pin_resp
|
||||
|
||||
test_file.unlink()
|
||||
Loading…
Reference in New Issue