skygpu
/
skynet
mirror of https://github.com/skygpu/skynet.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #45 from skygpu/uv_migration 

Switch to uv package manager
network-error-patch
Guillermo Rodriguez 2025-01-22 14:42:59 -03:00 committed by GitHub
parent b5f52b3b5b 8f4429bd06
commit 2e50b1a542
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 2933 additions and 4530 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -11,3 +11,4 @@ docs
ipfs-docker-data ipfs-docker-data
ipfs-staging ipfs-staging
weights weights
*.png

25
docker/Dockerfile.runtime
View File

@ -1,25 +0,0 @@
from python:3.11
env DEBIAN_FRONTEND=noninteractive
run apt-get update && apt-get install -y \
git
run curl -sSL https://install.python-poetry.org | python3 -
env PATH "/root/.local/bin:$PATH"
copy . /skynet
workdir /skynet
env POETRY_VIRTUALENVS_PATH /skynet/.venv
run poetry install
workdir /root/target
copy docker/entrypoint.sh /entrypoint.sh
entrypoint ["/entrypoint.sh"]
cmd ["skynet", "--help"]

46
docker/Dockerfile.runtime+cuda-py310
View File

@ -1,46 +0,0 @@
from nvidia/cuda:11.8.0-devel-ubuntu20.04
from python:3.10
env DEBIAN_FRONTEND=noninteractive
run apt-get update && apt-get install -y \
git \
clang \
cmake \
ffmpeg \
libsm6 \
libxext6 \
ninja-build
env CC /usr/bin/clang
env CXX /usr/bin/clang++
# install llvm10 as required by llvm-lite
run git clone https://github.com/llvm/llvm-project.git -b llvmorg-10.0.1
workdir /llvm-project
# this adds a commit from 12.0.0 that fixes build on newer compilers
run git cherry-pick -n b498303066a63a203d24f739b2d2e0e56dca70d1
run cmake -S llvm -B build -G Ninja -DCMAKE_BUILD_TYPE=Release
run ninja -C build install # -j8
run curl -sSL https://install.python-poetry.org | python3 -
env PATH "/root/.local/bin:$PATH"
copy . /skynet
workdir /skynet
env POETRY_VIRTUALENVS_PATH /skynet/.venv
run poetry install --with=cuda -v
workdir /root/target
env PYTORCH_CUDA_ALLOC_CONF max_split_size_mb:128
env NVIDIA_VISIBLE_DEVICES=all
copy docker/entrypoint.sh /entrypoint.sh
entrypoint ["/entrypoint.sh"]
cmd ["skynet", "--help"]

46
docker/Dockerfile.runtime+cuda-py311
View File

@ -1,46 +0,0 @@
from nvidia/cuda:11.8.0-devel-ubuntu20.04
from python:3.11
env DEBIAN_FRONTEND=noninteractive
run apt-get update && apt-get install -y \
git \
clang \
cmake \
ffmpeg \
libsm6 \
libxext6 \
ninja-build
env CC /usr/bin/clang
env CXX /usr/bin/clang++
# install llvm10 as required by llvm-lite
run git clone https://github.com/llvm/llvm-project.git -b llvmorg-10.0.1
workdir /llvm-project
# this adds a commit from 12.0.0 that fixes build on newer compilers
run git cherry-pick -n b498303066a63a203d24f739b2d2e0e56dca70d1
run cmake -S llvm -B build -G Ninja -DCMAKE_BUILD_TYPE=Release
run ninja -C build install # -j8
run curl -sSL https://install.python-poetry.org | python3 -
env PATH "/root/.local/bin:$PATH"
copy . /skynet
workdir /skynet
env POETRY_VIRTUALENVS_PATH /skynet/.venv
run poetry install --with=cuda -v
workdir /root/target
env PYTORCH_CUDA_ALLOC_CONF max_split_size_mb:128
env NVIDIA_VISIBLE_DEVICES=all
copy docker/entrypoint.sh /entrypoint.sh
entrypoint ["/entrypoint.sh"]
cmd ["skynet", "--help"]

46
docker/Dockerfile.runtime+cuda-py312
View File

@ -1,42 +1,40 @@
from nvidia/cuda:12.4.1-devel-ubuntu22.04 from nvidia/cuda:12.4.1-devel-ubuntu22.04
from python:3.12
env DEBIAN_FRONTEND=noninteractive env DEBIAN_FRONTEND=noninteractive
run apt-get update && apt-get install -y \ run apt-get update && apt-get install -y \
git \ git \
llvm \ curl \
ffmpeg \ libgl1 \
libsm6 \ libglib2.0-0 \
libxext6 \ libglu1-mesa \
ninja-build libgl1-mesa-glx
# env CC /usr/bin/clang env PATH="/opt/uv:$PATH"
# env CXX /usr/bin/clang++
#
# # install llvm10 as required by llvm-lite
# run git clone https://github.com/llvm/llvm-project.git -b llvmorg-10.0.1
# workdir /llvm-project
# # this adds a commit from 12.0.0 that fixes build on newer compilers
# run git cherry-pick -n b498303066a63a203d24f739b2d2e0e56dca70d1
# run cmake -S llvm -B build -G Ninja -DCMAKE_BUILD_TYPE=Release
# run ninja -C build install # -j8
run curl -sSL https://install.python-poetry.org | python3 - arg USER_ID=1000
arg GROUP_ID=1000
env PATH "/root/.local/bin:$PATH" run groupadd -g $GROUP_ID skynet \
&& useradd -l -u $USER_ID -g skynet -s /bin/bash skynet \
&& mkdir -p /home/skynet \
&& chown -R skynet:skynet /home/skynet
copy . /skynet run curl -LsSf https://astral.sh/uv/install.sh | env UV_UNMANAGED_INSTALL="/opt/uv" sh
workdir /skynet run chown -R skynet:skynet /opt/uv
env POETRY_VIRTUALENVS_PATH /skynet/.venv run ls /opt/uv -lah
run poetry install --with=cuda -v user skynet
workdir /root/target workdir /home/skynet
env PYTORCH_CUDA_ALLOC_CONF max_split_size_mb:128 run uv venv --python 3.12
workdir /home/skynet/target
env PYTORCH_CUDA_ALLOC_CONF=max_split_size_mb:128
env NVIDIA_VISIBLE_DEVICES=all env NVIDIA_VISIBLE_DEVICES=all
copy docker/entrypoint.sh /entrypoint.sh copy docker/entrypoint.sh /entrypoint.sh

25
docker/Dockerfile.runtime+frontend
View File

@ -1,25 +0,0 @@
from python:3.11
env DEBIAN_FRONTEND=noninteractive
run apt-get update && apt-get install -y \
git
run curl -sSL https://install.python-poetry.org | python3 -
env PATH "/root/.local/bin:$PATH"
copy . /skynet
workdir /skynet
env POETRY_VIRTUALENVS_PATH /skynet/.venv
run poetry install --with=frontend -v
workdir /root/target
copy docker/entrypoint.sh /entrypoint.sh
entrypoint ["/entrypoint.sh"]
cmd ["skynet", "--help"]

6
docker/build_docker.sh
View File

@ -1,7 +1,3 @@
docker build \ docker build \
-t guilledk/skynet:runtime-cuda-py312 \ -t guilledk/skynet:runtime-cuda-py312 \
-f docker/Dockerfile.runtime+cuda-py312 . -f docker/Dockerfile.runtime+cuda-py312 . --progress=plain
# docker build \
# -t guilledk/skynet:runtime-cuda \
# -f docker/Dockerfile.runtime+cuda-py311 .

7
docker/entrypoint.sh
View File

@ -1,8 +1,5 @@
#!/bin/sh #!/bin/sh
export VIRTUAL_ENV='/skynet/.venv' uv sync
poetry env use $VIRTUAL_ENV/bin/python
poetry install exec uv run "$@"
exec poetry run "$@"

6
docker/push_docker.sh
View File

@ -1,5 +1 @@
docker push guilledk/skynet:runtime docker push guilledk/skynet:runtime-cuda-py312
docker push guilledk/skynet:runtime-frontend
docker push guilledk/skynet:runtime-cuda
docker push guilledk/skynet:runtime-cuda-py311
docker push guilledk/skynet:runtime-cuda-py310

4276
poetry.lock generated
View File

File diff suppressed because it is too large Load Diff

2
poetry.toml
View File

@ -1,2 +0,0 @@
[virtualenvs]
in-project = true

149
pyproject.toml
View File

@ -1,78 +1,85 @@
[tool.poetry] [project]
name = 'skynet' name = "skynet"
version = '0.1a13' version = "0.1a13"
description = 'Decentralized compute platform' description = "Decentralized compute platform"
authors = ['Guillermo Rodriguez <guillermo@telos.net>'] authors = [{ name = "Guillermo Rodriguez", email = "guillermo@telos.net" }]
license = 'AGPL' requires-python = ">=3.10,<3.13"
readme = 'README.md' readme = "README.md"
license = "AGPL-3.0-or-later"
dependencies = [
"pytz~=2023.3.post1",
"trio>=0.22.2,<0.23",
"Pillow>=10.0.1,<11",
"docker>=6.1.3,<7",
"py-leap",
"toml>=0.10.2,<0.11",
"msgspec>=0.19.0,<0.20",
"numpy<2.1",
"protobuf>=5.29.3,<6",
"zstandard>=0.23.0,<0.24",
"click>=8.1.8,<9",
"httpx>=0.28.1,<0.29",
]
[tool.poetry.dependencies] [project.scripts]
python = '>=3.10,<3.13' skynet = "skynet.cli:skynet"
pytz = '^2023.3.post1' txt2img = "skynet.cli:txt2img"
trio = '^0.22.2' img2img = "skynet.cli:img2img"
Pillow = '^10.0.1' upscale = "skynet.cli:upscale"
docker = '^6.1.3' inpaint = "skynet.cli:inpaint"
py-leap = {git = 'https://github.com/guilledk/py-leap.git', rev = 'v0.1a32'}
toml = '^0.10.2'
msgspec = "^0.19.0"
numpy = "<2.1"
protobuf = "^5.29.3"
zstandard = "^0.23.0"
click = "^8.1.8"
httpx = "^0.28.1"
[tool.poetry.group.frontend] [dependency-groups]
optional = true frontend = [
"triopg>=0.6.0,<0.7",
"aiohttp>=3.8.5,<4",
"psycopg2-binary>=2.9.7,<3",
"pyTelegramBotAPI>=4.14.0,<5",
"discord.py>=2.3.2,<3",
]
dev = [
"pdbpp>=0.10.3,<0.11",
"pytest>=7.4.2,<8",
"pytest-trio>=0.8.0,<0.9",
]
cuda = [
"torch==2.5.1+cu121",
"scipy==1.15.1",
"numba==0.60.0",
"quart>=0.19.3,<0.20",
"triton==3.1.0",
"xformers>=0.0.29,<0.0.30",
"hypercorn>=0.14.4,<0.15",
"diffusers==0.32.1",
"quart-trio>=0.11.0,<0.12",
"torchvision==0.20.1+cu121",
"accelerate==0.34.0",
"transformers==4.48.0",
"huggingface-hub>=0.27.1,<0.28",
"invisible-watermark>=0.2.0,<0.3",
"bitsandbytes>=0.45.0,<0.46",
"basicsr>=1.4.2,<2",
"realesrgan>=0.3.0,<0.4",
"sentencepiece>=0.2.0",
]
[tool.poetry.group.frontend.dependencies] [tool.uv]
triopg = {version = '^0.6.0'} default-groups = [
aiohttp = {version = '^3.8.5'} "frontend",
psycopg2-binary = {version = '^2.9.7'} "dev",
pyTelegramBotAPI = {version = '^4.14.0'} "cuda",
'discord.py' = {version = '^2.3.2'} ]
[tool.poetry.group.dev] [[tool.uv.index]]
optional = true name = "torch"
url = "https://download.pytorch.org/whl/cu121"
explicit = true
[tool.poetry.group.dev.dependencies] [tool.uv.sources]
pdbpp = {version = '^0.10.3'} torch = { index = "torch" }
pytest = {version = '^7.4.2'} triton = { index = "torch" }
pytest-trio = "^0.8.0" torchvision = { index = "torch" }
py-leap = { git = "https://github.com/guilledk/py-leap.git", rev = "v0.1a32" }
[tool.poetry.group.cuda]
optional = true
[tool.poetry.group.cuda.dependencies]
torch = {version = '2.5.1+cu121', source = 'torch'}
scipy = {version = '1.15.1'}
numba = {version = '0.60.0'}
quart = {version = '^0.19.3'}
triton = {version = '3.1.0', source = 'torch'}
xformers = {version = '^0.0.29'}
hypercorn = {version = '^0.14.4'}
diffusers = {version = '0.32.1'}
quart-trio = {version = '^0.11.0'}
torchvision = {version = '0.20.1+cu121', source = 'torch'}
accelerate = {version = '0.34.0'}
transformers = {version = '4.48.0'}
huggingface-hub = {version = '^0.27.1'}
invisible-watermark = {version = '^0.2.0'}
bitsandbytes = "^0.45.0"
basicsr = "^1.4.2"
realesrgan = "^0.3.0"
[[tool.poetry.source]]
name = 'torch'
url = 'https://download.pytorch.org/whl/cu121'
priority = 'explicit'
[build-system] [build-system]
requires = ['poetry-core', 'cython'] requires = ["hatchling"]
build-backend = 'poetry.core.masonry.api' build-backend = "hatchling.build"
[tool.poetry.scripts]
skynet = 'skynet.cli:skynet'
txt2img = 'skynet.cli:txt2img'
img2img = 'skynet.cli:img2img'
upscale = 'skynet.cli:upscale'
inpaint = 'skynet.cli:inpaint'

2828
uv.lock 100644
View File

File diff suppressed because it is too large Load Diff