skygpu
/
skynet
mirror of https://github.com/skygpu/skynet.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #45 from skygpu/uv_migration 

Switch to uv package manager
network-error-patch
Guillermo Rodriguez 2025-01-22 14:42:59 -03:00 committed by GitHub
parent b5f52b3b5b 8f4429bd06
commit 2e50b1a542
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 2933 additions and 4530 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -11,3 +11,4 @@ docs
ipfs-docker-data
ipfs-staging
weights
*.png

25
docker/Dockerfile.runtime
View File

@ -1,25 +0,0 @@
from python:3.11
env DEBIAN_FRONTEND=noninteractive
run apt-get update && apt-get install -y \
git
run curl -sSL https://install.python-poetry.org | python3 -
env PATH "/root/.local/bin:$PATH"
copy . /skynet
workdir /skynet
env POETRY_VIRTUALENVS_PATH /skynet/.venv
run poetry install
workdir /root/target
copy docker/entrypoint.sh /entrypoint.sh
entrypoint ["/entrypoint.sh"]
cmd ["skynet", "--help"]

46
docker/Dockerfile.runtime+cuda-py310
View File

@ -1,46 +0,0 @@
from nvidia/cuda:11.8.0-devel-ubuntu20.04
from python:3.10
env DEBIAN_FRONTEND=noninteractive
run apt-get update && apt-get install -y \
git \
clang \
cmake \
ffmpeg \
libsm6 \
libxext6 \
ninja-build
env CC /usr/bin/clang
env CXX /usr/bin/clang++
# install llvm10 as required by llvm-lite
run git clone https://github.com/llvm/llvm-project.git -b llvmorg-10.0.1
workdir /llvm-project
# this adds a commit from 12.0.0 that fixes build on newer compilers
run git cherry-pick -n b498303066a63a203d24f739b2d2e0e56dca70d1
run cmake -S llvm -B build -G Ninja -DCMAKE_BUILD_TYPE=Release
run ninja -C build install # -j8
run curl -sSL https://install.python-poetry.org | python3 -
env PATH "/root/.local/bin:$PATH"
copy . /skynet
workdir /skynet
env POETRY_VIRTUALENVS_PATH /skynet/.venv
run poetry install --with=cuda -v
workdir /root/target
env PYTORCH_CUDA_ALLOC_CONF max_split_size_mb:128
env NVIDIA_VISIBLE_DEVICES=all
copy docker/entrypoint.sh /entrypoint.sh
entrypoint ["/entrypoint.sh"]
cmd ["skynet", "--help"]

46
docker/Dockerfile.runtime+cuda-py311
View File

@ -1,46 +0,0 @@
from nvidia/cuda:11.8.0-devel-ubuntu20.04
from python:3.11
env DEBIAN_FRONTEND=noninteractive
run apt-get update && apt-get install -y \
git \
clang \
cmake \
ffmpeg \
libsm6 \
libxext6 \
ninja-build
env CC /usr/bin/clang
env CXX /usr/bin/clang++
# install llvm10 as required by llvm-lite
run git clone https://github.com/llvm/llvm-project.git -b llvmorg-10.0.1
workdir /llvm-project
# this adds a commit from 12.0.0 that fixes build on newer compilers
run git cherry-pick -n b498303066a63a203d24f739b2d2e0e56dca70d1
run cmake -S llvm -B build -G Ninja -DCMAKE_BUILD_TYPE=Release
run ninja -C build install # -j8
run curl -sSL https://install.python-poetry.org | python3 -
env PATH "/root/.local/bin:$PATH"
copy . /skynet
workdir /skynet
env POETRY_VIRTUALENVS_PATH /skynet/.venv
run poetry install --with=cuda -v
workdir /root/target
env PYTORCH_CUDA_ALLOC_CONF max_split_size_mb:128
env NVIDIA_VISIBLE_DEVICES=all
copy docker/entrypoint.sh /entrypoint.sh
entrypoint ["/entrypoint.sh"]
cmd ["skynet", "--help"]

46
docker/Dockerfile.runtime+cuda-py312
View File

@ -1,42 +1,40 @@
from nvidia/cuda:12.4.1-devel-ubuntu22.04
from python:3.12
env DEBIAN_FRONTEND=noninteractive
run apt-get update && apt-get install -y \
git \
llvm \
ffmpeg \
libsm6 \
libxext6 \
ninja-build
curl \
libgl1 \
libglib2.0-0 \
libglu1-mesa \
libgl1-mesa-glx
# env CC /usr/bin/clang
# env CXX /usr/bin/clang++
#
# # install llvm10 as required by llvm-lite
# run git clone https://github.com/llvm/llvm-project.git -b llvmorg-10.0.1
# workdir /llvm-project
# # this adds a commit from 12.0.0 that fixes build on newer compilers
# run git cherry-pick -n b498303066a63a203d24f739b2d2e0e56dca70d1
# run cmake -S llvm -B build -G Ninja -DCMAKE_BUILD_TYPE=Release
# run ninja -C build install # -j8
env PATH="/opt/uv:$PATH"
run curl -sSL https://install.python-poetry.org | python3 -
arg USER_ID=1000
arg GROUP_ID=1000
env PATH "/root/.local/bin:$PATH"
run groupadd -g $GROUP_ID skynet \
&& useradd -l -u $USER_ID -g skynet -s /bin/bash skynet \
&& mkdir -p /home/skynet \
&& chown -R skynet:skynet /home/skynet
copy . /skynet
run curl -LsSf https://astral.sh/uv/install.sh | env UV_UNMANAGED_INSTALL="/opt/uv" sh
workdir /skynet
run chown -R skynet:skynet /opt/uv
env POETRY_VIRTUALENVS_PATH /skynet/.venv
run ls /opt/uv -lah
run poetry install --with=cuda -v
user skynet
workdir /root/target
workdir /home/skynet
env PYTORCH_CUDA_ALLOC_CONF max_split_size_mb:128
run uv venv --python 3.12
workdir /home/skynet/target
env PYTORCH_CUDA_ALLOC_CONF=max_split_size_mb:128
env NVIDIA_VISIBLE_DEVICES=all
copy docker/entrypoint.sh /entrypoint.sh

25
docker/Dockerfile.runtime+frontend
View File

@ -1,25 +0,0 @@
from python:3.11
env DEBIAN_FRONTEND=noninteractive
run apt-get update && apt-get install -y \
git
run curl -sSL https://install.python-poetry.org | python3 -
env PATH "/root/.local/bin:$PATH"
copy . /skynet
workdir /skynet
env POETRY_VIRTUALENVS_PATH /skynet/.venv
run poetry install --with=frontend -v
workdir /root/target
copy docker/entrypoint.sh /entrypoint.sh
entrypoint ["/entrypoint.sh"]
cmd ["skynet", "--help"]

6
docker/build_docker.sh
View File

@ -1,7 +1,3 @@
docker build \
-t guilledk/skynet:runtime-cuda-py312 \
-f docker/Dockerfile.runtime+cuda-py312 .
# docker build \
# -t guilledk/skynet:runtime-cuda \
# -f docker/Dockerfile.runtime+cuda-py311 .
-f docker/Dockerfile.runtime+cuda-py312 . --progress=plain

7
docker/entrypoint.sh
View File

@ -1,8 +1,5 @@
#!/bin/sh
export VIRTUAL_ENV='/skynet/.venv'
poetry env use $VIRTUAL_ENV/bin/python
uv sync
poetry install
exec poetry run "$@"
exec uv run "$@"

6
docker/push_docker.sh
View File

@ -1,5 +1 @@
docker push guilledk/skynet:runtime
docker push guilledk/skynet:runtime-frontend
docker push guilledk/skynet:runtime-cuda
docker push guilledk/skynet:runtime-cuda-py311
docker push guilledk/skynet:runtime-cuda-py310
docker push guilledk/skynet:runtime-cuda-py312

4276
poetry.lock generated
View File

File diff suppressed because it is too large Load Diff

2
poetry.toml
View File

@ -1,2 +0,0 @@
[virtualenvs]
in-project = true

149
pyproject.toml
View File

@ -1,78 +1,85 @@
[tool.poetry]
name = 'skynet'
version = '0.1a13'
description = 'Decentralized compute platform'
authors = ['Guillermo Rodriguez <guillermo@telos.net>']
license = 'AGPL'
readme = 'README.md'
[project]
name = "skynet"
version = "0.1a13"
description = "Decentralized compute platform"
authors = [{ name = "Guillermo Rodriguez", email = "guillermo@telos.net" }]
requires-python = ">=3.10,<3.13"
readme = "README.md"
license = "AGPL-3.0-or-later"
dependencies = [
"pytz~=2023.3.post1",
"trio>=0.22.2,<0.23",
"Pillow>=10.0.1,<11",
"docker>=6.1.3,<7",
"py-leap",
"toml>=0.10.2,<0.11",
"msgspec>=0.19.0,<0.20",
"numpy<2.1",
"protobuf>=5.29.3,<6",
"zstandard>=0.23.0,<0.24",
"click>=8.1.8,<9",
"httpx>=0.28.1,<0.29",
]
[tool.poetry.dependencies]
python = '>=3.10,<3.13'
pytz = '^2023.3.post1'
trio = '^0.22.2'
Pillow = '^10.0.1'
docker = '^6.1.3'
py-leap = {git = 'https://github.com/guilledk/py-leap.git', rev = 'v0.1a32'}
toml = '^0.10.2'
msgspec = "^0.19.0"
numpy = "<2.1"
protobuf = "^5.29.3"
zstandard = "^0.23.0"
click = "^8.1.8"
httpx = "^0.28.1"
[project.scripts]
skynet = "skynet.cli:skynet"
txt2img = "skynet.cli:txt2img"
img2img = "skynet.cli:img2img"
upscale = "skynet.cli:upscale"
inpaint = "skynet.cli:inpaint"
[tool.poetry.group.frontend]
optional = true
[dependency-groups]
frontend = [
"triopg>=0.6.0,<0.7",
"aiohttp>=3.8.5,<4",
"psycopg2-binary>=2.9.7,<3",
"pyTelegramBotAPI>=4.14.0,<5",
"discord.py>=2.3.2,<3",
]
dev = [
"pdbpp>=0.10.3,<0.11",
"pytest>=7.4.2,<8",
"pytest-trio>=0.8.0,<0.9",
]
cuda = [
"torch==2.5.1+cu121",
"scipy==1.15.1",
"numba==0.60.0",
"quart>=0.19.3,<0.20",
"triton==3.1.0",
"xformers>=0.0.29,<0.0.30",
"hypercorn>=0.14.4,<0.15",
"diffusers==0.32.1",
"quart-trio>=0.11.0,<0.12",
"torchvision==0.20.1+cu121",
"accelerate==0.34.0",
"transformers==4.48.0",
"huggingface-hub>=0.27.1,<0.28",
"invisible-watermark>=0.2.0,<0.3",
"bitsandbytes>=0.45.0,<0.46",
"basicsr>=1.4.2,<2",
"realesrgan>=0.3.0,<0.4",
"sentencepiece>=0.2.0",
]
[tool.poetry.group.frontend.dependencies]
triopg = {version = '^0.6.0'}
aiohttp = {version = '^3.8.5'}
psycopg2-binary = {version = '^2.9.7'}
pyTelegramBotAPI = {version = '^4.14.0'}
'discord.py' = {version = '^2.3.2'}
[tool.uv]
default-groups = [
"frontend",
"dev",
"cuda",
]
[tool.poetry.group.dev]
optional = true
[[tool.uv.index]]
name = "torch"
url = "https://download.pytorch.org/whl/cu121"
explicit = true
[tool.poetry.group.dev.dependencies]
pdbpp = {version = '^0.10.3'}
pytest = {version = '^7.4.2'}
pytest-trio = "^0.8.0"
[tool.poetry.group.cuda]
optional = true
[tool.poetry.group.cuda.dependencies]
torch = {version = '2.5.1+cu121', source = 'torch'}
scipy = {version = '1.15.1'}
numba = {version = '0.60.0'}
quart = {version = '^0.19.3'}
triton = {version = '3.1.0', source = 'torch'}
xformers = {version = '^0.0.29'}
hypercorn = {version = '^0.14.4'}
diffusers = {version = '0.32.1'}
quart-trio = {version = '^0.11.0'}
torchvision = {version = '0.20.1+cu121', source = 'torch'}
accelerate = {version = '0.34.0'}
transformers = {version = '4.48.0'}
huggingface-hub = {version = '^0.27.1'}
invisible-watermark = {version = '^0.2.0'}
bitsandbytes = "^0.45.0"
basicsr = "^1.4.2"
realesrgan = "^0.3.0"
[[tool.poetry.source]]
name = 'torch'
url = 'https://download.pytorch.org/whl/cu121'
priority = 'explicit'
[tool.uv.sources]
torch = { index = "torch" }
triton = { index = "torch" }
torchvision = { index = "torch" }
py-leap = { git = "https://github.com/guilledk/py-leap.git", rev = "v0.1a32" }
[build-system]
requires = ['poetry-core', 'cython']
build-backend = 'poetry.core.masonry.api'
[tool.poetry.scripts]
skynet = 'skynet.cli:skynet'
txt2img = 'skynet.cli:txt2img'
img2img = 'skynet.cli:img2img'
upscale = 'skynet.cli:upscale'
inpaint = 'skynet.cli:inpaint'
requires = ["hatchling"]
build-backend = "hatchling.build"

2828
uv.lock 100644
View File

File diff suppressed because it is too large Load Diff