skygpu
/
skynet
mirror of https://github.com/skygpu/skynet.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Improve worker ipfs input data parallelizm

pull/26/head
Guillermo Rodriguez 2023-10-08 09:54:31 -03:00
parent aa1d52dba0
commit d749dc4f57
No known key found for this signature in database
GPG Key ID: EC3AB66D5D83B392
1 changed files with 22 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
skynet/dgpu/network.py
View File

@ -12,7 +12,7 @@ import asks
import trio
import anyio
from PIL import Image
from PIL import Image, UnidentifiedImageError
from leap.cleos import CLEOS
from leap.sugar import Checksum256, Name, asset_from_str
@ -263,43 +263,35 @@ class SkynetGPUConnector:
ipfs_link = f'https://ipfs.{DEFAULT_DOMAIN}/ipfs/{ipfs_hash}'
ipfs_link_legacy = ipfs_link + '/image.png'
async def get_and_set_results(link: str):
results[link] = await get_ipfs_file(link)
def get_image_from_resp(resp):
with Image.open(io.BytesIO(resp.raw)):
return resp.raw
async with trio.open_nursery() as n:
async def get_and_set_results(link: str):
res = await get_ipfs_file(link, timeout=1)
logging.info(f'got response from {link}')
if not res or res.status_code != 200:
logging.warning(f'couldn\'t get ipfs binary data at {link}!')
else:
try:
with Image.open(io.BytesIO(res.raw)):
results[link] = res.raw
n.cancel_scope.cancel()
except UnidentifiedImageError:
logging.warning(f'couldn\'t get ipfs binary data at {link}!')
n.start_soon(
get_and_set_results, ipfs_link)
n.start_soon(
get_and_set_results, ipfs_link_legacy)
png_img = None
resp = results[ipfs_link_legacy]
if not resp or resp.status_code != 200:
logging.warning(f'couldn\'t get ipfs binary data at {ipfs_link_legacy}!')
if ipfs_link_legacy in results:
png_img = results[ipfs_link_legacy]
else:
try:
png_img = get_image_from_resp(resp)
except UnidentifiedImageError:
logging.warning(f'couldn\'t get ipfs binary data at {ipfs_link_legacy}!')
if not png_img:
resp = results[ipfs_link]
if not resp or resp.status_code != 200:
logging.warning(f'couldn\'t get ipfs binary data at {ipfs_link}!')
else:
try:
png_img = get_image_from_resp(resp)
except UnidentifiedImageError:
logging.warning(f'couldn\'t get ipfs binary data at {ipfs_link}!')
...
png_img = None
if ipfs_link in results:
png_img = results[ipfs_link]
if not png_img:
raise DGPUComputeError('Couldn\'t gather input data from ipfs')