diff --git a/skynet/cli.py b/skynet/cli.py index 6e4db55..03b36ae 100644 --- a/skynet/cli.py +++ b/skynet/cli.py @@ -1,6 +1,7 @@ #!/usr/bin/python import os +import time import json import logging import random @@ -376,21 +377,23 @@ def telegram( @run.command() -@click.option('--loglevel', '-l', default='warning', help='logging level') +@click.option('--loglevel', '-l', default='INFO', help='logging level') @click.option( '--container', '-c', default='ipfs_host') @click.option( '--hyperion-url', '-n', default='http://127.0.0.1:42001') -def pinner(loglevel, container): +def pinner(loglevel, container, hyperion_url): + logging.basicConfig(level=loglevel) dclient = docker.from_env() - container = dclient.containers.get(conatiner) + container = dclient.containers.get(container) ipfs_node = IPFSDocker(container) + hyperion = HyperionAPI(hyperion_url) last_pinned: dict[str, datetime] = {} def cleanup_pinned(now: datetime): - for cid in last_pinned.keys(): + for cid in set(last_pinned.keys()): ts = last_pinned[cid] if now - ts > timedelta(minutes=1): del last_pinned[cid] @@ -398,7 +401,7 @@ def pinner(loglevel, container): try: while True: # get all submits in the last minute - now = dateimte.now() + now = datetime.now() half_min_ago = now - timedelta(seconds=30) submits = hyperion.get_actions( account='telos.gpu', @@ -422,7 +425,11 @@ def pinner(loglevel, container): ipfs_node.pin(cid) + logging.info(f'pinned {cid}') + cleanup_pinned(now) + time.sleep(1) + except KeyboardInterrupt: ... diff --git a/skynet/ipfs.py b/skynet/ipfs.py index 1e56b52..002622c 100644 --- a/skynet/ipfs.py +++ b/skynet/ipfs.py @@ -42,6 +42,14 @@ class IPFSDocker: def open_ipfs_node(): dclient = docker.from_env() + staging_dir = (Path().resolve() / 'ipfs-docker-staging').mkdir( + parents=True, exist_ok=True) + data_dir = (Path().resolve() / 'ipfs-docker-data').mkdir( + parents=True, exist_ok=True) + + export_target = '/export' + data_target = '/data/ipfs' + container = dclient.containers.run( 'ipfs/go-ipfs:latest', name='skynet-ipfs', @@ -51,25 +59,17 @@ def open_ipfs_node(): '5001/tcp': ('127.0.0.1', 5001) }, mounts=[ - Mount( - '/export', - str(Path().resolve() / 'tmp/ipfs-docker-staging'), - 'bind' - ), - Mount( - '/data/ipfs', - str(Path().resolve() / 'tmp/ipfs-docker-data'), - 'bind' - ) + Mount(export_target, str(staging_dir), 'bind'), + Mount(data_target, str(data_dir), 'bind') ], detach=True, remove=True ) uid = os.getuid() gid = os.getgid() - ec, out = container.exec_run(['chown', f'{uid}:{gid}', '-R', '/export']) + ec, out = container.exec_run(['chown', f'{uid}:{gid}', '-R', export_target]) assert ec == 0 - ec, out = container.exec_run(['chown', f'{uid}:{gid}', '-R', '/data/ipfs']) + ec, out = container.exec_run(['chown', f'{uid}:{gid}', '-R', data_target]) assert ec == 0 try: