From 93c81fa4d1c4dc877dde0c81a5664e78cb4db635 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 8 Mar 2023 15:14:39 -0500 Subject: [PATCH] Start `piker.service` sub-package For now just moves everything that was in `piker._daemon` to a subpkg module but a reorg is coming pronto! --- piker/__init__.py | 8 ++++---- piker/brokers/cli.py | 21 +++++++++++++-------- piker/brokers/core.py | 2 +- piker/clearing/_client.py | 7 +++++-- piker/cli/__init__.py | 14 ++++++++------ piker/data/_ahab.py | 14 +++++++++----- piker/data/_sampling.py | 8 ++++---- piker/data/cli.py | 4 ++-- piker/data/feed.py | 2 +- piker/{_daemon.py => service/__init__.py} | 13 ++++++------- piker/ui/_app.py | 2 +- piker/ui/_exec.py | 2 +- piker/ui/cli.py | 2 +- tests/conftest.py | 12 ++++++------ tests/test_databases.py | 2 +- tests/test_services.py | 3 +-- 16 files changed, 64 insertions(+), 52 deletions(-) rename piker/{_daemon.py => service/__init__.py} (99%) diff --git a/piker/__init__.py b/piker/__init__.py index d08c2dbc..6ebeec3d 100644 --- a/piker/__init__.py +++ b/piker/__init__.py @@ -1,5 +1,5 @@ # piker: trading gear for hackers. -# Copyright 2020-eternity Tyler Goodlet (in stewardship for piker0) +# Copyright 2020-eternity Tyler Goodlet (in stewardship for pikers) # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -14,11 +14,11 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -""" +''' piker: trading gear for hackers. -""" -from ._daemon import open_piker_runtime +''' +from .service import open_piker_runtime from .data.feed import open_feed __all__ = [ diff --git a/piker/brokers/cli.py b/piker/brokers/cli.py index 0d84384d..f86c679e 100644 --- a/piker/brokers/cli.py +++ b/piker/brokers/cli.py @@ -29,8 +29,15 @@ import tractor from ..cli import cli from .. import watchlists as wl from ..log import get_console_log, colorize_json, get_logger -from .._daemon import maybe_spawn_brokerd, maybe_open_pikerd -from ..brokers import core, get_brokermod, data +from ..service import ( + maybe_spawn_brokerd, + maybe_open_pikerd, +) +from ..brokers import ( + core, + get_brokermod, + data, +) log = get_logger('cli') DEFAULT_BROKER = 'questrade' @@ -60,6 +67,7 @@ def get_method(client, meth_name: str): print_ok('found!.') return method + async def run_method(client, meth_name: str, **kwargs): method = get_method(client, meth_name) print('running...', end='', flush=True) @@ -67,19 +75,20 @@ async def run_method(client, meth_name: str, **kwargs): print_ok(f'done! result: {type(result)}') return result + async def run_test(broker_name: str): brokermod = get_brokermod(broker_name) total = 0 passed = 0 failed = 0 - print(f'getting client...', end='', flush=True) + print('getting client...', end='', flush=True) if not hasattr(brokermod, 'get_client'): print_error('fail! no \'get_client\' context manager found.') return async with brokermod.get_client(is_brokercheck=True) as client: - print_ok(f'done! inside client context.') + print_ok('done! inside client context.') # check for methods present on brokermod method_list = [ @@ -130,7 +139,6 @@ async def run_test(broker_name: str): total += 1 - # check for methods present con brokermod.Client and their # results @@ -180,7 +188,6 @@ def brokercheck(config, broker): trio.run(run_test, broker) - @cli.command() @click.option('--keys', '-k', multiple=True, help='Return results only for these keys') @@ -335,8 +342,6 @@ def contracts(ctx, loglevel, broker, symbol, ids): brokermod = get_brokermod(broker) get_console_log(loglevel) - - contracts = trio.run(partial(core.contracts, brokermod, symbol)) if not ids: # just print out expiry dates which can be used with diff --git a/piker/brokers/core.py b/piker/brokers/core.py index af5da3a1..3e9e1614 100644 --- a/piker/brokers/core.py +++ b/piker/brokers/core.py @@ -28,7 +28,7 @@ import trio from ..log import get_logger from . import get_brokermod -from .._daemon import maybe_spawn_brokerd +from ..service import maybe_spawn_brokerd from .._cacheables import open_cached_client diff --git a/piker/clearing/_client.py b/piker/clearing/_client.py index 0a40b548..7d03406a 100644 --- a/piker/clearing/_client.py +++ b/piker/clearing/_client.py @@ -29,8 +29,11 @@ from tractor.trionics import broadcast_receiver from ..log import get_logger from ..data.types import Struct -from .._daemon import maybe_open_emsd -from ._messages import Order, Cancel +from ..service import maybe_open_emsd +from ._messages import ( + Order, + Cancel, +) from ..brokers import get_brokermod if TYPE_CHECKING: diff --git a/piker/cli/__init__.py b/piker/cli/__init__.py index 9b6f225c..b4d13505 100644 --- a/piker/cli/__init__.py +++ b/piker/cli/__init__.py @@ -19,16 +19,18 @@ CLI commons. ''' import os -from pprint import pformat -from functools import partial import click import trio import tractor -from ..log import get_console_log, get_logger, colorize_json +from ..log import ( + get_console_log, + get_logger, + colorize_json, +) from ..brokers import get_brokermod -from .._daemon import ( +from ..service import ( _default_registry_host, _default_registry_port, ) @@ -68,7 +70,7 @@ def pikerd( ''' - from .._daemon import open_pikerd + from ..service import open_pikerd log = get_console_log(loglevel) if pdb: @@ -171,7 +173,7 @@ def cli( @click.pass_obj def services(config, tl, ports): - from .._daemon import ( + from ..service import ( open_piker_runtime, _default_registry_port, _default_registry_host, diff --git a/piker/data/_ahab.py b/piker/data/_ahab.py index 2c0230f1..38d4a9e7 100644 --- a/piker/data/_ahab.py +++ b/piker/data/_ahab.py @@ -426,11 +426,15 @@ async def open_ahabd( finally: # TODO: ensure loglevel can be set and teardown logs are # reported if possible on error or cancel.. - with trio.CancelScope(shield=True): - await cntr.cancel( - log_msg_key=conf['log_msg_key'], - stop_predicate=stop_lambda, - ) + # XXX WARNING: currently shielding here can result in hangs + # on ctl-c from user.. ideally we can avoid a cancel getting + # consumed and not propagating whilst still doing teardown + # logging.. + # with trio.CancelScope(shield=True): + await cntr.cancel( + log_msg_key=conf['log_msg_key'], + stop_predicate=stop_lambda, + ) async def start_ahab( diff --git a/piker/data/_sampling.py b/piker/data/_sampling.py index a5df96cc..f44304bf 100644 --- a/piker/data/_sampling.py +++ b/piker/data/_sampling.py @@ -42,7 +42,7 @@ from ..log import ( get_logger, get_console_log, ) -from .._daemon import maybe_spawn_daemon +from ..service import maybe_spawn_daemon if TYPE_CHECKING: from ._sharedmem import ( @@ -68,8 +68,8 @@ class Sampler: This non-instantiated type is meant to be a singleton within a `samplerd` actor-service spawned once by the user wishing to - time-step sample real-time quote feeds, see - ``._daemon.maybe_open_samplerd()`` and the below + time-step-sample (real-time) quote feeds, see + ``.service.maybe_open_samplerd()`` and the below ``register_with_sampler()``. ''' @@ -379,7 +379,7 @@ async def spawn_samplerd( update and increment count write and stream broadcasting. ''' - from piker._daemon import Services + from piker.service import Services dname = 'samplerd' log.info(f'Spawning `{dname}`') diff --git a/piker/data/cli.py b/piker/data/cli.py index 994b9da4..7c8b9a68 100644 --- a/piker/data/cli.py +++ b/piker/data/cli.py @@ -137,7 +137,7 @@ def storesh( ''' from piker.data.marketstore import open_tsdb_client - from piker._daemon import open_piker_runtime + from piker.service import open_piker_runtime async def main(): nonlocal symbols @@ -187,7 +187,7 @@ def storage( ''' from piker.data.marketstore import open_tsdb_client - from piker._daemon import open_piker_runtime + from piker.service import open_piker_runtime async def main(): nonlocal symbols diff --git a/piker/data/feed.py b/piker/data/feed.py index 906f4bb4..a31e955a 100644 --- a/piker/data/feed.py +++ b/piker/data/feed.py @@ -58,7 +58,7 @@ from ..log import ( get_logger, get_console_log, ) -from .._daemon import ( +from ..service import ( maybe_spawn_brokerd, check_for_service, ) diff --git a/piker/_daemon.py b/piker/service/__init__.py similarity index 99% rename from piker/_daemon.py rename to piker/service/__init__.py index 1a8576a1..6788b764 100644 --- a/piker/_daemon.py +++ b/piker/service/__init__.py @@ -19,6 +19,8 @@ Structured, daemon tree service management. """ from __future__ import annotations +from pprint import pformat +from functools import partial import os from typing import ( Optional, @@ -35,14 +37,11 @@ import tractor import trio from trio_typing import TaskStatus -from .log import ( +from ..log import ( get_logger, get_console_log, ) -from .brokers import get_brokermod - -from pprint import pformat -from functools import partial +from ..brokers import get_brokermod log = get_logger(__name__) @@ -669,7 +668,7 @@ async def spawn_brokerd( ) # non-blocking setup of brokerd service nursery - from .data import _setup_persistent_brokerd + from ..data import _setup_persistent_brokerd await Services.start_service_task( dname, @@ -732,7 +731,7 @@ async def spawn_emsd( ) # non-blocking setup of clearing service - from .clearing._ems import _setup_persistent_emsd + from ..clearing._ems import _setup_persistent_emsd await Services.start_service_task( 'emsd', diff --git a/piker/ui/_app.py b/piker/ui/_app.py index 3be073e7..9978dbe3 100644 --- a/piker/ui/_app.py +++ b/piker/ui/_app.py @@ -24,7 +24,7 @@ from types import ModuleType from PyQt5.QtCore import QEvent import trio -from .._daemon import maybe_spawn_brokerd +from ..service import maybe_spawn_brokerd from . import _event from ._exec import run_qtractor from ..data.feed import install_brokerd_search diff --git a/piker/ui/_exec.py b/piker/ui/_exec.py index d8eabb70..19663cac 100644 --- a/piker/ui/_exec.py +++ b/piker/ui/_exec.py @@ -49,7 +49,7 @@ from qdarkstyle import DarkPalette import trio from outcome import Error -from .._daemon import ( +from ..service import ( maybe_open_pikerd, get_tractor_runtime_kwargs, ) diff --git a/piker/ui/cli.py b/piker/ui/cli.py index a72c2f5c..9b8385f2 100644 --- a/piker/ui/cli.py +++ b/piker/ui/cli.py @@ -24,7 +24,7 @@ import tractor from ..cli import cli from .. import watchlists as wl -from .._daemon import maybe_spawn_brokerd +from ..service import maybe_spawn_brokerd _config_dir = click.get_app_dir('piker') diff --git a/tests/conftest.py b/tests/conftest.py index 8218ec16..68d392aa 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,7 +1,6 @@ from contextlib import asynccontextmanager as acm from functools import partial import os -from typing import AsyncContextManager from pathlib import Path from shutil import rmtree @@ -11,7 +10,7 @@ from piker import ( # log, config, ) -from piker._daemon import ( +from piker.service import ( Services, ) from piker.clearing._client import open_ems @@ -88,7 +87,7 @@ async def _open_test_pikerd( ''' import random - from piker._daemon import maybe_open_pikerd + from piker.service import maybe_open_pikerd if reg_addr is None: port = random.randint(6e3, 7e3) @@ -151,8 +150,9 @@ async def _open_test_pikerd_and_ems( fqsn, mode=mode, loglevel=loglevel, - ) as ems_services): - yield (services, ems_services) + ) as ems_services, + ): + yield (services, ems_services) @pytest.fixture @@ -168,7 +168,7 @@ def open_test_pikerd_and_ems( mode, loglevel, open_test_pikerd - ) + ) @pytest.fixture(scope='module') diff --git a/tests/test_databases.py b/tests/test_databases.py index 4eb444f3..7fcee34a 100644 --- a/tests/test_databases.py +++ b/tests/test_databases.py @@ -3,7 +3,7 @@ import trio from typing import AsyncContextManager -from piker._daemon import Services +from piker.service import Services from piker.log import get_logger from elasticsearch import Elasticsearch diff --git a/tests/test_services.py b/tests/test_services.py index 763b438e..29e613e3 100644 --- a/tests/test_services.py +++ b/tests/test_services.py @@ -9,8 +9,7 @@ import pytest import trio import tractor -from piker.log import get_logger -from piker._daemon import ( +from piker.service import ( find_service, Services, )