From 0cccdd01b50b2d5353e4a8d0b5d58b919ca3c531 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 5 Apr 2018 23:15:24 -0400 Subject: [PATCH 1/6] Only log when the network first goes down --- piker/brokers/core.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/piker/brokers/core.py b/piker/brokers/core.py index ae58c837..fac9d72c 100644 --- a/piker/brokers/core.py +++ b/piker/brokers/core.py @@ -49,15 +49,21 @@ async def quote(brokermod: ModuleType, tickers: [str]) -> dict: async def wait_for_network(get_quotes, sleep=1): """Wait until the network comes back up. """ + down = False while True: try: with trio.move_on_after(1) as cancel_scope: - return await get_quotes() + quotes = await get_quotes() + if down: + log.warn("Network is back up") + return quotes if cancel_scope.cancelled_caught: log.warn("Quote query timed out") continue except socket.gaierror: - log.warn(f"Network is down waiting for reestablishment...") + if not down: # only report/log network down once + log.warn(f"Network is down waiting for re-establishment...") + down = True await trio.sleep(sleep) From 7e92df435288e2b1786150848df7c8c2654c5b38 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 6 Apr 2018 15:00:11 -0400 Subject: [PATCH 2/6] Move built-in lists to respective module --- piker/cli.py | 26 +++----------------------- piker/watchlists.py | 6 +++++- 2 files changed, 8 insertions(+), 24 deletions(-) diff --git a/piker/cli.py b/piker/cli.py index 65377937..82fdf138 100644 --- a/piker/cli.py +++ b/piker/cli.py @@ -120,35 +120,15 @@ def watch(loglevel, broker, rate, name): from .ui.watchlist import _async_main log = get_console_log(loglevel) # activate console logging brokermod = get_brokermod(broker) - - watchlists_base = { - 'cannabis': [ - 'EMH.VN', 'LEAF.TO', 'HVT.VN', 'HMMJ.TO', 'APH.TO', - 'CBW.VN', 'TRST.CN', 'VFF.TO', 'ACB.TO', 'ABCN.VN', - 'APH.TO', 'MARI.CN', 'WMD.VN', 'LEAF.TO', 'THCX.VN', - 'WEED.TO', 'NINE.VN', 'RTI.VN', 'SNN.CN', 'ACB.TO', - 'OGI.VN', 'IMH.VN', 'FIRE.VN', 'EAT.CN', - 'WMD.VN', 'HEMP.VN', 'CALI.CN', 'RQB.CN', 'MPX.CN', - 'SEED.TO', 'HMJR.TO', 'CMED.TO', 'PAS.VN', - 'CRON', - ], - 'dad': ['GM', 'TSLA', 'DOL.TO', 'CIM', 'SPY', 'SHOP.TO'], - 'pharma': ['ATE.VN'], - 'indexes': ['SPY', 'DAX', 'QQQ', 'DIA'], - } watchlist_from_file = wl.ensure_watchlists(_watchlists_data_path) - watchlists = wl.merge_watchlist(watchlist_from_file, watchlists_base) - # broker_conf_path = os.path.join( - # click.get_app_dir('piker'), 'watchlists.json') - # from piker.testing import _quote_streamer as brokermod + watchlists = wl.merge_watchlist(watchlist_from_file, wl._builtins) broker_limit = getattr(brokermod, '_rate_limit', float('inf')) + if broker_limit < rate: rate = broker_limit log.warn(f"Limiting {brokermod.__name__} query rate to {rate}/sec") + trio.run(_async_main, name, watchlists[name], brokermod, rate) - # broker_conf_path = os.path.join( - # click.get_app_dir('piker'), 'watchlists.json') - # from piker.testing import _quote_streamer as brokermod @cli.group() diff --git a/piker/watchlists.py b/piker/watchlists.py index ae777904..0f71a49c 100644 --- a/piker/watchlists.py +++ b/piker/watchlists.py @@ -6,12 +6,16 @@ from .log import get_logger log = get_logger(__name__) +_builtins = { + 'indexes': ['SPY', 'DAX', 'QQQ', 'DIA'], +} + def write_sorted_json(watchlist, path): for key in watchlist: watchlist[key] = sorted(list(set(watchlist[key]))) with open(path, 'w') as f: - json.dump(watchlist, f, sort_keys=True) + json.dump(watchlist, f, sort_keys=True, indent=4) def make_config_dir(dir_path): From 6b72d04427d1c6821db0897f3324ff9f86d23925 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 6 Apr 2018 15:07:47 -0400 Subject: [PATCH 3/6] Change watchlist write function name --- piker/cli.py | 10 +++++----- piker/watchlists.py | 2 +- tests/test_cli.py | 4 ++-- tests/test_watchlists.py | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/piker/cli.py b/piker/cli.py index 82fdf138..94d132ee 100644 --- a/piker/cli.py +++ b/piker/cli.py @@ -159,7 +159,7 @@ def show(ctx, name): @click.pass_context def load(ctx, data): try: - wl.write_sorted_json(json.loads(data), ctx.obj['path']) + wl.write_to_file(json.loads(data), ctx.obj['path']) except (json.JSONDecodeError, IndexError): click.echo('You have passed an invalid text respresentation of a ' 'JSON object. Try again.') @@ -172,7 +172,7 @@ def load(ctx, data): def add(ctx, name, ticker_name): watchlist = wl.add_ticker(name, ticker_name, ctx.obj['watchlist']) - wl.write_sorted_json(watchlist, ctx.obj['path']) + wl.write_to_file(watchlist, ctx.obj['path']) @watchlists.command(help='remove ticker from watchlist') @@ -181,7 +181,7 @@ def add(ctx, name, ticker_name): @click.pass_context def remove(ctx, name, ticker_name): watchlist = wl.remove_ticker(name, ticker_name, ctx.obj['watchlist']) - wl.write_sorted_json(watchlist, ctx.obj['path']) + wl.write_to_file(watchlist, ctx.obj['path']) @watchlists.command(help='delete watchlist group') @@ -189,7 +189,7 @@ def remove(ctx, name, ticker_name): @click.pass_context def delete(ctx, name): watchlist = wl.delete_group(name, ctx.obj['watchlist']) - wl.write_sorted_json(watchlist, ctx.obj['path']) + wl.write_to_file(watchlist, ctx.obj['path']) @watchlists.command(help='merge a watchlist from another user') @@ -198,7 +198,7 @@ def delete(ctx, name): def merge(ctx, watchlist_to_merge): merged_watchlist = wl.merge_watchlist(json.loads(watchlist_to_merge), ctx.obj['watchlist']) - wl.write_sorted_json(merged_watchlist, ctx.obj['path']) + wl.write_to_file(merged_watchlist, ctx.obj['path']) @watchlists.command(help='dump text respresentation of a watchlist to console') diff --git a/piker/watchlists.py b/piker/watchlists.py index 0f71a49c..6eac4052 100644 --- a/piker/watchlists.py +++ b/piker/watchlists.py @@ -11,7 +11,7 @@ _builtins = { } -def write_sorted_json(watchlist, path): +def write_to_file(watchlist, path): for key in watchlist: watchlist[key] = sorted(list(set(watchlist[key]))) with open(path, 'w') as f: diff --git a/tests/test_cli.py b/tests/test_cli.py index 10e5668f..44c57f0e 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -112,7 +112,7 @@ def piker_dir(temp_dir): 'pharma': ['ATE.VN'], 'indexes': ['SPY', 'DAX', 'QQQ', 'DIA'], } - wl.write_sorted_json(watchlist, json_file_path) + wl.write_to_file(watchlist, json_file_path) yield json_file_path @@ -201,7 +201,7 @@ def test_watchlists_are_merged(capfd, piker_dir): 'pharma': ['ATE.VN', 'MALI', 'PERCOCET'], 'drugs': ['CRACK'] } - wl.write_sorted_json(orig_watchlist, piker_dir) + wl.write_to_file(orig_watchlist, piker_dir) run(f"piker watchlists -d {piker_dir} merge", list_to_merge) out = wl.ensure_watchlists(piker_dir) assert out == expected_out diff --git a/tests/test_watchlists.py b/tests/test_watchlists.py index f198860f..7245e989 100644 --- a/tests/test_watchlists.py +++ b/tests/test_watchlists.py @@ -30,7 +30,7 @@ def test_watchlist_is_sorted_no_dups_and_saved_to_file(piker_dir): wl_temp = {'test': ['TEST.CN', 'AAA'], 'AA': ['TEST.CN', 'TEST.CN'], 'AA': ['TEST.CN']} wl_sort = {'AA': ['TEST.CN'], 'test': ['AAA', 'TEST.CN']} - wl.write_sorted_json(wl_temp, piker_dir) + wl.write_to_file(wl_temp, piker_dir) temp_sorted = wl.ensure_watchlists(piker_dir) assert temp_sorted == wl_sort @@ -60,7 +60,7 @@ def test_watchlist_is_read_from_file(piker_dir): wl_temp = wl.ensure_watchlists(piker_dir) assert wl_temp == {} wl_temp2 = {"AA": ["TEST.CN"]} - wl.write_sorted_json(wl_temp2, piker_dir) + wl.write_to_file(wl_temp2, piker_dir) assert wl_temp2 == wl.ensure_watchlists(piker_dir) From 381df2815d736f08fda2856fb2e944d7569b433b Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 10 Apr 2018 14:12:06 -0400 Subject: [PATCH 4/6] Expose remove errors to caller --- piker/watchlists.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/piker/watchlists.py b/piker/watchlists.py index 6eac4052..7fc532ba 100644 --- a/piker/watchlists.py +++ b/piker/watchlists.py @@ -36,10 +36,9 @@ def add_ticker(name, ticker_name, watchlist): def remove_ticker(name, ticker_name, watchlist): - if name in watchlist: - watchlist[name].remove(str(ticker_name).upper()) - if watchlist[name] == []: - del watchlist[name] + watchlist[name].remove(str(ticker_name).upper()) + if watchlist[name] == []: + del watchlist[name] return watchlist From 577ca43c02dc195eda8b151b9f98645c9a4502fd Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 10 Apr 2018 14:13:00 -0400 Subject: [PATCH 5/6] Include built-in lists in show, log errors from remove --- piker/cli.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/piker/cli.py b/piker/cli.py index 94d132ee..171a71d8 100644 --- a/piker/cli.py +++ b/piker/cli.py @@ -149,7 +149,7 @@ def watchlists(ctx, loglevel, config_dir): @click.argument('name', nargs=1, required=False) @click.pass_context def show(ctx, name): - watchlist = ctx.obj['watchlist'] + watchlist = wl.merge_watchlist(ctx.obj['watchlist'], wl._builtins) click.echo(colorize_json( watchlist if name is None else watchlist[name])) @@ -180,8 +180,18 @@ def add(ctx, name, ticker_name): @click.argument('ticker_name', nargs=1, required=True) @click.pass_context def remove(ctx, name, ticker_name): - watchlist = wl.remove_ticker(name, ticker_name, ctx.obj['watchlist']) - wl.write_to_file(watchlist, ctx.obj['path']) + try: + watchlist = wl.remove_ticker(name, ticker_name, ctx.obj['watchlist']) + except KeyError: + log.error(f"No watchlist with name `{name}` could be found?") + except ValueError: + if name in wl._builtins and ticker_name in wl._builtins[name]: + log.error(f"Can not remove ticker `{ticker_name}` from built-in " + f"list `{name}`") + else: + log.error(f"Ticker `{ticker_name}` not found in list `{name}`") + else: + wl.write_to_file(watchlist, ctx.obj['path']) @watchlists.command(help='delete watchlist group') From 2070f292b105309cd1c1e4c13c792e52d984be7b Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 10 Apr 2018 14:13:58 -0400 Subject: [PATCH 6/6] Update tests --- tests/test_cli.py | 89 +++++++++++++++++++++------------------- tests/test_watchlists.py | 8 ++++ 2 files changed, 54 insertions(+), 43 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 44c57f0e..6ec88150 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -6,11 +6,8 @@ import subprocess import pytest import tempfile import os.path -import logging import piker.watchlists as wl -import piker.cli as cli -from piker.log import colorize_json def run(cmd, *args): @@ -104,71 +101,77 @@ def temp_dir(): @pytest.fixture -def piker_dir(temp_dir): +def ex_watchlists(): + """Made up watchlists to use for expected outputs. + """ + watchlists = { + 'dad': list(sorted(['GM', 'TSLA', 'DOL.TO', 'CIM', 'SPY', 'SHOP.TO'])), + 'pharma': ['ATE.VN'], + } + return watchlists + + +@pytest.fixture +def ex_watchlists_wbi(ex_watchlists): + """Made up watchlists + built-in list(s) to use for expected outputs. + """ + with_builtins = ex_watchlists.copy() + with_builtins.update(wl._builtins) + return with_builtins + + +@pytest.fixture +def piker_dir(temp_dir, ex_watchlists): wl.make_config_dir(temp_dir) json_file_path = os.path.join(temp_dir, 'watchlists.json') - watchlist = { - 'dad': ['GM', 'TSLA', 'DOL.TO', 'CIM', 'SPY', 'SHOP.TO'], - 'pharma': ['ATE.VN'], - 'indexes': ['SPY', 'DAX', 'QQQ', 'DIA'], - } - wl.write_to_file(watchlist, json_file_path) + # push test watchlists to file without built-ins + to_write = ex_watchlists.copy() + wl.write_to_file(to_write, json_file_path) yield json_file_path -def test_show_watchlists(capfd, piker_dir): - """Ensure a watchlist is printed. +def test_show_watchlists(capfd, piker_dir, ex_watchlists_wbi): + """Ensure all watchlists are printed as json to stdout. + + (Can't seem to get pretty formatting to work, pytest thing?) """ - expected_out = json.dumps({ - 'dad': ['CIM', 'DOL.TO', 'GM', 'SHOP.TO', 'SPY', 'TSLA'], - 'indexes': ['DAX', 'DIA', 'QQQ', 'SPY'], - 'pharma': ['ATE.VN'], - }, indent=4) + expected_out = json.dumps(ex_watchlists_wbi, indent=4, sort_keys=True) run(f"piker watchlists -d {piker_dir} show") out, err = capfd.readouterr() assert out.strip() == expected_out -def test_dump_watchlists(capfd, piker_dir): - """Ensure watchlist is dumped. +def test_dump_watchlists(capfd, piker_dir, ex_watchlists): + """Ensure watchlist is dumped without built-in lists. """ - expected_out = json.dumps({ - 'dad': ['CIM', 'DOL.TO', 'GM', 'SHOP.TO', 'SPY', 'TSLA'], - 'indexes': ['DAX', 'DIA', 'QQQ', 'SPY'], - 'pharma': ['ATE.VN'], - }) + expected_out = json.dumps(ex_watchlists) run(f"piker watchlists -d {piker_dir} dump") out, err = capfd.readouterr() assert out.strip() == expected_out -def test_ticker_added_to_watchlists(capfd, piker_dir): - expected_out = { - 'dad': ['CIM', 'DOL.TO', 'GM', 'SHOP.TO', 'SPY', 'TSLA'], - 'indexes': ['DAX', 'DIA', 'QQQ', 'SPY'], - 'pharma': ['ATE.VN', 'CRACK'], - } +def test_ticker_added_to_watchlists(capfd, piker_dir, ex_watchlists): + ex_watchlists['pharma'].append('CRACK') run(f"piker watchlists -d {piker_dir} add pharma CRACK") out = wl.ensure_watchlists(piker_dir) - assert out == expected_out + assert out == ex_watchlists -def test_ticker_removed_from_watchlists(capfd, piker_dir): - expected_out = { - 'dad': ['CIM', 'DOL.TO', 'GM', 'SHOP.TO', 'SPY', 'TSLA'], - 'indexes': ['DAX', 'DIA', 'SPY'], - 'pharma': ['ATE.VN'], - } - run(f"piker watchlists -d {piker_dir} remove indexes QQQ") +def test_ticker_removed_from_watchlists(capfd, piker_dir, ex_watchlists): + expected_out = ex_watchlists.copy() + expected_out['dad'].remove('SPY') + run(f"piker watchlists -d {piker_dir} remove dad SPY") out = wl.ensure_watchlists(piker_dir) assert out == expected_out + # removing a non-entry should be a no-op + run(f"piker watchlists -d {piker_dir} remove dad SPY") + out = wl.ensure_watchlists(piker_dir) -def test_group_deleted_from_watchlists(capfd, piker_dir): - expected_out = { - 'dad': ['CIM', 'DOL.TO', 'GM', 'SHOP.TO', 'SPY', 'TSLA'], - 'indexes': ['DAX', 'DIA', 'QQQ', 'SPY'], - } + +def test_group_deleted_from_watchlists(capfd, piker_dir, ex_watchlists): + expected_out = ex_watchlists.copy() + expected_out.pop('pharma') run(f"piker watchlists -d {piker_dir} delete pharma") out = wl.ensure_watchlists(piker_dir) assert out == expected_out diff --git a/tests/test_watchlists.py b/tests/test_watchlists.py index 7245e989..31511034 100644 --- a/tests/test_watchlists.py +++ b/tests/test_watchlists.py @@ -83,6 +83,14 @@ def test_ticker_is_removed(): assert wl_temp == {'test': ['TEST2.CN']} assert not wl_temp.get('test2') + # verify trying to remove from nonexistant list + with pytest.raises(KeyError): + wl.remove_ticker('doggy', 'TEST.CN', wl_temp) + + # verify trying to remove non-existing ticker + with pytest.raises(ValueError): + wl.remove_ticker('test', 'TEST.CN', wl_temp) + def test_group_is_deleted(): """Check that watchlist group is removed.