From 17a4fe4b2fa0c82a114979242b760ae02b42fac7 Mon Sep 17 00:00:00 2001 From: Guillermo Rodriguez Date: Wed, 15 Feb 2023 15:00:57 -0300 Subject: [PATCH] Trim unnecesary stuff left from marketstore copy, also fix elastic config name for docker build, add elasticsearch to dependencies --- dockering/elastic/Dockerfile | 2 +- .../{elasticsearch.yaml => elasticsearch.yml} | 0 piker/data/elasticsearch.py | 191 +----------------- setup.py | 5 +- 4 files changed, 16 insertions(+), 182 deletions(-) rename dockering/elastic/{elasticsearch.yaml => elasticsearch.yml} (100%) diff --git a/dockering/elastic/Dockerfile b/dockering/elastic/Dockerfile index 091b8d08..16f84cb2 100644 --- a/dockering/elastic/Dockerfile +++ b/dockering/elastic/Dockerfile @@ -8,4 +8,4 @@ COPY elasticsearch.yml /usr/share/elasticsearch/config/ RUN printf "password" | ./bin/elasticsearch-keystore add -f -x "bootstrap.password" -EXPOSE 9200 \ No newline at end of file +EXPOSE 9200 diff --git a/dockering/elastic/elasticsearch.yaml b/dockering/elastic/elasticsearch.yml similarity index 100% rename from dockering/elastic/elasticsearch.yaml rename to dockering/elastic/elasticsearch.yml diff --git a/piker/data/elasticsearch.py b/piker/data/elasticsearch.py index 6fa9c1d6..28028350 100644 --- a/piker/data/elasticsearch.py +++ b/piker/data/elasticsearch.py @@ -65,49 +65,6 @@ _config = { 'log_level': 'debug', } -_yaml_config = ''' -# piker's ``elasticsearch`` config. - -# mount this config using: -# sudo docker run \ - # -itd \ - # --rm \ - # --network=host \ - # --mount type=bind,source="$(pwd)"/elastic,target=/usr/share/elasticsearch/data \ - # --env "elastic_username=elastic" \ - # --env "elastic_password=password" \ - # --env "xpack.security.enabled=false" \ - # elastic - -root_directory: data -port: {port} -log_level: {log_level} -queryable: true -stop_grace_period: 0 -wal_rotate_interval: 5 -stale_threshold: 5 -enable_add: true -enable_remove: false - -triggers: - - module: ondiskagg.so - on: "*/1Sec/OHLCV" - config: - # filter: "nasdaq" - destinations: - - 1Min - - 5Min - - 15Min - - 1H - - 1D - - - module: stream.so - on: '*/*/*' - # config: - # filter: "nasdaq" - -'''.format(**_config) - def start_elasticsearch( client: docker.DockerClient, @@ -122,156 +79,30 @@ def start_elasticsearch( The equivalent cli cmd to this code is: sudo docker run \ - -itd \ - --rm \ - --network=host \ - --mount type=bind,source="$(pwd)"/elastic,target=/usr/share/elasticsearch/data \ - --env "elastic_username=elastic" \ - --env "elastic_password=password" \ - --env "xpack.security.enabled=false" \ - elastic + -itd \ + --rm \ + --network=host \ + --mount type=bind,source="$(pwd)"/elastic,target=/usr/share/elasticsearch/data \ + --env "elastic_username=elastic" \ + --env "elastic_password=password" \ + --env "xpack.security.enabled=false" \ + elastic ''' - import os import docker - from .. import config get_console_log('info', name=__name__) - esdir = os.path.join(config._config_dir, 'elasticsearch') - - # create dirs when dne - if not os.path.isdir(config._config_dir): - os.mkdir(config._config_dir) - - if not os.path.isdir(esdir): - os.mkdir(esdir) - - yml_file = os.path.join(esdir, 'es.yml') - if not os.path.isfile(yml_file): - log.warning( - f'No `elasticsearch` config exists?: {yml_file}\n' - 'Generating new file from template:\n' - f'{_yaml_config}\n' - ) - with open(yml_file, 'w') as yf: - yf.write(_yaml_config) - - # create a mount from user's local piker config dir into container - config_dir_mnt = docker.types.Mount( - target='/etc', - source=esdir, - type='bind', - ) - - # create a user config subdir where the elasticsearch - # backing filesystem database can be persisted. - persistent_data_dir = os.path.join( - esdir, 'data', - ) - if not os.path.isdir(persistent_data_dir): - os.mkdir(persistent_data_dir) - - data_dir_mnt = docker.types.Mount( - target='/data', - source=persistent_data_dir, - type='bind', - ) - dcntr: DockerContainer = client.containers.run( 'elastic', - ports={ - '9200':9200, - }, - mounts=[ - config_dir_mnt, - data_dir_mnt, - ], - # log_config=log_cf, + network='host', detach=True, - # stop_signal='SIGINT', - # init=True, - # remove=True, + remove=True, ) return ( dcntr, - _config, - + {}, # expected startup and stop msgs "launching listener for all services...", "exiting...", ) - - -_tick_tbk_ids: tuple[str, str] = ('1Sec', 'TICK') -_tick_tbk: str = '{}/' + '/'.join(_tick_tbk_ids) - -_tick_dt = [ - # these two are required for as a "primary key" - ('Epoch', 'i8'), - ('Nanoseconds', 'i4'), - ('IsTrade', 'i1'), - ('IsBid', 'i1'), - ('Price', 'f4'), - ('Size', 'f4') -] - -_quote_dt = [ - # these two are required for as a "primary key" - ('Epoch', 'i8'), - ('Nanoseconds', 'i4'), - - ('Tick', 'i4'), # (-1, 0, 1) = (on bid, same, on ask) - # ('fill_time', 'f4'), - ('Last', 'f4'), - ('Bid', 'f4'), - ('Bsize', 'i8'), - ('Asize', 'i8'), - ('Ask', 'f4'), - ('Size', 'i8'), - ('Volume', 'i8'), - # ('brokerd_ts', 'i64'), - # ('VWAP', 'f4') -] - -_quote_tmp = {}.fromkeys(dict(_quote_dt).keys(), np.nan) -_tick_map = { - 'Up': 1, - 'Equal': 0, - 'Down': -1, - None: np.nan, -} - -_ohlcv_dt = [ - # these two are required for as a "primary key" - ('Epoch', 'i8'), - # ('Nanoseconds', 'i4'), - - # ohlcv sampling - ('Open', 'f4'), - ('High', 'f4'), - ('Low', 'f4'), - ('Close', 'f4'), - ('Volume', 'f4'), -] - - -ohlc_key_map = bidict({ - 'Epoch': 'time', - 'Open': 'open', - 'High': 'high', - 'Low': 'low', - 'Close': 'close', - 'Volume': 'volume', -}) - -# # map of seconds ints to "time frame" accepted keys -tf_in_1s = bidict({ - 1: '1Sec', - 60: '1Min', - 60*5: '5Min', - 60*15: '15Min', - 60*30: '30Min', - 60*60: '1H', - 60*60*24: '1D', -}) diff --git a/setup.py b/setup.py index bd6363c5..2a686cc5 100755 --- a/setup.py +++ b/setup.py @@ -85,7 +85,10 @@ setup( 'tsdb': [ 'docker', ], - + 'es': [ + 'docker', + 'elasticsearch' + ] }, tests_require=['pytest'], python_requires=">=3.10",