# piker: trading gear for hackers
# Copyright (C) 2018-present  Tyler Goodlet (in stewardship of pikers)

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <https://www.gnu.org/licenses/>.

"""
Storage middle-ware CLIs.

"""
from __future__ import annotations
from datetime import (
    UTC,
    datetime,
)
import json
from pathlib import Path
import sys
import time
from types import ModuleType
from typing import (
    Annotated,
    TYPE_CHECKING,
)

import polars as pl
import numpy as np
import tractor
# import pendulum
from rich.console import Console
from rich.table import Table
import trio
# from rich.markdown import Markdown
import typer

import piker as piker_pkg
from piker.service import open_piker_runtime
from piker.cli import cli
from tractor.ipc._shm import ShmArray
from piker import tsp
from piker import config
from . import log
from . import (
    open_storage_client,
    StorageClient,
)
from ._audit import audit_ohlcv_parquet
from .nativedb import (
    iter_native_series,
    mk_ohlcv_shm_keyed_filepath,
    NativeSeriesRef,
)

if TYPE_CHECKING:
    from piker.ui._remote_ctl import AnnotClient


store = typer.Typer(no_args_is_help=True)
_shm_periods: dict[str, int] = {
    'hist': 60,
    'rt': 1,
}


def _format_utc(timestamp: float) -> str|None:
    '''
    Format a timestamp without aborting on corrupt epochs.

    '''
    try:
        return datetime.fromtimestamp(
            timestamp,
            tz=UTC,
        ).isoformat()
    except (
        OSError,
        OverflowError,
        ValueError,
    ):
        return None


def _shm_period_and_invalid_count(
    times: np.ndarray,

) -> tuple[float|None, int]:
    '''
    Infer cadence without treating unpublished slots as samples.

    '''
    valid: np.ndarray = (
        np.isfinite(times)
        &
        (times > 0)
    )
    invalid_count: int = int(np.count_nonzero(~valid))
    published: np.ndarray = times[valid]
    if published.size < 2:
        return None, invalid_count

    steps: np.ndarray = np.diff(published)
    positive_steps: np.ndarray = steps[steps > 0]
    if not positive_steps.size:
        return None, invalid_count

    periods, counts = np.unique(
        positive_steps,
        return_counts=True,
    )
    period: float = float(periods[np.argmax(counts)])
    return period, invalid_count


def _summarize_shm_frame(
    ref: tsp.ShmBufferRef,
    frame: np.ndarray,
    max_gaps: int,

) -> dict:
    '''
    Summarize one immutable OHLCV SHM snapshot.

    '''
    times: np.ndarray = frame['time']
    observed_period_s, invalid_count = (
        _shm_period_and_invalid_count(times)
    )
    period_s: int = _shm_periods[ref.kind]
    valid: np.ndarray = (
        np.isfinite(times)
        &
        (times > 0)
    )
    published: np.ndarray = times[valid]
    steps: np.ndarray = np.diff(published)
    duplicate_steps: int = int(np.count_nonzero(steps == 0))
    reversed_steps: int = int(np.count_nonzero(steps < 0))
    gaps: list[dict] = []
    if published.size > 1:
        indexes: np.ndarray = np.flatnonzero(steps > period_s)
        ranked: list[int] = sorted(
            indexes.tolist(),
            key=lambda index: steps[index],
            reverse=True,
        )
        for index in ranked[:max_gaps]:
            left: float = float(published[index])
            right: float = float(published[index + 1])
            gaps.append({
                'delta_s': float(steps[index]),
                'left_timestamp': left,
                'left_utc': _format_utc(left),
                'right_timestamp': right,
                'right_utc': _format_utc(right),
            })

    invalid: np.ndarray = np.flatnonzero(~valid)
    return {
        'shm_name': ref.path.name,
        'service': ref.service,
        'generation': ref.generation,
        'fqme': ref.fqme,
        'kind': ref.kind,
        'rows': len(frame),
        'first_index': (
            int(frame['index'][0])
            if len(frame)
            else None
        ),
        'last_index': (
            int(frame['index'][-1])
            if len(frame)
            else None
        ),
        'first_timestamp': (
            float(published[0])
            if published.size
            else None
        ),
        'last_timestamp': (
            float(published[-1])
            if published.size
            else None
        ),
        'period_s': period_s,
        'observed_period_s': observed_period_s,
        'invalid_count': invalid_count,
        'duplicate_step_count': duplicate_steps,
        'reversed_step_count': reversed_steps,
        'invalid_index_bounds': (
            [
                int(frame['index'][invalid[0]]),
                int(frame['index'][invalid[-1]]),
            ]
            if invalid.size
            else None
        ),
        'gap_count': len(np.flatnonzero(steps > period_s)),
        'largest_gaps': gaps,
    }


def _render_audit_report(report: dict) -> None:
    '''
    Render a compact human summary and explicit gap endpoints.

    '''
    result: dict = report['result']
    timestamps: dict = report['timestamps']
    gaps: dict = report['gaps']
    source: dict = report['source']

    fqme: str = report['fqme']
    period_s: int = report['period_s']
    table = Table(title=f'{fqme} @ {period_s}s')
    table.add_column('Check')
    table.add_column('Value')
    table.add_row('Path', source['path'])
    table.add_row('SHA-256', source['sha256'])
    table.add_row('Rows', str(report['rows']['count']))
    table.add_row('Minimum UTC', str(timestamps.get('minimum_utc')))
    table.add_row('Maximum UTC', str(timestamps.get('maximum_utc')))
    table.add_row('Structural OK', str(result['structural_ok']))
    table.add_row('Gap free', str(result['gap_free']))
    table.add_row('Gap count', str(gaps['count']))
    table.add_row('Sub-period steps', str(gaps['subperiod_count']))
    table.add_row('Gap details truncated', str(
        gaps['details_truncated']
    ))
    violations: list[str] = result['violations']
    table.add_row(
        'Violations',
        ', '.join(violations) if violations else 'none',
    )

    console = Console()
    console.print(table)
    intervals: list[dict] = [
        *gaps['intervals'],
        *gaps['subperiod_intervals'],
    ]
    if not intervals:
        return

    gap_table = Table(title='Timestamp cadence deviations')
    gap_table.add_column('Left UTC')
    gap_table.add_column('Right UTC')
    gap_table.add_column('Delta (s)')
    gap_table.add_column('Missing')
    gap_table.add_column('Aligned')
    for gap in intervals:
        gap_table.add_row(
            str(gap['left_utc']),
            str(gap['right_utc']),
            str(gap['delta_s']),
            str(gap.get('missing_samples')),
            str(gap.get('period_multiple')),
        )
    console.print(gap_table)


def _native_series_payload(
    pattern: str|None,

) -> tuple[Path, list[dict]]:
    '''
    Collect exact NativeDB series metadata for CLI presentation.

    '''
    datadir: Path = config.get_conf_dir() / 'nativedb'
    refs: list[NativeSeriesRef] = list(iter_native_series(datadir))
    if pattern is not None:
        needle: str = pattern.casefold()
        refs = [
            ref
            for ref in refs
            if needle in ref.fqme.casefold()
        ]

    payload: list[dict] = []
    for ref in refs:
        try:
            size_bytes: int = ref.path.stat().st_size
        except FileNotFoundError:
            continue
        payload.append({
            'fqme': ref.fqme,
            'period_s': ref.period_s,
            'path': str(ref.path),
            'size_bytes': size_bytes,
        })
    payload.sort(key=lambda item: (
        item['fqme'],
        item['period_s'],
    ))

    return datadir, payload


@store.command()
def ls(
    pattern: Annotated[
        str|None,
        typer.Argument(
            help='Optional case-insensitive FQME substring.',
        ),
    ] = None,
    all_series: Annotated[
        bool,
        typer.Option(
            '--all',
            help='List all FQMEs (the default).',
        ),
    ] = False,
) -> None:
    '''
    List NativeDB FQMEs and their available sample periods.

    '''
    if (
        all_series
        and
        pattern is not None
    ):
        raise typer.BadParameter('Pass a pattern or --all, not both')

    datadir, payload = _native_series_payload(pattern)
    periods_by_fqme: dict[str, list[int]] = {}
    for item in payload:
        periods_by_fqme.setdefault(item['fqme'], []).append(
            item['period_s']
        )

    table = Table(title=f'NativeDB series @ {datadir}')
    table.add_column('FQME')
    table.add_column('Periods')
    for fqme, periods in periods_by_fqme.items():
        table.add_row(
            fqme,
            ', '.join(
                f'{period}s'
                for period in sorted(periods)
            ),
        )
    Console().print(table)


@store.command()
def series(
    pattern: Annotated[
        str|None,
        typer.Argument(
            help='Optional case-insensitive FQME substring.',
        ),
    ] = None,
    all_series: Annotated[
        bool,
        typer.Option(
            '--all',
            help='List all series (the default).',
        ),
    ] = False,
    json_output: Annotated[
        bool,
        typer.Option(
            '--json',
            help='Emit machine-readable JSON.',
        ),
    ] = False,
) -> None:
    '''
    List detailed NativeDB sub-series without opening a runtime.

    '''
    if (
        all_series
        and
        pattern is not None
    ):
        raise typer.BadParameter('Pass a pattern or --all, not both')
    datadir, payload = _native_series_payload(pattern)
    if json_output:
        typer.echo(json.dumps(payload, indent=2, sort_keys=True))
        return

    table = Table(title=f'NativeDB series @ {datadir}')
    table.add_column('FQME')
    table.add_column('Period')
    table.add_column('Bytes', justify='right')
    table.add_column('Path')
    for item in payload:
        table.add_row(
            item['fqme'],
            f'{item["period_s"]}s',
            str(item['size_bytes']),
            item['path'],
        )
    Console().print(table)


@store.command(no_args_is_help=True)
def audit(
    fqme: str,
    period: int = typer.Option(
        60,
        '--period',
        min=1,
        help='Expected sampling period in seconds.',
    ),
    json_output: bool = typer.Option(
        False,
        '--json',
        help='Emit machine-readable JSON.',
    ),
    output: Path|None = typer.Option(
        None,
        '--output',
        '-o',
        help='Write JSON to a new file without replacing it.',
    ),
    snapshot: Path|None = typer.Option(
        None,
        '--snapshot',
        help='Copy the exact audited bytes to a new file.',
    ),
    max_gaps: int = typer.Option(
        100,
        '--max-gaps',
        min=0,
        help='Maximum gap details to include.',
    ),
    strict: bool = typer.Option(
        False,
        '--strict',
        help='Exit nonzero for structural defects or any gap.',
    ),
) -> None:
    '''
    Audit one NativeDB Parquet without normalizing or mutating it.

    Positive gaps are reported as unclassified evidence. They are not
    automatically treated as corruption or expected venue closures.

    '''
    if (
        fqme in {'.', '..'}
        or
        Path(fqme).name != fqme
        or
        '/' in fqme
        or
        '\\' in fqme
    ):
        typer.echo(f'Unsafe FQME path component: {fqme!r}', err=True)
        raise typer.Exit(code=2)
    datadir: Path = config.get_conf_dir() / 'nativedb'
    path: Path = mk_ohlcv_shm_keyed_filepath(
        fqme,
        period,
        datadir,
    )
    destinations: list[Path] = [
        destination
        for destination in (output, snapshot)
        if destination is not None
    ]
    resolved_source: Path = path.resolve()
    if path.is_symlink():
        typer.echo(
            f'Refusing symlinked audit source: {path}',
            err=True,
        )
        raise typer.Exit(code=2)
    resolved_destinations: list[Path] = []
    for destination in destinations:
        if destination.is_symlink():
            typer.echo(
                f'Refusing symlinked output: {destination}',
                err=True,
            )
            raise typer.Exit(code=2)
        resolved: Path = destination.resolve()
        if resolved == resolved_source:
            typer.echo(
                f'Refusing to replace audited source: {resolved}',
                err=True,
            )
            raise typer.Exit(code=2)
        if destination.exists():
            typer.echo(
                f'Refusing to replace existing output: '
                f'{destination}',
                err=True,
            )
            raise typer.Exit(code=2)
        if not destination.parent.is_dir():
            typer.echo(
                f'Output directory does not exist: '
                f'{destination.parent}',
                err=True,
            )
            raise typer.Exit(code=2)
        resolved_destinations.append(resolved)
    if len(set(resolved_destinations)) != len(resolved_destinations):
        typer.echo(
            'JSON output and snapshot paths must differ',
            err=True,
        )
        raise typer.Exit(code=2)

    try:
        report: dict = audit_ohlcv_parquet(
            path,
            fqme,
            period,
            max_gaps,
            snapshot,
        )
    except (
        FileNotFoundError,
        OSError,
        pl.exceptions.PolarsError,
        ValueError,
    ) as err:
        typer.echo(
            f'Unable to audit {path}: {err}',
            err=True,
        )
        raise typer.Exit(code=2) from err

    report['runtime'] = {
        'piker_file': str(Path(piker_pkg.__file__).resolve()),
        'executable': str(Path(sys.argv[0]).resolve()),
    }
    payload: str = json.dumps(
        report,
        allow_nan=False,
        indent=2,
        sort_keys=True,
    )
    if output is not None:
        try:
            with output.open('x') as output_file:
                output_file.write(f'{payload}\n')
        except OSError as err:
            typer.echo(
                f'Unable to write audit output: {err}',
                err=True,
            )
            raise typer.Exit(code=2) from err

    if json_output:
        typer.echo(payload)
    else:
        _render_audit_report(report)

    if (
        strict
        and
        not report['result']['qualification_ok']
    ):
        raise typer.Exit(code=1)


# TODO: like ls but takes in a pattern and matches
# @store.command()
# def search(
#     patt: str,
#     backends: list[str] = typer.Argument(
#         default=None,
#         help='Storage backends to query, default is all.'
#     ),
# ):
#     ...


@store.command(no_args_is_help=True)
def delete(
    symbols: list[str],

    backend: str = typer.Option(
        default=None,
        help='Storage backend to update'
    ),
    # TODO: expose this as flagged multi-option?
    timeframes: list[int] = [1, 60],
):
    '''
    Delete a storage backend's time series for (table) keys provided as
    ``symbols``.

    '''
    from . import open_storage_client

    async def main(symbols: list[str]):
        async with (
            open_piker_runtime(
                'tsdb_storage',
            ),
            open_storage_client(backend) as (_, client),
            trio.open_nursery() as n,
        ):
            # spawn queries as tasks for max conc!
            for fqme in symbols:
                for tf in timeframes:
                    n.start_soon(
                        client.delete_ts,
                        fqme,
                        tf,
                    )

    trio.run(main, symbols)


@store.command(no_args_is_help=True)
def anal(
    fqme: str,
    period: int = 60,
    pdb: bool = False,

) -> np.ndarray:
    '''
    Anal-ysis is when you take the data do stuff to it.

    NOTE: This ONLY loads the offline timeseries data (by default
    from a parquet file) NOT the in-shm version you might be seeing
    in a chart.

    '''
    async def main():
        async with (
            open_piker_runtime(
                # are you a bear or boi?
                'tsdb_polars_anal',
                debug_mode=pdb,
            ),
            open_storage_client() as (
                mod,
                client,
            ),
        ):
            syms: list[str] = await client.list_keys()
            log.info(f'{len(syms)} FOUND for {mod.name}')

            history: ShmArray  # np buffer format
            (
                history,
                first_dt,
                last_dt,
            ) = await client.load(
                fqme,
                period,
            )
            assert first_dt < last_dt

            null_segs: tuple = tsp.get_null_segs(
                frame=history,
                period=period,
            )
            # TODO: do tsp queries to backcend to fill i missing
            # history and then prolly write it to tsdb!

            shm_df: pl.DataFrame = await client.as_df(
                fqme,
                period,
            )

            df: pl.DataFrame  # with dts
            deduped: pl.DataFrame  # deduplicated dts
            (
                df,
                deduped,
                diff,
            ) = tsp.dedupe(
                shm_df,
                period=period,
            )

            write_edits: bool = True
            if (
                write_edits
                and (
                    diff
                    or null_segs
                )
            ):
                await tractor.pause()
                await client.write_ohlcv(
                    fqme,
                    ohlcv=deduped,
                    timeframe=period,
                )

            else:
                # TODO: something better with tab completion..
                # is there something more minimal but nearly as
                # functional as ipython?
                await tractor.pause()
                assert not null_segs

    trio.run(main)


def _inspect_shm(
    fqme: str,
    shm_name: Annotated[
        str|None,
        typer.Option(
            '--shm-name',
            help='Inspect only this exact SHM object name.',
        ),
    ] = None,
    max_gaps: Annotated[
        int,
        typer.Option(
            '--max-gaps',
            min=0,
            help='Maximum largest gaps to show per buffer.',
        ),
    ] = 10,
    json_output: Annotated[
        bool,
        typer.Option(
            '--json',
            help='Emit machine-readable JSON.',
        ),
    ] = False,
    pdb: bool = False,
) -> None:
    '''
    Inspect matching OHLCV SHM buffers without mutating them.

    '''
    refs: list[tsp.ShmBufferRef] = [
        ref
        for ref in tsp.iter_shm_buffer_refs(fqme=fqme)
        if (
            shm_name is None
            or
            ref.path.name == shm_name
        )
    ]
    if not refs:
        typer.echo(
            f'No exact OHLCV SHM buffers found for {fqme!r}',
            err=True,
        )
        raise typer.Exit(code=2)

    ref_by_name: dict[str, tsp.ShmBufferRef] = {
        ref.path.name: ref
        for ref in refs
    }
    reports: list[dict] = []

    async def main() -> None:
        async with open_piker_runtime(
            'shm_inspector',
            debug_mode=pdb,
        ):
            for (
                shmfile,
                shm,
                _shm_df,
            ) in tsp.iter_dfs_from_shms(
                fqme,
                shm_name=shm_name,
                refs=refs,
            ):
                ref: tsp.ShmBufferRef = ref_by_name[shmfile.name]
                reports.append(_summarize_shm_frame(
                    ref,
                    shm.array.copy(),
                    max_gaps,
                ))

    trio.run(main)
    if not reports:
        typer.echo(
            'Selected SHM buffers disappeared before inspection',
            err=True,
        )
        raise typer.Exit(code=2)
    if json_output:
        typer.echo(json.dumps(reports, indent=2, sort_keys=True))
        return

    table = Table(title=f'OHLCV SHM buffers for {fqme}')
    table.add_column('Kind')
    table.add_column('Generation')
    table.add_column('Rows', justify='right')
    table.add_column('Period')
    table.add_column('Invalid', justify='right')
    table.add_column('Order', justify='right')
    table.add_column('Gaps', justify='right')
    table.add_column('SHM name')
    for report in reports:
        period_s: int = report['period_s']
        table.add_row(
            report['kind'],
            report['generation'],
            str(report['rows']),
            f'{period_s}s',
            str(report['invalid_count']),
            str(
                report['duplicate_step_count']
                +
                report['reversed_step_count']
            ),
            str(report['gap_count']),
            report['shm_name'],
        )
    Console().print(table)

    for report in reports:
        gaps: list[dict] = report['largest_gaps']
        if not gaps:
            continue
        gap_table = Table(
            title=f'Largest gaps: {report["shm_name"]}'
        )
        gap_table.add_column('Delta (s)')
        gap_table.add_column('Left UTC')
        gap_table.add_column('Right UTC')
        for gap in gaps:
            gap_table.add_row(
                str(gap['delta_s']),
                str(gap['left_utc']),
                str(gap['right_utc']),
            )
        Console().print(gap_table)


@store.command(no_args_is_help=True)
def shm(
    fqme: str,
    write_parquet: Annotated[
        bool,
        typer.Option(
            '--write-parquet',
            help='Persist repaired frames to NativeDB.',
        ),
    ] = False,
    reload_parquet_to_shm: Annotated[
        bool,
        typer.Option(
            '--reload-parquet-to-shm',
            help='Reload persisted repairs into SHM.',
        ),
    ] = False,
    pdb: bool = False,  # --pdb passed?
    max_gaps: Annotated[
        int|None,
        typer.Option(
            '--max-gaps',
            min=0,
            help=(
                'Maximum largest gaps in read-only output '
                '(default: 10).'
            ),
            show_default=False,
        ),
    ] = None,
    json_output: Annotated[
        bool,
        typer.Option(
            '--json',
            help='Emit read-only output as JSON.',
        ),
    ] = False,
    shm_name: Annotated[
        str|None,
        typer.Option(
            '--shm-name',
            help='Select only this exact SHM object name.',
        ),
    ] = None,

) -> None:
    '''
    Inspect volatile OHLCV SHM, with explicit repair options.

    '''
    if (
        reload_parquet_to_shm
        and
        not write_parquet
    ):
        raise typer.BadParameter(
            '--reload-parquet-to-shm requires --write-parquet'
        )
    if (
        write_parquet
        and (
            json_output
            or
            max_gaps is not None
        )
    ):
        raise typer.BadParameter(
            '--json and --max-gaps apply only to read-only output'
        )
    if not write_parquet:
        _inspect_shm(
            fqme,
            shm_name=shm_name,
            max_gaps=(
                10
                if max_gaps is None
                else max_gaps
            ),
            json_output=json_output,
            pdb=pdb,
        )
        return

    if (
        shm_name is not None
        and
        not any(
            ref.path.name == shm_name
            for ref in tsp.iter_shm_buffer_refs(fqme=fqme)
        )
    ):
        typer.echo(
            f'No exact OHLCV SHM buffer named {shm_name!r}',
            err=True,
        )
        raise typer.Exit(code=2)

    async def main() -> bool:
        from piker.ui._remote_ctl import (
            open_annot_client,
        )
        actl: AnnotClient
        mod: ModuleType
        client: StorageClient
        async with (
            open_piker_runtime(
                'polars_boi',
                enable_modules=['piker.data._sharedmem'],
                debug_mode=pdb,
            ),
            open_storage_client() as (
                mod,
                client,
            ),
            open_annot_client() as actl,
        ):
            shm_df: pl.DataFrame | None = None
            tf2aids: dict[float, dict] = {}

            iter_kwargs: dict = {}
            if shm_name is not None:
                iter_kwargs['shm_name'] = shm_name
            for (
                shmfile,
                shm,
                # parquet_path,
                shm_df,
            ) in tsp.iter_dfs_from_shms(fqme, **iter_kwargs):

                times: np.ndarray = shm_df['time'].to_numpy()
                (
                    observed_period_s,
                    invalid_count,
                ) = _shm_period_and_invalid_count(times)
                if observed_period_s is None:
                    log.warning(
                        f'Could not infer a positive sample period '
                        f'for {shmfile.name}; skipping buffer\n'
                    )
                    continue
                kind: str = shmfile.suffix.removeprefix('.')
                period_s: int = _shm_periods[kind]
                if observed_period_s != period_s:
                    log.warning(
                        f'Modal step {observed_period_s}s in '
                        f'{period_s}s {shmfile.name}; using kind '
                        f'for storage identity\n'
                    )
                log.info(
                    f'Processing shm buffer:\n'
                    f'  file: {shmfile.name}\n'
                    f'  period: {period_s}s\n'
                )

                # TODO: call null-seg fixer somehow?
                if invalid_count:
                #     async with (
                #         trio.open_nursery() as tn,
                #         mod.open_history_client(
                #             mkt,
                #         ) as (get_hist, config),
                #     ):
                #         nulls_detected: trio.Event = await tn.start(partial(
                #             tsp.maybe_fill_null_segments,

                #             shm=shm,
                #             timeframe=timeframe,
                #             get_hist=get_hist,
                #             sampler_stream=sampler_stream,
                #             mkt=mkt,
                #         ))
                    if tractor.runtime._state.is_debug_mode():
                        await tractor.pause()
                    log.warning(
                        f'Found {invalid_count} invalid SHM row(s) '
                        f'in {shmfile.name}; skipping persistence '
                        f'and reload\n'
                    )
                    continue

                # over-write back to shm?
                wdts: pl.DataFrame  # with dts
                deduped: pl.DataFrame  # deduplicated dts
                (
                    wdts,
                    deduped,
                    diff,
                    valid_races,
                    dq_issues,
                ) = tsp.dedupe_ohlcv_smart(
                    shm_df,
                )

                # Report duplicate analysis
                if diff > 0:
                    log.info(
                        f'Removed {diff} duplicate timestamp(s)\n'
                    )
                    if valid_races is not None:
                        identical: int = (
                            valid_races
                            .filter(pl.col('identical_bars'))
                            .height
                        )
                        monotonic: int = valid_races.height - identical
                        log.info(
                            f'Valid race conditions: {valid_races.height}\n'
                            f'  - Identical bars: {identical}\n'
                            f'  - Volume monotonic: {monotonic}\n'
                        )

                    if dq_issues is not None:
                        log.warning(
                            f'DATA QUALITY ISSUES from provider: '
                            f'{dq_issues.height} timestamp(s)\n'
                            f'{dq_issues}\n'
                        )

                # detect gaps from in expected (uniform OHLC) sample period
                step_gaps: pl.DataFrame = tsp.detect_time_gaps(
                    deduped,
                    expect_period=period_s,
                )

                # TODO: by default we always want to mark these up
                # with rects showing up/down gaps Bo
                venue_gaps: pl.DataFrame = tsp.detect_time_gaps(
                    deduped,
                    expect_period=period_s,

                    # TODO: actually pull the exact duration
                    # expected for each venue operational period?
                    # gap_dt_unit='day',
                    gap_dt_unit='day',
                    gap_thresh=1,
                )

                # TODO: find the disjoint set of step gaps from
                # venue (closure) set!
                # -[ ] do a set diff by checking for the unique
                #    gap set only in the step_gaps?
                if (
                    not venue_gaps.is_empty()
                    or (
                        not step_gaps.is_empty()
                        # XXX, i presume i put this bc i was guarding
                        # for ib venue gaps?
                        # and
                        # period_s < 60
                    )
                ):
                    # write repaired ts to parquet-file?
                    if write_parquet:
                        start: float = time.time()
                        path: Path = await client.write_ohlcv(
                            fqme,
                            ohlcv=deduped,
                            timeframe=period_s,
                        )
                        write_delay: float = round(
                            time.time() - start,
                            ndigits=6,
                        )

                        # read back from fs
                        start: float = time.time()
                        read_df: pl.DataFrame = pl.read_parquet(path)
                        read_delay: float = round(
                            time.time() - start,
                            ndigits=6,
                        )
                        log.info(
                            f'parquet write took {write_delay} secs\n'
                            f'file path: {path}'
                            f'parquet read took {read_delay} secs\n'
                            f'polars df: {read_df}'
                        )

                        if reload_parquet_to_shm:
                            new = tsp.pl2np(
                                deduped,
                                dtype=shm.array.dtype,
                            )
                            # since normally readonly
                            shm._array.setflags(
                                write=int(1),
                            )
                            shm.push(
                                new,
                                prepend=True,
                                start=new['index'][-1],
                                update_first=False,  # don't update ._first
                            )

                    do_markup_gaps: bool = True
                    if do_markup_gaps:
                        aids: dict = await tsp._annotate.markup_gaps(
                            fqme,
                            period_s,
                            actl,
                            deduped,
                            step_gaps,
                        )
                        # last chance manual overwrites in REPL
                        # await tractor.pause()
                        if not aids:
                            log.warning(
                                f'No gaps were found !?\n'
                                f'fqme: {fqme!r}\n'
                                f'timeframe: {period_s!r}\n'
                                f"WELL THAT'S GOOD NOOZ!\n"
                            )
                        tf2aids[period_s] = aids

                else:
                    # No significant gaps to handle, but may have had
                    # duplicates removed (valid race conditions are ok)
                    if diff > 0 and dq_issues is not None:
                        log.warning(
                            'Found duplicates with data quality issues '
                            'but no significant time gaps!\n'
                        )

            if shm_df is None:
                log.error(
                    f'No matching shm buffers for {fqme} ?'

                )
                return False

            await tractor.pause()
            log.info('Exiting TSP shm anal-izer!')
            return True

    if not trio.run(main):
        raise typer.Exit(code=2)


typer_click_object = typer.main.get_command(store)
cli.add_command(typer_click_object, 'store')
