# piker: trading gear for hackers
# Copyright (C) Tyler Goodlet (in stewardship for pikers)

'''
Read-only OHLCV persistence qualification.

'''
from datetime import (
    UTC,
    datetime,
)
from hashlib import sha256
import os
from pathlib import Path
import shutil
from stat import S_ISREG
from typing import BinaryIO

import numpy as np
import polars as pl

from piker.data import def_iohlcv_fields


AUDIT_SCHEMA: str = 'piker.nativedb.audit/v1'
_value_fields: tuple[str, ...] = (
    'open',
    'high',
    'low',
    'close',
    'volume',
)
_price_fields: tuple[str, ...] = (
    'open',
    'high',
    'low',
    'close',
)


def _utc_str(timestamp: float|int|None) -> str|None:
    '''
    Render an epoch timestamp as deterministic UTC.

    '''
    if timestamp is None:
        return None
    try:
        dt: datetime = datetime.fromtimestamp(timestamp, tz=UTC)
    except (
        OSError,
        OverflowError,
        ValueError,
    ):
        return None
    return dt.isoformat().replace('+00:00', 'Z')


def _number(value: float|int) -> float|int:
    '''
    Convert a NumPy number to a JSON-native scalar.

    '''
    if isinstance(value, (int, np.integer)):
        return int(value)
    value = float(value)
    return int(value) if value.is_integer() else value


def _sha256(file: BinaryIO) -> str:
    '''
    Hash and rewind one already-open file snapshot.

    '''
    digest = sha256()
    for chunk in iter(
        lambda: file.read(1024 * 1024),
        b'',
    ):
        digest.update(chunk)
    file.seek(0)
    return digest.hexdigest()


def _numeric_counts(series: pl.Series) -> dict[str, int|None]:
    '''
    Count null and non-finite values without coercing text.

    '''
    null_count: int = series.null_count()
    if not series.dtype.is_numeric():
        return {
            'null': null_count,
            'nan': None,
            'positive_infinity': None,
            'negative_infinity': None,
            'nonfinite': None,
        }

    numeric: pl.Series = series.cast(pl.Float64, strict=False)
    nan_count: int = int(numeric.is_nan().sum() or 0)
    pos_inf_count: int = int(
        (numeric == float('inf')).sum() or 0
    )
    neg_inf_count: int = int(
        (numeric == float('-inf')).sum() or 0
    )
    cast_null_count: int = max(
        0,
        numeric.null_count() - null_count,
    )
    return {
        'null': null_count,
        'nan': nan_count,
        'positive_infinity': pos_inf_count,
        'negative_infinity': neg_inf_count,
        'nonfinite': (
            null_count
            + nan_count
            + pos_inf_count
            + neg_inf_count
            + cast_null_count
        ),
    }


def _numeric_values(
    series: pl.Series,
) -> tuple[np.ndarray, np.ndarray]:
    '''
    Preserve integer precision and return an explicit valid mask.

    '''
    if series.dtype.is_integer():
        valid: np.ndarray = series.is_not_null().to_numpy()
        values: np.ndarray = series.fill_null(0).to_numpy()
        return values, valid

    values = series.to_numpy()
    try:
        valid = np.isfinite(values)
    except TypeError:
        values = series.cast(pl.Float64).to_numpy()
        valid = np.isfinite(values)
    return values, valid


def _empty_gaps(
    period_s: int,
    verifiable: bool = False,
) -> dict:
    '''
    Return the stable gap report shape for unauditable timestamps.

    '''
    return {
        'basis': 'sorted_unique_valid_timestamps',
        'expected_period_s': period_s,
        'verifiable': verifiable,
        'count': 0,
        'aligned_count': 0,
        'misaligned_count': 0,
        'subperiod_count': 0,
        'missing_samples_total': 0,
        'details_count': 0,
        'details_truncated': False,
        'intervals': [],
        'subperiod_intervals': [],
    }


def _audit_timestamps(
    df: pl.DataFrame,
    period_s: int,
    max_gaps: int,

) -> tuple[dict, dict]:
    '''
    Audit physical timestamp ordering and chronological gaps.

    '''
    if 'time' not in df.columns:
        return (
            {
                'present': False,
                'numeric': False,
                'strictly_increasing': False,
            },
            _empty_gaps(period_s),
        )

    series: pl.Series = df['time']
    counts: dict[str, int|None] = _numeric_counts(series)
    if not series.dtype.is_numeric():
        return (
            {
                'present': True,
                'numeric': False,
                **counts,
                'strictly_increasing': False,
            },
            _empty_gaps(period_s),
        )

    values, finite = _numeric_values(series)
    valid: np.ndarray = values[finite]
    unique: np.ndarray = np.unique(valid)
    adjacent_valid: np.ndarray = finite[:-1] & finite[1:]
    deltas: np.ndarray = np.diff(values)
    physical_deltas: np.ndarray = deltas[adjacent_valid]

    zero_delta_count: int = int(np.count_nonzero(
        physical_deltas == 0
    ))
    negative_delta_count: int = int(np.count_nonzero(
        physical_deltas < 0
    ))
    first: float|int|None = None
    last: float|int|None = None
    if values.size:
        if finite[0]:
            first = _number(values[0])
        if finite[-1]:
            last = _number(values[-1])

    minimum: float|int|None = None
    maximum: float|int|None = None
    if valid.size:
        minimum = _number(np.min(valid))
        maximum = _number(np.max(valid))

    chronological: np.ndarray = unique[unique > 0]
    gap_deltas: np.ndarray = np.diff(chronological)
    gap_indexes: np.ndarray = np.flatnonzero(
        gap_deltas > period_s
    )
    gap_values: np.ndarray = gap_deltas[gap_indexes]
    integer_gaps: bool = np.issubdtype(
        gap_values.dtype,
        np.integer,
    )
    if integer_gaps:
        aligned: np.ndarray = gap_values % period_s == 0
    else:
        aligned = np.isclose(
            gap_values % period_s,
            0,
        )
    aligned_count: int = int(np.count_nonzero(aligned))
    if integer_gaps:
        missing_total: int = sum(
            int(delta) // period_s - 1
            for delta in gap_values[aligned]
        )
    else:
        missing_total = sum(
            int(round(float(delta) / period_s)) - 1
            for delta in gap_values[aligned]
        )
    subperiod_indexes: np.ndarray = np.flatnonzero(
        gap_deltas < period_s
    )
    intervals: list[dict] = []
    for gap_index in gap_indexes[:max_gaps]:
        left: float = chronological[gap_index]
        right: float = chronological[gap_index + 1]
        delta: float = right - left
        period_multiple: bool = bool(np.isclose(
            delta % period_s,
            0,
        ))
        missing_samples: int|None = None
        if period_multiple:
            missing_samples = (
                int(delta) // period_s - 1
                if integer_gaps
                else int(round(float(delta) / period_s)) - 1
            )

        left_value: float|int = _number(left)
        right_value: float|int = _number(right)
        intervals.append({
            'left_timestamp': left_value,
            'right_timestamp': right_value,
            'left_utc': _utc_str(left_value),
            'right_utc': _utc_str(right_value),
            'delta_s': _number(delta),
            'period_multiple': period_multiple,
            'missing_samples': missing_samples,
            'classification': 'unclassified',
        })

    subperiod_intervals: list[dict] = []
    remaining_details: int = max(0, max_gaps - len(intervals))
    for step_index in subperiod_indexes[:remaining_details]:
        left = _number(chronological[step_index])
        right = _number(chronological[step_index + 1])
        subperiod_intervals.append({
            'left_timestamp': left,
            'right_timestamp': right,
            'left_utc': _utc_str(left),
            'right_utc': _utc_str(right),
            'delta_s': _number(right - left),
        })

    gap_count: int = gap_indexes.size
    gaps: dict = {
        'basis': 'sorted_unique_valid_timestamps',
        'expected_period_s': period_s,
        'verifiable': chronological.size >= 2,
        'count': gap_count,
        'aligned_count': aligned_count,
        'misaligned_count': gap_count - aligned_count,
        'missing_samples_total': missing_total,
        'subperiod_count': subperiod_indexes.size,
        'details_count': (
            len(intervals)
            + len(subperiod_intervals)
        ),
        'details_truncated': bool(
            gap_count > len(intervals)
            or
            subperiod_indexes.size > len(subperiod_intervals)
        ),
        'intervals': intervals,
        'subperiod_intervals': subperiod_intervals,
    }
    timestamps: dict = {
        'present': True,
        'numeric': True,
        'first_in_file': first,
        'last_in_file': last,
        'minimum': minimum,
        'maximum': maximum,
        'minimum_utc': _utc_str(minimum),
        'maximum_utc': _utc_str(maximum),
        **counts,
        'zero': int(np.count_nonzero(valid == 0)),
        'negative': int(np.count_nonzero(valid < 0)),
        'fractional': int(np.count_nonzero(
            valid != np.floor(valid)
        )),
        'unique': unique.size,
        'duplicate_excess': valid.size - unique.size,
        'adjacent_zero_delta': zero_delta_count,
        'negative_delta': negative_delta_count,
        'non_positive_delta': (
            zero_delta_count
            + negative_delta_count
        ),
        'strictly_increasing': bool(
            values.size > 0
            and
            values.size == valid.size
            and
            np.all(np.diff(values) > 0)
        ),
    }
    return timestamps, gaps


def _audit_index(df: pl.DataFrame) -> dict:
    '''
    Verify persisted indexes match physical row positions.

    '''
    if 'index' not in df.columns:
        return {
            'present': False,
            'numeric': False,
            'canonical_from_zero': False,
        }

    series: pl.Series = df['index']
    counts: dict[str, int|None] = _numeric_counts(series)
    if not series.dtype.is_numeric():
        return {
            'present': True,
            'numeric': False,
            'dtype': str(series.dtype),
            **counts,
            'canonical_from_zero': False,
        }

    values, finite = _numeric_values(series)
    expected: np.ndarray = np.arange(df.height)
    mismatch_count: int = int(np.count_nonzero(
        ~finite
        |
        (values != expected)
    ))
    valid: np.ndarray = values[finite]
    unique_count: int = np.unique(valid).size
    return {
        'present': True,
        'numeric': True,
        'dtype': str(series.dtype),
        'first': (
            _number(values[0])
            if values.size and finite[0]
            else None
        ),
        'last': (
            _number(values[-1])
            if values.size and finite[-1]
            else None
        ),
        **counts,
        'duplicate_excess': valid.size - unique_count,
        'non_unit_step': int(np.count_nonzero(
            np.diff(valid) != 1
        )),
        'row_position_mismatch': mismatch_count,
        'contiguous': bool(
            values.size > 0
            and
            valid.size == values.size
            and
            np.all(np.diff(values) == 1)
        ),
        'canonical_from_zero': bool(
            values.size > 0
            and
            values.size == expected.size
            and
            mismatch_count == 0
        ),
    }


def audit_ohlcv_frame(
    df: pl.DataFrame,
    fqme: str,
    period_s: int,
    max_gaps: int = 100,

) -> dict:
    '''
    Inspect a raw persisted frame without normalizing evidence.

    '''
    if period_s < 1:
        raise ValueError('Audit period must be positive')
    if max_gaps < 0:
        raise ValueError('Maximum gap details must be non-negative')

    expected_columns: list[str] = [
        name
        for name, _ in def_iohlcv_fields
    ]
    expected_dtypes: dict[str, str] = {
        name: str(pl.Int64 if field_type is int else pl.Float64)
        for name, field_type in def_iohlcv_fields
    }
    actual_dtypes: dict[str, str] = {
        name: str(dtype)
        for name, dtype in df.schema.items()
    }
    missing: list[str] = sorted(
        set(expected_columns).difference(df.columns)
    )
    extra: list[str] = sorted(
        set(df.columns).difference(expected_columns)
    )
    column_order_ok: bool = df.columns == expected_columns
    dtypes_ok: bool = all(
        actual_dtypes.get(name) == dtype
        for name, dtype in expected_dtypes.items()
    )

    timestamps, gaps = _audit_timestamps(
        df,
        period_s,
        max_gaps,
    )
    index: dict = _audit_index(df)

    nulls: dict[str, int|None] = {}
    nans: dict[str, int|None] = {}
    infinities: dict[str, int|None] = {}
    all_values_finite: bool = True
    for field in _value_fields:
        if field not in df.columns:
            nulls[field] = None
            nans[field] = None
            infinities[field] = None
            all_values_finite = False
            continue

        counts = _numeric_counts(df[field])
        nulls[field] = counts['null']
        nans[field] = counts['nan']
        pos_inf: int|None = counts['positive_infinity']
        neg_inf: int|None = counts['negative_infinity']
        infinities[field] = (
            None
            if pos_inf is None or neg_inf is None
            else pos_inf + neg_inf
        )
        if counts['nonfinite'] != 0:
            all_values_finite = False

    all_zero_price_rows: int|None = None
    epoch_sentinel_rows: int|None = None
    if all(
        field in df.columns
        and
        df[field].dtype.is_numeric()
        for field in _price_fields
    ):
        zero_prices: pl.Expr = pl.all_horizontal([
            pl.col(field) == 0
            for field in _price_fields
        ]).fill_null(False)
        all_zero_price_rows = int(
            df.select(zero_prices).to_series().sum()
            or 0
        )
        if (
            'time' in df.columns
            and
            df['time'].dtype.is_numeric()
        ):
            epoch_sentinel_rows = int(
                df.select(
                    (pl.col('time') <= period_s)
                    &
                    zero_prices
                ).to_series().sum()
                or 0
            )

    schema_ok: bool = bool(
        not missing
        and
        not extra
        and
        column_order_ok
        and
        dtypes_ok
    )
    violations: list[str] = []
    if df.is_empty():
        violations.append('empty_frame')
    if missing:
        violations.append('missing_columns')
    if extra:
        violations.append('extra_columns')
    if not column_order_ok:
        violations.append('column_order')
    if not dtypes_ok:
        violations.append('canonical_dtypes')
    if not all_values_finite:
        violations.append('nonfinite_ohlcv')
    if epoch_sentinel_rows:
        violations.append('epoch_ohlc_sentinel')
    if timestamps.get('nonfinite') != 0:
        violations.append('nonfinite_timestamps')
    if (
        timestamps.get('zero', 0)
        or
        timestamps.get('negative', 0)
    ):
        violations.append('nonpositive_timestamps')
    if timestamps.get('fractional', 0):
        violations.append('fractional_timestamps')
    if timestamps.get('duplicate_excess', 0):
        violations.append('duplicate_timestamps')
    if not timestamps.get('strictly_increasing', False):
        violations.append('timestamp_order')
    if not index.get('canonical_from_zero', False):
        violations.append('index_not_canonical')
    if gaps.get('misaligned_count', 0):
        violations.append('misaligned_time_gap')
    if gaps.get('subperiod_count', 0):
        violations.append('subperiod_time_step')

    warnings: list[str] = []
    if gaps['count']:
        warnings.append('positive_time_gaps_unclassified')
    unclassified_zero_rows: int|None = None
    if all_zero_price_rows is not None:
        unclassified_zero_rows = (
            all_zero_price_rows
            -
            (epoch_sentinel_rows or 0)
        )
        if unclassified_zero_rows:
            warnings.append('all_zero_ohlc_prices_unclassified')

    structural_ok: bool = not violations
    gap_free: bool|None = (
        gaps['count'] == 0
        if gaps['verifiable']
        else None
    )
    return {
        'audit_schema': AUDIT_SCHEMA,
        'fqme': fqme,
        'period_s': period_s,
        'schema': {
            'columns': df.columns,
            'dtypes': actual_dtypes,
            'expected_columns': expected_columns,
            'expected_dtypes': expected_dtypes,
            'missing_columns': missing,
            'extra_columns': extra,
            'column_order_ok': column_order_ok,
            'canonical_dtypes_ok': dtypes_ok,
            'canonical': schema_ok,
        },
        'rows': {
            'count': df.height,
            'empty': df.is_empty(),
        },
        'timestamps': timestamps,
        'index': index,
        'values': {
            'null_by_column': nulls,
            'nan_by_column': nans,
            'infinity_by_column': infinities,
            'all_finite': all_values_finite,
            'all_zero_price_rows': all_zero_price_rows,
            'epoch_sentinel_rows': epoch_sentinel_rows,
            'unclassified_zero_price_rows': unclassified_zero_rows,
        },
        'gaps': gaps,
        'result': {
            'structural_ok': structural_ok,
            'gap_free': gap_free,
            'qualification_ok': bool(structural_ok and gap_free),
            'violations': violations,
            'warnings': warnings,
        },
    }


def audit_ohlcv_parquet(
    path: Path,
    fqme: str,
    period_s: int,
    max_gaps: int = 100,
    snapshot: Path|None = None,

) -> dict:
    '''
    Read and audit one Parquet file without touching storage state.

    '''
    path = path.expanduser().absolute()
    descriptor: int = os.open(
        path,
        os.O_RDONLY
        | os.O_CLOEXEC
        | os.O_NOFOLLOW
        | os.O_NONBLOCK,
    )
    stat = os.fstat(descriptor)
    if not S_ISREG(stat.st_mode):
        os.close(descriptor)
        raise ValueError('Audit source must be a regular file')
    with os.fdopen(descriptor, 'rb') as file:
        checksum: str = _sha256(file)
        if snapshot is not None:
            snapshot_is_symlink: bool = snapshot.is_symlink()
            snapshot = snapshot.resolve()
            if snapshot == path:
                raise ValueError(
                    'Snapshot path can not replace source'
                )
            if snapshot_is_symlink:
                raise ValueError(
                    'Snapshot path can not be a symlink'
                )
            file.seek(0)
            created: bool = False
            try:
                with snapshot.open('xb') as snapshot_file:
                    created = True
                    os.fchmod(snapshot_file.fileno(), 0o444)
                    shutil.copyfileobj(file, snapshot_file)
            except BaseException:
                if created:
                    snapshot.unlink(missing_ok=True)
                raise
        file.seek(0)
        frame: pl.DataFrame = pl.read_parquet(file)
    report: dict = audit_ohlcv_frame(
        frame,
        fqme,
        period_s,
        max_gaps,
    )
    generated_at: float = datetime.now(tz=UTC).timestamp()
    report['generated_at_utc'] = _utc_str(generated_at)
    report['source'] = {
        'path': str(path),
        'size_bytes': stat.st_size,
        'mtime_ns': stat.st_mtime_ns,
        'sha256': checksum,
    }
    return report
