"""
Data Quality Validators for ML Engine
======================================
Provides validation utilities to ensure data quality before ML training.

This module implements:
- Gap detection in time series data
- Outlier detection using statistical methods
- OHLC consistency validation
- Comprehensive quality reports

Author: ML Pipeline (NEXUS v4.0)
Created: 2026-01-25
"""

from typing import Optional, List, Dict, Any, Tuple
from dataclasses import dataclass, field
from datetime import datetime, timedelta
from enum import Enum

import numpy as np
import pandas as pd
from loguru import logger


class ValidationSeverity(Enum):
    """Severity levels for validation issues."""
    INFO = "info"
    WARNING = "warning"
    ERROR = "error"
    CRITICAL = "critical"


@dataclass
class ValidationIssue:
    """Represents a single validation issue found in the data."""
    severity: ValidationSeverity
    category: str
    message: str
    affected_rows: int = 0
    affected_columns: List[str] = field(default_factory=list)
    details: Dict[str, Any] = field(default_factory=dict)

    def __repr__(self) -> str:
        return f"[{self.severity.value.upper()}] {self.category}: {self.message}"


@dataclass
class QualityReport:
    """Comprehensive data quality report."""
    timestamp: str
    total_rows: int
    total_columns: int
    date_range: Tuple[str, str]
    issues: List[ValidationIssue]
    metrics: Dict[str, Any]
    is_valid: bool

    def __repr__(self) -> str:
        status = "VALID" if self.is_valid else "INVALID"
        issues_by_severity = {}
        for issue in self.issues:
            sev = issue.severity.value
            issues_by_severity[sev] = issues_by_severity.get(sev, 0) + 1

        issue_summary = ", ".join([f"{k}: {v}" for k, v in issues_by_severity.items()])

        return (
            f"QualityReport({status})\n"
            f"  Rows: {self.total_rows:,}, Columns: {self.total_columns}\n"
            f"  Date range: {self.date_range[0]} to {self.date_range[1]}\n"
            f"  Issues: {issue_summary or 'None'}"
        )

    def to_dict(self) -> Dict[str, Any]:
        """Convert report to dictionary."""
        return {
            'timestamp': self.timestamp,
            'total_rows': self.total_rows,
            'total_columns': self.total_columns,
            'date_range': self.date_range,
            'is_valid': self.is_valid,
            'issues': [
                {
                    'severity': i.severity.value,
                    'category': i.category,
                    'message': i.message,
                    'affected_rows': i.affected_rows,
                    'affected_columns': i.affected_columns,
                    'details': i.details
                }
                for i in self.issues
            ],
            'metrics': self.metrics
        }


class DataValidator:
    """
    Comprehensive data quality validator for trading data.

    Provides methods to detect:
    - Temporal gaps in time series
    - Statistical outliers
    - OHLC consistency issues
    - Missing values and data quality problems

    Usage:
        validator = DataValidator()

        # Individual validations
        gaps = validator.validate_gaps(df)
        outliers = validator.validate_outliers(df, ['close', 'volume'])
        consistency = validator.validate_consistency(df)

        # Full quality report
        report = validator.generate_quality_report(df)
        if not report.is_valid:
            for issue in report.issues:
                print(issue)
    """

    def __init__(
        self,
        expected_frequency: str = '5min',
        outlier_threshold: float = 3.0,
        max_gap_tolerance: int = 3,
        min_volume_threshold: float = 0.0
    ):
        """
        Initialize the data validator.

        Args:
            expected_frequency: Expected time frequency (e.g., '5min', '15min', '1h')
            outlier_threshold: Z-score threshold for outlier detection
            max_gap_tolerance: Maximum allowed consecutive gaps before warning
            min_volume_threshold: Minimum acceptable volume (0 to disable)
        """
        self.expected_frequency = expected_frequency
        self.outlier_threshold = outlier_threshold
        self.max_gap_tolerance = max_gap_tolerance
        self.min_volume_threshold = min_volume_threshold

        # Frequency mapping to timedelta
        self._freq_to_delta = {
            '1min': timedelta(minutes=1),
            '5min': timedelta(minutes=5),
            '15min': timedelta(minutes=15),
            '30min': timedelta(minutes=30),
            '1h': timedelta(hours=1),
            '4h': timedelta(hours=4),
            '1d': timedelta(days=1),
            'd': timedelta(days=1),
        }

    def validate_gaps(
        self,
        df: pd.DataFrame,
        frequency: Optional[str] = None
    ) -> List[ValidationIssue]:
        """
        Detect gaps in time series data.

        Args:
            df: DataFrame with DatetimeIndex
            frequency: Expected frequency (uses default if None)

        Returns:
            List of ValidationIssue objects for detected gaps
        """
        issues = []

        if not isinstance(df.index, pd.DatetimeIndex):
            issues.append(ValidationIssue(
                severity=ValidationSeverity.ERROR,
                category="gaps",
                message="DataFrame does not have a DatetimeIndex",
                details={'index_type': str(type(df.index))}
            ))
            return issues

        if len(df) < 2:
            return issues

        freq = frequency or self.expected_frequency
        expected_delta = self._freq_to_delta.get(freq)

        if expected_delta is None:
            logger.warning(f"Unknown frequency: {freq}")
            expected_delta = df.index.to_series().diff().median()

        # Calculate time differences
        time_diffs = df.index.to_series().diff()

        # Identify gaps (differences larger than expected)
        tolerance = expected_delta * 1.5  # Allow some tolerance
        gap_mask = time_diffs > tolerance
        gaps = df.index[gap_mask]

        if len(gaps) == 0:
            return issues

        # Analyze gaps
        gap_sizes = time_diffs[gap_mask]
        total_gaps = len(gaps)
        max_gap = gap_sizes.max()
        avg_gap = gap_sizes.mean()

        # Count consecutive gaps
        gap_indices = np.where(gap_mask)[0]
        consecutive_groups = np.split(
            gap_indices,
            np.where(np.diff(gap_indices) != 1)[0] + 1
        )
        max_consecutive = max(len(g) for g in consecutive_groups) if consecutive_groups else 0

        # Determine severity
        if max_consecutive > self.max_gap_tolerance * 3:
            severity = ValidationSeverity.ERROR
        elif max_consecutive > self.max_gap_tolerance:
            severity = ValidationSeverity.WARNING
        else:
            severity = ValidationSeverity.INFO

        # Create detailed gap report
        gap_details = []
        for i, (gap_time, gap_size) in enumerate(zip(gaps, gap_sizes)):
            if i < 10:  # Limit detailed reporting
                gap_details.append({
                    'timestamp': str(gap_time),
                    'gap_size': str(gap_size),
                    'gap_minutes': gap_size.total_seconds() / 60
                })

        issues.append(ValidationIssue(
            severity=severity,
            category="gaps",
            message=f"Detected {total_gaps} gaps in time series (max consecutive: {max_consecutive})",
            affected_rows=total_gaps,
            details={
                'total_gaps': total_gaps,
                'max_gap': str(max_gap),
                'avg_gap': str(avg_gap),
                'max_consecutive': max_consecutive,
                'expected_frequency': freq,
                'sample_gaps': gap_details
            }
        ))

        return issues

    def validate_outliers(
        self,
        df: pd.DataFrame,
        columns: Optional[List[str]] = None,
        threshold: Optional[float] = None,
        method: str = 'zscore'
    ) -> List[ValidationIssue]:
        """
        Detect statistical outliers in specified columns.

        Args:
            df: DataFrame with data
            columns: Columns to check (defaults to numeric columns)
            threshold: Z-score threshold for outlier detection
            method: Detection method ('zscore', 'iqr', 'mad')

        Returns:
            List of ValidationIssue objects for detected outliers
        """
        issues = []
        threshold = threshold or self.outlier_threshold

        # Determine columns to check
        if columns is None:
            columns = df.select_dtypes(include=[np.number]).columns.tolist()

        for col in columns:
            if col not in df.columns:
                continue

            series = df[col].dropna()
            if len(series) < 10:
                continue

            # Detect outliers based on method
            if method == 'zscore':
                outlier_mask = self._detect_zscore_outliers(series, threshold)
            elif method == 'iqr':
                outlier_mask = self._detect_iqr_outliers(series, threshold)
            elif method == 'mad':
                outlier_mask = self._detect_mad_outliers(series, threshold)
            else:
                outlier_mask = self._detect_zscore_outliers(series, threshold)

            n_outliers = outlier_mask.sum()

            if n_outliers == 0:
                continue

            outlier_pct = n_outliers / len(series) * 100

            # Determine severity based on percentage
            if outlier_pct > 5:
                severity = ValidationSeverity.ERROR
            elif outlier_pct > 1:
                severity = ValidationSeverity.WARNING
            else:
                severity = ValidationSeverity.INFO

            # Get outlier statistics
            outlier_values = series[outlier_mask]

            issues.append(ValidationIssue(
                severity=severity,
                category="outliers",
                message=f"Column '{col}' has {n_outliers} outliers ({outlier_pct:.2f}%)",
                affected_rows=n_outliers,
                affected_columns=[col],
                details={
                    'method': method,
                    'threshold': threshold,
                    'outlier_count': n_outliers,
                    'outlier_percentage': outlier_pct,
                    'min_outlier': float(outlier_values.min()),
                    'max_outlier': float(outlier_values.max()),
                    'column_mean': float(series.mean()),
                    'column_std': float(series.std()),
                    'sample_outliers': outlier_values.head(10).tolist()
                }
            ))

        return issues

    def _detect_zscore_outliers(
        self,
        series: pd.Series,
        threshold: float
    ) -> pd.Series:
        """Detect outliers using Z-score method."""
        mean = series.mean()
        std = series.std()
        if std == 0:
            return pd.Series(False, index=series.index)
        z_scores = np.abs((series - mean) / std)
        return z_scores > threshold

    def _detect_iqr_outliers(
        self,
        series: pd.Series,
        threshold: float
    ) -> pd.Series:
        """Detect outliers using IQR method."""
        q1 = series.quantile(0.25)
        q3 = series.quantile(0.75)
        iqr = q3 - q1
        lower_bound = q1 - threshold * iqr
        upper_bound = q3 + threshold * iqr
        return (series < lower_bound) | (series > upper_bound)

    def _detect_mad_outliers(
        self,
        series: pd.Series,
        threshold: float
    ) -> pd.Series:
        """Detect outliers using Median Absolute Deviation."""
        median = series.median()
        mad = np.median(np.abs(series - median))
        if mad == 0:
            return pd.Series(False, index=series.index)
        modified_z = 0.6745 * (series - median) / mad
        return np.abs(modified_z) > threshold

    def validate_consistency(
        self,
        df: pd.DataFrame
    ) -> List[ValidationIssue]:
        """
        Verify OHLC data consistency.

        Checks:
        - High >= Low for all rows
        - High >= Open and High >= Close
        - Low <= Open and Low <= Close
        - Non-negative volume

        Args:
            df: DataFrame with OHLC columns

        Returns:
            List of ValidationIssue objects for consistency violations
        """
        issues = []

        required_cols = ['open', 'high', 'low', 'close']
        missing_cols = [c for c in required_cols if c not in df.columns]

        if missing_cols:
            issues.append(ValidationIssue(
                severity=ValidationSeverity.WARNING,
                category="consistency",
                message=f"Missing OHLC columns: {missing_cols}",
                affected_columns=missing_cols
            ))
            return issues

        # Check High >= Low
        hl_violations = df['high'] < df['low']
        if hl_violations.any():
            n_violations = hl_violations.sum()
            issues.append(ValidationIssue(
                severity=ValidationSeverity.ERROR,
                category="consistency",
                message=f"High < Low in {n_violations} rows",
                affected_rows=n_violations,
                affected_columns=['high', 'low'],
                details={
                    'violation_indices': df.index[hl_violations][:10].tolist()
                }
            ))

        # Check High >= Open and High >= Close
        ho_violations = df['high'] < df['open']
        hc_violations = df['high'] < df['close']
        high_violations = ho_violations | hc_violations

        if high_violations.any():
            n_violations = high_violations.sum()
            issues.append(ValidationIssue(
                severity=ValidationSeverity.ERROR,
                category="consistency",
                message=f"High is not the highest price in {n_violations} rows",
                affected_rows=n_violations,
                affected_columns=['high', 'open', 'close'],
                details={
                    'high_lt_open': int(ho_violations.sum()),
                    'high_lt_close': int(hc_violations.sum())
                }
            ))

        # Check Low <= Open and Low <= Close
        lo_violations = df['low'] > df['open']
        lc_violations = df['low'] > df['close']
        low_violations = lo_violations | lc_violations

        if low_violations.any():
            n_violations = low_violations.sum()
            issues.append(ValidationIssue(
                severity=ValidationSeverity.ERROR,
                category="consistency",
                message=f"Low is not the lowest price in {n_violations} rows",
                affected_rows=n_violations,
                affected_columns=['low', 'open', 'close'],
                details={
                    'low_gt_open': int(lo_violations.sum()),
                    'low_gt_close': int(lc_violations.sum())
                }
            ))

        # Check non-negative volume
        if 'volume' in df.columns:
            neg_volume = df['volume'] < 0
            if neg_volume.any():
                issues.append(ValidationIssue(
                    severity=ValidationSeverity.ERROR,
                    category="consistency",
                    message=f"Negative volume in {neg_volume.sum()} rows",
                    affected_rows=int(neg_volume.sum()),
                    affected_columns=['volume']
                ))

            # Check zero volume (warning only)
            zero_volume = df['volume'] == 0
            if zero_volume.any() and self.min_volume_threshold > 0:
                pct_zero = zero_volume.sum() / len(df) * 100
                issues.append(ValidationIssue(
                    severity=ValidationSeverity.WARNING,
                    category="consistency",
                    message=f"Zero volume in {zero_volume.sum()} rows ({pct_zero:.1f}%)",
                    affected_rows=int(zero_volume.sum()),
                    affected_columns=['volume']
                ))

        # Check for duplicate timestamps
        if isinstance(df.index, pd.DatetimeIndex):
            duplicates = df.index.duplicated()
            if duplicates.any():
                n_dups = duplicates.sum()
                issues.append(ValidationIssue(
                    severity=ValidationSeverity.ERROR,
                    category="consistency",
                    message=f"Found {n_dups} duplicate timestamps",
                    affected_rows=n_dups,
                    details={
                        'duplicate_timestamps': df.index[duplicates][:10].tolist()
                    }
                ))

        return issues

    def validate_missing_values(
        self,
        df: pd.DataFrame
    ) -> List[ValidationIssue]:
        """
        Check for missing values in the data.

        Args:
            df: DataFrame to check

        Returns:
            List of ValidationIssue objects for missing value issues
        """
        issues = []

        # Check each column for missing values
        missing_counts = df.isnull().sum()
        total_rows = len(df)

        for col, missing in missing_counts.items():
            if missing == 0:
                continue

            pct_missing = missing / total_rows * 100

            # Determine severity
            if pct_missing > 10:
                severity = ValidationSeverity.ERROR
            elif pct_missing > 1:
                severity = ValidationSeverity.WARNING
            else:
                severity = ValidationSeverity.INFO

            issues.append(ValidationIssue(
                severity=severity,
                category="missing_values",
                message=f"Column '{col}' has {missing} missing values ({pct_missing:.2f}%)",
                affected_rows=missing,
                affected_columns=[col],
                details={
                    'missing_count': missing,
                    'missing_percentage': pct_missing
                }
            ))

        return issues

    def generate_quality_report(
        self,
        df: pd.DataFrame,
        symbol: str = 'unknown',
        timeframe: str = 'unknown'
    ) -> QualityReport:
        """
        Generate a comprehensive data quality report.

        Args:
            df: DataFrame to analyze
            symbol: Symbol identifier for the report
            timeframe: Timeframe identifier for the report

        Returns:
            QualityReport with all validation results
        """
        logger.info(f"Generating quality report for {symbol} ({timeframe})")

        all_issues = []

        # Run all validations
        all_issues.extend(self.validate_gaps(df))
        all_issues.extend(self.validate_outliers(df))
        all_issues.extend(self.validate_consistency(df))
        all_issues.extend(self.validate_missing_values(df))

        # Calculate metrics
        metrics = self._calculate_metrics(df)
        metrics['symbol'] = symbol
        metrics['timeframe'] = timeframe

        # Determine date range
        if isinstance(df.index, pd.DatetimeIndex) and len(df) > 0:
            date_range = (str(df.index.min()), str(df.index.max()))
        else:
            date_range = ('N/A', 'N/A')

        # Determine if data is valid (no ERROR or CRITICAL issues)
        is_valid = not any(
            issue.severity in (ValidationSeverity.ERROR, ValidationSeverity.CRITICAL)
            for issue in all_issues
        )

        report = QualityReport(
            timestamp=datetime.now().isoformat(),
            total_rows=len(df),
            total_columns=len(df.columns),
            date_range=date_range,
            issues=all_issues,
            metrics=metrics,
            is_valid=is_valid
        )

        # Log summary
        logger.info(f"Quality report generated:")
        logger.info(f"  Valid: {is_valid}")
        logger.info(f"  Issues: {len(all_issues)}")

        return report

    def _calculate_metrics(self, df: pd.DataFrame) -> Dict[str, Any]:
        """Calculate summary metrics for the data."""
        metrics = {
            'row_count': len(df),
            'column_count': len(df.columns),
            'memory_mb': df.memory_usage(deep=True).sum() / 1024 / 1024,
            'columns': list(df.columns),
        }

        # Numeric column statistics
        numeric_cols = df.select_dtypes(include=[np.number]).columns
        for col in numeric_cols[:10]:  # Limit to first 10
            metrics[f'{col}_mean'] = float(df[col].mean()) if not df[col].isna().all() else None
            metrics[f'{col}_std'] = float(df[col].std()) if not df[col].isna().all() else None

        # Time-based metrics if DatetimeIndex
        if isinstance(df.index, pd.DatetimeIndex) and len(df) > 1:
            time_diffs = df.index.to_series().diff().dropna()
            metrics['avg_interval_minutes'] = time_diffs.mean().total_seconds() / 60
            metrics['trading_days'] = len(df.index.normalize().unique())

        return metrics


def validate_trading_data(
    df: pd.DataFrame,
    symbol: str = 'unknown',
    timeframe: str = '5min'
) -> QualityReport:
    """
    Convenience function to validate trading data.

    Args:
        df: DataFrame with OHLCV data
        symbol: Symbol identifier
        timeframe: Data timeframe

    Returns:
        QualityReport with validation results
    """
    validator = DataValidator(expected_frequency=timeframe)
    return validator.generate_quality_report(df, symbol, timeframe)


if __name__ == "__main__":
    # Test the validators
    print("Testing DataValidator...")

    # Create sample data with some issues
    np.random.seed(42)
    n = 1000

    dates = pd.date_range('2024-01-01', periods=n, freq='5min')
    price = 2650 + np.cumsum(np.random.randn(n) * 0.5)

    df = pd.DataFrame({
        'open': price,
        'high': price + np.abs(np.random.randn(n)) * 2,
        'low': price - np.abs(np.random.randn(n)) * 2,
        'close': price + np.random.randn(n) * 0.5,
        'volume': np.random.randint(100, 1000, n)
    }, index=dates)

    # Introduce some issues
    # Add gaps
    df = df.drop(df.index[100:110])  # Create a gap

    # Add outliers
    df.iloc[200, df.columns.get_loc('close')] = 5000  # Outlier

    # Add OHLC inconsistency
    df.iloc[300, df.columns.get_loc('high')] = df.iloc[300]['low'] - 1  # High < Low

    # Add missing values
    df.iloc[400:405, df.columns.get_loc('volume')] = np.nan

    # Test individual validators
    validator = DataValidator(expected_frequency='5min')

    print("\n" + "=" * 60)
    print("Testing Gap Detection")
    print("=" * 60)
    gaps = validator.validate_gaps(df)
    for issue in gaps:
        print(issue)

    print("\n" + "=" * 60)
    print("Testing Outlier Detection")
    print("=" * 60)
    outliers = validator.validate_outliers(df, ['close', 'volume'])
    for issue in outliers:
        print(issue)

    print("\n" + "=" * 60)
    print("Testing Consistency Validation")
    print("=" * 60)
    consistency = validator.validate_consistency(df)
    for issue in consistency:
        print(issue)

    print("\n" + "=" * 60)
    print("Testing Missing Values")
    print("=" * 60)
    missing = validator.validate_missing_values(df)
    for issue in missing:
        print(issue)

    print("\n" + "=" * 60)
    print("Full Quality Report")
    print("=" * 60)
    report = validator.generate_quality_report(df, 'XAUUSD', '5min')
    print(report)

    print("\nAll Issues:")
    for issue in report.issues:
        print(f"  {issue}")

    print("\nTest complete!")
