"""
Phase 2 Pipeline - Complete Integration
Unified pipeline for Phase 2 trading signal generation
"""

import logging
from dataclasses import dataclass, field
from datetime import datetime
from pathlib import Path
from typing import Dict, List, Optional, Any, Tuple
import pandas as pd
import numpy as np
import yaml

from ..data.targets import Phase2TargetBuilder, RRConfig, HorizonConfig
from ..data.validators import DataLeakageValidator, WalkForwardValidator
from ..models.range_predictor import RangePredictor
from ..models.tp_sl_classifier import TPSLClassifier
from ..models.signal_generator import SignalGenerator, TradingSignal
from ..backtesting.rr_backtester import RRBacktester, BacktestConfig
from ..backtesting.metrics import MetricsCalculator, TradingMetrics
from ..utils.audit import Phase1Auditor
from ..utils.signal_logger import SignalLogger


logger = logging.getLogger(__name__)


@dataclass
class PipelineConfig:
    """Configuration for Phase 2 pipeline"""
    # Data paths
    data_path: str = "data/processed"
    model_path: str = "models/phase2"
    output_path: str = "outputs/phase2"

    # Instrument settings
    symbol: str = "XAUUSD"
    timeframe_base: str = "5m"

    # Horizons (in bars of base timeframe)
    horizons: List[int] = field(default_factory=lambda: [3, 12])  # 15m, 1h
    horizon_names: List[str] = field(default_factory=lambda: ["15m", "1h"])

    # R:R configurations
    rr_configs: List[Dict[str, float]] = field(default_factory=lambda: [
        {"sl": 5.0, "tp": 10.0, "name": "rr_2_1"},
        {"sl": 5.0, "tp": 15.0, "name": "rr_3_1"}
    ])

    # ATR settings
    atr_period: int = 14
    atr_bins: List[float] = field(default_factory=lambda: [0.25, 0.5, 1.0])

    # Training settings
    train_split: float = 0.7
    val_split: float = 0.15
    walk_forward_folds: int = 5
    min_fold_size: int = 1000

    # Model settings
    use_gpu: bool = True
    n_estimators: int = 500
    max_depth: int = 6
    learning_rate: float = 0.05

    # Signal generation
    min_confidence: float = 0.55
    min_prob_tp: float = 0.50

    # Logging
    enable_signal_logging: bool = True
    log_format: str = "jsonl"

    @classmethod
    def from_yaml(cls, config_path: str) -> 'PipelineConfig':
        """Load config from YAML file"""
        with open(config_path, 'r') as f:
            config_dict = yaml.safe_load(f)
        return cls(**config_dict)


class Phase2Pipeline:
    """
    Complete Phase 2 Pipeline for trading signal generation.

    This pipeline integrates:
    1. Data validation and audit
    2. Target calculation (ΔHigh/ΔLow, bins, TP/SL labels)
    3. Model training (RangePredictor, TPSLClassifier)
    4. Signal generation
    5. Backtesting
    6. Signal logging for LLM fine-tuning
    """

    def __init__(self, config: Optional[PipelineConfig] = None):
        """Initialize pipeline with configuration"""
        self.config = config or PipelineConfig()

        # Create output directories
        Path(self.config.model_path).mkdir(parents=True, exist_ok=True)
        Path(self.config.output_path).mkdir(parents=True, exist_ok=True)

        # Initialize components
        self.target_builder = None
        self.range_predictor = None
        self.tpsl_classifier = None
        self.signal_generator = None
        self.backtester = None
        self.signal_logger = None

        # State
        self.is_trained = False
        self.training_metrics = {}
        self.backtest_results = {}

    def initialize_components(self):
        """Initialize all pipeline components"""
        logger.info("Initializing Phase 2 pipeline components...")

        # Build RR configs
        rr_configs = [
            RRConfig(
                name=cfg["name"],
                sl_distance=cfg["sl"],
                tp_distance=cfg["tp"]
            )
            for cfg in self.config.rr_configs
        ]

        # Build horizon configs
        horizon_configs = [
            HorizonConfig(
                name=name,
                bars=bars,
                minutes=bars * 5  # 5m base timeframe
            )
            for name, bars in zip(self.config.horizon_names, self.config.horizons)
        ]

        # Initialize target builder
        self.target_builder = Phase2TargetBuilder(
            rr_configs=rr_configs,
            horizon_configs=horizon_configs,
            atr_period=self.config.atr_period,
            atr_bins=self.config.atr_bins
        )

        # Initialize models
        self.range_predictor = RangePredictor(
            horizons=self.config.horizon_names,
            n_estimators=self.config.n_estimators,
            max_depth=self.config.max_depth,
            learning_rate=self.config.learning_rate,
            use_gpu=self.config.use_gpu
        )

        self.tpsl_classifier = TPSLClassifier(
            rr_configs=[cfg["name"] for cfg in self.config.rr_configs],
            horizons=self.config.horizon_names,
            n_estimators=self.config.n_estimators,
            max_depth=self.config.max_depth,
            learning_rate=self.config.learning_rate,
            use_gpu=self.config.use_gpu
        )

        # Initialize signal logger
        if self.config.enable_signal_logging:
            self.signal_logger = SignalLogger(
                output_dir=f"{self.config.output_path}/signals"
            )

        logger.info("Pipeline components initialized")

    def audit_data(self, df: pd.DataFrame) -> Dict[str, Any]:
        """
        Run Phase 1 audit on input data.

        Args:
            df: Input DataFrame

        Returns:
            Audit results dictionary
        """
        logger.info("Running Phase 1 audit...")

        auditor = Phase1Auditor(df)
        report = auditor.run_full_audit()

        audit_results = {
            "passed": report.passed,
            "score": report.overall_score,
            "issues": report.issues,
            "warnings": report.warnings,
            "label_audit": {
                "future_values_used": report.label_audit.future_values_used if report.label_audit else None,
                "current_bar_in_labels": report.label_audit.current_bar_in_labels if report.label_audit else None
            },
            "leakage_check": {
                "has_leakage": report.leakage_check.has_leakage if report.leakage_check else None,
                "leaky_features": report.leakage_check.leaky_features if report.leakage_check else []
            }
        }

        if not report.passed:
            logger.warning(f"Audit issues found: {report.issues}")

        return audit_results

    def prepare_data(
        self,
        df: pd.DataFrame,
        feature_columns: List[str]
    ) -> Tuple[pd.DataFrame, pd.DataFrame]:
        """
        Prepare data with Phase 2 targets.

        Args:
            df: Input DataFrame with OHLCV data
            feature_columns: List of feature column names

        Returns:
            Tuple of (features DataFrame, targets DataFrame)
        """
        logger.info("Preparing Phase 2 targets...")

        # Calculate targets
        df_with_targets = self.target_builder.build_all_targets(df)

        # Get target columns
        target_cols = [col for col in df_with_targets.columns
                       if any(x in col for x in ['delta_high', 'delta_low', 'bin_high',
                                                   'bin_low', 'tp_first', 'atr'])]

        # Validate no leakage
        validator = DataLeakageValidator()
        validation = validator.validate_temporal_split(
            df_with_targets, feature_columns, target_cols,
            train_end_idx=int(len(df_with_targets) * self.config.train_split)
        )

        if not validation.passed:
            logger.error(f"Data leakage detected: {validation.details}")
            raise ValueError("Data leakage detected in preparation")

        # Remove rows with NaN targets (at the end due to horizon)
        df_clean = df_with_targets.dropna(subset=target_cols)

        features = df_clean[feature_columns]
        targets = df_clean[target_cols]

        logger.info(f"Prepared {len(features)} samples with {len(target_cols)} targets")

        return features, targets

    def train(
        self,
        features: pd.DataFrame,
        targets: pd.DataFrame,
        walk_forward: bool = True
    ) -> Dict[str, Any]:
        """
        Train all Phase 2 models.

        Args:
            features: Feature DataFrame
            targets: Target DataFrame
            walk_forward: Use walk-forward validation

        Returns:
            Training metrics dictionary
        """
        logger.info("Training Phase 2 models...")

        # Split data
        n_samples = len(features)
        train_end = int(n_samples * self.config.train_split)
        val_end = int(n_samples * (self.config.train_split + self.config.val_split))

        X_train = features.iloc[:train_end]
        X_val = features.iloc[train_end:val_end]
        X_test = features.iloc[val_end:]

        # Prepare target arrays for each model
        metrics = {}

        # Train RangePredictor for each horizon
        logger.info("Training RangePredictor models...")
        for horizon in self.config.horizon_names:
            y_high_train = targets[f'delta_high_{horizon}'].iloc[:train_end]
            y_low_train = targets[f'delta_low_{horizon}'].iloc[:train_end]
            y_high_val = targets[f'delta_high_{horizon}'].iloc[train_end:val_end]
            y_low_val = targets[f'delta_low_{horizon}'].iloc[train_end:val_end]

            # Regression targets
            range_metrics = self.range_predictor.train(
                X_train.values, y_high_train.values, y_low_train.values,
                X_val.values, y_high_val.values, y_low_val.values,
                horizon=horizon
            )
            metrics[f'range_{horizon}'] = range_metrics

            # Classification targets (bins)
            if f'bin_high_{horizon}' in targets.columns:
                y_bin_high_train = targets[f'bin_high_{horizon}'].iloc[:train_end]
                y_bin_low_train = targets[f'bin_low_{horizon}'].iloc[:train_end]
                y_bin_high_val = targets[f'bin_high_{horizon}'].iloc[train_end:val_end]
                y_bin_low_val = targets[f'bin_low_{horizon}'].iloc[train_end:val_end]

                bin_metrics = self.range_predictor.train_bin_classifiers(
                    X_train.values, y_bin_high_train.values, y_bin_low_train.values,
                    X_val.values, y_bin_high_val.values, y_bin_low_val.values,
                    horizon=horizon
                )
                metrics[f'bins_{horizon}'] = bin_metrics

        # Train TPSLClassifier for each R:R config and horizon
        logger.info("Training TPSLClassifier models...")
        for rr_cfg in self.config.rr_configs:
            rr_name = rr_cfg["name"]
            for horizon in self.config.horizon_names:
                target_col = f'tp_first_{rr_name}_{horizon}'
                if target_col in targets.columns:
                    y_train = targets[target_col].iloc[:train_end]
                    y_val = targets[target_col].iloc[train_end:val_end]

                    tpsl_metrics = self.tpsl_classifier.train(
                        X_train.values, y_train.values,
                        X_val.values, y_val.values,
                        rr_config=rr_name,
                        horizon=horizon
                    )
                    metrics[f'tpsl_{rr_name}_{horizon}'] = tpsl_metrics

        self.training_metrics = metrics
        self.is_trained = True

        # Initialize signal generator with trained models
        self.signal_generator = SignalGenerator(
            range_predictor=self.range_predictor,
            tpsl_classifier=self.tpsl_classifier,
            symbol=self.config.symbol,
            min_confidence=self.config.min_confidence
        )

        logger.info("Phase 2 models trained successfully")
        return metrics

    def generate_signals(
        self,
        features: pd.DataFrame,
        current_prices: pd.Series,
        horizons: Optional[List[str]] = None,
        rr_config: str = "rr_2_1"
    ) -> List[TradingSignal]:
        """
        Generate trading signals for given features.

        Args:
            features: Feature DataFrame
            current_prices: Series of current prices
            horizons: Horizons to generate for (default: all)
            rr_config: R:R configuration to use

        Returns:
            List of TradingSignal objects
        """
        if not self.is_trained:
            raise RuntimeError("Pipeline must be trained before generating signals")

        horizons = horizons or self.config.horizon_names
        signals = []

        for i in range(len(features)):
            for horizon in horizons:
                signal = self.signal_generator.generate_signal(
                    features=features.iloc[i].to_dict(),
                    current_price=current_prices.iloc[i],
                    horizon=horizon,
                    rr_config=rr_config
                )
                if signal:
                    signals.append(signal)

        # Log signals if enabled
        if self.signal_logger and signals:
            for signal in signals:
                self.signal_logger.log_signal(signal.to_dict())

        return signals

    def backtest(
        self,
        df: pd.DataFrame,
        signals: List[TradingSignal],
        initial_capital: float = 10000.0,
        risk_per_trade: float = 0.02
    ) -> Dict[str, Any]:
        """
        Run backtest on generated signals.

        Args:
            df: OHLCV DataFrame
            signals: List of trading signals
            initial_capital: Starting capital
            risk_per_trade: Risk per trade as fraction

        Returns:
            Backtest results dictionary
        """
        logger.info(f"Running backtest on {len(signals)} signals...")

        # Initialize backtester
        backtest_config = BacktestConfig(
            initial_capital=initial_capital,
            risk_per_trade=risk_per_trade,
            commission=0.0,
            slippage=0.0
        )

        self.backtester = RRBacktester(config=backtest_config)

        # Convert signals to backtest format
        trades_data = []
        for signal in signals:
            trades_data.append({
                'timestamp': signal.timestamp,
                'direction': signal.direction,
                'entry_price': signal.entry_price,
                'stop_loss': signal.stop_loss,
                'take_profit': signal.take_profit,
                'horizon_minutes': signal.horizon_minutes,
                'prob_tp_first': signal.prob_tp_first
            })

        # Run backtest
        result = self.backtester.run_backtest(df, trades_data)

        self.backtest_results = {
            'total_trades': result.total_trades,
            'winning_trades': result.winning_trades,
            'winrate': result.winrate,
            'profit_factor': result.profit_factor,
            'net_profit': result.net_profit,
            'max_drawdown': result.max_drawdown,
            'max_drawdown_pct': result.max_drawdown_pct,
            'sharpe_ratio': result.sharpe_ratio,
            'sortino_ratio': result.sortino_ratio
        }

        logger.info(f"Backtest complete: {result.total_trades} trades, "
                   f"Winrate: {result.winrate:.1%}, PF: {result.profit_factor:.2f}")

        return self.backtest_results

    def save_models(self, path: Optional[str] = None):
        """Save trained models"""
        path = path or self.config.model_path
        Path(path).mkdir(parents=True, exist_ok=True)

        self.range_predictor.save(f"{path}/range_predictor")
        self.tpsl_classifier.save(f"{path}/tpsl_classifier")

        # Save config
        with open(f"{path}/config.yaml", 'w') as f:
            yaml.dump(self.config.__dict__, f)

        logger.info(f"Models saved to {path}")

    def load_models(self, path: Optional[str] = None):
        """Load trained models"""
        path = path or self.config.model_path

        self.range_predictor.load(f"{path}/range_predictor")
        self.tpsl_classifier.load(f"{path}/tpsl_classifier")

        # Initialize signal generator
        self.signal_generator = SignalGenerator(
            range_predictor=self.range_predictor,
            tpsl_classifier=self.tpsl_classifier,
            symbol=self.config.symbol,
            min_confidence=self.config.min_confidence
        )

        self.is_trained = True
        logger.info(f"Models loaded from {path}")

    def save_signals_for_finetuning(
        self,
        formats: List[str] = ["jsonl", "openai", "anthropic"]
    ) -> Dict[str, Path]:
        """
        Save logged signals in various formats for LLM fine-tuning.

        Args:
            formats: Output formats to generate

        Returns:
            Dictionary mapping format names to file paths
        """
        if not self.signal_logger:
            raise RuntimeError("Signal logging not enabled")

        output_files = {}

        if "jsonl" in formats:
            output_files["jsonl"] = self.signal_logger.save_jsonl()

        if "openai" in formats:
            output_files["openai"] = self.signal_logger.save_openai_format()

        if "anthropic" in formats:
            output_files["anthropic"] = self.signal_logger.save_anthropic_format()

        return output_files

    def get_summary(self) -> Dict[str, Any]:
        """Get pipeline summary"""
        return {
            "config": {
                "symbol": self.config.symbol,
                "timeframe": self.config.timeframe_base,
                "horizons": self.config.horizon_names,
                "rr_configs": [cfg["name"] for cfg in self.config.rr_configs]
            },
            "is_trained": self.is_trained,
            "training_metrics": self.training_metrics,
            "backtest_results": self.backtest_results,
            "signals_logged": len(self.signal_logger.conversations) if self.signal_logger else 0
        }


def run_phase2_pipeline(
    data_path: str,
    config_path: Optional[str] = None,
    output_path: str = "outputs/phase2"
) -> Dict[str, Any]:
    """
    Convenience function to run the complete Phase 2 pipeline.

    Args:
        data_path: Path to input data
        config_path: Optional path to config YAML
        output_path: Output directory

    Returns:
        Pipeline results dictionary
    """
    # Load config
    if config_path:
        config = PipelineConfig.from_yaml(config_path)
    else:
        config = PipelineConfig(output_path=output_path)

    # Initialize pipeline
    pipeline = Phase2Pipeline(config)
    pipeline.initialize_components()

    # Load data
    df = pd.read_parquet(data_path)

    # Run audit
    audit_results = pipeline.audit_data(df)
    if not audit_results["passed"]:
        logger.warning("Audit issues detected, proceeding with caution")

    # Get feature columns (exclude OHLCV and target-like columns)
    exclude_patterns = ['open', 'high', 'low', 'close', 'volume',
                        'delta_', 'bin_', 'tp_first', 'target']
    feature_cols = [col for col in df.columns
                    if not any(p in col.lower() for p in exclude_patterns)]

    # Prepare data
    features, targets = pipeline.prepare_data(df, feature_cols)

    # Train models
    training_metrics = pipeline.train(features, targets)

    # Generate signals on test set
    test_start = int(len(features) * (config.train_split + config.val_split))
    test_features = features.iloc[test_start:]
    test_prices = df['close'].iloc[test_start:test_start + len(test_features)]

    signals = pipeline.generate_signals(test_features, test_prices)

    # Run backtest
    backtest_results = pipeline.backtest(df.iloc[test_start:], signals)

    # Save models
    pipeline.save_models()

    # Save signals for fine-tuning
    if config.enable_signal_logging:
        pipeline.save_signals_for_finetuning()

    return pipeline.get_summary()


# Export
__all__ = [
    'Phase2Pipeline',
    'PipelineConfig',
    'run_phase2_pipeline'
]
