"""
Backtesting Report Generator
============================
Generate comprehensive backtesting reports for ML model validation.

Supports:
- Individual strategy reports
- Ensemble reports with strategy weights
- Comparison reports across multiple strategies

Output formats:
- Markdown (human-readable)
- JSON (machine-readable)

Author: ML-Specialist (NEXUS v4.0)
Version: 1.0.0
Created: 2026-01-25
"""

import json
from datetime import datetime, timedelta
from pathlib import Path
from typing import Dict, List, Optional, Any, Union
from dataclasses import dataclass, field
import numpy as np
import pandas as pd
from loguru import logger


@dataclass
class ReportSection:
    """Individual section of a report."""

    title: str
    content: str
    data: Optional[Dict[str, Any]] = None
    subsections: List['ReportSection'] = field(default_factory=list)


@dataclass
class PerformanceThresholds:
    """Thresholds for pass/fail evaluation."""

    min_sharpe_ratio: float = 1.0
    min_win_rate: float = 0.45
    min_profit_factor: float = 1.2
    max_drawdown_pct: float = 20.0
    min_trades: int = 30
    min_calmar_ratio: float = 0.5


@dataclass
class StrategyReport:
    """Complete strategy backtest report."""

    strategy_name: str
    generated_at: datetime
    backtest_period: Dict[str, str]
    executive_summary: Dict[str, Any]
    performance_metrics: Dict[str, float]
    trade_statistics: Dict[str, Any]
    drawdown_analysis: Dict[str, Any]
    returns_analysis: Dict[str, Any]
    recommendations: List[str]
    passed_validation: bool
    sections: List[ReportSection] = field(default_factory=list)
    raw_data: Dict[str, Any] = field(default_factory=dict)

    def to_dict(self) -> Dict[str, Any]:
        """Convert report to dictionary."""
        return {
            'strategy_name': self.strategy_name,
            'generated_at': self.generated_at.isoformat(),
            'backtest_period': self.backtest_period,
            'executive_summary': self.executive_summary,
            'performance_metrics': self.performance_metrics,
            'trade_statistics': self.trade_statistics,
            'drawdown_analysis': self.drawdown_analysis,
            'returns_analysis': self.returns_analysis,
            'recommendations': self.recommendations,
            'passed_validation': self.passed_validation,
            'raw_data': self.raw_data
        }


@dataclass
class EnsembleReport:
    """Report for ensemble strategy backtest."""

    ensemble_name: str
    generated_at: datetime
    strategy_weights: Dict[str, float]
    individual_results: Dict[str, Dict[str, Any]]
    combined_metrics: Dict[str, float]
    weight_analysis: Dict[str, Any]
    diversification_benefit: float
    recommendations: List[str]
    passed_validation: bool

    def to_dict(self) -> Dict[str, Any]:
        """Convert report to dictionary."""
        return {
            'ensemble_name': self.ensemble_name,
            'generated_at': self.generated_at.isoformat(),
            'strategy_weights': self.strategy_weights,
            'individual_results': self.individual_results,
            'combined_metrics': self.combined_metrics,
            'weight_analysis': self.weight_analysis,
            'diversification_benefit': self.diversification_benefit,
            'recommendations': self.recommendations,
            'passed_validation': self.passed_validation
        }


@dataclass
class ComparisonReport:
    """Report comparing multiple strategies."""

    generated_at: datetime
    strategies_compared: List[str]
    rankings: Dict[str, List[Dict[str, Any]]]
    correlation_analysis: Dict[str, Any]
    best_strategy: Dict[str, str]
    regime_analysis: Dict[str, Dict[str, str]]
    recommendations: List[str]

    def to_dict(self) -> Dict[str, Any]:
        """Convert report to dictionary."""
        return {
            'generated_at': self.generated_at.isoformat(),
            'strategies_compared': self.strategies_compared,
            'rankings': self.rankings,
            'correlation_analysis': self.correlation_analysis,
            'best_strategy': self.best_strategy,
            'regime_analysis': self.regime_analysis,
            'recommendations': self.recommendations
        }


class ReportGenerator:
    """
    Generate comprehensive backtesting reports.

    Creates detailed reports including:
    - Executive summary with pass/fail status
    - Performance metrics tables
    - Trade statistics breakdown
    - Drawdown analysis
    - Monthly and weekly returns
    - Strategy-specific recommendations

    Usage:
        generator = ReportGenerator()
        report = generator.generate_strategy_report(
            strategy_name='MRD',
            backtest_result=result,
            metrics=metrics
        )
        generator.export_to_markdown(report, 'report.md')
    """

    def __init__(
        self,
        thresholds: Optional[PerformanceThresholds] = None,
        include_raw_data: bool = True,
        include_trade_list: bool = False
    ):
        """
        Initialize report generator.

        Args:
            thresholds: Performance thresholds for pass/fail evaluation
            include_raw_data: Whether to include raw data in reports
            include_trade_list: Whether to include full trade list
        """
        self.thresholds = thresholds or PerformanceThresholds()
        self.include_raw_data = include_raw_data
        self.include_trade_list = include_trade_list

        logger.info("ReportGenerator initialized")

    def generate_strategy_report(
        self,
        strategy_name: str,
        backtest_result: Any,
        metrics: Optional[Dict[str, Any]] = None,
        trades: Optional[List[Any]] = None,
        equity_curve: Optional[Union[pd.Series, np.ndarray]] = None,
        additional_info: Optional[Dict[str, Any]] = None
    ) -> StrategyReport:
        """
        Generate comprehensive report for a single strategy backtest.

        Args:
            strategy_name: Name of the strategy
            backtest_result: BacktestResult object or dictionary
            metrics: Optional pre-computed metrics dictionary
            trades: Optional list of trades
            equity_curve: Optional equity curve data
            additional_info: Optional additional context

        Returns:
            StrategyReport with all analysis
        """
        logger.info(f"Generating report for strategy: {strategy_name}")

        result_dict = self._normalize_result(backtest_result)
        metrics_dict = metrics if metrics else self._extract_metrics(result_dict)
        trade_list = trades if trades else self._extract_trades(result_dict)
        equity_data = equity_curve if equity_curve is not None else self._extract_equity_curve(result_dict)

        backtest_period = self._compute_backtest_period(trade_list, result_dict)
        performance_metrics = self._compile_performance_metrics(metrics_dict, result_dict)
        trade_statistics = self._compute_trade_statistics(trade_list, metrics_dict)
        drawdown_analysis = self._analyze_drawdown(equity_data, metrics_dict)
        returns_analysis = self._analyze_returns(equity_data, trade_list)

        passed_validation = self._evaluate_pass_fail(performance_metrics, trade_statistics)

        executive_summary = self._create_executive_summary(
            strategy_name=strategy_name,
            performance_metrics=performance_metrics,
            trade_statistics=trade_statistics,
            passed_validation=passed_validation
        )

        recommendations = self._generate_recommendations(
            strategy_name=strategy_name,
            performance_metrics=performance_metrics,
            trade_statistics=trade_statistics,
            drawdown_analysis=drawdown_analysis,
            passed_validation=passed_validation
        )

        raw_data = {}
        if self.include_raw_data:
            raw_data['equity_curve'] = self._serialize_equity_curve(equity_data)
            if self.include_trade_list:
                raw_data['trades'] = self._serialize_trades(trade_list)
            if additional_info:
                raw_data['additional_info'] = additional_info

        report = StrategyReport(
            strategy_name=strategy_name,
            generated_at=datetime.now(),
            backtest_period=backtest_period,
            executive_summary=executive_summary,
            performance_metrics=performance_metrics,
            trade_statistics=trade_statistics,
            drawdown_analysis=drawdown_analysis,
            returns_analysis=returns_analysis,
            recommendations=recommendations,
            passed_validation=passed_validation,
            raw_data=raw_data
        )

        logger.info(f"Report generated: {strategy_name} - {'PASSED' if passed_validation else 'FAILED'}")
        return report

    def generate_ensemble_report(
        self,
        ensemble_result: Any,
        strategy_results: Dict[str, Any],
        strategy_weights: Optional[Dict[str, float]] = None,
        ensemble_name: str = "Ensemble"
    ) -> EnsembleReport:
        """
        Generate report for ensemble strategy combining multiple models.

        Args:
            ensemble_result: Combined backtest result for ensemble
            strategy_results: Dictionary mapping strategy names to their results
            strategy_weights: Dictionary of strategy weights
            ensemble_name: Name for the ensemble

        Returns:
            EnsembleReport with combined analysis
        """
        logger.info(f"Generating ensemble report: {ensemble_name}")

        ensemble_dict = self._normalize_result(ensemble_result)
        ensemble_metrics = self._extract_metrics(ensemble_dict)

        individual_results = {}
        individual_sharpes = []

        for strategy_name, result in strategy_results.items():
            result_dict = self._normalize_result(result)
            metrics = self._extract_metrics(result_dict)
            individual_results[strategy_name] = {
                'sharpe_ratio': metrics.get('sharpe_ratio', 0),
                'sortino_ratio': metrics.get('sortino_ratio', 0),
                'win_rate': metrics.get('winrate', metrics.get('win_rate', 0)),
                'profit_factor': metrics.get('profit_factor', 0),
                'total_trades': metrics.get('total_trades', 0),
                'net_profit': metrics.get('net_profit', 0),
                'max_drawdown_pct': metrics.get('max_drawdown_pct', 0)
            }
            individual_sharpes.append(metrics.get('sharpe_ratio', 0))

        if strategy_weights is None:
            num_strategies = len(strategy_results)
            strategy_weights = {name: 1.0 / num_strategies for name in strategy_results.keys()}

        weight_analysis = self._analyze_strategy_weights(
            individual_results=individual_results,
            strategy_weights=strategy_weights
        )

        ensemble_sharpe = ensemble_metrics.get('sharpe_ratio', 0)
        diversification_benefit = self._compute_diversification_benefit(
            individual_sharpes=individual_sharpes,
            ensemble_sharpe=ensemble_sharpe
        )

        combined_metrics = {
            'sharpe_ratio': ensemble_sharpe,
            'sortino_ratio': ensemble_metrics.get('sortino_ratio', 0),
            'win_rate': ensemble_metrics.get('winrate', ensemble_metrics.get('win_rate', 0)),
            'profit_factor': ensemble_metrics.get('profit_factor', 0),
            'total_trades': ensemble_metrics.get('total_trades', 0),
            'net_profit': ensemble_metrics.get('net_profit', 0),
            'max_drawdown_pct': ensemble_metrics.get('max_drawdown_pct', 0),
            'calmar_ratio': ensemble_metrics.get('calmar_ratio', 0)
        }

        passed_validation = self._evaluate_pass_fail(combined_metrics, {'total_trades': combined_metrics['total_trades']})

        recommendations = self._generate_ensemble_recommendations(
            combined_metrics=combined_metrics,
            individual_results=individual_results,
            weight_analysis=weight_analysis,
            diversification_benefit=diversification_benefit,
            passed_validation=passed_validation
        )

        report = EnsembleReport(
            ensemble_name=ensemble_name,
            generated_at=datetime.now(),
            strategy_weights=strategy_weights,
            individual_results=individual_results,
            combined_metrics=combined_metrics,
            weight_analysis=weight_analysis,
            diversification_benefit=diversification_benefit,
            recommendations=recommendations,
            passed_validation=passed_validation
        )

        logger.info(f"Ensemble report generated: {ensemble_name}")
        return report

    def generate_comparison_report(
        self,
        all_results: Dict[str, Any],
        regime_data: Optional[Dict[str, Any]] = None
    ) -> ComparisonReport:
        """
        Generate comparison report across multiple strategies.

        Args:
            all_results: Dictionary mapping strategy names to backtest results
            regime_data: Optional market regime data for regime-specific analysis

        Returns:
            ComparisonReport with rankings and comparisons
        """
        logger.info(f"Generating comparison report for {len(all_results)} strategies")

        strategy_metrics = {}
        for name, result in all_results.items():
            result_dict = self._normalize_result(result)
            strategy_metrics[name] = self._extract_metrics(result_dict)

        rankings = {
            'by_sharpe_ratio': self._rank_strategies(strategy_metrics, 'sharpe_ratio', descending=True),
            'by_sortino_ratio': self._rank_strategies(strategy_metrics, 'sortino_ratio', descending=True),
            'by_profit_factor': self._rank_strategies(strategy_metrics, 'profit_factor', descending=True),
            'by_win_rate': self._rank_strategies(strategy_metrics, 'winrate', descending=True, fallback_key='win_rate'),
            'by_max_drawdown': self._rank_strategies(strategy_metrics, 'max_drawdown_pct', descending=False),
            'by_calmar_ratio': self._rank_strategies(strategy_metrics, 'calmar_ratio', descending=True)
        }

        correlation_analysis = self._compute_correlation_analysis(all_results)

        best_strategy = {
            'overall_sharpe': rankings['by_sharpe_ratio'][0]['strategy'] if rankings['by_sharpe_ratio'] else 'N/A',
            'overall_risk_adjusted': rankings['by_calmar_ratio'][0]['strategy'] if rankings['by_calmar_ratio'] else 'N/A',
            'lowest_drawdown': rankings['by_max_drawdown'][0]['strategy'] if rankings['by_max_drawdown'] else 'N/A',
            'highest_win_rate': rankings['by_win_rate'][0]['strategy'] if rankings['by_win_rate'] else 'N/A'
        }

        regime_analysis = {}
        if regime_data:
            regime_analysis = self._analyze_regime_performance(all_results, regime_data)

        recommendations = self._generate_comparison_recommendations(
            rankings=rankings,
            correlation_analysis=correlation_analysis,
            best_strategy=best_strategy,
            regime_analysis=regime_analysis
        )

        report = ComparisonReport(
            generated_at=datetime.now(),
            strategies_compared=list(all_results.keys()),
            rankings=rankings,
            correlation_analysis=correlation_analysis,
            best_strategy=best_strategy,
            regime_analysis=regime_analysis,
            recommendations=recommendations
        )

        logger.info("Comparison report generated")
        return report

    def export_to_markdown(
        self,
        report: Union[StrategyReport, EnsembleReport, ComparisonReport],
        filepath: Union[str, Path]
    ) -> None:
        """
        Export report to markdown format.

        Args:
            report: Report object to export
            filepath: Output file path
        """
        filepath = Path(filepath)
        filepath.parent.mkdir(parents=True, exist_ok=True)

        if isinstance(report, StrategyReport):
            content = self._strategy_report_to_markdown(report)
        elif isinstance(report, EnsembleReport):
            content = self._ensemble_report_to_markdown(report)
        elif isinstance(report, ComparisonReport):
            content = self._comparison_report_to_markdown(report)
        else:
            raise ValueError(f"Unknown report type: {type(report)}")

        with open(filepath, 'w', encoding='utf-8') as f:
            f.write(content)

        logger.info(f"Report exported to markdown: {filepath}")

    def export_to_json(
        self,
        report: Union[StrategyReport, EnsembleReport, ComparisonReport],
        filepath: Union[str, Path]
    ) -> None:
        """
        Export report to JSON format.

        Args:
            report: Report object to export
            filepath: Output file path
        """
        filepath = Path(filepath)
        filepath.parent.mkdir(parents=True, exist_ok=True)

        report_dict = report.to_dict()

        with open(filepath, 'w', encoding='utf-8') as f:
            json.dump(report_dict, f, indent=2, default=str)

        logger.info(f"Report exported to JSON: {filepath}")

    def _normalize_result(self, result: Any) -> Dict[str, Any]:
        """Normalize backtest result to dictionary format."""
        if isinstance(result, dict):
            return result
        if hasattr(result, 'to_dict'):
            return result.to_dict()
        if hasattr(result, '__dict__'):
            return vars(result)
        return {'result': result}

    def _extract_metrics(self, result_dict: Dict[str, Any]) -> Dict[str, Any]:
        """Extract metrics from result dictionary."""
        if 'metrics' in result_dict and isinstance(result_dict['metrics'], dict):
            metrics = result_dict['metrics'].copy()
            for key in ['sharpe_ratio', 'sortino_ratio', 'profit_factor', 'win_rate', 'winrate',
                        'max_drawdown', 'max_drawdown_pct', 'total_trades', 'net_profit', 'calmar_ratio']:
                if key in result_dict and key not in metrics:
                    metrics[key] = result_dict[key]
            return metrics
        return {k: v for k, v in result_dict.items() if not k.startswith('_') and not callable(v)}

    def _extract_trades(self, result_dict: Dict[str, Any]) -> List[Any]:
        """Extract trades from result dictionary."""
        if 'trades' in result_dict:
            return result_dict['trades']
        return []

    def _extract_equity_curve(self, result_dict: Dict[str, Any]) -> Optional[np.ndarray]:
        """Extract equity curve from result dictionary."""
        if 'equity_curve' in result_dict:
            ec = result_dict['equity_curve']
            if isinstance(ec, pd.Series):
                return ec.values
            if isinstance(ec, np.ndarray):
                return ec
            if isinstance(ec, list):
                return np.array(ec)
        return None

    def _compute_backtest_period(self, trades: List[Any], result_dict: Dict[str, Any]) -> Dict[str, str]:
        """Compute backtest period from trades or result."""
        start_date = None
        end_date = None

        if trades:
            entry_times = []
            exit_times = []
            for t in trades:
                if hasattr(t, 'entry_time'):
                    if t.entry_time:
                        entry_times.append(t.entry_time)
                elif isinstance(t, dict) and 'entry_time' in t:
                    if t['entry_time']:
                        entry_times.append(t['entry_time'])

                if hasattr(t, 'exit_time'):
                    if t.exit_time:
                        exit_times.append(t.exit_time)
                elif isinstance(t, dict) and 'exit_time' in t:
                    if t['exit_time']:
                        exit_times.append(t['exit_time'])

            if entry_times:
                start_date = min(entry_times)
            if exit_times:
                end_date = max(exit_times)

        if start_date is None and 'start_date' in result_dict:
            start_date = result_dict['start_date']
        if end_date is None and 'end_date' in result_dict:
            end_date = result_dict['end_date']

        return {
            'start': str(start_date) if start_date else 'N/A',
            'end': str(end_date) if end_date else 'N/A',
            'duration_days': self._compute_duration_days(start_date, end_date)
        }

    def _compute_duration_days(self, start_date: Any, end_date: Any) -> str:
        """Compute duration in days between dates."""
        if start_date is None or end_date is None:
            return 'N/A'
        try:
            if isinstance(start_date, str):
                start_date = datetime.fromisoformat(start_date.replace('Z', '+00:00'))
            if isinstance(end_date, str):
                end_date = datetime.fromisoformat(end_date.replace('Z', '+00:00'))
            delta = end_date - start_date
            return str(delta.days)
        except Exception:
            return 'N/A'

    def _compile_performance_metrics(self, metrics: Dict[str, Any], result_dict: Dict[str, Any]) -> Dict[str, float]:
        """Compile all performance metrics."""
        compiled = {}

        metric_keys = [
            ('sharpe_ratio', 0.0),
            ('sortino_ratio', 0.0),
            ('calmar_ratio', 0.0),
            ('profit_factor', 0.0),
            ('net_profit', 0.0),
            ('gross_profit', 0.0),
            ('gross_loss', 0.0),
            ('max_drawdown', 0.0),
            ('max_drawdown_pct', 0.0),
            ('total_profit_pct', 0.0),
            ('roi', 0.0)
        ]

        for key, default in metric_keys:
            value = metrics.get(key, result_dict.get(key, default))
            compiled[key] = float(value) if value is not None else default

        win_rate = metrics.get('winrate', metrics.get('win_rate', result_dict.get('win_rate', 0)))
        compiled['win_rate'] = float(win_rate) if win_rate is not None else 0.0

        return compiled

    def _compute_trade_statistics(self, trades: List[Any], metrics: Dict[str, Any]) -> Dict[str, Any]:
        """Compute detailed trade statistics."""
        total_trades = len(trades) if trades else metrics.get('total_trades', 0)
        winning_trades = metrics.get('winning_trades', 0)
        losing_trades = metrics.get('losing_trades', 0)

        if trades and (winning_trades == 0 and losing_trades == 0):
            for t in trades:
                pnl = t.pnl if hasattr(t, 'pnl') else t.get('pnl', 0) if isinstance(t, dict) else 0
                if pnl > 0:
                    winning_trades += 1
                elif pnl < 0:
                    losing_trades += 1

        stats = {
            'total_trades': total_trades,
            'winning_trades': winning_trades,
            'losing_trades': losing_trades,
            'breakeven_trades': total_trades - winning_trades - losing_trades,
            'win_rate': winning_trades / total_trades if total_trades > 0 else 0,
            'avg_win': metrics.get('avg_win', 0),
            'avg_loss': metrics.get('avg_loss', 0),
            'largest_win': metrics.get('largest_win', 0),
            'largest_loss': metrics.get('largest_loss', 0),
            'avg_trade': metrics.get('avg_trade', 0),
            'avg_trade_duration': metrics.get('avg_trade_duration', 0),
            'max_consecutive_wins': metrics.get('max_consecutive_wins', 0),
            'max_consecutive_losses': metrics.get('max_consecutive_losses', 0)
        }

        if trades:
            longs = sum(1 for t in trades if self._get_trade_direction(t) == 'long')
            shorts = sum(1 for t in trades if self._get_trade_direction(t) == 'short')
            stats['long_trades'] = longs
            stats['short_trades'] = shorts

        return stats

    def _get_trade_direction(self, trade: Any) -> str:
        """Get trade direction from trade object."""
        if hasattr(trade, 'direction'):
            return trade.direction
        if hasattr(trade, 'side'):
            return trade.side
        if isinstance(trade, dict):
            return trade.get('direction', trade.get('side', 'unknown'))
        return 'unknown'

    def _analyze_drawdown(self, equity_curve: Optional[np.ndarray], metrics: Dict[str, Any]) -> Dict[str, Any]:
        """Analyze drawdown characteristics."""
        analysis = {
            'max_drawdown': metrics.get('max_drawdown', 0),
            'max_drawdown_pct': metrics.get('max_drawdown_pct', 0),
            'max_drawdown_duration': metrics.get('max_drawdown_duration', 0),
            'avg_drawdown': 0,
            'drawdown_periods': 0,
            'recovery_factor': 0
        }

        if equity_curve is not None and len(equity_curve) > 1:
            running_max = np.maximum.accumulate(equity_curve)
            drawdown = (running_max - equity_curve) / running_max
            drawdown = np.nan_to_num(drawdown, nan=0.0, posinf=0.0, neginf=0.0)

            analysis['max_drawdown_pct'] = float(np.max(drawdown) * 100)
            analysis['avg_drawdown'] = float(np.mean(drawdown[drawdown > 0]) * 100) if np.any(drawdown > 0) else 0

            in_drawdown = drawdown > 0.01
            drawdown_changes = np.diff(in_drawdown.astype(int))
            analysis['drawdown_periods'] = int(np.sum(drawdown_changes == 1))

            net_profit = equity_curve[-1] - equity_curve[0] if len(equity_curve) > 1 else 0
            max_dd = analysis['max_drawdown'] if analysis['max_drawdown'] > 0 else 1
            analysis['recovery_factor'] = float(net_profit / max_dd) if max_dd > 0 else 0

        return analysis

    def _analyze_returns(self, equity_curve: Optional[np.ndarray], trades: List[Any]) -> Dict[str, Any]:
        """Analyze returns distribution."""
        analysis = {
            'monthly_returns': {},
            'weekly_returns': {},
            'daily_stats': {
                'best_day': 0,
                'worst_day': 0,
                'avg_daily_return': 0,
                'daily_volatility': 0
            }
        }

        if equity_curve is not None and len(equity_curve) > 1:
            returns = np.diff(equity_curve) / equity_curve[:-1]
            returns = np.nan_to_num(returns, nan=0.0, posinf=0.0, neginf=0.0)

            if len(returns) > 0:
                analysis['daily_stats']['best_day'] = float(np.max(returns) * 100)
                analysis['daily_stats']['worst_day'] = float(np.min(returns) * 100)
                analysis['daily_stats']['avg_daily_return'] = float(np.mean(returns) * 100)
                analysis['daily_stats']['daily_volatility'] = float(np.std(returns) * 100)

        if trades:
            monthly_pnl = {}
            weekly_pnl = {}

            for t in trades:
                exit_time = None
                pnl = 0

                if hasattr(t, 'exit_time') and hasattr(t, 'pnl'):
                    exit_time = t.exit_time
                    pnl = t.pnl
                elif isinstance(t, dict):
                    exit_time = t.get('exit_time')
                    pnl = t.get('pnl', 0)

                if exit_time and pnl:
                    if isinstance(exit_time, str):
                        try:
                            exit_time = datetime.fromisoformat(exit_time.replace('Z', '+00:00'))
                        except Exception:
                            continue

                    month_key = exit_time.strftime('%Y-%m')
                    week_key = exit_time.strftime('%Y-W%W')

                    monthly_pnl[month_key] = monthly_pnl.get(month_key, 0) + pnl
                    weekly_pnl[week_key] = weekly_pnl.get(week_key, 0) + pnl

            analysis['monthly_returns'] = {k: float(v) for k, v in sorted(monthly_pnl.items())}
            analysis['weekly_returns'] = {k: float(v) for k, v in sorted(weekly_pnl.items())}

        return analysis

    def _evaluate_pass_fail(self, performance_metrics: Dict[str, float], trade_statistics: Dict[str, Any]) -> bool:
        """Evaluate if strategy passes validation thresholds."""
        total_trades = trade_statistics.get('total_trades', 0)
        if total_trades < self.thresholds.min_trades:
            return False

        sharpe = performance_metrics.get('sharpe_ratio', 0)
        if sharpe < self.thresholds.min_sharpe_ratio:
            return False

        win_rate = performance_metrics.get('win_rate', 0)
        if win_rate < self.thresholds.min_win_rate:
            return False

        profit_factor = performance_metrics.get('profit_factor', 0)
        if profit_factor < self.thresholds.min_profit_factor:
            return False

        max_dd = performance_metrics.get('max_drawdown_pct', 100)
        if max_dd > self.thresholds.max_drawdown_pct:
            return False

        return True

    def _create_executive_summary(
        self,
        strategy_name: str,
        performance_metrics: Dict[str, float],
        trade_statistics: Dict[str, Any],
        passed_validation: bool
    ) -> Dict[str, Any]:
        """Create executive summary section."""
        return {
            'strategy': strategy_name,
            'status': 'PASSED' if passed_validation else 'FAILED',
            'key_metrics': {
                'sharpe_ratio': round(performance_metrics.get('sharpe_ratio', 0), 2),
                'win_rate': f"{performance_metrics.get('win_rate', 0) * 100:.1f}%",
                'profit_factor': round(performance_metrics.get('profit_factor', 0), 2),
                'total_trades': trade_statistics.get('total_trades', 0),
                'net_profit': round(performance_metrics.get('net_profit', 0), 2),
                'max_drawdown': f"{performance_metrics.get('max_drawdown_pct', 0):.1f}%"
            },
            'verdict': self._generate_verdict(performance_metrics, trade_statistics, passed_validation)
        }

    def _generate_verdict(
        self,
        performance_metrics: Dict[str, float],
        trade_statistics: Dict[str, Any],
        passed_validation: bool
    ) -> str:
        """Generate human-readable verdict."""
        if passed_validation:
            sharpe = performance_metrics.get('sharpe_ratio', 0)
            if sharpe >= 2.0:
                return "Excellent performance. Strategy is ready for live trading with full position size."
            elif sharpe >= 1.5:
                return "Good performance. Strategy is suitable for live trading with moderate position size."
            else:
                return "Acceptable performance. Strategy passes minimum thresholds, consider paper trading first."
        else:
            issues = []
            if trade_statistics.get('total_trades', 0) < self.thresholds.min_trades:
                issues.append("insufficient trades")
            if performance_metrics.get('sharpe_ratio', 0) < self.thresholds.min_sharpe_ratio:
                issues.append("low Sharpe ratio")
            if performance_metrics.get('win_rate', 0) < self.thresholds.min_win_rate:
                issues.append("low win rate")
            if performance_metrics.get('profit_factor', 0) < self.thresholds.min_profit_factor:
                issues.append("low profit factor")
            if performance_metrics.get('max_drawdown_pct', 100) > self.thresholds.max_drawdown_pct:
                issues.append("excessive drawdown")

            return f"Strategy does not meet minimum requirements: {', '.join(issues)}."

    def _generate_recommendations(
        self,
        strategy_name: str,
        performance_metrics: Dict[str, float],
        trade_statistics: Dict[str, Any],
        drawdown_analysis: Dict[str, Any],
        passed_validation: bool
    ) -> List[str]:
        """Generate actionable recommendations."""
        recommendations = []

        if not passed_validation:
            recommendations.append("Do not deploy this strategy to live trading until issues are resolved.")

        sharpe = performance_metrics.get('sharpe_ratio', 0)
        if sharpe < 1.0:
            recommendations.append("Consider adding filters to improve signal quality and increase Sharpe ratio.")
        elif sharpe > 2.5:
            recommendations.append("Excellent Sharpe ratio. Verify results are not due to overfitting.")

        win_rate = performance_metrics.get('win_rate', 0)
        profit_factor = performance_metrics.get('profit_factor', 0)

        if win_rate < 0.5 and profit_factor > 1.5:
            recommendations.append("Low win rate but good profit factor suggests the strategy relies on large winners. Ensure psychological readiness for losing streaks.")
        elif win_rate > 0.6 and profit_factor < 1.2:
            recommendations.append("High win rate but low profit factor. Review risk:reward ratios and consider widening take profit targets.")

        max_dd = drawdown_analysis.get('max_drawdown_pct', 0)
        if max_dd > 15:
            recommendations.append(f"Maximum drawdown of {max_dd:.1f}% is significant. Consider position sizing reduction or adding filters during adverse conditions.")

        total_trades = trade_statistics.get('total_trades', 0)
        if total_trades < 100:
            recommendations.append("Limited trade count. Results may not be statistically significant. Extend backtest period.")

        consecutive_losses = trade_statistics.get('max_consecutive_losses', 0)
        if consecutive_losses >= 8:
            recommendations.append(f"Strategy experienced {consecutive_losses} consecutive losses. Ensure adequate capital to withstand losing streaks.")

        if not recommendations:
            recommendations.append("Strategy shows strong performance across all metrics. Continue monitoring in forward testing.")

        return recommendations

    def _analyze_strategy_weights(
        self,
        individual_results: Dict[str, Dict[str, Any]],
        strategy_weights: Dict[str, float]
    ) -> Dict[str, Any]:
        """Analyze strategy weight contribution."""
        analysis = {
            'weight_distribution': strategy_weights,
            'contribution_analysis': {},
            'suggested_weights': {}
        }

        total_sharpe = sum(r.get('sharpe_ratio', 0) * strategy_weights.get(name, 0)
                         for name, r in individual_results.items())

        for name, result in individual_results.items():
            sharpe = result.get('sharpe_ratio', 0)
            weight = strategy_weights.get(name, 0)
            contribution = sharpe * weight / total_sharpe if total_sharpe > 0 else 0

            analysis['contribution_analysis'][name] = {
                'weight': weight,
                'sharpe': sharpe,
                'weighted_contribution': round(contribution * 100, 1)
            }

        sharpe_sum = sum(max(0, r.get('sharpe_ratio', 0)) for r in individual_results.values())
        for name, result in individual_results.items():
            sharpe = max(0, result.get('sharpe_ratio', 0))
            suggested = sharpe / sharpe_sum if sharpe_sum > 0 else 1.0 / len(individual_results)
            analysis['suggested_weights'][name] = round(suggested, 3)

        return analysis

    def _compute_diversification_benefit(
        self,
        individual_sharpes: List[float],
        ensemble_sharpe: float
    ) -> float:
        """Compute diversification benefit of ensemble."""
        if not individual_sharpes:
            return 0.0

        avg_individual = np.mean(individual_sharpes)
        if avg_individual <= 0:
            return 0.0

        benefit = (ensemble_sharpe - avg_individual) / avg_individual * 100
        return float(round(benefit, 2))

    def _generate_ensemble_recommendations(
        self,
        combined_metrics: Dict[str, float],
        individual_results: Dict[str, Dict[str, Any]],
        weight_analysis: Dict[str, Any],
        diversification_benefit: float,
        passed_validation: bool
    ) -> List[str]:
        """Generate recommendations for ensemble."""
        recommendations = []

        if not passed_validation:
            recommendations.append("Ensemble does not meet minimum validation criteria.")

        if diversification_benefit < 0:
            recommendations.append("Negative diversification benefit. Consider removing highly correlated strategies.")
        elif diversification_benefit < 10:
            recommendations.append("Low diversification benefit. Strategies may be too correlated.")
        elif diversification_benefit > 30:
            recommendations.append("Excellent diversification benefit. Ensemble is well-diversified.")

        for name, analysis in weight_analysis.get('contribution_analysis', {}).items():
            if analysis.get('weighted_contribution', 0) < 5:
                recommendations.append(f"Strategy '{name}' has minimal contribution. Consider removing or increasing weight.")

        if not recommendations:
            recommendations.append("Ensemble shows healthy diversification and performance.")

        return recommendations

    def _rank_strategies(
        self,
        strategy_metrics: Dict[str, Dict[str, Any]],
        metric_key: str,
        descending: bool = True,
        fallback_key: Optional[str] = None
    ) -> List[Dict[str, Any]]:
        """Rank strategies by a specific metric."""
        rankings = []

        for name, metrics in strategy_metrics.items():
            value = metrics.get(metric_key)
            if value is None and fallback_key:
                value = metrics.get(fallback_key, 0)
            if value is None:
                value = 0

            rankings.append({
                'strategy': name,
                'value': float(value),
                'metric': metric_key
            })

        rankings.sort(key=lambda x: x['value'], reverse=descending)

        for i, r in enumerate(rankings):
            r['rank'] = i + 1

        return rankings

    def _compute_correlation_analysis(self, all_results: Dict[str, Any]) -> Dict[str, Any]:
        """Compute correlation analysis between strategies."""
        analysis = {
            'correlation_matrix': {},
            'high_correlations': [],
            'diversification_score': 0
        }

        strategy_returns = {}
        for name, result in all_results.items():
            result_dict = self._normalize_result(result)
            ec = self._extract_equity_curve(result_dict)
            if ec is not None and len(ec) > 1:
                returns = np.diff(ec) / ec[:-1]
                returns = np.nan_to_num(returns, nan=0.0, posinf=0.0, neginf=0.0)
                strategy_returns[name] = returns

        if len(strategy_returns) < 2:
            return analysis

        names = list(strategy_returns.keys())
        min_len = min(len(r) for r in strategy_returns.values())

        for i, name_i in enumerate(names):
            analysis['correlation_matrix'][name_i] = {}
            for j, name_j in enumerate(names):
                returns_i = strategy_returns[name_i][:min_len]
                returns_j = strategy_returns[name_j][:min_len]

                if len(returns_i) > 1:
                    corr = np.corrcoef(returns_i, returns_j)[0, 1]
                    corr = 0.0 if np.isnan(corr) else corr
                else:
                    corr = 0.0

                analysis['correlation_matrix'][name_i][name_j] = round(corr, 3)

                if i < j and abs(corr) > 0.7:
                    analysis['high_correlations'].append({
                        'pair': (name_i, name_j),
                        'correlation': round(corr, 3)
                    })

        correlations = []
        for i, name_i in enumerate(names):
            for j, name_j in enumerate(names):
                if i < j:
                    correlations.append(analysis['correlation_matrix'][name_i][name_j])

        if correlations:
            avg_corr = np.mean(np.abs(correlations))
            analysis['diversification_score'] = round((1 - avg_corr) * 100, 1)

        return analysis

    def _analyze_regime_performance(
        self,
        all_results: Dict[str, Any],
        regime_data: Dict[str, Any]
    ) -> Dict[str, Dict[str, str]]:
        """Analyze performance by market regime."""
        regimes = regime_data.get('regimes', ['low_volatility', 'high_volatility', 'trending', 'ranging'])
        regime_analysis = {}

        for regime in regimes:
            best_strategy = None
            best_metric = float('-inf')

            for name, result in all_results.items():
                result_dict = self._normalize_result(result)
                metrics = self._extract_metrics(result_dict)

                metrics_by_regime = metrics.get('metrics_by_volatility', {})
                regime_metrics = metrics_by_regime.get(regime, {})
                sharpe = regime_metrics.get('sharpe_ratio', metrics.get('sharpe_ratio', 0))

                if sharpe > best_metric:
                    best_metric = sharpe
                    best_strategy = name

            regime_analysis[regime] = {
                'best_strategy': best_strategy or 'N/A',
                'sharpe_ratio': round(best_metric, 2) if best_metric > float('-inf') else 0
            }

        return regime_analysis

    def _generate_comparison_recommendations(
        self,
        rankings: Dict[str, List[Dict[str, Any]]],
        correlation_analysis: Dict[str, Any],
        best_strategy: Dict[str, str],
        regime_analysis: Dict[str, Dict[str, str]]
    ) -> List[str]:
        """Generate recommendations for strategy comparison."""
        recommendations = []

        top_sharpe = rankings['by_sharpe_ratio'][0] if rankings['by_sharpe_ratio'] else None
        if top_sharpe:
            recommendations.append(
                f"'{top_sharpe['strategy']}' has the highest Sharpe ratio ({top_sharpe['value']:.2f}). "
                f"Consider this as the primary strategy."
            )

        if correlation_analysis.get('high_correlations'):
            for pair_info in correlation_analysis['high_correlations'][:3]:
                pair = pair_info['pair']
                corr = pair_info['correlation']
                recommendations.append(
                    f"Strategies '{pair[0]}' and '{pair[1]}' have high correlation ({corr:.2f}). "
                    f"Running both provides limited diversification."
                )

        diversification_score = correlation_analysis.get('diversification_score', 0)
        if diversification_score < 30:
            recommendations.append(
                f"Low diversification score ({diversification_score:.0f}%). "
                f"Consider adding uncorrelated strategies."
            )
        elif diversification_score > 70:
            recommendations.append(
                f"Good diversification score ({diversification_score:.0f}%). "
                f"Strategies are relatively uncorrelated."
            )

        if not recommendations:
            recommendations.append("Review individual strategy metrics to make deployment decisions.")

        return recommendations

    def _serialize_equity_curve(self, equity_curve: Optional[np.ndarray]) -> List[float]:
        """Serialize equity curve for JSON export."""
        if equity_curve is None:
            return []
        return [float(x) for x in equity_curve]

    def _serialize_trades(self, trades: List[Any]) -> List[Dict[str, Any]]:
        """Serialize trades for JSON export."""
        serialized = []
        for t in trades:
            if hasattr(t, 'to_dict'):
                serialized.append(t.to_dict())
            elif isinstance(t, dict):
                serialized.append(t)
            else:
                serialized.append(str(t))
        return serialized

    def _strategy_report_to_markdown(self, report: StrategyReport) -> str:
        """Convert strategy report to markdown."""
        lines = [
            f"# Backtest Report: {report.strategy_name}",
            "",
            f"**Generated:** {report.generated_at.strftime('%Y-%m-%d %H:%M:%S')}",
            f"**Status:** {'PASSED' if report.passed_validation else 'FAILED'}",
            "",
            "---",
            "",
            "## Executive Summary",
            "",
        ]

        summary = report.executive_summary
        lines.extend([
            f"- **Strategy:** {summary.get('strategy', 'N/A')}",
            f"- **Status:** {summary.get('status', 'N/A')}",
            f"- **Verdict:** {summary.get('verdict', 'N/A')}",
            "",
            "### Key Metrics",
            ""
        ])

        key_metrics = summary.get('key_metrics', {})
        for k, v in key_metrics.items():
            lines.append(f"- **{k.replace('_', ' ').title()}:** {v}")

        lines.extend([
            "",
            "---",
            "",
            "## Performance Metrics",
            "",
            "| Metric | Value |",
            "|--------|-------|"
        ])

        for metric, value in report.performance_metrics.items():
            if isinstance(value, float):
                formatted = f"{value:.4f}"
            else:
                formatted = str(value)
            lines.append(f"| {metric.replace('_', ' ').title()} | {formatted} |")

        lines.extend([
            "",
            "---",
            "",
            "## Trade Statistics",
            "",
            "| Statistic | Value |",
            "|-----------|-------|"
        ])

        for stat, value in report.trade_statistics.items():
            if isinstance(value, float):
                formatted = f"{value:.4f}"
            else:
                formatted = str(value)
            lines.append(f"| {stat.replace('_', ' ').title()} | {formatted} |")

        lines.extend([
            "",
            "---",
            "",
            "## Drawdown Analysis",
            "",
        ])

        for key, value in report.drawdown_analysis.items():
            if isinstance(value, float):
                formatted = f"{value:.4f}"
            else:
                formatted = str(value)
            lines.append(f"- **{key.replace('_', ' ').title()}:** {formatted}")

        lines.extend([
            "",
            "---",
            "",
            "## Returns Analysis",
            "",
            "### Daily Statistics",
            ""
        ])

        daily_stats = report.returns_analysis.get('daily_stats', {})
        for key, value in daily_stats.items():
            if isinstance(value, float):
                formatted = f"{value:.4f}"
            else:
                formatted = str(value)
            lines.append(f"- **{key.replace('_', ' ').title()}:** {formatted}")

        if report.returns_analysis.get('monthly_returns'):
            lines.extend([
                "",
                "### Monthly Returns",
                ""
            ])
            for month, value in list(report.returns_analysis['monthly_returns'].items())[-12:]:
                lines.append(f"- **{month}:** ${value:,.2f}")

        lines.extend([
            "",
            "---",
            "",
            "## Recommendations",
            ""
        ])

        for i, rec in enumerate(report.recommendations, 1):
            lines.append(f"{i}. {rec}")

        lines.extend([
            "",
            "---",
            "",
            "*Report generated by ML-Engine Backtesting Module*"
        ])

        return "\n".join(lines)

    def _ensemble_report_to_markdown(self, report: EnsembleReport) -> str:
        """Convert ensemble report to markdown."""
        lines = [
            f"# Ensemble Backtest Report: {report.ensemble_name}",
            "",
            f"**Generated:** {report.generated_at.strftime('%Y-%m-%d %H:%M:%S')}",
            f"**Status:** {'PASSED' if report.passed_validation else 'FAILED'}",
            f"**Diversification Benefit:** {report.diversification_benefit:.1f}%",
            "",
            "---",
            "",
            "## Strategy Weights",
            "",
            "| Strategy | Weight |",
            "|----------|--------|"
        ]

        for strategy, weight in report.strategy_weights.items():
            lines.append(f"| {strategy} | {weight:.1%} |")

        lines.extend([
            "",
            "---",
            "",
            "## Combined Metrics",
            "",
            "| Metric | Value |",
            "|--------|-------|"
        ])

        for metric, value in report.combined_metrics.items():
            if isinstance(value, float):
                formatted = f"{value:.4f}"
            else:
                formatted = str(value)
            lines.append(f"| {metric.replace('_', ' ').title()} | {formatted} |")

        lines.extend([
            "",
            "---",
            "",
            "## Individual Strategy Performance",
            ""
        ])

        for strategy, metrics in report.individual_results.items():
            lines.append(f"### {strategy}")
            lines.append("")
            for k, v in metrics.items():
                if isinstance(v, float):
                    formatted = f"{v:.4f}"
                else:
                    formatted = str(v)
                lines.append(f"- **{k.replace('_', ' ').title()}:** {formatted}")
            lines.append("")

        lines.extend([
            "---",
            "",
            "## Weight Analysis",
            "",
            "### Contribution Analysis",
            ""
        ])

        for strategy, analysis in report.weight_analysis.get('contribution_analysis', {}).items():
            lines.append(f"- **{strategy}:** {analysis.get('weighted_contribution', 0):.1f}% contribution")

        lines.extend([
            "",
            "### Suggested Weights",
            ""
        ])

        for strategy, weight in report.weight_analysis.get('suggested_weights', {}).items():
            lines.append(f"- **{strategy}:** {weight:.1%}")

        lines.extend([
            "",
            "---",
            "",
            "## Recommendations",
            ""
        ])

        for i, rec in enumerate(report.recommendations, 1):
            lines.append(f"{i}. {rec}")

        lines.extend([
            "",
            "---",
            "",
            "*Report generated by ML-Engine Backtesting Module*"
        ])

        return "\n".join(lines)

    def _comparison_report_to_markdown(self, report: ComparisonReport) -> str:
        """Convert comparison report to markdown."""
        lines = [
            "# Strategy Comparison Report",
            "",
            f"**Generated:** {report.generated_at.strftime('%Y-%m-%d %H:%M:%S')}",
            f"**Strategies Compared:** {', '.join(report.strategies_compared)}",
            "",
            "---",
            "",
            "## Best Strategies",
            ""
        ]

        for category, strategy in report.best_strategy.items():
            lines.append(f"- **{category.replace('_', ' ').title()}:** {strategy}")

        lines.extend([
            "",
            "---",
            "",
            "## Rankings",
            ""
        ])

        for ranking_type, rankings in report.rankings.items():
            lines.append(f"### {ranking_type.replace('_', ' ').title()}")
            lines.append("")
            lines.append("| Rank | Strategy | Value |")
            lines.append("|------|----------|-------|")
            for r in rankings:
                lines.append(f"| {r['rank']} | {r['strategy']} | {r['value']:.4f} |")
            lines.append("")

        lines.extend([
            "---",
            "",
            "## Correlation Analysis",
            "",
            f"**Diversification Score:** {report.correlation_analysis.get('diversification_score', 0):.1f}%",
            ""
        ])

        if report.correlation_analysis.get('high_correlations'):
            lines.append("### High Correlations")
            lines.append("")
            for pair_info in report.correlation_analysis['high_correlations']:
                pair = pair_info['pair']
                corr = pair_info['correlation']
                lines.append(f"- {pair[0]} <-> {pair[1]}: {corr:.3f}")
            lines.append("")

        if report.regime_analysis:
            lines.extend([
                "---",
                "",
                "## Regime Analysis",
                "",
                "| Regime | Best Strategy | Sharpe |",
                "|--------|---------------|--------|"
            ])
            for regime, analysis in report.regime_analysis.items():
                lines.append(f"| {regime.replace('_', ' ').title()} | {analysis['best_strategy']} | {analysis['sharpe_ratio']:.2f} |")
            lines.append("")

        lines.extend([
            "---",
            "",
            "## Recommendations",
            ""
        ])

        for i, rec in enumerate(report.recommendations, 1):
            lines.append(f"{i}. {rec}")

        lines.extend([
            "",
            "---",
            "",
            "*Report generated by ML-Engine Backtesting Module*"
        ])

        return "\n".join(lines)
