"""
Movement Magnitude Predictor
============================
Predicts price movement magnitude in USD for asymmetric trading opportunities.

Key Concept:
- Normal Gold movement: ~$5 USD in a period
- Good opportunity: Predicted high=$10-15, predicted low=$5 -> Long with 1:2 RR
- Identifies asymmetric moves for favorable risk:reward trades

Horizons:
- 5m candles -> 15 min prediction (3 bars)
- 15m candles -> 60 min prediction (4 bars)

Enhanced with:
- Sample weighting by movement magnitude
- Session and volatility-based attention
- Improved session features (cyclical encoding)

Author: ML-Specialist (NEXUS v4.0)
Date: 2026-01-04
Version: 2.0.0 (2026-01-05) - Added weighting support
"""

import numpy as np
import pandas as pd
from dataclasses import dataclass, field
from typing import Dict, List, Optional, Tuple, Any
from pathlib import Path
import joblib
import yaml
from loguru import logger
from datetime import datetime

try:
    from xgboost import XGBRegressor
    HAS_XGBOOST = True
except ImportError:
    HAS_XGBOOST = False
    logger.warning("XGBoost not available")

from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score

# Import weighting modules
try:
    from ..training.sample_weighting import SampleWeighter, SampleWeightConfig
    from ..training.session_volatility_weighting import (
        SessionVolatilityWeighter, SessionWeightConfig, create_session_features
    )
    HAS_WEIGHTING = True
except ImportError:
    HAS_WEIGHTING = False
    logger.warning("Weighting modules not available, using uniform weights")


@dataclass
class MovementPrediction:
    """Prediction of price movement magnitude"""
    timeframe: str              # '5m' or '15m'
    horizon_minutes: int        # 15 or 60

    # Predictions in USD
    predicted_high_usd: float   # Max upward move predicted
    predicted_low_usd: float    # Max downward move predicted (positive value)

    # Reference volatility
    baseline_move_usd: float    # Normal/average move for this period
    std_move_usd: float         # Std dev of moves

    # Opportunity assessment
    asymmetry_ratio: float      # high/low ratio (>1.5 = bullish, <0.67 = bearish)
    opportunity_score: float    # How good is the opportunity (0-1)
    suggested_direction: str    # 'LONG', 'SHORT', 'NEUTRAL'
    suggested_rr: float         # Suggested Risk:Reward ratio

    # Confidence
    confidence: float           # Model confidence (0-1)
    timestamp: Optional[datetime] = None

    def to_dict(self) -> Dict:
        return {
            'timeframe': self.timeframe,
            'horizon_minutes': self.horizon_minutes,
            'predicted_high_usd': round(self.predicted_high_usd, 2),
            'predicted_low_usd': round(self.predicted_low_usd, 2),
            'baseline_move_usd': round(self.baseline_move_usd, 2),
            'std_move_usd': round(self.std_move_usd, 2),
            'asymmetry_ratio': round(self.asymmetry_ratio, 3),
            'opportunity_score': round(self.opportunity_score, 3),
            'suggested_direction': self.suggested_direction,
            'suggested_rr': round(self.suggested_rr, 2),
            'confidence': round(self.confidence, 3),
            'timestamp': self.timestamp.isoformat() if self.timestamp else None
        }


@dataclass
class MovementMetrics:
    """Metrics for movement magnitude prediction"""
    timeframe: str
    horizon_minutes: int
    target_type: str  # 'high' or 'low'

    # Error metrics in USD
    mae_usd: float = 0.0
    rmse_usd: float = 0.0

    # Relative metrics
    mape: float = 0.0
    r2: float = 0.0

    # Trading-specific
    asymmetry_accuracy: float = 0.0  # How often we correctly predict asymmetry
    avg_predicted_rr: float = 0.0     # Average R:R when we predict opportunity
    profitable_signals: float = 0.0   # % of signals that would be profitable

    n_samples: int = 0

    def to_dict(self) -> Dict:
        return {
            'timeframe': self.timeframe,
            'horizon_minutes': self.horizon_minutes,
            'target_type': self.target_type,
            'mae_usd': round(self.mae_usd, 4),
            'rmse_usd': round(self.rmse_usd, 4),
            'mape': round(self.mape, 4),
            'r2': round(self.r2, 4),
            'asymmetry_accuracy': round(self.asymmetry_accuracy, 4),
            'avg_predicted_rr': round(self.avg_predicted_rr, 4),
            'profitable_signals': round(self.profitable_signals, 4),
            'n_samples': self.n_samples
        }


class MovementMagnitudePredictor:
    """
    Predicts price movement magnitude in absolute USD for trading opportunities.

    Strategy:
    - Predict max high and max low movements in USD over a horizon
    - Calculate asymmetry ratio (high/low)
    - When asymmetry > threshold, signal opportunity with favorable R:R

    Example for Gold:
    - Normal 15-min range: $5
    - Predict: High=$12, Low=$4 -> Asymmetry=3.0 -> LONG opportunity, 1:3 RR
    - Predict: High=$4, Low=$10 -> Asymmetry=0.4 -> SHORT opportunity, 1:2.5 RR
    """

    # Configuration for each timeframe/horizon combination
    HORIZON_CONFIGS = {
        '5m_15min': {
            'timeframe': '5m',
            'horizon_minutes': 15,
            'bars_ahead': 3,
            'feature_windows': [6, 12, 24, 48],  # 30m, 1h, 2h, 4h
            'min_samples': 10000,
        },
        '15m_60min': {
            'timeframe': '15m',
            'horizon_minutes': 60,
            'bars_ahead': 4,
            'feature_windows': [4, 8, 16, 32],  # 1h, 2h, 4h, 8h
            'min_samples': 5000,
        },
        '15m_45min': {
            'timeframe': '15m',
            'horizon_minutes': 45,
            'bars_ahead': 3,
            'feature_windows': [4, 8, 16, 32],
            'min_samples': 5000,
        }
    }

    def __init__(
        self,
        horizons: Optional[List[str]] = None,
        use_gpu: bool = True,
        asymmetry_threshold: float = 1.5,  # Ratio threshold for opportunity
        min_move_usd: float = 3.0,  # Minimum move to consider (noise filter)
        use_sample_weighting: bool = True,
        use_session_weighting: bool = False,  # Disabled by default - only ATR volatility weighting
        sample_weight_config: Optional[Dict] = None,
        session_weight_config: Optional[Dict] = None,
    ):
        """
        Initialize Movement Magnitude Predictor.

        Args:
            horizons: List of horizon configs to use (default: all)
            use_gpu: Whether to use GPU acceleration
            asymmetry_threshold: Minimum asymmetry ratio to signal opportunity
            min_move_usd: Minimum USD move to filter noise
            use_sample_weighting: Enable movement-based sample weighting
            use_session_weighting: Enable session/hour-based weighting (disabled by default)
            sample_weight_config: Configuration for SampleWeighter
            session_weight_config: Configuration for SessionVolatilityWeighter (uses ATR by default)
        """
        self.horizons = horizons or list(self.HORIZON_CONFIGS.keys())
        self.use_gpu = use_gpu
        self.asymmetry_threshold = asymmetry_threshold
        self.min_move_usd = min_move_usd

        self.models: Dict[str, Dict[str, Any]] = {}  # {horizon: {'high': model, 'low': model}}
        self.metrics: Dict[str, MovementMetrics] = {}
        self.baseline_stats: Dict[str, Dict] = {}  # Historical movement statistics
        self.feature_columns: List[str] = []
        self._is_trained = False

        # Weighting configuration
        self.use_sample_weighting = use_sample_weighting and HAS_WEIGHTING
        self.use_session_weighting = use_session_weighting and HAS_WEIGHTING

        # Initialize weighters if available
        if HAS_WEIGHTING:
            sample_cfg = sample_weight_config or {'min_movement_threshold': min_move_usd, 'min_rr_ratio': asymmetry_threshold}
            if isinstance(sample_cfg, dict):
                sample_cfg = SampleWeightConfig(**sample_cfg)
            self.sample_weighter = SampleWeighter(sample_cfg)

            session_cfg = session_weight_config or {}
            if isinstance(session_cfg, dict):
                session_cfg = SessionWeightConfig(**session_cfg)
            self.session_weighter = SessionVolatilityWeighter(session_cfg)

            logger.info(f"Sample weighting: {self.use_sample_weighting}, Session weighting: {self.use_session_weighting}")
        else:
            self.sample_weighter = None
            self.session_weighter = None

        # XGBoost config
        self.xgb_config = self._default_xgb_config()

        logger.info(f"Initialized MovementMagnitudePredictor with horizons: {self.horizons}")
        logger.info(f"Asymmetry threshold: {asymmetry_threshold}, Min move: ${min_move_usd}")

    def _default_xgb_config(self) -> Dict:
        """Default XGBoost config optimized for magnitude prediction"""
        config = {
            'n_estimators': 500,
            'max_depth': 7,
            'learning_rate': 0.02,
            'subsample': 0.8,
            'colsample_bytree': 0.8,
            'min_child_weight': 10,
            'reg_alpha': 0.1,
            'reg_lambda': 1.0,
            'objective': 'reg:squarederror',
            'random_state': 42,
            'n_jobs': -1,
        }

        if self.use_gpu and HAS_XGBOOST:
            config['tree_method'] = 'hist'
            config['device'] = 'cuda'

        return config

    def create_targets(
        self,
        df: pd.DataFrame,
        horizon_key: str
    ) -> pd.DataFrame:
        """
        Create target variables for movement magnitude prediction.

        Targets:
        - target_high_usd: Maximum upward move from current close in horizon
        - target_low_usd: Maximum downward move from current close in horizon (positive)
        - target_asymmetry: high/low ratio (for validation)

        Args:
            df: OHLCV DataFrame
            horizon_key: Key from HORIZON_CONFIGS

        Returns:
            DataFrame with target columns added
        """
        config = self.HORIZON_CONFIGS[horizon_key]
        bars_ahead = config['bars_ahead']
        horizon_mins = config['horizon_minutes']

        logger.info(f"Creating targets for {horizon_key}: {bars_ahead} bars = {horizon_mins} min")

        df = df.copy()

        # Get close, high, low
        close = df['close'].values
        high = df['high'].values
        low = df['low'].values

        n = len(df)
        target_high_usd = np.zeros(n)
        target_low_usd = np.zeros(n)

        for i in range(n - bars_ahead):
            current_close = close[i]

            # Look at next 'bars_ahead' bars
            future_highs = high[i+1:i+1+bars_ahead]
            future_lows = low[i+1:i+1+bars_ahead]

            # Max high move (in USD)
            max_high = np.max(future_highs)
            target_high_usd[i] = max_high - current_close

            # Max low move (in USD, positive value)
            min_low = np.min(future_lows)
            target_low_usd[i] = current_close - min_low

        # Mark last bars as NaN (no future data)
        target_high_usd[-bars_ahead:] = np.nan
        target_low_usd[-bars_ahead:] = np.nan

        df[f'target_high_usd_{horizon_key}'] = target_high_usd
        df[f'target_low_usd_{horizon_key}'] = target_low_usd

        # Calculate asymmetry (with small epsilon to avoid division by zero)
        eps = 0.01
        df[f'target_asymmetry_{horizon_key}'] = target_high_usd / (target_low_usd + eps)

        # Log statistics
        valid_high = target_high_usd[~np.isnan(target_high_usd)]
        valid_low = target_low_usd[~np.isnan(target_low_usd)]

        logger.info(f"  Target high USD: mean={valid_high.mean():.2f}, std={valid_high.std():.2f}")
        logger.info(f"  Target low USD: mean={valid_low.mean():.2f}, std={valid_low.std():.2f}")

        # Store baseline statistics
        self.baseline_stats[horizon_key] = {
            'mean_high': float(valid_high.mean()),
            'std_high': float(valid_high.std()),
            'mean_low': float(valid_low.mean()),
            'std_low': float(valid_low.std()),
            'mean_total_range': float((valid_high + valid_low).mean()),
            'std_total_range': float((valid_high + valid_low).std()),
        }

        return df

    def create_movement_features(
        self,
        df: pd.DataFrame,
        horizon_key: str
    ) -> pd.DataFrame:
        """
        Create features specifically designed for movement magnitude prediction.

        Features focus on:
        - Recent volatility patterns
        - Range expansions/contractions
        - Momentum characteristics
        - Time-based patterns
        """
        config = self.HORIZON_CONFIGS[horizon_key]
        windows = config['feature_windows']

        df = df.copy()

        close = df['close']
        high = df['high']
        low = df['low']
        volume = df['volume'] if 'volume' in df.columns else pd.Series(1, index=df.index)

        # 1. Range-based features (key for movement prediction)
        df['bar_range_usd'] = high - low
        df['bar_range_pct'] = (high - low) / close

        for w in windows:
            # Average range over window
            df[f'avg_range_usd_{w}'] = df['bar_range_usd'].rolling(w).mean()
            df[f'max_range_usd_{w}'] = df['bar_range_usd'].rolling(w).max()
            df[f'min_range_usd_{w}'] = df['bar_range_usd'].rolling(w).min()

            # Range expansion/contraction
            df[f'range_zscore_{w}'] = (
                df['bar_range_usd'] - df['bar_range_usd'].rolling(w).mean()
            ) / (df['bar_range_usd'].rolling(w).std() + 0.001)

            # Range percentile
            df[f'range_pctl_{w}'] = df['bar_range_usd'].rolling(w).apply(
                lambda x: (x[-1] > x[:-1]).sum() / len(x[:-1]) if len(x) > 1 else 0.5
            )

        # 2. High/Low asymmetry features
        df['high_body'] = high - close.shift(1).clip(lower=df['open'])  # High from previous close/open
        df['low_body'] = close.shift(1).clip(lower=df['open']) - low

        for w in windows:
            df[f'avg_high_move_{w}'] = df['high_body'].rolling(w).mean()
            df[f'avg_low_move_{w}'] = df['low_body'].rolling(w).mean()
            df[f'high_low_ratio_{w}'] = df[f'avg_high_move_{w}'] / (df[f'avg_low_move_{w}'] + 0.001)

        # 3. Momentum features
        for w in windows:
            df[f'momentum_{w}'] = close.pct_change(w)
            df[f'momentum_abs_{w}'] = close.pct_change(w).abs()

            # Rate of change of range
            df[f'range_roc_{w}'] = df['bar_range_usd'].pct_change(w)

        # 4. ATR and volatility
        true_range = pd.concat([
            high - low,
            (high - close.shift(1)).abs(),
            (low - close.shift(1)).abs()
        ], axis=1).max(axis=1)

        for w in windows:
            df[f'atr_{w}'] = true_range.rolling(w).mean()
            df[f'atr_pct_{w}'] = df[f'atr_{w}'] / close

            # Volatility clustering
            df[f'vol_clustering_{w}'] = true_range.rolling(w).std() / (true_range.rolling(w).mean() + 0.001)

        # 5. Price position features
        for w in windows:
            rolling_high = high.rolling(w).max()
            rolling_low = low.rolling(w).min()
            df[f'price_position_{w}'] = (close - rolling_low) / (rolling_high - rolling_low + 0.001)

            # Distance from extremes
            df[f'dist_from_high_{w}'] = rolling_high - close
            df[f'dist_from_low_{w}'] = close - rolling_low

        # 6. Volume features (if available)
        if 'volume' in df.columns and df['volume'].sum() > 0:
            for w in windows:
                df[f'volume_ma_{w}'] = volume.rolling(w).mean()
                df[f'volume_ratio_{w}'] = volume / (df[f'volume_ma_{w}'] + 1)

                # Volume-weighted range
                df[f'vol_range_{w}'] = (df['bar_range_usd'] * volume).rolling(w).sum() / (volume.rolling(w).sum() + 1)

        # 7. Time features
        if isinstance(df.index, pd.DatetimeIndex):
            df['hour'] = df.index.hour
            df['day_of_week'] = df.index.dayofweek
            df['is_london'] = ((df['hour'] >= 8) & (df['hour'] < 16)).astype(int)
            df['is_ny'] = ((df['hour'] >= 13) & (df['hour'] < 21)).astype(int)
            df['is_overlap'] = ((df['hour'] >= 13) & (df['hour'] < 16)).astype(int)

        # 8. Candlestick pattern features
        df['body_size'] = (close - df['open']).abs()
        df['upper_wick'] = high - close.clip(lower=df['open'])
        df['lower_wick'] = close.clip(upper=df['open']) - low
        df['body_to_range'] = df['body_size'] / (df['bar_range_usd'] + 0.001)

        for w in [3, 6, 12]:
            df[f'avg_body_size_{w}'] = df['body_size'].rolling(w).mean()
            df[f'bullish_candles_{w}'] = (close > df['open']).rolling(w).sum() / w

        return df

    def compute_sample_weights(
        self,
        df: pd.DataFrame,
        horizon_key: str
    ) -> np.ndarray:
        """
        Compute combined sample weights from movement magnitude and session/volatility.

        Args:
            df: DataFrame with OHLCV data and targets
            horizon_key: Current horizon being trained

        Returns:
            Array of sample weights
        """
        n_samples = len(df)
        weights = np.ones(n_samples)

        target_high_col = f'target_high_usd_{horizon_key}'
        target_low_col = f'target_low_usd_{horizon_key}'

        # 1. Movement-based weights
        if self.use_sample_weighting and self.sample_weighter is not None:
            if target_high_col in df.columns and target_low_col in df.columns:
                df_temp = df.copy()
                df_temp['target_high'] = df[target_high_col]
                df_temp['target_low'] = df[target_low_col]

                movement_weights, valid_mask = self.sample_weighter.compute_sample_weights(
                    df_temp, 'target_high', 'target_low'
                )
                weights = weights * movement_weights
                logger.info(f"Applied movement weights: mean={movement_weights.mean():.3f}")

        # 2. Session and volatility weights
        if self.use_session_weighting and self.session_weighter is not None:
            session_weights = self.session_weighter.compute_combined_weights(df)
            weights = weights * session_weights
            logger.info(f"Applied session weights: mean={session_weights.mean():.3f}")

        # Normalize to mean=1
        if weights.mean() > 0:
            weights = weights / weights.mean()

        logger.info(f"Combined weights: min={weights.min():.3f}, max={weights.max():.3f}, mean={weights.mean():.3f}")

        return weights

    def fit(
        self,
        df_train: pd.DataFrame,
        feature_columns: Optional[List[str]] = None,
        horizons: Optional[List[str]] = None,
        sample_weight: Optional[np.ndarray] = None
    ) -> Dict[str, MovementMetrics]:
        """
        Train the movement magnitude predictor.

        Args:
            df_train: Training DataFrame with OHLCV data
            feature_columns: Feature columns to use (if None, auto-generate)
            horizons: Which horizons to train (default: all)
            sample_weight: Pre-computed sample weights (optional, auto-computed if None)

        Returns:
            Dictionary of training metrics
        """
        horizons = horizons or self.horizons
        all_metrics = {}

        for horizon_key in horizons:
            logger.info(f"\n{'='*60}")
            logger.info(f"Training for horizon: {horizon_key}")
            logger.info(f"{'='*60}")

            config = self.HORIZON_CONFIGS[horizon_key]

            # Create movement features
            df = self.create_movement_features(df_train.copy(), horizon_key)

            # Add enhanced session features if weighting module available
            if HAS_WEIGHTING and isinstance(df.index, pd.DatetimeIndex):
                df = create_session_features(df)

            # Create targets
            df = self.create_targets(df, horizon_key)

            # Determine feature columns
            if feature_columns is None:
                exclude_cols = ['open', 'high', 'low', 'close', 'volume', 'vwap']
                exclude_cols += [c for c in df.columns if c.startswith('target_')]
                self.feature_columns = [
                    c for c in df.columns
                    if c not in exclude_cols
                    and df[c].dtype in ['float64', 'float32', 'int64', 'int32']
                ]
            else:
                self.feature_columns = feature_columns

            logger.info(f"Using {len(self.feature_columns)} features")

            # Prepare training data
            target_high_col = f'target_high_usd_{horizon_key}'
            target_low_col = f'target_low_usd_{horizon_key}'

            df_valid = df.dropna(subset=[target_high_col, target_low_col] + self.feature_columns)

            if len(df_valid) < config['min_samples']:
                logger.warning(f"Insufficient samples: {len(df_valid)} < {config['min_samples']}")
                continue

            X = df_valid[self.feature_columns].values
            y_high = df_valid[target_high_col].values
            y_low = df_valid[target_low_col].values

            # Clean inf and very large values
            X = np.nan_to_num(X, nan=0.0, posinf=0.0, neginf=0.0)
            X = np.clip(X, -1e10, 1e10)

            logger.info(f"Training samples: {len(X)}")

            # Compute sample weights if not provided
            if sample_weight is not None:
                weights = sample_weight
            elif self.use_sample_weighting or self.use_session_weighting:
                weights = self.compute_sample_weights(df_valid, horizon_key)
            else:
                weights = None

            # Initialize models for this horizon
            self.models[horizon_key] = {}

            # Prepare fit params
            fit_params = {}
            if weights is not None:
                fit_params['sample_weight'] = weights

            # Train HIGH model
            logger.info("Training HIGH movement model..." + (" (weighted)" if weights is not None else ""))
            model_high = XGBRegressor(**self.xgb_config)
            model_high.fit(X, y_high, **fit_params)
            self.models[horizon_key]['high'] = model_high

            # Evaluate HIGH
            pred_high = model_high.predict(X)
            metrics_high = self._calculate_metrics(
                y_high, pred_high, horizon_key, 'high', config['horizon_minutes']
            )
            all_metrics[f'{horizon_key}_high'] = metrics_high

            logger.info(f"  HIGH - MAE: ${metrics_high.mae_usd:.2f}, R²: {metrics_high.r2:.4f}")

            # Train LOW model
            logger.info("Training LOW movement model..." + (" (weighted)" if weights is not None else ""))
            model_low = XGBRegressor(**self.xgb_config)
            model_low.fit(X, y_low, **fit_params)
            self.models[horizon_key]['low'] = model_low

            # Evaluate LOW
            pred_low = model_low.predict(X)
            metrics_low = self._calculate_metrics(
                y_low, pred_low, horizon_key, 'low', config['horizon_minutes']
            )
            all_metrics[f'{horizon_key}_low'] = metrics_low

            logger.info(f"  LOW - MAE: ${metrics_low.mae_usd:.2f}, R²: {metrics_low.r2:.4f}")

            # Calculate asymmetry accuracy
            actual_asymmetry = y_high / (y_low + 0.01)
            pred_asymmetry = pred_high / (pred_low + 0.01)

            # When we predict asymmetry > threshold, how often is actual also > threshold?
            bullish_signals = pred_asymmetry > self.asymmetry_threshold
            bearish_signals = pred_asymmetry < (1 / self.asymmetry_threshold)

            if bullish_signals.sum() > 0:
                bullish_accuracy = (actual_asymmetry[bullish_signals] > 1.0).mean()
                logger.info(f"  Bullish signals: {bullish_signals.sum()}, Accuracy: {bullish_accuracy:.2%}")

            if bearish_signals.sum() > 0:
                bearish_accuracy = (actual_asymmetry[bearish_signals] < 1.0).mean()
                logger.info(f"  Bearish signals: {bearish_signals.sum()}, Accuracy: {bearish_accuracy:.2%}")

        self.metrics = all_metrics
        self._is_trained = True

        logger.info(f"\nTraining complete. Models trained for {len(self.models)} horizons.")

        return all_metrics

    def _calculate_metrics(
        self,
        y_true: np.ndarray,
        y_pred: np.ndarray,
        horizon_key: str,
        target_type: str,
        horizon_minutes: int
    ) -> MovementMetrics:
        """Calculate comprehensive metrics for movement prediction"""

        mae = mean_absolute_error(y_true, y_pred)
        rmse = np.sqrt(mean_squared_error(y_true, y_pred))
        r2 = r2_score(y_true, y_pred)

        # MAPE with protection against zeros
        mask = y_true > 0.1  # Only calculate for meaningful movements
        if mask.sum() > 0:
            mape = np.mean(np.abs(y_pred[mask] - y_true[mask]) / y_true[mask])
        else:
            mape = 0.0

        return MovementMetrics(
            timeframe=self.HORIZON_CONFIGS[horizon_key]['timeframe'],
            horizon_minutes=horizon_minutes,
            target_type=target_type,
            mae_usd=mae,
            rmse_usd=rmse,
            mape=mape,
            r2=r2,
            n_samples=len(y_true)
        )

    def predict(
        self,
        df: pd.DataFrame,
        feature_columns: Optional[List[str]] = None
    ) -> List[MovementPrediction]:
        """
        Generate movement magnitude predictions.

        Args:
            df: DataFrame with OHLCV data
            feature_columns: Feature columns (use training columns if None)

        Returns:
            List of MovementPrediction objects
        """
        if not self._is_trained:
            raise ValueError("Model not trained. Call fit() first.")

        predictions = []
        feature_cols = feature_columns or self.feature_columns

        for horizon_key, models in self.models.items():
            config = self.HORIZON_CONFIGS[horizon_key]

            # Create features for this horizon
            df_feat = self.create_movement_features(df.copy(), horizon_key)

            # Check feature availability
            missing_features = set(feature_cols) - set(df_feat.columns)
            if missing_features:
                logger.warning(f"Missing features for {horizon_key}: {missing_features}")
                continue

            # Get valid rows
            df_valid = df_feat.dropna(subset=feature_cols)
            if len(df_valid) == 0:
                continue

            X = df_valid[feature_cols].values

            # Clean inf and very large values
            X = np.nan_to_num(X, nan=0.0, posinf=0.0, neginf=0.0)
            X = np.clip(X, -1e10, 1e10)

            # Predict
            pred_high = models['high'].predict(X)
            pred_low = models['low'].predict(X)

            # Get baseline stats
            stats = self.baseline_stats.get(horizon_key, {
                'mean_high': 5.0, 'std_high': 3.0,
                'mean_low': 5.0, 'std_low': 3.0
            })

            # Create predictions
            for idx, (ph, pl) in enumerate(zip(pred_high, pred_low)):
                # Calculate asymmetry
                asymmetry = ph / (pl + 0.01)

                # Determine direction and opportunity
                if asymmetry > self.asymmetry_threshold and ph > self.min_move_usd:
                    direction = 'LONG'
                    suggested_rr = ph / pl if pl > 0.5 else ph / 0.5
                    opportunity_score = min(1.0, (asymmetry - 1) / 2)
                elif asymmetry < (1 / self.asymmetry_threshold) and pl > self.min_move_usd:
                    direction = 'SHORT'
                    suggested_rr = pl / ph if ph > 0.5 else pl / 0.5
                    opportunity_score = min(1.0, (1/asymmetry - 1) / 2)
                else:
                    direction = 'NEUTRAL'
                    suggested_rr = 1.0
                    opportunity_score = 0.0

                # Calculate confidence based on prediction vs baseline
                high_confidence = 1 - abs(ph - stats['mean_high']) / (stats['std_high'] * 3 + 0.001)
                low_confidence = 1 - abs(pl - stats['mean_low']) / (stats['std_low'] * 3 + 0.001)
                confidence = max(0, min(1, (high_confidence + low_confidence) / 2))

                pred = MovementPrediction(
                    timeframe=config['timeframe'],
                    horizon_minutes=config['horizon_minutes'],
                    predicted_high_usd=float(ph),
                    predicted_low_usd=float(pl),
                    baseline_move_usd=stats['mean_high'],
                    std_move_usd=stats['std_high'],
                    asymmetry_ratio=float(asymmetry),
                    opportunity_score=float(opportunity_score),
                    suggested_direction=direction,
                    suggested_rr=float(suggested_rr),
                    confidence=float(confidence),
                    timestamp=df_valid.index[idx] if isinstance(df_valid.index, pd.DatetimeIndex) else None
                )
                predictions.append(pred)

        return predictions

    def evaluate_oos(
        self,
        df_test: pd.DataFrame,
        feature_columns: Optional[List[str]] = None
    ) -> Dict[str, MovementMetrics]:
        """
        Evaluate model on out-of-sample data.

        Args:
            df_test: Test DataFrame
            feature_columns: Feature columns to use

        Returns:
            Dictionary of OOS metrics
        """
        if not self._is_trained:
            raise ValueError("Model not trained. Call fit() first.")

        oos_metrics = {}
        feature_cols = feature_columns or self.feature_columns

        for horizon_key, models in self.models.items():
            config = self.HORIZON_CONFIGS[horizon_key]

            # Create features and targets
            df = self.create_movement_features(df_test.copy(), horizon_key)
            df = self.create_targets(df, horizon_key)

            target_high_col = f'target_high_usd_{horizon_key}'
            target_low_col = f'target_low_usd_{horizon_key}'

            df_valid = df.dropna(subset=[target_high_col, target_low_col] + feature_cols)

            if len(df_valid) == 0:
                logger.warning(f"No valid OOS samples for {horizon_key}")
                continue

            X = df_valid[feature_cols].values
            y_high = df_valid[target_high_col].values
            y_low = df_valid[target_low_col].values

            # Clean inf and very large values
            X = np.nan_to_num(X, nan=0.0, posinf=0.0, neginf=0.0)
            X = np.clip(X, -1e10, 1e10)

            # Predict
            pred_high = models['high'].predict(X)
            pred_low = models['low'].predict(X)

            # Calculate metrics
            metrics_high = self._calculate_metrics(
                y_high, pred_high, horizon_key, 'high', config['horizon_minutes']
            )
            metrics_low = self._calculate_metrics(
                y_low, pred_low, horizon_key, 'low', config['horizon_minutes']
            )

            oos_metrics[f'{horizon_key}_high'] = metrics_high
            oos_metrics[f'{horizon_key}_low'] = metrics_low

            # Calculate trading metrics
            actual_asymmetry = y_high / (y_low + 0.01)
            pred_asymmetry = pred_high / (pred_low + 0.01)

            # Asymmetry prediction accuracy
            asymmetry_correct = (
                ((pred_asymmetry > self.asymmetry_threshold) & (actual_asymmetry > 1.0)) |
                ((pred_asymmetry < 1/self.asymmetry_threshold) & (actual_asymmetry < 1.0)) |
                ((pred_asymmetry >= 1/self.asymmetry_threshold) & (pred_asymmetry <= self.asymmetry_threshold) &
                 (actual_asymmetry >= 0.67) & (actual_asymmetry <= 1.5))
            )

            metrics_high.asymmetry_accuracy = asymmetry_correct.mean()
            metrics_low.asymmetry_accuracy = asymmetry_correct.mean()

            logger.info(f"\nOOS Metrics for {horizon_key}:")
            logger.info(f"  HIGH - MAE: ${metrics_high.mae_usd:.2f}, R²: {metrics_high.r2:.4f}")
            logger.info(f"  LOW - MAE: ${metrics_low.mae_usd:.2f}, R²: {metrics_low.r2:.4f}")
            logger.info(f"  Asymmetry Accuracy: {metrics_high.asymmetry_accuracy:.2%}")

        return oos_metrics

    def save(self, path: str):
        """Save model to disk"""
        path = Path(path)
        path.mkdir(parents=True, exist_ok=True)

        # Save models
        for horizon_key, models in self.models.items():
            joblib.dump(models['high'], path / f'{horizon_key}_high.joblib')
            joblib.dump(models['low'], path / f'{horizon_key}_low.joblib')

        # Helper to convert numpy types to native Python
        def convert_numpy(obj):
            if isinstance(obj, dict):
                return {k: convert_numpy(v) for k, v in obj.items()}
            elif isinstance(obj, list):
                return [convert_numpy(v) for v in obj]
            elif isinstance(obj, np.ndarray):
                return obj.tolist()
            elif isinstance(obj, (np.integer, np.int64, np.int32)):
                return int(obj)
            elif isinstance(obj, (np.floating, np.float64, np.float32)):
                return float(obj)
            elif isinstance(obj, np.bool_):
                return bool(obj)
            return obj

        # Save metadata
        metadata = {
            'horizons': self.horizons,
            'feature_columns': self.feature_columns,
            'baseline_stats': convert_numpy(self.baseline_stats),
            'asymmetry_threshold': float(self.asymmetry_threshold),
            'min_move_usd': float(self.min_move_usd),
            'metrics': {k: convert_numpy(v.to_dict()) for k, v in self.metrics.items()},
            'saved_at': datetime.now().isoformat()
        }

        with open(path / 'metadata.yaml', 'w') as f:
            yaml.dump(metadata, f, default_flow_style=False)

        logger.info(f"Model saved to {path}")

    def load(self, path: str):
        """Load model from disk"""
        path = Path(path)

        # Load metadata
        with open(path / 'metadata.yaml', 'r') as f:
            metadata = yaml.safe_load(f)

        self.horizons = metadata['horizons']
        self.feature_columns = metadata['feature_columns']
        self.baseline_stats = metadata['baseline_stats']
        self.asymmetry_threshold = metadata.get('asymmetry_threshold', 1.5)
        self.min_move_usd = metadata.get('min_move_usd', 3.0)

        # Load models
        self.models = {}
        for horizon_key in self.horizons:
            high_path = path / f'{horizon_key}_high.joblib'
            low_path = path / f'{horizon_key}_low.joblib'

            if high_path.exists() and low_path.exists():
                self.models[horizon_key] = {
                    'high': joblib.load(high_path),
                    'low': joblib.load(low_path)
                }

        self._is_trained = True
        logger.info(f"Model loaded from {path}")


def calculate_standard_variance(
    df: pd.DataFrame,
    timeframe: str,
    lookback_periods: int = 100
) -> Dict[str, float]:
    """
    Calculate standard variance/movement statistics for a given timeframe.

    Returns baseline statistics like:
    - mean_range: Average bar range in USD
    - std_range: Standard deviation of bar range
    - typical_high_move: Typical upward movement
    - typical_low_move: Typical downward movement
    """
    high = df['high'].values
    low = df['low'].values
    close = df['close'].values

    # Bar range
    bar_range = high - low

    # High/low moves from previous close
    high_move = high[1:] - close[:-1]
    low_move = close[:-1] - low[1:]

    stats = {
        'timeframe': timeframe,
        'mean_range': float(np.mean(bar_range[-lookback_periods:])),
        'std_range': float(np.std(bar_range[-lookback_periods:])),
        'median_range': float(np.median(bar_range[-lookback_periods:])),
        'p75_range': float(np.percentile(bar_range[-lookback_periods:], 75)),
        'p90_range': float(np.percentile(bar_range[-lookback_periods:], 90)),
        'mean_high_move': float(np.mean(high_move[-lookback_periods:])),
        'mean_low_move': float(np.mean(low_move[-lookback_periods:])),
        'std_high_move': float(np.std(high_move[-lookback_periods:])),
        'std_low_move': float(np.std(low_move[-lookback_periods:])),
    }

    return stats
