Trading AI Model Hyperparameter Optimization System

Development of a hyperparameter optimization system for a trading AI model

The AI trading model is hypersensitive to hyperparameters: the search space is vast, and classic grid search is unacceptable in terms of time. Specialized optimization methods for financial problems take into account the temporal structure of data and market volatility.

Specifics of financial tasks

Why standard AutoML doesn't work for trading:

• Standard cross-validation violates temporal causality (data leakage)
• The accuracy metric is useless - you need Sharpe Ratio, Calmar, Max Drawdown
• The market regime is changing: optimal parameters for 2022 may be fatal in 2024
• Overfitting is especially dangerous: in real trading, curve fitting = loss of capital

Walk-Forward Validation is the only fair method:

import numpy as np
import pandas as pd
from typing import Callable

def walk_forward_optimization(
    price_data: pd.DataFrame,
    strategy_func: Callable,
    param_space: dict,
    in_sample_months: int = 12,
    out_sample_months: int = 3
) -> dict:
    """
    WFO: обучаем на IS периоде, тестируем на OOS — и так скользим вперёд.
    Метрика = агрегированный OOS Sharpe ratio по всем периодам.
    """
    results = []
    total_months = len(price_data) // 21  # ~21 торговый день в месяце

    for start_month in range(0, total_months - in_sample_months, out_sample_months):
        is_end = start_month + in_sample_months
        oos_end = is_end + out_sample_months

        if oos_end > total_months:
            break

        is_data = price_data.iloc[start_month * 21: is_end * 21]
        oos_data = price_data.iloc[is_end * 21: oos_end * 21]

        # Оптимизация на IS периоде
        best_params = optimize_on_period(strategy_func, is_data, param_space)

        # Тест на OOS
        oos_returns = strategy_func(oos_data, **best_params)
        oos_sharpe = calculate_sharpe(oos_returns, annualization=252)

        results.append({
            'period_start': is_end,
            'best_params': best_params,
            'oos_sharpe': oos_sharpe,
            'oos_returns': oos_returns
        })

    return {
        'wfo_results': results,
        'avg_oos_sharpe': np.mean([r['oos_sharpe'] for r in results]),
        'sharpe_stability': np.std([r['oos_sharpe'] for r in results]),
        'profitable_periods': sum(1 for r in results if r['oos_sharpe'] > 0) / len(results)
    }

Optuna for financial tasks

Bayesian Optimization with Custom Metrics:

import optuna

def optimize_trading_model(train_data: pd.DataFrame,
                             val_data: pd.DataFrame) -> dict:
    def objective(trial):
        params = {
            'n_estimators': trial.suggest_int('n_estimators', 100, 500),
            'max_depth': trial.suggest_int('max_depth', 3, 8),
            'learning_rate': trial.suggest_float('learning_rate', 1e-3, 0.1, log=True),
            'min_samples_leaf': trial.suggest_int('min_samples_leaf', 50, 500),
            'feature_fraction': trial.suggest_float('feature_fraction', 0.5, 1.0),

            # Специфичные для финансов параметры
            'lookback_window': trial.suggest_int('lookback_window', 5, 60),
            'prediction_horizon': trial.suggest_categorical('prediction_horizon', [1, 5, 10, 20]),
            'threshold_long': trial.suggest_float('threshold_long', 0.001, 0.01),
            'threshold_short': trial.suggest_float('threshold_short', -0.01, -0.001)
        }

        # Обучаем и тестируем
        model = train_model(train_data, params)
        signals = generate_signals(val_data, model, params)
        returns = backtest_signals(val_data, signals)

        # Составная метрика: Sharpe с штрафом за drawdown
        sharpe = calculate_sharpe(returns)
        max_dd = calculate_max_drawdown(returns)

        # Штраф за чрезмерную торговлю (транзакционные издержки)
        trade_count = signals.abs().sum()
        cost_penalty = trade_count * 0.0001  # 1 bp за сделку

        # Optuna максимизирует: sharpe - drawdown_penalty - cost_penalty
        return sharpe - abs(max_dd) * 0.5 - cost_penalty

    study = optuna.create_study(
        direction='maximize',
        sampler=optuna.samplers.TPESampler(seed=42),
        pruner=optuna.pruners.HyperbandPruner()
    )
    study.optimize(objective, n_trials=200, timeout=3600)

    return {
        'best_params': study.best_params,
        'best_value': study.best_value,
        'n_trials': len(study.trials)
    }

Adaptive optimization to the market regime

Mode detection and parameter selection:

from hmmlearn import hmm

class RegimeAwareOptimizer:
    """
    Разные режимы рынка (тренд/флэт/волатильность) требуют разных гиперпараметров.
    HMM определяет режим → выбираем предварительно оптимизированный набор параметров.
    """
    def __init__(self, n_regimes=3):
        self.regime_model = hmm.GaussianHMM(n_components=n_regimes, covariance_type='full')
        self.regime_params = {}  # {режим: best_params}

    def fit_regimes(self, returns: np.ndarray):
        features = np.column_stack([
            returns,
            np.abs(returns),                         # волатильность
            pd.Series(returns).rolling(20).std().values  # rolling vol
        ])
        self.regime_model.fit(features[~np.isnan(features).any(axis=1)])

    def optimize_per_regime(self, price_data, strategy_func, param_space):
        """Для каждого режима — отдельная WFO оптимизация"""
        regimes = self.get_current_regime(price_data)

        for regime_id in range(self.regime_model.n_components):
            regime_data = price_data[regimes == regime_id]
            if len(regime_data) > 500:
                self.regime_params[regime_id] = optimize_trading_model(
                    regime_data[:len(regime_data)//2],
                    regime_data[len(regime_data)//2:]
                )['best_params']

    def get_current_regime(self, recent_data: pd.DataFrame) -> int:
        features = extract_regime_features(recent_data.tail(20))
        return self.regime_model.predict(features)[-1]

Timeframe: Walk-forward validation + Optuna basic optimization + backtest — 3-4 weeks. Regime-aware optimization, adaptive re-optimization, multi-objective Pareto front-end — 6-8 weeks.