AI Candlestick Pattern Analysis Model Development

AI Model for Candlestick Pattern Analysis on Charts

Recognizing Japanese candlestick patterns is a CV task, but with an important caveat: the pattern itself is not a trading signal. A break of resistance at high volume with a confirming "hammer" is meaningful. A "hammer" in the middle of a flat range without volume is noise. Therefore, the model should recognize not an isolated pattern, but a pattern in context.

Approaches to the Task

Approach 1: CV on chart screenshots — train a detector on PNG/JPEG images. Simple, but loses numeric OHLCV data. Accuracy is limited by resolution and chart style.

Approach 2: ML on numeric features — extract geometric candle features and train XGBoost/LightGBM. Faster, more interpretable, independent of visualization.

Approach 3: Hybrid — numeric features + chart rendering → multimodal model. Best accuracy, high complexity.

Approach 2: Numeric Features (Recommended)

import numpy as np
import pandas as pd
from typing import Optional

class CandlestickFeatureExtractor:
    """
    Extract geometric and relative features of candles.
    All features are normalized to ATR (Average True Range) —
    this makes them scale-invariant.
    """

    def compute_candle_features(
        self,
        df: pd.DataFrame,   # OHLCV DataFrame
        lookback: int = 5   # number of previous candles
    ) -> pd.DataFrame:
        """
        Features of a single candle:
        - body_ratio: (close-open) / ATR — body size
        - upper_shadow_ratio: upper shadow / ATR
        - lower_shadow_ratio: lower shadow / ATR
        - body_position: body position in high-low range
        - gap: gap from previous close / ATR
        - volume_ratio: volume / MA(volume, 20)
        """
        atr = self._calculate_atr(df, period=14)

        features = pd.DataFrame(index=df.index)

        for i in range(lookback):
            shift = i + 1
            c = df.shift(shift) if i > 0 else df

            body = c['close'] - c['open']
            total_range = c['high'] - c['low'] + 1e-8

            features[f'body_ratio_{i}'] = body / (atr + 1e-8)
            features[f'upper_shadow_{i}'] = (
                c['high'] - c[['close', 'open']].max(axis=1)
            ) / (atr + 1e-8)
            features[f'lower_shadow_{i}'] = (
                c[['close', 'open']].min(axis=1) - c['low']
            ) / (atr + 1e-8)
            features[f'body_pos_{i}'] = (
                (c[['close', 'open']].min(axis=1) - c['low']) / total_range
            )
            if i == 0:
                features[f'gap_{i}'] = (
                    (c['open'] - df['close'].shift(1)) / (atr + 1e-8)
                )
            features[f'vol_ratio_{i}'] = c['volume'] / (
                c['volume'].rolling(20).mean() + 1e-8
            )

        # Context features
        features['trend_5'] = (
            df['close'] - df['close'].shift(5)
        ) / (atr + 1e-8)
        features['trend_20'] = (
            df['close'] - df['close'].shift(20)
        ) / (atr + 1e-8)
        features['volatility_norm'] = atr / df['close']

        return features.fillna(0)

    def _calculate_atr(self, df: pd.DataFrame, period: int = 14) -> pd.Series:
        high_low   = df['high'] - df['low']
        high_close = (df['high'] - df['close'].shift()).abs()
        low_close  = (df['low']  - df['close'].shift()).abs()
        true_range = pd.concat(
            [high_low, high_close, low_close], axis=1
        ).max(axis=1)
        return true_range.ewm(span=period, adjust=False).mean()

Pattern Labeling and Training

import talib   # TA-Lib for classical patterns
import lightgbm as lgb
from sklearn.model_selection import TimeSeriesSplit
from sklearn.metrics import f1_score

def label_patterns(df: pd.DataFrame) -> pd.DataFrame:
    """
    Auto-labeling patterns via TA-Lib.
    Values: 0 = no pattern, 100 = bullish, -100 = bearish.
    """
    patterns = {
        'hammer':        talib.CDLHAMMER,
        'doji':          talib.CDLDOJI,
        'engulfing':     talib.CDLENGULFING,
        'morning_star':  talib.CDLMORNINGSTAR,
        'evening_star':  talib.CDLEVENINGSTAR,
        'shooting_star': talib.CDLSHOOTINGSTAR,
        'harami':        talib.CDLHARAMI,
        'three_white':   talib.CDL3WHITESOLDIERS,
    }

    for name, func in patterns.items():
        df[f'pattern_{name}'] = func(
            df['open'].values, df['high'].values,
            df['low'].values,  df['close'].values
        )

    # Target variable: significant move in 3 candles
    df['target'] = np.where(
        df['close'].shift(-3) > df['close'] * 1.005, 1,   # +0.5% = bullish
        np.where(
            df['close'].shift(-3) < df['close'] * 0.995, -1,  # -0.5% = bearish
            0  # flat
        )
    )
    return df

def train_pattern_classifier(
    features: pd.DataFrame,
    labels: pd.Series
) -> lgb.Booster:
    """
    TimeSeriesSplit is mandatory for financial data.
    Cannot use random split (future leakage).
    """
    tscv = TimeSeriesSplit(n_splits=5)
    models = []

    params = {
        'objective': 'multiclass',
        'num_class': 3,           # -1, 0, 1
        'learning_rate': 0.05,
        'n_estimators': 500,
        'max_depth': 6,
        'min_child_samples': 50,  # important for finance: avoid overfitting
        'subsample': 0.8,
        'colsample_bytree': 0.8,
        'reg_lambda': 1.0,
        'metric': 'multi_logloss',
        'verbose': -1
    }

    for fold, (train_idx, val_idx) in enumerate(tscv.split(features)):
        X_train = features.iloc[train_idx]
        y_train = labels.iloc[train_idx] + 1   # shift: -1,0,1 → 0,1,2
        X_val   = features.iloc[val_idx]
        y_val   = labels.iloc[val_idx] + 1

        train_data = lgb.Dataset(X_train, label=y_train)
        val_data   = lgb.Dataset(X_val,   label=y_val)

        model = lgb.train(
            params,
            train_data,
            valid_sets=[val_data],
            callbacks=[lgb.early_stopping(50), lgb.log_evaluation(100)]
        )

        preds = model.predict(X_val).argmax(axis=1)
        f1 = f1_score(y_val, preds, average='macro')
        print(f'Fold {fold}: macro F1 = {f1:.4f}')
        models.append(model)

    return models

Important Warning

A pattern by itself predicts movement with accuracy barely above 50%. In tests on 10 years of SPY data: model accuracy ~58% with macro F1 ~0.41. This is not a trading system — it is one of the signals. Real gain comes from an ensemble: pattern + volume analysis + RSI/MACD context + market regime.

Timeline