Synthetic Tabular Data Generation System

Development of a system for generating synthetic tabular data

Synthetic tabular data is the most commonly used type of synthetic data in business. Financial transactions, medical records, user profiles—all of this structured data is necessary for training models but is limited by compliance requirements.

Selecting a generation method

CTGAN: Implementation Details

import pandas as pd
from ctgan import CTGAN
import numpy as np

def train_ctgan_synthesizer(
    data: pd.DataFrame,
    discrete_columns: list,
    epochs: int = 300
) -> CTGAN:

    synthesizer = CTGAN(
        embedding_dim=128,
        generator_dim=(256, 256),
        discriminator_dim=(256, 256),
        batch_size=500,
        epochs=epochs,
        verbose=True,
        pac=10,  # Packing size для улучшения режима
    )

    synthesizer.fit(data, discrete_columns=discrete_columns)
    return synthesizer

# Пример: финансовые транзакции
financial_data = pd.read_parquet("transactions.parquet")
discrete_cols = ['merchant_category', 'transaction_type', 'currency', 'is_fraud']

synth = train_ctgan_synthesizer(financial_data, discrete_cols)

# Генерация с сохранением соотношений классов
n_real = len(financial_data)
synthetic = synth.sample(n_real * 5)  # 5x синтетических данных

# Проверка соотношения fraud (важно для несбалансированных датасетов)
print(f"Real fraud rate: {financial_data['is_fraud'].mean():.4f}")
print(f"Synthetic fraud rate: {synthetic['is_fraud'].mean():.4f}")

Handling specific data types

from sdv.single_table import CTGANSynthesizer
from sdv.metadata import SingleTableMetadata

# Специальная обработка временных рядов и ID
metadata = SingleTableMetadata()
metadata.add_column('user_id', sdtype='id', regex_format='user_[0-9]{6}')
metadata.add_column('email', sdtype='email')
metadata.add_column('phone', sdtype='phone_number')
metadata.add_column('ip_address', sdtype='ipv4_address')
metadata.add_column('amount', sdtype='numerical', computer_representation='Float')
metadata.add_column('created_at', sdtype='datetime',
                    datetime_format='%Y-%m-%d %H:%M:%S')
metadata.add_column('category', sdtype='categorical')

synthesizer = CTGANSynthesizer(metadata, epochs=500)
synthesizer.fit(real_df)

Conditional generation

# Генерация только определённых сегментов
# Например: только premium клиенты с транзакциями > $1000
synthetic_premium = synthesizer.sample(
    num_rows=50_000,
    conditions=[
        Condition({'customer_tier': 'premium', 'amount_gt_1000': True})
    ]
)

Validation of statistical quality

from scipy.stats import ks_2samp
import matplotlib.pyplot as plt

def validate_synthetic_quality(real: pd.DataFrame, synthetic: pd.DataFrame) -> dict:
    results = {}

    for col in real.select_dtypes(include=np.number).columns:
        ks_stat, p_value = ks_2samp(real[col].dropna(), synthetic[col].dropna())
        results[col] = {
            'real_mean': real[col].mean(),
            'synthetic_mean': synthetic[col].mean(),
            'real_std': real[col].std(),
            'synthetic_std': synthetic[col].std(),
            'ks_stat': ks_stat,
            'distribution_match': p_value > 0.05
        }

    # Корреляционная матрица
    real_corr = real.select_dtypes(np.number).corr()
    synth_corr = synthetic.select_dtypes(np.number).corr()
    corr_diff = (real_corr - synth_corr).abs().mean().mean()
    results['correlation_mae'] = corr_diff  # Цель: < 0.05

    return results

For most ML model optimization problems, synthetic data generated by CTGAN with a score > 0.85 according to SDMetrics allows achieving 95-98% of the model quality compared to training on real data of the same volume.