import warnings
warnings.filterwarnings('ignore')
!pip install --upgrade pandas==2.2.2 tensorflow==2.19.0 scikeras scikit-optimize yfinance pandas_ta quantstats shap--quiet

import warnings
warnings.filterwarnings('ignore')

import numpy as np
import pandas as pd
import yfinance as yf
import pandas_ta as ta  # technical analysis helpers

# Machine Learning / optimization
from sklearn.model_selection import TimeSeriesSplit, GridSearchCV, RandomizedSearchCV
from sklearn.preprocessing import StandardScaler
from sklearn.svm import SVC
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.metrics import classification_report, confusion_matrix, accuracy_score
from sklearn.pipeline import Pipeline
from sklearn.base import clone

import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout
from tensorflow.keras.optimizers import Adam
from scikeras.wrappers import KerasClassifier

from skopt import BayesSearchCV
from skopt.space import Real, Categorical, Integer

import matplotlib.pyplot as plt
import seaborn as sns

sns.set_style("whitegrid")

# -------------------------------------------------------------------------
# CONFIGURATION - OPTIMIZED FOR SPEED
# -------------------------------------------------------------------------
# Set to True for a quick test run with few iterations.
# Set to False for a full, thorough search (will take a long time).
FAST_MODE = True
# -------------------------------------------------------------------------

RANDOM_STATE = 42
np.random.seed(RANDOM_STATE)
tf.random.set_seed(RANDOM_STATE)

# --- Define search parameters based on FAST_MODE ---
if FAST_MODE:
    print("--- RUNNING IN ULTRA-FAST MODE ---")
    # Skip expensive operations
    SKIP_GRID_SEARCH = True
    SKIP_RANDOM_SEARCH = True  
    SKIP_NN = True
    # SVM parameters
    svm_c_grid = [1]
    svm_gamma_grid = [0.01]
    svm_kernel_grid = ['rbf']
    svm_random_iters = 2
    svm_bayes_iters = 3
    # NN parameters
    nn_search_iters = 2
    nn_epochs = 3
    # LDA parameters
    lda_shrinkage_grid = [0.0, 0.5, 1.0]
    n_splits = 3
else:
    print("--- RUNNING IN FULL_MODE (this will take time) ---")
    # Run everything
    SKIP_GRID_SEARCH = False
    SKIP_RANDOM_SEARCH = False
    SKIP_NN = False
    # SVM parameters
    svm_c_grid = [0.1, 1, 10, 100]
    svm_gamma_grid = [0.001, 0.01, 0.1, 1]
    svm_kernel_grid = ['rbf', 'linear']
    svm_random_iters = 20  # Reduced from 50
    svm_bayes_iters = 20   # Reduced from 50
    # NN parameters
    nn_search_iters = 10   # Reduced from 20
    nn_epochs = 30         # Reduced from 50
    # LDA parameters
    lda_shrinkage_grid = np.arange(0.0, 1.01, 0.1)  # 11 values instead of 21
    n_splits = 5

# -------------------------------------------------------------------------
# 1) DATA ACQUISITION - OPTIMIZED
# -------------------------------------------------------------------------
print("Downloading data...")
ticker = 'SPY'
# Use smaller date range for faster testing
if FAST_MODE:
    data = yf.download(ticker, start='2020-01-01', end='2023-12-31', auto_adjust=False)
else:
    data = yf.download(ticker, start='2010-01-01', end='2023-12-31', auto_adjust=False)

# Flatten MultiIndex columns if yfinance returned them
if isinstance(data.columns, pd.MultiIndex):
    data.columns = data.columns.droplevel(1)

print("Raw data shape:", data.shape)

# Reset index so 'Date' becomes a column
data = data.reset_index()

# -------------------------------------------------------------------------
# 2) FEATURE ENGINEERING - Technical indicators
# -------------------------------------------------------------------------
print("Calculating technical indicators...")
# use a time-indexed DF for pandas_ta
data_indexed = data.set_index('Date')

# compute indicators - only essential ones
rsi = ta.rsi(data_indexed['Close'], length=14)
macd = ta.macd(data_indexed['Close'], fast=12, slow=26, signal=9)
bbands = ta.bbands(data_indexed['Close'], length=20)
obv = ta.obv(data_indexed['Close'], data_indexed['Volume'])

# helper to find columns by pattern(s)
def find_col(df, patterns):
    """Return the first column name in df whose string contains any pattern in patterns (case-insensitive)."""
    if df is None:
        return None
    for pat in patterns:
        for c in df.columns:
            if pat.lower() in str(c).lower():
                return c
    return None

# --- Add RSI safely ---
if isinstance(rsi, pd.Series):
    data['RSI_14'] = rsi.values
elif isinstance(rsi, pd.DataFrame) and len(rsi.columns) >= 1:
    data['RSI_14'] = rsi.iloc[:, 0].values

# --- MACD: robust column detection ---
macd_upper = find_col(macd, ['MACD', 'macd'])
macd_signal = find_col(macd, ['MACDs', 'signal'])
macd_hist = find_col(macd, ['MACDh', 'hist', 'histogram'])

# fallback: positional if names not found
if macd_upper is None or macd_signal is None or macd_hist is None:
    if isinstance(macd, pd.DataFrame) and macd.shape[1] >= 3:
        macd_upper, macd_signal, macd_hist = macd.columns[:3]

# assign MACD columns if available
if macd_upper is not None:
    data['MACD'] = macd[macd_upper].values
if macd_signal is not None:
    data['MACD_signal'] = macd[macd_signal].values
if macd_hist is not None:
    data['MACD_hist'] = macd[macd_hist].values

# --- Bollinger Bands: robust detection ---
bb_upper_col = find_col(bbands, ['BBU', 'upper', 'bb_u', 'bb_upper'])
bb_middle_col = find_col(bbands, ['BBM', 'middle', 'bb_m', 'bb_middle'])
bb_lower_col = find_col(bbands, ['BBL', 'lower', 'bb_l', 'bb_lower'])

# fallback to positional order if pattern search fails but there are >=3 cols
if bb_upper_col is None or bb_middle_col is None or bb_lower_col is None:
    if isinstance(bbands, pd.DataFrame) and bbands.shape[1] >= 3:
        bb_upper_col, bb_middle_col, bb_lower_col = bbands.columns[:3]

# assign BB columns
data['BB_upper'] = bbands[bb_upper_col].values
data['BB_middle'] = bbands[bb_middle_col].values
data['BB_lower'] = bbands[bb_lower_col].values

# --- OBV ---
if isinstance(obv, pd.Series):
    data['OBV'] = obv.values
elif isinstance(obv, pd.DataFrame) and obv.shape[1] >= 1:
    data['OBV'] = obv.iloc[:, 0].values

# --- Simplified engineered features ---
data['return_lag_1'] = data['Close'].pct_change(1)
data['return_lag_2'] = data['Close'].pct_change(2)
data['volatility_10d'] = data['Close'].pct_change().rolling(window=10).std()

# --- Target ---
data['target'] = (data['Close'].shift(-1) > data['Close']).astype(int)

# -------------------------------------------------------------------------
# 3) CLEANING and PREPARATION
# -------------------------------------------------------------------------
# Drop rows with NaNs created by indicators
data.dropna(inplace=True)
data.reset_index(drop=True, inplace=True)

# Drop non-features
features_to_drop = ['Date', 'Open', 'High', 'Low', 'Close', 'Adj Close', 'Volume']
for col in features_to_drop:
    if col in data.columns:
        data.drop(columns=col, inplace=True)

# Align X and y
X = data.drop(columns=['target'])
y = data['target']

print("\nPrepared data shapes:")
print("X shape:", X.shape)
print("y shape:", y.shape)
print("Feature columns:", list(X.columns))

# In FAST_MODE, use smaller dataset for testing
if FAST_MODE and len(X) > 1500:
    X = X[-1000:]
    y = y[-1000:]
    print(f"Reduced dataset to {len(X)} samples for faster testing")

# -------------------------------------------------------------------------
# 4) TIME-SERIES SPLIT
# -------------------------------------------------------------------------
tscv = TimeSeriesSplit(n_splits=n_splits)

# Quick visualization only in full mode
if not FAST_MODE:
    fig, ax = plt.subplots(figsize=(12, 3))
    for i, (train_idx, test_idx) in enumerate(tscv.split(X)):
        ax.fill_betweenx([i-0.4, i+0.4], train_idx[0], train_idx[-1], color='tab:blue', alpha=0.6, label='Train' if i == 0 else "")
        ax.fill_betweenx([i-0.4, i+0.4], test_idx[0], test_idx[-1], color='tab:orange', alpha=0.8, label='Test' if i == 0 else "")
    ax.set_title('TimeSeriesSplit Cross-Validation Folds')
    ax.set_xlabel('Index')
    ax.set_ylabel('Fold')
    ax.legend()
    plt.tight_layout()
    plt.show()

# -------------------------------------------------------------------------
# 5) SVM PIPELINE & SEARCHES - OPTIMIZED
# -------------------------------------------------------------------------
svm_pipeline = Pipeline([
    ('scaler', StandardScaler()),
    ('svm', SVC(random_state=RANDOM_STATE))
])

# Only run Bayesian Search in FAST_MODE (most efficient)
if not SKIP_GRID_SEARCH:
    svm_param_grid = {
        'svm__C': svm_c_grid,
        'svm__gamma': svm_gamma_grid,
        'svm__kernel': svm_kernel_grid
    }

    print("Running Grid Search for SVM...")
    grid_search = GridSearchCV(svm_pipeline, svm_param_grid, cv=tscv, n_jobs=-1, verbose=1)
    grid_search.fit(X, y)
    print("Grid Search best score:", grid_search.best_score_)
    print("Grid Search best params:", grid_search.best_params_)
else:
    print("Skipping Grid Search for SVM")
    grid_search = None

if not SKIP_RANDOM_SEARCH:
    svm_param_dist = {
        'svm__C': np.logspace(-3, 3, 7),  # Reduced from 10
        'svm__gamma': np.logspace(-4, 1, 6),  # Reduced from 10
        'svm__kernel': ['rbf', 'linear']  # Removed 'poly'
    }
    print("Running Randomized Search for SVM...")
    random_search = RandomizedSearchCV(svm_pipeline, svm_param_dist, n_iter=svm_random_iters, 
                                     cv=tscv, n_jobs=-1, verbose=1, random_state=RANDOM_STATE)
    random_search.fit(X, y)
    print("Random Search best score:", random_search.best_score_)
    print("Random Search best params:", random_search.best_params_)
else:
    print("Skipping Randomized Search for SVM")
    random_search = None

# Always run Bayesian (most efficient)
svm_param_space = {
    'svm__C': Real(1e-3, 1e3, prior='log-uniform'),
    'svm__gamma': Real(1e-4, 1e1, prior='log-uniform'),
    'svm__kernel': Categorical(['rbf', 'linear'])  # Removed 'poly'
}

print("Running Bayesian Optimization for SVM...")
bayes_search = BayesSearchCV(svm_pipeline, search_spaces=svm_param_space, n_iter=svm_bayes_iters, 
                           cv=tscv, n_jobs=-1, verbose=1, random_state=RANDOM_STATE)
bayes_search.fit(X, y)
print("Bayes Search best score:", bayes_search.best_score_)
print("Bayes Search best params:", bayes_search.best_params_)

# -------------------------------------------------------------------------
# 6) NEURAL NETWORK - SKIPPED IN FAST_MODE
# -------------------------------------------------------------------------
if not SKIP_NN:
    def create_nn_model(learning_rate=0.001, dropout_rate=0.2, num_layers=2, neurons_per_layer=32, l2_reg=0.01):
        model = Sequential()
        model.add(Dense(neurons_per_layer, input_dim=X.shape[1], activation='relu',
                        kernel_regularizer=tf.keras.regularizers.l2(l2_reg)))
        model.add(Dropout(dropout_rate))
        for _ in range(num_layers - 1):
            model.add(Dense(max(4, neurons_per_layer // 2), activation='relu',
                            kernel_regularizer=tf.keras.regularizers.l2(l2_reg)))
            model.add(Dropout(dropout_rate))
        model.add(Dense(1, activation='sigmoid'))
        optimizer = Adam(learning_rate=learning_rate)
        model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy'])
        return model

    nn_classifier = KerasClassifier(model=create_nn_model, epochs=nn_epochs, verbose=0, random_state=RANDOM_STATE)
    nn_pipeline = Pipeline([
        ('scaler', StandardScaler()),
        ('model', nn_classifier)
    ])

    nn_param_space = {
        'model__learning_rate': Real(1e-5, 1e-1, prior='log-uniform'),
        'model__dropout_rate': Real(0.1, 0.5, prior='uniform'),
        'model__num_layers': Integer(1, 3),
        'model__neurons_per_layer': Integer(16, 64),  # Reduced max from 128
        'model__l2_reg': Real(1e-4, 1e-1, prior='log-uniform'),
        'model__batch_size': Categorical([32, 64]),  # Reduced options
    }

    print("Running Bayesian Optimization for Neural Network...")
    nn_bayes_search = BayesSearchCV(
        nn_pipeline,
        search_spaces=nn_param_space,
        n_iter=nn_search_iters,
        cv=tscv,
        n_jobs=1,  # Keras doesn't play well with n_jobs > 1
        verbose=1,
        random_state=RANDOM_STATE,
        scoring='accuracy'
    )
    nn_bayes_search.fit(X, y)
    print("NN Bayes best score:", nn_bayes_search.best_score_)
    print("NN Bayes best params:", nn_bayes_search.best_params_)
else:
    print("Skipping Neural Network for speed")
    nn_bayes_search = None

# -------------------------------------------------------------------------
# 7) LDA - OPTIMIZED
# -------------------------------------------------------------------------
lda_pipeline = Pipeline([
    ('scaler', StandardScaler()),
    ('lda', LinearDiscriminantAnalysis())
])

lda_param_grid = {
    'lda__solver': ['lsqr', 'eigen'],
    'lda__shrinkage': lda_shrinkage_grid
}

print("Running Grid Search for LDA...")
lda_grid_search = GridSearchCV(lda_pipeline, lda_param_grid, cv=tscv, n_jobs=-1, verbose=1)
lda_grid_search.fit(X, y)
print("LDA best score:", lda_grid_search.best_score_)
print("LDA best params:", lda_grid_search.best_params_)

# -------------------------------------------------------------------------
# 8) FINAL TRAIN / TEST and EVALUATION
# -------------------------------------------------------------------------
train_size = int(len(X) * 0.8)
X_train, X_test = X[:train_size], X[train_size:]
y_train, y_test = y[:train_size], y[train_size:]

# Use best SVM from Bayesian search (most efficient method)
best_svm_params = dict(bayes_search.best_params_)
final_svm_params = {k.replace('svm__', ''): v for k, v in best_svm_params.items()}

final_model = Pipeline([
    ('scaler', StandardScaler()),
    ('svm', SVC(**final_svm_params, random_state=RANDOM_STATE))
])

print("Training final SVM model with optimized hyperparameters...")
final_model.fit(X_train, y_train)
y_pred = final_model.predict(X_test)
accuracy = accuracy_score(y_test, y_pred)
print(f"Final model test accuracy: {accuracy:.4f}")
print(classification_report(y_test, y_pred))

# Confusion matrix
cm = confusion_matrix(y_test, y_pred)
plt.figure(figsize=(6, 5))
sns.heatmap(cm, annot=True, fmt='d', cmap='Blues',
            xticklabels=['Down/Flat', 'Up'],
            yticklabels=['Down/Flat', 'Up'])
plt.title('Figure 1: Confusion Matrix (Final SVM)')
plt.ylabel('Actual')
plt.xlabel('Predicted')
plt.show()

# -------------------------------------------------------------------------
# 9) Comparison of CV results
# -------------------------------------------------------------------------
models_comparison = {
    'SVM (Bayesian)': bayes_search.best_score_,
    'LDA': lda_grid_search.best_score_,
}

# Only add if they were run
if grid_search is not None:
    models_comparison['SVM (Grid Search)'] = grid_search.best_score_
if random_search is not None:
    models_comparison['SVM (Random Search)'] = random_search.best_score_
if nn_bayes_search is not None:
    models_comparison['Neural Network'] = nn_bayes_search.best_score_

comparison_df = pd.DataFrame(list(models_comparison.items()), columns=['Model', 'CV Score']).sort_values('CV Score', ascending=False)
print("\nModels comparison (CV scores):")
print(comparison_df)

plt.figure(figsize=(10, 5))
bars = plt.bar(comparison_df['Model'], comparison_df['CV Score'])
plt.title('Figure 2: Comparison of Model CV Scores')
plt.ylabel('CV Accuracy')
plt.xticks(rotation=45)
for bar in bars:
    plt.text(bar.get_x() + bar.get_width()/2., bar.get_height(), f"{bar.get_height():.4f}", ha='center', va='bottom')
plt.tight_layout()
plt.show()

# -------------------------------------------------------------------------
# 10) FEATURE IMPORTANCE via LDA coefficients
# -------------------------------------------------------------------------
best_lda = lda_grid_search.best_estimator_.named_steps['lda']
feature_importance = pd.DataFrame({
    'feature': X.columns,
    'importance': np.abs(best_lda.coef_[0]) if hasattr(best_lda, 'coef_') else np.zeros(len(X.columns))
})
feature_importance = feature_importance.sort_values('importance', ascending=False)

plt.figure(figsize=(10, 6))
plt.barh(feature_importance['feature'][:10], feature_importance['importance'][:10])
plt.gca().invert_yaxis()
plt.title('Figure 3: Top 10 features (LDA absolute coeff)')
plt.xlabel('Absolute Coefficient')
plt.tight_layout()
plt.show()

print("\nTop 10 most important features (LDA):")
print(feature_importance.head(10))

2025-10-20 22:34:15.440155: I external/local_xla/xla/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used.
2025-10-20 22:34:15.479622: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
2025-10-20 22:34:16.368580: I external/local_xla/xla/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used.

--- RUNNING IN ULTRA-FAST MODE ---
Downloading data...

[*********************100%***********************]  1 of 1 completed

Raw data shape: (1006, 6)
Calculating technical indicators...

Prepared data shapes:
X shape: (973, 11)
y shape: (973,)
Feature columns: ['RSI_14', 'MACD', 'MACD_signal', 'MACD_hist', 'BB_upper', 'BB_middle', 'BB_lower', 'OBV', 'return_lag_1', 'return_lag_2', 'volatility_10d']
Skipping Grid Search for SVM
Skipping Randomized Search for SVM
Running Bayesian Optimization for SVM...
Fitting 3 folds for each of 1 candidates, totalling 3 fits
Fitting 3 folds for each of 1 candidates, totalling 3 fits
Fitting 3 folds for each of 1 candidates, totalling 3 fits
Bayes Search best score: 0.5185185185185185
Bayes Search best params: OrderedDict({'svm__C': 105.76211650904162, 'svm__gamma': 2.6096146808538574, 'svm__kernel': 'rbf'})
Skipping Neural Network for speed
Running Grid Search for LDA...
Fitting 3 folds for each of 6 candidates, totalling 18 fits
LDA best score: 0.4732510288065843
LDA best params: {'lda__shrinkage': 1.0, 'lda__solver': 'lsqr'}
Training final SVM model with optimized hyperparameters...
Final model test accuracy: 0.5436
              precision    recall  f1-score   support

           0       0.44      0.21      0.29        84
           1       0.57      0.79      0.66       111

    accuracy                           0.54       195
   macro avg       0.51      0.50      0.48       195
weighted avg       0.51      0.54      0.50       195

Models comparison (CV scores):
            Model  CV Score
0  SVM (Bayesian)  0.518519
1             LDA  0.473251

Top 10 most important features (LDA):
           feature  importance
4         BB_upper    0.096574
8     return_lag_1    0.078948
5        BB_middle    0.075201
10  volatility_10d    0.065124
2      MACD_signal    0.060802
0           RSI_14    0.059915
6         BB_lower    0.054014
1             MACD    0.053771
3        MACD_hist    0.008221
9     return_lag_2    0.007151

import warnings
warnings.filterwarnings('ignore')

import numpy as np
import pandas as pd
import yfinance as yf
import pandas_ta as ta  # technical analysis helpers

# Machine Learning / optimization
from sklearn.model_selection import TimeSeriesSplit
from sklearn.preprocessing import StandardScaler
from sklearn.svm import SVR
from sklearn.linear_model import Lasso, Ridge
from sklearn.metrics import mean_squared_error
from sklearn.pipeline import Pipeline

import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense
from tensorflow.keras.optimizers import Adam

import matplotlib.pyplot as plt
import seaborn as sns

sns.set_style("whitegrid")

# -------------------------------------------------------------------------
# CONFIGURATION
# -------------------------------------------------------------------------
# Set to True for a quick test run with few iterations.
# Set to False for a full, thorough search (will take a long time).
FAST_MODE = True
# -------------------------------------------------------------------------

RANDOM_STATE = 42
np.random.seed(RANDOM_STATE)
tf.random.set_seed(RANDOM_STATE)

# --- Define search parameters based on FAST_MODE ---
if FAST_MODE:
    print("--- RUNNING IN FAST_MODE ---")
    svr_gamma_points = 15
    reg_alpha_points = 30
    nn_width_range = [1, 2, 4, 6, 8, 10, 12, 14, 16, 24, 32, 64]
    nn_epochs = 100
else:
    print("--- RUNNING IN FULL_MODE (this will take time) ---")
    svr_gamma_points = 30
    reg_alpha_points = 100
    nn_width_range = [1, 2, 3, 4, 5, 6, 8, 10, 12, 14, 16, 20, 24, 32, 48, 64, 96, 128, 192, 256]
    nn_epochs = 500


# -------------------------------------------------------------------------
# 1) DATA ACQUISITION & FEATURE ENGINEERING
# -------------------------------------------------------------------------
ticker = 'SPY'
data = yf.download(ticker, start='2010-01-01', end='2023-12-31', auto_adjust=False)
if isinstance(data.columns, pd.MultiIndex):
    data.columns = data.columns.droplevel(1)
data = data.reset_index()

# --- Features (same as before) ---
data_indexed = data.set_index('Date')
data['RSI_14'] = ta.rsi(data_indexed['Close'], length=14).values
macd = ta.macd(data_indexed['Close'], fast=12, slow=26, signal=9)
data['MACD'] = macd.iloc[:, 0].values
data['MACD_hist'] = macd.iloc[:, 1].values
bbands = ta.bbands(data_indexed['Close'], length=20)
data['BB_upper'] = bbands.iloc[:, 2].values
data['BB_middle'] = bbands.iloc[:, 1].values
data['BB_lower'] = bbands.iloc[:, 0].values
data['OBV'] = ta.obv(data_indexed['Close'], data_indexed['Volume']).values
data['ATR_14'] = ta.atr(data_indexed['High'], data_indexed['Low'], data_indexed['Close'], length=14).values
for lag in range(1, 6):
    data[f'return_lag_{lag}'] = data['Close'].pct_change(lag)
data['volatility_20d'] = data['Close'].pct_change().rolling(window=20).std()

# --- TARGET (MODIFIED FOR REGRESSION) ---
# We predict the next day's percentage return, matching the MSE math.
# Multiply by 100 to make MSE values more readable.
data['target'] = data['Close'].pct_change(1).shift(-1) * 100

print("Data preparation...")
data.dropna(inplace=True)
data.reset_index(drop=True, inplace=True)

# --- Define X and y ---
features_to_drop = ['Date', 'Open', 'High', 'Low', 'Close', 'Adj Close', 'Volume', 'target']
X = data.drop(columns=[col for col in features_to_drop if col in data.columns])
y = data['target']

# --- Train/Test Split ---
train_size = int(len(X) * 0.8)
X_train, X_test = X.iloc[:train_size], X.iloc[train_size:]
y_train, y_test = y.iloc[:train_size], y.iloc[train_size:]

# --- Scale Data (CRUCIAL) ---
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)
n_features = X.shape[1]

print(f"Data ready. n_features: {n_features}, n_train: {len(y_train)}, n_test: {len(y_test)}")

# -------------------------------------------------------------------------
# 2) SECTION 1: THE CLASSIC BIAS-VARIANCE U-CURVE
# -------------------------------------------------------------------------
print("\nRunning Section 1: Classic Bias-Variance U-Curve (SVR)...")

# We vary 'gamma' to control complexity.
# Low gamma = simple model (high bias)
# High gamma = complex model (high variance)
gamma_range = np.logspace(-4, 1, svr_gamma_points)
train_errors = []
test_errors = []

for gamma in gamma_range:
    model = SVR(kernel='rbf', gamma=gamma, C=1.0) # C=1 is a reasonable default
    model.fit(X_train_scaled, y_train)

    y_pred_train = model.predict(X_train_scaled)
    y_pred_test = model.predict(X_test_scaled)

    train_errors.append(mean_squared_error(y_train, y_pred_train))
    test_errors.append(mean_squared_error(y_test, y_pred_test))

plt.figure(figsize=(10, 6))
plt.plot(gamma_range, train_errors, 'b-o', label='Train Error (Bias Proxy)')
plt.plot(gamma_range, test_errors, 'r-o', label='Test Error (Total Error)')
plt.xscale('log')
plt.xlabel('Model Complexity (SVR Gamma)')
plt.ylabel('Mean Squared Error (MSE)')
plt.title('Figure 4: The Classic Bias-Variance Tradeoff')
plt.legend()
plt.ylim(0, np.mean(y_test**2) * 1.5) # Cap y-axis at 1.5x variance of y
plt.fill_between(gamma_range, train_errors, test_errors, color='orange', alpha=0.2, label='Variance Proxy (Train-Test Gap)')
plt.axvline(x=gamma_range[np.argmin(test_errors)], color='grey', linestyle='--', label=f'Optimal Gamma')
plt.legend()
plt.show()

# -------------------------------------------------------------------------
# 3) SECTION 2: REGULARIZATION (L1 & L2)
# -------------------------------------------------------------------------
print("\nRunning Section 2: Regularization Paths (Lasso/Ridge)...")

alpha_range = np.logspace(-5, 2, reg_alpha_points)

lasso_coefs = []
lasso_test_mse = []
ridge_coefs = []
ridge_test_mse = []

for alpha in alpha_range:
    # Lasso (L1)
    lasso = Lasso(alpha=alpha, random_state=RANDOM_STATE, max_iter=1000)
    lasso.fit(X_train_scaled, y_train)
    lasso_coefs.append(lasso.coef_)
    lasso_test_mse.append(mean_squared_error(y_test, lasso.predict(X_test_scaled)))

    # Ridge (L2)
    ridge = Ridge(alpha=alpha, random_state=RANDOM_STATE)
    ridge.fit(X_train_scaled, y_train)
    ridge_coefs.append(ridge.coef_)
    ridge_test_mse.append(mean_squared_error(y_test, ridge.predict(X_test_scaled)))

# --- Plot Regularization Results ---
fig, axs = plt.subplots(2, 2, figsize=(16, 12))
fig.suptitle('Figure 5: Regularization Impact on Bias-Variance Tradeoff', fontsize=16)

# Plot 1: Lasso (L1) Coefficient Path
axs[0, 0].plot(alpha_range, lasso_coefs)
axs[0, 0].set_xscale('log')
axs[0, 0].set_title('A) L1 (Lasso) Coefficient Path')
axs[0, 0].set_xlabel('Penalty (alpha)')
axs[0, 0].set_ylabel('Coefficient Weight')

# Plot 2: Lasso (L1) MSE vs. Alpha
axs[0, 1].plot(alpha_range, lasso_test_mse, 'r-o')
axs[0, 1].set_xscale('log')
axs[0, 1].set_title('B) L1 (Lasso) Test Error')
axs[0, 1].set_xlabel('Penalty (alpha)')
axs[0, 1].set_ylabel('Test MSE')
axs[0, 1].axvline(x=alpha_range[np.argmin(lasso_test_mse)], color='grey', linestyle='--', label='Optimal Alpha')
axs[0, 1].legend()
axs[0, 1].set_ylim(min(lasso_test_mse) * 0.95, min(lasso_test_mse) * 1.5) # Zoom in

# Plot 3: Ridge (L2) Coefficient Path
axs[1, 0].plot(alpha_range, ridge_coefs)
axs[1, 0].set_xscale('log')
axs[1, 0].set_title('C) L2 (Ridge) Coefficient Path')
axs[1, 0].set_xlabel('Penalty (alpha)')
axs[1, 0].set_ylabel('Coefficient Weight')

# Plot 4: Ridge (L2) MSE vs. Alpha
axs[1, 1].plot(alpha_range, ridge_test_mse, 'r-o')
axs[1, 1].set_xscale('log')
axs[1, 1].set_title('D) L2 (Ridge) Test Error')
axs[1, 1].set_xlabel('Penalty (alpha)')
axs[1, 1].set_ylabel('Test MSE')
axs[1, 1].axvline(x=alpha_range[np.argmin(ridge_test_mse)], color='grey', linestyle='--', label='Optimal Alpha')
axs[1, 1].legend()
axs[1, 1].set_ylim(min(ridge_test_mse) * 0.95, min(ridge_test_mse) * 1.5) # Zoom in

plt.tight_layout(rect=[0, 0.03, 1, 0.95])
plt.show()

# -------------------------------------------------------------------------
# 4) SECTION 3: THE DOUBLE DESCENT PHENOMENON
# -------------------------------------------------------------------------
print("\nRunning Section 3: Double Descent (Neural Network)...")
print(f"(Training {len(nn_width_range)} models for {nn_epochs} epochs each...)")

# We need a small dataset to find the "interpolation threshold" (p ≈ n)
n_samples = 200
X_train_sub = X_train_scaled[:n_samples]
y_train_sub = y_train.iloc[:n_samples]

# Estimate interpolation threshold: p ≈ n
# For a 1-layer NN, p ≈ (n_features * width) + width + (width * 1) + 1
# p ≈ (n_features + 2) * width
# p ≈ n_samples  =>  (17 + 2) * width ≈ 200 => 19 * width ≈ 200
interpolation_threshold_width = n_samples / (n_features + 2)
print(f"Interpolation Threshold (p≈n) estimated at width ≈ {interpolation_threshold_width:.1f}")

nn_train_errors = []
nn_test_errors = []

for width in nn_width_range:
    # Define a simple 1-hidden-layer NN
    # NO regularization (no dropout, no L1/L2) is key to seeing the effect
    model = Sequential([
        Dense(width, activation='relu', input_dim=n_features),
        Dense(1)  # Linear output for regression
    ])

    model.compile(optimizer=Adam(learning_rate=0.001), loss='mse')

    # Train to interpolation (or close to it)
    model.fit(X_train_sub, y_train_sub, epochs=nn_epochs, verbose=0, batch_size=32)

    # Evaluate
    train_mse = model.evaluate(X_train_sub, y_train_sub, verbose=0)
    test_mse = model.evaluate(X_test_scaled, y_test, verbose=0)

    nn_train_errors.append(train_mse)
    nn_test_errors.append(test_mse)

    if width % 8 == 0 or width == 1:
        print(f"  Width: {width:3d} | Train MSE: {train_mse:7.4f} | Test MSE: {test_mse:7.4f}")


plt.figure(figsize=(12, 7))
plt.plot(nn_width_range, nn_train_errors, 'b-o', label='Train Error')
plt.plot(nn_width_range, nn_test_errors, 'r-o', label='Test Error')
plt.xscale('log')
plt.yscale('log')
plt.title('Figure 6: The Double Descent Phenomenon')
plt.xlabel('Model Capacity (Width of Hidden Layer)')
plt.ylabel('Mean Squared Error (Log Scale)')

# Highlight the two regimes
plt.axvline(x=interpolation_threshold_width, color='grey', linestyle='--',
            label=f'Interpolation Threshold (p≈n)\nWidth ≈ {interpolation_threshold_width:.1f}')
plt.axvspan(min(nn_width_range), interpolation_threshold_width, alpha=0.1, color='blue', label='Classical Regime')
plt.axvspan(interpolation_threshold_width, max(nn_width_range), alpha=0.1, color='green', label='Modern Regime')
plt.legend()
plt.show()

--- RUNNING IN FAST_MODE ---

[*********************100%***********************]  1 of 1 completed

Data preparation...
Data ready. n_features: 14, n_train: 2790, n_test: 698

Running Section 1: Classic Bias-Variance U-Curve (SVR)...

Running Section 2: Regularization Paths (Lasso/Ridge)...

Running Section 3: Double Descent (Neural Network)...
(Training 12 models for 100 epochs each...)
Interpolation Threshold (p≈n) estimated at width ≈ 12.5

2025-10-20 22:34:31.243612: E external/local_xla/xla/stream_executor/cuda/cuda_platform.cc:51] failed call to cuInit: INTERNAL: CUDA error: Failed call to cuInit: UNKNOWN ERROR (303)

  Width:   1 | Train MSE:  1.3753 | Test MSE:  1.2711
  Width:   8 | Train MSE:  1.2898 | Test MSE:  2.5647
  Width:  16 | Train MSE:  1.2056 | Test MSE:  4.3503
  Width:  24 | Train MSE:  1.2509 | Test MSE:  1.8374
  Width:  32 | Train MSE:  1.1483 | Test MSE:  7.0487
  Width:  64 | Train MSE:  1.0316 | Test MSE:  5.3992

import warnings
warnings.filterwarnings('ignore')

import numpy as np
import pandas as pd
import yfinance as yf
import pandas_ta as ta  # technical analysis helpers

# Machine Learning / optimization
from sklearn.model_selection import TimeSeriesSplit
from sklearn.preprocessing import StandardScaler
from sklearn.svm import SVC
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import classification_report, confusion_matrix, accuracy_score
from sklearn.pipeline import Pipeline
from sklearn.base import clone # Import clone

# Ensemble Methods
from sklearn.ensemble import RandomForestClassifier # (Bagging)
from sklearn.ensemble import AdaBoostClassifier # (Boosting)
# StackingClassifier is no longer used
# from sklearn.ensemble import StackingClassifier 

# Neural Network
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.regularizers import l2
from scikeras.wrappers import KerasClassifier

import matplotlib.pyplot as plt
import seaborn as sns

sns.set_style("whitegrid")

# -------------------------------------------------------------------------
# CONFIGURATION
# -------------------------------------------------------------------------
# Set to True for a quick test run with few iterations/estimators.
# Set to False for a full, thorough run (will take longer).
FAST_MODE = True
# -------------------------------------------------------------------------

RANDOM_STATE = 42
np.random.seed(RANDOM_STATE)
tf.random.set_seed(RANDOM_STATE)

# --- Define run parameters based on FAST_MODE ---
if FAST_MODE:
    print("--- RUNNING IN FAST_MODE ---")
    nn_epochs = 10
    rf_estimators = 50
    ada_estimators = 50
else:
    print("--- RUNNING IN FULL_MODE (this will take time) ---")
    nn_epochs = 50
    rf_estimators = 200
    ada_estimators = 200

# -------------------------------------------------------------------------
# 1) DATA ACQUISITION & FEATURE ENGINEERING
# -------------------------------------------------------------------------
print("Fetching and preparing data...")
ticker = 'SPY'
data = yf.download(ticker, start='2010-01-01', end='2023-12-31', auto_adjust=False)
if isinstance(data.columns, pd.MultiIndex):
    data.columns = data.columns.droplevel(1)
data = data.reset_index()

# --- Features (same as before) ---
data_indexed = data.set_index('Date')
data['RSI_14'] = ta.rsi(data_indexed['Close'], length=14).values
macd = ta.macd(data_indexed['Close'], fast=12, slow=26, signal=9)
data['MACD'] = macd.iloc[:, 0].values
data['MACD_hist'] = macd.iloc[:, 1].values
bbands = ta.bbands(data_indexed['Close'], length=20)
data['BB_upper'] = bbands.iloc[:, 2].values
data['BB_middle'] = bbands.iloc[:, 1].values
data['BB_lower'] = bbands.iloc[:, 0].values
data['OBV'] = ta.obv(data_indexed['Close'], data_indexed['Volume']).values
data['ATR_14'] = ta.atr(data_indexed['High'], data_indexed['Low'], data_indexed['Close'], length=14).values
for lag in range(1, 6):
    data[f'return_lag_{lag}'] = data['Close'].pct_change(lag)
data['volatility_20d'] = data['Close'].pct_change().rolling(window=20).std()

# --- TARGET (CLASSIFICATION) ---
# We predict the binary direction: 1 for 'Up', 0 for 'Down' or 'Flat'
data['target'] = (data['Close'].shift(-1) > data['Close']).astype(int)

# --- Cleaning and Splitting ---
data.dropna(inplace=True)
data.reset_index(drop=True, inplace=True)

features_to_drop = ['Date', 'Open', 'High', 'Low', 'Close', 'Adj Close', 'Volume', 'target']
X = data.drop(columns=[col for col in features_to_drop if col in data.columns])
y = data['target']

# --- Standard 80/20 Train/Test Split ---
# We will use the *raw* (unscaled) X_train/X_test for pipelines
train_size = int(len(X) * 0.8)
X_train, X_test = X.iloc[:train_size], X.iloc[train_size:]
y_train, y_test = y.iloc[:train_size], y.iloc[train_size:]

# --- Scaled data for non-pipeline models ---
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)

# --- TimeSeriesSplit for CV in Stacking ---
n_splits = 5
tscv = TimeSeriesSplit(n_splits=n_splits)

print(f"Data ready. X_train shape: {X_train.shape}, X_test shape: {X_test.shape}")


# -------------------------------------------------------------------------
# 2) DEFINE "THE SPECIALISTS" (BASE MODELS)
# -------------------------------------------------------------------------
# We define our three diverse models as pipelines, using
# "optimal" params found in previous steps.

# Specialist 1: SVM (The Pattern-Recognizer)
# Using assumed best params from a Bayesian search
svm_params = {'C': 10.0, 'gamma': 0.01, 'kernel': 'rbf', 'probability': True}
svm_pipeline = Pipeline([
    ('scaler', StandardScaler()),
    ('svm', SVC(**svm_params, random_state=RANDOM_STATE))
])

# Specialist 2: Neural Network (The Deep-Analyst)
# Keras model creation function
def create_nn_model(learning_rate=0.001, dropout_rate=0.2, l2_reg=0.01, num_layers=2, neurons_per_layer=64):
    model = Sequential()
    model.add(Dense(neurons_per_layer, input_dim=X.shape[1], activation='relu',
                    kernel_regularizer=l2(l2_reg)))
    model.add(Dropout(dropout_rate))
    for _ in range(num_layers - 1):
        model.add(Dense(max(4, neurons_per_layer // 2), activation='relu',
                        kernel_regularizer=l2(l2_reg)))
        model.add(Dropout(dropout_rate))
    model.add(Dense(1, activation='sigmoid'))
    optimizer = Adam(learning_rate=learning_rate)
    model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy'])
    return model

# Using assumed best params from a Bayesian search
nn_params = {
    'model__learning_rate': 0.001,
    'model__dropout_rate': 0.2,
    'model__l2_reg': 0.01,
    'model__num_layers': 2,
    'model__neurons_per_layer': 64,
    'batch_size': 32
}
nn_classifier = KerasClassifier(
    model=create_nn_model,
    epochs=nn_epochs,
    verbose=0,
    random_state=RANDOM_STATE,
    **nn_params
)
nn_pipeline = Pipeline([
    ('scaler', StandardScaler()),
    ('model', nn_classifier)
])

# Specialist 3: LDA (The Cautious Economist)
# Using assumed best params from a Grid search
lda_params = {'solver': 'lsqr', 'shrinkage': 0.1}
lda_pipeline = Pipeline([
    ('scaler', StandardScaler()),
    ('lda', LinearDiscriminantAnalysis(**lda_params))
])

# List of our specialist estimators for Stacking
base_estimators = [
    ('svm', svm_pipeline),
    ('nn', nn_pipeline),
    ('lda', lda_pipeline)
]

# -------------------------------------------------------------------------
# 3) IMPLEMENT ENSEMBLE STRATEGIES
# -------------------------------------------------------------------------

# Dictionary to store results
model_scores = {}

# --- Strategy 1: Bagging (The Democratic Poll) ---
print("\nTraining Strategy 1: Bagging (Random Forest)...")
rf_model = RandomForestClassifier(
    n_estimators=rf_estimators,
    random_state=RANDOM_STATE,
    n_jobs=-1,
    max_depth=10 # Control tree depth to prevent overfitting
)
# Tree models don't strictly need scaling, but we use scaled data
# for a fair comparison with other scaled, non-pipelined models.
rf_model.fit(X_train_scaled, y_train)
y_pred_rf = rf_model.predict(X_test_scaled)
rf_accuracy = accuracy_score(y_test, y_pred_rf)
model_scores['Bagging (RandomForest)'] = rf_accuracy
print(f"Random Forest Test Accuracy: {rf_accuracy:.4f}")
# print(classification_report(y_test, y_pred_rf))


# --- Strategy 2: Boosting (The Focused Task Force) ---
print("\nTraining Strategy 2: Boosting (AdaBoost)...")
ada_model = AdaBoostClassifier(
    n_estimators=ada_estimators,
    random_state=RANDOM_STATE,
    learning_rate=0.1
)
ada_model.fit(X_train_scaled, y_train)
y_pred_ada = ada_model.predict(X_test_scaled)
ada_accuracy = accuracy_score(y_test, y_pred_ada)
model_scores['Boosting (AdaBoost)'] = ada_accuracy
print(f"AdaBoost Test Accuracy: {ada_accuracy:.4f}")
# print(classification_report(y_test, y_pred_ada))


# --- Strategy 3: Stacking (The Expert Council) - MANUAL IMPLEMENTATION ---
print("\nTraining Strategy 3: Stacking (Manual Time-Series Method)...")
# The "Chairperson" (meta-learner)
meta_learner = LogisticRegression(C=1.0, random_state=RANDOM_STATE, n_jobs=-1)

# Lists to store out-of-fold predictions (meta-features) and corresponding targets
meta_X_train_list = []
meta_y_train_list = []

print("Generating out-of-fold predictions for meta-learner...")
# Loop through each TimeSeriesSplit fold
for fold_n, (train_idx, val_idx) in enumerate(tscv.split(X_train)):
    print(f"  ...Processing Fold {fold_n+1}/{n_splits}")
    X_fold_train, X_fold_val = X_train.iloc[train_idx], X_train.iloc[val_idx]
    y_fold_train, y_fold_val = y_train.iloc[train_idx], y_train.iloc[val_idx]

    # Store predictions for this fold
    fold_meta_features = []
    
    # Train each base model and get predictions on the validation set
    for name, model_pipeline in base_estimators:
        cloned_model = clone(model_pipeline)
        cloned_model.fit(X_fold_train, y_fold_train)
        
        # Use predict_proba for more info (probability of class 1)
        # Ensure model has predict_proba (like LDA) or handle it
        if hasattr(cloned_model, "predict_proba"):
            preds = cloned_model.predict_proba(X_fold_val)[:, 1]
        else: # Fallback for models like base SVM without probability=True
            preds = cloned_model.predict(X_fold_val)
            
        fold_meta_features.append(preds)

    # Add this fold's predictions (meta-features) to the list
    meta_X_train_list.append(np.column_stack(fold_meta_features))
    # Add this fold's true targets to the list
    meta_y_train_list.append(y_fold_val)

# Concatenate all out-of-fold predictions to create the meta-learner's training set
Z_train = np.concatenate(meta_X_train_list)
y_meta_train = np.concatenate(meta_y_train_list)

print(f"\nMeta-learner training on {Z_train.shape[0]} out-of-fold samples.")

# Train the "Chairperson" (meta-learner) on the out-of-fold predictions
meta_learner.fit(Z_train, y_meta_train)

# --- Generate predictions for the final X_test ---
# 1. Train all base models on the *entire* X_train dataset
# 2. Generate predictions (meta-features) on X_test
# 3. Feed these meta-features to the *trained* meta_learner

print("Training base models on full data for final test prediction...")
Z_test_list = []
for name, model_pipeline in base_estimators:
    # We must re-fit the models on the *full* training data
    cloned_model = clone(model_pipeline)
    cloned_model.fit(X_train, y_train)
    
    if hasattr(cloned_model, "predict_proba"):
        preds = cloned_model.predict_proba(X_test)[:, 1]
    else:
        preds = cloned_model.predict(X_test)
        
    Z_test_list.append(preds)

# Create the meta-feature matrix for the test set
Z_test = np.column_stack(Z_test_list)

print("Generating final stacking predictions...")
# Get the final prediction from the meta-learner
y_pred_stack = meta_learner.predict(Z_test)
stack_accuracy = accuracy_score(y_test, y_pred_stack)
model_scores['Stacking (Manual)'] = stack_accuracy # Updated name
print(f"Stacking Test Accuracy: {stack_accuracy:.4f}")

print("\nStacking Model Classification Report (Test Set):")
print(classification_report(y_test, y_pred_stack))

# Confusion matrix for the recommended model
cm = confusion_matrix(y_test, y_pred_stack)
plt.figure(figsize=(6, 5))
sns.heatmap(cm, annot=True, fmt='d', cmap='Blues',
            xticklabels=['Down/Flat', 'Up'],
            yticklabels=['Down/Flat', 'Up'])
plt.title('Figure 7: Confusion Matrix (Stacking Model)')
plt.ylabel('Actual')
plt.xlabel('Predicted')
plt.show()

# -------------------------------------------------------------------------
# 4) FINAL COMPARISON
# -------------------------------------------------------------------------
print("\nTraining individual 'specialists' for comparison...")

# Train and score each base model individually
# We fit on *raw* X_train, as they are pipelines
svm_pipeline.fit(X_train, y_train)
model_scores['SVM (Specialist)'] = accuracy_score(y_test, svm_pipeline.predict(X_test))
print(f"Individual SVM Accuracy: {model_scores['SVM (Specialist)']:.4f}")

lda_pipeline.fit(X_train, y_train)
model_scores['LDA (Specialist)'] = accuracy_score(y_test, lda_pipeline.predict(X_test))
print(f"Individual LDA Accuracy: {model_scores['LDA (Specialist)']:.4f}")

# The NN pipeline takes a bit longer
nn_pipeline.fit(X_train, y_train)
model_scores['NN (Specialist)'] = accuracy_score(y_test, nn_pipeline.predict(X_test))
print(f"Individual NN Accuracy: {model_scores['NN (Specialist)']:.4f}")


# --- Plot Comparison Chart ---
comparison_df = pd.DataFrame(list(model_scores.items()), columns=['Model', 'Test Accuracy'])
comparison_df = comparison_df.sort_values('Test Accuracy', ascending=False)

print("\n--- Final Model Comparison ---")
print(comparison_df.to_string(index=False))

plt.figure(figsize=(10, 6))
bars = sns.barplot(
    data=comparison_df,
    x='Test Accuracy',
    y='Model',
    palette='viridis'
)
plt.title('Figure 8: Ensemble vs. Individual Model Performance')
plt.xlabel('Test Set Accuracy')
plt.ylabel('Model')
plt.xlim(min(model_scores.values()) * 0.95, max(model_scores.values()) * 1.05)

# Add text labels
for bar in bars.patches:
    bars.annotate(
        f'{bar.get_width():.4f}',
        (bar.get_width(), bar.get_y() + bar.get_height() / 2),
        ha='left', va='center',
        size=10, xytext=(5, 0),
        textcoords='offset points'
    )

plt.tight_layout()
plt.show()

[*********************100%***********************]  1 of 1 completed

--- RUNNING IN FAST_MODE ---
Fetching and preparing data...
Data ready. X_train shape: (2791, 14), X_test shape: (698, 14)

Training Strategy 1: Bagging (Random Forest)...
Random Forest Test Accuracy: 0.4756

Training Strategy 2: Boosting (AdaBoost)...

AdaBoost Test Accuracy: 0.5201

Training Strategy 3: Stacking (Manual Time-Series Method)...
Generating out-of-fold predictions for meta-learner...
  ...Processing Fold 1/5

2025-10-20 22:35:05.520299: E tensorflow/core/framework/node_def_util.cc:680] NodeDef mentions attribute use_unbounded_threadpool which is not in the op definition: Op<name=MapDataset; signature=input_dataset:variant, other_arguments: -> handle:variant; attr=f:func; attr=Targuments:list(type),min=0; attr=output_types:list(type),min=1; attr=output_shapes:list(shape),min=1; attr=use_inter_op_parallelism:bool,default=true; attr=preserve_cardinality:bool,default=false; attr=force_synchronous:bool,default=false; attr=metadata:string,default=""> This may be expected if your graph generating binary is newer  than this binary. Unknown attributes will be ignored. NodeDef: {{node ParallelMapDatasetV2/_15}}

  ...Processing Fold 2/5
  ...Processing Fold 3/5
  ...Processing Fold 4/5
  ...Processing Fold 5/5

2025-10-20 22:35:12.407510: E tensorflow/core/framework/node_def_util.cc:680] NodeDef mentions attribute use_unbounded_threadpool which is not in the op definition: Op<name=MapDataset; signature=input_dataset:variant, other_arguments: -> handle:variant; attr=f:func; attr=Targuments:list(type),min=0; attr=output_types:list(type),min=1; attr=output_shapes:list(shape),min=1; attr=use_inter_op_parallelism:bool,default=true; attr=preserve_cardinality:bool,default=false; attr=force_synchronous:bool,default=false; attr=metadata:string,default=""> This may be expected if your graph generating binary is newer  than this binary. Unknown attributes will be ignored. NodeDef: {{node ParallelMapDatasetV2/_15}}

Meta-learner training on 2325 out-of-fold samples.
Training base models on full data for final test prediction...
Generating final stacking predictions...
Stacking Test Accuracy: 0.5201

Stacking Model Classification Report (Test Set):
              precision    recall  f1-score   support

           0       0.00      0.00      0.00       335
           1       0.52      1.00      0.68       363

    accuracy                           0.52       698
   macro avg       0.26      0.50      0.34       698
weighted avg       0.27      0.52      0.36       698

Training individual 'specialists' for comparison...
Individual SVM Accuracy: 0.5244
Individual LDA Accuracy: 0.5186

2025-10-20 22:35:17.634494: E tensorflow/core/framework/node_def_util.cc:680] NodeDef mentions attribute use_unbounded_threadpool which is not in the op definition: Op<name=MapDataset; signature=input_dataset:variant, other_arguments: -> handle:variant; attr=f:func; attr=Targuments:list(type),min=0; attr=output_types:list(type),min=1; attr=output_shapes:list(shape),min=1; attr=use_inter_op_parallelism:bool,default=true; attr=preserve_cardinality:bool,default=false; attr=force_synchronous:bool,default=false; attr=metadata:string,default=""> This may be expected if your graph generating binary is newer  than this binary. Unknown attributes will be ignored. NodeDef: {{node ParallelMapDatasetV2/_15}}

Model	Hyperparameter	Function in Financial Task	Ideal Range/Value
SVM	$C$ (Regularization)	Overfitting to market noise is balanced with model complexity.	$[10^{-3}, 10^3]$
	$\gamma$ (Kernel Width)	Indicates a single data point's influence range; this is important for localizing patterns.	$[10^{-4}, 10]$
	$kernel$	Identifies non-linear relationships in the data.	RBF (for intricate patterns)
Neural Network	Learning Rate	Regulates step size for convergence; crucial for training stability.	$[10^{-5}, 10^{-1}]$
	Layers / Neurons	Assesses the model's ability to learn market complexity.	1–5 layers; 16–512 neurons/layer
	Dropout	Prevents overfitting to temporal artifacts by randomly turning off neurons.	$[0.1, 0.5]$
LDA	shrinkage	Maintains covariance estimates in high-dimensional data.	$[0, 1]$
	solver	Assures numerical stability in the covariance matrix computation.	`'svd'` (when features > samples)

Method	Core Principle	Primary Goal	Base Learner Type	Combination Method	Key Advantage in Finance
Bagging	Trains many models at the same time, each on a slightly different random sample (bootstrap) of the data.	To reduce variance and make the model more stable.	Homogeneous (e.g., all Decision Trees)	Lets all the models vote (classification) or averages their results (regression).	Makes the model more stable and less likely to "overfit" or just memorize market noise.
Boosting	Builds models one after another, where each new model tries to fix the mistakes the previous one made.	To reduce bias (by fixing errors) and variance.	Homogeneous (e.g., simple "weak" learners)	A weighted sum, giving more power to the predictions from better models.	Creates a very accurate predictor by focusing on the hard-to-predict cases.
Stacking	Trains several different models, then trains a final "meta-model" that learns how to best combine their predictions.	To get the best possible predictive performance by mixing models.	Heterogeneous (e.g., SVM, NN, Trees)	A final "meta-learner" model makes the call based on what the first-level models said.	Dynamically picks the best parts of each model, making it great for complex forecasts where no single model is best.

Reviewer	Author	Review of Work (Issue)	What Was Great	Suggestions for Improvement
Student A	Student C	Issue 3: Ensemble Learning	• Awesome Analogies: The "Expert Council" metaphor makes stacking super easy to understand. • Covered All the Bases: You explained Bagging, Boosting, and Stacking really well, with all the right details. • Clear Recommendation: The reasoning for picking Stacking makes total sense and answers the main question perfectly.	1. Tiny Formatting Tweak: In the Boosting math, maybe use $\exp$ in LaTeX? Just for consistency. 2. One More Detail: It might be good to add a note that the final models in stacking get retrained on all the data before you use them.
Student B	Student A	Issue 1: Optimizing Hyperparameters	• Good Financial Focus: I liked how you kept bringing it back to temporal validation and finance-specific settings. • Clear Process: The "Strategic, Intelligent, Validation" framework was a really clear and smart way to explain it. • Helpful Summary: That comparison table at the end is a great, quick reference.	1. Connecting the Dots: Maybe try to link the first abstract formula more directly to how cross-validation actually uses it. 2. Readability: For the "Critical Questions" part, you could use bullet points to make it a bit faster to read.
Student C	Student B	Issue 2: Bias-Variance Tradeoff	• Really Deep Dive: Loved how you connected the classic theory to newer stuff like "Double Descent." • Great Framing: The "Two Types of Strategy Failure" idea was a perfect way to explain under/overfitting. • Spot-on Analogy: Describing regularization as "portfolio construction for factors" was perfect for a finance crowd.	A Small Structural Idea: Maybe add a quick, simple summary of "double descent" to the non-technical part? It would help prep the strategists for the more advanced idea later.

Introduction¶

Optimizing Hyperparameters¶

Technical Section¶

Model-Specific Hyperparameter Spaces¶

Optimization Methodologies¶

Financial Validation Procedure¶

Non-Technical Section¶

Optimizing the Bias-Variance Tradeoff¶

Technical Section¶

Mathematical Decomposition of Mean Squared Error (MSE)¶

The tradeoff with Regularization¶

The Double Descent Phenomenon¶

Non-Technical Section¶

Finding the "Just Right" Strategy¶

Regularization: The Built-in Bouncer¶

When "Too Complicated" Becomes Good¶

Practical Applications to Investment Management¶

Applying Ensemble Learning: Bagging, Boosting, or Stacking¶

Technical Section¶

Homogeneous Ensembles: Bagging and Boosting¶

Heterogeneous Ensembles: Stacking (Stacked Generalization)¶

Non-Technical Section¶

Peer Reviews¶

Marketing Alpha¶

Conclusion¶

References¶