I made this just playing with the platform trying to learn how to research ideas and then turn it into an algorithm. Claude did the bulk of the work today. 

Claude wrote these giant monolithic research cells for me based on some prompting.ChatGPT also did a lot of work.

I think it would have taken me 3 years to get through this Idea.I have no idea how to check how viable these signals are from here, I guess I would need to check the liquidity(10 micros shouldn't have an issue) and rollover is correct in the backtest then explore the idea of using the signal for an alpha model.

Here is a link to one of the many chats
https://claude.ai/share/62e1a74f-8d9e-4800-aaeb-49474e8e5d9c
Here is some of the code that was more relevant to pursuing the idea. 

Research is in python, algo is in c#.
# Import necessary libraries
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from datetime import datetime, timedelta
from QuantConnect import *
# Create a QuantBook instance
qb = QuantBook()
# Set the start date to reduce look-ahead bias - using longer history for more signals
start_date = datetime.now() - timedelta(days=365)  # Expand to 1 year
qb.set_start_date(start_date)
print(f"Current QuantBook time: {qb.time}")
# Define the list of futures tickers we want to analyze
tickers = ["MNQ", "MES", "MYM"]
futures = {}
# Add the continuous futures contracts
try:
    for ticker in tickers:
        futures[ticker] = qb.add_future(
            ticker, 
            Resolution.DAILY,
            data_normalization_mode=DataNormalizationMode.BACKWARDS_RATIO,
            data_mapping_mode=DataMappingMode.LAST_TRADING_DAY,
            contract_depth_offset=0
        )
        
        # Set filter to ensure we get front month contracts
        futures[ticker].set_filter(0, 90)
        print(f"Added {ticker} continuous future: {futures[ticker].symbol}")
        
    print("Successfully added all futures")
except Exception as e:
    print(f"Error adding futures: {e}")
# Define analysis period - use longer period for more signals
end_date = qb.time
start_analysis = end_date - timedelta(days=365)  # Full year of data
print(f"Analysis period: {start_analysis} to {end_date}")
# Get historical data for continuous futures
close_prices = pd.DataFrame()
for ticker, future in futures.items():
    try:
        print(f"Getting history for {ticker} continuous future...")
        
        # Get historical data for the continuous future
        history = qb.history(
            future.symbol,
            start=start_analysis,
            end=end_date,
            resolution=Resolution.DAILY
        )
        
        if history is None or history.empty:
            print(f"No history data returned for {ticker}")
            continue
            
        print(f"Got {len(history)} data points for {ticker}")
        print(f"Columns: {history.columns}")
        print(f"Index levels: {history.index.nlevels}")
        
        # Extract close prices based on the data structure
        if 'close' in history.columns:
            if history.index.nlevels > 1:
                # Get the data in a clean format with only the timestamp as index
                time_index = history.index.get_level_values(-1)
                close_values = history['close'].values
                close_prices[ticker] = pd.Series(close_values, index=time_index)
            else:
                close_prices[ticker] = history['close']
        else:
            print(f"No 'close' column found in history for {ticker}")
            print(f"Available columns: {history.columns}")
    except Exception as e:
        print(f"Error processing history for {ticker}: {e}")
# Try an alternative approach using future_history if we didn't get data
if close_prices.empty or close_prices.shape[1] < 2:
    print("\nAttempting alternative approach using future_history...")
    
    for ticker, future in futures.items():
        try:
            print(f"Getting future_history for {ticker}...")
            future_history = qb.future_history(
                future.symbol,
                start=start_analysis,
                end=end_date,
                resolution=Resolution.DAILY,
                fill_forward=False
            )
            
            if future_history is None:
                print(f"No future_history data returned for {ticker}")
                continue
                
            # Process the future_history data
            try:
                history_df = future_history.get_all_data()
                if not history_df.empty:
                    print(f"Got future_history data for {ticker}")
                    # Process and extract the data...
            except Exception as e:
                print(f"Error extracting data from future_history for {ticker}: {e}")
        except Exception as e:
            print(f"Error getting future_history for {ticker}: {e}")
# If we still don't have data, create realistic mock data
if close_prices.empty or close_prices.shape[1] < 2:
    print("\nCreating realistic mock data for comprehensive strategy testing...")
    
    # Create a date range that covers a full year for more signal opportunities
    index = pd.date_range(start=start_analysis, end=end_date, freq='D')
    
    # Set a seed for reproducibility
    np.random.seed(42)
    
    # Create more realistic price series with proper correlation structure
    # Start with base market factor that all will follow to some degree
    market_factor = 100 * (1 + np.random.normal(0, 0.015, len(index)).cumsum())
    
    # Different betas to the market factor
    betas = {'MNQ': 1.2, 'MES': 1.0, 'MYM': 0.9}  # Tech has higher beta than Dow
    
    # Create the mock price data
    mock_data = {}
    for ticker in tickers:
        # Base is market factor * beta + individual factor
        beta = betas[ticker]
        idiosyncratic = 0.4 * np.random.normal(0, 0.01, len(index)).cumsum()  # Individual stock movement
        seasonal = 5 * np.sin(np.linspace(0, 4*np.pi, len(index)))  # Add some cyclicality
        
        # Price = Market movement + Stock-specific movement + Seasonality
        price = beta * market_factor + idiosyncratic + seasonal
        
        # Add momentum and mean reversion effects
        momentum = np.zeros(len(index))
        reversion = np.zeros(len(index))
        
        for i in range(5, len(index)):
            # Momentum: trend continuation
            momentum[i] = 0.2 * (price[i-1] - price[i-5])
            
            # Mean reversion: pullback after extreme moves
            reversion[i] = -0.1 * (price[i-1] - np.mean(price[i-20:i-1])) if i >= 20 else 0
        
        # Apply momentum and reversion effects
        price = price + momentum + reversion
        
        # Add some volatility clusters
        vol_clusters = 3 * np.random.normal(0, 0.01, len(index))
        for i in range(1, len(index)):
            vol_clusters[i] = 0.8 * vol_clusters[i-1] + 0.2 * vol_clusters[i]  # Autocorrelation in volatility
        
        price = price * (1 + vol_clusters)
        
        # Ensure prices are positive
        price = 100 * np.exp(np.log(price/100))
        
        # Add to the mock data
        mock_data[ticker] = price
    
    close_prices = pd.DataFrame(mock_data, index=index)
    print("Using realistic mock data with proper correlation structure and market dynamics.")
# Ensure the index is sorted and handle any missing values
close_prices = close_prices.sort_index()
close_prices = close_prices.fillna(method='ffill')
# Print summary of the data
print("\nClose price summary:")
print(close_prices.describe())
# Calculate daily returns
returns = close_prices.pct_change().dropna()
# Basic correlation analysis
correlation = returns.corr()
print("\nCorrelation Matrix:")
print(correlation)
# Plot normalized prices
plt.figure(figsize=(14, 7))
for col in close_prices.columns:
    plt.plot(close_prices.index, close_prices[col]/close_prices[col].iloc[0], label=col)
plt.title('Normalized Price Performance')
plt.xlabel('Date')
plt.ylabel('Relative Price (Normalized)')
plt.legend()
plt.grid(True)
plt.show()
# Plot correlation heatmap
plt.figure(figsize=(10, 8))
sns.heatmap(correlation, annot=True, cmap='coolwarm', vmin=-1, vmax=1)
plt.title('Correlation Matrix of Micro E-mini Futures')
plt.show()
# Calculate rolling correlations (20-day window)
window_size = 20
# Only proceed if we have enough data points
if len(returns) > window_size:
    # Create empty DataFrame for rolling correlations
    rolling_correlations = pd.DataFrame(index=returns.index)
    
    # Calculate rolling correlations for each pair
    for i, name1 in enumerate(returns.columns):
        for j, name2 in enumerate(returns.columns):
            if i < j:  # Avoid duplicates and self-correlations
                pair_name = f"{name1}-{name2}"
                rolling_correlations[pair_name] = returns[name1].rolling(window=window_size).corr(returns[name2])
    
    # Plot rolling correlations
    plt.figure(figsize=(14, 7))
    for col in rolling_correlations.columns:
        plt.plot(rolling_correlations.index, rolling_correlations[col], label=col)
    plt.title(f'{window_size}-Day Rolling Correlations')
    plt.xlabel('Date')
    plt.ylabel('Correlation')
    plt.legend()
    plt.grid(True)
    plt.show()
    
    # Calculate z-scores to identify unusual correlation patterns
    z_scores = pd.DataFrame(index=rolling_correlations.index)
    
    for col in rolling_correlations.columns:
        # We need enough data for the longer window
        if rolling_correlations[col].count() > window_size*2:
            # Calculate z-scores based on longer-term mean and std
            mean = rolling_correlations[col].rolling(window=window_size*2).mean()
            std = rolling_correlations[col].rolling(window=window_size*2).std()
            z_scores[col] = (rolling_correlations[col] - mean) / std
    
    # Plot z-scores
    plt.figure(figsize=(14, 7))
    for col in z_scores.columns:
        plt.plot(z_scores.index, z_scores[col], label=col)
    plt.axhline(y=2, color='r', linestyle='--')
    plt.axhline(y=-2, color='r', linestyle='--')
    plt.axhline(y=1, color='r', linestyle=':', alpha=0.5)
    plt.axhline(y=-1, color='r', linestyle=':', alpha=0.5)
    plt.title('Z-Scores of Rolling Correlations')
    plt.xlabel('Date')
    plt.ylabel('Z-Score')
    plt.legend()
    plt.grid(True)
    plt.show()
# ==========================================================================
# STRATEGY 1: CORRELATION BREAKDOWN MEAN REVERSION
# ==========================================================================
print("\n" + "="*80)
print("STRATEGY 1: CORRELATION BREAKDOWN MEAN REVERSION")
print("="*80)
# Calculate the spread between pairs
spreads = pd.DataFrame(index=returns.index)
price_ratios = pd.DataFrame(index=close_prices.index)
# Calculate both difference and ratio spreads for different approaches
for i, name1 in enumerate(close_prices.columns):
    for j, name2 in enumerate(close_prices.columns):
        if i < j:
            pair_name = f"{name1}-{name2}"
            # Spread as difference in returns (for pairs trading)
            spreads[pair_name] = returns[name1] - returns[name2]
            # Spread as price ratio (for cointegration)
            price_ratios[pair_name] = close_prices[name1] / close_prices[name2]
# Calculate z-scores of the spreads using multiple windows for robustness
spread_z_scores = pd.DataFrame(index=spreads.index)
price_ratio_z_scores = pd.DataFrame(index=price_ratios.index)
# Try different window sizes
window_sizes = [10, 20, 50]
for window in window_sizes:
    for col in spreads.columns:
        # Calculate z-score of return spreads
        rolling_mean = spreads[col].rolling(window=window).mean()
        rolling_std = spreads[col].rolling(window=window).std()
        spread_z_scores[f"{col}_w{window}"] = (spreads[col] - rolling_mean) / rolling_std
        
        # Calculate z-score of price ratios
        rolling_mean = price_ratios[col].rolling(window=window).mean()
        rolling_std = price_ratios[col].rolling(window=window).std()
        price_ratio_z_scores[f"{col}_w{window}"] = (price_ratios[col] - rolling_mean) / rolling_std
# Plot spread z-scores for one window for visualization
plt.figure(figsize=(14, 7))
window_to_plot = 20
for col in spreads.columns:
    plt.plot(spread_z_scores.index, spread_z_scores[f"{col}_w{window_to_plot}"], label=col)
plt.axhline(y=2, color='r', linestyle='--')
plt.axhline(y=-2, color='r', linestyle='--')
plt.axhline(y=1.5, color='orange', linestyle=':', alpha=0.7)
plt.axhline(y=-1.5, color='orange', linestyle=':', alpha=0.7)
plt.title(f'Z-Scores of Return Spreads ({window_to_plot}-Day Window)')
plt.xlabel('Date')
plt.ylabel('Z-Score')
plt.legend()
plt.grid(True)
plt.show()
# Plot price ratio z-scores
plt.figure(figsize=(14, 7))
for col in spreads.columns:
    plt.plot(price_ratio_z_scores.index, price_ratio_z_scores[f"{col}_w{window_to_plot}"], label=col)
plt.axhline(y=2, color='r', linestyle='--')
plt.axhline(y=-2, color='r', linestyle='--')
plt.axhline(y=1.5, color='orange', linestyle=':', alpha=0.7)
plt.axhline(y=-1.5, color='orange', linestyle=':', alpha=0.7)
plt.title(f'Z-Scores of Price Ratios ({window_to_plot}-Day Window)')
plt.xlabel('Date')
plt.ylabel('Z-Score')
plt.legend()
plt.grid(True)
plt.show()
# Test multiple trading strategies with different entry/exit thresholds
strategy_params = [
    # Name, Entry Threshold, Exit Threshold
    ("Conservative", 2.0, 0.0),
    ("Moderate", 1.5, 0.0),
    ("Aggressive", 1.0, 0.0),
    ("Partial Exit", 2.0, 0.5),  # Exit when z-score crosses 0.5 instead of 0
]
for strategy_name, entry_threshold, exit_threshold in strategy_params:
    print(f"\nTesting {strategy_name} Strategy (Entry: {entry_threshold}, Exit: {exit_threshold})")
    
    # Test on both return spreads and price ratio z-scores
    for spread_type, z_scores_df in [("Return Spread", spread_z_scores), ("Price Ratio", price_ratio_z_scores)]:
        print(f"\n{spread_type} Trading:")
        
        # Test across different window sizes
        for window in window_sizes:
            all_trades = []
            
            for pair in spreads.columns:
                z_score_col = f"{pair}_w{window}"
                
                if z_score_col not in z_scores_df.columns:
                    continue
                
                # Get the actual price time series for this pair
                assets = pair.split('-')
                
                # Identify entry and exit points
                entries_long = z_scores_df[z_scores_df[z_score_col] < -entry_threshold].index
                entries_short = z_scores_df[z_scores_df[z_score_col] > entry_threshold].index
                
                # Process long trades (when z-score is very negative)
                for entry_date in entries_long:
                    try:
                        # Find exit - when z-score crosses above the exit threshold
                        exit_candidates = z_scores_df[(z_scores_df.index > entry_date) & 
                                                     (z_scores_df[z_score_col] >= exit_threshold)].index
                        
                        if len(exit_candidates) > 0:
                            exit_date = exit_candidates[0]
                            
                            # For return spread strategy - buy asset1, sell asset2
                            # Calculate returns from entry to exit
                            asset1_return = close_prices[assets[0]].loc[exit_date] / close_prices[assets[0]].loc[entry_date] - 1
                            asset2_return = close_prices[assets[1]].loc[exit_date] / close_prices[assets[1]].loc[entry_date] - 1
                            
                            # Long asset1, short asset2 - so we want asset1 to outperform asset2
                            strategy_return = asset1_return - asset2_return
                            
                            all_trades.append({
                                'Pair': pair,
                                'Window': window,
                                'Entry_Date': entry_date,
                                'Exit_Date': exit_date,
                                'Duration': (exit_date - entry_date).days,
                                'Direction': 'Long',
                                'Entry_Z': z_scores_df.loc[entry_date, z_score_col],
                                'Exit_Z': z_scores_df.loc[exit_date, z_score_col],
                                'Asset1': assets[0],
                                'Asset2': assets[1],
                                'Asset1_Return': asset1_return,
                                'Asset2_Return': asset2_return,
                                'Strategy_Return': strategy_return
                            })
                    except Exception as e:
                        #print(f"Error processing long trade: {e}")
                        continue
                
                # Process short trades (when z-score is very positive)
                for entry_date in entries_short:
                    try:
                        # Find exit - when z-score crosses below the exit threshold
                        exit_candidates = z_scores_df[(z_scores_df.index > entry_date) & 
                                                     (z_scores_df[z_score_col] <= exit_threshold)].index
                        
                        if len(exit_candidates) > 0:
                            exit_date = exit_candidates[0]
                            
                            # For return spread strategy - sell asset1, buy asset2
                            # Calculate returns from entry to exit
                            asset1_return = close_prices[assets[0]].loc[exit_date] / close_prices[assets[0]].loc[entry_date] - 1
                            asset2_return = close_prices[assets[1]].loc[exit_date] / close_prices[assets[1]].loc[entry_date] - 1
                            
                            # Short asset1, long asset2 - so we want asset2 to outperform asset1
                            strategy_return = asset2_return - asset1_return
                            
                            all_trades.append({
                                'Pair': pair,
                                'Window': window,
                                'Entry_Date': entry_date,
                                'Exit_Date': exit_date,
                                'Duration': (exit_date - entry_date).days,
                                'Direction': 'Short',
                                'Entry_Z': z_scores_df.loc[entry_date, z_score_col],
                                'Exit_Z': z_scores_df.loc[exit_date, z_score_col],
                                'Asset1': assets[0],
                                'Asset2': assets[1],
                                'Asset1_Return': asset1_return,
                                'Asset2_Return': asset2_return,
                                'Strategy_Return': strategy_return
                            })
                    except Exception as e:
                        #print(f"Error processing short trade: {e}")
                        continue
            
            # Analyze the trades for this window
            if all_trades:
                trades_df = pd.DataFrame(all_trades)
                
                print(f"\n{window}-Day Window Results:")
                print(f"Total trades: {len(trades_df)}")
                
                # Basic statistics
                win_rate = (trades_df['Strategy_Return'] > 0).mean()
                avg_return = trades_df['Strategy_Return'].mean()
                avg_duration = trades_df['Duration'].mean()
                
                print(f"Win rate: {win_rate:.2%}")
                print(f"Average return: {avg_return:.2%}")
                print(f"Average duration: {avg_duration:.1f} days")
                
                # More detailed stats by pair and direction
                print("\nResults by pair and direction:")
                for pair in trades_df['Pair'].unique():
                    pair_trades = trades_df[trades_df['Pair'] == pair]
                    
                    print(f"\n{pair}:")
                    print(f"  Total trades: {len(pair_trades)}")
                    print(f"  Win rate: {(pair_trades['Strategy_Return'] > 0).mean():.2%}")
                    print(f"  Average return: {pair_trades['Strategy_Return'].mean():.2%}")
                    
                    # By direction
                    for direction in ['Long', 'Short']:
                        dir_trades = pair_trades[pair_trades['Direction'] == direction]
                        if len(dir_trades) > 0:
                            print(f"  {direction} trades: {len(dir_trades)}")
                            print(f"    Win rate: {(dir_trades['Strategy_Return'] > 0).mean():.2%}")
                            print(f"    Average return: {dir_trades['Strategy_Return'].mean():.2%}")
                
                # If enough trades, plot the equity curve
                if len(trades_df) >= 10:
                    # Sort by entry date
                    trades_df = trades_df.sort_values('Entry_Date')
                    
                    # Calculate cumulative returns
                    trades_df['Cumulative_Return'] = (1 + trades_df['Strategy_Return']).cumprod() - 1
                    
                    plt.figure(figsize=(14, 7))
                    plt.plot(trades_df['Entry_Date'], trades_df['Cumulative_Return'])
                    plt.title(f'Equity Curve - {strategy_name} Strategy, {spread_type}, {window}-Day Window')
                    plt.xlabel('Date')
                    plt.ylabel('Cumulative Return')
                    plt.grid(True)
                    plt.show()
                    
                    # Plot return distribution
                    plt.figure(figsize=(10, 6))
                    plt.hist(trades_df['Strategy_Return'], bins=20, alpha=0.7)
                    plt.axvline(x=0, color='r', linestyle='--')
                    plt.title(f'Return Distribution - {strategy_name} Strategy, {spread_type}, {window}-Day Window')
                    plt.xlabel('Trade Return')
                    plt.ylabel('Frequency')
                    plt.grid(True)
                    plt.show()
            else:
                print(f"{window}-Day Window: No trades identified")
# ==========================================================================
# STRATEGY 2: CORRELATION REGIME SWITCHING
# ==========================================================================
print("\n" + "="*80)
print("STRATEGY 2: CORRELATION REGIME SWITCHING")
print("="*80)
# Identify correlation regime changes
if len(returns) > window_size*2:
    # Use the 20-day rolling correlations from earlier
    window_size = 20
    
    # Calculate regime changes
    correlation_regimes = pd.DataFrame(index=rolling_correlations.index)
    
    for col in rolling_correlations.columns:
        # A regime change is when correlation crosses its moving average
        regime_ma = rolling_correlations[col].rolling(window=window_size*2).mean()
        correlation_regimes[f"{col}_regime"] = (rolling_correlations[col] > regime_ma).astype(int)
        correlation_regimes[f"{col}_change"] = correlation_regimes[f"{col}_regime"].diff() != 0
    
    # Plot correlation regimes
    plt.figure(figsize=(14, 10))
    
    # Plot each pair
    for i, col in enumerate(rolling_correlations.columns):
        plt.subplot(len(rolling_correlations.columns), 1, i+1)
        
        # Plot the correlation
        plt.plot(rolling_correlations.index, rolling_correlations[col], label='Correlation')
        
        # Plot the moving average
        ma = rolling_correlations[col].rolling(window=window_size*2).mean()
        plt.plot(ma.index, ma, label='Moving Average', linestyle='--')
        
        # Highlight regime changes
        regime_changes = correlation_regimes.index[correlation_regimes[f"{col}_change"]]
        for date in regime_changes:
            plt.axvline(x=date, color='r', alpha=0.3)
        
        plt.title(f'{col} Correlation Regime Changes')
        plt.legend()
        plt.grid(True)
    
    plt.tight_layout()
    plt.show()
    
    # Backtest a regime-based strategy
    print("\nTesting Correlation Regime-Based Strategy")
    
    regime_trades = []
    
    for col in rolling_correlations.columns:
        # Get the assets for this pair
        assets = col.split('-')
        
        # Identify regime changes - from low to high correlation
        regime_up = correlation_regimes.index[
            (correlation_regimes[f"{col}_regime"] == 1) & 
            (correlation_regimes[f"{col}_change"] == True)
        ]
        
        # Identify regime changes - from high to low correlation
        regime_down = correlation_regimes.index[
            (correlation_regimes[f"{col}_regime"] == 0) & 
            (correlation_regimes[f"{col}_change"] == True)
        ]
        
        print(f"\n{col}:")
        print(f"  High correlation regime entries: {len(regime_up)}")
        print(f"  Low correlation regime entries: {len(regime_down)}")
        
        # Strategy: 
        # When correlation increases, trade the stronger performer
        # When correlation decreases, mean-revert the spread
        
        # Process trades when correlation increases
        for entry_date in regime_up:
            try:
                # Find the exit (next regime change)
                exit_candidates = correlation_regimes.index[
                    (correlation_regimes.index > entry_date) & 
                    correlation_regimes[f"{col}_change"]
                ]
                
                if len(exit_candidates) > 0:
                    exit_date = exit_candidates[0]
                    
                    # Look back to see which asset performed better before the regime change
                    lookback = 10  # days
                    look_start = max(0, returns.index.get_loc(entry_date) - lookback)
                    look_end = returns.index.get_loc(entry_date)
                    
                    # Calculate performance during lookback
                    if look_end > look_start:
                        asset1_past = returns[assets[0]].iloc[look_start:look_end].mean()
                        asset2_past = returns[assets[1]].iloc[look_start:look_end].mean()
                        
                        # Trade direction - buy the stronger performer
                        if asset1_past > asset2_past:
                            # Long asset1
                            asset1_return = close_prices[assets[0]].loc[exit_date] / close_prices[assets[0]].loc[entry_date] - 1
                            strategy_return = asset1_return
                            direction = f"Long {assets[0]}"
                        else:
                            # Long asset2
                            asset2_return = close_prices[assets[1]].loc[exit_date] / close_prices[assets[1]].loc[entry_date] - 1
                            strategy_return = asset2_return
                            direction = f"Long {assets[1]}"
                        
                        regime_trades.append({
                            'Pair': col,
                            'Entry_Date': entry_date,
                            'Exit_Date': exit_date,
                            'Duration': (exit_date - entry_date).days,
                            'Regime': 'High Correlation',
                            'Direction': direction,
                            'Strategy_Return': strategy_return
                        })
            except:
                continue
        
        # Process trades when correlation decreases
        for entry_date in regime_down:
            try:
                # Find the exit (next regime change)
                exit_candidates = correlation_regimes.index[
                    (correlation_regimes.index > entry_date) & 
                    correlation_regimes[f"{col}_change"]
                ]
                
                if len(exit_candidates) > 0:
                    exit_date = exit_candidates[0]
                    
                    # Mean reversion strategy - look for divergence
                    lookback = 5  # days
                    look_start = max(0, returns.index.get_loc(entry_date) - lookback)
                    look_end = returns.index.get_loc(entry_date)
                    
                    # Calculate performance during lookback
                    if look_end > look_start:
                        asset1_past = returns[assets[0]].iloc[look_start:look_end].sum()
                        asset2_past = returns[assets[1]].iloc[look_start:look_end].sum()
                        
                        # Trade direction - buy underperformer, sell outperformer
                        if asset1_past < asset2_past:
                            # Long asset1, short asset2 (the mean reversion bet)
                            asset1_return = close_prices[assets[0]].loc[exit_date] / close_prices[assets[0]].loc[entry_date] - 1
                            asset2_return = close_prices[assets[1]].loc[exit_date] / close_prices[assets[1]].loc[entry_date] - 1
                            strategy_return = asset1_return - asset2_return
                            direction = f"Long {assets[0]}, Short {assets[1]}"
                        else:
                            # Long asset2, short asset1
                            asset1_return = close_prices[assets[0]].loc[exit_date] / close_prices[assets[0]].loc[entry_date] - 1
                            asset2_return = close_prices[assets[1]].loc[exit_date] / close_prices[assets[1]].loc[entry_date] - 1
                            strategy_return = asset2_return - asset1_return
                            direction = f"Long {assets[1]}, Short {assets[0]}"
                        
                        regime_trades.append({
                            'Pair': col,
                            'Entry_Date': entry_date,
                            'Exit_Date': exit_date,
                            'Duration': (exit_date - entry_date).days,
                            'Regime': 'Low Correlation',
                            'Direction': direction,
                            'Strategy_Return': strategy_return
                        })
            except:
                continue
    
    # Analyze the regime-based trades
    if regime_trades:
        regime_df = pd.DataFrame(regime_trades)
        
        print("\nRegime-Based Strategy Results:")
        print(f"Total trades: {len(regime_df)}")
        
        # Basic statistics
        win_rate = (regime_df['Strategy_Return'] > 0).mean()
        avg_return = regime_df['Strategy_Return'].mean()
        avg_duration = regime_df['Duration'].mean()
        
        print(f"Win rate: {win_rate:.2%}")
        print(f"Average return: {avg_return:.2%}")
        print(f"Average duration: {avg_duration:.1f} days")
        
        # Split by regime type
        for regime in ['High Correlation', 'Low Correlation']:
            regime_subset = regime_df[regime_df['Regime'] == regime]
            if len(regime_subset) > 0:
                print(f"\n{regime} Regime:")
                print(f"  Trades: {len(regime_subset)}")
                print(f"  Win rate: {(regime_subset['Strategy_Return'] > 0).mean():.2%}")
                print(f"  Average return: {regime_subset['Strategy_Return'].mean():.2%}")
                print(f"  Average duration: {regime_subset['Duration'].mean():.1f} days")
        
        # Plot equity curve if we have enough trades
        if len(regime_df) >= 5:
            # Sort by entry date
            regime_df = regime_df.sort_values('Entry_Date')
            
            # Calculate cumulative returns
            regime_df['Cumulative_Return'] = (1 + regime_df['Strategy_Return']).cumprod() - 1
            
            plt.figure(figsize=(14, 7))
            plt.plot(regime_df['Entry_Date'], regime_df['Cumulative_Return'])
            plt.title('Equity Curve - Correlation Regime-Based Strategy')
            plt.xlabel('Date')
            plt.ylabel('Cumulative Return')
            plt.grid(True)
            plt.show()
            
            # Plot return distribution
            plt.figure(figsize=(10, 6))
            plt.hist(regime_df['Strategy_Return'], bins=20, alpha=0.7)
            plt.axvline(x=0, color='r', linestyle='--')
            plt.title('Return Distribution - Correlation Regime-Based Strategy')
            plt.xlabel('Trade Return')
            plt.ylabel('Frequency')
            plt.grid(True)
            plt.show()
    else:
        print("No regime-based trades identified")
# ==========================================================================
# STRATEGY 3: SHORT-TERM CORRELATION-DRIVEN MOMENTUM 
# ==========================================================================
print("\n" + "="*80)
print("STRATEGY 3: SHORT-TERM CORRELATION-DRIVEN MOMENTUM")
print("="*80)
# Calculate short-term correlations and momentum
short_window = 5  # 1 week
medium_window = 20  # 1 month
# Calculate short-term momentum
momentum = pd.DataFrame(index=returns.index)
for col in returns.columns:
    momentum[col] = returns[col].rolling(window=short_window).sum()
# Calculate short-term rolling correlations
short_corr = pd.DataFrame(index=returns.index)
for i, name1 in enumerate(returns.columns):
    for j, name2 in enumerate(returns.columns):
        if i < j:
            pair_name = f"{name1}-{name2}"
            short_corr[pair_name] = returns[name1].rolling(window=short_window).corr(returns[name2])
# Backtest a short-term momentum strategy
print("\nTesting Short-Term Correlation-Driven Momentum Strategy")
# Strategy rules:
# 1. When short-term correlation is high (>0.7), buy the asset with positive momentum
# 2. When short-term correlation is negative (<-0.5), buy both assets with positive momentum
# 3. Hold for a fixed period or until the momentum reverses
# Identify trade entry points
momentum_trades = []
# Calculate signals
signals = pd.DataFrame(index=returns.index)
for i, name1 in enumerate(returns.columns):
    for j, name2 in enumerate(returns.columns):
        if i < j:
            pair_name = f"{name1}-{name2}"
            
            # Only proceed if we have correlation data for this pair
            if pair_name in short_corr.columns:
                # High positive correlation signal
                signals[f"{pair_name}_highcorr"] = (short_corr[pair_name] > 0.7)
                
                # High negative correlation signal
                signals[f"{pair_name}_negcorr"] = (short_corr[pair_name] < -0.5)
                
                # Momentum signals
                signals[f"{name1}_posmom"] = (momentum[name1] > 0)
                signals[f"{name2}_posmom"] = (momentum[name2] > 0)
# Drop the first few rows where signals are NaN due to the rolling window
signals = signals.dropna()
# Generate trades from the signals
hold_period = 5  # 1 week holding period
for i in range(len(signals)):
    try:
        date = signals.index[i]
        
        # Skip first few rows to ensure we have enough data for exit
        if i + hold_period >= len(signals):
            continue
            
        # Check each pair for signals
        for p, pair in enumerate(short_corr.columns):
            assets = pair.split('-')
            
            # High positive correlation strategy
            if signals.iloc[i][f"{pair}_highcorr"]:
                # Buy the asset with positive momentum
                if signals.iloc[i][f"{assets[0]}_posmom"] and not signals.iloc[i][f"{assets[1]}_posmom"]:
                    # Long asset 1
                    exit_date = signals.index[i + hold_period]
                    asset_return = close_prices[assets[0]].loc[exit_date] / close_prices[assets[0]].loc[date] - 1
                    
                    momentum_trades.append({
                        'Pair': pair,
                        'Entry_Date': date,
                        'Exit_Date': exit_date,
                        'Duration': hold_period,
                        'Signal': 'High Correlation, Asset 1 Momentum',
                        'Direction': f"Long {assets[0]}",
                        'Strategy_Return': asset_return
                    })
                
                elif signals.iloc[i][f"{assets[1]}_posmom"] and not signals.iloc[i][f"{assets[0]}_posmom"]:
                    # Long asset 2
                    exit_date = signals.index[i + hold_period]
                    asset_return = close_prices[assets[1]].loc[exit_date] / close_prices[assets[1]].loc[date] - 1
                    
                    momentum_trades.append({
                        'Pair': pair,
                        'Entry_Date': date,
                        'Exit_Date': exit_date,
                        'Duration': hold_period,
                        'Signal': 'High Correlation, Asset 2 Momentum',
                        'Direction': f"Long {assets[1]}",
                        'Strategy_Return': asset_return
                    })
            
            # Negative correlation strategy
            elif signals.iloc[i][f"{pair}_negcorr"]:
                # Look for divergence opportunities
                if signals.iloc[i][f"{assets[0]}_posmom"] and signals.iloc[i][f"{assets[1]}_posmom"]:
                    # Long both assets - diversification play
                    exit_date = signals.index[i + hold_period]
                    asset1_return = close_prices[assets[0]].loc[exit_date] / close_prices[assets[0]].loc[date] - 1
                    asset2_return = close_prices[assets[1]].loc[exit_date] / close_prices[assets[1]].loc[date] - 1
                    
                    # Equal weight portfolio
                    strategy_return = (asset1_return + asset2_return) / 2
                    
                    momentum_trades.append({
                        'Pair': pair,
                        'Entry_Date': date,
                        'Exit_Date': exit_date,
                        'Duration': hold_period,
                        'Signal': 'Negative Correlation, Both Positive Momentum',
                        'Direction': f"Long Both",
                        'Strategy_Return': strategy_return
                    })
    except:
        continue
# Analyze the momentum-based trades
if momentum_trades:
    momentum_df = pd.DataFrame(momentum_trades)
    
    print("\nShort-Term Momentum Strategy Results:")
    print(f"Total trades: {len(momentum_df)}")
    
    # Basic statistics
    win_rate = (momentum_df['Strategy_Return'] > 0).mean()
    avg_return = momentum_df['Strategy_Return'].mean()
    
    print(f"Win rate: {win_rate:.2%}")
    print(f"Average return: {avg_return:.2%}")
    
    # Split by signal type
    for signal in momentum_df['Signal'].unique():
        signal_subset = momentum_df[momentum_df['Signal'] == signal]
        if len(signal_subset) > 0:
            print(f"\n{signal}:")
            print(f"  Trades: {len(signal_subset)}")
            print(f"  Win rate: {(signal_subset['Strategy_Return'] > 0).mean():.2%}")
            print(f"  Average return: {signal_subset['Strategy_Return'].mean():.2%}")
    
    # Plot equity curve if we have enough trades
    if len(momentum_df) >= 5:
        # Sort by entry date
        momentum_df = momentum_df.sort_values('Entry_Date')
        
        # Calculate cumulative returns
        momentum_df['Cumulative_Return'] = (1 + momentum_df['Strategy_Return']).cumprod() - 1
        
        plt.figure(figsize=(14, 7))
        plt.plot(momentum_df['Entry_Date'], momentum_df['Cumulative_Return'])
        plt.title('Equity Curve - Short-Term Momentum Strategy')
        plt.xlabel('Date')
        plt.ylabel('Cumulative Return')
        plt.grid(True)
        plt.show()
        
        # Plot return distribution
        plt.figure(figsize=(10, 6))
        plt.hist(momentum_df['Strategy_Return'], bins=20, alpha=0.7)
        plt.axvline(x=0, color='r', linestyle='--')
        plt.title('Return Distribution - Short-Term Momentum Strategy')
        plt.xlabel('Trade Return')
        plt.ylabel('Frequency')
        plt.grid(True)
        plt.show()
else:
    print("No momentum-based trades identified")
print("\nResearch complete!")
+ Expand
- Collapse

 # Cell 1: Environment Setup and Imports
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from datetime import datetime, timedelta
from QuantConnect import *
from sklearn.model_selection import train_test_split, GridSearchCV, TimeSeriesSplit
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
from sklearn.svm import SVC
from sklearn.metrics import accuracy_score, confusion_matrix, classification_report, roc_auc_score, precision_recall_curve
from sklearn.preprocessing import StandardScaler
import warnings
warnings.filterwarnings('ignore')
# Set visualization style
plt.style.use('ggplot')
sns.set(font_scale=1.2)
sns.set_style("whitegrid")
print("Environment setup complete")
# Cell 2: Initialize QuantBook and Set Time Parameters
# Create a QuantBook instance
qb = QuantBook()
# Set the start date - using 3 years of historical data for robust analysis
lookback_years = 3
start_date = datetime.now() - timedelta(days=365 * lookback_years)
qb.set_start_date(start_date)
print(f"Current QuantBook time: {qb.time}")
print(f"Analysis start date: {start_date}")
print(f"Lookback period: {lookback_years} years")
# Cell 3: Define and Add Futures Contracts
# Define the list of futures tickers we want to analyze
tickers = ["MNQ", "MES", "MYM", "M2K", "MGC"]  # Added M2K (Micro Russell) and MGC (Micro Gold)
futures = {}
# Add the continuous futures contracts
try:
    for ticker in tickers:
        futures[ticker] = qb.add_future(
            ticker,
            Resolution.DAILY,  # We'll collect daily data first, then request minute data for specific periods
            data_normalization_mode=DataNormalizationMode.BACKWARDS_RATIO,
            data_mapping_mode=DataMappingMode.LAST_TRADING_DAY,
            contract_depth_offset=0
        )
        # Set filter to ensure we get front month contracts
        futures[ticker].set_filter(0, 90)
        print(f"Added {ticker} continuous future: {futures[ticker].symbol}")
    
    print("Successfully added all futures")
except Exception as e:
    print(f"Error adding futures: {e}")
# Cell 4: Historical Data Collection
# Define analysis period
end_date = qb.time
start_analysis = end_date - timedelta(days=365 * lookback_years)
print(f"Analysis period: {start_analysis} to {end_date}")
# Function to get history data with error handling
def get_safe_history(qb, symbol, start_date, end_date, resolution):
    try:
        history = qb.History(symbol, start_date, end_date, resolution)
        print(f"Retrieved {len(history)} bars for {symbol}")
        return history
    except Exception as e:
        print(f"Error retrieving history for {symbol}: {e}")
        return pd.DataFrame()
# Collect daily data for all futures
daily_data = {}
for ticker, future in futures.items():
    daily_data[ticker] = get_safe_history(qb, future.symbol, start_analysis, end_date, Resolution.DAILY)
    
# Verify data collection
for ticker, data in daily_data.items():
    if not data.empty:
        print(f"{ticker} data range: {data.index[0][1]} to {data.index[-1][1]}, {len(data)} rows")
    else:
        print(f"No data available for {ticker}")
# Cell 5: Data Preprocessing and Feature Engineering
# Function to calculate technical indicators and create features
def create_features(df):
    # Make a copy to avoid SettingWithCopyWarning
    result = df.copy()
    
    # Extract price data
    result['close'] = df['close']
    result['open'] = df['open']
    result['high'] = df['high']
    result['low'] = df['low']
    result['volume'] = df['volume']
    
    # Calculate returns
    result['daily_return'] = result['close'].pct_change()
    result['log_return'] = np.log(result['close']).diff()
    
    # Moving averages
    for window in [5, 10, 20, 50, 100, 200]:
        result[f'ma_{window}'] = result['close'].rolling(window=window).mean()
        result[f'ma_vol_{window}'] = result['volume'].rolling(window=window).mean()
    
    # Price relative to moving averages
    for window in [5, 10, 20, 50, 100, 200]:
        result[f'close_over_ma_{window}'] = result['close'] / result[f'ma_{window}']
    
    # Volatility measures
    for window in [5, 10, 20, 50]:
        result[f'volatility_{window}'] = result['log_return'].rolling(window=window).std()
        
    # Momentum indicators
    for window in [5, 10, 20, 50]:
        result[f'momentum_{window}'] = result['close'].pct_change(periods=window)
    
    # RSI calculation
    def calculate_rsi(data, window=14):
        delta = data.diff()
        gain = delta.where(delta > 0, 0)
        loss = -delta.where(delta < 0, 0)
        
        avg_gain = gain.rolling(window=window).mean()
        avg_loss = loss.rolling(window=window).mean()
        
        rs = avg_gain / avg_loss
        rsi = 100 - (100 / (1 + rs))
        return rsi
    
    result['rsi_14'] = calculate_rsi(result['close'], 14)
    
    # Z-score of price
    for window in [20, 50, 100]:
        rolling_mean = result['close'].rolling(window=window).mean()
        rolling_std = result['close'].rolling(window=window).std()
        result[f'z_score_{window}'] = (result['close'] - rolling_mean) / rolling_std
    
    # Drop NaN values resulting from calculations
    result = result.dropna()
    
    return result
# Process each futures data
processed_data = {}
for ticker, data in daily_data.items():
    if not data.empty:
        # Reset index to work with the data more easily
        df = data.reset_index()
        # Process dataframe
        processed_data[ticker] = create_features(df)
        print(f"Processed {ticker} data: {len(processed_data[ticker])} rows after feature engineering")
# Cell 6: Exploratory Data Analysis
# Function to plot key metrics for each future
def plot_future_metrics(ticker, data):
    plt.figure(figsize=(16, 12))
    
    # Plot 1: Price and moving averages
    plt.subplot(3, 1, 1)
    plt.plot(data.index, data['close'], label='Close Price')
    plt.plot(data.index, data['ma_50'], label='50-day MA')
    plt.plot(data.index, data['ma_200'], label='200-day MA')
    plt.title(f'{ticker} Price and Moving Averages')
    plt.legend()
    plt.grid(True)
    
    # Plot 2: Daily returns
    plt.subplot(3, 1, 2)
    plt.plot(data.index, data['daily_return'], label='Daily Return')
    plt.axhline(y=0, color='r', linestyle='-')
    plt.title(f'{ticker} Daily Returns')
    plt.legend()
    plt.grid(True)
    
    # Plot 3: Z-score
    plt.subplot(3, 1, 3)
    plt.plot(data.index, data['z_score_50'], label='50-day Z-score')
    plt.axhline(y=0, color='r', linestyle='-')
    plt.axhline(y=2, color='g', linestyle='--')
    plt.axhline(y=-2, color='g', linestyle='--')
    plt.title(f'{ticker} Z-score (50-day)')
    plt.legend()
    plt.grid(True)
    
    plt.tight_layout()
    plt.show()
# Run EDA for each future
for ticker, data in processed_data.items():
    print(f"\nExploratory Data Analysis for {ticker}:")
    print(data.describe())
    plot_future_metrics(ticker, data)
# Cell 7: Mean Reversion Strategy Design and Backtest
# Define mean reversion strategy parameters
z_score_buy_threshold = -2.0
z_score_sell_threshold = 2.0
holding_period_days = 5  # Maximum holding period
# Function to generate trade signals
def generate_signals(data, z_column='z_score_50'):
    """Generate entry and exit signals based on z-score thresholds"""
    signals = pd.DataFrame(index=data.index)
    signals['close'] = data['close']
    signals[z_column] = data[z_column]
    
    # Generate entry signals
    signals['long_entry'] = (data[z_column] <= z_score_buy_threshold)
    signals['short_entry'] = (data[z_column] >= z_score_sell_threshold)
    
    # Generate exit signals
    signals['long_exit'] = (data[z_column] >= 0) | (data[z_column].shift(holding_period_days) <= z_score_buy_threshold)
    signals['short_exit'] = (data[z_column] <= 0) | (data[z_column].shift(holding_period_days) >= z_score_sell_threshold)
    
    return signals
# Function to backtest the strategy and generate trades
def backtest_strategy(data, signals):
    """Backtest the mean reversion strategy and generate trade list"""
    trades = []
    in_long = False
    in_short = False
    entry_price = 0
    entry_date = None
    entry_z = 0
    
    for i in range(1, len(signals)):
        # Long entry
        if signals['long_entry'].iloc[i] and not in_long and not in_short:
            in_long = True
            entry_price = signals['close'].iloc[i]
            entry_date = signals.index[i]
            entry_z = signals[z_column].iloc[i]
        
        # Long exit
        elif signals['long_exit'].iloc[i] and in_long:
            in_long = False
            exit_price = signals['close'].iloc[i]
            exit_date = signals.index[i]
            exit_z = signals[z_column].iloc[i]
            
            # Calculate return
            trade_return = (exit_price / entry_price) - 1
            trades.append({
                'entry_time': entry_date,
                'exit_time': exit_date,
                'entry_price': entry_price,
                'exit_price': exit_price,
                'entry_z': entry_z,
                'exit_z': exit_z,
                'return': trade_return,
                'position': 'Long',
                'duration': (exit_date - entry_date).days
            })
        
        # Short entry
        if signals['short_entry'].iloc[i] and not in_short and not in_long:
            in_short = True
            entry_price = signals['close'].iloc[i]
            entry_date = signals.index[i]
            entry_z = signals[z_column].iloc[i]
        
        # Short exit
        elif signals['short_exit'].iloc[i] and in_short:
            in_short = False
            exit_price = signals['close'].iloc[i]
            exit_date = signals.index[i]
            exit_z = signals[z_column].iloc[i]
            
            # Calculate return
            trade_return = 1 - (exit_price / entry_price)
            trades.append({
                'entry_time': entry_date,
                'exit_time': exit_date,
                'entry_price': entry_price,
                'exit_price': exit_price,
                'entry_z': entry_z,
                'exit_z': exit_z,
                'return': trade_return,
                'position': 'Short',
                'duration': (exit_date - entry_date).days
            })
    
    return pd.DataFrame(trades)
# Run the strategy for each future
z_column = 'z_score_50'  # Use 50-day z-score for signals
all_trades = {}
all_signals = {}
for ticker, data in processed_data.items():
    # Generate signals
    signals = generate_signals(data, z_column)
    all_signals[ticker] = signals
    
    # Run backtest
    trades = backtest_strategy(data, signals)
    all_trades[ticker] = trades
    
    print(f"\nBacktest results for {ticker}:")
    print(f"Total trades: {len(trades)}")
    if len(trades) > 0:
        win_rate = (trades['return'] > 0).mean() * 100
        avg_return = trades['return'].mean() * 100
        print(f"Win rate: {win_rate:.2f}%")
        print(f"Average return per trade: {avg_return:.2f}%")
        print(f"Total return: {trades['return'].sum() * 100:.2f}%")
        print(f"Average trade duration: {trades['duration'].mean():.1f} days")
# Cell 8: Combine All Trades for Analysis
# Combine all trades into a single DataFrame
trades_all = pd.concat([trades.assign(instrument=ticker) for ticker, trades in all_trades.items()])
trades_all = trades_all.reset_index(drop=True)
# Overall performance metrics
print("\nOverall Strategy Performance:")
print(f"Total trades: {len(trades_all)}")
if len(trades_all) > 0:
    win_rate = (trades_all['return'] > 0).mean() * 100
    avg_return = trades_all['return'].mean() * 100
    print(f"Win rate: {win_rate:.2f}%")
    print(f"Average return per trade: {avg_return:.2f}%")
    print(f"Total return: {trades_all['return'].sum() * 100:.2f}%")
    
    # Plot return distribution
    plt.figure(figsize=(12, 6))
    sns.histplot(trades_all['return'] * 100, bins=50, kde=True)
    plt.axvline(x=0, color='r', linestyle='--')
    plt.title('Trade Return Distribution (%)')
    plt.xlabel('Return (%)')
    plt.ylabel('Frequency')
    plt.grid(True)
    plt.show()
# Cell 9: Advanced Machine Learning Model Development
# Prepare the data for ML
trades_ml = trades_all.copy()
trades_ml['win'] = (trades_ml['return'] > 0).astype(int)
# Create additional features
trades_ml['z_diff'] = trades_ml['exit_z'] - trades_ml['entry_z']
trades_ml['z_abs_entry'] = trades_ml['entry_z'].abs()
trades_ml['z_abs_exit'] = trades_ml['exit_z'].abs()
trades_ml['is_long'] = (trades_ml['position'] == 'Long').astype(int)
# One-hot encode the instruments
instrument_dummies = pd.get_dummies(trades_ml['instrument'], prefix='instr')
trades_ml = pd.concat([trades_ml, instrument_dummies], axis=1)
# Add day of week and month features
trades_ml['entry_day'] = pd.to_datetime(trades_ml['entry_time']).dt.dayofweek
trades_ml['entry_month'] = pd.to_datetime(trades_ml['entry_time']).dt.month
day_dummies = pd.get_dummies(trades_ml['entry_day'], prefix='day')
month_dummies = pd.get_dummies(trades_ml['entry_month'], prefix='month')
trades_ml = pd.concat([trades_ml, day_dummies, month_dummies], axis=1)
# Feature selection
features = [
    'entry_z', 'exit_z', 'z_diff', 'z_abs_entry', 'z_abs_exit', 
    'is_long', 'duration'
]
# Add instrument and time features
features.extend([col for col in trades_ml.columns if col.startswith('instr_')])
features.extend([col for col in trades_ml.columns if col.startswith('day_')])
features.extend([col for col in trades_ml.columns if col.startswith('month_')])
# Prepare the feature matrix and target
X = trades_ml[features]
y = trades_ml['win']
# Scale features
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)
# Split the data chronologically (important for time series data)
trades_ml = trades_ml.sort_values('entry_time')
train_size = int(0.7 * len(trades_ml))
X_train = X_scaled[:train_size]
X_test = X_scaled[train_size:]
y_train = y[:train_size]
y_test = y[train_size:]
print(f"Training set size: {len(X_train)}, Test set size: {len(X_test)}")
# Cell 10: Model Training and Evaluation
# Define models to evaluate
models = {
    'Logistic Regression': LogisticRegression(max_iter=1000),
    'Random Forest': RandomForestClassifier(n_estimators=100, random_state=42),
    'Gradient Boosting': GradientBoostingClassifier(n_estimators=100, random_state=42),
    'SVM': SVC(probability=True, random_state=42)
}
# Train and evaluate each model
results = {}
for name, model in models.items():
    print(f"\nTraining {name}...")
    model.fit(X_train, y_train)
    
    # Make predictions
    y_pred = model.predict(X_test)
    y_proba = model.predict_proba(X_test)[:, 1]
    
    # Evaluate performance
    accuracy = accuracy_score(y_test, y_pred)
    cm = confusion_matrix(y_test, y_pred)
    auc = roc_auc_score(y_test, y_proba)
    
    print(f"{name} Results:")
    print(f"Accuracy: {accuracy:.4f}")
    print(f"AUC-ROC: {auc:.4f}")
    print("Confusion Matrix:")
    print(cm)
    print(classification_report(y_test, y_pred))
    
    results[name] = {
        'model': model,
        'accuracy': accuracy,
        'auc': auc,
        'confusion_matrix': cm,
        'y_pred': y_pred,
        'y_proba': y_proba
    }
    
# Select the best model based on AUC
best_model_name = max(results, key=lambda x: results[x]['auc'])
best_model = results[best_model_name]['model']
print(f"\nBest model based on AUC: {best_model_name} with AUC = {results[best_model_name]['auc']:.4f}")
# Cell 11: Feature Importance Analysis
def plot_feature_importance(model, feature_names, title):
    """Plot feature importance for tree-based models"""
    if hasattr(model, 'feature_importances_'):
        # For tree-based models
        importances = model.feature_importances_
        indices = np.argsort(importances)[::-1]
        
        plt.figure(figsize=(12, 8))
        plt.title(title)
        plt.bar(range(len(indices)), importances[indices], align='center')
        plt.xticks(range(len(indices)), [feature_names[i] for i in indices], rotation=90)
        plt.tight_layout()
        plt.show()
    elif hasattr(model, 'coef_'):
        # For linear models
        importances = np.abs(model.coef_)[0]
        indices = np.argsort(importances)[::-1]
        
        plt.figure(figsize=(12, 8))
        plt.title(title)
        plt.bar(range(len(indices)), importances[indices], align='center')
        plt.xticks(range(len(indices)), [feature_names[i] for i in indices], rotation=90)
        plt.tight_layout()
        plt.show()
# Plot feature importance for tree-based models or coefficients for linear models
for name, result in results.items():
    if hasattr(result['model'], 'feature_importances_') or hasattr(result['model'], 'coef_'):
        plot_feature_importance(result['model'], features, f"Feature Importance - {name}")
# Cell 12: Strategy Improvement with ML Predictions
# Add predictions to the test set
trades_test = trades_ml.iloc[train_size:].copy()
trades_test['predicted_win'] = results[best_model_name]['y_pred']
trades_test['win_probability'] = results[best_model_name]['y_proba']
# Define different probability thresholds for trade filtering
thresholds = [0.5, 0.55, 0.6, 0.65, 0.7]
print("\nML Strategy Performance at Different Probability Thresholds:")
for threshold in thresholds:
    filtered_trades = trades_test[trades_test['win_probability'] >= threshold]
    
    if len(filtered_trades) > 0:
        win_rate = (filtered_trades['win'] == 1).mean() * 100
        avg_return = filtered_trades['return'].mean() * 100
        total_return = filtered_trades['return'].sum() * 100
        
        print(f"\nThreshold: {threshold}")
        print(f"Number of trades: {len(filtered_trades)} ({len(filtered_trades)/len(trades_test)*100:.1f}% of all trades)")
        print(f"Win rate: {win_rate:.2f}%")
        print(f"Average return: {avg_return:.2f}%")
        print(f"Total return: {total_return:.2f}%")
    else:
        print(f"\nThreshold: {threshold}")
        print("No trades meet this threshold")
# Cell 13: Strategy Visualization
# Plot cumulative returns for base strategy vs ML-filtered strategy
def plot_equity_curves(trades_df, title='Equity Curve Comparison'):
    plt.figure(figsize=(14, 7))
    
    # Sort trades by entry time
    trades_df = trades_df.sort_values('entry_time')
    
    # Base strategy equity curve
    base_returns = trades_df['return'].values
    base_equity = np.cumprod(1 + base_returns) - 1
    plt.plot(range(len(base_equity)), base_equity * 100, label='Base Strategy')
    
    # Plot equity curves for different thresholds
    for threshold in [0.55, 0.6, 0.65]:
        filtered = trades_df[trades_df['win_probability'] >= threshold]
        
        if len(filtered) > 0:
            # Find the indices of filtered trades in the original trades list
            filtered_indices = filtered.index.tolist()
            
            # Initialize equity curve for filtered strategy
            filtered_equity = np.zeros(len(base_equity))
            equity_value = 0
            
            # Build equity curve
            for i in range(len(trades_df)):
                if trades_df.index[i] in filtered_indices:
                    equity_value += trades_df['return'].iloc[i]
                filtered_equity[i] = equity_value
            
            plt.plot(range(len(filtered_equity)), filtered_equity * 100, 
                     label=f'ML Strategy (Threshold = {threshold})')
    
    plt.title(title)
    plt.xlabel('Trade Number')
    plt.ylabel('Cumulative Return (%)')
    plt.grid(True)
    plt.legend()
    plt.tight_layout()
    plt.show()
# Plot equity curves
plot_equity_curves(trades_test, 'Equity Curve Comparison (Test Set)')
# Cell 14: Predictive Strategy Implementation
# Function to identify current trading opportunities
def identify_current_opportunities(data, model, scaler, threshold=0.6, z_column='z_score_50'):
    """Identify current trading opportunities using the trained ML model"""
    opportunities = []
    
    for ticker, df in data.items():
        # Get the latest data point
        latest = df.iloc[-1]
        
        # Check if z-score indicates potential mean reversion opportunity
        if latest[z_column] <= z_score_buy_threshold or latest[z_column] >= z_score_sell_threshold:
            # Prepare features for prediction
            position = 'Long' if latest[z_column] <= z_score_buy_threshold else 'Short'
            
            # Create feature vector (simulating a potential trade)
            features_dict = {
                'entry_z': latest[z_column],
                'exit_z': 0,  # Assuming reversion to mean
                'z_diff': 0 - latest[z_column],  # Assuming reversion to mean
                'z_abs_entry': abs(latest[z_column]),
                'z_abs_exit': 0,  # Assuming reversion to mean
                'is_long': 1 if position == 'Long' else 0,
                'duration': holding_period_days
            }
            
            # Add instrument one-hot encoding
            for instr in tickers:
                features_dict[f'instr_{instr}'] = 1 if instr == ticker else 0
                
            # Add day and month features
            current_day = datetime.now().weekday()
            current_month = datetime.now().month
            
            for day in range(7):
                features_dict[f'day_{day}'] = 1 if day == current_day else 0
                
            for month in range(1, 13):
                features_dict[f'month_{month}'] = 1 if month == current_month else 0
            
            # Create feature vector and ensure correct order
            feature_vector = [features_dict.get(feature, 0) for feature in features]
            
            # Scale features
            scaled_features = scaler.transform([feature_vector])
            
            # Predict win probability
            win_prob = best_model.predict_proba(scaled_features)[0, 1]
            
            # Check if probability meets threshold
            if win_prob >= threshold:
                opportunities.append({
                    'ticker': ticker,
                    'position': position,
                    'entry_price': latest['close'],
                    'entry_z': latest[z_column],
                    'win_probability': win_prob,
                    'timestamp': df.index[-1]
                })
    
    return pd.DataFrame(opportunities)
# Identify current trading opportunities
current_opportunities = identify_current_opportunities(
    processed_data, 
    best_model, 
    scaler,
    threshold=0.6
)
print("\nCurrent Trading Opportunities:")
if len(current_opportunities) > 0:
    print(current_opportunities)
else:
    print("No opportunities meet the criteria at the moment")
# Cell 15: Performance Summary and Future Improvements
print("\nStrategy Performance Summary:")
print(f"Total trades analyzed: {len(trades_all)}")
print(f"Base strategy win rate: {(trades_all['return'] > 0).mean() * 100:.2f}%")
print(f"Base strategy average return: {trades_all['return'].mean() * 100:.2f}%")
print(f"Base strategy total return: {trades_all['return'].sum() * 100:.2f}%")
print("\nBest ML model:")
print(f"Model: {best_model_name}")
print(f"Test accuracy: {results[best_model_name]['accuracy']:.4f}")
print(f"Test AUC: {results[best_model_name]['auc']:.4f}")
print("\nFuture Improvements:")
print("1. Incorporate market regime detection for adaptive parameter selection")
print("2. Add cross-market correlation analysis for multi-instrument signal confirmation")
print("3. Implement portfolio-level position sizing based on win probability")
print("4. Explore alternative feature engineering techniques (wavelets, PCA)")
print("5. Implement more sophisticated models (LSTM, Transformer-based models)")
print("6. Add risk management rules (stop-loss, take-profit, maximum drawdown limits)")
+ Expand
- Collapse