Merge Prices and Indicators in One Single Dataframe in Research Environment

Thanks for submitting the code used. The indexing issue comes primarily due to the fact that the historical data is multi-indexed. Since only one asset is being analyzed, it made sense to remove the symbol index from the OHLC DataFrame. This can be done via .unstack() or .reset_index(). The notebook below has the code above running and another indicator added was to show how multiple indicators can be created and implemented into DataFrames together.

• Notebook

Thank you!! I understand now :)

Emilio

Hi All,

I know this is an old thread but I'm having the same issue. Access to the code Gurumeher shared above seems to be denied. In the main.py accessing OHLC data and Indicators with rollingwindows and merging two dataframes seems tricky because of the index mismatch. Does anyone have an idea on merging price data and indicators in the main.py.

from AlgorithmImports import *
import pandas as pd
import numpy as np
import pickle
from datetime import datetime
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import StratifiedShuffleSplit
from sklearn.preprocessing import StandardScaler

class ScikitLearnExampleAlgorithm(QCAlgorithm):
    
    def Initialize(self):
        self.SetStartDate(2023, 2, 11)
        self.SetEndDate(2023, 2, 23)
        self.SetCash("USDT", 4000)
        self.symbol = self.AddCryptoFuture("BTCUSDT", Resolution.Hour, Market.Binance, leverage= 2).Symbol
        self.SetBrokerageModel(BrokerageName.BinanceFutures, AccountType.Margin)
        self.SetBenchmark(self.symbol)
        self.open_trades = []
        self.ticket = None
        self.trade_period = 24   
        self.train_time = self.Time
        self.SetWarmUp(100, Resolution.Hour)     
        
        self.training_length = 2*30*24
        self.training_data = RollingWindow[TradeBar](self.training_length)  
        history = self.History[TradeBar](self.symbol, self.training_length, Resolution.Hour)
        for trade_bar in history:
            self.training_data.Add(trade_bar)

        self.ad = self.AD(self.symbol)
        self.ad_window = RollingWindow[IndicatorDataPoint](self.training_length) 
        self.ad.Updated += self.adUpdated
        
        self.obv = self.OBV(self.symbol)
        self.obv_window = RollingWindow[IndicatorDataPoint](self.training_length)
        self.obv.Updated += self.obvUpdated

        self.model_key = 'RFC_240223_v0' 
        if self.ObjectStore.ContainsKey(self.model_key):
            deserializedObject = bytes(self.ObjectStore.ReadBytes(self.model_key))
            self.model = pickle.loads(deserializedObject)
        else:
	        self.Train(self.my_training_method)
            
        self.Train(self.DateRules.EveryDay(), self.TimeRules.At(3,10), self.my_training_method)

    def OnWarmupFinished(self):
        self.EnableAutomaticIndicatorWarmUp = True 

    def my_training_method(self):
        self.Log(f'Start training at {self.Time}')
        self.model_is_training = True 
        features_scaled, labels = self.get_features_and_labels()

        sss = StratifiedShuffleSplit(n_splits=2, test_size=0.3, random_state=42)
        sss.get_n_splits(features_scaled, labels)
        for train_index, test_index in sss.split(features_scaled, labels):
            X_train, X_test = features_scaled[train_index], features_scaled[test_index]
            y_train, y_test = labels[train_index], labels[test_index]

        RFC = RandomForestClassifier(n_estimators=50, n_jobs=-1, )
        self.model = RFC
        self.model.fit(X_train, y_train)
        self.model_is_training = False
        self.train_time = self.Time
        
    def get_features_and_labels(self):

        ad = np.array([x.Value for x in self.ad_window]).reshape(-1,1)
        obv = np.array([x.Value for x in self.obv_window]).reshape(-1,1)
        self.indicator_df = pd.DataFrame(np.hstack((ad, obv)))

        ohlcv_df = self.PandasConverter.GetDataFrame[TradeBar](list(self.training_data)[::-1])
        
        df = pd.merge(self.indicator_df, ohlcv_df, on = ohlcv_df.index)
        df['target'] = ((df.shift(-24)['close']) >= df['close']).astype(int)
        df = df.dropna(axis=0)

        features = df.iloc[:, :-1].values  
        labels = df.iloc[:, -1].values   

        # Scaling
        sc = StandardScaler()
        features_scaled = sc.fit_transform(features)

        return features_scaled, labels

    def OnData(self, slice: Slice) -> None:

        if self.symbol in slice.Bars:
            self.training_data.Add(slice.Bars[self.symbol])
            self.bar = slice.Bars[self.symbol]

        features_scaled, _ = self.get_features_and_labels()
        Y_pred = self.model.predict(features_scaled[-1].reshape(1,-1))  
        prob = self.model.predict_proba(features_scaled[-1].reshape(1,-1)) 

        self.Log("{0} : {1}".format(Y_pred, prob))
        order_size = 50
        order_quantity = order_size / self.bar.Close

        if not self.Portfolio.Invested:
            if Y_pred == 1:
                self.ticket = self.MarketOrder(self.symbol, order_quantity)
                self.open_trades.append(self.ticket)
                self.Log("{0}: {1}: {2}".format(self.ticket.Time, len(self.open_trades), self.train_time))
            else:
                return
        else:
            if Y_pred == 1:
                self.ticket = self.MarketOrder(self.symbol, order_quantity)
                self.open_trades.append(self.ticket)
                self.Log("{0}: {1}: {2}".format(self.ticket.Time, len(self.open_trades), self.train_time))

            oldest_trade = self.open_trades[0]
            quantity_filled = oldest_trade.QuantityFilled
            time_delta = self.UtcTime - oldest_trade.Time
        
            if time_delta.total_seconds() >= (self.trade_period*60*60) - 1:
                self.MarketOrder(self.symbol, -quantity_filled)
                self.open_trades.pop(0)

    def adUpdated(self, sender, updated):
        self.ad_window.Add(updated)
        
    def obvUpdated(self, sender, updated):
        self.obv_window.Add(updated)

    def OnEndOfAlgorithm(self):
        serializedObject = pickle.dumps(self.model)
        self.ObjectStore.SaveBytes(self.model_key, serializedObject)

I found the necessary code piece in the documents. Thanks anyways