Hi All

Would appreciate some advice on how to tackle this use case. Trying to do coarse and then fine filter selection on equities, and then to rank the results further with custom indicators (via the standard SymbolData) pattern, but I keep running into run time errors with missing data: 

The thing I want to do is use the market index as a point of reference (ideally without actually adding into my tradeable universe - I don't want to trade the market index, just use it for strategy condition assessment).  I suspect this is a pretty common use case, but I haven't been able to figure out how to do it without running into these data issues.

This is the error:

Runtime Error: 'SPY' wasn't found in the Slice object, likely because there was no-data at this moment in time and it wasn't possible to fillforward historical data. Please check the data exists before accessing it with data.ContainsKey("SPY") in Slice.cs:line 329.

Needless to say, I am checking the slice data, and in fact, have layered in extensive Debug statements to trace the program execution, in an attempt to figure out where the issue is arising, but it's not clear. (The stack trace certain doesn't offer any info).  Can anyone assist with this? Code embedded for easy replication (can't attach a backtest - execution has an error).

cheers

 

#region imports
from AlgorithmImports import *
#endregion

import numpy as np
import pandas as pd
from scipy.stats import linregress
from collections import deque
from datetime import timedelta
from datetime import datetime
import math


class ExponentialMomentumRanker(PythonIndicator):
    
    def momentum(self, closes):
        returns = np.log(closes)
        x = np.arange(len(returns))
        slope, _, rvalue, _, _ = linregress(x, returns)
        annualised_slope = (np.power(np.exp(slope), 252)-1)*100
        return annualised_slope * (rvalue ** 2)  # annualize slope and multiply by R^2

    def __init__(self, period):
        self.Time = datetime.min
        self.period = period
        self.closes = deque(maxlen=period)  # queue for storing closes
        self.Value = 0
    
    def __repr__(self):
        return f"Exp Momentum -> IsReady: {self.IsReady}. Time: {self.Time}. Value: {self.Value}." 

    def IsReady(self):
        return (self.Value > 0) # return(len(self.closes)==self.closes.maxlen)
    
    def Update(self, bar:TradeBar):
        self.closes.append(bar.Close)
        self.Time = bar.Time
        self.Value = 0

        if len(self.closes) == self.closes.maxlen:
            self.Value = self.momentum(np.array([*self.closes])) # convert deque to list then to np.array
        
        return (self.Value > 0)

class MomentumRanker(PythonIndicator):
    
    def momentum(self, closes):
        returns = np.log(closes)
        x = np.arange(len(returns))
        slope, _, rvalue, _, _ = linregress(x, returns)
        return ((1 + slope) ** 252) * (rvalue ** 2)  # annualize slope and multiply by R^2

    def __init__(self, period):
        self.Time = datetime.min
        self.period = period
        self.closes = deque(maxlen=period)  # queue for storing closes
        self.Value = 0
    
    def __repr__(self):
        return f"Momentum -> IsReady: {self.IsReady}. Time: {self.Time}. Value: {self.Value}." 

    def IsReady(self):
        return (self.Value > 0) # return(len(self.closes)==self.closes.maxlen)
    
    def Update(self, bar:TradeBar):
        self.closes.append(bar.Close)
        self.Time = bar.Time
    
        self.Value = 0

        if len(self.closes) == self.closes.maxlen:
            self.Value = self.momentum(np.array([*self.closes])) # convert deque to list then to np.array
        
        return (self.Value > 0)

class SymbolData:
    def __init__(self, algorithm, symbol, filter_period):
        self.algorithm = algorithm
        self.symbol = symbol

        self.filter_period = filter_period
        self.volatility_period = 20
        self.momentum_period = 90 

        self.momentum = MomentumRanker(self.momentum_period)
        self.atr = self.algorithm.ATR(symbol, self.volatility_period)
        self.filter = self.algorithm.SMA(symbol, self.filter_period)

        algorithm.RegisterIndicator(self.symbol, self.momentum)
        algorithm.RegisterIndicator(self.symbol, self.filter)
        algorithm.RegisterIndicator(self.symbol, self.atr)
        
        algorithm.WarmUpIndicator(self.symbol, self.atr)
        algorithm.WarmUpIndicator(self.symbol, self.filter)

        # warmup momentum indicators
        trade_bars = algorithm.History[TradeBar](symbol, self.momentum_period, Resolution.Daily)
        for bar in trade_bars:
            self.momentum.Update(bar)
    
    def UpdateIndicators(self, data:TradeBar): 
        # update the indicators
        self.momentum.Update(data)
        self.filter.Update(data.Time, data.Close)  # SMA takes time, IndicatorDataPoint, may need to construct and IndicatorDataPoint
        self.atr.Update(data)                   

    def IsReady(self):
        return self.momentum.IsReady and self.atr.IsReady and self.filter.IsReady

   

class MomentumEffectAlgorithm(QCAlgorithm):

    def Initialize(self):
        self.SetStartDate(2016, 1, 1)  # Set Start Date
        self.SetEndDate(2017, 1, 1)    # Set End Date       
        self.SetCash(100000)           # Set Strategy Cash

        self.UniverseSettings.Resolution = Resolution.Daily
        self.UniverseSettings.DataNormalizationMode = DataNormalizationMode.Adjusted
        self.UniverseSettings.MinimumTimeInUniverse = timedelta(days=1)

        # self.ActiveSecurities dictionary of all the securities currently in universe
        self.symbol_dictionary = {} # dictionary for holding SymbolData key by symbol
        self.excluded_securities = ['SPY']

        # adjust these in line with Clenow's approach - want approx 20 stocks in the portfolio
        self.num_coarse = 500 # Number of symbols selected at Coarse Selection
        self.num_fine = 20    # Number of symbols selected at Fine Selection
        self.num_positions = 10     # Number of symbols with open positions
        self.risk_factor = 0.001    # targeting 10 basis point move per day
        self.momentum_threshold = 0.0  # TODO: check the value to see if this is reasonable
        
        self.index_filter_period = 200

        # variables to control the portfolio rebalance and the stock selection reranking
        self.month = -1
        self.dayofweek = 3  # day of week to rerank
        self.rebalance = False
        self.rerank = False

        # create the equities universe

        # self.AddUniverse(self.Universe.DollarVolume.Top(500))
        # self.AddUniverse(self.Universe.ETF("SPY", Market.USA, self.UniverseSettings))
        self.AddUniverse(self.CoarseSelectionFunction, self.FineSelectionFunction)

        # this is causing serious problems. I don't want to trade SPY, just to use it and its 
        # 200 sma as a filter on the  market, but it ends up in Portfolio holdings and screwing up the data feed 
        # via the factor file adjustments or not in Slice errors, seemingly no matter how much I check the OnData slice

        self.market = self.AddIndex("SPY", Resolution.Daily).Symbol

        self.SetBenchmark(self.market)

        # set Brokerage model and Fee Structure
        self.SetBrokerageModel(BrokerageName.InteractiveBrokersBrokerage, AccountType.Margin)

        # set Free Cash to 2.5%
        self.Settings.FreePortfolioValuePercentage = 0.025
        

    # my methods

    def CalculateRiskParityPositionSize(self, symbol, atr)-> float:
        amountToRisk = self.Portfolio.TotalPortfolioValue * self.risk_factor
        quantity = self.CalculateOrderQuantity(symbol, 0.01) # get the quanity of shares for holding of 1%
        shares = amountToRisk / atr
        rounded_holding = 0.0
        if quantity != 0:
            holding_percent = (shares/quantity)/100
            rounded_holding = round(holding_percent, 3) # round down to 0.001
        return rounded_holding

    # QC methods
    def OnWarmUpFinished(self) -> None:
        self.Log("Equities Momentum Algorithm Ready")

    
    # this article explains when universe selection is done (12:00am each day) and the order of the calls
    # coarse, fine, onsecuritieschanged, then ondata
    # https://www.quantconnect.com/forum/discussion/6485/onsecuritieschanged-questions/p1

    # may eventually switch this out for small or mid cap stocks
    def CoarseSelectionFunction(self, coarse):
        '''Drop securities which have no fundamental data or have too low prices. Select those with highest by dollar volume'''
        
        if not self.Time.weekday() == 1: # monday
            return Universe.Unchanged

        self.rerank = True

        selected = sorted([x for x in coarse if x.HasFundamentalData and x.Price > 5], key=lambda x: x.DollarVolume, reverse=True)
        return [x.Symbol for x in selected[:self.num_coarse]]

    # may not really want to do anything here - might be that coarse filtering is enough
    def FineSelectionFunction(self, fine):
        '''Select security with highest market cap, and common stock'''
        self.Debug(f'{self.Time} FineSelectionFunction')
        filtered = sorted(fine, key=lambda f: f.MarketCap and 
                                              f.SecurityReference.SecurityType == 'ST00000001', # stock is common stock, ST00000002 is preferred stock, 
                                              reverse=True)
        
        return [x.Symbol for x in filtered[:self.num_fine]] 


    def OnData(self, data):

        sliceData = []
        for symbol in data.keys():
            sliceData.append(f'{symbol.ID}')
            
        self.Debug(f'{self.Time} OnData - Slice contains {sliceData}')

        if self.market not in self.symbol_dictionary.keys() and data.Bars.ContainsKey(self.market):
            self.symbol_dictionary[self.market] = SymbolData(self, self.market, 200)
        
        if (self.dayofweek != self.Time.weekday()):
            self.rerank = True
            
        else:
            return

        if (self.month != self.Time.month):
            self.rebalance = True
            self.month = self.Time.month

        self.Debug(f'OnData: Update Indicators in Symbol Dictionary')

        symbolUpdates = []
        
        for symbol in self.Securities.keys():
            symbString = f'{symbol.ID}'

            if data.Bars.ContainsKey(symbol):
                if symbol in self.symbol_dictionary.keys():
                    self.symbol_dictionary[symbol].UpdateIndicators(data.Bars[symbol])
                    symbString + " updated"
                else:
                    self.symbol_dictionary[symbol] = SymbolData(self, symbol, 100)
                    symbString + " created"
            else:
               symbString + " data missing" 
            symbolUpdates.append(symbString)  

        self.Debug(f'OnData: Symbol updates {symbolUpdates}')

        # rerank weekly, rebalance monthly
        if not self.rerank:
            return

        liquidated = []
        purchased = []

        self.Log('OnData: Momentum Reranking')

        # rerank securities based on momentum rank above threshold and close above sma filter
        mom_list = [key for key in self.symbol_dictionary.keys() if self.symbol_dictionary[key].momentum.IsReady]
        mom_threshold = [key for key in mom_list if self.symbol_dictionary[key].momentum.Current.Value >= self.momentum_threshold]
        sorted_mom = sorted(mom_threshold, key=lambda x: self.symbol_dictionary[x].momentum.Current.Value, reverse=True)

        # filter selected to chose only those with a price above their sma 100 
        for symbol in sorted_mom:
            if self.symbol_dictionary[symbol].filter.IsReady:
                if data[symbol].Close < self.symbol_dictionary[symbol].filter.Current.Value:
                    sorted_mom.remove(symbol)
        
        selected = sorted_mom[:self.num_positions] 
        
        momentum_selection = []
        for symbol in selected:
            momentum_selection.append(symbol.ID.ToString().split(' ')[0])
        self.Log(f'Selected by Momentum: {momentum_selection}')

        # Sell
        # Liquidate securities that are in our portfolio, but no longer in the top momentum rankings
        # and have closed below the filter
        for symbol, holding in self.Portfolio.items():
            if (symbol not in selected) and (holding.Quantity > 0):
                close = data[symbol].Close
                filterVal = self.symbol_dictionary[symbol].filter.Current.Value
                if close < filterVal:
                    self.Liquidate(symbol, f'{str(symbol)} as closed {close} is below filter {filterVal}')
                    liquidated.append(symbol.ID.ToString().split(' ')[0])
                      
        # log which ones were removed and current margin remaining now
        self.Log(f'OnData: {liquidated} liquidated. Current Margin: {self.Portfolio.MarginRemaining}')

        # Buy those we don't already have
        buying = (data[self.market].Close > self.symbol_dictionary[self.market].filter.Current.Value)
        self.Log(f'OnData: Buying = {buying} [{data[self.market].Close}, {self.symbol_dictionary[self.market].filter.Current.Value}]')

        # if we pass our index filter, buy selected securities (check not already in holdings first though)
        if buying:
            buffer = (self.Portfolio.TotalPortfolioValue / 100) # 1% buffer
            for symbol in selected:
                if (not symbol in self.Portfolio.keys()) and (not symbol in self.excluded_securities):
                    remaining_margin = self.Portfolio.MarginRemaining
                    if remaining_margin > buffer:
                        holding = self.CalculateRiskParityPositionSize(symbol, self.volatility[symbol].Current.Value)
                        self.SetHoldings(symbol, holding)
                        purchased.append(f"{symbol.ID.ToString().split(' ')[0]} {holding:.2f}")
            # log which ones were purchased
            self.Log(f'OnData: {purchased} purchased based on momentum rankings')
        
        # check current holdings
        holdings = []
        for symbol, holding in self.Portfolio.items(): 
            if holding.Quantity > 0:
                holdings.append(f"{symbol.ID.ToString().split(' ')[0]} {holding.Quantity:.2f}")
        self.Log(f'Portfolio Holdings: {holdings}')

        portfolio_update = f'Total: {self.Portfolio.TotalPortfolioValue}, Cash: {self.Portfolio.Cash}, Margin Used: {self.Portfolio.TotalMarginUsed}, Margin Remaining {self.Portfolio.MarginRemaining}'
        self.Log(f'Portfolio Value: {portfolio_update}')

        if not self.rebalance:
            return

        rebalanced = []

        for symbol, security_holding in self.Portfolio.items():
            # quantitylog = f'{str(symbol)}, qty = {security_holding.Quantity}'
            symbol = security_holding.Symbol
            if symbol in self.symbol_dictionary.keys():
                holding = self.CalculateRiskParityPositionSize(symbol, self.symbol_dictionary[symbol].atr.Current.Value)
                self.SetHoldings(symbol, holding)
                rebalanced.append(f"{symbol.ID.ToString().split(' ')[0]} {holding},")
            else:
               self.Debug(f'Error {str(symbol)} in portfolio holdings, but not in symbol_dictionary') 

        self.Log(f'Portfolio Weights: {rebalanced}')

        self.rebalance = False
        self.rerank = False

 
    def OnSecuritiesChanged(self, changes):

        # shifted the changes from OnSecuritiesChanged to be processed here via self.changes
        addedSymbols = []
        removedSymbols = []

        self.Debug(f'{self.Time:} OnSecuritiesChanged')
        # Clean up securities list and indicator data for removed securities
        for security in changes.RemovedSecurities:
            symbol = security.Symbol
            self.Liquidate(symbol, f'{self.Time} {str(symbol)} no longer in Universe')  
            self.RemoveSecurity(symbol)
            if symbol in self.symbol_dictionary.keys():
                self.symbol_dictionary.pop(symbol) 
                removedSymbols.append(symbol.ID.ToString().split(' ')[0])

        # Create indicators and warm them up for securities newly added to the universe
        for security in changes.AddedSecurities:
            symbol = security.Symbol
            self.AddSecurity(symbol)
            if symbol not in self.symbol_dictionary.keys(): 
                addedSymbols.append(symbol.ID.ToString().split(' ')[0])

        self.Debug(f'SecuritiesChanged: Added: {addedSymbols}')
        self.Debug(f'SecuritiesChanged: Removed: {removedSymbols}')