Overall Statistics
Total Trades
0
Average Win
0%
Average Loss
0%
Compounding Annual Return
0%
Drawdown
0%
Expectancy
0
Net Profit
0%
Sharpe Ratio
0
Probabilistic Sharpe Ratio
0%
Loss Rate
0%
Win Rate
0%
Profit-Loss Ratio
0
Alpha
0
Beta
0
Annual Standard Deviation
0
Annual Variance
0
Information Ratio
0
Tracking Error
0
Treynor Ratio
0
Total Fees
$0.00
Estimated Strategy Capacity
$0
Lowest Capacity Asset
from AlgorithmImports import *
from HelperFuncs import *
import pandas as pd

# 2nd Alpha Model based on Cash Price - Only active 2 days in a week (Day after Midweek & weekly reporting)
    # - if cash fell since last time while future finished more than say $20 above cash today, generate a negative Insight for tom
    # - Confidence & Magnitude depends on How far we are from expiry as well as how big is the basis.
    # - Period is for full 6 hours

# 3rd Alpha Model based on COT NSL - Only applicable Once a week 
# 4th Alpha Model based on WWPA Weekly Inventory Data - Only applicable Once a week 

# Can even seperate this Alpha to 1. Nasdaq performance  & 2. BID/ASK/WAP comparison with yesterday & Traded Price


class NasdaqAlpha(AlphaModel):
    
    def __init__(self, DataDict):
        
        self.DataDict = DataDict # Passed the Data Dictionary that contains all Futures
        self.ModelData = pd.DataFrame() # Contains percentChangeSeconds

        # Insights Related Variables       
        self.insights = []
        self.InsightDirectionDict = lambda s: {0:InsightDirection.Up, 1:InsightDirection.Down, 2:InsightDirection.Flat}[s]
        self.insightPeriod = None
        self.insightsTimeBySymbol = {} # dict to hold insight time
        self.previousBeforeMarketOpenSignals = None
        self.previousAfterMarketOpenSignals = None

    def updateModelData(self):
        
        for symbol in self.DataDict.keys():
            # self.yesterdayPercentChange only updates during Lumber hours
            if self.DataDict[symbol].yesterdayPercentChange:               
                # Captures every Second
                self.ModelData.loc[self.DataDict[symbol].algo.Time,symbol] = list(self.DataDict[symbol]._yesterdayPercentChange.values())[0]            
                # self.DataDict[symbol].algo.Debug(f"In AlphaModel{self.DataDict[symbol].algo.Time}:{self.ModelData.tail(1)}")
                
                # Chart Added
                self.DataDict[symbol].algo.Plot("NQ vs LBS", symbol, self.ModelData.loc[self.DataDict[symbol].algo.Time,symbol])

    def clearModelData(self):
        self.ModelData = pd.DataFrame()

    def saveModelData(self):
        dailyFileName =  f"ModelData_{self.DataDict['Lumber'].algo.Time.date()}"
        self.DataDict['Lumber'].algo.Debug(f"dailyFileName:{dailyFileName}")
        self.DataDict['Lumber'].algo.Debug(f"Length of ModelData b4 saving:{len(self.ModelData)}")
        self.DataDict['Lumber'].algo.ObjectStore.Save(f"{14153031}/{dailyFileName}", self.ModelData.to_json(date_unit='s'))
        

    # Handles security changes in from your universe model.
    def OnSecuritiesChanged(self, algorithm: QCAlgorithm, changes: SecurityChanges) -> None:
        
        # This code runs everytime there is a contract rollover but does it know which symbol has switched
        for changed_event in algorithm.CurrentSlice.SymbolChangedEvents.Values:
            algorithm.Debug(f"Contract rollover from (AM- OnSecuritiesChanged) {changed_event.OldSymbol} to {changed_event.NewSymbol}, Time:{algorithm.Time}")
            pass
        
        # Everytime Contract rollsOver (any Lumber or Nasdaq), the following line runs, but ...
        # algorithm.Debug(f"Count AM: {changes.Count}, changes:{changes}")
        
        # .. this code only runs Once at the very start not when future contracts roll!
        for security in changes.AddedSecurities:
            algorithm.Debug(f"In OnSecuritiesChanged(AM) @ DateTime:{algorithm.Time}, Mapped/ID: {security.Mapped}, Canonical: {security.Mapped.Canonical} \
            #  Symbol: {security.Symbol}, Value: {security.Mapped.Value}, SecurityType: {getSecurityType(security.Mapped.SecurityType)}")                          
            
            # Capture Nasdaq %age Change before Lumber Opens  
            algorithm.Schedule.On(algorithm.DateRules.EveryDay('/LBS'), algorithm.TimeRules.At(9,59,51),self.updateModelData)
            # Capture %age Change for Nasdaq & Lumber during Lumber Market Hours
            algorithm.Schedule.On(algorithm.DateRules.EveryDay('/LBS'), algorithm.TimeRules.Every(timedelta(seconds=1)),self.updateModelData)
            # Store Daily file from ModelData
            # algorithm.Schedule.On(algorithm.DateRules.EveryDay('/LBS'), algorithm.TimeRules.At(16,10,00),self.saveModelData)
            # clear self.ModelData
            algorithm.Schedule.On(algorithm.DateRules.EveryDay('/LBS'), algorithm.TimeRules.At(23,50,00),self.clearModelData)


        for security in changes.RemovedSecurities:
            pass


    def Update(self, algorithm, data):
        
        # https://www.quantconnect.com/docs/v2/writing-algorithms/datasets/quantconnect/us-futures-security-master#05-Data-Point-Attributes
        # This Code runs everytime there is a futures rollover
        # BEAWARE SymbolChangedEvents may also be linked to other events?
        # algorithm.Debug(f"data.SymbolChangedEvents.Keys:{data.SymbolChangedEvents.Keys}")

        # if data.Keys[0] == self.lumberDataClass.Symbol:
        for changed_event in data.SymbolChangedEvents.Values:
            algorithm.Debug(f"Contract rollover (AM- Update Method) from {changed_event.OldSymbol} to {changed_event.NewSymbol}")
            security  = data.Keys[0]             
            algorithm.Debug(f"In Update (AM) @ DateTime:{algorithm.Time}, security:{security}, ID:{security.ID},Canonical:{security.Canonical} \
                + Value:{security.Value},Underlying:{security.Underlying}. SecurityType:{getSecurityType(security.SecurityType)}")

        # or (algorithm.Time < self.DataDict['Lumber'].algo.signalStartTime or algorithm.Time > self.DataDict['Lumber'].algo.signalEndTime)

        # self.Signal only returns value if this is the first time or if position signal has changed
        insightPosition = self.Signal(algorithm)        
        if insightPosition is None: 
            return []

        # TODO:  Active for 5 Minutes after Market open - To Change
        
        # Before Market Open insightPeriod should be long enough so doesn't expire before Market Opens. 
        self.insightPeriod = timedelta(seconds = (self.DataDict['Lumber'].currentClose - timedelta(hours=1, minutes =5) - algorithm.Time).total_seconds())
        # Insight(symbol, period, type, direction, magnitude=None, confidence=None, sourceModel=None, weight=None)
        nasdaqPercent = list(self.DataDict['Nasdaq']._yesterdayPercentChange.values())[0]
        self.insights.append(Insight(symbol = self.DataDict['Lumber'].Mapped, period = self.insightPeriod, \
        type = InsightType.Price, direction = insightPosition, \
        magnitude  = self.NasdaqPercentChangeToMagnitude(nasdaqPercent,self.DataDict['Lumber'].algo.lumberBetaToNasdaq), \
        confidence = self.NasdaqPercentChangeToConfidence(nasdaqPercent), \
        sourceModel=None, weight=None))

        for insight in self.insights:
            algorithm.Debug(f"Insight:{insight}::{algorithm.Time}")

        return self.insights


    def Signal(self, algorithm):    
        symbol = 'Lumber'
        # Before Market Opens:            
        if self.DataDict['Nasdaq'].yesterdayPercentChange and algorithm.Time <= self.DataDict[symbol].currentOpen: # If Not Empty
            nasdaqPercent = list(self.DataDict['Nasdaq']._yesterdayPercentChange.values())[0]
            lumberPercent = list(self.DataDict[symbol]._yesterdayPercentChange.values())[0] if self.DataDict[symbol].yesterdayPercentChange else None
            
            # For now just using 1 condition in Checklist - see if 2 conditions are better
            check_list = [self.DataDict[symbol].bestBidMapped['price'],self.DataDict[symbol].bestAskMapped['price'],self.DataDict[symbol].WAPMapped]           
            condLongPreMarketOpen  = nasdaqPercent > self.DataDict[symbol].algo.nasdaqCutoff and sum(c > self.DataDict[symbol]._MappedYesterday.Current.Value for c in check_list) >=1
            condShortPreMarketOpen = nasdaqPercent < -self.DataDict[symbol].algo.nasdaqCutoff and sum(c < self.DataDict[symbol]._MappedYesterday.Current.Value for c in check_list) >=1

            # The following code is to check if the Signal changed before market Open - if changed (or first time only) then return the InsightDirection else return None
            if (condLongPreMarketOpen, condShortPreMarketOpen) != self.previousBeforeMarketOpenSignals and self.ShouldEmitInsight(algorithm.Time,symbol):
                self.previousBeforeMarketOpenSignals = (condLongPreMarketOpen, condShortPreMarketOpen)
                # returns 1 for Long, 0 for Flat, -1 for Short
                return self.InsightDirectionDict([i for i, x in enumerate([condLongPreMarketOpen,condShortPreMarketOpen,not (condLongPreMarketOpen or condShortPreMarketOpen)]) if x][0])
                
        # After Market Opens to say 1 hour before Market Closes - minutes + 5 since it shows extended timing. 
        elif self.DataDict[symbol].yesterdayPercentChange and algorithm.Time > self.DataDict[symbol].currentOpen \
            and algorithm.Time < (self.DataDict[symbol].currentClose - timedelta(hours=1, minutes =5)):
            nasdaqPercent = list(self.DataDict['Nasdaq']._yesterdayPercentChange.values())[0]
            lumberPercent = list(self.DataDict[symbol]._yesterdayPercentChange.values())[0] if self.DataDict[symbol].yesterdayPercentChange else None
            
            # For now just using 1 condition in Checklist - see if 2 conditions are better
            check_list = [self.DataDict[symbol].bestBidMapped['price'],self.DataDict[symbol].bestAskMapped['price'],self.DataDict[symbol].WAPMapped]           
        
            condLongPostMarketOpen = nasdaqPercent > self.DataDict[symbol].algo.nasdaqCutoff \
            and sum(c > self.DataDict[symbol]._tradeContinuous.price for c in check_list) >=1 \
            and lumberPercent > 0 and lumberPercent < 0.5 * (nasdaqPercent*self.DataDict[symbol].algo.lumberBetaToNasdaq)

            condShortPostMarketOpen = nasdaqPercent < -self.DataDict[symbol].algo.nasdaqCutoff \
            and sum(c < self.DataDict[symbol]._tradeContinuous.price for c in check_list) >=1 \
            and lumberPercent < 0 and lumberPercent > 0.5 * (nasdaqPercent*self.DataDict[symbol].algo.lumberBetaToNasdaq)

            # The following code is to check if the Signal changed after market Open - if changed (or first time only) then return the InsightDirection else return None
            if (condLongPostMarketOpen, condShortPostMarketOpen) != self.previousAfterMarketOpenSignals and self.ShouldEmitInsight(algorithm.Time,symbol):
                self.previousAfterMarketOpenSignals = (condLongPostMarketOpen, condShortPostMarketOpen)
                # returns 1 for Long, 0 for Flat, -1 for Short
                return self.InsightDirectionDict([i for i, x in enumerate([condLongPostMarketOpen,condShortPostMarketOpen,not (condLongPostMarketOpen or condShortPostMarketOpen)]) if x][0])             

        return None
        
    # After every 1 seconds    
    def ShouldEmitInsight(self, utcTime, symbol):
        generatedTimeUtc = self.insightsTimeBySymbol.get(symbol)
        if generatedTimeUtc is not None:
            # we previously emitted a insight for this symbol, check it's period to see if we should emit another insight        
            if utcTime - generatedTimeUtc < timedelta(seconds=1):
                return False
        # we either haven't emitted a insight for this symbol or the previous insight's period has expired, so emit a new insight now for this symbol
        self.insightsTimeBySymbol[symbol] = utcTime
        return True


    # See Notes - Higher the NasdaqYesterday% Change, Higher confidence in Alpha Insight
    def NasdaqPercentChangeToConfidence(self, PercentChange):
        scores = {5: 0.9, 4: 0.75, 3: 0.6, 2: 0.45, 1: 0.3, 0: 0.15}
        abs_percent_change = abs(PercentChange)
        return scores[min(scores.keys(), key=lambda x: abs(x - abs_percent_change))]

    # Change this to Lambda func
    def NasdaqPercentChangeToMagnitude(self, PercentChange, multiplier):
        magnitude = PercentChange * multiplier
        return magnitude




        # After Market Opens - Till say 30 minutes before Close
        # Change to tradeMapped once we find a way to calculate it efficient;ly not using History
        # elif algorithm.Time >= self.DataDict[symbol].currentOpen and self.DataDict[symbol].tradeContinuous['price'] is not None:

        #     symbol = 'Lumber'
        #     algorithm.Debug(f"Algorithm Time:{algorithm.Time}")
        #     algorithm.Debug(f"OpenTime for:{symbol} ::{self.DataDict[symbol].currentOpen}")
        #     algorithm.Debug(f"Is AlgoTime >= OpenTime:{algorithm.Time >= self.DataDict[symbol].currentOpen}")
                        
        #     algorithm.Debug(f"Trade for:{symbol} ::{self.DataDict[symbol].tradeContinuous['price']}")
        #     algorithm.Debug(f"yesterdayPercentChange for Lumber ::{self.DataDict[symbol].yesterdayPercentChange}")



    # def Signal(self, algorithm):    
    #     for symbol in self.DataDict.keys():
    #         if self.DataDict[symbol].yesterdayPercentChange is not None:
    #             # algorithm.Debug(f"yesterdayPercentChange for:{symbol} Time:{algorithm.Time}::{self.DataDict[symbol]._yesterdayPercentChange}")

    #             if symbol == 'Lumber':
    #                 # Change to tradeMapped once we find a way to calculate it efficient;ly not using History
    #                 algorithm.Debug(f"Trade for:{symbol} ::{self.DataDict[symbol].tradeContinuous['price']}")


from AlgorithmImports import *
from HelperFuncs import *
import pandas as pd


class BasisAlpha(AlphaModel):
    
    def __init__(self, DataDict):
        self.insights = []
        self.securities = []
        self.insightPeriod = None
        self.insightsTimeBySymbol = {} # dict to hold insight time
        self.insightCollection = InsightCollection()
        self.DataDict = DataDict # Passed the Data Dictionary that contains all Futures
        self.InsightDirectionDict = lambda s: {0:InsightDirection.Up, 1:InsightDirection.Down, 2:InsightDirection.Flat}[s]
   
    # Handles security changes in from your universe model.
    def OnSecuritiesChanged(self, algorithm: QCAlgorithm, changes: SecurityChanges) -> None:
        
        # This code runs everytime there is a contract rollover but does it know which symbol has switched
        for changed_event in algorithm.CurrentSlice.SymbolChangedEvents.Values:
            algorithm.Debug(f"Contract rollover from (AM- OnSecuritiesChanged) {changed_event.OldSymbol} to {changed_event.NewSymbol}, Time:{algorithm.Time}")
            pass
        
        # Everytime Contract rollsOver (any Lumber or Nasdaq), the following line runs, but ...
        # algorithm.Debug(f"Count AM: {changes.Count}, changes:{changes}")
        
        # .. this code only runs Once at the very start not when future contracts roll!
        for security in changes.AddedSecurities:
            algorithm.Debug(f"In OnSecuritiesChanged(AM) @ DateTime:{algorithm.Time}, Mapped/ID: {security.Mapped}, Canonical: {security.Mapped.Canonical} \
            #  Symbol: {security.Symbol}, Value: {security.Mapped.Value}, SecurityType: {getSecurityType(security.Mapped.SecurityType)}")                          
                       
        for security in changes.RemovedSecurities:
            pass


    def Update(self, algorithm, data):
       # https://www.quantconnect.com/docs/v2/writing-algorithms/datasets/quantconnect/us-futures-security-master#05-Data-Point-Attributes

        # This Code runs everytime there is a futures rollover
        # BEAWARE SymbolChangedEvents may also be linked to other events?
        # algorithm.Debug(f"data.SymbolChangedEvents.Keys:{data.SymbolChangedEvents.Keys}")

        # if data.Keys[0] == self.lumberDataClass.Symbol:
        for changed_event in data.SymbolChangedEvents.Values:
            algorithm.Debug(f"Contract rollover (AM- Update Method) from {changed_event.OldSymbol} to {changed_event.NewSymbol}")
            security  = data.Keys[0]                           
            self.securities.append(security.Value)
            algorithm.Debug(f"In Update (AM) @ DateTime:{algorithm.Time}, security:{security}, ID:{security.ID},Canonical:{security.Canonical} \
                + Value:{security.Value},Underlying:{security.Underlying}. SecurityType:{getSecurityType(security.SecurityType)}")


        insightPosition = self.Signal(algorithm)        
        if insightPosition is None: 
            return []
        

        self.insightPeriod = timedelta(hours=6)
      
        self.insights.append(Insight(symbol = str(self.DataDict['Lumber'].Mapped), period = self.insightPeriod, type = InsightType.Price, direction = insightPosition,magnitude  = None, confidence =None , sourceModel=None, weight=None))

        # for insight in self.insights:
        #     algorithm.Debug(f"Insight:{insight}::{algorithm.Time}")

        return self.insights

   

    def Signal(self, algorithm):
        # key for cash data is 
        symbol='Lumber'
        key="CASH.Cash"

        #get cash data form data dict
        cash=self.DataDict[symbol].custom ["CASH.Cash"]

        #check if cash custom data is avaliable
        if cash:
            #check if data point for previous cash report is avaliable
            #check if report avaliable date and algo date are same
            if cash["last"] and cash['updated_at'].strftime('%Y-%m-%d') == algorithm.Time.strftime('%Y-%m-%d'):
                algorithm.Debug("Condition met with cash data{}")
                # check if data for long 
                condLong  = cash["data"]["Settlephys"] > cash["last"]["Settlephys"]
                condShort = cash["data"]["Settlephys"] < cash["last"]["Settlephys"]

                algorithm.Debug(f"long:{condLong},short:{condShort},previous_phy:{cash['last']['Settlephys']},newphy:{ cash['data']['Settlephys']}")
                if  self.ShouldEmitInsight(algorithm.Time,symbol):
                    # returns 1 for Long, 0 for Flat, -1 for Short
                    return self.InsightDirectionDict([i for i, x in enumerate([condLong,condShort,not (condLong or condShort)]) if x][0])             
        else:
            return None
      
    
    # After every 1 sec    
    def ShouldEmitInsight(self, utcTime, symbol):
        generatedTimeUtc = self.insightsTimeBySymbol.get(symbol)
        if generatedTimeUtc is not None:
            # we previously emitted a insight for this symbol, check it's period to see if we should emit another insight        
            if utcTime - generatedTimeUtc < timedelta(hours=1):
                return False
        # we either haven't emitted a insight for this symbol or the previous insight's period has expired, so emit a new insight now for this symbol
        self.insightsTimeBySymbol[symbol] = utcTime
        return True

     

       
        
          
       
from AlgorithmImports import *
import pandas as pd


class CFTCAlpha(AlphaModel):
    
    def __init__(self, DataDict):
        self.insights = []
        self.securities = []
        self.insightPeriod = None
        self.insightsTimeBySymbol = {} # dict to hold insight time
        self.insightCollection = InsightCollection()
        self.DataDict = DataDict # Passed the Data Dictionary that contains all Futures
        self.InsightDirectionDict = lambda s: {0:InsightDirection.Up, 1:InsightDirection.Down, 2:InsightDirection.Flat}[s]
   
    # Handles security changes in from your universe model.
    def OnSecuritiesChanged(self, algorithm: QCAlgorithm, changes: SecurityChanges) -> None:
        
        # This code runs everytime there is a contract rollover but does it know which symbol has switched
        for changed_event in algorithm.CurrentSlice.SymbolChangedEvents.Values:
            algorithm.Debug(f"Contract rollover from (AM- OnSecuritiesChanged) {changed_event.OldSymbol} to {changed_event.NewSymbol}, Time:{algorithm.Time}")
            pass
        
        # Everytime Contract rollsOver (any Lumber or Nasdaq), the following line runs, but ...
        # algorithm.Debug(f"Count AM: {changes.Count}, changes:{changes}")
        
        # .. this code only runs Once at the very start not when future contracts roll!
        for security in changes.AddedSecurities:
            algorithm.Debug(f"In OnSecuritiesChanged(AM) @ DateTime:{algorithm.Time}, Mapped/ID: {security.Mapped}, Canonical: {security.Mapped.Canonical} \
            #  Symbol: {security.Symbol}, Value: {security.Mapped.Value}, SecurityType: {getSecurityType(security.Mapped.SecurityType)}")                          
                       
        for security in changes.RemovedSecurities:
            pass


    def Update(self, algorithm, data):
       # https://www.quantconnect.com/docs/v2/writing-algorithms/datasets/quantconnect/us-futures-security-master#05-Data-Point-Attributes

        # This Code runs everytime there is a futures rollover
        # BEAWARE SymbolChangedEvents may also be linked to other events?
        # algorithm.Debug(f"data.SymbolChangedEvents.Keys:{data.SymbolChangedEvents.Keys}")

        # if data.Keys[0] == self.lumberDataClass.Symbol:
        for changed_event in data.SymbolChangedEvents.Values:
            algorithm.Debug(f"Contract rollover (AM- Update Method) from {changed_event.OldSymbol} to {changed_event.NewSymbol}")
            security  = data.Keys[0]                           
            self.securities.append(security.Value)
            algorithm.Debug(f"In Update (AM) @ DateTime:{algorithm.Time}, security:{security}, ID:{security.ID},Canonical:{security.Canonical} \
                + Value:{security.Value},Underlying:{security.Underlying}. SecurityType:{getSecurityType(security.SecurityType)}")


        insightPosition = self.Signal(algorithm)        
        if insightPosition is None: 
            return []
        

        self.insightPeriod = timedelta(hours=6)
      
        self.insights.append(Insight(symbol = str(self.DataDict['Lumber'].Mapped), period = self.insightPeriod, type = InsightType.Price, direction = insightPosition,magnitude  = None, confidence =None , sourceModel=None, weight=None))

        # for insight in self.insights:
        #     algorithm.Debug(f"Insight:{insight}::{algorithm.Time}")

        return self.insights

   

    def Signal(self, algorithm):
        # key for disaggregated data is 
        symbol='Lumber'
        key="DISAGGREGATED.Disaggregated"

        #get disaggregated data form data dict
        data=self.DataDict[symbol].custom

        #check if custom data is avaliable
        if data and data[key]:

            disaggregated=data[key]
            #check if data point for previous disaggregated report is avaliable
            #check if report avaliable date and algo date are same
            if disaggregated["last"] and disaggregated['updated_at'].strftime('%Y-%m-%d') == algorithm.Time.strftime('%Y-%m-%d'):
                algorithm.Debug("Condition met with disaggregated data{}")
                # check if data for long 
                condLong  = (disaggregated["data"]["ProdMercPositionsLongAll"]-disaggregated["data"]["ProdMercPositionsShortAll"]) > (disaggregated["last"]["ProdMercPositionsLongAll"]-disaggregated["last"]["ProdMercPositionsShortAll"])
                condShort = (disaggregated["data"]["ProdMercPositionsLongAll"]-disaggregated["data"]["ProdMercPositionsShortAll"]) <(disaggregated["last"]["ProdMercPositionsLongAll"]-disaggregated["last"]["ProdMercPositionsShortAll"])

                algorithm.Debug(f"long:{condLong},short:{condShort},previous_phy:{disaggregated['last']['Settlephys']},newphy:{ disaggregated['data']['Settlephys']}")
                if  self.ShouldEmitInsight(algorithm.Time,symbol):
                    # returns 1 for Long, 0 for Flat, -1 for Short
                    return self.InsightDirectionDict([i for i, x in enumerate([condLong,condShort,not (condLong or condShort)]) if x][0])             
        else:
            return None
      
    
    # After every 1 hour    
    def ShouldEmitInsight(self, utcTime, symbol):
        generatedTimeUtc = self.insightsTimeBySymbol.get(symbol)
        if generatedTimeUtc is not None:
            # we previously emitted a insight for this symbol, check it's period to see if we should emit another insight        
            if utcTime - generatedTimeUtc < timedelta(hours=1):
                return False
        # we either haven't emitted a insight for this symbol or the previous insight's period has expired, so emit a new insight now for this symbol
        self.insightsTimeBySymbol[symbol] = utcTime
        return True

     

       
        
          
       
from AlgorithmImports import *
from HelperFuncs import *
import pandas as pd


class FundamentalsAlpha(AlphaModel):
    
    def __init__(self, DataDict):
        self.insights = []
        self.securities = []
        self.insightPeriod = None
        self.insightsTimeBySymbol = {} # dict to hold insight time
        self.insightCollection = InsightCollection()
        self.DataDict = DataDict # Passed the Data Dictionary that contains all Futures
        self.InsightDirectionDict = lambda s: {0:InsightDirection.Up, 1:InsightDirection.Down, 2:InsightDirection.Flat}[s]
   
    # Handles security changes in from your universe model.
    def OnSecuritiesChanged(self, algorithm: QCAlgorithm, changes: SecurityChanges) -> None:
        
        # This code runs everytime there is a contract rollover but does it know which symbol has switched
        for changed_event in algorithm.CurrentSlice.SymbolChangedEvents.Values:
            algorithm.Debug(f"Contract rollover from (AM- OnSecuritiesChanged) {changed_event.OldSymbol} to {changed_event.NewSymbol}, Time:{algorithm.Time}")
            pass
        
        # Everytime Contract rollsOver (any Lumber or Nasdaq), the following line runs, but ...
        # algorithm.Debug(f"Count AM: {changes.Count}, changes:{changes}")
        
        # .. this code only runs Once at the very start not when future contracts roll!
        for security in changes.AddedSecurities:
            algorithm.Debug(f"In OnSecuritiesChanged(AM) @ DateTime:{algorithm.Time}, Mapped/ID: {security.Mapped}, Canonical: {security.Mapped.Canonical} \
            #  Symbol: {security.Symbol}, Value: {security.Mapped.Value}, SecurityType: {getSecurityType(security.Mapped.SecurityType)}")                          
                       
        for security in changes.RemovedSecurities:
            pass


    def Update(self, algorithm, data):
       # https://www.quantconnect.com/docs/v2/writing-algorithms/datasets/quantconnect/us-futures-security-master#05-Data-Point-Attributes

        # This Code runs everytime there is a futures rollover
        # BEAWARE SymbolChangedEvents may also be linked to other events?
        # algorithm.Debug(f"data.SymbolChangedEvents.Keys:{data.SymbolChangedEvents.Keys}")

        # if data.Keys[0] == self.lumberDataClass.Symbol:
        for changed_event in data.SymbolChangedEvents.Values:
            algorithm.Debug(f"Contract rollover (AM- Update Method) from {changed_event.OldSymbol} to {changed_event.NewSymbol}")
            security  = data.Keys[0]                           
            self.securities.append(security.Value)
            algorithm.Debug(f"In Update (AM) @ DateTime:{algorithm.Time}, security:{security}, ID:{security.ID},Canonical:{security.Canonical} \
                + Value:{security.Value},Underlying:{security.Underlying}. SecurityType:{getSecurityType(security.SecurityType)}")


        insightPosition = self.Signal(algorithm)        
        if insightPosition is None: 
            return []
        

        self.insightPeriod = timedelta(hours=6)
      
        self.insights.append(Insight(symbol = str(self.DataDict['Lumber'].Mapped), period = self.insightPeriod, type = InsightType.Price, direction = insightPosition,magnitude  = None, confidence =None , sourceModel=None, weight=None))

        # for insight in self.insights:
        #     algorithm.Debug(f"Insight:{insight}::{algorithm.Time}")

        return self.insights

   

    def Signal(self, algorithm):
        # key for western data is 
        symbol='Lumber'
        key="WESTERN.Western"
        #get western data form data dict
        data=self.DataDict[symbol].custom

        #check if western custom data is avaliable
        if data and data[key]:
            western=data[key]
            #check if data point for previous western report is avaliable
            #check if report avaliable date and algo date are same
            if western["last"] and western['updated_at'].strftime('%Y-%m-%d') == algorithm.Time.strftime('%Y-%m-%d'):
                algorithm.Debug("Condition met with western data{}")
                # check if data for long 
                condLong  = western["data"]["Orders"] > western["last"]["Orders"]
                condShort = western["data"]["Orders"] < western["last"]["Orders"]

                algorithm.Debug(f"long:{condLong},short:{condShort},previous_order:{western['last']['Orders']},new_order:{ western['data']['Orders']}")
                if  self.ShouldEmitInsight(algorithm.Time,symbol):
                    # returns 1 for Long, 0 for Flat, -1 for Short
                    return self.InsightDirectionDict([i for i, x in enumerate([condLong,condShort,not (condLong or condShort)]) if x][0])             
        else:
            return None
      
    
    # After every 1 hour    
    def ShouldEmitInsight(self, utcTime, symbol):
        generatedTimeUtc = self.insightsTimeBySymbol.get(symbol)
        if generatedTimeUtc is not None:
            # we previously emitted a insight for this symbol, check it's period to see if we should emit another insight        
            if utcTime - generatedTimeUtc < timedelta(hours=1):
                return False
        # we either haven't emitted a insight for this symbol or the previous insight's period has expired, so emit a new insight now for this symbol
        self.insightsTimeBySymbol[symbol] = utcTime
        return True
from AlgorithmImports import *
from HelperFuncs import *
import pandas as pd
import math

# TO DO: Use following functions in only 1 of the Alpha Models
        # updateModelData
        # clearModelData
        # saveModelData
        # & In scheduler
        # changed_event in data.SymbolChangedEvents.Values: in Update Function

class NasdaqAlpha(AlphaModel):
    Name = "NasdaqAlpha"
    symbol = 'Lumber'

    
    def __init__(self, DataDict):
        
        self.DataDict = DataDict # Passed the Data Dictionary that contains all Futures
        self.ModelData = pd.DataFrame() # Contains percentChangeSeconds

        # Insights Related Variables       
        self.InsightDirectionDict = lambda s: {0:InsightDirection.Up, 1:InsightDirection.Down, 2:InsightDirection.Flat}[s]
        self.previousBeforeMarketOpenSignals = None
        self.previousAfterMarketOpenSignals = None
        self.BeforeMarketOpenFlag = None
        self.ShouldEmitInsight = RunAfterInterval(interval_seconds= 1 ) # Class from HelperFuncs
        
        # Signal Paremeters - To be Optimised
        self.nasdaqCutoff = 1 # Nasdaq Cutoff before we go Long/Short    
        self.lumberBetaToNasdaq = 1.5 # This should be calculated based on regression coefficient        
        self.percentOfPotential = 0.5 # Cutoff to enter Trade before Signal has played out
        self.temp = 0

    # Handles security changes in from your universe model.
    def OnSecuritiesChanged(self, algorithm: QCAlgorithm, changes: SecurityChanges) -> None:
        
        # This code runs everytime there is a contract rollover but does it know which symbol has switched
        for changed_event in algorithm.CurrentSlice.SymbolChangedEvents.Values:
            algorithm.Debug(f"Contract rollover from (AM- OnSecuritiesChanged) {changed_event.OldSymbol} to {changed_event.NewSymbol}, Time:{algorithm.Time}")
            pass
        
        # Everytime Contract rollsOver (any Lumber or Nasdaq), the following line runs, but ...
        # algorithm.Debug(f"Count AM: {changes.Count}, changes:{changes}")
        
        # .. this code only runs Once at the very start not when future contracts roll!
        for security in changes.AddedSecurities:
            # algorithm.Debug(f"In OnSecuritiesChanged(AM) @ DateTime:{algorithm.Time}, Mapped/ID: {security.Mapped}, Canonical: {security.Mapped.Canonical} \
            # #  Symbol: {security.Symbol}, Value: {security.Mapped.Value}, SecurityType: {getSecurityType(security.Mapped.SecurityType)}")                          
            pass
            # Capture Nasdaq %age Change before Lumber Opens  
            # algorithm.Schedule.On(algorithm.DateRules.EveryDay('/LBS'), algorithm.TimeRules.At(9,59,51),self.updateModelData)
            # Capture %age Change for Nasdaq & Lumber during Lumber Market Hours
            # algorithm.Schedule.On(algorithm.DateRules.EveryDay('/LBS'), algorithm.TimeRules.Every(timedelta(seconds=1)),self.updateModelData)
            # Store Daily file from ModelData
            # algorithm.Schedule.On(algorithm.DateRules.EveryDay('/LBS'), algorithm.TimeRules.At(16,10,00),self.saveModelData)
            # clear self.ModelData
            # algorithm.Schedule.On(algorithm.DateRules.EveryDay('/LBS'), algorithm.TimeRules.At(23,50,00),self.clearModelData)

        for security in changes.RemovedSecurities:
            pass

    def Update(self, algorithm, data):
        
        # https://www.quantconnect.com/docs/v2/writing-algorithms/datasets/quantconnect/us-futures-security-master#05-Data-Point-Attributes
        # This Code runs everytime there is a futures rollover
        # algorithm.Debug(f"data.SymbolChangedEvents.Keys:{data.SymbolChangedEvents.Keys}")

        # if data.Keys[0] == self.lumberDataClass.Symbol:
        for changed_event in data.SymbolChangedEvents.Values:
            algorithm.Debug(f"Contract rollover (AM- Update Method) from {changed_event.OldSymbol} to {changed_event.NewSymbol}")
            security  = data.Keys[0]             
            algorithm.Debug(f"In Update (AM) @ DateTime:{algorithm.Time}, security:{security}, ID:{security.ID},Canonical:{security.Canonical} \
                + Value:{security.Value},Underlying:{security.Underlying}. SecurityType:{getSecurityType(security.SecurityType)}")

        # Constructing Insight
        insight = None
        # 1. Insight Position - Up, Down or Flat        
        if (insightPosition := self.Signal(algorithm)) is None or not self.ShouldEmitInsight.run(self.symbol, algorithm.UtcTime):
            return []

        # 2. Insight Period: 1 Second Long Insights 
        if self.BeforeMarketOpenFlag:
            # insightPeriod = timedelta(seconds = (self.DataDict[self.symbol].currentOpen - algorithm.Time).total_seconds()) + timedelta(seconds = 60)
            insightPeriod = timedelta(seconds = (self.DataDict[self.symbol].currentClose - self.DataDict[self.symbol].currentOpen).total_seconds()) - timedelta(hours = 1)
        else:
            insightPeriod = timedelta(seconds = (self.DataDict[self.symbol].currentClose - timedelta(hours=1, minutes =5) - algorithm.Time).total_seconds())
        
        # 3. Insight Symbol
        mappedSymbol = self.DataDict[self.symbol].Mapped
        nasdaqPercent = list(self.DataDict['Nasdaq']._yesterdayPercentChange.values())[0]

        # 4. Insight Magnitude
        insightMagnitude = round(NasdaqPercentChangeToMagnitude(nasdaqPercent,self.lumberBetaToNasdaq)/100,3)
        
        # 5. Insight Confidence
        insightConfidence = NasdaqPercentChangeToConfidence(nasdaqPercent)

        insight = Insight(symbol = mappedSymbol, period = insightPeriod, type = InsightType.Price, direction = insightPosition, \
        magnitude  = insightMagnitude, confidence = insightConfidence, sourceModel=self.Name, weight=None)

        algorithm.Debug(f"Insight:{insight}::{algorithm.Time}")

        return [insight]


    def Signal(self, algorithm):    
        # Before Market Opens: Remember Exchange stops accepting orders 30 seoconds b4 Market Opens, yesterdayPercentChange available only after SignalStartTime        
        if self.DataDict['Nasdaq'].yesterdayPercentChange and algorithm.Time < (self.DataDict[self.symbol].currentOpen- timedelta(seconds = 30)):   
            
            self.BeforeMarketOpenFlag = True
            nasdaqPercent = list(self.DataDict['Nasdaq']._yesterdayPercentChange.values())[0]            
            condLongPreMarketOpen  = nasdaqPercent > self.nasdaqCutoff
            condShortPreMarketOpen = nasdaqPercent < -self.nasdaqCutoff

            # Only Send new Insight if Changed from before else return None
            if (condLongPreMarketOpen, condShortPreMarketOpen) != self.previousBeforeMarketOpenSignals:
                # algorithm.Debug(f"1 Works in NASDAQALPHA :::{algorithm.Time}")
                self.previousBeforeMarketOpenSignals = (condLongPreMarketOpen, condShortPreMarketOpen)
                # returns 1 for Long, 0 for Flat, -1 for Short
                return self.InsightDirectionDict([i for i, x in enumerate([condLongPreMarketOpen,condShortPreMarketOpen,not (condLongPreMarketOpen or condShortPreMarketOpen)]) if x][0])
               

        # After Market Opens to say 1 hour before Market Closes - minutes + 5 since it shows extended timing. 
        # Note that if no Traded Value for Lumber, yesterdayPercentChange will be empty and this code won't execute
        elif self.DataDict[self.symbol].yesterdayPercentChange and algorithm.Time >= self.DataDict[self.symbol].currentOpen and algorithm.Time < (self.DataDict[self.symbol].currentClose - timedelta(hours=0, minutes =5)):
            
            self.BeforeMarketOpenFlag = False
            nasdaqPercent = list(self.DataDict['Nasdaq']._yesterdayPercentChange.values())[0]
            lumberPercent = list(self.DataDict[self.symbol]._yesterdayPercentChange.values())[0] if self.DataDict[self.symbol].yesterdayPercentChange else None
            
            # Note if lumberPercent is None, it won't affect the following 2 conditions:
            condLongPostMarketOpen = nasdaqPercent > self.nasdaqCutoff and (lumberPercent is None or lumberPercent > 0) \
            and lumberPercent < self.percentOfPotential * (nasdaqPercent*self.lumberBetaToNasdaq)

            condShortPostMarketOpen = nasdaqPercent < -self.nasdaqCutoff and (lumberPercent is None or lumberPercent < 0) \
            and lumberPercent > self.percentOfPotential * (nasdaqPercent*self.lumberBetaToNasdaq)

            # The following code is to check if the Signal changed after market Open - if changed (or first time only) then return the InsightDirection else return None
            if (condLongPostMarketOpen, condShortPostMarketOpen) != self.previousAfterMarketOpenSignals:

                self.temp += 1 
                algorithm.Debug(f"Start:{self.temp} at {algorithm.Time}")
                algorithm.Debug(f"condLongPostMarketOpen:{condLongPostMarketOpen} at {algorithm.Time}")
                algorithm.Debug(f"condShortPostMarketOpen:{condShortPostMarketOpen} at {algorithm.Time}")
                algorithm.Debug(f"nasdaqPercent:{nasdaqPercent} at {algorithm.Time}")
                algorithm.Debug(f"lumberPercent:{lumberPercent} at {algorithm.Time}")
                algorithm.Debug(f"percentOfPotential * (nasdaqPercent*lumberBetaToNasdaq):{self.percentOfPotential * (nasdaqPercent*self.lumberBetaToNasdaq)} at {algorithm.Time}")
                algorithm.Debug(f"Return from InsighTdICT:{self.InsightDirectionDict([i for i, x in enumerate([condLongPostMarketOpen,condShortPostMarketOpen,not (condLongPostMarketOpen or condShortPostMarketOpen)]) if x][0])} at {algorithm.Time}")
                algorithm.Debug(f"Stop:{self.temp} at {algorithm.Time}")
                # algorithm.Debug(f"2 Works in NASDAQALPHA :::{algorithm.Time}")

                self.previousAfterMarketOpenSignals = (condLongPostMarketOpen, condShortPostMarketOpen)
                # returns 1 for Long, 0 for Flat, -1 for Short
                return self.InsightDirectionDict([i for i, x in enumerate([condLongPostMarketOpen,condShortPostMarketOpen,not (condLongPostMarketOpen or condShortPostMarketOpen)]) if x][0])

        return None


    # ModelData Related Functions
    def updateModelData(self):
        
        for symbol in self.DataDict.keys():
            # self.yesterdayPercentChange only updates during Lumber hours
            if self.DataDict[symbol].yesterdayPercentChange:               
                # Captures every Second
                self.ModelData.loc[self.DataDict[symbol].algo.Time,symbol] = list(self.DataDict[symbol]._yesterdayPercentChange.values())[0]            
                # self.DataDict[symbol].algo.Debug(f"In AlphaModel{self.DataDict[symbol].algo.Time}:{self.ModelData.tail(1)}")
                
                # Chart Added
                self.DataDict[symbol].algo.Plot("NQ vs LBS", symbol, self.ModelData.loc[self.DataDict[symbol].algo.Time,symbol])

    def clearModelData(self):
        self.ModelData = pd.DataFrame()

    def saveModelData(self):
        dailyFileName =  f"ModelData_{self.DataDict[self.symbol].algo.Time.date()}"
        self.DataDict[self.symbol].algo.Debug(f"dailyFileName:{dailyFileName}")
        self.DataDict[self.symbol].algo.Debug(f"Length of ModelData b4 saving:{len(self.ModelData)}")
        self.DataDict[self.symbol].algo.ObjectStore.Save(f"{14153031}/{dailyFileName}", self.ModelData.to_json(date_unit='s'))
from AlgorithmImports import *
from HelperFuncs import *
from itertools import groupby

# TO CHECK - Only First 10 k alphas are being shown in Backtest Insight Tab - For now using TimeInterval of 2 seconds to see everything - change to 1

# Only use for Entry Condition before Market Opens. AfterMarketOpen Insight can be backtested later for Market Order?
class OrderBookAlpha(AlphaModel):
    Name = "OrderBook"
    symbol = 'Lumber'

    def __init__(self, DataDict):
        
        self.DataDict = DataDict # Passed the Data Dictionary that contains all Futures

        # Insights Related Variables       
        self.InsightDirectionDict = lambda s: {0:InsightDirection.Up, 1:InsightDirection.Down, 2:InsightDirection.Flat}[s]
        self.previousBeforeMarketOpenSignals = None
        self.NumberOfConditions = 2  # Using 2 conditions out of 3 (BID,ASK, WAP) compared to YesterdayClose (Before Open) & Last Traded Price (After Open)
        self.BeforeMarketOpenFlag = None
        self.ShouldEmitInsight = RunAfterInterval(interval_seconds= 2) # Class from HelperFuncs

    # Handles security changes in from your universe model.
    def OnSecuritiesChanged(self, algorithm: QCAlgorithm, changes: SecurityChanges) -> None:
        
        # This code runs everytime there is a contract rollover but does it know which symbol has switched
        for changed_event in algorithm.CurrentSlice.SymbolChangedEvents.Values:
            algorithm.Debug(f"Contract rollover from (AM- OnSecuritiesChanged) {changed_event.OldSymbol} to {changed_event.NewSymbol}, Time:{algorithm.Time}")
            pass
        
        # This code only runs Once at the very start not when future contracts roll!
        for security in changes.AddedSecurities:
            # algorithm.Debug(f"In OnSecuritiesChanged(AM) @ DateTime:{algorithm.Time}, Mapped/ID: {security.Mapped}, Canonical: {security.Mapped.Canonical} \
            # #  Symbol: {security.Symbol}, Value: {security.Mapped.Value}, SecurityType: {getSecurityType(security.Mapped.SecurityType)}")                          
            pass

        for security in changes.RemovedSecurities:
            pass


    def Update(self, algorithm, data):
        
        for changed_event in data.SymbolChangedEvents.Values:
            pass
        
        # Constructing Insight
        insight = None
        # 1. Insight Position - Up, Down or Flat
        if (insightPosition := self.Signal(algorithm)) is not None and self.ShouldEmitInsight.run(self.symbol, algorithm.UtcTime):
             pass
        else:
            try:
                # algorithm.Debug(f"Close:{self.DataDict[self.symbol].currentClose} at {algorithm.Time}")
                # algorithm.Debug(f"Open:{self.DataDict[self.symbol].currentOpen} at {algorithm.Time}")
                if algorithm.Time > (self.DataDict[self.symbol].currentClose - timedelta(hours=0, minutes =55)):
                    # algorithm.Debug(f"Last 55 minutes at {algorithm.Time}")
                    pass
            except:
                pass
            return []
        # 2. Insight Period: 1 Second Long Insights            
        if self.BeforeMarketOpenFlag:
            insightPeriod = timedelta(seconds = (self.DataDict[self.symbol].currentOpen - algorithm.Time).total_seconds()) + timedelta(seconds = 2)
        else:
            insightPeriod = timedelta(seconds = 1)
        # 3. Insight Symbol
        mappedSymbol = self.DataDict[self.symbol].Mapped

        # 5. Insight Confidence - Using Distance Formula
        
        insightConfidence = NasdaqPercentChangeToConfidence(nasdaqPercent)
        
        insight = Insight(symbol = mappedSymbol, period = insightPeriod, \
        type = InsightType.Price, direction = insightPosition, \
        magnitude  = None, confidence = None, sourceModel= self.Name, weight=None)       

        return [insight]


    def Signal(self, algorithm):    

        # Before Market Opens: Remember Exchange stops accepting orders 30 seoconds b4 Market Opens, yesterdayPercentChange available only after SignalStartTime        
        if self.DataDict['Nasdaq'].yesterdayPercentChange and algorithm.Time < (self.DataDict[self.symbol].currentOpen- timedelta(seconds = 30)):           
            self.BeforeMarketOpenFlag = True
            check_list = [self.DataDict[self.symbol].bestBidMapped['price'],self.DataDict[self.symbol].bestAskMapped['price'],self.DataDict[self.symbol].WAPMapped]           
            condLongPreMarketOpen  = sum(c > self.DataDict[self.symbol]._MappedYesterday.Current.Value for c in check_list) >= self.NumberOfConditions
            condShortPreMarketOpen = sum(c < self.DataDict[self.symbol]._MappedYesterday.Current.Value for c in check_list) >= self.NumberOfConditions
            # The following code is to check if the Signal changed before 30 seconds market Open - if changed (or first time only) then return the InsightDirection else return None
            if (condLongPreMarketOpen, condShortPreMarketOpen) != self.previousBeforeMarketOpenSignals:
                self.previousBeforeMarketOpenSignals = (condLongPreMarketOpen, condShortPreMarketOpen)
                # Returns 1 for Long, 0 for Flat, -1 for Short
                return self.InsightDirectionDict([i for i, x in enumerate([condLongPreMarketOpen,condShortPreMarketOpen,not (condLongPreMarketOpen or condShortPreMarketOpen)]) if x][0])
          
        # After Market Opens to say 1 hour before Market Closes: minutes + 5 since it shows extended timing. 
        # Change tradeContinuous to tradeMapped once we figure out efficient way to calculate Mapped Trade
        elif self.DataDict[self.symbol].yesterdayPercentChange and algorithm.Time >= self.DataDict[self.symbol].currentOpen and algorithm.Time < (self.DataDict[self.symbol].currentClose - timedelta(hours=0, minutes =5)):

            # algorithm.Debug(f"currentOpen:{self.DataDict[self.symbol].currentOpen} at {algorithm.Time}")
            # algorithm.Debug(f"currentClose:  {(self.DataDict[self.symbol].currentClose)} at {algorithm.Time}")
            # algorithm.Debug(f"currentClose - 5 minutes:  {(self.DataDict[self.symbol].currentClose - timedelta(minutes =5))} at {algorithm.Time}")
            # algorithm.Debug(f"currentClose - 1 hour:  {(self.DataDict[self.symbol].currentClose - timedelta(hours=1))} at {algorithm.Time}")
            # algorithm.Debug(f"currentClose - 1 hour & 5 minutes:  {(self.DataDict[self.symbol].currentClose - timedelta(hours=1, minutes =5))} at {algorithm.Time}")

            self.BeforeMarketOpenFlag = False
            check_list = [self.DataDict[self.symbol].bestBidMapped['price'],self.DataDict[self.symbol].bestAskMapped['price'],self.DataDict[self.symbol].WAPMapped]                   
            condLongPostMarketOpen = sum(c > self.DataDict[self.symbol]._tradeContinuous.price for c in check_list) >= self.NumberOfConditions
            condShortPostMarketOpen = sum(c < self.DataDict[self.symbol]._tradeContinuous.price for c in check_list) >= self.NumberOfConditions
               
            return self.InsightDirectionDict([i for i, x in enumerate([condLongPostMarketOpen,condShortPostMarketOpen,not (condLongPostMarketOpen or condShortPostMarketOpen)]) if x][0])             

        return None
        
from AlgorithmImports import *
import math
from collections import deque
from EnumFunc import *
from functools import wraps


# A skeleton class for storing Futures Data - Shared by all Models

class DataClass:

    def __init__(self, Algorithm, continuous_contract):
        
        self.algo = Algorithm # This is Our Instance of QCAlgorithm
        # self.algo.UniverseSettings.ExtendedMarketHours = True

        # https://www.quantconnect.com/docs/v2/writing-algorithms/securities/asset-classes/futures/requesting-data#09-Properties
        self.continuous_contract = continuous_contract

        # Symbols for securities:
        self.Symbol = self.continuous_contract.Symbol 
        self.tickSize = self.continuous_contract.SymbolProperties.MinimumPriceVariation
        
        # Index, askclose, askhigh, asklow, askopen, asksize, bidclose, bidhigh, bidlow, bidopen, bidsize, close, high, low, open, volume)
        # Index, Value:(Timestamp('2022-11-15 00:00:00'), 'LBS Y3IN813DW4QP', Timestamp('2022-10-26 19:00:00'))
        self.yesterday = dotdict(dict.fromkeys(['open', 'high', 'low', 'close', 'volume']))
        
        # Make sure to use self.Mapped and not self.Symbol since using BackwardPanama
        self.lastBid = dotdict(dict.fromkeys(['time', 'price', 'size']))
        self.lastAsk = dotdict(dict.fromkeys(['time','price', 'size']))
        
        self._bestBid = dotdict(dict.fromkeys(['time', 'price', 'size']))
        self._bestAsk = dotdict(dict.fromkeys(['time','price', 'size']))
        self._trade = dotdict(dict.fromkeys(['time','price', 'size']))

        # self.trade = dotdict(dict.fromkeys(['time','price', 'size']))

        self._WAP = None
                

        # https://www.quantconnect.com/docs/v2/writing-algorithms/indicators/supported-indicators/average-true-range
        # This should be yesterday's
        # self.atr = self.algo.ATR(self.Symbol, Resolution.Daily)    
        
        # quoteEvent is triggered everytime a new Quote is placed not only when the best quote is changed/placed- I think?
        # self.quoteEvent = TickQuoteBarConsolidator(1)
        # self.quoteEvent.DataConsolidated += self.quoteEventHandler
        # self.algo.SubscriptionManager.AddConsolidator(self.Symbol, self.quoteEvent)

        # tradeEvent is triggered everytime a Trade is placed?
        # self.tradeEvent = TickConsolidator(1)
        # self.tradeEvent.DataConsolidated += self.tradeEventHandler
        # self.algo.SubscriptionManager.AddConsolidator(self.Symbol, self.tradeEvent)

        # To Update Yesterday: At Midnight the contract Rolls and pulls the correct data for Lumber - The issue is other futures may have different Open Close timings
        # self.algo.Schedule.On(self.algo.DateRules.EveryDay(), self.algo.TimeRules.Midnight, self.updateYesterday)

        # Update at a specific Time - Symbol agnostic - This is 10 minutes before Lumber Open - Change when daylight savings
        # self.algo.Schedule.On(self.algo.DateRules.EveryDay(), self.algo.TimeRules.At(8,59), self.updateYesterday)        

        # Get Yesterday's Settlement - Make it a Property
        # self.Securities[self.lumber.Mapped].Expiry # Get contract's expiry?

        # Update Yesterday Price 10 minutes before Market Opens - Can even use 'LBS' in TimeRules.AfterMarketOpen
        self.algo.Schedule.On(self.algo.DateRules.EveryDay(self.Symbol), self.algo.TimeRules.AfterMarketOpen(self.Symbol, -10), self.updateYesterday)

        # Reset BBO dicts 10 minutes after market close - Not sure if it impacts extendedMarketHours
        self.algo.Schedule.On(self.algo.DateRules.EveryDay(self.Symbol), self.algo.TimeRules.BeforeMarketClose(self.Symbol, -10), self.resetTickDict)

        # Nasdaq - closes at 4 pm & opens at 8:30 Chicago, Also trades at 11 pm till not sure when?
        # OnSecuritiesChanged: Date:2022-12-19 23:00:00, security:MNQ Y6URRFPZ86BL


    def quoteEventHandler(self, sender: object, Qbar: QuoteBar) -> None:

        if self.algo.CurrentSlice.Ticks.ContainsKey(self.Symbol) and self.algo.CurrentSlice.Ticks[self.Symbol] is not None:
            ticks = self.algo.CurrentSlice.Ticks[self.Symbol]
            for tick in ticks:
                tick_type = getTickType(tick.TickType)
                if tick_type == 'Quote':
                                        
                    if int(getattr(self.algo.Securities[self.Mapped], 'AskSize')) != 0: 
                        for key,prop in zip(list(self._bestAsk.keys()),['LocalTime','AskPrice', 'AskSize']):
                            setattr(self._bestAsk,str(key),getattr(self.algo.Securities[self.Mapped], prop))

                    if int(getattr(self.algo.Securities[self.Mapped], 'BidSize')) != 0: 
                        for key,prop in zip(list(self._bestBid.keys()),['LocalTime','BidPrice', 'BidSize']):
                            setattr(self._bestBid,str(key),getattr(self.algo.Securities[self.Mapped], prop))

                    # Code to store Last Bid & Ask - switching it off for the time being
                    # if int(getattr(tick, 'AskPrice')) != 0: 
                    #     # In DateTime - microseconds missing leading zeros in our dict versus reported
                    #     for key,prop in zip(list(self.lastAsk.keys()),['EndTime','AskPrice', 'AskSize']):
                    #         setattr(self.lastAsk,str(key),getattr(tick, prop))
                        
                    # if int(getattr(tick, 'BidPrice')) != 0: 
                    #     for key,prop in zip(list(self.lastBid.keys()),['EndTime','BidPrice', 'BidSize']):
                    #         setattr(self.lastBid,str(key),getattr(tick, prop))



    def tradeEventHandler(self, sender: object, Tbar: TradeBar) -> None:
        
        if self.algo.CurrentSlice.Ticks.ContainsKey(self.Symbol) and self.algo.CurrentSlice.Ticks[self.Symbol] is not None:
            ticks = self.algo.CurrentSlice.Ticks[self.Symbol]
            for tick in ticks:
                tick_type = getTickType(tick.TickType)
                if tick_type == 'Trade':
                    self.algo.tradeCount += 1
                    if int(getattr(tick, 'Quantity')) != 0:
                        for key,prop in zip(list(self.trade.keys()),['EndTime','Price','Quantity']):
                            setattr(self.trade,str(key),getattr(tick, prop))                            


    # Resetting Best Bid/Ask at MarketClose
    def resetTickDict(self):

        # Resetting After Market Close
        # self.lastAsk = self.lastAsk.fromkeys(self.lastAsk, None)
        # self.lastBid = self.lastBid.fromkeys(self.lastBid, None)

        self._bestAsk = self._bestAsk.fromkeys(self._bestAsk, None)
        self._bestBid = self._bestBid.fromkeys(self._bestBid, None)
        self._trade = self._trade.fromkeys(self._trade, None)
        self._WAP = None

        # self.algo.Debug(f"Ask Reset:{self.Symbol} :{self.algo.Time}:{self._bestAsk}")
        # self.algo.Debug(f"Bid Reset :{self.Symbol} :{self.algo.Time}:{self._bestBid}")
        # self.algo.Debug(f"Trade Reset :{self.Symbol} :{self.algo.Time}:{self._trade}")


    def updateYesterday(self):

        # self.algo.Debug(f"Updated Yesterday price for:{self.Symbol} @ {self.algo.Time}")
        
        # Stores Contracts Yesterday's OHCLV
        # Note that Volume doesn't match with reported in our Database (atleast for Lumber)
        
        if self.Mapped is not None:
            # self.algo.Debug(f"ctrctHistory Symbol:{self.Mapped},{self.continuous_contract}")

            ctrctHistory = self.algo.History(self.Mapped, 1, Resolution.Daily )   
            for bar in ctrctHistory.itertuples():
                for property in ['open', 'high', 'low', 'close', 'volume']:
                    if not math.isnan(float(getattr(bar, property))): # Required since updateYesterday is called even on Non trading days
                        self.yesterday[str(property)] = getattr(bar, property)

                # for prop in dir(bar):
                #     value = getattr(bar, prop)
                #     self.algo.Debug(f"prop:{prop}, value:{value}")
                        # Checking if we get same data
                        # if self.algo.CurrentSlice[self.Mapped].High is not None:
                        #     self.algo.Debug(f"{self.algo.CurrentSlice[self.Mapped].High}")


    # RealOnly RealTime 
    @property
    def Mapped(self):
        return getattr(self.continuous_contract, 'Mapped')


    @property
    def Canonical(self):
        return getattr(self.Symbol, 'Canonical')

    # @property
    def Underlying(self):
        return getattr(self, 'Underlying')


    @property
    def bestBid(self):

        # Not using this since somehow when we club 2 futures, this wasn't updating correctly or giving None
        # Also using getattr(self.algo.Securities[self.Mapped], prop)) instead of 
        # if self.algo.CurrentSlice.Ticks.ContainsKey(self.Symbol) and self.algo.CurrentSlice.Ticks[self.Symbol] is not None:
        #     ticks = self.algo.CurrentSlice.Ticks[self.Symbol]
        #     for tick in ticks:
        #         tick_type = getTickType(tick.TickType)
        #         if tick_type == 'Quote' and int(getattr(self.algo.Securities[self.Mapped], 'BidSize')) != 0: 
        #             for key,prop in zip(list(self._bestBid.keys()),['LocalTime','BidPrice', 'BidSize']):
        #                 setattr(self._bestBid,str(key),getattr(self.algo.Securities[self.Mapped], prop))

        # Only updating if Bid price and Size changed so as to keep the original Time of last update
        if int(getattr(self.algo.Securities[self.Mapped], 'BidSize')) != 0 and not self.alreadyUpdated('BidPrice', 'BidSize', self._bestBid):
            for key,prop in zip(list(self._bestBid.keys()),['LocalTime','BidPrice', 'BidSize']):
                setattr(self._bestBid,str(key),getattr(self.algo.Securities[self.Mapped], prop))

        return self._bestBid



    @property
    def bestAsk(self):
        
        # Only updating if Ask price and Size changed so as to keep the original Time of last update
        if int(getattr(self.algo.Securities[self.Mapped], 'AskSize')) != 0 and not self.alreadyUpdated('AskPrice', 'AskSize', self._bestAsk):
            for key,prop in zip(list(self._bestBid.keys()),['LocalTime','AskPrice', 'AskSize']):
                setattr(self._bestAsk,str(key),getattr(self.algo.Securities[self.Mapped], prop))

        return self._bestAsk


    # This is not entirely correct as someone may have canceled and others may have added a bid with net being affect 0 
    def alreadyUpdated(self, price, size, dictionary):
        return getattr(self.algo.Securities[self.Mapped], price) == dictionary.price and getattr(self.algo.Securities[self.Mapped], size) == dictionary.size
        
    
    @property
    def trade(self):
        
        # TO DO: Need to make it work like Quotes - Ask/Bid - getattr(self.algo.Securities[self.Mapped], prop))

        # Ideally this should use self.Mapped but it doesn't take that. 

        if self.algo.CurrentSlice.Ticks.ContainsKey(self.Symbol) and self.algo.CurrentSlice.Ticks[self.Symbol] is not None:
            ticks = self.algo.CurrentSlice.Ticks[self.Symbol]

            for tick in ticks:
                tick_type = getTickType(tick.TickType)
                if tick_type == 'Trade' and int(getattr(tick, 'Quantity')) != 0:
                    for key,prop in zip(list(self._trade.keys()),['Time','Price','Quantity']):
                        setattr(self._trade,str(key),getattr(tick, prop))
                        # setattr(self._trade,str(key),getattr(self.algo.Securities[self.Mapped], prop))

        # https://www.quantconnect.com/docs/v2/writing-algorithms/historical-data/history-requests

        # end_time = self.algo.Time
        # start_time = end_time  - timedelta(seconds=1)
        
        
        # ticks = self.algo.History[Tick](self.Mapped, start_time, end_time, Resolution.Tick)
        # count = 0
        # # self.algo.Debug(f"type of ticks:{type(Ticks)}")
        # for tick in ticks:
        #     if getTickType(tick.TickType) == 'Trade':
        #         self.algo.Debug(f"tick{count}:{tick}")
        #         count +=1
            
                # if getTickType(tick.TickType) == 'Trade':
                #     for prop in dir(tick):
                #         try:
                #                 value = getattr(tick, prop)
                #                 self.algo.Debug(f"Tprop:{prop}, Tvalue:{value}")
                #         except:
                #             pass

                #         tick.set_Symbol(self.Mapped)

            # tick_type = getTickType(tick.TickType)
            #  and int(getattr(tick, 'Quantity')) != 0
            # if tick_type == 'Trade':
            #     count +=1
            #     dic = {k:getattr(tick, k) for k in ['EndTime','Price','Quantity'] if getattr(tick, k) is not None}
            #     # self.algo.Debug(f"Tickcount:{count}, time:{self.algo.Time} {dic}")

            #     for key,prop in zip(list(self._trade.keys()),['EndTime','Price','Quantity']):                       
            #         setattr(self._trade,str(key),getattr(tick, prop))
        
        return self._trade


    @property
    def WAP(self):

        if None not in (self.bestAsk.size,self.bestBid.size):
            self._WAP = round(((self.bestBid.price * self.bestAsk.size) + (self.bestAsk.price * self.bestBid.size))/(self.bestBid.size + self.bestAsk.size),1)
            
        return self._WAP
        
        
    # @property
    # def Bid(self):
    #     return self._Bid
        
    # @Bid.setter
    # def Bid(self, bid):
    #     # if bid == 0 or bid is None:
    #         # self._Bid = 0  # Pull this from other rolling?
    #         # pass
    #     # self._Bid = bid

    #     # self.WAP = None # Invalidate previously Calculated WAP
    #     # return self._Bid
    #     pass





# Links to documentation pertaining 

# Time Modeling
# https://www.quantconnect.com/docs/v2/writing-algorithms/key-concepts/time-modeling/timeslices#03-Properties

# Futures - Handling Data
# https://www.quantconnect.com/docs/v2/writing-algorithms/securities/asset-classes/futures/handling-data


# To get the current Slice object, define an OnData method or use the CurrentSlice property of your algorithm (outside of the OnData method).
# If the data object doesn't contain any market data but it contains auxiliary data, the slice.ContainsKey(symbol) method can return true while slice[symbol] returns None.


class dotdict(dict):
    """dot.notation access to dictionary attributes"""
    __getattr__ = dict.get
    __setattr__ = dict.__setitem__
    __delattr__ = dict.__delitem__

# region imports
from AlgorithmImports import *
from AlphaModel import *
from DataClass import *
import math 

class NasdaqStrategy(QCAlgorithm):

    def Initialize(self):

        self.SetStartDate(2022,12,19)
        self.SetEndDate(2023,1,5)
        self.SetCash(100000)
        self.SetTimeZone(TimeZones.Chicago)  # If TimeZone is Chicago - Algo Time and Data End Time are same at 1800 hours
        self.SetWarmUp(timedelta(days=1))

        self.Data = {}
        # FutureSymbols = {'Lumber':Futures.Forestry.RandomLengthLumber,'Nasdaq':Futures.Indices.NASDAQ100EMini}
        FutureSymbols = {'Lumber':Futures.Forestry.RandomLengthLumber}

        # For Nasdaq we need DataMappingMode.OpenInterest or LastTradingDay - How to change?
        
        for key, value in FutureSymbols.items():
            
            dataMappingMode_ = DataMappingMode.FirstDayMonth if key == 'Lumber' \
            else DataMappingMode.OpenInterest if key == 'Nasdaq' else DataMappingMode.LastTradingDay

            # dataMappingMode_ = DataMappingMode.FirstDayMonth if key == 'Lumber' \
            # else DataMappingMode.LastTradingDay
            
            # BackwardsPanamaCanal

            future = self.AddFuture(value, Resolution.Tick,dataMappingMode = dataMappingMode_, contractDepthOffset=0, 
            dataNormalizationMode = DataNormalizationMode.Raw, extendedMarketHours=True, fillDataForward = True)
            self.Data[key] = DataClass(self, future) # Initiating DataClass for each Future & Passing our instance of QCAlgorithm Class
        
        
        # NOT SURE IF WORKING CORRECTLY
        # To trade the contract in the same time step you subscribe to the contract - Since GetLastKnowPrice makes a history request, it may slow the algorithm down.
        # self.SetSecurityInitializer(lambda x: x.SetMarketPrice(self.GetLastKnownPrice(x)))
        
        # Set alpha model - This also gives you access to the AlphaModel Instance
        # self.alpha = NasdaqAlpha(self.Data['Lumber'])  
        # self.SetAlpha(self.alpha)

        # Printing Yesterday data 
        # self.Schedule.On(self.DateRules.EveryDay(self.Data['Lumber'].Symbol), self.TimeRules.AfterMarketOpen(self.Data['Lumber'].Symbol, -1), self.beforeLumberOpen)
        self.Schedule.On(self.DateRules.EveryDay(), self.TimeRules.At(8,55), self.beforeLumberOpen)
        # self.TimeRules.BeforeMarketClose(symbol: Symbol, minutesBeforeClose: float = 0, extendedMarketOpen: bool = False)
        self.tradeCount = 0

        # Check Entry Condition only 30 seconds prior to Market Open
        self.entryTimeStart = self.Time.replace(hour=9, minute=0, second=0, microsecond=55000)
        self.entryTimeEnd   = self.Time.replace(hour=9, minute=1, second=0, microsecond=0)

       
    def beforeLumberOpen(self):

        if self.IsWarmingUp: return
        
        # Works - Get Yesterdays' data keeping in mind Rolled Over Contracts
        for symbol in self.Data.keys():
            # self.Debug(f"Yester Time:{self.Time}, {self.Data[symbol].Mapped}.yesterday:{self.Data[symbol].yesterday}")
            pass
        
        # Works
        # for security in self.ActiveSecurities.Values:
        #     self.Debug(f"self.Time:{self.Time} ActiveSecurity:{security}")


    def OnSecuritiesChanged(self, changes: SecurityChanges) -> None:
        
        # Only runs at the start of program not when Contract rolls 
        for security in changes.AddedSecurities:
            if len(changes.AddedSecurities) == 0: return
            self.Debug(f"In Main: OnSecuritiesChanged: Date:{self.Time}, security:{changes.AddedSecurities[0].Mapped}")

    def OnEndOfDay(self, symbol):
        # self.Debug(f"OnEndOfDay self.Time:{self.Time}, Symbol:{self.Data['Lumber'].Mapped.Value}")
        pass
                        
    def OnData(self, data):    

        # Useless
        # self.Debug("Hello")
        # for chain in data.FutureChains:
        #     contracts = list(chain.Value)
        #     self.Debug(f"Last contracts: {contracts}")
        #     for contract in contracts:
        #         self.Debug(f"contract:{contract}, Last Price: {contract.LastPrice}")
        
        # https://www.quantconnect.com/docs/v2/writing-algorithms/historical-data/warm-up-periods#03-Warm-Up-Vs-Reality
        #  In OnData: Don't run if the indicators aren't ready
        if self.IsWarmingUp: return
        
        # Below code important to compare Ticks with whats reported from DataClass
        tickTradeProps = ['LastPrice','Quantity']
        tickQuoteProps = ['BidPrice','BidSize','AskPrice','AskSize']
        # tickOIProps = ['Value']  # Not Getting Data for OpenInterest
        
        # for k in data.Ticks.Keys:
        #     self.Debug(f"keys:{k}")
        #     self.Debug(f"Underlying:{k.Underlying}")
        #     if data.Ticks.ContainsKey(k) and data.Ticks[k] is not None:
        #         self.Debug("yes")



        for security in data.Keys:

            # if self.Time < self.entryTimeStart or self.Time > self.entryTimeEnd: return
            # pass
            # self.Debug(f"security:{security}")
            # for prop in dir(security):
            #     value = getattr(security, prop)
            #     self.Debug(f"prop:{prop}, value:{value}")

            if security == self.Data['Lumber'].Mapped:
                self.Debug(f"security Matched:{security}")
                
            if data.Ticks.ContainsKey(security) and data.Ticks[security] is not None:
                ticks = data.Ticks[security]
                for tick in ticks:
                    previous_time = self.Time
                    tick_type = getTickType(tick.TickType)
                    if tick_type == 'Trade' and int(getattr(tick, 'Quantity')) != 0:

                        # This is for Trade Data
                        dic = {k:getattr(tick, k) for k in tickTradeProps if getattr(tick, k) is not None}
                        self.Debug(f"TradeTick:{self.Time} {dic}")
                        # self.Debug(f"self.Data['Lumber'].Underlying:{self.Data['Lumber'].Underlying} ")
                                             
                    # elif tick_type == 'Quote':
                        # This if for Last Bid & Ask
                        # dic = {k:getattr(tick, k) for k in tickQuoteProps if getattr(tick, k) is not None}
                        # self.Debug(f"*QuoteTick*: {self.Time}, security:{security},  {dic}")

                        # This if for Best Bid & Ask
                        # dic2 = {k:getattr(self.Securities[security], k) for k in tickQuoteProps} 
                        # self.Debug(f"+QuoteTick2+:{self.Time}, security:{security},  {dic2}")
                        
            
        for symbol in self.Data.keys():
            
            if self.Time < self.entryTimeStart or self.Time > self.entryTimeEnd: return

            # self.Debug(f"SYMBOL:{symbol}")            
            # self.Debug(f"++BestBid:{symbol}:{self.Time}:{self.Data[symbol].bestBid}")
            # self.Debug(f"^^BestAsk:{symbol}:{self.Time}:{self.Data[symbol].bestAsk}")
            
            self.Debug(f"**trade:{symbol}:{self.Time}:{self.Data[symbol].trade}")
            # self.Debug(f"**WAP:{symbol}:{self.Time}:{self.Data[symbol].WAP}")            
            # self.Debug(f"**ATR:{symbol}:{self.Time}:{self.Data[symbol].atr.Current.Value}")

            # self.Debug(f"**LastAsk:{self.Data[symbol].lastAsk}")            
            # self.Debug(f"**LastBid:{self.Data[symbol].lastBid}")

            # self.Debug(f" BestBid == LastBid:{self.Data[symbol].bestBid.price == self.Data[symbol].lastBid.price}")
            # self.Debug(f" BestAsk == LastAsk:{self.Data[symbol].bestAsk.price == self.Data[symbol].lastAsk.price}")
                    

        # Just to check if FuncSecuritySeeder gets us GetLastKnownPrices
        # for security in self.ActiveSecurities.Values:
            # self.Debug(f"Todays Close:{self.Securities[security.Symbol].Close}, Date:{self.Time}, security:{security}")
            # if security == 'LBS Y54QLJOCEN7L':
            
            # self.Debug(f"Mapped:{self.Data['Lumber'].Mapped}")
            # self.Debug(f"security == self.Data['Lumber'].Mapped:{security.Symbol == self.Data['Lumber'].Mapped}")
            # if security.Symbol == self.Data['Lumber'].Mapped:
            #     self.Debug(f"Todays Close:{self.Securities[security.Symbol].Close}, Date:{self.Time}, security:{security}")
            #     self.Debug("Yes")
            #     b = self.Securities[self.Data['Lumber'].Mapped]
            #     for prop in dir(b):
            #         value = getattr(b, prop)
            #         self.Debug(f"prop11:{prop}, value:{value}")  


        # WORKS - Remove from here since updated several times in a day
        # self.Debug(f"Todays ____Close:{self.Securities[self.Data['Lumber'].Mapped].Price}, Date:{self.Time}")
       
 
        
        if data.Ticks.ContainsKey(self.Data['Lumber'].Mapped):
            if data.Ticks[self.Data['Lumber'].Symbol] is not None:                
                self.Debug(f"**MAIN** Date:{self.Time}, MappedValue:{self.Data['Lumber'].Mapped.Value}, MappedID:{self.Data['Lumber'].Mapped.ID}, ") 


        #         b = data.Bars[self.Data['Lumber'].Mapped] 
        #         for prop in dir(b):
        #             value = getattr(b, prop)
        #             self.algo.Debug(f"prop:{prop}, value:{value}")         


        # Works
        # if data.Ticks.ContainsKey(self.Data['Lumber'].Symbol):
        #     if data.Ticks[self.Data['Lumber'].Symbol] is not None:                
                # self.Debug(f"**MAIN** Date:{self.Time}, MappedValue:{self.Data['Lumber'].Mapped.Value}, MappedID:{self.Data['Lumber'].Mapped.ID}, ")             

        # https://www.quantconnect.com/docs/v2/writing-algorithms/datasets/quantconnect/us-futures-security-master#05-Data-Point-Attributes        
        for symbol in self.Data.keys():
            if data.SymbolChangedEvents.ContainsKey(self.Data[symbol].Symbol):
                symbolChangedEvent = data.SymbolChangedEvents[self.Data[symbol].Symbol]
                self.Debug(f"In MAIN Symbol changed: {symbolChangedEvent.OldSymbol} -> {symbolChangedEvent.NewSymbol} \
                EndTime:{symbolChangedEvent.EndTime} DataType:{getDataType(symbolChangedEvent.DataType)}, Expiry: {self.Securities[self.Data[symbol].Mapped].Expiry}")

        

        # if data[self.Data['Lumber'].Mapped] is not None:
        #     self.Debug(f"data123Map: {data[self.Data['Lumber'].Mapped]}")


             
    # When your algorithm stops executing, LEAN calls the OnEndOfAlgorithm method.
    def OnEndOfAlgorithm(self) -> None:
        # self.Debug(f"self.Alpha.securities:{self.alpha.securities}")
        # self.Debug(f"self.count:{self.count}")
        self.Debug(f"self.TradeCount:{self.tradeCount}")
        


    def OnWarmUpFinished(self) -> None:
        self.Debug(f"Algorithm Ready@{self.Time}")
        pass


        
from AlgorithmImports import *
from AlgorithmImports import *

class Cash(PythonData):
    def GetSource(self,
         config: SubscriptionDataConfig,
         date: datetime,
         isLive: bool) -> SubscriptionDataSource:
        source="https://northamerica-northeast1-cross-asset-other-commodities.cloudfunctions.net/http_api?src=cwpenergy&data=cash&match=true"
        return SubscriptionDataSource(source, SubscriptionTransportMedium.RemoteFile,FileFormat.UnfoldingCollection)

    def Reader(self,
         config: SubscriptionDataConfig,
         line: str,
         date: datetime,
         isLive: bool) -> BaseData:
    
        objects = []
        data = json.loads(line)

        if len(data) == 0:
            return None

        for datum in data:
            
            index = Cash()
            index.Symbol = config.Symbol
            index.Time = datetime.strptime(datum["pubdate"], "%Y-%m-%d")
            index.EndTime = index.Time + timedelta(1)
            index.Value= float(datum["settle_fin"])
            index["code"] = str(datum["code"])
            index["year"] =int(datum["year"])
            index["month"]=int(datum["month"])
            index["week"]=int(datum["week"])
            index["date"]=datetime.strptime(datum["pubdate"], '%Y-%m-%d')
            index["cmeexpiry"]=datetime.strptime(datum["cme_exp"], '%Y-%m-%d')
            index["basis"]=float(datum["basis"])
            index["settlefin"]=float(datum["settle_fin"])
            index["settlephys"]=float(datum["settle_phys"])
            index["shortdesc"]=str(datum["shortdesc"])
            index["season"]=str(datum["season"])
            index["length"]=str(datum["length"])
            index["grade"]=str(datum["grade"])
            index["size"]=str(datum["size"])
            index["season"]=str(datum["season"])
            index["pricepoint"]=str(datum["pricepoint"])
            index["dest"]=str(datum["dest"])
            index["origin"]=str(datum["origin"])
            index["finish"]=str(datum["finish"])
            
            objects.append(index)

        return BaseDataCollection(objects[-1].EndTime, config.Symbol, objects)
      
class Settled(PythonData):
    ''' Settled Price Lumber'''

    def GetSource(self, config, date, isLive):
        source = "http://northamerica-northeast1-cross-asset-other-commodities.cloudfunctions.net/http_api?src=cwpenergy&data=settledprice&contract=1"
        return SubscriptionDataSource(source, SubscriptionTransportMedium.RemoteFile,FileFormat.UnfoldingCollection)

    def Reader(self, config, line, date, isLive):
        objects = []
        data = json.loads(line)
        for datum in data:
            try:
                index = Settled()
                index.Symbol = config.Symbol
                index.Time = datetime.strptime(datum["date"], "%Y-%m-%d")
                index.EndTime = index.Time + timedelta(1)
                index.Value=float(datum["settle_price"])
                index["date"]=datum["date"]
                index["ranking"] = str(datum["ranking"])
                index["openingprice"] =float(datum["opening_price"])
                index["dayhighprice"]=float(datum["day_high_price"])
                index["daylowprice"]=float(datum["day_low_price"])
                index["settleprice"]=float(datum["settle_price"])
                index["prevdayvol"]=float(datum["prev_day_vol"])
                index["prevdayopeninterest"]=float(datum["prev_day_open_interest"])
                index["mmy"]=datetime.strptime(datum["mmy"], '%Y-%m-%d')
                index["maturity"]=datetime.strptime(datum["maturity_date"], '%Y-%m-%d')

                objects.append(index)

            except Exception as e:
                # Do nothing, possible error in json decoding
                raise e

        return BaseDataCollection(objects[0].Time,objects[-1].EndTime, config.Symbol, objects)


class Western(PythonData):
   
    def GetSource(self, config, date, isLive):
        source = "https://northamerica-northeast1-cross-asset-other-commodities.cloudfunctions.net/http_api?src=lumberdatasources&data=wwpa&tbl=barometer_western"
        return SubscriptionDataSource(source, SubscriptionTransportMedium.RemoteFile,FileFormat.UnfoldingCollection)

    def Reader(self, config, line, date, isLive):
        objects = []
        data = json.loads(line)
        for datum in data:
            try:
                index = Western()
                index.Symbol = config.Symbol
                index.Time = datetime.strptime(datum["Timestamp"], "%Y-%m-%d")
                index.EndTime=index.Time + timedelta(1)
                index.Value=float(datum["Inventories"])
                index["date"]=datum["Timestamp"]
                index["Inventories"] = str(datum["Inventories"])
                index["Month"] =int(datum["Month"])
                index["Orders"]=int(datum["Orders"])
                index["Production"]=int(datum["Production"])
                index["Shipments"]=int(datum["Shipments"])
                index["Unfilled"]=int(datum["Unfilled"])
                index["Year"]=int(datum["Year"])
                index["Day"]=int(datum["Day"])
               
                objects.append(index)

            except Exception as e:
                # Do nothing, possible error in json decoding
                raise e

        return BaseDataCollection(objects[0].Time,objects[-1].Time, config.Symbol, objects)


class Disaggregated(PythonData):
    '''Disaggregated Custom Data Class'''
    def GetSource(self, config: SubscriptionDataConfig, date: datetime, isLiveMode: bool) -> SubscriptionDataSource:
        report="disaggregated_fut"
        source = f"https://northamerica-northeast1-cross-asset-other-commodities.cloudfunctions.net/http_api?src=lumberdatasources&data=cot&tbl=disaggregated&report={report}"
        return SubscriptionDataSource(source, SubscriptionTransportMedium.RemoteFile,FileFormat.UnfoldingCollection)

    def Reader(self, config: SubscriptionDataConfig, line: str, date: datetime, isLiveMode: bool) -> BaseData:

        objects = []
        data = json.loads(line)

        if len(data) == 0:
            return None

        for datum in data:

            try:
            
                index = Disaggregated()
                index.Symbol = config.Symbol
                index.Time = datetime.strptime(datum["Date"], "%Y-%m-%d")
                index.EndTime = index.Time + timedelta(1)
                index.Value= float(datum["M_Money_Positions_Long_All"])
                index["Date"]=datum["Date"]
                index["Mmoneypositionslongall"]=float(datum["M_Money_Positions_Long_All"])
                index["MMoneyPositionsLongOld"]=float(datum["M_Money_Positions_Long_Old"])
                index["MMoneyPositionsLongOther"]=float(datum["M_Money_Positions_Long_Other"])
                index["MMoneyPositionsShortAll"]=float(datum["M_Money_Positions_Short_All"])
                index["MMoneyPositionsShortOld"]=float(datum["M_Money_Positions_Short_Old"])
                index["MMoneyPositionsShortOther"]=float(datum["M_Money_Positions_Short_Other"])
                index["MMoneyPositionsSpreadAll"]=float(datum["M_Money_Positions_Spread_All"])
                index["MMoneyPositionsSpreadOld"]=float(datum["M_Money_Positions_Spread_Old"])
                index["MMoneyPositionsSpreadOther"]=float(datum["M_Money_Positions_Spread_Other"])
                index["NonReptPositionsLongAll"]=float(datum["NonRept_Positions_Long_All"])
                index["NonReptPositionsLongOld"]=float(datum["NonRept_Positions_Long_Old"])
                index["NonReptPositionsLongOther"]=float(datum["NonRept_Positions_Long_Other"])
                index["NonReptPositionsShortAll"]=str(datum["NonRept_Positions_Short_All"])
                index["NonReptPositionsShortOld"]=float(datum["NonRept_Positions_Short_Old"])
                index["NonReptPositionsShortOther"]=float(datum["NonRept_Positions_Short_Other"])
                index["OpenInterestAll"]=float(datum["Open_Interest_All"])
                index["OpenInterestOld"]=float(datum["Open_Interest_Old"])
                index["OpenInterestOther"]=float(datum["Open_Interest_Other"])
                index["OtherReptPositionsLongAll"]=float(datum["Other_Rept_Positions_Long_All"])
                index["OtherReptPositionsLongOld"]=float(datum["Other_Rept_Positions_Long_Old"])
                index["OtherReptPositionsShortAll"]=float(datum["Other_Rept_Positions_Short_All"])
                index["OtherReptPositionsShortOld"]=float(datum["Other_Rept_Positions_Short_Old"])
                index["OtherReptPositionsShortOther"]=float(datum["Other_Rept_Positions_Short_Other"])
                index["OtherReptPositionsSpreadAll"]=float(datum["Other_Rept_Positions_Spread_All"])
                index["OtherReptPositionsSpreadOld"]=float(datum["Other_Rept_Positions_Spread_Old"])
                index["OtherReptPositionsSpreadOther"]=float(datum["Other_Rept_Positions_Spread_Other"])
                index["ProdMercPositionsLongAll"]=float(datum["Prod_Merc_Positions_Long_All"])
                index["ProdMercPositionsLongOld"]=float(datum["Prod_Merc_Positions_Long_Old"])
                index["ProdMercPositionsLongOther"]=float(datum["Prod_Merc_Positions_Long_Other"])
                index["ProdMercPositionsShortAll"]=float(datum["Prod_Merc_Positions_Short_All"])
                index["ProdMercPositionsShortOld"]=float(datum["Prod_Merc_Positions_Short_Old"])
                index["ProdMercPositionsShortOther"]=float(datum["Prod_Merc_Positions_Short_Other"])
                index["SwapPositionsLongOther"]=float(datum["Swap_Positions_Long_Other"])

                objects.append(index)

            except Exception as e:
                # Do nothing, possible error in json decoding
                raise e

        return BaseDataCollection(objects[-1].EndTime, config.Symbol, objects)


mapped_attr ={

    "DISAGGREGATED.Disaggregated":
    [
    "Date",
    "Mmoneypositionslongall",
    "Mmoneypositionslongold",
    "Mmoneypositionslongother",
    "Mmoneypositionsshortall",
    "Mmoneypositionsshortold",
    "Mmoneypositionsshortother",
    "Mmoneypositionsspreadall",
    "Mmoneypositionsspreadold",
    "Mmoneypositionsspreadother",
    "Nonreptpositionslongall",
    "Nonreptpositionslongold",
    "Nonreptpositionslongother",
    "Nonreptpositionsshortall",
    "Nonreptpositionsshortold",
    "Nonreptpositionsshortother",
    "Openinterestall",
    "Openinterestold",
    "Openinterestother",
    "Otherreptpositionslongall",
    "Otherreptpositionslongold",

    "Otherreptpositionsshortall",
    "Otherreptpositionsshortold",
    "Otherreptpositionsshortother",
    "Otherreptpositionsspreadall",
    "Otherreptpositionsspreadold",
    "Otherreptpositionsspreadother",
    "Prodmercpositionslongall",
    "Prodmercpositionslongold",
    "Prodmercpositionslongother",
    "Prodmercpositionsshortall",
    "Prodmercpositionsshortold",
    "Prodmercpositionsshortother",
    "Swappositionslongother"],
      

    'WESTERN.Western':['Date',
    'Inventories',
    'Month',
    'Orders',
    'Production',
    'Shipments',
    'Unfilled',
    'Year',
    'Day'] ,

    "CASH.Cash":[
    'Code',
    'Year',
    'Month',
    'Week',
    'Date',
    "Cmeexpiry",
    "Basis",
    "Settlefin",
    "Settlephys",
    "Shortdesc",
    "Season",
    "Length",
    "Grade",
    "Size",
    "Season",
    "Pricepoint",
    "Dest",
    "Origin",
    "Finish",
   
   ],


   "SETTLED.Settled":[
    'Date',
    'Ranking',
    'Dayhighprice',
    'Daylowprice',
    'Settleprice',
    'Openingprice',
    'Prevdayvol',
    'Prevdayopeninterest',
    'Mmy',
    'Maturity',
    ],


}



#function to create dict from objects
def create_dict(obj):

    attr=mapped_attr[str(obj.Symbol)]

    dct={}
    dct["Symbol"]=str(obj.Symbol)
    dct["Value"]=float(obj.Value)   
    
    for at in attr:
        dct[at]=getattr(obj,at)

    return dct

from AlgorithmImports import *
import math
from collections import deque
from HelperFuncs import *
from functools import wraps
import pandas as pd
import numpy as np
from Custom import create_dict
from QuantConnect.Data.Consolidators import CalendarInfo


class DataClass:

    def __init__(self, Algorithm, continuous_contract,customData=None):
        
        self.algo = Algorithm # This is Our Instance of QCAlgorithm
        self.continuous_contract = continuous_contract
        self.Symbol = self.continuous_contract.Symbol 
        self.tradePriceUpdated = False

        #custom data
        self.customData=customData
        #check if there is custom data for security
        if self.customData:
            #dict to hold custom data points
            self.customDataDict={}
            
            #create empty key for each security value is none for all
            # dict values are updated in Update function below
            for custom in self.customData:
                self.customDataDict[str(custom)]=None

        # Contract Properties
        self.tickSize = self.continuous_contract.SymbolProperties.MinimumPriceVariation

        # Bids, Asks, Trade Prices for both Mapped & Continuous        
        self._bestBidContinuous = dotdict(dict.fromkeys(['time', 'price', 'size']))
        self._bestBidMapped = dotdict(dict.fromkeys(['time', 'price', 'size']))

        self._bestAskContinuous = dotdict(dict.fromkeys(['time','price', 'size']))
        self._bestAskMapped = dotdict(dict.fromkeys(['time','price', 'size']))

        self._tradeContinuous = dotdict(dict.fromkeys(['time','price', 'size']))
        self._tradeMapped = dotdict(dict.fromkeys(['time','price', 'size']))

        self._WAPMapped = None            
        self._yesterdayPercentChange = {}
        self.RollingYesterdayPercentChange = deque(maxlen=6*60*60) # 6 hours * 60 minutes * 60 seconds

        # Market Timings
        self._currentOpen = None
        self._currentClose = None
        self._nextOpen = None

        # Indicators
        self.atrPeriod = 2
        self._ContinuousATR = AverageTrueRange(self.atrPeriod, MovingAverageType.Simple) 
        self._MappedATR = AverageTrueRange(self.atrPeriod, MovingAverageType.Simple)

        self._ContinuousMA2 = SimpleMovingAverage(2)
        self._MappedMA2 = SimpleMovingAverage(2)
        self._MappedYesterday = SimpleMovingAverage(1)
        self._ContinuousYesterday = SimpleMovingAverage(1)

        # History DFs - Using so only call History when not already called before for other Indicators
        self.mappedTradeDailyHistory = None
        self.mappedTradeHistory = None        
        self.continuousDailyHistory = None
        self.continuousHistory = None
        
        # Schedulers

        # Update Yesterday Price 10 minutes before Market Opens - Can even use 'LBS' in TimeRules.AfterMarketOpen
        self.algo.Schedule.On(self.algo.DateRules.EveryDay(self.Symbol), self.algo.TimeRules.AfterMarketOpen(self.Symbol, -10), self.updateDailyIndicators)
        
        # Reset BBO dicts 10 minutes after market close
        self.algo.Schedule.On(self.algo.DateRules.EveryDay(self.Symbol), self.algo.TimeRules.BeforeMarketClose(self.Symbol, -10), self.resetTickDict)

        # Capture Nasdaq %age Change before Lumber Opens  
        # self.algo.Schedule.On(self.algo.DateRules.EveryDay(self.Symbol), self.algo.TimeRules.At(9,59,51),self.updateRollingSeries)
        # Capture %age Change for Nasdaq & Lumber during Lumber Market Hours
        # self.algo.Schedule.On(self.algo.DateRules.EveryDay(self.Symbol), self.algo.TimeRules.Every(timedelta(seconds=1)),self.updateRollingSeries)

        # Calls the property (self.tradeContinuous/tradeMapped) everytime there is a trade so as to update it. 
        self.tradeEvent = TickConsolidator(1)
        self.tradeEvent.DataConsolidated += self.updateTradePrice
        # Not sure if using self.Symbol will only show Trades for front Month or for all Mapped symbols
        self.algo.SubscriptionManager.AddConsolidator(self.Symbol, self.tradeEvent)

    # Change to self.tradeMapped once we find a better way to calculate it rather than using History
    def updateTradePrice(self, sender: object, Tbar: TradeBar) -> None:
        if self.algo.Time < self.algo.signalStartTime or self.algo.Time > self.algo.signalEndTime: return                
        
        self.tradePriceUpdated = True
        self._yesterdayPercentChange = {}
        # self.tradeMapped
        self.tradeContinuous

    @property
    def yesterdayPercentChange(self):
        if self.algo.Time < self.algo.signalStartTime or self.algo.Time > self.algo.signalEndTime: return    

        # Change self._tradeContinuous with self._tradeMapped once we have more efficient way to get self._tradeMapped rather than using History
        # Can use continuous incase using DataNormalizationMode.Raw
        if None not in (self._tradeContinuous.price,self._MappedYesterday.Current.Value) and self.tradePriceUpdated:
            self._yesterdayPercentChange[self._tradeContinuous.time] = round(((self._tradeContinuous.price - self._MappedYesterday.Current.Value)/self._MappedYesterday.Current.Value)*100,2)                   
            self.tradePriceUpdated = False # Ensures that unless new Trade value is updated in UpdateTradePrice func, no need to recalculate
        return self._yesterdayPercentChange        
    
    # To be removed - but first see all other varibles, their uses etc. Compare output with the One in AlphaModel etc. 
    # Moved to Alpha Model - call it updateModelData or percentChangeSeconds
    def updateRollingSeries(self):  

        # self.yesterdayPercentChange only updates during Lumber hours but the above statement is used for 'else' below:
        if self.yesterdayPercentChange:
            
            # Capctures only when data available
            self.RollingYesterdayPercentChange.appendleft(self._yesterdayPercentChange)                                 
            
            # Captures every Second
            self.algo.percentChangeDF.loc[self.algo.Time,self.Symbol] = list(self._yesterdayPercentChange.values())[0]            
            self.algo.Debug(f"{self.algo.Time}:{self.algo.percentChangeDF.tail(1)}")
            
            # Get Latest RollingYesterdayPercentChange fo respective Symbols - only updates when yesterdayPercentChange exists for a symbol
            self.algo.Debug(f"##{self.algo.Time}:{self.Symbol}:{self.RollingYesterdayPercentChange[0]}")
            self.algo.Debug(f"***{self.algo.Time}:{self.Symbol}:{list(self.RollingYesterdayPercentChange[0].values())[0]}")

            # Max, Min and rolling Max for RollingYesterdayPercentChange
            # self.algo.Debug(f"MAX{self.algo.Time}:{self.Symbol}:{max(max(d.values()) for d in self.RollingYesterdayPercentChange)}")           
            # Find the maximum value over the last n data points (in this case, n=3)
            # max_n_value = max(max(d.values()) for d in list(self.RollingYesterdayPercentChange)[:3])
            # self.algo.Debug(f"MIN{self.algo.Time}:{self.Symbol}:{min(min(d.values()) for d in self.RollingYesterdayPercentChange)}")

            # To Do Adjustments to percentChangeDF    
            # self.algo.percentChangeDF= self.algo.percentChangeDF.fillna(method='ffill')
            # corr = self.algo.percentChangeDF['/NQ'].corr(self.algo.percentChangeDF['/LBS'])
            # self.algo.Debug(f"{self.algo.Time}:Correlation{corr}")


        




    # Gets Updated from main
    def Update(self, data):

        # below code updates the custom data dict for particulat data 
        
        #check if custom data is avaliable for security
        if self.customData:
            # Iterate through custom data list
            for key in self.customData:
               #check if data contains particular custom data symbol
               if data.ContainsKey(key):

                    #check if there is any earlier update in custom data dict for this custom symbol
                    if self.customDataDict[str(key)] == None:
                        #set last to None
                        last=None
                    
                    else:
                        #set last to current data in dict
                        last=self.customDataDict[str(key)]["data"]
                    
                    # update the dict with new data
                    # value for dict contains a dict with 
                    # update_at:time at which data is received
                    # data:updated data point for symbol
                    # last:previous data point for symbol
                    self.customDataDict[str(key)]={"updated_at":self.algo.Time,"data":create_dict(data[key]), "last":last }
                    self.algo.Debug(f"{self.customDataDict}")






        if data.SymbolChangedEvents.ContainsKey(self.Symbol):
            # Reset ONLY Mapped Indicators (ones based on Mapped contracts) - This will load History when contract switches
            self._MappedATR.Reset() 
            self._MappedMA2.Reset() 
            self._MappedYesterday.Reset() 
            
            # NOT letting register New Symbol - asking to subscribe to it first.
            # symbolChangedEvent = data.SymbolChangedEvents[self.Symbol]
            # self.algo.SubscriptionManager.AddConsolidator(symbolChangedEvent.NewSymbol, self.tradeEvent)
            # self.algo.SubscriptionManager.RemoveConsolidator(symbolChangedEvent.OldSymbol, self.tradeEvent)
        
        # if self.algo.Time <= self.algo.signalStartTime or self.algo.Time >= self.algo.signalEndTime: return


    @property
    def ATR(self):
        # self.algo.Debug(f"self._MappedATR.IsReady:{self._MappedATR.IsReady} at {self.algo.Time}")
        # self.algo.Debug(f"# of samples{self._MappedATR.Samples}")
        return self._MappedATR.Current.Value


    # Resetting Best Bid/Ask at MarketClose
    def resetTickDict(self):

        # 10 minutes after close
        # NG - 17:10 on 27th (Start date on 26th) 
        # LBS - 16:15 on 27th (Start date on 26th)
        # self.algo.Debug(f"symbol:{self.Symbol},{self.algo.Time}")

        # Emoty the deque
        self.RollingYesterdayPercentChange = deque(maxlen=6*60*60) # 6 hours * 60 minutes * 60 seconds
        
        self._bestAskContinuous = self._bestAskContinuous.fromkeys(self._bestAskContinuous, None)
        self._bestBidContinuous = self._bestBidContinuous.fromkeys(self._bestBidContinuous, None)

        self._bestAskMapped = self._bestAskMapped.fromkeys(self._bestAskMapped, None)
        self._bestBidMapped = self._bestBidMapped.fromkeys(self._bestBidMapped, None)

        self._tradeContinuous = self._tradeContinuous.fromkeys(self._tradeContinuous, None)
        self._tradeMapped = self._tradeMapped.fromkeys(self._tradeMapped, None)

        self._WAPMapped = None
        self._yesterdayPercentChange = {}

        self._currentOpen = None
        self._currentClose = None
        self._nextOpen = None
        

    def getMappedTradeHistory(self,days):
        history = self.algo.History[TradeBar](self.Mapped, days, Resolution.Daily)
        # https://www.quantconnect.com/docs/v2/writing-algorithms/historical-data/rolling-window#09-Cast-to-Other-Types
        return self.algo.PandasConverter.GetDataFrame[TradeBar](list(history)) 


    def getContinuousHistory(self,days):
        return self.algo.History(tickers=[self.Symbol], 
        start=self.startBDate(self.algo.Time,days).date(), # Using self.startBDate since if we simply use startdate minus timedelta(self.atrperiod), it doesn't keep holidays in count
        end=self.algo.Time,
        resolution=Resolution.Daily, 
        fillForward=True, 
        extendedMarket=True, 
        dataMappingMode=DataMappingMode.FirstDayMonth,
        dataNormalizationMode = self.algo.DataNormalizationMode, # Got this from Main
        contractDepthOffset=0)


    def updateDailyIndicators(self):        
        
        # NG - 9:20 on 27th (Start date on 26th)
        # LBS - 9:50 on 27th (Start date on 26th)
        # self.algo.Debug(f"symbol:{self.Symbol},{self.algo.Time}")

        # Resetting Every Day
        self.mappedTradeDailyHistory = None
        self.mappedTradeHistory = None        
        self.continuousDailyHistory = None
        self.continuousHistory = None
        
        self.updateDailyMappedIndicators(indicator = self._MappedATR, period = self.atrPeriod+1, updateType =  'TradeBar')        
        self.updateDailyMappedIndicators(indicator = self._MappedYesterday, period = 1, updateType = 'DailyClose')
        self.updateDailyMappedIndicators(indicator = self._MappedMA2, period = 2, updateType = 'DailyClose')
        
        self.updateDailyContinuousIndicators(indicator = self._ContinuousATR, period = self.atrPeriod+1, updateType =  'TradeBar')
        self.updateDailyContinuousIndicators(indicator = self._ContinuousMA2, period = 2, updateType = 'DailyClose')       
        self.updateDailyContinuousIndicators(indicator = self._ContinuousYesterday, period = 1, updateType = 'DailyClose') 


    def updateDailyMappedIndicators(self, indicator, period, updateType): 
        
        # MAPPED INDICATORS 
        if self.Mapped is not None:
            # ATR Update - # All Manual Indicators Updated Manually Here
            if not indicator.IsReady: # Runs (1) At start (2) When contract Rolls
                if self.mappedTradeHistory is None or len(self.mappedTradeHistory) < period:
                    self.mappedTradeHistory = self.getMappedTradeHistory(days = period)                
                
                mappedTradeHistory_ = self.mappedTradeHistory[-period:]
                
                for bar in mappedTradeHistory_.itertuples():
                    if updateType == 'TradeBar':
                        tradebar = TradeBar(bar.Index[1], self.Mapped, float(bar.open), float(bar.high), float(bar.low), float(bar.close), float(bar.volume))
                        # self.algo.Debug(f"Bar1:{tradebar} @ {bar.Index[1]}")
                        indicator.Update(tradebar)
                        # self.algo.Debug(f"1stManual:{tradebar},{bar.Index[1]}")
                    elif updateType == 'DailyClose':
                        indicator.Update(bar.Index[1], bar.close)

            else: # Indicator is ready so just updating it for the day.   
                if self.mappedTradeDailyHistory is None or self.mappedTradeDailyHistory.empty:
                    self.mappedTradeDailyHistory = self.getMappedTradeHistory(days = 1) 
                               
                for bar in self.mappedTradeDailyHistory.itertuples():   
                    if updateType == 'TradeBar':                 
                        tradebar = TradeBar(bar.Index[1], self.Mapped, float(bar.open), float(bar.high), float(bar.low), float(bar.close), float(bar.volume))
                        indicator.Update(tradebar)
                        # self.algo.Debug(f"2ndManual:{tradebar},{bar.Index[1]}")
                    elif updateType == 'DailyClose':
                        indicator.Update(bar.Index[1], bar.close)
                        

    def updateDailyContinuousIndicators(self, indicator, period, updateType): 

        # CONTINUOUS INDICATORS
        if self.Symbol is not None:

            # ATR Update - # All Manual Indicators Updated Manually Here
            if not indicator.IsReady: # Runs (1) At start OnLY     
                if self.continuousHistory is None or len(self.continuousHistory) < period:
                    self.continuousHistory = self.getContinuousHistory(days=period)
                
                continuousHistory_ = self.continuousHistory[-period:]

                for bar in continuousHistory_.itertuples(): 
                    if updateType == 'TradeBar':
                        tradebar = TradeBar(bar.Index[2], bar.Index[1], float(bar.open), float(bar.high), float(bar.low), float(bar.close), float(bar.volume))
                        indicator.Update(tradebar)
                    elif updateType == 'DailyClose':
                        indicator.Update(bar.Index[2], bar.close)
                        
                    # self.algo.Debug(f"1stAuto:{tradebar},{bar.Index[2]}")


            else: # Indicator is ready so just updating it for the day.     
                if self.continuousDailyHistory is None or self.continuousDailyHistory.empty:
                    self.continuousDailyHistory = self.getContinuousHistory(days=1)                            
                
                for bar in self.continuousDailyHistory.itertuples():
                    if updateType == 'TradeBar':
                        # self.algo.Debug(f"2ndAuto:{tradebar},{bar.Index[2]}")
                        tradebar = TradeBar(bar.Index[2], bar.Index[1], float(bar.open), float(bar.high), float(bar.low), float(bar.close), float(bar.volume))
                        indicator.Update(tradebar)
                    elif updateType == 'DailyClose':
                        indicator.Update(bar.Index[2], bar.close)


    @property
    def bestBidMapped(self):

        # Only updating if Bid price and Size changed so as to keep the original Time of last update
        if int(getattr(self.algo.Securities[self.Mapped], 'BidSize')) != 0 and not self.alreadyUpdated('BidPrice', 'BidSize', self._bestBidMapped):
            for key,prop in zip(list(self._bestBidMapped.keys()),['LocalTime','BidPrice', 'BidSize']):
                setattr(self._bestBidMapped,str(key),getattr(self.algo.Securities[self.Mapped], prop))

        return self._bestBidMapped


    @property
    def bestAskMapped(self):        
        # Only updating if Ask price and Size changed so as to keep the original Time of last update
        if int(getattr(self.algo.Securities[self.Mapped], 'AskSize')) != 0 and not self.alreadyUpdated('AskPrice', 'AskSize', self._bestAskMapped):
            for key,prop in zip(list(self._bestAskMapped.keys()),['LocalTime','AskPrice', 'AskSize']):
                setattr(self._bestAskMapped,str(key),getattr(self.algo.Securities[self.Mapped], prop))
        return self._bestAskMapped

    
    @property
    def bestBidContinuous(self):
        # Only updating if Bid price and Size changed so as to keep the original Time of last update
        if int(getattr(self.algo.Securities[self.Symbol], 'BidSize')) != 0 and not self.alreadyUpdated('BidPrice', 'BidSize', self._bestBidContinuous):
            for key,prop in zip(list(self._bestBidContinuous.keys()),['LocalTime','BidPrice', 'BidSize']):
                if key == 'price':
                    setattr(self._bestBidContinuous,str(key),round(getattr(self.algo.Securities[self.Symbol], prop),1))
                else:
                    setattr(self._bestBidContinuous,str(key),getattr(self.algo.Securities[self.Symbol], prop))
        return self._bestBidContinuous



    @property
    def bestAskContinuous(self):       
        # Only updating if Ask price and Size changed so as to keep the original Time of last update
        if int(getattr(self.algo.Securities[self.Symbol], 'AskSize')) != 0 and not self.alreadyUpdated('AskPrice', 'AskSize', self._bestAskContinuous):
            for key,prop in zip(list(self._bestAskContinuous.keys()),['LocalTime','AskPrice', 'AskSize']):
                if key == 'price':
                    setattr(self._bestAskContinuous,str(key),round(getattr(self.algo.Securities[self.Symbol], prop),1))
                else:
                    setattr(self._bestAskContinuous,str(key),getattr(self.algo.Securities[self.Symbol], prop))
        return self._bestAskContinuous

   
    @property
    def tradeContinuous(self):        
        if self.algo.CurrentSlice.Ticks.ContainsKey(self.Symbol) and self.algo.CurrentSlice.Ticks[self.Symbol] is not None:
            ticks = self.algo.CurrentSlice.Ticks[self.Symbol]
            for tick in ticks:
                tick_type = getTickType(tick.TickType)
                if tick_type == 'Trade' and int(getattr(tick, 'Quantity')) != 0 and int(getattr(tick, 'Price')) is not None:
                    for key,prop in zip(list(self._tradeContinuous.keys()),['Time','Price','Quantity']):
                        if key == 'price':
                            setattr(self._tradeContinuous,str(key),round(getattr(tick, prop),1))
                        else:
                            setattr(self._tradeContinuous,str(key),getattr(tick, prop))
                else:
                    pass
        return self._tradeContinuous

    @property
    def tradeMapped(self):

        # Keep these seconds equal to scheduler ??
        # history = self.algo.History[Tick](self.Mapped, timedelta(seconds=60), Resolution.Tick)

        history = self.algo.History[Tick](self.Mapped, timedelta(microseconds=1), Resolution.Tick)
        history = self.algo.PandasConverter.GetDataFrame[Tick](list(history)) 
        for bar in history.itertuples():
            if hasattr(bar,'quantity'):
                if getattr(bar,'quantity') >= 1:
                    self._tradeMapped['time'] = bar.Index[1].to_pydatetime().strftime('%Y-%m-%d %H:%M:%S.%f')
                    self._tradeMapped['price'] = bar.lastprice
                    self._tradeMapped['size'] = bar.quantity

        return self._tradeMapped


    @property
    def WAPMapped(self):
        if None not in (self.bestAskMapped.size,self.bestBidMapped.size):
            self._WAPMapped = round(((self.bestBidMapped.price * self.bestAskMapped.size) + (self.bestAskMapped.price * self.bestBidMapped.size))/(self.bestBidMapped.size + self.bestAskMapped.size),1)
            
        return self._WAPMapped


    @property
    def currentOpen(self):
        marketHoursString = str(self.algo.Securities[self.Mapped].Exchange.Hours.GetMarketHours(self.algo.Time))
        marketOpenString = marketHoursString.split("Market: ")[2].split(" | ")[0].split("-")[0].strip()
        marketOpen = datetime.strptime(marketOpenString, '%H:%M:%S').time()
        # Added extra hour to Make it work with NewYork Time/ Algorithm Time
        self._currentOpen = datetime.combine(self.algo.Time.date(),marketOpen) + timedelta(hours=1) 
        return self._currentOpen 

    @property
    def currentClose(self):
        marketHoursString = str(self.algo.Securities[self.Mapped].Exchange.Hours.GetMarketHours(self.algo.Time))
        marketCloseString = marketHoursString.split("Market: ")[2].split(" | ")[0].split("-")[1].strip()
        marketClose = datetime.strptime(marketCloseString, '%H:%M:%S').time()
        # Added extra hour to Make it work with NewYork Time/ Algorithm Time
        self._currentClose = datetime.combine(self.algo.Time.date(),marketClose) + timedelta(hours=1)     
        return self._currentClose 


    # @property
    # def currentOpen(self):
    #     hours = self.algo.Securities[self.Mapped].Exchange.Hours
    #     # Added extra hour to Make it work with NewYork Time/ Algorithm Time
    #     # False is for not including extended Market Hours
    #     self._currentOpen = hours.GetNextMarketOpen(self.algo.Time + timedelta(hours=24), False) + timedelta(hours=1) 
    #     return self._currentOpen  # 
    

    # Needs to be Checked
    @property
    def nextOpen(self):
        hours = self.algo.Securities[self.Mapped].Exchange.Hours
        # Added extra hour to Make it work with NewYork Time/ Algorithm Time
        self._nextOpen = hours.GetNextMarketOpen(self.algo.Time, False) + timedelta(hours=1) 
        return self._nextOpen  # 


    # RealOnly RealTime 
    @property
    def Mapped(self):
        return getattr(self.continuous_contract, 'Mapped')

    @property
    def Canonical(self):
        return getattr(self.Symbol, 'Canonical')


    @property
    def custom(self):
        if self.customData:
            return self.customDataDict
        else:
            return None
    # @property
    # def timezone(self):
    #     return self.algo.MarketHoursDatabase.GetDataTimeZone(Market.CME, self.Symbol, SecurityType.Future)


    # Genric Functions - Combine both together
    def businessDay(self,dt):
        return 1 if getattr(self.algo.TradingCalendar.GetTradingDay(dt),'BusinessDay') else 0
    
    def startBDate(self,end_dt, lag):  
        count = 0      
        start_dt = end_dt 
        while count!= lag:
            start_dt -= timedelta(days=1)
            count += self.businessDay(start_dt)
        return start_dt
    
    
    # This is not entirely correct as someone may have canceled and others may have added a bid with net being affect 0 
    def alreadyUpdated(self, price, size, dictionary):
        return getattr(self.algo.Securities[self.Mapped], price) == dictionary.price and getattr(self.algo.Securities[self.Mapped], size) == dictionary.size
        

class dotdict(dict):
    """dot.notation access to dictionary attributes"""
    __getattr__ = dict.get
    __setattr__ = dict.__setitem__
    __delattr__ = dict.__delitem__

from AlgorithmImports import *
# https://github.com/QuantConnect/Lean/blob/master/Common/Orders/OrderTypes.cs#L87

def get_order_status_name(index):
   return { 
       0: 'New',
       1: 'Submitted',
       2: 'PartiallyFilled',
       3: 'Filled',
       4: 'None',
       5: 'Canceled',
       6: 'None',
       7: 'Invalid',
       8: 'CancelPending',
       9: 'UpdateSubmitted '
    }[index]


def get_order_direction_name(index):
    return {
        0: 'Buy',
        1: 'Sell',
        2: 'Hold',
    }[index]


def get_order_type_name(index):
    return {
        0: 'Market',
        1: 'Limit',
        2: 'StopMarket',
        3: 'StopLimit',
        4: 'MarketOnOpen',
        5: 'MarketOnClose',
        6: 'OptionExercise',
        7: 'LimitIfTouched'
    }[index]

# This dictionary is Used to detrmine the Upside of Lumber Prices versus Yesterday's Settle:
# From there we assume, TP, 
def get_upside(index):
    return {
        '1.5%':10,
        '2%':12,
        '3%':15
    }[index]


# https://www.quantconnect.com/docs/v2/writing-algorithms/securities/key-concepts
def getSecurityType(index):
       return {
        0: 'Base',
        1: 'Equity',
        2: 'Option',
        3: 'Commodity',
        4: 'Forex',
        5: 'Future',
        6: 'Cfd',
        7: 'Crypto',
        8: 'FutureOption',
        9: 'Index',
        10: 'IndexOption',
        'CryptoFuture': 'CryptoFuture'
        
    }[index]

def getDataType(index):
       return {
        0: 'Base',
        1: 'TradeBar',
        2: 'Tick',
        3: 'Auxiliary', # Data associated with an instrument
        4: 'QuoteBar',
        5: 'OptionChain',
        6: 'FuturesChain'
    }[index]

def getTickType(index):
       return {
        0: 'Trade',
        1: 'Quote',
        2: 'OpenInterest' # Open Interest type tick object (for options, futures)
    }[index]


# See Notes - Higher the NasdaqYesterday% Change, Higher confidence in Alpha Insight
# Copied from GPT - default_score is if PercentChange is below 1%
def NasdaqPercentChangeToConfidence(PercentChange, default_score=0.15):
    scores = {0: 0.15, 1: 0.30, 2: 0.45, 3: 0.6, 4: 0.75, 5: 0.9}
    abs_percent_change = abs(PercentChange)
    return scores.get(next((k for k in reversed(sorted(scores.keys())) if abs_percent_change >= k), None), default_score)

# Func - Returns the product of 2 numbers
NasdaqPercentChangeToMagnitude = lambda PercentChange, multiplier: PercentChange * multiplier

# List - Insight Properties
insight_Properties = ['Symbol','GeneratedTimeUtc','CloseTimeUtc','Direction','EstimatedValue','Weight','Id','Magnitude','Confidence','Period','Score','IsActive','IsExpired','SourceModel']      


# This Class is used to run a function after a certain interval of time
class RunAfterInterval:
    def __init__(self, interval_seconds):
        self.interval_seconds = interval_seconds
        self.last_run_time_by_key = {}

    def run(self, key, current_time):
        last_run_time = self.last_run_time_by_key.get(key)
        if last_run_time is not None:
            elapsed_time_since_last_run = current_time - last_run_time
            if elapsed_time_since_last_run < timedelta(seconds=self.interval_seconds):
                return False
        self.last_run_time_by_key[key] = current_time
        return True
#region imports
from AlgorithmImports import *
#endregion

# NEED HELP
# 1. Done: Get Yesterday's Close Price for Lumber Contracts - 2 ways to acheive. 
        # 1a. Use history fuction everyday - Works Fine
        # 2a. Use Identity Indicator - Its partially correct when using Resolution.Daily but incorrect On days when the contract rolls.
        # i.e. it still shows yesterday's close for previous contract on RollOverDays. Also would require a day or 2 of warmuP
        #   Secondly, if I use Resolution.Tick in Lumber Contract definition, it shows 2 days ago prices when called before market Open


# 3. Find trading days when both Nasdaq and Lumber were open (keeping in mind yesterday's data) and only trade when both are available atleast for backtesting
# https://www.quantconnect.com/docs/v2/writing-algorithms/securities/asset-classes/futures/market-hours/cme/mnq
# https://www.quantconnect.com/docs/v2/writing-algorithms/securities/asset-classes/futures/market-hours/cme/lbs

# 4. Nasdaq data not available on first day since it gets added at 2300 hours while Lumber gets added at 00 hours.

# 5. Didn't check if Nasdaq prices are fine or if Nasdaq yesterday contract prices match the rolled over contract

# 6. Whats the difference between getting Bids & Asks from ticks like below - Is the first one Time&Sales (for quotes) while second one gives Best Bids/Asks?
                # ticks = self.algo.CurrentSlice.Ticks[self.Symbol]
                # for tick in ticks:
                #         tick.AskPrice 

        # versus getting bids/asks from self.Securities[security].AskPrice

# 7.         # Questions on quoteEvent (TickQuoteBarConsolidator): 
        # 1. Does this tell you total number of best bids/asks at any moment in time or are these just seperate orders and their corresponding sizes? 
        # 2. Quotes are for mapped rolled contracts?
        # 3. Also these are not all quotes as placed but only Top Bids/Asks. So for example, they are only generated if someone has topped a previous bid/offer?






# Some other references:

# 1. Tick Data related
        # https://www.quantconnect.com/docs/v2/writing-algorithms/securities/asset-classes/futures/handling-data#07-Ticks
        # LastPrice - Alias for "Value" - the last sale for this asset. Same as Price 
        # IsFillForward - True if this is a fill forward piece of data 
        # 'Time','EndTime' - for Trade Tick is same as self.Time but for time use this property only
        # 'Symbol' - Not available but has other sub properties 
        # ,'SaleCondition','Suspicious' - Not relevant
        
        # tickTradeProps = ['Price','Quantity']
        # tickQuoteProps = ['BidPrice','BidSize','AskPrice','AskSize']
        # tickOIProps = ['Value']  # Not Getting Data for OpenInterest

# Other Notes to self:
        # 1. Since Resolution is for Ticks, we won't be getting Trade & Quote Bars
        # 2. 'LocalTime': Local time for this market
        # 3. Properties of self.Securities - https://www.quantconnect.com/docs/v2/writing-algorithms/securities/properties#02-Security-Properties
        
        # 4. Nasdaq - closes at 4 pm & opens at 8:30 Chicago, Also trades at 11 pm till not sure when?
                # OnSecuritiesChanged: Date:2022-12-19 23:00:00, security:MNQ Y6URRFPZ86BL

        # 5. # Links to documentation pertaining 

                # Time Modeling
                # https://www.quantconnect.com/docs/v2/writing-algorithms/key-concepts/time-modeling/timeslices#03-Properties

                # Futures - Handling Data
                # https://www.quantconnect.com/docs/v2/writing-algorithms/securities/asset-classes/futures/handling-data


                # To get the current Slice object, define an OnData method or use the CurrentSlice property of your algorithm (outside of the OnData method).
                # If the data object doesn't contain any market data but it contains auxiliary data, the slice.ContainsKey(symbol) method can return true while slice[symbol] returns None.


# Indicator Help

# In QuantConnect/Lean, we have shortcut methods for indicators, they belong to the QCAlgorithm class (use self) and name are upper-cased. These helper methods create a new instance of a indicator object and hook it up to a data consolidator so that the indicator is automatically updated by the engine.
# Since these methods create a new instance, we just should only to call it once (normally in Initialize) and assign it to a class variable to be accessed throughout the algorithm.

# 1. AverageTrueRange:
# https://www.quantconnect.com/forum/discussion/7972/using-atr-and-other-039-complex-039-indicators-with-history/p1
# https://www.quantconnect.com/forum/discussion/11457/warmup-atr-with-history
# https://www.quantconnect.com/docs/v2/writing-algorithms/indicators/supported-indicators/average-true-range


# 2. Aplying Indicators to the continuous contract prices:
        # we don't need to reset the indicator when the contract rolls over.
        # The indicator will just continue to reflect the prices of the continuous contract.

# 3. Aplying Indicators to Specific Mapped contract prices:
        #  reset the indicator if we were applying the indicator to a specific contract
        #  and then we switch to a new contract after the rollover.

# 4. Common Mistakes - Creating Automatic Indicators in a Dynamic Universe: - You can't currently remove the consolidator that LEAN creates to update automatic indicators. 
        # If you add consolidators to a dynamic universe, the consolidators build up over time and slow down your algorithm. 
        # To avoid issues, if you algorithm has a dynamic universe, use manual indicators.
        # However, I think since expired contracts expire (say 15 days after we roll over), the consolidators won't have data to fill so won't be as slow?

# https://www.quantconnect.com/docs/v2/writing-algorithms/indicators/manual-indicators#05-Automatic-Updates
# 5. Automatic Updates for Manual Indicators: To configure automatic updates, create a consolidator and then call the RegisterIndicator method. 
        # If your algorithm has a dynamic universe, save a reference to the consolidator so you can remove it when the universe removes the security. 
        # If you register an indicator for automatic updates, don't call the indicator's Update method or else the indicator will receive double updates.

# 6. Looks like if you need an Indicator before market open, then may have to use manual indicators??
        # Since "Once your algorithm reaches the EndTime of a data point, LEAN sends the data to your OnData method. 
                # For bar data, this is the beginning of the next period. "
        # How about if we use consolidated data in Automatic Indicators - consolidated from much lower time frame
        # Not sure but it may also be that using the above solution, the indicator keeps getting updated through the day

        # The consolidators can update your indicators at each time step or with aggregated bars. By default, LEAN updates data point indicators with the close price of the consolidated bars, but you can change it to a custom data field.



# Insights NOTES

        # 1. Insight Properties
        # https://www.quantconnect.com/docs/v2/writing-algorithms/algorithm-framework/alpha/key-concepts#06-Insights



        # 2. InsightCollection class is a helper class to manage insights
        # https://www.quantconnect.com/docs/v2/writing-algorithms/algorithm-framework/alpha/key-concepts#07-Insight-Collection
        




# Insight Related Variables 

        # I think they probably changed it:
        # Direction score - when your buy (sell) insight closed, was the security higher (lower) than when you issued the insight? (binary yes/no)
        # Magnitude score - when your insight closed, had the security moved by as much as you had predicted? (continuous 0->1)

        # 1. Magnitude - Get from Regression Analysis - The insight magnitude should be signed.
        # 2. Confidence as below (based on NAasdaq absolute yesterdayPercentChange)
        #   0 - 1% = 15%
        #   1 - 2% = 30%
        #   2 - 3% = 45%
        #   3 - 4% = 60%
        #   4 - 5% = 75%
        #   >5%    = 90%


        # From Old Discussions - perhaps not as valid now?
        # Direction: You predicted up relative to starting price; if its > starting price you get 1. If price < start: 0. Sum(Backtest) => Average Score.
        # Magnitude: You predicted up 10%; price moves up 5% at insight end time. You get magnitude score of 50%. Sum(Backtest) => Average Score.
        # Magnitude is a harder one to set; though arguably more valuable as most firms want an expected return curve. With a universe of magnitude scores, they can build this expected return. 
        # The average insight value being $0 means winning insights earned the same as losing ones lost. This balance of signals nets $0. It is possible to have a near $0 algorithm and still have a positive return curve!
        
        # INISGHT PERIOD - For those of us who know the actual timestamp we want to predict for, how should we go about determining the period to pass into the insight initializer?
        # https://www.quantconnect.com/forum/discussion/4873/insight-expiry-time/p1

        # Buy and hold can be accomplished using Insights. To do this, we can set the Insight duration to an arbitrarily long duration. Furthermore, if an Up Insight is followed by another Up Insight before the previous Insight expires, the previous position is held. This means, as long as we keep emitting insights with the same direction for a stock, we buy and hold that stock until an Insight is either not emitted or the Insight Direction changes for that stock (a Flat insight liquidates, while a Down Insight flips the position). In the algorithm I posted earlier, the latter method is used (note: I accidentally used a Down Insight instead of a Flat Insight, so I'll attach a fixed version).

        # 1. You state we can set an arbitrarily long expiration date for the insight, but the code you posted appears to have expiration set at month-end. That's not long enough. How could we set the expiration for 30 years? 
                # As long as we follow Insight by another Insight with the same direction, the holdings in the previous Insight is extended. To have a 30 year Insight, set the duration to 30 * 252 = 7560 days (~252 trading days in a year). 

        # 2. If the insight expiration is set at 30 years, the stock would be sold if at any point during this timespan a down insight is emitted for that same stock, is that correct? That would be ideal.
                # This is true, however, the best way to liquidate a position is to use a Flat Insight. A Down Insight will liquidate a position, but then also take a short position (please see my previous comment on the Down Insight). When an Up/Down Insight is followed by a Flat Insight, the position will be liquidated.

        # 3. You state: "as long as we keep emitting insights with the same direction for a stock, we buy and hold that stock until an Insight is either not emitted or the Insight Direction changes for that stock" --> If an up insight is emitted for a stock and is therefore purchased, I want that stock to be continued to be held even if we don't continue emitting up direction insights for that stock. For example, if a stock has a ROIC over 0.7 and is in the bottom quartile for EV-to-EBIT, then an up insight is emitted and the stock is initially purchased. But going forward, now that the stock is being held, it will be sold only if ROIC goes below 0.7, but not if it's no longer in the bottom quartile for EV-to-EBIT. We only care about EV-to-EBIT for the initial purchase, but it's not a metric that's looked at to determine when to sell it. How could this be done with insights?
                # Emit an arbitrarily long insight (e.g. 99999999 days) and emit a Flat Insight when the ROIC is below .7, (Fine Fundamental data can now be accessed through the Security object, see this thread)


# ALPHA Model
                # 2nd Alpha Model based on Cash Price - Only active 2 days in a week (Day after Midweek & weekly reporting)
                # - if cash fell since last time while future finished more than say $20 above cash today, generate a negative Insight for tom
                # - Confidence & Magnitude depends on How far we are from expiry as well as how big is the basis.
                # - Period is for full 6 hours

                # 3rd Alpha Model based on COT NSL - Only applicable Once a week 
                # 4th Alpha Model based on WWPA Weekly Inventory Data - Only applicable Once a week 
                
                # TO DO: Use following functions in only 1 of the Alpha Models
                        # updateModelData
                        # clearModelData
                        # saveModelData
                        # & In scheduler
                        # changed_event in data.SymbolChangedEvents.Values: in Update Function
 
from AlgorithmImports import *
from itertools import groupby
from HelperFuncs import *

class SourcePortfolioConstructionModel(PortfolioConstructionModel):
    symbol = 'Lumber'

    def __init__(self, DataDict):
        
        self.DataDict = DataDict # Passed the Data Dictionary that contains all Futures

        self.insightCollection = InsightCollection()
        self.ShouldPrint = RunAfterInterval(interval_seconds = 1)  # Class from HelperFuncs

    # REQUIRED: Will determine the target percent for each insight
    def DetermineTargetPercent(self, activeInsights: List[Insight]) -> Dict[Insight, float]:
        return []

    # Gets the target insights to calculate a portfolio target percent for, they will be piped to DetermineTargetPercent()
    def GetTargetInsights(self) -> List[Insight]:
        return []

    # To combine the active insights differently, override the GetTargetInsights, and return all active insights.
    # https://www.quantconnect.com/docs/v2/writing-algorithms/algorithm-framework/portfolio-construction/key-concepts#04-Multi-Alpha-Algorithms
    # def GetTargetInsights(self) -> List[Insight]:
    #     return list(self.InsightCollection.GetActiveInsights(self.Algorithm.UtcTime))

    # Create list of PortfolioTarget objects from Insights
    def CreateTargets(self, algorithm: QCAlgorithm, insights: List[Insight]) -> List[PortfolioTarget]:
        
        # Not Sure if its correct to use here
        if algorithm.Time < self.DataDict[self.symbol].algo.signalStartTime or algorithm.Time > self.DataDict[self.symbol].algo.signalEndTime: 
            algorithm.Debug("After Signal End Returning Empty Targets")
            return []

        ActiveInsights = []
        LastActiveInsights = []
        MappedCtrct = self.DataDict[self.symbol].Mapped

        # Add Insights to InsightCollection
        for insight in insights:
            if self.ShouldCreateTargetForInsight(insight):
                self.insightCollection.Add(insight)           
        
        # To check if an insight exists in the InsightCollection
        # algorithm.Debug(f"{self.insightCollection.Contains(insight)}")
        
        # To iterate through the InsightCollection
        # for insight in self.insightCollection.GetEnumerator():
        #     algorithm.Debug(f"self.insightCollection.GetEnumerator:{insight}")

        # HasActiveInsight for a Symbol 
        if MappedCtrct is not None:
            # algorithm.Debug(f"MappedCtrct:{MappedCtrct}")
            if self.insightCollection.ContainsKey(MappedCtrct):
                hasActiveInsights = self.insightCollection.HasActiveInsights(MappedCtrct, algorithm.UtcTime)

        # Remove Expired Insights from InsightCollection
        ExpiredInsights = self.insightCollection.RemoveExpiredInsights(algorithm.UtcTime)
        for count, insight in enumerate(ExpiredInsights):
            algorithm.Debug(f"ExpiredInsights#{count+1}:{insight} @ {algorithm.Time}")

        # Get ActiveInsights
        ActiveInsights = self.insightCollection.GetActiveInsights(algorithm.UtcTime)       

        # Remove duplicate Insights from ActiveInsights - Check if neccessary
        ActiveInsights = list(dict.fromkeys(ActiveInsights))

        # Print Active Insight
        for count, insight in enumerate(ActiveInsights[:]):
            # algorithm.Debug(f"ActiveInsights#{count+1}:{insight} @ {algorithm.Time}") 
            pass
                    
        # Get LastActive Insight
        for symbol, g in groupby(ActiveInsights, lambda x:x.Symbol):
            LastActiveInsights.append(sorted(g, key = lambda x:x.GeneratedTimeUtc)[-1])        
        
        # Print Total ActiveInsights, LastActiveInsights & NextExpiryTime
        
        if self.ShouldPrint.run(MappedCtrct,algorithm.UtcTime) and len(ActiveInsights[:]) > 0:
            algorithm.Debug(f"Total ActiveInsights:{len(ActiveInsights[:])} at {algorithm.Time}") 
            # var1 += 1
            # algorithm.Debug(f"var in ShouldPrint:{var1} at {algorithm.Time}") 
            
            # Get the next Insight expiry time
            NextExpiryTime = self.insightCollection.GetNextExpiryTime() 
            if NextExpiryTime is not None: NextExpiryTime = NextExpiryTime - timedelta(hours=5)
            algorithm.Debug(f"NextExpiryTime:{NextExpiryTime}")
            
            for insight in LastActiveInsights:
                algorithm.Debug(f"LastActiveInsight:{insight}") # from Model:{insight.SourceModel}
                # pass

            # Still Some empty values are not being printed for IsExpired & IsActive as they are bound methods
            InsightProperties = []            
            insight_values = [(prop, getattr(insight, prop)) for prop in insight_Properties if hasattr(insight, prop) \
                and getattr(insight, prop) is not None and (not isinstance(getattr(insight, prop), str) or bool(getattr(insight, prop)))]           
            # Convert UTC time to Eastern time for the "GeneratedTimeUtc" and "CloseTimeUtc" properties
            insight_values = [(key, value - timedelta(hours=5)) if key in ["GeneratedTimeUtc", "CloseTimeUtc"] \
                else (key, value) for key, value in insight_values]        
            InsightProperties.extend(insight_values)
            for key, value in InsightProperties:
                if key in ["GeneratedTimeUtc", "CloseTimeUtc"]:
                    algorithm.Debug(f"Key: {key}, Value: {value}")
        
            if len(ExpiredInsights) > 0:
                algorithm.Debug(f"Total ExpiredInsights:{len(ExpiredInsights)} at {algorithm.Time}") 


        return []


    # Determine if the portfolio construction model should create a target for this insight
    def ShouldCreateTargetForInsight(self, insight: Insight) -> bool:
        return True

    # Determines if the portfolio should be rebalanced base on the provided rebalancing func
    def IsRebalanceDue(self, insights: List[Insight], algorithmUtc: datetime) -> bool:
        return True

    # OPTIONAL: Security change details
    def OnSecuritiesChanged(self, algorithm: QCAlgorithm, changes: SecurityChanges) -> None:
        # Security additions and removals are pushed here.
        # This can be used for setting up algorithm state.
        # changes.AddedSecurities:
        # changes.RemovedSecurities:
        pass

from AlgorithmImports import *
from DataClass import *

from Alphas.NasdaqAlpha import *
from Alphas.OrderBookAlpha import *
from Alphas.BasisAlpha import *
from Alphas.FundamentalsAlpha import *
from Alphas.CFTCAlpha import *
from PortfolioConstruction import *

from Custom import *
import math 
import pandas as pd
from System.Drawing import Color
import pytz

class NasdaqStrategy(QCAlgorithm):

    def Initialize(self):

        self.SetBrokerageModel(BrokerageName.InteractiveBrokersBrokerage, AccountType.Margin)
        self.DefaultOrderProperties = InteractiveBrokersOrderProperties()
        
        # Order is valid until filled or the market closes
        # self.DefaultOrderProperties.TimeInForce = TimeInForce.Day
        # self.DefaultOrderProperties.OutsideRegularTradingHours = True
        
        # Set a Limit Order to be good until Lumber Close
        self.order_properties = OrderProperties()

        # self.SetStartDate(2023,2,15)        
        self.SetStartDate(2022,12,22)
        self.SetEndDate(2022,12,29)
        # self.SetEndDate(2023,1,5)
        self.SetCash(100000)
        
        self.DataNormalizationMode = DataNormalizationMode.Raw

        # TradeBuilder tracks the trades of your algorithm and calculates some statistics.
        # https://www.quantconnect.com/docs/v2/writing-algorithms/trading-and-orders/trade-statistics
        tradeBuilder = TradeBuilder(FillGroupingMethod.FillToFill,FillMatchingMethod.FIFO)    
        self.SetTradeBuilder(tradeBuilder)

        #custom Lumber data
        self.cash=self.AddData(Cash,"CASH",Resolution.Daily).Symbol
        self.western=self.AddData(Western,"WESTERN",Resolution.Daily).Symbol
        self.settle=self.AddData(Settled,"SETTLED",Resolution.Daily).Symbol
        self.disaggregated=self.AddData(Disaggregated,"DISAGGREGATED",Resolution.Daily).Symbol

        #custom data mapping dict using this to elminate if else statement
        # key=Symbol as in FuturesSymbol dict ,value=list of custom data
        customMapping={"Lumber":[self.cash,self.western,self.settle,self.disaggregated],"Nasdaq":None}
        # customMapping={"Lumber":None,"Nasdaq":None}

        self.Data = {}
        FutureSymbols = {'Lumber':Futures.Forestry.RandomLengthLumber,'Nasdaq':Futures.Indices.NASDAQ100EMini}
        # FutureSymbols = {'Lumber':Futures.Forestry.RandomLengthLumber}
        # FutureSymbols = {'Nasdaq':Futures.Indices.NASDAQ100EMini}
        
        for key, value in FutureSymbols.items():
            
            dataMappingMode_ = DataMappingMode.FirstDayMonth if key == 'Lumber' \
            else DataMappingMode.OpenInterest if key == 'Nasdaq' else DataMappingMode.LastTradingDay

            # Use BackwardsPanamaCanal or BackwardsRatio or Raw for DataNormalizationMode when "Trade" PROPERTY is able to map to Mapped contract
            future = self.AddFuture(value, Resolution.Tick,dataMappingMode = dataMappingMode_, contractDepthOffset=0, 
            dataNormalizationMode = self.DataNormalizationMode, extendedMarketHours=True, fillDataForward = True)
            
            # passing custom data to dataclass
            self.Data[key] = DataClass(self, future,customMapping[key]) # Initiating DataClass for each Future & Passing our instance of QCAlgorithm Class
        
        # Set alpha model - This also gives you access to the AlphaModel Instance 
        self.NasdaqAlpha = NasdaqAlpha(self.Data)  # Also Pass the QCAlgorithmn Instance 
        self.BasisModel  = BasisAlpha(self.Data) 
        self.OrderBookAlpha = OrderBookAlpha(self.Data) 
        self.FundamentalAlpha=FundamentalsAlpha(self.Data)
        self.CFTCAlpha=CFTCAlpha(self.Data)

        # self.AddAlpha(self.NasdaqAlpha)
        # self.AddAlpha(self.OrderBookAlpha)
        self.AddAlpha(self.BasisModel)
        # self.AddAlpha(self.FundamentalAlpha)

        # self.SetPortfolioConstruction(SourcePortfolioConstructionModel(self.Data))
        # self.Settings.RebalancePortfolioOnInsightChanges = False
        # self.Settings.RebalancePortfolioOnSecurityChanges = False


        # If I include Nasdaq, this should be on otherwise I get "Object reference not set to an instance of an object."
        # self.SetWarmUp(timedelta(days=1), Resolution.Tick)
        # self.SetWarmUp(timedelta(days=1), Resolution.Daily)

        
        # Printing Indicator's data 
        self.Schedule.On(self.DateRules.EveryDay(self.Data['Lumber'].Symbol), self.TimeRules.AfterMarketOpen(self.Data['Lumber'].Symbol, -1), self.beforeLumberOpen)        

        # Ideally signal Start & End Time should be picked up from self.Data['Lumber'].currentOpen & currentClose but they won't be initialised at this point
        # Algorithm starts calculating singal values between these 2 times everyday - Market doesn't accept orders 30 seconds b4 Market Open
        self.signalStartTimeDict = {'hour':9,'minute':59,'second':25,'microSecond':0}

        # But due to this we won't be able to view eod close data
        self.signalEndTimeDict = {'hour':16,'minute':5,'second':0,'microSecond':0}

        self.signalStartTime = self.Time.replace(hour=self.signalStartTimeDict['hour'], minute=self.signalStartTimeDict['minute'], second=self.signalStartTimeDict['second'], microsecond=self.signalStartTimeDict['microSecond'])
        self.signalEndTime   = self.Time.replace(hour=self.signalEndTimeDict['hour'], minute=self.signalEndTimeDict['minute'], second=self.signalEndTimeDict['second'], microsecond=self.signalEndTimeDict['microSecond'])

        self.SetTimeZone(TimeZones.NewYork)  # This is a daylight savings time zone.
        # If TimeZone is Chicago - Algo Time and Data End Time are same at 1800 hours
       
       # Remove this once verified
        self.percentChangeDF = pd.DataFrame()

        # Chart
        chart = Chart("NQ vs LBS")
        # How to Add both Index & Color to Series
        chart.AddSeries(Series(name = "Nasdaq", type = SeriesType.Line,  unit = '%', color = Color.Green))
        chart.AddSeries(Series(name = "Lumber", type = SeriesType.Line,  unit = '%', color = Color.Red))
        self.AddChart(chart)

        # Trading Related Global Variables
        self.maxContractLimit = 5


    def beforeLumberOpen(self):

        self.signalStartTime = self.Time.replace(hour=self.signalStartTimeDict['hour'], minute=self.signalStartTimeDict['minute'], second=self.signalStartTimeDict['second'], microsecond=self.signalStartTimeDict['microSecond'])
        self.signalEndTime   = self.Time.replace(hour=self.signalEndTimeDict['hour'], minute=self.signalEndTimeDict['minute'], second=self.signalEndTimeDict['second'], microsecond=self.signalEndTimeDict['microSecond'])

        # if self.IsWarmingUp: return
        
        # Works - Get Indicator's data - For mapped Indicators, keeping in mind Rolled Over Contracts
        for symbol in self.Data.keys():

            # if self.Data[symbol].yesterdayPercentChange is not None:
            #     self.Debug(f"yesterdayPercentChange:{symbol}:{self.Time}:{self.Data[symbol].yesterdayPercentChange}")

            pass
            # self.Debug(f"Yester Time:{self.Time}, {self.Data[symbol].Mapped}.yesterday:{self.Data[symbol].yesterday}")
            # self.Debug(f"{symbol}: Mapped ATR property:{self.Data[symbol].ATR}, {self.Time}")               
            # self.Debug(f"{symbol}: Continuous ATR property:{round(self.Data[symbol]._ContinuousATR.Current.Value,2)}, {self.Time}")    
            # self.Debug(f"{symbol}: Mapped MA2 property:{self.Data[symbol]._MappedMA2.Current.Value}, {self.Time}")    
            # self.Debug(f"{symbol}: Continuous MA2 property:{round(self.Data[symbol]._ContinuousMA2.Current.Value,2)}, {self.Time}") 
            # self.Debug(f"{symbol}: Mapped Yesterday:{self.Data[symbol]._MappedYesterday.Current.Value}, {self.Time}") 
            # self.Debug(f"{symbol}: Timezone:{self.Data[symbol].timezone}") 
            # self.Debug(f"{symbol}: Continuous Yesterday:{self.Data[symbol]._ContinuousYesterday.Current.Value}, {self.Time}") 

        # Works
        # for security in self.ActiveSecurities.Values:
        #     self.Debug(f"self.Time:{self.Time} ActiveSecurity:{security}")


    def OnSecuritiesChanged(self, changes: SecurityChanges) -> None:
        
        # NG - at 00:00:00 on 27th when StartDate is 26th
        # LBS - at 01:00:00 on 27th when StartDate is 26th

        # Only runs at the start of program not when Contract rolls 
        for security in changes.AddedSecurities:
            if len(changes.AddedSecurities) == 0: return
            self.Debug(f"In Main: OnSecuritiesChanged: Date:{self.Time}, security:{changes.AddedSecurities[0].Mapped}")


    def OnData(self, data):    
        
        # https://www.quantconnect.com/docs/v2/writing-algorithms/historical-data/warm-up-periods#03-Warm-Up-Vs-Reality
        # if self.IsWarmingUp: return
        
        # Below code important to compare Ticks with whats reported from DataClass
        tickTradeProps = ['LastPrice','Quantity']
        tickQuoteProps = ['BidPrice','BidSize','AskPrice','AskSize']
        # tickOIProps = ['Value']  # Not Getting Data for OpenInterest
        
        for security in data.Keys:           
            
            # if self.Time < self.signalStartTime or self.Time > self.signalEndTime: return # temporary
            
            if data.Ticks.ContainsKey(security) and data.Ticks[security] is not None:
                ticks = data.Ticks[security]
                for tick in ticks:
                    tick_type = getTickType(tick.TickType)
                    if tick_type == 'Trade' and int(getattr(tick, 'Quantity')) != 0:
                        # For Trade Data
                        dic = {k:getattr(tick, k) for k in tickTradeProps if getattr(tick, k) is not None}
                        # self.Debug(f"TradeTick:{self.Time} {dic}, security:{security}")          
                    elif tick_type == 'Quote':
                        # For Best Bid & Ask
                        # dic2 = {k:getattr(self.Securities[security], k) for k in tickQuoteProps} 
                        dic2 = {k:round(getattr(self.Securities[security], k),1) if k in ['BidPrice','AskPrice'] else getattr(self.Securities[security], k) for k in tickQuoteProps} 
                        # self.Debug(f"QuoteTick:{self.Time}, security:{security},  {dic2}")

        # prev_dt = None
        # for symbol in data.Keys: 
        for symbol in self.Data.keys(): 

            # sending updates to dataclass of each security
            # checking for security and custom data symbol within the dataclass
            
            self.Data[symbol].Update(data)



            # if data.ContainsKey(self.Data[symbol].Symbol):
            #     self.Data[symbol].Update(data)
            
            if self.Time < self.signalStartTime or self.Time > self.signalEndTime: return # temporary
            

            # self.Debug(f"{self.Time}:{self.percentChangeDF.tail(1)}")



            # self.Debug(f"++BestBidMapped:{symbol}:{self.Time}:{self.Data[symbol].bestBidMapped}")
            # self.Debug(f"^^BestAskMapped:{symbol}:{self.Time}:{self.Data[symbol].bestAskMapped}")   
            # self.Debug(f"++BestBidContinuous:{symbol}:{self.Time}:{self.Data[symbol].bestBidContinuous}")
            # self.Debug(f"^^BestAskContinuous:{symbol}:{self.Time}:{self.Data[symbol].bestAskContinuous}")   
            
            # if self.Data[symbol].tradeMapped != prev_dt:
            # self.Debug(f"**tradeMapped:{symbol}:{self.Data[symbol].tradeMapped}")
            # prev_dt = self.Data[symbol].tradeMapped

            # self.Debug(f"**tradeContinuous:{symbol}:{self.Data[symbol].tradeContinuous}")


            # self.Debug(f"**WAP:{symbol}:{self.Time}:{self.Data[symbol].WAP}")       
            # if self.Data[symbol].yesterdayPercentChange is not None:
            #     self.Debug(f"yesterdayPercentChange:{symbol}:{self.Time}:{self.Data[symbol].yesterdayPercentChange}")

            # if self.Data[symbol].RollingYesterdayPercentChange:
            #     self.Debug(f"RollingYesterdayPercentChange:{symbol}:{self.Time}:{self.Data[symbol].RollingYesterdayPercentChange[0]}")

            
            # WORKS - Updated several times in a day - Doesn't give last traded price - gives bid or ask
            # self.Debug(f"Todays ____Close:{self.Securities[self.Data['Lumber'].Mapped].Price}, Date:{self.Time}")
       
        # Works
        # if data.Ticks.ContainsKey(self.Data['Lumber'].Symbol):
        #     if data.Ticks[self.Data['Lumber'].Symbol] is not None:                
                # self.Debug(f"**MAIN** Date:{self.Time}, MappedValue:{self.Data['Lumber'].Mapped.Value}, MappedID:{self.Data['Lumber'].Mapped.ID}, ")             

        # https://www.quantconnect.com/docs/v2/writing-algorithms/datasets/quantconnect/us-futures-security-master#05-Data-Point-Attributes        
        for symbol in self.Data.keys():
            if data.SymbolChangedEvents.ContainsKey(self.Data[symbol].Symbol):
                symbolChangedEvent = data.SymbolChangedEvents[self.Data[symbol].Symbol]
                self.Debug(f"In MAIN Symbol changed: {symbolChangedEvent.OldSymbol} -> {symbolChangedEvent.NewSymbol} \
                EndTime:{symbolChangedEvent.EndTime} DataType:{getDataType(symbolChangedEvent.DataType)}, Expiry: {self.Securities[self.Data[symbol].Mapped].Expiry}")

                          
    # When your algorithm stops executing, LEAN calls the OnEndOfAlgorithm method.
    def OnEndOfAlgorithm(self) -> None:
        # self.Debug(f"self.Alpha.securities:{self.alpha.securities}")
        # self.Debug(f"Total Rows DataFrame:{len(self.percentChangeDF.index)}")
        # self.ObjectStore.Save(f"{14153031}/self.alpha.ModelData", df.reset_index().to_json(date_unit='s'))

        # self.ObjectStore.Save(f"{14153031}/sampleDF", self.alpha.ModelData.to_json(date_unit='s'))
        pass       

    # OnEndOfDay notifies when (Time) each security has finished trading for the day
    def OnEndOfDay(self, symbol: Symbol) -> None:
        # For LBS: Finished Trading on 2023-01-03 15:55:00 
        # For NBS: Finished Trading on 2023-01-03 16:50:00

        # self.Debug(f"Finished Trading on {self.Time} for security {symbol}")

        # dailyFileName =  f"ModelData_{self.Time.date()}"
        # self.ObjectStore.Save(f"{14153031}/{dailyFileName}", self.alpha.ModelData.to_json(date_unit='s'))

        # The OnEndOfDay method is called 10 minutes before closing to allow you to close out your position(s).
        # Clear ModelData DataFrame at end of Day - Issue it it happens 10 minutes or more before market closes so can't capture eod data
        # self.alpha.ModelData = pd.DataFrame()

        # self.Debug(f"OnEndOfDay self.Time:{self.Time}, Symbol:{self.Data['Lumber'].Mapped.Value}")
        pass
                

    def OnWarmUpFinished(self) -> None:
        self.Debug(f"Algorithm Ready@{self.Time}")
        pass