from System import *
from QuantConnect import *
from QuantConnect.Indicators import *
from QuantConnect.Data import *
from QuantConnect.Data.Market import *
from QuantConnect.Orders import *
from QuantConnect.Algorithm import *
from QuantConnect.Algorithm.Framework import *
from QuantConnect.Algorithm.Framework.Execution import *
from QuantConnect.Algorithm.Framework.Portfolio import *

import numpy as np
import tweepy
import statistics

# import datetime
from datetime import timedelta, datetime

class ScheduledExecutionModel(ExecutionModel):
    '''Execution model that submits orders while the current market price is more favorable that the current volume weighted average price.'''

    def __init__(self, algorithm, *args, **kwargs):
        super().__init__()
        '''Initializes a new instance of the VolumeWeightedAveragePriceExecutionModel class'''
        self.targetsCollection = PortfolioTargetCollection()
        self.symbolData = {}

        # Gets or sets the maximum order quantity as a percentage of the current bar's volume.
        # This defaults to 0.01m = 1%. For example, if the current bar's volume is 100,
        # then the maximum order size would equal 1 share.
        self.MaximumOrderQuantityPercentVolume = 0.1 # 0.1

        # Gets or sets the maximum spread compare to current price in percentage.
        self.acceptingSpreadPercent = 0.005
        
    def Execute(self, algorithm, targets):
        '''Executes market orders if the standard deviation of price is more
       than the configured number of deviations in the favorable direction.
       Args:
           algorithm: The algorithm instance
           targets: The portfolio targets'''

        # update the complete set of portfolio targets with the new targets
        self.targetsCollection.AddRange(targets)

        # for performance we check count value, OrderByMarginImpact and ClearFulfilled are expensive to call
        if self.targetsCollection.Count > 0:
            
            for target in self.targetsCollection.OrderByMarginImpact(algorithm):
                symbol = target.Symbol

                # calculate remaining quantity to be ordered
                unorderedQuantity = OrderSizing.GetUnorderedQuantity(algorithm, target)
                
                # fetch our symbol data containing our VWAP indicator
                data = self.symbolData.get(symbol, None)
                if data is None: return
                
                # check order entry conditions  
                if self.PriceIsFavorable(data, unorderedQuantity):
                    # adjust order size to respect maximum order size based on a percentage of current volume
                    orderSize = OrderSizing.GetOrderSizeForPercentVolume(data.Security, self.MaximumOrderQuantityPercentVolume, unorderedQuantity)
                    
                    max_quantity = algorithm.CalculateOrderQuantity(symbol, 0.95)
          
                    # suggested amount divided by the total possible amount
                    try:
                        order_percent = round(float(orderSize/max_quantity), 4)
                    except:
                        if max_quantity == 0:
                            order_percent = 0.0
                        else:
                            cash = algorithm.Portfolio.Cash
                            max_quantity = int(cash/algorithm.Securities[symbol].Close)
                            order_percent = round(float(orderSize/max_quantity), 4)
                    
                    breakup, breakdown = self.breakouts(symbol, algorithm)
                    
                    if ((orderSize != 0) and (abs(order_percent) > 0.05)):
                        coef = abs(order_percent) * 0.5
                        if algorithm.Portfolio[symbol].Invested:
                            if coef < 0.25:
                                coef = int(abs(float(order_percent))/0.05)*10
                        if breakup:
                            coef = coef * 20
                        orderSize = OrderSizing.GetOrderSizeForPercentVolume(data.Security, self.MaximumOrderQuantityPercentVolume*coef, unorderedQuantity)
                        projected_cost = round(algorithm.Securities[symbol].Close * orderSize, 3) * 1.05
                        
                        order_pct = round(float(orderSize)/algorithm.Portfolio.Cash, 4)
                        algorithm.Plot("Order Percent", str(symbol), float(order_pct))
                        if (algorithm.Portfolio.Cash > projected_cost):
                            algorithm.MarketOrder(symbol, orderSize)
            self.targetsCollection.ClearFulfilled(algorithm)

    def OnSecuritiesChanged(self, algorithm, changes):
        '''Event fired each time the we add/remove securities from the data feed
        Args:
            algorithm: The algorithm instance that experienced the change in securities
            changes: The security additions and removals from the algorithm'''
        for removed in changes.RemovedSecurities:
            # clean up removed security data
            if removed.Symbol in self.symbolData:
                if self.IsSafeToRemove(algorithm, removed.Symbol):
                    data = self.symbolData.pop(removed.Symbol)
                    algorithm.SubscriptionManager.RemoveConsolidator(removed.Symbol, data.Consolidator)

        for added in changes.AddedSecurities:
            if added.Symbol not in self.symbolData:
                self.symbolData[added.Symbol] = SymbolData(algorithm, added)

    def PriceIsFavorable(self, data, unorderedQuantity):
        '''Determines if the current price is more than the configured
       number of standard deviations away from the mean in the favorable direction.'''
        if unorderedQuantity > 0:
            if data.Security.BidPrice < data.VWAP:
                return True
        else:
            if data.Security.AskPrice > data.VWAP:
                return True

        return False

    def SpreadIsFavorable(self, data):
        '''Determines if the spread is in desirable range.'''
        # Price has to be larger than zero to avoid zero division error, or negative price causing the spread percentage < 0 by error
        # Has to be in opening hours of exchange to avoid extreme spread in OTC period
        return data.Security.Exchange.ExchangeOpen \
            and data.Security.Price > 0 and data.Security.AskPrice > 0 and data.Security.BidPrice > 0 \
            and (data.Security.AskPrice - data.Security.BidPrice) / data.Security.Price <= self.acceptingSpreadPercent

    def IsSafeToRemove(self, algorithm, symbol):
        '''Determines if it's safe to remove the associated symbol data'''
        # confirm the security isn't currently a member of any universe
        return not any([kvp.Value.ContainsMember(symbol) for kvp in algorithm.UniverseManager])

    def breakouts(self, symbol, algorithm):
        self.lookback = 20
        self.ceiling, self.floor = 30, 10
        
        close = algorithm.History(symbol, 31, Resolution.Daily)["close"]
        todayvol = np.std(close[1:31])
        yesterdayvol = np.std(close[0:30])
        deltavol = (todayvol - yesterdayvol) / todayvol
        self.lookback = round(self.lookback * (1 + deltavol))

        # Account for upper/lower limit of lockback length
        if self.lookback > self.ceiling:
            self.lookback = self.ceiling
        elif self.lookback < self.floor:
            self.lookback = self.floor

        # List of daily highs
        self.high = algorithm.History(symbol, self.lookback, Resolution.Daily)["high"]
        self.low = algorithm.History(symbol, self.lookback, Resolution.Daily)["low"]
        
        # Buy in case of breakout upward
        breakup = True
        breakout_condition1 = (algorithm.Securities[symbol].Close >= max(self.high[:-1])) and (round(float(algorithm.Securities[symbol].Close - algorithm.Securities[symbol].Open)/algorithm.Securities[symbol].Open, 4) >= 0.01)
        breakout_condition2 = (round(float(algorithm.Securities[symbol].Close - algorithm.Securities[symbol].Open)/algorithm.Securities[symbol].Open, 4) >= 0.02)
        breakdown = True
        if breakout_condition1 or breakout_condition2:
            breakup = True
        else:
            breakup = False
        if algorithm.Securities[symbol].Close <= min(self.low[:-1]):
            breakdown = True
        else:
            breakdown = False
        
        return breakup, breakdown
            
    def UpdateTickets(self, algorithm, symbol, orderSize):
        close = algorithm.Securities[symbol].Close
        algorithm.LimitOrder(symbol, -int(orderSize/2), close * 1.025)
        
class SymbolData:
    def __init__(self, algorithm, security):
        self.Security = security
        self.Consolidator = algorithm.ResolveConsolidator(security.Symbol, security.Resolution)
        name = algorithm.CreateIndicatorName(security.Symbol, "VWAP", security.Resolution)
        self.vwap = IntradayVwap(name)
        algorithm.RegisterIndicator(security.Symbol, self.vwap, self.Consolidator)

    @property
    def VWAP(self):
       return self.vwap.Value
       
    def dispose(self, algorithm):
        algorithm.SubscriptionManager.RemoveConsolidator(security.Symbol, self.consolidator)

class IntradayVwap:
    '''Defines the canonical intraday VWAP indicator'''
    def __init__(self, name):
        self.Name = name
        self.Value = 0.0
        self.lastDate = datetime.min
        self.sumOfVolume = 0.0
        self.sumOfPriceTimesVolume = 0.0

    @property
    def IsReady(self):
        return self.sumOfVolume > 0.0

    def Update(self, input):
        '''Computes the new VWAP'''
        success, volume, averagePrice = self.GetVolumeAndAveragePrice(input)
        if not success:
            return self.IsReady

        # reset vwap on daily boundaries
        if self.lastDate != input.EndTime.date():
            self.sumOfVolume = 0.0
            self.sumOfPriceTimesVolume = 0.0
            self.lastDate = input.EndTime.date()

        # running totals for Σ PiVi / Σ Vi
        self.sumOfVolume += volume
        self.sumOfPriceTimesVolume += averagePrice * volume

        if self.sumOfVolume == 0.0:
           # if we have no trade volume then use the current price as VWAP
           self.Value = input.Value
           return self.IsReady

        self.Value = self.sumOfPriceTimesVolume / self.sumOfVolume
        return self.IsReady

    def GetVolumeAndAveragePrice(self, input):
        '''Determines the volume and price to be used for the current input in the VWAP computation'''

        if type(input) is Tick:
            if input.TickType == TickType.Trade:
                return True, float(input.Quantity), float(input.LastPrice)

        if type(input) is TradeBar:
            if not input.IsFillForward:
                averagePrice = float(input.Open + input.High + input.Low + input.Close) / 4
                return True, float(input.Volume), averagePrice

        return False, 0.0, 0.0
        

import pandas as pd
import numpy as np
from scipy.optimize import minimize


class myTrailingStopRiskManagementModel:
    '''
    Credit goes to: Alex Catarino and many of his friends at QuantConnect
    
    https://github.com/QuantConnect/Lean/blob/master/Algorithm.Framework/Risk/TrailingStopRiskManagementModel.py
    
    Description:
        Limits the maximum possible loss measured from the highest unrealized profit
    '''
    
    def __init__(self, maximumDrawdownPercent = 0.08):
        '''initializes the class
        Args: maximumDrawdownPercent: The maximum percentage drawdown allowed for algorithm portfolio compared with the highest unrealized profit, defaults to 5% drawdown
        '''
        self.maximumDrawdownPercent = -abs(maximumDrawdownPercent)
        self.trailingHighs = dict()

    def setDD(self, maximumDrawdownPercent = 0.08):
        '''allows to change the drawdown
        Args: maximumDrawdownPercent: The maximum percentage drawdown allowed for algorithm portfolio compared with the highest unrealized profit, defaults to 5% drawdown
        '''
        self.maximumDrawdownPercent = -abs(maximumDrawdownPercent)

    def setWTtoZeroIfDDtooHigh(self, algorithm, targets=None):
        '''If drawdown is too high, set wt[symbol] to zero
           algo.wt[symbol] = weights which will be set to 0 in case drawdown exceeds the maximum    
        '''
        for kvp in algorithm.Securities:
            symbol = kvp.Key
            security = kvp.Value
            
            # Remove from trailingHighs dict if not invested
            if not security.Invested:
                self.trailingHighs.pop(symbol, None)
                continue
            
            # Add newly invested securities to trailingHighs dict
            if symbol not in self.trailingHighs:
                self.trailingHighs[symbol] = security.Holdings.AveragePrice
                continue
            
            # Check for new highs and update trailingHighs dict
            if self.trailingHighs[symbol] < security.High:
                self.trailingHighs[symbol] = security.High
                continue
            
            # Calc the drawdown
            securityHigh = self.trailingHighs[symbol]
            drawdown = (security.Low / securityHigh) - 1
            
            # If drawdown is too high, set symbol weight to zero
            if drawdown < self.maximumDrawdownPercent:
                algorithm.wt[symbol] = 0
        
        return
    
    
    
class myPortfolioOptimizer:
    '''
    Credit goes to: Emilio Freire / InnoQuantivity
    
    https://innoquantivity.com/blogs/inno-blog/portfolio-optimization-quantconnect-research-algorithm
    https://www.quantconnect.com/forum/discussion/8128/portfolio-optimization-research-amp-algorithm-for-better-workflows/p1/comment-22952
    
    Description:
        Implementation of a custom optimizer that calculates the weights for each asset to optimize a given objective function
    Details:
        Optimization can be:
            - Equal Weighting
            - Maximize Portfolio Return
            - Minimize Portfolio Standard Deviation
            - Mean-Variance (minimize Standard Deviation given a target return)
            - Maximize Portfolio Sharpe Ratio
            - Maximize Portfolio Sortino Ratio
            - Risk Parity Portfolio
        Constraints:
            - Weights must be between some given boundaries
            - Weights must sum to 1
    '''
    def __init__(self, 
                 minWeight = 0,
                 maxWeight = 1):
        '''
        Description:
            Initialize the CustomPortfolioOptimizer
        Args:
            minWeight(float): The lower bound on portfolio weights
            maxWeight(float): The upper bound on portfolio weights
        '''
        self.minWeight = minWeight
        self.maxWeight = maxWeight
        
        
    def CalcWeights(self, algorithm, symbols, objectiveFunction='riskParity', lookback=63, targetReturn=None):
        '''
        Description:
            Calculate weights from daily returns, return a pandas Series
        '''
        history = np.log10(algorithm.History(symbols, lookback, Resolution.Daily)['close'].unstack(level = 0))
        returnsDf = history.pct_change().dropna()
        returnsDf.columns = [algorithm.AddEquity(i).Symbol.Value for i in list(returnsDf.columns)]
        weights = self.Optimize(objectiveFunction, returnsDf, targetReturn)
        return pd.Series(weights, index=returnsDf.columns, name='weights')
        
        
    def Optimize(self, objFunction, dailyReturnsDf, targetReturn = None):
        '''
        Description:
            Perform portfolio optimization given a series of returns
        Args:
            objFunction: The objective function to optimize (equalWeighting, maxReturn, minVariance, meanVariance, maxSharpe, maxSortino, riskParity)
            dailyReturnsDf: DataFrame of historical daily arithmetic returns
        Returns:
            Array of double with the portfolio weights (size: K x 1)
        '''
        # initial weights: equally weighted
        size = dailyReturnsDf.columns.size # K x 1
        self.initWeights = np.array(size * [1. / size])
        
        # get sample covariance matrix
        covariance = dailyReturnsDf.cov()
        # get the sample covariance matrix of only negative returns for sortino ratio
        negativeReturnsDf = dailyReturnsDf[dailyReturnsDf < 0]
        covarianceNegativeReturns = negativeReturnsDf.cov()
        
        if objFunction == 'equalWeighting':
            return self.initWeights
        
        bounds = tuple((self.minWeight, self.maxWeight) for x in range(size))
        constraints = [{'type': 'eq', 'fun': lambda x: np.sum(x) - 1.0}]
        
        if objFunction == 'meanVariance':
            # if no target return is provided, use the resulting from equal weighting
            if targetReturn is None:
                targetReturn = self.CalculateAnnualizedPortfolioReturn(dailyReturnsDf, self.initWeights)
            constraints.append( {'type': 'eq', 'fun': lambda weights:
                                self.CalculateAnnualizedPortfolioReturn(dailyReturnsDf, weights) - targetReturn} )
        
        opt = minimize(lambda weights: self.ObjectiveFunction(objFunction, dailyReturnsDf,
                                                                covariance, covarianceNegativeReturns,
                                                                weights),
                                                            x0 = self.initWeights,
                                                            bounds = bounds,
                                                            constraints = constraints,
                                                            method = 'SLSQP')
        return opt['x']
        
        
    def ObjectiveFunction(self, objFunction, dailyReturnsDf, covariance, covarianceNegativeReturns, weights):
        
        '''
        Description:
            Compute the objective function
        Args:
            objFunction: The objective function to optimize (equalWeighting, maxReturn, minVariance, meanVariance,
                                                                maxSharpe, maxSortino, riskParity)
            dailyReturnsDf: DataFrame of historical daily returns
            covariance: Sample covariance
            covarianceNegativeReturns: Sample covariance matrix of only negative returns
            weights: Portfolio weights
        '''
        if objFunction == 'maxReturn':
            f = self.CalculateAnnualizedPortfolioReturn(dailyReturnsDf, weights)
            return -f # convert to negative to be minimized
        elif objFunction == 'minVariance':
            f = self.CalculateAnnualizedPortfolioStd(covariance, weights)
            return f
        elif objFunction == 'meanVariance':
            f = self.CalculateAnnualizedPortfolioStd(covariance, weights)
            return f
        elif objFunction == 'maxSharpe':
            f = self.CalculateAnnualizedPortfolioSharpeRatio(dailyReturnsDf, covariance, weights)
            return -f # convert to negative to be minimized
        elif objFunction == 'maxSortino':
            f = self.CalculateAnnualizedPortfolioSortinoRatio(dailyReturnsDf, covarianceNegativeReturns, weights)
            return -f # convert to negative to be minimized
        elif objFunction == 'riskParity':
            f = self.CalculateRiskParityFunction(covariance, weights)
            return f
        else:
            raise ValueError(f'PortfolioOptimizer.ObjectiveFunction: objFunction input has to be one of equalWeighting,'
             + ' maxReturn, minVariance, meanVariance, maxSharpe, maxSortino or riskParity')
        
        
    def CalculateAnnualizedPortfolioReturn(self, dailyReturnsDf, weights):
        
        annualizedPortfolioReturns = np.sum( ((1 + dailyReturnsDf.mean())**252 - 1) * weights )
        
        return annualizedPortfolioReturns
        
            
    def CalculateAnnualizedPortfolioStd(self, covariance, weights):
        
        annualizedPortfolioStd = np.sqrt( np.dot(weights.T, np.dot(covariance * 252, weights)) )
        
        if annualizedPortfolioStd == 0:
            raise ValueError(f'PortfolioOptimizer.CalculateAnnualizedPortfolioStd: annualizedPortfolioStd cannot be zero. Weights: {weights}')
            
        return annualizedPortfolioStd
        
        
    def CalculateAnnualizedPortfolioNegativeStd(self, covarianceNegativeReturns, weights):
    
        annualizedPortfolioNegativeStd = np.sqrt( np.dot(weights.T, np.dot(covarianceNegativeReturns * 252, weights)) )        
        
        if annualizedPortfolioNegativeStd == 0:
            raise ValueError(f'PortfolioOptimizer.CalculateAnnualizedPortfolioNegativeStd: annualizedPortfolioNegativeStd cannot be zero. Weights: {weights}')
        
        return annualizedPortfolioNegativeStd
        
        
    def CalculateAnnualizedPortfolioSharpeRatio(self, dailyReturnsDf, covariance, weights):
        
        annualizedPortfolioReturn = self.CalculateAnnualizedPortfolioReturn(dailyReturnsDf, weights)
        annualizedPortfolioStd = self.CalculateAnnualizedPortfolioStd(covariance, weights)
        annualizedPortfolioSharpeRatio = annualizedPortfolioReturn / annualizedPortfolioStd
            
        return annualizedPortfolioSharpeRatio
        
        
    def CalculateAnnualizedPortfolioSortinoRatio(self, dailyReturnsDf, covarianceNegativeReturns, weights):
        
        annualizedPortfolioReturn = self.CalculateAnnualizedPortfolioReturn(dailyReturnsDf, weights)
        annualizedPortfolioNegativeStd = self.CalculateAnnualizedPortfolioNegativeStd(covarianceNegativeReturns, weights)
        annualizedPortfolioSortinoRatio = annualizedPortfolioReturn / annualizedPortfolioNegativeStd
            
        return annualizedPortfolioSortinoRatio
        
        
    def CalculateRiskParityFunction(self, covariance, weights):
        
        ''' Spinu formulation for risk parity portfolio '''
        
        assetsRiskBudget = self.initWeights
        portfolioVolatility = self.CalculateAnnualizedPortfolioStd(covariance, weights)
        
        x = weights / portfolioVolatility
        riskParity = (np.dot(x.T, np.dot(covariance, x)) / 2) - np.dot(assetsRiskBudget.T, np.log(x))
            
        return riskParity

from System import *
from QuantConnect import *
from QuantConnect.Algorithm import *
from QuantConnect.Indicators import *

import sys
import tweepy
from tweepy import OAuthHandler
from time import sleep
import json

# Important - Don't name file 'tweepy.py'

# Used below to supress problems and continue instead of try/except/continue
from contextlib import suppress

# --- # --------------------------------------------------------------------- # --- #
# --- # --------------------------------------------------------------------- # --- #

# Set True to follow a defined list of twitter users.
# Set False to stream from all accounts based on defined keywords.

FollowerMode = True

# --- # --------------------------------------------------------------------- # --- #
# --- # --------------------------------------------------------------------- # --- #

# --- # ---------------------------------- # --- #
# --- # ---- Follower Mode Dictionary ---- # --- #
# --- # ---------------------------------- # --- #

# Lookup Twitter ID's here http://gettwitterid.com/ by entering the accounts @Usernamehandle (without the @)
# It doesn't care what you name them. The names are only displayed to the console.

idsdict = {'TT3Private'}

# --- # --------------------------------------------------------------------- # --- #
# --- # --------------------------------------------------------------------- # --- #

# --- # ------------------------------ # --- #
# --- # ---- Search Mode Keywords ---- # --- #
# --- # ------------------------------ # --- #

# --- # ----------------- # --- #
# --- # SEARCH BY KEYWORD # --- #
# --- # ----------------- # --- #

# Example
# search = ['breaking news']

# --- # ---------- # --- #
# --- # SEARCH ALL # --- #
# --- # ---------- # --- #

# [' '] and [''] yields no results. The only way to truly stream all of the tweets (unfiltered)
# requires a connection to the firehose(https://developer.twitter.com/en/docs/tweets/sample-realtime/overview/decahose.html), 
# which is granted only in specific use enterprise cases by Twitter.

search = ['.','a','@','\'','this','to',':(','?','!','$',
          'h','+','_','-','#','b','you', 'c',',','the',
          'i','/','lol','at','this','need','and','RT',
          'if','1', 'd','e','f','g'] # Feel free to expand on this. I believe there's a limit on how much you can add.


# --- # -------------------- # --- #
# --- # SEARCH BY USER INPUT # --- #
# --- # -------------------- # --- #

# search = [input('Enter keyword\n\n')]

# --- # -------------------------------------------------------------- # --- #
# --- # -------------------------------------------------------------- # --- #



# --- # ---------------------- # --- #
# --- # --- AUTHENTICATION --- # --- #
# --- # ---------------------- # --- #

consumer_key = self.GetParameter("Consumer_Api")
consumer_secret = self.GetParameter("Consumer_Api_Secret")
access_token = self.GetParameter("Access_Token")
access_token_secret = self.GetParameter("Access_Token_Secret")

auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
api = tweepy.API(auth)

# --- # -------------------------------------------------------------- # --- #
# --- # -------------------------------------------------------------- # --- #

print('Listening for tweets...\n')

if FollowerMode == True:
    # gets all IDs from 'idsdict' and converts them to strings
    ids = [str(i) for i in list(idsdict.values())]

# Main Stream Listener Class
class MyStreamListener(tweepy.StreamListener):
    global ids
    global FollowerMode
    tweets = 0
    
    # on_status is a built in tweepy method to fetch tweets.
    # on_data is another one and shows more detailed information for analytical reasons,
    # but be aware that you will have to parse the json manually like data['text'], data['user']['location'], etc.
    # print out status or data(if using on_data) and run the script to fetch the full JSON to see everything that it can do.
    # You can find a good example of that here https://github.com/varadhbhatnagar/Emoyto
    def on_status(self, status):
        """
        # Use this if you plan to use the json functionality below. 
        # If you use this, tab the rest of this class below over by one indent(4 spaces)
        # with open ('tweets.json', 'a', encoding='utf-8') as f:        

        # Supress errors so if that specific tweet has an issue for whatever reason, it will skip it. Similar to try/except.
        # Don't use this if you want to debug/look for issues.
        """

        with suppress(Exception):
            
            userid = str(status.user.id)
            
            # "userid in ids" mentioned below removes all of the mentions and retweets and makes sure it only comes from the original account.
            # Tweepy has no built in way to exclude that to my knowledge based on stackoverflow answers.
            
            if FollowerMode == True and userid in ids:
                
                # You can do this for example - " if status.place.country == 'United States': ",
                # but most people don't have their country listed. status.user.location often shows 'state' or 'city, state' and/or country,
                # but their location is user set so it can really be something made up like 'outer space'. If it's that important,
                # you could always try and use an API to see if it's a valid location.
                
                print('-' * 80)
                
                # Prints the name for this ID that's defined in 'idsdict'
                with suppress(Exception):
                    print(list(idsdict.keys())[list(idsdict.values()).index(int(userid))])
                
                print('User: ' + status.user.screen_name)
                # Attempt to display location and/or country if it exists
                with suppress(Exception):
                    if status.user.location != None and status.user.location != 'None':
                        print('Location: ' + status.user.location)
                with suppress(Exception):
                    print('Country: ' + status.place.country)
                
                # Checks to see if tweet is 'extended'/long. If it is, it will display the full tweet.
                try:
                    text = status.extended_tweet['full_text']
                except AttributeError:
                    text = status.text
                print('Tweet: ' + text)
                sleep(0.015)
                
            elif FollowerMode == False:
                
                print('-' * 80)
                print('User: ' + status.user.screen_name)
                with suppress(Exception):
                    if status.user.location != None and status.user.location != 'None':
                        print('Location: ' + status.user.location)
                with suppress(Exception):
                    print('Country: ' + status.place.country)
                        
                try:
                    text = status.extended_tweet['full_text']
                except AttributeError:
                    text = status.text
                print('Tweet: ' + text)
                
                # Prevents the display from hiccups and keeps the scrolling smooth when scanning all
                sleep(0.016)
                            
            # --- # --------------------------------------------------------------------- # --- #
            # --- # --------------------------------------------------------------------- # --- #
            
            # Optional - Write tweet into json file. You can store just tweets for example
            # Make sure to un-comment the 'with f.open' above and tab the rest of the class below it.
          
            #json_str = json.dumps(status._json)
            #f.write(status.text + '\n')
            
            # --- # --------------------------------------------------------------------- # --- #
            # --- # --------------------------------------------------------------------- # --- #
            
            #  # Optional - Print something out every certain number of tweets to show how many tweets have came through.
            
            #  MyStreamListener.tweets += 1
            # if MyStreamListener.tweets % 1000 == 0:
            #     print(str(MyStreamListener.tweets) + ' Tweets')
            #     for i in range(15):
            #         print(f'|||||||||||||||||||||||||||||||||||----- {MyStreamListener.tweets} ------||||||||||||||||||||||||||||||||||||||| \n')
            #     sleep(1)
                
# Define the listener
listener = MyStreamListener()
stream = tweepy.Stream(auth, listener)


if FollowerMode == True:
    stream.filter(follow=ids)
else:
    stream.filter(languages=["en"], track = search )

from System import *
from QuantConnect import *
from QuantConnect.Algorithm import *
from QuantConnect.Indicators import *

import tweepy, statistics
from datetime import datetime, timedelta, date
import numpy as np
from scipy import stats
from AlgorithmImports import *
from sklearn.linear_model import LinearRegression, LogisticRegression
from sklearn.tree import ExtraTreeRegressor, ExtraTreeClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error, r2_score, mean_absolute_error
from sklearn import preprocessing

from helpers import myTrailingStopRiskManagementModel

class DualMomentumWithOutDaysAlphaModel(AlphaModel):

    def __init__(self, algorithm, VOLA = 126, BASE_RET = 83, resolution = Resolution.Daily, *args, **kwargs):
        super().__init__()
        self.VOLA = VOLA
        self.BASE_RET = BASE_RET
        self.resolution = Resolution.Daily # resolution
        self.MKT = algorithm.AddEquity('SPY', resolution).Symbol
        self.SLV = algorithm.AddEquity('SLV', resolution).Symbol
        self.GLD = algorithm.AddEquity('GLD', resolution).Symbol
        self.XLI = algorithm.AddEquity('XLI', resolution).Symbol
        self.XLU = algorithm.AddEquity('XLU', resolution).Symbol
        self.DBB = algorithm.AddEquity('DBB', resolution).Symbol
        self.UUP = algorithm.AddEquity('UUP', resolution).Symbol

        self.count = self.BASE_RET
        self.outday = 5

        pairs = [self.MKT, self.SLV, self.GLD, self.XLI, self.XLU, self.DBB, self.UUP]
        for symbol in pairs:
            self.consolidator = TradeBarConsolidator(timedelta(days=1))
            self.consolidator.DataConsolidated += self.consolidation_handler
            algorithm.SubscriptionManager.AddConsolidator(symbol, self.consolidator)

        self.history = np.log10(algorithm.History(pairs, self.VOLA + 1, self.resolution))
        # self.history = self.history['close'].unstack(level=0).dropna()

        self.predictionInterval = Time.Multiply(Extensions.ToTimeSpan(self.resolution), 1)
        resolutionString = Extensions.GetEnumString(resolution, Resolution)
        self.Name = f"{self.__class__.__name__}({resolutionString})"

        # Force alpha to only produce insights Daily at 11.10am
        self.set_flag = False
        algorithm.Schedule.On(algorithm.DateRules.EveryDay(),
                              algorithm.TimeRules.AfterMarketOpen('SPY', 60),
                              self.SetFlag)
                              
    def SetFlag(self):
        self.set_flag = True

    def consolidation_handler(self, sender, consolidated):
        self.history.loc[consolidated.EndTime, consolidated.Symbol] = consolidated.Close
        self.history = self.history.iloc[-(self.VOLA + 1):]
        
    def Update(self, algorithm, _data):
        if algorithm.IsWarmingUp or not self.set_flag:
            return []

        self.set_flag = False
        insights = []
        
        # Volatility
        vola = self.history[self.MKT].pct_change().std() * np.sqrt(252)
        wait_days = int(vola * self.BASE_RET)
        period = int((1.0 - vola) * self.BASE_RET)
        r = self.history.pct_change(period).iloc[-1]

        exit_market = r[self.SLV] < r[self.GLD] and r[self.XLI] < r[self.XLU] and r[self.DBB] < r[self.UUP]
        
        # # ML Model
        # pairs = [self.MKT, self.SLV, self.GLD, self.XLI, self.XLU, self.DBB, self.UUP]
        # data = self.history
        # data['vola'] = data[self.MKT].pct_change().rolling(126).std() * np.sqrt(252)
        # data['wait_days'] = data['vola'].map(lambda x: int(int(x)*self.BASE_RET), na_action='ignore')
        # data['period'] = data['vola'].map(lambda x: int((1.0 - int(x)) * self.BASE_RET), na_action='ignore')
        # market_signals = list()
        # for i in range(0, len(data['period'])):
        #     if 'na' not in str(data['period'][i]):
        #         count = int(i)
        #         period = int(data['period'][i])
        #         r = data[count:count+126].pct_change(period).iloc[-1]
        #         exit_market = (r[self.SLV] < r[self.GLD] and r[self.XLI] < r[self.XLU] and r[self.DBB] < r[self.UUP])
        #         market_signals.append(exit_market)
        #     else:
        #         market_signals.append('na')
                
        # data['market_signals'] = market_signals
        # del market_signals
        
        # data['Market_Change'] = data[self.MKT].pct_change()
        # # # Load LabelEncoder to process string variables
        # le = preprocessing.LabelEncoder()
        
        # ml_data = data.dropna()
        # pairs = [self.MKT, self.SLV, self.GLD, self.XLI, self.XLU, self.DBB, self.UUP, 'vola', 'wait_days', 'period', 'Market_Change']
        # X = ml_data.drop(['market_signals'], axis=1)
        # y = np.ravel(ml_data[['market_signals']].astype(str).apply(le.fit_transform))
        # #algorithm.Debug(str(X.shape) + ";" + str(y.shape))
        # X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
        
        # model = ExtraTreeClassifier(max_depth = 40,
        #                      max_features = 'sqrt',
        #                      criterion = 'gini',
        #                      class_weight = 'balanced',
        #                      random_state = 42)
                             
        # model.fit(X_train, y_train)
        
        # y_pred = model.predict(X_test)
        # algorithm.Plot("Model Score", "Score", model.score(X_test, y_test))
        # algorithm.Plot("MAE:", "Mae", mean_absolute_error(y_test, y_pred))
        # algorithm.Plot("Coef of Determination:", "r-squared", r2_score(y_test, y_pred))
        # algorithm.Plot("Actual vs Pred", "Actual", r2_score(y_test, y_pred))
        
        # clf_results = metrics.precision_recall_fscore_support(y_test, y_pred)
        # algorithm.Plot("Confusion_Matrix", "Precision", clf_results[0].round(2))
        # algorithm.Plot("Confusion_Matrix", "Recall", clf_results[1].round(2))
        # algorithm.Plot("Confusion_Matrix", "f1-score", clf_results[2].round(2))
        # algorithm.Plot("Confusion_Matrix", "Support", clf_results[3].round(2))
        
        # signal_actual = np.log(self.history[self.MKT]).iloc[-1]
        # signal_pred = model.predict(np.ravel(np.log(self.history[pairs]).iloc[-1]).reshape(1, -1))
        
        # prev = np.log(self.history[self.MKT]).iloc[-2]
        # signal_actual = 1 if signal_actual > prev else -1
        # signal_pred = 1 if signal_pred > prev else -1
        
        # agreement= 1 if signal_actual == signal_pred else -1
        
        # algorithm.Plot("Actual vs Pred", "Actual", float(signal_actual))
        # algorithm.Plot("Actual vs Pred", "Pred", float(signal_pred))
        # algorithm.Plot("Actual vs Pred", "Agreement", float(agreement))

        direction = InsightDirection.Down
            
        if (exit_market):
            #algorithm.Plot("In vs Out", "Market", -1)
            direction = InsightDirection.Down
            self.outday = self.count
        elif (self.count >= wait_days + self.outday):
                #algorithm.Plot("In vs Out", "Market", 1)
                direction = InsightDirection.Up
        else:
            direciton = InsightDirection.Flat
        self.count += 1
        
        # algorithm.Plot("Wait Days", "Actual", self.count)
        # algorithm.Plot("Wait Days", "Expected", float(wait_days + self.outday))
        algorithm.Plot("Market Volatility", str(self.MKT), float(vola))
        if direction == InsightDirection.Down:
            val = -1.0
        elif direction == InsightDirection.Up:
            val = 1.0
        else:
            val = 0.0
        algorithm.Plot("IN vs Out", str(self.MKT), float(val))        
                                                                    
        insights.append(Insight.Price(self.MKT, self.predictionInterval, direction))

        return insights

from itertools import groupby
import tweepy
from datetime import datetime, timedelta, date
import time
import pandas as pd
import numpy as np
import re, math
import scipy
from math import ceil
from collections import deque
from itertools import chain
from pytz import timezone

from System import *
from QuantConnect import *
from QuantConnect.Algorithm import *
from QuantConnect.Indicators import *
from QuantConnect.Data.Market import TradeBar
from QuantConnect.Algorithm.Framework.Execution import StandardDeviationExecutionModel, VolumeWeightedAveragePriceExecutionModel
from QuantConnect.Algorithm.Framework.Risk import MaximumDrawdownPercentPortfolio, MaximumUnrealizedProfitPercentPerSecurity, MaximumDrawdownPercentPerSecurity, TrailingStopRiskManagementModel

from dual_momentum_with_out_days_alpha import DualMomentumWithOutDaysAlphaModel
from portfolio_management import PortfolioManagementModel
from trade_execution import ScheduledExecutionModel

VOLA = 126; BASE_RET = 83; RET = 252; EXCL = 21; LEV = 1.00;

class HorizontalQuantumCoil(QCAlgorithm):

    def Initialize(self):
        self.Portfolio.MarginCallModel = MarginCallModel.Null
        self.SetStartDate(2008, 1, 1)
        self.SetCash(10000)
        self.added_cash = 115
        self.upkeep = 28
        self.simulate_live = False
        self.SetWarmUp(timedelta(252))
        self.Settings.FreePortfolioValuePercentage = 0.05
        # self.SetBrokerageModel(BrokerageName.AlphaStreams)
        self.SetAlpha(DualMomentumWithOutDaysAlphaModel(self, VOLA, BASE_RET, Resolution.Daily))

        stonks = ['IYW', 'FDN', 'QQQ', 'IWM', 'SPY', 'VTI', 'DIA', 'IWF', 'TLT', 'TLH', 'IEI', 'IEF']
        # lev_stonks = ['TQQQ', 'URTY', 'SPXL', 'TMF', 'AGQ', 'UGL']
        # stonks = ['ITOT', 'IVV', 'IJH', 'IJR', 'XT', 'IHAK', 'IWFH', 'IDNA', 'IRBO', 'TECB', 'BFTR', 'BTEK', 'BMED']
        symbols = []
        for stonk in stonks:
            val = Symbol.Create(stonk, SecurityType.Equity, Market.USA)
            symbols.append(val)
        self.SetUniverseSelection(ManualUniverseSelectionModel(symbols))
        self.UniverseSettings.Resolution = Resolution.Daily
        self.SetPortfolioConstruction(PortfolioManagementModel(self, RET, EXCL, LEV, Resolution.Daily))
        self.SetExecution(ScheduledExecutionModel(self))
        #self.SetRiskManagement(MaximumUnrealizedProfitPercentPerSecurity(maximumUnrealizedProfitPercent = 0.10))
        #self.SetRiskManagement(MaximumDrawdownPercentPerSecurity(maximumDrawdownPercent = 0.03))
        #self.SetRiskManagement(MaximumDrawdownPercentPortfolio(maximumDrawdownPercent = 0.04, isTrailing = True))
        #self.SetRiskManagement(TrailingStopRiskManagementModel(maximumDrawdownPercent = 0.04))

        self.createPlots("SPY")
    
        if self.simulate_live:
            self.Schedule.On(self.DateRules.Every(DayOfWeek.Monday), \
                             self.TimeRules.BeforeMarketClose("SPY", 0), \
                             self.AddCash)
    
            self.Schedule.On(self.DateRules.MonthStart("SPY"), \
                             self.TimeRules.BeforeMarketClose("SPY", 0), \
                             self.UpKeep)
            
    
    def AddCash(self):
        self.Portfolio.SetCash(self.Portfolio.Cash + self.added_cash)

    def UpKeep(self):
        self.Portfolio.SetCash(self.Portfolio.Cash - self.upkeep)        

    def consolidation_handler(self, sender, consolidated):
        self.history.loc[consolidated.EndTime, consolidated.Symbol] = consolidated.Close
        self.history = self.history.iloc[-(VOLA + 1):]

    def createPlots(self, benchmark):
        self.__benchmark = benchmark

        self.__plot_every_n_days = 5
        self.__plot_every_n_days_i = 0
        plot = Chart('Performance')
        plot.AddSeries(Series(self.__benchmark, SeriesType.Line, 0, '%'))
        plot.AddSeries(Series("Algorithm", SeriesType.Line, 0, '%'))
        self.AddChart(plot)

        self.ResetPlot()

    def ResetPlot(self):
        self.year = self.Time.year
        self.__cost_portfolio = None
        self.__cost_benchmark = None

    def CalculateBenchmarkPerformance(self):
        price = self.Securities[self.__benchmark].Price
        if self.__cost_benchmark == None:
            self.__cost_benchmark = price
        return 100.0 * ((price / self.__cost_benchmark) - 1.0)
        
    def CalculatePortfolioPerformance(self):
        if self.__cost_portfolio == None:
            self.__cost_portfolio = self.Portfolio.TotalPortfolioValue
        return 100.0 * ((self.Portfolio.TotalPortfolioValue / self.__cost_portfolio) - 1.0)

    def OnEndOfDay(self):
        if self.IsWarmingUp or not self.Securities[self.__benchmark].HasData:
            return

        if self.Time.year != self.year:
            self.ResetPlot()
            self.__plot_every_n_days_i == -1

        self.__plot_every_n_days_i += 1
        if self.__plot_every_n_days_i % self.__plot_every_n_days != 0:
            return
        
        self.Plot('Performance', self.__benchmark, self.CalculateBenchmarkPerformance())
        self.Plot('Performance', "Algorithm", self.CalculatePortfolioPerformance())

    # def OnData(self, slice):
        
    #     action, symbol, strike, direction = self.grab_tweet()
    #     self.TradeOptions(slice, action, symbol, strike, direction)

    def user_filter(user, i):
        if 'TwinsightPro' in user:
            if '#ALERT' in i.text:
                return True
        else:
            return False

    def grab_tweet(self):
        eastern = timezone('US/Eastern')
        delta = timedelta(minutes=30)
        
        users = ['@TwinsightPro']
        
        for user in users:
            for i in self.api.user_timeline(id = '@TwinsightPro'):
                dtime = i._json.get('created_at')
                new_datetime = datetime.strftime(datetime.strptime(dtime,'%a %b %d %H:%M:%S +0000 %Y'), '%Y-%b-%d %H:%M:%S')
                tweet_time = datetime.strptime(new_datetime, '%Y-%b-%d %H:%M:%S').astimezone(eastern)
                _time = datetime.now().astimezone(eastern)
                time_diff = math.floor((_time - tweet_time) / timedelta(minutes=1))
                
                _is_retweet = i.retweeted
                _is_quote = i.is_quote_status
                _is_reply = i.in_reply_to_screen_name
                if time_diff < 5:
                    if not _is_retweet:
                        if not _is_quote:
                            if not _is_reply:
                                if user_filter(user, i):
                                    user = user.replace('@', '')
                                    tweet = i.text.split(' ')
                                    action = 'Buy' if tweet[1] == 'Bought' else 'Sell'
                                    symbol = tweet[2]
                                    strike = tweet[3][:-1]
                                    direction = 0 if tweet[3][-1]  == 'C' else 1
                                    price = tweet[4]
                                    return action, symbol, strike, direction
        
    def TradeOptions(self, slice, action, symbol, strike, direction):
        if slice.OptionChains.Count == 0: return   
        for i in slice.OptionChains:
            if i.Key != symbol: continue
            chain = i.Value
            call = [x for x in chain if (x.Right == direction) and (x.Strike == strike)] # filter the call options contracts
            # sorted the contracts according to their expiration dates and choose the ATM options
            contracts = sorted(sorted(call, key = lambda x: x.Expiry, reverse=True), 
                                            key = lambda x: abs(chain.Underlying.Price - x.Strike))
    
            if len(contracts) == 0: return    
            contract = contracts[0]
            self.call = contract.Symbol
            if action == 'Buy':
                self.Buy(self.call, 1)
            if action == 'Sell':
                self.Sell(self.call, 1) # short the call options

                
    def MarketOpen(self):
        return self.Time.hour != 0 and self.Time.minute == 1

from System import *
from QuantConnect import *
from QuantConnect.Algorithm import *
from QuantConnect.Indicators import *
from QuantConnect.Data.Market import TradeBar

import pandas as pd
import numpy as np
from scipy import stats
import statistics
from operator import itemgetter

from helpers import myPortfolioOptimizer

class PortfolioManagementModel(PortfolioConstructionModel):
    def __init__(self,
                 algorithm,
                 RET=252,
                 EXCL=21,
                 LEV=1.00,
                 resolution = Resolution.Daily,
                 *args, **kwargs):
        super().__init__()
        self.resolution = resolution
        self.RET = RET
        self.EXCL = EXCL
        self.LEV = LEV
        self.VOLA = 126
        self.STK1 = algorithm.AddEquity('SPY', self.resolution).Symbol # SPXL/SPY
        self.STK2 = algorithm.AddEquity('QQQ', self.resolution).Symbol # TQQQ/QQQ
        self.STK3 = algorithm.AddEquity('IWM', self.resolution).Symbol # URTY/IWM
        self.STK4 = algorithm.AddEquity('FDN', self.resolution).Symbol # FDN/FDN
        self.STK5 = algorithm.AddEquity('VTI', self.resolution).Symbol # AGQ/VTI
        self.STK6 = algorithm.AddEquity('DIA', self.resolution).Symbol # AGQ/VTI
        self.STK7 = algorithm.AddEquity('IWF', self.resolution).Symbol # AGQ/VTI
        self.BND1 = algorithm.AddEquity('TLH', self.resolution).Symbol # TMF/TLH
        self.BND2 = algorithm.AddEquity('TLT', self.resolution).Symbol # UGL/TLT
        self.BND3 = algorithm.AddEquity('IEI', self.resolution).Symbol # TMF/TLH
        self.BND4 = algorithm.AddEquity('IEF', self.resolution).Symbol # UGL/TLT
        self.STOCKS = [self.STK1, self.STK2, self.STK3, self.STK4, self.STK5, self.STK6, self.STK7]
        self.BONDS = [self.BND1, self.BND2, self.BND3, self.BND4]
        self.ASSETS = self.STOCKS + self.BONDS
        self.data = dict()
        
            
        for symbol in self.ASSETS:
            self.consolidator = TradeBarConsolidator(timedelta(days=1))
            self.consolidator.DataConsolidated += self.consolidation_handler
            algorithm.SubscriptionManager.AddConsolidator(symbol, self.consolidator)
            
        self.history = algorithm.History(self.ASSETS, self.VOLA + 1, self.resolution)
        self.pfo = myPortfolioOptimizer(minWeight=0, maxWeight=1)

        for symbol in self.ASSETS:
            algorithm.Securities[symbol].SetLeverage(1)
        
 
    def consolidation_handler(self, sender, consolidated):
        self.history.loc[consolidated.EndTime, consolidated.Symbol] = consolidated.Close
        self.history = self.history.iloc[-(self.VOLA + 1):]
        
    def OnSecuritiesChanged(self, algorithm, changes):
        addedSymbols = []
        for security in changes.AddedSecurities:
            addedSymbols.append(security.Symbol)
            if security.Symbol not in self.data:
                self.data[security.Symbol] = SymbolData(security.Symbol, algorithm)
   
        if len(addedSymbols) > 0:
            history = algorithm.History(addedSymbols, self.VOLA + 1, self.resolution).loc[addedSymbols]
            for symbol in addedSymbols:
                try:
                    self.data[symbol].Warmup(history.loc[symbol])
                except:
                    algorithm.Debug(str(symbol))
                    continue
        
    def returns_custom(self, symbol, timeframe):
        prices = np.log10(algorithm.History(symbol, TimeSpan.FromDays(self.VOLA), self.resolution).close)
        return round((prices[-timeframe] - prices[-5]) / prices[-5], 4)        

    def rel_vol(self, symbol):
        volume = self.data[symbol].sma.Current.Value
        volumew = self.data[symbol].smaw.Current.Value
        volume_ratio = round(float(volumew - volume)/volumew, 4)
        return volume_ratio

    def custom_filter(self, symbol, filter_type = 'both'):
        if filter_type == 'both':  # and (self.data[symbol].vol_slope >= 0) # self.data[symbol].superTrend.Current.Value
            if ((self.data[symbol].roc_slope >= 0.0) and (self.data[symbol].vol_slope >= 0.0)) and (self.data[symbol].vpnIndicator or self.data[symbol].breakout): 
                return True
            else:
                return False
        if filter_type == 'either': #  and (self.data[symbol].vol_slope >= 0)
            if ((self.data[symbol].roc_slope >= 0.0) and (self.data[symbol].vol_slope >= 0.0)) or (self.data[symbol].vpnIndicator or self.data[symbol].breakout):
                return True
            else:
                return False
        if filter_type == 'slope':
            if ((self.data[symbol].roc_slope > 0.0) and (self.data[symbol].vol_slope > 0.0)):
                return True
            else:
                return False
        if filter_type == 'breakout':
            if self.data[symbol].breakout:
                return True
            else:
                return False
                
    def CreateTargets(self, algorithm, insights):
        if algorithm.IsWarmingUp:
            return []

        targets = []
        
        # We expect at most only one active insight since we only
        # generate insights for one equity.
        assert len(insights) <= 1
        if len(insights) == 1:
            insight = insights[0]

            _in = round(statistics.median([round(statistics.median([self.data[symbol].Signal_fast.Current.Value, self.data[symbol].Signal_med.Current.Value, self.data[symbol].Signal_slow.Current.Value]), 4)
                                for symbol in [self.STK1, self.STK2, self.STK3, self.STK4]]), 4)
            _out = round(statistics.median([round(statistics.median([self.data[symbol].Signal_fast.Current.Value, self.data[symbol].Signal_med.Current.Value, self.data[symbol].Signal_slow.Current.Value]), 4)
                                for symbol in [self.BND1, self.BND2, self.BND3, self.BND4]]), 4)
            bleh = (_in -_out)/2   
            
            if insight.Direction != InsightDirection.Flat:
                if insight.Direction == InsightDirection.Down:
                    self.bull = False
                else:
                    self.bull = True
                
                algorithm.Plot("Volatility", self.STK1, float(self.data[self.STK1].volatility))
                selected = list()
                if self.bull: # , self.data[symbol].median_roc, self.data[symbol].volatility # self.data[symbol].superTrend.Current.Value
                    stocks = [(symbol, self.data[symbol].median_roc, self.data[symbol].vpnScale, (self.data[symbol].Signal_fast.Current.Value/max([self.data[self.STK1].Signal_fast.Current.Value, 0.05])), self.data[symbol].volatility) for symbol in self.STOCKS if self.custom_filter(symbol, filter_type = 'both') is True]
                    stocks.sort(key=itemgetter(1, 2, 3, 4), reverse=True)
                    for sec, roc, vpn, vola, sig in stocks[:1]:
                        selected.append(sec)
                    if len(selected) < 2: # , self.data[symbol].vpnScale, self.data[symbol].volatility
                        stocks = [(symbol, self.data[symbol].median_roc, self.data[symbol].vpnScale, (self.data[symbol].Signal_fast.Current.Value/max([self.data[self.STK1].Signal_fast.Current.Value, 0.05])), self.data[symbol].volatility) for symbol in self.STOCKS if self.custom_filter(symbol, filter_type = 'either') is True]
                        stocks.sort(key=itemgetter(1, 2, 3, 4), reverse=True)
                        for sec, roc, vpn, vola, sig in stocks:
                            if (len(selected) < 2) and (sec not in selected):
                                selected.append(sec)
                elif not self.bull:
                    bonds = [(symbol, self.data[symbol].median_roc, self.data[symbol].vpnScale, (self.data[symbol].Signal_fast.Current.Value/max([self.data[self.BND3].Signal_fast.Current.Value, 0.05])), self.data[symbol].volatility) for symbol in self.BONDS if self.custom_filter(symbol, filter_type = 'both') is True]
                    bonds.sort(key=itemgetter(1, 2, 3, 4), reverse=True)
                    for sec, roc, vpn, vola, sig in bonds[:1]:
                        selected.append(sec)
                    if len(selected) < 2:
                        # selected.append(self.BND3)
                        # selected.append(self.BND4)
                        bonds = [(symbol, self.data[symbol].median_roc, self.data[symbol].vpnScale, (self.data[symbol].Signal_fast.Current.Value/max([self.data[self.BND3].Signal_fast.Current.Value, 0.05])), self.data[symbol].volatility) for symbol in self.BONDS if self.custom_filter(symbol, filter_type = 'either') is True]
                        bonds.sort(key=itemgetter(1, 2, 3, 4), reverse=True)
                        for sec, roc, vpn, vola, sig in bonds:
                            if (len(selected) < 2) and (sec not in selected):
                                selected.append(sec)
            
            if insight.Direction == InsightDirection.Flat: 
                return []
            
            for asset in self.ASSETS:
                if asset in selected:
                    # if not algorithm.Portfolio[asset].Invested:
                    #     targets.append(PortfolioTarget.Percent(algorithm, asset, 0.25))
                    # elif algorithm.Portfolio[asset].Invested:
                    targets.append(PortfolioTarget.Percent(algorithm, asset, 0.5))
                else:
                    targets.append(PortfolioTarget.Percent(algorithm, asset, 0.0))
        return targets

class SymbolData(object):
    def __init__(self, symbol, algorithm):
        self.Symbol = symbol
        self.resolution = Resolution.Daily
        self.lookback = 20
        self.ceiling = 30
        self.floor = 10
        self.breakout = False
        self.fir = 0.00
        self.EXCL = 21
        self.scale = 0.00
        self.is_uptrend = False
        self.volatility = 0.00
        self.tolerance = 0.95
        self.vol_slope = 0.00
        self.roc_slope = 0.00
        self.median_roc = 0.00
        self.median_roc_safe = 0.00
        self.fast = VolumeWeightedAveragePriceIndicator(8)
        self.fast_ema_window = RollingWindow[float](5)
        self.slow = VolumeWeightedAveragePriceIndicator(14)
        self.slow_ema_window = RollingWindow[float](14)
        self.sma = SimpleMovingAverage(21)
        self.smaw = SimpleMovingAverage(5)
        self.roc = RateOfChange(5)
        self.roc_fast = RateOfChange(3)
        self.roc_med = RateOfChange(8)
        self.roc_long = RateOfChange(63)
        self.vol_roc = RateOfChange(5)
        self.vol_roc_fast = RateOfChange(3)
        self.vol_roc_med = RateOfChange(8)
        self.vol_roc_long = RateOfChange(63)
        self.roc_window = RollingWindow[float](5)
        self.vol_window = RollingWindow[float](5)
        self.prices_window = RollingWindow[float](41)
        self.high_window = RollingWindow[float](41)
        self.superTrend = SuperTrend(10, 3, movingAverageType = MovingAverageType.Wilders)
        self.psar = ParabolicStopAndReverse(0.2, 0.2, 0.2, MovingAverageType.Exponential)
        self.macd = MovingAverageConvergenceDivergence(12, 26, 9, MovingAverageType.Exponential)
        self.kama = KaufmanAdaptiveMovingAverage(10, MovingAverageType.Exponential)
        self.vpnIndicator = False
        self.vpnScale = 0.00
        self.vpn_period = 10
        self.atr = AverageTrueRange(self.vpn_period, MovingAverageType.Exponential)
        self.vpn_vol_window = RollingWindow[float](self.vpn_period)
        self.vpn_hlc_window = RollingWindow[float](self.vpn_period)
        self.vpn_list = RollingWindow[float](6)
        self.obvm = OnBalanceVolume(7, MovingAverageType.Exponential)
        self.obvmEMA = IndicatorExtensions.EMA(self.obvm, 3)
        self.obvmSignal = IndicatorExtensions.Over(IndicatorExtensions.Minus(self.obvm, self.obvmEMA), self.obvmEMA)
        self.rocSignal = IndicatorExtensions.Over(self.roc, self.roc_med)
        self.volSignal = IndicatorExtensions.Over(self.vol_roc, self.vol_roc_med)
        self.Signal_fast = IndicatorExtensions.Over(IndicatorExtensions.Minus(self.roc_fast, self.roc), IndicatorExtensions.Over(IndicatorExtensions.Plus(self.roc_fast, self.roc), 2.0))
        self.Signal_med = IndicatorExtensions.Over(IndicatorExtensions.Minus(self.roc, self.roc_med), IndicatorExtensions.Over(IndicatorExtensions.Plus(self.roc, self.roc_med), 2.0))
        self.Signal_slow = IndicatorExtensions.Over(IndicatorExtensions.Minus(self.roc_med, self.roc_long), IndicatorExtensions.Over(IndicatorExtensions.Plus(self.roc_med, self.roc_long), 2.0))
        self.signal_window = RollingWindow[float](5) # round(float(fast - slow) / ((fast + slow) / 2.0), 4)
        self.signal_median = 0.00
        
        self.consolidator = algorithm.ResolveConsolidator(symbol, self.resolution)
        algorithm.RegisterIndicator(symbol, self.roc, self.consolidator)

        self.consolidator = algorithm.ResolveConsolidator(symbol, self.resolution)
        algorithm.RegisterIndicator(symbol, self.roc_fast, self.consolidator)

        self.consolidator = algorithm.ResolveConsolidator(symbol, self.resolution)
        algorithm.RegisterIndicator(symbol, self.roc_med, self.consolidator)

        self.consolidator = algorithm.ResolveConsolidator(symbol, self.resolution)
        algorithm.RegisterIndicator(symbol, self.roc_long, self.consolidator)

        self.consolidator = algorithm.ResolveConsolidator(symbol, self.resolution)
        algorithm.RegisterIndicator(symbol, self.vol_roc, self.consolidator)

        self.consolidator = algorithm.ResolveConsolidator(symbol, self.resolution)
        algorithm.RegisterIndicator(symbol, self.vol_roc_fast, self.consolidator)

        self.consolidator = algorithm.ResolveConsolidator(symbol, self.resolution)
        algorithm.RegisterIndicator(symbol, self.vol_roc_med, self.consolidator)

        self.consolidator = algorithm.ResolveConsolidator(symbol, self.resolution)
        algorithm.RegisterIndicator(symbol, self.vol_roc_long, self.consolidator)

        self.consolidator = algorithm.ResolveConsolidator(symbol, self.resolution)
        algorithm.RegisterIndicator(symbol, self.fast, self.consolidator)
        
        self.consolidator = algorithm.ResolveConsolidator(symbol, self.resolution)
        algorithm.RegisterIndicator(symbol, self.slow, self.consolidator)

        self.consolidator = algorithm.ResolveConsolidator(symbol, self.resolution)
        algorithm.RegisterIndicator(symbol, self.smaw, self.consolidator)
        
        self.consolidator = algorithm.ResolveConsolidator(symbol, self.resolution)
        algorithm.RegisterIndicator(symbol, self.sma, self.consolidator)

        self.consolidator = algorithm.ResolveConsolidator(symbol, self.resolution)    
        algorithm.RegisterIndicator(symbol, self.superTrend, self.consolidator)
        
        self.consolidator = algorithm.ResolveConsolidator(symbol, self.resolution)    
        algorithm.RegisterIndicator(symbol, self.psar, self.consolidator)

        self.consolidator = algorithm.ResolveConsolidator(symbol, self.resolution)    
        algorithm.RegisterIndicator(symbol, self.macd, self.consolidator)

        self.consolidator = algorithm.ResolveConsolidator(symbol, self.resolution)    
        algorithm.RegisterIndicator(symbol, self.kama, self.consolidator)
        
        self.consolidator = algorithm.ResolveConsolidator(symbol, self.resolution)    
        algorithm.RegisterIndicator(symbol, self.atr, self.consolidator)   
        
        self.consolidator = algorithm.ResolveConsolidator(symbol, self.resolution)
        algorithm.RegisterIndicator(symbol, self.obvm, self.consolidator)

        self.consolidator = algorithm.ResolveConsolidator(symbol, self.resolution)
        algorithm.RegisterIndicator(symbol, self.obvmEMA, self.consolidator)
        
        # Warm up
        history = algorithm.History(symbol, 126, self.resolution)
        if history.empty or 'close' not in history.columns:
            return
        for index, row in history.loc[symbol].iterrows():
            tradeBar = TradeBar(index, row['open'], row['high'], row['low'], row['close'], row['volume'])
            self.roc.Update(index, row['close']) # index, row['close']
            self.roc_fast.Update(index, row['close'])
            self.roc_med.Update(index, row['close'])
            self.roc_long.Update(index, row['close'])
            self.rocSignal.Update(index, row['close'])
            self.vol_roc.Update(index, row['volume'])
            self.vol_roc_fast.Update(index, row['volume'])
            self.vol_roc_med.Update(index, row['volume'])
            self.vol_roc_long.Update(index, row['volume'])
            self.volSignal.Update(index, row['volume'])
            self.atr.Update(tradeBar)
            self.superTrend.Update(tradeBar)
            self.psar.Update(tradeBar)
            self.macd.Update(index, row['close'])
            self.fast.Update(tradeBar)
            self.slow.Update(tradeBar)
            self.smaw.Update(index, row['volume'])
            self.sma.Update(index, row['volume'])
            self.obvm.Update(tradeBar)
            self.kama.Update(index, row['close'])
            
            self.roc_window.Add(statistics.median([self.roc.Current.Value*1.25, self.roc_fast.Current.Value*1.5, self.roc_med.Current.Value, self.roc_long.Current.Value]))
            if self.roc_window.IsReady:
                roc_sum = sum(list(self.roc_window))
                roc_len = len(list(self.roc_window))
                self.roc_slope = round(float(roc_sum)/roc_len, 4)
            
            self.vol_window.Add(statistics.median([self.vol_roc.Current.Value*1.25, self.vol_roc_fast.Current.Value*1.5, self.vol_roc_med.Current.Value, self.vol_roc_long.Current.Value]))
            if self.vol_window.IsReady:
                vol_sum = sum(list(self.vol_window))
                vol_len = len(list(self.vol_window))
                self.vol_slope = round(float(vol_sum)/vol_len, 4)
            
            self.prices_window.Add(statistics.median([self.roc.Current.Value*1.25, self.roc_fast.Current.Value*1.5, self.roc_med.Current.Value, self.roc_long.Current.Value]))
            if self.prices_window.IsReady:
                prices = np.log10(list(self.prices_window))
                frames = [i for i in range(-2, -21, -2)]
                self.median_roc = round(statistics.median([round(float(prices[i] - prices[i-5]/ prices[i-5]), 4) if prices[i-5] != 0 else 0 for i in frames]), 4)
                
                #vola_list = pd.DataFrame(list(self.prices_window)).pct_change().rolling(2).std() * np.sqrt(252)
                #self.volatility = vola_list.pct_change().iloc[-1][0]
                    
            # # power_weighted_moving_average
                self.power = 1
                    
                weights = np.array([(i+1)**self.power for i in range(len(prices))])
                    
                self.fir = sum(prices * weights)/sum(weights)
                
            # OnBalanceVolumeModified
            # if self.obvm.Current.Value > self.obvmEMA.Current.Value:
            #     self.obvmSignal = True
            # else:
            #     self.obvmSignal = False

            # _in vs _out signal
            self.signal_window.Add(round(statistics.median([self.Signal_fast.Current.Value*2.5, self.Signal_med.Current.Value, self.Signal_slow.Current.Value]), 4))
            if self.signal_window.IsReady:
                sig_sum = round(sum(list(self.signal_window)), 4)
                sig_len = len(list(self.signal_window))
                if sig_sum == 0:
                    self.signal_median == 0.00
                else:
                    self.signal_median = round(sig_sum/sig_len, 4)
            
            self.high_window.Add(row['high'])
            if self.high_window.IsReady and self.prices_window.IsReady:
                close = list(self.prices_window)
                todayvol = np.std(close[1:31])
                yesterdayvol = np.std(close[0:30])
                deltavol = (todayvol - yesterdayvol) / todayvol
                self.lookback = round(self.lookback * (1 + deltavol))
                
                # Account for upper/lower limit of lockback length
                if self.lookback > self.ceiling:
                    self.lookback = self.ceiling
                elif self.lookback < self.floor:
                    self.lookback = self.floor
                    
                high = list(self.high_window)

                # Buy in case of breakout
                #if not algorithm.Securities[symbol].Invested and \
                breakout_condition1 = (algorithm.Securities[symbol].Close >= max(high[:-1])) and (round(float(algorithm.Securities[symbol].Close - algorithm.Securities[symbol].Open)/algorithm.Securities[symbol].Open, 4) >= -0.01)
                breakout_condition2 = (round(float(algorithm.Securities[symbol].Close - algorithm.Securities[symbol].Open)/algorithm.Securities[symbol].Open, 4) >= 0.02)
                if breakout_condition1 or breakout_condition2:
                    self.breakout = True
            
            fast = self.fast.Current.Value
            slow = self.slow.Current.Value
            self.is_uptrend = (fast > (slow * self.tolerance)) and (row[tradeBar] > (fast * self.tolerance))
            
        if self.is_uptrend:
            self.scale = round(float(fast - slow) / ((fast + slow) / 2.0), 4)
        else:
            self.scale = 0

        # VPN Indicator
        iATR = 0.1
        ema_smooth = 3
        vp = 0.0
        vn = 0.0
        vtot = 0.0
        dist = self.atr.Current.Value * iATR
        self.vpn_vol_window.Add(row['volume'])
        self.vpn_hlc_window.Add(statistics.median([row['high'], row['low'], row['close']]))
        if self.vpn_vol_window.IsReady and self.vpn_hlc_window.IsReady:
            vpn_vol_window = list(self.vpn_vol_window)
            vpn_hlc_window = list(self.vpn_hlc_window)
            for i in range(0, self.vpn_period - 1):
                if (vpn_hlc_window[i] > vpn_hlc_window[i+1] + dist):
                    vp += vpn_vol_window[i]
                elif (vpn_hlc_window[i] < vpn_hlc_window[i+1] - dist):
                    vn += vpn_vol_window[i]
                vtot += vpn_vol_window[i]
                
            vpn_val = (((vp - vn) / (vtot/self.vpn_period)) / self.vpn_period) * 100
                
            self.vpn_list.Add(vpn_val)

        if self.vpn_list.IsReady:
            vpn_ema = pd.DataFrame(list(self.vpn_list)).ewm(span=3, adjust=False).mean().iloc[-1][0]
            vpn_scale = self.vpn_list[2]
            vpnIndicator = (vpn_scale > (vpn_ema * self.tolerance)) and (row[tradeBar] > (vpn_scale * self.tolerance)) # True if vpn_scale >= vpn_ema else False
            self.vpnIndicator = vpnIndicator
        if self.vpnIndicator:
            vpnScale = round(float(vpn_scale - vpn_ema) / ((vpn_scale + vpn_ema) / 2.0), 4)
            self.vpnScale = vpnScale
        else:
            self.vpnScale = 0

from itertools import groupby
import tweepy
from datetime import datetime, timedelta, date
import time
import pandas as pd
import numpy as np
import re, math
import scipy
from math import ceil
from collections import deque
from itertools import chain

from System import *
from QuantConnect import *
from QuantConnect.Algorithm import *
from QuantConnect.Indicators import *
from QuantConnect.Data.Market import TradeBar
from QuantConnect.Algorithm.Framework.Execution import StandardDeviationExecutionModel, VolumeWeightedAveragePriceExecutionModel
from QuantConnect.Algorithm.Framework.Risk import MaximumDrawdownPercentPortfolio, MaximumUnrealizedProfitPercentPerSecurity, MaximumDrawdownPercentPerSecurity, TrailingStopRiskManagementModel

from dual_momentum_with_out_days_alpha import DualMomentumWithOutDaysAlphaModel
from trade_execution import ScheduledExecutionModel
from portfolio_management import PortfolioManagementModel

VOLA = 126; BASE_RET = 83; RET = 252; EXCL = 21; LEV = 1.00;

class HorizontalQuantumCoil(QCAlgorithm):

    def Initialize(self):
        self.Portfolio.MarginCallModel = MarginCallModel.Null
        self.SetStartDate(2008, 1, 1)
        self.SetCash(100000)
        self.added_cash = 115
        self.upkeep = 28
        self.simulate_live = False
        self.SetWarmUp(timedelta(252))
        self.Settings.FreePortfolioValuePercentage = 0.05
        #self.SetBrokerageModel(BrokerageName.AlphaStreams)
        self.SetAlpha(DualMomentumWithOutDaysAlphaModel(self, VOLA, BASE_RET, Resolution.Daily))

        stonks = ['FDN', 'QQQ', 'IWM', 'SPY', 'VTI', 'DIA', 'IWF', 'TLT', 'TLH', 'IEI', 'IEF']
        # lev_stonks = ['UDOW', 'TQQQ', 'URTY', 'SPXL', 'TMF', 'AGQ', 'UGL']
        # stonks = ['ITOT', 'IVV', 'IJH', 'IJR', 'XT', 'IHAK', 'IWFH', 'IDNA', 'IRBO', 'TECB', 'BFTR', 'BTEK', 'BMED']
        symbols = []
        for stonk in stonks:
            val = Symbol.Create(stonk, SecurityType.Equity, Market.USA)
            symbols.append(val)
        self.SetUniverseSelection(ManualUniverseSelectionModel(symbols))
        self.UniverseSettings.Resolution = Resolution.Daily
        self.SetPortfolioConstruction(PortfolioManagementModel(self, RET, EXCL, LEV, Resolution.Daily))
        self.SetExecution(ScheduledExecutionModel(self))
        # self.SetRiskManagement(MaximumUnrealizedProfitPercentPerSecurity(maximumUnrealizedProfitPercent = 0.10))
        # self.SetRiskManagement(MaximumDrawdownPercentPerSecurity(maximumDrawdownPercent = 0.075))
        # self.SetRiskManagement(MaximumDrawdownPercentPortfolio(maximumDrawdownPercent = 0.05, isTrailing = True))

        self.createPlots("SPY")
    
        # if self.simulate_live:
        #     self.Schedule.On(self.DateRules.Every(DayOfWeek.Monday), \
        #                      self.TimeRules.BeforeMarketClose("SPY", 0), \
        #                      self.AddCash)
    
        #     self.Schedule.On(self.DateRules.MonthStart("SPY"), \
        #                      self.TimeRules.BeforeMarketClose("SPY", 0), \
        #                      self.UpKeep)
    
    def AddCash(self):
        self.Portfolio.SetCash(self.Portfolio.Cash + self.added_cash)

    def UpKeep(self):
        self.Portfolio.SetCash(self.Portfolio.Cash - self.upkeep)        

    def consolidation_handler(self, sender, consolidated):
        self.history.loc[consolidated.EndTime, consolidated.Symbol] = consolidated.Close
        self.history = self.history.iloc[-(VOLA + 1):]

    def createPlots(self, benchmark):
        self.__benchmark = benchmark

        self.__plot_every_n_days = 5
        self.__plot_every_n_days_i = 0
        plot = Chart('Performance')
        plot.AddSeries(Series(self.__benchmark, SeriesType.Line, 0, '%'))
        plot.AddSeries(Series("Algorithm", SeriesType.Line, 0, '%'))
        self.AddChart(plot)

        self.ResetPlot()

    def ResetPlot(self):
        self.year = self.Time.year
        self.__cost_portfolio = None
        self.__cost_benchmark = None

    def CalculateBenchmarkPerformance(self):
        price = self.Securities[self.__benchmark].Price
        if self.__cost_benchmark == None:
            self.__cost_benchmark = price
        return 100.0 * ((price / self.__cost_benchmark) - 1.0)

    def CalculatePortfolioPerformance(self):
        if self.__cost_portfolio == None:
            self.__cost_portfolio = self.Portfolio.TotalPortfolioValue
        return 100.0 * ((self.Portfolio.TotalPortfolioValue / self.__cost_portfolio) - 1.0)

    def OnEndOfDay(self):
        if self.IsWarmingUp or not self.Securities[self.__benchmark].HasData:
            return

        if self.Time.year != self.year:
            self.ResetPlot()
            self.__plot_every_n_days_i == -1

        self.__plot_every_n_days_i += 1
        if self.__plot_every_n_days_i % self.__plot_every_n_days != 0:
            return
        
        #self.Plot('Performance', self.__benchmark, self.CalculateBenchmarkPerformance())
        #self.Plot('Performance', "Algorithm", self.CalculatePortfolioPerformance())
    
    def UpdateTickets(self):
        invested = [x.Key for x in self.Portfolio if x.Value.Invested]
        for symbol in invested:
            if self.breakout(symbol):
                close = self.Securities[symbol].Close
                quantity = self.Portfolio[symbol].Quantity
                self.StopMarketOrder(symbol, quantity, close * 0.95)
                self.LimitOrder(symbol, int(quantity/2), close * 1.05)
      
    def breakout(self, symbol):
        self.lookback = 20
        self.ceiling, self.floor = 30, 10
        
        close = self.History(symbol, 31, Resolution.Daily)["close"]
        todayvol = np.std(close[1:31])
        yesterdayvol = np.std(close[0:30])
        deltavol = (todayvol - yesterdayvol) / todayvol
        self.lookback = round(self.lookback * (1 + deltavol))

        # Account for upper/lower limit of lockback length
        if self.lookback > self.ceiling:
            self.lookback = self.ceiling
        elif self.lookback < self.floor:
            self.lookback = self.floor

        # List of daily highs
        self.high = self.History(symbol, self.lookback, Resolution.Daily)["high"]
        
        # Buy in case of breakout
        #if not self.Securities[symbol].Invested and \
        if self.Securities[symbol].Close >= max(self.high[:-1]):
            return True

from System import *
from QuantConnect import *

from sklearn.ensemble import ExtraTreesClassifier, ExtraTreesRegressor
from sklearn.model_selection import train_test_split, cross_validate, cross_val_score, cross_val_predict
import tweepy, statistics
from datetime import datetime, timedelta, date
import numpy as np
import pickle
from scipy import stats

from helpers import myTrailingStopRiskManagementModel

class DualMomentumWithOutDaysAlphaModel(AlphaModel):

    def __init__(self, algorithm, VOLA = 126, BASE_RET = 83, resolution = Resolution.Daily, *args, **kwargs):
        super().__init__()
        self.algorithm = algorithm
        self.VOLA = VOLA
        self.BASE_RET = BASE_RET
        self.resolution = resolution
        self.MKT = algorithm.AddEquity('SPY', resolution).Symbol
        self.SLV = algorithm.AddEquity('SLV', resolution).Symbol
        self.GLD = algorithm.AddEquity('GLD', resolution).Symbol
        self.XLI = algorithm.AddEquity('XLI', resolution).Symbol
        self.XLU = algorithm.AddEquity('XLU', resolution).Symbol
        self.DBB = algorithm.AddEquity('DBB', resolution).Symbol
        self.UUP = algorithm.AddEquity('UUP', resolution).Symbol

        self.count = self.BASE_RET
        self.outday = 5
        self.data = dict()

        pairs = [self.MKT, self.SLV, self.GLD, self.XLI, self.XLU, self.DBB, self.UUP]
        for symbol in pairs:
            self.consolidator = TradeBarConsolidator(timedelta(days=1))
            self.consolidator.DataConsolidated += self.consolidation_handler
            algorithm.SubscriptionManager.AddConsolidator(symbol, self.consolidator)

        self.history = algorithm.History(pairs, self.VOLA + 1, self.resolution)
        
        # Set TrainingMethod to be executed immediately
        #algorithm.Train(self.TrainingMethod)

        self.predictionInterval = Time.Multiply(Extensions.ToTimeSpan(self.resolution), 1)
        resolutionString = Extensions.GetEnumString(resolution, Resolution)
        self.Name = f"{self.__class__.__name__}({resolutionString})"

        # Force alpha to only produce insights Daily at 11.10am
        self.set_flag = False
        algorithm.Schedule.On(algorithm.DateRules.EveryDay(),
                              algorithm.TimeRules.AfterMarketOpen('SPY', 60),
                              self.SetFlag)
        
        #algorithm.Train(algorithm.DateRules.Every(DayOfWeek.Monday), algorithm.TimeRules.At(1, 0), self.TrainingMethod)
                                
                              
    def SetFlag(self):
        self.set_flag = True

    def consolidation_handler(self, sender, consolidated):
        self.history.loc[consolidated.EndTime, consolidated.Symbol] = consolidated.Close
        self.history = self.history.iloc[-(self.VOLA + 1):]
        
    def OnSecuritiesChanged(self, algorithm, changes):
        addedSymbols = []
        for security in changes.AddedSecurities:
            addedSymbols.append(security.Symbol)
            if security.Symbol not in self.data:
                self.data[security.Symbol] = SymbolData(security.Symbol, algorithm)
   
        if len(addedSymbols) > 0:
            history = self.algorithm.History(addedSymbols, self.VOLA + 1, Resolution.Daily).loc[addedSymbols]
            for symbol in addedSymbols:
                try:
                    self.data[symbol].Warmup(history.loc[symbol])
                except:
                    self.algorithm.Debug(str(symbol))
                    continue
        
    def TrainingMethod(self):

        self.classifier = ExtraTreesClassifier(n_estimators=500,
                                          criterion = "gini",
                                          min_samples_split = 5,
                                          random_state = 1990)
        
        pairs = [self.MKT, self.SLV, self.GLD, self.XLI, self.XLU, self.DBB, self.UUP]

        if self.history is None: return
        df = self.algorithm.History(pairs, 252, Resolution.Daily)

        # Prepare data
        returns = df.unstack(level=1).close.transpose().dropna()
        returns['Volatility'] = returns[self.MKT].pct_change().rolling(self.VOLA).std() * np.sqrt(252)
        returns["Wait_Days"] = returns['Volatility'].apply(lambda x: self.wait_days(x))
        returns["Period"] = returns['Volatility'].apply(lambda x: self.period(x))
        returns["Exit_Market"] = returns["Period"].apply(lambda x: self.exit_market(x))
        
        returns = returns.dropna()
        
        y_clf = returns["Exit_Market"].apply(lambda x: 1 if True else 0).values
        
        del returns['Exit_Market']
        
        X = returns
        
        #y_clf = X["Exit_Market"].apply(lambda x: 1 if True else 0).values
        X_train, X_test, y_train_clf, y_test_clf = train_test_split(X, y_clf, test_size = 0.2, random_state = 1990)
        
        self.classifier.fit(X_train, y_train_clf)

        
    def Update(self, algorithm, _data):
        if algorithm.IsWarmingUp or not self.set_flag:
            return []

        self.set_flag = False
        insights = []
        
        #if self.classifier is None: return
        # Volatility
        vola = self.history[self.MKT].pct_change().std() * np.sqrt(252)
        wait_days = int(vola * self.BASE_RET)
        period = int((1.0 - vola) * self.BASE_RET)
        r = self.history.pct_change(period).iloc[-1]

        exit_market = r[self.SLV] < r[self.GLD] and r[self.XLI] < r[self.XLU] and r[self.DBB] < r[self.UUP]
        
        # pairs = [self.MKT, self.SLV, self.GLD, self.XLI, self.XLU, self.DBB, self.UUP]
        # data = self.history[pairs]
        # data['Volatility'] = vola
        # data['Wait_Days'] = int(vola * self.BASE_RET)
        # data['Period'] = int((1.0 - vola) * self.BASE_RET)
        
        # exit_market = self.classifier.predict(data.tail(1))
        self.algorithm.Debug(self.data[self.MKT].prices)

        direction = InsightDirection.Down
            
        if (exit_market):
            #algorithm.Plot("In vs Out", "Market", -1)
            direction = InsightDirection.Down
            self.outday = self.count
        else:
            if (self.count >= wait_days + self.outday):
                #algorithm.Plot("In vs Out", "Market", 1)
                direction = InsightDirection.Up
        self.count += 1
        
        algorithm.Plot("Exit", str(self.MKT), int(exit_market))
        
        # algorithm.Plot("Wait Days", "Actual", self.count)
        # algorithm.Plot("Wait Days", "Expected", wait_days + self.outday)
                                                                    
        insights.append(Insight.Price(self.MKT, self.predictionInterval, direction))

        return insights

class SymbolData(object):
    def __init__(self, symbol, algorithm):
        self.Symbol = symbol
        self.VOLA = 126
        self.prices = RollingWindow[float](126)
        self.prices_window = RollingWindow[float](126)
        self.algorithm = algorithm
        
        self.consolidator = algorithm.ResolveConsolidator(symbol, Resolution.Daily)
        
        # Warm up
        history = self.algorithm.History(symbol, 126, Resolution.Daily)
        if history.empty or 'close' not in history.columns:
            return
        for index, row in history.loc[symbol].iterrows():
            self.prices_window.Add(row['close'])
            if self.prices_window.IsReady:
                self.prices = list(self.prices_window)
                
        
    def wait_days(self, x):
        try:
            val = int(x * self.BASE_RET)
        except:
            val = np.nan
        return val

    def period(self, x):
        try:
            val = int((1.0 - x) * self.BASE_RET)
        except:
            val = np.nan
        return val
 
    def exit_market(self, x):
        try:
            x = int(x)
            r = self.history.pct_change(x).iloc[-1]
            cond1 = r[self.SLV] < r[self.GLD]
            cond2 = r[self.XLI] < r[self.XLU]
            cond3 = r[self.DBB] < r[self.UUP]
            exit_market = cond1 and cond2 and cond3
        except:
            exit_market = np.nan
        return exit_market