#insights.append(Insight(symbol, self.insightsTimeDelta, InsightType.Price, symbolData.InsightDirection, None,None, None,0.1))
                #algorithm.Log(f"{symbol}\tMOM\t[{symbolData.fmom}]\t{round(symbolData.mom.Current.Value,2)}\tKAMA\t[{symbolData.fkama}]\t{round(symbolData.kama.Current.Value,2)}\
                #                \tPrice\t{symbolData.price}\tROC\t[{symbolData.froc}]\t{round(symbolData.roc.Current.Value,4)}\tEMA\t[{symbolData.fema}]\tEMA-13\t{round(symbolData.ema13.Current.Value,2)}\
                #                \tEMA-63\t{round(symbolData.ema63.Current.Value,2)}\tEMA-150\t{round(symbolData.ema150.Current.Value,2)}\taction\t{symbolData.InsightDirection}")

            
          #self.SetPortfolioConstruction(EqualWeightingPortfolioConstructionModel())
        #self.SetPortfolioConstruction(MeanVarianceOptimizationPortfolioConstructionModel(param.resolution,PortfolioBias.LongShort,1,63,param.resolution,0.02,MaximumSharpeRatioPortfolioOptimizer(0,1,0)))
        
        
            # self.rebalancingPeriod  = Expiry.EndOfMonth
            #pcm = InsightWeightingPortfolioConstructionModel(lambda time: param.rebalancingPeriod(time))
            
                        #self.InsightDirection   =   InsightDirection.Up
                                    #self.InsightDirection = InsightDirection.Flat # liqudates position - work around InsightDirection.Down which may sell and then short

import numpy as np
import pandas as pd
from datetime import datetime, date
from datetime import datetime, timedelta
from PortfolioOptimizerClass import PortfolioOptimizer

from System.Drawing import Color

class ModelA(AlphaModel): 
    
    def __init__(self, param):

        self.param                  = param
        self.symbolDataBySymbol     = {}
        self.modelResolution        = param.resolution
        self.insightsTimeDelta      = param.timedelta
        self.objectiveFunction      = param.pcmObjectiveFunction
        self.lookbackOptimization   = param.pcmLookbackOptimization
        self.portOpt                = PortfolioOptimizer(minWeight = 0, maxWeight = 1)
        
    def OnSecuritiesChanged(self, algorithm, changes):
            for added in changes.AddedSecurities:
                symbolData = self.symbolDataBySymbol.get(added.Symbol)
                if symbolData is None:
                    symbolData = SymbolData(added.Symbol, algorithm, self.param)
                    self.symbolDataBySymbol[added.Symbol] = symbolData
 
    def Update(self, algorithm, data):
        
        insights=[]
        liquidate=[]

        invested = [ x.Symbol.Value for x in algorithm.Portfolio.Values if x.Invested ] # can we make this easier via key query?
       
        for symbol, symbolData in self.symbolDataBySymbol.items():
 
            isInvested= str(symbol) in invested  
            
            if symbol != self.param.benchmark:

                symbolData.getInsight(algorithm.Securities[symbol].Price, isInvested) # Latest known price; we are at 12:00 and the last trade at 10.57 
            
                if symbolData.trade:
                    
                    if symbolData.liquidate:
                        invested.remove(str(symbol))
                        liquidate.append(str(symbol))
                        algorithm.Debug(f"sell {str(symbol)}")
                    else:
                        invested.append(str(symbol))
                        algorithm.Debug(f"buy {str(symbol)}")
                    

        # calculate optimal weights
        if invested:

            weights = self.CalculateOptimalWeights(algorithm, invested, self.objectiveFunction, self.lookbackOptimization)

            for symbol in invested:

                weight = weights[str(symbol)]
                insights.append(Insight.Price(symbol, self.insightsTimeDelta, InsightDirection.Up,
                                 None, None, None, weight))
        if liquidate:
            for symbol in liquidate:
                insights.append(Insight.Price(symbol, self.insightsTimeDelta, InsightDirection.Flat,
                                 None, None, None, 1))
            
        return insights

    def CalculateOptimalWeights(self, algorithm, symbols, objectiveFunction, lookbackOptimization):
            
        # get historical close prices
        historyClosePrices = algorithm.History(symbols, lookbackOptimization, Resolution.Daily)['close'].unstack(level = 0)
        
        # calculate daily returns
        returnsDf = historyClosePrices.pct_change().dropna()
        # rename the columns in the dataframe in order to have tickers and not symbol strings
        columnsList = list(returnsDf.columns)
        returnsDf.rename(columns = {columnsList[i]: algorithm.ActiveSecurities[columnsList[i]].Symbol.Value for i in range(len(columnsList))}, inplace = True)
        
        # calculate optimal weights
        weights = self.portOpt.Optimize(objectiveFunction, returnsDf)
        # convert the weights to a pandas Series
        weights = pd.Series(weights, index = returnsDf.columns, name = 'weights')
        
        return weights
        
class FrameworkAlgorithm(QCAlgorithm):
    
    def Initialize(self):

        param=paramData()
        symbols             =   [Symbol.Create(x, SecurityType.Equity, Market.USA) for x in param.tickers]
        
        self.SetStartDate(param.dateFrom[0],param.dateFrom[1],param.dateFrom[2])   
        self.SetEndDate(param.dateTo[0],param.dateTo[1],param.dateTo[2])   
        self.SetCash(param.cash)           
     
        self.SetBenchmark(param.benchmarkTicker)
        param.setBenchmark(self.AddEquity(param.benchmarkTicker,param.resolution).Symbol)
        
        self.UniverseSettings.Resolution = param.resolution
        self.SetWarmUp(timedelta(param.warmup)) 
        self.SetUniverseSelection(ManualUniverseSelectionModel(symbols))
        self.SetBrokerageModel(AlphaStreamsBrokerageModel()) # learn more about this
        self.SetAlpha(ModelA(param))
                                
        myPCM = InsightWeightingPortfolioConstructionModel(rebalance = timedelta(days=252), portfolioBias = PortfolioBias.Long)
        myPCM.RebalanceOnInsightChanges = False
        myPCM.RebalanceOnSecurityChanges = False 
        self.SetPortfolioConstruction(myPCM)
        
        self.SetRiskManagement(NullRiskManagementModel()) # MaximumDrawdownPercentPerSecurity(param.maxDrawDown) > drop in profit from the max >> done daily / TODO: redo hourly? or 
        self.SetExecution(ImmediateExecutionModel())
        self.Schedule.On(self.DateRules.EveryDay(), self.TimeRules.Every(TimeSpan.FromMinutes(param.runEveryXminutes)), self.hourlyHouseKeeping)

    def hourlyHouseKeeping(self):
        
            # Fail Safe - If our strategy is losing than acceptable (something is wrong)
            # Strategy suddenly losing money or logic problem/bug we did't catch when testing
            pnl= sum([self.Portfolio[symbol].NetProfit for symbol in self.Portfolio.Keys])
            #if self.LiveMode:
            if pnl < -5000: # can't pass value via parametr? 
                self.Log(f"Fallback event triggered, liquidating with total portfolio loss of {pnl}")
                self.Liquidate()
                self.Quit()
            
            dt=int(self.Time.hour)
            if dt >9 and dt<18: # if not set still prints out of hours for self.IsMarketOpen("SPY")
                if (self.IsMarketOpen("SPY") and self.Portfolio.Invested):
                    self.Log("\n\nPortfolio")
                    summary = {}
    
                    invested = [ x.Symbol.Value for x in self.Portfolio.Values if x.Invested ]
                    for symbol in invested:
                    
                        hold_val    = round(self.Portfolio[symbol].HoldingsValue, 2)
                        abs_val     = round(self.Portfolio[symbol].AbsoluteHoldingsValue, 2)
                        pnl         = round(self.Portfolio[symbol].UnrealizedProfit, 2)
                        qty         = self.Portfolio[symbol].Quantity
                        price       = self.Portfolio[symbol].Price
                        
                        summary[symbol]=[hold_val,abs_val,pnl,qty,price]
                        
                    df=pd.DataFrame(summary)
                    df.index = ['hold_val', 'abs_val', 'pnl', 'qty','price']
                    df=df.T
                    hold_val_total= abs(df['hold_val']).sum()
                    df = df.assign(weight=abs(df['hold_val'])/hold_val_total)
                    self.Log(df)
                    self.Log("\n\n")

class paramData:
    def __init__(self):
        self.dateFrom                   = (2020,9,10)    
        self.dateTo                     = (2020,11,10)   
        self.cash                       = 50000 # how to top this up after going live?
        self.warmup                     = 28 # starts from self.dateFrom
        self.resolution                 = Resolution.Daily  #10-11, etc Daily data is midnight to mifnight, 12AM EST 
        self.tickers                    = ["MSFT"] # how do I change this on request?
        self.fallback_barrier           = -1000
        self.timedelta                  = timedelta(hours=240)
        self.maxDrawDown                = 0.02
        self.runEveryXminutes           = 60 # Schedule frequency
        self.benchmarkTicker            = 'SPY' # can be ticker as a part of the dictionary ["MSFT:SPY"]
        self.pcmObjectiveFunction       = 'equalWeighting'
        self.pcmLookbackOptimization    = 63

        
    def setBenchmark(self, symbol):
        self.benchmark          =  symbol
        
class SymbolData:
    
    def __init__(self, symbol, algorithm, param):
        
        self.symbol             = symbol
        self.algorithm          = algorithm
        self.param              = param
        self.resolution         = param.resolution
        self.price              = 0.00
        self.kama               = algorithm.KAMA(symbol, 10,2,30, self.resolution)
        self.variationRate      = 1.03 # tolerance level to avoid buy and immediate sell scenario
        self.mom                = algorithm.MOM(symbol, 14, self.resolution)
        self.roc                = algorithm.ROC(symbol, 9, self.resolution) 
        self.ema13              = algorithm.EMA(symbol, 13, self.resolution)
        self.ema63              = algorithm.EMA(symbol, 63, self.resolution)
        self.ema150             = algorithm.EMA(symbol, 150, self.resolution)
        self.fkama              = False
        self.fmom               = False
        self.froc               = False
        self.fema               = False
        self.rsStock            = False
        self.rsIdx              = False
        
        #algorithm.STD()
        
        
        # Chart Plotting 
        self.kama.Updated       += self.getRSL
        self.kama.Updated       += self.OnSymbolDataUpdate
        self.dataPlot = Chart('Detail'+str(self.symbol))
        self.dataPlot.AddSeries(Series('Price', SeriesType.Line, '$'))
        self.dataPlot.AddSeries(Series('Kama', SeriesType.Line, '$'))
        self.dataPlot.AddSeries(Series('MOM', SeriesType.Line, ''))
        self.dataPlot.AddSeries(Series('EMA13', SeriesType.Line, '$'))
        self.dataPlot.AddSeries(Series('EMA63', SeriesType.Line, '$'))
        self.dataPlot.AddSeries(Series('EMA150', SeriesType.Line, '$'))
        self.dataPlot.AddSeries(Series('ROC', SeriesType.Line, ''))
        self.dataPlot.AddSeries(Series('RS-stock', SeriesType.Line, ''))
        self.dataPlot.AddSeries(Series('RS-idx', SeriesType.Line, ''))
        self.dataPlot.AddSeries(Series('Buy', SeriesType.Scatter, '$', Color.Green,ScatterMarkerSymbol.Circle))
        self.dataPlot.AddSeries(Series('Sell', SeriesType.Scatter, '$', Color.Red,ScatterMarkerSymbol.Circle))
        self.algorithm.AddChart(self.dataPlot)
        
    def getInsight(self, price, isInvested):    
        
        self.price              = price
        self.fkama_buy          = self.price>self.kama.Current.Value
        self.fkama_sell         = self.price*1/self.variationRate<self.kama.Current.Value
        self.fmom               = self.mom.Current.Value>0
        self.froc               = self.roc.Current.Value>0
        self.fema               = self.ema13.Current.Value>self.ema63.Current.Value>self.ema150.Current.Value

        self.trade              = False
        self.liquidate          = False
        
        # and self.fkama_buy and self.fema and self.froc
        
        if not isInvested and self.fmom:
            self.trade              =   True
            self.algorithm.Plot('Detail'+str(self.symbol),'Buy', self.price)

        # or self.fkama_sell  or not self.fema or not self.froc
        if isInvested and (not self.fmom):
            self.trade              =   True
            self.liquidate          =   True
            self.algorithm.Plot('Detail'+str(self.symbol),'Sell',self.price)
    

    def OnSymbolDataUpdate(self, sender, updated):
        
        self.algorithm.Plot('Detail'+str(self.symbol),'Price', self.price)
        self.algorithm.Plot('Detail'+str(self.symbol),'Kama', self.kama.Current.Value)
        self.algorithm.Plot('Detail'+str(self.symbol),'ROC', self.roc.Current.Value)
        self.algorithm.Plot('Detail'+str(self.symbol),'MOM', self.mom.Current.Value)
        self.algorithm.Plot('Detail'+str(self.symbol),'EMA13', self.ema13.Current.Value)
        self.algorithm.Plot('Detail'+str(self.symbol),'EMA63', self.ema63.Current.Value)
        self.algorithm.Plot('Detail'+str(self.symbol),'EMA150', self.ema150.Current.Value)
        
    def getRSL(self, sender, updated):

        # lookback days : algo weight 
        days = {40:0.5,80:0.25,160:0.25}

        rs = {}

        for symbol in [self.symbol,self.param.benchmark]:
 
            result =[]
            
            df=pd.DataFrame(self.algorithm.History(symbol, 300, Resolution.Daily)) 
            df=df.iloc[::-1]
            df=df.reset_index(level=0, drop=True)
            
            symbol = str(symbol)
            for x in days:
                result.append([symbol, x, df.iloc[0]['close'], df.iloc[x-1]['close'],days[x]])

            df = pd.DataFrame(result,columns=['Symbol','Days','Ref_Price','Close_Price','Weight'],dtype=float)
            df = df.assign(Rsl=(df['Ref_Price'])/df['Close_Price']*df['Weight'])
            
            rs[symbol] = (abs(df['Rsl']).sum()*1000)-1000
        
        self.rsStock    = rs[str(self.symbol)]
        self.rsIdx      = rs[str(self.param.benchmark)]
        
        self.algorithm.Plot('Detail'+str(self.symbol),'RS-stock',self.rsStock )
        self.algorithm.Plot('Detail'+str(self.symbol),'RS-idx', self.rsIdx)

from clr import AddReference
AddReference("QuantConnect.Research")
#clr.AddReference('QuantConnect.Research')
from QuantConnect.Research import QuantBook

class RelativeStrengthLineCalc():

    def getRSL(self, ref_date, symbols):

        self.rsl_target_days     = [40,80,160] 
        self.rsl_target_weights  = [0.5,0.25,0.25]

        qb = QuantBook()
    
        date_end = datetime(ref_date)
        date_start = date_end - timedelta(days=300)
        
        for symbol in symbols:
            
            smbl = qb.AddEquity(symbol) # add equity data
            result =[]
            
            history = qb.History(smbl.Symbol, date_start, date_end, Resolution.Daily)
            df=pd.DataFrame(history)
            df=df.iloc[::-1]
            df=df.reset_index(level=0, drop=True)
            
            i=0
            for x in rsl_target_days:
                result.append([symbol, x, df.iloc[0]['close'], df.iloc[x-1]['close'],rsl_target_weights[i]])
                i=i+1
            df = pd.DataFrame(result,columns=['Symbol','Days','Ref_Price','Close_Price','Weight'],dtype=float)
            df = df.assign(Rsl=(df['Ref_Price'])/df['Close_Price']*df['Weight'])
            rsl=(abs(df['Rsl']).sum()*1000)-1000
            
        return  rsl

class RelativeStrengthLineCalc():

    def getRSL():

        rsl_target_days     = [40,80,160] 
        rsl_target_weights  = [0.5,0.25,0.25]
        
        return  1

import pandas as pd
import numpy as np
from scipy.optimize import minimize

class PortfolioOptimizer:
    
    '''
    Description:
        Implementation of a custom optimizer that calculates the weights for each asset to optimize a given objective function
    Details:
        Optimization can be:
            - Equal Weighting
            - Maximize Portfolio Return
            - Minimize Portfolio Standard Deviation
            - Mean-Variance (minimize Standard Deviation given a target return)
            - Maximize Portfolio Sharpe Ratio
            - Maximize Portfolio Sortino Ratio
            - Risk Parity Portfolio
        Constraints:
            - Weights must be between some given boundaries
            - Weights must sum to 1
    '''
    
    def __init__(self, 
                 minWeight = 0,
                 maxWeight = 1):
                     
        '''
        Description:
            Initialize the CustomPortfolioOptimizer
        Args:
            minWeight(float): The lower bound on portfolio weights
            maxWeight(float): The upper bound on portfolio weights
        '''
        
        self.minWeight = minWeight
        self.maxWeight = maxWeight

    def Optimize(self, objFunction, dailyReturnsDf, targetReturn = None):
        
        '''
        Description:
            Perform portfolio optimization given a series of returns
        Args:
            objFunction: The objective function to optimize (equalWeighting, maxReturn, minVariance, meanVariance, maxSharpe, maxSortino, riskParity)
            dailyReturnsDf: DataFrame of historical daily arithmetic returns
        Returns:
            Array of double with the portfolio weights (size: K x 1)
        '''
        
        # initial weights: equally weighted
        size = dailyReturnsDf.columns.size # K x 1
        self.initWeights = np.array(size * [1. / size])

        # get sample covariance matrix
        covariance = dailyReturnsDf.cov()
        # get the sample covariance matrix of only negative returns for sortino ratio
        negativeReturnsDf = dailyReturnsDf[dailyReturnsDf < 0]
        covarianceNegativeReturns = negativeReturnsDf.cov()
        
        if objFunction == 'equalWeighting':
            return self.initWeights
        
        bounds = tuple((self.minWeight, self.maxWeight) for x in range(size))
        constraints = [{'type': 'eq', 'fun': lambda x: np.sum(x) - 1.0}]
        
        if objFunction == 'meanVariance':
            # if no target return is provided, use the resulting from equal weighting
            if targetReturn is None:
                targetReturn = self.CalculateAnnualizedPortfolioReturn(dailyReturnsDf, self.initWeights)
            constraints.append( {'type': 'eq', 'fun': lambda weights:
                                self.CalculateAnnualizedPortfolioReturn(dailyReturnsDf, weights) - targetReturn} )

        opt = minimize(lambda weights: self.ObjectiveFunction(objFunction, dailyReturnsDf,
                                                                covariance, covarianceNegativeReturns,
                                                                weights),
                        x0 = self.initWeights,
                        bounds = bounds,
                        constraints = constraints,
                        method = 'SLSQP')

        return opt['x']

    def ObjectiveFunction(self, objFunction, dailyReturnsDf, covariance, covarianceNegativeReturns, weights):
        
        '''
        Description:
            Compute the objective function
        Args:
            objFunction: The objective function to optimize (equalWeighting, maxReturn, minVariance, meanVariance,
                                                                maxSharpe, maxSortino, riskParity)
            dailyReturnsDf: DataFrame of historical daily returns
            covariance: Sample covariance
            covarianceNegativeReturns: Sample covariance matrix of only negative returns
            weights: Portfolio weights
        '''
    
        if objFunction == 'maxReturn':
            f = self.CalculateAnnualizedPortfolioReturn(dailyReturnsDf, weights)
            return -f # convert to negative to be minimized
        elif objFunction == 'minVariance':
            f = self.CalculateAnnualizedPortfolioStd(covariance, weights)
            return f
        elif objFunction == 'meanVariance':
            f = self.CalculateAnnualizedPortfolioStd(covariance, weights)
            return f
        elif objFunction == 'maxSharpe':
            f = self.CalculateAnnualizedPortfolioSharpeRatio(dailyReturnsDf, covariance, weights)
            return -f # convert to negative to be minimized
        elif objFunction == 'maxSortino':
            f = self.CalculateAnnualizedPortfolioSortinoRatio(dailyReturnsDf, covarianceNegativeReturns, weights)
            return -f # convert to negative to be minimized
        elif objFunction == 'riskParity':
            f = self.CalculateRiskParityFunction(covariance, weights)
            return f
        else:
            raise ValueError(f'PortfolioOptimizer.ObjectiveFunction: objFunction input has to be one of equalWeighting,'
             + ' maxReturn, minVariance, meanVariance, maxSharpe, maxSortino or riskParity')
        
    def CalculateAnnualizedPortfolioReturn(self, dailyReturnsDf, weights):
        
        annualizedPortfolioReturns = np.sum( ((1 + dailyReturnsDf.mean())**252 - 1) * weights )
        
        return annualizedPortfolioReturns
            
    def CalculateAnnualizedPortfolioStd(self, covariance, weights):
        
        annualizedPortfolioStd = np.sqrt( np.dot(weights.T, np.dot(covariance * 252, weights)) )
        
        if annualizedPortfolioStd == 0:
            raise ValueError(f'PortfolioOptimizer.CalculateAnnualizedPortfolioStd: annualizedPortfolioStd cannot be zero. Weights: {weights}')
            
        return annualizedPortfolioStd
    
    def CalculateAnnualizedPortfolioNegativeStd(self, covarianceNegativeReturns, weights):
    
        annualizedPortfolioNegativeStd = np.sqrt( np.dot(weights.T, np.dot(covarianceNegativeReturns * 252, weights)) )        
    
        if annualizedPortfolioNegativeStd == 0:
            raise ValueError(f'PortfolioOptimizer.CalculateAnnualizedPortfolioNegativeStd: annualizedPortfolioNegativeStd cannot be zero. Weights: {weights}')
        
        return annualizedPortfolioNegativeStd
        
    def CalculateAnnualizedPortfolioSharpeRatio(self, dailyReturnsDf, covariance, weights):
        
        annualizedPortfolioReturn = self.CalculateAnnualizedPortfolioReturn(dailyReturnsDf, weights)
        annualizedPortfolioStd = self.CalculateAnnualizedPortfolioStd(covariance, weights)
        annualizedPortfolioSharpeRatio = annualizedPortfolioReturn / annualizedPortfolioStd
            
        return annualizedPortfolioSharpeRatio
    
    def CalculateAnnualizedPortfolioSortinoRatio(self, dailyReturnsDf, covarianceNegativeReturns, weights):
        
        annualizedPortfolioReturn = self.CalculateAnnualizedPortfolioReturn(dailyReturnsDf, weights)
        annualizedPortfolioNegativeStd = self.CalculateAnnualizedPortfolioNegativeStd(covarianceNegativeReturns, weights)
        annualizedPortfolioSortinoRatio = annualizedPortfolioReturn / annualizedPortfolioNegativeStd
            
        return annualizedPortfolioSortinoRatio
    
    def CalculateRiskParityFunction(self, covariance, weights):
        
        ''' Spinu formulation for risk parity portfolio '''
        
        assetsRiskBudget = self.initWeights
        portfolioVolatility = self.CalculateAnnualizedPortfolioStd(covariance, weights)
        
        x = weights / portfolioVolatility
        riskParity = (np.dot(x.T, np.dot(covariance, x)) / 2) - np.dot(assetsRiskBudget.T, np.log(x))
            
        return riskParity