# https://quantpedia.com/strategies/idiosyncratic-momentum-in-stocks/
#
# The investment universe consists of all NYSE, AMEX and NASDAQ stocks with a capitalization higher than the 20th percentile for 
# market capitalization among NYSE stocks (large caps). The excess monthly return over a risk-free rate for each stock is calculated. 
# The last three years of data are used to estimate the CAPM regression for each stock (using the market factor from the Kenneth French 
# data library as an independent variable). The idiosyncratic momentum is calculated as the cumulative idiosyncratic return (residuals 
# from the regression) over the 11-month period (month t-12 to month t-2). Stocks are sorted into quintiles based on the idiosyncratic
# momentum, and the investor goes long on stocks from the top quintile, and he/she goes short on stocks from the bottom quintile. Stocks 
# are weighted equally, and the portfolio is rebalanced on a monthly basis.

import fk_tools
import numpy as np
from scipy import stats
from collections import deque

class IdiosyncraticMomentumStocks(QCAlgorithm):

    def Initialize(self):
        self.SetStartDate(2007, 6, 1)  
        self.SetCash(100000) 

        self.risk_free_symbol = 'BIL'   # Data started 2007-05-30
        self.market_symbol = 'SPY'
        self.AddEquity(self.risk_free_symbol, Resolution.Daily)
        self.AddEquity(self.market_symbol, Resolution.Daily)
        
        self.coarse_count = 1000
        
        self.period = 36

        # Monthly price data.
        self.data = {}
        
        # Monthly residuals for stocks.
        self.residual = {}
        
        self.long = []
        self.short = []
        
        self.selection_flag = False
        self.rebalance_flag = False
        self.UniverseSettings.Resolution = Resolution.Daily
        self.AddUniverse(self.CoarseSelectionFunction)
        
        self.Schedule.On(self.DateRules.MonthEnd(self.market_symbol), self.TimeRules.AfterMarketOpen(self.market_symbol), self.Selection)

    def OnSecuritiesChanged(self, changes):
        for security in changes.AddedSecurities:
            security.SetFeeModel(fk_tools.CustomFeeModel(self))

    def CoarseSelectionFunction(self, coarse):
        if not self.selection_flag:
            return Universe.Unchanged
        
        self.selection_flag = False
        
        selected = sorted([x for x in coarse if x.HasFundamentalData and x.Market == 'usa' and x.Price > 5],
            key=lambda x: x.DollarVolume, reverse=True)[:self.coarse_count]
        
        selected_symbols = [x.Symbol for x in selected]
        
        # Store monthly data for universe.
        for stock in selected:
            symbol = stock.Symbol
            
            if symbol not in self.data:
                self.data[symbol] = deque(maxlen = self.period)
                
            price = stock.Price
            if price != 0:
                self.data[symbol].append(price)
        
        # Make sure we have consecutive monthly data and residual data.
        symbols_to_delete = []
        for symbol in self.data:
            if symbol in [self.market_symbol, self.risk_free_symbol]: continue
            if symbol not in selected_symbols:
                symbols_to_delete.append(symbol)
        for symbol in symbols_to_delete:
            del self.data[symbol]
            if symbol in self.residual:
                del self.residual[symbol]
        
        # Risk free etf data or market data is not ready.
        if len(self.data[self.market_symbol]) != self.data[self.market_symbol].maxlen: return []
        if len(self.data[self.risk_free_symbol]) != self.data[self.risk_free_symbol].maxlen: return []
        
        # Price data ready stocks.
        data_ready_stocks = [x for x in self.data.items() if x[0] not in [self.market_symbol, self.risk_free_symbol] and len(x[1]) == x[1].maxlen]
        
        if len(data_ready_stocks) == 0: return []
        
        market_prices = np.array([x for x in self.data[self.market_symbol]])
        market_returns = (market_prices[1:] - market_prices[:-1]) / market_prices[:-1]
        
        risk_free_prices = np.array([x for x in self.data[self.risk_free_symbol]])
        risk_free_returns = (risk_free_prices[1:] - risk_free_prices[:-1]) / risk_free_prices[:-1]
        
        market_factor = market_returns - risk_free_returns
        
        for symbol, data in data_ready_stocks:
            stock_prices = np.array([x for x in data])
            stock_returns = (stock_prices[1:] - stock_prices[:-1]) / stock_prices[:-1]
            
            excess_return = stock_returns - risk_free_returns
            
            # Y = α + (β ∗ X)
            # intercept = alpha
            # slope = beta
            slope, intercept, r_value, p_value, std_err = stats.linregress(market_factor, excess_return)
            
            actual_value = excess_return[-1]
            estimate_value = intercept + (slope * market_factor[-1])
            
            residual = actual_value - estimate_value
            if symbol not in self.residual:
                self.residual[symbol] = deque(maxlen = 12)
            self.residual[symbol].append(residual)
        
        # Symbols with 12 months of residual data.
        residual_ready_symbols = [x for x in self.residual.items() if len(x[1]) == x[1].maxlen]
        if len(residual_ready_symbols) == 0: return []
        
        # Sorted (month t-12 to month t-2) by idiosyncratic return.
        sorted_by_idiosyncratic_momentum = sorted(residual_ready_symbols, key = lambda x: sum([y for y in x[1]][:-2]), reverse = True)
        quintile = int(len(sorted_by_idiosyncratic_momentum) / 5)
        
        self.long = [x[0] for x in sorted_by_idiosyncratic_momentum[:quintile]]
        self.short = [x[0] for x in sorted_by_idiosyncratic_momentum[-quintile:]]
        
        self.rebalance_flag = True
        
        return self.long + self.short
    
    def OnData(self, data):
        if not self.rebalance_flag:
            return
        self.rebalance_flag = False

        # Trade execution
        count = len(self.long + self.short)
        if count == 0: return

        stocks_invested = [x.Key for x in self.Portfolio if x.Value.Invested]
        for symbol in stocks_invested:
            if symbol not in self.long + self.short:
                self.Liquidate(symbol)

        for symbol in self.long:
            if self.Securities[symbol].Price != 0:  # Prevent error message.
                self.SetHoldings(symbol, 0.9 / count)
        for symbol in self.short:
            if self.Securities[symbol].Price != 0:  # Prevent error message.
                self.SetHoldings(symbol, -0.9 / count)

        self.long.clear()
        self.short.clear()
    
    def Selection(self):
        self.selection_flag = True
        
        # Store Risk free etf price and market price.
        for symbol in [self.market_symbol, self.risk_free_symbol]:
            if symbol not in self.data:
                self.data[symbol] = deque(maxlen = self.period)                
                
            if self.Securities.ContainsKey(symbol):
                price = self.Securities[symbol].Price
                if price != 0:
                    self.data[symbol].append(price)

import numpy as np
from scipy.optimize import minimize

sp100_stocks = ['AAPL','MSFT','AMZN','FB','BRK.B','GOOGL','GOOG','JPM','JNJ','V','PG','XOM','UNH','BAC','MA','T','DIS','INTC','HD','VZ','MRK','PFE','CVX','KO','CMCSA','CSCO','PEP','WFC','C','BA','ADBE','WMT','CRM','MCD','MDT','BMY','ABT','NVDA','NFLX','AMGN','PM','PYPL','TMO','COST','ABBV','ACN','HON','NKE','UNP','UTX','NEE','IBM','TXN','AVGO','LLY','ORCL','LIN','SBUX','AMT','LMT','GE','MMM','DHR','QCOM','CVS','MO','LOW','FIS','AXP','BKNG','UPS','GILD','CHTR','CAT','MDLZ','GS','USB','CI','ANTM','BDX','TJX','ADP','TFC','CME','SPGI','COP','INTU','ISRG','CB','SO','D','FISV','PNC','DUK','SYK','ZTS','MS','RTN','AGN','BLK']

def Return(values):
    return (values[-1] - values[0]) / values[0]
    
def Volatility(values):
    values = np.array(values)
    returns = (values[1:] - values[:-1]) / values[:-1]
    return np.std(returns)  

# Custom fee model
class CustomFeeModel(FeeModel):
    def GetOrderFee(self, parameters):
        fee = parameters.Security.Price * parameters.Order.AbsoluteQuantity * 0.00005
        return OrderFee(CashAmount(fee, "USD"))

# Quandl free data
class QuandlFutures(PythonQuandl):
    def __init__(self):
        self.ValueColumnName = "settle"

# Quandl short interest data.
class QuandlFINRA_ShortVolume(PythonQuandl):
    def __init__(self):
        self.ValueColumnName = 'SHORTVOLUME'    # also 'TOTALVOLUME' is accesible

# Quantpedia data
# NOTE: IMPORTANT: Data order must be ascending (datewise)
class QuantpediaFutures(PythonData):
    def GetSource(self, config, date, isLiveMode):
        return SubscriptionDataSource("data.quantpedia.com/backtesting_data/futures/{0}.csv".format(config.Symbol.Value), SubscriptionTransportMedium.RemoteFile, FileFormat.Csv)

    def Reader(self, config, line, date, isLiveMode):
        data = QuantpediaFutures()
        data.Symbol = config.Symbol
        
        if not line[0].isdigit(): return None
        split = line.split(';')
        
        data.Time = datetime.strptime(split[0], "%d.%m.%Y") + timedelta(days=1)
        data['settle'] = float(split[1])
        data.Value = float(split[1])

        return data
        
# NOTE: Manager for new trades. It's represented by certain count of equally weighted brackets for long and short positions.
# If there's a place for new trade, it will be managed for time of holding period.
class TradeManager():
    def __init__(self, algorithm, long_size, short_size, holding_period):
        self.algorithm = algorithm  # algorithm to execute orders in.
        
        self.long_size = long_size
        self.short_size = short_size
        self.weight = 1 / (self.long_size + self.short_size)
        
        self.long_len = 0
        self.short_len = 0
    
        # Arrays of ManagedSymbols
        self.symbols = []
        
        self.holding_period = holding_period    # Days of holding.
    
    # Add stock symbol object
    def Add(self, symbol, long_flag):
        # Open new long trade.
        managed_symbol = ManagedSymbol(symbol, self.holding_period, long_flag)
        
        if long_flag:
            # If there's a place for it.
            if self.long_len < self.long_size:
                self.symbols.append(managed_symbol)
                self.algorithm.SetHoldings(symbol, self.weight)
                self.long_len += 1
        # Open new short trade.
        else:
            # If there's a place for it.
            if self.long_len < self.short_size:
                self.symbols.append(managed_symbol)
                self.algorithm.SetHoldings(symbol, - self.weight)
                self.short_len += 1
    
    # Decrement holding period and liquidate symbols.
    def TryLiquidate(self):
        symbols_to_delete = []
        for managed_symbol in self.symbols:
            managed_symbol.days_to_liquidate -= 1
            
            # Liquidate.
            if managed_symbol.days_to_liquidate == 0:
                symbols_to_delete.append(managed_symbol)
                self.algorithm.Liquidate(managed_symbol.symbol)
                if managed_symbol.long_flag: self.long_len -= 1
                else: self.short_len -= 1

        # Remove symbols from management.
        for managed_symbol in symbols_to_delete:
            self.symbols.remove(managed_symbol)

class ManagedSymbol():
    def __init__(self, symbol, days_to_liquidate, long_flag):
        self.symbol = symbol
        self.days_to_liquidate = days_to_liquidate
        self.long_flag = long_flag
        
class PortfolioOptimization(object):
    def __init__(self, df_return, risk_free_rate, num_assets):
        self.daily_return = df_return
        self.risk_free_rate = risk_free_rate
        self.n = num_assets # numbers of risk assets in portfolio
        self.target_vol = 0.05

    def annual_port_return(self, weights):
        # calculate the annual return of portfolio
        return np.sum(self.daily_return.mean() * weights) * 252

    def annual_port_vol(self, weights):
        # calculate the annual volatility of portfolio
        return np.sqrt(np.dot(weights.T, np.dot(self.daily_return.cov() * 252, weights)))

    def min_func(self, weights):
        # method 1: maximize sharp ratio
        return - self.annual_port_return(weights) / self.annual_port_vol(weights)
        
        # method 2: maximize the return with target volatility
        #return - self.annual_port_return(weights) / self.target_vol

    def opt_portfolio(self):
        # maximize the sharpe ratio to find the optimal weights
        cons = ({'type': 'eq', 'fun': lambda x: np.sum(x) - 1})
        bnds = tuple((0, 1) for x in range(2)) + tuple((0, 0.25) for x in range(self.n - 2))
        opt = minimize(self.min_func,                               # object function
                       np.array(self.n * [1. / self.n]),            # initial value
                       method='SLSQP',                              # optimization method
                       bounds=bnds,                                 # bounds for variables 
                       constraints=cons)                            # constraint conditions
                      
        opt_weights = opt['x']
 
        return opt_weights