#region imports
from AlgorithmImports import *
#endregion

class SymbolData():
    def __init__(self, daily_period: int, monthly_period: int) -> None:
        self._daily_price: RollingWindow = RollingWindow[float](daily_period)
        self._monthly_price: RollingWindow = RollingWindow[float](monthly_period)
    
    def update_daily_price(self, price: float) -> None:
        self._daily_price.add(price)

    def update_monthly_price(self, price: float) -> None:
        self._monthly_price.add(price)

    def get_vol(self) -> float:
        daily_returns: np.ndarray = pd.Series(list(self._daily_price)[::-1]).pct_change().values[1:]
        return np.std(daily_returns) * np.sqrt(252)

    def get_momentum(self) -> float:
        # skip latest month
        return self._monthly_price[1] / self._monthly_price[self._monthly_price.count - 1] - 1

    def is_ready(self) -> bool:
        return self._daily_price.is_ready and self._monthly_price.is_ready

# Custom fee model.
class CustomFeeModel(FeeModel):
    def GetOrderFee(self, parameters: OrderFeeParameters) -> OrderFee:
        fee: float = parameters.Security.Price * parameters.Order.AbsoluteQuantity * 0.00005
        return OrderFee(CashAmount(fee, "USD"))

# https://quantpedia.com/strategies/leveraging-the-low-volatility-effect/
# 
# The investment universe for this strategy consists of the top 1,000 U.S. stocks by market capitalization.
# Coarse Selection: Individual instruments are selected based on historical volatility, momentum, and value characteristics. Initially, the strategy screens for the 
# 500 stocks with the lowest 3-year historical volatility. From these, stocks are further evaluated and ranked based on their 12-1 month price momentum and net payout
# yield (measuring value). The top 100 stocks with the highest combined momentum and value scores are selected for inclusion in the portfolio.
# (Data Sources: U.S. bond data and credit spreads are sourced from the FRED database, with bond returns calculated using the 10-year government bond yield. The equity 
# and 1-month T-bill rate are from the Kenneth French data library, and the low-volatility data are from the Robeco website.)
# Follow the approach described on pg. 9: Fifth case: Downside protection: construct a portfolio with a 30% long position in low-volatility stocks combined with a -50% 
# short position in speculative stocks. (The result shall be portfolio with a downside beta of around -0.5 and generally strong defensive properties versus equities.)
# Trading Rules:
# Buy rules involve selecting 100 stocks from the low-volatility subset that rank highest combined with momentum and value scores using leverage 0.3.
# Stocks selected for shorting are the 'speculative stocks' described by Blitz and Van Vliet (2018), characterized by high volatility, weak net payout yield, and poor 
# 12-1-month momentum, the worst 100 stocks with leverage of 0.5.
# Sell and (buy to) cover rules are applied during monthly rebalancing when stocks that no longer meet the criteria are removed from the portfolio.
# (Left 0.2 capital can be in risk-free assets or cash.)
# The portfolio is constructed by equal-weighting the top and bottom 100 selected stocks (all 200 in equal weight), ensuring diversification and minimizing concentration 
# risk. The strategy includes monthly rebalancing to maintain alignment with the selection criteria.
# 
# Implementation changes:
#	- BIL ETF is used as risk free asset.
#   - long stocks use 0.5 leverage and short stocks use 0.3 leverage as we suspect that there is typo in the paper

# region imports
from AlgorithmImports import *
import data_tools
from numpy import isnan
# endregion

class LeveragingTheLowVolatilityEffect(QCAlgorithm):

    def initialize(self) -> None:
        self.set_start_date(2010, 1, 1)
        self.set_cash(100_000)

        self._excluded_tickers: List[str] = ['IIGP', 'FHRI', 'HRC', 'GTNR', 'CLRO', 'RELM', 'SKH']
        self._exchange_codes: List[str] = ['NYS', 'NAS', 'ASE']

        leverage: int = 4
        self._vol_period: int = 3 * 252
        self._mom_period: int = 12
        self._short_weight: float = .3
        self._long_weight: float = .5
        self._risk_free_weight: float = 1 - (self._short_weight + self._long_weight)

        self._risk_free_asset: Symbol = self.add_equity('BIL', Resolution.DAILY).symbol

        market: Symbol = self.add_equity('SPY', Resolution.DAILY).symbol

        self._stock_count: int = 100
        self._fundamental_count: int = 1_000
        self._fundamental_sorting_key = lambda x: x.market_cap

        self._data: Dict[Symbol, data_tools.SymbolData] = {}
        self._weight: Dict[Symbol, float] = {}

        self._selection_flag: bool = False
        self._rebalance_flag: bool = False
        self.universe_settings.leverage = leverage
        self.universe_settings.resolution = Resolution.DAILY
        self.add_universe(self.fundamental_selection_function)
        self.settings.daily_precise_end_time = False
        self.settings.minimum_order_margin_portfolio_percentage = 0.

        self.schedule.on(
            self.date_rules.month_start(market),
            self.time_rules.after_market_open(market),
            self.selection
        )

    def on_securities_changed(self, changes: SecurityChanges) -> None:
        for security in changes.added_securities:
            security.set_fee_model(data_tools.CustomFeeModel())

    def fundamental_selection_function(self, fundamental: List[Fundamental]) -> List[Symbol]:
        # update the data every day
        for stock in fundamental:
            symbol: Symbol = stock.symbol
            
            if symbol in self._data:
                self._data[symbol].update_daily_price(stock.adjusted_price)
                # store monthly price
                if self._selection_flag:
                    self._data[symbol].update_monthly_price(stock.adjusted_price)

        if not self._selection_flag:
            return Universe.UNCHANGED

        selected: List[Fundamental] = [
            x for x in fundamental
            if x.has_fundamental_data
            and x.market_cap != 0
            and not isnan(x.valuation_ratios.payout_ratio)
            and x.valuation_ratios.payout_ratio != 0
            and x.security_reference.exchange_id in self._exchange_codes
            and x.symbol.value not in self._excluded_tickers
        ]
        
        if len(selected) > self._fundamental_count:
            selected = [
                x for x in sorted(selected, key=self._fundamental_sorting_key, reverse=True)[:self._fundamental_count]
            ]

        payout_yield: Dict[Symbol, float] = {}

        # price warmup
        for stock in selected:
            symbol: Symbol = stock.symbol

            if symbol not in self._data:
                self._data[symbol] = data_tools.SymbolData(self._vol_period, self._mom_period)
                history: DataFrame = self.history(TradeBar, symbol, self._vol_period, Resolution.DAILY)
                if history.empty:
                    self.log(f"Not enough data for {symbol} yet.")
                    continue
                data: Series = history.loc[symbol]
                monthly_data: Series = data.groupby(pd.Grouper(freq='MS')).first()
                for time, row in data.iterrows():
                    self._data[symbol].update_daily_price(row.close)
                for time, row in monthly_data.iterrows():
                    self._data[symbol].update_monthly_price(row.close)

        sorted_vol: List[Fundamental] = sorted([
            x for x in selected if self._data[x.symbol].is_ready()
        ], key=lambda x: self._data[x.symbol].get_vol())

        if len(sorted_vol) == 0:
            self.log(f'Not enough data for further calculation - volatility sorted stock count: {len(sorted_vol)}')
            return Universe.UNCHANGED

        # Divide stocks sorted by volatilities into half.
        quantile: int = len(sorted_vol) // 2
        low_vol: List[Fundamental] = sorted_vol[:quantile]
        high_vol: List[Fundamental] = sorted_vol[-quantile:]

        if all(len(vol_port) >= self._stock_count for vol_port in [low_vol, high_vol]):
            for i, vol_port in enumerate([high_vol, low_vol]):
                # Rank based on momentum and net payout yield.
                sorted_momentum: List[Symbol] = sorted(list(map(
                    lambda x: x.symbol, vol_port 
                )), key=lambda x: self._data[x].get_momentum(), reverse=bool(i))

                momentum_ranks: Dict[Symbol, int] = {
                    symbol: rank for rank, symbol in enumerate(sorted_momentum)
                }

                sorted_payout_yield: List[Symbol] = sorted(vol_port, key=lambda x: x.valuation_ratios.payout_ratio, reverse=bool(i))
                value_ranks: Dict[Symbol, int] = {
                    stock.symbol: rank for rank, stock in enumerate(sorted_payout_yield)
                }

                aggregated_ranks: List[Symbol] = sorted({
                    x.symbol: momentum_ranks[x.symbol] + value_ranks[x.symbol] for x in vol_port
                }.items(), key=lambda x: x[1])[:self._stock_count]

                for symbol, rank in aggregated_ranks:
                    weight: float = self._short_weight if i == 0 else self._long_weight
                    self._weight[symbol] = (-1 * ((-1) ** i)) * weight / len(aggregated_ranks)
        else:
            self.log(f'Not enough data for further calculation - low volatility portfolio count: {len(low_vol)}; high volatility portfolio count: {len(high_vol)}')

        if len(selected) == 0:
            return Universe.UNCHANGED

        return list(self._weight.keys())

    def on_data(self, slice: Slice) -> None:
        # order execution
        if not self._selection_flag:
            return
        self._selection_flag = False

        portfolio: List[PortfolioTarget] = [
            PortfolioTarget(symbol, w) for symbol, w in self._weight.items() if slice.contains_key(symbol) and slice[symbol]
        ]

        if not slice.contains_key(self._risk_free_asset) or not slice[self._risk_free_asset]:
            return
        portfolio.append(PortfolioTarget(self._risk_free_asset, self._risk_free_weight))

        self.set_holdings(portfolio, True)

    def selection(self) -> None:
        self._selection_flag = True
        self._weight.clear()