# https://quantpedia.com/strategies/short-interest-effect-long-short-version/
#
# All stocks from NYSE, AMEX, and NASDAQ are part of the investment universe. Stocks are then sorted each month into short-interest deciles based on
# the ratio of short interest to shares outstanding. The investor then goes long on the decile with the lowest short ratio and short on the decile 
# with the highest short ratio. The portfolio is rebalanced monthly, and stocks in the portfolio are weighted equally.

from AlgorithmImports import *
from io import StringIO
from typing import List, Dict
from numpy import isnan

class ShortInterestEffect(QCAlgorithm):

    def Initialize(self) -> None:
        self.SetStartDate(2017, 1, 1)
        self.SetCash(100_000)

        self.quantile: int = 10
        self.leverage: int = 5

        self.weight: Dict[Symbol, float] = {}

        # source: https://www.finra.org/finra-data/browse-catalog/equity-short-interest/data
        text: str = self.Download('data.quantpedia.com/backtesting_data/economic/short_volume.csv')

        self.short_volume_df: DataFrame = pd.read_csv(StringIO(text), delimiter=';')
        self.short_volume_df['date'] = pd.to_datetime(self.short_volume_df['date']).dt.date
        self.short_volume_df.set_index('date', inplace=True)

        self.selection_flag: bool = False
        self.UniverseSettings.Resolution = Resolution.Daily
        self.AddUniverse(self.FundamentalSelectionFunction)
        self.Settings.MinimumOrderMarginPortfolioPercentage = 0.
        self.recent_month = -1

    def OnSecuritiesChanged(self, changes: SecurityChanges) -> None:
        for security in changes.AddedSecurities:
            security.SetFeeModel(CustomFeeModel())
            security.SetLeverage(self.leverage)

    def FundamentalSelectionFunction(self, fundamental: List[Fundamental]) -> List[Symbol]:
        # monthly rebalance
        if self.recent_month == self.Time.month:
            return Universe.Unchanged
        self.recent_month = self.Time.month
        self.selection_flag = True

        # check last date on custom data
        if self.Time.date() > self.short_volume_df.index[-1] or self.Time.date() < self.short_volume_df.index[0]:
            self.Liquidate()
            return Universe.Unchanged

        selected: List[Fundamental] = [
            x for x in fundamental 
            if x.HasFundamentalData 
            and x.Market == 'usa'
            and x.CompanyProfile.SharesOutstanding != 0
            and x.Symbol.Value in self.short_volume_df.columns
        ]

        short_interest: Dict[Symbol, float] = {}

        # calculate short interest
        for stock in selected:
            symbol: Symbol = stock.Symbol
            ticker: str = symbol.Value

            if ticker in self.short_volume_df.columns:
                if isnan(self.short_volume_df[self.short_volume_df.index <= self.Time.date()][ticker][-1]):
                    continue
                short_interest[symbol] = self.short_volume_df[self.short_volume_df.index <= self.Time.date()][ticker][-1] / stock.CompanyProfile.SharesOutstanding

        if len(short_interest) >= self.quantile:
            # sorting by short interest ratio
            sorted_short_interest: List[Fundamental] = sorted(short_interest, key = short_interest.get)
            quantile: int = int(len(sorted_short_interest) / self.quantile)
            long: List[Fundamental] = sorted_short_interest[:quantile]
            short: List[Fundamental] = sorted_short_interest[-quantile:]

            # equaly weighting
            for i, portfolio in enumerate([long, short]):
                for symbol in portfolio:
                    self.weight[symbol] = ((-1) ** i) / len(portfolio)

        return list(self.weight.keys())

    def OnData(self, data: Slice) -> None:
        if not self.selection_flag:
            return
        self.selection_flag = False
        
        # trade execution
        portfolio:List[PortfolioTarget] = [PortfolioTarget(symbol, w) for symbol, w in self.weight.items() if symbol in data and data[symbol]]
        self.SetHoldings(portfolio, True)
        self.weight.clear()

# Custom fee model.
class CustomFeeModel(FeeModel):
    def GetOrderFee(self, parameters: OrderFeeParameters) -> OrderFee:
        fee: float = parameters.Security.Price * parameters.Order.AbsoluteQuantity * 0.00005
        return OrderFee(CashAmount(fee, "USD"))