### Fundamental backtester 2
# returns 800 stocks with largest market cap that have: available fundamdal data including desired fundamental, dollar volume > $5m
# returns 50 stocks with the desired fundamental data either even or odd from top 100
# buys these and holds till next rebalance then repeats to rebalence porfolio every X months

# Current desired fundamental data:
# operation_ratios.roe.one_year
# &
# valuation_ratios.earning_yield

#Previos:
# valuation_ratios # pe_ratio (EPS/Price) #earning_yield (Adjusted Close Price/ EPS) # ps_ratio (SalesPerShare / Price) # sales_yield (Adjusted close price / Sales Per Share) # pb_ratio (Adjusted close price / Book Value Per Share)
# valuation_ratios # trailing_dividend_yield # buy_back_yield # total_yield
# pe_ratio/diluted_eps_growth.one_year (PEG) valuation_ratios.pe_ratio/earning_ratios.diluted_eps_growth.five_years (5y PEG)
# earning_ratios # diluted_eps_growth.one_year # diluted_eps_growth.five_years
# operation_ratios. # revenue_growth.one_year # revenue_growth.five_years # roe: roe.one_year # roa: roa.one_year # roic: roic.one_year

# region imports
from AlgorithmImports import *
# endregion

class FundamentalBacktester1(QCAlgorithm):
    def initialize(self):

        ### SELECT SCENARIO RULES

        # even or odd (1 = odd | 2 = even)
        self.even_odd = 1

        # backtest period (1 = 2/1999-2/2024 | 2 = 6/1999-6/2024 B | 3 = testing)
        self.scenario = 1

        # number of months between each portfolio rebalance
        self.rebalance_every = 3


        # inverse? (1 = yes | 0 = no)
        self.inverse = 0


        # number of stocks that will be returned from coarse selection filter (used in fuction)
        self.num_coarse = 800
        # number of stocks that will be returned from fine selection filter (used in fuction)
        self.num_fine = 50


        if self.scenario == 1:
            self.SetStartDate(1999, 2, 23)
            self.SetEndDate(2024, 2, 23)
            month_day = 12
        elif self.scenario == 2:
            self.SetStartDate(1999, 6, 3) 
            self.SetEndDate(2024, 6, 4)
            month_day = 2
        elif self.scenario == 3:
            self.SetStartDate(1999, 2, 3)
            self.SetEndDate(2002, 2, 4)
            month_day = 3
        else:
            raise ValueError("Invalid scenario")

        if self.inverse == 1:
            self.std_true = False
            self.std_false = True
        elif self.inverse == 0:
            self.std_true = True
            self.std_false = False
        else:
            raise ValueError("Invalid inverse value")

        # Universe settings
        self.set_cash(1000000)
        self.universe_settings.resolution = Resolution.DAILY
        self.settings.daily_precise_end_time = False
        self.settings.minimum_order_margin_portfolio_percentage = 0

        # portolio constuction model
        self.set_portfolio_construction(EqualWeightingPortfolioConstructionModel(lambda time: None))
        self.settings.rebalance_portfolio_on_insight_changes = False
        self.settings.rebalance_portfolio_on_security_changes = True
        # universe selection model
        self.set_universe_selection(FineFundamentalUniverseSelectionModel(self.coarse_filter, self.fine_filter))

        #Set benchmark
        self.spy = self.add_equity("SPY", Resolution.DAILY).symbol
        self.set_benchmark("SPY")

        # rebalance schedule
        self.schedule.on(self.date_rules.month_start("SPY", month_day), self.time_rules.after_market_open("SPY", 120), self.rebalance)
        self.rebalance_count = self.rebalance_every

        # initalise variables  
        self.day_counter = 1
        self.filtered_stocks = []
        self.PV = []
        self.charts = []
        self.monthly_rebalance = False
        self.rebalanced = False
        self.no_data_day = False
        self.blacklist = []
        self.whitelist = []
        self.set_security_initializer(self.custom_security_initializer)



    # set fee stucture
    def custom_security_initializer(self, security): 
        security.set_fee_model(CustomPercentageFeeModel())

    # fuction to chaeck for 1y histrical data
    def has_historical_data(self, symbol):
        history = self.history(symbol, 365, Resolution.DAILY)
        return not history.empty



    # coarse_filter fuction runs every day at midnight
    # filters stock universe, returns course universe, up to 1k sybmols with the largest market cap (min 2b), that have available fundamdal data and dollar volume > $5m (all ~ inflation adusted)
    def coarse_filter(self, coarse):
        # If the rebalance flag is not set, do not change the universe (skip call)
        if not self.monthly_rebalance:
            self.rebalanced = False
            return Universe.UNCHANGED 

        self.rebalanced = True
        self.monthly_rebalance = False  
        remove_symbol = []
        sorted_coarse = []
        filtered_coarse = []
        returned_coarse = []

        current_year = self.time.year
        years_to_2024 = 2024 - current_year
        inflation_adjust = 1 - (years_to_2024 * 0.03)

        sorted_coarse = sorted([
            x for x in coarse
            if x.symbol.value not in self.blacklist
            and x.price > 0
            and x.has_fundamental_data

            and x.operation_ratios.roe.one_year > 0
            and x.valuation_ratios.earning_yield > 0

            and x.dollar_volume > inflation_adjust * 3000000
            and x.market_cap > inflation_adjust * 30000000],
            key=lambda x: x.market_cap, reverse=True)

        # blacklist any symbols with no histoical data
        for coarse_symbol in sorted_coarse:
            symbol = coarse_symbol.symbol
            if symbol not in self.whitelist and symbol not in self.blacklist:
                if self.has_historical_data(symbol):
                    self.whitelist.append(symbol)
                else:
                    self.blacklist.append(symbol)
                    remove_symbol.append(symbol)
                    self.Debug(f"No data for '{symbol}' added to blacklist")
            if symbol not in self.blacklist:
                filtered_coarse.append(coarse_symbol)

        returned_coarse = [x.symbol for x in filtered_coarse[:self.num_coarse]]
        if len(returned_coarse) == 0:
            return Universe.UNCHANGED 

        last_symbol = filtered_coarse[len(returned_coarse) - 1]
        self.Debug("CoarseFilter returned "+str(len(returned_coarse))+" securities of "+str(len(filtered_coarse))+" securities meeting requirements. min Market Cap: "+str(last_symbol.market_cap))

        # Return the filtered simbols
        return returned_coarse



    # fine_filter fuction runs every day at midnight
    # filters Course filter output (up to 1k sybmols) returns, returns fine universe, 50 sybmols with desired fundamental quality (ther odd or evan symbols from top 100)
    def fine_filter(self, fine):
        self.filtered_stocks = []

        # save ranks of funamental fundamental_1 values
        fundamental_1 = sorted(fine, key=lambda x: x.operation_ratios.roe.one_year, reverse=self.std_true)
        fundamental_1_ranks = {stock.symbol: rank for rank, stock in enumerate(fundamental_1)}
        # save ranks of funamental fundamental_2 values
        fundamental_2 = sorted(fine, key=lambda x: x.valuation_ratios.earning_yield, reverse=self.std_true)
        fundamental_2_ranks = {stock.symbol: rank for rank, stock in enumerate(fundamental_2)}
        
        # save put these values in a variable: combined_ranks[0] = stock/ticker, combined_ranks[1] = fundamtal_1, combined_ranks[2] = fundamtal_2, combined_ranks[3] = fundamtal_1+fundamtal_2
        combined_ranks = [
            (stock, fundamental_1_ranks[stock.symbol], fundamental_2_ranks[stock.symbol], 
            fundamental_1_ranks[stock.symbol]*1 + fundamental_2_ranks[stock.symbol]*1)
            for stock in fine]

        # sort by combined fundamntal ranks 
        # x[1] = fundamtal_1, x[2] = fundamtal_2, x[3] = fundamtal_1+fundamtal_2
        combined_ranks.sort(key=lambda x: x[1])

        # Save the sorted combined ranks to filtered_fine
        filtered_fine = combined_ranks[:self.num_fine * 2]

        # Filter for even or odd numbered ranks
        if self.even_odd == 1:
            even_or_odd_ranked_stocks = [stock[0].Symbol for i, stock in enumerate(filtered_fine) if i % 2 != 0]
        elif self.even_odd == 2:
            even_or_odd_ranked_stocks = [stock[0].Symbol for i, stock in enumerate(filtered_fine) if i % 2 == 0]
        else:
            raise ValueError("Invalid evan_or_odd value")

        # Limit the selection to 'self.num_fine'
        self.filtered_stocks = even_or_odd_ranked_stocks[:self.num_fine]

        # Save the first and last stock in the filtered stocks for debugging
        first_1 = filtered_fine[0]
        last_1 = filtered_fine[len(self.filtered_stocks) - 1]
        first_symbol = first_1[0]
        last_symbol = last_1[0]

        self.Debug("FineFilter returned "+str(len(self.filtered_stocks)) +" securities."
        +"      operation_ratios.roe.one_year: first (1st): "+str(first_symbol.operation_ratios.roe.one_year)
        +"  last (50th): "+str(last_symbol.operation_ratios.roe.one_year) 
        +"      valuation_ratios.earning_yield: first (1st): "+str(first_symbol.valuation_ratios.earning_yield)
        +"  last (50th): "+str(last_symbol.valuation_ratios.earning_yield))
        return self.filtered_stocks



    # on_data fuction runs every day at midnight
    # creates buy and sell insights based on fine_filter output
    def on_data(self, data):
        self.day_counter += 1

        # Add stock data
        for symbol in self.charts:
            if not data.contains_key(symbol):
                self.add_equity(symbol, Resolution.DAILY)

        # Check if it's November 8, 2022
        if self.Time.date() == datetime(2022, 11, 8).date():
            self.prices_on_nov_8 = {}  # Dictionary to store prices of held stocks on Nov 8
            for symbol in self.PV:
                if data.contains_key(symbol) and data[symbol] is not None:
                    self.prices_on_nov_8[symbol] = data[symbol].close
                    self.Debug(f"Price of {symbol} on Nov 8, 2022: {data[symbol].close}")

        # Check if it's November 10, 2022
        if self.Time.date() == datetime(2022, 11, 10).date():
            for symbol in self.PV:
                if symbol in self.prices_on_nov_8:
                    price_on_nov_8 = self.prices_on_nov_8[symbol]
                    if data.contains_key(symbol) and data[symbol] is not None:
                        price_on_nov_10 = data[symbol].close
                        percent_return = ((price_on_nov_10 - price_on_nov_8) / price_on_nov_8) * 100
                        self.Debug(f"Price of {symbol} on Nov 10, 2022: {price_on_nov_10}, "
                                f"Return since Nov 8: {percent_return:.2f}%")

        # returns out of function if not just rebalenced
        if not self.rebalanced:
            return

        insights = []
        self.open_counter = 0
        self.close_counter = 0
        debug_output = ""

        # Create close insights for symbols in self.PV (current holdings) & not in self.filtered_stocks (fine_filter output)
        for symbol in list(self.PV):
            if symbol not in self.filtered_stocks:
                if not data.contains_key(symbol):
                    continue
                insights.append(Insight.price(symbol, timedelta(days=7560), InsightDirection.FLAT))
                self.PV.remove(symbol)
                self.close_counter += 1
        '''
                if data[symbol] is not None:
                    close_price = data[symbol].close
                    debug_output += f"| Close {symbol} @ {close_price} |"
                else:
                    debug_output += f"| Close {symbol} @ price ERROR |"
        self.Debug(f"{debug_output}")
        '''
        debug_output = ""

        # Create open insights for symbols in self.filtered_stocks (fine_filter output) & not in self.PV (current holdings)
        for symbol in self.filtered_stocks:
            if symbol not in self.PV:
                if symbol not in self.charts:
                    self.charts.append(symbol)
                if not data.contains_key(symbol):
                    continue
                insights.append(Insight.price(symbol, timedelta(days=7560), InsightDirection.UP))
                self.PV.append(symbol)
                self.open_counter += 1
        '''       
                if data[symbol] is not None:
                    open_price = data[symbol].close
                    debug_output += f"| Open {symbol} @ {open_price} |"
                else:
                    debug_output += f"| Open {symbol} @ price ERROR  |"
        self.Debug(f"{debug_output}")
        '''

        # log number of opened, closed, and held securities
        self.Debug("Currently holding "+str(len(self.PV))+" Securities: Opened "+str(self.open_counter)+" Closed "+str(self.close_counter))

        # send buy and sell insights (managed by EqualWeightingPortfolioConstructionModel)
        self.emit_insights(*insights)

    # this is called according to Schedule.On it is being used to initiates a rebalence of the portfolio
    def rebalance(self):
        if self.rebalance_count == self.rebalance_every:
            self.monthly_rebalance = True
            self.rebalance_count = 1
        else:
            self.rebalance_count += 1
    

# set fee stucture
class CustomPercentageFeeModel(FeeModel):
    def get_order_fee(self, parameters):
        order = parameters.order
        value = order.absolute_quantity * parameters.security.price
        fee = value * 0.002
        return OrderFee(CashAmount(fee, "USD"))