#region imports
    using System;
    using System.Collections;
    using System.Collections.Generic;
    using System.Linq;
    using System.Globalization;
    using System.Drawing;
    using QuantConnect;
    using QuantConnect.Algorithm.Framework;
    using QuantConnect.Algorithm.Framework.Selection;
    using QuantConnect.Algorithm.Framework.Alphas;
    using QuantConnect.Algorithm.Framework.Portfolio;
    using QuantConnect.Algorithm.Framework.Execution;
    using QuantConnect.Algorithm.Framework.Risk;
    using QuantConnect.Parameters;
    using QuantConnect.Benchmarks;
    using QuantConnect.Brokerages;
    using QuantConnect.Util;
    using QuantConnect.Interfaces;
    using QuantConnect.Algorithm;
    using QuantConnect.Indicators;
    using QuantConnect.Data;
    using QuantConnect.Data.Consolidators;
    using QuantConnect.Data.Custom;
    using QuantConnect.DataSource;
    using QuantConnect.Data.Fundamental;
    using QuantConnect.Data.Market;
    using QuantConnect.Data.UniverseSelection;
    using QuantConnect.Notifications;
    using QuantConnect.Orders;
    using QuantConnect.Orders.Fees;
    using QuantConnect.Orders.Fills;
    using QuantConnect.Orders.Slippage;
    using QuantConnect.Scheduling;
    using QuantConnect.Securities;
    using QuantConnect.Securities.Equity;
    using QuantConnect.Securities.Future;
    using QuantConnect.Securities.Option;
    using QuantConnect.Securities.Forex;
    using QuantConnect.Securities.Interfaces;
    using QuantConnect.Storage;
    using QuantConnect.Data.Custom.AlphaStreams;
    using QCAlgorithmFramework = QuantConnect.Algorithm.QCAlgorithm;
    using QCAlgorithmFrameworkBridge = QuantConnect.Algorithm.QCAlgorithm;
#endregion
namespace QuantConnect.Algorithm.CSharp
{
    public class SecuritiesFilter : QCAlgorithm
    {
		// === GENERAL PARAMETERS ===

		// day to start the run
        private readonly string start = "10/25/2023 00:00:00.000 AM";

		// day to start the run
        private readonly string end = "10/26/2023 00:00:00.000 AM";

		// whether to use adjusted data
		private readonly bool useAdjusted = false;

        // [enable] volume filtering. true = enabled, false = disabled
        private readonly bool useVolumeFiltering = true;

        // MIN volume for universe selection
        // in example this was 69M
        private readonly int minvol = 16300000;

        // MAX volume for universe selection
        // in example this was 70M
        private readonly int maxvol = 16500000;

        // [enable] open filtering. true = enabled, false = disabled
        private readonly bool useOpenFiltering = true;

        // averaging modulus parameter
        // will average $open every N days (1 for every day, 2 for every other day, etc.)
        private readonly int openAvgMod = 1;

        // this parameter will omit an $open values less than the provided amount from any averaging calculations
        private readonly decimal openAvgMin = 0;

        // MIN $OPEN value for filtering securities
        private readonly decimal minopen = 0;

        // MAX $OPEN value for filtering securities
        private readonly decimal maxopen = 1000000000;

		// === UNIVERSE PARAMETERS ===

        // lookback time (in days)
        private readonly int lookback = 2;

        // [enable] volume filtering for hoc/loc. true = enabled, false = disabled
        private readonly bool useVolumeFilteringHocLoc = false;

        // volume for counting hoc/loc
        private readonly int W = 100000;

        // [enable] $Open filtering for hoc/loc. true = enabled, false = disabled
        private readonly bool useOpenFilteringHocLoc = true;

        // $Open for counting hoc/loc
		private readonly int X = 0;

		// values for condition 2: HIGH <= OPEN + Y (% or Tick)
		private readonly bool useTick1 = false; 	// false = %; true = tick
		private readonly decimal Y = 0.004m;		    // 0.01 = 1%

        private readonly decimal percentLong = 0.70m;

		// values for condition 3: LOW >= OPEN - Z (% or Tick)
		private readonly bool useTick2 = false; 	// false = %; true = tick
		private readonly decimal Z = 0.004m;		    // 0.01 = 1%

        private readonly decimal percentShort = 0.30m;

        private DateTime startDate;
        private DateTime endDate;
        private decimal low = Decimal.MaxValue;

        List<Symbol> universe = new List<Symbol> {};
        
        public Dictionary<Symbol, decimal> volTotal = new Dictionary<Symbol, decimal>();
        public Dictionary<Symbol, int> volCount = new Dictionary<Symbol, int>();
        public Dictionary<Symbol, decimal[]> openValueAvgs = new Dictionary<Symbol, decimal[]>();
        public Dictionary<String, decimal[]> openValues;

		// Initialize the scanner
       	public override void Initialize()
        {
            Log("time,ticker,count,days,hoc,loc,hoc_adj,loc_adj,hoc_o,loc_o,hoc_change,loc_change,dd_max,pl,ot_quantity_avg,ot_price_avg,ot_avg,vol_avg"); // note hoc = high <= open count; loc = low >= open count.

            if(useAdjusted)
                UniverseSettings.DataNormalizationMode = DataNormalizationMode.Adjusted;
            else
                UniverseSettings.DataNormalizationMode = DataNormalizationMode.Raw;

            startDate = DateTime.Parse(start);
            endDate = DateTime.Parse(end);
            DateTime endDateM1 = endDate;

            SetStartDate(startDate.Year, startDate.Month, startDate.Day);
            SetEndDate(endDate.Year, endDate.Month, endDate.Day);
            SetCash(100000);

			// define universe
			AddUniverse(CoarseFilterFunction);

            // schedule event
            AddEquity("SPY", Resolution.Daily);

            // load all stored open values
            if(ObjectStore.ContainsKey("openvalues")) {
                openValues = ObjectStore.ReadJson<Dictionary<String, decimal[]>>("openvalues");
            } else
                openValues = new Dictionary<String, decimal[]>();

            // scan all data at end of looping for volume average conditional and $open conditional 
            Schedule.On(DateRules.On(endDateM1.Year, endDateM1.Month, endDateM1.Day), TimeRules.AfterMarketOpen("SPY", 0), Scan);
        }

        // loops through all securities in the list and filters based on average volume
        // then checks getOpen conditional for validation before perform HOC/LOC conditionals
        private void Scan() {
            List<Symbol> filter1 = new List<Symbol>();
            Dictionary<Symbol, IEnumerable<TradeBar>> filter2 = new Dictionary<Symbol, IEnumerable<TradeBar>>();

            // loop through all retrieved symbols in the coarse universe
            foreach(Symbol symbol in volCount.Keys) {
                
                // calculate average volume
                decimal vol_avg = 0.0m;
                if(volCount.ContainsKey(symbol) && volCount[symbol] > 0 && volTotal.ContainsKey(symbol)) {
                    vol_avg = volTotal[symbol] / volCount[symbol];
                }

                // check volume conditional
                if(useVolumeFiltering && (vol_avg < minvol || vol_avg > maxvol))
                    continue;

                filter1.Add(symbol);
            }

            // check for $open avg conditional in all values
            if(useOpenFiltering) {
                // loop through all symbols
                foreach(Symbol symbol in filter1) {
                    // get daily history for each ticker
                    var startDate = DateTime.Parse(start);
                    var endDate = DateTime.Parse(end);

                    decimal open_volTotal = 0.0m;
                    decimal open_price_total = 0.0m;
                    decimal open_total = 0.0m;
                    decimal open_count = 0.0m;

                    // loop through each bar in history
                    var history = History(symbol, startDate, endDate, Resolution.Daily);
                    int i = 0;
                    foreach(var bar in history) {
                        // check for day modulus
                        if(++i % openAvgMod == 0) {
                            // retrieve daily $Open value
                            decimal[] open = getOpen(symbol, bar.Time);

                            // determine if $open is less than openAvgMin
                            // if so, omit
                            if(open[2] < openAvgMin)
                                continue; 

                            open_volTotal += open[0];
                            open_price_total += open[1];
                            open_total += open[2];
                            open_count++;
                        }
                    }

                    // define open_avg values
                    decimal open_vol_avg = 0.0m;
                    decimal open_price_avg = 0.0m;
                    decimal open_avg = 0.0m;
                    if(open_count > 0) {
                        open_vol_avg = open_volTotal / open_count;
                        open_price_avg = open_price_total / open_count;
                        open_avg = open_total / open_count;
                        openValueAvgs[symbol] = new decimal[]{open_vol_avg, open_price_avg, open_avg};
                    } else {
                        openValueAvgs[symbol] = new decimal[]{0.0m, 0.0m, 0.0m};
                    }

                    // validate that $open is within provided range, if not then skip
                    if(open_avg < minopen || open_avg > maxopen)
                        continue;

                    // add symbol to values list
                    var enumerable = History(symbol, lookback, Resolution.Daily);
                    var LookbackHistory = enumerable.ToList();
                    filter2[symbol] = LookbackHistory;
                }
            } 
            
            // if no open filter, add all symbols to filter2

            else {
                foreach(Symbol symbol in filter1) {
                    // add symbol to values list
                    var enumerable = History(symbol, lookback, Resolution.Daily);
                    var LookbackHistory = enumerable.ToList();
                    filter2[symbol] = LookbackHistory;
                }
            }

            // call logging function
            Logging(filter2);
        }

        // returns stored value from object storage
        private decimal[] getOpenObjStore(String key) {
            // return value if it exists
            if(openValues.ContainsKey(key)) {
                return openValues[key];
            }

            // otherwise return null
            return null;
        }

        // save an object to object storage
        private void saveOpenObjStore(String key, decimal[] values) {
            // add values to map
            openValues[key] = values;
        }

        // formatting for open object in key storage
        private String formatObjStoreKey(Symbol symbol, DateTime day) {
            return $"{symbol.ToString()}-{day.Year}-{day.Month}-{day.Day}";
        }

        // retrieve open value based on symbol and date
        private decimal[] getOpen(Symbol symbol, DateTime day) {
            String key = formatObjStoreKey(symbol, day);

            // check to see if object store contains values, if so retrieve
            decimal[] open = getOpenObjStore(key);
            if(open != null)
                return open;

            // retrieve open
            DateTime start = new DateTime(day.Year, day.Month, day.Day, 9, 30, 0);
            DateTime end = new DateTime(day.Year, day.Month, day.Day, 9, 30, 1);
            var ticks = History<Tick>(symbol, start, end, Resolution.Tick);
            open = getOpen(ticks);

            // if open is null, retrieve the next 30 seconds of ticks for processing
            if(open == null) {
                // retrieve open
                start = new DateTime(day.Year, day.Month, day.Day, 9, 30, 1);
                end = new DateTime(day.Year, day.Month, day.Day, 9, 30, 30);
                ticks = History<Tick>(symbol, start, end, Resolution.Tick);
                open = getOpen(ticks);
            }

            // if still not found, set to 0
            if(open == null) {
                open = new decimal[]{0, 0, 0};
            }

            saveOpenObjStore(key, open);
            return open;
        }

        // used to retrieve the $open value
        private decimal[] getOpen(IEnumerable<Tick> ticks) {
            // retrieve tick data
            decimal quantity = 0;
            decimal price = 0.0m;
            List<Tick> openings = new List<Tick>();
                
            foreach(var tick in ticks) {
                if(tick.SaleCondition == "4000000" || tick.SaleCondition == "4000001") {
                    openings.Add(tick);
                }
            }

            if(openings.Count() == 0) {
                return null;
            }

            // find tick with largest trade price
            foreach(Tick t in openings) {
                if(t.Quantity > quantity) {
                    quantity = t.Quantity;
                    price = t.Price;
                }
            }

            return new decimal[]{quantity,price,quantity*price};
        }

        private void Logging(Dictionary<Symbol, IEnumerable<TradeBar>> values) {

            foreach(KeyValuePair<Symbol, IEnumerable<TradeBar>> pair in values) {

                var history = values[pair.Key].ToList();
                var symbol = pair.Key;

                // if no history is found 
                if(history.Count() == 0)
                    continue;

                // get number of days high <= open and low >= open
                int hoc = 0;
                int loc = 0;
                decimal hoc_o = 0.0m;
                decimal loc_o = 0.0m;
                int hoc_oc = 0;
                int loc_oc = 0;

                // DD/PNL parameters
                decimal dd = 0.0m;
                decimal dd_max = 0.0m;
                decimal pl = 0.0m;

                int days = 0;

                for(int i = 0; i < history.Count(); i++) {
                    if(i == 0)
                        continue;

                    var bar = history[i];

                    // check volume conditional
                    if(useVolumeFilteringHocLoc && (bar.Volume < W))
                        continue;

                    // check $open conditional
                    if(useOpenFilteringHocLoc) {
                        // retrieve daily $Open value and check conditional
                        var open = getOpen(symbol, bar.Time);

                        if(open[2] < X)
                            continue;
                    }

                    // increment volume days
                    days++;

                    // change values for HOC and LOC to be applied to PL and DD
                    decimal hoc_change = 0.0m, loc_change = 0.0m;
                    
                    // check for HOC
                    if(bar.High <= bar.Open + (useTick1 ? Y : bar.Open * Y)) {
                        hoc++;

                        // if not last day, retrieve o-(o+1)%
                        if(i != history.Count() - 1) {
                            var next = history[i + 1];
                            decimal change = (bar.Open - next.Open) / bar.Open;
                            hoc_o += change;
                            if(change > 0)
                                hoc_oc++;

                            hoc_change = -(next.Open - bar.Open) / bar.Open;
                        }
                    } else {
                        hoc_change = -(useTick1 ? Math.Abs(Y / bar.Open) : Y);
                    }

                    // check for LOC
                    if(bar.Low >= bar.Open - (useTick2 ? Z : bar.Open * Z)) {
                        loc++;

                         // if not last day, retrieve -(o-(o+1))%
                        if(i != history.Count() - 1) {
                            var next = history[i + 1];
                            decimal change = (next.Open - bar.Open) / bar.Open;
                            loc_o += change;
                            if(change > 0)
                                loc_oc++;

                            loc_change = (next.Open - bar.Open) / bar.Open;
                        }
                    } else {
                        loc_change = -(useTick2 ? Math.Abs(-Z / bar.Open) : Z);
                    }
                    
                    // apply percent long and percent short calculations
                    hoc_change *= percentShort;
                    loc_change *= percentLong;

                    // apply PL and DD changes
                    pl += (hoc_change + loc_change);
                    dd += (hoc_change + loc_change);

                    // check for if max
                    if(dd < dd_max)
                        dd_max = dd;
                    // check for if >0
                    if(dd > 0)
                        dd = 0.0m;
                }

                // if no days then skip
                if(days == 0)
                    continue;

                // retrieve last bar to get volume
                var last = history.Last();

                // retrieve volume average
                decimal vol_avg = 0.0m;
                if(volCount[symbol] > 0)
                    vol_avg = volTotal[symbol] / volCount[symbol];


                decimal[] open_values = new decimal[] {0.0m, 0.0m, 0.0m};
                if(openValueAvgs.ContainsKey(symbol)) 
                    open_values = openValueAvgs[symbol];

                int rounding = 2;
                Log($"{Time},{history.Last().Symbol.ToString().Split(" ")[0]},"+                                                                        // time, ticker,
                    $"{history.Count()},{days},{hoc},{loc},"+                                                                                           // days, hoc, loc,
                    $"{Math.Round((hoc*100.0)/days, rounding)},{Math.Round((loc*100.0)/days, rounding)},"+                                              // hoc_adj, loc_adj,
                    $"{Math.Round(hoc_o*100, rounding)},{Math.Round(loc_o*100, rounding)},"+                                                            // hoc_o, loc_o
                    $"{(hoc == 0 ? 0 : Math.Round((hoc_oc*100.0)/hoc, rounding))},{(loc == 0 ? 0 : Math.Round((loc_oc*100.0)/loc, rounding))},"+        // hoc_change, loc_change
                    $"{Math.Round(dd_max*100.0m, rounding)},{Math.Round(pl*100.0m/(days/252.0m), rounding)},"+                                          // dd_max, pl
                    $"{Math.Round(open_values[0], 2)},{Math.Round(open_values[1], 2)},{Math.Round(open_values[2], 2)},{Math.Round(vol_avg, 2)}");               // $open_vol_avg, $open_price_avg, $open_avg, vol_avg
            }

            // reset all lists
            values.Clear();

            // after finished save all open values
            Log("Saving open values...");
            ObjectStore.SaveJson<Dictionary<String, decimal[]>>("openvalues", openValues);
            Log("Open values have been saved.");
        }

		public IEnumerable<Symbol> CoarseFilterFunction(IEnumerable<CoarseFundamental> coarse) {

            foreach(CoarseFundamental c in coarse) {
                // add to volume total and to volume count
                if(!volTotal.ContainsKey(c.Symbol))
                    volTotal[c.Symbol] = 0;
                if(!volCount.ContainsKey(c.Symbol))
                    volCount[c.Symbol] = 0;
                volTotal[c.Symbol] += c.Volume;
                volCount[c.Symbol] += 1;
            }

            return Universe.Unchanged;
		}
    }
}