Historical Data

Universe Data

Introduction

Universe selection is the process of selecting a basket of assets you may trade. Dynamic universe selection increase diversification and decrease selection bias in your algorithm. To get the history of a universe, call the Historyhistory method with the universe object. Set the flatten argument to True to organize the data in a DataFrame for easy pandas wrangling. Note that the Historyhistory method doesn't filter the universe with your filter function. It simply returns all of the assets and data points in the universe dataset.

US Equity Fundamentals

The following example gets the history of a US Equity fundamental universe:

public class USEquityFundamentalUniverseHistoryAlgorithm : QCAlgorithm
{
    public override void Initialize()
    {
        SetStartDate(2024, 12, 23);
        // Add a universe of US Equities based on the Fundamentals dataset.
        var universe = AddUniverse(fundamentals => fundamentals.Select(f => f.Symbol));
        // Get 5 days of history for the universe.
        var history = History(universe, TimeSpan.FromDays(5));
        // Iterate through each day of the universe history.
        foreach (var fundamentals in history)
        {
            // Iterate through each asset in the universe on this day and access its data point attributes.
            foreach (Fundamental f in fundamentals)
            {
                var symbol = f.Symbol;
                var t = f.EndTime;
                var peRatio = f.ValuationRatios.PERatio;
            }
        }
    }
}
class USEquityFundamentalUniverseHistoryAlgorithm(QCAlgorithm):

    def initialize(self) -> None:
        self.set_start_date(2024, 12, 23)
        # Add a universe of US Equities based on the Fundamentals dataset.
        universe = self.add_universe(lambda fundamentals: [f.symbol for f in fundamentals])
        # Get 5 days of history for the universe.
        history = self.history(universe, timedelta(5), flatten=True)
DataFrame of the last 5 days of a US Equity fundamental universe.
# Get the PE Ratio of the first row in the DataFrame.
pe_ratio = history.iloc[0].valuationratios.pe_ratio

US Equity ETF Constituents

The following example gets the history of a US Equity ETF constituents universe:

public class ETFConstituentUniverseHistoryAlgorithm : QCAlgorithm
{
    public override void Initialize()
    {
        SetStartDate(2024, 12, 23);
        // Add a universe of US Equities based on the constituents of an ETF.
        var universe = AddUniverse(Universe.ETF("SPY");
        // Get 5 days of history for the universe.
        var history = History(universe, TimeSpan.FromDays(5));
        // Iterate through each day of the universe history.
        foreach (var constituents in history)
        {
            // Select the 2 assets with the smallest weights in the ETF on this day.
            var dailyLargest = constituents.Select(c => c as ETFConstituentData).OrderBy(c => c.Weight).Take(2);
        }
    }
}
class ETFConstituentUniverseHistoryAlgorithm(QCAlgorithm):

    def initialize(self) -> None:
        self.set_start_date(2024, 12, 23)
        # Add a universe of US Equities based on the constituents of an ETF.
        universe = self.add_universe(self.universe.etf('SPY'))
        # Get 5 days of history for the universe.
        history = self.history(universe, timedelta(5), flatten=True)
DataFrame of the last 5 days of a US Equity ETF Constituents universe.
# Select the 2 assets with the smallest weights in the ETF each day.
daily_smallest = history.groupby('time').apply(lambda x: x.nsmallest(2, 'weight')).reset_index(level=1, drop=True).weight
time        symbol            
2024-12-19  AMTMW YM37RIGZUD0L    0.000053
            NWSVV VHJF6S7EZRL1    0.000068
2024-12-20  AMTMW YM37RIGZUD0L    0.000051
            NWSVV VHJF6S7EZRL1    0.000069
2024-12-21  AMTMW YM37RIGZUD0L    0.000048
            NWSVV VHJF6S7EZRL1    0.000069
Name: weight, dtype: float64

US Equity Alternative Data

The following example gets the history of a US Equity alternative data universe:

public class USEquityAltDataUniverseHistoryAlgorithm : QCAlgorithm
{
    public override void Initialize()
    {
        SetStartDate(2024, 12, 23);
        // Add a universe of US Equities based on an alternative dataset.
        var universe = AddUniverse<BrainStockRankingUniverse>();
        // Get 5 days of history for the universe.
        var history = History(universe, TimeSpan.FromDays(5));
        // Iterate through each day of the universe history.
        foreach (var altCoarse in history)
        {
            // Select the asset with the greatest value each day.
            var dailyWinner = altCoarse
                .Select(c => c as BrainStockRankingUniverse)
                .OrderByDescending(c => c.Value)
                .First();
        }
    }
}
class USEquityAltDataUniverseHistoryAlgorithm(QCAlgorithm):

    def initialize(self) -> None:
        self.set_start_date(2024, 12, 23)
        # Add a universe of US Equities based on an alternative dataset.
        universe = self.add_universe(BrainStockRankingUniverse)
        # Get 5 days of history for the universe.
        history = self.history(universe, timedelta(5), flatten=True)
DataFrame of the last 5 days of a US Equity alternative data universe.
# Select the asset with the greatest value each day.
daily_winner = history.groupby('time').apply(lambda x: x.nlargest(1, 'value')).reset_index(level=1, drop=True).value
time        symbol          
2024-12-18  FIC R735QTJ8XC9X    0.054204
2024-12-19  FIC R735QTJ8XC9X    0.073250
2024-12-20  FIC R735QTJ8XC9X    0.065142
2024-12-21  FIC R735QTJ8XC9X    0.065142
2024-12-22  FIC R735QTJ8XC9X    0.065142
2024-12-23  FIC R735QTJ8XC9X    0.065142
Name: value, dtype: float64

Equity Options

The following example gets the history of an Equity Options universe:

public class EquityOptionUniverseHistoryAlgorithm : QCAlgorithm
{
    public override void Initialize()
    {
        SetStartDate(2024, 12, 23);
        // Add a universe of Equity Option contracts.
        var option = AddOption("SPY");
        // Get 5 days of history for the universe.
        var history = History<OptionUniverse>(option.Symbol, TimeSpan.FromDays(5));
        // Iterate through each day of the universe history.
        foreach (var chain in history)
        {
            var t = chain.EndTime;
            // Select contracts that have open interest > 0 on this day.
            var openContracts = chain.Data
                .Select(contract => contract as OptionUniverse)
                .Where(contract => contract.OpenInterest > 0m);
        }
    }
}
class EquityOptionUniverseHistoryAlgorithm(QCAlgorithm):

    def initialize(self) -> None:
        self.set_start_date(2024, 12, 23)
        # Add a universe of Equity Option contracts.
        option = self.add_option('SPY')
        # Get 5 days of history for the universe.
        history = self.history(option.symbol, timedelta(5), flatten=True)
DataFrame of the last 5 days of a US Equity Options universe.
# Find the open interest value at the 99th percentile.
high_open_interest = history.openinterest.quantile(0.99)

Index Options

The following example gets the history of an Index Options universe:

public class IndexOptionsUniverseHistoryAlgorithm : QCAlgorithm
{
    public override void Initialize()
    {
        SetStartDate(2024, 12, 23);
        // Add a universe of Index Option contracts.
        var option = AddIndexOption("VIX");
        // Get 5 days of history for the universe.
        var history = History<OptionUniverse>(option.Symbol, TimeSpan.FromDays(5));
        // Iterate through each day of the universe history.
        foreach (var chain in history)
        {
            var t = chain.EndTime;
            // Calculate the trading volume across all the contracts on this day.
            var dailyVolume = chain.Data
                .Select(contract => contract as OptionUniverse)
                .Sum(contract => contract.Volume);
        }
    }
}
class IndexOptionsUniverseHistoryAlgorithm(QCAlgorithm):

    def initialize(self) -> None:
        self.set_start_date(2024, 12, 23)
        # Add a universe of Index Option contracts.
        option = self.add_index_option('VIX')
        # Get 5 days of history for the universe.
        history = self.history(option.symbol, timedelta(5), flatten=True)
DataFrame of the last 5 days of an Index Options universe.
# Get the daily trading volume across all the contracts.
daily_volume = history.groupby('time').apply(lambda x: x.volume.sum())
time
2024-12-19    1285753.0
2024-12-20    1374216.0
2024-12-21     833384.0
dtype: float64

Crypto

The following example gets the history of a Crypto universe:

public class CryptoUniverseHistoryAlgorithm : QCAlgorithm
{
    public override void Initialize()
    {
        SetStartDate(2024, 12, 23);
        // Add a universe of Cryptocurrencies.
        var universe = AddUniverse(CryptoUniverse.Coinbase(data => data.Select(x => x.Symbol)));
        // Get 5 days of history for the universe.
        var history = History(universe, TimeSpan.FromDays(5));
        // Iterate through each day of the universe history.
        foreach (var universeDay in history)
        {
            var t = universeDay.EndTime;
            // Select the Crypto with the largest daily return on this day.
            var topGainer = universeDay
                .Select(c => c as CryptoUniverse)
                .OrderByDescending(c => (c.Close - c.Open) / c.Open)
                .First();
        }
    }
}
class CryptoUniverseHistoryAlgorithm(QCAlgorithm):

    def initialize(self) -> None:
        self.set_start_date(2024, 12, 23)
        # Add a universe of Cryptocurrencies.
        universe = self.add_universe(CryptoUniverse.coinbase(lambda data: [x.symbol for x in data]))
        # Get 5 days of history for the universe.
        history = self.history(universe, timedelta(5), flatten=True)
DataFrame of the last 5 days of a Crypto universe.
# Get the Crypto with the largest daily return each day.
history['roc'] = (history['close'] - history['open']) / history['open']
top_gainers = history.loc[history.groupby('time')['roc'].idxmax()]
DataFrame of the daily top gainers in the Crypto universe.

Crypto Market Cap

The following example gets the history of a Crypto market cap universe:

public class CryptoMarketCapUniverseHistoryAlgorithm : QCAlgorithm
{
    public override void Initialize()
    {
        SetStartDate(2024, 12, 24);
        // Add a universe of Cryptocurrencies.
        var universe = AddUniverse<CoinGecko>();
        // Get 365 days of history for the universe.
        var history = History(universe, TimeSpan.FromDays(365));
        // Iterate through each day of the universe history.
        foreach (var universeDay in history)
        {
            var t = universeDay.EndTime;
            // Select the Crypto with the largest market cap on this day.
            var largestCoin = universeDay.Select(c => c as CoinGecko).OrderByDescending(c => c.MarketCap).First();
        }
    }
}
class CryptoMarketCapUniverseHistoryAlgorithm(QCAlgorithm):

    def initialize(self) -> None:
        self.set_start_date(2024, 12, 24)
        # Add a universe of Cryptocurrencies.
        universe = self.add_universe(CoinGeckoUniverse)
        # Get 355 days of history for the universe.
        history = self.history(universe, timedelta(365), flatten=True)
DataFrame of the last 365 days of a Crypto market cap universe.
# Select the largest Crypto each day.
largest_coin = history.drop('time', axis=1).groupby('time').apply(
    lambda x: x.nlargest(1, 'marketcap')
).reset_index(level=1, drop=True).marketcap
time        symbol          
2023-12-25  TAO.CoinGecko 2S    1.608797e+09
2023-12-26  TAO.CoinGecko 2S    1.604877e+09
2023-12-27  TAO.CoinGecko 2S    1.519449e+09
2023-12-28  TAO.CoinGecko 2S    1.878762e+09
2023-12-29  TAO.CoinGecko 2S    1.804409e+09
                                    ...     
2024-12-19  TAO.CoinGecko 2S    1.749002e+09
2024-12-20  TAO.CoinGecko 2S    1.749002e+09
2024-12-21  TAO.CoinGecko 2S    1.749002e+09
2024-12-22  TSM.CoinGecko 2S             NaN
2024-12-23  TSM.CoinGecko 2S             NaN
Name: marketcap, Length: 365, dtype: float64

Custom Data

The following example gets the history of a custom data universe:

// Define the custom data class outside of the algorithm class.
public class StockDataSource : BaseData
{
    public List<string> Symbols { get; set; } = new();
    public override DateTime EndTime => Time.AddDays(1);

    public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
    {
        return new SubscriptionDataSource(
            "https://raw.githubusercontent.com/QuantConnect/Documentation/master/Resources/datasets/custom-data/csv-universe-example.csv", 
            SubscriptionTransportMedium.RemoteFile, 
            FileFormat.Csv
        );
    }

    public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode)
    {
        var stocks = new StockDataSource();
        stocks.Symbol = config.Symbol;
        try
        {
            var csv = line.Split(',');
            stocks.Time = DateTime.ParseExact(csv[0], "yyyyMMdd", null);
            stocks.Symbols.AddRange(csv.Skip(1));
        }
        catch { return null; }
        return stocks;
    }
}

public class CustomDataUniverseHistoryAlgorithm : QCAlgorithm
{
    public override void Initialize()
    {
        SetStartDate(2018, 1, 10);
        // Add a custom data universe.
        var universe = AddUniverse<StockDataSource>("StockDataSource", Resolution.Daily);
        // Get the universe history.
        var history = History(universe, new DateTime(2018, 1, 1), new DateTime(2018, 1, 10)).Cast<StockDataSource>();
        // Iterate through each day in the universe history and count the number of constituents.
        foreach (var stockData in history)
        {
            var t = stockData.Time;
            var size = stockData.Symbols.Count;
        }
    }
}
# Define the custom data class outside of the algorithm class.    
class StockDataSource(PythonData):

    def get_source(self, config: SubscriptionDataConfig, date: datetime, is_live: bool) -> SubscriptionDataSource:
        return SubscriptionDataSource(
            "https://raw.githubusercontent.com/QuantConnect/Documentation/master/Resources/datasets/custom-data/csv-universe-example.csv", 
            SubscriptionTransportMedium.REMOTE_FILE, 
            FileFormat.CSV
        )

    def reader(self,config: SubscriptionDataConfig, line: str, date: datetime, is_live: bool) -> BaseData:
        if not (line.strip() and line[0].isdigit()): 
            return None
        stocks = StockDataSource()
        stocks.symbol = config.symbol
        try:
            csv = line.split(',')
            stocks.time = datetime.strptime(csv[0], "%Y%m%d")
            stocks.end_time = stocks.time + timedelta(days=1)
            stocks["Symbols"] = csv[1:]
        except ValueError:
            return None
        return stocks


class CustomDataUniverseHistoryAlgorithm(QCAlgorithm):

    def initialize(self) -> None:
        self.set_start_date(2018, 1, 10)
        # Add a custom data universe.
        universe = self.add_universe(StockDataSource)
        # Get the universe history.
        history = self.history(universe, datetime(2018, 1, 1), datetime(2018, 1, 10), flatten=True)
DataFrame of the custom data universe.
# Count the number of assets in the universe each day.
universe_size = history.apply(lambda row: len(row.symbols), axis=1).reset_index(level=0, drop=True)
time
2018-01-04    5
2018-01-05    5
2018-01-06    5
2018-01-07    5
2018-01-08    5
2018-01-09    5
2018-01-10    5
dtype: int64

You can also see our Videos. You can also get in touch with us via Discord.

Did you find this page helpful?

Contribute to the documentation: