Historical Data
Universe Data
Introduction
Universe selection is the process of selecting a basket of assets you may trade.
Dynamic universe selection increase diversification and decrease selection bias in your algorithm.
To get the history of a universe, call the History
history
method with the universe object.
Set the flatten
argument to True
to organize the data in a DataFrame for easy pandas wrangling.
Note that the History
history
method doesn't filter the universe with your filter function.
It simply returns all of the assets and data points in the universe dataset.
US Equity Fundamentals
The following example gets the history of a US Equity fundamental universe:
public class USEquityFundamentalUniverseHistoryAlgorithm : QCAlgorithm { public override void Initialize() { SetStartDate(2024, 12, 23); // Add a universe of US Equities based on the Fundamentals dataset. var universe = AddUniverse(fundamentals => fundamentals.Select(f => f.Symbol)); // Get 5 days of history for the universe. var history = History(universe, TimeSpan.FromDays(5)); // Iterate through each day of the universe history. foreach (var fundamentals in history) { // Iterate through each asset in the universe on this day and access its data point attributes. foreach (Fundamental f in fundamentals) { var symbol = f.Symbol; var t = f.EndTime; var peRatio = f.ValuationRatios.PERatio; } } } }
class USEquityFundamentalUniverseHistoryAlgorithm(QCAlgorithm): def initialize(self) -> None: self.set_start_date(2024, 12, 23) # Add a universe of US Equities based on the Fundamentals dataset. universe = self.add_universe(lambda fundamentals: [f.symbol for f in fundamentals]) # Get 5 days of history for the universe. history = self.history(universe, timedelta(5), flatten=True)
# Get the PE Ratio of the first row in the DataFrame. pe_ratio = history.iloc[0].valuationratios.pe_ratio
US Equity ETF Constituents
The following example gets the history of a US Equity ETF constituents universe:
public class ETFConstituentUniverseHistoryAlgorithm : QCAlgorithm { public override void Initialize() { SetStartDate(2024, 12, 23); // Add a universe of US Equities based on the constituents of an ETF. var universe = AddUniverse(Universe.ETF("SPY"); // Get 5 days of history for the universe. var history = History(universe, TimeSpan.FromDays(5)); // Iterate through each day of the universe history. foreach (var constituents in history) { // Select the 2 assets with the smallest weights in the ETF on this day. var dailyLargest = constituents.Select(c => c as ETFConstituentData).OrderBy(c => c.Weight).Take(2); } } }
class ETFConstituentUniverseHistoryAlgorithm(QCAlgorithm): def initialize(self) -> None: self.set_start_date(2024, 12, 23) # Add a universe of US Equities based on the constituents of an ETF. universe = self.add_universe(self.universe.etf('SPY')) # Get 5 days of history for the universe. history = self.history(universe, timedelta(5), flatten=True)
# Select the 2 assets with the smallest weights in the ETF each day. daily_smallest = history.groupby('time').apply(lambda x: x.nsmallest(2, 'weight')).reset_index(level=1, drop=True).weight
time symbol 2024-12-19 AMTMW YM37RIGZUD0L 0.000053 NWSVV VHJF6S7EZRL1 0.000068 2024-12-20 AMTMW YM37RIGZUD0L 0.000051 NWSVV VHJF6S7EZRL1 0.000069 2024-12-21 AMTMW YM37RIGZUD0L 0.000048 NWSVV VHJF6S7EZRL1 0.000069 Name: weight, dtype: float64
US Equity Alternative Data
The following example gets the history of a US Equity alternative data universe:
public class USEquityAltDataUniverseHistoryAlgorithm : QCAlgorithm { public override void Initialize() { SetStartDate(2024, 12, 23); // Add a universe of US Equities based on an alternative dataset. var universe = AddUniverse<BrainStockRankingUniverse>(); // Get 5 days of history for the universe. var history = History(universe, TimeSpan.FromDays(5)); // Iterate through each day of the universe history. foreach (var altCoarse in history) { // Select the asset with the greatest value each day. var dailyWinner = altCoarse .Select(c => c as BrainStockRankingUniverse) .OrderByDescending(c => c.Value) .First(); } } }
class USEquityAltDataUniverseHistoryAlgorithm(QCAlgorithm): def initialize(self) -> None: self.set_start_date(2024, 12, 23) # Add a universe of US Equities based on an alternative dataset. universe = self.add_universe(BrainStockRankingUniverse) # Get 5 days of history for the universe. history = self.history(universe, timedelta(5), flatten=True)
# Select the asset with the greatest value each day. daily_winner = history.groupby('time').apply(lambda x: x.nlargest(1, 'value')).reset_index(level=1, drop=True).value
time symbol 2024-12-18 FIC R735QTJ8XC9X 0.054204 2024-12-19 FIC R735QTJ8XC9X 0.073250 2024-12-20 FIC R735QTJ8XC9X 0.065142 2024-12-21 FIC R735QTJ8XC9X 0.065142 2024-12-22 FIC R735QTJ8XC9X 0.065142 2024-12-23 FIC R735QTJ8XC9X 0.065142 Name: value, dtype: float64
Equity Options
The following example gets the history of an Equity Options universe:
public class EquityOptionUniverseHistoryAlgorithm : QCAlgorithm { public override void Initialize() { SetStartDate(2024, 12, 23); // Add a universe of Equity Option contracts. var option = AddOption("SPY"); // Get 5 days of history for the universe. var history = History<OptionUniverse>(option.Symbol, TimeSpan.FromDays(5)); // Iterate through each day of the universe history. foreach (var chain in history) { var t = chain.EndTime; // Select contracts that have open interest > 0 on this day. var openContracts = chain.Data .Select(contract => contract as OptionUniverse) .Where(contract => contract.OpenInterest > 0m); } } }
class EquityOptionUniverseHistoryAlgorithm(QCAlgorithm): def initialize(self) -> None: self.set_start_date(2024, 12, 23) # Add a universe of Equity Option contracts. option = self.add_option('SPY') # Get 5 days of history for the universe. history = self.history(option.symbol, timedelta(5), flatten=True)
# Find the open interest value at the 99th percentile. high_open_interest = history.openinterest.quantile(0.99)
Index Options
The following example gets the history of an Index Options universe:
public class IndexOptionsUniverseHistoryAlgorithm : QCAlgorithm { public override void Initialize() { SetStartDate(2024, 12, 23); // Add a universe of Index Option contracts. var option = AddIndexOption("VIX"); // Get 5 days of history for the universe. var history = History<OptionUniverse>(option.Symbol, TimeSpan.FromDays(5)); // Iterate through each day of the universe history. foreach (var chain in history) { var t = chain.EndTime; // Calculate the trading volume across all the contracts on this day. var dailyVolume = chain.Data .Select(contract => contract as OptionUniverse) .Sum(contract => contract.Volume); } } }
class IndexOptionsUniverseHistoryAlgorithm(QCAlgorithm): def initialize(self) -> None: self.set_start_date(2024, 12, 23) # Add a universe of Index Option contracts. option = self.add_index_option('VIX') # Get 5 days of history for the universe. history = self.history(option.symbol, timedelta(5), flatten=True)
# Get the daily trading volume across all the contracts. daily_volume = history.groupby('time').apply(lambda x: x.volume.sum())
time 2024-12-19 1285753.0 2024-12-20 1374216.0 2024-12-21 833384.0 dtype: float64
Crypto
The following example gets the history of a Crypto universe:
public class CryptoUniverseHistoryAlgorithm : QCAlgorithm { public override void Initialize() { SetStartDate(2024, 12, 23); // Add a universe of Cryptocurrencies. var universe = AddUniverse(CryptoUniverse.Coinbase(data => data.Select(x => x.Symbol))); // Get 5 days of history for the universe. var history = History(universe, TimeSpan.FromDays(5)); // Iterate through each day of the universe history. foreach (var universeDay in history) { var t = universeDay.EndTime; // Select the Crypto with the largest daily return on this day. var topGainer = universeDay .Select(c => c as CryptoUniverse) .OrderByDescending(c => (c.Close - c.Open) / c.Open) .First(); } } }
class CryptoUniverseHistoryAlgorithm(QCAlgorithm): def initialize(self) -> None: self.set_start_date(2024, 12, 23) # Add a universe of Cryptocurrencies. universe = self.add_universe(CryptoUniverse.coinbase(lambda data: [x.symbol for x in data])) # Get 5 days of history for the universe. history = self.history(universe, timedelta(5), flatten=True)
# Get the Crypto with the largest daily return each day. history['roc'] = (history['close'] - history['open']) / history['open'] top_gainers = history.loc[history.groupby('time')['roc'].idxmax()]
Crypto Market Cap
The following example gets the history of a Crypto market cap universe:
public class CryptoMarketCapUniverseHistoryAlgorithm : QCAlgorithm { public override void Initialize() { SetStartDate(2024, 12, 24); // Add a universe of Cryptocurrencies. var universe = AddUniverse<CoinGecko>(); // Get 365 days of history for the universe. var history = History(universe, TimeSpan.FromDays(365)); // Iterate through each day of the universe history. foreach (var universeDay in history) { var t = universeDay.EndTime; // Select the Crypto with the largest market cap on this day. var largestCoin = universeDay.Select(c => c as CoinGecko).OrderByDescending(c => c.MarketCap).First(); } } }
class CryptoMarketCapUniverseHistoryAlgorithm(QCAlgorithm): def initialize(self) -> None: self.set_start_date(2024, 12, 24) # Add a universe of Cryptocurrencies. universe = self.add_universe(CoinGeckoUniverse) # Get 355 days of history for the universe. history = self.history(universe, timedelta(365), flatten=True)
# Select the largest Crypto each day. largest_coin = history.drop('time', axis=1).groupby('time').apply( lambda x: x.nlargest(1, 'marketcap') ).reset_index(level=1, drop=True).marketcap
time symbol 2023-12-25 TAO.CoinGecko 2S 1.608797e+09 2023-12-26 TAO.CoinGecko 2S 1.604877e+09 2023-12-27 TAO.CoinGecko 2S 1.519449e+09 2023-12-28 TAO.CoinGecko 2S 1.878762e+09 2023-12-29 TAO.CoinGecko 2S 1.804409e+09 ... 2024-12-19 TAO.CoinGecko 2S 1.749002e+09 2024-12-20 TAO.CoinGecko 2S 1.749002e+09 2024-12-21 TAO.CoinGecko 2S 1.749002e+09 2024-12-22 TSM.CoinGecko 2S NaN 2024-12-23 TSM.CoinGecko 2S NaN Name: marketcap, Length: 365, dtype: float64
Custom Data
The following example gets the history of a custom data universe:
// Define the custom data class outside of the algorithm class. public class StockDataSource : BaseData { public List<string> Symbols { get; set; } = new(); public override DateTime EndTime => Time.AddDays(1); public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode) { return new SubscriptionDataSource( "https://raw.githubusercontent.com/QuantConnect/Documentation/master/Resources/datasets/custom-data/csv-universe-example.csv", SubscriptionTransportMedium.RemoteFile, FileFormat.Csv ); } public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode) { var stocks = new StockDataSource(); stocks.Symbol = config.Symbol; try { var csv = line.Split(','); stocks.Time = DateTime.ParseExact(csv[0], "yyyyMMdd", null); stocks.Symbols.AddRange(csv.Skip(1)); } catch { return null; } return stocks; } } public class CustomDataUniverseHistoryAlgorithm : QCAlgorithm { public override void Initialize() { SetStartDate(2018, 1, 10); // Add a custom data universe. var universe = AddUniverse<StockDataSource>("StockDataSource", Resolution.Daily); // Get the universe history. var history = History(universe, new DateTime(2018, 1, 1), new DateTime(2018, 1, 10)).Cast<StockDataSource>(); // Iterate through each day in the universe history and count the number of constituents. foreach (var stockData in history) { var t = stockData.Time; var size = stockData.Symbols.Count; } } }
# Define the custom data class outside of the algorithm class. class StockDataSource(PythonData): def get_source(self, config: SubscriptionDataConfig, date: datetime, is_live: bool) -> SubscriptionDataSource: return SubscriptionDataSource( "https://raw.githubusercontent.com/QuantConnect/Documentation/master/Resources/datasets/custom-data/csv-universe-example.csv", SubscriptionTransportMedium.REMOTE_FILE, FileFormat.CSV ) def reader(self,config: SubscriptionDataConfig, line: str, date: datetime, is_live: bool) -> BaseData: if not (line.strip() and line[0].isdigit()): return None stocks = StockDataSource() stocks.symbol = config.symbol try: csv = line.split(',') stocks.time = datetime.strptime(csv[0], "%Y%m%d") stocks.end_time = stocks.time + timedelta(days=1) stocks["Symbols"] = csv[1:] except ValueError: return None return stocks class CustomDataUniverseHistoryAlgorithm(QCAlgorithm): def initialize(self) -> None: self.set_start_date(2018, 1, 10) # Add a custom data universe. universe = self.add_universe(StockDataSource) # Get the universe history. history = self.history(universe, datetime(2018, 1, 1), datetime(2018, 1, 10), flatten=True)
# Count the number of assets in the universe each day. universe_size = history.apply(lambda row: len(row.symbols), axis=1).reset_index(level=0, drop=True)
time 2018-01-04 5 2018-01-05 5 2018-01-06 5 2018-01-07 5 2018-01-08 5 2018-01-09 5 2018-01-10 5 dtype: int64