Using Universe Selection With alternative Dataset

Hi Garrison

For dynamic universe, the Benzinga News Feed data has to be subscribe in the OnSecuritiesChanged method by:

def OnSecuritiesChanged(self, changes):
    for added in changes.AddedSecurities:
       self.AddData(BenzingaNews, added.Symbol)

as Initialize method will only run once at the start of the algorithm.

Best
Louis

You are awesome! I got it to work, Now how would I go about setting the universe coarse/fine to a custom metric such as marketCap / Dollarvolume < 10,000 ? I have tried following the documentation here: https://www.quantconnect.com/docs/algorithm-reference/universes

But It doesn't tell how to define the symbols before it gets to the class object as i am getting errors noting that.

from datetime import datetime, timedelta

### <summary>
### Benzinga is a provider of news data. Their news is made in-house
### and covers stock related news such as corporate events.
### </summary>
class BenzingaNewsAlgorithm(QCAlgorithm):

    def Initialize(self):
        self.words = {
            "bad": -0.5, "good": 0.5,
            "negative": -0.5, "great": 0.5,
            "growth": 0.5, "fail": -0.5,
            "failed": -0.5, "success": 0.5,
            "nailed": 0.5, "beat": 0.5,
            "missed": -0.5
        }

        self.lastTrade = datetime(1, 1, 1)
        self.SetStartDate(2018, 6, 5)
        self.SetEndDate(2018, 8, 4)
        self.SetCash(100000)
        self.SetUniverseSelection(SelectionData())

    #def MyUniverse(self, algorithm, coarse):
        #filteredByVolume = [c for c in coarse if c.DollarVolume > 100000]
        #return [c.Symbol for c in coarse]

    #def MyUniverse(self, algorithm, fine):
        #filteredByMktCap = [x for x in fine if 1e8 < x.MarketCap < 1e9] #between 100-1 bill
        #filteredByMktCap = [x for x in fine if (x.MarketCap / c.DollarVolume) < 8000] #new test
        #return [x.Symbol for x in filteredByMktCap]

    
        
    def OnSecuritiesChanged(self, changes):
        for added in changes.AddedSecurities:
            self.AddData(BenzingaNews, added.Symbol)
            
        for r in changes.RemovedSecurities:
            # If removed from the universe, liquidate and remove the custom data from the algorithm
            self.Liquidate(r.Symbol)
            self.RemoveSecurity(Symbol.CreateBase(BenzingaNews, r.Symbol, Market.USA))
            
    def OnData(self, data):
        
        if (self.Time - self.lastTrade) < timedelta(days=5):
            return

        # Get rid of our holdings after 5 days, and start fresh
        self.Liquidate()

        # Get all Benzinga data and loop over it
        for article in data.Get(BenzingaNews).Values:
            selectedSymbol = None

            # Use loop instead of list comprehension for clarity purposes

            # Select the same Symbol we're getting a data point for
            # from the articles list so that we can get the sentiment of the article
            # We use the underlying Symbol because the Symbols included in the `Symbols` property
            # are equity Symbols.
            for symbol in article.Symbols:
                if symbol == article.Symbol.Underlying:
                    selectedSymbol = symbol
                    break

            if selectedSymbol is None:
                raise Exception(f"Could not find current Symbol {article.Symbol.Underlying} even though it should exist")

            # The intersection of the article contents and the pre-defined words are the words that are included in both collections
            intersection = set(article.Contents.lower().split(" ")).intersection(list(self.words.keys()))
            # Get the words, then get the aggregate sentiment
            sentimentSum = sum([self.words[i] for i in intersection])

            if sentimentSum >= 0.5:
                self.Log(f"Longing {article.Symbol.Underlying} with sentiment score of {sentimentSum}")
                self.SetHoldings(article.Symbol.Underlying, sentimentSum / 5)

                self.lastTrade = self.Time

            if sentimentSum <= -0.5:
                self.Log(f"Shorting {article.Symbol.Underlying} with sentiment score of {sentimentSum}")
                self.SetHoldings(article.Symbol.Underlying, sentimentSum / 5)

                self.lastTrade = self.Time

class SelectionData(object):
    def __init__(self, symbol):
        self.symbol = symbol
        self.DollarVolume = 0
        self.Marketcap_ratio = 0
        self.MarketCap = 0

    def update(self, DollarVolume, MarketCap):
        self.DollarVolume = DollarVolume
        self.MarketCap = MarketCap
        if self.DollarVolume > 100000:
           
            self.Marketcap_ratio = MarketCap / DollarVolume

Sorry Use this example instead, forgot the second part, still getting the same error:

from datetime import datetime, timedelta

### <summary>
### Benzinga is a provider of news data. Their news is made in-house
### and covers stock related news such as corporate events.
### </summary>
class BenzingaNewsAlgorithm(QCAlgorithm):

    def Initialize(self):
        self.words = {
            "bad": -0.5, "good": 0.5,
            "negative": -0.5, "great": 0.5,
            "growth": 0.5, "fail": -0.5,
            "failed": -0.5, "success": 0.5,
            "nailed": 0.5, "beat": 0.5,
            "missed": -0.5
        }

        self.lastTrade = datetime(1, 1, 1)
        self.SetStartDate(2018, 6, 5)
        self.SetEndDate(2018, 8, 4)
        self.SetCash(100000)
        self.SetUniverseSelection(SelectionData())

    #def MyUniverse(self, algorithm, coarse):
        #filteredByVolume = [c for c in coarse if c.DollarVolume > 100000]
        #return [c.Symbol for c in coarse]

    #def MyUniverse(self, algorithm, fine):
        #filteredByMktCap = [x for x in fine if 1e8 < x.MarketCap < 1e9] #between 100-1 bill
        #filteredByMktCap = [x for x in fine if (x.MarketCap / c.DollarVolume) < 8000] #new test
        #return [x.Symbol for x in filteredByMktCap]

    
        
    def OnSecuritiesChanged(self, changes):
        for added in changes.AddedSecurities:
            self.AddData(BenzingaNews, added.Symbol)
            
        for r in changes.RemovedSecurities:
            # If removed from the universe, liquidate and remove the custom data from the algorithm
            self.Liquidate(r.Symbol)
            self.RemoveSecurity(Symbol.CreateBase(BenzingaNews, r.Symbol, Market.USA))
            
    def OnData(self, data):
        
        if (self.Time - self.lastTrade) < timedelta(days=5):
            return

        # Get rid of our holdings after 5 days, and start fresh
        self.Liquidate()

        # Get all Benzinga data and loop over it
        for article in data.Get(BenzingaNews).Values:
            selectedSymbol = None

            # Use loop instead of list comprehension for clarity purposes

            # Select the same Symbol we're getting a data point for
            # from the articles list so that we can get the sentiment of the article
            # We use the underlying Symbol because the Symbols included in the `Symbols` property
            # are equity Symbols.
            for symbol in article.Symbols:
                if symbol == article.Symbol.Underlying:
                    selectedSymbol = symbol
                    break

            if selectedSymbol is None:
                raise Exception(f"Could not find current Symbol {article.Symbol.Underlying} even though it should exist")

            # The intersection of the article contents and the pre-defined words are the words that are included in both collections
            intersection = set(article.Contents.lower().split(" ")).intersection(list(self.words.keys()))
            # Get the words, then get the aggregate sentiment
            sentimentSum = sum([self.words[i] for i in intersection])

            if sentimentSum >= 0.5:
                self.Log(f"Longing {article.Symbol.Underlying} with sentiment score of {sentimentSum}")
                self.SetHoldings(article.Symbol.Underlying, sentimentSum / 5)

                self.lastTrade = self.Time

            if sentimentSum <= -0.5:
                self.Log(f"Shorting {article.Symbol.Underlying} with sentiment score of {sentimentSum}")
                self.SetHoldings(article.Symbol.Underlying, sentimentSum / 5)

                self.lastTrade = self.Time

class SelectionData(object):
    def __init__(self, symbol):
        self.symbol = symbol
        self.DollarVolume = 0
        self.Marketcap_ratio = 0
        self.MarketCap = 0

    def update(self, DollarVolume, MarketCap):
        self.DollarVolume = DollarVolume
        self.MarketCap = MarketCap
        self.Marketcap_ratio = MarketCap / DollarVolume
        
    def CoarseFilterFunction(self, coarse):
        for c in coarse:
            if c.Symbol not in self.stateData:
                self.stateData[c.Symbol] = SelectionData(c.Symbol, 10)
            avg = self.stateData[c.Symbol]
            avg.update(c.EndTime, c.AdjustedPrice, c.DollarVolume)

        # filter the values of selectionData(sd) above SMA
        values = [sd for sd in self.stateData.values() if sd.Marketcap_ratio < 10000 and sd.DollarVolume > 100000]
        
        # sort sd by the largest % jump in volume.
        #values.sort(key=lambda sd: sd.volume_ratio, reverse=True)

        # return the top 10 symbol objects
        return [ sd.symbol for sd in values ]   

Louis Szeto 

Hi Garrison Whipple,
The Universe selection method in the snippet is incorrect. The attribute MarketCap is a property of the FineFundamental object; to get the above method to work, we recommend implementing different functions for coarse and fine selection. We recommend going through the following example algorithm, which combines coarse and fine filters for universe selection.
Best,
Varad Kabade

Varad kabade where does it combine them? the example you gave has them filtered seperately.

Hi Garrison

They are combined. Only the symbols returned from Coarse Universe Selection will relay to the Fine Universe Selection method. Check this doc to see how universe selection works.

Best
Louis

Louis Szeto are you saying what I am trying to do is not possible?

Hi Garrison Whipple,
The filtering that you are trying to do is possible. You just have to break down the filter into a coarse and fine filter function.
Best,
Varad Kabade

Louis Szeto Varad kabade I did try that already but I don't understand how to use the DollarVolume variable from the coarse filter with the marketcap from the fine filter at the same time.

Do I somehow pass the variables to another function or do I pass the coarse variable to the fine function similar to what I have below?

from datetime import datetime, timedelta

### <summary>
### Benzinga is a provider of news data. Their news is made in-house
### and covers stock related news such as corporate events.
### </summary>
class BenzingaNewsAlgorithm(QCAlgorithm):

    def Initialize(self):
        self.words = {
            "bad": -0.5, "good": 0.5,
            "negative": -0.5, "great": 0.5,
            "growth": 0.5, "fail": -0.5,
            "failed": -0.5, "success": 0.5,
            "nailed": 0.5, "beat": 0.5,
            "missed": -0.5
        }

        self.lastTrade = datetime(1, 1, 1)
        self.SetStartDate(2018, 6, 5)
        self.SetEndDate(2018, 8, 4)
        self.SetCash(100000)
        self.AddUniverse(self.CoarseSelectionFunction, self.FineSelectionFunction)

    def CoarseSelectionFunction(self, coarse):
        filteredByVolume = [c for c in coarse if c.DollarVolume > 100000]
        return [c.Symbol for c in coarse]

    def FineSelectionFunction(self, DollarVolume, fine):
        #filteredByMktCap = [x for x in fine if 1000000 < x.MarketCap ] 
        filteredByMktCap = [x for x in fine if (x.MarketCap / x.DollarVolume) < 1000 and 1000000 < x.MarketCap]
        return [x.Symbol for x in filteredByMktCap]

    
        
    def OnSecuritiesChanged(self, changes):
        for added in changes.AddedSecurities:
            self.AddData(BenzingaNews, added.Symbol)
            
        for r in changes.RemovedSecurities:
            # If removed from the universe, liquidate and remove the custom data from the algorithm
            self.Liquidate(r.Symbol)
            self.RemoveSecurity(Symbol.CreateBase(BenzingaNews, r.Symbol, Market.USA))
            
    def OnData(self, data):
        
        if (self.Time - self.lastTrade) < timedelta(days=5):
            return

        # Get rid of our holdings after 5 days, and start fresh
        self.Liquidate()

        # Get all Benzinga data and loop over it
        for article in data.Get(BenzingaNews).Values:
            selectedSymbol = None

            # Use loop instead of list comprehension for clarity purposes

            # Select the same Symbol we're getting a data point for
            # from the articles list so that we can get the sentiment of the article
            # We use the underlying Symbol because the Symbols included in the `Symbols` property
            # are equity Symbols.
            for symbol in article.Symbols:
                if symbol == article.Symbol.Underlying:
                    selectedSymbol = symbol
                    break

            if selectedSymbol is None:
                raise Exception(f"Could not find current Symbol {article.Symbol.Underlying} even though it should exist")

            # The intersection of the article contents and the pre-defined words are the words that are included in both collections
            intersection = set(article.Contents.lower().split(" ")).intersection(list(self.words.keys()))
            # Get the words, then get the aggregate sentiment
            sentimentSum = sum([self.words[i] for i in intersection])

            if sentimentSum >= 0.5:
                self.Log(f"Longing {article.Symbol.Underlying} with sentiment score of {sentimentSum}")
                self.SetHoldings(article.Symbol.Underlying, sentimentSum / 5)

                self.lastTrade = self.Time

            if sentimentSum <= -0.5:
                self.Log(f"Shorting {article.Symbol.Underlying} with sentiment score of {sentimentSum}")
                self.SetHoldings(article.Symbol.Underlying, sentimentSum / 5)

                self.lastTrade = self.Time


        

Hi Garrison

Only the symbols from the Coarse Universe Selection method will pass on for Fine filtering, so the Fine Universe Selection method will not have access to .DollarVolume attribute as in Coarse Selection. However, since the working order is the first filter by Coarse then Fine, we can use the updated dollar volume in Fine by storing it in a dictionary:

def CoarseSelectionFunction(self, coarse):
    self.dollarVolume = {}
    for c in coarse:
        if c.DollarVolume > 100000 and c.HasFundamentalData:
            self.dollarVolume[c.Symbol] = c.DollarVolume
    return list(self.dollarVolume.keys())

def FineSelectionFunction(self, DollarVolume, fine):
    filteredByMktCap = [x for x in fine if (x.MarketCap / self.dollarVolume[x.Symbol]) < 1000 and 1000000 < x.MarketCap]
    return [x.Symbol for x in filteredByMktCap]

Best
Louis