Search code examples
pythontwittertweepytwitter-streaming-api

Twitters streaming API, sample tweets, getting rate limitations?


I am quite new to Twitter API and Tweepy and I am confused with the rate-limiting concept, I am using the streaming API and I want to gather sample tweets without using any filters such as hashtags or location, some sources state I should not get rate limited with sample tweets as I am getting 1% of tweets and some state otherwise. I keep getting error 420 very often and I was wondering if there is a way to avoid it or make it smoother? Thank you so much for your help

My code:

import json
from tweepy.streaming import StreamListener
from tweepy import OAuthHandler
from tweepy import Stream
from textblob import TextBlob
from elasticsearch import Elasticsearch
from datetime import datetime

# import twitter keys and tokens
from config import *

# create instance of elasticsearch
es = Elasticsearch()

indexName = "test_new_fields"

consumer_key = ''
consumer_secret = ''
    access_token = ''
    access_token_secret = ''

class TweetStreamListener(StreamListener):

    hashtags = []

    # on success
    def on_data(self, data):

        # decode json
        dict_data = json.loads(data) # data is a json string

        # print(data) # to print the twitter json string
        print(dict_data)

        # pass tweet into TextBlob
        tweet = TextBlob(dict_data["text"])

        # determine if sentiment is positive, negative, or neutral
        if tweet.sentiment.polarity < 0:
            sentiment = "negative"
        elif tweet.sentiment.polarity == 0:
            sentiment = "neutral"
        else:
            sentiment = "positive"

        # output polarity sentiment and tweet text
        print (str(tweet.sentiment.polarity) + " " + sentiment + " " + dict_data["text"])

        try:    
            #check if there r any hashtags
            if len(dict_data["entities"]["hashtags"]) != 0: 

                    hashtags = dict_data["entities"]["hashtags"]
            #if no hashtags add empty 
            else:
                hashtags= []
        except:
             pass

    es.indices.put_settings(index=indexName, body={"index.blocks.write":False})
        # add text and sentiment info to elasticsearch
        es.index(index=indexName,
                 doc_type="test-type",
                 body={"author": dict_data["user"]["screen_name"],
                       "date": dict_data["created_at"], # unfortunately this gets stored as a string
                       "location": dict_data["user"]["location"], # user location
                       "followers": dict_data["user"]["followers_count"],
                       "friends": dict_data["user"]["friends_count"],
                       "time_zone": dict_data["user"]["time_zone"],
                       "lang": dict_data["user"]["lang"],
                       #"timestamp": float(dict_data["timestamp_ms"]), # double not recognised as date 
                       "timestamp": dict_data["timestamp_ms"],
                       "datetime": datetime.now(),
                       "message": dict_data["text"],
                       "hashtags": hashtags,
                       "polarity": tweet.sentiment.polarity,
                       "subjectivity": tweet.sentiment.subjectivity,
                       # handle geo data
                       #"coordinates": dict_data[coordinates],
                       "sentiment": sentiment})
        return True

    # on failure

    def on_error(self, error):

        print "error: " + str(error)

if __name__ == '__main__':

    # create instance of the tweepy tweet stream listener
    listener = TweetStreamListener()

    # set twitter keys/tokens
    auth = OAuthHandler(consumer_key, consumer_secret)
    auth.set_access_token(access_token, access_token_secret)


    while True:
        try:
              #create instance of the tweepy stream
            stream = Stream(auth, listener)  
            # search twitter for sample tweets
            stream.sample()
        except KeyError:
            pass

Solution

  • Ok, I have found the solution for this problem, changing the method from on_data to on_status removed all the issues causing the error 420.