# Code inspired by = http://adilmoujahid.com/posts/2014/07/twitter-analytics/
#Import the necessary methods from tweepy library
from tweepy.streaming import StreamListener
from tweepy import OAuthHandler
from tweepy import Stream
#Variables that contains the user credentials to access Twitter API
access_token = '73090441-N74od'
access_token_secret = 'A########9'
consumer_key = 'Q#############1'
consumer_secret = 'S#############x'
#This is a basic listener that just prints received tweets to stdout.
class StdOutListener(StreamListener):
def on_data(self, data):
print data
return True
def on_error(self, status):
print status
if __name__ == '__main__':
#This handles Twitter authetification and the connection to Twitter Streaming API
l = StdOutListener()
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
stream = Stream(auth, l)
#This line filter Twitter Streams to capture data by the keywords: 'python', 'javascript', 'ruby'
stream.filter(track=['#rstats , #python, JavaScript , Perl , JAVA, PHP , RSTUDIO, Tutorial , oreilly , oreilly , Springer , Python , Pearson , Elsevier , Bertelsmann AG , McGrawHill , Wiley , APRESS , CAMBRIDGE , HARVARD , HarperCollins, HBR , Kadokawa Publishing'])
for tweet in stream:
print tweet['text']
# Note only 1 JSON Dump shown below ----
{"created_at":"Tue Jan 12 01:47:10 +0000 2016","id":686726079554269185,"id_str":"686726079554269185","text":"@Tom_Dart Talk to Tip Top Cafe's Jim Scott, who says 'armed society is a civil society,' allows open carry https:\/\/t.co\/10VHLJKbW4 via @mySA","source":"\u003ca href=\"http:\/\/twitter.com\" rel=\"nofollow\"\u003eTwitter Web Client\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":151412907,"in_reply_to_user_id_str":"151412907","in_reply_to_screen_name":"Tom_Dart","user":{"id":1599217352,"id_str":"1599217352","name":"Arthur Cavazos","screen_name":"artcavazos1","location":null,"url":null,"description":"Veteran public relations executive and business owner with a sterling reputation built on solid values, experience, and strategic thinking.","protected":false,"verified":false,"followers_count":901,"friends_count":1976,"listed_count":11,"favourites_count":320,"statuses_count":2435,"created_at":"Tue Jul 16 20:29:31 +0000 2013","utc_offset":null,"time_zone":null,"geo_enabled":false,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"C0DEED","profile_background_image_url":"http:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_image_url_https":"https:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_tile":false,"profile_link_color":"0084B4","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"profile_image_url":"http:\/\/pbs.twimg.com\/profile_images\/378800000143871263\/80968f059be13a992f9da992acba42bb_normal.jpeg","profile_image_url_https":"https:\/\/pbs.twimg.com\/profile_images\/378800000143871263\/80968f059be13a992f9da992acba42bb_normal.jpeg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/1599217352\/1398272181","default_profile":true,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"is_quote_status":false,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[],"urls":[{"url":"https:\/\/t.co\/10VHLJKbW4","expanded_url":"http:\/\/www.mysanantonio.com\/news\/local\/article\/San-Antonio-staple-Tip-Top-Cafe-to-allow-open-6740579.php?cmpid=twitter-desktop","display_url":"mysanantonio.com\/news\/local\/art\u2026","indices":[107,130]}],"user_mentions":[{"screen_name":"Tom_Dart","name":"Tom Dart","id":151412907,"id_str":"151412907","indices":[0,9]},{"screen_name":"mySA","name":"mySA","id":9830752,"id_str":"9830752","indices":[135,140]}],"symbols":[]},"favorited":false,"retweeted":false,"possibly_sensitive":false,"filter_level":"low","lang":"en","timestamp_ms":"1452563230776"}
In [27]:#Import the necessary methods from tweepy library from tweepy.streaming import StreamListener from tweepy import OAuthHandler from tweepy import Stream import json import pandas as pd %matplotlib inline import matplotlib.pyplot as plt import matplotlib matplotlib.style.use('ggplot') #Variables that contains the user credentials to access Twitter API access_token = '73090441-N74orPfAdWYhOvWUHpKFkUkejTqSmOeTjadhfX0Yd' access_token_secret = 'Ah7WiI4tvUcEVoBDXbQJECRSHfUHrCWE78TmU2G3nMGE9' consumer_key = 'Q3M3ZNo3QL9PDAjOoZCMKowM1' consumer_secret = 'SOenR4B7JzF6KMi5WtbSpAYX9MLlzSy6mlud3c3nn0on5yu3Qx'In [28]:tweets_data_path = 'C:/STAT/BRIDGE/Module-4/_______Python Own/Wk-1/Twitter1/_Tech_1.txt' tweets_data = [] tweets_file = open(tweets_data_path, "r") for line in tweets_file: try: tweet = json.loads(line) tweets_data.append(tweet) except: continueIn [29]:print len(tweets_data)In [30]:tweets = pd.DataFrame() tweets['tweet'] = map(lambda tweet: tweet['text'] if 'text' in tweet else None, tweets_data) tweets['created_at'] = map(lambda tweet: tweet['created_at'] if 'created_at' in tweet else None, tweets_data) tweets['user_id'] = map(lambda tweet: tweet['user']['id'] if 'user' in tweet else None, tweets_data) tweets['id_str'] = map(lambda tweet: tweet['user']['id_str'] if 'user' in tweet else None, tweets_data) tweets['username'] = map(lambda tweet: tweet['user']['name'] if 'user' in tweet else None, tweets_data) tweets['screen_name'] = map(lambda tweet: tweet['user']['screen_name'] if 'user' in tweet else None, tweets_data) tweets['location'] = map(lambda tweet: tweet['user']['location'] if 'user' in tweet else None, tweets_data) tweets['followers_count'] = map(lambda tweet: tweet['user']['followers_count'] if 'user' in tweet else None, tweets_data) tweets['friends_count'] = map(lambda tweet: tweet['user']['friends_count'] if 'user' in tweet else None, tweets_data) tweets['created_at'] = map(lambda tweet: tweet['user']['created_at'] if 'user' in tweet else None, tweets_data) tweets['user_lang'] = map(lambda tweet: tweet['user']['lang'] if 'user' in tweet else None, tweets_data) tweets['following'] = map(lambda tweet: tweet['user']['following'] if 'user' in tweet else None, tweets_data) tweets['geo'] = map(lambda tweet: tweet['geo'], tweets_data) # tweets['coordinates'] = map(lambda tweet: tweet['coordinates'], tweets_data) tweets['retweet_count'] = map(lambda tweet: tweet['retweet_count'], tweets_data) tweets['favorite_count'] = map(lambda tweet: tweet['favorite_count'], tweets_data) tweets['lang'] = map(lambda tweet: tweet['lang'], tweets_data) # if k1 in d and k2 in d[k1] tweets['country'] = map(lambda tweet: tweet['place']['country'] if 'country' in tweet else None, tweets_data) tweets['country'] = map(lambda tweet: tweet['place']['country'] if tweet['place'] != None else None, tweets_data) print (tweets)In [31]:tweets_by_loc = tweets['location'].value_counts() print len(tweets_by_loc)In [33]:tweets_by_loc = tweets['location'].value_counts() # determines most frequent tweet countries print tweets_by_loc # prints out countries and how many tweets came from each in descending orderIn [34]:tweets_by_lang = tweets['user_lang'].value_counts() # determines most frequent tweet countries print tweets_by_lang # prints out countries and how many tweets came from each in descending orderIn [35]:tweets_by_lang = tweets['user_lang'].value_counts() fig, ax = plt.subplots() ax.tick_params(axis='x', labelsize=15) ax.tick_params(axis='y', labelsize=10) ax.set_xlabel('User_lang', fontsize=15) ax.set_ylabel('Number of tweets' , fontsize=15) ax.set_title('Top 25 user_lang', fontsize=25, fontweight='bold',color='black') tweets_by_lang[:25].plot(ax=ax, kind='bar', color='black')Out[35]:In [36]:tweets_by_loc = tweets['location'].value_counts() fig, ax = plt.subplots() ax.tick_params(axis='x', labelsize=15) ax.tick_params(axis='y', labelsize=10) ax.set_xlabel('Locations', fontsize=15) ax.set_ylabel('Number of tweets' , fontsize=15) ax.set_title('Top 25 Locations', fontsize=25, fontweight='bold',color='black') tweets_by_loc[:25].plot(ax=ax, kind='bar', color='black')Out[36]:In [38]:tweets_by_followers_count = tweets['followers_count'].value_counts() # Need to re- check seems something wrong here ... fig, ax = plt.subplots() ax.tick_params(axis='x', labelsize=15) ax.tick_params(axis='y', labelsize=10) ax.set_xlabel('followers_count', fontsize=15) ax.set_ylabel(' Count' , fontsize=15) ax.set_title('Top 15 followers_count', fontsize=25, fontweight='bold',color='black') tweets_by_followers_count[:15].plot(ax=ax, kind='bar', color='black')Out[38]:In [24]:import reIn [45]:def word_in_text(word, text): word = word.lower() text = text.lower() match = re.search(word, text) if match: return True return False def extract_link(text): regex = r'https?://[^\s<>"]+|www\.[^\s<>"]+' match = re.search(regex, text) if match: return match.group() return '' if __name__ == '__main__': tweets_data_path = 'C:/STAT/BRIDGE/Module-4/_______Python Own/Wk-1/Twitter1/_Tech_1.txt' tweets_data = [] tweets_file = open(tweets_data_path, "r") for line in tweets_file: try: tweet = json.loads(line) tweets_data.append(tweet) except: continue tweets = pd.DataFrame() texts = [] langs = [] countries = [] for line, tweet in enumerate(tweets_data): #print line, tweet try: texts.append(tweet['text']) # langs.append(tweet['lang']) # countries.append(tweet['place']['country'] if tweet['place'] != None else None) except: print "Error line %d" % (line) tweets['text'] = texts # tweets['lang'] = langs # tweets['country'] = countries #Mining tweets['python'] = tweets['text'].apply(lambda tweet: word_in_text('python', tweet)) tweets['javascript'] = tweets['text'].apply(lambda tweet: word_in_text('javascript', tweet)) tweets['ruby'] = tweets['text'].apply(lambda tweet: word_in_text('ruby', tweet)) tweets['programming'] = tweets['text'].apply(lambda tweet: word_in_text('programming', tweet)) tweets['tutorial'] = tweets['text'].apply(lambda tweet: word_in_text('tutorial', tweet)) tweets['relevant'] = tweets['text'].apply(lambda tweet: word_in_text('programming', tweet) or word_in_text('tutorial', tweet)) tweets['link'] = tweets['text'].apply(lambda tweet: extract_link(tweet)) tweets_relevant = tweets[tweets['relevant'] == True] tweets_relevant_with_link = tweets_relevant[tweets_relevant['link'] != ''] print tweets_relevant_with_link[tweets_relevant_with_link['python'] == True]['link'] print tweets_relevant_with_link[tweets_relevant_with_link['javascript'] == True]['link'] print tweets_relevant_with_link[tweets_relevant_with_link['ruby'] == True]['link'] plt.show()
No comments:
Post a Comment