Monday 25 January 2016
A Quick hack of the -Domino Data Lab .... http://blog.dominodatalab.com/geographic-visualization-with-rs-ggmaps/
I Sincerely thought id found the bug till i scrolled down on their visitor comments to see that misiti- https://disqus.com/by/misiti3780/ , had pointed it out earlier , well here is the post anyways :)
My results are different as i have plotted on - mydata$OT.Number
mydata = read.csv("vehicle-accidents.csv")
mydata$State <- as.character(mydata$State)
mydata$MV.Number = as.numeric(mydata$MV.Number)
mydata = mydata[mydata$State != "Alaska", ]
mydata = mydata[mydata$State != "Hawaii", ]
#
for (i in 1:nrow(mydata)) {
latlon = geocode(mydata[i,1])
mydata$lon[i] = as.numeric(latlon[1])
mydata$lat[i] = as.numeric(latlon[2])
}
#-
mv_num_collisions = data.frame(mydata$OT.Number, mydata$lon, mydata$lat)
# MV Number to OT Number
#
#
colnames(mv_num_collisions) = c('collisions','lon','lat')
#
usa_center = as.numeric(geocode("United States"))
#
USAMap = ggmap(get_googlemap(center=usa_center, scale=2, zoom=4), extent="normal")
str(USAMap)
#
circle_scale_amt=0.75
USAMap +
geom_point(aes(x=lon, y=lat), color="red",data=mv_num_collisions, alpha=0.4,size=mv_num_collisions$collisions*circle_scale_amt) +
scale_size_continuous(range=range(mv_num_collisions$collisions))
List of 9
$ data :'data.frame': 4 obs. of 2 variables:
..$ lon: num [1:4] -123.8 -67.5 -123.8 -67.5
..$ lat: num [1:4] 11.7 11.7 56.1 56.1
..- attr(*, "out.attrs")=List of 2
.. ..$ dim : Named int [1:2] 2 2
.. .. ..- attr(*, "names")= chr [1:2] "lon" "lat"
.. ..$ dimnames:List of 2
.. .. ..$ lon: chr [1:2] "lon=-123.79395" "lon= -67.54395"
.. .. ..$ lat: chr [1:2] "lat=11.73830" "lat=56.09656"
$ layers :List of 3
..$ :Classes 'LayerInstance', 'Layer', 'ggproto' <ggproto object: Class LayerInstance, Layer>
aes_params: list
compute_aesthetics: function
compute_geom_1: function
compute_geom_2: function
compute_position: function
compute_statistic: function
data: waiver
draw_geom: function
geom: <ggproto object: Class GeomBlank, Geom>
aesthetics: function
default_aes: uneval
draw_group: function
draw_key: function
draw_layer: function
draw_panel: function
extra_params: na.rm
handle_na: function
non_missing_aes:
parameters: function
required_aes:
setup_data: function
use_defaults: function
super: <ggproto object: Class Geom>
geom_params: list
inherit.aes: TRUE
map_statistic: function
mapping: NULL
position: <ggproto object: Class PositionIdentity, Position>
compute_layer: function
compute_panel: function
required_aes:
setup_data: function
setup_params: function
super: <ggproto object: Class Position>
print: function
show.legend: NA
stat: <ggproto object: Class StatIdentity, Stat>
compute_group: function
compute_layer: function
compute_panel: function
default_aes: uneval
extra_params: na.rm
non_missing_aes:
parameters: function
required_aes:
retransform: TRUE
setup_data: function
setup_params: function
super: <ggproto object: Class Stat>
stat_params: list
subset: NULL
super: <ggproto object: Class Layer>
..$ :Classes 'LayerInstance', 'Layer', 'ggproto' <ggproto object: Class LayerInstance, Layer>
aes_params: list
compute_aesthetics: function
compute_geom_1: function
compute_geom_2: function
compute_position: function
compute_statistic: function
data: waiver
draw_geom: function
geom: <ggproto object: Class GeomRasterAnn, Geom>
aesthetics: function
default_aes: uneval
draw_group: function
draw_key: function
draw_layer: function
draw_panel: function
extra_params:
handle_na: function
non_missing_aes:
parameters: function
required_aes:
setup_data: function
use_defaults: function
super: <ggproto object: Class Geom>
geom_params: list
inherit.aes: TRUE
map_statistic: function
mapping: NULL
position: <ggproto object: Class PositionIdentity, Position>
compute_layer: function
compute_panel: function
required_aes:
setup_data: function
setup_params: function
super: <ggproto object: Class Position>
print: function
show.legend: NA
stat: <ggproto object: Class StatIdentity, Stat>
compute_group: function
compute_layer: function
compute_panel: function
default_aes: uneval
extra_params: na.rm
non_missing_aes:
parameters: function
required_aes:
retransform: TRUE
setup_data: function
setup_params: function
super: <ggproto object: Class Stat>
stat_params: list
subset: NULL
super: <ggproto object: Class Layer>
..$ :Classes 'LayerInstance', 'Layer', 'ggproto' <ggproto object: Class LayerInstance, Layer>
aes_params: list
compute_aesthetics: function
compute_geom_1: function
compute_geom_2: function
compute_position: function
compute_statistic: function
data: data.frame
draw_geom: function
geom: <ggproto object: Class GeomRect, Geom>
aesthetics: function
default_aes: uneval
draw_group: function
draw_key: function
draw_layer: function
draw_panel: function
extra_params: na.rm
handle_na: function
non_missing_aes:
parameters: function
required_aes: xmin xmax ymin ymax
setup_data: function
use_defaults: function
super: <ggproto object: Class Geom>
geom_params: list
inherit.aes: FALSE
map_statistic: function
mapping: uneval
position: <ggproto object: Class PositionIdentity, Position>
compute_layer: function
compute_panel: function
required_aes:
setup_data: function
setup_params: function
super: <ggproto object: Class Position>
print: function
show.legend: FALSE
stat: <ggproto object: Class StatIdentity, Stat>
compute_group: function
compute_layer: function
compute_panel: function
default_aes: uneval
extra_params: na.rm
non_missing_aes:
parameters: function
required_aes:
retransform: TRUE
setup_data: function
setup_params: function
super: <ggproto object: Class Stat>
stat_params: list
subset: NULL
super: <ggproto object: Class Layer>
$ scales :Classes 'ScalesList', 'ggproto' <ggproto object: Class ScalesList>
add: function
clone: function
find: function
get_scales: function
has_scale: function
input: function
n: function
non_position_scales: function
scales: list
super: <ggproto object: Class ScalesList>
$ mapping :List of 2
..$ x: symbol lon
..$ y: symbol lat
$ theme : list()
$ coordinates:Classes 'CoordMap', 'Coord', 'ggproto' <ggproto object: Class CoordMap, Coord>
aspect: function
distance: function
is_linear: function
labels: function
limits: list
orientation: NULL
params: list
projection: mercator
range: function
render_axis_h: function
render_axis_v: function
render_bg: function
render_fg: function
train: function
transform: function
super: <ggproto object: Class CoordMap, Coord>
$ facet :List of 1
..$ shrink: logi TRUE
..- attr(*, "class")= chr [1:2] "null" "facet"
$ plot_env :<environment: 0x000000001fd04120>
$ labels :List of 6
..$ x : chr "lon"
..$ y : chr "lat"
..$ xmin: chr "xmin"
..$ xmax: chr "xmax"
..$ ymin: chr "ymin"
..$ ymax: chr "ymax"
- attr(*, "class")= chr [1:2] "gg" "ggplot"
> scale_size_continuous(range=range(mv_num_collisions$collisions))
<ScaleContinuous>
Range:
Limits: 0 -- 1
> #
> USAMap +
+ geom_point(aes(x=lon, y=lat), data=mv_num_collisions, col="orange", alpha=0.4) +
+ scale_size_continuous(range=range(mv_num_collisions$collisions))
Error in layer(data = data, mapping = mapping, stat = stat, geom = GeomPoint, :
object 'circle_scale_amt' not found
>
Wednesday 13 January 2016
Initial Analysis of Tweets - State of the Union Address by President Barack Obama
https://twitter.com/search?src=typd&q=%23SOTU
Initial Analysis of the Tweets around the State of the Union Address by President Barack Obama
a detailed analysis shall follow soon ....
Some of the Tweets which have been mined can be seen below -
Initial Analysis of the Tweets around the State of the Union Address by President Barack Obama
a detailed analysis shall follow soon ....
Some of the Tweets which have been mined can be seen below -
tweet \ 0 RT @bungdan: @KevinMKruse @dandrezner @CombatC... 1 RT @MatthewACherry: At least now we know who t... 2 RT @Debi129: We pledge to make sure no man bor... 3 @NigeriaNewsdesk @todayng Obama has better thi... 4 Obama sold missiles to Cuba that can do harm t... 5 @BCfromBlo And this is Paul Ryan's characteriz... 6 RT @LorettaLynch: Great to talk #CriminalJusti... 7 @GovHaleySC Seems like the "siren" is you Hale... 8 RT @whitehouseostp: Liked #SOTU? Then tune in ... 9 RT @_bateeeeeey: #SOTU Paul Ryan looking like ... 10 #3: 31ピース クムクムパズル 映画妖怪ウォッチ エンマ大王と5つの物語だニャン! U... 11 RT @ForestWhitaker: I share @POTUS's optimism ... 12 Lol nice https://t.co/32j8DE4rii 13 RT @ambivertido: Obama en su primer y último D...
.............
21 @nikkihaley have you seen what's happening to ... 22 #2PeasInAPod @HillaryClinton will import milli... 23 RT @Ronaldodom22: I am leaving an #EmptySeat f... 24 RT @AshRenee7745: Without watching the #SOTU I... 25 https://t.co/i3G1960sWl vía @Etsy #usa #uk #ca... 26 RT @JaySekulow: #Iran captures our soldiers &a... 27 RT @RedComunismo: Barack Obama, ganador del No... 28 RT @SkyNewsBreak: U.S. military says 10 sailor... 29 RT @Sultanknish: Nikki Haley Didn't Have a Pro... ... ... 6180 President Obama's final State of the Union add... 6181 RT @CSDHI: .@GerardLAUTON 3 exécutions par jou... 6182 Bruselas estudiará en los próximos meses la op... 6183 RT @cjwerleman: Are we really going to lecture... 6184 RT @lara_legunda: Creo que mi mama usa más el ... 6185 Trump vs. Cruz https://t.co/o0Cwqobi1q @youths... 6186 OIL FOR THE LAMPS OF CHINA PAT O'BRIEN JEAN ...
AWS.Amazon.com RStudio server on UBUNTU 14.04 #rstats
AWS.Amazon.com RStudio server on UBUNTU 14.04 #rstats
|
Monday 11 January 2016
Twitter API - LIVE Stream Mining PART - II
#Import the necessary methods from tweepy library
from tweepy.streaming import StreamListener
from tweepy import OAuthHandler
from tweepy import Stream
#Variables that contains the user credentials to access Twitter API
access_token = '7###############d'
access_token_secret = 'A#############
consumer_key = 'Q###########1'
consumer_secret = 'S##################x'
#This is a basic listener that just prints received tweets to stdout.
class StdOutListener(StreamListener):
def on_data(self, data):
print data
return True
def on_error(self, status):
print status
if __name__ == '__main__':
#This handles Twitter authentication and the connection to Twitter Streaming API
l = StdOutListener()
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
stream = Stream(auth, l)
#Key word - OBAMA ....
stream.filter(track=['OBAMA '])
for tweet in stream:
print tweet['text']
#DATA Mining Tweets which mention - OBAMA .....only 1 tweets JSON dump shown rest all truncated .............
{"created_at":"Mon Jan 11 21:36:39 +0000 2016","id":686663033926594562,"id_str":"686663033926594562","text":"@McCormackJohn @JonathanLanday Will be awkward when they get up to clap when Obama talks about helping poor, climate change, Iran deal, etc.","source":"\u003ca href=\"http:\/\/twitter.com\" rel=\"nofollow\"\u003eTwitter Web Client\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":686662207858438145,"in_reply_to_status_id_str":"686662207858438145","in_reply_to_user_id":27689907,"in_reply_to_user_id_str":"27689907","in_reply_to_screen_name":"McCormackJohn","user":{"id":2208906530,"id_str":"2208906530","name":"Ali Ahmadi","screen_name":"AliR_Ahmadi","location":"Tehran-NYC","url":null,"description":"Researcher and writer focusing on Iran, the Middle East and sources of US foreign policy towards the region. Personal account.","protected":false,"verified":false,"followers_count":878,"friends_count":739,"listed_count":38,"favourites_count":1374,"statuses_count":9729,"created_at":"Fri Nov 22 12:41:04 +0000 2013","utc_offset":-28800,"time_zone":"Pacific Time (US & Canada)","geo_enabled":false,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"C0DEED","profile_background_image_url":"http:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_image_url_https":"https:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_tile":false,"profile_link_color":"0084B4","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"profile_image_url":"http:\/\/pbs.twimg.com\/profile_images\/573155028506181632\/gV1_lfDs_normal.jpeg","profile_image_url_https":"https:\/\/pbs.twimg.com\/profile_images\/573155028506181632\/gV1_lfDs_normal.jpeg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/2208906530\/1428808169","default_profile":true,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"is_quote_status":false,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[],"urls":[],"user_mentions":[{"screen_name":"McCormackJohn","name":"John McCormack","id":27689907,"id_str":"27689907","indices":[0,14]},{"screen_name":"JonathanLanday","name":"Jonathan Landay","id":47408060,"id_str":"47408060","indices":[15,30]}],"symbols":[]},"favorited":false,"retweeted":false,"filter_level":"low","lang":"en","timestamp_ms":"1452548199527"}
In [3]:#Import the necessary methods from tweepy library from tweepy.streaming import StreamListener from tweepy import OAuthHandler from tweepy import Stream import json import pandas as pd %matplotlib inline import matplotlib.pyplot as plt import matplotlib matplotlib.style.use('ggplot') #Variables that contains the user credentials to access Twitter API access_token = '7###########' access_token_secret = 'A############# consumer_key = 'Q########### consumer_secret = 'SO##################In [4]:tweets_data_path = 'C:/...../Twitter1/_US_Pres.txt' tweets_data = [] tweets_file = open(tweets_data_path, "r") for line in tweets_file: try: tweet = json.loads(line) tweets_data.append(tweet) except: continueIn [5]:print len(tweets_data)In [6]:tweets = pd.DataFrame() tweets['tweet'] = map(lambda tweet: tweet['text'] if 'text' in tweet else None, tweets_data) tweets['created_at'] = map(lambda tweet: tweet['created_at'] if 'created_at' in tweet else None, tweets_data) tweets['user_id'] = map(lambda tweet: tweet['user']['id'] if 'user' in tweet else None, tweets_data) tweets['id_str'] = map(lambda tweet: tweet['user']['id_str'] if 'user' in tweet else None, tweets_data) tweets['username'] = map(lambda tweet: tweet['user']['name'] if 'user' in tweet else None, tweets_data) tweets['screen_name'] = map(lambda tweet: tweet['user']['screen_name'] if 'user' in tweet else None, tweets_data) tweets['location'] = map(lambda tweet: tweet['user']['location'] if 'user' in tweet else None, tweets_data) tweets['followers_count'] = map(lambda tweet: tweet['user']['followers_count'] if 'user' in tweet else None, tweets_data) tweets['friends_count'] = map(lambda tweet: tweet['user']['friends_count'] if 'user' in tweet else None, tweets_data) tweets['created_at'] = map(lambda tweet: tweet['user']['created_at'] if 'user' in tweet else None, tweets_data) tweets['user_lang'] = map(lambda tweet: tweet['user']['lang'] if 'user' in tweet else None, tweets_data) tweets['following'] = map(lambda tweet: tweet['user']['following'] if 'user' in tweet else None, tweets_data) tweets['geo'] = map(lambda tweet: tweet['geo'], tweets_data) # tweets['coordinates'] = map(lambda tweet: tweet['coordinates'], tweets_data) tweets['retweet_count'] = map(lambda tweet: tweet['retweet_count'], tweets_data) tweets['favorite_count'] = map(lambda tweet: tweet['favorite_count'], tweets_data) tweets['lang'] = map(lambda tweet: tweet['lang'], tweets_data) # if k1 in d and k2 in d[k1] tweets['country'] = map(lambda tweet: tweet['place']['country'] if 'country' in tweet else None, tweets_data) tweets['country'] = map(lambda tweet: tweet['place']['country'] if tweet['place'] != None else None, tweets_data) print (tweets)In [7]:tweets_by_loc = tweets['location'].value_counts() print len(tweets_by_loc)In [8]:tweets_by_loc = tweets['location'].value_counts() # determines most frequent tweet Locations print tweets_by_loc # prints out Locations and how many tweets came from each Location - in descending orderIn [9]:tweets_by_lang = tweets['user_lang'].value_counts() # determines most frequent tweet countries print tweets_by_lang # prints Tweet Freq. in each used LanguageIn [11]:tweets_by_lang = tweets['user_lang'].value_counts() fig, ax = plt.subplots() ax.tick_params(axis='x', labelsize=15) ax.tick_params(axis='y', labelsize=10) ax.set_xlabel('User_lang', fontsize=15) ax.set_ylabel('Number of tweets' , fontsize=15) ax.set_title('Top 5 user_lang', fontsize=25, fontweight='bold',color='black') tweets_by_lang[:5].plot(ax=ax, kind='bar', color='black') # # Top Five user Languages are - English - Espanol , French , En-Gb ? and Presumably Italian . #Out[11]:In [12]:tweets_by_loc = tweets['location'].value_counts() fig, ax = plt.subplots() ax.tick_params(axis='x', labelsize=15) ax.tick_params(axis='y', labelsize=10) ax.set_xlabel('Locations', fontsize=15) ax.set_ylabel('Number of tweets' , fontsize=15) ax.set_title('Top 25 Locations', fontsize=25, fontweight='bold',color='black') tweets_by_loc[:25].plot(ax=ax, kind='bar', color='black')Out[12]:In [13]:tweets_by_followers_count = tweets['followers_count'].value_counts() # Need to re- check seems something wrong here ... fig, ax = plt.subplots() ax.tick_params(axis='x', labelsize=15) ax.tick_params(axis='y', labelsize=10) ax.set_xlabel('followers_count', fontsize=15) ax.set_ylabel(' Count' , fontsize=15) ax.set_title('Top 15 followers_count', fontsize=25, fontweight='bold',color='black') tweets_by_followers_count[:15].plot(ax=ax, kind='bar', color='black')Out[13]:In [14]:import reIn [16]:def word_in_text(word, text): word = word.lower() text = text.lower() match = re.search(word, text) if match: return True return False def extract_link(text): regex = r'https?://[^\s<>"]+|www\.[^\s<>"]+' match = re.search(regex, text) if match: return match.group() return '' if __name__ == '__main__': tweets_data_path = 'C:/..../Twitter1/_US_Pres.txt' tweets_data = [] tweets_file = open(tweets_data_path, "r") for line in tweets_file: try: tweet = json.loads(line) tweets_data.append(tweet) except: continue tweets = pd.DataFrame() texts = [] langs = [] countries = [] for line, tweet in enumerate(tweets_data): #print line, tweet try: texts.append(tweet['text']) # langs.append(tweet['lang']) # countries.append(tweet['place']['country'] if tweet['place'] != None else None) except: print "Error line %d" % (line) tweets['text'] = texts # tweets['lang'] = langs # tweets['country'] = countries #Mining tweets['python'] = tweets['text'].apply(lambda tweet: word_in_text('python', tweet)) tweets['javascript'] = tweets['text'].apply(lambda tweet: word_in_text('javascript', tweet)) tweets['ruby'] = tweets['text'].apply(lambda tweet: word_in_text('ruby', tweet)) tweets['programming'] = tweets['text'].apply(lambda tweet: word_in_text('programming', tweet)) tweets['tutorial'] = tweets['text'].apply(lambda tweet: word_in_text('tutorial', tweet)) tweets['relevant'] = tweets['text'].apply(lambda tweet: word_in_text('programming', tweet) or word_in_text('tutorial', tweet)) tweets['link'] = tweets['text'].apply(lambda tweet: extract_link(tweet)) tweets_relevant = tweets[tweets['relevant'] == True] tweets_relevant_with_link = tweets_relevant[tweets_relevant['link'] != ''] print tweets_relevant_with_link[tweets_relevant_with_link['python'] == True]['link'] print tweets_relevant_with_link[tweets_relevant_with_link['javascript'] == True]['link'] print tweets_relevant_with_link[tweets_relevant_with_link['ruby'] == True]['link'] plt.show()In [17]:import tweepy auth = tweepy.OAuthHandler(consumer_key, consumer_secret) auth.set_access_token(access_token, access_token_secret) api = tweepy.API(auth) ids = [] current_cursor = "" for page in tweepy.Cursor(api.friends_ids, screen_name="DhankarRohit").pages(): cursor = tweepy.Cursor(api.followers_ids, screen_name="DhankarRohit", cursor = current_cursor) current_cursor = cursor.iterator.next_cursor # print repr(cursor) # int current_cursor ids.extend(page) # print page # print ids print len(ids)In [ ]:
Subscribe to:
Posts (Atom)