Sentiment Analysis - #iPython #Dato #GraphLab Create - Rotten Tomatoes - Public Kaggle Data

In [1]:

import graphlab
import os 
os.getcwd()

Out[1]:

'C:\\Anaconda2\\envs\\dato-env'

In [2]:

df_1=graphlab.SFrame("train_1.tsv")

[INFO] Start server at: ipc:///tmp/graphlab_server-7380 - Server binary: C:\Anaconda2\envs\dato-env\lib\site-packages\graphlab\unity_server.exe - Server log: C:\Users\Rohit\AppData\Local\Temp\graphlab_server_1455190805.log.0
[INFO] GraphLab Server Version: 1.8.1

PROGRESS: Finished parsing file C:\Anaconda2\envs\dato-env\train_1.tsv
PROGRESS: Parsing completed. Parsed 100 lines in 0.483028 secs.
------------------------------------------------------
Inferred types from first line of file as 
column_type_hints=[long,long,str,long]
If parsing fails due to incorrect types, you can correct
the inferred type list above and pass it to read_csv in
the column_type_hints argument
------------------------------------------------------
PROGRESS: Finished parsing file C:\Anaconda2\envs\dato-env\train_1.tsv
PROGRESS: Parsing completed. Parsed 156060 lines in 0.227013 secs.

In [3]:

convert_func = lambda x:1 if x>3 else 0

In [4]:

df_1[100:110]

Out[4]:

PhraseId	SentenceId	Phrase	Sentiment
101	3	would have a hard time sitting through this one ...	1
102	3	would have a hard time sitting through this one ...	0
103	3	would	2
104	3	have a hard time sitting through this one ...	0
105	3	have	2
106	3	a hard time sitting through this one ...	1
107	3	a hard time	1
108	3	hard time	1
109	3	hard	2
110	3	time	2

[10 rows x 4 columns]

In [5]:

df_1[1000:1010]
# within 1000 Rows of the DF - the Sentence ID ahs moved from - 3 to 36 and Phrase ID 110 to 1001

Out[5]:

PhraseId	SentenceId	Phrase	Sentiment
1001	36	to avoid	1
1002	36	avoid	0
1003	37	It almost feels as if the movie is more interested ...	1
1004	37	almost feels as if the movie is more interested ...	0
1005	37	feels as if the movie is more interested in ...	1
1006	37	feels as if the movie is more interested in ...	1
1007	37	feels	2
1008	37	as if the movie is more interested in ...	1
1009	37	if the movie is more interested in ...	1
1010	37	if	2

[10 rows x 4 columns]

In [6]:

df_1['Target'] = df_1['Sentiment'].apply(convert_func)
#
# Creates a Variable named Target - adds a column at end of DF named Target .
# Values of Sentiment are converted as defined in the - "convert_func"
# 
df_1[100:110]

Out[6]:

PhraseId	SentenceId	Phrase	Sentiment	Target
101	3	would have a hard time sitting through this one ...	1	0
102	3	would have a hard time sitting through this one ...	0	0
103	3	would	2	0
104	3	have a hard time sitting through this one ...	0	0
105	3	have	2	0
106	3	a hard time sitting through this one ...	1	0
107	3	a hard time	1	0
108	3	hard time	1	0
109	3	hard	2	0
110	3	time	2	0

[10 rows x 5 columns]

In [8]:

df_1['word_count'] = graphlab.text_analytics.count_words(df_1['Phrase'])
#

In [12]:

df_1['Phrase'][1:2]
# Seen below we get 1 Row of the DataType String [str] , printed from the Variable Phrase in our Data Frame 
# As seen Frequency of Ocurrence of all Words is - ONCE -- besides  'the' which occurs Twice...

Out[12]:

dtype: str
Rows: 1
['A series of escapades demonstrating the adage that what is good for the goose']

In [9]:

df_1['word_count'][1:2]
# As seen below - a Row of the Dictionary , the dtype: dict - is printed within the Squiggly Braces ..
# Seen below the Frequency of Ocurrence of the Words is -- 1L -- for every word besides 'the' :2L

Out[9]:

dtype: dict
Rows: 1
[{'a': 1L, 'what': 1L, 'good': 1L, 'escapades': 1L, 'for': 1L, 'that': 1L, 'series': 1L, 'is': 1L, 'goose': 1L, 'adage': 1L, 'demonstrating': 1L, 'of': 1L, 'the': 2L}]

In [13]:

df_1['Phrase'][2:4]
# Two Rows are printed out - Row 1 has Two Words == 'A series'
# Row 2 has One Word == 'A'

Out[13]:

dtype: str
Rows: 2
['A series', 'A']

In [10]:

df_1['word_count'][2:4]
# Seen below - Two Row's of Dictionary printed , One Row each within the Squiggly Braces.
# Square Brackets surround COMPLETE output we have sought from the - Data Type == dtype: dict. 

Out[10]:

dtype: dict
Rows: 2
[{'a': 1L, 'series': 1L}, {'a': 1L}]

In [11]:

df_1['word_count'][2:10]

Out[11]:

dtype: dict
Rows: 8
[{'a': 1L, 'series': 1L}, {'a': 1L}, {'series': 1L}, {'what': 1L, 'good': 1L, 'for': 1L, 'escapades': 1L, 'that': 1L, 'of': 1L, 'is': 1L, 'goose': 1L, 'adage': 1L, 'demonstrating': 1L, 'the': 2L}, {'of': 1L}, {'what': 1L, 'good': 1L, 'escapades': 1L, 'for': 1L, 'that': 1L, 'is': 1L, 'goose': 1L, 'adage': 1L, 'demonstrating': 1L, 'the': 2L}, {'escapades': 1L}, {'what': 1L, 'good': 1L, 'for': 1L, 'that': 1L, 'is': 1L, 'goose': 1L, 'adage': 1L, 'demonstrating': 1L, 'the': 2L}]

Mongo DB Best Practices from recent usage - .explain()

Some practical tips from recent usage - .explain()

dt1_1_EXPLAIN = db.col1.find({"created_at": {"$gte":dt1_1 , "$lt": dt1_2}}).explain()
    print " Tweets Slot -1 :",dt1_1_EXPLAIN
    print #

 Tweets Slot -1 : {u'executionStats': {u'executionTimeMillis': 219, u'nReturned': 334, u'totalKeysExamined': 0, u'allPlansExecution': [], u'executionSuccess': True, u'executionStages': {u'needYield': 0, u'direction': u'forward', u'saveState': 47, u'restoreState': 47, u'isEOF': 1, u'docsExamined': 6015, u'nReturned': 334, u'needTime': 5682, u'filter': {u'$and': [{u'created_at': {u'$lt': datetime.datetime(2016, 2, 1, 6, 0)}}, {u'created_at': {u'$gte': datetime.datetime(2016, 2, 1, 1, 0)}}]}, u'executionTimeMillisEstimate': 0, u'invalidates': 0, u'works': 6017, u'advanced': 334, u'stage': u'COLLSCAN'}, u'totalDocsExamined': 6015}, u'queryPlanner': {u'parsedQuery': {u'$and': [{u'created_at': {u'$lt': datetime.datetime(2016, 2, 1, 6, 0)}}, {u'created_at': {u'$gte': datetime.datetime(2016, 2, 1, 1, 0)}}]}, u'rejectedPlans': [], u'namespace': u'db1.col1', u'winningPlan': {u'filter': {u'$and': [{u'created_at': {u'$lt': datetime.datetime(2016, 2, 1, 6, 0)}}, {u'created_at': {u'$gte': datetime.datetime(2016, 2, 1, 1, 0)}}]}, u'direction': u'forward', u'stage': u'COLLSCAN'}, u'indexFilterSet': False, u'plannerVersion': 1}, u'ok': 1.0, u'serverInfo': {u'host': u'Rohit-VAIO', u'version': u'3.2.1', u'port': 27017, u'gitVersion': u'a14d55980c2cdc565d4704a7e3ad37e4e535c1b2'}}

Twitter Analysis of Tweets - Time Series Plots #Python #rstats #ggplot2 #plotly

TimeSeries of Actual Tweets for @IndiGo6E and @flyspicejet

#Plotly

require(plotly)

require(ggplot2)

#

ZZ <- read.csv("C:/STAT/foo.csv")

#

pp <- ggplot(ZZ, aes(x=Time_Slot, y= Count_of_Tweets, group=Tweet_Category))

pp + geom_line(aes(colour = Tweet_Category)) 

#

QQ<-ggplotly(pp+geom_line(aes(colour = Tweet_Category)))

QQ

Links to live interactive plots -- 

http://rpubs.com/rohitdhankaranalytics/Indigo_JetAirways_Spicejet_08FEB

http://rpubs.com/rohitdhankaranalytics/Indigo_and_Spicejet_01FEB

Comparison SpiceJet and IndiGo6E 

                 

                   Comparison SpiceJet and IndiGo6E 

Date time Formats in Python - datetime.datetime.now().strftime ("%A,%Y-%m-%d,%H:%M%p")

In [1]:

import time
import datetime
import datetime

start = datetime.datetime.now()


i =0 
while i < 90000:
    i = i + 1
    
end = datetime.datetime.now()

print(end - start)

0:00:00.003000

In [2]:

import time
import datetime
import datetime

start = datetime.datetime.now()


i =0 
while i < 90000:
    i = i + 1
    
end = datetime.datetime.now()

print(end - start)

0:00:00.014000

In [3]:

import time
import datetime
import datetime

start = datetime.datetime.now()


i =0 
while i < 99000000:
    i = i + 1
    
end = datetime.datetime.now()

print(end - start)

0:00:11.576000

In [4]:

import time
import datetime
import datetime

start = datetime.datetime.now()
print start

2016-02-05 19:44:52.974000

In [5]:

import time
import datetime
import datetime

start = datetime.datetime.now()
print start

2016-02-05 19:45:12.613000

In [1]:

import time
import datetime
import datetime
start =datetime.datetime.now().strftime("%A, %d. %B %Y %I:%M%p")
print start

Saturday, 06. February 2016 06:20PM

In [2]:

import time
import datetime
start =datetime.datetime.now().strftime('%Y-%m-%d %X')
print start

2016-02-06 18:22:54

In [6]:

import time
import datetime
start =datetime.datetime.now().strftime("%d,%m,%y, %X")
print start

06,02,16, 18:24:56

In [8]:

import time
import datetime
start =datetime.datetime.now().strftime ("%Y,%m,%d,%H,%M")
print start

2016,02,06,18,27

In [9]:

import time
import datetime
start =datetime.datetime.now().strftime ("%Y,%m,%d,%H:%M")
print start

2016,02,06,18:27

In [10]:

import time
import datetime
start =datetime.datetime.now().strftime ("%Y-%m-%d,%H:%M")
print start

2016-02-06,18:27

In [11]:

import time
import datetime
start =datetime.datetime.now().strftime ("%A,%Y-%m-%d,%H:%M")
print start

Saturday,2016-02-06,18:29

In [12]:

import time
import datetime
start =datetime.datetime.now().strftime ("%A,%Y-%m-%d,%H:%M%p")
print start
# %p

Saturday,2016-02-06,18:29PM

In [13]:

import time
import datetime
start =datetime.datetime.now().strftime ("%A,%Y-%m-%d,%H:%M%P")
print start
# %P

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-13-7937915b6b5f> in <module>()
      1 import time
      2 import datetime
----> 3 start =datetime.datetime.now().strftime ("%A,%Y-%m-%d,%H:%M%P")
      4 print start
      5 # %P

ValueError: Invalid format string

In [ ]: