import requests
import numpy as np
import pandas as pd
import tweepy
import json
import matplotlib.pyplot as plt
from timeit import default_timer as timer
import warnings
warnings.filterwarnings('ignore')

url= 'https://d17h27t6h515a5.cloudfront.net/topher/2017/August/599fd2ad_image-predictions/image-predictions.tsv'
response = requests.get(url)
with open('image-predictions.tsv',mode='wb')as file:
    file.write(response.content)

dogs_rates_archive=pd.read_csv('twitter-archive-enhanced.csv')
image_predictions=pd.read_csv('image-predictions.tsv',sep= '\t')

import tweepy

consumer_key = 'YOUR CONSUMER KEY'
consumer_secret = 'YOUR CONSUMER SECRET'
access_token = 'YOUR ACCESS TOKEN'
access_secret = 'YOUR ACCESS SECRET'

auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_secret)

api = tweepy.API(auth, parser=tweepy.parsers.JSONParser(), 
                 wait_on_rate_limit=True,wait_on_rate_limit_notify=True)

start = timer()

tweet_list = []
tweet_errors = []
for tweet_id in dogs_rates_archive.tweet_id:
    try:
        tweet = api.get_status(tweet_id, tweet_mode = 'extended')
        tweet_list.append(tweet)
      
        
    except Exception as e:
        print(str(tweet_id) + '_' + str(e))
        tweet_errors.append(tweet_id)
end = timer()
print(end-start)   
   

with open('tweet_json.txt', 'w') as fb:
    json.dump(tweet_list, fb)

with open('tweet_json.txt') as file:
    tweet_list = json.load(file)
tweets_df = pd.DataFrame(tweet_list, columns = ['id','favorite_count', 'retweet_count'])

tweets_df.to_csv('tweets_df.csv',index = False)

tweets_df = pd.read_csv('tweets_df.csv')

dogs_rates_archive
       

image_predictions

tweets_df

dogs_rates_archive.info()

image_predictions.info()

tweets_df.info()

dogs_rates_archive.duplicated().sum()

image_predictions.duplicated().sum()

tweets_df.duplicated().sum()

dogs_rates_archive.tweet_id.value_counts()

dogs_rates_archive_clean = dogs_rates_archive.copy()
image_predictions_clean = image_predictions.copy()
tweets_df_clean = tweets_df.copy()

mask1 = dogs_rates_archive_clean.name.str.isupper()
mask2 = dogs_rates_archive_clean.name.str.islower()
column_name = 'name'
dogs_rates_archive_clean.loc[(mask1 | mask2), column_name] = np.nan

dogs_rates_archive_clean['name'].replace('None', np.nan, inplace=True)

dogs_rates_archive_clean

#drop rows where denominator != 10
dogs_rates_archive_clean.drop(dogs_rates_archive_clean[dogs_rates_archive_clean.rating_denominator != 10].index, inplace = True)

#drop rows rhere numerator more than 15
dogs_rates_archive_clean.drop(dogs_rates_archive_clean[dogs_rates_archive_clean.rating_numerator > 15].index, inplace = True)

dogs_rates_archive_clean.rating_numerator

dogs_rates_archive_clean.rating_denominator

dogs_rates_archive_clean = dogs_rates_archive_clean[pd.isnull(dogs_rates_archive_clean.retweeted_status_id)]

dogs_rates_archive_clean.retweeted_status_id.value_counts(dropna=False)

dogs_rates_archive_clean.drop(columns=['in_reply_to_status_id', 'in_reply_to_user_id', 'retweeted_status_id', 'retweeted_status_user_id', 'retweeted_status_timestamp'], inplace = True)

dogs_rates_archive_clean

dogs_rates_archive_clean.astype({'timestamp': 'datetime64', 'tweet_id': 'object'},copy = False).dtypes

image_predictions_clean.rename(columns={'p1': 'Prediction1', 'p2': 'Prediction2', 'p3': 'Prediction3'}, inplace=True)
image_predictions_clean.rename(columns={'p1_conf': 'Prediction1_conf', 'p1_dog': 'Prediction1_dog'}, inplace=True)
image_predictions_clean.rename(columns={'p2_conf': 'Prediction2_conf', 'p2_dog': 'Prediction2_dog'}, inplace=True)
image_predictions_clean.rename(columns={'p3_conf': 'Prediction3_conf', 'p3_dog': 'Prediction3_dog'}, inplace=True)

image_predictions_clean.astype({'tweet_id': 'object'},copy = False).dtypes

image_predictions_clean.info()

prediction = []
confidence = []
def most_confident_prediction(dataframe):
    if dataframe['Prediction1_dog'] == True:
        prediction.append(dataframe['Prediction1'])
        confidence.append(dataframe['Prediction1_conf'])
    elif dataframe['Prediction2_dog'] == True:
        prediction.append(dataframe['Prediction2'])
        confidence.append(dataframe['Prediction2_conf'])
    elif dataframe['Prediction3_dog'] == True:
        prediction.append(dataframe['Prediction3'])
        confidence.append(dataframe['Prediction3_conf'])   
    else:
        prediction.append('NAN')
        confidence.append(0)

image_predictions_clean.apply(most_confident_prediction, axis=1)
image_predictions_clean['prediction'] = prediction
image_predictions_clean['confidence'] = confidence
        
    

image_predictions_clean

dogs_rates_archive_clean = pd.merge(dogs_rates_archive_clean, image_predictions_clean[['tweet_id', 'prediction', 'confidence']],
                                   on = 'tweet_id', how = 'inner')

dogs_rates_archive_clean.info()

dogs_rates_archive_clean['stages'] = dogs_rates_archive_clean.doggo + dogs_rates_archive_clean.floofer + dogs_rates_archive_clean.pupper + dogs_rates_archive_clean.puppo
dogs_rates_archive_clean.stages.loc[dogs_rates_archive_clean.stages == 'NoneNoneNoneNone'] = None
dogs_rates_archive_clean.stages.loc[dogs_rates_archive_clean.stages == 'doggoNoneNoneNone'] = 'doggo'
dogs_rates_archive_clean.stages.loc[dogs_rates_archive_clean.stages == 'NoneNonepupperNone'] = 'pupper'
dogs_rates_archive_clean.stages.loc[dogs_rates_archive_clean.stages == 'NoneNoneNonepuppo'] = 'puppo'
dogs_rates_archive_clean.stages.loc[dogs_rates_archive_clean.stages == 'NoneflooferNoneNone'] = 'floofer'
dogs_rates_archive_clean.stages.loc[dogs_rates_archive_clean.stages == 'doggoNoneNonepuppo'] = 'doggo'
dogs_rates_archive_clean.stages.loc[dogs_rates_archive_clean.stages == 'doggoNonepupperNone'] = 'doggo'
dogs_rates_archive_clean.stages.loc[dogs_rates_archive_clean.stages == 'doggoflooferNoneNone'] = 'doggo'



dogs_rates_archive_clean.drop(columns=['floofer', 'puppo', 'doggo', 'pupper'], inplace = True)

dogs_rates_archive_clean.info()

dogs_rates_archive_clean.astype({'stages': 'category'},copy = False).dtypes

tweets_df_clean.rename(columns={'id':'tweet_id'}, inplace=True)

dogs_rates_archive_clean = pd.merge(dogs_rates_archive_clean, tweets_df_clean,
                                   on = 'tweet_id', how = 'inner')

dogs_rates_archive_clean.info()

dogs_rates_archive_clean.to_csv('twitter_archive_master.csv',index = False)

df=pd.read_csv('twitter_archive_master.csv')

df.head()

#get the number of eveery dog stage
df_stages = df.groupby('stages')['tweet_id'].count()
df_stages

#plot the count of edog stages in a bar graph
plt.pie(df_stages,labels=['doggo','floofer','pupper','puppo'])
plt.title('The distribution of dog stages');

# plot the ralationship between retweet count an favourite counts
plt.scatter(df['favorite_count'],df['retweet_count'])
plt.title('the relation between likes count and retweet count')
plt.xlabel('likes count')
plt.ylabel('retweett count');

# calculate the correlation coeffecient
df['favorite_count'].corr(df['retweet_count'])