import pandas as pd

# importing filed query results
df = pd.read_csv('QueryResults.csv', parse_dates=['CreationDate'])

df.info()

import numpy as np

# clean the tags column for list use by removing/replacing greater/less than chars and removing last comma
df['Tags'] = df.Tags.str.replace('<', '').str.replace('>', ',' )#.str.rstrip(',')

# fill missing values with zeros
df = df.fillna(0)

# rationalize columns to appropiate data type as we deal with low integer values and not floats
df = df.astype({'ViewCount': np.int64, 'AnswerCount': np.int64, 'FavoriteCount': np.int64})

df.info()

# one large list
taglist = df.Tags.tolist()
# remove the zeros
taglist = [i for i in taglist if i != 0 and i != '']
# union
string = ''.join(taglist)
# split into elements by comma
taglist = string.split(",")
taglist = taglist[:-1] #last element always empty
len(taglist)

taglist[-4:]

tagfreq = pd.Series(taglist).value_counts().sort_index().reset_index().reset_index(drop=True)
tagfreq.columns = ['Tag', 'TagCount']
tagfreq = tagfreq[tagfreq['Tag'] != '']
tagfreq = tagfreq.set_index('Tag')

# tag assignment frequency
tagfreq = tagfreq.sort_values('TagCount', ascending=False)#[0:20]

tagfreq.index[0:20]
#tagfreq

df.iloc[0,:].Tags.split(",")

%%time
# cumulated question viewcount with tag assigned
tagfreq['ViewCount'] = 0

def tag_counter(itag, df, targetcol=5, count=0):
    for row in df.itertuples(index=False):
        if itag in row[targetcol].split(","):
            count += row.ViewCount
    return count
                                                
for itag in tagfreq.index:
    tagfreq.at[itag,'ViewCount'] = tag_counter(itag, df[df.PostTypeId == 1])
    
tagfreq.iloc[0:20,:]

# Wall time: 5min 38s | for index, row in df.iterrows():

tagfreq.info()

import matplotlib.pyplot as plt
import seaborn as sns
import random
sns.set_style('dark')
sns.set(rc={"figure.figsize":(12, 8)})

def plot_certain_tags(df=tagfreq, x='TagCount', y='ViewCount', h=0, i=30):
    sns.scatterplot(x=x, y=y, data=df.iloc[h:i,:],linewidth=1, alpha = 1,edgecolor="black")
    for x,y,label,num in zip(df['TagCount'][h:i], df.ViewCount[h:i], df.index[h:i], np.arange(h,i)):
        plt.annotate(label, (x,y), textcoords="offset points", xytext=(0,10*[-1,1][random.randrange(2)]), ha='center')
    plt.show()

plot_certain_tags(h=0, i=20)

plot_certain_tags(h=3, i=25)

plot_certain_tags(h=25, i=50)

tagfreq.corr()

TC20 = tagfreq.index[0:20]
VC20 = tagfreq.sort_values('ViewCount', ascending=False).index[0:20]

# tags that are in TC20 & VC20
set(TC20).intersection(VC20)

# tags that are in Top20 only in one list
set(TC20) ^ set(VC20)
set(TC20).symmetric_difference(VC20)

# tags that are in TC20 only
set(np.setdiff1d(TC20,VC20))

# tags that are in VC20 only
set(np.setdiff1d(VC20,TC20))

%%time
# Custom One Hot Encoding

tagset = set(taglist)

def DSSE_one_hot_encoder(tagset, targetcol=5, df=df):
    encoded_df = pd.DataFrame(columns=list(tagset))
    for question in df.itertuples():
        question_set = set(question[targetcol+1].rstrip(',').split(","))
        unc_list = list(tagset - question_set)
        com_list = list(tagset.intersection(question_set))
        d1 = {key:0 for key in unc_list} # dict comprehension
        d2 = {key:1 for key in com_list}
        d1.update(d2)
        encoded_df.loc[question.Index] = d1
    return encoded_df

testdf = DSSE_one_hot_encoder(tagset, targetcol=5, df=df[df.PostTypeId == 1])
testdf.sample(3)

%%time
from mlxtend.frequent_patterns import apriori, association_rules

# Apriori
Apriori_tags = apriori(testdf, min_support=0.00035, use_colnames=True, verbose=1)
print(Apriori_tags.head(7))
len(Apriori_tags)

chosen_metric = 'lift'
thresholds = {'lift': 15, 'conviction': 1, 'leverage': 0.0001}

rules = association_rules(Apriori_tags, metric=chosen_metric, min_threshold=thresholds[chosen_metric])
rules.sort_values(chosen_metric, ascending=False)

# collect all tags where the tag 'deep-learning' is in any of antecedents or consequents sets

def collect_associated_tags(df=rules, targetcols=(0,1), source_tag='deep-learning'):
    target_tags = set()
    for rule in rules.itertuples(index=False):
        rule_tag_set = set(rule[targetcols[0]])
        rule_tag_set.update(set(rule[targetcols[1]]))
        if source_tag in rule_tag_set:
            target_tags.update(rule_tag_set)
    target_tags.discard(source_tag)
    return target_tags

associated_tags = collect_associated_tags()
print(len(associated_tags))
associated_tags

associated_tags = associated_tags - set(['machine-learning', 'data-augmentation', 'arima', 'data-science-model', 'forecasting', 'gradient-descent', 'mini-batch-gradient-descent', 'python', 'time-series', 'validation'])
associated_tags

allquest = pd.read_csv('all_questions_Jan22.csv', parse_dates=['CreationDate'])

# processing tags

# comma separate
allquest['Tags'] = allquest.Tags.str.replace('<', '').str.replace('>', ',' )#.str.rstrip(',')
# one large list
taglist = allquest.Tags.tolist()
# remove the zeros
taglist = [i for i in taglist if i != 0 and i != '']
# union
string = ''.join(taglist)
# split into elements by comma
taglist = string.split(",")  
len(taglist)

# counting tag assignments
alltime_tagfreq = pd.Series(taglist).value_counts().sort_index().reset_index().reset_index(drop=True)
alltime_tagfreq.columns = ['Tag', 'TagCount']
alltime_tagfreq = alltime_tagfreq[alltime_tagfreq['Tag'] != '']
alltime_tagfreq = alltime_tagfreq.set_index('Tag')
alltime_tagfreq = alltime_tagfreq.sort_values('TagCount', ascending=False)#[0:20]

alltime_tagfreq.index[0:20]

alltime_tagfreq.head()

%%time
# counting cumulated question views with tag assigned

alltime_tagfreq['ViewCount'] = 0
                                             
for itag in alltime_tagfreq.index:
    alltime_tagfreq.at[itag,'ViewCount'] = tag_counter(itag=itag, df=allquest, targetcol=2)
    
alltime_tagfreq.iloc[0:20,:]

def label_deep_learning(qtags, associated_tags=associated_tags):
    if (True in [(t in associated_tags) for t in qtags]): return True 
    else: return False

%%time
allquest['deep-learning'] = allquest.Tags.apply(lambda x: label_deep_learning(x.split(",")))

allquest.head()

quarterly = allquest[allquest['deep-learning'] == True].resample('Q', on='CreationDate').count()
quarterly.sample(2)

import matplotlib.pyplot as plt
from matplotlib.dates import DateFormatter
import matplotlib.dates as mdates
import seaborn as sns

ax = quarterly.plot(kind='bar', use_index=True, y='Id', rot=50)
ax.legend(["Number of Questions"]);
x_labels = quarterly.index.strftime('%m/%y')
ax.set_xticklabels(x_labels)
plt.title('Number of posted Deep-Learning Questions on Data Science Stack Exchange')
plt.show()

nonquarterly = allquest[allquest['deep-learning'] == False].resample('Q', on='CreationDate').count()
nonquarterly = nonquarterly.drop(['CreationDate','Tags','ViewCount','deep-learning'], axis=1)
nonquarterly.tail(2)

# create a dictonary
data = {"non_dl": nonquarterly.Id,
        "dl": quarterly.Id}

combined_quarterly = pd.DataFrame(data )
combined_quarterly.drop(combined_quarterly.tail(1).index,inplace=True) # drop last incomplete quarter
combined_quarterly.tail(2)

ax = combined_quarterly.plot(kind='bar', use_index=True, stacked=True, rot=50)
ax.legend(["Non DL-Questions", "DL Questions"]);
x_labels = combined_quarterly.index.strftime('%m/%y')
ax.set_xticklabels(x_labels)
plt.title('Number of posted Data Science Questions on Data Science Stack Exchange')
plt.show()

combined_quarterly.loc['2019-03-31':,:].sum(axis=1).mean().round()

combined_quarterly.loc['2019-03-31':,:].sum(axis=1).std().round()

comb_ratio_quarterly = combined_quarterly.copy()
comb_ratio_quarterly['total'] = comb_ratio_quarterly.sum(axis=1) # row summing

comb_ratio_quarterly.non_dl = comb_ratio_quarterly.non_dl /  comb_ratio_quarterly.total
comb_ratio_quarterly.dl = comb_ratio_quarterly.dl /  comb_ratio_quarterly.total
comb_ratio_quarterly = comb_ratio_quarterly.drop('total', axis=1)
comb_ratio_quarterly.drop(comb_ratio_quarterly.tail(1).index,inplace=True) # drop last incomplete quarter

comb_ratio_quarterly.sample(2)

import matplotlib.ticker as mtick

ax = comb_ratio_quarterly.plot(kind='bar', use_index=True, stacked=True, rot=50)
ax.legend(["Non Deep-Learning Questions", "Deep-Learning Questions"]);
x_labels = comb_ratio_quarterly.index.strftime('%m/%y')
ax.set_xticklabels(x_labels)
ax.yaxis.set_major_formatter(mtick.PercentFormatter(1.0))
plt.title('Ratios of posted Questions on Data Science Stack Exchange')
plt.legend(["Non Deep-Learning Questions", "Deep-Learning Questions"], bbox_to_anchor=(1,1), loc="upper center")
plt.show()