Process Analytics & Process Mining with pm4py ~ Case study of an 'Order to Cash' process

import os
os.environ["PATH"] += os.pathsep + 'C://Program Files//Graphviz//bin'

import pandas as pd
import numpy as np
from datetime import date
from IPython.display import Markdown, display
import matplotlib.pyplot as plt
import seaborn as sns
sns.set_style('darkgrid')

import pm4py
import pyreadr

print('\n'.join(f'{m.__name__}=={m.__version__}' for m in globals().values() if getattr(m, '__version__', None)))

import sys
print('current env:' , sys.exec_prefix.split(os.sep)[-1] )

quotations = pyreadr.read_r('otc_quotations.RDS')

quotations = quotations[None] 

quotations.head()

# Inspect quotations
quotations.info()

offer_history = quotations.melt(id_vars='quotation_id')
offer_history

offer_history[['activity','info']] = offer_history.variable.str.split('_', expand=True)
offer_history.drop('variable', axis=1, inplace=True)
offer_history = offer_history[['quotation_id', 'activity', 'info', 'value']]

offer_history

recode = {"info":  {"at": "timestamp", "by": "resource"}}
offer_history_recode = offer_history.replace(recode)
offer_history_recode['info'] = offer_history_recode['info'].astype('category')
offer_history_recode

offer_history_spread = offer_history_recode.pivot( index=['quotation_id', 'activity'], columns='info', values='value').reset_index()

offer_history_spread

validations = pyreadr.read_r('otc_validations.RDS')

validations = validations[None]
validations

validate_history = validations
validate_history['activity'] = "Validate"
validate_history['action'] = validate_history.quotation_id + '-validate'
validate_history

validate_history_gathered = validate_history.melt(id_vars=['quotation_id','resource', 'activity','action'],
                                                  var_name='lifecycle', value_name='timestamp'
                                                 )

validate_history_gathered

recode = {"lifecycle":  {"started": "start", "completed": "complete"}}
validate_history_recode = validate_history_gathered.replace(recode)
validate_history_recode['lifecycle'] = validate_history_recode['lifecycle'].astype('category')
validate_history_recode

offer_history_recode = offer_history_spread
offer_history_recode['lifecycle'] = "complete"
offer_history_recode['increment_num'] = range(0+1, len(offer_history_recode.quotation_id)+1)
offer_history_recode['action'] = offer_history_recode.quotation_id.astype(str) + '-' + offer_history_recode.increment_num.astype(str)
offer_history_recode.drop('increment_num', axis=1, inplace=True)

offer_history_recode = offer_history_recode.dropna() # to arrive at 7735 rows - undocumented in case study

# concat quotations + validations histories to sales history

sales_history2 = pd.concat([offer_history_recode,validate_history_recode])
sales_history2

sales_history = pyreadr.read_r('otc_sales_history.RDS')
sales_history = sales_history[None]
sales_history

order_history = pyreadr.read_r('otc_order_history.RDS')
order_history = order_history[None]
order_history.rename(columns = {'time':'timestamp'}, inplace = True)
order_history

otc = pd.concat([sales_history, order_history])
#otc[otc.activity == 'Receive Sales Order']
otc.info()

otc_event_log = otc.copy()
otc_event_log["timestamp"] = pd.to_datetime(otc_event_log["timestamp"])
otc_event_log = otc_event_log.sort_values(by=["timestamp"]) # https://aginic.com/blog/your-guide-to-process-mining/
otc_event_log['case_id'] = otc_event_log.quotation_id.fillna('') + '-' + otc_event_log.sales_order_id.fillna('')
#df['new_col'] = df['foodstuff'].fillna('') + df['type'].fillna('')
#otc_event_log['case_id'] = otc_event_log['quotation_id'].combine(otc_event_log['sales_order_id'], lambda a, b: ((a or "") + (b or "")) or None, None)
otc_event_log

len(otc_event_log.sales_order_id.unique())

# only slice cases with sales orders
otc_event_log_only_so = otc_event_log[~otc_event_log.sales_order_id.isnull()]
otc_event_log_only_so.sales_order_id.isnull().sum()

# pm4py needs for later applied methods caseid column in the first column of the dataframe

# get rid of unused cols
otc_event_log_only_so2 = otc_event_log_only_so.copy().drop(['lifecycle', 'status'], axis=1)

first_column_to_be = otc_event_log_only_so2.pop('sales_order_id')
otc_event_log_only_so2.insert(0, 'sales_order_id', first_column_to_be)
  
otc_event_log_only_so2.shape

#otc_event_log[otc_event_log.activity == 'Receive Request For Quotation']

from pm4py.objects.conversion.log import converter as log_converter

#parameters = {log_converter.Variants.TO_EVENT_LOG.value.Parameters.CASE_ID_KEY: 'sales_order_id'}
#parameters = {log_converter.Variants.TO_EVENT_LOG.value.Parameters.ACTIVITY_KEY: 'activity'}
#parameters = {log_converter.Variants.TO_EVENT_LOG.value.Parameters.TIMESTAMP_KEY: 'timestamp'}
#event_log = log_converter.apply(otc, parameters=parameters, variant=log_converter.Variants.TO_EVENT_LOG)

event_log = pm4py.format_dataframe(otc_event_log_only_so2, case_id='sales_order_id', activity_key='activity', timestamp_key='timestamp')

type(event_log)

start_activities = pm4py.get_start_activities(event_log)
end_activities = pm4py.get_end_activities(event_log)
print("Start activities: {}\nEnd activities: {}".format(start_activities, end_activities))

# process mining 
from pm4py.algo.discovery.alpha import algorithm as alpha_miner
from pm4py.algo.discovery.inductive import algorithm as inductive_miner
from pm4py.algo.discovery.heuristics import algorithm as heuristics_miner
from pm4py.algo.discovery.dfg import algorithm as dfg_discovery

# viz
from pm4py.visualization.petrinet import visualizer as pn_visualizer
from pm4py.visualization.process_tree import visualizer as pt_visualizer
from pm4py.visualization.heuristics_net import visualizer as hn_visualizer
from pm4py.visualization.dfg import visualizer as dfg_visualization

# misc 
from pm4py.objects.conversion.process_tree import converter as pt_converter

# alpha miner
net, initial_marking, final_marking = alpha_miner.apply(event_log)

# Visualise
gviz = pn_visualizer.apply(net, initial_marking, final_marking)
pn_visualizer.view(gviz)

# heuristics miner
heu_net = heuristics_miner.apply_heu(event_log)

# viz
gviz = hn_visualizer.apply(heu_net)
hn_visualizer.view(gviz)

from pm4py.algo.filtering.log.start_activities import start_activities_filter
log_start = start_activities_filter.get_start_activities(event_log)
log_start

type(event_log)

event_log.info()

process_tree = pm4py.discover_process_tree_inductive(event_log)
pm4py.view_process_tree(process_tree, #bgcolor='white'
                        #parameters={'bgcolor': 'white'}
                       )

otc_event_log_only_so = pm4py.format_dataframe(otc_event_log_only_so.copy(), case_id='sales_order_id', activity_key='activity', timestamp_key='timestamp')

from pm4py.algo.discovery.dfg import algorithm as dfg_discovery
dfg = dfg_discovery.apply(event_log)

from pm4py.visualization.dfg import visualizer as dfg_visualization
gviz = dfg_visualization.apply(dfg, log=event_log, variant=dfg_visualization.Variants.FREQUENCY)
dfg_visualization.view(gviz)

from pm4py.algo.discovery.dfg import algorithm as dfg_discovery
from pm4py.visualization.dfg import visualizer as dfg_visualization

dfg = dfg_discovery.apply(event_log, variant=dfg_discovery.Variants.PERFORMANCE)
gviz = dfg_visualization.apply(dfg, log=event_log, variant=dfg_visualization.Variants.PERFORMANCE)
dfg_visualization.view(gviz)

# drop some unneeeded cols
otc_event_log_only_so2 = otc_event_log_only_so.drop(['lifecycle', 'status'], axis=1)

# undocumented(?) pm4py requirement: column sequence in df is important
first_column_to_be = otc_event_log_only_so2.pop('sales_order_id')
otc_event_log_only_so2.insert(0, 'sales_order_id', first_column_to_be)
otc_event_log_only_so2.shape

#parameters = {log_converter.Variants.TO_EVENT_LOG.value.Parameters.CASE_ID_KEY: 'case:concept:name'} # identify the case_id_key name (if not change it will simply be the mane of the coloumn)
#event_log_unfiltered = log_converter.apply(otc_event_log_only_so2, parameters=parameters, variant=log_converter.Variants.TO_EVENT_LOG)

#otc_event_log_only_so2.head()

# before 'between filter'
otc_event_log_only_so2[otc_event_log_only_so2['case:concept:name'] == 'order-17-56542'].sort_values(by=['sales_order_id', 'timestamp'])

# filter df
filtered_otc_event_log = pm4py.filter_between(otc_event_log_only_so2, "Send Invoice", "Payment Received")
filtered_otc_event_log.shape

# after filter
filtered_otc_event_log = pm4py.filter_between(otc_event_log_only_so2, "Send Invoice", "Payment Received")
filtered_otc_event_log.sort_values(by=['sales_order_id', 'timestamp']).head()

parameters = {log_converter.Variants.TO_EVENT_LOG.value.Parameters.CASE_ID_KEY: 'case:concept:name'} # identify the case_id_key name (if not change it will simply be the mane of the coloumn)
event_log__otc = log_converter.apply(filtered_otc_event_log, parameters=parameters, variant=log_converter.Variants.TO_EVENT_LOG)

event_log__otc

from pm4py.statistics.traces.generic.log import case_statistics

all_case_durations = case_statistics.get_all_case_durations(event_log__otc, parameters={case_statistics.Parameters.TIMESTAMP_KEY: "time:timestamp"}
                                                           )

all_case_durations_days = pd.Series(all_case_durations)/86400 # days

from matplotlib.pyplot import figure

figure(figsize=(6, 9), dpi=80)
sns.boxplot(data=all_case_durations_days)
plt.title("Distribution of case durations of subprocess 'Payment'")
plt.ylabel("case duration in days")
plt.xlabel("")
plt.show()

# it seems no title can be parametrized through view_dotted_chart method
pm4py.view_dotted_chart(event_log__otc)