A/B Testing

import os
import glob
import pandas as pd
import re

filepath = r"CrazyEgg"

df_list = []

for subdir, dirs, files in os.walk(filepath):
    # Find all CSV files in the current directory
    csv_files = glob.glob(os.path.join(subdir, "*.csv"))
    for file in csv_files:
        # Read the CSV file
        df = pd.read_csv(file)
        # Optionally, add a column to track the source file
        df["source_file"] = file
        # Append the dataframe to the list
        df_list.append(df)

# Concatenate all dataframes into one
crazy_egg = pd.concat(df_list, ignore_index=True)

# Extract filename using regex
pattern = r".*[\\/](?P<filename>[^\\/]+)$"
crazy_egg["source_file"] = crazy_egg["source_file"].apply(
    lambda x: re.sub(pattern, r"\1", x)
)

# Creating webpage ID column from filenames.
webpage_pattern = r"(?<=- )\w+"
crazy_egg["webpage"] = crazy_egg["source_file"].apply(
    lambda x: re.search(webpage_pattern, x).group(0)
    if re.search(webpage_pattern, x)
    else None
)

# Create base total click through number table
total = crazy_egg[crazy_egg["Snapshot information"].fillna("").str.contains("created")][
    ["webpage", "Snapshot information"]
]
total.rename(columns={"Snapshot information": "total clicks"}, inplace=True)
click_pattern = r"\d+(?= clicks)"
total["total clicks"] = total["total clicks"].apply(
    lambda x: re.search(click_pattern, x).group(0)
    if re.search(click_patterh, x)
    else None
)
total["total clicks"] = total["total clicks"].astype(int)

# Create home page click through number
home_clicks = crazy_egg[crazy_egg["Name"].str.contains("Home")][
    ["webpage", "No. clicks"]
]
home_clicks.rename(columns={"No. clicks": "homepage clicks"}, inplace=True)
home_clicks["homepage clicks"] = home_clicks["homepage clicks"].astype(int)

# Create element click through number
element_clicks = crazy_egg[crazy_egg["Name"] == crazy_egg["webpage"].str.upper()][
    ["webpage", "No. clicks"]
]
element_clicks.rename(columns={"No. clicks": "element clicks"}, inplace=True)
element_clicks["element clicks"] = element_clicks["element clicks"].astype(int)

# Combine create final click through number table
click_through = pd.merge(total, home_clicks)
click_through["adjusted clicks"] = (
    click_through["total clicks"] - click_through["homepage clicks"]
)
click_through = pd.merge(click_through, element_clicks)

click_through

library(datasets)
library(tidyverse)

filepath <- "CrazyEgg"

# Getting the filenames of the .csv's and put in a data frame.
crazy_egg <- data.frame(filename = list.files(
  path = filepath, pattern = "*.csv",
  full.names = TRUE,
  recursive = TRUE
))

# Consolidate all csv files.
crazy_egg$raw <- map(as.character(crazy_egg$filename), read_csv)

# Regex pattern to extract filenames
pattern <- ".*/([^/\\\\]+)$"
crazy_egg$filename <- sub(pattern, "\\1", crazy_egg$filename)

# Creating webpage name from filenames (Interact, Connect, Learn, Help, Services).
crazy_egg$webpage <- str_extract(
  string = crazy_egg$filename,
  pattern = "(?<=- )\\w+"
)

# Unnesting data frames to make one big dataframe.
crazy_egg <- unnest(crazy_egg)

# Making Column names more readable.
colnames(crazy_egg) <- make.names(colnames(crazy_egg))

# Extracting click count and create click-through base table.
click_through <- filter(crazy_egg, grepl("created", Snapshot.information)) %>%
  select(webpage, Snapshot.information)
colnames(click_through) <- c("webpage", "clicks")

click_through$clicks <- str_extract(
  string = click_through$clicks,
  pattern = "\\d+(?= clicks)"
) %>%
  as.numeric()

# Extracting homepage clicks.
click_through$home_page_clicks <- filter(crazy_egg, grepl("Home", Name)) %>%
  select(No..clicks) %>%
  unlist()

# Homepage adjusted clicks.
click_through <- mutate(click_through, adjusted_clicks = clicks - home_page_clicks)

# Target clicks (e.g., Interact, Connect, Learn, Help, Services button).
click_through$target_clicks <- filter(crazy_egg, grepl(paste(toupper(click_through$webpage), collapse = "|"), Name)) %>%
  select(No..clicks) %>%
  unlist()

click_through <- mutate(click_through, click_rate = target_clicks / adjusted_clicks)

click_through

import plotly.express as px

click_through["click_through_rate"] = (
    click_through["element clicks"] / click_through["adjusted clicks"]
)

fig = px.bar(
    click_through,
    x="webpage",
    y="click_through_rate",
    text=click_through["click_through_rate"].apply(
        lambda x: f"{x*100:.2f}%"
    ),  # Format as percentage with two decimal places
    title="Click-through Rate of Each Variation",
)

# Update layout to customize the plot
fig.update_layout(
    title={
        "text": "Click-through Rate of Each Variation",
        "y": 0.9,
        "x": 0.5,
        "xanchor": "center",
        "yanchor": "top",
    },
    xaxis_title="Webpage",
    yaxis_title="Click-through Rate (%)",
    autosize=False,
    width=600,
    height=600,
)

# Update y-axis tick labels to percentage format
fig.update_yaxes(
    tickformat=".1%", range=[0, 0.04]
)  # Tick format to display as percentage with two decimal places

# Update hover template to show more decimal places
fig.update_traces(hovertemplate="Webpage: %{x}<br>Click-through Rate: %{y:.4f}")

# Show the figure
fig.show()

import statsmodels.stats.proportion as prop
from statsmodels.stats.multitest import multipletests

# Assigning names to successes and trials
successes = click_through["element clicks"].values
trials = click_through["adjusted clicks"].values
webpages = click_through["webpage"].values

# Perform pairwise proportion tests
results = []
for i in range(len(successes)):
    for j in range(i + 1, len(successes)):
        success1 = successes[i]
        trial1 = trials[i]
        success2 = successes[j]
        trial2 = trials[j]
        count = np.array([success1, success2])
        nobs = np.array([trial1, trial2])

        # Perform two independent binomial samples
        z_stat, p_value = prop.test_proportions_2indep(
            success1, trial1, success2, trial2
        )
        # Store results
        result = {
            "Webpage A": webpages[i],
            "Webpage B": webpages[j],
            "z_statistic": z_stat,
            "p_value": p_value,
        }
        results.append(result)

# Convert results to DataFrame
results_df = pd.DataFrame(results)

# Adjust p-values using Bonferroni correction
p_adjusted_bonf = multipletests(results_df["p_value"], method="bonferroni")[1]
results_df["p_adjusted_bonf"] = p_adjusted_bonf

# Adjust p-values using Benjamini-Hochberg (BH) correction
p_adjusted_bh = multipletests(results_df["p_value"], method="fdr_bh")[1]
results_df["p_adjusted_bh"] = p_adjusted_bh

# Display results
results_df["test_result"] = results_df["p_value"].apply(
    lambda p: "Reject" if p < 0.05 else "Failed to reject"
)
results_df["test_result_adjusted_bonf"] = results_df["p_adjusted_bonf"].apply(
    lambda p: "Reject" if p < 0.05 else "Failed to reject"
)
results_df["test_result_adjusted_bh"] = results_df["p_adjusted_bh"].apply(
    lambda p: "Reject" if p < 0.05 else "Failed to reject"
)
results_df

successes <- click_through$element_clicks
trials <- click_through$adjusted_clicks
names(successes) <- click_through$webpage
names(trials) <- click_through$webpage

# Pair-wise proportion test with Bonferroni Correction
pairwise.prop.test(successes,
                   trials,
                   p.adjust.method = "bonferroni"
)

# Pair-wise proportion test with Benjamini-Hochberg (BH) Correction
pairwise.prop.test(successes,
                   trials,
                   p.adjust.method = "BH"
)