pdf(NULL) # suppress Rplots.pdf in batch mode
# ============================================================================
# PRE-REGISTERED HYPOTHESIS 8: BASELINE COMPARISONS
# GPT-4V and Human baseline performance across reference images
# ============================================================================

rm(list = ls())
set.seed(1324)

# === LIBRARIES ===
library(tidyverse)
library(here)
library(boot)

# === DATA IMPORT ===
input_path <- here("similarity_exp_replication", "inputs", "preReg_H8_baselines_data_TRIMMED.csv")
if (!file.exists(input_path)) {
  stop(
    "\n",
    "============================================================\n",
    "  MISSING INPUT FILE\n",
    "  Expected: ", input_path, "\n",
    "  This file is not included in the replication package.\n",
    "============================================================\n"
  )
}

main_df <- read_csv(input_path) %>%
  mutate(subject = factor(if_else(human, "Human", "GPT-4"), levels = c("Human", "GPT-4")))

# === AGGREGATED SUMMARY ===
agg_df <- main_df %>%
  group_by(ref_image, subject, dalle_version) %>%
  summarise(mean_dreamsim = mean(dreamsim, na.rm = TRUE),
            mean_cosine   = mean(cosinesim, na.rm = TRUE),
            sd_dreamsim   = sd(dreamsim, na.rm = TRUE),
            sd_cosine     = sd(cosinesim, na.rm = TRUE),
            n_count       = n(),
            .groups = "drop")

write_csv(agg_df, here("similarity_exp_replication", "inputs",
                       "aggDF_H8_baselines_data.csv"))

# === HELPER FUNCTIONS ===
bootstrap_ci <- function(values, R = 1000) {
  mean_fun <- function(data, indices) mean(data[indices])
  boot_res <- boot(values, statistic = mean_fun, R = R)
  ci_perc  <- boot.ci(boot_res, type = "perc")
  c(lower = ci_perc$percent[4], upper = ci_perc$percent[5])
}

process_data <- function(data, similarity_measure) {
  summarize_df <- data %>%
    group_by(ref_image, subject) %>%
    summarize(mean_score = mean(.data[[similarity_measure]]),
              .groups = "drop") %>%
    mutate(ref_image_fct = factor(ref_image, levels = ref_image[order(mean_score)]))

  ci_df <- data %>%
    group_by(ref_image, subject) %>%
    summarize(ci_lower = bootstrap_ci(.data[[similarity_measure]])["lower"],
              ci_upper = bootstrap_ci(.data[[similarity_measure]])["upper"],
              .groups = "drop")

  summarize_df %>%
    left_join(ci_df, by = c("ref_image", "subject")) %>%
    arrange(desc(mean_score))
}

ref_image_labels <- c(
  "design1" = "Design 1", "design2" = "Design 2", "design3" = "Design 3",
  "design4" = "Design 4", "design5" = "Design 5",
  "photography1" = "Photography 1", "photography2" = "Photography 2",
  "photography3" = "Photography 3", "photography4" = "Photography 4",
  "photography5" = "Photography 5",
  "business1" = "Business 1", "business2" = "Business 2",
  "business3" = "Business 3", "business4" = "Business 4",
  "business5" = "Business 5"
)

# === GPT-4V BASELINES ===
plot_df <- main_df %>% filter(human == FALSE)

# Dreamsim
similarity_measure <- "dreamsim"
summarize_df <- process_data(plot_df, similarity_measure)
sorted_ref_images <- summarize_df$ref_image
print(sorted_ref_images)

ggplot(summarize_df, aes(x = ref_image_fct, y = mean_score)) +
  geom_point() +
  geom_errorbar(aes(ymin = ci_lower, ymax = ci_upper), width = 0.1) +
  scale_x_discrete(labels = ref_image_labels) +
  theme_minimal() +
  labs(title    = "Ranking Image Task Difficulty by GPT-4V Performance (Dreamsim)",
       subtitle = "Mean and Bootstrapped 95% CIs",
       x = "Target Image",
       y = "(1 - Dreamsim) Performance Metric") +
  theme(axis.text.x = element_text(angle = 45, hjust = 1)) +
  coord_flip()

# Cosine similarity
similarity_measure <- "cosinesim"
summarize_df <- process_data(plot_df, similarity_measure)
sorted_ref_images <- summarize_df$ref_image
print(sorted_ref_images)

ggplot(summarize_df, aes(x = ref_image_fct, y = mean_score)) +
  geom_point() +
  geom_errorbar(aes(ymin = ci_lower, ymax = ci_upper), width = 0.1) +
  scale_x_discrete(labels = ref_image_labels) +
  theme_minimal() +
  labs(title    = "Ranking Image Task Difficulty by GPT-4V Performance (Cosine Similarity)",
       subtitle = "Mean and Bootstrapped 95% CIs",
       x = "Target Image",
       y = "Cosine Similarity") +
  theme(axis.text.x = element_text(angle = 45, hjust = 1)) +
  coord_flip()

# === HUMAN BASELINES ===
plot_df <- main_df %>% filter(human == TRUE)

# Dreamsim
similarity_measure <- "dreamsim"
summarize_df <- process_data(plot_df, similarity_measure)
sorted_ref_images <- summarize_df$ref_image
print(sorted_ref_images)

ggplot(summarize_df, aes(x = ref_image_fct, y = mean_score)) +
  geom_point() +
  geom_errorbar(aes(ymin = ci_lower, ymax = ci_upper), width = 0.1) +
  scale_x_discrete(labels = ref_image_labels) +
  theme_minimal() +
  labs(title    = "Ranking Image Task Difficulty by Human Performance (Dreamsim)",
       subtitle = "Mean and Bootstrapped 95% CIs",
       x = "Target Image",
       y = "(1 - Dreamsim) Performance Metric") +
  theme(axis.text.x = element_text(angle = 45, hjust = 1)) +
  coord_flip()

# Cosine similarity
similarity_measure <- "cosinesim"
summarize_df <- process_data(plot_df, similarity_measure)
sorted_ref_images <- summarize_df$ref_image
print(sorted_ref_images)

ggplot(summarize_df, aes(x = ref_image_fct, y = mean_score)) +
  geom_point() +
  geom_errorbar(aes(ymin = ci_lower, ymax = ci_upper), width = 0.1) +
  scale_x_discrete(labels = ref_image_labels) +
  theme_minimal() +
  labs(title    = "Ranking Image Task Difficulty by Human Performance (Cosine Similarity)",
       subtitle = "Mean and Bootstrapped 95% CIs",
       x = "Target Image",
       y = "Cosine Similarity") +
  theme(axis.text.x = element_text(angle = 45, hjust = 1)) +
  coord_flip()
