user_study_analysis/bdd_study.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e9acc6a8-6211-43d2-9baa-6ae5d8e5d164",
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "import math\n",
    "import os\n",
    "import pickle\n",
    "import random\n",
    "import sys\n",
    "from random import randint\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import requests\n",
    "import statsmodels\n",
    "from PIL import Image\n",
    "from scipy import stats\n",
    "from scipy.stats import ttest_ind, ttest_rel\n",
    "from sklearn.metrics import classification_report\n",
    "from statsmodels.stats import multitest\n",
    "from tabulate import tabulate\n",
    "from tqdm import tqdm\n",
    "\n",
    "\"\"\"\n",
    "array of time taken for each task j. [0, 10, 200, 20, ...]\n",
    "helper for code chunk that parses data\n",
    "\"\"\"\n",
    "\n",
    "\n",
    "def get_each_time_taken(response, time_string):\n",
    "    raw_times = response[time_string]\n",
    "    clean_times = []  # first time will be dropped (unless we know what time they)\n",
    "    i = 1\n",
    "    while i < len(raw_times):\n",
    "        clean_time = raw_times[i] - raw_times[i - 1]\n",
    "        # print(clean_time_noai)\n",
    "        # print(clean_time_noai)\n",
    "        if (\n",
    "            clean_time < 120000\n",
    "        ):  # if people leave mid-task, then comeback (if more than 2minutes spent on 1 task, don't add to clean time)\n",
    "            clean_times.append(clean_time)\n",
    "        i += 1\n",
    "    # print(clean_times_noai)\n",
    "    return clean_times\n",
    "\n",
    "\n",
    "def time_add_to_table(times, col_num):\n",
    "    # print(times)\n",
    "    s_times = [x / 1000 for x in times]\n",
    "    mean, std = np.mean(s_times), np.std(s_times) / np.sqrt(len(s_times))\n",
    "    r_mean = round(mean, 3)\n",
    "    r_std = round(std, 3)\n",
    "    time_row[col_num] = (r_mean, r_std)\n",
    "\n",
    "\n",
    "def ai_rel_add_to_table(ai_rel, col_num):\n",
    "    mean, std = np.mean(ai_rel), np.std(ai_rel) / np.sqrt(len(ai_rel))\n",
    "    r_mean = round(mean, 3)\n",
    "    r_std = round(std, 3)\n",
    "    ai_reliance_row[col_num] = (r_mean, r_std)\n",
    "\n",
    "\n",
    "def mean_std_add_to_table(accuracies, col_num):\n",
    "    mean, std = np.mean(accuracies), np.std(accuracies) / np.sqrt(len(accuracies))\n",
    "    r_mean = round(mean, 3)\n",
    "    r_std = round(std, 3)\n",
    "    mean_row[col_num] = (r_mean, r_std)\n",
    "\n",
    "\n",
    "def p_t_add_to_table(accuracies, col_num):\n",
    "    t_stat, p_val = ttest_ind(accuracies, accuracies_hum_ai_1)\n",
    "    r_t_stat = round(t_stat, 3)\n",
    "    r_p_val = round(p_val, 3)\n",
    "    p_row[col_num] = (r_p_val, r_t_stat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "af5ece7b-70ad-4d2f-b2e6-eddb04f4cc89",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pickle.load(open(\"../data/bdd_study_data.pkl\", \"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ba8563ad-c182-4076-86e4-dbdb87d9a3d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "accuracies_ai_all = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9e286968-6828-4a8f-bd4f-281c8a6baf18",
   "metadata": {},
   "outputs": [],
   "source": [
    "tasks = data[\"tasks\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e5eb31e6-c9b1-4277-b7b5-94b923a1b291",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Human+AI baseline\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7280bd33-44e3-43a5-bd88-29884253580a",
   "metadata": {},
   "outputs": [],
   "source": [
    "responses = data[\"human+ai baseline\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d751bddc-91be-45e8-bfaf-a267de92aaf0",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "accuracies_hum_no_ai = []\n",
    "accuracies_hum_ai = []\n",
    "accuracies_ai_with_ai = []\n",
    "accuracies_ai_no_ai = []\n",
    "failed_attention_check = []\n",
    "ai_reliance = []\n",
    "\n",
    "# time variables\n",
    "ai_reliance = []\n",
    "times_hum_no_ai = []\n",
    "times_hum_with_ai = []\n",
    "\n",
    "for i in range(len(responses)):\n",
    "    # find the matching task\n",
    "    id_task = responses[i][\"id\"].split(\"-\")[0]\n",
    "    task = tasks[id_task]\n",
    "    user_answers_no_ai_raw = responses[i][\"testing_answers_noai\"]\n",
    "    user_answers_ai_raw = responses[i][\"testing_answers_withai\"]\n",
    "\n",
    "    # task\n",
    "    labels_no_ai = task[\"testing_withairec_labels\"]  # humans with\n",
    "    labels_ai = task[\"testing_withai_label\"]\n",
    "    ai_no_ai = task[\"testing_withairec_ai_answers_raw\"]\n",
    "    ai_ai = task[\"testing_withai_ai_answers_raw\"]\n",
    "    att_check_no_ai = task[\"testing_attentioncheck\"]\n",
    "    att_check_ai = task[\"testing_withai_attentioncheck\"]\n",
    "\n",
    "    # time\n",
    "    response = responses[i]\n",
    "    clean_times_noai = get_each_time_taken(\n",
    "        response, \"testing_times_noai\"\n",
    "    )  # [example1 time, example2 time, ...]\n",
    "    clean_times_withai = get_each_time_taken(response, \"testing_times_withai\")\n",
    "    times_hum_no_ai.append(\n",
    "        sum(clean_times_noai) / len(clean_times_noai)\n",
    "    )  # [avg time per example by person1, by person2, ...]\n",
    "    times_hum_with_ai.append(sum(clean_times_withai) / len(clean_times_withai))\n",
    "\n",
    "    # initialize to compile user answers\n",
    "    user_answers_no_ai = []\n",
    "    user_answers_ai = []\n",
    "    reliance_ai = 0\n",
    "    for j in range(len(user_answers_no_ai_raw)):\n",
    "        if user_answers_no_ai_raw[j] == \"yes\":\n",
    "            user_answers_no_ai.append(1)\n",
    "        else:\n",
    "            user_answers_no_ai.append(0)\n",
    "\n",
    "    for j in range(len(user_answers_ai_raw)):\n",
    "        if user_answers_ai_raw[j] == \"yes\":\n",
    "            user_answers_ai.append(1)\n",
    "        elif user_answers_ai_raw[j] == \"no\":\n",
    "            user_answers_ai.append(0)\n",
    "        else:\n",
    "            reliance_ai += 1\n",
    "            user_answers_ai.append(ai_ai[j])\n",
    "    # check if the attention check is correct\n",
    "    failed_checks = 0\n",
    "    for j in range(len(att_check_no_ai)):\n",
    "        if att_check_no_ai[j] == 1:\n",
    "            if user_answers_no_ai[j] != labels_no_ai[j]:\n",
    "                failed_checks += 1\n",
    "    if failed_checks >= np.sum(att_check_no_ai):\n",
    "        continue\n",
    "    failed_checks = 0\n",
    "    for j in range(len(att_check_ai)):\n",
    "        if att_check_ai[j] == 1:\n",
    "            if user_answers_ai[j] != labels_ai[j]:\n",
    "                failed_checks += 1\n",
    "    failed_attention_check.append(failed_checks)\n",
    "    if failed_checks >= np.sum(att_check_ai):\n",
    "        continue\n",
    "\n",
    "    accuracies_hum_no_ai.append(accuracy_score(labels_no_ai, user_answers_no_ai))\n",
    "    accuracies_hum_ai.append(accuracy_score(labels_ai, user_answers_ai))\n",
    "    accuracies_ai_with_ai.append(accuracy_score(labels_ai, ai_ai))\n",
    "    accuracies_ai_no_ai.append(accuracy_score(labels_no_ai, ai_no_ai))\n",
    "    if accuracy_score(labels_ai, user_answers_ai) == 0:\n",
    "        print(user_answers_ai)\n",
    "        print(labels_ai)\n",
    "    ai_reliance.append(reliance_ai / len(user_answers_ai))\n",
    "\n",
    "accuracies_hum_no_ai_1 = accuracies_hum_no_ai.copy()\n",
    "accuracies_hum_ai_1 = accuracies_hum_ai.copy()\n",
    "ai_reliance_1 = ai_reliance.copy()\n",
    "accuracies_ai_no_ai_1 = accuracies_ai_no_ai.copy()\n",
    "accuracies_ai_all.append(accuracies_ai_no_ai_1)\n",
    "\n",
    "# print(times_hum_no_ai)\n",
    "print(len(accuracies_hum_ai_1))\n",
    "print(f\"Human only accuracy: {np.mean(accuracies_hum_no_ai_1)}\")\n",
    "print(f\"Human + AI accuracy: {np.mean(accuracies_hum_ai_1)}\")\n",
    "print(f\" AI (no ai) accuracy: {np.mean(accuracies_ai_no_ai_1)}\")\n",
    "print(f\" AI (with AI) accuracy: {np.mean(accuracies_ai_with_ai)}\")\n",
    "\n",
    "print(f\"AI reliance: {np.mean(ai_reliance_1)}\")\n",
    "print(f\"Failed attention checks: {np.mean(failed_attention_check)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10a5d05e-4c0c-4b4a-95be-999c7495309c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# initialize table\n",
    "\n",
    "num_col = 7\n",
    "empty_row = [None] * num_col\n",
    "# 1: human, 2: human+ai, 3: human+ai+teach+rec, 4: human+ai+teach, 5: human+ai+teach baseline, 6:human+ai+rec, 7: ai only\n",
    "\n",
    "mean_row = [\"Accuracy (mean, std dev)\"] + empty_row\n",
    "p_row = [\"Test with Human+AI (p-value, t-value)\"] + empty_row\n",
    "ai_reliance_row = [\"AI reliance\"] + empty_row\n",
    "time_row = [\"Time per example\"] + empty_row\n",
    "ai_acc_row = [\"Accuracy when using AI\"] + empty_row\n",
    "no_ai_acc_row = [\"Accuracy when not using AI\"] + empty_row\n",
    "\n",
    "table = [\n",
    "    [\n",
    "        \"Metric\",\n",
    "        \"Human\",\n",
    "        \"Human+AI\",\n",
    "        \"Human+AI+Teach+Rec\",\n",
    "        \"Human+AI+Teach\",\n",
    "        \"Human+AI+Teach baseline\",\n",
    "        \"Human+AI+Rec\",\n",
    "        \"AI only\",\n",
    "    ],\n",
    "    mean_row,\n",
    "    p_row,\n",
    "    ai_reliance_row,\n",
    "    time_row,\n",
    "    ai_acc_row,\n",
    "    no_ai_acc_row,\n",
    "]\n",
    "\n",
    "headers = table.pop(0)\n",
    "\n",
    "print(tabulate(table, headers=headers))\n",
    "# print(tabulate(table, headers=headers, tablefmt=\"latex\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "03735d2c-f6c1-4c83-8d37-382532add096",
   "metadata": {},
   "outputs": [],
   "source": [
    "# populate table for human and human_ai columns\n",
    "mean_std_add_to_table(accuracies_hum_no_ai_1, 1)\n",
    "mean_std_add_to_table(accuracies_hum_ai_1, 2)\n",
    "\n",
    "ai_reliance_row[1] = \"N/A\"\n",
    "ai_rel_add_to_table(ai_reliance_1, 2)\n",
    "\n",
    "time_add_to_table(times_hum_no_ai, 1)\n",
    "time_add_to_table(times_hum_with_ai, 2)\n",
    "\n",
    "# human and human_ai pairwise t test\n",
    "\n",
    "t_stat1, p_val1 = ttest_rel(accuracies_hum_no_ai_1, accuracies_hum_ai_1)\n",
    "r_t_stat1 = round(t_stat1, 3)\n",
    "r_p_val1 = round(p_val1, 3)\n",
    "p_row[1] = (r_p_val1, r_t_stat1)\n",
    "p_row[2] = (r_p_val1, r_t_stat1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3875f6ce-18ff-4854-8d31-b36f03ecb160",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "# Human + AI + Teach (ours)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dad3ee77-0344-49e7-874f-670e46308877",
   "metadata": {},
   "outputs": [],
   "source": [
    "responses = data[\"human+ai+teach(ours)\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d79f266a-8524-4929-8131-327946b4a4d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "accuracies_hum_no_ai = []\n",
    "accuracies_hum_ai = []\n",
    "accuracies_ai_with_ai = []\n",
    "accuracies_ai_no_ai = []\n",
    "failed_attention_check = []\n",
    "ai_reliance_3 = []\n",
    "ai_reliance_4 = []\n",
    "\n",
    "# time\n",
    "times_hum_no_ai = []\n",
    "times_hum_with_ai = []\n",
    "\n",
    "for i in range(len(responses)):\n",
    "    # find the matching task\n",
    "    id_task = responses[i][\"id\"].split(\"-\")[0]\n",
    "    task = tasks[id_task]\n",
    "    user_answers_no_ai_raw = responses[i][\"testing_answers_withairec\"]\n",
    "    user_answers_ai_raw = responses[i][\"testing_answers_withai\"]\n",
    "    # task\n",
    "    labels_no_ai = task[\"testing_withairec_labels\"]\n",
    "    labels_ai = task[\"testing_withai_label\"]\n",
    "    ai_no_ai = task[\"testing_withairec_ai_answers_raw\"]\n",
    "    ai_ai = task[\"testing_withai_ai_answers_raw\"]\n",
    "    att_check_no_ai = task[\"testing_attentioncheck\"]\n",
    "    att_check_ai = task[\"testing_withai_attentioncheck\"]\n",
    "    user_answers_no_ai = []\n",
    "    user_answers_ai = []\n",
    "    reliance_ai_3 = 0\n",
    "    reliance_ai_4 = 0\n",
    "\n",
    "    # time\n",
    "    response = responses[i]\n",
    "    clean_times_noai = get_each_time_taken(\n",
    "        response, \"testing_times_withairec\"\n",
    "    )  # [example1 time, example2 time, ...]\n",
    "    clean_times_withai = get_each_time_taken(response, \"testing_times_withai\")\n",
    "    times_hum_no_ai.append(\n",
    "        sum(clean_times_noai) / len(clean_times_noai)\n",
    "    )  # [avg time per example by person1, by person2, ...]\n",
    "    times_hum_with_ai.append(sum(clean_times_withai) / len(clean_times_withai))\n",
    "\n",
    "    for j in range(len(user_answers_no_ai_raw)):\n",
    "        if user_answers_no_ai_raw[j] == \"yes\":\n",
    "            user_answers_no_ai.append(1)\n",
    "        elif user_answers_no_ai_raw == \"no\":\n",
    "            user_answers_no_ai.append(0)\n",
    "        else:\n",
    "            reliance_ai_3 += 1\n",
    "            user_answers_no_ai.append(ai_no_ai[j])\n",
    "\n",
    "    for j in range(len(user_answers_ai_raw)):\n",
    "        if user_answers_ai_raw[j] == \"yes\":\n",
    "            user_answers_ai.append(1)\n",
    "        elif user_answers_ai_raw[j] == \"no\":\n",
    "            user_answers_ai.append(0)\n",
    "        else:\n",
    "            reliance_ai_4 += 1\n",
    "            user_answers_ai.append(ai_ai[j])\n",
    "    # check if the attention check is correct\n",
    "    failed_checks = 0\n",
    "\n",
    "    for j in range(len(att_check_no_ai) - 1):\n",
    "        if att_check_no_ai[j] == 1:\n",
    "            if user_answers_no_ai[j] != labels_no_ai[j]:\n",
    "                failed_checks += 1\n",
    "    if failed_checks >= np.sum(att_check_no_ai):\n",
    "        continue\n",
    "    failed_checks = 0\n",
    "    for j in range(len(att_check_ai)):\n",
    "        if att_check_ai[j] == 1:\n",
    "            if user_answers_ai[j] != labels_ai[j]:\n",
    "                failed_checks += 1\n",
    "    if failed_checks >= np.sum(att_check_ai):\n",
    "        continue\n",
    "    failed_attention_check.append(failed_checks)\n",
    "\n",
    "    try:\n",
    "        if accuracy_score(labels_no_ai, user_answers_no_ai) == 0.5:\n",
    "            continue\n",
    "        accuracies_hum_no_ai.append(accuracy_score(labels_no_ai, user_answers_no_ai))\n",
    "        accuracies_hum_ai.append(accuracy_score(labels_ai, user_answers_ai))\n",
    "\n",
    "        accuracies_ai_with_ai.append(accuracy_score(labels_ai, ai_ai))\n",
    "        accuracies_ai_no_ai.append(accuracy_score(labels_no_ai, ai_no_ai))\n",
    "\n",
    "        if accuracy_score(labels_ai, user_answers_ai) == 0:\n",
    "            print(user_answers_ai)\n",
    "            print(labels_ai)\n",
    "        ai_reliance_4.append(reliance_ai_4 / len(user_answers_ai))\n",
    "\n",
    "        if accuracy_score(labels_no_ai, user_answers_no_ai) == 0:\n",
    "            print(user_answers_no_ai)\n",
    "            print(labels_no_ai)\n",
    "        ai_reliance_3.append(reliance_ai_3 / len(user_answers_no_ai))\n",
    "\n",
    "    except:\n",
    "        print(\"some error\")\n",
    "        continue\n",
    "\n",
    "\n",
    "accuracies_hum_no_ai_teach_rec = accuracies_hum_no_ai.copy()\n",
    "accuracies_hum_no_ai_teach = accuracies_hum_ai.copy()\n",
    "accuracies_ai_no_ai_2 = accuracies_ai_no_ai.copy()\n",
    "accuracies_ai_with_ai_2 = accuracies_ai_with_ai.copy()\n",
    "accuracies_ai_all.append(accuracies_ai_with_ai_2)\n",
    "\n",
    "print(f\"Human + AI + Teach + rec accuracy: {np.mean(accuracies_hum_no_ai_teach_rec)}\")\n",
    "print(f\"Human + AI + Teach accuracy: {np.mean(accuracies_hum_no_ai_teach)}\")\n",
    "print(f\" AI (with rec) accuracy: {np.mean(accuracies_ai_no_ai_2)}\")\n",
    "print(f\" AI (no rec) accuracy: {np.mean(accuracies_ai_with_ai_2)}\")\n",
    "\n",
    "print(f\"AI reliance: {np.mean(ai_reliance_3)}\")\n",
    "print(f\"AI reliance: {np.mean(ai_reliance_4)}\")\n",
    "print(f\"Failed attention checks: {np.mean(failed_attention_check)}\")\n",
    "# show the distribution of accuracies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1dcffbd1-5caf-49aa-8309-ff7ee239e775",
   "metadata": {},
   "outputs": [],
   "source": [
    "# populate table for 3: human+ai+teach+rec t-test, 4: human+ai+teach\n",
    "mean_std_add_to_table(accuracies_hum_no_ai_teach_rec, 3)\n",
    "p_t_add_to_table(accuracies_hum_no_ai_teach_rec, 3)\n",
    "\n",
    "mean_std_add_to_table(accuracies_hum_no_ai_teach, 4)\n",
    "p_t_add_to_table(accuracies_hum_no_ai_teach, 4)\n",
    "\n",
    "ai_rel_add_to_table(ai_reliance_3, 3)\n",
    "ai_rel_add_to_table(ai_reliance_4, 4)\n",
    "\n",
    "time_add_to_table(times_hum_no_ai, 3)\n",
    "time_add_to_table(times_hum_with_ai, 4)\n",
    "\n",
    "print(tabulate(table, headers=headers))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "86c0541b-3e4e-49fd-b091-3cf64669631e",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "# Human + AI + Teach Baseline\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "084edbd5-75e3-481c-87c1-fdab2c3fc340",
   "metadata": {},
   "outputs": [],
   "source": [
    "responses = data[\"human+ai+teach baseline\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4bbd3195-cfe8-45f8-a355-c91e017cd58a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "accuracies_hum_no_ai = []\n",
    "accuracies_hum_ai = []\n",
    "accuracies_ai_with_ai = []\n",
    "accuracies_ai_no_ai = []\n",
    "failed_attention_check = []\n",
    "ai_reliance = []\n",
    "times_hum_with_ai = []\n",
    "for i in range(len(responses)):\n",
    "    # find the matching task\n",
    "    id_task = responses[i][\"id\"].split(\"-\")[0]\n",
    "    task = tasks[id_task]\n",
    "    user_answers_ai_raw = responses[i][\"testing_answers_withai\"]\n",
    "    # task\n",
    "    labels_no_ai = task[\"testing_withairec_labels\"]\n",
    "    labels_ai = task[\"testing_withai_label\"]\n",
    "    ai_no_ai = task[\"testing_withairec_ai_answers_raw\"]\n",
    "    ai_ai = task[\"testing_withai_ai_answers_raw\"]\n",
    "    att_check_no_ai = task[\"testing_attentioncheck\"]\n",
    "    att_check_ai = task[\"testing_withai_attentioncheck\"]\n",
    "    user_answers_no_ai = []\n",
    "    user_answers_ai = []\n",
    "    reliance_ai = 0\n",
    "\n",
    "    # time\n",
    "    response = responses[i]\n",
    "    clean_times_withai = get_each_time_taken(response, \"testing_times_withai\")\n",
    "    times_hum_with_ai.append(sum(clean_times_withai) / len(clean_times_withai))\n",
    "\n",
    "    for j in range(len(user_answers_ai_raw)):\n",
    "        if user_answers_ai_raw[j] == \"yes\":\n",
    "            user_answers_ai.append(1)\n",
    "        elif user_answers_ai_raw[j] == \"no\":\n",
    "            user_answers_ai.append(0)\n",
    "        else:\n",
    "            reliance_ai += 1\n",
    "            user_answers_ai.append(ai_ai[j])\n",
    "    # check if the attention check is correct\n",
    "    failed_checks = 0\n",
    "\n",
    "    for j in range(len(att_check_ai) - 1):\n",
    "        if att_check_ai[j] == 1:\n",
    "            if user_answers_ai[j] != labels_ai[j]:\n",
    "                failed_checks += 1\n",
    "\n",
    "    if failed_checks >= np.sum(att_check_ai):\n",
    "        continue\n",
    "    failed_attention_check.append(failed_checks)\n",
    "    try:\n",
    "        accuracies_hum_ai.append(accuracy_score(labels_ai, user_answers_ai))\n",
    "        accuracies_ai_with_ai.append(accuracy_score(labels_ai, ai_ai))\n",
    "        if accuracy_score(labels_ai, user_answers_ai) == 0:\n",
    "            print(user_answers_ai)\n",
    "            print(labels_ai)\n",
    "        ai_reliance.append(reliance_ai / len(user_answers_ai))\n",
    "    except:\n",
    "        print(\"some error\")\n",
    "        continue\n",
    "\n",
    "accuracies_hum_ai_base = accuracies_hum_ai.copy()\n",
    "accuracies_ai_only = accuracies_ai_with_ai.copy()\n",
    "ai_reliance_5 = ai_reliance.copy()\n",
    "accuracies_ai_only_3 = accuracies_ai_only.copy()\n",
    "accuracies_ai_all.append(accuracies_ai_only_3)\n",
    "\n",
    "print(\n",
    "    f\"Human + AI + Teach Baseline: {np.mean(accuracies_hum_ai_base)}\"\n",
    ")  # 1 treatment, just teach baseline\n",
    "print(f\" AI (no rec) accuracy: {np.mean(accuracies_ai_only_3)}\")\n",
    "\n",
    "print(f\"AI reliance: {np.mean(ai_reliance_5)}\")\n",
    "print(f\"Failed attention checks: {np.mean(failed_attention_check)}\")\n",
    "print(len(accuracies_hum_ai_base))\n",
    "print(failed_attention_check)\n",
    "# show the distribution of accuracies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5fede5e-ebc1-4157-8e60-d62eb2613d7a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 5: human+ai+teach baseline\n",
    "mean_std_add_to_table(accuracies_hum_ai_base, 5)\n",
    "p_t_add_to_table(accuracies_hum_ai_base, 5)\n",
    "\n",
    "ai_rel_add_to_table(ai_reliance_5, 5)\n",
    "\n",
    "time_add_to_table(times_hum_with_ai, 5)\n",
    "\n",
    "print(tabulate(table, headers=headers))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9a24634f-0482-4e1a-807e-83a1529f4ced",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "# Human + AI + REC\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3a9b9f61-c03a-4085-82d2-e4a7ce5e4809",
   "metadata": {},
   "outputs": [],
   "source": [
    "responses = data[\"human+ai rec\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f88b87da-5e7a-4c6b-ab4e-6f636b2bc5a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "accuracies_hum_no_ai = []\n",
    "accuracies_hum_ai = []\n",
    "accuracies_ai_with_ai = []\n",
    "accuracies_ai_no_ai = []\n",
    "failed_attention_check = []\n",
    "ai_reliance = []\n",
    "# time\n",
    "times_hum_ai = []\n",
    "\n",
    "for i in range(len(responses)):\n",
    "    # find the matching task\n",
    "    id_task = responses[i][\"id\"].split(\"-\")[0]\n",
    "    task = tasks[id_task]\n",
    "    user_answers_ai_raw = responses[i][\"testing_answers_withairec\"]\n",
    "    # task\n",
    "    labels_no_ai = task[\"testing_withairec_labels\"]\n",
    "    labels_ai = task[\"testing_withai_label\"]\n",
    "    ai_no_ai = task[\"testing_withairec_ai_answers_raw\"]\n",
    "    ai_ai = task[\"testing_withai_ai_answers_raw\"]\n",
    "    att_check_no_ai = task[\"testing_attentioncheck\"]\n",
    "    att_check_ai = task[\"testing_withai_attentioncheck\"]\n",
    "    user_answers_no_ai = []\n",
    "    user_answers_ai = []\n",
    "    reliance_ai = 0\n",
    "\n",
    "    # time\n",
    "    response = responses[i]\n",
    "    clean_times_withai = get_each_time_taken(response, \"testing_times_withairec\")\n",
    "    times_hum_with_ai.append(sum(clean_times_withai) / len(clean_times_withai))\n",
    "\n",
    "    for j in range(len(user_answers_ai_raw)):\n",
    "        if user_answers_ai_raw[j] == \"yes\":\n",
    "            user_answers_ai.append(1)\n",
    "        elif user_answers_ai_raw[j] == \"no\":\n",
    "            user_answers_ai.append(0)\n",
    "        else:\n",
    "            reliance_ai += 1\n",
    "            user_answers_ai.append(ai_no_ai[j])\n",
    "    # check if the attention check is correct\n",
    "    failed_checks = 0\n",
    "\n",
    "    for j in range(len(att_check_ai) - 1):\n",
    "        if att_check_ai[j] == 1:\n",
    "            if user_answers_ai[j] != labels_no_ai[j]:\n",
    "                failed_checks += 1\n",
    "    if failed_checks >= np.sum(att_check_ai):\n",
    "        continue\n",
    "    failed_attention_check.append(failed_checks)\n",
    "    try:\n",
    "        accuracies_hum_ai.append(accuracy_score(labels_no_ai, user_answers_ai))\n",
    "        accuracies_ai_with_ai.append(accuracy_score(labels_no_ai, ai_no_ai))\n",
    "        if accuracy_score(labels_ai, user_answers_ai) == 0:\n",
    "            print(user_answers_ai)\n",
    "            print(labels_ai)\n",
    "        ai_reliance.append(reliance_ai / len(user_answers_ai))\n",
    "    except:\n",
    "        print(\"some error\")\n",
    "        continue\n",
    "\n",
    "accuracies_hum_ai_rec = accuracies_hum_ai.copy()\n",
    "accuracies_ai_with_ai4 = accuracies_ai_with_ai.copy()\n",
    "accuracies_ai_all.append(accuracies_ai_with_ai4)\n",
    "print(len(accuracies_hum_ai_rec))\n",
    "print(\n",
    "    f\"Human + AI + REC Basline: {np.mean(accuracies_hum_ai_rec)}\"\n",
    ")  # just rec (no teaching)\n",
    "print(f\" AI (no rec) accuracy: {np.mean(accuracies_ai_with_ai4)}\")\n",
    "\n",
    "print(f\"AI reliance: {np.mean(ai_reliance)}\")\n",
    "print(f\"Failed attention checks: {np.mean(failed_attention_check)}\")\n",
    "# show the distribution of accuracies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2ac4698f-634a-47c9-9e23-2ca3131fedfe",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 6: human+ai+teach baseline\n",
    "mean_std_add_to_table(accuracies_hum_ai_rec, 6)\n",
    "p_t_add_to_table(accuracies_hum_ai_rec, 6)\n",
    "ai_rel_add_to_table(ai_reliance, 6)\n",
    "time_add_to_table(times_hum_with_ai, 6)\n",
    "print(tabulate(table, headers=headers))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aeb34145",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 6: human+ai+teach baseline\n",
    "mean_std_add_to_table(accuracies_hum_ai_rec, 6)\n",
    "p_t_add_to_table(accuracies_hum_ai_rec, 6)\n",
    "ai_rel_add_to_table(ai_reliance, 6)\n",
    "time_add_to_table(times_hum_with_ai, 6)\n",
    "print(tabulate(table, headers=headers))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e045232c-c26e-429b-8015-e2e1142c8526",
   "metadata": {
    "tags": []
   },
   "source": [
    "# SUMMARY\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "84741a1d-2772-4b00-ad08-ed77e15e3d9b",
   "metadata": {},
   "outputs": [],
   "source": [
    "accuracies_ai_all_flat = [\n",
    "    item for nested_list in accuracies_ai_all for item in nested_list\n",
    "]\n",
    "data[\"ai only\"] = accuracies_ai_all_flat\n",
    "# 7: AI only\n",
    "mean_std_add_to_table(accuracies_ai_all_flat, 7)\n",
    "p_t_add_to_table(accuracies_ai_all_flat, 7)\n",
    "print(tabulate(table, headers=headers))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b60bb000-2e2c-4f38-b9de-3ba6df2d0f7c",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(tabulate(table, headers=headers))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "88f42b31-7780-4174-9f2f-e7a824843c95",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(tabulate(table, headers=headers, tablefmt=\"latex_booktabs\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0caa4f1d-49c8-4821-99ac-ff7d855d842e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# List of array names for reference\n",
    "array_names = [\n",
    "    \"accuracies_hum_ai_rec\",\n",
    "    \"accuracies_ai_all_flat\",\n",
    "    \"accuracies_hum_ai_base\",\n",
    "    \"accuracies_hum_no_ai_teach_rec\",\n",
    "    \"accuracies_hum_no_ai_teach\",\n",
    "    \"accuracies_hum_no_ai_1\",\n",
    "    \"accuracies_hum_ai_1\",\n",
    "]\n",
    "names_array = [\n",
    "    \"Rec\",\n",
    "    \"AI only\",\n",
    "    \"Onboard(baseline)\",\n",
    "    \"Onboard(ours)+Rec\",\n",
    "    \"Onboard(ours)\",\n",
    "    \"Human\",\n",
    "    \"Human-AI\",\n",
    "]\n",
    "\n",
    "\n",
    "# Create a list to hold the results\n",
    "results = []\n",
    "\n",
    "j = -1\n",
    "# Perform pairwise t-tests\n",
    "for i in range(len(array_names) - 1):\n",
    "    p_value = stats.ttest_ind(eval(array_names[i]), eval(array_names[j])).pvalue\n",
    "    results.append([names_array[i], names_array[j], p_value])\n",
    "\n",
    "\n",
    "tests_to_add = []\n",
    "\n",
    "for test in tests_to_add:\n",
    "    i = test[0]\n",
    "    j = test[1]\n",
    "    p_value = stats.ttest_ind(eval(array_names[i]), eval(array_names[j])).pvalue\n",
    "    results.append([names_array[i], names_array[j], p_value])\n",
    "\n",
    "\n",
    "# Create a pandas DataFrame to organize the results\n",
    "results_df = pd.DataFrame(results, columns=[\"Array 1\", \"Array 2\", \"p-value\"])\n",
    "p_values = results_df[\"p-value\"].values\n",
    "\n",
    "# Perform FDR correction using Benjamini-Hochberg method\n",
    "rejected, adjusted_p_values = multitest.fdrcorrection(\n",
    "    p_values, alpha=0.05, method=\"indep\"\n",
    ")\n",
    "\n",
    "# Add adjusted p-values to the DataFrame\n",
    "results_df[\"adjusted_p-value\"] = adjusted_p_values\n",
    "# Display the results\n",
    "print(results_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ca88aad3-078c-46cd-b302-147e3d466d38",
   "metadata": {},
   "source": [
    "# qualitative\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "31778b60-bd03-4a70-9b35-4456f2fce22a",
   "metadata": {},
   "outputs": [],
   "source": [
    "responses_teach = data[\"human+ai+teach(ours)\"]\n",
    "responses = data[\"human+ai baseline\"]\n",
    "outake_teach = [\n",
    "    responses_teach[i][\"outake_quest1\"] for i in range(len(responses_teach))\n",
    "]\n",
    "outake_base = [responses[i][\"outake_quest1\"] for i in range(len(responses))]\n",
    "\n",
    "from sentence_transformers import SentenceTransformer\n",
    "\n",
    "model = SentenceTransformer(\"sentence-transformers/all-MiniLM-L6-v2\")\n",
    "# batch size of 16\n",
    "all_responses = outake_teach + outake_base\n",
    "embeddings = model.encode(all_responses, batch_size=16)\n",
    "embeddings = np.array(embeddings)\n",
    "\n",
    "# cluster_scores is 2d array where each element is [1,0] if it is in outake_teach and [0,1] if it is in outake_base\n",
    "cluster_scores = np.zeros((len(all_responses), 2))\n",
    "cluster_labels = np.zeros(len(all_responses))\n",
    "\n",
    "for i in range(len(outake_teach)):\n",
    "    cluster_scores[i][0] = 1\n",
    "    cluster_labels[i] = int(1)\n",
    "for i in range(len(outake_base)):\n",
    "    cluster_scores[i + len(outake_teach)][1] = 1\n",
    "\n",
    "\n",
    "def get_text_embedding(string):\n",
    "    return model.encode([string], batch_size=16)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3ae14d43-6609-43f2-ac97-1913b05c62b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "\n",
    "sys.path.append(\"../src\")\n",
    "logging.getLogger().setLevel(logging.INFO)\n",
    "\n",
    "from describers.itterative_describe import *\n",
    "\n",
    "# OPEN AI KEY\n",
    "keyfile = open(\"../keys.txt\", \"r\")\n",
    "# read the file\n",
    "key = keyfile.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c6e86181-0cba-46a0-914b-fee50d376135",
   "metadata": {},
   "outputs": [],
   "source": [
    "class IterativeRegionDescribeAnalysis(IterativeRegionDescribe):\n",
    "    def __init__(\n",
    "        self,\n",
    "        descriptions,\n",
    "        embeddings,\n",
    "        cluster_scores,\n",
    "        cluster_labels,\n",
    "        open_ai_key,\n",
    "        get_text_embedding_fn,\n",
    "        n_rounds=5,\n",
    "        initial_positive_set_size=15,\n",
    "        initial_negative_set_size=5,\n",
    "        chat_correct=False,\n",
    "    ):\n",
    "        super().__init__(\n",
    "            descriptions,\n",
    "            embeddings,\n",
    "            cluster_scores,\n",
    "            cluster_labels,\n",
    "            open_ai_key,\n",
    "            get_text_embedding_fn,\n",
    "            n_rounds,\n",
    "            initial_positive_set_size,\n",
    "            initial_negative_set_size,\n",
    "            chat_correct,\n",
    "        )  # Calling the superclass constructor\n",
    "        self.pre_instruction = (\n",
    "            \"I will provide you with a set of descriptions of points that belong to a region and a set of descriptions of point that do not belong to the region.\"\n",
    "            + \"Your task is to summarize the points inside the region in one or two sentences detailed while making sure the summary contrast to points outside the region. Please compare to outside the region.\"\n",
    "            + \"Your   summary should be able to allow a person to distinguish between points inside and outside the region while describing the region really well.\"\n",
    "            + \"The summary should be no more than 100 words, it should be accurate, detailed, concise, distinguishing and precise.\"\n",
    "            + \"Example: \\n\"\n",
    "            + \"inside the region: \\n two cows and two sheep grazing in a pasture. \\n the sheep is standing near a tree. \\n outside the region:  the cows are lying on the grass beside the water.\\n\"\n",
    "            + \"summary: The region consists of descriptions that have have sheep in them outside in nature, it could have cows but must have sheep. \\n End of Example \\n\"\n",
    "        )\n",
    "        self.post_instruction = \"summary:\"\n",
    "\n",
    "    def get_completion(self, prompt, history=[]):\n",
    "        while True:\n",
    "            try:\n",
    "                if len(history) == 0:\n",
    "                    messages = [\n",
    "                        {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n",
    "                        {\"role\": \"user\", \"content\": prompt},\n",
    "                    ]\n",
    "                else:\n",
    "                    messages = history\n",
    "\n",
    "                response = openai.ChatCompletion.create(\n",
    "                    model=\"gpt-4\", messages=messages\n",
    "                )\n",
    "                logging.info(\"Called OPENAI API\")\n",
    "\n",
    "                return response[\"choices\"][0][\"message\"][\"content\"]\n",
    "            except:\n",
    "                print(\"pausing\")\n",
    "                time.sleep(0.3)\n",
    "                continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2ec2972a-b243-4f15-961e-68c6b7d393a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "itt_desriber = IterativeRegionDescribeAnalysis(\n",
    "    all_responses,\n",
    "    embeddings,\n",
    "    cluster_scores,\n",
    "    cluster_labels,\n",
    "    key,\n",
    "    get_text_embedding,\n",
    "    0,\n",
    "    initial_positive_set_size=len(outake_teach),\n",
    "    initial_negative_set_size=len(outake_base),\n",
    ")\n",
    "\n",
    "itt_des = itt_desriber.describe_region(1)\n",
    "itt_des[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "89772653-139a-4f18-9278-b48e3254e7d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "itt_desriber = IterativeRegionDescribeAnalysis(\n",
    "    all_responses,\n",
    "    embeddings,\n",
    "    cluster_scores,\n",
    "    cluster_labels,\n",
    "    key,\n",
    "    get_text_embedding,\n",
    "    0,\n",
    "    initial_positive_set_size=len(outake_base),\n",
    "    initial_negative_set_size=len(outake_teach),\n",
    ")\n",
    "\n",
    "itt_des = itt_desriber.describe_region(0)\n",
    "itt_des[0]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:teach_vision] *",
   "language": "python",
   "name": "conda-env-teach_vision-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}