5-one-shot-and-linear-probing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "from umap import UMAP\n",
    "from sklearn.decomposition import PCA\n",
    "\n",
    "from sklearn.model_selection import train_test_split, StratifiedShuffleSplit, GroupShuffleSplit\n",
    "\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.neural_network import MLPClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from source.constants import RANDOM_SEED\n",
    "from source.constants import DATA_DIR, FEATURE_VECTORS_SAVE_DIR, ANNOTATIONS_SAVE_DIR\n",
    "from source.constants import ALL_TISSUE_TYPES, ALL_CANCER_TYPES, ALL_IMG_NORMS, ALL_EXTRACTOR_MODELS, ALL_DIMENSIONALITY_REDUCTION_METHODS, ALL_CLUSTERING_ALGORITHMS, ALL_DISTANCE_METRICS\n",
    "from source.constants import ORIGINAL_2_PRETTY_MODEL_NAMES\n",
    "\n",
    "print(f\"DATA_DIR: {DATA_DIR}\")\n",
    "print(f\"FEATURE_VECTORS_SAVE_DIR: {FEATURE_VECTORS_SAVE_DIR}\")\n",
    "print(f\"ANNOTATIONS_SAVE_DIR: {ANNOTATIONS_SAVE_DIR}\")\n",
    "\n",
    "print(\"ALL_TISSUE_TYPES:\", ALL_TISSUE_TYPES)\n",
    "print(\"ALL_CANCER_TYPES:\", ALL_CANCER_TYPES)\n",
    "print(\"ALL_EXTRACTOR_MODELS:\", ALL_EXTRACTOR_MODELS)\n",
    "print(\"ALL_IMG_NORMS:\", ALL_IMG_NORMS)\n",
    "print(\"ALL_DIMENSIONALITY_REDUCTION_METHODS:\",\n",
    "      ALL_DIMENSIONALITY_REDUCTION_METHODS)\n",
    "print(\"ALL_CLUSTERING_ALGORITHMS:\", ALL_CLUSTERING_ALGORITHMS)\n",
    "print(\"ALL_DISTANCE_METRICS:\", ALL_DISTANCE_METRICS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO: Set the cancer type, extractor name, and image normalization method\n",
    "\n",
    "# CANCER_TYPES = ['lung_aca', 'lung_n', 'lung_scc']\n",
    "# CANCER_TYPES = ['colon_aca', 'colon_n']\n",
    "CANCER_TYPES = ['lung_aca', 'lung_n', 'lung_scc', 'colon_aca', 'colon_n']\n",
    "\n",
    "# should be the same for every extractor for comparability\n",
    "IMG_NORM = 'imagenet'\n",
    "# IMG_NORM = 'resize_only'\n",
    "\n",
    "EXTRACTOR_NAME = 'imagenet_resnet18-last-layer' # worst\n",
    "# EXTRACTOR_NAME = 'owkin-phikon'               # medium\n",
    "# EXTRACTOR_NAME = 'UNI'                        # best\n",
    "\n",
    "# use validation set and early stopping, or just use train set until loss convergence\n",
    "use_val_set = False\n",
    "# decrease training size twice each time\n",
    "test_sizes = [0.2, 0.8, 0.95]\n",
    "# cross-validation splits\n",
    "n_splits = 10\n",
    "\n",
    "assert set(CANCER_TYPES).issubset(set(ALL_CANCER_TYPES))\n",
    "assert EXTRACTOR_NAME in ALL_EXTRACTOR_MODELS\n",
    "assert IMG_NORM in ALL_IMG_NORMS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_knn():\n",
    "    return KNeighborsClassifier(n_neighbors=1)\n",
    "\n",
    "\n",
    "def get_mlp(use_val_set):\n",
    "    if use_val_set:\n",
    "        return MLPClassifier(random_state=RANDOM_SEED, hidden_layer_sizes=(), max_iter=1000, early_stopping=True, validation_fraction=0.2)\n",
    "    else:\n",
    "        return MLPClassifier(random_state=RANDOM_SEED, hidden_layer_sizes=(), max_iter=1000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load the features, labels, and clusters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_sorted_cluster_labels(annotations_df, ids_2_imgpaths):\n",
    "    assert len(set(ids_2_imgpaths.values())) == len(list(ids_2_imgpaths.values())\n",
    "                                                    ), \"Can only reverse a bijective mapping, duplicate values found.\"\n",
    "    img_paths_2_int_ids = {v: int(k) for k, v in ids_2_imgpaths.items()}\n",
    "\n",
    "    cluster_labels = annotations_df['cluster_label'].values.astype(int)\n",
    "    img_paths = annotations_df['img_path'].values\n",
    "    img_paths_2_cluster_labels = dict(zip(img_paths, cluster_labels))\n",
    "\n",
    "    assert set(img_paths_2_int_ids.keys()) == set(img_paths_2_cluster_labels.keys(\n",
    "    )), \"The img_paths in the annotations_df must be the same as the img_paths in the ids_2_imgpaths dictionary.\"\n",
    "\n",
    "    intids_2_cluster_labels = {\n",
    "        img_paths_2_int_ids[img_path]: img_paths_2_cluster_labels[img_path]\n",
    "        for img_path in img_paths_2_cluster_labels.keys()\n",
    "    }\n",
    "\n",
    "    cluster_labels_sorted_by_intids = [intids_2_cluster_labels[i] for i in sorted(intids_2_cluster_labels.keys())]\n",
    "    cluster_labels_sorted_by_intids = np.array(cluster_labels_sorted_by_intids)\n",
    "\n",
    "    assert all(np.unique(cluster_labels_sorted_by_intids) == np.arange(cluster_labels_sorted_by_intids.max()+1))\n",
    "\n",
    "    return cluster_labels_sorted_by_intids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = []\n",
    "y = []\n",
    "all_cluster_labels = []\n",
    "\n",
    "current_max_plus_one = 0\n",
    "for i, cancer_type in enumerate(CANCER_TYPES):\n",
    "    feature_path = f\"{FEATURE_VECTORS_SAVE_DIR}/{cancer_type}/{EXTRACTOR_NAME}/{IMG_NORM}/features.npy\"\n",
    "    with open(f\"{FEATURE_VECTORS_SAVE_DIR}/{cancer_type}/{EXTRACTOR_NAME}/{IMG_NORM}/ids_2_img_paths.json\", 'r') as f:\n",
    "        ids_2_imgpaths = json.load(f)\n",
    "    annotations_path = f\"{ANNOTATIONS_SAVE_DIR}/{cancer_type}/UNI/resize_only/final_clusters.csv\"\n",
    "    features_arr = np.load(feature_path)\n",
    "    labels = np.full(features_arr.shape[0], fill_value=i)\n",
    "    X.append(features_arr)\n",
    "    y.append(labels)\n",
    "\n",
    "    annotations_df = pd.read_csv(annotations_path)\n",
    "    sorted_cluster_labels = get_sorted_cluster_labels(annotations_df, ids_2_imgpaths)\n",
    "    print(f\"cancer type {cancer_type} has {len(np.unique(sorted_cluster_labels))} clusters\")\n",
    "    cancer_cluster_labels = current_max_plus_one + sorted_cluster_labels\n",
    "    all_cluster_labels.append(cancer_cluster_labels)\n",
    "\n",
    "    # update the current_max_plus_one\n",
    "    current_max_plus_one += sorted_cluster_labels.max() + 1\n",
    "\n",
    "X = np.concatenate(X)\n",
    "y = np.concatenate(y)\n",
    "all_cluster_labels = np.concatenate(all_cluster_labels)\n",
    "\n",
    "# check that currnet_max_plus_one works as expected - i.e. no overlap between the cluster labels of different cancer types\n",
    "assert len(set(all_cluster_labels[:5000]).intersection(\n",
    "    set(all_cluster_labels[5000:]))) == 0\n",
    "assert len(set(all_cluster_labels[:10000]).intersection(\n",
    "    set(all_cluster_labels[10000:]))) == 0\n",
    "print(\"Total groups:\", len(set(all_cluster_labels)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Classes Visualisation in Feature Space"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dim_reduction = PCA(n_components=2, random_state=RANDOM_SEED)\n",
    "# dim_reduction = UMAP(n_components=2)\n",
    "\n",
    "np.random.seed(RANDOM_SEED)\n",
    "random_order = np.random.permutation(X.shape[0])\n",
    "X_random_order = X[random_order]\n",
    "y_random_order = y[random_order]\n",
    "all_cluster_labels_random_order = all_cluster_labels[random_order]\n",
    "\n",
    "X_random_order_reduced = dim_reduction.fit_transform(X_random_order)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot X_umap with y as the color\n",
    "\n",
    "plt.figure(figsize=(5, 5))\n",
    "sns.scatterplot(\n",
    "    x=X_random_order_reduced[:, 0],\n",
    "    y=X_random_order_reduced[:, 1],\n",
    "    hue=[CANCER_TYPES[i] for i in y_random_order],\n",
    "    palette='tab20',\n",
    "    s=10\n",
    ")\n",
    "# plt.title(f\"Feature vectors: {IMG_NORM} normalisation + {EXTRACTOR_NAME} extraction\")\n",
    "plt.title(ORIGINAL_2_PRETTY_MODEL_NAMES[EXTRACTOR_NAME])\n",
    "plt.legend(title='Cancer type',)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Precision@1 and Precision@5 for the original dataset restricted to tissue type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from source.eval_utils import precision_at_1, precision_at_k\n",
    "\n",
    "# y_connectivity_matrix = (y[:, np.newaxis] == y[np.newaxis, :]).astype(int)\n",
    "# plt.imshow(y_connectivity_matrix, cmap='gray')\n",
    "# plt.show()\n",
    "# print(\"Precision@1\", precision_at_1(X, y_connectivity_matrix))\n",
    "# print(\"Precision@5\", precision_at_k(X, y_connectivity_matrix, k=5))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Precision@1 and Precision@5 for the cleaned dataset restricted to tissue type (take 1 example from each cluster)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_cluster_labels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Simple classification of the original dataset, no cross-validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "knn = get_knn()\n",
    "mlp = get_mlp(use_val_set)\n",
    "\n",
    "for test_size in test_sizes:\n",
    "    X_train, X_test, y_train, y_test = train_test_split(\n",
    "        X, y, test_size=test_size, stratify=y, random_state=RANDOM_SEED)\n",
    "\n",
    "    # Fit and score KNN\n",
    "    knn.fit(X_train, y_train)\n",
    "    y_pred_knn = knn.predict(X_test)\n",
    "    score_knn = knn.score(X_test, y_test)\n",
    "    print(f\"KNN - Test size: {test_size:.4}, score: {score_knn:.4}\")\n",
    "\n",
    "    # Fit and score MLP\n",
    "    mlp.fit(X_train, y_train)\n",
    "    y_pred_mlp = mlp.predict(X_test)\n",
    "    score_mlp = mlp.score(X_test, y_test)\n",
    "    print(f\"MLP - Test size: {test_size}, score: {score_mlp:.4}\")\n",
    "\n",
    "    print()\n",
    "\n",
    "# print(\"mlp.out_activation_:\", mlp.out_activation_)\n",
    "# print(\"len(mlp.coefs_):\", len(mlp.coefs_))\n",
    "# for i in range(len(mlp.coefs_)):\n",
    "#     print(f\"\\tmlp.coefs_[{i}].shape:\", mlp.coefs_[i].shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## N-samples Cross-validation of the original LC25000 dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "knn = get_knn()\n",
    "mlp = get_mlp(use_val_set)\n",
    "\n",
    "original_test_size_2_scores = {}\n",
    "\n",
    "for test_size in test_sizes:\n",
    "    knn_scores = []\n",
    "    mlp_scores = []\n",
    "\n",
    "    sss = StratifiedShuffleSplit(\n",
    "        n_splits=n_splits, test_size=test_size, random_state=RANDOM_SEED)\n",
    "\n",
    "    for train_index, test_index in sss.split(X, y):\n",
    "        X_train, X_test = X[train_index], X[test_index]\n",
    "        y_train, y_test = y[train_index], y[test_index]\n",
    "\n",
    "        # Evaluate KNN\n",
    "        knn.fit(X_train, y_train)\n",
    "        score_knn = knn.score(X_test, y_test)\n",
    "        knn_scores.append(score_knn)\n",
    "\n",
    "        # Evaluate MLP\n",
    "        mlp.fit(X_train, y_train)\n",
    "        score_mlp = mlp.score(X_test, y_test)\n",
    "        mlp_scores.append(score_mlp)\n",
    "\n",
    "    original_test_size_2_scores[test_size] = {\n",
    "        'knn': knn_scores,\n",
    "        'mlp': mlp_scores,\n",
    "    }\n",
    "\n",
    "    print(f\"KNN - Test size: {test_size:.4}, mean score: {np.mean(knn_scores):.4}, std: {np.std(knn_scores):.4}\")\n",
    "    print(f\"MLP - Test size: {test_size}, mean score: {np.mean(mlp_scores):.4}, std: {np.std(mlp_scores):.4}\")\n",
    "    print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## N-samples Grouped Cross-validation on LC25000-clean grouped by clusters (prototypes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "knn = get_knn()\n",
    "mlp = get_mlp(use_val_set)\n",
    "\n",
    "clean_test_size_2_scores = {}\n",
    "\n",
    "for test_size in test_sizes:\n",
    "    knn_scores = []\n",
    "    mlp_scores = []\n",
    "\n",
    "    gss = GroupShuffleSplit(\n",
    "        n_splits=n_splits, test_size=test_size, random_state=RANDOM_SEED)\n",
    "\n",
    "    for train_index, test_index in gss.split(X, y, groups=all_cluster_labels):\n",
    "        X_train, X_test = X[train_index], X[test_index]\n",
    "        y_train, y_test = y[train_index], y[test_index]\n",
    "        all_cluster_labels_train, all_cluster_labels_test = all_cluster_labels[train_index], all_cluster_labels[test_index]\n",
    "        assert len(set(all_cluster_labels_train).intersection(all_cluster_labels_test)) == 0, \"Overlap between the cluster labels of the training and test sets\"\n",
    "\n",
    "        # Evaluate KNN\n",
    "        knn.fit(X_train, y_train)\n",
    "        score_knn = knn.score(X_test, y_test)\n",
    "        knn_scores.append(score_knn)\n",
    "\n",
    "        # Evaluate MLP\n",
    "        mlp.fit(X_train, y_train)\n",
    "        score_mlp = mlp.score(X_test, y_test)\n",
    "        mlp_scores.append(score_mlp)\n",
    "\n",
    "    clean_test_size_2_scores[test_size] = {\n",
    "        'knn': knn_scores,\n",
    "        'mlp': mlp_scores,\n",
    "    }\n",
    "\n",
    "    print(f\"KNN - Test size: {test_size:.4}, mean score: {np.mean(knn_scores):.4}, std: {np.std(knn_scores):.4}\")\n",
    "    print(f\"MLP - Test size: {test_size}, mean score: {np.mean(mlp_scores):.4}, std: {np.std(mlp_scores):.4}\")\n",
    "    print()\n",
    "\n",
    "    # break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Combined Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_test_size_2_scores = {\n",
    "    'original': original_test_size_2_scores,\n",
    "    'clean': clean_test_size_2_scores,\n",
    "}\n",
    "all_test_size_2_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_mean_scores(test_size_2_scores):\n",
    "    return {\n",
    "        test_size: {\n",
    "            cls: f\"{np.mean(scores[cls]).round(3)} $\\pm$ {np.std(scores[cls]).round(3)}\"\n",
    "            for cls in scores.keys()\n",
    "        }\n",
    "        for test_size, scores in test_size_2_scores.items()\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Image Normalization\", IMG_NORM)\n",
    "print(\"Extractor Name:\", EXTRACTOR_NAME)\n",
    "print(\"Using validation set:\", use_val_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.DataFrame(get_mean_scores(original_test_size_2_scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.DataFrame(get_mean_scores(clean_test_size_2_scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "lc25k-cleaning",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}