adv3_evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "sys.path.insert(0,'/code/')\n",
    "\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import torch\n",
    "import torchvision.transforms as T\n",
    "from torchvision.datasets import ImageNet, CIFAR100, CIFAR10, STL10, ImageFolder\n",
    "import pandas as pd\n",
    "from torchvision.utils import save_image\n",
    "\n",
    "import multiprocessing as mp\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "from datasets.imagenette import ImageNette\n",
    "from metrics.hamming_distance import hamming_distance\n",
    "from adv3_robustness_check.py import get_rotation_angles, get_translation_tuples\n",
    "from utils.transforms import Rotate, Translate, ChangeSaturation, ChangeHue, ChangeContrast, ChangeBrightness, \\\n",
    "    JpegCompression, HorizontalFlipping, BlackBorder, CenterCrop, VerticalFlipping\n",
    "from scipy.interpolate import griddata\n",
    "\n",
    "sns.set_style(\"ticks\", {'axes.grid': True})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "DATASET = 'imagenet_train'\n",
    "MAX_ROT_ANGLE = 64\n",
    "ROT_LOG_BASE = 2\n",
    "ROT_STEPS = 7\n",
    "MAX_TRANS = 64\n",
    "TRANS_LOG_BASE = 2\n",
    "TRANS_STEPS = 7\n",
    "\n",
    "HASH_DIR = f'dataset_hashes/{DATASET}'\n",
    "PLOT_DIR = f'robustness_plots/{DATASET}'\n",
    "if not os.path.exists(PLOT_DIR):\n",
    "    os.makedirs(PLOT_DIR)\n",
    "EXAMPLE_IMG_CLASS = 'flamingo'\n",
    "EXAMPLE_IMG_IDX = 2\n",
    "EXAMPLE_IMG_DIR = f'robustness_plots/{DATASET}/example_imgs'\n",
    "if not os.path.exists(EXAMPLE_IMG_DIR):\n",
    "    os.makedirs(EXAMPLE_IMG_DIR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def get_dataset(dataset_name: str):\n",
    "    transforms = T.Compose([\n",
    "        T.Resize((360, 360)),\n",
    "        T.ToTensor()\n",
    "    ])\n",
    "    if dataset_name.lower() == 'stl10':\n",
    "        dataset = STL10(root='data', split='train', download=True, transform=transforms)\n",
    "    elif dataset_name.lower() == 'imagenette':\n",
    "        dataset = ImageNette(root='data', train=True, download=True, transform=transforms)\n",
    "    elif dataset_name.lower() == 'imagenette_val':\n",
    "        dataset = ImageNette(root='data', train=False, download=True, transform=transforms)\n",
    "    elif dataset_name.lower() == 'cifar10':\n",
    "        dataset = CIFAR10(root='data', train=True, download=True, transform=transforms)\n",
    "    elif dataset_name.lower() == 'cifar100':\n",
    "        dataset = CIFAR100(root='data', train=True, download=True, transform=transforms)\n",
    "    elif dataset_name.lower() == 'imagenet_test':\n",
    "        dataset = ImageFolder(root='data/ILSVRC2012_test', transform=transforms)\n",
    "    elif dataset_name.lower() == 'imagenet_train':\n",
    "        dataset = ImageNet(root='data/ILSVRC2012', split='train', transform=transforms)\n",
    "    elif dataset_name.lower() == 'imagenet_val':\n",
    "        dataset = ImageNet(root='data/ILSVRC2012', split='val', transform=transforms)\n",
    "    else:\n",
    "        raise RuntimeError(f'Dataset with name {dataset_name} was not found.')\n",
    "\n",
    "    return dataset\n",
    "    \n",
    "\n",
    "def get_hashes_from_csv(file_path):\n",
    "    df = pd.read_csv(file_path)\n",
    "    bin_hashes = []\n",
    "    for bit_string in df['hash_bin']:\n",
    "        bin_hashes.append(list(bit_string))\n",
    "    bin_hashes = np.array(bin_hashes, dtype=int)\n",
    "\n",
    "    return bin_hashes\n",
    "\n",
    "\n",
    "def plot_corr_mat(df, min_corr_val=None, max_corr_val=None):\n",
    "    corr_mat = df.corr()\n",
    "    corr_values = np.sort(np.unique(corr_mat.values.flatten()))[:-1]\n",
    "    print(f'Min Corr. Value: {corr_values.min()}')\n",
    "    print(f'Max Corr. Value: {corr_values.max()}')\n",
    "    sns.heatmap(\n",
    "        corr_mat,\n",
    "        cmap=sns.diverging_palette(250, 15, as_cmap=True),\n",
    "        vmax=max_corr_val if max_corr_val else corr_values.max(),\n",
    "        center=0,\n",
    "        vmin=min_corr_val if min_corr_val else corr_values.min()\n",
    "    )\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "def get_augmented_hashes_and_hamming_dist(filenames, augment_values, original_hashes, return_bin_hashes=False, num_processes=2):\n",
    "    return_hash_dict = {}\n",
    "    return_hamming_dict = {}\n",
    "\n",
    "    with tqdm(total=len(filenames)) as pbar:\n",
    "        with mp.Pool(num_processes) as pool:\n",
    "            for i, bin_hashes in enumerate(pool.imap(get_hashes_from_csv, filenames)):\n",
    "                return_hash_dict[augment_values[i]] = bin_hashes\n",
    "                return_hamming_dict[augment_values[i]] = hamming_distance(torch.tensor(bin_hashes), original_hashes).numpy()\n",
    "                pbar.update()\n",
    "\n",
    "    if return_bin_hashes:\n",
    "        return return_hash_dict, return_hamming_dict\n",
    "\n",
    "    return return_hamming_dict\n",
    "\n",
    "\n",
    "def print_mean_and_std_for_keys(given_dict):\n",
    "    for key in given_dict.keys():\n",
    "        print(f'Mean Hamming Distance for {key}: {given_dict[key].mean()}')\n",
    "        print(f'Standard Deviation Hamming Distance for {key}: {given_dict[key].std()}')\n",
    "        \n",
    "\n",
    "def check_for_non_altered_hashes(given_dict, key, return_indices=False):\n",
    "    non_altered_hash_indices = (given_dict[key] == 0).nonzero()[0]\n",
    "    print(f'{len(non_altered_hash_indices)} ({float(len(non_altered_hash_indices)) / float(len(given_dict[key])):5.4f}) hashes were not altered when using key {key}')\n",
    "\n",
    "    if return_indices:\n",
    "        return non_altered_hash_indices\n",
    "\n",
    "\n",
    "def plot_example_img_with_transformation(dataset, img_idx, img_class=None, transformation=None, file_path=None):\n",
    "    if transformation is not None:\n",
    "        transforms = T.Compose([\n",
    "            T.Resize((360, 360)),\n",
    "            T.ToTensor(),\n",
    "            transformation\n",
    "        ])\n",
    "        dataset.transform = transforms\n",
    "    base_index = 0\n",
    "    if img_class is not None:\n",
    "        base_index = np.argmax(np.array(dataset.targets) == dataset.class_to_idx[img_class])\n",
    "    img_idx += base_index\n",
    "    print(f'{dataset.classes[dataset[img_idx][1]][0]}:')\n",
    "    image = dataset[img_idx][0].permute(1,2,0).numpy()\n",
    "    plt.imshow(image)\n",
    "    plt.show()\n",
    "\n",
    "    if file_path is not None:\n",
    "        save_image(dataset[img_idx][0], file_path)\n",
    "\n",
    "    dataset.transform = T.Compose([\n",
    "            T.Resize((360, 360)),\n",
    "            T.ToTensor()\n",
    "    ])\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = get_dataset(DATASET)\n",
    "\n",
    "plot_example_img_with_transformation(dataset, EXAMPLE_IMG_IDX, img_class=EXAMPLE_IMG_CLASS, file_path=os.path.join(EXAMPLE_IMG_DIR, 'original.png'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "bin_hashes_orig = torch.tensor(get_hashes_from_csv(os.path.join(HASH_DIR, f'{DATASET}_original.csv')))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# Rotation Robustness"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# get the rotation angles\n",
    "angles = get_rotation_angles(MAX_ROT_ANGLE, ROT_LOG_BASE, ROT_STEPS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "rot_hamming = get_augmented_hashes_and_hamming_dist(\n",
    "    [os.path.join(HASH_DIR, 'rotation', f'{DATASET}_rotation_{angle}.csv') for angle in angles],\n",
    "    angles,\n",
    "    bin_hashes_orig,\n",
    "    num_processes=15\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "df = pd.DataFrame(rot_hamming)\n",
    "df = df.melt()\n",
    "df = df.rename(columns={'variable': 'Angle', 'value': 'Normalized Hamming Distance'})\n",
    "sns.set(rc={'figure.figsize':(8,5)})\n",
    "sns.set_style(\"whitegrid\")\n",
    "fig = sns.lineplot(data=df, marker='o', markersize=8, x=\"Angle\", y=\"Normalized Hamming Distance\", ci='sd')\n",
    "fig.set_xlabel(\"Angle\", fontsize = 21, fontweight='bold')\n",
    "fig.set_ylabel(\"Norm. Hamming Distance\", fontsize = 21, fontweight='bold')\n",
    "plt.xticks([-64, -32, -16,-4, 4, 16, 32, 64])\n",
    "plt.yticks([0.0, 0.1, 0.2, 0.3, 0.4, 0.5])\n",
    "fig.set(ylim=(-0.025, 0.55))\n",
    "fig.set_yticklabels(fig.get_yticks(), size = 18, fontweight='bold')\n",
    "fig.set_xticklabels(fig.get_xticks(), size = 18, fontweight='bold')\n",
    "plt.tight_layout() \n",
    "plt.savefig(f'{PLOT_DIR}/{DATASET}_rotation_robustness.pdf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "check_for_non_altered_hashes(rot_hamming, angles[6])\n",
    "check_for_non_altered_hashes(rot_hamming, angles[8])\n",
    "check_for_non_altered_hashes(rot_hamming, angles[4])\n",
    "check_for_non_altered_hashes(rot_hamming, angles[10])\n",
    "check_for_non_altered_hashes(rot_hamming, angles[0])\n",
    "\n",
    "# plot the single image that does not change\n",
    "idx = check_for_non_altered_hashes(rot_hamming, angles[14], return_indices=True)[0]\n",
    "print(f'{dataset.classes[dataset[idx][1]][0]}:')\n",
    "plt.imshow(dataset[idx][0].permute(1,2,0).numpy())\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "print_mean_and_std_for_keys(rot_hamming)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_example_img_with_transformation(dataset, EXAMPLE_IMG_IDX, img_class=EXAMPLE_IMG_CLASS, transformation=Rotate(angles[-1]), file_path=os.path.join(EXAMPLE_IMG_DIR, 'rotation.png'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# Translation Robustness"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "translations = get_translation_tuples(MAX_TRANS, TRANS_LOG_BASE, TRANS_STEPS)\n",
    "trans_hamming = get_augmented_hashes_and_hamming_dist(\n",
    "    [os.path.join(HASH_DIR, 'translation', f'{DATASET}_translation_{trans[0]}_{trans[1]}.csv') for trans in translations],\n",
    "    translations,\n",
    "    bin_hashes_orig,\n",
    "    num_processes=24\n",
    ")\n",
    "# add the hamming distance without any translation\n",
    "trans_hamming[(0, 0)] = np.zeros_like(trans_hamming[(1, 1)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "df = pd.DataFrame(trans_hamming)\n",
    "df = pd.DataFrame(df.mean()).transpose()\n",
    "df = df.melt()\n",
    "sns.set_style(\"ticks\", {'axes.grid': False})\n",
    "x_values = df['variable_0'].to_numpy()\n",
    "y_values = df['variable_1'].to_numpy()\n",
    "z_values = df['value'].to_numpy()\n",
    "xi = np.linspace(0, x_values.max(), 67, endpoint=True)[None, :]\n",
    "yi = np.linspace(0, y_values.max(), 67, endpoint=True)[:, None]\n",
    "scipy_linear = griddata((x_values, y_values), z_values, (xi, yi), rescale=True)\n",
    "sns.set(rc={'figure.figsize':(8,6)})\n",
    "cmap = sns.cubehelix_palette(as_cmap=True)\n",
    "labels = [i if i in [0, 4, 8, 16, 32, 64] else None for i in range(66)]\n",
    "ax = sns.heatmap(scipy_linear, cmap=cmap, vmax=0.25, vmin=0,\n",
    "    xticklabels=labels, yticklabels=labels, rasterized=True)\n",
    "ax.tick_params(left=False, bottom=False)\n",
    "cbar = ax.collections[0].colorbar\n",
    "cbar.set_label(label='Norm. Hamming Distance', weight='bold')\n",
    "cbar.set_ticks([0, 0.05, 0.1, 0.15, 0.2, 0.25])\n",
    "for label in cbar.ax.get_yticklabels():\n",
    "    label.set_fontsize(18)\n",
    "    label.set_fontweight('bold')\n",
    "ax.scatter(x_values + 0.5, y_values + 0.5, s=15, c='None', edgecolors='yellow')\n",
    "ax.invert_yaxis()\n",
    "ax.figure.axes[-1].yaxis.label.set_size(18)\n",
    "ax.figure.axes[-1].tick_params(labelsize=18)\n",
    "\n",
    "ax.set_xticklabels(ax.get_xticklabels(), size=18, fontweight='bold') \n",
    "ax.set_yticklabels(ax.get_yticklabels(), size=18, fontweight='bold', rotation=0) \n",
    "\n",
    "plt.xlabel('Horizontal Translation', fontsize=21, fontweight='bold')\n",
    "plt.ylabel('Vertical Translation', fontsize=21, fontweight='bold')\n",
    "plt.tight_layout() \n",
    "plt.savefig(f'{PLOT_DIR}/{DATASET}_translation_robustness.pdf')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "print_mean_and_std_for_keys(trans_hamming)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot two images that do not change\n",
    "idx1, idx2 = check_for_non_altered_hashes(trans_hamming, translations[-1], return_indices=True)\n",
    "print(f'{dataset.classes[dataset[idx1][1]][0]}:')\n",
    "plt.imshow(dataset[idx1][0].permute(1,2,0))\n",
    "plt.show()\n",
    "print(f'{dataset.classes[dataset[idx2][1]][0]}:')\n",
    "plt.imshow(dataset[idx2][0].permute(1,2,0))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_example_img_with_transformation(dataset, EXAMPLE_IMG_IDX, img_class=EXAMPLE_IMG_CLASS, transformation=Translate(translations[-1]), file_path=os.path.join(EXAMPLE_IMG_DIR, 'translation.png'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# Hue Robustness"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "hues = list(range(-180, 180, 30))\n",
    "hue_hamming = get_augmented_hashes_and_hamming_dist(\n",
    "    [os.path.join(HASH_DIR, 'hue', f'{DATASET}_hue_{hue}.csv') for hue in hues],\n",
    "    hues,\n",
    "    bin_hashes_orig,\n",
    "    num_processes=12\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "hue_hamming[180] = hue_hamming[-180]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(hue_hamming)\n",
    "df = df.melt()\n",
    "df = df.rename(columns={'variable': 'HSV Angle', 'value': 'Normalized Hamming Distance'})\n",
    "sns.set(rc={'figure.figsize':(8,5)})\n",
    "sns.set_style(\"whitegrid\")\n",
    "fig = sns.lineplot(data=df, marker='o', markersize=8, x=\"HSV Angle\", y=\"Normalized Hamming Distance\", ci='sd')\n",
    "plt.xticks([-180, -120, -60, 0, 60, 120, 180])\n",
    "plt.yticks([0.0, 0.05, 0.1, 0.15, 0.2, 0.25])\n",
    "fig.set_xlabel(\"Hue Angle\", fontsize = 21, fontweight='bold')\n",
    "fig.set_ylabel(\"Norm. Hamming Distance\", fontsize = 21, fontweight='bold')\n",
    "fig.set_yticklabels(fig.get_yticks(), size = 18, fontweight='bold')\n",
    "fig.set_xticklabels(fig.get_xticks(), size = 18, fontweight='bold')\n",
    "yticklabels = []\n",
    "for item in fig.get_yticklabels():\n",
    "    fmt = '{:0.2f}'\n",
    "    item.set_text(fmt.format(float(item.get_text())))\n",
    "    yticklabels += [item]\n",
    "fig.set_yticklabels(yticklabels)\n",
    "\n",
    "plt.tight_layout() \n",
    "plt.savefig(f'{PLOT_DIR}/{DATASET}_hue_robustness.pdf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "print_mean_and_std_for_keys(hue_hamming)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "check_for_non_altered_hashes(hue_hamming, hues[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_example_img_with_transformation(dataset, EXAMPLE_IMG_IDX, img_class=EXAMPLE_IMG_CLASS, transformation=ChangeHue(hues[0]), file_path=os.path.join(EXAMPLE_IMG_DIR, 'hue.png'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# Brightness Robustness"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "brightness_values = list(np.linspace(0, 2, 9, endpoint=True))\n",
    "brightness_hamming = get_augmented_hashes_and_hamming_dist(\n",
    "    [os.path.join(HASH_DIR, 'brightness', f'{DATASET}_brightness_{brightness}.csv') for brightness in brightness_values],\n",
    "    brightness_values,\n",
    "    bin_hashes_orig,\n",
    "    num_processes=9\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(brightness_hamming)\n",
    "df = df.melt()\n",
    "df = df.rename(columns={'variable': 'Brightness Factor', 'value': 'Norm. Hamming Distance'})\n",
    "sns.set(rc={'figure.figsize':(8,5)})\n",
    "sns.set_style(\"whitegrid\")\n",
    "fig = sns.lineplot(data=df, marker='o', markersize=8, x=\"Brightness Factor\", y=\"Norm. Hamming Distance\", ci='sd')\n",
    "plt.yticks([0.0, 0.1, 0.2, 0.3, 0.4, 0.5])\n",
    "fig.set(ylabel=None)\n",
    "fig.set(ylim=(-0.025, 0.55))\n",
    "fig.set_xlabel(fig.get_xlabel(), fontsize = 21, fontweight='bold')\n",
    "#fig.set_ylabel(fig.get_ylabel(), fontsize = 21, fontweight='bold')\n",
    "fig.set_yticklabels([])\n",
    "fig.set_xticklabels(fig.get_xticks(), size = 18, fontweight='bold')\n",
    "plt.tight_layout() \n",
    "plt.savefig(f'{PLOT_DIR}/{DATASET}_brightness_robustness.pdf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "print_mean_and_std_for_keys(brightness_hamming)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "check_for_non_altered_hashes(brightness_hamming, brightness_values[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_example_img_with_transformation(dataset, EXAMPLE_IMG_IDX, img_class=EXAMPLE_IMG_CLASS, transformation=ChangeBrightness(brightness_values[-1]), file_path=os.path.join(EXAMPLE_IMG_DIR, 'brightness.png'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot two images that do not change\n",
    "idx1, idx2 = check_for_non_altered_hashes(brightness_hamming, brightness_values[-1], return_indices=True)[:2]\n",
    "print(f'{dataset.classes[dataset[idx1][1]][0]}:')\n",
    "plt.imshow(dataset[idx1][0].permute(1,2,0))\n",
    "plt.show()\n",
    "print(f'{dataset.classes[dataset[idx2][1]][0]}:')\n",
    "plt.imshow(dataset[idx2][0].permute(1,2,0))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# Contrast Robustness"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "contrast_values = list(np.linspace(0, 2, 9, endpoint=True))\n",
    "contrast_hamming = get_augmented_hashes_and_hamming_dist(\n",
    "    [os.path.join(HASH_DIR, 'contrast', f'{DATASET}_contrast_{contrast}.csv') for contrast in contrast_values],\n",
    "    contrast_values,\n",
    "    bin_hashes_orig,\n",
    "    num_processes=9\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(contrast_hamming)\n",
    "df = df.melt()\n",
    "df = df.rename(columns={'variable': 'Contrast Factor', 'value': 'Norm. Hamming Distance'})\n",
    "sns.set(rc={'figure.figsize':(8,5)})\n",
    "sns.set_style(\"whitegrid\")\n",
    "fig = sns.lineplot(data=df, marker='o', markersize=8, x=\"Contrast Factor\", y=\"Norm. Hamming Distance\", ci='sd')\n",
    "plt.yticks([0.0, 0.1, 0.2, 0.3, 0.4, 0.5])\n",
    "fig.set(ylabel=None)\n",
    "fig.set(ylim=(-0.025, 0.55))\n",
    "fig.set_xlabel(fig.get_xlabel(), fontsize = 21, fontweight='bold')\n",
    "fig.set_ylabel(fig.get_ylabel(), fontsize = 21, fontweight='bold')\n",
    "fig.set_yticklabels([])\n",
    "fig.set_xticklabels(fig.get_xticks(), size = 18, fontweight='bold')\n",
    "\n",
    "plt.tight_layout() \n",
    "plt.savefig(f'{PLOT_DIR}/{DATASET}_contrast_robustness.pdf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "check_for_non_altered_hashes(contrast_hamming, contrast_values[-1])\n",
    "# plot two images that do not change\n",
    "idx1, idx2 = check_for_non_altered_hashes(contrast_hamming, contrast_values[-1], return_indices=True)[:2]\n",
    "print(f'{dataset.classes[dataset[idx1][1]][0]}:')\n",
    "plt.imshow(dataset[idx1][0].permute(1,2,0))\n",
    "plt.show()\n",
    "print(f'{dataset.classes[dataset[idx2][1]][0]}:')\n",
    "plt.imshow(dataset[idx2][0].permute(1,2,0))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print_mean_and_std_for_keys(contrast_hamming)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_example_img_with_transformation(dataset, EXAMPLE_IMG_IDX, img_class=EXAMPLE_IMG_CLASS, transformation=ChangeContrast(contrast_values[-1]), file_path=os.path.join(EXAMPLE_IMG_DIR, 'contrast.png'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# Saturation Robustness"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "saturation_values = list(np.linspace(0, 2, 9, endpoint=True))\n",
    "saturation_hamming = get_augmented_hashes_and_hamming_dist(\n",
    "    [os.path.join(HASH_DIR, 'saturation', f'{DATASET}_saturation_{saturation}.csv') for saturation in saturation_values],\n",
    "    saturation_values,\n",
    "    bin_hashes_orig,\n",
    "    num_processes=9\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(saturation_hamming)\n",
    "df = df.melt()\n",
    "df = df.rename(columns={'variable': 'Saturation Factor', 'value': 'Norm. Hamming Distance'})\n",
    "sns.set(rc={'figure.figsize':(8,5)})\n",
    "sns.set_style(\"whitegrid\")\n",
    "fig = sns.lineplot(data=df, marker='o', markersize=8, x=\"Saturation Factor\", y=\"Norm. Hamming Distance\", ci='sd')\n",
    "plt.yticks([0.00, 0.01, 0.02, 0.03, 0.04, 0.05, 0.06])\n",
    "fig.set_xlabel(fig.get_xlabel(), fontsize = 21, fontweight='bold')\n",
    "fig.set_ylabel(fig.get_ylabel(), fontsize = 21, fontweight='bold')\n",
    "fig.set_yticklabels(fig.get_yticks(), size = 18, fontweight='bold')\n",
    "fig.set_xticklabels(fig.get_xticks(), size = 18, fontweight='bold')\n",
    "yticklabels = []\n",
    "for item in fig.get_yticklabels():\n",
    "    fmt = '{:0.2f}'\n",
    "    item.set_text(fmt.format(float(item.get_text())))\n",
    "    yticklabels += [item]\n",
    "fig.set_yticklabels(yticklabels)\n",
    "\n",
    "plt.tight_layout() \n",
    "plt.savefig(f'{PLOT_DIR}/{DATASET}_saturation_robustness.pdf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print_mean_and_std_for_keys(saturation_hamming)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "check_for_non_altered_hashes(saturation_hamming, saturation_values[-1])\n",
    "# plot two images that do not change\n",
    "idx1, idx2 = check_for_non_altered_hashes(saturation_hamming, saturation_values[-1], return_indices=True)[:2]\n",
    "print(f'{dataset.classes[dataset[idx1][1]][0]}:')\n",
    "plt.imshow(dataset[idx1][0].permute(1,2,0))\n",
    "plt.show()\n",
    "print(f'{dataset.classes[dataset[idx2][1]][0]}:')\n",
    "plt.imshow(dataset[idx2][0].permute(1,2,0))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_example_img_with_transformation(dataset, EXAMPLE_IMG_IDX, img_class=EXAMPLE_IMG_CLASS, transformation=ChangeSaturation(saturation_values[-1]), file_path=os.path.join(EXAMPLE_IMG_DIR, 'saturation.png'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# Compression Robustness"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "compression_values = [100] + list(\n",
    "    (100 - np.ceil(np.logspace(0, np.log(100) / np.log(1.5), 10, endpoint=True, base=1.5))).clip(0, 100)\n",
    ")\n",
    "compression_hamming = get_augmented_hashes_and_hamming_dist(\n",
    "    [os.path.join(HASH_DIR, 'compression', f'{DATASET}_compression_{compression}.csv') for compression in compression_values],\n",
    "    compression_values,\n",
    "    bin_hashes_orig,\n",
    "    num_processes=11\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(compression_hamming)\n",
    "df = df.melt()\n",
    "df = df.rename(columns={'variable': 'Compression Value', 'value': 'Norm. Hamming Distance'})\n",
    "sns.set(rc={'figure.figsize':(8,5)})\n",
    "sns.set_style(\"whitegrid\")\n",
    "fig = sns.lineplot(data=df, marker='o', markersize=8, x=\"Compression Value\", y=\"Norm. Hamming Distance\", ci='sd')\n",
    "plt.xticks([0, 40, 64, 78, 87, 92, 100])\n",
    "plt.yticks([0.0, 0.1, 0.2, 0.3, 0.4, 0.5])\n",
    "fig.set(ylabel=None)\n",
    "fig.set(ylim=(-0.025, 0.55))\n",
    "fig.set_xlabel(fig.get_xlabel(), fontsize = 21, fontweight='bold')\n",
    "fig.set_ylabel(None)\n",
    "fig.set_yticklabels([])\n",
    "fig.set_xticklabels(fig.get_xticks(), size = 18, fontweight='bold')\n",
    "\n",
    "plt.tight_layout() \n",
    "plt.savefig(f'{PLOT_DIR}/{DATASET}_compression_robustness.pdf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print_mean_and_std_for_keys(compression_hamming)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "check_for_non_altered_hashes(compression_hamming, compression_values[5])\n",
    "# plot two images that do not change\n",
    "idx1, idx2 = check_for_non_altered_hashes(compression_hamming, compression_values[5], return_indices=True)[-2:]\n",
    "print(f'{dataset.classes[dataset[idx1][1]][0]}:')\n",
    "plt.imshow(dataset[idx1][0].permute(1,2,0))\n",
    "plt.show()\n",
    "print(f'{dataset.classes[dataset[idx2][1]][0]}:')\n",
    "plt.imshow(dataset[idx2][0].permute(1,2,0))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_example_img_with_transformation(dataset, EXAMPLE_IMG_IDX, img_class=EXAMPLE_IMG_CLASS, transformation=JpegCompression(compression_values[5]), file_path=os.path.join(EXAMPLE_IMG_DIR, 'compression.png'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# Crop Robustness"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "crop_values = list(\n",
    "    filter(\n",
    "        lambda x: x != 359,\n",
    "        [360] + list(360 - np.append(np.logspace(0, 7, 8, base=2, endpoint=True, dtype=int), [180]))\n",
    "    )\n",
    ")\n",
    "crop_hamming = get_augmented_hashes_and_hamming_dist(\n",
    "    [os.path.join(HASH_DIR, 'crop', f'{DATASET}_crop_{crop}.csv') for crop in crop_values],\n",
    "    crop_values,\n",
    "    bin_hashes_orig,\n",
    "    num_processes=10\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(crop_hamming)\n",
    "df = df.melt()\n",
    "df = df.rename(columns={'variable': 'Center Crop Size', 'value': 'Norm. Hamming Distance'})\n",
    "sns.set(rc={'figure.figsize':(8,5)})\n",
    "sns.set_style(\"whitegrid\")\n",
    "fig = sns.lineplot(data=df, marker='o', markersize=8, x=\"Center Crop Size\", y=\"Norm. Hamming Distance\", ci='sd')\n",
    "plt.xticks([180, 232, 286, 328, 360])\n",
    "plt.yticks([0.0, 0.1, 0.2, 0.3, 0.4, 0.5])\n",
    "fig.set(ylim=(-0.025, 0.55))\n",
    "fig.invert_xaxis()\n",
    "fig.set_xlabel(fig.get_xlabel(), fontsize = 21, fontweight='bold')\n",
    "fig.set_ylabel(fig.get_ylabel(), fontsize = 21, fontweight='bold')\n",
    "fig.set_yticklabels(fig.get_yticks(), size = 18, fontweight='bold')\n",
    "fig.set_xticklabels(fig.get_xticks(), size = 18, fontweight='bold')\n",
    "\n",
    "plt.tight_layout() \n",
    "plt.savefig(f'{PLOT_DIR}/{DATASET}_crop_robustness.pdf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print_mean_and_std_for_keys(crop_hamming)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "check_for_non_altered_hashes(crop_hamming, crop_values[-2])\n",
    "# plot two images that do not change\n",
    "idx1, idx2 = check_for_non_altered_hashes(crop_hamming, crop_values[-2], return_indices=True)[:2]\n",
    "print(f'{dataset.classes[dataset[idx1][1]][0]}:')\n",
    "plt.imshow(dataset[idx1][0].permute(1,2,0))\n",
    "plt.show()\n",
    "print(f'{dataset.classes[dataset[idx2][1]][0]}:')\n",
    "plt.imshow(dataset[idx2][0].permute(1,2,0))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_example_img_with_transformation(dataset, EXAMPLE_IMG_IDX, img_class=EXAMPLE_IMG_CLASS, transformation=CenterCrop(crop_values[-2]), file_path=os.path.join(EXAMPLE_IMG_DIR, 'crop.png'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# Horizontal Flipping Robustness"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "hflip_hamming = get_augmented_hashes_and_hamming_dist(\n",
    "    [os.path.join(HASH_DIR, 'hflip', f'{DATASET}_hflip.csv')],\n",
    "    [0],\n",
    "    bin_hashes_orig,\n",
    "    num_processes=1\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "print(f'Average Hamming Distance: {hflip_hamming[0].mean()}')\n",
    "print(f'Standard Deviation Hamming Distance: {hflip_hamming[0].std()}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "check_for_non_altered_hashes(hflip_hamming, 0)\n",
    "# plot two images that do not change\n",
    "idx1, idx2 = check_for_non_altered_hashes(hflip_hamming, 0, return_indices=True)[:2]\n",
    "print(f'{dataset.classes[dataset[idx1][1]][0]}:')\n",
    "plt.imshow(dataset[idx1][0].permute(1,2,0))\n",
    "plt.show()\n",
    "print(f'{dataset.classes[dataset[idx2][1]][0]}:')\n",
    "plt.imshow(dataset[idx2][0].permute(1,2,0))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_example_img_with_transformation(dataset, EXAMPLE_IMG_IDX, img_class=EXAMPLE_IMG_CLASS, transformation=HorizontalFlipping(), file_path=os.path.join(EXAMPLE_IMG_DIR, 'hflip.png'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# Vertical Flipping Robustness"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "vflip_hamming = get_augmented_hashes_and_hamming_dist(\n",
    "    [os.path.join(HASH_DIR, 'vflip', f'{DATASET}_vflip.csv')],\n",
    "    [0],\n",
    "    bin_hashes_orig,\n",
    "    num_processes=1\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "print(f'Average Hamming Distance: {vflip_hamming[0].mean()}')\n",
    "print(f'Standard Deviation Hamming Distance: {vflip_hamming[0].std()}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "check_for_non_altered_hashes(vflip_hamming, 0)\n",
    "# plot two images that do not change\n",
    "idx1, idx2 = check_for_non_altered_hashes(vflip_hamming, 0, return_indices=True)[:2]\n",
    "print(f'{dataset.classes[dataset[idx1][1]][0]}:')\n",
    "plt.imshow(dataset[idx1][0].permute(1,2,0))\n",
    "plt.show()\n",
    "print(f'{dataset.classes[dataset[idx2][1]][0]}:')\n",
    "plt.imshow(dataset[idx2][0].permute(1,2,0))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_example_img_with_transformation(dataset, EXAMPLE_IMG_IDX, img_class=EXAMPLE_IMG_CLASS, transformation=VerticalFlipping(), file_path=os.path.join(EXAMPLE_IMG_DIR, 'vflip.png'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# Downsizing Robustness"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "downsizing_values = list(\n",
    "        filter(\n",
    "            lambda x: x != 359,\n",
    "            [360] + list(360 - np.append(np.logspace(0, 7, 8, base=2, endpoint=True, dtype=int), [180]))\n",
    "        )\n",
    "    )\n",
    "downsizing_hamming = get_augmented_hashes_and_hamming_dist(\n",
    "    [os.path.join(HASH_DIR, 'downsizing', f'{DATASET}_downsizing_{size}.csv') for size in downsizing_values],\n",
    "    downsizing_values,\n",
    "    bin_hashes_orig,\n",
    "    num_processes=9\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(downsizing_hamming)\n",
    "df = df.melt()\n",
    "df = df.rename(columns={'variable': 'Image Size', 'value': 'Norm. Hamming Distance'})\n",
    "sns.set(rc={'figure.figsize':(8,5)})\n",
    "sns.set_style(\"whitegrid\")\n",
    "fig = sns.lineplot(data=df, marker='o', markersize=8, x=\"Image Size\", y=\"Norm. Hamming Distance\", ci='sd')\n",
    "fig.invert_xaxis()\n",
    "plt.xticks([360, 328, 296, 232, 180])\n",
    "fig.set(ylabel=None)\n",
    "fig.set(ylim=(-0.025, 0.55))\n",
    "fig.set_xlabel(fig.get_xlabel(), fontsize = 21, fontweight='bold')\n",
    "fig.set_ylabel(None)\n",
    "fig.set_yticklabels([])\n",
    "fig.set_xticklabels(fig.get_xticks(), size = 18, fontweight='bold')\n",
    "\n",
    "plt.tight_layout() \n",
    "plt.savefig(f'{PLOT_DIR}/{DATASET}_downsizing_robustness.pdf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "print_mean_and_std_for_keys(downsizing_hamming)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "check_for_non_altered_hashes(downsizing_hamming, downsizing_values[1])\n",
    "# plot two images that do not change\n",
    "idx1, idx2 = check_for_non_altered_hashes(downsizing_hamming, downsizing_values[1], return_indices=True)[:2]\n",
    "print(f'{dataset.classes[dataset[idx1][1]][0]}:')\n",
    "plt.imshow(dataset[idx1][0].permute(1,2,0))\n",
    "plt.show()\n",
    "print(f'{dataset.classes[dataset[idx2][1]][0]}:')\n",
    "plt.imshow(dataset[idx2][0].permute(1,2,0))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_example_img_with_transformation(dataset, EXAMPLE_IMG_IDX, img_class=EXAMPLE_IMG_CLASS, transformation=BlackBorder(downsizing_values[-1]), file_path=os.path.join(EXAMPLE_IMG_DIR, 'downsizing.png'))"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "d4d1e4263499bec80672ea0156c357c1ee493ec2b1c70f0acce89fc37c4a6abe"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "name": "python",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}