{
"cells": [
{
"cell_type": "markdown",
"metadata": {
"collapsed": true
},
"source": [
"# Presenting SOTA results on CIMA dataset\n",
"\n",
"This notebook serves as visualisation for State-of-the-Art methods on CIMA dataset\n",
"\n",
"_Note: In case you want to get some further evaluation related to new submission, you may contact JB._"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/jb/.local/lib/python3.6/site-packages/dask/config.py:161: YAMLLoadWarning: calling yaml.load() without Loader=... is deprecated, as the default Loader is unsafe. Please read https://msg.pyyaml.org/load for full details.\n",
" data = yaml.load(f.read()) or {}\n"
]
}
],
"source": [
"%matplotlib inline\n",
"%load_ext autoreload\n",
"%autoreload 2\n",
"\n",
"import os, sys\n",
"import glob, json\n",
"import shutil\n",
"\n",
"import tqdm\n",
"import pandas as pd\n",
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
"import seaborn as sns\n",
"\n",
"sys.path += [os.path.abspath('.'), os.path.abspath('..')] # Add path to root\n",
"from birl.utilities.data_io import update_path\n",
"from birl.utilities.evaluate import compute_ranking\n",
"from birl.utilities.drawing import RadarChart, draw_scatter_double_scale\n",
"from bm_ANHIR.generate_regist_pairs import VAL_STATUS_TRAIN, VAL_STATUS_TEST\n",
"from bm_ANHIR.evaluate_submission import COL_TISSUE"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"This notebook serves for computing extended statistics (e.g. metrics inliding ranks) and visualie some more statistics.\n",
"\n",
"\n",
"You can run the notebook to see result on both scales `10k` and `full`.\n",
"To do so you need to unzip paticular archive in `bm_CIMA` to a separate folder and point-out this path as `PATH_RESULTS` bellow."
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"# folder with all participants submissions\n",
"PATH_RESULTS = os.path.join(update_path('bm_CIMA'), 'size-10k')\n",
"# temporary folder for unzipping submissions\n",
"PATH_TEMP = os.path.abspath(os.path.expanduser('~/Desktop/CIMA_size-10k'))\n",
"# configuration needed for recomputing detail metrics\n",
"PATH_DATASET = os.path.join(update_path('bm_ANHIR'), 'dataset_ANHIR')\n",
"PATH_TABLE = os.path.join(update_path('bm_CIMA'), 'dataset_CIMA_10k.csv')\n",
"# landmarks provided to participants, in early ANHIR stage we provided only 20% points per image pair\n",
"PATH_LNDS_PROVIDED = os.path.join(PATH_DATASET, 'landmarks_all')\n",
"# complete landmarks dataset\n",
"PATH_LNDS_COMPLATE = os.path.join(PATH_DATASET, 'landmarks_all')\n",
"# baseline for normalization of computing time\n",
"PATH_COMP_BM = os.path.join(PATH_DATASET, 'computer-performances_cmpgrid-71.json')\n",
"FIELD_TISSUE = 'type-tissue'\n",
"\n",
"# configuration for Pandas tables\n",
"pd.set_option(\"display.max_columns\", 25)\n",
"assert os.path.isdir(PATH_TEMP)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Some initial replacement and name adjustments"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"# simplify the metrics names according paper\n",
"METRIC_LUT = {'Average-': 'A', 'Rank-': 'R', 'Median-': 'M', 'Max-': 'S'}\n",
"\n",
"def col_metric_rename(col):\n",
" for m in METRIC_LUT:\n",
" col = col.replace(m, METRIC_LUT[m])\n",
" return col"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Parse and load submissions"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Extract metrics from particular submissions\n",
"\n",
"All submissions are expected to be as a zip archives in single folder. The archive name is the author name."
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
],
"source": [
"# Find all archives and unzip them to the same folder.\n",
"archive_paths = sorted(glob.glob(os.path.join(PATH_RESULTS, '*.zip')))\n",
"submission_dirs = []\n",
"\n",
"for path_zip in tqdm.tqdm(archive_paths, desc='unzipping'):\n",
" sub = os.path.join(PATH_TEMP, os.path.splitext(os.path.basename(path_zip))[0])\n",
" os.system('unzip -o \"%s\" -d \"%s\"' % (path_zip, sub))\n",
" sub_ins = glob.glob(os.path.join(sub, '*'))\n",
" # if the zip subfolder contain only one folder move it up\n",
" if len(sub_ins) == 1:\n",
" [shutil.move(p, sub) for p in glob.glob(os.path.join(sub_ins[0], '*'))]\n",
" submission_dirs.append(sub)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Parse submissions and compute the final metrics. This can be computed just once.\n",
"\n",
"**NOTE:** you can skip this step if you have already computed metrics in JSON files"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"scrolled": false
},
"outputs": [],
"source": [
"import bm_ANHIR.evaluate_submission\n",
"\n",
"bm_ANHIR.evaluate_submission.REQUIRE_OVERLAP_INIT_TARGET = False\n",
"\n",
"tqdm_bar = tqdm.tqdm(total=len(submission_dirs))\n",
"for path_sub in submission_dirs:\n",
" tqdm_bar.set_description(path_sub)\n",
" # run the evaluation with details\n",
" path_json = bm_ANHIR.evaluate_submission.main(\n",
" path_experiment=path_sub, path_table=PATH_TABLE, path_dataset=PATH_LNDS_PROVIDED,\n",
" path_reference=PATH_LNDS_COMPLATE, path_comp_bm=PATH_COMP_BM, path_output=path_sub,\n",
" min_landmarks=1., details=True, allow_inverse=True)\n",
" # rename the metrics by the participant\n",
" shutil.copy(os.path.join(path_sub, 'metrics.json'),\n",
" os.path.join(PATH_RESULTS, os.path.basename(path_sub) + '.json'))\n",
" tqdm_bar.update()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Load parsed measures from each experiment"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Users: dict_keys(['ANTs', 'DROP', 'Elastix', 'RNiftyReg', 'RVSS', 'bUnwarpJ-SIFT', 'bUnwarpJ'])\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n"
]
}
],
"source": [
"submission_paths = sorted(glob.glob(os.path.join(PATH_RESULTS, '*.json')))\n",
"submissions = {}\n",
"# loading all participants metrics\n",
"for path_sub in tqdm.tqdm(submission_paths, desc='loading'):\n",
" with open(path_sub, 'r') as fp:\n",
" metrics = json.load(fp)\n",
" # rename tissue types accoding new LUT\n",
" for case in metrics['cases']:\n",
" metrics['cases'][case][FIELD_TISSUE] = metrics['cases'][case][FIELD_TISSUE]\n",
" m_agg = {stat: metrics['aggregates'][stat] for stat in metrics['aggregates']}\n",
" metrics['aggregates'] = m_agg\n",
" submissions[os.path.splitext(os.path.basename(path_sub))[0]] = metrics\n",
"\n",
"print ('Users: %r' % submissions.keys())"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Fields: dict_keys(['aggregates', 'cases', 'computer', 'submission-time', 'required-landmarks'])\n",
"required-landmarks: [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]\n",
"found tissues: ['lung-lesion', 'lung-lobes', 'mammary-gland']\n"
]
}
],
"source": [
"# split the particular fields inside the measured items\n",
"users = list(submissions.keys())\n",
"print ('Fields: %r' % submissions[users[0]].keys())\n",
"user_aggreg = {u: submissions[u]['aggregates'] for u in users}\n",
"user_computer = {u: submissions[u]['computer'] for u in users}\n",
"user_cases = {u: submissions[u]['cases'] for u in users}\n",
"print ('required-landmarks: %r' % [submissions[u]['required-landmarks'] for u in users])\n",
"tissues = set(user_cases[users[0]][cs][FIELD_TISSUE] for cs in user_cases[users[0]])\n",
"print ('found tissues: %r' % sorted(tissues))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Define colors and markers later used in charts"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"METHODS = sorted(submissions.keys())\n",
"# https://seaborn.pydata.org/tutorial/color_palettes.html\n",
"# https://www.codecademy.com/articles/seaborn-design-ii\n",
"COLOR_PALETTE = \"Set1\"\n",
"METHOD_CMAP = sns.color_palette(COLOR_PALETTE, len(submissions))\n",
"METHOD_COLORS = {m: METHOD_CMAP[i] for i, m in enumerate(METHODS)}\n",
"\n",
"def list_methods_colors(methods):\n",
" return [METHOD_COLORS[m] for m in methods]\n",
"\n",
"def cmap_methods(method):\n",
" return METHOD_COLORS[m]\n",
"\n",
"# define cyclic buffer of markers for methods\n",
"# https://matplotlib.org/3.1.1/api/markers_api.html\n",
"METHOD_MARKERS = dict(zip(submissions.keys(), list('.*^v<>pPhHXdD')))\n",
"# METHOD_MARKERS = dict(zip(submissions.keys(), list('.1234+xosD^v<>')))\n",
"\n",
"def list_methods_markers(methods):\n",
" return [METHOD_MARKERS[m] for m in methods]\n",
"\n",
"# display(pd.DataFrame([METHOD_COLORS, METHOD_MARKERS]).T)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Compute ranked measures"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Extend the aggregated statistic by Rank measures such as compute ranking over all cases for each selected field and average it"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"for field, field_agg in [('rTRE-Median', 'Median-rTRE'),\n",
" ('rTRE-Max', 'Max-rTRE')]:\n",
" # Compute ranking per user in selected metric `field` over all dataset\n",
" user_cases = compute_ranking(user_cases, field)\n",
" for user in users:\n",
" # iterate over Robust or all cases\n",
" for robust in [True, False]:\n",
" # chose inly robyst if it is required\n",
" vals = [user_cases[user][cs][field + '_rank'] for cs in user_cases[user]\n",
" if (robust and user_cases[user][cs]['Robustness']) or (not robust)]\n",
" s_robust = '-Robust' if robust else ''\n",
" user_aggreg[user]['Average-Rank-' + field_agg + s_robust] = np.mean(vals)\n",
" user_aggreg[user]['STD-Rank-' + field_agg + s_robust] = np.std(vals)\n",
" # iterate over all tissue kinds\n",
" for tissue in tissues:\n",
" vals = [user_cases[user][cs][field + '_rank'] for cs in user_cases[user]\n",
" if user_cases[user][cs][FIELD_TISSUE] == tissue]\n",
" user_aggreg[user]['Average-Rank-' + field_agg + '__tissue_' + tissue + '__All'] = np.mean(vals)\n",
" user_aggreg[user]['STD-Rank-' + field_agg + '__tissue_' + tissue + '__All'] = np.std(vals)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Show the raw table with **global** statistic (joint training and testing/evaluation)."
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"scrolled": true
},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" Average-used-landmarks | \n",
" Average-Robustness | \n",
" STD-Robustness | \n",
" Median-Robustness | \n",
" Average-Rank-Median-rTRE | \n",
" Average-Rank-Max-rTRE | \n",
" Average-Median-rTRE | \n",
" STD-Median-rTRE | \n",
" Median-Median-rTRE | \n",
" Average-Max-rTRE | \n",
" STD-Max-rTRE | \n",
" Median-Max-rTRE | \n",
" Average-Average-rTRE | \n",
" STD-Average-rTRE | \n",
" Median-Average-rTRE | \n",
" Average-Norm-Time | \n",
" STD-Norm-Time | \n",
" Median-Norm-Time | \n",
" STD-Rank-Median-rTRE | \n",
" STD-Rank-Max-rTRE | \n",
"
\n",
" \n",
" \n",
" \n",
" | ANTs | \n",
" 1.0 | \n",
" 0.790214 | \n",
" 0.248232 | \n",
" 0.892468 | \n",
" 3.342593 | \n",
" 2.824074 | \n",
" 0.023031 | \n",
" 0.020008 | \n",
" 0.016662 | \n",
" 0.055601 | \n",
" 0.036617 | \n",
" 0.050011 | \n",
" 0.024131 | \n",
" 0.019472 | \n",
" 0.018600 | \n",
" 52.167229 | \n",
" 26.893003 | \n",
" 60.916335 | \n",
" 1.564431 | \n",
" 1.489648 | \n",
"
\n",
" \n",
" | DROP | \n",
" 1.0 | \n",
" 0.842451 | \n",
" 0.304637 | \n",
" 0.986842 | \n",
" 2.296296 | \n",
" 2.564815 | \n",
" 0.025039 | \n",
" 0.051129 | \n",
" 0.005084 | \n",
" 0.062894 | \n",
" 0.079209 | \n",
" 0.029880 | \n",
" 0.026894 | \n",
" 0.051019 | \n",
" 0.006521 | \n",
" 1.856635 | \n",
" 0.844369 | \n",
" 2.160855 | \n",
" 2.042417 | \n",
" 2.148527 | \n",
"
\n",
" \n",
" | bUnwarpJ | \n",
" 1.0 | \n",
" 0.740683 | \n",
" 0.277490 | \n",
" 0.831250 | \n",
" 3.601852 | \n",
" 3.824074 | \n",
" 0.028214 | \n",
" 0.020833 | \n",
" 0.030013 | \n",
" 0.066614 | \n",
" 0.036561 | \n",
" 0.064172 | \n",
" 0.029630 | \n",
" 0.020194 | \n",
" 0.031437 | \n",
" 2.989572 | \n",
" 1.125316 | \n",
" 2.933392 | \n",
" 1.855621 | \n",
" 1.762935 | \n",
"
\n",
" \n",
" | RNiftyReg | \n",
" 1.0 | \n",
" 0.680507 | \n",
" 0.296854 | \n",
" 0.702044 | \n",
" 4.222222 | \n",
" 4.064815 | \n",
" 0.031981 | \n",
" 0.019345 | \n",
" 0.032205 | \n",
" 0.068474 | \n",
" 0.034884 | \n",
" 0.068234 | \n",
" 0.033036 | \n",
" 0.019073 | \n",
" 0.034479 | \n",
" 0.364217 | \n",
" 0.598157 | \n",
" 0.045782 | \n",
" 1.517389 | \n",
" 1.760014 | \n",
"
\n",
" \n",
" | Elastix | \n",
" 1.0 | \n",
" 0.795504 | \n",
" 0.164782 | \n",
" 0.807692 | \n",
" 4.527778 | \n",
" 5.000000 | \n",
" 0.037900 | \n",
" 0.028974 | \n",
" 0.034742 | \n",
" 0.082920 | \n",
" 0.044742 | \n",
" 0.075468 | \n",
" 0.039706 | \n",
" 0.028133 | \n",
" 0.037397 | \n",
" 4.024381 | \n",
" 0.754311 | \n",
" 4.254518 | \n",
" 1.807691 | \n",
" 1.563472 | \n",
"
\n",
" \n",
" | RVSS | \n",
" 1.0 | \n",
" 0.574520 | \n",
" 0.289345 | \n",
" 0.541241 | \n",
" 4.981481 | \n",
" 4.462963 | \n",
" 0.059515 | \n",
" 0.158266 | \n",
" 0.032394 | \n",
" 0.094764 | \n",
" 0.165354 | \n",
" 0.063392 | \n",
" 0.060600 | \n",
" 0.157822 | \n",
" 0.034269 | \n",
" 1.499862 | \n",
" 0.825750 | \n",
" 1.229713 | \n",
" 1.747917 | \n",
" 1.734326 | \n",
"
\n",
" \n",
" | bUnwarpJ-SIFT | \n",
" 1.0 | \n",
" 0.496952 | \n",
" 0.383557 | \n",
" 0.540145 | \n",
" 5.027778 | \n",
" 5.259259 | \n",
" 0.074338 | \n",
" 0.079285 | \n",
" 0.042298 | \n",
" 0.135266 | \n",
" 0.122401 | \n",
" 0.096839 | \n",
" 0.077425 | \n",
" 0.080656 | \n",
" 0.046481 | \n",
" 2.659866 | \n",
" 1.165022 | \n",
" 2.708990 | \n",
" 1.863183 | \n",
" 1.796964 | \n",
"
\n",
" \n",
"
\n",
"
\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" Average-Median-rTRE | \n",
" STD-Median-rTRE | \n",
" Median-Median-rTRE | \n",
" Average-Max-rTRE | \n",
" STD-Max-rTRE | \n",
" Median-Max-rTRE | \n",
" Average-Average-rTRE | \n",
" STD-Average-rTRE | \n",
" Median-Average-rTRE | \n",
" Average-Norm-Time | \n",
" STD-Norm-Time | \n",
" Median-Norm-Time | \n",
" Average-Rank-Median-rTRE | \n",
" STD-Rank-Median-rTRE | \n",
" Average-Rank-Max-rTRE | \n",
" STD-Rank-Max-rTRE | \n",
"
\n",
" \n",
" \n",
" \n",
" | ANTs | \n",
" 0.019158 | \n",
" 0.017901 | \n",
" 0.012060 | \n",
" 0.051248 | \n",
" 0.036556 | \n",
" 0.041127 | \n",
" 0.020362 | \n",
" 0.017289 | \n",
" 0.014229 | \n",
" 50.306482 | \n",
" 27.266031 | \n",
" 55.401263 | \n",
" 3.342593 | \n",
" 1.564431 | \n",
" 2.824074 | \n",
" 1.489648 | \n",
"
\n",
" \n",
" | DROP | \n",
" 0.006222 | \n",
" 0.007302 | \n",
" 0.004133 | \n",
" 0.035815 | \n",
" 0.034673 | \n",
" 0.024499 | \n",
" 0.008198 | \n",
" 0.008946 | \n",
" 0.005886 | \n",
" 1.926173 | \n",
" 0.821015 | \n",
" 2.271720 | \n",
" 2.029412 | \n",
" 1.768215 | \n",
" 2.313725 | \n",
" 1.935126 | \n",
"
\n",
" \n",
" | bUnwarpJ | \n",
" 0.024697 | \n",
" 0.020917 | \n",
" 0.021041 | \n",
" 0.062631 | \n",
" 0.038681 | \n",
" 0.057320 | \n",
" 0.026341 | \n",
" 0.020400 | \n",
" 0.023998 | \n",
" 2.945704 | \n",
" 1.038554 | \n",
" 2.902659 | \n",
" 3.601852 | \n",
" 1.855621 | \n",
" 3.824074 | \n",
" 1.762935 | \n",
"
\n",
" \n",
" | RNiftyReg | \n",
" 0.029002 | \n",
" 0.020507 | \n",
" 0.030274 | \n",
" 0.066526 | \n",
" 0.038920 | \n",
" 0.066252 | \n",
" 0.030126 | \n",
" 0.020228 | \n",
" 0.032280 | \n",
" 0.382616 | \n",
" 0.612680 | \n",
" 0.045494 | \n",
" 4.188679 | \n",
" 1.505419 | \n",
" 4.028302 | \n",
" 1.750782 | \n",
"
\n",
" \n",
" | Elastix | \n",
" 0.035819 | \n",
" 0.028515 | \n",
" 0.032164 | \n",
" 0.081753 | \n",
" 0.045233 | \n",
" 0.071432 | \n",
" 0.037716 | \n",
" 0.027670 | \n",
" 0.034383 | \n",
" 4.007976 | \n",
" 0.758227 | \n",
" 3.978500 | \n",
" 4.527778 | \n",
" 1.807691 | \n",
" 5.000000 | \n",
" 1.563472 | \n",
"
\n",
" \n",
" | RVSS | \n",
" 0.029296 | \n",
" 0.024597 | \n",
" 0.021571 | \n",
" 0.064435 | \n",
" 0.045133 | \n",
" 0.050208 | \n",
" 0.030293 | \n",
" 0.024747 | \n",
" 0.023583 | \n",
" 1.456545 | \n",
" 0.747360 | \n",
" 1.343542 | \n",
" 4.913462 | \n",
" 1.743728 | \n",
" 4.375000 | \n",
" 1.705125 | \n",
"
\n",
" \n",
" | bUnwarpJ-SIFT | \n",
" 0.032093 | \n",
" 0.018648 | \n",
" 0.034173 | \n",
" 0.076951 | \n",
" 0.034319 | \n",
" 0.073679 | \n",
" 0.034356 | \n",
" 0.017777 | \n",
" 0.034625 | \n",
" 2.896436 | \n",
" 1.167257 | \n",
" 2.824024 | \n",
" 4.505882 | \n",
" 1.766509 | \n",
" 4.800000 | \n",
" 1.761016 | \n",
"
\n",
" \n",
"
\n",
"