# -*- coding: utf-8 -*-
"""Developer convenience functions for ibs (detections).

TODO: need to split up into sub modules:
    consistency_checks
    feasibility_fixes
    move the export stuff to dbio

    then there are also convineience functions that need to be ordered at least
    within this file
"""
import logging
from os.path import abspath, expanduser, join

import cv2
import numpy as np
import tqdm
import utool as ut
import vtool as vt

from wbia.control import controller_inject

# Inject utool functions
(print, rrr, profile) = ut.inject2(__name__, '[other.detectfuncs]')
logger = logging.getLogger('wbia')

SAMPLES = 1000
AP_SAMPLE_POINTS = [_ / float(SAMPLES) for _ in range(0, SAMPLES + 1)]

# Must import class before injection
CLASS_INJECT_KEY, register_ibs_method = controller_inject.make_ibs_register_decorator(
    __name__
)

GLOBAL_GT_CACHE_DICT = {}
GLOBAL_PRED_CACHE_DICT = {}


def _resize(image, t_width=None, t_height=None, verbose=False):
    if verbose:
        logger.info(
            'RESIZING WITH t_width = {!r} and t_height = {!r}'.format(t_width, t_height)
        )
    height, width = image.shape[:2]
    if t_width is None and t_height is None:
        return image
    elif t_width is not None and t_height is not None:
        pass
    elif t_width is None:
        t_width = (width / height) * float(t_height)
    elif t_height is None:
        t_height = (height / width) * float(t_width)
    t_width, t_height = float(t_width), float(t_height)
    t_width, t_height = int(np.around(t_width)), int(np.around(t_height))
    assert t_width > 0 and t_height > 0, 'target size too small'
    assert (
        t_width <= width * 10 and t_height <= height * 10
    ), 'target size too large (capped at 1000%)'
    # interpolation = cv2.INTER_LANCZOS4
    interpolation = cv2.INTER_LINEAR
    return cv2.resize(image, (t_width, t_height), interpolation=interpolation)


@register_ibs_method
def get_species_nice_mapping(ibs, species):
    from wbia.constants import SPECIES_MAPPING

    species_code, species_nice = SPECIES_MAPPING.get(species, (species, species))
    while species_code is None:
        species_code, species_nice = SPECIES_MAPPING[species_nice]
    return species_nice


def simple_code(label):
    from wbia.constants import PARTS_MAPPING, SPECIES_MAPPING, YAWALIAS

    if label == 'ignore':
        return 'IGNORE'

    for key in PARTS_MAPPING:
        if key in label:
            part_code, part_nice = PARTS_MAPPING[key]
            while part_code is None:
                part_code, part_nice = PARTS_MAPPING[part_nice]
            assert part_code is not None
            label = label.replace(key, part_code)

    for key in SPECIES_MAPPING:
        if key in label:
            species_code, species_nice = SPECIES_MAPPING[key]
            while species_code is None:
                species_code, species_nice = SPECIES_MAPPING[species_nice]
            assert species_code is not None
            label = label.replace(key, species_code)

    for key in sorted(YAWALIAS.keys(), key=len, reverse=True):
        value = YAWALIAS[key]
        label = label.replace(key, value)

    return label


##########################################################################################


def general_precision_recall_algo(
    ibs,
    label_list,
    confidence_list,
    category='positive',
    samples=SAMPLES,
    index_list=None,
    filter_fn_func=None,
    **kwargs,
):
    def errors(zipped, conf, category, filter_fn_func_, gid_aids_mapping):
        tp, tn, fp, fn = 0.0, 0.0, 0.0, 0.0
        fn_filter_converted = 0
        fn_filter_total = 0
        for label, confidence, index in zipped:
            if label == category:
                if conf <= confidence:
                    tp += 1
                else:
                    fn_filter_total += 1
                    if None not in [filter_fn_func_, index]:
                        values = (
                            index,
                            label,
                            confidence,
                            category,
                            conf,
                            zipped,
                        )
                        flag = filter_fn_func_(ibs, 1, values, gid_aids_mapping)
                        if flag:
                            fn += 1
                        else:
                            fn_filter_converted += 1
                            tp += 1
                    else:
                        fn += 1
            else:
                if conf <= confidence:
                    fp += 1
                else:
                    tn += 1
        # if filter_fn_func_ is not None:
        #     print('fn_filter_converted = %d / %d' % (fn_filter_converted, fn_filter_total, ))
        return tp, tn, fp, fn

    print('Using filter_fn_func = {!r}'.format(filter_fn_func))
    if index_list is None:
        index_list = [None] * len(label_list)
        gid_aids_mapping = {}
    else:
        aids_list = ibs.get_image_aids(index_list)
        gid_aids_mapping = dict(zip(index_list, aids_list))
    zipped = list(zip(label_list, confidence_list, index_list))

    conf_list = [_ / float(samples) for _ in range(0, int(samples) + 1)]
    conf_dict = {}
    for conf in tqdm.tqdm(conf_list):
        conf_dict[conf] = errors(zipped, conf, category, filter_fn_func, gid_aids_mapping)

    conf_list_ = [-1.0, -1.0]
    pr_list = [1.0, 0.0]
    re_list = [0.0, 1.0]
    tpr_list = [0.0, 1.0]
    fpr_list = [0.0, 1.0]
    # conf_list_ = []
    # pr_list = []
    # re_list = []
    # tpr_list = []
    # fpr_list = []
    for conf in sorted(conf_dict.keys(), reverse=True):
        error_list = conf_dict[conf]
        tp, tn, fp, fn = error_list
        try:
            pr = tp / (tp + fp)
            re = tp / (tp + fn)
            tpr = tp / (tp + fn)
            fpr = fp / (fp + tn)
            conf_list_.append(conf)
            pr_list.append(pr)
            re_list.append(re)
            tpr_list.append(tpr)
            fpr_list.append(fpr)
        except ZeroDivisionError:
            logger.info(
                'Zero division error (%r) - tp: %r tn: %r fp: %r fn: %r'
                % (conf, tp, tn, fp, fn)
            )

    return conf_list_, pr_list, re_list, tpr_list, fpr_list


def general_interpolate_precision_recall(conf_list, re_list, pr_list):
    conf_list_, re_list_, pr_list_ = [], [], []
    zipped = zip(re_list, conf_list, pr_list)
    zipped = sorted(zipped, reverse=True)
    max_pr = None
    for re, conf, pr in zipped:
        if max_pr is None or pr > max_pr:
            if max_pr is not None:
                conf_list_.append(np.nan)
                re_list_.append(re)
                pr_list_.append(max_pr)
            max_pr = pr
        if pr < max_pr:
            pr = max_pr
        conf_list_.append(conf)
        re_list_.append(re)
        pr_list_.append(pr)
    return conf_list_, re_list_, pr_list_


def general_identify_operating_point(conf_list, x_list, y_list, target=(1.0, 1.0)):
    best_length = np.inf
    best_conf_list = []
    best_x_list = []
    best_y_list = []
    tx, ty = target
    for conf, x, y in sorted(zip(conf_list, x_list, y_list)):
        x_ = x
        y_ = y
        x_ = x_ - tx
        y_ = y_ - ty
        length = np.sqrt(x_ * x_ + y_ * y_)
        if length < best_length:
            best_length = length
            best_conf_list = [conf]
            best_x_list = [x]
            best_y_list = [y]
        elif length == best_length:
            flag_list = [abs(best_conf - conf) > 0.01 for best_conf in best_conf_list]
            if False in flag_list:
                continue
            best_conf_list.append(conf)
            best_x_list.append(x)
            best_y_list.append(y)

    return best_conf_list, best_x_list, best_y_list, best_length


def general_area_best_conf(
    conf_list,
    x_list,
    y_list,
    label='Unknown',
    color='b',
    marker='o',
    plot_point=True,
    interpolate=True,
    target=(1.0, 1.0),
    target_recall=None,
    force_target_recall=False,
    **kwargs,
):
    import matplotlib.pyplot as plt

    zipped = list(sorted(zip(x_list, y_list, conf_list)))
    x_list = [_[0] for _ in zipped]
    y_list = [_[1] for _ in zipped]
    conf_list = [_[2] for _ in zipped]

    if interpolate:
        conf_list, x_list, y_list = general_interpolate_precision_recall(
            conf_list, x_list, y_list
        )

    if interpolate:
        ap_list = []
        for AP_POINT in AP_SAMPLE_POINTS:
            for re, pr in sorted(zip(x_list, y_list)):
                if AP_POINT <= re:
                    ap_list.append(pr)
                    break
        ap = sum(ap_list) / len(ap_list)
    else:
        ap = np.trapz(y_list, x=x_list)

    tup1 = general_identify_operating_point(conf_list, x_list, y_list, target=target)
    best_conf_list, best_x_list, best_y_list, best_length = tup1

    tup2 = None
    if target_recall is not None:
        for x, y, conf in sorted(zip(x_list, y_list, conf_list)):
            if target_recall <= x and not np.isnan(conf):
                tup2 = [conf], [x], [y], None
                break

    if force_target_recall and tup2 is not None:
        tup1 = tup2

    if len(best_conf_list) > 1:
        logger.info(
            'WARNING: Multiple best operating points found {!r}'.format(best_conf_list)
        )

    assert len(best_conf_list) > 0
    best_conf = best_conf_list[0]

    if interpolate:
        # label = '%s [AP = %0.02f, OP = %0.02f]' % (label, ap * 100.0, best_conf)
        label = '{} [AP = {:0.02f}]'.format(label, ap * 100.0)
    else:
        label = '{} [AUC = {:0.02f}]'.format(label, ap * 100.0)

    linestyle = '--' if kwargs.get('line_dotted', False) else '-'
    plt.plot(x_list, y_list, color=color, linestyle=linestyle, label=label)

    if plot_point:
        plt.plot(best_x_list, best_y_list, color=color, marker=marker)

    return ap, best_conf, tup1, tup2


def general_confusion_matrix_algo(
    label_correct_list,
    label_predict_list,
    category_list,
    category_mapping,
    fig_,
    axes_,
    fuzzy_dict=None,
    conf=None,
    conf_list=None,
    size=10,
    **kwargs,
):
    # import matplotlib.colors as colors
    import matplotlib.pyplot as plt

    suppressed_label = 'SUP'
    if conf is not None:
        assert conf_list is not None
        category_list.append(suppressed_label)
        index = len(category_list) - 1
        category_mapping[suppressed_label] = index
        if fuzzy_dict is not None:
            fuzzy_dict[index] = set()

    if category_mapping is not None:
        index_list = [category_mapping[category] for category in category_list]
        zipped = list(sorted(zip(index_list, category_list)))
        category_list = [_[1] for _ in zipped]

    # Get the number of categories
    num_categories = len(category_list)

    # Build the confusion matrix
    confusion_matrix = np.zeros((num_categories, num_categories))
    zipped = zip(label_correct_list, label_predict_list)
    suppressed = 0.0
    suppressed_correct = 0.0
    suppressed_fuzzy = 0.0
    for index, (label_correct, label_predict) in enumerate(zipped):
        if conf is not None:
            conf_ = conf_list[index]
            if conf_ < conf:
                if label_correct != label_predict:
                    suppressed_correct += 1
                if fuzzy_dict is not None:
                    x = category_mapping[label_correct]
                    y = category_mapping[label_predict]
                    if not (y in fuzzy_dict[x] or x in fuzzy_dict[y]):
                        suppressed_fuzzy += 1
                label_predict = suppressed_label
                suppressed += 1
        # Perform any mapping that needs to be done
        correct_ = category_mapping[label_correct]
        predict_ = category_mapping[label_predict]
        # Add to the confidence matrix
        confusion_matrix[correct_][predict_] += 1

    # Normalize the confusion matrix using the rows
    row_normalizer = np.sum(confusion_matrix, axis=1)
    confusion_normalized = np.array((confusion_matrix.T / row_normalizer).T)

    # Draw the confusion matrix
    res = axes_.imshow(confusion_normalized, cmap=plt.cm.jet, interpolation='nearest')

    correct = suppressed_correct
    fuzzy = suppressed_fuzzy
    total = 0.0
    for x in range(num_categories):
        for y in range(num_categories):
            number = int(confusion_matrix[x][y])
            if x == y:
                correct += number
            if fuzzy_dict is not None and (y in fuzzy_dict[x] or x in fuzzy_dict[y]):
                fuzzy += number
            total += number
            axes_.annotate(
                str(number),
                xy=(y, x),
                horizontalalignment='center',
                verticalalignment='center',
                size=size,
            )

    cb = fig_.colorbar(res)  # NOQA
    # cb.set_clim(0.0, 1.0)
    plt.xticks(np.arange(num_categories), category_list, rotation=90)
    plt.yticks(np.arange(num_categories), category_list)
    margin_small = 0.1
    margin_large = 0.9
    plt.subplots_adjust(
        left=margin_small, right=margin_large, bottom=margin_small, top=margin_large
    )

    correct_rate = correct / total
    fuzzy_rate = fuzzy / total
    return correct_rate, fuzzy_rate


def general_intersection_over_union(bbox1, bbox2, return_components=False):
    intersection_xtl = max(bbox1['xtl'], bbox2['xtl'])
    intersection_ytl = max(bbox1['ytl'], bbox2['ytl'])
    intersection_xbr = min(bbox1['xbr'], bbox2['xbr'])
    intersection_ybr = min(bbox1['ybr'], bbox2['ybr'])

    intersection_w = intersection_xbr - intersection_xtl
    intersection_h = intersection_ybr - intersection_ytl

    if intersection_w <= 0 or intersection_h <= 0:
        intersection = 0.0
    else:
        intersection_w = max(0.0, intersection_xbr - intersection_xtl)
        intersection_h = max(0.0, intersection_ybr - intersection_ytl)
        intersection = intersection_w * intersection_h

    union = (
        (bbox1['width'] * bbox1['height'])
        + (bbox2['width'] * bbox2['height'])
        - intersection
    )

    if return_components:
        return intersection, union
    else:
        return intersection / union


def general_overlap(gt_list, pred_list):
    overlap = np.zeros((len(gt_list), len(pred_list)), dtype=np.float32)
    for i, gt in enumerate(gt_list):
        for j, pred in enumerate(pred_list):
            overlap[i, j] = general_intersection_over_union(gt, pred)
    return overlap


def general_tp_fp_fn(gt_list, pred_list, min_overlap, **kwargs):
    overlap = general_overlap(gt_list, pred_list)
    num_gt, num_pred = overlap.shape
    if num_gt == 0:
        tp = 0.0
        fp = num_pred
        fn = 0.0
    elif num_pred == 0:
        tp = 0.0
        fp = 0.0
        fn = num_gt
    else:
        pred_index_list = range(num_pred)
        gt_index_list = np.argmax(overlap, axis=0)
        max_overlap_list = np.max(overlap, axis=0)
        confidence_list = [pred.get('confidence', None) for pred in pred_list]
        assert None not in confidence_list
        zipped = zip(confidence_list, max_overlap_list, pred_index_list, gt_index_list)
        pred_conf_list = [
            (confidence, max_overlap, pred_index, gt_index)
            for confidence, max_overlap, pred_index, gt_index in zipped
        ]
        pred_conf_list = sorted(pred_conf_list, reverse=True)

        assignment_dict = {}
        for pred_conf, max_overlap, pred_index, gt_index in pred_conf_list:
            if max_overlap > min_overlap:
                if gt_index not in assignment_dict:
                    assignment_dict[gt_index] = pred_index

        tp = len(assignment_dict.keys())
        fp = num_pred - tp
        fn = num_gt - tp
        assert tp >= 0
        assert fp >= 0
        assert fn >= 0

    return tp, fp, fn


def general_get_imageset_gids(ibs, imageset_text, unique=True, **kwargs):
    imageset_id = ibs.get_imageset_imgsetids_from_text(imageset_text)
    test_gid_list = ibs.get_imageset_gids(imageset_id)
    if unique:
        test_gid_list = list(set(test_gid_list))
    return test_gid_list


def general_parse_gt_annots(
    ibs,
    aid_list,
    gid_list=None,
    include_parts=True,
    restrict_to_boundary=True,
    species_mapping={},
    gt_species_mapping={},
    **kwargs,
):
    if gid_list is None:
        gid_list = ibs.get_annot_gids(aid_list)

    species_mapping_ = species_mapping.copy()
    species_mapping_.update(gt_species_mapping)

    species_set = set()
    gt_list = []
    for gid, aid in zip(gid_list, aid_list):
        width, height = ibs.get_image_sizes(gid)

        is_tile = ibs.get_tile_flags(gid)
        reference_tile_gid = gid if is_tile else None
        bbox = ibs.get_annot_bboxes(aid, reference_tile_gid=reference_tile_gid)
        theta = ibs.get_annot_thetas(aid)

        # Transformation matrix
        R = vt.rotation_around_bbox_mat3x3(theta, bbox)
        # Get verticies of the annotation polygon
        verts = vt.verts_from_bbox(bbox, close=True)
        # Rotate and transform vertices
        xyz_pts = vt.add_homogenous_coordinate(np.array(verts).T)
        trans_pts = vt.remove_homogenous_coordinate(R.dot(xyz_pts))
        new_verts = np.round(trans_pts).astype(np.int).T.tolist()
        x_points = [pt[0] for pt in new_verts]
        y_points = [pt[1] for pt in new_verts]
        xtl = int(min(x_points))
        xbr = int(max(x_points))
        ytl = int(min(y_points))
        ybr = int(max(y_points))

        xtl = xtl / width
        ytl = ytl / height
        xbr = xbr / width
        ybr = ybr / height
        if restrict_to_boundary:
            xtl = max(0.0, min(1.0, xtl))
            ytl = max(0.0, min(1.0, ytl))
            xbr = max(0.0, min(1.0, xbr))
            ybr = max(0.0, min(1.0, ybr))

        species = ibs.get_annot_species_texts(aid)
        viewpoint = ibs.get_annot_viewpoints(aid)
        interest = ibs.get_annot_interest(aid)
        temp = {
            'gid': gid,
            'aid': aid,
            'xtl': xtl,
            'ytl': ytl,
            'xbr': xbr,
            'ybr': ybr,
            'width': xbr - xtl,
            'height': ybr - ytl,
            'class': species_mapping_.get(species, species),
            'viewpoint': viewpoint,
            'interest': interest,
            'confidence': 1.0,
        }
        species_set.add(temp['class'])
        gt_list.append(temp)

        part_rowid_list = ibs.get_annot_part_rowids(aid)
        if include_parts:
            for part_rowid in part_rowid_list:
                bbox = ibs.get_part_bboxes(
                    part_rowid, reference_tile_gid=reference_tile_gid
                )
                theta = ibs.get_part_thetas(part_rowid)

                # Transformation matrix
                R = vt.rotation_around_bbox_mat3x3(theta, bbox)
                # Get verticies of the annotation polygon
                verts = vt.verts_from_bbox(bbox, close=True)
                # Rotate and transform vertices
                xyz_pts = vt.add_homogenous_coordinate(np.array(verts).T)
                trans_pts = vt.remove_homogenous_coordinate(R.dot(xyz_pts))
                new_verts = np.round(trans_pts).astype(np.int).T.tolist()
                x_points = [pt[0] for pt in new_verts]
                y_points = [pt[1] for pt in new_verts]
                xtl = int(min(x_points))
                xbr = int(max(x_points))
                ytl = int(min(y_points))
                ybr = int(max(y_points))

                xtl = xtl / width
                ytl = ytl / height
                xbr = xbr / width
                ybr = ybr / height
                if restrict_to_boundary:
                    xtl = max(0.0, min(1.0, xtl))
                    ytl = max(0.0, min(1.0, ytl))
                    xbr = max(0.0, min(1.0, xbr))
                    ybr = max(0.0, min(1.0, ybr))

                tag = ibs.get_part_tag_text(part_rowid)

                if tag is None:
                    tag = species
                else:
                    tag = '{}+{}'.format(species, tag)

                temp = {
                    'gid': gid,
                    'aid': aid,
                    'part_id': part_rowid,
                    'xtl': xtl,
                    'ytl': ytl,
                    'xbr': xbr,
                    'ybr': ybr,
                    'width': xbr - xtl,
                    'height': ybr - ytl,
                    'class': tag,
                    'viewpoint': viewpoint,
                    'interest': interest,
                    'confidence': 1.0,
                }
                species_set.add(temp['class'])
                gt_list.append(temp)

    return gt_list, species_set


def general_parse_gt(ibs, test_gid_list=None, **kwargs):
    global GLOBAL_GT_CACHE_DICT

    kwargs_str = ut.repr3(kwargs)
    kwargs_cfgstr = ut.hash_data(kwargs_str)

    if test_gid_list is None:
        test_gid_list = general_get_imageset_gids(ibs, 'TEST_SET', **kwargs)

    uuid_list = ibs.get_image_uuids(test_gid_list)
    gid_list = ibs.get_image_gids_from_uuid(uuid_list)
    aids_list = ibs.get_image_aids(gid_list)

    species_set = set()
    gt_dict = {}
    zipped = list(zip(gid_list, aids_list, uuid_list))
    for gid, aid_list, uuid in tqdm.tqdm(zipped):
        cache_tag = '{}_{}'.format(
            uuid,
            kwargs_cfgstr,
        )
        if cache_tag not in GLOBAL_GT_CACHE_DICT:
            reference_tile_gid_list = [gid] * len(aid_list)
            gt_list, species_set = general_parse_gt_annots(
                ibs, aid_list, gid_list=reference_tile_gid_list, **kwargs
            )
            species_set = species_set | species_set
            GLOBAL_GT_CACHE_DICT[cache_tag] = gt_list

        gt_dict[uuid] = GLOBAL_GT_CACHE_DICT[cache_tag]

    # logger.info('General Parse GT species_set = %r' % (species_set, ))
    return gt_dict


##########################################################################################


def localizer_parse_pred_dirty(ibs, test_gid_list=None, species_mapping_={}, **kwargs):
    depc = ibs.depc_image

    uuid_list = ibs.get_image_uuids(test_gid_list)
    size_list = ibs.get_image_sizes(test_gid_list)

    # Unsure, but we need to call this multiple times?  Lazy loading bug?
    # depc.delete_property('localizations', test_gid_list, config=kwargs)

    bboxes_list = depc.get_property(
        'localizations', test_gid_list, 'bboxes', config=kwargs
    )
    # Get actual data
    bboxes_list = depc.get_property(
        'localizations', test_gid_list, 'bboxes', config=kwargs
    )
    thetas_list = depc.get_property(
        'localizations', test_gid_list, 'thetas', config=kwargs
    )
    confss_list = depc.get_property(
        'localizations', test_gid_list, 'confs', config=kwargs
    )
    classs_list = depc.get_property(
        'localizations', test_gid_list, 'classes', config=kwargs
    )

    length_list = [len(bbox_list) for bbox_list in bboxes_list]

    # Establish primitives
    test_gids_list = [
        [test_gid] * length for test_gid, length in zip(test_gid_list, length_list)
    ]
    sizes_list = [[size] * length for size, length in zip(size_list, length_list)]
    keeps_list = [[True] * length for length in length_list]
    features_list = [[None] * length for length in length_list]
    features_lazy_list = [[None] * length for length in length_list]
    viewpoints_list = [[None] * length for length in length_list]
    interests_list = [[None] * length for length in length_list]

    # Get features
    if kwargs.get('features', False):
        features_list = depc.get_property(
            'localizations_features', test_gid_list, 'vector', config=kwargs
        )

    if kwargs.get('features_lazy', False):
        from functools import partial

        def features_lazy_func(gid, offset):
            vector_list = depc.get_property(
                'localizations_features', gid, 'vector', config=kwargs
            )
            vector = vector_list[offset]
            return vector

        features_lazy_list = [
            [
                partial(features_lazy_func, test_gid, test_offset)
                for test_offset in range(length)
            ]
            for test_gid, length in zip(test_gid_list, length_list)
        ]

    # Get species and viewpoints labels
    if kwargs.get('labels', False):
        classs_list = depc.get_property(
            'localizations_labeler', test_gid_list, 'species', config=kwargs
        )
        viewpoints_list = depc.get_property(
            'localizations_labeler', test_gid_list, 'viewpoint', config=kwargs
        )

    # Get updated confidences for boxes
    if kwargs.get('classify', False):
        logger.info('Using alternate classifications')
        # depc.delete_property('localizations_classifier', test_gid_list, config=kwargs)
        confss_list = depc.get_property(
            'localizations_classifier', test_gid_list, 'score', config=kwargs
        )

    # Get updated confidences for boxes
    if kwargs.get('interest', False):
        logger.info('Using alternate AoI interest flags')
        interests_list = depc.get_property(
            'localizations_classifier', test_gid_list, 'score', config=kwargs
        )

    # Reformat results for json
    zipped_list_list = zip(
        keeps_list,
        test_gids_list,
        sizes_list,
        bboxes_list,
        thetas_list,
        confss_list,
        classs_list,
        viewpoints_list,
        interests_list,
        features_list,
        features_lazy_list,
    )
    results_list = [
        [
            {
                'gid': test_gid,
                'xtl': bbox[0] / width,
                'ytl': bbox[1] / height,
                'xbr': (bbox[0] + bbox[2]) / width,
                'ybr': (bbox[1] + bbox[3]) / height,
                'width': bbox[2] / width,
                'height': bbox[3] / height,
                'theta': theta,
                'confidence': conf,
                'class': species_mapping_.get(class_, class_),
                'viewpoint': viewpoint,
                'interest': None if interest is None else interest >= 0.84,
                'feature': feature,
                'feature_lazy': feature_lazy,
            }
            for keep_, test_gid, (
                width,
                height,
            ), bbox, theta, conf, class_, viewpoint, interest, feature, feature_lazy in zip(
                *zipped_list
            )
            if keep_
        ]
        for zipped_list in zipped_list_list
    ]

    pred_dict = {
        uuid_: result_list for uuid_, result_list in zip(uuid_list, results_list)
    }
    return pred_dict


def localizer_parse_pred(
    ibs, test_gid_list=None, species_mapping={}, pred_species_mapping={}, **kwargs
):
    global GLOBAL_PRED_CACHE_DICT

    if 'feature2_algo' not in kwargs:
        kwargs['feature2_algo'] = 'resnet'

    species_mapping_ = species_mapping.copy()
    species_mapping_.update(pred_species_mapping)

    kwargs_str = ut.repr3([species_mapping_, kwargs])
    kwargs_cfgstr = ut.hash_data(kwargs_str)

    if test_gid_list is None:
        test_gid_list = general_get_imageset_gids(ibs, 'TEST_SET', **kwargs)

    uuid_list = ibs.get_image_uuids(test_gid_list)
    gid_list = ibs.get_image_gids_from_uuid(uuid_list)

    dirty_gids = []
    for gid, uuid in zip(gid_list, uuid_list):
        cache_tag = '{}_{}'.format(
            uuid,
            kwargs_cfgstr,
        )
        if cache_tag not in GLOBAL_PRED_CACHE_DICT:
            dirty_gids.append(gid)

    pred_dict_dirty = localizer_parse_pred_dirty(
        ibs, dirty_gids, species_mapping_=species_mapping_, **kwargs
    )

    for dirty_uuid in pred_dict_dirty:
        cache_tag = '{}_{}'.format(
            dirty_uuid,
            kwargs_cfgstr,
        )
        GLOBAL_PRED_CACHE_DICT[cache_tag] = pred_dict_dirty[dirty_uuid]

    pred_dict = {}
    for gid, uuid in zip(gid_list, uuid_list):
        cache_tag = '{}_{}'.format(
            uuid,
            kwargs_cfgstr,
        )
        assert cache_tag in GLOBAL_PRED_CACHE_DICT
        pred_dict[uuid] = GLOBAL_PRED_CACHE_DICT[cache_tag]

    return pred_dict


def localizer_precision_recall_algo(
    ibs, samples=SAMPLES, test_gid_list=None, ignore_filter_func=None, **kwargs
):
    if test_gid_list is None:
        test_gid_list = general_get_imageset_gids(ibs, 'TEST_SET', **kwargs)

    test_uuid_list = ibs.get_image_uuids(test_gid_list)

    if ignore_filter_func is None:
        ignore_filter_func = _ignore_filter_identity_func

    logger.info('\tGather Ground-Truth')
    gt_dict = general_parse_gt(ibs, test_gid_list=test_gid_list, **kwargs)

    logger.info('\tGather Predictions')
    pred_dict = localizer_parse_pred(ibs, test_gid_list=test_gid_list, **kwargs)

    species_set = kwargs.get('species_set', None)
    if species_set is not None:
        # filter out any prefix ! to denote interest only
        species_set_ = {species.lstrip('!') for species in species_set}

        dict_list = [
            (gt_dict, 'Ground-Truth'),
            (pred_dict, 'Predictions'),
        ]
        for dict_, dict_tag in dict_list:
            for image_uuid in dict_:
                temp = []
                for val in dict_[image_uuid]:
                    if val.get('class', None) not in species_set_:
                        continue
                    if ignore_filter_func(ibs, val):
                        continue
                    temp.append(val)
                dict_[image_uuid] = temp

    values = localizer_tp_fp(test_uuid_list, gt_dict, pred_dict, **kwargs)
    conf_list, tp_list, fp_list, total = values

    conf_list_ = [-1.0, -1.0]
    pr_list = [1.0, 0.0]
    re_list = [0.0, 1.0]
    for conf, tp, fp in zip(conf_list, tp_list, fp_list):
        try:
            pr = tp / (tp + fp)
            re = tp / total
        except ZeroDivisionError:
            continue
        conf_list_.append(conf)
        pr_list.append(pr)
        re_list.append(re)

    return conf_list_, pr_list, re_list


def localizer_assign(gt_list, pred, min_overlap):
    best_overlap = min_overlap
    best_index = None
    for index, gt in enumerate(gt_list):
        if gt['class'] != pred['class']:
            continue

        overlap = general_intersection_over_union(gt, pred)
        if overlap < best_overlap:
            continue

        best_overlap = overlap
        best_index = index

    if best_index is None:
        best_overlap = None

    return best_index, best_overlap


def localizer_assignments(pred_list, gt_list, gt_list_=[], min_overlap=0.5):
    pred_list = sorted(pred_list, key=lambda pred: pred['confidence'], reverse=True)

    index_list = list(range(len(gt_list)))
    match_list = []
    for pred in pred_list:
        flag = False

        match_index, best_overlap = localizer_assign(gt_list, pred, min_overlap)
        match_index_, best_overlap_ = localizer_assign(gt_list_, pred, min_overlap)

        if match_index is not None:
            flag = True
            del gt_list[match_index]
        elif match_index_ is not None:
            flag = None

        match_index_external = (
            None if match_index is None else index_list.pop(match_index)
        )

        if flag is not None:
            match_list += [(pred['confidence'], flag, match_index_external, best_overlap)]

    return match_list


def localizer_tp_fp(
    uuid_list, gt_dict, pred_dict, min_overlap=0.5, return_match_dict=False, **kwargs
):
    total = 0.0

    interest_species_set = set()
    species_set = kwargs.get('species_set', None)
    if species_set is not None:
        for species in species_set:
            if species.startswith('!'):
                species = species.lstrip('!')
                interest_species_set.add(species)

    match_list = []
    match_dict = {}
    for image_uuid in uuid_list:
        gt_list = []
        gt_list_ = []
        pred_list = pred_dict[image_uuid]
        for gt in gt_dict[image_uuid]:
            species = gt['class']
            interest = gt['interest']
            if species in interest_species_set and not interest:
                gt_list_.append(gt)
            else:
                gt_list.append(gt)

        total_ = len(gt_list)
        total += total_

        # Match predictions
        match_list_ = localizer_assignments(pred_list, gt_list, gt_list_, min_overlap)
        match_dict[image_uuid] = (
            match_list_,
            total_,
        )
        for match_ in match_list_:
            match_list.append(match_)

    # sort matches by confidence from high to low
    match_list = sorted(match_list, key=lambda match: match[0], reverse=True)

    conf_list = []
    tp_list = []
    fp_list = []

    tp_counter = 0
    fp_counter = 0
    for conf, flag, index, overlap in match_list:
        if flag:
            tp_counter += 1
        else:
            fp_counter += 1
        conf_list.append(conf)
        tp_list.append(tp_counter)
        fp_list.append(fp_counter)

    # logger.info('\t tps  [:10]     : %r' % (tp_list[:10], ))
    # logger.info('\t fps  [:10]     : %r' % (fp_list[:10], ))
    # logger.info('\t con  [:10]     : %r' % (conf_list[:10], ))
    # logger.info('\t tps [-10:]     : %r' % (tp_list[-10:], ))
    # logger.info('\t fps [-10:]     : %r' % (fp_list[-10:], ))
    # logger.info('\t con [-10:]     : %r' % (conf_list[-10:], ))
    # logger.info('\t num_annotations: %r' % (total, ))

    if return_match_dict:
        return conf_list, tp_list, fp_list, total, match_dict
    else:
        return conf_list, tp_list, fp_list, total


def localizer_precision_recall_algo_plot(ibs, **kwargs):
    label = kwargs['label']
    logger.info('Processing Precision-Recall for: {!r}'.format(label))
    conf_list, pr_list, re_list = localizer_precision_recall_algo(ibs, **kwargs)
    return general_area_best_conf(conf_list, re_list, pr_list, **kwargs)


def _ignore_filter_identity_func(*args, **kwargs):
    return False


def localizer_iou_recall_algo(
    ibs, samples=100, test_gid_list=None, ignore_filter_func=None, **kwargs
):

    assert 'min_overlap' not in kwargs

    if test_gid_list is None:
        test_gid_list = general_get_imageset_gids(ibs, 'TEST_SET', **kwargs)

    test_uuid_list = ibs.get_image_uuids(test_gid_list)

    if ignore_filter_func is None:
        ignore_filter_func = _ignore_filter_identity_func

    logger.info('\tGather Ground-Truth')
    gt_dict = general_parse_gt(ibs, test_gid_list=test_gid_list, **kwargs)

    logger.info('\tGather Predictions')
    pred_dict = localizer_parse_pred(ibs, test_gid_list=test_gid_list, **kwargs)

    species_set = kwargs.get('species_set', None)
    if species_set is not None:
        # filter out any prefix ! to denote interest only
        species_set_ = {species.lstrip('!') for species in species_set}

        dict_list = [
            (gt_dict, 'Ground-Truth'),
            (pred_dict, 'Predictions'),
        ]

        for dict_, dict_tag in dict_list:
            for image_uuid in dict_:
                temp = []
                for val in dict_[image_uuid]:
                    if val.get('class', None) not in species_set_:
                        continue
                    if ignore_filter_func(ibs, val):
                        continue
                    temp.append(val)
                dict_[image_uuid] = temp

    target = (1.0, 1.0)
    iou_list = [_ / float(samples) for _ in range(0, int(samples) + 1)]

    conf_list_ = []
    iou_list_ = []
    recall_list = []
    for iou in tqdm.tqdm(iou_list):
        values = localizer_tp_fp(
            test_uuid_list, gt_dict, pred_dict, min_overlap=iou, **kwargs
        )
        conf_list, tp_list, fp_list, total = values

        conf_list_ = []
        pr_list = []
        re_list = []
        for conf, tp, fp in zip(conf_list, tp_list, fp_list):
            try:
                pr = tp / (tp + fp)
                re = tp / total
            except ZeroDivisionError:
                continue
            conf_list_.append(conf)
            pr_list.append(pr)
            re_list.append(re)

        best_tup = general_identify_operating_point(
            conf_list, re_list, pr_list, target=target
        )
        best_conf_list, best_re_list, best_pr_list, best_length = best_tup
        if len(best_conf_list) > 1:
            logger.info(
                'WARNING: Multiple best operating points found {!r}'.format(
                    best_conf_list
                )
            )
        assert len(best_conf_list) > 0

        best_re_index = np.argmax(best_re_list)
        best_re = best_re_list[best_re_index]
        best_conf = best_conf_list[best_re_index]

        conf_list_.append(best_conf)
        iou_list_.append(iou)
        recall_list.append(best_re)

    return conf_list_, iou_list_, recall_list


def localizer_iou_recall_algo_plot(ibs, **kwargs):
    label = kwargs['label']
    logger.info('Processing IoU-Recall for: {!r}'.format(label))
    conf_list, iou_list, recall_list = localizer_iou_recall_algo(ibs, **kwargs)
    return general_area_best_conf(
        conf_list, iou_list, recall_list, interpolate=False, **kwargs
    )


# def localizer_iou_precision_algo_plot(ibs, **kwargs):
#     label = kwargs['label']
#     logger.info('Processing Precision-Recall for: %r' % (label, ))
#     conf_list, iou_list, pr_list, re_list = localizer_iou_precision_recall_algo(ibs, **kwargs)
# return general_area_best_conf(conf_list, iou_list, re_list, **kwargs)


def localizer_confusion_matrix_algo_plot(
    ibs,
    label=None,
    target_conf=None,
    test_gid_list=None,
    ignore_filter_func=None,
    **kwargs,
):
    if test_gid_list is None:
        test_gid_list = general_get_imageset_gids(ibs, 'TEST_SET', **kwargs)

    test_uuid_list = ibs.get_image_uuids(test_gid_list)

    if ignore_filter_func is None:
        ignore_filter_func = _ignore_filter_identity_func

    logger.info('\tGather Ground-Truth')
    gt_dict = general_parse_gt(ibs, test_gid_list=test_gid_list, **kwargs)

    logger.info('\tGather Predictions')
    pred_dict = localizer_parse_pred(ibs, test_gid_list=test_gid_list, **kwargs)

    species_set = kwargs.get('species_set', None)
    if species_set is not None:
        # filter out any prefix ! to denote interest only
        species_set_ = {species.lstrip('!') for species in species_set}

        dict_list = [
            (gt_dict, 'Ground-Truth'),
            (pred_dict, 'Predictions'),
        ]
        for dict_, dict_tag in dict_list:
            for image_uuid in dict_:
                temp = []
                for val in dict_[image_uuid]:
                    if val.get('class', None) not in species_set_:
                        continue
                    if ignore_filter_func(ibs, val):
                        continue
                    temp.append(val)
                dict_[image_uuid] = temp

    values = localizer_tp_fp(test_uuid_list, gt_dict, pred_dict, **kwargs)
    conf_list, tp_list, fp_list, total = values

    best_conf = None
    best_accuracy = None
    best_args = None
    for conf, tp, fp in sorted(zip(conf_list, tp_list, fp_list)):
        fn = total - tp
        accuracy = tp / (tp + fp + fn)

        if target_conf is None:
            if best_accuracy is None or accuracy > best_accuracy:
                best_conf = conf
                best_accuracy = accuracy
                best_args = (tp, fp, fn)
        else:
            if target_conf <= conf:
                best_conf = conf
                best_accuracy = accuracy
                best_args = (tp, fp, fn)
                break

    try:
        assert None not in [best_conf, best_accuracy, best_args]
    except AssertionError:
        ut.embed()
        return np.nan, (np.nan, None)

    logger.info(
        'Processing Confusion Matrix for: %r (Conf = %0.02f, Accuracy = %0.02f)'
        % (label, best_conf, best_accuracy)
    )
    tp, fp, fn = best_args

    label_list = []
    prediction_list = []
    for _ in range(int(tp)):
        label_list.append('positive')
        prediction_list.append('positive')
    for _ in range(int(fp)):
        label_list.append('negative')
        prediction_list.append('positive')
    for _ in range(int(fn)):
        label_list.append('positive')
        prediction_list.append('negative')

    category_list = ['positive', 'negative']
    category_mapping = {
        'positive': 0,
        'negative': 1,
    }
    values = general_confusion_matrix_algo(
        label_list, prediction_list, category_list, category_mapping, size=20, **kwargs
    )
    return best_conf, values


@register_ibs_method
def localizer_precision_recall(
    ibs,
    config_dict=None,
    output_path=None,
    test_gid_list=None,
    ignore_filter_func=None,
    overwrite_config_keys=False,
    **kwargs,
):
    if config_dict is None:
        if test_gid_list is not None:
            logger.info('Using %d test gids' % (len(test_gid_list),))

        # species_mapping = {  # NOQA
        #     'giraffe_masai'       : 'giraffe',
        #     'giraffe_reticulated' : 'giraffe',
        #     'zebra_grevys'        : 'zebra',
        #     'zebra_plains'        : 'zebra',
        # }

        config_dict = {
            # 'seaturtle': (
            #     [
            #         {'label': 'Sea Turtle',        'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.2, 'species_set' : set(['turtle_green', 'turtle_hawksbill'])},
            #         {'label': 'Sea Turtle Heads',  'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.2, 'species_set' : set(['turtle_green+head', 'turtle_hawksbill+head'])},
            #         {'label': 'Green',             'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.2, 'species_set' : set(['turtle_green'])},
            #         {'label': 'Green Heads',       'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.2, 'species_set' : set(['turtle_green+head'])},
            #         {'label': 'Hawksbill',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.2, 'species_set' : set(['turtle_hawksbill'])},
            #         {'label': 'Hawksbill Heads',   'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.2, 'species_set' : set(['turtle_hawksbill+head'])},
            #     ],
            #     {'BEST_INDEX': 0},
            # ),
            # '!seaturtle': (
            #     [
            #         {'label': '! Sea Turtle',        'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.2, 'species_set' : set(['!turtle_green', '!turtle_hawksbill'])},
            #         {'label': '! Sea Turtle Heads',  'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.2, 'species_set' : set(['!turtle_green+head', '!turtle_hawksbill+head'])},
            #         {'label': '! Green',             'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.2, 'species_set' : set(['!turtle_green'])},
            #         {'label': '! Green Heads',       'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.2, 'species_set' : set(['!turtle_green+head'])},
            #         {'label': '! Hawksbill',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.2, 'species_set' : set(['!turtle_hawksbill'])},
            #         {'label': '! Hawksbill Heads',   'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.2, 'species_set' : set(['!turtle_hawksbill+head'])},
            #     ],
            #     {'BEST_INDEX': 0},
            # ),
            # 'hawksbills': (
            #     [
            #         {'label': 'Hawksbill NMS 0%',          'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.00, 'species_set' : set(['turtle_hawksbill'])},
            #         {'label': 'Hawksbill NMS 10%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['turtle_hawksbill'])},
            #         {'label': 'Hawksbill NMS 20%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.20, 'species_set' : set(['turtle_hawksbill'])},
            #         {'label': 'Hawksbill NMS 30%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['turtle_hawksbill'])},
            #         {'label': 'Hawksbill NMS 40%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.40, 'species_set' : set(['turtle_hawksbill'])},
            #         {'label': 'Hawksbill NMS 50%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['turtle_hawksbill'])},
            #         {'label': 'Hawksbill NMS 60%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.60, 'species_set' : set(['turtle_hawksbill'])},
            #         {'label': 'Hawksbill NMS 70%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['turtle_hawksbill'])},
            #         {'label': 'Hawksbill NMS 80%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.80, 'species_set' : set(['turtle_hawksbill'])},
            #         {'label': 'Hawksbill NMS 90%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['turtle_hawksbill'])},
            #         {'label': 'Hawksbill NMS 100%',        'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 1.00, 'species_set' : set(['turtle_hawksbill'])},
            #     ],
            #     {},
            # ),
            # 'hawsbills+heads': (
            #     [
            #         {'label': 'Hawksbill Head NMS 0%',          'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.00, 'species_set' : set(['turtle_hawksbill+head'])},
            #         {'label': 'Hawksbill Head NMS 10%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['turtle_hawksbill+head'])},
            #         {'label': 'Hawksbill Head NMS 20%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.20, 'species_set' : set(['turtle_hawksbill+head'])},
            #         {'label': 'Hawksbill Head NMS 30%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['turtle_hawksbill+head'])},
            #         {'label': 'Hawksbill Head NMS 40%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.40, 'species_set' : set(['turtle_hawksbill+head'])},
            #         {'label': 'Hawksbill Head NMS 50%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['turtle_hawksbill+head'])},
            #         {'label': 'Hawksbill Head NMS 60%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.60, 'species_set' : set(['turtle_hawksbill+head'])},
            #         {'label': 'Hawksbill Head NMS 70%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['turtle_hawksbill+head'])},
            #         {'label': 'Hawksbill Head NMS 80%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.80, 'species_set' : set(['turtle_hawksbill+head'])},
            #         {'label': 'Hawksbill Head NMS 90%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['turtle_hawksbill+head'])},
            #         {'label': 'Hawksbill Head NMS 100%',        'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 1.00, 'species_set' : set(['turtle_hawksbill+head'])},
            #     ],
            #     {},
            # ),
            # 'hammerhead': (
            #     [
            #         {'label': 'Hammerhead NMS 0%',            'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.00, 'species_set' : set(['shark_hammerhead'])},
            #         {'label': 'Hammerhead NMS 10%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['shark_hammerhead'])},
            #         {'label': 'Hammerhead NMS 20%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.20, 'species_set' : set(['shark_hammerhead'])},
            #         {'label': 'Hammerhead NMS 30%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['shark_hammerhead'])},
            #         {'label': 'Hammerhead NMS 40%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.40, 'species_set' : set(['shark_hammerhead'])},
            #         {'label': 'Hammerhead NMS 50%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['shark_hammerhead'])},
            #         {'label': 'Hammerhead NMS 60%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.60, 'species_set' : set(['shark_hammerhead'])},
            #         {'label': 'Hammerhead NMS 70%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['shark_hammerhead'])},
            #         {'label': 'Hammerhead NMS 80%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.80, 'species_set' : set(['shark_hammerhead'])},
            #         {'label': 'Hammerhead NMS 90%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['shark_hammerhead'])},
            #         {'label': 'Hammerhead NMS 100%',          'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 1.00, 'species_set' : set(['shark_hammerhead'])},
            #     ],
            #     {},
            # ),
            # '!hammerhead': (
            #     [
            #         {'label': 'Hammerhead NMS 40%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.40, 'species_set' : set(['shark_hammerhead'])},
            #         {'label': 'Hammerhead ! NMS 0%',          'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.00, 'species_set' : set(['!shark_hammerhead'])},
            #         {'label': 'Hammerhead ! NMS 10%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['!shark_hammerhead'])},
            #         {'label': 'Hammerhead ! NMS 20%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.20, 'species_set' : set(['!shark_hammerhead'])},
            #         {'label': 'Hammerhead ! NMS 30%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['!shark_hammerhead'])},
            #         {'label': 'Hammerhead ! NMS 40%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.40, 'species_set' : set(['!shark_hammerhead'])},
            #         {'label': 'Hammerhead ! NMS 50%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['!shark_hammerhead'])},
            #         {'label': 'Hammerhead ! NMS 60%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.60, 'species_set' : set(['!shark_hammerhead'])},
            #         {'label': 'Hammerhead ! NMS 70%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['!shark_hammerhead'])},
            #         {'label': 'Hammerhead ! NMS 80%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.80, 'species_set' : set(['!shark_hammerhead'])},
            #         {'label': 'Hammerhead ! NMS 90%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['!shark_hammerhead'])},
            #         {'label': 'Hammerhead ! NMS 100%',        'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 1.00, 'species_set' : set(['!shark_hammerhead'])},
            #     ],
            #     {'offset_color': 1},
            # ),
            # 'ggr2-giraffe-lightnet': (
            #     [
            #         {'label': 'Giraffe NMS 0%',          'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.00, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
            #         {'label': 'Giraffe NMS 10%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.10, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
            #         {'label': 'Giraffe NMS 20%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.20, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
            #         {'label': 'Giraffe NMS 30%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.30, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
            #         {'label': 'Giraffe NMS 40%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.40, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
            #         {'label': 'Giraffe NMS 50%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.50, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
            #         {'label': 'Giraffe NMS 60%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.60, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
            #         {'label': 'Giraffe NMS 70%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.70, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
            #         {'label': 'Giraffe NMS 80%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.80, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
            #         {'label': 'Giraffe NMS 90%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.90, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
            #         {'label': 'Giraffe NMS 100%',        'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 1.00, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
            #     ],
            #     {},
            # ),
            # 'ggr2-zebra-lightnet': (
            #     [
            #         {'label': 'Zebra NMS 0%',          'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.00, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
            #         {'label': 'Zebra NMS 10%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.10, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
            #         {'label': 'Zebra NMS 20%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.20, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
            #         {'label': 'Zebra NMS 30%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.30, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
            #         {'label': 'Zebra NMS 40%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.40, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
            #         {'label': 'Zebra NMS 50%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.50, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
            #         {'label': 'Zebra NMS 60%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.60, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
            #         {'label': 'Zebra NMS 70%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.70, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
            #         {'label': 'Zebra NMS 80%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.80, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
            #         {'label': 'Zebra NMS 90%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.90, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
            #         {'label': 'Zebra NMS 100%',        'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 1.00, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
            #     ],
            #     {},
            # ),
            # 'ggr2-!giraffe-lightnet': (
            #     [
            #         {'label': 'Giraffe ! NMS 0%',          'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.00, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
            #         {'label': 'Giraffe ! NMS 10%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.10, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
            #         {'label': 'Giraffe ! NMS 20%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.20, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
            #         {'label': 'Giraffe ! NMS 30%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.30, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
            #         {'label': 'Giraffe ! NMS 40%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.40, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
            #         {'label': 'Giraffe ! NMS 50%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.50, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
            #         {'label': 'Giraffe ! NMS 60%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.60, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
            #         {'label': 'Giraffe ! NMS 70%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.70, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
            #         {'label': 'Giraffe ! NMS 80%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.80, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
            #         {'label': 'Giraffe ! NMS 90%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.90, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
            #         {'label': 'Giraffe ! NMS 100%',        'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 1.00, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
            #     ],
            #     {},
            # ),
            # 'ggr2-!zebra-lightnet': (
            #     [
            #         {'label': 'Zebra ! NMS 0%',          'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.00, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
            #         {'label': 'Zebra ! NMS 10%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.10, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
            #         {'label': 'Zebra ! NMS 20%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.20, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
            #         {'label': 'Zebra ! NMS 30%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.30, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
            #         {'label': 'Zebra ! NMS 40%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.40, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
            #         {'label': 'Zebra ! NMS 50%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.50, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
            #         {'label': 'Zebra ! NMS 60%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.60, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
            #         {'label': 'Zebra ! NMS 70%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.70, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
            #         {'label': 'Zebra ! NMS 80%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.80, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
            #         {'label': 'Zebra ! NMS 90%',         'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.90, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
            #         {'label': 'Zebra ! NMS 100%',        'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 1.00, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
            #     ],
            #     {},
            # ),
            # 'ggr2-giraffe-azure': (
            #     [
            #         {'label': 'Giraffe NMS 0%',          'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.00, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
            #         {'label': 'Giraffe NMS 10%',         'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.10, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
            #         {'label': 'Giraffe NMS 20%',         'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.20, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
            #         {'label': 'Giraffe NMS 30%',         'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.30, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
            #         {'label': 'Giraffe NMS 40%',         'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.40, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
            #         {'label': 'Giraffe NMS 50%',         'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.50, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
            #         {'label': 'Giraffe NMS 60%',         'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.60, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
            #         {'label': 'Giraffe NMS 70%',         'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.70, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
            #         {'label': 'Giraffe NMS 80%',         'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.80, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
            #         {'label': 'Giraffe NMS 90%',         'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.90, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
            #         {'label': 'Giraffe NMS 100%',        'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 1.00, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
            #     ],
            #     {},
            # ),
            # 'ggr2-zebra-azure': (
            #     [
            #         {'label': 'Zebra NMS 0%',          'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.00, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
            #         {'label': 'Zebra NMS 10%',         'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.10, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
            #         {'label': 'Zebra NMS 20%',         'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.20, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
            #         {'label': 'Zebra NMS 30%',         'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.30, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
            #         {'label': 'Zebra NMS 40%',         'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.40, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
            #         {'label': 'Zebra NMS 50%',         'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.50, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
            #         {'label': 'Zebra NMS 60%',         'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.60, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
            #         {'label': 'Zebra NMS 70%',         'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.70, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
            #         {'label': 'Zebra NMS 80%',         'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.80, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
            #         {'label': 'Zebra NMS 90%',         'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.90, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
            #         {'label': 'Zebra NMS 100%',        'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 1.00, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
            #     ],
            #     {},
            # ),
            # 'ggr2-!giraffe-azure': (
            #     [
            #         {'label': 'Giraffe ! NMS 0%',          'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.00, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
            #         {'label': 'Giraffe ! NMS 10%',         'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.10, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
            #         {'label': 'Giraffe ! NMS 20%',         'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.20, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
            #         {'label': 'Giraffe ! NMS 30%',         'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.30, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
            #         {'label': 'Giraffe ! NMS 40%',         'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.40, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
            #         {'label': 'Giraffe ! NMS 50%',         'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.50, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
            #         {'label': 'Giraffe ! NMS 60%',         'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.60, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
            #         {'label': 'Giraffe ! NMS 70%',         'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.70, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
            #         {'label': 'Giraffe ! NMS 80%',         'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.80, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
            #         {'label': 'Giraffe ! NMS 90%',         'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.90, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
            #         {'label': 'Giraffe ! NMS 100%',        'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 1.00, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
            #     ],
            #     {},
            # ),
            # 'ggr2-!zebra-azure': (
            #     [
            #         {'label': 'Zebra ! NMS 0%',          'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.00, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
            #         {'label': 'Zebra ! NMS 10%',         'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.10, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
            #         {'label': 'Zebra ! NMS 20%',         'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.20, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
            #         {'label': 'Zebra ! NMS 30%',         'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.30, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
            #         {'label': 'Zebra ! NMS 40%',         'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.40, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
            #         {'label': 'Zebra ! NMS 50%',         'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.50, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
            #         {'label': 'Zebra ! NMS 60%',         'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.60, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
            #         {'label': 'Zebra ! NMS 70%',         'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.70, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
            #         {'label': 'Zebra ! NMS 80%',         'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.80, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
            #         {'label': 'Zebra ! NMS 90%',         'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.90, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
            #         {'label': 'Zebra ! NMS 100%',        'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 1.00, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
            #     ],
            #     {},
            # ),
            # 'lynx': (
            #     [
            #         {'label': 'Lynx NMS 0%',            'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'lynx', 'weight_filepath' : 'lynx', 'nms': True, 'nms_thresh': 0.00, 'species_set' : set(['lynx'])},
            #         {'label': 'Lynx NMS 10%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'lynx', 'weight_filepath' : 'lynx', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['lynx'])},
            #         {'label': 'Lynx NMS 20%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'lynx', 'weight_filepath' : 'lynx', 'nms': True, 'nms_thresh': 0.20, 'species_set' : set(['lynx'])},
            #         {'label': 'Lynx NMS 30%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'lynx', 'weight_filepath' : 'lynx', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['lynx'])},
            #         {'label': 'Lynx NMS 40%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'lynx', 'weight_filepath' : 'lynx', 'nms': True, 'nms_thresh': 0.40, 'species_set' : set(['lynx'])},
            #         {'label': 'Lynx NMS 50%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'lynx', 'weight_filepath' : 'lynx', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['lynx'])},
            #         {'label': 'Lynx NMS 60%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'lynx', 'weight_filepath' : 'lynx', 'nms': True, 'nms_thresh': 0.60, 'species_set' : set(['lynx'])},
            #         {'label': 'Lynx NMS 70%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'lynx', 'weight_filepath' : 'lynx', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['lynx'])},
            #         {'label': 'Lynx NMS 80%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'lynx', 'weight_filepath' : 'lynx', 'nms': True, 'nms_thresh': 0.80, 'species_set' : set(['lynx'])},
            #         {'label': 'Lynx NMS 90%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'lynx', 'weight_filepath' : 'lynx', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['lynx'])},
            #         {'label': 'Lynx NMS 100%',          'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'lynx', 'weight_filepath' : 'lynx', 'nms': True, 'nms_thresh': 1.00, 'species_set' : set(['lynx'])},
            #     ],
            #     {},
            # ),
            # 'jaguar': (
            #     [
            #         {'label': 'Jaguar NMS 0%',            'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.00, 'species_set' : set(['jaguar'])},
            #         {'label': 'Jaguar NMS 10%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['jaguar'])},
            #         {'label': 'Jaguar NMS 20%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.20, 'species_set' : set(['jaguar'])},
            #         {'label': 'Jaguar NMS 30%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['jaguar'])},
            #         {'label': 'Jaguar NMS 40%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.40, 'species_set' : set(['jaguar'])},
            #         {'label': 'Jaguar NMS 50%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['jaguar'])},
            #         {'label': 'Jaguar NMS 60%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.60, 'species_set' : set(['jaguar'])},
            #         {'label': 'Jaguar NMS 70%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['jaguar'])},
            #         {'label': 'Jaguar NMS 80%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.80, 'species_set' : set(['jaguar'])},
            #         {'label': 'Jaguar NMS 90%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['jaguar'])},
            #         {'label': 'Jaguar NMS 100%',          'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 1.00, 'species_set' : set(['jaguar'])},
            #     ],
            #     {},
            # ),
            # '!jaguar': (
            #     [
            #         {'label': 'Jaguar NMS 0%',            'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.00, 'species_set' : set(['!jaguar'])},
            #         {'label': 'Jaguar NMS 10%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['!jaguar'])},
            #         {'label': 'Jaguar NMS 20%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.20, 'species_set' : set(['!jaguar'])},
            #         {'label': 'Jaguar NMS 30%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['!jaguar'])},
            #         {'label': 'Jaguar NMS 40%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.40, 'species_set' : set(['!jaguar'])},
            #         {'label': 'Jaguar NMS 50%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['!jaguar'])},
            #         {'label': 'Jaguar NMS 60%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.60, 'species_set' : set(['!jaguar'])},
            #         {'label': 'Jaguar NMS 70%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['!jaguar'])},
            #         {'label': 'Jaguar NMS 80%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.80, 'species_set' : set(['!jaguar'])},
            #         {'label': 'Jaguar NMS 90%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['!jaguar'])},
            #         {'label': 'Jaguar NMS 100%',          'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 1.00, 'species_set' : set(['!jaguar'])},
            #     ],
            #     {},
            # ),
            # 'manta': (
            #     [
            #         {'label': 'Manta NMS 0%',            'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.00, 'species_set' : set(['manta_ray_giant'])},
            #         {'label': 'Manta NMS 10%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['manta_ray_giant'])},
            #         {'label': 'Manta NMS 20%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.20, 'species_set' : set(['manta_ray_giant'])},
            #         {'label': 'Manta NMS 30%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['manta_ray_giant'])},
            #         {'label': 'Manta NMS 40%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.40, 'species_set' : set(['manta_ray_giant'])},
            #         {'label': 'Manta NMS 50%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['manta_ray_giant'])},
            #         {'label': 'Manta NMS 60%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.60, 'species_set' : set(['manta_ray_giant'])},
            #         {'label': 'Manta NMS 70%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['manta_ray_giant'])},
            #         {'label': 'Manta NMS 80%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.80, 'species_set' : set(['manta_ray_giant'])},
            #         {'label': 'Manta NMS 90%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['manta_ray_giant'])},
            #         {'label': 'Manta NMS 100%',          'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 1.00, 'species_set' : set(['manta_ray_giant'])},
            #     ],
            #     {},
            # ),
            # '!manta': (
            #     [
            #         {'label': 'Manta NMS 0%',            'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.00, 'species_set' : set(['!manta_ray_giant'])},
            #         {'label': 'Manta NMS 10%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['!manta_ray_giant'])},
            #         {'label': 'Manta NMS 20%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.20, 'species_set' : set(['!manta_ray_giant'])},
            #         {'label': 'Manta NMS 30%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['!manta_ray_giant'])},
            #         {'label': 'Manta NMS 40%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.40, 'species_set' : set(['!manta_ray_giant'])},
            #         {'label': 'Manta NMS 50%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['!manta_ray_giant'])},
            #         {'label': 'Manta NMS 60%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.60, 'species_set' : set(['!manta_ray_giant'])},
            #         {'label': 'Manta NMS 70%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['!manta_ray_giant'])},
            #         {'label': 'Manta NMS 80%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.80, 'species_set' : set(['!manta_ray_giant'])},
            #         {'label': 'Manta NMS 90%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['!manta_ray_giant'])},
            #         {'label': 'Manta NMS 100%',          'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 1.00, 'species_set' : set(['!manta_ray_giant'])},
            #     ],
            #     {},
            # ),
            # 'giraffe': (
            #     [
            #         {'label': 'Giraffe NMS 10%',                 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'giraffe_v1', 'weight_filepath' : 'giraffe_v1', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['giraffe_masai', 'giraffe_reticulated'])},
            #         {'label': 'Giraffe NMS 30%',                 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'giraffe_v1', 'weight_filepath' : 'giraffe_v1', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['giraffe_masai', 'giraffe_reticulated'])},
            #         {'label': 'Giraffe NMS 50%',                 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'giraffe_v1', 'weight_filepath' : 'giraffe_v1', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['giraffe_masai', 'giraffe_reticulated'])},
            #         {'label': 'Giraffe NMS 70%',                 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'giraffe_v1', 'weight_filepath' : 'giraffe_v1', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['giraffe_masai', 'giraffe_reticulated'])},
            #         {'label': 'Giraffe NMS 90%',                 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'giraffe_v1', 'weight_filepath' : 'giraffe_v1', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['giraffe_masai', 'giraffe_reticulated'])},
            #         {'label': 'Masai Giraffe NMS 10%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'giraffe_v1', 'weight_filepath' : 'giraffe_v1', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['giraffe_masai'])},
            #         {'label': 'Masai Giraffe NMS 30%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'giraffe_v1', 'weight_filepath' : 'giraffe_v1', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['giraffe_masai'])},
            #         {'label': 'Masai Giraffe NMS 50%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'giraffe_v1', 'weight_filepath' : 'giraffe_v1', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['giraffe_masai'])},
            #         {'label': 'Masai Giraffe NMS 70%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'giraffe_v1', 'weight_filepath' : 'giraffe_v1', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['giraffe_masai'])},
            #         {'label': 'Masai Giraffe NMS 90%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'giraffe_v1', 'weight_filepath' : 'giraffe_v1', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['giraffe_masai'])},
            #         {'label': 'Reticulated Giraffe NMS 10%',     'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'giraffe_v1', 'weight_filepath' : 'giraffe_v1', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['giraffe_reticulated'])},
            #         {'label': 'Reticulated Giraffe NMS 30%',     'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'giraffe_v1', 'weight_filepath' : 'giraffe_v1', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['giraffe_reticulated'])},
            #         {'label': 'Reticulated Giraffe NMS 50%',     'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'giraffe_v1', 'weight_filepath' : 'giraffe_v1', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['giraffe_reticulated'])},
            #         {'label': 'Reticulated Giraffe NMS 70%',     'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'giraffe_v1', 'weight_filepath' : 'giraffe_v1', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['giraffe_reticulated'])},
            #         {'label': 'Reticulated Giraffe NMS 90%',     'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'giraffe_v1', 'weight_filepath' : 'giraffe_v1', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['giraffe_reticulated'])},
            #     ],
            #     {},
            # ),
            # 'spotted_skunk_v0': (
            #     [
            #         {'label': 'Spotted Skunk NMS 0%',            'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.00, 'species_set' : set(['skunk_spotted'])},
            #         {'label': 'Spotted Skunk NMS 10%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['skunk_spotted'])},
            #         {'label': 'Spotted Skunk NMS 20%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.20, 'species_set' : set(['skunk_spotted'])},
            #         {'label': 'Spotted Skunk NMS 30%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['skunk_spotted'])},
            #         {'label': 'Spotted Skunk NMS 40%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.40, 'species_set' : set(['skunk_spotted'])},
            #         {'label': 'Spotted Skunk NMS 50%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['skunk_spotted'])},
            #         {'label': 'Spotted Skunk NMS 60%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.60, 'species_set' : set(['skunk_spotted'])},
            #         {'label': 'Spotted Skunk NMS 70%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['skunk_spotted'])},
            #         {'label': 'Spotted Skunk NMS 80%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.80, 'species_set' : set(['skunk_spotted'])},
            #         {'label': 'Spotted Skunk NMS 90%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['skunk_spotted'])},
            #         {'label': 'Spotted Skunk NMS 100%',          'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 1.00, 'species_set' : set(['skunk_spotted'])},
            #     ],
            #     {},
            # ),
            # '!spotted_skunk_v0': (
            #     [
            #         {'label': 'Spotted Skunk NMS 0%',            'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.00, 'species_set' : set(['!skunk_spotted'])},
            #         {'label': 'Spotted Skunk NMS 10%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['!skunk_spotted'])},
            #         {'label': 'Spotted Skunk NMS 20%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.20, 'species_set' : set(['!skunk_spotted'])},
            #         {'label': 'Spotted Skunk NMS 30%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['!skunk_spotted'])},
            #         {'label': 'Spotted Skunk NMS 40%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.40, 'species_set' : set(['!skunk_spotted'])},
            #         {'label': 'Spotted Skunk NMS 50%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['!skunk_spotted'])},
            #         {'label': 'Spotted Skunk NMS 60%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.60, 'species_set' : set(['!skunk_spotted'])},
            #         {'label': 'Spotted Skunk NMS 70%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['!skunk_spotted'])},
            #         {'label': 'Spotted Skunk NMS 80%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.80, 'species_set' : set(['!skunk_spotted'])},
            #         {'label': 'Spotted Skunk NMS 90%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['!skunk_spotted'])},
            #         {'label': 'Spotted Skunk NMS 100%',          'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 1.00, 'species_set' : set(['!skunk_spotted'])},
            #     ],
            #     {},
            # ),
            # 'nassau_grouper_v0': (
            #     [
            #         {'label': 'Nassau Grouper NMS 0%',            'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.00, 'species_set' : set(['grouper_nassau'])},
            #         {'label': 'Nassau Grouper NMS 10%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['grouper_nassau'])},
            #         {'label': 'Nassau Grouper NMS 20%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.20, 'species_set' : set(['grouper_nassau'])},
            #         {'label': 'Nassau Grouper NMS 30%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['grouper_nassau'])},
            #         {'label': 'Nassau Grouper NMS 40%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.40, 'species_set' : set(['grouper_nassau'])},
            #         {'label': 'Nassau Grouper NMS 50%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['grouper_nassau'])},
            #         {'label': 'Nassau Grouper NMS 60%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.60, 'species_set' : set(['grouper_nassau'])},
            #         {'label': 'Nassau Grouper NMS 70%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['grouper_nassau'])},
            #         {'label': 'Nassau Grouper NMS 80%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.80, 'species_set' : set(['grouper_nassau'])},
            #         {'label': 'Nassau Grouper NMS 90%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['grouper_nassau'])},
            #         {'label': 'Nassau Grouper NMS 100%',          'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 1.00, 'species_set' : set(['grouper_nassau'])},
            #     ],
            #     {},
            # ),
            # '!nassau_grouper_v0': (
            #     [
            #         {'label': 'Nassau Grouper! NMS 0%',            'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.00, 'species_set' : set(['!grouper_nassau'])},
            #         {'label': 'Nassau Grouper! NMS 10%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['!grouper_nassau'])},
            #         {'label': 'Nassau Grouper! NMS 20%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.20, 'species_set' : set(['!grouper_nassau'])},
            #         {'label': 'Nassau Grouper! NMS 30%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['!grouper_nassau'])},
            #         {'label': 'Nassau Grouper! NMS 40%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.40, 'species_set' : set(['!grouper_nassau'])},
            #         {'label': 'Nassau Grouper! NMS 50%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['!grouper_nassau'])},
            #         {'label': 'Nassau Grouper! NMS 60%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.60, 'species_set' : set(['!grouper_nassau'])},
            #         {'label': 'Nassau Grouper! NMS 70%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['!grouper_nassau'])},
            #         {'label': 'Nassau Grouper! NMS 80%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.80, 'species_set' : set(['!grouper_nassau'])},
            #         {'label': 'Nassau Grouper! NMS 90%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['!grouper_nassau'])},
            #         {'label': 'Nassau Grouper! NMS 100%',          'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 1.00, 'species_set' : set(['!grouper_nassau'])},
            #     ],
            #     {},
            # ),
            # 'spotted_dolphin_v0': (
            #     [
            #         {'label': 'Spotted DolphinNMS 0%',            'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.00, 'species_set' : set(['dolphin_spotted'])},
            #         {'label': 'Spotted DolphinNMS 10%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['dolphin_spotted'])},
            #         {'label': 'Spotted DolphinNMS 20%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.20, 'species_set' : set(['dolphin_spotted'])},
            #         {'label': 'Spotted DolphinNMS 30%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['dolphin_spotted'])},
            #         {'label': 'Spotted DolphinNMS 40%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.40, 'species_set' : set(['dolphin_spotted'])},
            #         {'label': 'Spotted DolphinNMS 50%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['dolphin_spotted'])},
            #         {'label': 'Spotted DolphinNMS 60%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.60, 'species_set' : set(['dolphin_spotted'])},
            #         {'label': 'Spotted DolphinNMS 70%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['dolphin_spotted'])},
            #         {'label': 'Spotted DolphinNMS 80%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.80, 'species_set' : set(['dolphin_spotted'])},
            #         {'label': 'Spotted DolphinNMS 90%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['dolphin_spotted'])},
            #         {'label': 'Spotted DolphinNMS 100%',          'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 1.00, 'species_set' : set(['dolphin_spotted'])},
            #     ],
            #     {},
            # ),
            # '!spotted_dolphin_v0': (
            #     [
            #         {'label': 'Spotted Dolphin! NMS 0%',            'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.00, 'species_set' : set(['!dolphin_spotted'])},
            #         {'label': 'Spotted Dolphin! NMS 10%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['!dolphin_spotted'])},
            #         {'label': 'Spotted Dolphin! NMS 20%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.20, 'species_set' : set(['!dolphin_spotted'])},
            #         {'label': 'Spotted Dolphin! NMS 30%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['!dolphin_spotted'])},
            #         {'label': 'Spotted Dolphin! NMS 40%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.40, 'species_set' : set(['!dolphin_spotted'])},
            #         {'label': 'Spotted Dolphin! NMS 50%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['!dolphin_spotted'])},
            #         {'label': 'Spotted Dolphin! NMS 60%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.60, 'species_set' : set(['!dolphin_spotted'])},
            #         {'label': 'Spotted Dolphin! NMS 70%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['!dolphin_spotted'])},
            #         {'label': 'Spotted Dolphin! NMS 80%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.80, 'species_set' : set(['!dolphin_spotted'])},
            #         {'label': 'Spotted Dolphin! NMS 90%',           'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['!dolphin_spotted'])},
            #         {'label': 'Spotted Dolphin! NMS 100%',          'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 1.00, 'species_set' : set(['!dolphin_spotted'])},
            #     ],
            #     {},
            # ),
            'seadragon_weedy_v1': (
                [
                    {
                        'label': 'Weedy Body NMS 0%',
                        'grid': False,
                        'algo': 'lightnet',
                        'config_filepath': 'seadragon_v1',
                        'weight_filepath': 'seadragon_v1',
                        'nms': True,
                        'nms_thresh': 0.00,
                        'species_set': {'seadragon_leafy'},
                    },
                    {
                        'label': 'Weedy Body NMS 10%',
                        'grid': False,
                        'algo': 'lightnet',
                        'config_filepath': 'seadragon_v1',
                        'weight_filepath': 'seadragon_v1',
                        'nms': True,
                        'nms_thresh': 0.10,
                        'species_set': {'seadragon_leafy'},
                    },
                    {
                        'label': 'Weedy Body NMS 20%',
                        'grid': False,
                        'algo': 'lightnet',
                        'config_filepath': 'seadragon_v1',
                        'weight_filepath': 'seadragon_v1',
                        'nms': True,
                        'nms_thresh': 0.20,
                        'species_set': {'seadragon_leafy'},
                    },
                    {
                        'label': 'Weedy Body NMS 30%',
                        'grid': False,
                        'algo': 'lightnet',
                        'config_filepath': 'seadragon_v1',
                        'weight_filepath': 'seadragon_v1',
                        'nms': True,
                        'nms_thresh': 0.30,
                        'species_set': {'seadragon_leafy'},
                    },
                    {
                        'label': 'Weedy Body NMS 40%',
                        'grid': False,
                        'algo': 'lightnet',
                        'config_filepath': 'seadragon_v1',
                        'weight_filepath': 'seadragon_v1',
                        'nms': True,
                        'nms_thresh': 0.40,
                        'species_set': {'seadragon_leafy'},
                    },
                    {
                        'label': 'Weedy Body NMS 50%',
                        'grid': False,
                        'algo': 'lightnet',
                        'config_filepath': 'seadragon_v1',
                        'weight_filepath': 'seadragon_v1',
                        'nms': True,
                        'nms_thresh': 0.50,
                        'species_set': {'seadragon_leafy'},
                    },
                    {
                        'label': 'Weedy Body NMS 60%',
                        'grid': False,
                        'algo': 'lightnet',
                        'config_filepath': 'seadragon_v1',
                        'weight_filepath': 'seadragon_v1',
                        'nms': True,
                        'nms_thresh': 0.60,
                        'species_set': {'seadragon_leafy'},
                    },
                    {
                        'label': 'Weedy Body NMS 70%',
                        'grid': False,
                        'algo': 'lightnet',
                        'config_filepath': 'seadragon_v1',
                        'weight_filepath': 'seadragon_v1',
                        'nms': True,
                        'nms_thresh': 0.70,
                        'species_set': {'seadragon_leafy'},
                    },
                    {
                        'label': 'Weedy Body NMS 80%',
                        'grid': False,
                        'algo': 'lightnet',
                        'config_filepath': 'seadragon_v1',
                        'weight_filepath': 'seadragon_v1',
                        'nms': True,
                        'nms_thresh': 0.80,
                        'species_set': {'seadragon_leafy'},
                    },
                    {
                        'label': 'Weedy Body NMS 90%',
                        'grid': False,
                        'algo': 'lightnet',
                        'config_filepath': 'seadragon_v1',
                        'weight_filepath': 'seadragon_v1',
                        'nms': True,
                        'nms_thresh': 0.90,
                        'species_set': {'seadragon_leafy'},
                    },
                    {
                        'label': 'Weedy Body NMS 100%',
                        'grid': False,
                        'algo': 'lightnet',
                        'config_filepath': 'seadragon_v1',
                        'weight_filepath': 'seadragon_v1',
                        'nms': True,
                        'nms_thresh': 1.00,
                        'species_set': {'seadragon_leafy'},
                    },
                ],
                {},
            ),
            'seadragon_leafy_v1': (
                [
                    {
                        'label': 'Leafy Body NMS 0%',
                        'grid': False,
                        'algo': 'lightnet',
                        'config_filepath': 'seadragon_v1',
                        'weight_filepath': 'seadragon_v1',
                        'nms': True,
                        'nms_thresh': 0.00,
                        'species_set': {'seadragon_weedy'},
                    },
                    {
                        'label': 'Leafy Body NMS 10%',
                        'grid': False,
                        'algo': 'lightnet',
                        'config_filepath': 'seadragon_v1',
                        'weight_filepath': 'seadragon_v1',
                        'nms': True,
                        'nms_thresh': 0.10,
                        'species_set': {'seadragon_weedy'},
                    },
                    {
                        'label': 'Leafy Body NMS 20%',
                        'grid': False,
                        'algo': 'lightnet',
                        'config_filepath': 'seadragon_v1',
                        'weight_filepath': 'seadragon_v1',
                        'nms': True,
                        'nms_thresh': 0.20,
                        'species_set': {'seadragon_weedy'},
                    },
                    {
                        'label': 'Leafy Body NMS 30%',
                        'grid': False,
                        'algo': 'lightnet',
                        'config_filepath': 'seadragon_v1',
                        'weight_filepath': 'seadragon_v1',
                        'nms': True,
                        'nms_thresh': 0.30,
                        'species_set': {'seadragon_weedy'},
                    },
                    {
                        'label': 'Leafy Body NMS 40%',
                        'grid': False,
                        'algo': 'lightnet',
                        'config_filepath': 'seadragon_v1',
                        'weight_filepath': 'seadragon_v1',
                        'nms': True,
                        'nms_thresh': 0.40,
                        'species_set': {'seadragon_weedy'},
                    },
                    {
                        'label': 'Leafy Body NMS 50%',
                        'grid': False,
                        'algo': 'lightnet',
                        'config_filepath': 'seadragon_v1',
                        'weight_filepath': 'seadragon_v1',
                        'nms': True,
                        'nms_thresh': 0.50,
                        'species_set': {'seadragon_weedy'},
                    },
                    {
                        'label': 'Leafy Body NMS 60%',
                        'grid': False,
                        'algo': 'lightnet',
                        'config_filepath': 'seadragon_v1',
                        'weight_filepath': 'seadragon_v1',
                        'nms': True,
                        'nms_thresh': 0.60,
                        'species_set': {'seadragon_weedy'},
                    },
                    {
                        'label': 'Leafy Body NMS 70%',
                        'grid': False,
                        'algo': 'lightnet',
                        'config_filepath': 'seadragon_v1',
                        'weight_filepath': 'seadragon_v1',
                        'nms': True,
                        'nms_thresh': 0.70,
                        'species_set': {'seadragon_weedy'},
                    },
                    {
                        'label': 'Leafy Body NMS 80%',
                        'grid': False,
                        'algo': 'lightnet',
                        'config_filepath': 'seadragon_v1',
                        'weight_filepath': 'seadragon_v1',
                        'nms': True,
                        'nms_thresh': 0.80,
                        'species_set': {'seadragon_weedy'},
                    },
                    {
                        'label': 'Leafy Body NMS 90%',
                        'grid': False,
                        'algo': 'lightnet',
                        'config_filepath': 'seadragon_v1',
                        'weight_filepath': 'seadragon_v1',
                        'nms': True,
                        'nms_thresh': 0.90,
                        'species_set': {'seadragon_weedy'},
                    },
                    {
                        'label': 'Leafy Body NMS 100%',
                        'grid': False,
                        'algo': 'lightnet',
                        'config_filepath': 'seadragon_v1',
                        'weight_filepath': 'seadragon_v1',
                        'nms': True,
                        'nms_thresh': 1.00,
                        'species_set': {'seadragon_weedy'},
                    },
                ],
                {},
            ),
            'seadragon_weedy_head_v1': (
                [
                    {
                        'label': 'Weedy Head NMS 0%',
                        'grid': False,
                        'algo': 'lightnet',
                        'config_filepath': 'seadragon_v1',
                        'weight_filepath': 'seadragon_v1',
                        'nms': True,
                        'nms_thresh': 0.00,
                        'species_set': {'seadragon_leafy+head'},
                    },
                    {
                        'label': 'Weedy Head NMS 10%',
                        'grid': False,
                        'algo': 'lightnet',
                        'config_filepath': 'seadragon_v1',
                        'weight_filepath': 'seadragon_v1',
                        'nms': True,
                        'nms_thresh': 0.10,
                        'species_set': {'seadragon_leafy+head'},
                    },
                    {
                        'label': 'Weedy Head NMS 20%',
                        'grid': False,
                        'algo': 'lightnet',
                        'config_filepath': 'seadragon_v1',
                        'weight_filepath': 'seadragon_v1',
                        'nms': True,
                        'nms_thresh': 0.20,
                        'species_set': {'seadragon_leafy+head'},
                    },
                    {
                        'label': 'Weedy Head NMS 30%',
                        'grid': False,
                        'algo': 'lightnet',
                        'config_filepath': 'seadragon_v1',
                        'weight_filepath': 'seadragon_v1',
                        'nms': True,
                        'nms_thresh': 0.30,
                        'species_set': {'seadragon_leafy+head'},
                    },
                    {
                        'label': 'Weedy Head NMS 40%',
                        'grid': False,
                        'algo': 'lightnet',
                        'config_filepath': 'seadragon_v1',
                        'weight_filepath': 'seadragon_v1',
                        'nms': True,
                        'nms_thresh': 0.40,
                        'species_set': {'seadragon_leafy+head'},
                    },
                    {
                        'label': 'Weedy Head NMS 50%',
                        'grid': False,
                        'algo': 'lightnet',
                        'config_filepath': 'seadragon_v1',
                        'weight_filepath': 'seadragon_v1',
                        'nms': True,
                        'nms_thresh': 0.50,
                        'species_set': {'seadragon_leafy+head'},
                    },
                    {
                        'label': 'Weedy Head NMS 60%',
                        'grid': False,
                        'algo': 'lightnet',
                        'config_filepath': 'seadragon_v1',
                        'weight_filepath': 'seadragon_v1',
                        'nms': True,
                        'nms_thresh': 0.60,
                        'species_set': {'seadragon_leafy+head'},
                    },
                    {
                        'label': 'Weedy Head NMS 70%',
                        'grid': False,
                        'algo': 'lightnet',
                        'config_filepath': 'seadragon_v1',
                        'weight_filepath': 'seadragon_v1',
                        'nms': True,
                        'nms_thresh': 0.70,
                        'species_set': {'seadragon_leafy+head'},
                    },
                    {
                        'label': 'Weedy Head NMS 80%',
                        'grid': False,
                        'algo': 'lightnet',
                        'config_filepath': 'seadragon_v1',
                        'weight_filepath': 'seadragon_v1',
                        'nms': True,
                        'nms_thresh': 0.80,
                        'species_set': {'seadragon_leafy+head'},
                    },
                    {
                        'label': 'Weedy Head NMS 90%',
                        'grid': False,
                        'algo': 'lightnet',
                        'config_filepath': 'seadragon_v1',
                        'weight_filepath': 'seadragon_v1',
                        'nms': True,
                        'nms_thresh': 0.90,
                        'species_set': {'seadragon_leafy+head'},
                    },
                    {
                        'label': 'Weedy Head NMS 100%',
                        'grid': False,
                        'algo': 'lightnet',
                        'config_filepath': 'seadragon_v1',
                        'weight_filepath': 'seadragon_v1',
                        'nms': True,
                        'nms_thresh': 1.00,
                        'species_set': {'seadragon_leafy+head'},
                    },
                ],
                {},
            ),
            'seadragon_leafy_head_v1': (
                [
                    {
                        'label': 'Leafy Head NMS 0%',
                        'grid': False,
                        'algo': 'lightnet',
                        'config_filepath': 'seadragon_v1',
                        'weight_filepath': 'seadragon_v1',
                        'nms': True,
                        'nms_thresh': 0.00,
                        'species_set': {'seadragon_weedy+head'},
                    },
                    {
                        'label': 'Leafy Head NMS 10%',
                        'grid': False,
                        'algo': 'lightnet',
                        'config_filepath': 'seadragon_v1',
                        'weight_filepath': 'seadragon_v1',
                        'nms': True,
                        'nms_thresh': 0.10,
                        'species_set': {'seadragon_weedy+head'},
                    },
                    {
                        'label': 'Leafy Head NMS 20%',
                        'grid': False,
                        'algo': 'lightnet',
                        'config_filepath': 'seadragon_v1',
                        'weight_filepath': 'seadragon_v1',
                        'nms': True,
                        'nms_thresh': 0.20,
                        'species_set': {'seadragon_weedy+head'},
                    },
                    {
                        'label': 'Leafy Head NMS 30%',
                        'grid': False,
                        'algo': 'lightnet',
                        'config_filepath': 'seadragon_v1',
                        'weight_filepath': 'seadragon_v1',
                        'nms': True,
                        'nms_thresh': 0.30,
                        'species_set': {'seadragon_weedy+head'},
                    },
                    {
                        'label': 'Leafy Head NMS 40%',
                        'grid': False,
                        'algo': 'lightnet',
                        'config_filepath': 'seadragon_v1',
                        'weight_filepath': 'seadragon_v1',
                        'nms': True,
                        'nms_thresh': 0.40,
                        'species_set': {'seadragon_weedy+head'},
                    },
                    {
                        'label': 'Leafy Head NMS 50%',
                        'grid': False,
                        'algo': 'lightnet',
                        'config_filepath': 'seadragon_v1',
                        'weight_filepath': 'seadragon_v1',
                        'nms': True,
                        'nms_thresh': 0.50,
                        'species_set': {'seadragon_weedy+head'},
                    },
                    {
                        'label': 'Leafy Head NMS 60%',
                        'grid': False,
                        'algo': 'lightnet',
                        'config_filepath': 'seadragon_v1',
                        'weight_filepath': 'seadragon_v1',
                        'nms': True,
                        'nms_thresh': 0.60,
                        'species_set': {'seadragon_weedy+head'},
                    },
                    {
                        'label': 'Leafy Head NMS 70%',
                        'grid': False,
                        'algo': 'lightnet',
                        'config_filepath': 'seadragon_v1',
                        'weight_filepath': 'seadragon_v1',
                        'nms': True,
                        'nms_thresh': 0.70,
                        'species_set': {'seadragon_weedy+head'},
                    },
                    {
                        'label': 'Leafy Head NMS 80%',
                        'grid': False,
                        'algo': 'lightnet',
                        'config_filepath': 'seadragon_v1',
                        'weight_filepath': 'seadragon_v1',
                        'nms': True,
                        'nms_thresh': 0.80,
                        'species_set': {'seadragon_weedy+head'},
                    },
                    {
                        'label': 'Leafy Head NMS 90%',
                        'grid': False,
                        'algo': 'lightnet',
                        'config_filepath': 'seadragon_v1',
                        'weight_filepath': 'seadragon_v1',
                        'nms': True,
                        'nms_thresh': 0.90,
                        'species_set': {'seadragon_weedy+head'},
                    },
                    {
                        'label': 'Leafy Head NMS 100%',
                        'grid': False,
                        'algo': 'lightnet',
                        'config_filepath': 'seadragon_v1',
                        'weight_filepath': 'seadragon_v1',
                        'nms': True,
                        'nms_thresh': 1.00,
                        'species_set': {'seadragon_weedy+head'},
                    },
                ],
                {},
            ),
        }

    for config_key in config_dict:
        config_list, config = config_dict[config_key]

        for key in kwargs:
            config[key] = kwargs[key]

        # Backwards compatibility hack
        if test_gid_list is not None:
            for config_ in config_list:
                if overwrite_config_keys or 'test_gid_list' not in config_:
                    config_['test_gid_list'] = test_gid_list

        if ignore_filter_func is not None:
            for config_ in config_list:
                if overwrite_config_keys or 'ignore_filter_func' not in config_:
                    config_['ignore_filter_func'] = ignore_filter_func

        ibs.localizer_precision_recall_algo_display(
            config_list, config_tag=config_key, output_path=output_path, **config
        )


@register_ibs_method
def localizer_precision_recall_algo_display(
    ibs,
    config_list,
    config_tag='',
    min_overlap=0.5,
    figsize=(40, 9),
    target_recall=0.8,
    BEST_INDEX=None,
    offset_color=0,
    write_images=False,
    plot_point=True,
    output_path=None,
    plot_iou_recall=True,
    **kwargs,
):
    import matplotlib.pyplot as plt

    import wbia.plottool as pt

    if output_path is None:
        output_path = abspath(expanduser(join('~', 'Desktop')))

    color_list_ = []
    for _ in range(offset_color):
        color_list_ += [(0.2, 0.2, 0.2)]

    color_list = pt.distinct_colors(len(config_list) - len(color_list_), randomize=False)
    color_list = color_list_ + color_list

    fig_ = plt.figure(figsize=figsize, dpi=400)

    ######################################################################################

    axes_ = plt.subplot(141)
    axes_.set_autoscalex_on(False)
    axes_.set_autoscaley_on(False)
    axes_.set_xlabel('Recall (Ground-Truth IOU >= {:0.02f})'.format(min_overlap))
    axes_.set_ylabel('Precision')
    axes_.set_xlim([0.0, 1.01])
    axes_.set_ylim([0.0, 1.01])

    ret_list = [
        localizer_precision_recall_algo_plot(
            ibs,
            color=color,
            min_overlap=min_overlap,
            plot_point=plot_point,
            target_recall=target_recall,
            **config,
        )
        for color, config in zip(color_list, config_list)
    ]

    area_list = [ret[0] for ret in ret_list]
    tup2_list = [ret[3] for ret in ret_list]

    best_index = (
        None if BEST_INDEX is None else BEST_INDEX
    )  # Match formatting of below, this is a silly conditional

    best_y = 0.0
    best_index_ = None
    valid_best_index = []
    for index, tup2 in enumerate(tup2_list):
        if tup2 is None:
            continue

        conf_list, x_list, y_list, length = tup2
        y = y_list[0]
        if best_y < y:
            valid_best_index.append(index)
            best_index_ = index
            best_y = y

    # If user defined best_index is invalid, don't use it
    if best_index is None:
        best_index = best_index_
    else:
        if best_index not in valid_best_index:
            best_index = None

    if best_index is not None:
        best_conf_list, best_x_list, best_y_list, best_length = tup2_list[best_index]
        color = 'xkcd:gold'
        marker = 'D'
        plt.plot(best_x_list, best_y_list, color=color, marker=marker)

    plt.title('Precision-Recall Curves', y=1.19)
    plt.legend(
        bbox_to_anchor=(0.0, 1.02, 1.0, 0.102),
        loc=3,
        ncol=2,
        mode='expand',
        borderaxespad=0.0,
    )

    ######################################################################################

    if plot_iou_recall:
        axes_ = plt.subplot(142)
        axes_.set_autoscalex_on(False)
        axes_.set_autoscaley_on(False)
        axes_.set_xlabel('IOU (Intersection / Union)')
        axes_.set_ylabel('Recall')
        axes_.set_xlim([0.0, 1.01])
        axes_.set_ylim([0.0, 1.01])

        ret_list = [
            localizer_iou_recall_algo_plot(ibs, color=color_, plot_point=False, **config_)
            for color_, config_ in zip(color_list, config_list)
        ]
        plt.plot(
            [min_overlap, min_overlap], [0.0, 1.0], color=(0.5, 0.5, 0.5), linestyle='--'
        )

        # area_list = [ ret[0] for ret in ret_list ]
        # tup2_list = [ ret[3] for ret in ret_list ]

        # best_index = None if BEST_INDEX is None else BEST_INDEX  # Match formatting of below, this is a silly conditional

        # best_y = 0.0
        # best_index_ = None
        # valid_best_index = []
        # for index, tup2 in enumerate(tup2_list):
        #     if tup2 is None:
        #         continue

        #     conf_list, x_list, y_list, length = tup2
        #     y = y_list[0]
        #     if best_y < y:
        #         valid_best_index.append(index)
        #         best_index_ = index
        #         best_y = y

        # # If user defined best_index is invalid, don't use it
        # if best_index is None:
        #     best_index = best_index_
        # else:
        #     if best_index not in valid_best_index:
        #         best_index = None

        # if best_index is not None:
        #     best_conf_list, best_x_list, best_y_list, best_length = tup2_list[best_index]
        #     color = 'xkcd:gold'
        #     marker = 'D'
        #     plt.plot(best_x_list, best_y_list, color=color, marker=marker)

        plt.title('Recall-IOU Curves', y=1.19)
        plt.legend(
            bbox_to_anchor=(0.0, 1.02, 1.0, 0.102),
            loc=3,
            ncol=2,
            mode='expand',
            borderaxespad=0.0,
        )

    ######################################################################################

    # axes_ = plt.subplot(153)
    # axes_.set_autoscalex_on(False)
    # axes_.set_autoscaley_on(False)
    # axes_.set_xlabel('IOU (Intersection / Union)')
    # axes_.set_ylabel('Precision')
    # axes_.set_xlim([0.0, 1.01])
    # axes_.set_ylim([0.0, 1.01])

    # ret_list = [
    #     localizer_iou_precision_algo_plot(ibs, color=color_, plot_point=False, **config_)
    #     for color_, config_ in zip(color_list, config_list)
    # ]

    # plt.title('Precision-IOU Curves', y=1.19)
    # plt.legend(bbox_to_anchor=(0.0, 1.02, 1.0, .102), loc=3, ncol=2, mode="expand",
    #            borderaxespad=0.0)

    ######################################################################################

    if best_index is not None:
        axes_ = plt.subplot(144)
        axes_.set_aspect(1)
        gca_ = plt.gca()
        gca_.grid(False)

        target_conf = best_conf_list[0]
        best_config = config_list[best_index]
        best_label = config_list[best_index]['label']
        best_area = area_list[best_index]

        values = localizer_confusion_matrix_algo_plot(
            ibs,
            min_overlap=min_overlap,
            fig_=fig_,
            axes_=axes_,
            target_conf=target_conf,
            **best_config,
        )
        best_conf, (correct_rate, _) = values

        axes_.set_xlabel('Predicted (Correct = {:0.02f}%)'.format(correct_rate * 100.0))
        axes_.set_ylabel('Ground-Truth')
        args = (
            target_recall,
            best_label,
            best_area,
            best_conf,
        )
        plt.title(
            'Confusion Matrix for Recall >= %0.02f\n(Algo: %s, mAP = %0.02f, OP = %0.02f)'
            % args,
            y=1.26,
        )

    ######################################################################################
    axes_ = plt.subplot(143)
    axes_.set_aspect(1)
    gca_ = plt.gca()
    gca_.grid(False)

    best_index = np.argmax(area_list) if BEST_INDEX is None else BEST_INDEX

    best_config = config_list[best_index]
    best_label = config_list[best_index]['label']
    best_area = area_list[best_index]

    values = localizer_confusion_matrix_algo_plot(
        ibs, min_overlap=min_overlap, fig_=fig_, axes_=axes_, **best_config
    )
    best_conf, (correct_rate, _) = values

    axes_.set_xlabel('Predicted (Correct = {:0.02f}%)'.format(correct_rate * 100.0))
    axes_.set_ylabel('Ground-Truth')
    args = (
        best_label,
        best_area,
        best_conf,
    )
    plt.title('Confusion Matrix\n(Algo: %s, mAP = %0.02f, OP = %0.02f)' % args, y=1.26)

    ######################################################################################
    if len(config_tag) > 0:
        config_tag = '{}-'.format(config_tag)

    fig_filename = '{}localizer-precision-recall-{:0.2f}.png'.format(
        config_tag, min_overlap
    )
    fig_path = join(output_path, fig_filename)
    plt.savefig(fig_path, bbox_inches='tight')

    return fig_path


@register_ibs_method
def localizer_precision_recall_algo_display_animate(ibs, config_list, **kwargs):
    for value in range(10):
        min_overlap = value / 10.0
        logger.info('Processing: {!r}'.format(min_overlap))
        ibs.localizer_precision_recall_algo_display(
            config_list, min_overlap=min_overlap, **kwargs
        )


# def localizer_classification_tp_tn_fp_fn(gt_list, pred_list, conf, min_overlap,
#                                          check_species=False,
#                                          check_viewpoint=False, **kwargs):
#     overlap = general_overlap(gt_list, pred_list)
#     num_gt, num_pred = overlap.shape

#     # Get confidences
#     conf_list = [pred['confidence'] for pred in pred_list]
#     pred_flag_list = [conf <= conf_ for conf_ in conf_list]

#     if num_gt == 0:
#         tp_list = [False] * len(pred_list)
#         tn_list = [not pred_flag for pred_flag in pred_flag_list]
#         fp_list = [    pred_flag for pred_flag in pred_flag_list]
#         fn_list = [False] * len(pred_list)
#     elif num_pred == 0:
#         tp_list = []
#         tn_list = []
#         fp_list = []
#         fn_list = []
#     else:
#         max_overlap = np.max(overlap, axis=0)
#         gt_flag_list = min_overlap < max_overlap

#         status_list = []
#         for gt_flag, pred_flag in zip(gt_flag_list, pred_flag_list):
#             if gt_flag and pred_flag:
#                 status_list.append('tp')
#             elif gt_flag and not pred_flag:
#                 status_list.append('fn')
#             elif not gt_flag and pred_flag:
#                 status_list.append('fp')
#             elif not gt_flag and not pred_flag:
#                 status_list.append('tn')
#             else:
#                 raise ValueError

#         tp_list = [status == 'tp' for status in status_list]
#         tn_list = [status == 'tn' for status in status_list]
#         fp_list = [status == 'fp' for status in status_list]
#         fn_list = [status == 'fn' for status in status_list]

#     return tp_list, tn_list, fp_list, fn_list


# def localizer_classification_confusion_matrix_algo_plot(ibs, color, conf,
#                                                         label=None,
#                                                         min_overlap=0.25,
#                                                         write_images=False,
#                                                         **kwargs):
#     logger.info('Processing Confusion Matrix for: %r (Conf = %0.02f)' % (label, conf, ))

#     test_gid_list = general_get_imageset_gids(ibs, 'TEST_SET', **kwargs)
#     test_uuid_list = ibs.get_image_uuids(test_gid_list)

#     logger.info('\tGather Ground-Truth')
#     gt_dict = general_parse_gt(ibs, test_gid_list=test_gid_list, **kwargs)

#     logger.info('\tGather Predictions')
#     pred_dict = localizer_parse_pred(ibs, test_gid_list=test_gid_list, **kwargs)

#     if write_images:
#         output_folder = 'localizer-classification-confusion-matrix-%0.2f-%0.2f-images' % (min_overlap, conf, )
#         output_path = abspath(expanduser(join('~', 'Desktop', output_folder)))
#         ut.ensuredir(output_path)

#     label_list = []
#     prediction_list = []
#     for index, (test_gid, test_uuid) in enumerate(zip(test_gid_list, test_uuid_list)):
#         if test_uuid in pred_dict:
#             gt_list = gt_dict[test_uuid]
#             pred_list = pred_dict[test_uuid]
#             values = localizer_classification_tp_tn_fp_fn(gt_list, pred_list, conf,
#                                                           min_overlap=min_overlap,
#                                                           **kwargs)
#             tp_list, tn_list, fp_list, fn_list = values
#             tp = tp_list.count(True)
#             tn = tn_list.count(True)
#             fp = fp_list.count(True)
#             fn = fn_list.count(True)

#             for _ in range(int(tp)):
#                 label_list.append('positive')
#                 prediction_list.append('positive')
#             for _ in range(int(tn)):
#                 label_list.append('negative')
#                 prediction_list.append('negative')
#             for _ in range(int(fp)):
#                 label_list.append('negative')
#                 prediction_list.append('positive')
#             for _ in range(int(fn)):
#                 label_list.append('positive')
#                 prediction_list.append('negative')

#             if write_images:
#                 test_image = ibs.get_images(test_gid)
#                 test_image = _resize(test_image, t_width=600, verbose=False)
#                 height_, width_, channels_ = test_image.shape

#                 for gt in gt_list:
#                     xtl = int(gt['xtl'] * width_)
#                     ytl = int(gt['ytl'] * height_)
#                     xbr = int(gt['xbr'] * width_)
#                     ybr = int(gt['ybr'] * height_)
#                     cv2.rectangle(test_image, (xtl, ytl), (xbr, ybr), (0, 0, 255))

#                 zipped = zip(pred_list, tp_list, tn_list, fp_list, fn_list)
#                 for pred, tp_, tn_, fp_, fn_ in zipped:
#                     if tp_:
#                         color = (0, 255, 0)
#                     elif fp_:
#                         continue
#                         # color = (255, 0, 0)
#                     elif fn_:
#                         color = (255, 0, 0)
#                     elif tn_:
#                         continue
#                     else:
#                         continue

#                     xtl = int(pred['xtl'] * width_)
#                     ytl = int(pred['ytl'] * height_)
#                     xbr = int(pred['xbr'] * width_)
#                     ybr = int(pred['ybr'] * height_)
#                     cv2.rectangle(test_image, (xtl, ytl), (xbr, ybr), color)

#                 status_str = 'success' if (fp + fn) == 0 else 'failure'
#                 status_val = tp - fp - fn
#                 args = (status_str, status_val, test_gid, tp, fp, fn, )
#                 output_filename = 'test_%s_%d_gid_%d_tp_%d_fp_%d_fn_%d.png' % args
#                 output_filepath = join(output_path, output_filename)
#                 cv2.imwrite(output_filepath, test_image)

#     category_list = ['positive', 'negative']
#     category_mapping = {
#         'positive': 0,
#         'negative': 1,
#     }
#     return general_confusion_matrix_algo(label_list, prediction_list, category_list,
#                                          category_mapping, size=20, **kwargs)


# @register_ibs_method
# def localizer_classifications_confusion_matrix_algo_display(ibs, conf,
#                                                             min_overlap=0.25,
#                                                             figsize=(24, 7),
#                                                             write_images=False,
#                                                             target_recall=0.9,
#                                                             plot_point=True,
#                                                             masking=False,
#                                                             **kwargs):
#     import matplotlib.pyplot as plt

#     fig_ = plt.figure(figsize=figsize)

#     config = {
#         'label'        : 'WIC',
#         'algo'         : '_COMBINED',
#         'species_set'  : set(['zebra']),
#         'classify'     : True,
#         'classifier_algo': 'svm',
#         'classifier_masking': masking,
#         'classifier_weight_filepath': '/home/jason/code/wbia/models-bootstrap/classifier.svm.image.zebra.pkl',
#     }

#     axes_ = plt.subplot(111)
#     axes_.set_aspect(1)
#     gca_ = plt.gca()
#     gca_.grid(False)

#     correct_rate, _ = localizer_classification_confusion_matrix_algo_plot(ibs, None, conf,
#                                                                           min_overlap=min_overlap,
#                                                                           write_images=write_images,
#                                                                           fig_=fig_, axes_=axes_,
#                                                                           **config)
#     axes_.set_xlabel('Predicted (Correct = %0.02f%%)' % (correct_rate * 100.0, ))
#     axes_.set_ylabel('Ground-Truth')
#     args = (min_overlap, conf, )
#     plt.title('Confusion Matrix (IoU %0.02f, Conf %0.02f)' % args, y=1.13)

#     # plt.show()
#     args = (min_overlap, conf, )
#     fig_filename = 'localizer-classification-confusion-matrix-%0.2f-%0.2f.png' % args
#     fig_path = abspath(expanduser(join('~', 'Desktop', fig_filename)))
#     plt.savefig(fig_path, bbox_inches='tight')


# @register_ibs_method
# def localizer_classifications_confusion_matrix_algo_display_animate(ibs, total=10, **kwargs):
#     for index in range(0, total + 1):
#         conf = index / total
#         ibs.localizer_classifications_confusion_matrix_algo_display(conf, **kwargs)


def classifier_cameratrap_precision_recall_algo(
    ibs, positive_imageset_id, negative_imageset_id, test_gid_list=None, **kwargs
):
    depc = ibs.depc_image

    if test_gid_list is None:
        test_gid_set_ = set(general_get_imageset_gids(ibs, 'TEST_SET'))
        test_gid_list = list(test_gid_set_)

    positive_gid_set = set(ibs.get_imageset_gids(positive_imageset_id))
    negative_gid_set = set(ibs.get_imageset_gids(negative_imageset_id))

    test_gid_set = []
    label_list = []
    for gid in test_gid_list:
        if gid in positive_gid_set:
            label = 'positive'
        elif gid in negative_gid_set:
            label = 'negative'
        else:
            # label = 'unknown'
            continue
        test_gid_set.append(gid)
        label_list.append(label)

    prediction_list = depc.get_property(
        'classifier', test_gid_set, 'class', config=kwargs
    )
    confidence_list = depc.get_property(
        'classifier', test_gid_set, 'score', config=kwargs
    )
    confidence_list = [
        confidence if prediction == 'positive' else 1.0 - confidence
        for prediction, confidence in zip(prediction_list, confidence_list)
    ]
    return general_precision_recall_algo(
        ibs, label_list, confidence_list, index_list=test_gid_set, **kwargs
    )


def classifier_cameratrap_precision_recall_algo_plot(ibs, **kwargs):
    label = kwargs['label']
    logger.info('Processing Precision-Recall for: {!r}'.format(label))
    (
        conf_list,
        pr_list,
        re_list,
        tpr_list,
        fpr_list,
    ) = classifier_cameratrap_precision_recall_algo(ibs, **kwargs)
    return general_area_best_conf(conf_list, re_list, pr_list, **kwargs)


def classifier_cameratrap_roc_algo_plot(ibs, target_recall=None, **kwargs):
    label = kwargs['label']
    logger.info('Processing ROC for: {!r}'.format(label))
    (
        conf_list,
        pr_list,
        re_list,
        tpr_list,
        fpr_list,
    ) = classifier_cameratrap_precision_recall_algo(ibs, **kwargs)
    return general_area_best_conf(
        conf_list, fpr_list, tpr_list, interpolate=False, target=(0.0, 1.0), **kwargs
    )


def classifier_cameratrap_confusion_matrix_algo_plot(
    ibs,
    label,
    color,
    conf,
    positive_imageset_id,
    negative_imageset_id,
    test_gid_list=None,
    output_cases=False,
    filter_fn_func=None,
    **kwargs,
):
    logger.info(
        'Processing Confusion Matrix for: {!r} (Conf = {:0.02f})'.format(label, conf)
    )
    depc = ibs.depc_image

    if test_gid_list is None:
        test_gid_set_ = set(general_get_imageset_gids(ibs, 'TEST_SET'))
        test_gid_list = list(test_gid_set_)

    positive_gid_set = set(ibs.get_imageset_gids(positive_imageset_id))
    negative_gid_set = set(ibs.get_imageset_gids(negative_imageset_id))

    test_gid_set = []
    label_list = []
    for gid in test_gid_list:
        if gid in positive_gid_set:
            label = 'positive'
        elif gid in negative_gid_set:
            label = 'negative'
        else:
            # label = 'unknown'
            continue
        test_gid_set.append(gid)
        label_list.append(label)

    prediction_list = depc.get_property(
        'classifier', test_gid_set, 'class', config=kwargs
    )
    confidence_list = depc.get_property(
        'classifier', test_gid_set, 'score', config=kwargs
    )
    confidence_list = [
        confidence if prediction == 'positive' else 1.0 - confidence
        for prediction, confidence in zip(prediction_list, confidence_list)
    ]
    prediction_list = [
        'positive' if confidence >= conf else 'negative' for confidence in confidence_list
    ]

    print('Using filter_fn_func = {!r}'.format(filter_fn_func))
    if filter_fn_func is not None:
        fn_filter_total = 0
        fn_filter_converted = 0

        aids_list = ibs.get_image_aids(test_gid_set)
        gid_aids_mapping = dict(zip(test_gid_set, aids_list))

        prediction_list_ = []
        zipped = list(zip(test_gid_set, label_list, prediction_list))
        for test_gid, label, prediction in zipped:
            prediction_ = prediction
            if label == 'positive' and prediction == 'negative':
                fn_filter_total += 1
                values = (test_gid, label, prediction, zipped)
                flag = filter_fn_func(ibs, 2, values, gid_aids_mapping)
                if not flag:
                    prediction_ = 'positive'
                    fn_filter_converted += 1
            prediction_list_.append(prediction_)

        print(
            'fn_filter_converted = %d / %d'
            % (
                fn_filter_converted,
                fn_filter_total,
            )
        )
        prediction_list = prediction_list_

    if output_cases:
        output_path = 'cameratrap-confusion-incorrect'
        output_path = abspath(expanduser(join('~', 'Desktop', output_path)))
        positive_path = join(output_path, 'positive')
        negative_path = join(output_path, 'negative')
        ut.delete(output_path)
        ut.ensuredir(output_path)
        ut.ensuredir(positive_path)
        ut.ensuredir(negative_path)

        interpolation = cv2.INTER_LANCZOS4
        warpkw = dict(interpolation=interpolation)
        for gid, label, prediction in zip(test_gid_set, label_list, prediction_list):
            if label == prediction:
                continue
            image = ibs.get_images(gid)
            image = cv2.resize(image, (192, 192), **warpkw)
            # Get path
            image_path = positive_path if label == 'positive' else negative_path
            image_filename = 'hardidx_%d_pred_%s_case_fail.jpg' % (gid, prediction)
            image_filepath = join(image_path, image_filename)
            # Save path
            cv2.imwrite(image_filepath, image)

    category_list = ['positive', 'negative']
    category_mapping = {
        'positive': 0,
        'negative': 1,
    }
    return general_confusion_matrix_algo(
        label_list, prediction_list, category_list, category_mapping, **kwargs
    )


@register_ibs_method
def classifier_cameratrap_precision_recall_algo_display(
    ibs,
    positive_imageset_id,
    negative_imageset_id,
    test_gid_list=None,
    figsize=(20, 20),
    config_list=None,
    target_recall=None,
    force_target_recall=False,
    offset_black=0,
    desired_index=None,
    filter_fn_func=None,
):
    import matplotlib.pyplot as plt

    import wbia.plottool as pt

    fig_ = plt.figure(figsize=figsize, dpi=400)

    if config_list is None:
        config_list = [
            # {'label': 'Initial Model (5%)  - IBEIS_CNN',  'classifier_algo': 'cnn',      'classifier_weight_filepath': 'ryan.wbia_cnn.v1'},
            # {'label': 'Initial Model (5%)  - DenseNet',   'classifier_algo': 'densenet', 'classifier_weight_filepath': 'ryan_densenet_v1'},
            # {'label': 'Initial Model (5%)  - DenseNet 0', 'classifier_algo': 'densenet', 'classifier_weight_filepath': 'ryan_densenet_v1:0'},
            # {'label': 'Initial Model (5%)  - DenseNet 1', 'classifier_algo': 'densenet', 'classifier_weight_filepath': 'ryan_densenet_v1:1'},
            # {'label': 'Initial Model (5%)  - DenseNet 2', 'classifier_algo': 'densenet', 'classifier_weight_filepath': 'ryan_densenet_v1:2'},
            {
                'label': 'Initial Model (10%) - DenseNet',
                'classifier_algo': 'densenet',
                'classifier_weight_filepath': 'ryan_densenet_v2',
            },
            # {'label': 'Initial Model (10%) - DenseNet 0', 'classifier_algo': 'densenet', 'classifier_weight_filepath': 'ryan_densenet_v2:0'},
            # {'label': 'Initial Model (10%) - DenseNet 1', 'classifier_algo': 'densenet', 'classifier_weight_filepath': 'ryan_densenet_v2:1'},
            # {'label': 'Initial Model (10%) - DenseNet 2', 'classifier_algo': 'densenet', 'classifier_weight_filepath': 'ryan_densenet_v2:2'},
            # {'label': 'Initial Model   (0%)', 'classifier_algo': 'cnn', 'classifier_weight_filepath': 'megan2.1'},
            # {'label': 'Retrained Model (1%)', 'classifier_algo': 'cnn', 'classifier_weight_filepath': 'megan2.2'},
            # {'label': 'Retrained Model (2%)', 'classifier_algo': 'cnn', 'classifier_weight_filepath': 'megan2.3'},
            # {'label': 'Retrained Model (3%)', 'classifier_algo': 'cnn', 'classifier_weight_filepath': 'megan2.4'},
            # {'label': 'Retrained Model (4%)', 'classifier_algo': 'cnn', 'classifier_weight_filepath': 'megan2.5'},
            # {'label': 'Retrained Model (5%)', 'classifier_algo': 'cnn', 'classifier_weight_filepath': 'megan2.6'},
            # {'label': 'Initial Model   (0%)',   'classifier_weight_filepath': 'megan1.1'},
            # {'label': 'Retrained Model (1%)',   'classifier_weight_filepath': 'megan1.2'},
            # {'label': 'Retrained Model (2%)',   'classifier_weight_filepath': 'megan1.3'},
            # {'label': 'Retrained Model (3%)',   'classifier_weight_filepath': 'megan1.4'},
            # {'label': 'Retrained Model (3.5%)', 'classifier_weight_filepath': 'megan1.5'},
            # {'label': 'Retrained Model (5%)',   'classifier_weight_filepath': 'megan1.6'},
        ]
    color_list = [(0, 0, 0)] * offset_black
    color_list += pt.distinct_colors(len(config_list) - len(color_list), randomize=False)

    axes_ = plt.subplot(221)
    axes_.set_autoscalex_on(False)
    axes_.set_autoscaley_on(False)
    axes_.set_xlabel('Recall')
    axes_.set_ylabel('Precision')
    axes_.set_xlim([0.0, 1.01])
    axes_.set_ylim([0.0, 1.01])
    ret_list = [
        classifier_cameratrap_precision_recall_algo_plot(
            ibs,
            color=color,
            positive_imageset_id=positive_imageset_id,
            negative_imageset_id=negative_imageset_id,
            test_gid_list=test_gid_list,
            target_recall=target_recall,
            force_target_recall=force_target_recall,
            filter_fn_func=filter_fn_func,
            **config,
        )
        for color, config in zip(color_list, config_list)
    ]

    conf_list = [ret[1] for ret in ret_list]
    if target_recall is None:
        # Get the highest AP
        area_list = [ret[0] for ret in ret_list]
    else:
        # Get the highest recall
        area_list = [ret[2][2][0] for ret in ret_list]

    if desired_index in ['argmax', None]:
        index = np.argmax(area_list)
    else:
        index = desired_index

    best_label1 = config_list[index]['label']
    best_config1 = config_list[index]
    best_color1 = color_list[index]
    best_area1 = area_list[index]
    best_conf1 = conf_list[index]
    plt.title(
        'Precision-Recall Curve (Best: {}, AP = {:0.02f})'.format(
            best_label1, best_area1
        ),
        y=1.10,
    )

    if target_recall is None:
        plt.title(
            'Precision-Recall Curve (Best: %s, AP = %0.04f)'
            % (
                best_label1,
                best_area1,
            ),
            y=1.16,
        )
    else:
        plt.title(
            'Precision-Recall Curve (Best: %s, Precision = %0.02f)'
            % (
                best_label1,
                best_area1,
            ),
            y=1.16,
        )

    plt.legend(
        bbox_to_anchor=(0.0, 1.02, 1.0, 0.102),
        loc=3,
        ncol=2,
        mode='expand',
        borderaxespad=0.0,
    )

    axes_ = plt.subplot(222)
    axes_.set_autoscalex_on(False)
    axes_.set_autoscaley_on(False)
    axes_.set_xlabel('False-Positive Rate')
    axes_.set_ylabel('True-Positive Rate')
    axes_.set_xlim([0.0, 1.01])
    axes_.set_ylim([0.0, 1.01])
    ret_list = [
        classifier_cameratrap_roc_algo_plot(
            ibs,
            color=color,
            positive_imageset_id=positive_imageset_id,
            negative_imageset_id=negative_imageset_id,
            test_gid_list=test_gid_list,
            filter_fn_func=filter_fn_func,
            **config,
        )
        for color, config in zip(color_list, config_list)
    ]
    area_list = [ret[0] for ret in ret_list]
    conf_list = [ret[1] for ret in ret_list]
    if desired_index in ['argmax', None]:
        index = np.argmax(area_list)
    else:
        index = desired_index
    # index = 0
    best_label2 = config_list[index]['label']
    best_config2 = config_list[index]
    best_color2 = color_list[index]
    best_area2 = area_list[index]
    best_conf2 = conf_list[index]
    plt.title(
        'ROC Curve (Best: {}, AP = {:0.02f})'.format(
            best_label2,
            best_area2,
        ),
        y=1.10,
    )
    plt.legend(
        bbox_to_anchor=(0.0, 1.02, 1.0, 0.102),
        loc=3,
        ncol=2,
        mode='expand',
        borderaxespad=0.0,
    )

    axes_ = plt.subplot(223)
    axes_.set_aspect(1)
    gca_ = plt.gca()
    gca_.grid(False)
    correct_rate, _ = classifier_cameratrap_confusion_matrix_algo_plot(
        ibs,
        color=best_color1,
        conf=best_conf1,
        fig_=fig_,
        axes_=axes_,
        positive_imageset_id=positive_imageset_id,
        negative_imageset_id=negative_imageset_id,
        test_gid_list=test_gid_list,
        output_cases=True,
        filter_fn_func=filter_fn_func,
        **best_config1,
    )
    axes_.set_xlabel('Predicted (Correct = {:0.02f}%)'.format(correct_rate * 100.0))
    axes_.set_ylabel('Ground-Truth')
    plt.title(
        'P-R Confusion Matrix (Model: %s, OP = %0.02f)'
        % (
            best_label1,
            best_conf1,
        ),
        y=1.12,
    )

    axes_ = plt.subplot(224)
    axes_.set_aspect(1)
    gca_ = plt.gca()
    gca_.grid(False)
    correct_rate, _ = classifier_cameratrap_confusion_matrix_algo_plot(
        ibs,
        color=best_color2,
        conf=best_conf2,
        fig_=fig_,
        axes_=axes_,
        positive_imageset_id=positive_imageset_id,
        negative_imageset_id=negative_imageset_id,
        test_gid_list=test_gid_list,
        filter_fn_func=filter_fn_func,
        **best_config2,
    )
    axes_.set_xlabel('Predicted (Correct = {:0.02f}%)'.format(correct_rate * 100.0))
    axes_.set_ylabel('Ground-Truth')
    plt.title(
        'ROC Confusion Matrix (Model: %s, OP = %0.02f)'
        % (
            best_label2,
            best_conf2,
        ),
        y=1.12,
    )

    if target_recall is None:
        fig_filename = 'classifier-cameratrap-precision-recall-roc.png'
    else:
        fig_filename = (
            'classifier-cameratrap-precision-recall-roc-target-recall-%0.02f.png'
            % (target_recall,)
        )

    if filter_fn_func is not None:
        fig_filename = fig_filename.replace('.png', '-recovery.png')

    fig_path = abspath(expanduser(join('~', 'Desktop', fig_filename)))
    plt.savefig(fig_path, bbox_inches='tight')


# def classifier_binary_precision_recall_algo(ibs, category_set, **kwargs):
#     depc = ibs.depc_image
#     test_gid_set = set(general_get_imageset_gids(ibs, 'TEST_SET'))
#     test_gid_set = list(test_gid_set)
#     aids_list = ibs.get_image_aids(test_gid_set)
#     species_set_list = [
#         set(ibs.get_annot_species_texts(aid_list))
#         for aid_list in aids_list
#     ]
#     label_list = [
#         'negative' if len(species_set & category_set) == 0 else 'positive'
#         for species_set in species_set_list
#     ]
#     prediction_list = depc.get_property('classifier', test_gid_set, 'class', config=kwargs)
#     confidence_list = depc.get_property('classifier', test_gid_set, 'score', config=kwargs)
#     confidence_list = [
#         confidence if prediction == 'positive' else 1.0 - confidence
#         for prediction, confidence  in zip(prediction_list, confidence_list)
#     ]
#     return general_precision_recall_algo(ibs, label_list, confidence_list)


# def classifier_binary_precision_recall_algo_plot(ibs, **kwargs):
#     label = kwargs['label']
#     logger.info('Processing Precision-Recall for: %r' % (label, ))
#     conf_list, pr_list, re_list, tpr_list, fpr_list = classifier_binary_precision_recall_algo(ibs, **kwargs)
#     return general_area_best_conf(conf_list, re_list, pr_list, **kwargs)


# def classifier_binary_roc_algo_plot(ibs, **kwargs):
#     label = kwargs['label']
#     logger.info('Processing ROC for: %r' % (label, ))
#     conf_list, pr_list, re_list, tpr_list, fpr_list = classifier_binary_precision_recall_algo(ibs, **kwargs)
#     return general_area_best_conf(conf_list, fpr_list, tpr_list, interpolate=False,
#                                   target=(0.0, 1.0), **kwargs)


# def classifier_binary_confusion_matrix_algo_plot(ibs, label, color, conf, category_set, **kwargs):
#     logger.info('Processing Confusion Matrix for: %r (Conf = %0.02f)' % (label, conf, ))
#     depc = ibs.depc_image
#     test_gid_set = set(general_get_imageset_gids(ibs, 'TEST_SET'))
#     test_gid_set = list(test_gid_set)
#     aids_list = ibs.get_image_aids(test_gid_set)
#     species_set_list = [
#         set(ibs.get_annot_species_texts(aid_list))
#         for aid_list in aids_list
#     ]
#     label_list = [
#         'negative' if len(species_set & category_set) == 0 else 'positive'
#         for species_set in species_set_list
#     ]
#     prediction_list = depc.get_property('classifier', test_gid_set, 'class', config=kwargs)
#     confidence_list = depc.get_property('classifier', test_gid_set, 'score', config=kwargs)
#     confidence_list = [
#         confidence if prediction == 'positive' else 1.0 - confidence
#         for prediction, confidence  in zip(prediction_list, confidence_list)
#     ]
#     prediction_list = [
#         'positive' if confidence >= conf else 'negative'
#         for confidence in confidence_list
#     ]

#     category_list = ['positive', 'negative']
#     category_mapping = {
#         'positive': 0,
#         'negative': 1,
#     }
#     return general_confusion_matrix_algo(label_list, prediction_list, category_list,
#                                          category_mapping, **kwargs)


# @register_ibs_method
# def classifier_binary_precision_recall_algo_display(ibs, figsize=(16, 16), **kwargs):
#     import matplotlib.pyplot as plt

#     fig_ = plt.figure(figsize=figsize)

#     # label = 'V1'
#     # species_list = ['zebra']
#     # kwargs['classifier_weight_filepath'] = 'coco_zebra'

#     label = 'V3'
#     species_list = ['zebra_plains', 'zebra_grevys']
#     kwargs['classifier_weight_filepath'] = 'v3_zebra'

#     category_set = set(species_list)

#     axes_ = plt.subplot(221)
#     axes_.set_autoscalex_on(False)
#     axes_.set_autoscaley_on(False)
#     axes_.set_xlabel('Recall')
#     axes_.set_ylabel('Precision')
#     axes_.set_xlim([0.0, 1.01])
#     axes_.set_ylim([0.0, 1.01])
#     area, best_conf1, _ = classifier_binary_precision_recall_algo_plot(ibs, label=label, color='r', category_set=category_set, **kwargs)
#     plt.title('Precision-Recall Curve (AP = %0.02f)' % (area, ), y=1.10)
#     plt.legend(bbox_to_anchor=(0.0, 1.02, 1.0, .102), loc=3, ncol=2, mode="expand",
#                borderaxespad=0.0)

#     axes_ = plt.subplot(222)
#     axes_.set_autoscalex_on(False)
#     axes_.set_autoscaley_on(False)
#     axes_.set_xlabel('False-Positive Rate')
#     axes_.set_ylabel('True-Positive Rate')
#     axes_.set_xlim([0.0, 1.01])
#     axes_.set_ylim([0.0, 1.01])
#     area, best_conf2, _ = classifier_binary_roc_algo_plot(ibs, label=label, color='r', category_set=category_set, **kwargs)
#     plt.title('ROC Curve (AP = %0.02f)' % (area, ), y=1.10)
#     plt.legend(bbox_to_anchor=(0.0, 1.02, 1.0, .102), loc=3, ncol=2, mode="expand",
#                borderaxespad=0.0)

#     axes_ = plt.subplot(223)
#     axes_.set_aspect(1)
#     gca_ = plt.gca()
#     gca_.grid(False)
#     correct_rate, _ = classifier_binary_confusion_matrix_algo_plot(ibs, label, 'r', conf=best_conf1, fig_=fig_, axes_=axes_, category_set=category_set, **kwargs)
#     axes_.set_xlabel('Predicted (Correct = %0.02f%%)' % (correct_rate * 100.0, ))
#     axes_.set_ylabel('Ground-Truth')
#     plt.title('P-R Confusion Matrix (OP = %0.02f)' % (best_conf1, ), y=1.12)

#     axes_ = plt.subplot(224)
#     axes_.set_aspect(1)
#     gca_ = plt.gca()
#     gca_.grid(False)
#     correct_rate, _ = classifier_binary_confusion_matrix_algo_plot(ibs, label, 'r', conf=best_conf2, fig_=fig_, axes_=axes_, category_set=category_set, **kwargs)
#     axes_.set_xlabel('Predicted (Correct = %0.02f%%)' % (correct_rate * 100.0, ))
#     axes_.set_ylabel('Ground-Truth')
#     plt.title('ROC Confusion Matrix (OP = %0.02f)' % (best_conf2, ), y=1.12)

#     fig_filename = 'classifier-precision-recall-roc.png'
#     fig_path = abspath(expanduser(join('~', 'Desktop', fig_filename)))
#     plt.savefig(fig_path, bbox_inches='tight')


def classifier2_precision_recall_algo(
    ibs,
    category,
    species_mapping={},
    output_path=None,
    test_gid_list=None,
    test_label_list=None,
    **kwargs,
):
    depc = ibs.depc_image
    if test_gid_list is None:
        test_gid_set = set(general_get_imageset_gids(ibs, 'TEST_SET'))
        test_gid_list = list(test_gid_set)

    if test_label_list is None:
        aids_list = ibs.get_image_aids(test_gid_list)
        species_list_list = list(map(ibs.get_annot_species_texts, aids_list))
        species_set_list = []
        for species_list in species_list_list:
            species_set = set()
            for species in species_list:
                species = species_mapping.get(species, species)
                species_set.add(species)
            species_set_list.append(species_set)
    else:
        species_set_list = [{label} for label in test_label_list]

    label_list = [
        'positive' if category in species_set_ else 'negative'
        for species_set_ in species_set_list
    ]

    confidence_dict_list = depc.get_property(
        'classifier_two', test_gid_list, 'scores', config=kwargs
    )
    confidence_list = [
        confidence_dict[category] for confidence_dict in confidence_dict_list
    ]

    if output_path is not None:
        ut.ensuredir(output_path)
        config_ = {
            'draw_annots': False,
            'thumbsize': (192, 192),
        }
        thumbnail_list = depc.get_property(
            'thumbnails', test_gid_list, 'img', config=config_
        )
        zipped = zip(
            test_gid_list, thumbnail_list, species_set_list, confidence_dict_list
        )
        for index, (test_gid, thumbnail, species_set, confidence_dict) in enumerate(
            zipped
        ):
            logger.info(index)
            x = ';'.join(species_set)
            y = []
            for key in confidence_dict:
                y.append('{}-{:0.04f}'.format(key, confidence_dict[key]))
            y = ';'.join(y)
            output_filename = 'image-index-{}-gid-{}-gt-{}-pred-{}.png'.format(
                index,
                test_gid,
                x,
                y,
            )
            output_filepath = join(output_path, output_filename)
            cv2.imwrite(output_filepath, thumbnail)

    kwargs.pop('category', None)
    return general_precision_recall_algo(ibs, label_list, confidence_list)


def classifier2_precision_recall_algo_plot(ibs, **kwargs):
    label = kwargs['label']
    logger.info('Processing Precision-Recall for: {!r}'.format(label))
    conf_list, pr_list, re_list, tpr_list, fpr_list = classifier2_precision_recall_algo(
        ibs, **kwargs
    )
    return general_area_best_conf(conf_list, re_list, pr_list, **kwargs)


def classifier2_roc_algo_plot(ibs, target_recall=None, **kwargs):
    label = kwargs['label']
    logger.info('Processing ROC for: {!r}'.format(label))
    conf_list, pr_list, re_list, tpr_list, fpr_list = classifier2_precision_recall_algo(
        ibs, **kwargs
    )
    return general_area_best_conf(
        conf_list, fpr_list, tpr_list, interpolate=False, target=(0.0, 1.0), **kwargs
    )


@register_ibs_method
def classifier2_precision_recall_algo_display(
    ibs,
    species_list=None,
    species_mapping={},
    nice_mapping={},
    test_gid_list=None,
    test_label_list=None,
    figsize=(20, 9),
    **kwargs,
):
    import matplotlib.pyplot as plt

    import wbia.plottool as pt

    depc = ibs.depc_image
    fig_ = plt.figure(figsize=figsize, dpi=400)  # NOQA

    # kwargs['classifier_two_weight_filepath'] = 'v3'
    # kwargs['classifier_two_weight_filepath'] = 'candidacy'
    # kwargs['classifier_two_weight_filepath'] = 'ggr2'

    is_labeled = test_label_list is not None

    kwargs['classifier_two_algo'] = 'densenet'
    kwargs['classifier_two_weight_filepath'] = 'flukebook_v1'

    test_gid_set = set(general_get_imageset_gids(ibs, 'TEST_SET'))
    test_gid_list_ = list(test_gid_set) if test_gid_list is None else test_gid_list
    test_label_list_ = test_label_list if is_labeled else [None] * len(test_gid_list_)

    zipped = list(zip(test_gid_list_, test_label_list_))
    test_gid_list_ = []
    test_label_list_ = []
    for test_gid_, test_label_ in zipped:
        if test_gid_ in test_gid_set:
            test_gid_list_.append(test_gid_)
            test_label_list_.append(test_label_)

    test_gid_list = test_gid_list_
    test_label_list = test_label_list_ if is_labeled else None

    # depc.delete_property('classifier_two', test_gid_list, config=kwargs)

    if species_list is None:
        test_gid = test_gid_list[0]
        confidence_dict = depc.get_property(
            'classifier_two', test_gid, 'scores', config=kwargs
        )
        species_list = confidence_dict.keys()

    category_set = sorted(species_list)

    config_list = []
    for category in category_set:
        category_nice = nice_mapping.get(category, category)
        config_dict = {
            'label': category_nice,
            'category': category,
        }
        config_dict.update(kwargs)
        config_list.append(config_dict)

    color_list_ = []
    color_list = pt.distinct_colors(len(config_list) - len(color_list_), randomize=False)
    color_list = color_list_ + color_list

    axes_ = plt.subplot(121)
    axes_.set_autoscalex_on(False)
    axes_.set_autoscaley_on(False)
    axes_.set_xlabel('Recall')
    axes_.set_ylabel('Precision')
    axes_.set_xlim([0.0, 1.01])
    axes_.set_ylim([0.0, 1.01])

    for color, config in zip(color_list, config_list):
        classifier2_precision_recall_algo_plot(
            ibs,
            color=color,
            test_gid_list=test_gid_list,
            test_label_list=test_label_list,
            species_mapping=species_mapping,
            **config,
        )
    plt.title('Precision-Recall Curves', y=1.19)
    plt.legend(
        bbox_to_anchor=(0.0, 1.02, 1.0, 0.102),
        loc=3,
        ncol=2,
        mode='expand',
        borderaxespad=0.0,
    )

    axes_ = plt.subplot(122)
    axes_.set_autoscalex_on(False)
    axes_.set_autoscaley_on(False)
    axes_.set_xlabel('False-Positive Rate')
    axes_.set_ylabel('True-Positive Rate')
    axes_.set_xlim([0.0, 1.01])
    axes_.set_ylim([0.0, 1.01])

    op_dict = {}
    for color, config in zip(color_list, config_list):
        values = classifier2_roc_algo_plot(
            ibs,
            color=color,
            test_gid_list=test_gid_list,
            test_label_list=test_label_list,
            species_mapping=species_mapping,
            **config,
        )
        ap, best_conf, tup1, tup2 = values
        op_dict[config['category']] = best_conf

    plt.title('ROC Curves', y=1.19)
    plt.legend(
        bbox_to_anchor=(0.0, 1.02, 1.0, 0.102),
        loc=3,
        ncol=2,
        mode='expand',
        borderaxespad=0.0,
    )

    if is_labeled:
        species_set_list = [{label} for label in test_label_list]
    else:
        aids_list = ibs.get_image_aids(test_gid_list)
        species_list_list = list(map(ibs.get_annot_species_texts, aids_list))
        species_set_list = []
        for species_list in species_list_list:
            species_set = set()
            for species in species_list:
                species = species_mapping.get(species, species)
                species_set.add(species)
            species_set_list.append(species_set)
    confidence_dict_list = depc.get_property(
        'classifier_two', test_gid_list, 'scores', config=kwargs
    )

    correct = 0
    for test_gid, confidence_dict, species_set in zip(
        test_gid_list, confidence_dict_list, species_set_list
    ):
        species_set_ = set()
        for key in confidence_dict:
            if op_dict[key] <= confidence_dict[key]:
                species_set_.add(key)
        if len(species_set ^ species_set_) == 0:
            correct += 1
        else:
            logger.info(test_gid, confidence_dict, species_set)
    logger.info('Accuracy: %0.04f' % (100.0 * correct / len(test_gid_list)))
    logger.info('\t using op_dict = {!r}'.format(op_dict))

    fig_filename = 'classifier2-precision-recall-roc.png'
    fig_path = abspath(expanduser(join('~', 'Desktop', fig_filename)))
    plt.savefig(fig_path, bbox_inches='tight')


def labeler_tp_tn_fp_fn(
    ibs,
    category_list,
    species_mapping={},
    viewpoint_mapping={},
    samples=SAMPLES,
    test_gid_set=None,
    **kwargs,
):
    def errors(zipped, conf, category):
        tp, tn, fp, fn = 0.0, 0.0, 0.0, 0.0
        for index, (label, confidence) in enumerate(zipped):
            if label == category:
                if conf <= confidence:
                    tp += 1
                else:
                    fn += 1
            else:
                if conf <= confidence:
                    fp += 1
                else:
                    tn += 1
        return tp, tn, fp, fn

    depc = ibs.depc_annot

    if test_gid_set is None:
        test_gid_set = set(general_get_imageset_gids(ibs, 'TEST_SET'))
        test_gid_set = list(test_gid_set)

    aids_list = ibs.get_image_aids(test_gid_set)
    aid_list = ut.flatten(aids_list)
    # Get annot species and viewpoints
    species_list = ibs.get_annot_species_texts(aid_list)
    viewpoint_list = ibs.get_annot_viewpoints(aid_list)
    # Filter aids with species of interest and undefined viewpoints

    species_list = [species_mapping.get(species, species) for species in species_list]
    viewpoint_list = [
        viewpoint_mapping.get(species, {}).get(viewpoint, viewpoint)
        for species, viewpoint in zip(species_list, viewpoint_list)
    ]

    flag_list = [
        species in category_list and viewpoint is not None
        for species, viewpoint in zip(species_list, viewpoint_list)
    ]

    if False in flag_list:
        aid_list = ut.compress(aid_list, flag_list)
        species_list = ut.compress(species_list, flag_list)
        viewpoint_list = ut.compress(viewpoint_list, flag_list)

    # Make ground-truth
    label_list = [
        '{}:{}'.format(species, viewpoint_)
        for species, viewpoint_ in zip(species_list, viewpoint_list)
    ]
    # Get predictions
    # depc.delete_property('labeler', aid_list, config=kwargs)
    probability_dict_list = depc.get_property('labeler', aid_list, 'probs', config=kwargs)

    viewpoint_label_mapping = {}
    for species in viewpoint_mapping:
        for viewpoint in viewpoint_mapping[species]:
            viewpoint_ = viewpoint_mapping[species][viewpoint]
            src_label = '{}:{}'.format(
                species,
                viewpoint,
            )
            dst_label = '{}:{}'.format(
                species,
                viewpoint_,
            )
            viewpoint_label_mapping[src_label] = dst_label

    for label in label_list:
        if label not in viewpoint_label_mapping:
            viewpoint_label_mapping[label] = label

    predict_label_list = []
    for label in probability_dict_list[0].keys():
        label_ = viewpoint_label_mapping.get(label, label)
        predict_label_list.append(label_)

    value1_list = set(label_list)
    value2_list = set(predict_label_list)
    print(value1_list)
    print(value2_list)
    assert len(value1_list - value2_list) == 0
    assert len(value2_list - value1_list) == 0

    conf_list = [_ / float(samples) for _ in range(0, int(samples) + 1)]
    label_dict = {}
    for key in value1_list:
        logger.info('\t{!r}'.format(key))
        conf_dict = {}
        confidence_list = [
            probability_dict[key] for probability_dict in probability_dict_list
        ]
        zipped = list(zip(label_list, confidence_list))
        for conf in conf_list:
            conf_dict[conf] = errors(zipped, conf, key)
        label_dict[key] = conf_dict
    return label_dict


def labeler_precision_recall_algo(ibs, category_list, label_dict, **kwargs):

    if category_list is None:
        category_list_ = label_dict.keys()
    else:
        category_list_ = []
        for category in category_list:
            for key in label_dict:
                if category in key or category is None:
                    category_list_.append(key)

    global_conf_dict = {}
    for category in category_list_:
        conf_dict = label_dict[category]
        for conf in conf_dict:
            new_list = conf_dict[conf]
            if conf not in global_conf_dict:
                global_conf_dict[conf] = new_list
            else:
                cur_list = global_conf_dict[conf]
                zipped_ = zip(cur_list, new_list)
                global_conf_dict[conf] = [cur + new for cur, new in zipped_]

    conf_list_ = [-1.0, -1.0]
    pr_list = [1.0, 0.0]
    re_list = [0.0, 1.0]
    tpr_list = [0.0, 1.0]
    fpr_list = [0.0, 1.0]
    # conf_list_ = []
    # pr_list = []
    # re_list = []
    # tpr_list = []
    # fpr_list = []
    for conf in sorted(global_conf_dict.keys(), reverse=True):
        error_list = global_conf_dict[conf]
        tp, tn, fp, fn = error_list
        try:
            pr = tp / (tp + fp)
            re = tp / (tp + fn)
            tpr = tp / (tp + fn)
            fpr = fp / (fp + tn)
            conf_list_.append(conf)
            pr_list.append(pr)
            re_list.append(re)
            tpr_list.append(tpr)
            fpr_list.append(fpr)
        except ZeroDivisionError:
            logger.info(
                'Zero division error (%r) - tp: %r tn: %r fp: %r fn: %r'
                % (conf, tp, tn, fp, fn)
            )

    return conf_list_, pr_list, re_list, tpr_list, fpr_list


def labeler_precision_recall_algo_plot(ibs, **kwargs):
    label = kwargs['label']
    category_list = kwargs['category_list']
    logger.info(
        'Processing Precision-Recall for: %r (category_list = %r)'
        % (label, category_list)
    )
    conf_list, pr_list, re_list, tpr_list, fpr_list = labeler_precision_recall_algo(
        ibs, **kwargs
    )
    return general_area_best_conf(conf_list, re_list, pr_list, **kwargs)


def labeler_roc_algo_plot(ibs, target_recall=None, **kwargs):
    label = kwargs['label']
    category_list = kwargs['category_list']
    logger.info(
        'Processing ROC for: {!r} (category_list = {!r})'.format(label, category_list)
    )
    conf_list, pr_list, re_list, tpr_list, fpr_list = labeler_precision_recall_algo(
        ibs, **kwargs
    )
    return general_area_best_conf(
        conf_list, fpr_list, tpr_list, interpolate=False, target=(0.0, 1.0), **kwargs
    )


def labeler_confusion_matrix_algo_plot(
    ibs,
    category_list,
    species_mapping={},
    viewpoint_mapping={},
    category_mapping=None,
    test_gid_set=None,
    **kwargs,
):
    logger.info('Processing Confusion Matrix')
    depc = ibs.depc_annot

    if test_gid_set is None:
        test_gid_set = set(general_get_imageset_gids(ibs, 'TEST_SET'))
        test_gid_set = list(test_gid_set)

    aids_list = ibs.get_image_aids(test_gid_set)
    aid_list = ut.flatten(aids_list)
    species_list = ibs.get_annot_species_texts(aid_list)
    viewpoint_list = ibs.get_annot_viewpoints(aid_list)
    label_list = [
        '%s:%s'
        % (
            species_mapping.get(species, species),
            viewpoint_mapping.get(species, {}).get(viewpoint, viewpoint),
        )
        for species, viewpoint in zip(species_list, viewpoint_list)
    ]
    temp_list = [
        (aid, label) for aid, label in zip(aid_list, label_list) if label in category_list
    ]
    aid_list = [_[0] for _ in temp_list]
    label_list = [_[1] for _ in temp_list]
    conf_list = depc.get_property('labeler', aid_list, 'score', config=kwargs)
    species_list = depc.get_property('labeler', aid_list, 'species', config=kwargs)
    viewpoint_list = depc.get_property('labeler', aid_list, 'viewpoint', config=kwargs)
    prediction_list = [
        '{}:{}'.format(species, viewpoint)
        for species, viewpoint in zip(species_list, viewpoint_list)
    ]

    category_list = list(map(simple_code, category_list))
    label_list = list(map(simple_code, label_list))
    prediction_list = list(map(simple_code, prediction_list))

    if category_mapping is None:
        category_mapping = {key: index for index, key in enumerate(category_list)}

    category_mapping = {
        simple_code(key): category_mapping[key] for key in category_mapping
    }
    return general_confusion_matrix_algo(
        label_list,
        prediction_list,
        category_list,
        category_mapping,
        conf_list=conf_list,
        size=8,
        **kwargs,
    )


@register_ibs_method
def labeler_precision_recall_algo_display(
    ibs,
    category_list=None,
    species_mapping={},
    viewpoint_mapping={},
    category_mapping=None,
    fuzzy_dict=None,
    figsize=(30, 9),
    test_gid_set=None,
    use_axis_aligned_chips=False,
    labeler_weight_filepath=None,
    config_list=None,
    **kwargs,
):
    import matplotlib.pyplot as plt

    from wbia import plottool as pt

    if category_list is None:

        if test_gid_set is None:
            test_gid_set = set(general_get_imageset_gids(ibs, 'TEST_SET'))
            test_gid_set = list(test_gid_set)

        aids_list = ibs.get_image_aids(test_gid_set)
        aid_list = ut.flatten(aids_list)
        species_list = ibs.get_annot_species_texts(aid_list)
        species_list = [species_mapping.get(species, species) for species in species_list]
        category_list = sorted(list(set(species_list)))

    logger.info('Compiling raw numbers...')
    kwargs['labeler_algo'] = 'densenet'
    if labeler_weight_filepath is None:
        # kwargs['labeler_weight_filepath'] = 'zebra_v1'
        # kwargs['labeler_weight_filepath'] = 'seaturtle'
        # kwargs['labeler_weight_filepath'] = 'giraffe_v1'
        # kwargs['labeler_weight_filepath'] = 'lynx_v3'
        # kwargs['labeler_weight_filepath'] = 'seaturtle_v3'
        # kwargs['labeler_weight_filepath'] = 'jaguar_v3'
        # kwargs['labeler_weight_filepath'] = 'hendrik_dorsal_v2'
        # kwargs['labeler_weight_filepath'] = 'spotted_skunk_v0'
        # kwargs['labeler_weight_filepath'] = 'nassau_grouper_v0'
        # kwargs['labeler_weight_filepath'] = 'spotted_dolphin_v0'
        # kwargs['labeler_weight_filepath'] = 'seadragon_v1'
        kwargs['labeler_weight_filepath'] = 'seadragon_v2'
    else:
        kwargs['labeler_weight_filepath'] = labeler_weight_filepath
    kwargs['labeler_axis_aligned'] = use_axis_aligned_chips

    label_dict = labeler_tp_tn_fp_fn(
        ibs,
        category_list,
        species_mapping=species_mapping,
        viewpoint_mapping=viewpoint_mapping,
        test_gid_set=test_gid_set,
        **kwargs,
    )

    if config_list is None:
        config_list = [
            # {'label': 'Giraffe',                'category_list': None},
            # {'label': 'Masai Giraffe',          'category_list': ['giraffe_masai']},
            # {'label': 'Reticulated Giraffe',    'category_list': ['giraffe_reticulated']},
            # {'label': 'Lynx',                   'category_list': ['lynx_pardinus']},
            # {'label': 'Sea Turtle',             'category_list': ['turtle_sea']},
            # {'label': 'Sea Turtle Head',        'category_list': ['turtle_sea+head']},
            # {'label': 'Manta',                  'category_list': ['manta_ray_giant']},
            # {'label': 'Jaguar',                 'category_list': ['jaguar']},
            # {'label': 'Dorsal Fin',             'category_list': ['dolphin_bottlenose_fin']},
            # {'label': 'Reticulated Giraffe',    'category_list': ['giraffe_reticulated']},
            # {'label': 'Sea Turtle',             'category_list': ['turtle_sea']},
            # {'label': 'Whale Fluke',            'category_list': ['whale_fluke']},
            # {'label': 'Grevy\'s Zebra',         'category_list': ['zebra_grevys']},
            # {'label': 'Plains Zebra',           'category_list': ['zebra_plains']},
            # {'label': 'Spotted Skunk',          'category_list': ['skunk_spotted']},
            # {'label': 'Nassau Grouper',           'category_list': ['grouper_nassau']},
            # {'label': 'Spotted Dolphin',           'category_list': ['dolphin_spotted']},
            # {'label': 'Spotted Dolphin',           'category_list': ['dolphin_spotted']},
            {'label': 'Weedy SD ', 'category_list': ['seadragon_weedy']},
            {'label': 'Weedy Head', 'category_list': ['seadragon_weedy+head']},
            {'label': 'Leafy SD ', 'category_list': ['seadragon_leafy']},
            {'label': 'Leafy Head', 'category_list': ['seadragon_leafy+head']},
        ]

    color_list = [(0.0, 0.0, 0.0)]
    color_list += pt.distinct_colors(len(config_list) - len(color_list), randomize=False)

    fig_ = plt.figure(figsize=figsize, dpi=400)  # NOQA

    axes_ = plt.subplot(131)
    axes_.set_autoscalex_on(False)
    axes_.set_autoscaley_on(False)
    axes_.set_xlabel('Recall')
    axes_.set_ylabel('Precision')
    axes_.set_xlim([0.0, 1.01])
    axes_.set_ylim([0.0, 1.01])
    area_list = []
    for color, config in zip(color_list, config_list):
        ret = labeler_precision_recall_algo_plot(
            ibs, label_dict=label_dict, color=color, **config
        )
        area = ret[0]
        area_list.append(area)
    plt.title('Precision-Recall Curve', y=1.19)
    plt.legend(
        bbox_to_anchor=(0.0, 1.02, 1.0, 0.102),
        loc=3,
        ncol=2,
        mode='expand',
        borderaxespad=0.0,
    )

    axes_ = plt.subplot(132)
    axes_.set_autoscalex_on(False)
    axes_.set_autoscaley_on(False)
    axes_.set_xlabel('False-Positive Rate')
    axes_.set_ylabel('True-Positive Rate')
    axes_.set_xlim([0.0, 1.01])
    axes_.set_ylim([0.0, 1.01])
    for color, config in zip(color_list, config_list):
        labeler_roc_algo_plot(ibs, label_dict=label_dict, color=color, **config)
    plt.title('ROC Curve', y=1.19)
    plt.legend(
        bbox_to_anchor=(0.0, 1.02, 1.0, 0.102),
        loc=3,
        ncol=2,
        mode='expand',
        borderaxespad=0.0,
    )

    key_list = sorted(label_dict.keys())

    fuzzy = fuzzy_dict is not None
    if not fuzzy:
        fuzzy_dict = {}
        for index1, label1 in enumerate(key_list):
            if label1 == 'ignore':
                fuzzy_list = []
            else:
                species, viewpoint = label1.strip().split(':')
                fuzzy_list = []
                for index2, label2 in enumerate(key_list):
                    if species in label2:
                        fuzzy_list.append(index2)
            fuzzy_dict[index1] = set(fuzzy_list)

    axes_ = plt.subplot(133)
    axes_.set_aspect(1)
    gca_ = plt.gca()
    gca_.grid(False)
    correct_rate, fuzzy_rate = labeler_confusion_matrix_algo_plot(
        ibs,
        key_list,
        species_mapping=species_mapping,
        viewpoint_mapping=viewpoint_mapping,
        category_mapping=category_mapping,
        fig_=fig_,
        axes_=axes_,
        fuzzy_dict=fuzzy_dict,
        test_gid_set=test_gid_set,
        **kwargs,
    )

    if fuzzy:
        axes_.set_xlabel(
            'Predicted (Correct = %0.02f%%, Fuzzy = %0.02f%%)'
            % (correct_rate * 100.0, fuzzy_rate * 100.0)
        )
    else:
        axes_.set_xlabel(
            'Predicted (Correct = %0.02f%%, Species = %0.02f%%)'
            % (correct_rate * 100.0, fuzzy_rate * 100.0)
        )

    axes_.set_ylabel('Ground-Truth')
    # area_list_ = area_list[1:]
    area_list_ = area_list
    mAP = sum(area_list_) / len(area_list_)
    args = (mAP * 100.0,)
    plt.title('Confusion Matrix\nmAP = %0.02f' % args, y=1.19)

    fig_filename = 'labeler-precision-recall-roc.png'
    fig_path = abspath(expanduser(join('~', 'Desktop', fig_filename)))
    plt.savefig(fig_path, bbox_inches='tight')


def canonical_precision_recall_algo(ibs, species, **kwargs):
    depc = ibs.depc_annot

    test_gid_set_ = set(general_get_imageset_gids(ibs, 'TEST_SET'))
    test_gid_list_ = list(test_gid_set_)
    test_aid_list_ = ut.flatten(ibs.get_image_aids(test_gid_list_))
    test_aid_list_ = ibs.filter_annotation_set(test_aid_list_, species=species)
    test_flag_list_ = ibs.get_annot_canonical(test_aid_list_)

    test_aid_set = []
    label_list = []
    for aid, flag in zip(test_aid_list_, test_flag_list_):
        if flag:
            label = 'positive'
        else:
            label = 'negative'
        test_aid_set.append(aid)
        label_list.append(label)

    prediction_list = depc.get_property(
        'classifier', test_aid_set, 'class', config=kwargs
    )
    confidence_list = depc.get_property(
        'classifier', test_aid_set, 'score', config=kwargs
    )
    confidence_list = [
        confidence if prediction == 'positive' else 1.0 - confidence
        for prediction, confidence in zip(prediction_list, confidence_list)
    ]
    return general_precision_recall_algo(ibs, label_list, confidence_list)


def canonical_precision_recall_algo_plot(ibs, **kwargs):
    label = kwargs['label']
    logger.info('Processing Precision-Recall for: {!r}'.format(label))
    conf_list, pr_list, re_list, tpr_list, fpr_list = canonical_precision_recall_algo(
        ibs, **kwargs
    )
    return general_area_best_conf(conf_list, re_list, pr_list, **kwargs)


def canonical_roc_algo_plot(ibs, target_recall=None, **kwargs):
    label = kwargs['label']
    logger.info('Processing ROC for: {!r}'.format(label))
    conf_list, pr_list, re_list, tpr_list, fpr_list = canonical_precision_recall_algo(
        ibs, **kwargs
    )
    return general_area_best_conf(
        conf_list, fpr_list, tpr_list, interpolate=False, target=(0.0, 1.0), **kwargs
    )


def canonical_confusion_matrix_algo_plot(
    ibs, label, color, conf, species, output_cases=False, **kwargs
):
    logger.info(
        'Processing Confusion Matrix for: {!r} (Conf = {:0.02f})'.format(label, conf)
    )
    depc = ibs.depc_annot

    test_gid_set_ = set(general_get_imageset_gids(ibs, 'TEST_SET'))
    test_gid_list_ = list(test_gid_set_)
    test_aid_list_ = ut.flatten(ibs.get_image_aids(test_gid_list_))
    test_aid_list_ = ibs.filter_annotation_set(test_aid_list_, species=species)
    test_flag_list_ = ibs.get_annot_canonical(test_aid_list_)

    test_aid_set = []
    label_list = []
    for aid, flag in zip(test_aid_list_, test_flag_list_):
        if flag:
            label = 'positive'
        else:
            label = 'negative'
        test_aid_set.append(aid)
        label_list.append(label)

    prediction_list = depc.get_property(
        'classifier', test_aid_set, 'class', config=kwargs
    )
    confidence_list = depc.get_property(
        'classifier', test_aid_set, 'score', config=kwargs
    )
    confidence_list = [
        confidence if prediction == 'positive' else 1.0 - confidence
        for prediction, confidence in zip(prediction_list, confidence_list)
    ]
    prediction_list = [
        'positive' if confidence >= conf else 'negative' for confidence in confidence_list
    ]

    if output_cases:
        output_path = 'canonical-confusion-incorrect'
        output_path = abspath(expanduser(join('~', 'Desktop', output_path)))
        positive_path = join(output_path, 'positive')
        negative_path = join(output_path, 'negative')
        ut.delete(output_path)
        ut.ensuredir(output_path)
        ut.ensuredir(positive_path)
        ut.ensuredir(negative_path)

        config = {
            'dim_size': (192, 192),
            'resize_dim': 'wh',
        }
        chip_list = ibs.depc_annot.get_property(
            'chips', test_aid_set, 'img', config=config
        )

        zipped = zip(test_aid_set, chip_list, label_list, prediction_list)
        for aid, chip, label, prediction in zipped:
            if label == prediction:
                continue
            # Get path
            image_path = positive_path if label == 'positive' else negative_path
            image_filename = 'hardidx_%d_pred_%s_case_fail.jpg' % (aid, prediction)
            image_filepath = join(image_path, image_filename)
            # Save path
            cv2.imwrite(image_filepath, chip)

    category_list = ['positive', 'negative']
    category_mapping = {
        'positive': 0,
        'negative': 1,
    }
    return general_confusion_matrix_algo(
        label_list, prediction_list, category_list, category_mapping, **kwargs
    )


@register_ibs_method
def canonical_precision_recall_algo_display(ibs, figsize=(20, 20)):
    import matplotlib.pyplot as plt

    from wbia import plottool as pt

    fig_ = plt.figure(figsize=figsize, dpi=400)

    config_list = [
        {
            'label': 'CA V1 Ensemble',
            'classifier_algo': 'densenet',
            'classifier_weight_filepath': 'canonical_zebra_grevys_v1',
            'species': 'zebra_grevys',
        },  # SMALLER DATASET
        {
            'label': 'CA V2 Ensemble',
            'classifier_algo': 'densenet',
            'classifier_weight_filepath': 'canonical_zebra_grevys_v2',
            'species': 'zebra_grevys',
        },  # BROKEN L/R AUGMENTATION
        {
            'label': 'CA V3 Ensemble',
            'classifier_algo': 'densenet',
            'classifier_weight_filepath': 'canonical_zebra_grevys_v3',
            'species': 'zebra_grevys',
        },  # LARGER DATASET, TOO HARSH AUGMENTATION
        {
            'label': 'CA V4 Ensemble',
            'classifier_algo': 'densenet',
            'classifier_weight_filepath': 'canonical_zebra_grevys_v4',
            'species': 'zebra_grevys',
        },  # BETTER AUGMENTATION
        # {'label': 'CA V4 Model 0',  'classifier_algo': 'densenet', 'classifier_weight_filepath': 'canonical_zebra_grevys_v4:0', 'species': 'zebra_grevys'},
        # {'label': 'CA V4 Model 1',  'classifier_algo': 'densenet', 'classifier_weight_filepath': 'canonical_zebra_grevys_v4:1', 'species': 'zebra_grevys'},
        # {'label': 'CA V4 Model 2',  'classifier_algo': 'densenet', 'classifier_weight_filepath': 'canonical_zebra_grevys_v4:2', 'species': 'zebra_grevys'},
    ]
    color_list = []
    # color_list = [(0, 0, 0)]
    color_list += pt.distinct_colors(len(config_list) - len(color_list), randomize=False)

    axes_ = plt.subplot(221)
    axes_.set_autoscalex_on(False)
    axes_.set_autoscaley_on(False)
    axes_.set_xlabel('Recall')
    axes_.set_ylabel('Precision')
    axes_.set_xlim([0.0, 1.01])
    axes_.set_ylim([0.0, 1.01])
    ret_list = [
        canonical_precision_recall_algo_plot(ibs, color=color, **config)
        for color, config in zip(color_list, config_list)
    ]
    area_list = [ret[0] for ret in ret_list]
    conf_list = [ret[1] for ret in ret_list]
    # index = np.argmax(area_list)
    index = -1
    best_label1 = config_list[index]['label']
    best_config1 = config_list[index]
    best_color1 = color_list[index]
    best_area1 = area_list[index]
    best_conf1 = conf_list[index]
    plt.title(
        'Precision-Recall Curve (Best: {}, AP = {:0.02f})'.format(
            best_label1, best_area1
        ),
        y=1.10,
    )
    plt.legend(
        bbox_to_anchor=(0.0, 1.02, 1.0, 0.102),
        loc=3,
        ncol=2,
        mode='expand',
        borderaxespad=0.0,
    )

    axes_ = plt.subplot(222)
    axes_.set_autoscalex_on(False)
    axes_.set_autoscaley_on(False)
    axes_.set_xlabel('False-Positive Rate')
    axes_.set_ylabel('True-Positive Rate')
    axes_.set_xlim([0.0, 1.01])
    axes_.set_ylim([0.0, 1.01])
    ret_list = [
        canonical_roc_algo_plot(ibs, color=color, **config)
        for color, config in zip(color_list, config_list)
    ]
    area_list = [ret[0] for ret in ret_list]
    conf_list = [ret[1] for ret in ret_list]
    # index = np.argmax(area_list)
    index = -1
    best_label2 = config_list[index]['label']
    best_config2 = config_list[index]
    best_color2 = color_list[index]
    best_area2 = area_list[index]
    best_conf2 = conf_list[index]
    plt.title(
        'ROC Curve (Best: {}, AP = {:0.02f})'.format(best_label2, best_area2), y=1.10
    )
    plt.legend(
        bbox_to_anchor=(0.0, 1.02, 1.0, 0.102),
        loc=3,
        ncol=2,
        mode='expand',
        borderaxespad=0.0,
    )

    axes_ = plt.subplot(223)
    axes_.set_aspect(1)
    gca_ = plt.gca()
    gca_.grid(False)
    correct_rate, _ = canonical_confusion_matrix_algo_plot(
        ibs,
        color=best_color1,
        conf=best_conf1,
        fig_=fig_,
        axes_=axes_,
        output_cases=True,
        **best_config1,
    )
    axes_.set_xlabel('Predicted (Correct = {:0.02f}%)'.format(correct_rate * 100.0))
    axes_.set_ylabel('Ground-Truth')
    plt.title('P-R Confusion Matrix (OP = {:0.02f})'.format(best_conf1), y=1.12)

    axes_ = plt.subplot(224)
    axes_.set_aspect(1)
    gca_ = plt.gca()
    gca_.grid(False)
    correct_rate, _ = canonical_confusion_matrix_algo_plot(
        ibs, color=best_color2, conf=best_conf2, fig_=fig_, axes_=axes_, **best_config2
    )
    axes_.set_xlabel('Predicted (Correct = {:0.02f}%)'.format(correct_rate * 100.0))
    axes_.set_ylabel('Ground-Truth')
    plt.title('ROC Confusion Matrix (OP = {:0.02f})'.format(best_conf2), y=1.12)

    fig_filename = 'canonical-precision-recall-roc.png'
    fig_path = abspath(expanduser(join('~', 'Desktop', fig_filename)))
    plt.savefig(fig_path, bbox_inches='tight')


def _canonical_get_boxes(ibs, gid_list, species):
    from wbia.web.appfuncs import CANONICAL_PART_TYPE

    aid_list = ut.flatten(ibs.get_image_aids(gid_list))
    aid_list = ibs.filter_annotation_set(aid_list, species=species)
    flag_list = ibs.get_annot_canonical(aid_list)
    part_rowids_list = ibs.get_annot_part_rowids(aid_list)
    part_types_list = list(map(ibs.get_part_types, part_rowids_list))

    aid_set = []
    bbox_set = []
    zipped = zip(aid_list, flag_list, part_rowids_list, part_types_list)
    for aid, flag, part_rowid_list, part_type_list in zipped:
        gid = ibs.get_annot_gids(aid)
        is_tile = ibs.get_tile_flags(gid)
        reference_tile_gid = gid if is_tile else None

        part_rowid_ = None
        if flag:
            for part_rowid, part_type in zip(part_rowid_list, part_type_list):
                if part_type == CANONICAL_PART_TYPE:
                    assert part_rowid_ is None, 'Cannot have multiple CA for one image'
                    part_rowid_ = part_rowid

        if part_rowid_ is not None:
            axtl, aytl, aw, ah = ibs.get_annot_bboxes(
                aid, reference_tile_gid=reference_tile_gid
            )
            axbr, aybr = axtl + aw, aytl + ah
            pxtl, pytl, pw, ph = ibs.get_part_bboxes(part_rowid_)
            pxbr, pybr = pxtl + pw, pytl + ph
            x0 = pxtl - axtl
            y0 = pytl - aytl
            x1 = axbr - pxbr
            y1 = aybr - pybr
            x0 = max(x0 / aw, 0.0)
            y0 = max(y0 / ah, 0.0)
            x1 = max(x1 / aw, 0.0)
            y1 = max(y1 / ah, 0.0)
            assert x0 + x1 < 0.99
            assert y0 + y1 < 0.99
            bbox = (x0, y0, x1, y1)
            aid_set.append(aid)
            bbox_set.append(bbox)

    return aid_set, bbox_set


def canonical_localization_deviation_plot(
    ibs, attribute, color, index, label=None, species=None, marker='o', **kwargs
):
    import random

    import matplotlib.pyplot as plt

    assert None not in [label, species]
    logger.info('Processing Deviation for: {!r}'.format(label))
    depc = ibs.depc_annot

    if attribute == 'x0':
        take_index = 0
    elif attribute == 'y0':
        take_index = 1
    elif attribute == 'x1':
        take_index = 2
    elif attribute == 'y1':
        take_index = 3
    else:
        raise ValueError('attribute not valid')

    test_gid_set_ = set(general_get_imageset_gids(ibs, 'TEST_SET'))
    test_gid_list_ = list(test_gid_set_)
    test_aid_set, test_bbox_set = _canonical_get_boxes(ibs, test_gid_list_, species)

    value_list = ut.take_column(test_bbox_set, take_index)
    prediction_list = depc.get_property(
        'canonical', test_aid_set, attribute, config=kwargs
    )

    x_list = []
    y_list = []
    overshoot = 0.0
    for value, prediction in zip(value_list, prediction_list):
        x = random.uniform(index, index + 1)
        y = value - prediction
        if y < 0:
            overshoot += 1
        x_list.append(x)
        y_list.append(y)
    mean = np.mean(y_list)
    std = np.std(y_list)
    overshoot /= len(y_list)

    label = '{} (Over: {:0.02f}, {:0.02f}+/-{:0.02f})'.format(label, overshoot, mean, std)
    plt.plot(
        x_list,
        y_list,
        color=color,
        linestyle='None',
        marker=marker,
        label=label,
        alpha=0.5,
    )

    plt.plot(
        [index, index + 1], [0.0, 0.0], color=(0.2, 0.2, 0.2), linestyle='-', alpha=0.3
    )
    if index % 4 == 3:
        plt.plot(
            [index + 1, index + 1],
            [-1.0, 1.0],
            color=(0.2, 0.2, 0.2),
            linestyle='--',
            alpha=0.1,
        )

    color = 'xkcd:gold'
    marker = 'D'
    plt.errorbar(
        [index + 0.5],
        [mean],
        [std],
        linestyle='None',
        color=color,
        marker=marker,
        zorder=999,
        barsabove=True,
    )
    # plt.plot([index + 0.5], [mean], color=color, marker=marker)


def canonical_localization_iou_plot(
    ibs, color, index, label=None, species=None, marker='o', threshold=0.75, **kwargs
):
    import random

    import matplotlib.pyplot as plt

    def _convert(bbox):
        x0, y0, x1, y1 = bbox
        retval = {
            'xtl': x0,
            'ytl': y0,
            'xbr': 1.0 - x1,
            'ybr': 1.0 - y1,
        }
        retval['width'] = retval['xbr'] - retval['xtl']
        retval['height'] = retval['ybr'] - retval['ytl']
        return retval

    assert None not in [label, species]
    logger.info('Processing IoU for: {!r}'.format(label))
    depc = ibs.depc_annot

    test_gid_set_ = set(general_get_imageset_gids(ibs, 'TEST_SET'))
    test_gid_list_ = list(test_gid_set_)
    test_aid_set, test_bbox_set = _canonical_get_boxes(ibs, test_gid_list_, species)

    prediction_list = depc.get_property('canonical', test_aid_set, None, config=kwargs)

    gt_list = [_convert(test_bbox) for test_bbox in test_bbox_set]
    pred_list = [_convert(prediction) for prediction in prediction_list]

    correct = 0.0
    x_list = []
    y_list = []
    for gt, pred in zip(gt_list, pred_list):
        overlap = general_overlap([gt], [pred])
        x = random.uniform(index, index + 1)
        y = overlap[0][0]
        if y >= threshold:
            correct += 1.0
        x_list.append(x)
        y_list.append(y)
    accuracy = correct / len(y_list)
    mean = np.mean(y_list)
    std = np.std(y_list)

    label = '{} (Acc: {:0.02f}, {:0.02f}+/-{:0.02f})'.format(label, accuracy, mean, std)
    plt.plot(
        x_list,
        y_list,
        color=color,
        linestyle='None',
        marker=marker,
        label=label,
        alpha=0.5,
    )

    for y_value in [0.5, 0.75, 0.9]:
        plt.plot(
            [index, index + 1],
            [y_value, y_value],
            color=(0.2, 0.2, 0.2),
            linestyle='-',
            alpha=0.3,
        )

    if index % 4 == 3:
        plt.plot(
            [index + 1, index + 1],
            [0.0, 1.0],
            color=(0.2, 0.2, 0.2),
            linestyle='--',
            alpha=0.1,
        )

    color = 'xkcd:gold'
    marker = 'D'
    plt.errorbar(
        [index + 0.5],
        [mean],
        [std],
        linestyle='None',
        color=color,
        marker=marker,
        zorder=999,
        barsabove=True,
    )
    # plt.plot([index + 0.5], [mean], color=color, marker=marker)

    return test_aid_set, test_bbox_set, prediction_list, y_list, accuracy


@register_ibs_method
def canonical_localization_iou_visualize(
    ibs,
    index,
    test_aid_set,
    test_bbox_set,
    prediction_list,
    overlap_list,
    color_list,
    label=None,
    species=None,
    **kwargs,
):
    assert None not in [label, species]
    assert len(color_list) == 4
    logger.info('Processing Renderings for: {!r}'.format(label))

    color_list_ = []
    for color in color_list:
        color_ = []
        for value in color:
            value_ = int(np.around(255.0 * value))
            color_ = [value_] + color_
        color_ = tuple(color_)
        color_list_.append(color_)
    color_list = color_list_

    output_path = expanduser(join('~', 'Desktop', 'canonical-regression-%d' % (index,)))
    ut.delete(output_path)
    ut.ensuredir(output_path)

    config = {
        'dim_size': 600,
        'resize_dim': 'maxwh',
    }
    chip_list = ibs.depc_annot.get_property('chips', test_aid_set, 'img', config=config)
    zipped = list(
        zip(test_aid_set, chip_list, test_bbox_set, prediction_list, overlap_list)
    )

    for test_aid, chip, test_bbox, prediction, overlap in zipped:
        h, w = chip.shape[:2]

        chipa = chip.copy()
        chipb = chip.copy()

        x0a, y0a, x1a, y1a = test_bbox
        x0b, y0b, x1b, y1b = prediction

        x0a = int(np.around(x0a * w))
        y0a = int(np.around(y0a * h))
        x1a = int(np.around(x1a * w))
        y1a = int(np.around(y1a * h))

        x0b = int(np.around(x0b * w))
        y0b = int(np.around(y0b * h))
        x1b = int(np.around(x1b * w))
        y1b = int(np.around(y1b * h))

        x1a = w - x1a
        x1b = w - x1b
        y1a = h - y1a
        y1b = h - y1b

        chipa = cv2.line(chipa, (x0a, y0a), (x0a, y1a), color_list[0], 3)
        chipa = cv2.line(chipa, (x0a, y0a), (x1a, y0a), color_list[1], 3)
        chipa = cv2.line(chipa, (x1a, y0a), (x1a, y1a), color_list[2], 3)
        chipa = cv2.line(chipa, (x0a, y1a), (x1a, y1a), color_list[3], 3)

        chipb = cv2.line(chipb, (x0b, y0b), (x0b, y1b), color_list[0], 3)
        chipb = cv2.line(chipb, (x0b, y0b), (x1b, y0b), color_list[1], 3)
        chipb = cv2.line(chipb, (x1b, y0b), (x1b, y1b), color_list[2], 3)
        chipb = cv2.line(chipb, (x0b, y1b), (x1b, y1b), color_list[3], 3)

        canvas = np.hstack((chipa, chipb))

        canvas_filepath = join(
            output_path,
            'canonical-regression-iou-{:0.02f}-aid-{}.jpg'.format(overlap, test_aid),
        )
        cv2.imwrite(canvas_filepath, canvas)


@register_ibs_method
def canonical_localization_precision_recall_algo_display(ibs, figsize=(20, 40)):
    import matplotlib.pyplot as plt

    from wbia import plottool as pt

    fig_ = plt.figure(figsize=figsize, dpi=400)  # NOQA

    config_list = [
        # {'label': 'CA V1 Ensemble', 'canonical_weight_filepath': 'canonical_zebra_grevys_v1',   'species': 'zebra_grevys'}, # OVER = 1.0, small dataset
        # {'label': 'CA V1 Model 0',  'canonical_weight_filepath': 'canonical_zebra_grevys_v1:0', 'species': 'zebra_grevys'}, # OVER = 1.0, small dataset
        # {'label': 'CA V1 Model 1',  'canonical_weight_filepath': 'canonical_zebra_grevys_v1:1', 'species': 'zebra_grevys'}, # OVER = 1.0, small dataset
        # {'label': 'CA V1 Model 2',  'canonical_weight_filepath': 'canonical_zebra_grevys_v1:2', 'species': 'zebra_grevys'}, # OVER = 1.0, small dataset
        # {'label': 'CA V2 Ensemble', 'canonical_weight_filepath': 'canonical_zebra_grevys_v2',   'species': 'zebra_grevys'}, # OVER = 1.0, large dataset
        # {'label': 'CA V2 Model 0',  'canonical_weight_filepath': 'canonical_zebra_grevys_v2:0', 'species': 'zebra_grevys'}, # OVER = 1.0, large dataset
        # {'label': 'CA V2 Model 1',  'canonical_weight_filepath': 'canonical_zebra_grevys_v2:1', 'species': 'zebra_grevys'}, # OVER = 1.0, large dataset
        # {'label': 'CA V2 Model 2',  'canonical_weight_filepath': 'canonical_zebra_grevys_v2:2', 'species': 'zebra_grevys'}, # OVER = 1.0, large dataset
        # {'label': 'CA V3 Ensemble', 'canonical_weight_filepath': 'canonical_zebra_grevys_v3',   'species': 'zebra_grevys'},  # OVER = 2.0
        # {'label': 'CA V3 Model 0',  'canonical_weight_filepath': 'canonical_zebra_grevys_v3:0', 'species': 'zebra_grevys'},  # OVER = 2.0
        # {'label': 'CA V3 Model 1',  'canonical_weight_filepath': 'canonical_zebra_grevys_v3:1', 'species': 'zebra_grevys'},  # OVER = 2.0
        # {'label': 'CA V3 Model 2',  'canonical_weight_filepath': 'canonical_zebra_grevys_v3:2', 'species': 'zebra_grevys'},  # OVER = 2.0
        {
            'label': 'CA V5-1.0 Ens.',
            'canonical_weight_filepath': 'canonical_zebra_grevys_v5',
            'species': 'zebra_grevys',
        },  # OVER = 1.0
        {
            'label': 'CA V5-1.0 M0',
            'canonical_weight_filepath': 'canonical_zebra_grevys_v5:0',
            'species': 'zebra_grevys',
        },  # OVER = 1.0
        {
            'label': 'CA V5-1.0 M1',
            'canonical_weight_filepath': 'canonical_zebra_grevys_v5:1',
            'species': 'zebra_grevys',
        },  # OVER = 1.0
        {
            'label': 'CA V5-1.0 M2',
            'canonical_weight_filepath': 'canonical_zebra_grevys_v5:2',
            'species': 'zebra_grevys',
        },  # OVER = 1.0
        {
            'label': 'CA V6-2.0 Ens.',
            'canonical_weight_filepath': 'canonical_zebra_grevys_v6',
            'species': 'zebra_grevys',
        },  # OVER = 2.0
        {
            'label': 'CA V6-2.0 M0',
            'canonical_weight_filepath': 'canonical_zebra_grevys_v6:0',
            'species': 'zebra_grevys',
        },  # OVER = 2.0
        {
            'label': 'CA V6-2.0 M1',
            'canonical_weight_filepath': 'canonical_zebra_grevys_v6:1',
            'species': 'zebra_grevys',
        },  # OVER = 2.0
        {
            'label': 'CA V6-2.0 M2',
            'canonical_weight_filepath': 'canonical_zebra_grevys_v6:2',
            'species': 'zebra_grevys',
        },  # OVER = 2.0
        {
            'label': 'CA V4-4.0 Ens.',
            'canonical_weight_filepath': 'canonical_zebra_grevys_v4',
            'species': 'zebra_grevys',
        },  # OVER = 4.0
        {
            'label': 'CA V4-4.0 M0',
            'canonical_weight_filepath': 'canonical_zebra_grevys_v4:0',
            'species': 'zebra_grevys',
        },  # OVER = 4.0
        {
            'label': 'CA V4-4.0 M1',
            'canonical_weight_filepath': 'canonical_zebra_grevys_v4:1',
            'species': 'zebra_grevys',
        },  # OVER = 4.0
        {
            'label': 'CA V4-4.0 M2',
            'canonical_weight_filepath': 'canonical_zebra_grevys_v4:2',
            'species': 'zebra_grevys',
        },  # OVER = 4.0
    ]
    color_list = []
    # color_list = [(0, 0, 0)]
    color_list += pt.distinct_colors(len(config_list) - len(color_list), randomize=False)

    min_, max_ = -1.0, 1.0

    axes_ = plt.subplot(321)
    axes_.grid(True, which='major')
    axes_.grid(False, which='minor')
    axes_.set_autoscalex_on(False)
    axes_.set_autoscaley_on(False)
    axes_.get_xaxis().set_ticks([])
    axes_.set_ylabel('GT - Pred Deviation (in percentages)')
    axes_.set_xlim([0.0, len(config_list)])
    axes_.set_ylim([min_, max_])
    axes_.fill_between([0.0, len(config_list)], -1, 0, facecolor='red', alpha=0.1)
    for index, (color, config) in enumerate(zip(color_list, config_list)):
        canonical_localization_deviation_plot(
            ibs, 'x0', color=color, index=index, **config
        )

    plt.title('X0 Deviation Scatter Plot')
    plt.legend(
        bbox_to_anchor=(0.0, 1.04, 1.0, 0.102),
        loc=3,
        ncol=2,
        mode='expand',
        borderaxespad=0.0,
    )

    axes_ = plt.subplot(322)
    axes_.grid(True, which='major')
    axes_.grid(False, which='minor')
    axes_.set_autoscalex_on(False)
    axes_.set_autoscaley_on(False)
    axes_.get_xaxis().set_ticks([])
    axes_.set_ylabel('GT - Pred Deviation (in percentages)')
    axes_.set_xlim([0.0, len(config_list)])
    axes_.set_ylim([min_, max_])
    axes_.fill_between([0.0, len(config_list)], -1, 0, facecolor='red', alpha=0.1)
    for index, (color, config) in enumerate(zip(color_list, config_list)):
        canonical_localization_deviation_plot(
            ibs, 'x1', color=color, index=index, **config
        )

    plt.title('Y0 Deviation Scatter Plot')
    plt.legend(
        bbox_to_anchor=(0.0, 1.04, 1.0, 0.102),
        loc=3,
        ncol=2,
        mode='expand',
        borderaxespad=0.0,
    )

    axes_ = plt.subplot(323)
    axes_.grid(True, which='major')
    axes_.grid(False, which='minor')
    axes_.set_autoscalex_on(False)
    axes_.set_autoscaley_on(False)
    axes_.get_xaxis().set_ticks([])
    axes_.set_ylabel('GT - Pred Deviation (in percentages)')
    axes_.set_xlim([0.0, len(config_list)])
    axes_.set_ylim([min_, max_])
    axes_.fill_between([0.0, len(config_list)], -1, 0, facecolor='red', alpha=0.1)
    for index, (color, config) in enumerate(zip(color_list, config_list)):
        canonical_localization_deviation_plot(
            ibs, 'y0', color=color, index=index, **config
        )

    plt.title('X1 Deviation Scatter Plot')
    plt.legend(
        bbox_to_anchor=(0.0, 1.04, 1.0, 0.102),
        loc=3,
        ncol=2,
        mode='expand',
        borderaxespad=0.0,
    )

    axes_ = plt.subplot(324)
    axes_.grid(True, which='major')
    axes_.grid(False, which='minor')
    axes_.set_autoscalex_on(False)
    axes_.set_autoscaley_on(False)
    axes_.get_xaxis().set_ticks([])
    axes_.set_ylabel('GT - Pred Deviation (in percentages)')
    axes_.set_xlim([0.0, len(config_list)])
    axes_.set_ylim([min_, max_])
    axes_.fill_between([0.0, len(config_list)], -1, 0, facecolor='red', alpha=0.1)
    for index, (color, config) in enumerate(zip(color_list, config_list)):
        canonical_localization_deviation_plot(
            ibs, 'y1', color=color, index=index, **config
        )

    plt.title('Y1 Deviation Scatter Plot')
    plt.legend(
        bbox_to_anchor=(0.0, 1.04, 1.0, 0.102),
        loc=3,
        ncol=2,
        mode='expand',
        borderaxespad=0.0,
    )

    axes_ = plt.subplot(325)
    axes_.grid(True, which='major')
    axes_.grid(False, which='minor')
    axes_.set_autoscalex_on(False)
    axes_.set_autoscaley_on(False)
    axes_.get_xaxis().set_ticks([])
    axes_.set_ylabel('GT - Pred Deviation (in percentages)')
    axes_.set_xlim([0.0, len(config_list)])
    axes_.set_ylim([min_, max_])
    axes_.fill_between([0.0, len(config_list)], -1, 0, facecolor='red', alpha=0.1)

    assert len(config_list) % 4 == 0
    rounds = len(config_list) // 4
    colors = pt.distinct_colors(4, randomize=False)

    attribute_list = []
    color_list_ = []
    for _ in range(rounds):
        attribute_list += ['x0', 'y0', 'x1', 'y1']
        color_list_ += colors

    for index, (attribute, color_) in enumerate(zip(attribute_list, color_list_)):
        index_ = (index // 4) * 4
        config_ = config_list[index_].copy()
        config_['label'] = '{} {}'.format(config_['label'], attribute)
        canonical_localization_deviation_plot(
            ibs, attribute, color=color_, index=index, **config_
        )

    plt.title('Ensemble Deviation Scatter Plot')
    plt.legend(
        bbox_to_anchor=(0.0, 1.04, 1.0, 0.102),
        loc=3,
        ncol=2,
        mode='expand',
        borderaxespad=0.0,
    )

    axes_ = plt.subplot(326)
    axes_.grid(True, which='major')
    axes_.grid(False, which='minor')
    axes_.set_autoscalex_on(False)
    axes_.set_autoscaley_on(False)
    axes_.get_xaxis().set_ticks([])
    axes_.set_ylabel('IoU')
    axes_.set_xlim([0.0, len(config_list)])
    axes_.set_ylim([0.0, 1.0])

    for index, (color, config) in enumerate(zip(color_list, config_list)):
        values_ = canonical_localization_iou_plot(ibs, color=color, index=index, **config)
        if index % 4 == 0:
            config_ = config_list[index]
            test_aid_set, test_bbox_set, prediction_list, y_list, accuracy = values_
            ibs.canonical_localization_iou_visualize(
                index,
                test_aid_set,
                test_bbox_set,
                prediction_list,
                y_list,
                colors,
                **config_,
            )

    plt.title('IoU Scatter Plot')
    plt.legend(
        bbox_to_anchor=(0.0, 1.04, 1.0, 0.102),
        loc=3,
        ncol=2,
        mode='expand',
        borderaxespad=0.0,
    )

    fig_filename = 'canonical-localization-deviance.png'
    fig_path = abspath(expanduser(join('~', 'Desktop', fig_filename)))
    plt.savefig(fig_path, bbox_inches='tight')


@register_ibs_method
def background_accuracy_display(ibs, category_list, test_gid_set=None, output_path=None):

    if output_path is None:
        output_path = abspath(expanduser(join('~', 'Desktop', 'background')))
        ut.ensuredir(output_path)

    if test_gid_set is None:
        test_gid_set = set(general_get_imageset_gids(ibs, 'TEST_SET'))
        test_gid_set = list(test_gid_set)
    aids_list = ibs.get_image_aids(test_gid_set)
    aid_list = ut.flatten(aids_list)
    species_list = ibs.get_annot_species_texts(aid_list)

    aid_list = [
        aid for aid, species in zip(aid_list, species_list) if species in category_list
    ]
    species_list = ibs.get_annot_species_texts(aid_list)
    gid_list = ibs.get_annot_gids(aid_list)

    config2_ = {'fw_detector': 'cnn'}
    hough_cpath_list = ibs.get_annot_probchip_fpath(aid_list, config2_=config2_)
    image_list = [vt.imread(hough_cpath) for hough_cpath in hough_cpath_list]
    chip_list = ibs.get_annot_chips(aid_list, config2_=config2_)
    zipped = zip(aid_list, gid_list, species_list, image_list, chip_list)
    for index, (aid, gid, species, image, chip) in enumerate(zipped):
        logger.info(index)
        mask = vt.resize_mask(image, chip)
        blended = vt.blend_images_multiply(chip, mask)
        blended *= 255.0
        blended = np.around(blended)
        blended[blended < 0] = 0
        blended[blended > 255] = 255
        blended = blended.astype(np.uint8)

        canvas = np.hstack((chip, mask, blended))
        output_filepath = join(
            output_path, 'background.%s.%d.%d.png' % (species, gid, aid)
        )
        cv2.imwrite(output_filepath, canvas)


def aoi2_precision_recall_algo(ibs, category_list=None, test_gid_set_=None, **kwargs):
    depc = ibs.depc_annot
    if test_gid_set_ is None:
        test_gid_set_ = general_get_imageset_gids(ibs, 'TEST_SET')
    test_aid_list_ = list(set(ut.flatten(ibs.get_image_aids(test_gid_set_))))
    species_list = ibs.get_annot_species_texts(test_aid_list_)
    interest_list = ibs.get_annot_interest(test_aid_list_)

    test_aid_list = []
    label_list = []
    for test_aid, species, interest in zip(test_aid_list_, species_list, interest_list):
        if category_list is not None:
            if species not in category_list:
                continue
        if interest is None:
            continue
        label = 'positive' if interest else 'negative'
        test_aid_list.append(test_aid)
        label_list.append(label)

    prediction_list = depc.get_property('aoi_two', test_aid_list, 'class', config=kwargs)
    confidence_list = depc.get_property('aoi_two', test_aid_list, 'score', config=kwargs)
    confidence_list = [
        confidence if prediction == 'positive' else 1.0 - confidence
        for prediction, confidence in zip(prediction_list, confidence_list)
    ]
    return general_precision_recall_algo(ibs, label_list, confidence_list, **kwargs)


def aoi2_precision_recall_algo_plot(ibs, **kwargs):
    label = kwargs['label']
    logger.info('Processing Precision-Recall for: {!r}'.format(label))
    conf_list, pr_list, re_list, tpr_list, fpr_list = aoi2_precision_recall_algo(
        ibs, **kwargs
    )
    return general_area_best_conf(conf_list, re_list, pr_list, **kwargs)


def aoi2_roc_algo_plot(ibs, target_recall=None, **kwargs):
    label = kwargs['label']
    logger.info('Processing ROC for: {!r}'.format(label))
    conf_list, pr_list, re_list, tpr_list, fpr_list = aoi2_precision_recall_algo(
        ibs, **kwargs
    )
    return general_area_best_conf(
        conf_list, fpr_list, tpr_list, interpolate=False, target=(0.0, 1.0), **kwargs
    )


def aoi2_confusion_matrix_algo_plot(
    ibs,
    label,
    color,
    conf,
    output_cases=False,
    category_list=None,
    test_gid_set_=None,
    **kwargs,
):
    logger.info(
        'Processing Confusion Matrix for: {!r} (Conf = {:0.02f})'.format(label, conf)
    )
    depc = ibs.depc_annot
    if test_gid_set_ is None:
        test_gid_set_ = general_get_imageset_gids(ibs, 'TEST_SET')
    test_aid_list_ = list(set(ut.flatten(ibs.get_image_aids(test_gid_set_))))
    species_list = ibs.get_annot_species_texts(test_aid_list_)
    interest_list = ibs.get_annot_interest(test_aid_list_)

    test_aid_list = []
    label_list = []
    for test_aid, species, interest in zip(test_aid_list_, species_list, interest_list):
        if category_list is not None:
            if species not in category_list:
                continue
        if interest is None:
            continue
        label = 'positive' if interest else 'negative'
        test_aid_list.append(test_aid)
        label_list.append(label)

    prediction_list = depc.get_property('aoi_two', test_aid_list, 'class', config=kwargs)
    confidence_list = depc.get_property('aoi_two', test_aid_list, 'score', config=kwargs)
    confidence_list = [
        confidence if prediction == 'positive' else 1.0 - confidence
        for prediction, confidence in zip(prediction_list, confidence_list)
    ]
    prediction_list = [
        'positive' if confidence >= conf else 'negative' for confidence in confidence_list
    ]

    if output_cases:
        output_path = 'aoi2-confusion-incorrect'
        output_path = abspath(expanduser(join('~', 'Desktop', output_path)))
        ut.delete(output_path)
        ut.ensuredir(output_path)

        manifest_dict = {}
        test_gid_list = ibs.get_annot_gids(test_aid_list)
        zipped = zip(test_gid_list, test_aid_list, label_list, prediction_list)
        for test_gid, test_aid, label, prediction in zipped:
            if test_gid not in manifest_dict:
                manifest_dict[test_gid] = {}
            assert test_aid not in manifest_dict[test_gid]
            manifest_dict[test_gid][test_aid] = (
                label,
                prediction,
            )

        for test_gid in manifest_dict:
            image = ibs.get_images(test_gid)
            w, h = ibs.get_image_sizes(test_gid)
            image = _resize(image, t_width=600, verbose=False)
            height_, width_, channels_ = image.shape

            is_tile = ibs.get_tile_flags(test_gid)
            reference_tile_gid = test_gid if is_tile else None

            for test_aid in manifest_dict[test_gid]:
                label, prediction = manifest_dict[test_gid][test_aid]
                bbox = ibs.get_annot_bboxes(
                    test_aid, reference_tile_gid=reference_tile_gid
                )
                xtl, ytl, width, height = bbox
                xbr = xtl + width
                ybr = ytl + height

                xtl = int(np.round((xtl / w) * width_))
                ytl = int(np.round((ytl / h) * height_))
                xbr = int(np.round((xbr / w) * width_))
                ybr = int(np.round((ybr / h) * height_))
                if label == 'positive':
                    color = (255, 99, 46)
                else:
                    color = (127, 255, 127)
                cv2.rectangle(image, (xtl, ytl), (xbr, ybr), color, 4)
                if prediction == 'positive':
                    color = (255, 99, 46)
                else:
                    color = (127, 255, 127)
                cv2.rectangle(image, (xtl - 4, ytl - 4), (xbr + 4, ybr + 4), color, 4)

            image_filename = 'image_%d.png' % (test_gid,)
            image_filepath = join(output_path, image_filename)
            cv2.imwrite(image_filepath, image)

    category_list = ['positive', 'negative']
    category_mapping = {
        'positive': 0,
        'negative': 1,
    }
    return general_confusion_matrix_algo(
        label_list, prediction_list, category_list, category_mapping, size=20, **kwargs
    )


@register_ibs_method
def aoi2_precision_recall_algo_display(
    ibs, test_gid_list=None, output_cases=False, figsize=(20, 20)
):
    import matplotlib.pyplot as plt

    import wbia.plottool as pt

    fig_ = plt.figure(figsize=figsize)

    test_gid_set = None if test_gid_list is None else sorted(set(test_gid_list))
    config_list = [
        # {'label': 'All Species',         'aoi_two_weight_filepath': 'ggr2', 'category_list': None},
        # {'label': 'Masai Giraffe',       'aoi_two_weight_filepath': 'ggr2', 'category_list': ['giraffe_masai']},
        # {'label': 'Reticulated Giraffe', 'aoi_two_weight_filepath': 'ggr2', 'category_list': ['giraffe_reticulated']},
        # {'label': 'Sea Turtle',          'aoi_two_weight_filepath': 'ggr2', 'category_list': ['turtle_sea']},
        # {'label': 'Whale Fluke',         'aoi_two_weight_filepath': 'ggr2', 'category_list': ['whale_fluke']},
        # {'label': 'Grevy\'s Zebra',      'aoi_two_weight_filepath': 'ggr2', 'category_list': ['zebra_grevys']},
        # {'label': 'Plains Zebra',        'aoi_two_weight_filepath': 'ggr2', 'category_list': ['zebra_plains']},
        # {'label': 'Hammerhead',        'aoi_two_weight_filepath': 'hammerhead', 'category_list': ['shark_hammerhead']},
        {
            'label': 'Jaguar',
            'aoi_two_weight_filepath': 'jaguar',
            'category_list': ['jaguar'],
        },
    ]
    color_list = [(0, 0, 0)]
    color_list += pt.distinct_colors(len(config_list) - len(color_list), randomize=False)

    axes_ = plt.subplot(221)
    axes_.set_autoscalex_on(False)
    axes_.set_autoscaley_on(False)
    axes_.set_xlabel('Recall')
    axes_.set_ylabel('Precision')
    axes_.set_xlim([0.0, 1.01])
    axes_.set_ylim([0.0, 1.01])
    ret_list = [
        aoi2_precision_recall_algo_plot(
            ibs, color=color, test_gid_set_=test_gid_set, **config
        )
        for color, config in zip(color_list, config_list)
    ]
    area_list = [ret[0] for ret in ret_list]
    conf_list = [ret[1] for ret in ret_list]
    # index = np.argmax(area_list)
    index = 0
    best_label1 = config_list[index]['label']
    best_config1 = config_list[index]
    best_color1 = color_list[index]
    best_area1 = area_list[index]
    best_conf1 = conf_list[index]
    plt.title(
        'Precision-Recall Curve (Best: {}, AP = {:0.02f})'.format(
            best_label1, best_area1
        ),
        y=1.10,
    )
    plt.legend(
        bbox_to_anchor=(0.0, 1.02, 1.0, 0.102),
        loc=3,
        ncol=2,
        mode='expand',
        borderaxespad=0.0,
    )

    axes_ = plt.subplot(222)
    axes_.set_autoscalex_on(False)
    axes_.set_autoscaley_on(False)
    axes_.set_xlabel('False-Positive Rate')
    axes_.set_ylabel('True-Positive Rate')
    axes_.set_xlim([0.0, 1.01])
    axes_.set_ylim([0.0, 1.01])
    ret_list = [
        aoi2_roc_algo_plot(ibs, color=color, **config)
        for color, config in zip(color_list, config_list)
    ]
    area_list = [ret[0] for ret in ret_list]
    conf_list = [ret[1] for ret in ret_list]
    # index = np.argmax(area_list)
    index = 0
    best_label2 = config_list[index]['label']
    best_config2 = config_list[index]
    best_color2 = color_list[index]
    best_area2 = area_list[index]
    best_conf2 = conf_list[index]
    plt.title(
        'ROC Curve (Best: {}, AP = {:0.02f})'.format(best_label2, best_area2), y=1.10
    )
    plt.legend(
        bbox_to_anchor=(0.0, 1.02, 1.0, 0.102),
        loc=3,
        ncol=2,
        mode='expand',
        borderaxespad=0.0,
    )
    plt.plot([0.0, 1.0], [0.0, 1.0], color=(0.5, 0.5, 0.5), linestyle='--')
    axes_ = plt.subplot(223)
    axes_.set_aspect(1)
    gca_ = plt.gca()
    gca_.grid(False)
    correct_rate, _ = aoi2_confusion_matrix_algo_plot(
        ibs,
        color=best_color1,
        conf=best_conf1,
        fig_=fig_,
        axes_=axes_,
        output_cases=output_cases,
        test_gid_set_=test_gid_set,
        **best_config1,
    )
    axes_.set_xlabel('Predicted (Correct = {:0.02f}%)'.format(correct_rate * 100.0))
    axes_.set_ylabel('Ground-Truth')
    plt.title('P-R Confusion Matrix (OP = {:0.02f})'.format(best_conf1), y=1.12)

    axes_ = plt.subplot(224)
    axes_.set_aspect(1)
    gca_ = plt.gca()
    gca_.grid(False)
    correct_rate, _ = aoi2_confusion_matrix_algo_plot(
        ibs,
        color=best_color2,
        conf=best_conf2,
        fig_=fig_,
        axes_=axes_,
        test_gid_set_=test_gid_set,
        **best_config2,
    )
    axes_.set_xlabel('Predicted (Correct = {:0.02f}%)'.format(correct_rate * 100.0))
    axes_.set_ylabel('Ground-Truth')
    plt.title('ROC Confusion Matrix (OP = {:0.02f})'.format(best_conf2), y=1.12)

    fig_filename = 'aoi2-precision-recall-roc.png'
    fig_path = abspath(expanduser(join('~', 'Desktop', fig_filename)))
    plt.savefig(fig_path, bbox_inches='tight')


def detector_parse_gt(ibs, test_gid_list=None, **kwargs):
    if test_gid_list is None:
        test_gid_list = general_get_imageset_gids(ibs, 'TEST_SET', **kwargs)
    uuid_list = ibs.get_image_uuids(test_gid_list)
    gid_list = ibs.get_image_gids_from_uuid(uuid_list)

    gt_dict = {}
    for gid, uuid in zip(gid_list, uuid_list):
        is_tile = ibs.get_tile_flags(gid)
        reference_tile_gid = gid if is_tile else None
        width, height = ibs.get_image_sizes(gid)
        aid_list = ibs.get_image_aids(gid)
        bbox_list = ibs.get_annot_bboxes(aid_list, reference_tile_gid=reference_tile_gid)
        gt_list = []
        for aid, bbox in zip(aid_list, bbox_list):
            temp = {
                'gid': gid,
                'xtl': bbox[0] / width,
                'ytl': bbox[1] / height,
                'xbr': (bbox[0] + bbox[2]) / width,
                'ybr': (bbox[1] + bbox[3]) / height,
                'width': bbox[2] / width,
                'height': bbox[3] / height,
                'class': ibs.get_annot_species_texts(aid),
                'viewpoint': ibs.get_annot_viewpoints(aid),
                'confidence': 1.0,
            }
            gt_list.append(temp)
        gt_dict[uuid] = gt_list
    return gt_dict


# def detector_parse_pred(ibs, test_gid_list=None, **kwargs):
#     depc = ibs.depc_image

#     if test_gid_list is None:
#         test_gid_list = general_get_imageset_gids(ibs, 'TEST_SET', **kwargs)
#     uuid_list = ibs.get_image_uuids(test_gid_list)

#     # depc.delete_property('detections', test_gid_list, config=kwargs)
#     results_list = depc.get_property('detections', test_gid_list, None, config=kwargs)
#     size_list = ibs.get_image_sizes(test_gid_list)
#     zipped_list = zip(results_list)
#     # Reformat results for json
#     results_list = [
#         [
#             {
#                 'gid'        : test_gid,
#                 'xtl'        : bbox[0] / width,
#                 'ytl'        : bbox[1] / height,
#                 'width'      : bbox[2] / width,
#                 'height'     : bbox[3] / height,
#                 'theta'      : theta,  # round(theta, 4),
#                 'confidence' : conf,   # round(conf, 4),
#                 'class'      : class_,
#                 'viewpoint'  : viewpoint,
#             }
#             for bbox, theta, class_, viewpoint, conf in zip(*zipped[0][1:])
#         ]
#         for zipped, (width, height), test_gid in zip(zipped_list, size_list, test_gid_list)
#     ]

#     pred_dict = {
#         uuid_ : result_list
#         for uuid_, result_list in zip(uuid_list, results_list)
#     }
#     # logger.info(pred_dict)
#     return pred_dict


# def detector_precision_recall_algo(ibs, samples=SAMPLES, force_serial=FORCE_SERIAL, **kwargs):
#     test_gid_list = general_get_imageset_gids(ibs, 'TEST_SET', **kwargs)
#     uuid_list = ibs.get_image_uuids(test_gid_list)

#     logger.info('\tGather Ground-Truth')
#     gt_dict = detector_parse_gt(ibs, test_gid_list=test_gid_list)

#     logger.info('\tGather Predictions')
#     pred_dict = detector_parse_pred(ibs, test_gid_list=test_gid_list, **kwargs)

#     logger.info('\tGenerate Curves...')
#     conf_list = [ _ / float(samples) for _ in range(0, int(samples) + 1) ]
#     conf_list = sorted(conf_list, reverse=True)

#     uuid_list_list = [ uuid_list for _ in conf_list ]
#     gt_dict_list   = [ gt_dict   for _ in conf_list ]
#     pred_dict_list = [ pred_dict for _ in conf_list ]
#     kwargs_list    = [ kwargs    for _ in conf_list ]
#     arg_iter = zip(conf_list, uuid_list_list, gt_dict_list, pred_dict_list, kwargs_list)
#     pr_re_gen = ut.generate2(detector_precision_recall_algo_worker, arg_iter,
#                              nTasks=len(conf_list), ordered=True,
#                              chunksize=CHUNK_SIZE, force_serial=force_serial)

#     conf_list_ = [-1.0, -1.0]
#     pr_list = [1.0, 0.0]
#     re_list = [0.0, 1.0]
#     # conf_list_ = []
#     # pr_list = []
#     # re_list = []
#     for conf, pr, re in pr_re_gen:
#         conf_list_.append(conf)
#         pr_list.append(pr)
#         re_list.append(re)

#     logger.info('...complete')
#     return conf_list_, pr_list, re_list


# def detector_precision_recall_algo_worker(conf, uuid_list, gt_dict, pred_dict,
#                                           kwargs):
#     tp, fp, fn = 0.0, 0.0, 0.0
#     for index, uuid_ in enumerate(uuid_list):
#         if uuid_ in pred_dict:
#             pred_list = [
#                 pred
#                 for pred in pred_dict[uuid_]
#                 if pred['confidence'] >= conf
#             ]
#             tp_, fp_, fn_ = general_tp_fp_fn(gt_dict[uuid_], pred_list, **kwargs)
#             tp += tp_
#             fp += fp_
#             fn += fn_
#     pr = tp / (tp + fp)
#     re = tp / (tp + fn)
#     return (conf, pr, re)


# def detector_precision_recall_algo_plot(ibs, **kwargs):
#     label = kwargs['label']
#     logger.info('Processing Precision-Recall for: %r' % (label, ))
#     conf_list, pr_list, re_list = detector_precision_recall_algo(ibs, **kwargs)
#     return general_area_best_conf(conf_list, re_list, pr_list, **kwargs)


# def detector_confusion_matrix_algo_plot(ibs, label, color, conf, **kwargs):
#     logger.info('Processing Confusion Matrix for: %r (Conf = %0.02f)' % (label, conf, ))

#     test_gid_list = general_get_imageset_gids(ibs, 'TEST_SET', **kwargs)
#     uuid_list = ibs.get_image_uuids(test_gid_list)

#     logger.info('\tGather Ground-Truth')
#     gt_dict = detector_parse_gt(ibs, test_gid_list=test_gid_list)

#     logger.info('\tGather Predictions')
#     pred_dict = detector_parse_pred(ibs, test_gid_list=test_gid_list, **kwargs)

#     label_list = []
#     prediction_list = []
#     for index, uuid_ in enumerate(uuid_list):
#         if uuid_ in pred_dict:
#             gt_list = gt_dict[uuid_]
#             pred_list = [
#                 pred
#                 for pred in pred_dict[uuid_]
#                 if pred['confidence'] >= conf
#             ]
#             tp, fp, fn = general_tp_fp_fn(gt_list, pred_list, **kwargs)
#             for _ in range(int(tp)):
#                 label_list.append('positive')
#                 prediction_list.append('positive')
#             for _ in range(int(fp)):
#                 label_list.append('negative')
#                 prediction_list.append('positive')
#             for _ in range(int(fn)):
#                 label_list.append('positive')
#                 prediction_list.append('negative')

#     category_list = ['positive', 'negative']
#     category_mapping = {
#         'positive': 0,
#         'negative': 1,
#     }
#     return general_confusion_matrix_algo(label_list, prediction_list, category_list,
#                                          category_mapping, **kwargs)


# @register_ibs_method
# def detector_precision_recall_algo_display(ibs, min_overlap=0.5, figsize=(24, 7), **kwargs):
#     import matplotlib.pyplot as plt

#     fig_ = plt.figure(figsize=figsize)

#     axes_ = plt.subplot(131)
#     axes_.set_autoscalex_on(False)
#     axes_.set_autoscaley_on(False)
#     axes_.set_xlabel('Recall (Ground-Truth IOU >= %0.02f)' % (min_overlap, ))
#     axes_.set_ylabel('Precision')
#     axes_.set_xlim([0.0, 1.01])
#     axes_.set_ylim([0.0, 1.01])

#     kwargs_list = [
#         {
#             'min_overlap'            : min_overlap,
#             'classifier_sensitivity' : 0.64,
#             'localizer_grid'         : False,
#             'localizer_sensitivity'  : 0.16,
#             'labeler_sensitivity'    : 0.42,
#         },
#         {
#             'min_overlap'            : min_overlap,
#             'classifier_sensitivity' : 0.64,
#             'localizer_grid'         : False,
#             'localizer_sensitivity'  : 0.16,
#             'labeler_sensitivity'    : 0.42,
#             'check_species'          : True,
#         },
#         {
#             'min_overlap'            : min_overlap,
#             'classifier_sensitivity' : 0.64,
#             'localizer_grid'         : False,
#             'localizer_sensitivity'  : 0.16,
#             'labeler_sensitivity'    : 0.42,
#             'check_viewpoint'        : True,
#         },
#         {
#             'min_overlap'            : min_overlap,
#             'classifier_sensitivity' : 0.04,
#             'localizer_grid'         : True,
#             'localizer_sensitivity'  : 0.05,
#             'labeler_sensitivity'    : 0.39,
#         },
#         {
#             'min_overlap'            : min_overlap,
#             'classifier_sensitivity' : 0.04,
#             'localizer_grid'         : True,
#             'localizer_sensitivity'  : 0.05,
#             'labeler_sensitivity'    : 0.39,
#             'check_species'          : True,
#         },
#         {
#             'min_overlap'            : min_overlap,
#             'classifier_sensitivity' : 0.04,
#             'localizer_grid'         : True,
#             'localizer_sensitivity'  : 0.05,
#             'labeler_sensitivity'    : 0.39,
#             'check_viewpoint'        : True,
#         },
#     ]
#     label_list = [
#         'Opt L',
#         'Opt L+S',
#         'Opt L+S+V',
#         'Rec L',
#         'Rec L+S',
#         'Rec L+S+V',
#     ]
#     color_list = [
#         'r',
#         'b',
#         'g',
#         'k',
#         'y',
#         'c',
#     ]
#     ret_list = [
#         detector_precision_recall_algo_plot(ibs, label=label, color=color, **kwargs_)
#         for label, color, kwargs_ in zip(label_list, color_list, kwargs_list)
#     ]

#     area_list = [ ret[0] for ret in ret_list ]
#     conf_list = [ ret[1] for ret in ret_list ]
#     index = np.argmax(area_list)
#     best_label = label_list[index]
#     best_kwargs = kwargs_list[index]
#     best_area = area_list[index]
#     best_conf = conf_list[index]
#     plt.title('Precision-Recall Curve (Best: %s, AP = %0.02f)' % (best_label, best_area, ), y=1.20)
#     # Display graph
#     plt.legend(bbox_to_anchor=(0.0, 1.02, 1.0, .102), loc=3, ncol=2, mode="expand",
#                borderaxespad=0.0)

#     axes_ = plt.subplot(132)
#     axes_.set_aspect(1)
#     gca_ = plt.gca()
#     gca_.grid(False)
#     correct_rate, _ = detector_confusion_matrix_algo_plot(ibs, 'V1', 'r', conf=best_conf, fig_=fig_, axes_=axes_, **best_kwargs)
#     axes_.set_xlabel('Predicted (Correct = %0.02f%%)' % (correct_rate * 100.0, ))
#     axes_.set_ylabel('Ground-Truth')
#     plt.title('P-R Confusion Matrix (Algo: %s, OP = %0.02f)' % (best_label, best_conf, ), y=1.26)

#     best_index = None
#     best_conf = None
#     best_pr = 0.0
#     best_re = 0.0
#     tup_list  = [ ret[2] for ret in ret_list ]
#     for index, tup in enumerate(tup_list):
#         for conf, re, pr in zip(*tup):
#             if pr > best_pr:
#                 best_index = index
#                 best_conf = conf
#                 best_pr = pr
#                 best_re = re

#     if best_index is not None:
#         axes_ = plt.subplot(131)
#         plt.plot([best_re], [best_pr], 'yo')

#         best_label = label_list[best_index]
#         best_kwargs = kwargs_list[best_index]

#         axes_ = plt.subplot(133)
#         axes_.set_aspect(1)
#         gca_ = plt.gca()
#         gca_.grid(False)
#         correct_rate, _ = detector_confusion_matrix_algo_plot(ibs, 'V1', 'r', conf=best_conf, fig_=fig_, axes_=axes_, **best_kwargs)
#         axes_.set_xlabel('Predicted (Correct = %0.02f%%)' % (correct_rate * 100.0, ))
#         axes_.set_ylabel('Ground-Truth')
#         plt.title('P-R Confusion Matrix (Algo: %s, OP = %0.02f)' % (best_label, best_conf, ), y=1.26)

#     # plt.show()
#     fig_filename = 'detector-precision-recall-%0.2f.png' % (min_overlap, )
#     fig_path = abspath(expanduser(join('~', 'Desktop', fig_filename)))
#     plt.savefig(fig_path, bbox_inches='tight')


# @register_ibs_method
# def detector_metric_graphs(ibs, species_list=[]):
#     ibs.classifier_precision_recall_algo_display(species_list)
#     ibs.localizer_precision_recall_algo_display()
#     ibs.labeler_precision_recall_algo_display()
#     ibs.detector_precision_recall_algo_display()
