diff --git "a/scores/colpali-v1.3.json" "b/scores/colpali-v1.3.json" --- "a/scores/colpali-v1.3.json" +++ "b/scores/colpali-v1.3.json" @@ -1,1188 +1,1185 @@ { "metadata": { "model_name": "colpali-v1.3", - "report_generated_date": "2025-06-28T08:54:18.701533", - "model_size": 2.92, - "url": "https://huggingface.co/vidore/colpali-v1.3", - "data_source": "TIGER-Lab" + "report_generated_date": "2025-12-29T09:27:50.414129" }, "metrics": { "image": { "ImageNet-1K": { - "hit@1": 0.424, + "hit@1": 0.423, "hit@5": 0.63, - "hit@10": 0.695, - "ndcg_linear@1": 0.424, - "ndcg_linear@5": 0.5362278752072428, - "ndcg_linear@10": 0.5572292479510181, - "ndcg_exponential@1": 0.424, - "ndcg_exponential@5": 0.5362278752072428, - "ndcg_exponential@10": 0.5572292479510181, - "precision@1": 0.424, - "precision@5": 0.12599999999999997, - "precision@10": 0.0695, - "recall@1": 0.424, + "hit@10": 0.696, + "ndcg_linear@1": 0.423, + "ndcg_linear@5": 0.5352936104959322, + "ndcg_linear@10": 0.5565825422178999, + "ndcg_exponential@1": 0.423, + "ndcg_exponential@5": 0.5352936104959322, + "ndcg_exponential@10": 0.5565825422178999, + "precision@1": 0.423, + "precision@5": 0.126, + "precision@10": 0.0696, + "recall@1": 0.423, "recall@5": 0.63, - "recall@10": 0.695, - "f1@1": 0.424, + "recall@10": 0.696, + "f1@1": 0.423, "f1@5": 0.21000000000000005, - "f1@10": 0.1263636363636364, - "map@1": 0.424, - "map@5": 0.5047499999999999, - "map@10": 0.5134055555555556, - "mrr@1": 0.424, - "mrr@5": 0.5047499999999999, - "mrr@10": 0.5134055555555556, + "f1@10": 0.12654545454545457, + "map@1": 0.423, + "map@5": 0.5035666666666667, + "map@10": 0.512320634920635, + "mrr@1": 0.423, + "mrr@5": 0.5035666666666667, + "mrr@10": 0.512320634920635, "num_pred": 1000, "num_data": 1000 }, "N24News": { - "hit@1": 0.255, - "hit@5": 0.522, - "hit@10": 0.698, - "ndcg_linear@1": 0.255, - "ndcg_linear@5": 0.39237570348741835, - "ndcg_linear@10": 0.4489355518555218, - "ndcg_exponential@1": 0.255, - "ndcg_exponential@5": 0.39237570348741835, - "ndcg_exponential@10": 0.4489355518555218, - "precision@1": 0.255, - "precision@5": 0.10439999999999999, - "precision@10": 0.0698, - "recall@1": 0.255, - "recall@5": 0.522, - "recall@10": 0.698, - "f1@1": 0.255, - "f1@5": 0.17400000000000004, - "f1@10": 0.12690909090909094, - "map@1": 0.255, - "map@5": 0.34968333333333335, - "map@10": 0.3728150793650794, - "mrr@1": 0.255, - "mrr@5": 0.34968333333333335, - "mrr@10": 0.3728150793650794, + "hit@1": 0.256, + "hit@5": 0.521, + "hit@10": 0.699, + "ndcg_linear@1": 0.256, + "ndcg_linear@5": 0.3922347083033226, + "ndcg_linear@10": 0.4494795145102568, + "ndcg_exponential@1": 0.256, + "ndcg_exponential@5": 0.3922347083033226, + "ndcg_exponential@10": 0.4494795145102568, + "precision@1": 0.256, + "precision@5": 0.10419999999999999, + "precision@10": 0.0699, + "recall@1": 0.256, + "recall@5": 0.521, + "recall@10": 0.699, + "f1@1": 0.256, + "f1@5": 0.17366666666666675, + "f1@10": 0.12709090909090912, + "map@1": 0.256, + "map@5": 0.34981666666666666, + "map@10": 0.3732583333333333, + "mrr@1": 0.256, + "mrr@5": 0.34981666666666666, + "mrr@10": 0.3732583333333333, "num_pred": 1000, "num_data": 1000 }, "HatefulMemes": { - "hit@1": 0.506, + "hit@1": 0.509, "hit@5": 1.0, "hit@10": 1.0, - "ndcg_linear@1": 0.506, - "ndcg_linear@5": 0.8176792982643001, - "ndcg_linear@10": 0.8176792982643001, - "ndcg_exponential@1": 0.506, - "ndcg_exponential@5": 0.8176792982643001, - "ndcg_exponential@10": 0.8176792982643001, - "precision@1": 0.506, + "ndcg_linear@1": 0.509, + "ndcg_linear@5": 0.8187865090035858, + "ndcg_linear@10": 0.8187865090035858, + "ndcg_exponential@1": 0.509, + "ndcg_exponential@5": 0.8187865090035858, + "ndcg_exponential@10": 0.8187865090035858, + "precision@1": 0.509, "precision@5": 0.20000000000000004, "precision@10": 0.10000000000000002, - "recall@1": 0.506, + "recall@1": 0.509, "recall@5": 1.0, "recall@10": 1.0, - "f1@1": 0.506, + "f1@1": 0.509, "f1@5": 0.3333333333333333, "f1@10": 0.18181818181818182, - "map@1": 0.506, - "map@5": 0.753, - "map@10": 0.753, - "mrr@1": 0.506, - "mrr@5": 0.753, - "mrr@10": 0.753, + "map@1": 0.509, + "map@5": 0.7545, + "map@10": 0.7545, + "mrr@1": 0.509, + "mrr@5": 0.7545, + "mrr@10": 0.7545, "num_pred": 1000, "num_data": 1000 }, "VOC2007": { "hit@1": 0.698, - "hit@5": 0.896, - "hit@10": 0.964, + "hit@5": 0.897, + "hit@10": 0.961, "ndcg_linear@1": 0.698, - "ndcg_linear@5": 0.8066977050401656, - "ndcg_linear@10": 0.8286811826798018, + "ndcg_linear@5": 0.8072237462301984, + "ndcg_linear@10": 0.8280662973356593, "ndcg_exponential@1": 0.698, - "ndcg_exponential@5": 0.8066977050401656, - "ndcg_exponential@10": 0.8286811826798018, + "ndcg_exponential@5": 0.8072237462301984, + "ndcg_exponential@10": 0.8280662973356593, "precision@1": 0.698, - "precision@5": 0.1792, - "precision@10": 0.0964, + "precision@5": 0.1794, + "precision@10": 0.09610000000000002, "recall@1": 0.698, - "recall@5": 0.896, - "recall@10": 0.964, + "recall@5": 0.897, + "recall@10": 0.961, "f1@1": 0.698, - "f1@5": 0.2986666666666667, - "f1@10": 0.17527272727272725, + "f1@5": 0.299, + "f1@10": 0.1747272727272727, "map@1": 0.698, - "map@5": 0.7766666666666666, - "map@10": 0.7857361111111111, + "map@5": 0.7770166666666667, + "map@10": 0.7857007936507937, "mrr@1": 0.698, - "mrr@5": 0.7766666666666666, - "mrr@10": 0.7857361111111111, + "mrr@5": 0.7770166666666667, + "mrr@10": 0.7857007936507937, "num_pred": 1000, "num_data": 1000 }, "SUN397": { - "hit@1": 0.561, - "hit@5": 0.776, - "hit@10": 0.845, - "ndcg_linear@1": 0.561, - "ndcg_linear@5": 0.6766637891896082, - "ndcg_linear@10": 0.6988908085166319, - "ndcg_exponential@1": 0.561, - "ndcg_exponential@5": 0.6766637891896082, - "ndcg_exponential@10": 0.6988908085166319, - "precision@1": 0.561, - "precision@5": 0.15519999999999998, - "precision@10": 0.0845, - "recall@1": 0.561, - "recall@5": 0.776, - "recall@10": 0.845, - "f1@1": 0.561, - "f1@5": 0.2586666666666667, - "f1@10": 0.15363636363636363, - "map@1": 0.561, - "map@5": 0.6435333333333333, - "map@10": 0.6526531746031746, - "mrr@1": 0.561, - "mrr@5": 0.6435333333333333, - "mrr@10": 0.6526531746031746, + "hit@1": 0.562, + "hit@5": 0.777, + "hit@10": 0.847, + "ndcg_linear@1": 0.562, + "ndcg_linear@5": 0.677817870334025, + "ndcg_linear@10": 0.7003228872079948, + "ndcg_exponential@1": 0.562, + "ndcg_exponential@5": 0.677817870334025, + "ndcg_exponential@10": 0.7003228872079948, + "precision@1": 0.562, + "precision@5": 0.15539999999999998, + "precision@10": 0.0847, + "recall@1": 0.562, + "recall@5": 0.777, + "recall@10": 0.847, + "f1@1": 0.562, + "f1@5": 0.259, + "f1@10": 0.154, + "map@1": 0.562, + "map@5": 0.6447, + "map@10": 0.6539095238095237, + "mrr@1": 0.562, + "mrr@5": 0.6447, + "mrr@10": 0.6539095238095237, "num_pred": 1000, "num_data": 1000 }, "Place365": { - "hit@1": 0.275, + "hit@1": 0.277, "hit@5": 0.508, - "hit@10": 0.59, - "ndcg_linear@1": 0.275, - "ndcg_linear@5": 0.3968289651631725, - "ndcg_linear@10": 0.4233454986797219, - "ndcg_exponential@1": 0.275, - "ndcg_exponential@5": 0.3968289651631725, - "ndcg_exponential@10": 0.4233454986797219, - "precision@1": 0.275, + "hit@10": 0.589, + "ndcg_linear@1": 0.277, + "ndcg_linear@5": 0.39711451688496774, + "ndcg_linear@10": 0.42327378672949156, + "ndcg_exponential@1": 0.277, + "ndcg_exponential@5": 0.39711451688496774, + "ndcg_exponential@10": 0.42327378672949156, + "precision@1": 0.277, "precision@5": 0.1016, - "precision@10": 0.05899999999999999, - "recall@1": 0.275, + "precision@10": 0.058899999999999994, + "recall@1": 0.277, "recall@5": 0.508, - "recall@10": 0.59, - "f1@1": 0.275, + "recall@10": 0.589, + "f1@1": 0.277, "f1@5": 0.16933333333333336, - "f1@10": 0.1072727272727273, - "map@1": 0.275, - "map@5": 0.3599666666666667, - "map@10": 0.37090396825396826, - "mrr@1": 0.275, - "mrr@5": 0.3599666666666667, - "mrr@10": 0.37090396825396826, + "f1@10": 0.10709090909090913, + "map@1": 0.277, + "map@5": 0.36043333333333333, + "map@10": 0.3712055555555555, + "mrr@1": 0.277, + "mrr@5": 0.36043333333333333, + "mrr@10": 0.3712055555555555, "num_pred": 1000, "num_data": 1000 }, "ImageNet-A": { - "hit@1": 0.149, - "hit@5": 0.386, - "hit@10": 0.511, - "ndcg_linear@1": 0.149, - "ndcg_linear@5": 0.27296061872015115, - "ndcg_linear@10": 0.31360674858954923, - "ndcg_exponential@1": 0.149, - "ndcg_exponential@5": 0.27296061872015115, - "ndcg_exponential@10": 0.31360674858954923, - "precision@1": 0.149, - "precision@5": 0.0772, - "precision@10": 0.05109999999999999, - "recall@1": 0.149, - "recall@5": 0.386, - "recall@10": 0.511, - "f1@1": 0.149, - "f1@5": 0.12866666666666668, - "f1@10": 0.09290909090909094, - "map@1": 0.149, - "map@5": 0.23555, - "map@10": 0.25245833333333334, - "mrr@1": 0.149, - "mrr@5": 0.23555, - "mrr@10": 0.25245833333333334, + "hit@1": 0.152, + "hit@5": 0.387, + "hit@10": 0.513, + "ndcg_linear@1": 0.152, + "ndcg_linear@5": 0.2750227665307411, + "ndcg_linear@10": 0.31592709692679594, + "ndcg_exponential@1": 0.152, + "ndcg_exponential@5": 0.2750227665307411, + "ndcg_exponential@10": 0.31592709692679594, + "precision@1": 0.152, + "precision@5": 0.07740000000000001, + "precision@10": 0.0513, + "recall@1": 0.152, + "recall@5": 0.387, + "recall@10": 0.513, + "f1@1": 0.152, + "f1@5": 0.12900000000000003, + "f1@10": 0.09327272727272728, + "map@1": 0.152, + "map@5": 0.23795, + "map@10": 0.2549222222222222, + "mrr@1": 0.152, + "mrr@5": 0.23795, + "mrr@10": 0.2549222222222222, "num_pred": 1000, "num_data": 1000 }, "ImageNet-R": { - "hit@1": 0.646, - "hit@5": 0.885, - "hit@10": 0.949, - "ndcg_linear@1": 0.646, - "ndcg_linear@5": 0.7805740835658224, - "ndcg_linear@10": 0.8012652867618623, - "ndcg_exponential@1": 0.646, - "ndcg_exponential@5": 0.7805740835658224, - "ndcg_exponential@10": 0.8012652867618623, - "precision@1": 0.646, - "precision@5": 0.177, - "precision@10": 0.09490000000000003, - "recall@1": 0.646, - "recall@5": 0.885, - "recall@10": 0.949, - "f1@1": 0.646, - "f1@5": 0.295, - "f1@10": 0.17254545454545453, - "map@1": 0.646, - "map@5": 0.74515, - "map@10": 0.7536865079365079, - "mrr@1": 0.646, - "mrr@5": 0.74515, - "mrr@10": 0.7536865079365079, + "hit@1": 0.652, + "hit@5": 0.882, + "hit@10": 0.945, + "ndcg_linear@1": 0.652, + "ndcg_linear@5": 0.7815841228718511, + "ndcg_linear@10": 0.8022289459711497, + "ndcg_exponential@1": 0.652, + "ndcg_exponential@5": 0.7815841228718511, + "ndcg_exponential@10": 0.8022289459711497, + "precision@1": 0.652, + "precision@5": 0.1764, + "precision@10": 0.09450000000000001, + "recall@1": 0.652, + "recall@5": 0.882, + "recall@10": 0.945, + "f1@1": 0.652, + "f1@5": 0.294, + "f1@10": 0.17181818181818181, + "map@1": 0.652, + "map@5": 0.7475, + "map@10": 0.756181746031746, + "mrr@1": 0.652, + "mrr@5": 0.7475, + "mrr@10": 0.756181746031746, "num_pred": 1000, "num_data": 1000 }, "ObjectNet": { "hit@1": 0.456, - "hit@5": 0.684, - "hit@10": 0.761, + "hit@5": 0.685, + "hit@10": 0.762, "ndcg_linear@1": 0.456, - "ndcg_linear@5": 0.5770939235735987, - "ndcg_linear@10": 0.6022221460924772, + "ndcg_linear@5": 0.5778818242707744, + "ndcg_linear@10": 0.6029722227178727, "ndcg_exponential@1": 0.456, - "ndcg_exponential@5": 0.5770939235735987, - "ndcg_exponential@10": 0.6022221460924772, + "ndcg_exponential@5": 0.5778818242707744, + "ndcg_exponential@10": 0.6029722227178727, "precision@1": 0.456, - "precision@5": 0.13679999999999998, - "precision@10": 0.0761, + "precision@5": 0.137, + "precision@10": 0.0762, "recall@1": 0.456, - "recall@5": 0.684, - "recall@10": 0.761, + "recall@5": 0.685, + "recall@10": 0.762, "f1@1": 0.456, - "f1@5": 0.22800000000000006, - "f1@10": 0.13836363636363638, + "f1@5": 0.22833333333333336, + "f1@10": 0.13854545454545455, "map@1": 0.456, - "map@5": 0.5416166666666666, - "map@10": 0.5521202380952381, + "map@5": 0.5423, + "map@10": 0.5527623015873017, "mrr@1": 0.456, - "mrr@5": 0.5416166666666666, - "mrr@10": 0.5521202380952381, + "mrr@5": 0.5423, + "mrr@10": 0.5527623015873017, "num_pred": 1000, "num_data": 1000 }, "Country211": { "hit@1": 0.06, - "hit@5": 0.184, + "hit@5": 0.183, "hit@10": 0.277, "ndcg_linear@1": 0.06, - "ndcg_linear@5": 0.1201143319880854, - "ndcg_linear@10": 0.14945291808444336, + "ndcg_linear@5": 0.11990940831659781, + "ndcg_linear@10": 0.14962065682975323, "ndcg_exponential@1": 0.06, - "ndcg_exponential@5": 0.1201143319880854, - "ndcg_exponential@10": 0.14945291808444336, + "ndcg_exponential@5": 0.11990940831659781, + "ndcg_exponential@10": 0.14962065682975323, "precision@1": 0.06, - "precision@5": 0.0368, - "precision@10": 0.027700000000000002, + "precision@5": 0.036599999999999994, + "precision@10": 0.0277, "recall@1": 0.06, - "recall@5": 0.184, + "recall@5": 0.183, "recall@10": 0.277, "f1@1": 0.06, - "f1@5": 0.061333333333333344, - "f1@10": 0.05036363636363637, + "f1@5": 0.061000000000000006, + "f1@10": 0.050363636363636374, "map@1": 0.06, - "map@5": 0.09943333333333333, - "map@10": 0.11110952380952381, + "map@5": 0.09946666666666668, + "map@10": 0.11132301587301587, "mrr@1": 0.06, - "mrr@5": 0.09943333333333333, - "mrr@10": 0.11110952380952381, + "mrr@5": 0.09946666666666668, + "mrr@10": 0.11132301587301587, "num_pred": 1000, "num_data": 1000 }, "OK-VQA": { "hit@1": 0.094, - "hit@5": 0.213, + "hit@5": 0.21, "hit@10": 0.287, "ndcg_linear@1": 0.094, - "ndcg_linear@5": 0.15781428161378155, - "ndcg_linear@10": 0.18190804766263002, + "ndcg_linear@5": 0.1562016759199612, + "ndcg_linear@10": 0.18142777231549798, "ndcg_exponential@1": 0.094, - "ndcg_exponential@5": 0.15781428161378155, - "ndcg_exponential@10": 0.18190804766263002, + "ndcg_exponential@5": 0.1562016759199612, + "ndcg_exponential@10": 0.18142777231549798, "precision@1": 0.094, - "precision@5": 0.0426, + "precision@5": 0.042, "precision@10": 0.028700000000000003, "recall@1": 0.094, - "recall@5": 0.213, + "recall@5": 0.21, "recall@10": 0.287, "f1@1": 0.094, - "f1@5": 0.07100000000000001, - "f1@10": 0.052181818181818176, + "f1@5": 0.07, + "f1@10": 0.05218181818181819, "map@1": 0.094, - "map@5": 0.1394166666666667, - "map@10": 0.14945396825396823, + "map@5": 0.13826666666666665, + "map@10": 0.14886666666666667, "mrr@1": 0.094, - "mrr@5": 0.1394166666666667, - "mrr@10": 0.14945396825396823, + "mrr@5": 0.13826666666666665, + "mrr@10": 0.14886666666666667, "num_pred": 1000, "num_data": 1000 }, "A-OKVQA": { - "hit@1": 0.066, + "hit@1": 0.064, "hit@5": 0.179, - "hit@10": 0.246, - "ndcg_linear@1": 0.066, - "ndcg_linear@5": 0.12531105197988915, - "ndcg_linear@10": 0.1467885463447978, - "ndcg_exponential@1": 0.066, - "ndcg_exponential@5": 0.12531105197988915, - "ndcg_exponential@10": 0.1467885463447978, - "precision@1": 0.066, + "hit@10": 0.248, + "ndcg_linear@1": 0.064, + "ndcg_linear@5": 0.12444198173346062, + "ndcg_linear@10": 0.14655233357924113, + "ndcg_exponential@1": 0.064, + "ndcg_exponential@5": 0.12444198173346062, + "ndcg_exponential@10": 0.14655233357924113, + "precision@1": 0.064, "precision@5": 0.0358, - "precision@10": 0.0246, - "recall@1": 0.066, + "precision@10": 0.024800000000000003, + "recall@1": 0.064, "recall@5": 0.179, - "recall@10": 0.246, - "f1@1": 0.066, + "recall@10": 0.248, + "f1@1": 0.064, "f1@5": 0.05966666666666667, - "f1@10": 0.04472727272727273, - "map@1": 0.066, - "map@5": 0.10751666666666666, - "map@10": 0.11627380952380952, - "mrr@1": 0.066, - "mrr@5": 0.10751666666666666, - "mrr@10": 0.11627380952380952, + "f1@10": 0.04509090909090909, + "map@1": 0.064, + "map@5": 0.10635, + "map@10": 0.1153595238095238, + "mrr@1": 0.064, + "mrr@5": 0.10635, + "mrr@10": 0.1153595238095238, "num_pred": 1000, "num_data": 1000 }, "DocVQA": { "hit@1": 0.113, "hit@5": 0.218, - "hit@10": 0.275, + "hit@10": 0.28, "ndcg_linear@1": 0.113, - "ndcg_linear@5": 0.16693960663358462, - "ndcg_linear@10": 0.18543782310001622, + "ndcg_linear@5": 0.16701899550603547, + "ndcg_linear@10": 0.18711468928218436, "ndcg_exponential@1": 0.113, - "ndcg_exponential@5": 0.16693960663358462, - "ndcg_exponential@10": 0.18543782310001622, + "ndcg_exponential@5": 0.16701899550603547, + "ndcg_exponential@10": 0.18711468928218436, "precision@1": 0.113, - "precision@5": 0.0436, - "precision@10": 0.0275, + "precision@5": 0.04359999999999999, + "precision@10": 0.028, "recall@1": 0.113, "recall@5": 0.218, - "recall@10": 0.275, + "recall@10": 0.28, "f1@1": 0.113, - "f1@5": 0.07266666666666667, - "f1@10": 0.05, + "f1@5": 0.07266666666666668, + "f1@10": 0.050909090909090904, "map@1": 0.113, - "map@5": 0.15011666666666665, - "map@10": 0.15779087301587302, + "map@5": 0.15023333333333336, + "map@10": 0.15855873015873015, "mrr@1": 0.113, - "mrr@5": 0.15011666666666665, - "mrr@10": 0.15779087301587302, + "mrr@5": 0.15023333333333336, + "mrr@10": 0.15855873015873015, "num_pred": 1000, "num_data": 1000 }, "InfographicsVQA": { "hit@1": 0.05, - "hit@5": 0.132, - "hit@10": 0.185, + "hit@5": 0.133, + "hit@10": 0.186, "ndcg_linear@1": 0.05, - "ndcg_linear@5": 0.09377615908783146, - "ndcg_linear@10": 0.11104853082401721, + "ndcg_linear@5": 0.09415529476478424, + "ndcg_linear@10": 0.1113748664000343, "ndcg_exponential@1": 0.05, - "ndcg_exponential@5": 0.09377615908783146, - "ndcg_exponential@10": 0.11104853082401721, + "ndcg_exponential@5": 0.09415529476478424, + "ndcg_exponential@10": 0.1113748664000343, "precision@1": 0.05, - "precision@5": 0.026400000000000007, - "precision@10": 0.0185, + "precision@5": 0.026600000000000006, + "precision@10": 0.018600000000000002, "recall@1": 0.05, - "recall@5": 0.132, - "recall@10": 0.185, + "recall@5": 0.133, + "recall@10": 0.186, "f1@1": 0.05, - "f1@5": 0.044, - "f1@10": 0.03363636363636364, + "f1@5": 0.04433333333333334, + "f1@10": 0.03381818181818182, "map@1": 0.05, - "map@5": 0.08101666666666667, - "map@10": 0.08821825396825397, + "map@5": 0.08121666666666667, + "map@10": 0.08836825396825398, "mrr@1": 0.05, - "mrr@5": 0.08101666666666667, - "mrr@10": 0.08821825396825397, + "mrr@5": 0.08121666666666667, + "mrr@10": 0.08836825396825398, "num_pred": 1000, "num_data": 1000 }, "ChartQA": { "hit@1": 0.057, "hit@5": 0.103, - "hit@10": 0.132, + "hit@10": 0.131, "ndcg_linear@1": 0.057, - "ndcg_linear@5": 0.08038385840193323, - "ndcg_linear@10": 0.08987025554866532, + "ndcg_linear@5": 0.08120319524367657, + "ndcg_linear@10": 0.09024484818926976, "ndcg_exponential@1": 0.057, - "ndcg_exponential@5": 0.08038385840193323, - "ndcg_exponential@10": 0.08987025554866532, + "ndcg_exponential@5": 0.08120319524367657, + "ndcg_exponential@10": 0.09024484818926976, "precision@1": 0.057, "precision@5": 0.0206, - "precision@10": 0.0132, + "precision@10": 0.013099999999999999, "recall@1": 0.057, "recall@5": 0.103, - "recall@10": 0.132, + "recall@10": 0.131, "f1@1": 0.057, - "f1@5": 0.03433333333333334, - "f1@10": 0.024, + "f1@5": 0.034333333333333334, + "f1@10": 0.02381818181818182, "map@1": 0.057, - "map@5": 0.07295, - "map@10": 0.07692777777777778, + "map@5": 0.07396666666666667, + "map@10": 0.07769206349206348, "mrr@1": 0.057, - "mrr@5": 0.07295, - "mrr@10": 0.07692777777777778, + "mrr@5": 0.07396666666666667, + "mrr@10": 0.07769206349206348, "num_pred": 1000, "num_data": 1000 }, "Visual7W": { "hit@1": 0.061, - "hit@5": 0.194, - "hit@10": 0.278, + "hit@5": 0.193, + "hit@10": 0.276, "ndcg_linear@1": 0.061, - "ndcg_linear@5": 0.1279247131968282, - "ndcg_linear@10": 0.15503365326266325, + "ndcg_linear@5": 0.12703138243716874, + "ndcg_linear@10": 0.15384034899225726, "ndcg_exponential@1": 0.061, - "ndcg_exponential@5": 0.1279247131968282, - "ndcg_exponential@10": 0.15503365326266325, + "ndcg_exponential@5": 0.12703138243716874, + "ndcg_exponential@10": 0.15384034899225726, "precision@1": 0.061, - "precision@5": 0.038799999999999994, - "precision@10": 0.027800000000000002, + "precision@5": 0.0386, + "precision@10": 0.027600000000000003, "recall@1": 0.061, - "recall@5": 0.194, - "recall@10": 0.278, + "recall@5": 0.193, + "recall@10": 0.276, "f1@1": 0.061, - "f1@5": 0.06466666666666669, - "f1@10": 0.050545454545454546, + "f1@5": 0.06433333333333334, + "f1@10": 0.05018181818181819, "map@1": 0.061, - "map@5": 0.10625, - "map@10": 0.11740198412698413, + "map@5": 0.10543333333333334, + "map@10": 0.11647261904761905, "mrr@1": 0.061, - "mrr@5": 0.10625, - "mrr@10": 0.11740198412698413, + "mrr@5": 0.10543333333333334, + "mrr@10": 0.11647261904761905, "num_pred": 1000, "num_data": 1000 }, "ScienceQA": { - "hit@1": 0.163, - "hit@5": 0.311, - "hit@10": 0.412, - "ndcg_linear@1": 0.163, - "ndcg_linear@5": 0.2405291138606887, - "ndcg_linear@10": 0.2730768668127606, - "ndcg_exponential@1": 0.163, - "ndcg_exponential@5": 0.2405291138606887, - "ndcg_exponential@10": 0.2730768668127606, - "precision@1": 0.163, - "precision@5": 0.0622, - "precision@10": 0.0412, - "recall@1": 0.163, - "recall@5": 0.311, - "recall@10": 0.412, - "f1@1": 0.163, - "f1@5": 0.10366666666666668, - "f1@10": 0.07490909090909093, - "map@1": 0.163, - "map@5": 0.21715, - "map@10": 0.23050873015873014, - "mrr@1": 0.163, - "mrr@5": 0.21715, - "mrr@10": 0.23050873015873014, + "hit@1": 0.165, + "hit@5": 0.317, + "hit@10": 0.411, + "ndcg_linear@1": 0.165, + "ndcg_linear@5": 0.24345689994443648, + "ndcg_linear@10": 0.2735080991056756, + "ndcg_exponential@1": 0.165, + "ndcg_exponential@5": 0.24345689994443648, + "ndcg_exponential@10": 0.2735080991056756, + "precision@1": 0.165, + "precision@5": 0.0634, + "precision@10": 0.0411, + "recall@1": 0.165, + "recall@5": 0.317, + "recall@10": 0.411, + "f1@1": 0.165, + "f1@5": 0.10566666666666669, + "f1@10": 0.07472727272727274, + "map@1": 0.165, + "map@5": 0.21919999999999998, + "map@10": 0.2313972222222222, + "mrr@1": 0.165, + "mrr@5": 0.21919999999999998, + "mrr@10": 0.2313972222222222, "num_pred": 1000, "num_data": 1000 }, "VizWiz": { - "hit@1": 0.276, - "hit@5": 0.427, - "hit@10": 0.481, - "ndcg_linear@1": 0.276, - "ndcg_linear@5": 0.35678991716957487, - "ndcg_linear@10": 0.3745417135350773, - "ndcg_exponential@1": 0.276, - "ndcg_exponential@5": 0.35678991716957487, - "ndcg_exponential@10": 0.3745417135350773, - "precision@1": 0.276, - "precision@5": 0.0854, - "precision@10": 0.0481, - "recall@1": 0.276, - "recall@5": 0.427, - "recall@10": 0.481, - "f1@1": 0.276, - "f1@5": 0.14233333333333337, - "f1@10": 0.08745454545454547, - "map@1": 0.276, - "map@5": 0.3333833333333333, - "map@10": 0.34087460317460316, - "mrr@1": 0.276, - "mrr@5": 0.3333833333333333, - "mrr@10": 0.34087460317460316, + "hit@1": 0.274, + "hit@5": 0.428, + "hit@10": 0.482, + "ndcg_linear@1": 0.274, + "ndcg_linear@5": 0.35651801835640323, + "ndcg_linear@10": 0.3742578495525595, + "ndcg_exponential@1": 0.274, + "ndcg_exponential@5": 0.35651801835640323, + "ndcg_exponential@10": 0.3742578495525595, + "precision@1": 0.274, + "precision@5": 0.0856, + "precision@10": 0.04819999999999999, + "recall@1": 0.274, + "recall@5": 0.428, + "recall@10": 0.482, + "f1@1": 0.274, + "f1@5": 0.14266666666666672, + "f1@10": 0.08763636363636365, + "map@1": 0.274, + "map@5": 0.3327, + "map@10": 0.3401801587301587, + "mrr@1": 0.274, + "mrr@5": 0.3327, + "mrr@10": 0.3401801587301587, "num_pred": 1000, "num_data": 1000 }, "GQA": { - "hit@1": 0.083, - "hit@5": 0.217, - "hit@10": 0.322, - "ndcg_linear@1": 0.083, - "ndcg_linear@5": 0.14908635394128397, - "ndcg_linear@10": 0.18236063680906084, - "ndcg_exponential@1": 0.083, - "ndcg_exponential@5": 0.14908635394128397, - "ndcg_exponential@10": 0.18236063680906084, - "precision@1": 0.083, - "precision@5": 0.04340000000000001, - "precision@10": 0.03219999999999999, - "recall@1": 0.083, - "recall@5": 0.217, - "recall@10": 0.322, - "f1@1": 0.083, - "f1@5": 0.07233333333333335, - "f1@10": 0.058545454545454546, - "map@1": 0.083, - "map@5": 0.12696666666666667, - "map@10": 0.1403059523809524, - "mrr@1": 0.083, - "mrr@5": 0.12696666666666667, - "mrr@10": 0.1403059523809524, + "hit@1": 0.081, + "hit@5": 0.218, + "hit@10": 0.321, + "ndcg_linear@1": 0.081, + "ndcg_linear@5": 0.14867291844507147, + "ndcg_linear@10": 0.18134257898847744, + "ndcg_exponential@1": 0.081, + "ndcg_exponential@5": 0.14867291844507147, + "ndcg_exponential@10": 0.18134257898847744, + "precision@1": 0.081, + "precision@5": 0.0436, + "precision@10": 0.032100000000000004, + "recall@1": 0.081, + "recall@5": 0.218, + "recall@10": 0.321, + "f1@1": 0.081, + "f1@5": 0.07266666666666668, + "f1@10": 0.058363636363636374, + "map@1": 0.081, + "map@5": 0.1261, + "map@10": 0.13921706349206348, + "mrr@1": 0.081, + "mrr@5": 0.1261, + "mrr@10": 0.13921706349206348, "num_pred": 1000, "num_data": 1000 }, "TextVQA": { - "hit@1": 0.188, - "hit@5": 0.321, - "hit@10": 0.373, - "ndcg_linear@1": 0.188, - "ndcg_linear@5": 0.259901257561043, - "ndcg_linear@10": 0.2768789912780585, - "ndcg_exponential@1": 0.188, - "ndcg_exponential@5": 0.259901257561043, - "ndcg_exponential@10": 0.2768789912780585, - "precision@1": 0.188, - "precision@5": 0.0642, - "precision@10": 0.0373, - "recall@1": 0.188, - "recall@5": 0.321, - "recall@10": 0.373, - "f1@1": 0.188, - "f1@5": 0.10700000000000001, - "f1@10": 0.06781818181818182, - "map@1": 0.188, - "map@5": 0.23946666666666663, - "map@10": 0.2465706349206349, - "mrr@1": 0.188, - "mrr@5": 0.23946666666666663, - "mrr@10": 0.2465706349206349, + "hit@1": 0.189, + "hit@5": 0.324, + "hit@10": 0.375, + "ndcg_linear@1": 0.189, + "ndcg_linear@5": 0.2614314277281202, + "ndcg_linear@10": 0.27778059449750064, + "ndcg_exponential@1": 0.189, + "ndcg_exponential@5": 0.2614314277281202, + "ndcg_exponential@10": 0.27778059449750064, + "precision@1": 0.189, + "precision@5": 0.0648, + "precision@10": 0.0375, + "recall@1": 0.189, + "recall@5": 0.324, + "recall@10": 0.375, + "f1@1": 0.189, + "f1@5": 0.10800000000000003, + "f1@10": 0.06818181818181819, + "map@1": 0.189, + "map@5": 0.24055, + "map@10": 0.24721071428571428, + "mrr@1": 0.189, + "mrr@5": 0.24055, + "mrr@10": 0.24721071428571428, "num_pred": 1000, "num_data": 1000 }, "VisDial": { - "hit@1": 0.412, - "hit@5": 0.672, - "hit@10": 0.779, - "ndcg_linear@1": 0.412, - "ndcg_linear@5": 0.5492983318706365, - "ndcg_linear@10": 0.5836800643308898, - "ndcg_exponential@1": 0.412, - "ndcg_exponential@5": 0.5492983318706365, - "ndcg_exponential@10": 0.5836800643308898, - "precision@1": 0.412, - "precision@5": 0.13439999999999996, - "precision@10": 0.0779, - "recall@1": 0.412, - "recall@5": 0.672, - "recall@10": 0.779, - "f1@1": 0.412, - "f1@5": 0.22400000000000006, - "f1@10": 0.14163636363636362, - "map@1": 0.412, - "map@5": 0.5086166666666666, - "map@10": 0.522688492063492, - "mrr@1": 0.412, - "mrr@5": 0.5086166666666666, - "mrr@10": 0.522688492063492, + "hit@1": 0.411, + "hit@5": 0.676, + "hit@10": 0.78, + "ndcg_linear@1": 0.411, + "ndcg_linear@5": 0.5507396153581792, + "ndcg_linear@10": 0.5840587221655774, + "ndcg_exponential@1": 0.411, + "ndcg_exponential@5": 0.5507396153581792, + "ndcg_exponential@10": 0.5840587221655774, + "precision@1": 0.411, + "precision@5": 0.1352, + "precision@10": 0.078, + "recall@1": 0.411, + "recall@5": 0.676, + "recall@10": 0.78, + "f1@1": 0.411, + "f1@5": 0.2253333333333334, + "f1@10": 0.14181818181818184, + "map@1": 0.411, + "map@5": 0.5092166666666667, + "map@10": 0.5227924603174603, + "mrr@1": 0.411, + "mrr@5": 0.5092166666666667, + "mrr@10": 0.5227924603174603, "num_pred": 1000, "num_data": 1000 }, "CIRR": { "hit@1": 0.082, - "hit@5": 0.388, - "hit@10": 0.509, + "hit@5": 0.389, + "hit@10": 0.511, "ndcg_linear@1": 0.082, - "ndcg_linear@5": 0.24313243074338908, - "ndcg_linear@10": 0.2821629226981484, + "ndcg_linear@5": 0.2433262059864622, + "ndcg_linear@10": 0.2825983248057715, "ndcg_exponential@1": 0.082, - "ndcg_exponential@5": 0.24313243074338908, - "ndcg_exponential@10": 0.2821629226981484, + "ndcg_exponential@5": 0.2433262059864622, + "ndcg_exponential@10": 0.2825983248057715, "precision@1": 0.082, - "precision@5": 0.07759999999999999, - "precision@10": 0.050899999999999994, + "precision@5": 0.0778, + "precision@10": 0.05109999999999999, "recall@1": 0.082, - "recall@5": 0.388, - "recall@10": 0.509, + "recall@5": 0.389, + "recall@10": 0.511, "f1@1": 0.082, - "f1@5": 0.12933333333333338, - "f1@10": 0.09254545454545456, + "f1@5": 0.12966666666666668, + "f1@10": 0.09290909090909094, "map@1": 0.082, - "map@5": 0.19506666666666667, - "map@10": 0.21111547619047621, + "map@5": 0.19503333333333336, + "map@10": 0.21114404761904762, "mrr@1": 0.082, - "mrr@5": 0.19506666666666667, - "mrr@10": 0.21111547619047621, + "mrr@5": 0.19503333333333336, + "mrr@10": 0.21114404761904762, "num_pred": 1000, "num_data": 1000 }, "VisualNews_t2i": { - "hit@1": 0.501, - "hit@5": 0.682, - "hit@10": 0.756, - "ndcg_linear@1": 0.501, - "ndcg_linear@5": 0.5982691408786056, - "ndcg_linear@10": 0.6224873796120489, - "ndcg_exponential@1": 0.501, - "ndcg_exponential@5": 0.5982691408786056, - "ndcg_exponential@10": 0.6224873796120489, - "precision@1": 0.501, - "precision@5": 0.13639999999999997, - "precision@10": 0.0756, - "recall@1": 0.501, - "recall@5": 0.682, - "recall@10": 0.756, - "f1@1": 0.501, - "f1@5": 0.22733333333333336, - "f1@10": 0.1374545454545455, - "map@1": 0.501, - "map@5": 0.5702833333333333, - "map@10": 0.5804488095238095, - "mrr@1": 0.501, - "mrr@5": 0.5702833333333333, - "mrr@10": 0.5804488095238095, + "hit@1": 0.505, + "hit@5": 0.687, + "hit@10": 0.755, + "ndcg_linear@1": 0.505, + "ndcg_linear@5": 0.6005801654918551, + "ndcg_linear@10": 0.6225876660914509, + "ndcg_exponential@1": 0.505, + "ndcg_exponential@5": 0.6005801654918551, + "ndcg_exponential@10": 0.6225876660914509, + "precision@1": 0.505, + "precision@5": 0.13739999999999997, + "precision@10": 0.0755, + "recall@1": 0.505, + "recall@5": 0.687, + "recall@10": 0.755, + "f1@1": 0.505, + "f1@5": 0.22900000000000006, + "f1@10": 0.1372727272727273, + "map@1": 0.505, + "map@5": 0.5719166666666666, + "map@10": 0.5810055555555556, + "mrr@1": 0.505, + "mrr@5": 0.5719166666666666, + "mrr@10": 0.5810055555555556, "num_pred": 1000, "num_data": 1000 }, "VisualNews_i2t": { - "hit@1": 0.476, - "hit@5": 0.643, - "hit@10": 0.7, - "ndcg_linear@1": 0.476, - "ndcg_linear@5": 0.5668805073945691, - "ndcg_linear@10": 0.5857674998865605, - "ndcg_exponential@1": 0.476, - "ndcg_exponential@5": 0.5668805073945691, - "ndcg_exponential@10": 0.5857674998865605, - "precision@1": 0.476, - "precision@5": 0.1286, - "precision@10": 0.07, - "recall@1": 0.476, - "recall@5": 0.643, - "recall@10": 0.7, - "f1@1": 0.476, - "f1@5": 0.2143333333333334, - "f1@10": 0.1272727272727273, - "map@1": 0.476, - "map@5": 0.5413833333333332, - "map@10": 0.5494496031746032, - "mrr@1": 0.476, - "mrr@5": 0.5413833333333332, - "mrr@10": 0.5494496031746032, + "hit@1": 0.477, + "hit@5": 0.644, + "hit@10": 0.699, + "ndcg_linear@1": 0.477, + "ndcg_linear@5": 0.5678005770231732, + "ndcg_linear@10": 0.5858450045742124, + "ndcg_exponential@1": 0.477, + "ndcg_exponential@5": 0.5678005770231732, + "ndcg_exponential@10": 0.5858450045742124, + "precision@1": 0.477, + "precision@5": 0.1288, + "precision@10": 0.0699, + "recall@1": 0.477, + "recall@5": 0.644, + "recall@10": 0.699, + "f1@1": 0.477, + "f1@5": 0.21466666666666676, + "f1@10": 0.1270909090909091, + "map@1": 0.477, + "map@5": 0.5422833333333333, + "map@10": 0.5498849206349206, + "mrr@1": 0.477, + "mrr@5": 0.5422833333333333, + "mrr@10": 0.5498849206349206, "num_pred": 1000, "num_data": 1000 }, "MSCOCO_t2i": { - "hit@1": 0.592, + "hit@1": 0.586, "hit@5": 0.852, - "hit@10": 0.916, - "ndcg_linear@1": 0.592, - "ndcg_linear@5": 0.7326893679293205, - "ndcg_linear@10": 0.7533319711529448, - "ndcg_exponential@1": 0.592, - "ndcg_exponential@5": 0.7326893679293205, - "ndcg_exponential@10": 0.7533319711529448, - "precision@1": 0.592, + "hit@10": 0.917, + "ndcg_linear@1": 0.586, + "ndcg_linear@5": 0.7304749464507493, + "ndcg_linear@10": 0.7515792910896772, + "ndcg_exponential@1": 0.586, + "ndcg_exponential@5": 0.7304749464507493, + "ndcg_exponential@10": 0.7515792910896772, + "precision@1": 0.586, "precision@5": 0.1704, - "precision@10": 0.09160000000000003, - "recall@1": 0.592, + "precision@10": 0.09170000000000002, + "recall@1": 0.586, "recall@5": 0.852, - "recall@10": 0.916, - "f1@1": 0.592, + "recall@10": 0.917, + "f1@1": 0.586, "f1@5": 0.284, - "f1@10": 0.16654545454545452, - "map@1": 0.592, - "map@5": 0.6928, - "map@10": 0.7012888888888887, - "mrr@1": 0.592, - "mrr@5": 0.6928, - "mrr@10": 0.7012888888888887, + "f1@10": 0.16672727272727272, + "map@1": 0.586, + "map@5": 0.6898, + "map@10": 0.6985551587301587, + "mrr@1": 0.586, + "mrr@5": 0.6898, + "mrr@10": 0.6985551587301587, "num_pred": 1000, "num_data": 1000 }, "MSCOCO_i2t": { - "hit@1": 0.499, - "hit@5": 0.773, + "hit@1": 0.504, + "hit@5": 0.775, "hit@10": 0.866, - "ndcg_linear@1": 0.499, - "ndcg_linear@5": 0.6464553564135078, - "ndcg_linear@10": 0.6765142035632646, - "ndcg_exponential@1": 0.499, - "ndcg_exponential@5": 0.6464553564135078, - "ndcg_exponential@10": 0.6765142035632646, - "precision@1": 0.499, - "precision@5": 0.15460000000000002, + "ndcg_linear@1": 0.504, + "ndcg_linear@5": 0.6489251594467971, + "ndcg_linear@10": 0.6783867667599184, + "ndcg_exponential@1": 0.504, + "ndcg_exponential@5": 0.6489251594467971, + "ndcg_exponential@10": 0.6783867667599184, + "precision@1": 0.504, + "precision@5": 0.155, "precision@10": 0.08660000000000001, - "recall@1": 0.499, - "recall@5": 0.773, + "recall@1": 0.504, + "recall@5": 0.775, "recall@10": 0.866, - "f1@1": 0.499, - "f1@5": 0.2576666666666667, + "f1@1": 0.504, + "f1@5": 0.25833333333333336, "f1@10": 0.15745454545454543, - "map@1": 0.499, - "map@5": 0.6042000000000001, - "map@10": 0.616602380952381, - "mrr@1": 0.499, - "mrr@5": 0.6042000000000001, - "mrr@10": 0.616602380952381, + "map@1": 0.504, + "map@5": 0.6069166666666667, + "map@10": 0.6191083333333334, + "mrr@1": 0.504, + "mrr@5": 0.6069166666666667, + "mrr@10": 0.6191083333333334, "num_pred": 1000, "num_data": 1000 }, "NIGHTS": { - "hit@1": 0.655, + "hit@1": 0.654, "hit@5": 0.961, - "hit@10": 0.985, - "ndcg_linear@1": 0.655, - "ndcg_linear@5": 0.8290381357176521, - "ndcg_linear@10": 0.8368697653933878, - "ndcg_exponential@1": 0.655, - "ndcg_exponential@5": 0.8290381357176521, - "ndcg_exponential@10": 0.8368697653933878, - "precision@1": 0.655, + "hit@10": 0.984, + "ndcg_linear@1": 0.654, + "ndcg_linear@5": 0.8290002484202931, + "ndcg_linear@10": 0.8365616722375002, + "ndcg_exponential@1": 0.654, + "ndcg_exponential@5": 0.8290002484202931, + "ndcg_exponential@10": 0.8365616722375002, + "precision@1": 0.654, "precision@5": 0.1922, - "precision@10": 0.09850000000000003, - "recall@1": 0.655, + "precision@10": 0.0984, + "recall@1": 0.654, "recall@5": 0.961, - "recall@10": 0.985, - "f1@1": 0.655, + "recall@10": 0.984, + "f1@1": 0.654, "f1@5": 0.32033333333333336, - "f1@10": 0.17909090909090908, - "map@1": 0.655, - "map@5": 0.7840833333333334, - "map@10": 0.7873575396825397, - "mrr@1": 0.655, - "mrr@5": 0.7840833333333334, - "mrr@10": 0.7873575396825397, + "f1@10": 0.17890909090909088, + "map@1": 0.654, + "map@5": 0.784, + "map@10": 0.7871900793650793, + "mrr@1": 0.654, + "mrr@5": 0.784, + "mrr@10": 0.7871900793650793, "num_pred": 1000, "num_data": 1000 }, "WebQA": { - "hit@1": 0.538, + "hit@1": 0.542, "hit@5": 0.783, - "hit@10": 0.86, - "ndcg_linear@1": 0.538, - "ndcg_linear@5": 0.6737746946055273, - "ndcg_linear@10": 0.6988699132073182, - "ndcg_exponential@1": 0.538, - "ndcg_exponential@5": 0.6737746946055273, - "ndcg_exponential@10": 0.6988699132073182, - "precision@1": 0.538, - "precision@5": 0.15660000000000002, - "precision@10": 0.086, - "recall@1": 0.538, + "hit@10": 0.864, + "ndcg_linear@1": 0.542, + "ndcg_linear@5": 0.6753197575342229, + "ndcg_linear@10": 0.7018458007929147, + "ndcg_exponential@1": 0.542, + "ndcg_exponential@5": 0.6753197575342229, + "ndcg_exponential@10": 0.7018458007929147, + "precision@1": 0.542, + "precision@5": 0.1566, + "precision@10": 0.0864, + "recall@1": 0.542, "recall@5": 0.783, - "recall@10": 0.86, - "f1@1": 0.538, - "f1@5": 0.26100000000000007, - "f1@10": 0.15636363636363637, - "map@1": 0.538, - "map@5": 0.6369166666666666, - "map@10": 0.6473904761904762, - "mrr@1": 0.538, - "mrr@5": 0.6369166666666666, - "mrr@10": 0.6473904761904762, + "recall@10": 0.864, + "f1@1": 0.542, + "f1@5": 0.261, + "f1@10": 0.1570909090909091, + "map@1": 0.542, + "map@5": 0.6390166666666667, + "map@10": 0.6501611111111111, + "mrr@1": 0.542, + "mrr@5": 0.6390166666666667, + "mrr@10": 0.6501611111111111, "num_pred": 1000, "num_data": 1000 }, "FashionIQ": { "hit@1": 0.059, - "hit@5": 0.176, - "hit@10": 0.252, + "hit@5": 0.175, + "hit@10": 0.25, "ndcg_linear@1": 0.059, - "ndcg_linear@5": 0.11860251335553075, - "ndcg_linear@10": 0.1431080455663842, + "ndcg_linear@5": 0.11838034862218229, + "ndcg_linear@10": 0.1426640509887753, "ndcg_exponential@1": 0.059, - "ndcg_exponential@5": 0.11860251335553075, - "ndcg_exponential@10": 0.1431080455663842, + "ndcg_exponential@5": 0.11838034862218229, + "ndcg_exponential@10": 0.1426640509887753, "precision@1": 0.059, - "precision@5": 0.0352, - "precision@10": 0.0252, + "precision@5": 0.03500000000000001, + "precision@10": 0.025, "recall@1": 0.059, - "recall@5": 0.176, - "recall@10": 0.252, + "recall@5": 0.175, + "recall@10": 0.25, "f1@1": 0.059, - "f1@5": 0.05866666666666667, - "f1@10": 0.04581818181818183, + "f1@5": 0.05833333333333334, + "f1@10": 0.04545454545454546, "map@1": 0.059, - "map@5": 0.09975, - "map@10": 0.1098202380952381, + "map@5": 0.09973333333333334, + "map@10": 0.10976746031746032, "mrr@1": 0.059, - "mrr@5": 0.09975, - "mrr@10": 0.1098202380952381, + "mrr@5": 0.09973333333333334, + "mrr@10": 0.10976746031746032, "num_pred": 1000, "num_data": 1000 }, "Wiki-SS-NQ": { "hit@1": 0.805, - "hit@5": 0.948, + "hit@5": 0.951, "hit@10": 0.967, "ndcg_linear@1": 0.805, - "ndcg_linear@5": 0.8862260429565881, - "ndcg_linear@10": 0.8924625724131149, + "ndcg_linear@5": 0.8869677709275445, + "ndcg_linear@10": 0.8923573514521735, "ndcg_exponential@1": 0.805, - "ndcg_exponential@5": 0.8862260429565881, - "ndcg_exponential@10": 0.8924625724131149, + "ndcg_exponential@5": 0.8869677709275445, + "ndcg_exponential@10": 0.8923573514521735, "precision@1": 0.805, - "precision@5": 0.18960000000000002, + "precision@5": 0.1902, "precision@10": 0.09670000000000002, "recall@1": 0.805, - "recall@5": 0.948, + "recall@5": 0.951, "recall@10": 0.967, "f1@1": 0.805, - "f1@5": 0.316, + "f1@5": 0.317, "f1@10": 0.1758181818181818, "map@1": 0.805, - "map@5": 0.8652000000000001, - "map@10": 0.8678285714285714, + "map@5": 0.8652833333333334, + "map@10": 0.8676353174603174, "mrr@1": 0.805, - "mrr@5": 0.8652000000000001, - "mrr@10": 0.8678285714285714, + "mrr@5": 0.8652833333333334, + "mrr@10": 0.8676353174603174, "num_pred": 1000, "num_data": 1000 }, "OVEN": { - "hit@1": 0.5, - "hit@5": 0.702, - "hit@10": 0.769, - "ndcg_linear@1": 0.5, - "ndcg_linear@5": 0.6068469045790832, - "ndcg_linear@10": 0.6285492152170207, - "ndcg_exponential@1": 0.5, - "ndcg_exponential@5": 0.6068469045790832, - "ndcg_exponential@10": 0.6285492152170207, - "precision@1": 0.5, - "precision@5": 0.14039999999999997, - "precision@10": 0.07690000000000001, - "recall@1": 0.5, - "recall@5": 0.702, - "recall@10": 0.769, - "f1@1": 0.5, - "f1@5": 0.23400000000000007, - "f1@10": 0.13981818181818184, - "map@1": 0.5, - "map@5": 0.5752666666666667, - "map@10": 0.5842511904761905, - "mrr@1": 0.5, - "mrr@5": 0.5752666666666667, - "mrr@10": 0.5842511904761905, + "hit@1": 0.501, + "hit@5": 0.703, + "hit@10": 0.77, + "ndcg_linear@1": 0.501, + "ndcg_linear@5": 0.6077704055058198, + "ndcg_linear@10": 0.629431973833435, + "ndcg_exponential@1": 0.501, + "ndcg_exponential@5": 0.6077704055058198, + "ndcg_exponential@10": 0.629431973833435, + "precision@1": 0.501, + "precision@5": 0.1406, + "precision@10": 0.077, + "recall@1": 0.501, + "recall@5": 0.703, + "recall@10": 0.77, + "f1@1": 0.501, + "f1@5": 0.2343333333333334, + "f1@10": 0.14, + "map@1": 0.501, + "map@5": 0.5761666666666667, + "map@10": 0.5851095238095239, + "mrr@1": 0.501, + "mrr@5": 0.5761666666666667, + "mrr@10": 0.5851095238095239, "num_pred": 1000, "num_data": 1000 }, "EDIS": { - "hit@1": 0.647, - "hit@5": 0.84, - "hit@10": 0.895, - "ndcg_linear@1": 0.647, - "ndcg_linear@5": 0.7509000967566725, - "ndcg_linear@10": 0.7687390290441729, - "ndcg_exponential@1": 0.647, - "ndcg_exponential@5": 0.7509000967566725, - "ndcg_exponential@10": 0.7687390290441729, - "precision@1": 0.647, - "precision@5": 0.168, - "precision@10": 0.0895, - "recall@1": 0.647, - "recall@5": 0.84, - "recall@10": 0.895, - "f1@1": 0.647, - "f1@5": 0.2800000000000001, - "f1@10": 0.16272727272727272, - "map@1": 0.647, - "map@5": 0.7211333333333332, - "map@10": 0.7285333333333333, - "mrr@1": 0.647, - "mrr@5": 0.7211333333333332, - "mrr@10": 0.7285333333333333, + "hit@1": 0.652, + "hit@5": 0.841, + "hit@10": 0.892, + "ndcg_linear@1": 0.652, + "ndcg_linear@5": 0.753995461388632, + "ndcg_linear@10": 0.7706074658133776, + "ndcg_exponential@1": 0.652, + "ndcg_exponential@5": 0.753995461388632, + "ndcg_exponential@10": 0.7706074658133776, + "precision@1": 0.652, + "precision@5": 0.16820000000000002, + "precision@10": 0.08920000000000002, + "recall@1": 0.652, + "recall@5": 0.841, + "recall@10": 0.892, + "f1@1": 0.652, + "f1@5": 0.2803333333333334, + "f1@10": 0.16218181818181818, + "map@1": 0.652, + "map@5": 0.7249, + "map@10": 0.7318321428571428, + "mrr@1": 0.652, + "mrr@5": 0.7249, + "mrr@10": 0.7318321428571428, "num_pred": 1000, "num_data": 1000 }, "MSCOCO": { - "hit@1": 0.367, - "hit@5": 0.486, - "hit@10": 0.531, - "ndcg_linear@1": 0.367, - "ndcg_linear@5": 0.4311932611107677, - "ndcg_linear@10": 0.44568713259686477, - "ndcg_exponential@1": 0.367, - "ndcg_exponential@5": 0.4311932611107677, - "ndcg_exponential@10": 0.44568713259686477, - "precision@1": 0.367, - "precision@5": 0.0972, - "precision@10": 0.053099999999999994, - "recall@1": 0.367, - "recall@5": 0.486, - "recall@10": 0.531, - "f1@1": 0.367, - "f1@5": 0.16200000000000006, - "f1@10": 0.09654545454545456, - "map@1": 0.367, - "map@5": 0.41286666666666666, - "map@10": 0.4188095238095238, - "mrr@1": 0.367, - "mrr@5": 0.41286666666666666, - "mrr@10": 0.4188095238095238, + "hit@1": 0.365, + "hit@5": 0.49, + "hit@10": 0.529, + "ndcg_linear@1": 0.365, + "ndcg_linear@5": 0.4315919600253284, + "ndcg_linear@10": 0.4440937817947862, + "ndcg_exponential@1": 0.365, + "ndcg_exponential@5": 0.4315919600253284, + "ndcg_exponential@10": 0.4440937817947862, + "precision@1": 0.365, + "precision@5": 0.098, + "precision@10": 0.05289999999999999, + "recall@1": 0.365, + "recall@5": 0.49, + "recall@10": 0.529, + "f1@1": 0.365, + "f1@5": 0.16333333333333336, + "f1@10": 0.09618181818181819, + "map@1": 0.365, + "map@5": 0.41213333333333335, + "map@10": 0.41722222222222227, + "mrr@1": 0.365, + "mrr@5": 0.41213333333333335, + "mrr@10": 0.41722222222222227, "num_pred": 1000, "num_data": 1000 }, "RefCOCO": { - "hit@1": 0.645, - "hit@5": 0.85, - "hit@10": 0.892, - "ndcg_linear@1": 0.645, - "ndcg_linear@5": 0.7606183958914773, - "ndcg_linear@10": 0.7742292643426952, - "ndcg_exponential@1": 0.645, - "ndcg_exponential@5": 0.7606183958914773, - "ndcg_exponential@10": 0.7742292643426952, - "precision@1": 0.645, - "precision@5": 0.17, - "precision@10": 0.08920000000000002, - "recall@1": 0.645, - "recall@5": 0.85, - "recall@10": 0.892, - "f1@1": 0.645, - "f1@5": 0.2833333333333334, - "f1@10": 0.16218181818181818, - "map@1": 0.645, - "map@5": 0.7303, - "map@10": 0.7359353174603175, - "mrr@1": 0.645, - "mrr@5": 0.7303, - "mrr@10": 0.7359353174603175, + "hit@1": 0.65, + "hit@5": 0.849, + "hit@10": 0.895, + "ndcg_linear@1": 0.65, + "ndcg_linear@5": 0.7605874744462212, + "ndcg_linear@10": 0.7754158412762995, + "ndcg_exponential@1": 0.65, + "ndcg_exponential@5": 0.7605874744462212, + "ndcg_exponential@10": 0.7754158412762995, + "precision@1": 0.65, + "precision@5": 0.1698, + "precision@10": 0.08950000000000001, + "recall@1": 0.65, + "recall@5": 0.849, + "recall@10": 0.895, + "f1@1": 0.65, + "f1@5": 0.28300000000000003, + "f1@10": 0.16272727272727275, + "map@1": 0.65, + "map@5": 0.73075, + "map@10": 0.736845238095238, + "mrr@1": 0.65, + "mrr@5": 0.73075, + "mrr@10": 0.736845238095238, "num_pred": 1000, "num_data": 1000 }, "RefCOCO-Matching": { - "hit@1": 0.039, + "hit@1": 0.012, "hit@5": 1.0, "hit@10": 1.0, - "ndcg_linear@1": 0.039, - "ndcg_linear@5": 0.6472162824428851, - "ndcg_linear@10": 0.6472162824428851, - "ndcg_exponential@1": 0.039, - "ndcg_exponential@5": 0.6472162824428851, - "ndcg_exponential@10": 0.6472162824428851, - "precision@1": 0.039, + "ndcg_linear@1": 0.012, + "ndcg_linear@5": 0.6372513857893145, + "ndcg_linear@10": 0.6372513857893145, + "ndcg_exponential@1": 0.012, + "ndcg_exponential@5": 0.6372513857893145, + "ndcg_exponential@10": 0.6372513857893145, + "precision@1": 0.012, "precision@5": 0.20000000000000004, "precision@10": 0.10000000000000002, - "recall@1": 0.039, + "recall@1": 0.012, "recall@5": 1.0, "recall@10": 1.0, - "f1@1": 0.039, + "f1@1": 0.012, "f1@5": 0.3333333333333333, "f1@10": 0.18181818181818182, - "map@1": 0.039, - "map@5": 0.5225, - "map@10": 0.5225, - "mrr@1": 0.039, - "mrr@5": 0.5195, - "mrr@10": 0.5195, + "map@1": 0.012, + "map@5": 0.509, + "map@10": 0.509, + "mrr@1": 0.012, + "mrr@5": 0.506, + "mrr@10": 0.506, "num_pred": 1000, "num_data": 1000 }, "Visual7W-Pointing": { - "hit@1": 0.561, - "hit@5": 0.744, - "hit@10": 0.815, - "ndcg_linear@1": 0.561, - "ndcg_linear@5": 0.6565683187093909, - "ndcg_linear@10": 0.6796809170784373, - "ndcg_exponential@1": 0.561, - "ndcg_exponential@5": 0.6565683187093909, - "ndcg_exponential@10": 0.6796809170784373, - "precision@1": 0.561, - "precision@5": 0.14880000000000002, - "precision@10": 0.0815, - "recall@1": 0.561, - "recall@5": 0.744, - "recall@10": 0.815, - "f1@1": 0.561, - "f1@5": 0.24800000000000005, - "f1@10": 0.1481818181818182, - "map@1": 0.561, - "map@5": 0.6276666666666666, - "map@10": 0.6372944444444445, - "mrr@1": 0.561, - "mrr@5": 0.6276666666666666, - "mrr@10": 0.6372944444444445, + "hit@1": 0.563, + "hit@5": 0.739, + "hit@10": 0.817, + "ndcg_linear@1": 0.563, + "ndcg_linear@5": 0.6553438056757999, + "ndcg_linear@10": 0.6807315493269261, + "ndcg_exponential@1": 0.563, + "ndcg_exponential@5": 0.6553438056757999, + "ndcg_exponential@10": 0.6807315493269261, + "precision@1": 0.563, + "precision@5": 0.1478, + "precision@10": 0.08170000000000001, + "recall@1": 0.563, + "recall@5": 0.739, + "recall@10": 0.817, + "f1@1": 0.563, + "f1@5": 0.24633333333333338, + "f1@10": 0.14854545454545456, + "map@1": 0.563, + "map@5": 0.6276166666666666, + "map@10": 0.6381912698412698, + "mrr@1": 0.563, + "mrr@5": 0.6276166666666666, + "mrr@10": 0.6381912698412698, "num_pred": 1000, "num_data": 1000 } }, "video": { "K700": { - "hit@1": 0.234, - "hit@5": 0.411, - "hit@10": 0.47, - "ndcg_linear@1": 0.234, - "ndcg_linear@5": 0.3273147185066129, - "ndcg_linear@10": 0.3462654041192875, - "ndcg_exponential@1": 0.234, - "ndcg_exponential@5": 0.3273147185066129, - "ndcg_exponential@10": 0.3462654041192875, - "precision@1": 0.234, - "precision@5": 0.0822, - "precision@10": 0.047, - "recall@1": 0.234, - "recall@5": 0.411, - "recall@10": 0.47, - "f1@1": 0.234, - "f1@5": 0.13700000000000004, - "f1@10": 0.08545454545454546, - "map@1": 0.234, - "map@5": 0.2995, - "map@10": 0.30724523809523807, - "mrr@1": 0.234, - "mrr@5": 0.2995, - "mrr@10": 0.30724523809523807, + "hit@1": 0.233, + "hit@5": 0.412, + "hit@10": 0.468, + "ndcg_linear@1": 0.233, + "ndcg_linear@5": 0.3271827057551513, + "ndcg_linear@10": 0.345088700145194, + "ndcg_exponential@1": 0.233, + "ndcg_exponential@5": 0.3271827057551513, + "ndcg_exponential@10": 0.345088700145194, + "precision@1": 0.233, + "precision@5": 0.08239999999999999, + "precision@10": 0.046799999999999994, + "recall@1": 0.233, + "recall@5": 0.412, + "recall@10": 0.468, + "f1@1": 0.233, + "f1@5": 0.13733333333333336, + "f1@10": 0.0850909090909091, + "map@1": 0.233, + "map@5": 0.2990333333333333, + "map@10": 0.3063007936507936, + "mrr@1": 0.233, + "mrr@5": 0.2990333333333333, + "mrr@10": 0.3063007936507936, "num_pred": 1000, "num_data": 1000 }, "SmthSmthV2": { - "hit@1": 0.251, - "hit@5": 0.485, + "hit@1": 0.249, + "hit@5": 0.482, "hit@10": 0.592, - "ndcg_linear@1": 0.251, - "ndcg_linear@5": 0.37516969884345464, - "ndcg_linear@10": 0.4100591996704718, - "ndcg_exponential@1": 0.251, - "ndcg_exponential@5": 0.37516969884345464, - "ndcg_exponential@10": 0.4100591996704718, - "precision@1": 0.251, - "precision@5": 0.097, + "ndcg_linear@1": 0.249, + "ndcg_linear@5": 0.373915583266227, + "ndcg_linear@10": 0.4097855778412195, + "ndcg_exponential@1": 0.249, + "ndcg_exponential@5": 0.373915583266227, + "ndcg_exponential@10": 0.4097855778412195, + "precision@1": 0.249, + "precision@5": 0.09639999999999999, "precision@10": 0.059199999999999996, - "recall@1": 0.251, - "recall@5": 0.485, + "recall@1": 0.249, + "recall@5": 0.482, "recall@10": 0.592, - "f1@1": 0.251, - "f1@5": 0.1616666666666667, + "f1@1": 0.249, + "f1@5": 0.1606666666666667, "f1@10": 0.10763636363636366, - "map@1": 0.251, - "map@5": 0.33863333333333334, - "map@10": 0.35319563492063494, - "mrr@1": 0.251, - "mrr@5": 0.33863333333333334, - "mrr@10": 0.35319563492063494, + "map@1": 0.249, + "map@5": 0.3378333333333334, + "map@10": 0.35280396825396826, + "mrr@1": 0.249, + "mrr@5": 0.3378333333333334, + "mrr@10": 0.35280396825396826, "num_pred": 1000, "num_data": 1000 }, "HMDB51": { "hit@1": 0.248, - "hit@5": 0.447, - "hit@10": 0.578, + "hit@5": 0.451, + "hit@10": 0.579, "ndcg_linear@1": 0.248, - "ndcg_linear@5": 0.35078743678844715, - "ndcg_linear@10": 0.39292321098555394, + "ndcg_linear@5": 0.351959841317477, + "ndcg_linear@10": 0.39281728649317943, "ndcg_exponential@1": 0.248, - "ndcg_exponential@5": 0.35078743678844715, - "ndcg_exponential@10": 0.39292321098555394, + "ndcg_exponential@5": 0.351959841317477, + "ndcg_exponential@10": 0.39281728649317943, "precision@1": 0.248, - "precision@5": 0.0894, - "precision@10": 0.0578, + "precision@5": 0.09019999999999999, + "precision@10": 0.05789999999999999, "recall@1": 0.248, - "recall@5": 0.447, - "recall@10": 0.578, + "recall@5": 0.451, + "recall@10": 0.579, "f1@1": 0.248, - "f1@5": 0.14900000000000005, - "f1@10": 0.10509090909090911, + "f1@5": 0.15033333333333337, + "f1@10": 0.10527272727272728, "map@1": 0.248, - "map@5": 0.31906666666666667, - "map@10": 0.33631944444444445, + "map@5": 0.31939999999999996, + "map@10": 0.3359404761904761, "mrr@1": 0.248, - "mrr@5": 0.31906666666666667, - "mrr@10": 0.33631944444444445, + "mrr@5": 0.31939999999999996, + "mrr@10": 0.3359404761904761, "num_pred": 1000, "num_data": 1000 }, "UCF101": { - "hit@1": 0.494, - "hit@5": 0.675, - "hit@10": 0.752, - "ndcg_linear@1": 0.494, - "ndcg_linear@5": 0.5961521274121583, - "ndcg_linear@10": 0.6208342304490151, - "ndcg_exponential@1": 0.494, - "ndcg_exponential@5": 0.5961521274121583, - "ndcg_exponential@10": 0.6208342304490151, - "precision@1": 0.494, - "precision@5": 0.135, - "precision@10": 0.0752, - "recall@1": 0.494, - "recall@5": 0.675, - "recall@10": 0.752, - "f1@1": 0.494, - "f1@5": 0.22500000000000006, - "f1@10": 0.13672727272727275, - "map@1": 0.494, - "map@5": 0.56945, - "map@10": 0.5795154761904762, - "mrr@1": 0.494, - "mrr@5": 0.56945, - "mrr@10": 0.5795154761904762, + "hit@1": 0.493, + "hit@5": 0.674, + "hit@10": 0.751, + "ndcg_linear@1": 0.493, + "ndcg_linear@5": 0.5952214508540848, + "ndcg_linear@10": 0.619960778078545, + "ndcg_exponential@1": 0.493, + "ndcg_exponential@5": 0.5952214508540848, + "ndcg_exponential@10": 0.619960778078545, + "precision@1": 0.493, + "precision@5": 0.1348, + "precision@10": 0.0751, + "recall@1": 0.493, + "recall@5": 0.674, + "recall@10": 0.751, + "f1@1": 0.493, + "f1@5": 0.22466666666666674, + "f1@10": 0.13654545454545455, + "map@1": 0.493, + "map@5": 0.5685333333333333, + "map@10": 0.5786507936507937, + "mrr@1": 0.493, + "mrr@5": 0.5685333333333333, + "mrr@10": 0.5786507936507937, "num_pred": 1000, "num_data": 1000 }, "Breakfast": { "hit@1": 0.10854503464203233, - "hit@5": 0.5519630484988453, + "hit@5": 0.5565819861431871, "hit@10": 1.0, "ndcg_linear@1": 0.10854503464203233, - "ndcg_linear@5": 0.3325565915387777, - "ndcg_linear@10": 0.47501706939610605, + "ndcg_linear@5": 0.3350672289584835, + "ndcg_linear@10": 0.4759708589449622, "ndcg_exponential@1": 0.10854503464203233, - "ndcg_exponential@5": 0.3325565915387777, - "ndcg_exponential@10": 0.47501706939610605, + "ndcg_exponential@5": 0.3350672289584835, + "ndcg_exponential@10": 0.4759708589449622, "precision@1": 0.10854503464203233, - "precision@5": 0.11039260969976905, + "precision@5": 0.1113163972286374, "precision@10": 0.1, "recall@1": 0.10854503464203233, - "recall@5": 0.5519630484988453, + "recall@5": 0.5565819861431871, "recall@10": 1.0, "f1@1": 0.10854503464203233, - "f1@5": 0.18398768283294847, + "f1@5": 0.1855273287143957, "f1@10": 0.1818181818181818, "map@1": 0.10854503464203233, - "map@5": 0.2607005388760585, - "map@10": 0.3181110011364053, + "map@5": 0.26250962278675904, + "map@10": 0.3192327431357454, "mrr@1": 0.10854503464203233, - "mrr@5": 0.2607005388760585, - "mrr@10": 0.3181110011364053, + "mrr@5": 0.26250962278675904, + "mrr@10": 0.3192327431357454, "num_pred": 433, "num_data": 433 }, "MVBench": { - "hit@1": 0.337, + "hit@1": 0.33575, "hit@5": 1.0, "hit@10": 1.0, - "ndcg_linear@1": 0.337, - "ndcg_linear@5": 0.6944068117467572, - "ndcg_linear@10": 0.6944068117467572, - "ndcg_exponential@1": 0.337, - "ndcg_exponential@5": 0.6944068117467572, - "ndcg_exponential@10": 0.6944068117467572, - "precision@1": 0.337, + "ndcg_linear@1": 0.33575, + "ndcg_linear@5": 0.6939672504394047, + "ndcg_linear@10": 0.6939672504394047, + "ndcg_exponential@1": 0.33575, + "ndcg_exponential@5": 0.6939672504394047, + "ndcg_exponential@10": 0.6939672504394047, + "precision@1": 0.33575, "precision@5": 0.20000000000000004, "precision@10": 0.10000000000000002, - "recall@1": 0.337, + "recall@1": 0.33575, "recall@5": 1.0, "recall@10": 1.0, - "f1@1": 0.337, + "f1@1": 0.33575, "f1@5": 0.3333333333333333, "f1@10": 0.18181818181818182, - "map@1": 0.337, - "map@5": 0.5917374999999999, - "map@10": 0.5917374999999999, - "mrr@1": 0.337, - "mrr@5": 0.5917374999999999, - "mrr@10": 0.5917374999999999, + "map@1": 0.33575, + "map@5": 0.5911416666666667, + "map@10": 0.5911416666666667, + "mrr@1": 0.33575, + "mrr@5": 0.5911416666666667, + "mrr@10": 0.5911416666666667, "num_pred": 4000, "num_data": 4000 }, @@ -1191,11 +1188,11 @@ "hit@5": 1.0, "hit@10": 1.0, "ndcg_linear@1": 0.30592592592592593, - "ndcg_linear@5": 0.6721188097820419, - "ndcg_linear@10": 0.6721188097820419, + "ndcg_linear@5": 0.6721958358286271, + "ndcg_linear@10": 0.6721958358286271, "ndcg_exponential@1": 0.30592592592592593, - "ndcg_exponential@5": 0.6721188097820419, - "ndcg_exponential@10": 0.6721188097820419, + "ndcg_exponential@5": 0.6721958358286271, + "ndcg_exponential@10": 0.6721958358286271, "precision@1": 0.30592592592592593, "precision@5": 0.19999999999999996, "precision@10": 0.09999999999999998, @@ -1206,39 +1203,39 @@ "f1@5": 0.3333333333333334, "f1@10": 0.18181818181818188, "map@1": 0.30592592592592593, - "map@5": 0.5626543209876542, - "map@10": 0.5626543209876542, + "map@5": 0.5627469135802469, + "map@10": 0.5627469135802469, "mrr@1": 0.30592592592592593, - "mrr@5": 0.5626543209876542, - "mrr@10": 0.5626543209876542, + "mrr@5": 0.5627469135802469, + "mrr@10": 0.5627469135802469, "num_pred": 2700, "num_data": 2700 }, "NExTQA": { - "hit@1": 0.3520551144325082, + "hit@1": 0.3514712751050911, "hit@5": 1.0, "hit@10": 1.0, - "ndcg_linear@1": 0.3520551144325082, - "ndcg_linear@5": 0.677249628478771, - "ndcg_linear@10": 0.677249628478771, - "ndcg_exponential@1": 0.3520551144325082, - "ndcg_exponential@5": 0.677249628478771, - "ndcg_exponential@10": 0.677249628478771, - "precision@1": 0.3520551144325082, + "ndcg_linear@1": 0.3514712751050911, + "ndcg_linear@5": 0.6769708575415712, + "ndcg_linear@10": 0.6769708575415712, + "ndcg_exponential@1": 0.3514712751050911, + "ndcg_exponential@5": 0.6769708575415712, + "ndcg_exponential@10": 0.6769708575415712, + "precision@1": 0.3514712751050911, "precision@5": 0.20000000000000004, "precision@10": 0.10000000000000002, - "recall@1": 0.3520551144325082, + "recall@1": 0.3514712751050911, "recall@5": 1.0, "recall@10": 1.0, - "f1@1": 0.3520551144325082, + "f1@1": 0.3514712751050911, "f1@5": 0.3333333333333333, "f1@10": 0.1818181818181818, - "map@1": 0.3520551144325082, - "map@5": 0.5717772069126577, - "map@10": 0.5717772069126577, - "mrr@1": 0.3520551144325082, - "mrr@5": 0.5716020551144325, - "mrr@10": 0.5716020551144325, + "map@1": 0.3514712751050911, + "map@5": 0.5714054958742021, + "map@10": 0.5714054958742021, + "mrr@1": 0.3514712751050911, + "mrr@5": 0.571230344075977, + "mrr@10": 0.571230344075977, "num_pred": 8564, "num_data": 8564 }, @@ -1247,11 +1244,11 @@ "hit@5": 1.0, "hit@10": 1.0, "ndcg_linear@1": 0.384, - "ndcg_linear@5": 0.6958150458511005, - "ndcg_linear@10": 0.6958150458511005, + "ndcg_linear@5": 0.6955366690855039, + "ndcg_linear@10": 0.6955366690855039, "ndcg_exponential@1": 0.384, - "ndcg_exponential@5": 0.6958150458511005, - "ndcg_exponential@10": 0.6958150458511005, + "ndcg_exponential@5": 0.6955366690855039, + "ndcg_exponential@10": 0.6955366690855039, "precision@1": 0.384, "precision@5": 0.2, "precision@10": 0.1, @@ -1262,937 +1259,909 @@ "f1@5": 0.3333333333333333, "f1@10": 0.18181818181818182, "map@1": 0.384, - "map@5": 0.5960666666666666, - "map@10": 0.5960666666666666, + "map@5": 0.5957666666666667, + "map@10": 0.5957666666666667, "mrr@1": 0.384, - "mrr@5": 0.5960666666666666, - "mrr@10": 0.5960666666666666, + "mrr@5": 0.5957666666666667, + "mrr@10": 0.5957666666666667, "num_pred": 500, "num_data": 500 }, "ActivityNetQA": { - "hit@1": 0.513, + "hit@1": 0.514, "hit@5": 1.0, "hit@10": 1.0, - "ndcg_linear@1": 0.513, - "ndcg_linear@5": 0.8202627899893, - "ndcg_linear@10": 0.8202627899893, - "ndcg_exponential@1": 0.513, - "ndcg_exponential@5": 0.8202627899893, - "ndcg_exponential@10": 0.8202627899893, - "precision@1": 0.513, + "ndcg_linear@1": 0.514, + "ndcg_linear@5": 0.8206318602357284, + "ndcg_linear@10": 0.8206318602357284, + "ndcg_exponential@1": 0.514, + "ndcg_exponential@5": 0.8206318602357284, + "ndcg_exponential@10": 0.8206318602357284, + "precision@1": 0.514, "precision@5": 0.20000000000000004, "precision@10": 0.10000000000000002, - "recall@1": 0.513, + "recall@1": 0.514, "recall@5": 1.0, "recall@10": 1.0, - "f1@1": 0.513, + "f1@1": 0.514, "f1@5": 0.3333333333333333, "f1@10": 0.18181818181818182, - "map@1": 0.513, - "map@5": 0.7565, - "map@10": 0.7565, - "mrr@1": 0.513, - "mrr@5": 0.7565, - "mrr@10": 0.7565, + "map@1": 0.514, + "map@5": 0.757, + "map@10": 0.757, + "mrr@1": 0.514, + "mrr@5": 0.757, + "mrr@10": 0.757, "num_pred": 1000, "num_data": 1000 }, "DiDeMo": { - "hit@1": 0.22808764940239043, - "hit@5": 0.4302788844621514, + "hit@1": 0.2250996015936255, + "hit@5": 0.42729083665338646, "hit@10": 0.5169322709163346, - "ndcg_linear@1": 0.22808764940239043, - "ndcg_linear@5": 0.33396649038814646, - "ndcg_linear@10": 0.3621437829397595, - "ndcg_exponential@1": 0.22808764940239043, - "ndcg_exponential@5": 0.33396649038814646, - "ndcg_exponential@10": 0.3621437829397595, - "precision@1": 0.22808764940239043, - "precision@5": 0.08605577689243028, + "ndcg_linear@1": 0.2250996015936255, + "ndcg_linear@5": 0.3318127661864477, + "ndcg_linear@10": 0.36100445093637457, + "ndcg_exponential@1": 0.2250996015936255, + "ndcg_exponential@5": 0.3318127661864477, + "ndcg_exponential@10": 0.36100445093637457, + "precision@1": 0.2250996015936255, + "precision@5": 0.0854581673306773, "precision@10": 0.051693227091633466, - "recall@1": 0.22808764940239043, - "recall@5": 0.4302788844621514, + "recall@1": 0.2250996015936255, + "recall@5": 0.42729083665338646, "recall@10": 0.5169322709163346, - "f1@1": 0.22808764940239043, - "f1@5": 0.14342629482071717, - "f1@10": 0.09398768562115178, - "map@1": 0.22808764940239043, - "map@5": 0.30207503320053125, - "map@10": 0.3137928444950357, - "mrr@1": 0.22808764940239043, - "mrr@5": 0.30207503320053125, - "mrr@10": 0.3137928444950357, + "f1@1": 0.2250996015936255, + "f1@5": 0.1424302788844622, + "f1@10": 0.09398768562115177, + "map@1": 0.2250996015936255, + "map@5": 0.30011620185922977, + "map@10": 0.3122826155694681, + "mrr@1": 0.2250996015936255, + "mrr@5": 0.30011620185922977, + "mrr@10": 0.3122826155694681, "num_pred": 1004, "num_data": 1004 }, "MSR-VTT": { - "hit@1": 0.176, - "hit@5": 0.337, - "hit@10": 0.441, - "ndcg_linear@1": 0.176, - "ndcg_linear@5": 0.26090414668509254, - "ndcg_linear@10": 0.29375988725520075, - "ndcg_exponential@1": 0.176, - "ndcg_exponential@5": 0.26090414668509254, - "ndcg_exponential@10": 0.29375988725520075, - "precision@1": 0.176, - "precision@5": 0.0674, - "precision@10": 0.04409999999999999, - "recall@1": 0.176, - "recall@5": 0.337, - "recall@10": 0.441, - "f1@1": 0.176, - "f1@5": 0.11233333333333335, - "f1@10": 0.08018181818181819, - "map@1": 0.176, - "map@5": 0.23563333333333336, - "map@10": 0.24873690476190477, - "mrr@1": 0.176, - "mrr@5": 0.23563333333333336, - "mrr@10": 0.24873690476190477, + "hit@1": 0.177, + "hit@5": 0.338, + "hit@10": 0.433, + "ndcg_linear@1": 0.177, + "ndcg_linear@5": 0.2615445742457477, + "ndcg_linear@10": 0.291414934329041, + "ndcg_exponential@1": 0.177, + "ndcg_exponential@5": 0.2615445742457477, + "ndcg_exponential@10": 0.291414934329041, + "precision@1": 0.177, + "precision@5": 0.0676, + "precision@10": 0.0433, + "recall@1": 0.177, + "recall@5": 0.338, + "recall@10": 0.433, + "f1@1": 0.177, + "f1@5": 0.11266666666666668, + "f1@10": 0.07872727272727274, + "map@1": 0.177, + "map@5": 0.23616666666666666, + "map@10": 0.24799444444444446, + "mrr@1": 0.177, + "mrr@5": 0.23616666666666666, + "mrr@10": 0.24799444444444446, "num_pred": 1000, "num_data": 1000 }, "MSVD": { "hit@1": 0.4537313432835821, - "hit@5": 0.7164179104477612, - "hit@10": 0.7865671641791044, + "hit@5": 0.7149253731343284, + "hit@10": 0.7805970149253731, "ndcg_linear@1": 0.4537313432835821, - "ndcg_linear@5": 0.5961390786820799, - "ndcg_linear@10": 0.6187825557676115, + "ndcg_linear@5": 0.5962240575412671, + "ndcg_linear@10": 0.6178439891483547, "ndcg_exponential@1": 0.4537313432835821, - "ndcg_exponential@5": 0.5961390786820799, - "ndcg_exponential@10": 0.6187825557676115, + "ndcg_exponential@5": 0.5962240575412671, + "ndcg_exponential@10": 0.6178439891483547, "precision@1": 0.4537313432835821, - "precision@5": 0.14328358208955225, - "precision@10": 0.07865671641791044, + "precision@5": 0.14298507462686566, + "precision@10": 0.07805970149253731, "recall@1": 0.4537313432835821, - "recall@5": 0.7164179104477612, - "recall@10": 0.7865671641791044, + "recall@5": 0.7149253731343284, + "recall@10": 0.7805970149253731, "f1@1": 0.4537313432835821, - "f1@5": 0.2388059701492538, - "f1@10": 0.14301221166892814, + "f1@5": 0.23830845771144282, + "f1@10": 0.1419267299864315, "map@1": 0.4537313432835821, - "map@5": 0.555870646766169, - "map@10": 0.5651930822080076, + "map@5": 0.5564179104477612, + "map@10": 0.5655739161336177, "mrr@1": 0.4537313432835821, - "mrr@5": 0.555870646766169, - "mrr@10": 0.5651930822080076, + "mrr@5": 0.5564179104477612, + "mrr@10": 0.5655739161336177, "num_pred": 670, "num_data": 670 }, "VATEX": { "hit@1": 0.16681554265297008, - "hit@5": 0.32402858418937025, - "hit@10": 0.40576150066994193, + "hit@5": 0.3233586422510049, + "hit@10": 0.40643144260830727, "ndcg_linear@1": 0.16681554265297008, - "ndcg_linear@5": 0.24876051238618194, - "ndcg_linear@10": 0.2751625761091204, + "ndcg_linear@5": 0.248544218537269, + "ndcg_linear@10": 0.2754244076528015, "ndcg_exponential@1": 0.16681554265297008, - "ndcg_exponential@5": 0.24876051238618194, - "ndcg_exponential@10": 0.2751625761091204, + "ndcg_exponential@5": 0.248544218537269, + "ndcg_exponential@10": 0.2754244076528015, "precision@1": 0.16681554265297008, - "precision@5": 0.06480571683787406, - "precision@10": 0.0405761500669942, + "precision@5": 0.06467172845020099, + "precision@10": 0.04064314426083073, "recall@1": 0.16681554265297008, - "recall@5": 0.32402858418937025, - "recall@10": 0.40576150066994193, + "recall@5": 0.3233586422510049, + "recall@10": 0.40643144260830727, "f1@1": 0.16681554265297008, - "f1@5": 0.10800952806312343, - "f1@10": 0.07377481830362581, + "f1@5": 0.10778621408366833, + "f1@10": 0.07389662592878314, "map@1": 0.16681554265297008, - "map@5": 0.22386109870477894, - "map@10": 0.23473578057097488, + "map@5": 0.2237866607116272, + "map@10": 0.2348887329235699, "mrr@1": 0.16681554265297008, - "mrr@5": 0.22386109870477894, - "mrr@10": 0.23473578057097488, + "mrr@5": 0.2237866607116272, + "mrr@10": 0.2348887329235699, "num_pred": 4478, "num_data": 4478 }, "YouCook2": { - "hit@1": 0.05316137150047184, - "hit@5": 0.13337527524378737, - "hit@10": 0.18433469644542308, - "ndcg_linear@1": 0.05316137150047184, - "ndcg_linear@5": 0.09392223767158699, - "ndcg_linear@10": 0.11028455847469672, - "ndcg_exponential@1": 0.05316137150047184, - "ndcg_exponential@5": 0.09392223767158699, - "ndcg_exponential@10": 0.11028455847469672, - "precision@1": 0.05316137150047184, - "precision@5": 0.026675055048757473, - "precision@10": 0.01843346964454231, - "recall@1": 0.05316137150047184, - "recall@5": 0.13337527524378737, - "recall@10": 0.18433469644542308, - "f1@1": 0.05316137150047184, - "f1@5": 0.04445842508126246, - "f1@10": 0.03351539935371329, - "map@1": 0.05316137150047184, - "map@5": 0.08096885813148788, - "map@10": 0.08765359976432639, - "mrr@1": 0.05316137150047184, - "mrr@5": 0.08096885813148788, - "mrr@10": 0.08765359976432639, + "hit@1": 0.051588549858446055, + "hit@5": 0.13557722554262347, + "hit@10": 0.18150361748977667, + "ndcg_linear@1": 0.051588549858446055, + "ndcg_linear@5": 0.09410601925475856, + "ndcg_linear@10": 0.10891737056758491, + "ndcg_exponential@1": 0.051588549858446055, + "ndcg_exponential@5": 0.09410601925475856, + "ndcg_exponential@10": 0.10891737056758491, + "precision@1": 0.051588549858446055, + "precision@5": 0.027115445108524698, + "precision@10": 0.01815036174897767, + "recall@1": 0.051588549858446055, + "recall@5": 0.13557722554262347, + "recall@10": 0.18150361748977667, + "f1@1": 0.051588549858446055, + "f1@5": 0.045192408514207826, + "f1@10": 0.03300065772541394, + "map@1": 0.051588549858446055, + "map@5": 0.08052846807172066, + "map@10": 0.0866150381721316, + "mrr@1": 0.051588549858446055, + "mrr@5": 0.08052846807172066, + "mrr@10": 0.0866150381721316, "num_pred": 3179, "num_data": 3179 }, "QVHighlight": { - "hit@1": 0.1994459833795014, - "hit@5": 0.5115420129270545, + "hit@1": 0.2012927054478301, + "hit@5": 0.5096952908587258, "hit@10": 1.0, - "ndcg_linear@1": 0.1994459833795014, - "ndcg_linear@5": 0.35187697350343916, - "ndcg_linear@10": 0.5052964341688586, - "ndcg_exponential@1": 0.1994459833795014, - "ndcg_exponential@5": 0.35187697350343916, - "ndcg_exponential@10": 0.5052964341688586, - "precision@1": 0.1994459833795014, - "precision@5": 0.10230840258541089, + "ndcg_linear@1": 0.2012927054478301, + "ndcg_linear@5": 0.3513600523262989, + "ndcg_linear@10": 0.5054097568520425, + "ndcg_exponential@1": 0.2012927054478301, + "ndcg_exponential@5": 0.3513600523262989, + "ndcg_exponential@10": 0.5054097568520425, + "precision@1": 0.2012927054478301, + "precision@5": 0.10193905817174516, "precision@10": 0.09999999999999998, - "recall@1": 0.1994459833795014, - "recall@5": 0.5115420129270545, + "recall@1": 0.2012927054478301, + "recall@5": 0.5096952908587258, "recall@10": 1.0, - "f1@1": 0.1994459833795014, - "f1@5": 0.17051400430901817, + "f1@1": 0.2012927054478301, + "f1@5": 0.16989843028624194, "f1@10": 0.18181818181818185, - "map@1": 0.1994459833795014, - "map@5": 0.3000461680517082, - "map@10": 0.3607542247431444, - "mrr@1": 0.1994459833795014, - "mrr@5": 0.3000461680517082, - "mrr@10": 0.3607542247431444, + "map@1": 0.2012927054478301, + "map@5": 0.3, + "map@10": 0.3609920268507526, + "mrr@1": 0.2012927054478301, + "mrr@5": 0.3, + "mrr@10": 0.3609920268507526, "num_pred": 1083, "num_data": 1083 }, "Charades-STA": { "hit@1": 0.2902338376891334, - "hit@5": 0.6217331499312242, + "hit@5": 0.6231086657496562, "hit@10": 1.0, "ndcg_linear@1": 0.2902338376891334, - "ndcg_linear@5": 0.4587900118805714, - "ndcg_linear@10": 0.5779627989573906, + "ndcg_linear@5": 0.45897181233645623, + "ndcg_linear@10": 0.57779966739102, "ndcg_exponential@1": 0.2902338376891334, - "ndcg_exponential@5": 0.4587900118805714, - "ndcg_exponential@10": 0.5779627989573906, + "ndcg_exponential@5": 0.45897181233645623, + "ndcg_exponential@10": 0.57779966739102, "precision@1": 0.2902338376891334, - "precision@5": 0.12434662998624484, + "precision@5": 0.12462173314993122, "precision@10": 0.09999999999999998, "recall@1": 0.2902338376891334, - "recall@5": 0.6217331499312242, + "recall@5": 0.6231086657496562, "recall@10": 1.0, "f1@1": 0.2902338376891334, - "f1@5": 0.2072443833104081, + "f1@5": 0.2077028885832187, "f1@10": 0.18181818181818185, "map@1": 0.2902338376891334, - "map@5": 0.40529573590096285, - "map@10": 0.4526838933647737, + "map@5": 0.4051352590554791, + "map@10": 0.45243499050239083, "mrr@1": 0.2902338376891334, - "mrr@5": 0.40529573590096285, - "mrr@10": 0.4526838933647737, + "mrr@5": 0.4051352590554791, + "mrr@10": 0.45243499050239083, "num_pred": 727, "num_data": 727 }, "MomentSeeker": { - "hit@1": 0.2761111111111111, - "hit@5": 0.7705555555555555, + "hit@1": 0.2933832709113608, + "hit@5": 0.768414481897628, "hit@10": 1.0, - "ndcg_linear@1": 0.2761111111111111, - "ndcg_linear@5": 0.5263229165531387, - "ndcg_linear@10": 0.6067454069462798, - "ndcg_exponential@1": 0.2761111111111111, - "ndcg_exponential@5": 0.5263229165531387, - "ndcg_exponential@10": 0.6067454069462798, - "precision@1": 0.2761111111111111, - "precision@5": 0.1698888888888889, - "precision@10": 0.11261111111111113, - "recall@1": 0.2555092592592592, - "recall@5": 0.7533333333333333, - "recall@10": 0.997962962962963, - "f1@1": 0.2614717813051146, - "f1@5": 0.2724406303572971, - "f1@10": 0.19986989861989862, - "map@1": 0.2761111111111111, - "map@5": 0.4455154320987654, - "map@10": 0.4805357785861258, - "mrr@1": 0.2761111111111111, - "mrr@5": 0.45693518518518517, - "mrr@10": 0.48703725749559085, + "ndcg_linear@1": 0.2933832709113608, + "ndcg_linear@5": 0.5325610248551454, + "ndcg_linear@10": 0.6143019476531396, + "ndcg_exponential@1": 0.2933832709113608, + "ndcg_exponential@5": 0.5325610248551454, + "ndcg_exponential@10": 0.6143019476531396, + "precision@1": 0.2933832709113608, + "precision@5": 0.17116104868913862, + "precision@10": 0.11404494382022475, + "recall@1": 0.268986683312526, + "recall@5": 0.7487515605493134, + "recall@10": 0.9975031210986267, + "f1@1": 0.2761022927689594, + "f1@5": 0.27332666986225046, + "f1@10": 0.20191039713511624, + "map@1": 0.2933832709113608, + "map@5": 0.45449264807879036, + "map@10": 0.4901928867850372, + "mrr@1": 0.2933832709113608, + "mrr@5": 0.46794631710362056, + "mrr@10": 0.4980735885698432, + "num_pred": 1602, + "num_data": 1602 + }, + "MomentSeeker_1k8": { + "hit@1": 0.27944444444444444, + "hit@5": 0.7716666666666666, + "hit@10": 1.0, + "ndcg_linear@1": 0.27944444444444444, + "ndcg_linear@5": 0.5276215301571869, + "ndcg_linear@10": 0.607670798991691, + "ndcg_exponential@1": 0.27944444444444444, + "ndcg_exponential@5": 0.5276215301571869, + "ndcg_exponential@10": 0.607670798991691, + "precision@1": 0.27944444444444444, + "precision@5": 0.17, + "precision@10": 0.11255555555555559, + "recall@1": 0.2574537037037037, + "recall@5": 0.7541666666666667, + "recall@10": 0.9977777777777778, + "f1@1": 0.26387918871252203, + "f1@5": 0.2726522705689373, + "f1@10": 0.19978442853442852, + "map@1": 0.27944444444444444, + "map@5": 0.4468688271604938, + "map@10": 0.48172140468841856, + "mrr@1": 0.27944444444444444, + "mrr@5": 0.45893518518518517, + "mrr@10": 0.48882782186948853, "num_pred": 1800, "num_data": 1800 } }, "visdoc": { "ViDoRe_arxivqa": { - "hit@1": 0.76, - "hit@5": 0.872, - "hit@10": 0.916, - "ndcg_linear@1": 0.76, - "ndcg_linear@5": 0.8173769173240496, - "ndcg_linear@10": 0.8316391463184296, - "ndcg_exponential@1": 0.76, - "ndcg_exponential@5": 0.8173769173240496, - "ndcg_exponential@10": 0.8316391463184296, - "precision@1": 0.76, - "precision@5": 0.17439999999999997, - "precision@10": 0.0916, - "recall@1": 0.76, - "recall@5": 0.872, - "recall@10": 0.916, - "f1@1": 0.76, - "f1@5": 0.29066666666666674, - "f1@10": 0.16654545454545455, - "map@1": 0.76, - "map@5": 0.7994, - "map@10": 0.8053031746031745, - "mrr@1": 0.76, - "mrr@5": 0.7994, - "mrr@10": 0.8053031746031745, + "hit@1": 0.776, + "hit@5": 0.884, + "hit@10": 0.918, + "ndcg_linear@1": 0.776, + "ndcg_linear@5": 0.8333460501531986, + "ndcg_linear@10": 0.8443483473677361, + "ndcg_exponential@1": 0.776, + "ndcg_exponential@5": 0.8333460501531986, + "ndcg_exponential@10": 0.8443483473677361, + "precision@1": 0.776, + "precision@5": 0.1768, + "precision@10": 0.0918, + "recall@1": 0.776, + "recall@5": 0.884, + "recall@10": 0.918, + "f1@1": 0.776, + "f1@5": 0.29466666666666674, + "f1@10": 0.16690909090909092, + "map@1": 0.776, + "map@5": 0.8164333333333333, + "map@10": 0.8209817460317461, + "mrr@1": 0.776, + "mrr@5": 0.8164333333333333, + "mrr@10": 0.8209817460317461, "num_pred": 500, "num_data": 500 }, "ViDoRe_docvqa": { - "hit@1": 0.4878048780487805, - "hit@5": 0.6385809312638581, - "hit@10": 0.6895787139689579, - "ndcg_linear@1": 0.4878048780487805, - "ndcg_linear@5": 0.5663817406801894, - "ndcg_linear@10": 0.5828545584108937, - "ndcg_exponential@1": 0.4878048780487805, - "ndcg_exponential@5": 0.5663817406801894, - "ndcg_exponential@10": 0.5828545584108937, - "precision@1": 0.4878048780487805, - "precision@5": 0.12860310421286028, - "precision@10": 0.06962305986696231, - "recall@1": 0.4835550628233555, - "recall@5": 0.635439763488544, - "recall@10": 0.685328898743533, - "f1@1": 0.4844505088407527, - "f1@5": 0.21333233958771985, - "f1@10": 0.1261170462944299, - "map@1": 0.4878048780487805, - "map@5": 0.542461197339246, - "map@10": 0.5494527153063739, - "mrr@1": 0.4878048780487805, - "mrr@5": 0.545269770879527, - "mrr@10": 0.5523598352866645, + "hit@1": 0.5055432372505543, + "hit@5": 0.6740576496674058, + "hit@10": 0.7250554323725056, + "ndcg_linear@1": 0.5055432372505543, + "ndcg_linear@5": 0.5920251065887477, + "ndcg_linear@10": 0.6074340777001023, + "ndcg_exponential@1": 0.5055432372505543, + "ndcg_exponential@5": 0.5920251065887477, + "ndcg_exponential@10": 0.6074340777001023, + "precision@1": 0.5055432372505543, + "precision@5": 0.13658536585365852, + "precision@10": 0.07383592017738358, + "recall@1": 0.49951958610495195, + "recall@5": 0.6692165558019217, + "recall@10": 0.7155949741315595, + "f1@1": 0.5007106714423788, + "f1@5": 0.22575447030253348, + "f1@10": 0.1332221363263492, + "map@1": 0.5055432372505543, + "map@5": 0.5649174673564918, + "map@10": 0.5712415971562314, + "mrr@1": 0.5055432372505543, + "mrr@5": 0.5691796008869181, + "mrr@10": 0.5759872241579559, "num_pred": 451, "num_data": 451 }, "ViDoRe_infovqa": { - "hit@1": 0.791497975708502, - "hit@5": 0.8967611336032388, - "hit@10": 0.9311740890688259, - "ndcg_linear@1": 0.791497975708502, - "ndcg_linear@5": 0.8493870405671953, - "ndcg_linear@10": 0.8606301954292797, - "ndcg_exponential@1": 0.791497975708502, - "ndcg_exponential@5": 0.8493870405671953, - "ndcg_exponential@10": 0.8606301954292797, - "precision@1": 0.791497975708502, - "precision@5": 0.17975708502024293, - "precision@10": 0.09331983805668018, - "recall@1": 0.791497975708502, - "recall@5": 0.8967611336032388, - "recall@10": 0.930161943319838, - "f1@1": 0.791497975708502, - "f1@5": 0.299402352033931, - "f1@10": 0.16958041958041958, - "map@1": 0.791497975708502, - "map@5": 0.8334008097165991, - "map@10": 0.838223764539554, - "mrr@1": 0.791497975708502, - "mrr@5": 0.8333333333333333, - "mrr@10": 0.8383008804061435, + "hit@1": 0.7874493927125507, + "hit@5": 0.9109311740890689, + "hit@10": 0.9352226720647774, + "ndcg_linear@1": 0.7874493927125507, + "ndcg_linear@5": 0.8538468442001622, + "ndcg_linear@10": 0.8623147778297474, + "ndcg_exponential@1": 0.7874493927125507, + "ndcg_exponential@5": 0.8538468442001622, + "ndcg_exponential@10": 0.8623147778297474, + "precision@1": 0.7874493927125507, + "precision@5": 0.18218623481781376, + "precision@10": 0.0937246963562753, + "recall@1": 0.7874493927125507, + "recall@5": 0.9089068825910931, + "recall@10": 0.9342105263157895, + "f1@1": 0.7874493927125507, + "f1@5": 0.30345093502988246, + "f1@10": 0.1703165255796835, + "map@1": 0.7874493927125507, + "map@5": 0.8350877192982457, + "map@10": 0.8388366750208854, + "mrr@1": 0.7874493927125507, + "mrr@5": 0.8356275303643724, + "mrr@10": 0.8390391041706831, "num_pred": 494, "num_data": 494 }, "ViDoRe_tabfquad": { - "hit@1": 0.8178571428571428, - "hit@5": 0.9107142857142857, + "hit@1": 0.8, + "hit@5": 0.9214285714285714, "hit@10": 0.9535714285714286, - "ndcg_linear@1": 0.8178571428571428, - "ndcg_linear@5": 0.8693088177463265, - "ndcg_linear@10": 0.8831829118752064, - "ndcg_exponential@1": 0.8178571428571428, - "ndcg_exponential@5": 0.8693088177463265, - "ndcg_exponential@10": 0.8831829118752064, - "precision@1": 0.8178571428571428, - "precision@5": 0.18214285714285713, + "ndcg_linear@1": 0.8, + "ndcg_linear@5": 0.8662768913699691, + "ndcg_linear@10": 0.8764161722576012, + "ndcg_exponential@1": 0.8, + "ndcg_exponential@5": 0.8662768913699691, + "ndcg_exponential@10": 0.8764161722576012, + "precision@1": 0.8, + "precision@5": 0.18428571428571427, "precision@10": 0.09535714285714285, - "recall@1": 0.8178571428571428, - "recall@5": 0.9107142857142857, + "recall@1": 0.8, + "recall@5": 0.9214285714285714, "recall@10": 0.9535714285714286, - "f1@1": 0.8178571428571428, - "f1@5": 0.30357142857142866, + "f1@1": 0.8, + "f1@5": 0.3071428571428572, "f1@10": 0.17337662337662343, - "map@1": 0.8178571428571428, - "map@5": 0.8552976190476191, - "map@10": 0.8610359977324263, - "mrr@1": 0.8178571428571428, - "mrr@5": 0.8552976190476191, - "mrr@10": 0.8610359977324263, + "map@1": 0.8, + "map@5": 0.847797619047619, + "map@10": 0.8518353174603175, + "mrr@1": 0.8, + "mrr@5": 0.847797619047619, + "mrr@10": 0.8518353174603175, "num_pred": 280, "num_data": 280 }, "ViDoRe_tatdqa": { - "hit@1": 0.5771567436208992, - "hit@5": 0.8207776427703524, - "hit@10": 0.8845686512758202, - "ndcg_linear@1": 0.5771567436208992, - "ndcg_linear@5": 0.7086500725869258, - "ndcg_linear@10": 0.729683893384968, - "ndcg_exponential@1": 0.5771567436208992, - "ndcg_exponential@5": 0.7086500725869258, - "ndcg_exponential@10": 0.729683893384968, - "precision@1": 0.5771567436208992, - "precision@5": 0.1645200486026731, - "precision@10": 0.08876063183475091, - "recall@1": 0.5762454434993924, - "recall@5": 0.8198663426488457, - "recall@10": 0.8842648845686513, - "f1@1": 0.5765492102065614, - "f1@5": 0.2739397095411677, - "f1@10": 0.16128171140321815, - "map@1": 0.5771567436208992, - "map@5": 0.6712991089509924, - "map@10": 0.6801273891492604, - "mrr@1": 0.5771567436208992, - "mrr@5": 0.6716281895504252, - "mrr@10": 0.6803197747304673, + "hit@1": 0.5662211421628189, + "hit@5": 0.8238153098420413, + "hit@10": 0.8882138517618469, + "ndcg_linear@1": 0.5662211421628189, + "ndcg_linear@5": 0.7046747305562437, + "ndcg_linear@10": 0.7260668062891217, + "ndcg_exponential@1": 0.5662211421628189, + "ndcg_exponential@5": 0.7046747305562437, + "ndcg_exponential@10": 0.7260668062891217, + "precision@1": 0.5662211421628189, + "precision@5": 0.16512758201701092, + "precision@10": 0.08912515188335358, + "recall@1": 0.5653098420413123, + "recall@5": 0.8229040097205347, + "recall@10": 0.887910085054678, + "f1@1": 0.5656136087484812, + "f1@5": 0.2749522652317307, + "f1@10": 0.16194447512795027, + "map@1": 0.5662211421628189, + "map@5": 0.6650313892264076, + "map@10": 0.6741077455688635, + "mrr@1": 0.5662211421628189, + "mrr@5": 0.6653604698258404, + "mrr@10": 0.6742825319678297, "num_pred": 1646, "num_data": 1646 }, "ViDoRe_shiftproject": { - "hit@1": 0.61, - "hit@5": 0.88, - "hit@10": 0.95, - "ndcg_linear@1": 0.61, - "ndcg_linear@5": 0.7511606311644851, - "ndcg_linear@10": 0.774534113357672, - "ndcg_exponential@1": 0.61, - "ndcg_exponential@5": 0.7511606311644851, - "ndcg_exponential@10": 0.774534113357672, - "precision@1": 0.61, - "precision@5": 0.176, - "precision@10": 0.09499999999999999, - "recall@1": 0.61, - "recall@5": 0.88, - "recall@10": 0.95, - "f1@1": 0.61, - "f1@5": 0.2933333333333334, - "f1@10": 0.1727272727272728, - "map@1": 0.61, - "map@5": 0.7083333333333333, - "map@10": 0.7184126984126985, - "mrr@1": 0.61, - "mrr@5": 0.7083333333333333, - "mrr@10": 0.7184126984126985, + "hit@1": 0.62, + "hit@5": 0.9, + "hit@10": 0.96, + "ndcg_linear@1": 0.62, + "ndcg_linear@5": 0.7683175057251009, + "ndcg_linear@10": 0.7873731027127768, + "ndcg_exponential@1": 0.62, + "ndcg_exponential@5": 0.7683175057251009, + "ndcg_exponential@10": 0.7873731027127768, + "precision@1": 0.62, + "precision@5": 0.17999999999999997, + "precision@10": 0.09599999999999997, + "recall@1": 0.62, + "recall@5": 0.9, + "recall@10": 0.96, + "f1@1": 0.62, + "f1@5": 0.30000000000000004, + "f1@10": 0.17454545454545464, + "map@1": 0.62, + "map@5": 0.7241666666666667, + "map@10": 0.7318134920634921, + "mrr@1": 0.62, + "mrr@5": 0.7241666666666667, + "mrr@10": 0.7318134920634921, "num_pred": 100, "num_data": 100 }, "ViDoRe_syntheticDocQA_artificial_intelligence": { - "hit@1": 0.92, - "hit@5": 0.98, - "hit@10": 0.99, - "ndcg_linear@1": 0.92, - "ndcg_linear@5": 0.9565464876785729, - "ndcg_linear@10": 0.9601085595496531, - "ndcg_exponential@1": 0.92, - "ndcg_exponential@5": 0.9565464876785729, - "ndcg_exponential@10": 0.9601085595496531, - "precision@1": 0.92, - "precision@5": 0.19599999999999995, - "precision@10": 0.09899999999999999, - "recall@1": 0.92, - "recall@5": 0.98, - "recall@10": 0.99, - "f1@1": 0.92, - "f1@5": 0.32666666666666677, - "f1@10": 0.18000000000000008, - "map@1": 0.92, - "map@5": 0.9483333333333333, - "map@10": 0.95, - "mrr@1": 0.92, - "mrr@5": 0.9483333333333333, - "mrr@10": 0.95, + "hit@1": 0.95, + "hit@5": 0.97, + "hit@10": 1.0, + "ndcg_linear@1": 0.95, + "ndcg_linear@5": 0.9626185950714291, + "ndcg_linear@10": 0.972081615162328, + "ndcg_exponential@1": 0.95, + "ndcg_exponential@5": 0.9626185950714291, + "ndcg_exponential@10": 0.972081615162328, + "precision@1": 0.95, + "precision@5": 0.19399999999999995, + "precision@10": 0.09999999999999998, + "recall@1": 0.95, + "recall@5": 0.97, + "recall@10": 1.0, + "f1@1": 0.95, + "f1@5": 0.3233333333333334, + "f1@10": 0.1818181818181819, + "map@1": 0.95, + "map@5": 0.96, + "map@10": 0.9637777777777778, + "mrr@1": 0.95, + "mrr@5": 0.96, + "mrr@10": 0.9637777777777778, "num_pred": 100, "num_data": 100 }, "ViDoRe_syntheticDocQA_energy": { - "hit@1": 0.92, - "hit@5": 0.97, + "hit@1": 0.94, + "hit@5": 0.96, "hit@10": 0.97, - "ndcg_linear@1": 0.92, - "ndcg_linear@5": 0.9466649487518345, - "ndcg_linear@10": 0.9466649487518345, - "ndcg_exponential@1": 0.92, - "ndcg_exponential@5": 0.9466649487518345, - "ndcg_exponential@10": 0.9466649487518345, - "precision@1": 0.92, - "precision@5": 0.19399999999999995, + "ndcg_linear@1": 0.94, + "ndcg_linear@5": 0.95017782560806, + "ndcg_linear@10": 0.9533324743759173, + "ndcg_exponential@1": 0.94, + "ndcg_exponential@5": 0.95017782560806, + "ndcg_exponential@10": 0.9533324743759173, + "precision@1": 0.94, + "precision@5": 0.19199999999999995, "precision@10": 0.09699999999999998, - "recall@1": 0.92, - "recall@5": 0.97, + "recall@1": 0.94, + "recall@5": 0.96, "recall@10": 0.97, - "f1@1": 0.92, - "f1@5": 0.3233333333333334, + "f1@1": 0.94, + "f1@5": 0.32000000000000006, "f1@10": 0.17636363636363647, - "map@1": 0.92, - "map@5": 0.9390000000000001, - "map@10": 0.9390000000000001, - "mrr@1": 0.92, - "mrr@5": 0.9390000000000001, - "mrr@10": 0.9390000000000001, + "map@1": 0.94, + "map@5": 0.9470000000000001, + "map@10": 0.94825, + "mrr@1": 0.94, + "mrr@5": 0.9470000000000001, + "mrr@10": 0.94825, "num_pred": 100, "num_data": 100 }, "ViDoRe_syntheticDocQA_government_reports": { - "hit@1": 0.86, - "hit@5": 0.99, + "hit@1": 0.9, + "hit@5": 1.0, "hit@10": 1.0, - "ndcg_linear@1": 0.86, - "ndcg_linear@5": 0.9354743802857166, - "ndcg_linear@10": 0.9388077136190499, - "ndcg_exponential@1": 0.86, - "ndcg_exponential@5": 0.9354743802857166, - "ndcg_exponential@10": 0.9388077136190499, - "precision@1": 0.86, - "precision@5": 0.19799999999999998, + "ndcg_linear@1": 0.9, + "ndcg_linear@5": 0.9558532532593068, + "ndcg_linear@10": 0.9558532532593068, + "ndcg_exponential@1": 0.9, + "ndcg_exponential@5": 0.9558532532593068, + "ndcg_exponential@10": 0.9558532532593068, + "precision@1": 0.9, + "precision@5": 0.19999999999999996, "precision@10": 0.09999999999999998, - "recall@1": 0.86, - "recall@5": 0.99, + "recall@1": 0.9, + "recall@5": 1.0, "recall@10": 1.0, - "f1@1": 0.86, - "f1@5": 0.3300000000000001, + "f1@1": 0.9, + "f1@5": 0.3333333333333335, "f1@10": 0.1818181818181819, - "map@1": 0.86, - "map@5": 0.9166666666666665, - "map@10": 0.918095238095238, - "mrr@1": 0.86, - "mrr@5": 0.9166666666666665, - "mrr@10": 0.918095238095238, + "map@1": 0.9, + "map@5": 0.9408333333333333, + "map@10": 0.9408333333333333, + "mrr@1": 0.9, + "mrr@5": 0.9408333333333333, + "mrr@10": 0.9408333333333333, "num_pred": 100, "num_data": 100 }, "ViDoRe_syntheticDocQA_healthcare_industry": { - "hit@1": 0.91, + "hit@1": 0.94, "hit@5": 0.99, "hit@10": 1.0, - "ndcg_linear@1": 0.91, - "ndcg_linear@5": 0.9591650827500021, - "ndcg_linear@10": 0.9621753827066419, - "ndcg_exponential@1": 0.91, - "ndcg_exponential@5": 0.9591650827500021, - "ndcg_exponential@10": 0.9621753827066419, - "precision@1": 0.91, + "ndcg_linear@1": 0.94, + "ndcg_linear@5": 0.9702371901428584, + "ndcg_linear@10": 0.9731278384060371, + "ndcg_exponential@1": 0.94, + "ndcg_exponential@5": 0.9702371901428584, + "ndcg_exponential@10": 0.9731278384060371, + "precision@1": 0.94, "precision@5": 0.19799999999999998, "precision@10": 0.09999999999999998, - "recall@1": 0.91, + "recall@1": 0.94, "recall@5": 0.99, "recall@10": 1.0, - "f1@1": 0.91, + "f1@1": 0.94, "f1@5": 0.3300000000000001, "f1@10": 0.1818181818181819, - "map@1": 0.91, - "map@5": 0.9483333333333335, - "map@10": 0.9494444444444445, - "mrr@1": 0.91, - "mrr@5": 0.9483333333333335, - "mrr@10": 0.9494444444444445, + "map@1": 0.94, + "map@5": 0.9633333333333334, + "map@10": 0.9643333333333334, + "mrr@1": 0.94, + "mrr@5": 0.9633333333333334, + "mrr@10": 0.9643333333333334, "num_pred": 100, "num_data": 100 }, "ViDoRe_esg_reports_human_labeled_v2": { - "hit@1": 0.4230769230769231, - "hit@5": 0.75, + "hit@1": 0.4807692307692308, + "hit@5": 0.8076923076923077, "hit@10": 0.8653846153846154, - "ndcg_linear@1": 0.4102564102564103, - "ndcg_linear@5": 0.5129471948247265, - "ndcg_linear@10": 0.562683866445629, - "ndcg_exponential@1": 0.4065934065934066, - "ndcg_exponential@5": 0.5103409095742849, - "ndcg_exponential@10": 0.5610704127395229, - "precision@1": 0.4230769230769231, - "precision@5": 0.20769230769230768, - "precision@10": 0.1423076923076923, - "recall@1": 0.3169871794871795, - "recall@5": 0.5976190476190476, - "recall@10": 0.7128746253746254, - "f1@1": 0.3403846153846154, - "f1@5": 0.2807916756446169, - "f1@10": 0.21769245083724723, - "map@1": 0.4230769230769231, - "map@5": 0.45147435897435895, - "map@10": 0.4801182844932845, - "mrr@1": 0.4230769230769231, - "mrr@5": 0.5532051282051282, - "mrr@10": 0.5671474358974359, + "ndcg_linear@1": 0.46794871794871795, + "ndcg_linear@5": 0.5724593187964453, + "ndcg_linear@10": 0.6153373244446659, + "ndcg_exponential@1": 0.4642857142857143, + "ndcg_exponential@5": 0.5714500037722439, + "ndcg_exponential@10": 0.6137359218498117, + "precision@1": 0.4807692307692308, + "precision@5": 0.23076923076923073, + "precision@10": 0.1519230769230769, + "recall@1": 0.33685897435897433, + "recall@5": 0.6495421245421246, + "recall@10": 0.7500083250083249, + "f1@1": 0.3677021696252466, + "f1@5": 0.3105841054370467, + "f1@10": 0.23182502828996046, + "map@1": 0.4807692307692308, + "map@5": 0.506201923076923, + "map@10": 0.5375249287749287, + "mrr@1": 0.4807692307692308, + "mrr@5": 0.6176282051282052, + "mrr@10": 0.6259615384615385, "num_pred": 52, "num_data": 52 }, "ViDoRe_biomedical_lectures_v2_multilingual": { - "hit@1": 0.5171875, - "hit@5": 0.7984375, - "hit@10": 0.878125, - "ndcg_linear@1": 0.5171875, - "ndcg_linear@5": 0.5472043616227252, - "ndcg_linear@10": 0.5826984113580178, - "ndcg_exponential@1": 0.5171875, - "ndcg_exponential@5": 0.5472043616227253, - "ndcg_exponential@10": 0.5826984113580178, - "precision@1": 0.5171875, - "precision@5": 0.24874999999999997, - "precision@10": 0.16234375, - "recall@1": 0.3132504734848485, - "recall@5": 0.5801439155207537, - "recall@10": 0.6875872763878279, - "f1@1": 0.35874357413419916, - "f1@5": 0.3077863438644689, - "f1@10": 0.23607504359066858, - "map@1": 0.5171875, - "map@5": 0.4755620659722222, - "map@10": 0.49229102595899477, - "mrr@1": 0.5171875, - "mrr@5": 0.6226822916666667, - "mrr@10": 0.6335261656746032, + "hit@1": 0.521875, + "hit@5": 0.834375, + "hit@10": 0.9078125, + "ndcg_linear@1": 0.521875, + "ndcg_linear@5": 0.5624564804075678, + "ndcg_linear@10": 0.6008313149897274, + "ndcg_exponential@1": 0.521875, + "ndcg_exponential@5": 0.5624564804075678, + "ndcg_exponential@10": 0.6008313149897274, + "precision@1": 0.521875, + "precision@5": 0.2596875, + "precision@10": 0.17031250000000003, + "recall@1": 0.31464488636363636, + "recall@5": 0.5936782312356761, + "recall@10": 0.7096702715176979, + "f1@1": 0.3613783482142857, + "f1@5": 0.31864331675269175, + "f1@10": 0.2468319649781047, + "map@1": 0.521875, + "map@5": 0.4865646701388888, + "map@10": 0.5060831259152966, + "mrr@1": 0.521875, + "mrr@5": 0.6422135416666667, + "mrr@10": 0.652063492063492, "num_pred": 640, "num_data": 640 }, "ViDoRe_economics_reports_v2_multilingual": { - "hit@1": 0.5086206896551724, - "hit@5": 0.8577586206896551, - "hit@10": 0.9353448275862069, - "ndcg_linear@1": 0.5086206896551724, - "ndcg_linear@5": 0.48968646101987606, - "ndcg_linear@10": 0.4855016494653571, - "ndcg_exponential@1": 0.5086206896551724, - "ndcg_exponential@5": 0.48968646101987606, - "ndcg_exponential@10": 0.4855016494653571, - "precision@1": 0.5086206896551724, - "precision@5": 0.4456896551724138, - "precision@10": 0.3732758620689655, - "recall@1": 0.049608559736213115, - "recall@5": 0.2650305856260614, - "recall@10": 0.4040109375386366, - "f1@1": 0.0867748494784438, - "f1@5": 0.2676800330504101, - "f1@10": 0.31406186428354066, - "map@1": 0.5086206896551724, - "map@5": 0.38345545977011497, - "map@10": 0.3455301240866717, - "mrr@1": 0.5086206896551724, - "mrr@5": 0.6462643678160919, - "mrr@10": 0.6564638067870827, + "hit@1": 0.5603448275862069, + "hit@5": 0.8663793103448276, + "hit@10": 0.9310344827586207, + "ndcg_linear@1": 0.5603448275862069, + "ndcg_linear@5": 0.50197510377362, + "ndcg_linear@10": 0.4970396850788216, + "ndcg_exponential@1": 0.5603448275862069, + "ndcg_exponential@5": 0.50197510377362, + "ndcg_exponential@10": 0.4970396850788216, + "precision@1": 0.5603448275862069, + "precision@5": 0.45, + "precision@10": 0.37887931034482764, + "recall@1": 0.06318503146234863, + "recall@5": 0.2673293507532089, + "recall@10": 0.4025573138717478, + "f1@1": 0.10566340746705108, + "f1@5": 0.2706881004107509, + "f1@10": 0.3174565349524109, + "map@1": 0.5603448275862069, + "map@5": 0.39307112068965516, + "map@10": 0.35709007944652954, + "mrr@1": 0.5603448275862069, + "mrr@5": 0.6778735632183908, + "mrr@10": 0.6865643814997262, "num_pred": 232, "num_data": 232 }, "ViDoRe_esg_reports_v2_multilingual": { - "hit@1": 0.49122807017543857, + "hit@1": 0.543859649122807, "hit@5": 0.8070175438596491, - "hit@10": 0.8728070175438597, - "ndcg_linear@1": 0.49122807017543857, - "ndcg_linear@5": 0.5294466383615495, - "ndcg_linear@10": 0.5608887645448074, - "ndcg_exponential@1": 0.49122807017543857, - "ndcg_exponential@5": 0.5294466383615495, - "ndcg_exponential@10": 0.5608887645448074, - "precision@1": 0.49122807017543857, - "precision@5": 0.26754385964912286, - "precision@10": 0.17631578947368426, - "recall@1": 0.25065267335004177, - "recall@5": 0.5506109022556391, - "recall@10": 0.6676691729323309, - "f1@1": 0.29780297238554954, - "f1@5": 0.3099469090154475, - "f1@10": 0.2428282645000911, - "map@1": 0.49122807017543857, - "map@5": 0.4436610623781677, - "map@10": 0.4549566044741483, - "mrr@1": 0.49122807017543857, - "mrr@5": 0.6209795321637426, - "mrr@10": 0.6295652325257587, + "hit@10": 0.8596491228070176, + "ndcg_linear@1": 0.543859649122807, + "ndcg_linear@5": 0.5555717745758091, + "ndcg_linear@10": 0.5846151532955132, + "ndcg_exponential@1": 0.543859649122807, + "ndcg_exponential@5": 0.5555717745758092, + "ndcg_exponential@10": 0.5846151532955132, + "precision@1": 0.543859649122807, + "precision@5": 0.2789473684210527, + "precision@10": 0.1846491228070176, + "recall@1": 0.2800125313283208, + "recall@5": 0.5596595655806181, + "recall@10": 0.6738304093567252, + "f1@1": 0.33267210439624234, + "f1@5": 0.3177002789645578, + "f1@10": 0.24986479718026078, + "map@1": 0.543859649122807, + "map@5": 0.4741496101364523, + "map@10": 0.48322973231690336, + "mrr@1": 0.543859649122807, + "mrr@5": 0.6522660818713449, + "mrr@10": 0.6592087858535226, "num_pred": 228, "num_data": 228 }, "VisRAG_ArxivQA": { "hit@1": 0.7561274509803921, - "hit@5": 0.8553921568627451, - "hit@10": 0.8848039215686274, + "hit@5": 0.8566176470588235, + "hit@10": 0.8860294117647058, "ndcg_linear@1": 0.7561274509803921, - "ndcg_linear@5": 0.8087241849861875, - "ndcg_linear@10": 0.8184673504656934, + "ndcg_linear@5": 0.8090276953102516, + "ndcg_linear@10": 0.8185106453892687, "ndcg_exponential@1": 0.7561274509803921, - "ndcg_exponential@5": 0.8087241849861875, - "ndcg_exponential@10": 0.8184673504656934, + "ndcg_exponential@5": 0.8090276953102516, + "ndcg_exponential@10": 0.8185106453892687, "precision@1": 0.7561274509803921, - "precision@5": 0.17107843137254902, - "precision@10": 0.08848039215686274, + "precision@5": 0.1713235294117647, + "precision@10": 0.08860294117647056, "recall@1": 0.7561274509803921, - "recall@5": 0.8553921568627451, - "recall@10": 0.8848039215686274, + "recall@5": 0.8566176470588235, + "recall@10": 0.8860294117647058, "f1@1": 0.7561274509803921, - "f1@5": 0.28513071895424846, - "f1@10": 0.16087344028520503, + "f1@5": 0.2855392156862746, + "f1@10": 0.16109625668449204, "map@1": 0.7561274509803921, - "map@5": 0.7931985294117647, - "map@10": 0.7973593604108311, + "map@5": 0.7932598039215686, + "map@10": 0.7971599751011516, "mrr@1": 0.7561274509803921, - "mrr@5": 0.7931985294117647, - "mrr@10": 0.7973593604108311, + "mrr@5": 0.7932598039215686, + "mrr@10": 0.7971599751011516, "num_pred": 816, "num_data": 816 }, "VisRAG_ChartQA": { - "hit@1": 0.6825396825396826, + "hit@1": 0.6666666666666666, "hit@5": 0.8571428571428571, "hit@10": 0.9047619047619048, - "ndcg_linear@1": 0.6825396825396826, - "ndcg_linear@5": 0.781492660957272, - "ndcg_linear@10": 0.7972160024827536, - "ndcg_exponential@1": 0.6825396825396826, - "ndcg_exponential@5": 0.781492660957272, - "ndcg_exponential@10": 0.7972160024827536, - "precision@1": 0.6825396825396826, + "ndcg_linear@1": 0.6666666666666666, + "ndcg_linear@5": 0.7703775308700012, + "ndcg_linear@10": 0.7855478720284862, + "ndcg_exponential@1": 0.6666666666666666, + "ndcg_exponential@5": 0.7703775308700012, + "ndcg_exponential@10": 0.7855478720284862, + "precision@1": 0.6666666666666666, "precision@5": 0.17142857142857135, "precision@10": 0.09047619047619043, - "recall@1": 0.6825396825396826, + "recall@1": 0.6666666666666666, "recall@5": 0.8571428571428571, "recall@10": 0.9047619047619048, - "f1@1": 0.6825396825396826, + "f1@1": 0.6666666666666666, "f1@5": 0.28571428571428564, "f1@10": 0.16450216450216454, - "map@1": 0.6825396825396826, - "map@5": 0.7558201058201058, - "map@10": 0.7624968505920886, - "mrr@1": 0.6825396825396826, - "mrr@5": 0.7558201058201058, - "mrr@10": 0.7624968505920886, + "map@1": 0.6666666666666666, + "map@5": 0.7412698412698413, + "map@10": 0.7473922902494332, + "mrr@1": 0.6666666666666666, + "mrr@5": 0.7412698412698413, + "mrr@10": 0.7473922902494332, "num_pred": 63, "num_data": 63 }, "VisRAG_MP-DocVQA": { - "hit@1": 0.7918781725888325, - "hit@5": 0.9289340101522843, - "hit@10": 0.9509306260575296, - "ndcg_linear@1": 0.7918781725888325, - "ndcg_linear@5": 0.8676285427363506, - "ndcg_linear@10": 0.8748306491817542, - "ndcg_exponential@1": 0.7918781725888325, - "ndcg_exponential@5": 0.8676285427363506, - "ndcg_exponential@10": 0.8748306491817542, - "precision@1": 0.7918781725888325, - "precision@5": 0.18578680203045683, - "precision@10": 0.09509306260575294, - "recall@1": 0.7918781725888325, - "recall@5": 0.9289340101522843, - "recall@10": 0.9509306260575296, - "f1@1": 0.7918781725888325, - "f1@5": 0.3096446700507615, - "f1@10": 0.17289647746500544, - "map@1": 0.7918781725888325, - "map@5": 0.8469543147208122, - "map@10": 0.8499758278946096, - "mrr@1": 0.7918781725888325, - "mrr@5": 0.8469543147208122, - "mrr@10": 0.8499758278946096, + "hit@1": 0.7969543147208121, + "hit@5": 0.9306260575296108, + "hit@10": 0.9543147208121827, + "ndcg_linear@1": 0.7969543147208121, + "ndcg_linear@5": 0.8702738675898986, + "ndcg_linear@10": 0.8780499784693713, + "ndcg_exponential@1": 0.7969543147208121, + "ndcg_exponential@5": 0.8702738675898986, + "ndcg_exponential@10": 0.8780499784693713, + "precision@1": 0.7969543147208121, + "precision@5": 0.18612521150592212, + "precision@10": 0.09543147208121826, + "recall@1": 0.7969543147208121, + "recall@5": 0.9306260575296108, + "recall@10": 0.9543147208121827, + "f1@1": 0.7969543147208121, + "f1@5": 0.31020868584320366, + "f1@10": 0.1735117674203969, + "map@1": 0.7969543147208121, + "map@5": 0.8499717992103779, + "map@10": 0.8532464480434023, + "mrr@1": 0.7969543147208121, + "mrr@5": 0.8499717992103779, + "mrr@10": 0.8532464480434023, "num_pred": 591, "num_data": 591 }, "VisRAG_SlideVQA": { - "hit@1": 0.9388489208633094, + "hit@1": 0.9406474820143885, "hit@5": 0.9892086330935251, "hit@10": 0.9892086330935251, - "ndcg_linear@1": 0.9388489208633094, - "ndcg_linear@5": 0.9502736353073175, - "ndcg_linear@10": 0.9527678727475414, - "ndcg_exponential@1": 0.9388489208633094, - "ndcg_exponential@5": 0.9502736353073175, - "ndcg_exponential@10": 0.9527678727475414, - "precision@1": 0.9388489208633094, + "ndcg_linear@1": 0.9406474820143885, + "ndcg_linear@5": 0.9512919764736166, + "ndcg_linear@10": 0.9540917946274893, + "ndcg_exponential@1": 0.9406474820143885, + "ndcg_exponential@5": 0.9512919764736166, + "ndcg_exponential@10": 0.9540917946274893, + "precision@1": 0.9406474820143885, "precision@5": 0.24388489208633093, - "precision@10": 0.12320143884892086, - "recall@1": 0.8120503597122302, + "precision@10": 0.12338129496402876, + "recall@1": 0.814748201438849, "recall@5": 0.9739208633093526, - "recall@10": 0.9802158273381295, - "f1@1": 0.85431654676259, + "recall@10": 0.9811151079136691, + "f1@1": 0.8567146282973621, "f1@5": 0.38309352517985623, - "f1@10": 0.2163723566601265, - "map@1": 0.9388489208633094, - "map@5": 0.9327038369304556, - "map@10": 0.9343789254310837, - "mrr@1": 0.9388489208633094, - "mrr@5": 0.9612410071942447, - "mrr@10": 0.9612410071942447, + "f1@10": 0.216672116851973, + "map@1": 0.9406474820143885, + "map@5": 0.9344424460431654, + "map@10": 0.9362774066461115, + "mrr@1": 0.9406474820143885, + "mrr@5": 0.9617805755395683, + "mrr@10": 0.9617805755395683, "num_pred": 556, "num_data": 556 }, "VisRAG_InfoVQA": { - "hit@1": 0.7729805013927576, - "hit@5": 0.924791086350975, - "hit@10": 0.9512534818941504, - "ndcg_linear@1": 0.7729805013927576, - "ndcg_linear@5": 0.8569356912112719, - "ndcg_linear@10": 0.8656693780762971, - "ndcg_exponential@1": 0.7729805013927576, - "ndcg_exponential@5": 0.8569356912112719, - "ndcg_exponential@10": 0.8656693780762971, - "precision@1": 0.7729805013927576, - "precision@5": 0.18495821727019496, - "precision@10": 0.09512534818941502, - "recall@1": 0.7729805013927576, - "recall@5": 0.924791086350975, - "recall@10": 0.9512534818941504, - "f1@1": 0.7729805013927576, - "f1@5": 0.308263695450325, - "f1@10": 0.1729551785262092, - "map@1": 0.7729805013927576, - "map@5": 0.8340529247910862, - "map@10": 0.8377569969491975, - "mrr@1": 0.7729805013927576, - "mrr@5": 0.8340529247910862, - "mrr@10": 0.8377569969491975, + "hit@1": 0.775766016713092, + "hit@5": 0.9233983286908078, + "hit@10": 0.9526462395543176, + "ndcg_linear@1": 0.775766016713092, + "ndcg_linear@5": 0.8572896135683271, + "ndcg_linear@10": 0.8669656191518114, + "ndcg_exponential@1": 0.775766016713092, + "ndcg_exponential@5": 0.8572896135683271, + "ndcg_exponential@10": 0.8669656191518114, + "precision@1": 0.775766016713092, + "precision@5": 0.18467966573816152, + "precision@10": 0.09526462395543174, + "recall@1": 0.775766016713092, + "recall@5": 0.9233983286908078, + "recall@10": 0.9526462395543176, + "f1@1": 0.775766016713092, + "f1@5": 0.307799442896936, + "f1@10": 0.17320840719169414, + "map@1": 0.775766016713092, + "map@5": 0.834958217270195, + "map@10": 0.8390806694079677, + "mrr@1": 0.775766016713092, + "mrr@5": 0.834958217270195, + "mrr@10": 0.8390806694079677, "num_pred": 718, "num_data": 718 }, "VisRAG_PlotQA": { - "hit@1": 0.46349942062572425, - "hit@5": 0.7265353418308227, - "hit@10": 0.831981460023175, - "ndcg_linear@1": 0.46349942062572425, - "ndcg_linear@5": 0.6029827126488958, - "ndcg_linear@10": 0.6373604951260508, - "ndcg_exponential@1": 0.46349942062572425, - "ndcg_exponential@5": 0.6029827126488958, - "ndcg_exponential@10": 0.6373604951260508, - "precision@1": 0.46349942062572425, - "precision@5": 0.14530706836616453, - "precision@10": 0.08319814600231748, - "recall@1": 0.46349942062572425, - "recall@5": 0.7265353418308227, - "recall@10": 0.831981460023175, - "f1@1": 0.46349942062572425, - "f1@5": 0.24217844727694096, - "f1@10": 0.15126935636785002, - "map@1": 0.46349942062572425, - "map@5": 0.5619351100811124, - "map@10": 0.5762865787489194, - "mrr@1": 0.46349942062572425, - "mrr@5": 0.5619351100811124, - "mrr@10": 0.5762865787489194, + "hit@1": 0.4646581691772885, + "hit@5": 0.7242178447276941, + "hit@10": 0.8273464658169177, + "ndcg_linear@1": 0.4646581691772885, + "ndcg_linear@5": 0.6017663045326518, + "ndcg_linear@10": 0.6357072474940606, + "ndcg_exponential@1": 0.4646581691772885, + "ndcg_exponential@5": 0.6017663045326518, + "ndcg_exponential@10": 0.6357072474940606, + "precision@1": 0.4646581691772885, + "precision@5": 0.1448435689455388, + "precision@10": 0.08273464658169176, + "recall@1": 0.4646581691772885, + "recall@5": 0.7242178447276941, + "recall@10": 0.8273464658169177, + "f1@1": 0.4646581691772885, + "f1@5": 0.24140594824256473, + "f1@10": 0.15042663014853053, + "map@1": 0.4646581691772885, + "map@5": 0.5611432985708767, + "map@10": 0.5754947672386839, + "mrr@1": 0.4646581691772885, + "mrr@5": 0.5611432985708767, + "mrr@10": 0.5754947672386839, "num_pred": 863, "num_data": 863 }, "ViDoSeek-page": { - "hit@1": 0.023642732049036778, - "hit@5": 0.4141856392294221, - "hit@10": 0.6348511383537653, - "ndcg_linear@1": 0.023642732049036778, - "ndcg_linear@5": 0.2215998227373851, - "ndcg_linear@10": 0.29328787953138075, - "ndcg_exponential@1": 0.023642732049036778, - "ndcg_exponential@5": 0.2215998227373851, - "ndcg_exponential@10": 0.29328787953138075, - "precision@1": 0.023642732049036778, - "precision@5": 0.08283712784588441, - "precision@10": 0.06348511383537653, - "recall@1": 0.023642732049036778, - "recall@5": 0.4141856392294221, - "recall@10": 0.6348511383537653, - "f1@1": 0.023642732049036778, - "f1@5": 0.13806187974314071, - "f1@10": 0.11542747970068462, - "map@1": 0.023642732049036778, - "map@5": 0.15847927612375948, - "map@10": 0.1882595974758847, - "mrr@1": 0.023642732049036778, - "mrr@5": 0.15847927612375948, - "mrr@10": 0.1882595974758847, - "num_pred": 1142, - "num_data": 1142 - }, - "ViDoSeek-doc": { - "hit@1": 0.9973730297723292, - "hit@5": 0.999124343257443, - "hit@10": 0.999124343257443, - "ndcg_linear@1": 0.680385288966725, - "ndcg_linear@5": 0.8366080322665971, - "ndcg_linear@10": 0.8228477305778173, - "ndcg_exponential@1": 0.4539654741055791, - "ndcg_exponential@5": 0.7156185424033248, - "ndcg_exponential@10": 0.7459741664587741, - "precision@1": 0.9973730297723292, - "precision@5": 0.9252189141856392, - "precision@10": 0.8394045534150613, - "recall@1": 0.05880569019029663, - "recall@5": 0.272128058121516, - "recall@10": 0.48450076291066557, - "f1@1": 0.11031252785994464, - "f1@5": 0.4120671799650714, - "f1@10": 0.6005622170825162, - "map@1": 0.9973730297723292, - "map@5": 0.9193753648569761, - "map@10": 0.8319699422384248, - "mrr@1": 0.9973730297723292, - "mrr@5": 0.998029772329247, - "mrr@10": 0.998029772329247, + "hit@1": 0.6637478108581436, + "hit@5": 0.9001751313485113, + "hit@10": 0.9623467600700525, + "ndcg_linear@1": 0.6637478108581436, + "ndcg_linear@5": 0.7921833562565235, + "ndcg_linear@10": 0.8124634134168822, + "ndcg_exponential@1": 0.6637478108581436, + "ndcg_exponential@5": 0.7921833562565235, + "ndcg_exponential@10": 0.8124634134168823, + "precision@1": 0.6637478108581436, + "precision@5": 0.18003502626970225, + "precision@10": 0.09623467600700523, + "recall@1": 0.6637478108581436, + "recall@5": 0.9001751313485113, + "recall@10": 0.9623467600700525, + "f1@1": 0.6637478108581436, + "f1@5": 0.3000583771161705, + "f1@10": 0.17497213819455507, + "map@1": 0.6637478108581436, + "map@5": 0.7559836544074722, + "map@10": 0.7644601506685569, + "mrr@1": 0.6637478108581436, + "mrr@5": 0.7559836544074722, + "mrr@10": 0.7644601506685569, "num_pred": 1142, "num_data": 1142 }, "MMLongBench-page": { - "hit@1": 0.06563245823389022, - "hit@5": 0.2863961813842482, - "hit@10": 0.3914081145584726, - "ndcg_linear@1": 0.06563245823389022, - "ndcg_linear@5": 0.14173090844476535, - "ndcg_linear@10": 0.17778697624105239, - "ndcg_exponential@1": 0.06563245823389022, - "ndcg_exponential@5": 0.14173090844476535, - "ndcg_exponential@10": 0.17778697624105239, - "precision@1": 0.06563245823389022, - "precision@5": 0.06897374701670644, - "precision@10": 0.0513126491646778, - "recall@1": 0.037907716785998406, - "recall@5": 0.20580889873849303, - "recall@10": 0.3072328294881994, - "f1@1": 0.045729568760595, - "f1@5": 0.09743296504141655, - "f1@10": 0.08383696942332947, - "map@1": 0.06563245823389022, - "map@5": 0.10388988332007426, - "map@10": 0.1198034589067661, - "mrr@1": 0.06563245823389022, - "mrr@5": 0.1441129673826571, - "mrr@10": 0.15842093798537715, - "num_pred": 838, - "num_data": 838 - }, - "MMLongBench-doc": { - "hit@1": 0.7326968973747017, - "hit@5": 0.8186157517899761, - "hit@10": 0.847255369928401, - "ndcg_linear@1": 0.5103420843277645, - "ndcg_linear@5": 0.5251195991901183, - "ndcg_linear@10": 0.4977133933501543, - "ndcg_exponential@1": 0.35151721786566653, - "ndcg_exponential@5": 0.43679570479418406, - "ndcg_exponential@10": 0.44114798529060906, - "precision@1": 0.7326968973747017, - "precision@5": 0.5794749403341289, - "precision@10": 0.5013126491646778, - "recall@1": 0.024172213173396184, - "recall@5": 0.09148792471264464, - "recall@10": 0.15302173843210523, - "f1@1": 0.04631909415766511, - "f1@5": 0.15159508610283381, - "f1@10": 0.21973685679091384, - "map@1": 0.7326968973747017, - "map@5": 0.5482895783611774, - "map@10": 0.4582356896617039, - "mrr@1": 0.7326968973747017, - "mrr@5": 0.7665871121718377, - "mrr@10": 0.7705226919725726, + "hit@1": 0.47255369928400953, + "hit@5": 0.6670644391408115, + "hit@10": 0.720763723150358, + "ndcg_linear@1": 0.47255369928400953, + "ndcg_linear@5": 0.5318266783914419, + "ndcg_linear@10": 0.5540995400526493, + "ndcg_exponential@1": 0.47255369928400953, + "ndcg_exponential@5": 0.5318266783914419, + "ndcg_exponential@10": 0.5540995400526493, + "precision@1": 0.47255369928400953, + "precision@5": 0.16968973747016705, + "precision@10": 0.09832935560859189, + "recall@1": 0.37621462666212063, + "recall@5": 0.5886327044739933, + "recall@10": 0.6506970489070727, + "f1@1": 0.4051861357947277, + "f1@5": 0.2510751462492302, + "f1@10": 0.16384316362168364, + "map@1": 0.47255369928400953, + "map@5": 0.49050682842747284, + "map@10": 0.5013023209482679, + "mrr@1": 0.47255369928400953, + "mrr@5": 0.548926014319809, + "mrr@10": 0.5558406258286926, "num_pred": 838, "num_data": 838 }