diff --git "a/scores/VLM2Vec-V1-Qwen2VL-7B.json" "b/scores/VLM2Vec-V1-Qwen2VL-7B.json" --- "a/scores/VLM2Vec-V1-Qwen2VL-7B.json" +++ "b/scores/VLM2Vec-V1-Qwen2VL-7B.json" @@ -1,7 +1,7 @@ { "metadata": { "model_name": "VLM2Vec-V1-Qwen2VL-7B", - "report_generated_date": "2025-06-28T08:54:18.948514", + "report_generated_date": "2025-12-29T09:27:50.447394", "model_size": 8.29, "url": "https://huggingface.co/TIGER-Lab/VLM2Vec-Qwen2VL-7B", "data_source": "TIGER-Lab" @@ -9,1264 +9,1264 @@ "metrics": { "image": { "ImageNet-1K": { - "hit@1": 0.801, - "hit@5": 0.955, + "hit@1": 0.802, + "hit@5": 0.956, "hit@10": 0.985, - "ndcg_linear@1": 0.801, - "ndcg_linear@5": 0.8890958638060573, - "ndcg_linear@10": 0.8988280477833336, - "ndcg_exponential@1": 0.801, - "ndcg_exponential@5": 0.8890958638060573, - "ndcg_exponential@10": 0.8988280477833336, - "precision@1": 0.801, - "precision@5": 0.19100000000000003, + "ndcg_linear@1": 0.802, + "ndcg_linear@5": 0.8898085046078267, + "ndcg_linear@10": 0.8991939108818895, + "ndcg_exponential@1": 0.802, + "ndcg_exponential@5": 0.8898085046078267, + "ndcg_exponential@10": 0.8991939108818895, + "precision@1": 0.802, + "precision@5": 0.19120000000000004, "precision@10": 0.09850000000000002, - "recall@1": 0.801, - "recall@5": 0.955, + "recall@1": 0.802, + "recall@5": 0.956, "recall@10": 0.985, - "f1@1": 0.801, - "f1@5": 0.3183333333333333, + "f1@1": 0.802, + "f1@5": 0.31866666666666665, "f1@10": 0.17909090909090908, - "map@1": 0.801, - "map@5": 0.8666499999999998, - "map@10": 0.8706829365079366, - "mrr@1": 0.801, - "mrr@5": 0.8666499999999998, - "mrr@10": 0.8706829365079366, + "map@1": 0.802, + "map@5": 0.8672833333333333, + "map@10": 0.8711575396825396, + "mrr@1": 0.802, + "mrr@5": 0.8672833333333333, + "mrr@10": 0.8711575396825396, "num_pred": 1000, "num_data": 1000 }, "N24News": { - "hit@1": 0.797, - "hit@5": 0.976, - "hit@10": 0.995, - "ndcg_linear@1": 0.797, - "ndcg_linear@5": 0.8994033806649699, - "ndcg_linear@10": 0.9057582147877457, - "ndcg_exponential@1": 0.797, - "ndcg_exponential@5": 0.8994033806649699, - "ndcg_exponential@10": 0.9057582147877457, - "precision@1": 0.797, - "precision@5": 0.1952, - "precision@10": 0.09950000000000002, - "recall@1": 0.797, - "recall@5": 0.976, - "recall@10": 0.995, - "f1@1": 0.797, - "f1@5": 0.3253333333333333, - "f1@10": 0.18090909090909088, - "map@1": 0.797, - "map@5": 0.8732833333333333, - "map@10": 0.8760261904761905, - "mrr@1": 0.797, - "mrr@5": 0.8732833333333333, - "mrr@10": 0.8760261904761905, + "hit@1": 0.795, + "hit@5": 0.975, + "hit@10": 0.994, + "ndcg_linear@1": 0.795, + "ndcg_linear@5": 0.8984353583084826, + "ndcg_linear@10": 0.9048980771023707, + "ndcg_exponential@1": 0.795, + "ndcg_exponential@5": 0.8984353583084826, + "ndcg_exponential@10": 0.9048980771023707, + "precision@1": 0.795, + "precision@5": 0.19500000000000003, + "precision@10": 0.09940000000000002, + "recall@1": 0.795, + "recall@5": 0.975, + "recall@10": 0.994, + "f1@1": 0.795, + "f1@5": 0.325, + "f1@10": 0.1807272727272727, + "map@1": 0.795, + "map@5": 0.8722666666666666, + "map@10": 0.8751178571428572, + "mrr@1": 0.795, + "mrr@5": 0.8722666666666666, + "mrr@10": 0.8751178571428572, "num_pred": 1000, "num_data": 1000 }, "HatefulMemes": { - "hit@1": 0.697, + "hit@1": 0.703, "hit@5": 1.0, "hit@10": 1.0, - "ndcg_linear@1": 0.697, - "ndcg_linear@5": 0.8881717153321518, - "ndcg_linear@10": 0.8881717153321518, - "ndcg_exponential@1": 0.697, - "ndcg_exponential@5": 0.8881717153321518, - "ndcg_exponential@10": 0.8881717153321518, - "precision@1": 0.697, + "ndcg_linear@1": 0.703, + "ndcg_linear@5": 0.890386136810723, + "ndcg_linear@10": 0.890386136810723, + "ndcg_exponential@1": 0.703, + "ndcg_exponential@5": 0.890386136810723, + "ndcg_exponential@10": 0.890386136810723, + "precision@1": 0.703, "precision@5": 0.20000000000000004, "precision@10": 0.10000000000000002, - "recall@1": 0.697, + "recall@1": 0.703, "recall@5": 1.0, "recall@10": 1.0, - "f1@1": 0.697, + "f1@1": 0.703, "f1@5": 0.3333333333333333, "f1@10": 0.18181818181818182, - "map@1": 0.697, - "map@5": 0.8485, - "map@10": 0.8485, - "mrr@1": 0.697, - "mrr@5": 0.8485, - "mrr@10": 0.8485, + "map@1": 0.703, + "map@5": 0.8515, + "map@10": 0.8515, + "mrr@1": 0.703, + "mrr@5": 0.8515, + "mrr@10": 0.8515, "num_pred": 1000, "num_data": 1000 }, "VOC2007": { - "hit@1": 0.807, - "hit@5": 0.96, - "hit@10": 0.986, - "ndcg_linear@1": 0.807, - "ndcg_linear@5": 0.895310312084262, - "ndcg_linear@10": 0.9037763050889952, - "ndcg_exponential@1": 0.807, - "ndcg_exponential@5": 0.895310312084262, - "ndcg_exponential@10": 0.9037763050889952, - "precision@1": 0.807, - "precision@5": 0.19200000000000003, - "precision@10": 0.09860000000000001, - "recall@1": 0.807, - "recall@5": 0.96, - "recall@10": 0.986, - "f1@1": 0.807, - "f1@5": 0.32, - "f1@10": 0.17927272727272725, - "map@1": 0.807, - "map@5": 0.8731833333333334, - "map@10": 0.876711507936508, - "mrr@1": 0.807, - "mrr@5": 0.8731833333333334, - "mrr@10": 0.876711507936508, + "hit@1": 0.806, + "hit@5": 0.962, + "hit@10": 0.984, + "ndcg_linear@1": 0.806, + "ndcg_linear@5": 0.8949038692397862, + "ndcg_linear@10": 0.9020230845413058, + "ndcg_exponential@1": 0.806, + "ndcg_exponential@5": 0.8949038692397862, + "ndcg_exponential@10": 0.9020230845413058, + "precision@1": 0.806, + "precision@5": 0.19240000000000004, + "precision@10": 0.0984, + "recall@1": 0.806, + "recall@5": 0.962, + "recall@10": 0.984, + "f1@1": 0.806, + "f1@5": 0.3206666666666667, + "f1@10": 0.17890909090909088, + "map@1": 0.806, + "map@5": 0.87205, + "map@10": 0.8749908730158731, + "mrr@1": 0.806, + "mrr@5": 0.87205, + "mrr@10": 0.8749908730158731, "num_pred": 1000, "num_data": 1000 }, "SUN397": { - "hit@1": 0.774, - "hit@5": 0.963, - "hit@10": 0.986, - "ndcg_linear@1": 0.774, - "ndcg_linear@5": 0.8829846031383517, - "ndcg_linear@10": 0.8903614778184943, - "ndcg_exponential@1": 0.774, - "ndcg_exponential@5": 0.8829846031383517, - "ndcg_exponential@10": 0.8903614778184943, - "precision@1": 0.774, - "precision@5": 0.19260000000000002, - "precision@10": 0.09860000000000002, - "recall@1": 0.774, - "recall@5": 0.963, - "recall@10": 0.986, - "f1@1": 0.774, - "f1@5": 0.321, - "f1@10": 0.17927272727272725, - "map@1": 0.774, - "map@5": 0.85565, - "map@10": 0.8586599206349206, - "mrr@1": 0.774, - "mrr@5": 0.85565, - "mrr@10": 0.8586599206349206, + "hit@1": 0.773, + "hit@5": 0.962, + "hit@10": 0.987, + "ndcg_linear@1": 0.773, + "ndcg_linear@5": 0.8824472573399378, + "ndcg_linear@10": 0.8905281733953193, + "ndcg_exponential@1": 0.773, + "ndcg_exponential@5": 0.8824472573399378, + "ndcg_exponential@10": 0.8905281733953193, + "precision@1": 0.773, + "precision@5": 0.19240000000000004, + "precision@10": 0.09870000000000002, + "recall@1": 0.773, + "recall@5": 0.962, + "recall@10": 0.987, + "f1@1": 0.773, + "f1@5": 0.32066666666666666, + "f1@10": 0.17945454545454545, + "map@1": 0.773, + "map@5": 0.8552166666666667, + "map@10": 0.8585503968253968, + "mrr@1": 0.773, + "mrr@5": 0.8552166666666667, + "mrr@10": 0.8585503968253968, "num_pred": 1000, "num_data": 1000 }, "Place365": { - "hit@1": 0.374, - "hit@5": 0.635, - "hit@10": 0.736, - "ndcg_linear@1": 0.374, - "ndcg_linear@5": 0.5143511655288809, - "ndcg_linear@10": 0.5468121907618051, - "ndcg_exponential@1": 0.374, - "ndcg_exponential@5": 0.5143511655288809, - "ndcg_exponential@10": 0.5468121907618051, - "precision@1": 0.374, - "precision@5": 0.12699999999999997, - "precision@10": 0.0736, - "recall@1": 0.374, - "recall@5": 0.635, - "recall@10": 0.736, - "f1@1": 0.374, - "f1@5": 0.21166666666666673, - "f1@10": 0.1338181818181818, - "map@1": 0.374, - "map@5": 0.4741166666666667, - "map@10": 0.4874035714285715, - "mrr@1": 0.374, - "mrr@5": 0.4741166666666667, - "mrr@10": 0.4874035714285715, + "hit@1": 0.372, + "hit@5": 0.632, + "hit@10": 0.732, + "ndcg_linear@1": 0.372, + "ndcg_linear@5": 0.5133671680880231, + "ndcg_linear@10": 0.5457563377113038, + "ndcg_exponential@1": 0.372, + "ndcg_exponential@5": 0.5133671680880231, + "ndcg_exponential@10": 0.5457563377113038, + "precision@1": 0.372, + "precision@5": 0.12639999999999998, + "precision@10": 0.07319999999999999, + "recall@1": 0.372, + "recall@5": 0.632, + "recall@10": 0.732, + "f1@1": 0.372, + "f1@5": 0.21066666666666672, + "f1@10": 0.13309090909090912, + "map@1": 0.372, + "map@5": 0.47363333333333335, + "map@10": 0.48703809523809527, + "mrr@1": 0.372, + "mrr@5": 0.47363333333333335, + "mrr@10": 0.48703809523809527, "num_pred": 1000, "num_data": 1000 }, "ImageNet-A": { - "hit@1": 0.581, - "hit@5": 0.796, - "hit@10": 0.861, - "ndcg_linear@1": 0.581, - "ndcg_linear@5": 0.6937519245909641, - "ndcg_linear@10": 0.7147143901304502, - "ndcg_exponential@1": 0.581, - "ndcg_exponential@5": 0.6937519245909641, - "ndcg_exponential@10": 0.7147143901304502, - "precision@1": 0.581, - "precision@5": 0.15919999999999998, - "precision@10": 0.08610000000000001, - "recall@1": 0.581, - "recall@5": 0.796, - "recall@10": 0.861, - "f1@1": 0.581, - "f1@5": 0.26533333333333337, - "f1@10": 0.15654545454545454, - "map@1": 0.581, - "map@5": 0.6598833333333334, - "map@10": 0.6685023809523809, - "mrr@1": 0.581, - "mrr@5": 0.6598833333333334, - "mrr@10": 0.6685023809523809, + "hit@1": 0.576, + "hit@5": 0.792, + "hit@10": 0.859, + "ndcg_linear@1": 0.576, + "ndcg_linear@5": 0.6890150232759639, + "ndcg_linear@10": 0.710780077999174, + "ndcg_exponential@1": 0.576, + "ndcg_exponential@5": 0.6890150232759639, + "ndcg_exponential@10": 0.710780077999174, + "precision@1": 0.576, + "precision@5": 0.1584, + "precision@10": 0.0859, + "recall@1": 0.576, + "recall@5": 0.792, + "recall@10": 0.859, + "f1@1": 0.576, + "f1@5": 0.26400000000000007, + "f1@10": 0.1561818181818182, + "map@1": 0.576, + "map@5": 0.6549333333333333, + "map@10": 0.6639738095238096, + "mrr@1": 0.576, + "mrr@5": 0.6549333333333333, + "mrr@10": 0.6639738095238096, "num_pred": 1000, "num_data": 1000 }, "ImageNet-R": { - "hit@1": 0.739, - "hit@5": 0.887, - "hit@10": 0.934, - "ndcg_linear@1": 0.739, - "ndcg_linear@5": 0.8210169628332845, - "ndcg_linear@10": 0.8361295476873967, - "ndcg_exponential@1": 0.739, - "ndcg_exponential@5": 0.8210169628332845, - "ndcg_exponential@10": 0.8361295476873967, - "precision@1": 0.739, - "precision@5": 0.17740000000000003, - "precision@10": 0.09340000000000001, - "recall@1": 0.739, - "recall@5": 0.887, - "recall@10": 0.934, - "f1@1": 0.739, - "f1@5": 0.2956666666666667, - "f1@10": 0.1698181818181818, - "map@1": 0.739, - "map@5": 0.7987833333333333, - "map@10": 0.804970238095238, - "mrr@1": 0.739, - "mrr@5": 0.7987833333333333, - "mrr@10": 0.804970238095238, + "hit@1": 0.742, + "hit@5": 0.889, + "hit@10": 0.933, + "ndcg_linear@1": 0.742, + "ndcg_linear@5": 0.823265710255611, + "ndcg_linear@10": 0.8374215997530884, + "ndcg_exponential@1": 0.742, + "ndcg_exponential@5": 0.823265710255611, + "ndcg_exponential@10": 0.8374215997530884, + "precision@1": 0.742, + "precision@5": 0.1778, + "precision@10": 0.09330000000000001, + "recall@1": 0.742, + "recall@5": 0.889, + "recall@10": 0.933, + "f1@1": 0.742, + "f1@5": 0.2963333333333334, + "f1@10": 0.1696363636363636, + "map@1": 0.742, + "map@5": 0.8011333333333335, + "map@10": 0.8069333333333334, + "mrr@1": 0.742, + "mrr@5": 0.8011333333333335, + "mrr@10": 0.8069333333333334, "num_pred": 1000, "num_data": 1000 }, "ObjectNet": { - "hit@1": 0.401, + "hit@1": 0.405, "hit@5": 0.668, - "hit@10": 0.746, - "ndcg_linear@1": 0.401, - "ndcg_linear@5": 0.5464435109813197, - "ndcg_linear@10": 0.5715775298815604, - "ndcg_exponential@1": 0.401, - "ndcg_exponential@5": 0.5464435109813197, - "ndcg_exponential@10": 0.5715775298815604, - "precision@1": 0.401, + "hit@10": 0.743, + "ndcg_linear@1": 0.405, + "ndcg_linear@5": 0.5477983861450416, + "ndcg_linear@10": 0.5720916894949353, + "ndcg_exponential@1": 0.405, + "ndcg_exponential@5": 0.5477983861450416, + "ndcg_exponential@10": 0.5720916894949353, + "precision@1": 0.405, "precision@5": 0.13359999999999997, - "precision@10": 0.0746, - "recall@1": 0.401, + "precision@10": 0.07429999999999998, + "recall@1": 0.405, "recall@5": 0.668, - "recall@10": 0.746, - "f1@1": 0.401, - "f1@5": 0.2226666666666667, - "f1@10": 0.13563636363636367, - "map@1": 0.401, - "map@5": 0.5056499999999999, - "map@10": 0.5159642857142857, - "mrr@1": 0.401, - "mrr@5": 0.5056499999999999, - "mrr@10": 0.5159642857142857, + "recall@10": 0.743, + "f1@1": 0.405, + "f1@5": 0.22266666666666673, + "f1@10": 0.13509090909090912, + "map@1": 0.405, + "map@5": 0.5075333333333333, + "map@10": 0.5175869047619047, + "mrr@1": 0.405, + "mrr@5": 0.5075333333333333, + "mrr@10": 0.5175869047619047, "num_pred": 1000, "num_data": 1000 }, "Country211": { - "hit@1": 0.298, - "hit@5": 0.49, - "hit@10": 0.584, - "ndcg_linear@1": 0.298, - "ndcg_linear@5": 0.3990647573557871, - "ndcg_linear@10": 0.42924999002239606, - "ndcg_exponential@1": 0.298, - "ndcg_exponential@5": 0.3990647573557871, - "ndcg_exponential@10": 0.42924999002239606, - "precision@1": 0.298, - "precision@5": 0.098, - "precision@10": 0.058399999999999994, - "recall@1": 0.298, - "recall@5": 0.49, - "recall@10": 0.584, - "f1@1": 0.298, - "f1@5": 0.16333333333333336, - "f1@10": 0.1061818181818182, - "map@1": 0.298, - "map@5": 0.36893333333333334, - "map@10": 0.3812690476190476, - "mrr@1": 0.298, - "mrr@5": 0.36893333333333334, - "mrr@10": 0.3812690476190476, + "hit@1": 0.302, + "hit@5": 0.486, + "hit@10": 0.58, + "ndcg_linear@1": 0.302, + "ndcg_linear@5": 0.3987057264153874, + "ndcg_linear@10": 0.42903027584839515, + "ndcg_exponential@1": 0.302, + "ndcg_exponential@5": 0.3987057264153874, + "ndcg_exponential@10": 0.42903027584839515, + "precision@1": 0.302, + "precision@5": 0.0972, + "precision@10": 0.057999999999999996, + "recall@1": 0.302, + "recall@5": 0.486, + "recall@10": 0.58, + "f1@1": 0.302, + "f1@5": 0.16200000000000006, + "f1@10": 0.10545454545454547, + "map@1": 0.302, + "map@5": 0.3697833333333333, + "map@10": 0.3822551587301587, + "mrr@1": 0.302, + "mrr@5": 0.3697833333333333, + "mrr@10": 0.3822551587301587, "num_pred": 1000, "num_data": 1000 }, "OK-VQA": { "hit@1": 0.568, - "hit@5": 0.809, - "hit@10": 0.877, + "hit@5": 0.811, + "hit@10": 0.875, "ndcg_linear@1": 0.568, - "ndcg_linear@5": 0.7001532619839014, - "ndcg_linear@10": 0.7221931447993053, + "ndcg_linear@5": 0.7014435109813197, + "ndcg_linear@10": 0.722385032610882, "ndcg_exponential@1": 0.568, - "ndcg_exponential@5": 0.7001532619839014, - "ndcg_exponential@10": 0.7221931447993053, + "ndcg_exponential@5": 0.7014435109813197, + "ndcg_exponential@10": 0.722385032610882, "precision@1": 0.568, - "precision@5": 0.1618, - "precision@10": 0.08770000000000001, + "precision@5": 0.16219999999999998, + "precision@10": 0.08750000000000001, "recall@1": 0.568, - "recall@5": 0.809, - "recall@10": 0.877, + "recall@5": 0.811, + "recall@10": 0.875, "f1@1": 0.568, - "f1@5": 0.2696666666666667, - "f1@10": 0.15945454545454543, + "f1@5": 0.27033333333333337, + "f1@10": 0.1590909090909091, "map@1": 0.568, - "map@5": 0.6636, - "map@10": 0.6727349206349206, + "map@5": 0.66465, + "map@10": 0.6734380952380953, "mrr@1": 0.568, - "mrr@5": 0.6636, - "mrr@10": 0.6727349206349206, + "mrr@5": 0.66465, + "mrr@10": 0.6734380952380953, "num_pred": 1000, "num_data": 1000 }, "A-OKVQA": { - "hit@1": 0.473, + "hit@1": 0.475, "hit@5": 0.757, - "hit@10": 0.84, - "ndcg_linear@1": 0.473, - "ndcg_linear@5": 0.6242983847947655, - "ndcg_linear@10": 0.6518307287887928, - "ndcg_exponential@1": 0.473, - "ndcg_exponential@5": 0.6242983847947655, - "ndcg_exponential@10": 0.6518307287887928, - "precision@1": 0.473, - "precision@5": 0.15139999999999998, - "precision@10": 0.084, - "recall@1": 0.473, + "hit@10": 0.838, + "ndcg_linear@1": 0.475, + "ndcg_linear@5": 0.623798357995132, + "ndcg_linear@10": 0.6505642595296839, + "ndcg_exponential@1": 0.475, + "ndcg_exponential@5": 0.623798357995132, + "ndcg_exponential@10": 0.6505642595296839, + "precision@1": 0.475, + "precision@5": 0.1514, + "precision@10": 0.08380000000000001, + "recall@1": 0.475, "recall@5": 0.757, - "recall@10": 0.84, - "f1@1": 0.473, - "f1@5": 0.2523333333333334, - "f1@10": 0.15272727272727274, - "map@1": 0.473, - "map@5": 0.5801666666666667, - "map@10": 0.591940873015873, - "mrr@1": 0.473, - "mrr@5": 0.5801666666666667, - "mrr@10": 0.591940873015873, + "recall@10": 0.838, + "f1@1": 0.475, + "f1@5": 0.25233333333333335, + "f1@10": 0.15236363636363637, + "map@1": 0.475, + "map@5": 0.5796333333333333, + "map@10": 0.5910142857142857, + "mrr@1": 0.475, + "mrr@5": 0.5796333333333333, + "mrr@10": 0.5910142857142857, "num_pred": 1000, "num_data": 1000 }, "DocVQA": { - "hit@1": 0.897, + "hit@1": 0.888, "hit@5": 0.972, - "hit@10": 0.983, - "ndcg_linear@1": 0.897, - "ndcg_linear@5": 0.9407970419021967, - "ndcg_linear@10": 0.9443639462909889, - "ndcg_exponential@1": 0.897, - "ndcg_exponential@5": 0.9407970419021967, - "ndcg_exponential@10": 0.9443639462909889, - "precision@1": 0.897, + "hit@10": 0.984, + "ndcg_linear@1": 0.888, + "ndcg_linear@5": 0.9361424192588348, + "ndcg_linear@10": 0.939892880893341, + "ndcg_exponential@1": 0.888, + "ndcg_exponential@5": 0.9361424192588348, + "ndcg_exponential@10": 0.939892880893341, + "precision@1": 0.888, "precision@5": 0.19440000000000004, - "precision@10": 0.09830000000000001, - "recall@1": 0.897, + "precision@10": 0.09840000000000002, + "recall@1": 0.888, "recall@5": 0.972, - "recall@10": 0.983, - "f1@1": 0.897, + "recall@10": 0.984, + "f1@1": 0.888, "f1@5": 0.324, - "f1@10": 0.17872727272727268, - "map@1": 0.897, - "map@5": 0.9300666666666666, - "map@10": 0.9315428571428572, - "mrr@1": 0.897, - "mrr@5": 0.9300666666666666, - "mrr@10": 0.9315428571428572, + "f1@10": 0.17890909090909088, + "map@1": 0.888, + "map@5": 0.9239166666666667, + "map@10": 0.9253900793650793, + "mrr@1": 0.888, + "mrr@5": 0.9239166666666667, + "mrr@10": 0.9253900793650793, "num_pred": 1000, "num_data": 1000 }, "InfographicsVQA": { - "hit@1": 0.6, - "hit@5": 0.811, - "hit@10": 0.876, - "ndcg_linear@1": 0.6, - "ndcg_linear@5": 0.7145328601111749, - "ndcg_linear@10": 0.7355162717771352, - "ndcg_exponential@1": 0.6, - "ndcg_exponential@5": 0.7145328601111749, - "ndcg_exponential@10": 0.7355162717771352, - "precision@1": 0.6, - "precision@5": 0.16219999999999998, - "precision@10": 0.08760000000000001, - "recall@1": 0.6, - "recall@5": 0.811, - "recall@10": 0.876, - "f1@1": 0.6, - "f1@5": 0.27033333333333337, - "f1@10": 0.1592727272727273, - "map@1": 0.6, - "map@5": 0.6822166666666667, - "map@10": 0.6908571428571428, - "mrr@1": 0.6, - "mrr@5": 0.6822166666666667, - "mrr@10": 0.6908571428571428, + "hit@1": 0.59, + "hit@5": 0.813, + "hit@10": 0.873, + "ndcg_linear@1": 0.59, + "ndcg_linear@5": 0.7109184737405526, + "ndcg_linear@10": 0.7303255515338261, + "ndcg_exponential@1": 0.59, + "ndcg_exponential@5": 0.7109184737405526, + "ndcg_exponential@10": 0.7303255515338261, + "precision@1": 0.59, + "precision@5": 0.16260000000000002, + "precision@10": 0.08730000000000002, + "recall@1": 0.59, + "recall@5": 0.813, + "recall@10": 0.873, + "f1@1": 0.59, + "f1@5": 0.271, + "f1@10": 0.15872727272727274, + "map@1": 0.59, + "map@5": 0.6767, + "map@10": 0.6847134920634921, + "mrr@1": 0.59, + "mrr@5": 0.6767, + "mrr@10": 0.6847134920634921, "num_pred": 1000, "num_data": 1000 }, "ChartQA": { - "hit@1": 0.569, - "hit@5": 0.777, - "hit@10": 0.849, - "ndcg_linear@1": 0.569, - "ndcg_linear@5": 0.6801833263027208, - "ndcg_linear@10": 0.7034571893742657, - "ndcg_exponential@1": 0.569, - "ndcg_exponential@5": 0.6801833263027208, - "ndcg_exponential@10": 0.7034571893742657, - "precision@1": 0.569, - "precision@5": 0.15539999999999998, - "precision@10": 0.0849, - "recall@1": 0.569, - "recall@5": 0.777, - "recall@10": 0.849, - "f1@1": 0.569, - "f1@5": 0.259, - "f1@10": 0.15436363636363637, - "map@1": 0.569, - "map@5": 0.6479166666666667, - "map@10": 0.6575218253968254, - "mrr@1": 0.569, - "mrr@5": 0.6479166666666667, - "mrr@10": 0.6575218253968254, + "hit@1": 0.566, + "hit@5": 0.769, + "hit@10": 0.847, + "ndcg_linear@1": 0.566, + "ndcg_linear@5": 0.6762005941642151, + "ndcg_linear@10": 0.7013418103096569, + "ndcg_exponential@1": 0.566, + "ndcg_exponential@5": 0.6762005941642151, + "ndcg_exponential@10": 0.7013418103096569, + "precision@1": 0.566, + "precision@5": 0.15380000000000002, + "precision@10": 0.0847, + "recall@1": 0.566, + "recall@5": 0.769, + "recall@10": 0.847, + "f1@1": 0.566, + "f1@5": 0.25633333333333336, + "f1@10": 0.154, + "map@1": 0.566, + "map@5": 0.6451666666666668, + "map@10": 0.6554980158730159, + "mrr@1": 0.566, + "mrr@5": 0.6451666666666668, + "mrr@10": 0.6554980158730159, "num_pred": 1000, "num_data": 1000 }, "Visual7W": { "hit@1": 0.527, - "hit@5": 0.883, + "hit@5": 0.887, "hit@10": 0.946, "ndcg_linear@1": 0.527, - "ndcg_linear@5": 0.7233005346983259, - "ndcg_linear@10": 0.7441692360181532, + "ndcg_linear@5": 0.7249812239810781, + "ndcg_linear@10": 0.7445295476481599, "ndcg_exponential@1": 0.527, - "ndcg_exponential@5": 0.7233005346983259, - "ndcg_exponential@10": 0.7441692360181532, + "ndcg_exponential@5": 0.7249812239810781, + "ndcg_exponential@10": 0.7445295476481599, "precision@1": 0.527, - "precision@5": 0.17660000000000003, + "precision@5": 0.17740000000000003, "precision@10": 0.09460000000000002, "recall@1": 0.527, - "recall@5": 0.883, + "recall@5": 0.887, "recall@10": 0.946, "f1@1": 0.527, - "f1@5": 0.2943333333333334, + "f1@5": 0.2956666666666667, "f1@10": 0.172, "map@1": 0.527, - "map@5": 0.6695166666666666, - "map@10": 0.678415873015873, + "map@5": 0.6705166666666666, + "map@10": 0.6788535714285713, "mrr@1": 0.527, - "mrr@5": 0.6695166666666666, - "mrr@10": 0.678415873015873, + "mrr@5": 0.6705166666666666, + "mrr@10": 0.6788535714285713, "num_pred": 1000, "num_data": 1000 }, "ScienceQA": { - "hit@1": 0.385, - "hit@5": 0.724, - "hit@10": 0.84, - "ndcg_linear@1": 0.385, - "ndcg_linear@5": 0.5679727026484885, - "ndcg_linear@10": 0.6050846778437835, - "ndcg_exponential@1": 0.385, - "ndcg_exponential@5": 0.5679727026484885, - "ndcg_exponential@10": 0.6050846778437835, - "precision@1": 0.385, - "precision@5": 0.1448, - "precision@10": 0.084, - "recall@1": 0.385, - "recall@5": 0.724, - "recall@10": 0.84, - "f1@1": 0.385, - "f1@5": 0.24133333333333337, - "f1@10": 0.1527272727272727, - "map@1": 0.385, - "map@5": 0.5158333333333333, - "map@10": 0.5309234126984127, - "mrr@1": 0.385, - "mrr@5": 0.5158333333333333, - "mrr@10": 0.5309234126984127, + "hit@1": 0.382, + "hit@5": 0.73, + "hit@10": 0.838, + "ndcg_linear@1": 0.382, + "ndcg_linear@5": 0.5702914979912869, + "ndcg_linear@10": 0.6047184493740754, + "ndcg_exponential@1": 0.382, + "ndcg_exponential@5": 0.5702914979912869, + "ndcg_exponential@10": 0.6047184493740754, + "precision@1": 0.382, + "precision@5": 0.14599999999999996, + "precision@10": 0.0838, + "recall@1": 0.382, + "recall@5": 0.73, + "recall@10": 0.838, + "f1@1": 0.382, + "f1@5": 0.24333333333333337, + "f1@10": 0.15236363636363637, + "map@1": 0.382, + "map@5": 0.5168666666666667, + "map@10": 0.5307892857142857, + "mrr@1": 0.382, + "mrr@5": 0.5168666666666667, + "mrr@10": 0.5307892857142857, "num_pred": 1000, "num_data": 1000 }, "VizWiz": { - "hit@1": 0.399, - "hit@5": 0.59, - "hit@10": 0.669, - "ndcg_linear@1": 0.399, - "ndcg_linear@5": 0.5023670367904077, - "ndcg_linear@10": 0.52783003091099, - "ndcg_exponential@1": 0.399, - "ndcg_exponential@5": 0.5023670367904077, - "ndcg_exponential@10": 0.52783003091099, - "precision@1": 0.399, - "precision@5": 0.11799999999999998, - "precision@10": 0.0669, - "recall@1": 0.399, - "recall@5": 0.59, - "recall@10": 0.669, - "f1@1": 0.399, - "f1@5": 0.19666666666666674, - "f1@10": 0.12163636363636365, - "map@1": 0.399, - "map@5": 0.4730333333333333, - "map@10": 0.4834888888888888, - "mrr@1": 0.399, - "mrr@5": 0.4730333333333333, - "mrr@10": 0.4834888888888888, + "hit@1": 0.396, + "hit@5": 0.594, + "hit@10": 0.671, + "ndcg_linear@1": 0.396, + "ndcg_linear@5": 0.503130703098848, + "ndcg_linear@10": 0.5278197925554152, + "ndcg_exponential@1": 0.396, + "ndcg_exponential@5": 0.503130703098848, + "ndcg_exponential@10": 0.5278197925554152, + "precision@1": 0.396, + "precision@5": 0.1188, + "precision@10": 0.06709999999999999, + "recall@1": 0.396, + "recall@5": 0.594, + "recall@10": 0.671, + "f1@1": 0.396, + "f1@5": 0.19800000000000006, + "f1@10": 0.122, + "map@1": 0.396, + "map@5": 0.47275, + "map@10": 0.4828174603174603, + "mrr@1": 0.396, + "mrr@5": 0.47275, + "mrr@10": 0.4828174603174603, "num_pred": 1000, "num_data": 1000 }, "GQA": { - "hit@1": 0.551, - "hit@5": 0.799, - "hit@10": 0.875, - "ndcg_linear@1": 0.551, - "ndcg_linear@5": 0.685415636190356, - "ndcg_linear@10": 0.7100738369162211, - "ndcg_exponential@1": 0.551, - "ndcg_exponential@5": 0.685415636190356, - "ndcg_exponential@10": 0.7100738369162211, - "precision@1": 0.551, - "precision@5": 0.1598, - "precision@10": 0.0875, - "recall@1": 0.551, - "recall@5": 0.799, - "recall@10": 0.875, - "f1@1": 0.551, - "f1@5": 0.26633333333333337, - "f1@10": 0.15909090909090906, - "map@1": 0.551, - "map@5": 0.6473833333333333, - "map@10": 0.6576083333333334, - "mrr@1": 0.551, - "mrr@5": 0.6473833333333333, - "mrr@10": 0.6576083333333334, + "hit@1": 0.544, + "hit@5": 0.798, + "hit@10": 0.874, + "ndcg_linear@1": 0.544, + "ndcg_linear@5": 0.6822012147117849, + "ndcg_linear@10": 0.7070687661557895, + "ndcg_exponential@1": 0.544, + "ndcg_exponential@5": 0.6822012147117849, + "ndcg_exponential@10": 0.7070687661557895, + "precision@1": 0.544, + "precision@5": 0.15960000000000002, + "precision@10": 0.0874, + "recall@1": 0.544, + "recall@5": 0.798, + "recall@10": 0.874, + "f1@1": 0.544, + "f1@5": 0.266, + "f1@10": 0.15890909090909092, + "map@1": 0.544, + "map@5": 0.6433833333333333, + "map@10": 0.6538146825396826, + "mrr@1": 0.544, + "mrr@5": 0.6433833333333333, + "mrr@10": 0.6538146825396826, "num_pred": 1000, "num_data": 1000 }, "TextVQA": { "hit@1": 0.716, - "hit@5": 0.828, - "hit@10": 0.868, + "hit@5": 0.824, + "hit@10": 0.867, "ndcg_linear@1": 0.716, - "ndcg_linear@5": 0.7768651248159903, - "ndcg_linear@10": 0.7898048365401985, + "ndcg_linear@5": 0.7743414058017046, + "ndcg_linear@10": 0.7882931486689345, "ndcg_exponential@1": 0.716, - "ndcg_exponential@5": 0.7768651248159903, - "ndcg_exponential@10": 0.7898048365401985, + "ndcg_exponential@5": 0.7743414058017046, + "ndcg_exponential@10": 0.7882931486689345, "precision@1": 0.716, - "precision@5": 0.16560000000000002, - "precision@10": 0.08680000000000002, + "precision@5": 0.1648, + "precision@10": 0.08670000000000001, "recall@1": 0.716, - "recall@5": 0.828, - "recall@10": 0.868, + "recall@5": 0.824, + "recall@10": 0.867, "f1@1": 0.716, - "f1@5": 0.276, - "f1@10": 0.1578181818181818, + "f1@5": 0.27466666666666667, + "f1@10": 0.15763636363636363, "map@1": 0.716, - "map@5": 0.7597333333333334, - "map@10": 0.7650805555555555, + "map@5": 0.7577333333333334, + "map@10": 0.7635206349206349, "mrr@1": 0.716, - "mrr@5": 0.7597333333333334, - "mrr@10": 0.7650805555555555, + "mrr@5": 0.7577333333333334, + "mrr@10": 0.7635206349206349, "num_pred": 1000, "num_data": 1000 }, "VisDial": { - "hit@1": 0.819, - "hit@5": 0.96, + "hit@1": 0.816, + "hit@5": 0.958, "hit@10": 0.979, - "ndcg_linear@1": 0.819, - "ndcg_linear@5": 0.8974115327708008, - "ndcg_linear@10": 0.9036371535428991, - "ndcg_exponential@1": 0.819, - "ndcg_exponential@5": 0.8974115327708008, - "ndcg_exponential@10": 0.9036371535428991, - "precision@1": 0.819, - "precision@5": 0.19200000000000003, + "ndcg_linear@1": 0.816, + "ndcg_linear@5": 0.8955483989778521, + "ndcg_linear@10": 0.9024709427581272, + "ndcg_exponential@1": 0.816, + "ndcg_exponential@5": 0.8955483989778521, + "ndcg_exponential@10": 0.9024709427581272, + "precision@1": 0.816, + "precision@5": 0.19160000000000002, "precision@10": 0.09790000000000001, - "recall@1": 0.819, - "recall@5": 0.96, + "recall@1": 0.816, + "recall@5": 0.958, "recall@10": 0.979, - "f1@1": 0.819, - "f1@5": 0.32, + "f1@1": 0.816, + "f1@5": 0.3193333333333333, "f1@10": 0.17799999999999996, - "map@1": 0.819, - "map@5": 0.8763166666666666, - "map@10": 0.8789325396825397, - "mrr@1": 0.819, - "mrr@5": 0.8763166666666666, - "mrr@10": 0.8789325396825397, + "map@1": 0.816, + "map@5": 0.8744500000000001, + "map@10": 0.8773869047619048, + "mrr@1": 0.816, + "mrr@5": 0.8744500000000001, + "mrr@10": 0.8773869047619048, "num_pred": 1000, "num_data": 1000 }, "CIRR": { - "hit@1": 0.511, - "hit@5": 0.881, + "hit@1": 0.514, + "hit@5": 0.879, "hit@10": 0.932, - "ndcg_linear@1": 0.511, - "ndcg_linear@5": 0.7178882619237661, - "ndcg_linear@10": 0.7343346954480953, - "ndcg_exponential@1": 0.511, - "ndcg_exponential@5": 0.7178882619237661, - "ndcg_exponential@10": 0.7343346954480953, - "precision@1": 0.511, - "precision@5": 0.1762, + "ndcg_linear@1": 0.514, + "ndcg_linear@5": 0.7185630154281765, + "ndcg_linear@10": 0.735609039232058, + "ndcg_exponential@1": 0.514, + "ndcg_exponential@5": 0.7185630154281765, + "ndcg_exponential@10": 0.735609039232058, + "precision@1": 0.514, + "precision@5": 0.1758, "precision@10": 0.09320000000000002, - "recall@1": 0.511, - "recall@5": 0.881, + "recall@1": 0.514, + "recall@5": 0.879, "recall@10": 0.932, - "f1@1": 0.511, - "f1@5": 0.2936666666666667, + "f1@1": 0.514, + "f1@5": 0.293, "f1@10": 0.16945454545454544, - "map@1": 0.511, - "map@5": 0.6627000000000001, - "map@10": 0.6694619047619047, - "mrr@1": 0.511, - "mrr@5": 0.6627000000000001, - "mrr@10": 0.6694619047619047, + "map@1": 0.514, + "map@5": 0.66425, + "map@10": 0.6712313492063493, + "mrr@1": 0.514, + "mrr@5": 0.66425, + "mrr@10": 0.6712313492063493, "num_pred": 1000, "num_data": 1000 }, "VisualNews_t2i": { - "hit@1": 0.805, + "hit@1": 0.803, "hit@5": 0.919, - "hit@10": 0.943, - "ndcg_linear@1": 0.805, - "ndcg_linear@5": 0.86749427389276, - "ndcg_linear@10": 0.875346241736819, - "ndcg_exponential@1": 0.805, - "ndcg_exponential@5": 0.86749427389276, - "ndcg_exponential@10": 0.875346241736819, - "precision@1": 0.805, + "hit@10": 0.946, + "ndcg_linear@1": 0.803, + "ndcg_linear@5": 0.8671489226606173, + "ndcg_linear@10": 0.8758860194138534, + "ndcg_exponential@1": 0.803, + "ndcg_exponential@5": 0.8671489226606173, + "ndcg_exponential@10": 0.8758860194138534, + "precision@1": 0.803, "precision@5": 0.18380000000000002, - "precision@10": 0.09430000000000001, - "recall@1": 0.805, + "precision@10": 0.09460000000000002, + "recall@1": 0.803, "recall@5": 0.919, - "recall@10": 0.943, - "f1@1": 0.805, + "recall@10": 0.946, + "f1@1": 0.803, "f1@5": 0.30633333333333335, - "f1@10": 0.17145454545454544, - "map@1": 0.805, - "map@5": 0.8502000000000001, - "map@10": 0.8534948412698413, - "mrr@1": 0.805, - "mrr@5": 0.8502000000000001, - "mrr@10": 0.8534948412698413, + "f1@10": 0.17199999999999996, + "map@1": 0.803, + "map@5": 0.8497, + "map@10": 0.8533130952380952, + "mrr@1": 0.803, + "mrr@5": 0.8497, + "mrr@10": 0.8533130952380952, "num_pred": 1000, "num_data": 1000 }, "VisualNews_i2t": { "hit@1": 0.812, - "hit@5": 0.925, - "hit@10": 0.958, + "hit@5": 0.928, + "hit@10": 0.959, "ndcg_linear@1": 0.812, - "ndcg_linear@5": 0.87458496804873, - "ndcg_linear@10": 0.8854936582584427, + "ndcg_linear@5": 0.8758947802837561, + "ndcg_linear@10": 0.8862427341230226, "ndcg_exponential@1": 0.812, - "ndcg_exponential@5": 0.87458496804873, - "ndcg_exponential@10": 0.8854936582584427, + "ndcg_exponential@5": 0.8758947802837561, + "ndcg_exponential@10": 0.8862427341230226, "precision@1": 0.812, - "precision@5": 0.185, - "precision@10": 0.09580000000000001, + "precision@5": 0.18560000000000001, + "precision@10": 0.0959, "recall@1": 0.812, - "recall@5": 0.925, - "recall@10": 0.958, + "recall@5": 0.928, + "recall@10": 0.959, "f1@1": 0.812, - "f1@5": 0.30833333333333335, - "f1@10": 0.1741818181818182, + "f1@5": 0.30933333333333335, + "f1@10": 0.17436363636363633, "map@1": 0.812, - "map@5": 0.8575833333333333, - "map@10": 0.8622246031746031, + "map@5": 0.8583666666666666, + "map@10": 0.8628313492063492, "mrr@1": 0.812, - "mrr@5": 0.8575833333333333, - "mrr@10": 0.8622246031746031, + "mrr@5": 0.8583666666666666, + "mrr@10": 0.8628313492063492, "num_pred": 1000, "num_data": 1000 }, "MSCOCO_t2i": { - "hit@1": 0.772, - "hit@5": 0.955, - "hit@10": 0.977, - "ndcg_linear@1": 0.772, - "ndcg_linear@5": 0.8728498225281803, - "ndcg_linear@10": 0.8801390594643754, - "ndcg_exponential@1": 0.772, - "ndcg_exponential@5": 0.8728498225281803, - "ndcg_exponential@10": 0.8801390594643754, - "precision@1": 0.772, - "precision@5": 0.19100000000000003, - "precision@10": 0.09770000000000002, - "recall@1": 0.772, - "recall@5": 0.955, - "recall@10": 0.977, - "f1@1": 0.772, - "f1@5": 0.31833333333333336, - "f1@10": 0.17763636363636362, - "map@1": 0.772, - "map@5": 0.8451833333333334, - "map@10": 0.848293253968254, - "mrr@1": 0.772, - "mrr@5": 0.8451833333333334, - "mrr@10": 0.848293253968254, + "hit@1": 0.775, + "hit@5": 0.951, + "hit@10": 0.978, + "ndcg_linear@1": 0.775, + "ndcg_linear@5": 0.8733232405143404, + "ndcg_linear@10": 0.8823129266554056, + "ndcg_exponential@1": 0.775, + "ndcg_exponential@5": 0.8733232405143404, + "ndcg_exponential@10": 0.8823129266554056, + "precision@1": 0.775, + "precision@5": 0.19020000000000004, + "precision@10": 0.09780000000000001, + "recall@1": 0.775, + "recall@5": 0.951, + "recall@10": 0.978, + "f1@1": 0.775, + "f1@5": 0.317, + "f1@10": 0.17781818181818182, + "map@1": 0.775, + "map@5": 0.8470333333333333, + "map@10": 0.8508940476190475, + "mrr@1": 0.775, + "mrr@5": 0.8470333333333333, + "mrr@10": 0.8508940476190475, "num_pred": 1000, "num_data": 1000 }, "MSCOCO_i2t": { - "hit@1": 0.739, - "hit@5": 0.951, - "hit@10": 0.982, - "ndcg_linear@1": 0.739, - "ndcg_linear@5": 0.8562866446438291, - "ndcg_linear@10": 0.8665048964431262, - "ndcg_exponential@1": 0.739, - "ndcg_exponential@5": 0.8562866446438291, - "ndcg_exponential@10": 0.8665048964431262, - "precision@1": 0.739, - "precision@5": 0.1902, - "precision@10": 0.09820000000000002, - "recall@1": 0.739, - "recall@5": 0.951, - "recall@10": 0.982, - "f1@1": 0.739, - "f1@5": 0.317, - "f1@10": 0.1785454545454545, - "map@1": 0.739, - "map@5": 0.8243666666666666, - "map@10": 0.828697619047619, - "mrr@1": 0.739, - "mrr@5": 0.8243666666666666, - "mrr@10": 0.828697619047619, + "hit@1": 0.736, + "hit@5": 0.953, + "hit@10": 0.983, + "ndcg_linear@1": 0.736, + "ndcg_linear@5": 0.8563536459100087, + "ndcg_linear@10": 0.8660833601755094, + "ndcg_exponential@1": 0.736, + "ndcg_exponential@5": 0.8563536459100087, + "ndcg_exponential@10": 0.8660833601755094, + "precision@1": 0.736, + "precision@5": 0.19060000000000002, + "precision@10": 0.09830000000000001, + "recall@1": 0.736, + "recall@5": 0.953, + "recall@10": 0.983, + "f1@1": 0.736, + "f1@5": 0.31766666666666665, + "f1@10": 0.17872727272727268, + "map@1": 0.736, + "map@5": 0.8237666666666666, + "map@10": 0.8277968253968254, + "mrr@1": 0.736, + "mrr@5": 0.8237666666666666, + "mrr@10": 0.8277968253968254, "num_pred": 1000, "num_data": 1000 }, "NIGHTS": { - "hit@1": 0.676, - "hit@5": 0.976, + "hit@1": 0.675, + "hit@5": 0.974, "hit@10": 0.992, - "ndcg_linear@1": 0.676, - "ndcg_linear@5": 0.8478912308797101, - "ndcg_linear@10": 0.8531212757384757, - "ndcg_exponential@1": 0.676, - "ndcg_exponential@5": 0.8478912308797101, - "ndcg_exponential@10": 0.8531212757384757, - "precision@1": 0.676, - "precision@5": 0.19520000000000004, + "ndcg_linear@1": 0.675, + "ndcg_linear@5": 0.8465559189535962, + "ndcg_linear@10": 0.8524347620224809, + "ndcg_exponential@1": 0.675, + "ndcg_exponential@5": 0.8465559189535962, + "ndcg_exponential@10": 0.8524347620224809, + "precision@1": 0.675, + "precision@5": 0.1948, "precision@10": 0.09920000000000001, - "recall@1": 0.676, - "recall@5": 0.976, + "recall@1": 0.675, + "recall@5": 0.974, "recall@10": 0.992, - "f1@1": 0.676, - "f1@5": 0.32533333333333336, + "f1@1": 0.675, + "f1@5": 0.3246666666666666, "f1@10": 0.18036363636363634, - "map@1": 0.676, - "map@5": 0.8041166666666666, - "map@10": 0.8063059523809525, - "mrr@1": 0.676, - "mrr@5": 0.8041166666666666, - "mrr@10": 0.8063059523809525, + "map@1": 0.675, + "map@5": 0.8029666666666667, + "map@10": 0.8054238095238097, + "mrr@1": 0.675, + "mrr@5": 0.8029666666666667, + "mrr@10": 0.8054238095238097, "num_pred": 1000, "num_data": 1000 }, "WebQA": { "hit@1": 0.883, - "hit@5": 0.987, - "hit@10": 0.992, + "hit@5": 0.988, + "hit@10": 0.993, "ndcg_linear@1": 0.883, - "ndcg_linear@5": 0.9427751826133371, - "ndcg_linear@10": 0.9444926023847804, + "ndcg_linear@5": 0.9429878234151065, + "ndcg_linear@10": 0.944635631113984, "ndcg_exponential@1": 0.883, - "ndcg_exponential@5": 0.9427751826133371, - "ndcg_exponential@10": 0.9444926023847804, + "ndcg_exponential@5": 0.9429878234151065, + "ndcg_exponential@10": 0.944635631113984, "precision@1": 0.883, - "precision@5": 0.19740000000000005, - "precision@10": 0.09920000000000001, + "precision@5": 0.19760000000000005, + "precision@10": 0.09930000000000001, "recall@1": 0.883, - "recall@5": 0.987, - "recall@10": 0.992, + "recall@5": 0.988, + "recall@10": 0.993, "f1@1": 0.883, - "f1@5": 0.329, - "f1@10": 0.18036363636363634, + "f1@5": 0.3293333333333333, + "f1@10": 0.18054545454545454, "map@1": 0.883, - "map@5": 0.9276666666666668, - "map@10": 0.9284345238095238, + "map@5": 0.9276333333333333, + "map@10": 0.928331746031746, "mrr@1": 0.883, - "mrr@5": 0.9276666666666668, - "mrr@10": 0.9284345238095238, + "mrr@5": 0.9276333333333333, + "mrr@10": 0.928331746031746, "num_pred": 1000, "num_data": 1000 }, "FashionIQ": { - "hit@1": 0.171, + "hit@1": 0.168, "hit@5": 0.382, - "hit@10": 0.492, - "ndcg_linear@1": 0.171, - "ndcg_linear@5": 0.27952792558154677, - "ndcg_linear@10": 0.3150974167961427, - "ndcg_exponential@1": 0.171, - "ndcg_exponential@5": 0.27952792558154677, - "ndcg_exponential@10": 0.3150974167961427, - "precision@1": 0.171, - "precision@5": 0.0764, - "precision@10": 0.0492, - "recall@1": 0.171, + "hit@10": 0.504, + "ndcg_linear@1": 0.168, + "ndcg_linear@5": 0.2789338269270776, + "ndcg_linear@10": 0.3183037929754536, + "ndcg_exponential@1": 0.168, + "ndcg_exponential@5": 0.2789338269270776, + "ndcg_exponential@10": 0.3183037929754536, + "precision@1": 0.168, + "precision@5": 0.07640000000000001, + "precision@10": 0.0504, + "recall@1": 0.168, "recall@5": 0.382, - "recall@10": 0.492, - "f1@1": 0.171, - "f1@5": 0.12733333333333338, - "f1@10": 0.08945454545454547, - "map@1": 0.171, - "map@5": 0.24576666666666666, - "map@10": 0.26045158730158735, - "mrr@1": 0.171, - "mrr@5": 0.24576666666666666, - "mrr@10": 0.26045158730158735, + "recall@10": 0.504, + "f1@1": 0.168, + "f1@5": 0.12733333333333335, + "f1@10": 0.09163636363636365, + "map@1": 0.168, + "map@5": 0.2449166666666667, + "map@10": 0.2611269841269841, + "mrr@1": 0.168, + "mrr@5": 0.2449166666666667, + "mrr@10": 0.2611269841269841, "num_pred": 1000, "num_data": 1000 }, "Wiki-SS-NQ": { - "hit@1": 0.623, - "hit@5": 0.837, - "hit@10": 0.886, - "ndcg_linear@1": 0.623, - "ndcg_linear@5": 0.7404547633407098, - "ndcg_linear@10": 0.756365196101992, - "ndcg_exponential@1": 0.623, - "ndcg_exponential@5": 0.7404547633407098, - "ndcg_exponential@10": 0.756365196101992, - "precision@1": 0.623, - "precision@5": 0.1674, - "precision@10": 0.08860000000000001, - "recall@1": 0.623, - "recall@5": 0.837, - "recall@10": 0.886, - "f1@1": 0.623, - "f1@5": 0.279, - "f1@10": 0.16109090909090906, - "map@1": 0.623, - "map@5": 0.7080333333333333, - "map@10": 0.7146444444444444, - "mrr@1": 0.623, - "mrr@5": 0.7080333333333333, - "mrr@10": 0.7146444444444444, + "hit@1": 0.621, + "hit@5": 0.842, + "hit@10": 0.891, + "ndcg_linear@1": 0.621, + "ndcg_linear@5": 0.7416501630806183, + "ndcg_linear@10": 0.7574790845736645, + "ndcg_exponential@1": 0.621, + "ndcg_exponential@5": 0.7416501630806183, + "ndcg_exponential@10": 0.7574790845736645, + "precision@1": 0.621, + "precision@5": 0.1684, + "precision@10": 0.08910000000000001, + "recall@1": 0.621, + "recall@5": 0.842, + "recall@10": 0.891, + "f1@1": 0.621, + "f1@5": 0.28066666666666673, + "f1@10": 0.162, + "map@1": 0.621, + "map@5": 0.7079500000000001, + "map@10": 0.7144809523809523, + "mrr@1": 0.621, + "mrr@5": 0.7079500000000001, + "mrr@10": 0.7144809523809523, "num_pred": 1000, "num_data": 1000 }, "OVEN": { - "hit@1": 0.665, - "hit@5": 0.869, - "hit@10": 0.913, - "ndcg_linear@1": 0.665, - "ndcg_linear@5": 0.7770472120489863, - "ndcg_linear@10": 0.7914052412762739, - "ndcg_exponential@1": 0.665, - "ndcg_exponential@5": 0.7770472120489863, - "ndcg_exponential@10": 0.7914052412762739, - "precision@1": 0.665, - "precision@5": 0.1738, - "precision@10": 0.0913, - "recall@1": 0.665, - "recall@5": 0.869, - "recall@10": 0.913, - "f1@1": 0.665, - "f1@5": 0.2896666666666667, - "f1@10": 0.166, - "map@1": 0.665, - "map@5": 0.7461, - "map@10": 0.7521063492063491, - "mrr@1": 0.665, - "mrr@5": 0.7461, - "mrr@10": 0.7521063492063491, + "hit@1": 0.666, + "hit@5": 0.866, + "hit@10": 0.912, + "ndcg_linear@1": 0.666, + "ndcg_linear@5": 0.7768227251398907, + "ndcg_linear@10": 0.7918046317273635, + "ndcg_exponential@1": 0.666, + "ndcg_exponential@5": 0.7768227251398907, + "ndcg_exponential@10": 0.7918046317273635, + "precision@1": 0.666, + "precision@5": 0.17320000000000005, + "precision@10": 0.09120000000000002, + "recall@1": 0.666, + "recall@5": 0.866, + "recall@10": 0.912, + "f1@1": 0.666, + "f1@5": 0.28866666666666674, + "f1@10": 0.1658181818181818, + "map@1": 0.666, + "map@5": 0.7467333333333334, + "map@10": 0.7529873015873015, + "mrr@1": 0.666, + "mrr@5": 0.7467333333333334, + "mrr@10": 0.7529873015873015, "num_pred": 1000, "num_data": 1000 }, "EDIS": { - "hit@1": 0.857, - "hit@5": 0.976, - "hit@10": 0.992, - "ndcg_linear@1": 0.857, - "ndcg_linear@5": 0.9223976700829437, - "ndcg_linear@10": 0.9277225779585583, - "ndcg_exponential@1": 0.857, - "ndcg_exponential@5": 0.9223976700829437, - "ndcg_exponential@10": 0.9277225779585583, - "precision@1": 0.857, - "precision@5": 0.1952, - "precision@10": 0.09920000000000001, - "recall@1": 0.857, - "recall@5": 0.976, - "recall@10": 0.992, - "f1@1": 0.857, - "f1@5": 0.32533333333333336, - "f1@10": 0.18036363636363634, - "map@1": 0.857, - "map@5": 0.9043833333333334, - "map@10": 0.9066714285714286, - "mrr@1": 0.857, - "mrr@5": 0.9043833333333334, - "mrr@10": 0.9066714285714286, + "hit@1": 0.859, + "hit@5": 0.977, + "hit@10": 0.993, + "ndcg_linear@1": 0.859, + "ndcg_linear@5": 0.9234870982614233, + "ndcg_linear@10": 0.9286837832976022, + "ndcg_exponential@1": 0.859, + "ndcg_exponential@5": 0.9234870982614233, + "ndcg_exponential@10": 0.9286837832976022, + "precision@1": 0.859, + "precision@5": 0.19540000000000005, + "precision@10": 0.09930000000000001, + "recall@1": 0.859, + "recall@5": 0.977, + "recall@10": 0.993, + "f1@1": 0.859, + "f1@5": 0.3256666666666666, + "f1@10": 0.1805454545454545, + "map@1": 0.859, + "map@5": 0.9055166666666666, + "map@10": 0.9076757936507935, + "mrr@1": 0.859, + "mrr@5": 0.9055166666666666, + "mrr@10": 0.9076757936507935, "num_pred": 1000, "num_data": 1000 }, "MSCOCO": { - "hit@1": 0.757, - "hit@5": 0.903, - "hit@10": 0.937, - "ndcg_linear@1": 0.757, - "ndcg_linear@5": 0.8373195733124786, - "ndcg_linear@10": 0.8484886438288004, - "ndcg_exponential@1": 0.757, - "ndcg_exponential@5": 0.8373195733124786, - "ndcg_exponential@10": 0.8484886438288004, - "precision@1": 0.757, - "precision@5": 0.1806, - "precision@10": 0.09370000000000002, - "recall@1": 0.757, - "recall@5": 0.903, - "recall@10": 0.937, - "f1@1": 0.757, - "f1@5": 0.30100000000000005, - "f1@10": 0.17036363636363638, - "map@1": 0.757, - "map@5": 0.8152, - "map@10": 0.8199079365079365, - "mrr@1": 0.757, - "mrr@5": 0.8152, - "mrr@10": 0.8199079365079365, + "hit@1": 0.754, + "hit@5": 0.902, + "hit@10": 0.94, + "ndcg_linear@1": 0.754, + "ndcg_linear@5": 0.8356762559526358, + "ndcg_linear@10": 0.8479250139295443, + "ndcg_exponential@1": 0.754, + "ndcg_exponential@5": 0.8356762559526358, + "ndcg_exponential@10": 0.8479250139295443, + "precision@1": 0.754, + "precision@5": 0.18040000000000003, + "precision@10": 0.09400000000000001, + "recall@1": 0.754, + "recall@5": 0.902, + "recall@10": 0.94, + "f1@1": 0.754, + "f1@5": 0.3006666666666667, + "f1@10": 0.1709090909090909, + "map@1": 0.754, + "map@5": 0.8133166666666666, + "map@10": 0.81835, + "mrr@1": 0.754, + "mrr@5": 0.8133166666666666, + "mrr@10": 0.81835, "num_pred": 1000, "num_data": 1000 }, "RefCOCO": { - "hit@1": 0.876, + "hit@1": 0.871, "hit@5": 0.988, "hit@10": 0.996, - "ndcg_linear@1": 0.876, - "ndcg_linear@5": 0.9412592336534622, - "ndcg_linear@10": 0.9439672239410106, - "ndcg_exponential@1": 0.876, - "ndcg_exponential@5": 0.9412592336534622, - "ndcg_exponential@10": 0.9439672239410106, - "precision@1": 0.876, + "ndcg_linear@1": 0.871, + "ndcg_linear@5": 0.9391076576643925, + "ndcg_linear@10": 0.9418106425547139, + "ndcg_exponential@1": 0.871, + "ndcg_exponential@5": 0.9391076576643925, + "ndcg_exponential@10": 0.9418106425547139, + "precision@1": 0.871, "precision@5": 0.19760000000000003, "precision@10": 0.09960000000000002, - "recall@1": 0.876, + "recall@1": 0.871, "recall@5": 0.988, "recall@10": 0.996, - "f1@1": 0.876, + "f1@1": 0.871, "f1@5": 0.3293333333333333, "f1@10": 0.18109090909090905, - "map@1": 0.876, - "map@5": 0.9252166666666667, - "map@10": 0.9264051587301587, - "mrr@1": 0.876, - "mrr@5": 0.9252166666666667, - "mrr@10": 0.9264051587301587, + "map@1": 0.871, + "map@5": 0.92235, + "map@10": 0.9235325396825397, + "mrr@1": 0.871, + "mrr@5": 0.92235, + "mrr@10": 0.9235325396825397, "num_pred": 1000, "num_data": 1000 }, "RefCOCO-Matching": { - "hit@1": 0.846, - "hit@5": 0.997, + "hit@1": 0.844, + "hit@5": 0.996, "hit@10": 0.999, - "ndcg_linear@1": 0.846, - "ndcg_linear@5": 0.9419510559475468, - "ndcg_linear@10": 0.9426227280114405, - "ndcg_exponential@1": 0.846, - "ndcg_exponential@5": 0.9419510559475468, - "ndcg_exponential@10": 0.9426227280114405, - "precision@1": 0.846, - "precision@5": 0.19940000000000002, + "ndcg_linear@1": 0.844, + "ndcg_linear@5": 0.9406258094519571, + "ndcg_linear@10": 0.9416536887029587, + "ndcg_exponential@1": 0.844, + "ndcg_exponential@5": 0.9406258094519571, + "ndcg_exponential@10": 0.9416536887029587, + "precision@1": 0.844, + "precision@5": 0.19920000000000004, "precision@10": 0.09990000000000002, - "recall@1": 0.846, - "recall@5": 0.997, + "recall@1": 0.844, + "recall@5": 0.996, "recall@10": 0.999, - "f1@1": 0.846, - "f1@5": 0.3323333333333333, + "f1@1": 0.844, + "f1@5": 0.332, "f1@10": 0.18163636363636362, - "map@1": 0.846, - "map@5": 0.9229833333333334, - "map@10": 0.923275, - "mrr@1": 0.846, - "mrr@5": 0.9199833333333334, - "mrr@10": 0.920275, + "map@1": 0.844, + "map@5": 0.9215333333333333, + "map@10": 0.9219916666666667, + "mrr@1": 0.844, + "mrr@5": 0.9185333333333333, + "mrr@10": 0.9189916666666667, "num_pred": 1000, "num_data": 1000 }, "Visual7W-Pointing": { - "hit@1": 0.81, - "hit@5": 0.941, - "hit@10": 0.958, - "ndcg_linear@1": 0.81, - "ndcg_linear@5": 0.8826264957483172, - "ndcg_linear@10": 0.8881107664102801, - "ndcg_exponential@1": 0.81, - "ndcg_exponential@5": 0.8826264957483172, - "ndcg_exponential@10": 0.8881107664102801, - "precision@1": 0.81, - "precision@5": 0.18820000000000003, - "precision@10": 0.09580000000000001, - "recall@1": 0.81, - "recall@5": 0.941, - "recall@10": 0.958, - "f1@1": 0.81, - "f1@5": 0.31366666666666665, - "f1@10": 0.17418181818181816, - "map@1": 0.81, - "map@5": 0.8629666666666667, - "map@10": 0.8652210317460317, - "mrr@1": 0.81, - "mrr@5": 0.8629666666666667, - "mrr@10": 0.8652210317460317, + "hit@1": 0.807, + "hit@5": 0.943, + "hit@10": 0.96, + "ndcg_linear@1": 0.807, + "ndcg_linear@5": 0.8815429772236839, + "ndcg_linear@10": 0.8869711728042284, + "ndcg_exponential@1": 0.807, + "ndcg_exponential@5": 0.8815429772236839, + "ndcg_exponential@10": 0.8869711728042284, + "precision@1": 0.807, + "precision@5": 0.18860000000000002, + "precision@10": 0.09600000000000002, + "recall@1": 0.807, + "recall@5": 0.943, + "recall@10": 0.96, + "f1@1": 0.807, + "f1@5": 0.31433333333333335, + "f1@10": 0.17454545454545453, + "map@1": 0.807, + "map@5": 0.8609333333333334, + "map@10": 0.8631313492063492, + "mrr@1": 0.807, + "mrr@5": 0.8609333333333334, + "mrr@10": 0.8631313492063492, "num_pred": 1000, "num_data": 1000 } }, "video": { "K700": { - "hit@1": 0.355, - "hit@5": 0.6, - "hit@10": 0.679, - "ndcg_linear@1": 0.355, - "ndcg_linear@5": 0.4845102488689986, - "ndcg_linear@10": 0.510304213516978, - "ndcg_exponential@1": 0.355, - "ndcg_exponential@5": 0.4845102488689986, - "ndcg_exponential@10": 0.510304213516978, - "precision@1": 0.355, - "precision@5": 0.12, - "precision@10": 0.06789999999999999, - "recall@1": 0.355, - "recall@5": 0.6, - "recall@10": 0.679, - "f1@1": 0.355, - "f1@5": 0.20000000000000007, - "f1@10": 0.12345454545454547, - "map@1": 0.355, - "map@5": 0.44616666666666666, - "map@10": 0.4569583333333333, - "mrr@1": 0.355, - "mrr@5": 0.44616666666666666, - "mrr@10": 0.4569583333333333, + "hit@1": 0.354, + "hit@5": 0.604, + "hit@10": 0.681, + "ndcg_linear@1": 0.354, + "ndcg_linear@5": 0.4860890962425044, + "ndcg_linear@10": 0.5109916705720554, + "ndcg_exponential@1": 0.354, + "ndcg_exponential@5": 0.4860890962425044, + "ndcg_exponential@10": 0.5109916705720554, + "precision@1": 0.354, + "precision@5": 0.12079999999999998, + "precision@10": 0.0681, + "recall@1": 0.354, + "recall@5": 0.604, + "recall@10": 0.681, + "f1@1": 0.354, + "f1@5": 0.20133333333333336, + "f1@10": 0.12381818181818184, + "map@1": 0.354, + "map@5": 0.4469666666666667, + "map@10": 0.4572424603174603, + "mrr@1": 0.354, + "mrr@5": 0.4469666666666667, + "mrr@10": 0.4572424603174603, "num_pred": 1000, "num_data": 1000 }, "SmthSmthV2": { - "hit@1": 0.321, + "hit@1": 0.32, "hit@5": 0.593, - "hit@10": 0.695, - "ndcg_linear@1": 0.321, - "ndcg_linear@5": 0.464820244083625, - "ndcg_linear@10": 0.49815926994038934, - "ndcg_exponential@1": 0.321, - "ndcg_exponential@5": 0.464820244083625, - "ndcg_exponential@10": 0.49815926994038934, - "precision@1": 0.321, + "hit@10": 0.692, + "ndcg_linear@1": 0.32, + "ndcg_linear@5": 0.4641045566275634, + "ndcg_linear@10": 0.4967244336762492, + "ndcg_exponential@1": 0.32, + "ndcg_exponential@5": 0.4641045566275634, + "ndcg_exponential@10": 0.4967244336762492, + "precision@1": 0.32, "precision@5": 0.1186, - "precision@10": 0.0695, - "recall@1": 0.321, + "precision@10": 0.06919999999999998, + "recall@1": 0.32, "recall@5": 0.593, - "recall@10": 0.695, - "f1@1": 0.321, + "recall@10": 0.692, + "f1@1": 0.32, "f1@5": 0.19766666666666674, - "f1@10": 0.1263636363636364, - "map@1": 0.321, - "map@5": 0.4222333333333333, - "map@10": 0.43619880952380946, - "mrr@1": 0.321, - "mrr@5": 0.4222333333333333, - "mrr@10": 0.43619880952380946, + "f1@10": 0.12581818181818183, + "map@1": 0.32, + "map@5": 0.4213166666666667, + "map@10": 0.43512817460317454, + "mrr@1": 0.32, + "mrr@5": 0.4213166666666667, + "mrr@10": 0.43512817460317454, "num_pred": 1000, "num_data": 1000 }, "HMDB51": { - "hit@1": 0.422, - "hit@5": 0.684, - "hit@10": 0.83, - "ndcg_linear@1": 0.422, - "ndcg_linear@5": 0.5578823382949479, - "ndcg_linear@10": 0.605271745232144, - "ndcg_exponential@1": 0.422, - "ndcg_exponential@5": 0.5578823382949479, - "ndcg_exponential@10": 0.605271745232144, - "precision@1": 0.422, - "precision@5": 0.1368, - "precision@10": 0.083, - "recall@1": 0.422, - "recall@5": 0.684, - "recall@10": 0.83, - "f1@1": 0.422, - "f1@5": 0.22800000000000006, - "f1@10": 0.15090909090909094, - "map@1": 0.422, - "map@5": 0.5162333333333333, - "map@10": 0.5358904761904761, - "mrr@1": 0.422, - "mrr@5": 0.5162333333333333, - "mrr@10": 0.5358904761904761, + "hit@1": 0.419, + "hit@5": 0.688, + "hit@10": 0.826, + "ndcg_linear@1": 0.419, + "ndcg_linear@5": 0.5579125084620252, + "ndcg_linear@10": 0.6025433435063025, + "ndcg_exponential@1": 0.419, + "ndcg_exponential@5": 0.5579125084620252, + "ndcg_exponential@10": 0.6025433435063025, + "precision@1": 0.419, + "precision@5": 0.1376, + "precision@10": 0.08259999999999999, + "recall@1": 0.419, + "recall@5": 0.688, + "recall@10": 0.826, + "f1@1": 0.419, + "f1@5": 0.2293333333333334, + "f1@10": 0.1501818181818182, + "map@1": 0.419, + "map@5": 0.5149833333333333, + "map@10": 0.5334099206349207, + "mrr@1": 0.419, + "mrr@5": 0.5149833333333333, + "mrr@10": 0.5334099206349207, "num_pred": 1000, "num_data": 1000 }, "UCF101": { - "hit@1": 0.618, - "hit@5": 0.889, - "hit@10": 0.94, - "ndcg_linear@1": 0.618, - "ndcg_linear@5": 0.772430845929742, - "ndcg_linear@10": 0.7893494678275759, - "ndcg_exponential@1": 0.618, - "ndcg_exponential@5": 0.772430845929742, - "ndcg_exponential@10": 0.7893494678275759, - "precision@1": 0.618, - "precision@5": 0.1778, - "precision@10": 0.09400000000000001, - "recall@1": 0.618, - "recall@5": 0.889, - "recall@10": 0.94, - "f1@1": 0.618, - "f1@5": 0.2963333333333334, - "f1@10": 0.1709090909090909, - "map@1": 0.618, - "map@5": 0.7326833333333334, - "map@10": 0.7399230158730159, - "mrr@1": 0.618, - "mrr@5": 0.7326833333333334, - "mrr@10": 0.7399230158730159, + "hit@1": 0.621, + "hit@5": 0.892, + "hit@10": 0.942, + "ndcg_linear@1": 0.621, + "ndcg_linear@5": 0.7751530106630904, + "ndcg_linear@10": 0.7916543715427622, + "ndcg_exponential@1": 0.621, + "ndcg_exponential@5": 0.7751530106630904, + "ndcg_exponential@10": 0.7916543715427622, + "precision@1": 0.621, + "precision@5": 0.17840000000000003, + "precision@10": 0.09420000000000002, + "recall@1": 0.621, + "recall@5": 0.892, + "recall@10": 0.942, + "f1@1": 0.621, + "f1@5": 0.2973333333333334, + "f1@10": 0.17127272727272724, + "map@1": 0.621, + "map@5": 0.7353666666666668, + "map@10": 0.7423742063492064, + "mrr@1": 0.621, + "mrr@5": 0.7353666666666668, + "mrr@10": 0.7423742063492064, "num_pred": 1000, "num_data": 1000 }, "Breakfast": { - "hit@1": 0.23787528868360278, - "hit@5": 0.651270207852194, + "hit@1": 0.23556581986143188, + "hit@5": 0.648960739030023, "hit@10": 1.0, - "ndcg_linear@1": 0.23787528868360278, - "ndcg_linear@5": 0.4481888236256693, - "ndcg_linear@10": 0.5633176174385133, - "ndcg_exponential@1": 0.23787528868360278, - "ndcg_exponential@5": 0.4481888236256693, - "ndcg_exponential@10": 0.5633176174385133, - "precision@1": 0.23787528868360278, - "precision@5": 0.13025404157043877, + "ndcg_linear@1": 0.23556581986143188, + "ndcg_linear@5": 0.44622155068209507, + "ndcg_linear@10": 0.5619170052684441, + "ndcg_exponential@1": 0.23556581986143188, + "ndcg_exponential@5": 0.44622155068209507, + "ndcg_exponential@10": 0.5619170052684441, + "precision@1": 0.23556581986143188, + "precision@5": 0.12979214780600462, "precision@10": 0.1, - "recall@1": 0.23787528868360278, - "recall@5": 0.651270207852194, + "recall@1": 0.23556581986143188, + "recall@5": 0.648960739030023, "recall@10": 1.0, - "f1@1": 0.23787528868360278, - "f1@5": 0.21709006928406474, + "f1@1": 0.23556581986143188, + "f1@5": 0.21632024634334107, "f1@10": 0.1818181818181818, - "map@1": 0.23787528868360278, - "map@5": 0.3814087759815243, - "map@10": 0.4303328567762748, - "mrr@1": 0.23787528868360278, - "mrr@5": 0.3814087759815243, - "mrr@10": 0.4303328567762748, + "map@1": 0.23556581986143188, + "map@5": 0.3795996920708237, + "map@10": 0.4286364969390374, + "mrr@1": 0.23556581986143188, + "mrr@5": 0.3795996920708237, + "mrr@10": 0.4286364969390374, "num_pred": 433, "num_data": 433 }, "MVBench": { - "hit@1": 0.28475, + "hit@1": 0.2855, "hit@5": 1.0, "hit@10": 1.0, - "ndcg_linear@1": 0.28475, - "ndcg_linear@5": 0.6742128560665944, - "ndcg_linear@10": 0.6742128560665944, - "ndcg_exponential@1": 0.28475, - "ndcg_exponential@5": 0.6742128560665944, - "ndcg_exponential@10": 0.6742128560665944, - "precision@1": 0.28475, + "ndcg_linear@1": 0.2855, + "ndcg_linear@5": 0.6744549970304524, + "ndcg_linear@10": 0.6744549970304524, + "ndcg_exponential@1": 0.2855, + "ndcg_exponential@5": 0.6744549970304524, + "ndcg_exponential@10": 0.6744549970304524, + "precision@1": 0.2855, "precision@5": 0.20000000000000004, "precision@10": 0.10000000000000002, - "recall@1": 0.28475, + "recall@1": 0.2855, "recall@5": 1.0, "recall@10": 1.0, - "f1@1": 0.28475, + "f1@1": 0.2855, "f1@5": 0.3333333333333333, "f1@10": 0.18181818181818182, - "map@1": 0.28475, - "map@5": 0.5643916666666666, - "map@10": 0.5643916666666666, - "mrr@1": 0.28475, - "mrr@5": 0.5643916666666666, - "mrr@10": 0.5643916666666666, + "map@1": 0.2855, + "map@5": 0.5647249999999999, + "map@10": 0.5647249999999999, + "mrr@1": 0.2855, + "mrr@5": 0.5647249999999999, + "mrr@10": 0.5647249999999999, "num_pred": 4000, "num_data": 4000 }, "Video-MME": { - "hit@1": 0.27814814814814814, + "hit@1": 0.2803703703703704, "hit@5": 1.0, "hit@10": 1.0, - "ndcg_linear@1": 0.27814814814814814, - "ndcg_linear@5": 0.660078255498731, - "ndcg_linear@10": 0.660078255498731, - "ndcg_exponential@1": 0.27814814814814814, - "ndcg_exponential@5": 0.660078255498731, - "ndcg_exponential@10": 0.660078255498731, - "precision@1": 0.27814814814814814, + "ndcg_linear@1": 0.2803703703703704, + "ndcg_linear@5": 0.6604876544388624, + "ndcg_linear@10": 0.6604876544388624, + "ndcg_exponential@1": 0.2803703703703704, + "ndcg_exponential@5": 0.6604876544388624, + "ndcg_exponential@10": 0.6604876544388624, + "precision@1": 0.2803703703703704, "precision@5": 0.19999999999999996, "precision@10": 0.09999999999999998, - "recall@1": 0.27814814814814814, + "recall@1": 0.2803703703703704, "recall@5": 1.0, "recall@10": 1.0, - "f1@1": 0.27814814814814814, + "f1@1": 0.2803703703703704, "f1@5": 0.3333333333333334, "f1@10": 0.18181818181818188, - "map@1": 0.27814814814814814, - "map@5": 0.5465432098765431, - "map@10": 0.5465432098765431, - "mrr@1": 0.27814814814814814, - "mrr@5": 0.5465432098765431, - "mrr@10": 0.5465432098765431, + "map@1": 0.2803703703703704, + "map@5": 0.5471296296296296, + "map@10": 0.5471296296296296, + "mrr@1": 0.2803703703703704, + "mrr@5": 0.5471296296296296, + "mrr@10": 0.5471296296296296, "num_pred": 2700, "num_data": 2700 }, "NExTQA": { - "hit@1": 0.20294255021018215, + "hit@1": 0.20282578234469875, "hit@5": 1.0, "hit@10": 1.0, - "ndcg_linear@1": 0.20294255021018215, - "ndcg_linear@5": 0.604249894952272, - "ndcg_linear@10": 0.604249894952272, - "ndcg_exponential@1": 0.20294255021018215, - "ndcg_exponential@5": 0.604249894952272, - "ndcg_exponential@10": 0.604249894952272, - "precision@1": 0.20294255021018215, + "ndcg_linear@1": 0.20282578234469875, + "ndcg_linear@5": 0.6042123805082606, + "ndcg_linear@10": 0.6042123805082606, + "ndcg_exponential@1": 0.20282578234469875, + "ndcg_exponential@5": 0.6042123805082606, + "ndcg_exponential@10": 0.6042123805082606, + "precision@1": 0.20282578234469875, "precision@5": 0.20000000000000004, "precision@10": 0.10000000000000002, - "recall@1": 0.20294255021018215, + "recall@1": 0.20282578234469875, "recall@5": 1.0, "recall@10": 1.0, - "f1@1": 0.20294255021018215, + "f1@1": 0.20282578234469875, "f1@5": 0.3333333333333333, "f1@10": 0.1818181818181818, - "map@1": 0.20294255021018215, - "map@5": 0.4747664642690332, - "map@10": 0.4747664642690332, - "mrr@1": 0.20294255021018215, - "mrr@5": 0.4746029892573564, - "mrr@10": 0.4746029892573564, + "map@1": 0.20282578234469875, + "map@5": 0.47471197259847425, + "map@10": 0.47471197259847425, + "mrr@1": 0.20282578234469875, + "mrr@5": 0.47454849758679746, + "mrr@10": 0.47454849758679746, "num_pred": 8564, "num_data": 8564 }, "EgoSchema": { - "hit@1": 0.218, + "hit@1": 0.222, "hit@5": 1.0, "hit@10": 1.0, - "ndcg_linear@1": 0.218, - "ndcg_linear@5": 0.5992262804502428, - "ndcg_linear@10": 0.5992262804502428, - "ndcg_exponential@1": 0.218, - "ndcg_exponential@5": 0.5992262804502428, - "ndcg_exponential@10": 0.5992262804502428, - "precision@1": 0.218, + "ndcg_linear@1": 0.222, + "ndcg_linear@5": 0.6024123653626677, + "ndcg_linear@10": 0.6024123653626677, + "ndcg_exponential@1": 0.222, + "ndcg_exponential@5": 0.6024123653626677, + "ndcg_exponential@10": 0.6024123653626677, + "precision@1": 0.222, "precision@5": 0.2, "precision@10": 0.1, - "recall@1": 0.218, + "recall@1": 0.222, "recall@5": 1.0, "recall@10": 1.0, - "f1@1": 0.218, + "f1@1": 0.222, "f1@5": 0.3333333333333333, "f1@10": 0.18181818181818182, - "map@1": 0.218, - "map@5": 0.46926666666666667, - "map@10": 0.46926666666666667, - "mrr@1": 0.218, - "mrr@5": 0.46926666666666667, - "mrr@10": 0.46926666666666667, + "map@1": 0.222, + "map@5": 0.4734333333333333, + "map@10": 0.4734333333333333, + "mrr@1": 0.222, + "mrr@5": 0.4734333333333333, + "mrr@10": 0.4734333333333333, "num_pred": 500, "num_data": 500 }, @@ -1299,211 +1299,239 @@ "num_data": 1000 }, "DiDeMo": { - "hit@1": 0.29282868525896416, - "hit@5": 0.5348605577689243, - "hit@10": 0.6215139442231076, - "ndcg_linear@1": 0.29282868525896416, - "ndcg_linear@5": 0.42005110007964297, - "ndcg_linear@10": 0.4483095791534034, - "ndcg_exponential@1": 0.29282868525896416, - "ndcg_exponential@5": 0.42005110007964297, - "ndcg_exponential@10": 0.4483095791534034, - "precision@1": 0.29282868525896416, - "precision@5": 0.10697211155378486, - "precision@10": 0.062151394422310755, - "recall@1": 0.29282868525896416, - "recall@5": 0.5348605577689243, - "recall@10": 0.6215139442231076, - "f1@1": 0.29282868525896416, - "f1@5": 0.1782868525896415, - "f1@10": 0.1130025353132923, - "map@1": 0.29282868525896416, - "map@5": 0.3820053120849934, - "map@10": 0.3938029627521659, - "mrr@1": 0.29282868525896416, - "mrr@5": 0.3820053120849934, - "mrr@10": 0.3938029627521659, + "hit@1": 0.2958167330677291, + "hit@5": 0.5358565737051793, + "hit@10": 0.6195219123505976, + "ndcg_linear@1": 0.2958167330677291, + "ndcg_linear@5": 0.42201539490899526, + "ndcg_linear@10": 0.44942310423704845, + "ndcg_exponential@1": 0.2958167330677291, + "ndcg_exponential@5": 0.42201539490899526, + "ndcg_exponential@10": 0.44942310423704845, + "precision@1": 0.2958167330677291, + "precision@5": 0.10717131474103585, + "precision@10": 0.06195219123505976, + "recall@1": 0.2958167330677291, + "recall@5": 0.5358565737051793, + "recall@10": 0.6195219123505976, + "f1@1": 0.2958167330677291, + "f1@5": 0.17861885790172646, + "f1@10": 0.11264034770010867, + "map@1": 0.2958167330677291, + "map@5": 0.38426294820717133, + "map@10": 0.39577127995952693, + "mrr@1": 0.2958167330677291, + "mrr@5": 0.38426294820717133, + "mrr@10": 0.39577127995952693, "num_pred": 1004, "num_data": 1004 }, "MSR-VTT": { - "hit@1": 0.345, - "hit@5": 0.585, - "hit@10": 0.679, - "ndcg_linear@1": 0.345, - "ndcg_linear@5": 0.4705689385818224, - "ndcg_linear@10": 0.5008515039770559, - "ndcg_exponential@1": 0.345, - "ndcg_exponential@5": 0.4705689385818224, - "ndcg_exponential@10": 0.5008515039770559, - "precision@1": 0.345, - "precision@5": 0.11699999999999998, - "precision@10": 0.06789999999999999, - "recall@1": 0.345, - "recall@5": 0.585, - "recall@10": 0.679, - "f1@1": 0.345, - "f1@5": 0.19500000000000006, - "f1@10": 0.12345454545454547, - "map@1": 0.345, - "map@5": 0.4327, - "map@10": 0.4451373015873016, - "mrr@1": 0.345, - "mrr@5": 0.4327, - "mrr@10": 0.4451373015873016, + "hit@1": 0.347, + "hit@5": 0.578, + "hit@10": 0.681, + "ndcg_linear@1": 0.347, + "ndcg_linear@5": 0.46936690538466047, + "ndcg_linear@10": 0.5028654388795876, + "ndcg_exponential@1": 0.347, + "ndcg_exponential@5": 0.46936690538466047, + "ndcg_exponential@10": 0.5028654388795876, + "precision@1": 0.347, + "precision@5": 0.1156, + "precision@10": 0.0681, + "recall@1": 0.347, + "recall@5": 0.578, + "recall@10": 0.681, + "f1@1": 0.347, + "f1@5": 0.19266666666666674, + "f1@10": 0.12381818181818184, + "map@1": 0.347, + "map@5": 0.43326666666666663, + "map@10": 0.4472130952380952, + "mrr@1": 0.347, + "mrr@5": 0.43326666666666663, + "mrr@10": 0.4472130952380952, "num_pred": 1000, "num_data": 1000 }, "MSVD": { "hit@1": 0.4671641791044776, - "hit@5": 0.744776119402985, - "hit@10": 0.8313432835820895, + "hit@5": 0.7477611940298508, + "hit@10": 0.826865671641791, "ndcg_linear@1": 0.4671641791044776, - "ndcg_linear@5": 0.6143164752221116, - "ndcg_linear@10": 0.6423987059237167, + "ndcg_linear@5": 0.617134562960507, + "ndcg_linear@10": 0.6422898259536498, "ndcg_exponential@1": 0.4671641791044776, - "ndcg_exponential@5": 0.6143164752221116, - "ndcg_exponential@10": 0.6423987059237167, + "ndcg_exponential@5": 0.617134562960507, + "ndcg_exponential@10": 0.6422898259536498, "precision@1": 0.4671641791044776, - "precision@5": 0.14895522388059698, - "precision@10": 0.08313432835820894, + "precision@5": 0.14955223880597013, + "precision@10": 0.08268656716417909, "recall@1": 0.4671641791044776, - "recall@5": 0.744776119402985, - "recall@10": 0.8313432835820895, + "recall@5": 0.7477611940298508, + "recall@10": 0.826865671641791, "f1@1": 0.4671641791044776, - "f1@5": 0.24825870646766174, - "f1@10": 0.15115332428765269, + "f1@5": 0.24925373134328363, + "f1@10": 0.1503392130257802, "map@1": 0.4671641791044776, - "map@5": 0.5709203980099502, - "map@10": 0.5825675195451315, + "map@5": 0.5735820895522388, + "map@10": 0.5837153518123668, "mrr@1": 0.4671641791044776, - "mrr@5": 0.5709203980099502, - "mrr@10": 0.5825675195451315, + "mrr@5": 0.5735820895522388, + "mrr@10": 0.5837153518123668, "num_pred": 670, "num_data": 670 }, "VATEX": { - "hit@1": 0.25524787851719516, - "hit@5": 0.5064761054041983, - "hit@10": 0.6192496650290308, - "ndcg_linear@1": 0.25524787851719516, - "ndcg_linear@5": 0.38626309581533674, - "ndcg_linear@10": 0.42289114395020533, - "ndcg_exponential@1": 0.25524787851719516, - "ndcg_exponential@5": 0.38626309581533674, - "ndcg_exponential@10": 0.42289114395020533, - "precision@1": 0.25524787851719516, - "precision@5": 0.10129522108083966, - "precision@10": 0.06192496650290307, - "recall@1": 0.25524787851719516, - "recall@5": 0.5064761054041983, - "recall@10": 0.6192496650290308, - "f1@1": 0.25524787851719516, - "f1@5": 0.16882536846806612, - "f1@10": 0.11259084818709653, - "map@1": 0.25524787851719516, - "map@5": 0.34648280482358196, - "map@10": 0.3616905754411337, - "mrr@1": 0.25524787851719516, - "mrr@5": 0.34648280482358196, - "mrr@10": 0.3616905754411337, + "hit@1": 0.25368468066100935, + "hit@5": 0.5046895935685574, + "hit@10": 0.6161232693166592, + "ndcg_linear@1": 0.25368468066100935, + "ndcg_linear@5": 0.3852947301778341, + "ndcg_linear@10": 0.421559309338407, + "ndcg_exponential@1": 0.25368468066100935, + "ndcg_exponential@5": 0.3852947301778341, + "ndcg_exponential@10": 0.421559309338407, + "precision@1": 0.25368468066100935, + "precision@5": 0.10093791871371148, + "precision@10": 0.06161232693166592, + "recall@1": 0.25368468066100935, + "recall@5": 0.5046895935685574, + "recall@10": 0.6161232693166592, + "f1@1": 0.25368468066100935, + "f1@5": 0.1682298645228525, + "f1@10": 0.11202241260302895, + "map@1": 0.25368468066100935, + "map@5": 0.34572725919309216, + "map@10": 0.36082700610391544, + "mrr@1": 0.25368468066100935, + "mrr@5": 0.34572725919309216, + "mrr@10": 0.36082700610391544, "num_pred": 4478, "num_data": 4478 }, "YouCook2": { - "hit@1": 0.08996539792387544, - "hit@5": 0.21453287197231835, - "hit@10": 0.28908461780434097, - "ndcg_linear@1": 0.08996539792387544, - "ndcg_linear@5": 0.15381567274119107, - "ndcg_linear@10": 0.1779025046239055, - "ndcg_exponential@1": 0.08996539792387544, - "ndcg_exponential@5": 0.15381567274119107, - "ndcg_exponential@10": 0.1779025046239055, - "precision@1": 0.08996539792387544, - "precision@5": 0.04290657439446367, - "precision@10": 0.0289084617804341, - "recall@1": 0.08996539792387544, - "recall@5": 0.21453287197231835, - "recall@10": 0.28908461780434097, - "f1@1": 0.08996539792387544, - "f1@5": 0.07151095732410613, - "f1@10": 0.05256083960078927, - "map@1": 0.08996539792387544, - "map@5": 0.13384187899758834, - "map@10": 0.14376925458240336, - "mrr@1": 0.08996539792387544, - "mrr@5": 0.13384187899758834, - "mrr@10": 0.14376925458240336, + "hit@1": 0.09027996225228059, + "hit@5": 0.21610569361434415, + "hit@10": 0.2875117961623152, + "ndcg_linear@1": 0.09027996225228059, + "ndcg_linear@5": 0.15502066411414483, + "ndcg_linear@10": 0.17804207732399388, + "ndcg_exponential@1": 0.09027996225228059, + "ndcg_exponential@5": 0.15502066411414483, + "ndcg_exponential@10": 0.17804207732399388, + "precision@1": 0.09027996225228059, + "precision@5": 0.04322113872286882, + "precision@10": 0.028751179616231522, + "recall@1": 0.09027996225228059, + "recall@5": 0.21610569361434415, + "recall@10": 0.2875117961623152, + "f1@1": 0.09027996225228059, + "f1@5": 0.07203523120478139, + "f1@10": 0.05227487202951186, + "map@1": 0.09027996225228059, + "map@5": 0.13492712593058612, + "map@10": 0.1443897701683169, + "mrr@1": 0.09027996225228059, + "mrr@5": 0.13492712593058612, + "mrr@10": 0.1443897701683169, "num_pred": 3179, "num_data": 3179 }, "QVHighlight": { - "hit@1": 0.577100646352724, - "hit@5": 0.8947368421052632, + "hit@1": 0.5789473684210527, + "hit@5": 0.8938134810710988, "hit@10": 1.0, - "ndcg_linear@1": 0.577100646352724, - "ndcg_linear@5": 0.7461876317485063, - "ndcg_linear@10": 0.7806995799400307, - "ndcg_exponential@1": 0.577100646352724, - "ndcg_exponential@5": 0.7461876317485063, - "ndcg_exponential@10": 0.7806995799400307, - "precision@1": 0.577100646352724, - "precision@5": 0.17894736842105263, + "ndcg_linear@1": 0.5789473684210527, + "ndcg_linear@5": 0.7463670562563719, + "ndcg_linear@10": 0.7812106314776668, + "ndcg_exponential@1": 0.5789473684210527, + "ndcg_exponential@5": 0.7463670562563719, + "ndcg_exponential@10": 0.7812106314776668, + "precision@1": 0.5789473684210527, + "precision@5": 0.17876269621421975, "precision@10": 0.09999999999999998, - "recall@1": 0.577100646352724, - "recall@5": 0.8947368421052632, + "recall@1": 0.5789473684210527, + "recall@5": 0.8938134810710988, "recall@10": 1.0, - "f1@1": 0.577100646352724, - "f1@5": 0.2982456140350878, + "f1@1": 0.5789473684210527, + "f1@5": 0.2979378270236997, "f1@10": 0.18181818181818185, - "map@1": 0.577100646352724, - "map@5": 0.6967220683287165, - "map@10": 0.7112386228729719, - "mrr@1": 0.577100646352724, - "mrr@5": 0.6967220683287165, - "mrr@10": 0.7112386228729719, + "map@1": 0.5789473684210527, + "map@5": 0.6972914742997847, + "map@10": 0.7119593574579723, + "mrr@1": 0.5789473684210527, + "mrr@5": 0.6972914742997847, + "mrr@10": 0.7119593574579723, "num_pred": 1083, "num_data": 1083 }, "Charades-STA": { - "hit@1": 0.19807427785419532, - "hit@5": 0.5818431911966988, + "hit@1": 0.1856946354883081, + "hit@5": 0.5680880330123796, "hit@10": 1.0, - "ndcg_linear@1": 0.19807427785419532, - "ndcg_linear@5": 0.3931810758884146, - "ndcg_linear@10": 0.5256230138842131, - "ndcg_exponential@1": 0.19807427785419532, - "ndcg_exponential@5": 0.3931810758884146, - "ndcg_exponential@10": 0.5256230138842131, - "precision@1": 0.19807427785419532, - "precision@5": 0.11636863823933974, + "ndcg_linear@1": 0.1856946354883081, + "ndcg_linear@5": 0.383205411375633, + "ndcg_linear@10": 0.5203588369423373, + "ndcg_exponential@1": 0.1856946354883081, + "ndcg_exponential@5": 0.383205411375633, + "ndcg_exponential@10": 0.5203588369423373, + "precision@1": 0.1856946354883081, + "precision@5": 0.11361760660247593, "precision@10": 0.09999999999999998, - "recall@1": 0.19807427785419532, - "recall@5": 0.5818431911966988, + "recall@1": 0.1856946354883081, + "recall@5": 0.5680880330123796, "recall@10": 1.0, - "f1@1": 0.19807427785419532, - "f1@5": 0.19394773039889962, + "f1@1": 0.1856946354883081, + "f1@5": 0.18936267767079323, "f1@10": 0.18181818181818185, - "map@1": 0.19807427785419532, - "map@5": 0.3313388353966071, - "map@10": 0.38442446671470054, - "mrr@1": 0.19807427785419532, - "mrr@5": 0.3313388353966071, - "mrr@10": 0.38442446671470054, + "map@1": 0.1856946354883081, + "map@5": 0.3223062815222375, + "map@10": 0.37749667037837603, + "mrr@1": 0.1856946354883081, + "mrr@5": 0.3223062815222375, + "mrr@10": 0.37749667037837603, "num_pred": 727, "num_data": 727 }, "MomentSeeker": { + "hit@1": 0.4101123595505618, + "hit@5": 0.8926342072409488, + "hit@10": 1.0, + "ndcg_linear@1": 0.4101123595505618, + "ndcg_linear@5": 0.6600653127570068, + "ndcg_linear@10": 0.7033465172353367, + "ndcg_exponential@1": 0.4101123595505618, + "ndcg_exponential@5": 0.6600653127570068, + "ndcg_exponential@10": 0.7033465172353367, + "precision@1": 0.4101123595505618, + "precision@5": 0.19588014981273408, + "precision@10": 0.11448189762796505, + "recall@1": 0.38516957969205157, + "recall@5": 0.872477111943404, + "recall@10": 0.998491468997087, + "f1@1": 0.39288884925589046, + "f1@5": 0.3147117309102328, + "f1@10": 0.20249326897266975, + "map@1": 0.4101123595505618, + "map@5": 0.5833506727701484, + "map@10": 0.6041062057018673, + "mrr@1": 0.4101123595505618, + "mrr@5": 0.594548481065335, + "mrr@10": 0.609163644650536, + "num_pred": 1602, + "num_data": 1602 + }, + "MomentSeeker_1k8": { "hit@1": 0.3927777777777778, "hit@5": 0.8838888888888888, "hit@10": 1.0, "ndcg_linear@1": 0.3927777777777778, - "ndcg_linear@5": 0.6451319149954187, - "ndcg_linear@10": 0.6903053809896799, + "ndcg_linear@5": 0.6455154388019301, + "ndcg_linear@10": 0.6906841041633804, "ndcg_exponential@1": 0.3927777777777778, - "ndcg_exponential@5": 0.6451319149954187, - "ndcg_exponential@10": 0.6903053809896799, + "ndcg_exponential@5": 0.6455154388019301, + "ndcg_exponential@10": 0.6906841041633804, "precision@1": 0.3927777777777778, "precision@5": 0.19233333333333336, "precision@10": 0.11294444444444444, @@ -1514,685 +1542,629 @@ "f1@5": 0.31004053045719715, "f1@10": 0.20030318446985115, "map@1": 0.3927777777777778, - "map@5": 0.5665493827160493, - "map@10": 0.5877770980305702, + "map@5": 0.5670246913580247, + "map@10": 0.5882374154908876, "mrr@1": 0.3927777777777778, - "mrr@5": 0.5764259259259259, - "mrr@10": 0.592172619047619, + "mrr@5": 0.5769444444444445, + "mrr@10": 0.5926955467372134, "num_pred": 1800, "num_data": 1800 } }, "visdoc": { "ViDoRe_arxivqa": { - "hit@1": 0.506, - "hit@5": 0.684, - "hit@10": 0.736, - "ndcg_linear@1": 0.506, - "ndcg_linear@5": 0.6019848283346673, - "ndcg_linear@10": 0.618842382116875, - "ndcg_exponential@1": 0.506, - "ndcg_exponential@5": 0.6019848283346673, - "ndcg_exponential@10": 0.618842382116875, - "precision@1": 0.506, - "precision@5": 0.13679999999999998, - "precision@10": 0.0736, - "recall@1": 0.506, - "recall@5": 0.684, - "recall@10": 0.736, - "f1@1": 0.506, - "f1@5": 0.22800000000000006, - "f1@10": 0.1338181818181818, - "map@1": 0.506, - "map@5": 0.5745333333333333, - "map@10": 0.5815087301587302, - "mrr@1": 0.506, - "mrr@5": 0.5745333333333333, - "mrr@10": 0.5815087301587302, + "hit@1": 0.17, + "hit@5": 0.384, + "hit@10": 0.476, + "ndcg_linear@1": 0.17, + "ndcg_linear@5": 0.2816080650570383, + "ndcg_linear@10": 0.3113006675776414, + "ndcg_exponential@1": 0.17, + "ndcg_exponential@5": 0.2816080650570383, + "ndcg_exponential@10": 0.3113006675776414, + "precision@1": 0.17, + "precision@5": 0.0768, + "precision@10": 0.047599999999999996, + "recall@1": 0.17, + "recall@5": 0.384, + "recall@10": 0.476, + "f1@1": 0.17, + "f1@5": 0.12800000000000003, + "f1@10": 0.08654545454545456, + "map@1": 0.17, + "map@5": 0.24783333333333332, + "map@10": 0.2600603174603175, + "mrr@1": 0.17, + "mrr@5": 0.24783333333333332, + "mrr@10": 0.2600603174603175, "num_pred": 500, "num_data": 500 }, "ViDoRe_docvqa": { - "hit@1": 0.270509977827051, - "hit@5": 0.42572062084257206, - "hit@10": 0.5188470066518847, - "ndcg_linear@1": 0.270509977827051, - "ndcg_linear@5": 0.3471381693961765, - "ndcg_linear@10": 0.3761073342818233, - "ndcg_exponential@1": 0.270509977827051, - "ndcg_exponential@5": 0.3471381693961765, - "ndcg_exponential@10": 0.3761073342818233, - "precision@1": 0.270509977827051, - "precision@5": 0.08558758314855876, - "precision@10": 0.05232815964523281, - "recall@1": 0.2668144863266815, - "recall@5": 0.42165558019216554, - "recall@10": 0.5104212860310421, - "f1@1": 0.2679231337767923, - "f1@5": 0.14195966634991028, - "f1@10": 0.09464696427002414, - "map@1": 0.270509977827051, - "map@5": 0.32139689578713965, - "map@10": 0.33320047161510574, - "mrr@1": 0.270509977827051, - "mrr@5": 0.3251662971175166, - "mrr@10": 0.3374995600605356, + "hit@1": 0.11751662971175167, + "hit@5": 0.2638580931263858, + "hit@10": 0.3303769401330377, + "ndcg_linear@1": 0.11751662971175167, + "ndcg_linear@5": 0.18981713322425658, + "ndcg_linear@10": 0.20953486119486092, + "ndcg_exponential@1": 0.11751662971175167, + "ndcg_exponential@5": 0.18981713322425658, + "ndcg_exponential@10": 0.20953486119486092, + "precision@1": 0.11751662971175167, + "precision@5": 0.05365853658536586, + "precision@10": 0.03348115299334812, + "recall@1": 0.114560236511456, + "recall@5": 0.25849963045084995, + "recall@10": 0.3199186991869918, + "f1@1": 0.11529933481152993, + "f1@5": 0.08808157409305187, + "f1@10": 0.06017810718475908, + "map@1": 0.11751662971175167, + "map@5": 0.16596452328159644, + "map@10": 0.17373913349523104, + "mrr@1": 0.11751662971175167, + "mrr@5": 0.16929046563192904, + "mrr@10": 0.17776633935170522, "num_pred": 451, "num_data": 451 }, "ViDoRe_infovqa": { - "hit@1": 0.597165991902834, - "hit@5": 0.7894736842105263, - "hit@10": 0.8522267206477733, - "ndcg_linear@1": 0.597165991902834, - "ndcg_linear@5": 0.7037600234071207, - "ndcg_linear@10": 0.7251514237863242, - "ndcg_exponential@1": 0.597165991902834, - "ndcg_exponential@5": 0.7037600234071207, - "ndcg_exponential@10": 0.7251514237863242, - "precision@1": 0.597165991902834, - "precision@5": 0.15789473684210525, - "precision@10": 0.08542510121457489, - "recall@1": 0.597165991902834, - "recall@5": 0.7884615384615384, - "recall@10": 0.8522267206477733, - "f1@1": 0.597165991902834, - "f1@5": 0.26306149990360517, - "f1@10": 0.1552570236780763, - "map@1": 0.597165991902834, - "map@5": 0.6750674763832659, - "map@10": 0.6844386607544503, - "mrr@1": 0.597165991902834, - "mrr@5": 0.6752699055330635, - "mrr@10": 0.6843037079879185, + "hit@1": 0.3259109311740891, + "hit@5": 0.5566801619433198, + "hit@10": 0.6497975708502024, + "ndcg_linear@1": 0.3259109311740891, + "ndcg_linear@5": 0.44770496020448086, + "ndcg_linear@10": 0.47772007951643186, + "ndcg_exponential@1": 0.3259109311740891, + "ndcg_exponential@5": 0.44770496020448086, + "ndcg_exponential@10": 0.47772007951643186, + "precision@1": 0.3259109311740891, + "precision@5": 0.11133603238866396, + "precision@10": 0.06497975708502023, + "recall@1": 0.3259109311740891, + "recall@5": 0.555668016194332, + "recall@10": 0.6487854251012146, + "f1@1": 0.3259109311740891, + "f1@5": 0.18546365914786975, + "f1@10": 0.11811434179855235, + "map@1": 0.3259109311740891, + "map@5": 0.411774628879892, + "map@10": 0.42411156095366626, + "mrr@1": 0.3259109311740891, + "mrr@5": 0.4119770580296896, + "mrr@10": 0.42431399010346377, "num_pred": 494, "num_data": 494 }, "ViDoRe_tabfquad": { - "hit@1": 0.675, - "hit@5": 0.8714285714285714, - "hit@10": 0.9178571428571428, - "ndcg_linear@1": 0.675, - "ndcg_linear@5": 0.7818109382746695, - "ndcg_linear@10": 0.7971064823405059, - "ndcg_exponential@1": 0.675, - "ndcg_exponential@5": 0.7818109382746695, - "ndcg_exponential@10": 0.7971064823405059, - "precision@1": 0.675, - "precision@5": 0.17428571428571427, - "precision@10": 0.09178571428571428, - "recall@1": 0.675, - "recall@5": 0.8714285714285714, - "recall@10": 0.9178571428571428, - "f1@1": 0.675, - "f1@5": 0.2904761904761905, - "f1@10": 0.1668831168831169, - "map@1": 0.675, - "map@5": 0.7517857142857143, - "map@10": 0.7582624716553289, - "mrr@1": 0.675, - "mrr@5": 0.7517857142857143, - "mrr@10": 0.7582624716553289, + "hit@1": 0.08928571428571429, + "hit@5": 0.24642857142857144, + "hit@10": 0.38571428571428573, + "ndcg_linear@1": 0.08928571428571429, + "ndcg_linear@5": 0.1697572472048771, + "ndcg_linear@10": 0.2148548317795466, + "ndcg_exponential@1": 0.08928571428571429, + "ndcg_exponential@5": 0.1697572472048771, + "ndcg_exponential@10": 0.2148548317795466, + "precision@1": 0.08928571428571429, + "precision@5": 0.04928571428571429, + "precision@10": 0.038571428571428576, + "recall@1": 0.08928571428571429, + "recall@5": 0.24642857142857144, + "recall@10": 0.38571428571428573, + "f1@1": 0.08928571428571429, + "f1@5": 0.08214285714285716, + "f1@10": 0.07012987012987011, + "map@1": 0.08928571428571429, + "map@5": 0.14464285714285716, + "map@10": 0.1632865646258503, + "mrr@1": 0.08928571428571429, + "mrr@5": 0.14464285714285716, + "mrr@10": 0.1632865646258503, "num_pred": 280, "num_data": 280 }, "ViDoRe_tatdqa": { - "hit@1": 0.17071688942891858, - "hit@5": 0.37484811664641554, - "hit@10": 0.4756986634264885, - "ndcg_linear@1": 0.17071688942891858, - "ndcg_linear@5": 0.27623880236260684, - "ndcg_linear@10": 0.30892627107009063, - "ndcg_exponential@1": 0.17071688942891858, - "ndcg_exponential@5": 0.27623880236260684, - "ndcg_exponential@10": 0.30892627107009063, - "precision@1": 0.17071688942891858, - "precision@5": 0.07509113001215068, - "precision@10": 0.047691373025516404, - "recall@1": 0.1704131227217497, - "recall@5": 0.3745443499392467, - "recall@10": 0.4756986634264885, - "f1@1": 0.17051437829080598, - "f1@5": 0.12506509286582193, - "f1@10": 0.08667476711219119, - "map@1": 0.17071688942891858, - "map@5": 0.24375759416767923, - "map@10": 0.25726243514821884, - "mrr@1": 0.17071688942891858, - "mrr@5": 0.24383353584447146, - "mrr@10": 0.2572371212559548, + "hit@1": 0.02916160388821385, + "hit@5": 0.08626974483596597, + "hit@10": 0.13183475091130012, + "ndcg_linear@1": 0.02916160388821385, + "ndcg_linear@5": 0.05728572257407032, + "ndcg_linear@10": 0.07199640880580573, + "ndcg_exponential@1": 0.02916160388821385, + "ndcg_exponential@5": 0.05728572257407032, + "ndcg_exponential@10": 0.07199640880580573, + "precision@1": 0.02916160388821385, + "precision@5": 0.017253948967193195, + "precision@10": 0.01324422843256379, + "recall@1": 0.02916160388821385, + "recall@5": 0.08596597812879708, + "recall@10": 0.1312879708383961, + "f1@1": 0.02916160388821385, + "f1@5": 0.028727651449401148, + "f1@10": 0.02401229794911447, + "map@1": 0.02916160388821385, + "map@5": 0.047883758606723364, + "map@10": 0.05398614245212058, + "mrr@1": 0.02916160388821385, + "mrr@5": 0.04798501417577967, + "mrr@10": 0.05405528554070473, "num_pred": 1646, "num_data": 1646 }, "ViDoRe_shiftproject": { - "hit@1": 0.23, - "hit@5": 0.52, - "hit@10": 0.63, - "ndcg_linear@1": 0.23, - "ndcg_linear@5": 0.3862145807098324, - "ndcg_linear@10": 0.4228517985151625, - "ndcg_exponential@1": 0.23, - "ndcg_exponential@5": 0.3862145807098324, - "ndcg_exponential@10": 0.4228517985151625, - "precision@1": 0.23, - "precision@5": 0.10399999999999998, - "precision@10": 0.06299999999999999, - "recall@1": 0.23, - "recall@5": 0.52, - "recall@10": 0.63, - "f1@1": 0.23, - "f1@5": 0.17333333333333337, - "f1@10": 0.11454545454545455, - "map@1": 0.23, - "map@5": 0.34149999999999997, - "map@10": 0.3572420634920635, - "mrr@1": 0.23, - "mrr@5": 0.34149999999999997, - "mrr@10": 0.3572420634920635, + "hit@1": 0.01, + "hit@5": 0.02, + "hit@10": 0.03, + "ndcg_linear@1": 0.01, + "ndcg_linear@5": 0.016309297535714574, + "ndcg_linear@10": 0.019871369406794796, + "ndcg_exponential@1": 0.01, + "ndcg_exponential@5": 0.016309297535714574, + "ndcg_exponential@10": 0.019871369406794796, + "precision@1": 0.01, + "precision@5": 0.004, + "precision@10": 0.0030000000000000005, + "recall@1": 0.01, + "recall@5": 0.02, + "recall@10": 0.03, + "f1@1": 0.01, + "f1@5": 0.006666666666666667, + "f1@10": 0.005454545454545454, + "map@1": 0.01, + "map@5": 0.015, + "map@10": 0.016666666666666666, + "mrr@1": 0.01, + "mrr@5": 0.015, + "mrr@10": 0.016666666666666666, "num_pred": 100, "num_data": 100 }, "ViDoRe_syntheticDocQA_artificial_intelligence": { - "hit@1": 0.55, - "hit@5": 0.78, - "hit@10": 0.82, - "ndcg_linear@1": 0.55, - "ndcg_linear@5": 0.6765467489235274, - "ndcg_linear@10": 0.6892234506542988, - "ndcg_exponential@1": 0.55, - "ndcg_exponential@5": 0.6765467489235274, - "ndcg_exponential@10": 0.6892234506542988, - "precision@1": 0.55, - "precision@5": 0.15599999999999997, - "precision@10": 0.08199999999999999, - "recall@1": 0.55, - "recall@5": 0.78, - "recall@10": 0.82, - "f1@1": 0.55, - "f1@5": 0.26000000000000006, - "f1@10": 0.1490909090909091, - "map@1": 0.55, - "map@5": 0.6418333333333334, - "map@10": 0.6469285714285715, - "mrr@1": 0.55, - "mrr@5": 0.6418333333333334, - "mrr@10": 0.6469285714285715, + "hit@1": 0.11, + "hit@5": 0.24, + "hit@10": 0.3, + "ndcg_linear@1": 0.11, + "ndcg_linear@5": 0.18226320510026384, + "ndcg_linear@10": 0.20069743245230928, + "ndcg_exponential@1": 0.11, + "ndcg_exponential@5": 0.18226320510026384, + "ndcg_exponential@10": 0.20069743245230928, + "precision@1": 0.11, + "precision@5": 0.04800000000000001, + "precision@10": 0.03, + "recall@1": 0.11, + "recall@5": 0.24, + "recall@10": 0.3, + "f1@1": 0.11, + "f1@5": 0.08, + "f1@10": 0.05454545454545455, + "map@1": 0.11, + "map@5": 0.1628333333333333, + "map@10": 0.16987301587301587, + "mrr@1": 0.11, + "mrr@5": 0.1628333333333333, + "mrr@10": 0.16987301587301587, "num_pred": 100, "num_data": 100 }, "ViDoRe_syntheticDocQA_energy": { - "hit@1": 0.49, - "hit@5": 0.7, - "hit@10": 0.85, - "ndcg_linear@1": 0.49, - "ndcg_linear@5": 0.604188565752427, - "ndcg_linear@10": 0.654079614057483, - "ndcg_exponential@1": 0.49, - "ndcg_exponential@5": 0.604188565752427, - "ndcg_exponential@10": 0.654079614057483, - "precision@1": 0.49, - "precision@5": 0.13999999999999996, - "precision@10": 0.08499999999999996, - "recall@1": 0.49, - "recall@5": 0.7, - "recall@10": 0.85, - "f1@1": 0.49, - "f1@5": 0.23333333333333336, - "f1@10": 0.15454545454545457, - "map@1": 0.49, - "map@5": 0.572, - "map@10": 0.5934246031746032, - "mrr@1": 0.49, - "mrr@5": 0.572, - "mrr@10": 0.5934246031746032, + "hit@1": 0.19, + "hit@5": 0.28, + "hit@10": 0.35, + "ndcg_linear@1": 0.19, + "ndcg_linear@5": 0.23902854691238618, + "ndcg_linear@10": 0.2605574639476009, + "ndcg_exponential@1": 0.19, + "ndcg_exponential@5": 0.23902854691238618, + "ndcg_exponential@10": 0.2605574639476009, + "precision@1": 0.19, + "precision@5": 0.055999999999999994, + "precision@10": 0.035, + "recall@1": 0.19, + "recall@5": 0.28, + "recall@10": 0.35, + "f1@1": 0.19, + "f1@5": 0.09333333333333335, + "f1@10": 0.06363636363636363, + "map@1": 0.19, + "map@5": 0.2253333333333333, + "map@10": 0.2335833333333333, + "mrr@1": 0.19, + "mrr@5": 0.2253333333333333, + "mrr@10": 0.2335833333333333, "num_pred": 100, "num_data": 100 }, "ViDoRe_syntheticDocQA_government_reports": { - "hit@1": 0.42, - "hit@5": 0.78, - "hit@10": 0.88, - "ndcg_linear@1": 0.42, - "ndcg_linear@5": 0.6181938909073429, - "ndcg_linear@10": 0.6501901901304684, - "ndcg_exponential@1": 0.42, - "ndcg_exponential@5": 0.6181938909073429, - "ndcg_exponential@10": 0.6501901901304684, - "precision@1": 0.42, - "precision@5": 0.15599999999999997, - "precision@10": 0.088, - "recall@1": 0.42, - "recall@5": 0.78, - "recall@10": 0.88, - "f1@1": 0.42, - "f1@5": 0.26000000000000006, - "f1@10": 0.16000000000000003, - "map@1": 0.42, - "map@5": 0.5638333333333334, - "map@10": 0.5768253968253969, - "mrr@1": 0.42, - "mrr@5": 0.5638333333333334, - "mrr@10": 0.5768253968253969, + "hit@1": 0.08, + "hit@5": 0.19, + "hit@10": 0.24, + "ndcg_linear@1": 0.08, + "ndcg_linear@5": 0.1387735500015086, + "ndcg_linear@10": 0.15545500867351458, + "ndcg_exponential@1": 0.08, + "ndcg_exponential@5": 0.1387735500015086, + "ndcg_exponential@10": 0.15545500867351458, + "precision@1": 0.08, + "precision@5": 0.038000000000000006, + "precision@10": 0.024000000000000004, + "recall@1": 0.08, + "recall@5": 0.19, + "recall@10": 0.24, + "f1@1": 0.08, + "f1@5": 0.06333333333333334, + "f1@10": 0.04363636363636363, + "map@1": 0.08, + "map@5": 0.12166666666666666, + "map@10": 0.12885714285714286, + "mrr@1": 0.08, + "mrr@5": 0.12166666666666666, + "mrr@10": 0.12885714285714286, "num_pred": 100, "num_data": 100 }, "ViDoRe_syntheticDocQA_healthcare_industry": { - "hit@1": 0.54, - "hit@5": 0.82, - "hit@10": 0.89, - "ndcg_linear@1": 0.54, - "ndcg_linear@5": 0.698531474110489, - "ndcg_linear@10": 0.7212229678465993, - "ndcg_exponential@1": 0.54, - "ndcg_exponential@5": 0.698531474110489, - "ndcg_exponential@10": 0.7212229678465993, - "precision@1": 0.54, - "precision@5": 0.16399999999999998, - "precision@10": 0.08899999999999997, - "recall@1": 0.54, - "recall@5": 0.82, - "recall@10": 0.89, - "f1@1": 0.54, - "f1@5": 0.2733333333333334, - "f1@10": 0.16181818181818186, - "map@1": 0.54, - "map@5": 0.6573333333333333, - "map@10": 0.6667619047619047, - "mrr@1": 0.54, - "mrr@5": 0.6573333333333333, - "mrr@10": 0.6667619047619047, + "hit@1": 0.2, + "hit@5": 0.34, + "hit@10": 0.41, + "ndcg_linear@1": 0.2, + "ndcg_linear@5": 0.27613173317260925, + "ndcg_linear@10": 0.29860505322014796, + "ndcg_exponential@1": 0.2, + "ndcg_exponential@5": 0.27613173317260925, + "ndcg_exponential@10": 0.29860505322014796, + "precision@1": 0.2, + "precision@5": 0.068, + "precision@10": 0.040999999999999995, + "recall@1": 0.2, + "recall@5": 0.34, + "recall@10": 0.41, + "f1@1": 0.2, + "f1@5": 0.11333333333333337, + "f1@10": 0.07454545454545455, + "map@1": 0.2, + "map@5": 0.25483333333333336, + "map@10": 0.2640079365079365, + "mrr@1": 0.2, + "mrr@5": 0.25483333333333336, + "mrr@10": 0.2640079365079365, "num_pred": 100, "num_data": 100 }, "ViDoRe_esg_reports_human_labeled_v2": { "hit@1": 0.038461538461538464, "hit@5": 0.11538461538461539, - "hit@10": 0.23076923076923078, + "hit@10": 0.21153846153846154, "ndcg_linear@1": 0.038461538461538464, - "ndcg_linear@5": 0.06817512332990149, - "ndcg_linear@10": 0.09598030466462644, + "ndcg_linear@5": 0.0695082664438747, + "ndcg_linear@10": 0.09094552394424182, "ndcg_exponential@1": 0.038461538461538464, - "ndcg_exponential@5": 0.06817512332990149, - "ndcg_exponential@10": 0.09599487668438379, + "ndcg_exponential@5": 0.0695082664438747, + "ndcg_exponential@10": 0.09096079471620305, "precision@1": 0.038461538461538464, "precision@5": 0.023076923076923075, "precision@10": 0.026923076923076925, "recall@1": 0.038461538461538464, "recall@5": 0.09134615384615384, - "recall@10": 0.16506410256410253, + "recall@10": 0.1378205128205128, "f1@1": 0.038461538461538464, "f1@5": 0.03540903540903541, - "f1@10": 0.04333166833166832, + "f1@10": 0.04204449396757089, "map@1": 0.038461538461538464, - "map@5": 0.057211538461538466, - "map@10": 0.06807844932844934, + "map@5": 0.05881410256410257, + "map@10": 0.06732740638990639, "mrr@1": 0.038461538461538464, - "mrr@5": 0.06634615384615385, - "mrr@10": 0.08041819291819292, + "mrr@5": 0.06794871794871794, + "mrr@10": 0.08084554334554335, "num_pred": 52, "num_data": 52 }, "ViDoRe_biomedical_lectures_v2_multilingual": { - "hit@1": 0.05, - "hit@5": 0.1203125, - "hit@10": 0.1875, - "ndcg_linear@1": 0.05, - "ndcg_linear@5": 0.05064502600213018, - "ndcg_linear@10": 0.06329515996638582, - "ndcg_exponential@1": 0.05, - "ndcg_exponential@5": 0.05064502600213018, - "ndcg_exponential@10": 0.06329515996638582, - "precision@1": 0.05, - "precision@5": 0.025312500000000005, - "precision@10": 0.022031250000000002, - "recall@1": 0.027369791666666664, - "recall@5": 0.052070591020499116, - "recall@10": 0.08876272241851287, - "f1@1": 0.031846196338383835, - "f1@5": 0.029338837377899885, - "f1@10": 0.031196784815810553, - "map@1": 0.05, - "map@5": 0.03780381944444444, - "map@10": 0.041590970155423276, - "mrr@1": 0.05, - "mrr@5": 0.07333333333333333, - "mrr@10": 0.08225384424603174, + "hit@1": 0.0515625, + "hit@5": 0.125, + "hit@10": 0.1796875, + "ndcg_linear@1": 0.0515625, + "ndcg_linear@5": 0.05220711277961643, + "ndcg_linear@10": 0.06158573118703668, + "ndcg_exponential@1": 0.0515625, + "ndcg_exponential@5": 0.05220711277961643, + "ndcg_exponential@10": 0.06158573118703668, + "precision@1": 0.0515625, + "precision@5": 0.0265625, + "precision@10": 0.02109375, + "recall@1": 0.025364165552584673, + "recall@5": 0.054219028520499105, + "recall@10": 0.0826931539661319, + "f1@1": 0.029871369949494946, + "f1@5": 0.030726295405982908, + "f1@10": 0.029608775194712694, + "map@1": 0.0515625, + "map@5": 0.03837109375, + "map@10": 0.04077487392526455, + "mrr@1": 0.0515625, + "mrr@5": 0.07656249999999999, + "mrr@10": 0.08390438988095238, "num_pred": 640, "num_data": 640 }, "ViDoRe_economics_reports_v2_multilingual": { - "hit@1": 0.1810344827586207, - "hit@5": 0.4224137931034483, - "hit@10": 0.5646551724137931, - "ndcg_linear@1": 0.1810344827586207, - "ndcg_linear@5": 0.13875113955781276, - "ndcg_linear@10": 0.14758177623224894, - "ndcg_exponential@1": 0.1810344827586207, - "ndcg_exponential@5": 0.1387511395578127, - "ndcg_exponential@10": 0.14758177623224894, - "precision@1": 0.1810344827586207, - "precision@5": 0.11982758620689656, - "precision@10": 0.11551724137931037, - "recall@1": 0.012707666017446384, - "recall@5": 0.06501080124572646, - "recall@10": 0.12018603229901821, - "f1@1": 0.02288545832709266, - "f1@5": 0.06054340208235827, - "f1@10": 0.08247544424998887, - "map@1": 0.1810344827586207, - "map@5": 0.08251436781609195, - "map@10": 0.07246543932068357, - "mrr@1": 0.1810344827586207, - "mrr@5": 0.2608477011494253, - "mrr@10": 0.27921798029556655, + "hit@1": 0.21120689655172414, + "hit@5": 0.3922413793103448, + "hit@10": 0.5603448275862069, + "ndcg_linear@1": 0.21120689655172414, + "ndcg_linear@5": 0.13754531139868112, + "ndcg_linear@10": 0.14364117363111378, + "ndcg_exponential@1": 0.21120689655172414, + "ndcg_exponential@5": 0.13754531139868112, + "ndcg_exponential@10": 0.14364117363111378, + "precision@1": 0.21120689655172414, + "precision@5": 0.11034482758620691, + "precision@10": 0.10991379310344827, + "recall@1": 0.014577652109731016, + "recall@5": 0.06538782189639662, + "recall@10": 0.10854851210073217, + "f1@1": 0.0262271908912617, + "f1@5": 0.05648888336723445, + "f1@10": 0.07828681857184043, + "map@1": 0.21120689655172414, + "map@5": 0.08316810344827585, + "map@10": 0.07051148760110684, + "mrr@1": 0.21120689655172414, + "mrr@5": 0.27025862068965517, + "mrr@10": 0.29276477832512315, "num_pred": 232, "num_data": 232 }, "ViDoRe_esg_reports_v2_multilingual": { - "hit@1": 0.08333333333333333, - "hit@5": 0.2324561403508772, - "hit@10": 0.31140350877192985, - "ndcg_linear@1": 0.08333333333333333, - "ndcg_linear@5": 0.11890188811957679, - "ndcg_linear@10": 0.1391360634250716, - "ndcg_exponential@1": 0.08333333333333333, - "ndcg_exponential@5": 0.11890188811957679, - "ndcg_exponential@10": 0.1391360634250716, - "precision@1": 0.08333333333333333, - "precision@5": 0.05964912280701755, - "precision@10": 0.04649122807017544, - "recall@1": 0.05936716791979949, - "recall@5": 0.1361737677527151, - "recall@10": 0.18859649122807018, - "f1@1": 0.06306354967879468, - "f1@5": 0.07183557899502171, - "f1@10": 0.06509698173479336, - "map@1": 0.08333333333333333, - "map@5": 0.09392543859649122, - "map@10": 0.10152116402116403, - "mrr@1": 0.08333333333333333, - "mrr@5": 0.1355263157894737, - "mrr@10": 0.14601434140907826, + "hit@1": 0.07894736842105263, + "hit@5": 0.2149122807017544, + "hit@10": 0.30701754385964913, + "ndcg_linear@1": 0.07894736842105263, + "ndcg_linear@5": 0.11052525703816217, + "ndcg_linear@10": 0.13413038304953584, + "ndcg_exponential@1": 0.07894736842105263, + "ndcg_exponential@5": 0.11052525703816217, + "ndcg_exponential@10": 0.13413038304953584, + "precision@1": 0.07894736842105263, + "precision@5": 0.054385964912280704, + "precision@10": 0.04517543859649123, + "recall@1": 0.05790517961570593, + "recall@5": 0.1276106934001671, + "recall@10": 0.19002715121136174, + "f1@1": 0.06087056722265434, + "f1@5": 0.06654596979674379, + "f1@10": 0.06426085456556371, + "map@1": 0.07894736842105263, + "map@5": 0.08797392787524366, + "map@10": 0.0970358187134503, + "mrr@1": 0.07894736842105263, + "mrr@5": 0.12514619883040937, + "mrr@10": 0.13705792258423835, "num_pred": 228, "num_data": 228 }, "VisRAG_ArxivQA": { - "hit@1": 0.43504901960784315, - "hit@5": 0.6029411764705882, + "hit@1": 0.4362745098039216, + "hit@5": 0.6041666666666666, "hit@10": 0.6727941176470589, - "ndcg_linear@1": 0.43504901960784315, - "ndcg_linear@5": 0.5260731896593214, - "ndcg_linear@10": 0.5488301957135047, - "ndcg_exponential@1": 0.43504901960784315, - "ndcg_exponential@5": 0.5260731896593214, - "ndcg_exponential@10": 0.5488301957135047, - "precision@1": 0.43504901960784315, - "precision@5": 0.12058823529411763, + "ndcg_linear@1": 0.4362745098039216, + "ndcg_linear@5": 0.5284726795191211, + "ndcg_linear@10": 0.550951313243767, + "ndcg_exponential@1": 0.4362745098039216, + "ndcg_exponential@5": 0.5284726795191211, + "ndcg_exponential@10": 0.550951313243767, + "precision@1": 0.4362745098039216, + "precision@5": 0.12083333333333332, "precision@10": 0.06727941176470587, - "recall@1": 0.43504901960784315, - "recall@5": 0.6029411764705882, + "recall@1": 0.4362745098039216, + "recall@5": 0.6041666666666666, "recall@10": 0.6727941176470589, - "f1@1": 0.43504901960784315, - "f1@5": 0.2009803921568628, - "f1@10": 0.12232620320855617, - "map@1": 0.43504901960784315, - "map@5": 0.5003472222222222, - "map@10": 0.5098355314347962, - "mrr@1": 0.43504901960784315, - "mrr@5": 0.5003472222222222, - "mrr@10": 0.5098355314347962, + "f1@1": 0.4362745098039216, + "f1@5": 0.20138888888888892, + "f1@10": 0.12232620320855618, + "map@1": 0.4362745098039216, + "map@5": 0.5030228758169935, + "map@10": 0.5124635270774976, + "mrr@1": 0.4362745098039216, + "mrr@5": 0.5030228758169935, + "mrr@10": 0.5124635270774976, "num_pred": 816, "num_data": 816 }, "VisRAG_ChartQA": { - "hit@1": 0.6031746031746031, - "hit@5": 0.7936507936507936, + "hit@1": 0.5873015873015873, + "hit@5": 0.7777777777777778, "hit@10": 0.8412698412698413, - "ndcg_linear@1": 0.6031746031746031, - "ndcg_linear@5": 0.7023156150418144, - "ndcg_linear@10": 0.7176213783260175, - "ndcg_exponential@1": 0.6031746031746031, - "ndcg_exponential@5": 0.7023156150418144, - "ndcg_exponential@10": 0.7176213783260175, - "precision@1": 0.6031746031746031, - "precision@5": 0.15873015873015867, + "ndcg_linear@1": 0.5873015873015873, + "ndcg_linear@5": 0.6896212213195614, + "ndcg_linear@10": 0.710351941841642, + "ndcg_exponential@1": 0.5873015873015873, + "ndcg_exponential@5": 0.6896212213195614, + "ndcg_exponential@10": 0.710351941841642, + "precision@1": 0.5873015873015873, + "precision@5": 0.1555555555555555, "precision@10": 0.08412698412698409, - "recall@1": 0.6031746031746031, - "recall@5": 0.7936507936507936, + "recall@1": 0.5873015873015873, + "recall@5": 0.7777777777777778, "recall@10": 0.8412698412698413, - "f1@1": 0.6031746031746031, - "f1@5": 0.2645502645502646, + "f1@1": 0.5873015873015873, + "f1@5": 0.2592592592592593, "f1@10": 0.15295815295815296, - "map@1": 0.6031746031746031, - "map@5": 0.6722222222222223, - "map@10": 0.6784580498866214, - "mrr@1": 0.6031746031746031, - "mrr@5": 0.6722222222222223, - "mrr@10": 0.6784580498866214, + "map@1": 0.5873015873015873, + "map@5": 0.6603174603174603, + "map@10": 0.6689783320735703, + "mrr@1": 0.5873015873015873, + "mrr@5": 0.6603174603174603, + "mrr@10": 0.6689783320735703, "num_pred": 63, "num_data": 63 }, "VisRAG_MP-DocVQA": { - "hit@1": 0.4077834179357022, - "hit@5": 0.6395939086294417, - "hit@10": 0.7123519458544839, - "ndcg_linear@1": 0.4077834179357022, - "ndcg_linear@5": 0.5280813677279776, - "ndcg_linear@10": 0.5522909877969642, - "ndcg_exponential@1": 0.4077834179357022, - "ndcg_exponential@5": 0.5280813677279776, - "ndcg_exponential@10": 0.5522909877969642, - "precision@1": 0.4077834179357022, - "precision@5": 0.1279187817258883, - "precision@10": 0.07123519458544839, - "recall@1": 0.4077834179357022, - "recall@5": 0.6395939086294417, - "recall@10": 0.7123519458544839, - "f1@1": 0.4077834179357022, - "f1@5": 0.2131979695431472, - "f1@10": 0.1295185356099062, - "map@1": 0.4077834179357022, - "map@5": 0.4912859560067682, - "map@10": 0.5016725754035398, - "mrr@1": 0.4077834179357022, - "mrr@5": 0.4912859560067682, - "mrr@10": 0.5016725754035398, + "hit@1": 0.40947546531302875, + "hit@5": 0.637901861252115, + "hit@10": 0.7191201353637902, + "ndcg_linear@1": 0.40947546531302875, + "ndcg_linear@5": 0.5274288911977277, + "ndcg_linear@10": 0.5543111275643495, + "ndcg_exponential@1": 0.40947546531302875, + "ndcg_exponential@5": 0.5274288911977277, + "ndcg_exponential@10": 0.5543111275643495, + "precision@1": 0.40947546531302875, + "precision@5": 0.127580372250423, + "precision@10": 0.07191201353637901, + "recall@1": 0.40947546531302875, + "recall@5": 0.637901861252115, + "recall@10": 0.7191201353637902, + "f1@1": 0.40947546531302875, + "f1@5": 0.21263395375070504, + "f1@10": 0.13074911552068913, + "map@1": 0.40947546531302875, + "map@5": 0.49103214890016916, + "map@10": 0.5024950984341848, + "mrr@1": 0.40947546531302875, + "mrr@5": 0.49103214890016916, + "mrr@10": 0.5024950984341848, "num_pred": 591, "num_data": 591 }, "VisRAG_SlideVQA": { - "hit@1": 0.6258992805755396, + "hit@1": 0.6312949640287769, "hit@5": 0.85431654676259, - "hit@10": 0.8974820143884892, - "ndcg_linear@1": 0.6258992805755396, - "ndcg_linear@5": 0.727722018029152, - "ndcg_linear@10": 0.7442517359673964, - "ndcg_exponential@1": 0.6258992805755396, - "ndcg_exponential@5": 0.727722018029152, - "ndcg_exponential@10": 0.7442517359673964, - "precision@1": 0.6258992805755396, - "precision@5": 0.2079136690647482, - "precision@10": 0.10953237410071942, - "recall@1": 0.5314748201438849, - "recall@5": 0.822841726618705, - "recall@10": 0.8714028776978417, - "f1@1": 0.5629496402877698, - "f1@5": 0.3258821514217198, - "f1@10": 0.19236429038587313, - "map@1": 0.6258992805755396, - "map@5": 0.6828387290167865, - "map@10": 0.6904986724905788, - "mrr@1": 0.6258992805755396, - "mrr@5": 0.7175659472422062, - "mrr@10": 0.7235775665182139, + "hit@10": 0.8902877697841727, + "ndcg_linear@1": 0.6312949640287769, + "ndcg_linear@5": 0.7276335110394372, + "ndcg_linear@10": 0.7423518482999316, + "ndcg_exponential@1": 0.6312949640287769, + "ndcg_exponential@5": 0.7276335110394372, + "ndcg_exponential@10": 0.7423518482999316, + "precision@1": 0.6312949640287769, + "precision@5": 0.2071942446043165, + "precision@10": 0.10881294964028777, + "recall@1": 0.5350719424460432, + "recall@5": 0.8210431654676259, + "recall@10": 0.8633093525179856, + "f1@1": 0.567146282973621, + "f1@5": 0.32485440219253175, + "f1@10": 0.19102899498582956, + "map@1": 0.6312949640287769, + "map@5": 0.6831235011990408, + "map@10": 0.6902149708804385, + "mrr@1": 0.6312949640287769, + "mrr@5": 0.7191546762589929, + "mrr@10": 0.7242213372159415, "num_pred": 556, "num_data": 556 }, "VisRAG_InfoVQA": { - "hit@1": 0.6002785515320335, - "hit@5": 0.8245125348189415, - "hit@10": 0.8899721448467967, - "ndcg_linear@1": 0.6002785515320335, - "ndcg_linear@5": 0.7196103081823707, - "ndcg_linear@10": 0.7410456616380353, - "ndcg_exponential@1": 0.6002785515320335, - "ndcg_exponential@5": 0.7196103081823707, - "ndcg_exponential@10": 0.7410456616380353, - "precision@1": 0.6002785515320335, - "precision@5": 0.16490250696378828, - "precision@10": 0.08899721448467965, - "recall@1": 0.6002785515320335, - "recall@5": 0.8245125348189415, - "recall@10": 0.8899721448467967, - "f1@1": 0.6002785515320335, - "f1@5": 0.27483751160631387, - "f1@10": 0.16181311724487216, - "map@1": 0.6002785515320335, - "map@5": 0.6846564531104921, - "map@10": 0.6936557677852943, - "mrr@1": 0.6002785515320335, - "mrr@5": 0.6846564531104921, - "mrr@10": 0.6936557677852943, + "hit@1": 0.5877437325905293, + "hit@5": 0.8217270194986073, + "hit@10": 0.8871866295264624, + "ndcg_linear@1": 0.5877437325905293, + "ndcg_linear@5": 0.7129824715133523, + "ndcg_linear@10": 0.7346117304636293, + "ndcg_exponential@1": 0.5877437325905293, + "ndcg_exponential@5": 0.7129824715133523, + "ndcg_exponential@10": 0.7346117304636293, + "precision@1": 0.5877437325905293, + "precision@5": 0.16434540389972144, + "precision@10": 0.08871866295264623, + "recall@1": 0.5877437325905293, + "recall@5": 0.8217270194986073, + "recall@10": 0.8871866295264624, + "f1@1": 0.5877437325905293, + "f1@5": 0.27390900649953587, + "f1@10": 0.16130665991390228, + "map@1": 0.5877437325905293, + "map@5": 0.6767177344475395, + "map@10": 0.6859104876862536, + "mrr@1": 0.5877437325905293, + "mrr@5": 0.6767177344475395, + "mrr@10": 0.6859104876862536, "num_pred": 718, "num_data": 718 }, "VisRAG_PlotQA": { - "hit@1": 0.22363847045191193, - "hit@5": 0.4484356894553882, - "hit@10": 0.5654692931633836, - "ndcg_linear@1": 0.22363847045191193, - "ndcg_linear@5": 0.3434876959499088, - "ndcg_linear@10": 0.3813457018404914, - "ndcg_exponential@1": 0.22363847045191193, - "ndcg_exponential@5": 0.3434876959499088, - "ndcg_exponential@10": 0.3813457018404914, - "precision@1": 0.22363847045191193, - "precision@5": 0.08968713789107764, - "precision@10": 0.05654692931633834, - "recall@1": 0.22363847045191193, - "recall@5": 0.4484356894553882, - "recall@10": 0.5654692931633836, - "f1@1": 0.22363847045191193, - "f1@5": 0.14947856315179606, - "f1@10": 0.10281259875697883, - "map@1": 0.22363847045191193, - "map@5": 0.3085940517574354, - "map@10": 0.32422382239879344, - "mrr@1": 0.22363847045191193, - "mrr@5": 0.3085940517574354, - "mrr@10": 0.32422382239879344, + "hit@1": 0.2190034762456547, + "hit@5": 0.45770567786790267, + "hit@10": 0.5666280417149478, + "ndcg_linear@1": 0.2190034762456547, + "ndcg_linear@5": 0.34520168295443, + "ndcg_linear@10": 0.3803099872098485, + "ndcg_exponential@1": 0.2190034762456547, + "ndcg_exponential@5": 0.34520168295443, + "ndcg_exponential@10": 0.3803099872098485, + "precision@1": 0.2190034762456547, + "precision@5": 0.09154113557358054, + "precision@10": 0.056662804171494774, + "recall@1": 0.2190034762456547, + "recall@5": 0.45770567786790267, + "recall@10": 0.5666280417149478, + "f1@1": 0.2190034762456547, + "f1@5": 0.1525685592893009, + "f1@10": 0.10302328031180873, + "map@1": 0.2190034762456547, + "map@5": 0.3078408651989185, + "map@10": 0.3222580882488183, + "mrr@1": 0.2190034762456547, + "mrr@5": 0.3078408651989185, + "mrr@10": 0.3222580882488183, "num_pred": 863, "num_data": 863 }, "ViDoSeek-page": { - "hit@1": 0.0446584938704028, - "hit@5": 0.3905429071803853, - "hit@10": 0.5971978984238179, - "ndcg_linear@1": 0.0446584938704028, - "ndcg_linear@5": 0.22283683461024645, - "ndcg_linear@10": 0.2895264993048246, - "ndcg_exponential@1": 0.0446584938704028, - "ndcg_exponential@5": 0.22283683461024653, - "ndcg_exponential@10": 0.28952649930482466, - "precision@1": 0.0446584938704028, - "precision@5": 0.07810858143607707, - "precision@10": 0.05971978984238179, - "recall@1": 0.0446584938704028, - "recall@5": 0.3905429071803853, - "recall@10": 0.5971978984238179, - "f1@1": 0.0446584938704028, - "f1@5": 0.13018096906012844, - "f1@10": 0.10858143607705782, - "map@1": 0.0446584938704028, - "map@5": 0.1676444833625219, - "map@10": 0.19508763517082256, - "mrr@1": 0.0446584938704028, - "mrr@5": 0.1676444833625219, - "mrr@10": 0.19508763517082256, - "num_pred": 1142, - "num_data": 1142 - }, - "ViDoSeek-doc": { - "hit@1": 0.968476357267951, - "hit@5": 0.9912434325744308, - "hit@10": 0.9947460595446584, - "ndcg_linear@1": 0.6625802685347344, - "ndcg_linear@5": 0.7780718841793594, - "ndcg_linear@10": 0.7502728417738824, - "ndcg_exponential@1": 0.44408306229672245, - "ndcg_exponential@5": 0.6673711755462026, - "ndcg_exponential@10": 0.6819104114803104, - "precision@1": 0.968476357267951, - "precision@5": 0.8481611208406304, - "precision@10": 0.750875656742557, - "recall@1": 0.05711879676554371, - "recall@5": 0.24781081127789126, - "recall@10": 0.4314558336731064, - "f1@1": 0.10713328875583096, - "f1@5": 0.375858198266283, - "f1@10": 0.5355661581210087, - "map@1": 0.968476357267951, - "map@5": 0.831468184471687, - "map@10": 0.7255596248277478, - "mrr@1": 0.968476357267951, - "mrr@5": 0.9776999416228838, - "mrr@10": 0.9781680010007505, + "hit@1": 0.617338003502627, + "hit@5": 0.9019264448336253, + "hit@10": 0.9535901926444834, + "ndcg_linear@1": 0.617338003502627, + "ndcg_linear@5": 0.7739597969228037, + "ndcg_linear@10": 0.7908187102319941, + "ndcg_exponential@1": 0.617338003502627, + "ndcg_exponential@5": 0.7739597969228037, + "ndcg_exponential@10": 0.7908187102319941, + "precision@1": 0.617338003502627, + "precision@5": 0.18038528896672504, + "precision@10": 0.09535901926444831, + "recall@1": 0.617338003502627, + "recall@5": 0.9019264448336253, + "recall@10": 0.9535901926444834, + "f1@1": 0.617338003502627, + "f1@5": 0.30064214827787517, + "f1@10": 0.17338003502626975, + "map@1": 0.617338003502627, + "map@5": 0.7309252772913019, + "map@10": 0.7379750090345537, + "mrr@1": 0.617338003502627, + "mrr@5": 0.7309252772913019, + "mrr@10": 0.7379750090345537, "num_pred": 1142, "num_data": 1142 }, "MMLongBench-page": { - "hit@1": 0.06801909307875895, - "hit@5": 0.2386634844868735, - "hit@10": 0.3305489260143198, - "ndcg_linear@1": 0.06801909307875895, - "ndcg_linear@5": 0.11822134474246979, - "ndcg_linear@10": 0.15022040843373538, - "ndcg_exponential@1": 0.06801909307875895, - "ndcg_exponential@5": 0.11822134474246979, - "ndcg_exponential@10": 0.15022040843373538, - "precision@1": 0.06801909307875895, - "precision@5": 0.056324582338902154, - "precision@10": 0.04379474940334129, - "recall@1": 0.038664905102852594, - "recall@5": 0.16723964844489903, - "recall@10": 0.25455198143145635, - "f1@1": 0.04688230756249849, - "f1@5": 0.07989331008424089, - "f1@10": 0.07145515842987549, - "map@1": 0.06801909307875895, - "map@5": 0.0867468841156192, - "map@10": 0.10167219917667411, - "mrr@1": 0.06801909307875895, - "mrr@5": 0.12738663484486873, - "mrr@10": 0.13961955904080012, - "num_pred": 838, - "num_data": 838 - }, - "MMLongBench-doc": { - "hit@1": 0.5668257756563246, - "hit@5": 0.6885441527446301, - "hit@10": 0.747016706443914, - "ndcg_linear@1": 0.3981702466189339, - "ndcg_linear@5": 0.40462422635020384, - "ndcg_linear@10": 0.38948040985289195, - "ndcg_exponential@1": 0.27770201159222635, - "ndcg_exponential@5": 0.33626701840937107, - "ndcg_exponential@10": 0.34529122789195027, - "precision@1": 0.5668257756563246, - "precision@5": 0.4470167064439141, - "precision@10": 0.3953460620525059, - "recall@1": 0.017686024380438876, - "recall@5": 0.06659662623490364, - "recall@10": 0.11644775779833906, - "f1@1": 0.03396785786854894, - "f1@5": 0.11131694335938004, - "f1@10": 0.16871497861256565, - "map@1": 0.5668257756563246, - "map@5": 0.41111774065234685, - "map@10": 0.34533373426273184, - "mrr@1": 0.5668257756563246, - "mrr@5": 0.6124105011933175, - "mrr@10": 0.6201973709133614, + "hit@1": 0.2947494033412888, + "hit@5": 0.5107398568019093, + "hit@10": 0.5906921241050119, + "ndcg_linear@1": 0.2947494033412888, + "ndcg_linear@5": 0.3675669169600146, + "ndcg_linear@10": 0.3985819348589493, + "ndcg_exponential@1": 0.2947494033412888, + "ndcg_exponential@5": 0.3675669169600146, + "ndcg_exponential@10": 0.3985819348589493, + "precision@1": 0.2947494033412888, + "precision@5": 0.12816229116945108, + "precision@10": 0.07899761336515512, + "recall@1": 0.2267090010228435, + "recall@5": 0.4366267218594187, + "recall@10": 0.5220639146056807, + "f1@1": 0.24683168197488004, + "f1@5": 0.18929226435193028, + "f1@10": 0.1320617045637117, + "map@1": 0.2947494033412888, + "map@5": 0.3248776849642005, + "map@10": 0.3402303489348537, + "mrr@1": 0.2947494033412888, + "mrr@5": 0.37663086714399363, + "mrr@10": 0.3878480509148767, "num_pred": 838, "num_data": 838 }