diff --git "a/scores/gme-Qwen2-VL-7B-Instruct.json" "b/scores/gme-Qwen2-VL-7B-Instruct.json" --- "a/scores/gme-Qwen2-VL-7B-Instruct.json" +++ "b/scores/gme-Qwen2-VL-7B-Instruct.json" @@ -1,7 +1,7 @@ { "metadata": { "model_name": "gme-Qwen2-VL-7B-Instruct", - "report_generated_date": "2025-06-28T08:54:18.334782", + "report_generated_date": "2025-12-29T09:27:50.352865", "model_size": 8.29, "url": "https://huggingface.co/Alibaba-NLP/gme-Qwen2-VL-7B-Instruct", "data_source": "TIGER-Lab" @@ -9,198 +9,198 @@ "metrics": { "image": { "ImageNet-1K": { - "hit@1": 0.646, + "hit@1": 0.647, "hit@5": 0.864, - "hit@10": 0.914, - "ndcg_linear@1": 0.646, - "ndcg_linear@5": 0.7675656151573618, - "ndcg_linear@10": 0.7837300096011678, - "ndcg_exponential@1": 0.646, - "ndcg_exponential@5": 0.7675656151573618, - "ndcg_exponential@10": 0.7837300096011678, - "precision@1": 0.646, + "hit@10": 0.913, + "ndcg_linear@1": 0.647, + "ndcg_linear@5": 0.7674725727011493, + "ndcg_linear@10": 0.7832892255780918, + "ndcg_exponential@1": 0.647, + "ndcg_exponential@5": 0.7674725727011493, + "ndcg_exponential@10": 0.7832892255780918, + "precision@1": 0.647, "precision@5": 0.1728, - "precision@10": 0.09140000000000001, - "recall@1": 0.646, + "precision@10": 0.0913, + "recall@1": 0.647, "recall@5": 0.864, - "recall@10": 0.914, - "f1@1": 0.646, + "recall@10": 0.913, + "f1@1": 0.647, "f1@5": 0.288, - "f1@10": 0.16618181818181818, - "map@1": 0.646, - "map@5": 0.7350166666666667, - "map@10": 0.741684126984127, - "mrr@1": 0.646, - "mrr@5": 0.7350166666666667, - "mrr@10": 0.741684126984127, + "f1@10": 0.166, + "map@1": 0.647, + "map@5": 0.7349333333333333, + "map@10": 0.7414408730158729, + "mrr@1": 0.647, + "mrr@5": 0.7349333333333333, + "mrr@10": 0.7414408730158729, "num_pred": 1000, "num_data": 1000 }, "N24News": { - "hit@1": 0.505, - "hit@5": 0.838, - "hit@10": 0.93, - "ndcg_linear@1": 0.505, - "ndcg_linear@5": 0.6859312278730303, - "ndcg_linear@10": 0.7160982776534236, - "ndcg_exponential@1": 0.505, - "ndcg_exponential@5": 0.6859312278730303, - "ndcg_exponential@10": 0.7160982776534236, - "precision@1": 0.505, - "precision@5": 0.1676, - "precision@10": 0.09300000000000003, - "recall@1": 0.505, - "recall@5": 0.838, - "recall@10": 0.93, - "f1@1": 0.505, - "f1@5": 0.2793333333333334, - "f1@10": 0.1690909090909091, - "map@1": 0.505, - "map@5": 0.635, - "map@10": 0.6476924603174602, - "mrr@1": 0.505, - "mrr@5": 0.635, - "mrr@10": 0.6476924603174602, + "hit@1": 0.503, + "hit@5": 0.836, + "hit@10": 0.929, + "ndcg_linear@1": 0.503, + "ndcg_linear@5": 0.6843654925843411, + "ndcg_linear@10": 0.7148982450094127, + "ndcg_exponential@1": 0.503, + "ndcg_exponential@5": 0.6843654925843411, + "ndcg_exponential@10": 0.7148982450094127, + "precision@1": 0.503, + "precision@5": 0.1672, + "precision@10": 0.09290000000000001, + "recall@1": 0.503, + "recall@5": 0.836, + "recall@10": 0.929, + "f1@1": 0.503, + "f1@5": 0.27866666666666673, + "f1@10": 0.1689090909090909, + "map@1": 0.503, + "map@5": 0.6335166666666666, + "map@10": 0.646384126984127, + "mrr@1": 0.503, + "mrr@5": 0.6335166666666666, + "mrr@10": 0.646384126984127, "num_pred": 1000, "num_data": 1000 }, "HatefulMemes": { - "hit@1": 0.536, + "hit@1": 0.539, "hit@5": 1.0, "hit@10": 1.0, - "ndcg_linear@1": 0.536, - "ndcg_linear@5": 0.8287514056571564, - "ndcg_linear@10": 0.8287514056571564, - "ndcg_exponential@1": 0.536, - "ndcg_exponential@5": 0.8287514056571564, - "ndcg_exponential@10": 0.8287514056571564, - "precision@1": 0.536, + "ndcg_linear@1": 0.539, + "ndcg_linear@5": 0.829858616396442, + "ndcg_linear@10": 0.829858616396442, + "ndcg_exponential@1": 0.539, + "ndcg_exponential@5": 0.829858616396442, + "ndcg_exponential@10": 0.829858616396442, + "precision@1": 0.539, "precision@5": 0.20000000000000004, "precision@10": 0.10000000000000002, - "recall@1": 0.536, + "recall@1": 0.539, "recall@5": 1.0, "recall@10": 1.0, - "f1@1": 0.536, + "f1@1": 0.539, "f1@5": 0.3333333333333333, "f1@10": 0.18181818181818182, - "map@1": 0.536, - "map@5": 0.768, - "map@10": 0.768, - "mrr@1": 0.536, - "mrr@5": 0.768, - "mrr@10": 0.768, + "map@1": 0.539, + "map@5": 0.7695, + "map@10": 0.7695, + "mrr@1": 0.539, + "mrr@5": 0.7695, + "mrr@10": 0.7695, "num_pred": 1000, "num_data": 1000 }, "VOC2007": { - "hit@1": 0.803, - "hit@5": 0.966, + "hit@1": 0.801, + "hit@5": 0.967, "hit@10": 0.99, - "ndcg_linear@1": 0.803, - "ndcg_linear@5": 0.8956917699369619, - "ndcg_linear@10": 0.9035303593848166, - "ndcg_exponential@1": 0.803, - "ndcg_exponential@5": 0.8956917699369619, - "ndcg_exponential@10": 0.9035303593848166, - "precision@1": 0.803, - "precision@5": 0.19320000000000004, + "ndcg_linear@1": 0.801, + "ndcg_linear@5": 0.8950886881747205, + "ndcg_linear@10": 0.902530328125145, + "ndcg_exponential@1": 0.801, + "ndcg_exponential@5": 0.8950886881747205, + "ndcg_exponential@10": 0.902530328125145, + "precision@1": 0.801, + "precision@5": 0.19340000000000004, "precision@10": 0.09900000000000002, - "recall@1": 0.803, - "recall@5": 0.966, + "recall@1": 0.801, + "recall@5": 0.967, "recall@10": 0.99, - "f1@1": 0.803, - "f1@5": 0.322, + "f1@1": 0.801, + "f1@5": 0.32233333333333336, "f1@10": 0.18, - "map@1": 0.803, - "map@5": 0.8717333333333334, - "map@10": 0.8750126984126985, - "mrr@1": 0.803, - "mrr@5": 0.8717333333333334, - "mrr@10": 0.8750126984126985, + "map@1": 0.801, + "map@5": 0.8706333333333334, + "map@10": 0.8737043650793651, + "mrr@1": 0.801, + "mrr@5": 0.8706333333333334, + "mrr@10": 0.8737043650793651, "num_pred": 1000, "num_data": 1000 }, "SUN397": { - "hit@1": 0.695, - "hit@5": 0.93, - "hit@10": 0.964, - "ndcg_linear@1": 0.695, - "ndcg_linear@5": 0.8264697358260907, - "ndcg_linear@10": 0.8377506399258345, - "ndcg_exponential@1": 0.695, - "ndcg_exponential@5": 0.8264697358260907, - "ndcg_exponential@10": 0.8377506399258345, - "precision@1": 0.695, - "precision@5": 0.18600000000000003, - "precision@10": 0.0964, - "recall@1": 0.695, - "recall@5": 0.93, - "recall@10": 0.964, - "f1@1": 0.695, - "f1@5": 0.31, - "f1@10": 0.17527272727272725, - "map@1": 0.695, - "map@5": 0.7914666666666665, - "map@10": 0.7962904761904762, - "mrr@1": 0.695, - "mrr@5": 0.7914666666666665, - "mrr@10": 0.7962904761904762, + "hit@1": 0.694, + "hit@5": 0.933, + "hit@10": 0.965, + "ndcg_linear@1": 0.694, + "ndcg_linear@5": 0.8271653178704612, + "ndcg_linear@10": 0.8376751042078517, + "ndcg_exponential@1": 0.694, + "ndcg_exponential@5": 0.8271653178704612, + "ndcg_exponential@10": 0.8376751042078517, + "precision@1": 0.694, + "precision@5": 0.18660000000000002, + "precision@10": 0.09650000000000003, + "recall@1": 0.694, + "recall@5": 0.933, + "recall@10": 0.965, + "f1@1": 0.694, + "f1@5": 0.311, + "f1@10": 0.17545454545454545, + "map@1": 0.694, + "map@5": 0.7914833333333333, + "map@10": 0.7959170634920636, + "mrr@1": 0.694, + "mrr@5": 0.7914833333333333, + "mrr@10": 0.7959170634920636, "num_pred": 1000, "num_data": 1000 }, "Place365": { "hit@1": 0.391, - "hit@5": 0.695, - "hit@10": 0.786, + "hit@5": 0.694, + "hit@10": 0.784, "ndcg_linear@1": 0.391, - "ndcg_linear@5": 0.5529304236707225, - "ndcg_linear@10": 0.5829135990155158, + "ndcg_linear@5": 0.5524304236707226, + "ndcg_linear@10": 0.5822801209427516, "ndcg_exponential@1": 0.391, - "ndcg_exponential@5": 0.5529304236707225, - "ndcg_exponential@10": 0.5829135990155158, + "ndcg_exponential@5": 0.5524304236707226, + "ndcg_exponential@10": 0.5822801209427516, "precision@1": 0.391, - "precision@5": 0.13899999999999998, - "precision@10": 0.07859999999999999, + "precision@5": 0.1388, + "precision@10": 0.07840000000000001, "recall@1": 0.391, - "recall@5": 0.695, - "recall@10": 0.786, + "recall@5": 0.694, + "recall@10": 0.784, "f1@1": 0.391, - "f1@5": 0.23166666666666674, - "f1@10": 0.14290909090909093, + "f1@5": 0.2313333333333334, + "f1@10": 0.14254545454545456, "map@1": 0.391, - "map@5": 0.50565, - "map@10": 0.51835, + "map@5": 0.5053166666666667, + "map@10": 0.5180718253968254, "mrr@1": 0.391, - "mrr@5": 0.50565, - "mrr@10": 0.51835, + "mrr@5": 0.5053166666666667, + "mrr@10": 0.5180718253968254, "num_pred": 1000, "num_data": 1000 }, "ImageNet-A": { - "hit@1": 0.412, - "hit@5": 0.692, - "hit@10": 0.776, - "ndcg_linear@1": 0.412, - "ndcg_linear@5": 0.5641118126578005, - "ndcg_linear@10": 0.5913889513202993, - "ndcg_exponential@1": 0.412, - "ndcg_exponential@5": 0.5641118126578005, - "ndcg_exponential@10": 0.5913889513202993, - "precision@1": 0.412, - "precision@5": 0.1384, - "precision@10": 0.07759999999999999, - "recall@1": 0.412, - "recall@5": 0.692, - "recall@10": 0.776, - "f1@1": 0.412, - "f1@5": 0.23066666666666671, - "f1@10": 0.14109090909090913, - "map@1": 0.412, - "map@5": 0.5212666666666667, - "map@10": 0.532590873015873, - "mrr@1": 0.412, - "mrr@5": 0.5212666666666667, - "mrr@10": 0.532590873015873, + "hit@1": 0.406, + "hit@5": 0.695, + "hit@10": 0.771, + "ndcg_linear@1": 0.406, + "ndcg_linear@5": 0.5625033620655201, + "ndcg_linear@10": 0.5872719831479489, + "ndcg_exponential@1": 0.406, + "ndcg_exponential@5": 0.5625033620655201, + "ndcg_exponential@10": 0.5872719831479489, + "precision@1": 0.406, + "precision@5": 0.139, + "precision@10": 0.0771, + "recall@1": 0.406, + "recall@5": 0.695, + "recall@10": 0.771, + "f1@1": 0.406, + "f1@5": 0.23166666666666672, + "f1@10": 0.14018181818181819, + "map@1": 0.406, + "map@5": 0.5182, + "map@10": 0.5285365079365079, + "mrr@1": 0.406, + "mrr@5": 0.5182, + "mrr@10": 0.5285365079365079, "num_pred": 1000, "num_data": 1000 }, @@ -209,13 +209,13 @@ "hit@5": 0.96, "hit@10": 0.978, "ndcg_linear@1": 0.839, - "ndcg_linear@5": 0.9069125896441956, - "ndcg_linear@10": 0.9128589977894869, + "ndcg_linear@5": 0.906563082635375, + "ndcg_linear@10": 0.9125214559500125, "ndcg_exponential@1": 0.839, - "ndcg_exponential@5": 0.9069125896441956, - "ndcg_exponential@10": 0.9128589977894869, + "ndcg_exponential@5": 0.906563082635375, + "ndcg_exponential@10": 0.9125214559500125, "precision@1": 0.839, - "precision@5": 0.192, + "precision@5": 0.19200000000000003, "precision@10": 0.09780000000000001, "recall@1": 0.839, "recall@5": 0.96, @@ -224,515 +224,515 @@ "f1@5": 0.32, "f1@10": 0.17781818181818182, "map@1": 0.839, - "map@5": 0.8889333333333334, - "map@10": 0.8914634920634922, + "map@5": 0.8885, + "map@10": 0.89104126984127, "mrr@1": 0.839, - "mrr@5": 0.8889333333333334, - "mrr@10": 0.8914634920634922, + "mrr@5": 0.8885, + "mrr@10": 0.89104126984127, "num_pred": 1000, "num_data": 1000 }, "ObjectNet": { "hit@1": 0.69, - "hit@5": 0.859, - "hit@10": 0.915, + "hit@5": 0.86, + "hit@10": 0.919, "ndcg_linear@1": 0.69, - "ndcg_linear@5": 0.7820151814813446, - "ndcg_linear@10": 0.8002958335952403, + "ndcg_linear@5": 0.7832213711303225, + "ndcg_linear@10": 0.8022186363668451, "ndcg_exponential@1": 0.69, - "ndcg_exponential@5": 0.7820151814813446, - "ndcg_exponential@10": 0.8002958335952403, + "ndcg_exponential@5": 0.7832213711303225, + "ndcg_exponential@10": 0.8022186363668451, "precision@1": 0.69, - "precision@5": 0.1718, - "precision@10": 0.09150000000000001, + "precision@5": 0.172, + "precision@10": 0.09190000000000001, "recall@1": 0.69, - "recall@5": 0.859, - "recall@10": 0.915, + "recall@5": 0.86, + "recall@10": 0.919, "f1@1": 0.69, - "f1@5": 0.2863333333333334, - "f1@10": 0.16636363636363635, + "f1@5": 0.28666666666666674, + "f1@10": 0.1670909090909091, "map@1": 0.69, - "map@5": 0.7562333333333333, - "map@10": 0.7638757936507936, + "map@5": 0.75745, + "map@10": 0.765243253968254, "mrr@1": 0.69, - "mrr@5": 0.7562333333333333, - "mrr@10": 0.7638757936507936, + "mrr@5": 0.75745, + "mrr@10": 0.765243253968254, "num_pred": 1000, "num_data": 1000 }, "Country211": { - "hit@1": 0.248, + "hit@1": 0.246, "hit@5": 0.457, - "hit@10": 0.559, - "ndcg_linear@1": 0.248, - "ndcg_linear@5": 0.3569485635087294, - "ndcg_linear@10": 0.3901006229753951, - "ndcg_exponential@1": 0.248, - "ndcg_exponential@5": 0.3569485635087294, - "ndcg_exponential@10": 0.3901006229753951, - "precision@1": 0.248, + "hit@10": 0.557, + "ndcg_linear@1": 0.246, + "ndcg_linear@5": 0.35633363563916204, + "ndcg_linear@10": 0.3889270717052165, + "ndcg_exponential@1": 0.246, + "ndcg_exponential@5": 0.35633363563916204, + "ndcg_exponential@10": 0.3889270717052165, + "precision@1": 0.246, "precision@5": 0.09140000000000001, - "precision@10": 0.0559, - "recall@1": 0.248, + "precision@10": 0.05569999999999999, + "recall@1": 0.246, "recall@5": 0.457, - "recall@10": 0.559, - "f1@1": 0.248, + "recall@10": 0.557, + "f1@1": 0.246, "f1@5": 0.15233333333333338, - "f1@10": 0.10163636363636365, - "map@1": 0.248, - "map@5": 0.32385, - "map@10": 0.33762063492063493, - "mrr@1": 0.248, - "mrr@5": 0.32385, - "mrr@10": 0.33762063492063493, + "f1@10": 0.10127272727272728, + "map@1": 0.246, + "map@5": 0.3230166666666667, + "map@10": 0.3366075396825397, + "mrr@1": 0.246, + "mrr@5": 0.3230166666666667, + "mrr@10": 0.3366075396825397, "num_pred": 1000, "num_data": 1000 }, "OK-VQA": { - "hit@1": 0.332, - "hit@5": 0.617, - "hit@10": 0.717, - "ndcg_linear@1": 0.332, - "ndcg_linear@5": 0.485871864688508, - "ndcg_linear@10": 0.5182227215507165, - "ndcg_exponential@1": 0.332, - "ndcg_exponential@5": 0.485871864688508, - "ndcg_exponential@10": 0.5182227215507165, - "precision@1": 0.332, - "precision@5": 0.12340000000000001, - "precision@10": 0.07169999999999999, - "recall@1": 0.332, - "recall@5": 0.617, - "recall@10": 0.717, - "f1@1": 0.332, - "f1@5": 0.20566666666666672, - "f1@10": 0.13036363636363638, - "map@1": 0.332, - "map@5": 0.4420333333333333, - "map@10": 0.45539007936507936, - "mrr@1": 0.332, - "mrr@5": 0.4420333333333333, - "mrr@10": 0.45539007936507936, + "hit@1": 0.331, + "hit@5": 0.618, + "hit@10": 0.72, + "ndcg_linear@1": 0.331, + "ndcg_linear@5": 0.4860205770028855, + "ndcg_linear@10": 0.5188775083809797, + "ndcg_exponential@1": 0.331, + "ndcg_exponential@5": 0.4860205770028855, + "ndcg_exponential@10": 0.5188775083809797, + "precision@1": 0.331, + "precision@5": 0.12359999999999999, + "precision@10": 0.072, + "recall@1": 0.331, + "recall@5": 0.618, + "recall@10": 0.72, + "f1@1": 0.331, + "f1@5": 0.20600000000000004, + "f1@10": 0.13090909090909095, + "map@1": 0.331, + "map@5": 0.44189999999999996, + "map@10": 0.4553813492063492, + "mrr@1": 0.331, + "mrr@5": 0.44189999999999996, + "mrr@10": 0.4553813492063492, "num_pred": 1000, "num_data": 1000 }, "A-OKVQA": { - "hit@1": 0.21, + "hit@1": 0.208, "hit@5": 0.5, - "hit@10": 0.638, - "ndcg_linear@1": 0.21, - "ndcg_linear@5": 0.36039694414599044, - "ndcg_linear@10": 0.40511531278859386, - "ndcg_exponential@1": 0.21, - "ndcg_exponential@5": 0.36039694414599044, - "ndcg_exponential@10": 0.40511531278859386, - "precision@1": 0.21, + "hit@10": 0.641, + "ndcg_linear@1": 0.208, + "ndcg_linear@5": 0.35925003863291044, + "ndcg_linear@10": 0.4048838191737925, + "ndcg_exponential@1": 0.208, + "ndcg_exponential@5": 0.35925003863291044, + "ndcg_exponential@10": 0.4048838191737925, + "precision@1": 0.208, "precision@5": 0.1, - "precision@10": 0.0638, - "recall@1": 0.21, + "precision@10": 0.06409999999999999, + "recall@1": 0.208, "recall@5": 0.5, - "recall@10": 0.638, - "f1@1": 0.21, + "recall@10": 0.641, + "f1@1": 0.208, "f1@5": 0.1666666666666667, - "f1@10": 0.11600000000000002, - "map@1": 0.21, - "map@5": 0.31425, - "map@10": 0.3327503968253968, - "mrr@1": 0.21, - "mrr@5": 0.31425, - "mrr@10": 0.3327503968253968, + "f1@10": 0.11654545454545456, + "map@1": 0.208, + "map@5": 0.31276666666666664, + "map@10": 0.3316174603174603, + "mrr@1": 0.208, + "mrr@5": 0.31276666666666664, + "mrr@10": 0.3316174603174603, "num_pred": 1000, "num_data": 1000 }, "DocVQA": { - "hit@1": 0.414, - "hit@5": 0.597, - "hit@10": 0.664, - "ndcg_linear@1": 0.414, - "ndcg_linear@5": 0.5132212142752178, - "ndcg_linear@10": 0.5351036098010266, - "ndcg_exponential@1": 0.414, - "ndcg_exponential@5": 0.5132212142752178, - "ndcg_exponential@10": 0.5351036098010266, - "precision@1": 0.414, - "precision@5": 0.11939999999999998, - "precision@10": 0.06639999999999997, - "recall@1": 0.414, - "recall@5": 0.597, - "recall@10": 0.664, - "f1@1": 0.414, - "f1@5": 0.19900000000000007, - "f1@10": 0.12072727272727275, - "map@1": 0.414, - "map@5": 0.48518333333333336, - "map@10": 0.4943369047619047, - "mrr@1": 0.414, - "mrr@5": 0.48518333333333336, - "mrr@10": 0.4943369047619047, + "hit@1": 0.411, + "hit@5": 0.599, + "hit@10": 0.665, + "ndcg_linear@1": 0.411, + "ndcg_linear@5": 0.5127389968360239, + "ndcg_linear@10": 0.5340892602528474, + "ndcg_exponential@1": 0.411, + "ndcg_exponential@5": 0.5127389968360239, + "ndcg_exponential@10": 0.5340892602528474, + "precision@1": 0.411, + "precision@5": 0.11979999999999999, + "precision@10": 0.0665, + "recall@1": 0.411, + "recall@5": 0.599, + "recall@10": 0.665, + "f1@1": 0.411, + "f1@5": 0.19966666666666671, + "f1@10": 0.12090909090909094, + "map@1": 0.411, + "map@5": 0.48388333333333333, + "map@10": 0.49269444444444443, + "mrr@1": 0.411, + "mrr@5": 0.48388333333333333, + "mrr@10": 0.49269444444444443, "num_pred": 1000, "num_data": 1000 }, "InfographicsVQA": { - "hit@1": 0.203, - "hit@5": 0.383, - "hit@10": 0.467, - "ndcg_linear@1": 0.203, - "ndcg_linear@5": 0.2983142031321632, - "ndcg_linear@10": 0.3252840110636628, - "ndcg_exponential@1": 0.203, - "ndcg_exponential@5": 0.2983142031321632, - "ndcg_exponential@10": 0.3252840110636628, - "precision@1": 0.203, - "precision@5": 0.07659999999999999, - "precision@10": 0.0467, - "recall@1": 0.203, - "recall@5": 0.383, - "recall@10": 0.467, - "f1@1": 0.203, - "f1@5": 0.1276666666666667, - "f1@10": 0.08490909090909092, - "map@1": 0.203, - "map@5": 0.2702, - "map@10": 0.2812119047619048, - "mrr@1": 0.203, - "mrr@5": 0.2702, - "mrr@10": 0.2812119047619048, + "hit@1": 0.205, + "hit@5": 0.382, + "hit@10": 0.465, + "ndcg_linear@1": 0.205, + "ndcg_linear@5": 0.2988479614524779, + "ndcg_linear@10": 0.32564168724726633, + "ndcg_exponential@1": 0.205, + "ndcg_exponential@5": 0.2988479614524779, + "ndcg_exponential@10": 0.32564168724726633, + "precision@1": 0.205, + "precision@5": 0.07640000000000001, + "precision@10": 0.0465, + "recall@1": 0.205, + "recall@5": 0.382, + "recall@10": 0.465, + "f1@1": 0.205, + "f1@5": 0.12733333333333338, + "f1@10": 0.08454545454545456, + "map@1": 0.205, + "map@5": 0.2712166666666667, + "map@10": 0.2822400793650794, + "mrr@1": 0.205, + "mrr@5": 0.2712166666666667, + "mrr@10": 0.2822400793650794, "num_pred": 1000, "num_data": 1000 }, "ChartQA": { - "hit@1": 0.178, + "hit@1": 0.177, "hit@5": 0.257, - "hit@10": 0.301, - "ndcg_linear@1": 0.178, - "ndcg_linear@5": 0.21916174591757567, - "ndcg_linear@10": 0.23326245822360894, - "ndcg_exponential@1": 0.178, - "ndcg_exponential@5": 0.21916174591757567, - "ndcg_exponential@10": 0.23326245822360894, - "precision@1": 0.178, + "hit@10": 0.298, + "ndcg_linear@1": 0.177, + "ndcg_linear@5": 0.21881871686118, + "ndcg_linear@10": 0.23208444540466167, + "ndcg_exponential@1": 0.177, + "ndcg_exponential@5": 0.21881871686118, + "ndcg_exponential@10": 0.23208444540466167, + "precision@1": 0.177, "precision@5": 0.05140000000000001, - "precision@10": 0.030100000000000002, - "recall@1": 0.178, + "precision@10": 0.029799999999999997, + "recall@1": 0.177, "recall@5": 0.257, - "recall@10": 0.301, - "f1@1": 0.178, + "recall@10": 0.298, + "f1@1": 0.177, "f1@5": 0.08566666666666668, - "f1@10": 0.054727272727272736, - "map@1": 0.178, - "map@5": 0.20663333333333334, - "map@10": 0.21237777777777778, - "mrr@1": 0.178, - "mrr@5": 0.20663333333333334, - "mrr@10": 0.21237777777777778, + "f1@10": 0.054181818181818185, + "map@1": 0.177, + "map@5": 0.20615, + "map@10": 0.21162896825396826, + "mrr@1": 0.177, + "mrr@5": 0.20615, + "mrr@10": 0.21162896825396826, "num_pred": 1000, "num_data": 1000 }, "Visual7W": { "hit@1": 0.222, - "hit@5": 0.544, - "hit@10": 0.674, + "hit@5": 0.541, + "hit@10": 0.676, "ndcg_linear@1": 0.222, - "ndcg_linear@5": 0.3896222077826222, - "ndcg_linear@10": 0.43208253526314966, + "ndcg_linear@5": 0.38824307210566933, + "ndcg_linear@10": 0.4323162096670103, "ndcg_exponential@1": 0.222, - "ndcg_exponential@5": 0.3896222077826222, - "ndcg_exponential@10": 0.43208253526314966, + "ndcg_exponential@5": 0.38824307210566933, + "ndcg_exponential@10": 0.4323162096670103, "precision@1": 0.222, - "precision@5": 0.10879999999999998, - "precision@10": 0.06739999999999999, + "precision@5": 0.10819999999999999, + "precision@10": 0.0676, "recall@1": 0.222, - "recall@5": 0.544, - "recall@10": 0.674, + "recall@5": 0.541, + "recall@10": 0.676, "f1@1": 0.222, - "f1@5": 0.1813333333333334, - "f1@10": 0.12254545454545457, + "f1@5": 0.18033333333333337, + "f1@10": 0.12290909090909094, "map@1": 0.222, - "map@5": 0.33858333333333335, - "map@10": 0.3563468253968254, + "map@5": 0.3377166666666667, + "map@10": 0.35614603174603177, "mrr@1": 0.222, - "mrr@5": 0.33858333333333335, - "mrr@10": 0.3563468253968254, + "mrr@5": 0.3377166666666667, + "mrr@10": 0.35614603174603177, "num_pred": 1000, "num_data": 1000 }, "ScienceQA": { - "hit@1": 0.28, - "hit@5": 0.64, - "hit@10": 0.763, - "ndcg_linear@1": 0.28, - "ndcg_linear@5": 0.4740675519059985, - "ndcg_linear@10": 0.514010974748877, - "ndcg_exponential@1": 0.28, - "ndcg_exponential@5": 0.4740675519059985, - "ndcg_exponential@10": 0.514010974748877, - "precision@1": 0.28, - "precision@5": 0.12799999999999997, - "precision@10": 0.07629999999999999, - "recall@1": 0.28, - "recall@5": 0.64, - "recall@10": 0.763, - "f1@1": 0.28, - "f1@5": 0.21333333333333337, - "f1@10": 0.13872727272727275, - "map@1": 0.28, - "map@5": 0.4186333333333333, - "map@10": 0.43520793650793654, - "mrr@1": 0.28, - "mrr@5": 0.4186333333333333, - "mrr@10": 0.43520793650793654, + "hit@1": 0.282, + "hit@5": 0.639, + "hit@10": 0.764, + "ndcg_linear@1": 0.282, + "ndcg_linear@5": 0.4747518293419879, + "ndcg_linear@10": 0.5153321511993152, + "ndcg_exponential@1": 0.282, + "ndcg_exponential@5": 0.4747518293419879, + "ndcg_exponential@10": 0.5153321511993152, + "precision@1": 0.282, + "precision@5": 0.12779999999999997, + "precision@10": 0.07640000000000001, + "recall@1": 0.282, + "recall@5": 0.639, + "recall@10": 0.764, + "f1@1": 0.282, + "f1@5": 0.21300000000000005, + "f1@10": 0.13890909090909093, + "map@1": 0.282, + "map@5": 0.4199, + "map@10": 0.436731746031746, + "mrr@1": 0.282, + "mrr@5": 0.4199, + "mrr@10": 0.436731746031746, "num_pred": 1000, "num_data": 1000 }, "VizWiz": { - "hit@1": 0.39, - "hit@5": 0.613, - "hit@10": 0.673, - "ndcg_linear@1": 0.39, - "ndcg_linear@5": 0.5101413361528128, - "ndcg_linear@10": 0.5295620828845852, - "ndcg_exponential@1": 0.39, - "ndcg_exponential@5": 0.5101413361528128, - "ndcg_exponential@10": 0.5295620828845852, - "precision@1": 0.39, - "precision@5": 0.1226, - "precision@10": 0.0673, - "recall@1": 0.39, - "recall@5": 0.613, - "recall@10": 0.673, - "f1@1": 0.39, - "f1@5": 0.2043333333333334, - "f1@10": 0.12236363636363638, - "map@1": 0.39, - "map@5": 0.4757833333333333, - "map@10": 0.4838107142857143, - "mrr@1": 0.39, - "mrr@5": 0.4757833333333333, - "mrr@10": 0.4838107142857143, + "hit@1": 0.389, + "hit@5": 0.612, + "hit@10": 0.672, + "ndcg_linear@1": 0.389, + "ndcg_linear@5": 0.5095423840427541, + "ndcg_linear@10": 0.5290233133631975, + "ndcg_exponential@1": 0.389, + "ndcg_exponential@5": 0.5095423840427541, + "ndcg_exponential@10": 0.5290233133631975, + "precision@1": 0.389, + "precision@5": 0.1224, + "precision@10": 0.06719999999999998, + "recall@1": 0.389, + "recall@5": 0.612, + "recall@10": 0.672, + "f1@1": 0.389, + "f1@5": 0.20400000000000007, + "f1@10": 0.1221818181818182, + "map@1": 0.389, + "map@5": 0.47526666666666667, + "map@10": 0.4833555555555555, + "mrr@1": 0.389, + "mrr@5": 0.47526666666666667, + "mrr@10": 0.4833555555555555, "num_pred": 1000, "num_data": 1000 }, "GQA": { - "hit@1": 0.769, + "hit@1": 0.768, "hit@5": 0.941, - "hit@10": 0.965, - "ndcg_linear@1": 0.769, - "ndcg_linear@5": 0.865366522766234, - "ndcg_linear@10": 0.8730495254605352, - "ndcg_exponential@1": 0.769, - "ndcg_exponential@5": 0.865366522766234, - "ndcg_exponential@10": 0.8730495254605352, - "precision@1": 0.769, + "hit@10": 0.964, + "ndcg_linear@1": 0.768, + "ndcg_linear@5": 0.8654056760410835, + "ndcg_linear@10": 0.8727851790279448, + "ndcg_exponential@1": 0.768, + "ndcg_exponential@5": 0.8654056760410835, + "ndcg_exponential@10": 0.8727851790279448, + "precision@1": 0.768, "precision@5": 0.1882, - "precision@10": 0.0965, - "recall@1": 0.769, + "precision@10": 0.0964, + "recall@1": 0.768, "recall@5": 0.941, - "recall@10": 0.965, - "f1@1": 0.769, + "recall@10": 0.964, + "f1@1": 0.768, "f1@5": 0.31366666666666665, - "f1@10": 0.17545454545454545, - "map@1": 0.769, + "f1@10": 0.17527272727272725, + "map@1": 0.768, "map@5": 0.8397666666666667, - "map@10": 0.8428908730158731, - "mrr@1": 0.769, + "map@10": 0.8427769841269842, + "mrr@1": 0.768, "mrr@5": 0.8397666666666667, - "mrr@10": 0.8428908730158731, + "mrr@10": 0.8427769841269842, "num_pred": 1000, "num_data": 1000 }, "TextVQA": { - "hit@1": 0.468, + "hit@1": 0.464, "hit@5": 0.685, - "hit@10": 0.74, - "ndcg_linear@1": 0.468, - "ndcg_linear@5": 0.5857686784282851, - "ndcg_linear@10": 0.6036026979398256, - "ndcg_exponential@1": 0.468, - "ndcg_exponential@5": 0.5857686784282851, - "ndcg_exponential@10": 0.6036026979398256, - "precision@1": 0.468, + "hit@10": 0.743, + "ndcg_linear@1": 0.464, + "ndcg_linear@5": 0.5843173556347134, + "ndcg_linear@10": 0.6028441763626061, + "ndcg_exponential@1": 0.464, + "ndcg_exponential@5": 0.5843173556347134, + "ndcg_exponential@10": 0.6028441763626061, + "precision@1": 0.464, "precision@5": 0.137, - "precision@10": 0.074, - "recall@1": 0.468, + "precision@10": 0.07429999999999999, + "recall@1": 0.464, "recall@5": 0.685, - "recall@10": 0.74, - "f1@1": 0.468, - "f1@5": 0.22833333333333336, - "f1@10": 0.13454545454545455, - "map@1": 0.468, - "map@5": 0.5525333333333333, - "map@10": 0.5599246031746031, - "mrr@1": 0.468, - "mrr@5": 0.5525333333333333, - "mrr@10": 0.5599246031746031, + "recall@10": 0.743, + "f1@1": 0.464, + "f1@5": 0.2283333333333334, + "f1@10": 0.13509090909090912, + "map@1": 0.464, + "map@5": 0.5505833333333332, + "map@10": 0.5580916666666667, + "mrr@1": 0.464, + "mrr@5": 0.5505833333333332, + "mrr@10": 0.5580916666666667, "num_pred": 1000, "num_data": 1000 }, "VisDial": { - "hit@1": 0.608, - "hit@5": 0.843, - "hit@10": 0.914, - "ndcg_linear@1": 0.608, - "ndcg_linear@5": 0.733577948489228, - "ndcg_linear@10": 0.7570027510391492, - "ndcg_exponential@1": 0.608, - "ndcg_exponential@5": 0.733577948489228, - "ndcg_exponential@10": 0.7570027510391492, - "precision@1": 0.608, - "precision@5": 0.1686, - "precision@10": 0.09140000000000001, - "recall@1": 0.608, - "recall@5": 0.843, - "recall@10": 0.914, - "f1@1": 0.608, - "f1@5": 0.281, - "f1@10": 0.16618181818181818, - "map@1": 0.608, - "map@5": 0.6971499999999999, - "map@10": 0.7070952380952381, - "mrr@1": 0.608, - "mrr@5": 0.6971499999999999, - "mrr@10": 0.7070952380952381, + "hit@1": 0.609, + "hit@5": 0.845, + "hit@10": 0.917, + "ndcg_linear@1": 0.609, + "ndcg_linear@5": 0.7342769357072357, + "ndcg_linear@10": 0.7577878968005705, + "ndcg_exponential@1": 0.609, + "ndcg_exponential@5": 0.7342769357072357, + "ndcg_exponential@10": 0.7577878968005705, + "precision@1": 0.609, + "precision@5": 0.169, + "precision@10": 0.09170000000000002, + "recall@1": 0.609, + "recall@5": 0.845, + "recall@10": 0.917, + "f1@1": 0.609, + "f1@5": 0.2816666666666667, + "f1@10": 0.16672727272727275, + "map@1": 0.609, + "map@5": 0.6974833333333333, + "map@10": 0.7073238095238095, + "mrr@1": 0.609, + "mrr@5": 0.6974833333333333, + "mrr@10": 0.7073238095238095, "num_pred": 1000, "num_data": 1000 }, "CIRR": { "hit@1": 0.549, - "hit@5": 0.89, - "hit@10": 0.944, + "hit@5": 0.892, + "hit@10": 0.945, "ndcg_linear@1": 0.549, - "ndcg_linear@5": 0.7379230235182084, - "ndcg_linear@10": 0.7559089463717755, + "ndcg_linear@5": 0.7396286716682412, + "ndcg_linear@10": 0.7571843778230585, "ndcg_exponential@1": 0.549, - "ndcg_exponential@5": 0.7379230235182084, - "ndcg_exponential@10": 0.7559089463717755, + "ndcg_exponential@5": 0.7396286716682412, + "ndcg_exponential@10": 0.7571843778230585, "precision@1": 0.549, - "precision@5": 0.178, - "precision@10": 0.09440000000000003, + "precision@5": 0.17840000000000003, + "precision@10": 0.0945, "recall@1": 0.549, - "recall@5": 0.89, - "recall@10": 0.944, + "recall@5": 0.892, + "recall@10": 0.945, "f1@1": 0.549, - "f1@5": 0.2966666666666667, - "f1@10": 0.17163636363636361, + "f1@5": 0.29733333333333334, + "f1@10": 0.17181818181818181, "map@1": 0.549, - "map@5": 0.6865999999999999, - "map@10": 0.6943325396825396, + "map@5": 0.6881666666666666, + "map@10": 0.6956579365079365, "mrr@1": 0.549, - "mrr@5": 0.6865999999999999, - "mrr@10": 0.6943325396825396, + "mrr@5": 0.6881666666666666, + "mrr@10": 0.6956579365079365, "num_pred": 1000, "num_data": 1000 }, "VisualNews_t2i": { - "hit@1": 0.797, + "hit@1": 0.795, "hit@5": 0.915, "hit@10": 0.936, - "ndcg_linear@1": 0.797, - "ndcg_linear@5": 0.8627496822471115, - "ndcg_linear@10": 0.8696409132009587, - "ndcg_exponential@1": 0.797, - "ndcg_exponential@5": 0.8627496822471115, - "ndcg_exponential@10": 0.8696409132009587, - "precision@1": 0.797, + "ndcg_linear@1": 0.795, + "ndcg_linear@5": 0.8621940123889464, + "ndcg_linear@10": 0.8691305683347267, + "ndcg_exponential@1": 0.795, + "ndcg_exponential@5": 0.8621940123889464, + "ndcg_exponential@10": 0.8691305683347267, + "precision@1": 0.795, "precision@5": 0.18300000000000002, "precision@10": 0.09360000000000002, - "recall@1": 0.797, + "recall@1": 0.795, "recall@5": 0.915, "recall@10": 0.936, - "f1@1": 0.797, - "f1@5": 0.305, + "f1@1": 0.795, + "f1@5": 0.30500000000000005, "f1@10": 0.17018181818181818, - "map@1": 0.797, - "map@5": 0.8451166666666665, - "map@10": 0.8480198412698412, - "mrr@1": 0.797, - "mrr@5": 0.8451166666666665, - "mrr@10": 0.8480198412698412, + "map@1": 0.795, + "map@5": 0.8443333333333333, + "map@10": 0.8472777777777777, + "mrr@1": 0.795, + "mrr@5": 0.8443333333333333, + "mrr@10": 0.8472777777777777, "num_pred": 1000, "num_data": 1000 }, "VisualNews_i2t": { "hit@1": 0.836, "hit@5": 0.932, - "hit@10": 0.946, + "hit@10": 0.947, "ndcg_linear@1": 0.836, - "ndcg_linear@5": 0.8899078662440771, - "ndcg_linear@10": 0.8946317706132814, + "ndcg_linear@5": 0.8901341606296082, + "ndcg_linear@10": 0.8951063875148078, "ndcg_exponential@1": 0.836, - "ndcg_exponential@5": 0.8899078662440771, - "ndcg_exponential@10": 0.8946317706132814, + "ndcg_exponential@5": 0.8901341606296082, + "ndcg_exponential@10": 0.8951063875148078, "precision@1": 0.836, "precision@5": 0.18640000000000004, - "precision@10": 0.0946, + "precision@10": 0.0947, "recall@1": 0.836, "recall@5": 0.932, - "recall@10": 0.946, + "recall@10": 0.947, "f1@1": 0.836, "f1@5": 0.3106666666666667, - "f1@10": 0.172, + "f1@10": 0.17218181818181819, "map@1": 0.836, - "map@5": 0.8756666666666666, - "map@10": 0.8777309523809524, + "map@5": 0.8759333333333332, + "map@10": 0.8780559523809525, "mrr@1": 0.836, - "mrr@5": 0.8756666666666666, - "mrr@10": 0.8777309523809524, + "mrr@5": 0.8759333333333332, + "mrr@10": 0.8780559523809525, "num_pred": 1000, "num_data": 1000 }, "MSCOCO_t2i": { - "hit@1": 0.712, + "hit@1": 0.715, "hit@5": 0.921, - "hit@10": 0.96, - "ndcg_linear@1": 0.712, - "ndcg_linear@5": 0.8275075440748253, - "ndcg_linear@10": 0.8405579014938537, - "ndcg_exponential@1": 0.712, - "ndcg_exponential@5": 0.8275075440748253, - "ndcg_exponential@10": 0.8405579014938537, - "precision@1": 0.712, + "hit@10": 0.961, + "ndcg_linear@1": 0.715, + "ndcg_linear@5": 0.8285714725622172, + "ndcg_linear@10": 0.8418352208528531, + "ndcg_exponential@1": 0.715, + "ndcg_exponential@5": 0.8285714725622172, + "ndcg_exponential@10": 0.8418352208528531, + "precision@1": 0.715, "precision@5": 0.18420000000000003, - "precision@10": 0.09600000000000002, - "recall@1": 0.712, + "precision@10": 0.0961, + "recall@1": 0.715, "recall@5": 0.921, - "recall@10": 0.96, - "f1@1": 0.712, + "recall@10": 0.961, + "f1@1": 0.715, "f1@5": 0.307, - "f1@10": 0.17454545454545456, - "map@1": 0.712, - "map@5": 0.7959833333333334, - "map@10": 0.8016265873015873, - "mrr@1": 0.712, - "mrr@5": 0.7959833333333334, - "mrr@10": 0.8016265873015873, + "f1@10": 0.17472727272727276, + "map@1": 0.715, + "map@5": 0.7974166666666668, + "map@10": 0.8030861111111112, + "mrr@1": 0.715, + "mrr@5": 0.7974166666666668, + "mrr@10": 0.8030861111111112, "num_pred": 1000, "num_data": 1000 }, "MSCOCO_i2t": { - "hit@1": 0.577, - "hit@5": 0.858, - "hit@10": 0.939, - "ndcg_linear@1": 0.577, - "ndcg_linear@5": 0.7273292794403879, - "ndcg_linear@10": 0.7536734944900013, - "ndcg_exponential@1": 0.577, - "ndcg_exponential@5": 0.7273292794403879, - "ndcg_exponential@10": 0.7536734944900013, - "precision@1": 0.577, - "precision@5": 0.17160000000000003, - "precision@10": 0.09390000000000001, - "recall@1": 0.577, - "recall@5": 0.858, - "recall@10": 0.939, - "f1@1": 0.577, - "f1@5": 0.286, - "f1@10": 0.1707272727272727, - "map@1": 0.577, - "map@5": 0.6838500000000001, - "map@10": 0.6948119047619048, - "mrr@1": 0.577, - "mrr@5": 0.6838500000000001, - "mrr@10": 0.6948119047619048, + "hit@1": 0.574, + "hit@5": 0.859, + "hit@10": 0.938, + "ndcg_linear@1": 0.574, + "ndcg_linear@5": 0.7269839282082453, + "ndcg_linear@10": 0.7526718699387368, + "ndcg_exponential@1": 0.574, + "ndcg_exponential@5": 0.7269839282082453, + "ndcg_exponential@10": 0.7526718699387368, + "precision@1": 0.574, + "precision@5": 0.1718, + "precision@10": 0.09380000000000001, + "recall@1": 0.574, + "recall@5": 0.859, + "recall@10": 0.938, + "f1@1": 0.574, + "f1@5": 0.2863333333333334, + "f1@10": 0.17054545454545453, + "map@1": 0.574, + "map@5": 0.6830166666666666, + "map@10": 0.6937019841269841, + "mrr@1": 0.574, + "mrr@5": 0.6830166666666666, + "mrr@10": 0.6937019841269841, "num_pred": 1000, "num_data": 1000 }, @@ -741,11 +741,11 @@ "hit@5": 0.977, "hit@10": 0.995, "ndcg_linear@1": 0.676, - "ndcg_linear@5": 0.8498540680608963, - "ndcg_linear@10": 0.8560163381696798, + "ndcg_linear@5": 0.8496022739842777, + "ndcg_linear@10": 0.8558002810061565, "ndcg_exponential@1": 0.676, - "ndcg_exponential@5": 0.8498540680608963, - "ndcg_exponential@10": 0.8560163381696798, + "ndcg_exponential@5": 0.8496022739842777, + "ndcg_exponential@10": 0.8558002810061565, "precision@1": 0.676, "precision@5": 0.19540000000000005, "precision@10": 0.09950000000000002, @@ -756,433 +756,433 @@ "f1@5": 0.32566666666666666, "f1@10": 0.18090909090909088, "map@1": 0.676, - "map@5": 0.8062999999999999, - "map@10": 0.8090440476190477, + "map@5": 0.806, + "map@10": 0.808779761904762, "mrr@1": 0.676, - "mrr@5": 0.8062999999999999, - "mrr@10": 0.8090440476190477, + "mrr@5": 0.806, + "mrr@10": 0.808779761904762, "num_pred": 1000, "num_data": 1000 }, "WebQA": { - "hit@1": 0.914, + "hit@1": 0.913, "hit@5": 0.991, - "hit@10": 0.994, - "ndcg_linear@1": 0.914, - "ndcg_linear@5": 0.9583965355112005, - "ndcg_linear@10": 0.9593428375202905, - "ndcg_exponential@1": 0.914, - "ndcg_exponential@5": 0.9583965355112005, - "ndcg_exponential@10": 0.9593428375202905, - "precision@1": 0.914, + "hit@10": 0.993, + "ndcg_linear@1": 0.913, + "ndcg_linear@5": 0.9577833883184351, + "ndcg_linear@10": 0.958440625501207, + "ndcg_exponential@1": 0.913, + "ndcg_exponential@5": 0.9577833883184351, + "ndcg_exponential@10": 0.958440625501207, + "precision@1": 0.913, "precision@5": 0.19820000000000004, - "precision@10": 0.0994, - "recall@1": 0.914, + "precision@10": 0.09930000000000001, + "recall@1": 0.913, "recall@5": 0.991, - "recall@10": 0.994, - "f1@1": 0.914, + "recall@10": 0.993, + "f1@1": 0.913, "f1@5": 0.3303333333333333, - "f1@10": 0.1807272727272727, - "map@1": 0.914, - "map@5": 0.9472333333333334, - "map@10": 0.9476111111111111, - "mrr@1": 0.914, - "mrr@5": 0.9472333333333334, - "mrr@10": 0.9476111111111111, + "f1@10": 0.18054545454545454, + "map@1": 0.913, + "map@5": 0.9464333333333333, + "map@10": 0.9467111111111111, + "mrr@1": 0.913, + "mrr@5": 0.9464333333333333, + "mrr@10": 0.9467111111111111, "num_pred": 1000, "num_data": 1000 }, "FashionIQ": { - "hit@1": 0.378, - "hit@5": 0.645, - "hit@10": 0.742, - "ndcg_linear@1": 0.378, - "ndcg_linear@5": 0.518666678313658, - "ndcg_linear@10": 0.5498506170222675, - "ndcg_exponential@1": 0.378, - "ndcg_exponential@5": 0.518666678313658, - "ndcg_exponential@10": 0.5498506170222675, - "precision@1": 0.378, - "precision@5": 0.129, - "precision@10": 0.07419999999999999, - "recall@1": 0.378, - "recall@5": 0.645, - "recall@10": 0.742, - "f1@1": 0.378, - "f1@5": 0.21500000000000005, - "f1@10": 0.13490909090909092, - "map@1": 0.378, - "map@5": 0.4767, - "map@10": 0.48945396825396825, - "mrr@1": 0.378, - "mrr@5": 0.4767, - "mrr@10": 0.48945396825396825, + "hit@1": 0.376, + "hit@5": 0.644, + "hit@10": 0.743, + "ndcg_linear@1": 0.376, + "ndcg_linear@5": 0.5169380356278844, + "ndcg_linear@10": 0.5489044628249767, + "ndcg_exponential@1": 0.376, + "ndcg_exponential@5": 0.5169380356278844, + "ndcg_exponential@10": 0.5489044628249767, + "precision@1": 0.376, + "precision@5": 0.1288, + "precision@10": 0.07429999999999999, + "recall@1": 0.376, + "recall@5": 0.644, + "recall@10": 0.743, + "f1@1": 0.376, + "f1@5": 0.21466666666666676, + "f1@10": 0.13509090909090912, + "map@1": 0.376, + "map@5": 0.47473333333333334, + "map@10": 0.4879, + "mrr@1": 0.376, + "mrr@5": 0.47473333333333334, + "mrr@10": 0.4879, "num_pred": 1000, "num_data": 1000 }, "Wiki-SS-NQ": { - "hit@1": 0.782, + "hit@1": 0.784, "hit@5": 0.952, - "hit@10": 0.969, - "ndcg_linear@1": 0.782, - "ndcg_linear@5": 0.8778108529080692, - "ndcg_linear@10": 0.8834557157208128, - "ndcg_exponential@1": 0.782, - "ndcg_exponential@5": 0.8778108529080692, - "ndcg_exponential@10": 0.8834557157208128, - "precision@1": 0.782, + "hit@10": 0.968, + "ndcg_linear@1": 0.784, + "ndcg_linear@5": 0.8784074567178853, + "ndcg_linear@10": 0.8837323645935, + "ndcg_exponential@1": 0.784, + "ndcg_exponential@5": 0.8784074567178853, + "ndcg_exponential@10": 0.8837323645935, + "precision@1": 0.784, "precision@5": 0.19040000000000004, - "precision@10": 0.0969, - "recall@1": 0.782, + "precision@10": 0.09680000000000001, + "recall@1": 0.784, "recall@5": 0.952, - "recall@10": 0.969, - "f1@1": 0.782, - "f1@5": 0.31733333333333336, - "f1@10": 0.17618181818181816, - "map@1": 0.782, - "map@5": 0.85265, - "map@10": 0.8550654761904761, - "mrr@1": 0.782, - "mrr@5": 0.85265, - "mrr@10": 0.8550654761904761, + "recall@10": 0.968, + "f1@1": 0.784, + "f1@5": 0.3173333333333333, + "f1@10": 0.176, + "map@1": 0.784, + "map@5": 0.8534666666666667, + "map@10": 0.855754761904762, + "mrr@1": 0.784, + "mrr@5": 0.8534666666666667, + "mrr@10": 0.855754761904762, "num_pred": 1000, "num_data": 1000 }, "OVEN": { - "hit@1": 0.751, - "hit@5": 0.908, - "hit@10": 0.94, - "ndcg_linear@1": 0.751, - "ndcg_linear@5": 0.8387290889357423, - "ndcg_linear@10": 0.8494521089299062, - "ndcg_exponential@1": 0.751, - "ndcg_exponential@5": 0.8387290889357423, - "ndcg_exponential@10": 0.8494521089299062, - "precision@1": 0.751, - "precision@5": 0.1816, - "precision@10": 0.094, - "recall@1": 0.751, - "recall@5": 0.908, - "recall@10": 0.94, - "f1@1": 0.751, - "f1@5": 0.30266666666666675, - "f1@10": 0.1709090909090909, - "map@1": 0.751, - "map@5": 0.8152999999999999, - "map@10": 0.8199472222222222, - "mrr@1": 0.751, - "mrr@5": 0.8152999999999999, - "mrr@10": 0.8199472222222222, + "hit@1": 0.756, + "hit@5": 0.907, + "hit@10": 0.942, + "ndcg_linear@1": 0.756, + "ndcg_linear@5": 0.840336841173973, + "ndcg_linear@10": 0.8519584610947856, + "ndcg_exponential@1": 0.756, + "ndcg_exponential@5": 0.840336841173973, + "ndcg_exponential@10": 0.8519584610947856, + "precision@1": 0.756, + "precision@5": 0.18140000000000003, + "precision@10": 0.09420000000000002, + "recall@1": 0.756, + "recall@5": 0.907, + "recall@10": 0.942, + "f1@1": 0.756, + "f1@5": 0.3023333333333334, + "f1@10": 0.17127272727272724, + "map@1": 0.756, + "map@5": 0.8177833333333333, + "map@10": 0.8227615079365079, + "mrr@1": 0.756, + "mrr@5": 0.8177833333333333, + "mrr@10": 0.8227615079365079, "num_pred": 1000, "num_data": 1000 }, "EDIS": { - "hit@1": 0.96, + "hit@1": 0.961, "hit@5": 0.998, "hit@10": 0.999, - "ndcg_linear@1": 0.96, - "ndcg_linear@5": 0.9831897521142866, - "ndcg_linear@10": 0.9835052169910724, - "ndcg_exponential@1": 0.96, - "ndcg_exponential@5": 0.9831897521142866, - "ndcg_exponential@10": 0.9835052169910724, - "precision@1": 0.96, + "ndcg_linear@1": 0.961, + "ndcg_linear@5": 0.9833585691652171, + "ndcg_linear@10": 0.9836740340420028, + "ndcg_exponential@1": 0.961, + "ndcg_exponential@5": 0.9833585691652171, + "ndcg_exponential@10": 0.9836740340420028, + "precision@1": 0.961, "precision@5": 0.19960000000000003, "precision@10": 0.0999, - "recall@1": 0.96, + "recall@1": 0.961, "recall@5": 0.998, "recall@10": 0.999, - "f1@1": 0.96, + "f1@1": 0.961, "f1@5": 0.3326666666666666, "f1@10": 0.18163636363636362, - "map@1": 0.96, - "map@5": 0.978, - "map@10": 0.978125, - "mrr@1": 0.96, - "mrr@5": 0.978, - "mrr@10": 0.978125, + "map@1": 0.961, + "map@5": 0.97825, + "map@10": 0.978375, + "mrr@1": 0.961, + "mrr@5": 0.97825, + "mrr@10": 0.978375, "num_pred": 1000, "num_data": 1000 }, "MSCOCO": { "hit@1": 0.314, - "hit@5": 0.489, - "hit@10": 0.588, + "hit@5": 0.488, + "hit@10": 0.587, "ndcg_linear@1": 0.314, - "ndcg_linear@5": 0.40410548152820475, - "ndcg_linear@10": 0.43612276758399793, + "ndcg_linear@5": 0.4035438752165599, + "ndcg_linear@10": 0.4355890138757634, "ndcg_exponential@1": 0.314, - "ndcg_exponential@5": 0.40410548152820475, - "ndcg_exponential@10": 0.43612276758399793, + "ndcg_exponential@5": 0.4035438752165599, + "ndcg_exponential@10": 0.4355890138757634, "precision@1": 0.314, - "precision@5": 0.09779999999999998, - "precision@10": 0.0588, + "precision@5": 0.09759999999999999, + "precision@10": 0.0587, "recall@1": 0.314, - "recall@5": 0.489, - "recall@10": 0.588, + "recall@5": 0.488, + "recall@10": 0.587, "f1@1": 0.314, - "f1@5": 0.16300000000000006, - "f1@10": 0.10690909090909094, + "f1@5": 0.1626666666666667, + "f1@10": 0.10672727272727275, "map@1": 0.314, - "map@5": 0.37615, - "map@10": 0.38935198412698413, + "map@5": 0.37573333333333336, + "map@10": 0.38896825396825396, "mrr@1": 0.314, - "mrr@5": 0.37615, - "mrr@10": 0.38935198412698413, + "mrr@5": 0.37573333333333336, + "mrr@10": 0.38896825396825396, "num_pred": 1000, "num_data": 1000 }, "RefCOCO": { - "hit@1": 0.609, - "hit@5": 0.847, - "hit@10": 0.897, - "ndcg_linear@1": 0.609, - "ndcg_linear@5": 0.7376656227905393, - "ndcg_linear@10": 0.7540339140284914, - "ndcg_exponential@1": 0.609, - "ndcg_exponential@5": 0.7376656227905393, - "ndcg_exponential@10": 0.7540339140284914, - "precision@1": 0.609, - "precision@5": 0.1694, - "precision@10": 0.08970000000000002, - "recall@1": 0.609, - "recall@5": 0.847, - "recall@10": 0.897, - "f1@1": 0.609, - "f1@5": 0.2823333333333334, - "f1@10": 0.1630909090909091, - "map@1": 0.609, - "map@5": 0.7011166666666667, - "map@10": 0.7079869047619047, - "mrr@1": 0.609, - "mrr@5": 0.7011166666666667, - "mrr@10": 0.7079869047619047, + "hit@1": 0.612, + "hit@5": 0.846, + "hit@10": 0.902, + "ndcg_linear@1": 0.612, + "ndcg_linear@5": 0.7380986215243596, + "ndcg_linear@10": 0.756279445386705, + "ndcg_exponential@1": 0.612, + "ndcg_exponential@5": 0.7380986215243596, + "ndcg_exponential@10": 0.756279445386705, + "precision@1": 0.612, + "precision@5": 0.16920000000000002, + "precision@10": 0.09020000000000002, + "recall@1": 0.612, + "recall@5": 0.846, + "recall@10": 0.902, + "f1@1": 0.612, + "f1@5": 0.2820000000000001, + "f1@10": 0.164, + "map@1": 0.612, + "map@5": 0.70205, + "map@10": 0.7095968253968254, + "mrr@1": 0.612, + "mrr@5": 0.70205, + "mrr@10": 0.7095968253968254, "num_pred": 1000, "num_data": 1000 }, "RefCOCO-Matching": { - "hit@1": 0.784, + "hit@1": 0.787, "hit@5": 0.994, "hit@10": 0.998, - "ndcg_linear@1": 0.784, - "ndcg_linear@5": 0.9166266927028892, - "ndcg_linear@10": 0.9178536205656439, - "ndcg_exponential@1": 0.784, - "ndcg_exponential@5": 0.9166266927028892, - "ndcg_exponential@10": 0.9178536205656439, - "precision@1": 0.784, + "ndcg_linear@1": 0.787, + "ndcg_linear@5": 0.9176467974394421, + "ndcg_linear@10": 0.918873725302197, + "ndcg_exponential@1": 0.787, + "ndcg_exponential@5": 0.9176467974394421, + "ndcg_exponential@10": 0.918873725302197, + "precision@1": 0.787, "precision@5": 0.1988, "precision@10": 0.09980000000000001, - "recall@1": 0.784, + "recall@1": 0.787, "recall@5": 0.994, "recall@10": 0.998, - "f1@1": 0.784, + "f1@1": 0.787, "f1@5": 0.3313333333333333, "f1@10": 0.18145454545454545, - "map@1": 0.784, - "map@5": 0.8897833333333333, - "map@10": 0.8902511904761905, - "mrr@1": 0.784, - "mrr@5": 0.8867833333333333, - "mrr@10": 0.8872511904761905, + "map@1": 0.787, + "map@5": 0.8911666666666668, + "map@10": 0.8916345238095238, + "mrr@1": 0.787, + "mrr@5": 0.8881666666666668, + "mrr@10": 0.8886345238095238, "num_pred": 1000, "num_data": 1000 }, "Visual7W-Pointing": { "hit@1": 0.665, - "hit@5": 0.851, - "hit@10": 0.899, + "hit@5": 0.848, + "hit@10": 0.902, "ndcg_linear@1": 0.665, - "ndcg_linear@5": 0.7648497950534088, - "ndcg_linear@10": 0.7805119979199628, + "ndcg_linear@5": 0.7638196248863316, + "ndcg_linear@10": 0.781356047977634, "ndcg_exponential@1": 0.665, - "ndcg_exponential@5": 0.7648497950534088, - "ndcg_exponential@10": 0.7805119979199628, + "ndcg_exponential@5": 0.7638196248863316, + "ndcg_exponential@10": 0.781356047977634, "precision@1": 0.665, - "precision@5": 0.1702, - "precision@10": 0.08990000000000001, + "precision@5": 0.16960000000000003, + "precision@10": 0.0902, "recall@1": 0.665, - "recall@5": 0.851, - "recall@10": 0.899, + "recall@5": 0.848, + "recall@10": 0.902, "f1@1": 0.665, - "f1@5": 0.2836666666666667, - "f1@10": 0.16345454545454544, + "f1@5": 0.2826666666666667, + "f1@10": 0.164, "map@1": 0.665, - "map@5": 0.7361333333333332, - "map@10": 0.7426833333333333, + "map@5": 0.7357166666666667, + "map@10": 0.7430007936507936, "mrr@1": 0.665, - "mrr@5": 0.7361333333333332, - "mrr@10": 0.7426833333333333, + "mrr@5": 0.7357166666666667, + "mrr@10": 0.7430007936507936, "num_pred": 1000, "num_data": 1000 } }, "video": { "K700": { - "hit@1": 0.397, - "hit@5": 0.629, - "hit@10": 0.715, - "ndcg_linear@1": 0.397, - "ndcg_linear@5": 0.5194137751146535, - "ndcg_linear@10": 0.547354054247126, - "ndcg_exponential@1": 0.397, - "ndcg_exponential@5": 0.5194137751146535, - "ndcg_exponential@10": 0.547354054247126, - "precision@1": 0.397, - "precision@5": 0.1258, - "precision@10": 0.0715, - "recall@1": 0.397, - "recall@5": 0.629, - "recall@10": 0.715, - "f1@1": 0.397, - "f1@5": 0.20966666666666675, - "f1@10": 0.13, - "map@1": 0.397, - "map@5": 0.4830833333333333, - "map@10": 0.4946920634920635, - "mrr@1": 0.397, - "mrr@5": 0.4830833333333333, - "mrr@10": 0.4946920634920635, + "hit@1": 0.396, + "hit@5": 0.628, + "hit@10": 0.717, + "ndcg_linear@1": 0.396, + "ndcg_linear@5": 0.518482557057635, + "ndcg_linear@10": 0.5472085460484165, + "ndcg_exponential@1": 0.396, + "ndcg_exponential@5": 0.518482557057635, + "ndcg_exponential@10": 0.5472085460484165, + "precision@1": 0.396, + "precision@5": 0.1256, + "precision@10": 0.07169999999999999, + "recall@1": 0.396, + "recall@5": 0.628, + "recall@10": 0.717, + "f1@1": 0.396, + "f1@5": 0.20933333333333337, + "f1@10": 0.13036363636363638, + "map@1": 0.396, + "map@5": 0.48218333333333335, + "map@10": 0.49400873015873015, + "mrr@1": 0.396, + "mrr@5": 0.48218333333333335, + "mrr@10": 0.49400873015873015, "num_pred": 1000, "num_data": 1000 }, "SmthSmthV2": { - "hit@1": 0.306, - "hit@5": 0.539, - "hit@10": 0.621, - "ndcg_linear@1": 0.306, - "ndcg_linear@5": 0.4300547709738873, - "ndcg_linear@10": 0.45655616986633807, - "ndcg_exponential@1": 0.306, - "ndcg_exponential@5": 0.4300547709738873, - "ndcg_exponential@10": 0.45655616986633807, - "precision@1": 0.306, - "precision@5": 0.10779999999999998, - "precision@10": 0.062099999999999995, - "recall@1": 0.306, - "recall@5": 0.539, - "recall@10": 0.621, - "f1@1": 0.306, - "f1@5": 0.1796666666666667, - "f1@10": 0.11290909090909092, - "map@1": 0.306, - "map@5": 0.39380000000000004, - "map@10": 0.40473095238095236, - "mrr@1": 0.306, - "mrr@5": 0.39380000000000004, - "mrr@10": 0.40473095238095236, + "hit@1": 0.305, + "hit@5": 0.54, + "hit@10": 0.62, + "ndcg_linear@1": 0.305, + "ndcg_linear@5": 0.42983565221969305, + "ndcg_linear@10": 0.4557301176709403, + "ndcg_exponential@1": 0.305, + "ndcg_exponential@5": 0.42983565221969305, + "ndcg_exponential@10": 0.4557301176709403, + "precision@1": 0.305, + "precision@5": 0.108, + "precision@10": 0.06199999999999999, + "recall@1": 0.305, + "recall@5": 0.54, + "recall@10": 0.62, + "f1@1": 0.305, + "f1@5": 0.18000000000000005, + "f1@10": 0.11272727272727275, + "map@1": 0.305, + "map@5": 0.3932166666666667, + "map@10": 0.40392023809523814, + "mrr@1": 0.305, + "mrr@5": 0.3932166666666667, + "mrr@10": 0.40392023809523814, "num_pred": 1000, "num_data": 1000 }, "HMDB51": { - "hit@1": 0.479, - "hit@5": 0.715, - "hit@10": 0.805, - "ndcg_linear@1": 0.479, - "ndcg_linear@5": 0.6047040247872099, - "ndcg_linear@10": 0.6340238736498572, - "ndcg_exponential@1": 0.479, - "ndcg_exponential@5": 0.6047040247872099, - "ndcg_exponential@10": 0.6340238736498572, - "precision@1": 0.479, - "precision@5": 0.143, - "precision@10": 0.0805, - "recall@1": 0.479, - "recall@5": 0.715, - "recall@10": 0.805, - "f1@1": 0.479, - "f1@5": 0.23833333333333337, - "f1@10": 0.14636363636363636, - "map@1": 0.479, - "map@5": 0.5679833333333334, - "map@10": 0.5802178571428571, - "mrr@1": 0.479, - "mrr@5": 0.5679833333333334, - "mrr@10": 0.5802178571428571, + "hit@1": 0.48, + "hit@5": 0.716, + "hit@10": 0.804, + "ndcg_linear@1": 0.48, + "ndcg_linear@5": 0.6058099963486386, + "ndcg_linear@10": 0.6344712214492474, + "ndcg_exponential@1": 0.48, + "ndcg_exponential@5": 0.6058099963486386, + "ndcg_exponential@10": 0.6344712214492474, + "precision@1": 0.48, + "precision@5": 0.1432, + "precision@10": 0.0804, + "recall@1": 0.48, + "recall@5": 0.716, + "recall@10": 0.804, + "f1@1": 0.48, + "f1@5": 0.23866666666666675, + "f1@10": 0.1461818181818182, + "map@1": 0.48, + "map@5": 0.5691, + "map@10": 0.5810492063492063, + "mrr@1": 0.48, + "mrr@5": 0.5691, + "mrr@10": 0.5810492063492063, "num_pred": 1000, "num_data": 1000 }, "UCF101": { - "hit@1": 0.547, - "hit@5": 0.776, + "hit@1": 0.544, + "hit@5": 0.777, "hit@10": 0.848, - "ndcg_linear@1": 0.547, - "ndcg_linear@5": 0.671929263649622, - "ndcg_linear@10": 0.6952345321688619, - "ndcg_exponential@1": 0.547, - "ndcg_exponential@5": 0.671929263649622, - "ndcg_exponential@10": 0.6952345321688619, - "precision@1": 0.547, - "precision@5": 0.15519999999999998, + "ndcg_linear@1": 0.544, + "ndcg_linear@5": 0.6713653351622302, + "ndcg_linear@10": 0.6943731925037665, + "ndcg_exponential@1": 0.544, + "ndcg_exponential@5": 0.6713653351622302, + "ndcg_exponential@10": 0.6943731925037665, + "precision@1": 0.544, + "precision@5": 0.15539999999999998, "precision@10": 0.08480000000000001, - "recall@1": 0.547, - "recall@5": 0.776, + "recall@1": 0.544, + "recall@5": 0.777, "recall@10": 0.848, - "f1@1": 0.547, - "f1@5": 0.25866666666666677, + "f1@1": 0.544, + "f1@5": 0.259, "f1@10": 0.1541818181818182, - "map@1": 0.547, - "map@5": 0.637, - "map@10": 0.6466361111111112, - "mrr@1": 0.547, - "mrr@5": 0.637, - "mrr@10": 0.6466361111111112, + "map@1": 0.544, + "map@5": 0.6359000000000001, + "map@10": 0.6454234126984126, + "mrr@1": 0.544, + "mrr@5": 0.6359000000000001, + "mrr@10": 0.6454234126984126, "num_pred": 1000, "num_data": 1000 }, "Breakfast": { - "hit@1": 0.14318706697459585, - "hit@5": 0.5635103926096998, + "hit@1": 0.13856812933025403, + "hit@5": 0.5658198614318707, "hit@10": 1.0, - "ndcg_linear@1": 0.14318706697459585, - "ndcg_linear@5": 0.35021514635379236, - "ndcg_linear@10": 0.4904891071186407, - "ndcg_exponential@1": 0.14318706697459585, - "ndcg_exponential@5": 0.35021514635379236, - "ndcg_exponential@10": 0.4904891071186407, - "precision@1": 0.14318706697459585, - "precision@5": 0.11270207852193995, + "ndcg_linear@1": 0.13856812933025403, + "ndcg_linear@5": 0.35075022566110337, + "ndcg_linear@10": 0.49010744393766, + "ndcg_exponential@1": 0.13856812933025403, + "ndcg_exponential@5": 0.35075022566110337, + "ndcg_exponential@10": 0.49010744393766, + "precision@1": 0.13856812933025403, + "precision@5": 0.11316397228637413, "precision@10": 0.1, - "recall@1": 0.14318706697459585, - "recall@5": 0.5635103926096998, + "recall@1": 0.13856812933025403, + "recall@5": 0.5658198614318707, "recall@10": 1.0, - "f1@1": 0.14318706697459585, - "f1@5": 0.18783679753656665, + "f1@1": 0.13856812933025403, + "f1@5": 0.18860662047729027, "f1@10": 0.1818181818181818, - "map@1": 0.14318706697459585, - "map@5": 0.2807544264819091, - "map@10": 0.33812731405110164, - "mrr@1": 0.14318706697459585, - "mrr@5": 0.2807544264819091, - "mrr@10": 0.33812731405110164, + "map@1": 0.13856812933025403, + "map@5": 0.2805619707467283, + "map@10": 0.3374537189779684, + "mrr@1": 0.13856812933025403, + "mrr@5": 0.2805619707467283, + "mrr@10": 0.3374537189779684, "num_pred": 433, "num_data": 433 }, "MVBench": { - "hit@1": 0.46625, + "hit@1": 0.463, "hit@5": 1.0, "hit@10": 1.0, - "ndcg_linear@1": 0.46625, - "ndcg_linear@5": 0.7596556549662636, - "ndcg_linear@10": 0.7596556549662636, - "ndcg_exponential@1": 0.46625, - "ndcg_exponential@5": 0.7596556549662636, - "ndcg_exponential@10": 0.7596556549662636, - "precision@1": 0.46625, + "ndcg_linear@1": 0.463, + "ndcg_linear@5": 0.7584965935513933, + "ndcg_linear@10": 0.7584965935513933, + "ndcg_exponential@1": 0.463, + "ndcg_exponential@5": 0.7584965935513933, + "ndcg_exponential@10": 0.7584965935513933, + "precision@1": 0.463, "precision@5": 0.20000000000000004, "precision@10": 0.10000000000000002, - "recall@1": 0.46625, + "recall@1": 0.463, "recall@5": 1.0, "recall@10": 1.0, - "f1@1": 0.46625, + "f1@1": 0.463, "f1@5": 0.3333333333333333, "f1@10": 0.18181818181818182, - "map@1": 0.46625, - "map@5": 0.678425, - "map@10": 0.678425, - "mrr@1": 0.46625, - "mrr@5": 0.678425, - "mrr@10": 0.678425, + "map@1": 0.463, + "map@5": 0.6768624999999999, + "map@10": 0.6768624999999999, + "mrr@1": 0.463, + "mrr@5": 0.6768624999999999, + "mrr@10": 0.6768624999999999, "num_pred": 4000, "num_data": 4000 }, @@ -1191,11 +1191,11 @@ "hit@5": 1.0, "hit@10": 1.0, "ndcg_linear@1": 0.39185185185185184, - "ndcg_linear@5": 0.7181257548926895, - "ndcg_linear@10": 0.7181257548926895, + "ndcg_linear@5": 0.7180316280864446, + "ndcg_linear@10": 0.7180316280864446, "ndcg_exponential@1": 0.39185185185185184, - "ndcg_exponential@5": 0.7181257548926895, - "ndcg_exponential@10": 0.7181257548926895, + "ndcg_exponential@5": 0.7180316280864446, + "ndcg_exponential@10": 0.7180316280864446, "precision@1": 0.39185185185185184, "precision@5": 0.19999999999999996, "precision@10": 0.09999999999999998, @@ -1206,489 +1206,517 @@ "f1@5": 0.3333333333333334, "f1@10": 0.18181818181818188, "map@1": 0.39185185185185184, - "map@5": 0.6235802469135802, - "map@10": 0.6235802469135802, + "map@5": 0.6234567901234568, + "map@10": 0.6234567901234568, "mrr@1": 0.39185185185185184, - "mrr@5": 0.6235802469135802, - "mrr@10": 0.6235802469135802, + "mrr@5": 0.6234567901234568, + "mrr@10": 0.6234567901234568, "num_pred": 2700, "num_data": 2700 }, "NExTQA": { - "hit@1": 0.5354974311069594, + "hit@1": 0.5351471275105091, "hit@5": 1.0, "hit@10": 1.0, - "ndcg_linear@1": 0.5354974311069594, - "ndcg_linear@5": 0.77906035779538, - "ndcg_linear@10": 0.77906035779538, - "ndcg_exponential@1": 0.5354974311069594, - "ndcg_exponential@5": 0.77906035779538, - "ndcg_exponential@10": 0.77906035779538, - "precision@1": 0.5354974311069594, + "ndcg_linear@1": 0.5351471275105091, + "ndcg_linear@5": 0.7791659933445173, + "ndcg_linear@10": 0.7791659933445173, + "ndcg_exponential@1": 0.5351471275105091, + "ndcg_exponential@5": 0.7791659933445173, + "ndcg_exponential@10": 0.7791659933445173, + "precision@1": 0.5351471275105091, "precision@5": 0.20000000000000004, "precision@10": 0.10000000000000002, - "recall@1": 0.5354974311069594, + "recall@1": 0.5351471275105091, "recall@5": 1.0, "recall@10": 1.0, - "f1@1": 0.5354974311069594, + "f1@1": 0.5351471275105091, "f1@5": 0.3333333333333333, "f1@10": 0.1818181818181818, - "map@1": 0.5354974311069594, - "map@5": 0.7058578545850848, - "map@10": 0.7058578545850848, - "mrr@1": 0.5354974311069594, - "mrr@5": 0.7056632414759458, - "mrr@10": 0.7056632414759458, + "map@1": 0.5351471275105091, + "map@5": 0.7059687840572941, + "map@10": 0.7059687840572941, + "mrr@1": 0.5351471275105091, + "mrr@5": 0.7057741709481551, + "mrr@10": 0.7057741709481551, "num_pred": 8564, "num_data": 8564 }, "EgoSchema": { - "hit@1": 0.468, + "hit@1": 0.464, "hit@5": 1.0, "hit@10": 1.0, - "ndcg_linear@1": 0.468, - "ndcg_linear@5": 0.7452026017383512, - "ndcg_linear@10": 0.7452026017383512, - "ndcg_exponential@1": 0.468, - "ndcg_exponential@5": 0.7452026017383512, - "ndcg_exponential@10": 0.7452026017383512, - "precision@1": 0.468, + "ndcg_linear@1": 0.464, + "ndcg_linear@5": 0.7446648974205923, + "ndcg_linear@10": 0.7446648974205923, + "ndcg_exponential@1": 0.464, + "ndcg_exponential@5": 0.7446648974205923, + "ndcg_exponential@10": 0.7446648974205923, + "precision@1": 0.464, "precision@5": 0.2, "precision@10": 0.1, - "recall@1": 0.468, + "recall@1": 0.464, "recall@5": 1.0, "recall@10": 1.0, - "f1@1": 0.468, + "f1@1": 0.464, "f1@5": 0.3333333333333333, "f1@10": 0.18181818181818182, - "map@1": 0.468, - "map@5": 0.6607999999999999, - "map@10": 0.6607999999999999, - "mrr@1": 0.468, - "mrr@5": 0.6607999999999999, - "mrr@10": 0.6607999999999999, + "map@1": 0.464, + "map@5": 0.66, + "map@10": 0.66, + "mrr@1": 0.464, + "mrr@5": 0.66, + "mrr@10": 0.66, "num_pred": 500, "num_data": 500 }, "ActivityNetQA": { - "hit@1": 0.656, + "hit@1": 0.66, "hit@5": 1.0, "hit@10": 1.0, - "ndcg_linear@1": 0.656, - "ndcg_linear@5": 0.8730398352285815, - "ndcg_linear@10": 0.8730398352285815, - "ndcg_exponential@1": 0.656, - "ndcg_exponential@5": 0.8730398352285815, - "ndcg_exponential@10": 0.8730398352285815, - "precision@1": 0.656, + "ndcg_linear@1": 0.66, + "ndcg_linear@5": 0.8745161162142956, + "ndcg_linear@10": 0.8745161162142956, + "ndcg_exponential@1": 0.66, + "ndcg_exponential@5": 0.8745161162142956, + "ndcg_exponential@10": 0.8745161162142956, + "precision@1": 0.66, "precision@5": 0.20000000000000004, "precision@10": 0.10000000000000002, - "recall@1": 0.656, + "recall@1": 0.66, "recall@5": 1.0, "recall@10": 1.0, - "f1@1": 0.656, + "f1@1": 0.66, "f1@5": 0.3333333333333333, "f1@10": 0.18181818181818182, - "map@1": 0.656, - "map@5": 0.828, - "map@10": 0.828, - "mrr@1": 0.656, - "mrr@5": 0.828, - "mrr@10": 0.828, + "map@1": 0.66, + "map@5": 0.83, + "map@10": 0.83, + "mrr@1": 0.66, + "mrr@5": 0.83, + "mrr@10": 0.83, "num_pred": 1000, "num_data": 1000 }, "DiDeMo": { - "hit@1": 0.2639442231075697, - "hit@5": 0.4800796812749004, - "hit@10": 0.5876494023904383, - "ndcg_linear@1": 0.2639442231075697, - "ndcg_linear@5": 0.378795485959218, - "ndcg_linear@10": 0.41351240118071275, - "ndcg_exponential@1": 0.2639442231075697, - "ndcg_exponential@5": 0.378795485959218, - "ndcg_exponential@10": 0.41351240118071275, - "precision@1": 0.2639442231075697, - "precision@5": 0.09601593625498009, - "precision@10": 0.05876494023904382, - "recall@1": 0.2639442231075697, - "recall@5": 0.4800796812749004, - "recall@10": 0.5876494023904383, - "f1@1": 0.2639442231075697, - "f1@5": 0.16002656042496685, - "f1@10": 0.10684534588917059, - "map@1": 0.2639442231075697, - "map@5": 0.34510292164674633, - "map@10": 0.35938705495478407, - "mrr@1": 0.2639442231075697, - "mrr@5": 0.34510292164674633, - "mrr@10": 0.35938705495478407, + "hit@1": 0.26294820717131473, + "hit@5": 0.47908366533864544, + "hit@10": 0.5846613545816733, + "ndcg_linear@1": 0.26294820717131473, + "ndcg_linear@5": 0.37790448004535715, + "ndcg_linear@10": 0.4120411890002066, + "ndcg_exponential@1": 0.26294820717131473, + "ndcg_exponential@5": 0.37790448004535715, + "ndcg_exponential@10": 0.4120411890002066, + "precision@1": 0.26294820717131473, + "precision@5": 0.09581673306772907, + "precision@10": 0.058466135458167326, + "recall@1": 0.26294820717131473, + "recall@5": 0.47908366533864544, + "recall@10": 0.5846613545816733, + "f1@1": 0.26294820717131473, + "f1@5": 0.15969455511288183, + "f1@10": 0.10630206446939516, + "map@1": 0.26294820717131473, + "map@5": 0.34423970783532537, + "map@10": 0.3583194997786631, + "mrr@1": 0.26294820717131473, + "mrr@5": 0.34423970783532537, + "mrr@10": 0.3583194997786631, "num_pred": 1004, "num_data": 1004 }, "MSR-VTT": { "hit@1": 0.318, - "hit@5": 0.549, - "hit@10": 0.653, + "hit@5": 0.552, + "hit@10": 0.657, "ndcg_linear@1": 0.318, - "ndcg_linear@5": 0.43999853281714363, - "ndcg_linear@10": 0.4729767910864836, + "ndcg_linear@5": 0.44178643351431945, + "ndcg_linear@10": 0.47499258351006446, "ndcg_exponential@1": 0.318, - "ndcg_exponential@5": 0.43999853281714363, - "ndcg_exponential@10": 0.4729767910864836, + "ndcg_exponential@5": 0.44178643351431945, + "ndcg_exponential@10": 0.47499258351006446, "precision@1": 0.318, - "precision@5": 0.10979999999999998, - "precision@10": 0.0653, + "precision@5": 0.11040000000000001, + "precision@10": 0.0657, "recall@1": 0.318, - "recall@5": 0.549, - "recall@10": 0.653, + "recall@5": 0.552, + "recall@10": 0.657, "f1@1": 0.318, - "f1@5": 0.18300000000000005, - "f1@10": 0.11872727272727275, + "f1@5": 0.18400000000000005, + "f1@10": 0.11945454545454547, "map@1": 0.318, - "map@5": 0.40383333333333327, - "map@10": 0.4170674603174603, + "map@5": 0.40518333333333334, + "map@10": 0.4184579365079365, "mrr@1": 0.318, - "mrr@5": 0.40383333333333327, - "mrr@10": 0.4170674603174603, + "mrr@5": 0.40518333333333334, + "mrr@10": 0.4184579365079365, "num_pred": 1000, "num_data": 1000 }, "MSVD": { - "hit@1": 0.49701492537313435, - "hit@5": 0.7388059701492538, - "hit@10": 0.8298507462686567, - "ndcg_linear@1": 0.49701492537313435, - "ndcg_linear@5": 0.6297230187611768, - "ndcg_linear@10": 0.6594388806903934, - "ndcg_exponential@1": 0.49701492537313435, - "ndcg_exponential@5": 0.6297230187611768, - "ndcg_exponential@10": 0.6594388806903934, - "precision@1": 0.49701492537313435, - "precision@5": 0.14776119402985072, - "precision@10": 0.08298507462686566, - "recall@1": 0.49701492537313435, - "recall@5": 0.7388059701492538, - "recall@10": 0.8298507462686567, - "f1@1": 0.49701492537313435, - "f1@5": 0.24626865671641796, - "f1@10": 0.15088195386702855, - "map@1": 0.49701492537313435, - "map@5": 0.5930348258706468, - "map@10": 0.6054554607912818, - "mrr@1": 0.49701492537313435, - "mrr@5": 0.5930348258706468, - "mrr@10": 0.6054554607912818, + "hit@1": 0.4955223880597015, + "hit@5": 0.744776119402985, + "hit@10": 0.8328358208955224, + "ndcg_linear@1": 0.4955223880597015, + "ndcg_linear@5": 0.6310405160075232, + "ndcg_linear@10": 0.6594479819024751, + "ndcg_exponential@1": 0.4955223880597015, + "ndcg_exponential@5": 0.6310405160075232, + "ndcg_exponential@10": 0.6594479819024751, + "precision@1": 0.4955223880597015, + "precision@5": 0.14895522388059698, + "precision@10": 0.08328358208955224, + "recall@1": 0.4955223880597015, + "recall@5": 0.744776119402985, + "recall@10": 0.8328358208955224, + "f1@1": 0.4955223880597015, + "f1@5": 0.24825870646766174, + "f1@10": 0.15142469470827682, + "map@1": 0.4955223880597015, + "map@5": 0.5929601990049751, + "map@10": 0.6046452262497038, + "mrr@1": 0.4955223880597015, + "mrr@5": 0.5929601990049751, + "mrr@10": 0.6046452262497038, "num_pred": 670, "num_data": 670 }, "VATEX": { - "hit@1": 0.24877177311299686, - "hit@5": 0.4723090665475659, - "hit@10": 0.5792764627065654, - "ndcg_linear@1": 0.24877177311299686, - "ndcg_linear@5": 0.36560074904604023, - "ndcg_linear@10": 0.40032465920098576, - "ndcg_exponential@1": 0.24877177311299686, - "ndcg_exponential@5": 0.36560074904604023, - "ndcg_exponential@10": 0.40032465920098576, - "precision@1": 0.24877177311299686, - "precision@5": 0.09446181330951318, - "precision@10": 0.05792764627065654, - "recall@1": 0.24877177311299686, - "recall@5": 0.4723090665475659, - "recall@10": 0.5792764627065654, - "f1@1": 0.24877177311299686, - "f1@5": 0.15743635551585533, - "f1@10": 0.10532299321937554, - "map@1": 0.24877177311299686, - "map@5": 0.33029626321274375, - "map@10": 0.3447070156036212, - "mrr@1": 0.24877177311299686, - "mrr@5": 0.33029626321274375, - "mrr@10": 0.3447070156036212, + "hit@1": 0.2474318892362662, + "hit@5": 0.47029924073246987, + "hit@10": 0.5801697186243859, + "ndcg_linear@1": 0.2474318892362662, + "ndcg_linear@5": 0.3644765548491043, + "ndcg_linear@10": 0.4001848721884503, + "ndcg_exponential@1": 0.2474318892362662, + "ndcg_exponential@5": 0.3644765548491043, + "ndcg_exponential@10": 0.4001848721884503, + "precision@1": 0.2474318892362662, + "precision@5": 0.09405984814649397, + "precision@10": 0.05801697186243858, + "recall@1": 0.2474318892362662, + "recall@5": 0.47029924073246987, + "recall@10": 0.5801697186243859, + "f1@1": 0.2474318892362662, + "f1@5": 0.15676641357748994, + "f1@10": 0.10548540338625198, + "map@1": 0.2474318892362662, + "map@5": 0.32939556349560817, + "map@10": 0.34423743593015593, + "mrr@1": 0.2474318892362662, + "mrr@5": 0.32939556349560817, + "mrr@10": 0.34423743593015593, "num_pred": 4478, "num_data": 4478 }, "YouCook2": { - "hit@1": 0.09090909090909091, - "hit@5": 0.22805913809374018, - "hit@10": 0.30229631959735764, - "ndcg_linear@1": 0.09090909090909091, - "ndcg_linear@5": 0.16160064951123587, - "ndcg_linear@10": 0.18557172152328222, - "ndcg_exponential@1": 0.09090909090909091, - "ndcg_exponential@5": 0.16160064951123587, - "ndcg_exponential@10": 0.18557172152328222, - "precision@1": 0.09090909090909091, - "precision@5": 0.04561182761874803, - "precision@10": 0.030229631959735763, - "recall@1": 0.09090909090909091, - "recall@5": 0.22805913809374018, - "recall@10": 0.30229631959735764, - "f1@1": 0.09090909090909091, - "f1@5": 0.0760197126979134, - "f1@10": 0.05496296719951958, - "map@1": 0.09090909090909091, - "map@5": 0.13971898919995804, - "map@10": 0.149584825017351, - "mrr@1": 0.09090909090909091, - "mrr@5": 0.13971898919995804, - "mrr@10": 0.149584825017351, + "hit@1": 0.08933626926706512, + "hit@5": 0.22459893048128343, + "hit@10": 0.3048128342245989, + "ndcg_linear@1": 0.08933626926706512, + "ndcg_linear@5": 0.16013540075641206, + "ndcg_linear@10": 0.186193332475231, + "ndcg_exponential@1": 0.08933626926706512, + "ndcg_exponential@5": 0.16013540075641206, + "ndcg_exponential@10": 0.186193332475231, + "precision@1": 0.08933626926706512, + "precision@5": 0.044919786096256686, + "precision@10": 0.030481283422459895, + "recall@1": 0.08933626926706512, + "recall@5": 0.22459893048128343, + "recall@10": 0.3048128342245989, + "f1@1": 0.08933626926706512, + "f1@5": 0.07486631016042782, + "f1@10": 0.05542051531356344, + "map@1": 0.08933626926706512, + "map@5": 0.13880150990877635, + "map@10": 0.149619776609396, + "mrr@1": 0.08933626926706512, + "mrr@5": 0.13880150990877635, + "mrr@10": 0.149619776609396, "num_pred": 3179, "num_data": 3179 }, "QVHighlight": { - "hit@1": 0.5946445060018467, - "hit@5": 0.8901200369344414, + "hit@1": 0.5937211449676824, + "hit@5": 0.8855032317636196, "hit@10": 1.0, - "ndcg_linear@1": 0.5946445060018467, - "ndcg_linear@5": 0.7511101894816002, - "ndcg_linear@10": 0.7866485150531158, - "ndcg_exponential@1": 0.5946445060018467, - "ndcg_exponential@5": 0.7511101894816002, - "ndcg_exponential@10": 0.7866485150531158, - "precision@1": 0.5946445060018467, - "precision@5": 0.17802400738688828, + "ndcg_linear@1": 0.5937211449676824, + "ndcg_linear@5": 0.7487037926751408, + "ndcg_linear@10": 0.7859182896668794, + "ndcg_exponential@1": 0.5937211449676824, + "ndcg_exponential@5": 0.7487037926751408, + "ndcg_exponential@10": 0.7859182896668794, + "precision@1": 0.5937211449676824, + "precision@5": 0.1771006463527239, "precision@10": 0.09999999999999998, - "recall@1": 0.5946445060018467, - "recall@5": 0.8901200369344414, + "recall@1": 0.5937211449676824, + "recall@5": 0.8855032317636196, "recall@10": 1.0, - "f1@1": 0.5946445060018467, - "f1@5": 0.2967066789781472, + "f1@1": 0.5937211449676824, + "f1@5": 0.2951677439212066, "f1@10": 0.18181818181818185, - "map@1": 0.5946445060018467, - "map@5": 0.7049092028316406, - "map@10": 0.7195789180553723, - "mrr@1": 0.5946445060018467, - "mrr@5": 0.7049092028316406, - "mrr@10": 0.7195789180553723, + "map@1": 0.5937211449676824, + "map@5": 0.7031855955678671, + "map@10": 0.7186533585425552, + "mrr@1": 0.5937211449676824, + "mrr@5": 0.7031855955678671, + "mrr@10": 0.7186533585425552, "num_pred": 1083, "num_data": 1083 }, "Charades-STA": { - "hit@1": 0.14030261348005502, - "hit@5": 0.4814305364511692, + "hit@1": 0.13892709766162312, + "hit@5": 0.48418156808803303, "hit@10": 1.0, - "ndcg_linear@1": 0.14030261348005502, - "ndcg_linear@5": 0.30810403477419174, - "ndcg_linear@10": 0.47145234528957136, - "ndcg_exponential@1": 0.14030261348005502, - "ndcg_exponential@5": 0.30810403477419174, - "ndcg_exponential@10": 0.47145234528957136, - "precision@1": 0.14030261348005502, - "precision@5": 0.09628610729023383, + "ndcg_linear@1": 0.13892709766162312, + "ndcg_linear@5": 0.30890937384695283, + "ndcg_linear@10": 0.471352990611122, + "ndcg_exponential@1": 0.13892709766162312, + "ndcg_exponential@5": 0.30890937384695283, + "ndcg_exponential@10": 0.471352990611122, + "precision@1": 0.13892709766162312, + "precision@5": 0.09683631361760661, "precision@10": 0.09999999999999998, - "recall@1": 0.14030261348005502, - "recall@5": 0.4814305364511692, + "recall@1": 0.13892709766162312, + "recall@5": 0.48418156808803303, "recall@10": 1.0, - "f1@1": 0.14030261348005502, - "f1@5": 0.1604768454837231, + "f1@1": 0.13892709766162312, + "f1@5": 0.16139385602934436, "f1@10": 0.18181818181818185, - "map@1": 0.14030261348005502, - "map@5": 0.25176524530032096, - "map@10": 0.316674854260824, - "mrr@1": 0.14030261348005502, - "mrr@5": 0.25176524530032096, - "mrr@10": 0.316674854260824, + "map@1": 0.13892709766162312, + "map@5": 0.25199449793672624, + "map@10": 0.31652966092443396, + "mrr@1": 0.13892709766162312, + "mrr@5": 0.25199449793672624, + "mrr@10": 0.31652966092443396, "num_pred": 727, "num_data": 727 }, "MomentSeeker": { - "hit@1": 0.3738888888888889, - "hit@5": 0.8438888888888889, + "hit@1": 0.39325842696629215, + "hit@5": 0.8489388264669163, "hit@10": 1.0, - "ndcg_linear@1": 0.3738888888888889, - "ndcg_linear@5": 0.6176553256419134, - "ndcg_linear@10": 0.6753521684415025, - "ndcg_exponential@1": 0.3738888888888889, - "ndcg_exponential@5": 0.6176553256419134, - "ndcg_exponential@10": 0.6753521684415025, - "precision@1": 0.3738888888888889, - "precision@5": 0.1856666666666667, + "ndcg_linear@1": 0.39325842696629215, + "ndcg_linear@5": 0.629857797791427, + "ndcg_linear@10": 0.6864603012072749, + "ndcg_exponential@1": 0.39325842696629215, + "ndcg_exponential@5": 0.629857797791427, + "ndcg_exponential@10": 0.6864603012072749, + "precision@1": 0.39325842696629215, + "precision@5": 0.18863920099875156, + "precision@10": 0.11441947565543072, + "recall@1": 0.3661828963795256, + "recall@5": 0.8310445276737412, + "recall@10": 0.9983094049105286, + "f1@1": 0.3741146978974694, + "f1@5": 0.30179845816549944, + "f1@10": 0.20240542852902405, + "map@1": 0.39325842696629215, + "map@5": 0.5571258149535303, + "map@10": 0.5831409894378059, + "mrr@1": 0.39325842696629215, + "mrr@5": 0.5676550145651269, + "mrr@10": 0.588573063432614, + "num_pred": 1602, + "num_data": 1602 + }, + "MomentSeeker_1k8": { + "hit@1": 0.37277777777777776, + "hit@5": 0.8444444444444444, + "hit@10": 1.0, + "ndcg_linear@1": 0.37277777777777776, + "ndcg_linear@5": 0.6174041300789458, + "ndcg_linear@10": 0.6748147689610049, + "ndcg_exponential@1": 0.37277777777777776, + "ndcg_exponential@5": 0.6174041300789458, + "ndcg_exponential@10": 0.6748147689610049, + "precision@1": 0.37277777777777776, + "precision@5": 0.18577777777777776, "precision@10": 0.11288888888888891, - "recall@1": 0.35006944444444443, - "recall@5": 0.8276851851851853, + "recall@1": 0.34868055555555555, + "recall@5": 0.8285185185185187, "recall@10": 0.9984953703703704, - "f1@1": 0.35703615520282184, - "f1@5": 0.29815089231755904, + "f1@1": 0.3557398589065256, + "f1@5": 0.2983625325291992, "f1@10": 0.2002250064750065, - "map@1": 0.3738888888888889, - "map@5": 0.5426706790123457, - "map@10": 0.5690101410934745, - "mrr@1": 0.3738888888888889, - "mrr@5": 0.5520092592592593, - "mrr@10": 0.5737444885361552, + "map@1": 0.37277777777777776, + "map@5": 0.5421012345679012, + "map@10": 0.5682781084656084, + "mrr@1": 0.37277777777777776, + "mrr@5": 0.5514259259259259, + "mrr@10": 0.5730663580246913, "num_pred": 1800, "num_data": 1800 } }, "visdoc": { "ViDoRe_arxivqa": { - "hit@1": 0.818, - "hit@5": 0.912, + "hit@1": 0.822, + "hit@5": 0.922, "hit@10": 0.944, - "ndcg_linear@1": 0.818, - "ndcg_linear@5": 0.8685712615408394, - "ndcg_linear@10": 0.8789118315122615, - "ndcg_exponential@1": 0.818, - "ndcg_exponential@5": 0.8685712615408394, - "ndcg_exponential@10": 0.8789118315122615, - "precision@1": 0.818, - "precision@5": 0.1824, + "ndcg_linear@1": 0.822, + "ndcg_linear@5": 0.8752573196536307, + "ndcg_linear@10": 0.8824475200920486, + "ndcg_exponential@1": 0.822, + "ndcg_exponential@5": 0.8752573196536307, + "ndcg_exponential@10": 0.8824475200920486, + "precision@1": 0.822, + "precision@5": 0.18440000000000004, "precision@10": 0.09440000000000001, - "recall@1": 0.818, - "recall@5": 0.912, + "recall@1": 0.822, + "recall@5": 0.922, "recall@10": 0.944, - "f1@1": 0.818, - "f1@5": 0.304, + "f1@1": 0.822, + "f1@5": 0.30733333333333335, "f1@10": 0.17163636363636361, - "map@1": 0.818, - "map@5": 0.8540666666666666, - "map@10": 0.8583349206349207, - "mrr@1": 0.818, - "mrr@5": 0.8540666666666666, - "mrr@10": 0.8583349206349207, + "map@1": 0.822, + "map@5": 0.8597, + "map@10": 0.8627055555555555, + "mrr@1": 0.822, + "mrr@5": 0.8597, + "mrr@10": 0.8627055555555555, "num_pred": 500, "num_data": 500 }, "ViDoRe_docvqa": { - "hit@1": 0.4878048780487805, - "hit@5": 0.6541019955654102, - "hit@10": 0.7250554323725056, - "ndcg_linear@1": 0.4878048780487805, - "ndcg_linear@5": 0.5745938935482915, - "ndcg_linear@10": 0.597561200968465, - "ndcg_exponential@1": 0.4878048780487805, - "ndcg_exponential@5": 0.5745938935482915, - "ndcg_exponential@10": 0.597561200968465, - "precision@1": 0.4878048780487805, - "precision@5": 0.1334811529933481, - "precision@10": 0.0745011086474501, - "recall@1": 0.4866962305986696, - "recall@5": 0.6472283813747228, - "recall@10": 0.7167775314116778, - "f1@1": 0.48706577974870663, - "f1@5": 0.2198328851126556, - "f1@10": 0.13419485171148143, - "map@1": 0.4878048780487805, - "map@5": 0.5490422517861542, - "map@10": 0.5587257593355155, - "mrr@1": 0.4878048780487805, - "mrr@5": 0.5516629711751663, - "mrr@10": 0.5612703691971984, + "hit@1": 0.4789356984478936, + "hit@5": 0.6452328159645233, + "hit@10": 0.7161862527716186, + "ndcg_linear@1": 0.4789356984478936, + "ndcg_linear@5": 0.5659230892387825, + "ndcg_linear@10": 0.58847955781244, + "ndcg_exponential@1": 0.4789356984478936, + "ndcg_exponential@5": 0.5659230892387825, + "ndcg_exponential@10": 0.58847955781244, + "precision@1": 0.4789356984478936, + "precision@5": 0.13215077605321507, + "precision@10": 0.07339246119733923, + "recall@1": 0.47671840354767187, + "recall@5": 0.6386178861788617, + "recall@10": 0.7079822616407981, + "f1@1": 0.47745750184774577, + "f1@5": 0.2175870200260445, + "f1@10": 0.13222926371485133, + "map@1": 0.4789356984478936, + "map@5": 0.5400418822370041, + "map@10": 0.5493721183965086, + "mrr@1": 0.4789356984478936, + "mrr@5": 0.5432372505543237, + "mrr@10": 0.5527029880688418, "num_pred": 451, "num_data": 451 }, "ViDoRe_infovqa": { - "hit@1": 0.8765182186234818, - "hit@5": 0.951417004048583, - "hit@10": 0.9696356275303644, - "ndcg_linear@1": 0.8765182186234818, - "ndcg_linear@5": 0.9159739102039204, - "ndcg_linear@10": 0.9221606391623131, - "ndcg_exponential@1": 0.8765182186234818, - "ndcg_exponential@5": 0.9159739102039204, - "ndcg_exponential@10": 0.9221606391623131, - "precision@1": 0.8765182186234818, - "precision@5": 0.1902834008097166, - "precision@10": 0.09716599190283401, - "recall@1": 0.8748987854251012, - "recall@5": 0.9487854251012146, - "recall@10": 0.9680161943319838, - "f1@1": 0.8751686909581647, - "f1@5": 0.3167727009832273, - "f1@10": 0.17650595019016072, - "map@1": 0.8765182186234818, - "map@5": 0.9044197031039136, - "map@10": 0.9070215603110341, - "mrr@1": 0.8765182186234818, - "mrr@5": 0.9062415654520918, - "mrr@10": 0.9085542381595014, + "hit@1": 0.8927125506072875, + "hit@5": 0.9493927125506073, + "hit@10": 0.9676113360323887, + "ndcg_linear@1": 0.8927125506072875, + "ndcg_linear@5": 0.9217356389736178, + "ndcg_linear@10": 0.928195077659096, + "ndcg_exponential@1": 0.8927125506072875, + "ndcg_exponential@5": 0.9217356389736178, + "ndcg_exponential@10": 0.928195077659096, + "precision@1": 0.8927125506072875, + "precision@5": 0.18987854251012148, + "precision@10": 0.09696356275303644, + "recall@1": 0.8910931174089068, + "recall@5": 0.9477732793522267, + "recall@10": 0.965991902834008, + "f1@1": 0.8913630229419703, + "f1@5": 0.3161943319838057, + "f1@10": 0.17613789719052877, + "map@1": 0.8927125506072875, + "map@5": 0.9126518218623483, + "map@10": 0.9156400616926932, + "mrr@1": 0.8927125506072875, + "mrr@5": 0.9142712550607287, + "mrr@10": 0.9171390013495276, "num_pred": 494, "num_data": 494 }, "ViDoRe_tabfquad": { - "hit@1": 0.9071428571428571, - "hit@5": 0.9785714285714285, - "hit@10": 0.9857142857142858, - "ndcg_linear@1": 0.9071428571428571, - "ndcg_linear@5": 0.9464498663499118, - "ndcg_linear@10": 0.9489125110657739, - "ndcg_exponential@1": 0.9071428571428571, - "ndcg_exponential@5": 0.9464498663499118, - "ndcg_exponential@10": 0.9489125110657739, - "precision@1": 0.9071428571428571, - "precision@5": 0.1957142857142857, - "precision@10": 0.09857142857142856, - "recall@1": 0.9071428571428571, - "recall@5": 0.9785714285714285, - "recall@10": 0.9857142857142858, - "f1@1": 0.9071428571428571, - "f1@5": 0.32619047619047625, - "f1@10": 0.17922077922077925, - "map@1": 0.9071428571428571, - "map@5": 0.9356547619047619, - "map@10": 0.9367602040816324, - "mrr@1": 0.9071428571428571, - "mrr@5": 0.9356547619047619, - "mrr@10": 0.9367602040816324, + "hit@1": 0.8821428571428571, + "hit@5": 0.9607142857142857, + "hit@10": 0.9821428571428571, + "ndcg_linear@1": 0.8821428571428571, + "ndcg_linear@5": 0.9270757945477919, + "ndcg_linear@10": 0.9339630572354429, + "ndcg_exponential@1": 0.8821428571428571, + "ndcg_exponential@5": 0.9270757945477919, + "ndcg_exponential@10": 0.9339630572354429, + "precision@1": 0.8821428571428571, + "precision@5": 0.19214285714285714, + "precision@10": 0.0982142857142857, + "recall@1": 0.8821428571428571, + "recall@5": 0.9607142857142857, + "recall@10": 0.9821428571428571, + "f1@1": 0.8821428571428571, + "f1@5": 0.3202380952380953, + "f1@10": 0.1785714285714286, + "map@1": 0.8821428571428571, + "map@5": 0.9155952380952381, + "map@10": 0.9184112811791384, + "mrr@1": 0.8821428571428571, + "mrr@5": 0.9155952380952381, + "mrr@10": 0.9184112811791384, "num_pred": 280, "num_data": 280 }, "ViDoRe_tatdqa": { - "hit@1": 0.6221142162818954, - "hit@5": 0.8444714459295262, - "hit@10": 0.9106925880923451, - "ndcg_linear@1": 0.6221142162818954, - "ndcg_linear@5": 0.740984681776679, - "ndcg_linear@10": 0.7629989160213186, - "ndcg_exponential@1": 0.6221142162818954, - "ndcg_exponential@5": 0.740984681776679, - "ndcg_exponential@10": 0.7629989160213186, - "precision@1": 0.6221142162818954, - "precision@5": 0.16925880923450792, - "precision@10": 0.09143377885783718, - "recall@1": 0.6215066828675577, - "recall@5": 0.8435601458080194, - "recall@10": 0.9106925880923451, - "f1@1": 0.6217091940056704, - "f1@5": 0.28183764392755895, - "f1@10": 0.16613277366618803, - "map@1": 0.6221142162818954, - "map@5": 0.7066322397731876, - "map@10": 0.7159557079210785, - "mrr@1": 0.6221142162818954, - "mrr@5": 0.706946132037262, - "mrr@10": 0.71601405041563, + "hit@1": 0.6500607533414338, + "hit@5": 0.8669501822600243, + "hit@10": 0.9185905224787363, + "ndcg_linear@1": 0.6500607533414338, + "ndcg_linear@5": 0.7657576781612838, + "ndcg_linear@10": 0.7830528566422958, + "ndcg_exponential@1": 0.6500607533414338, + "ndcg_exponential@5": 0.7657576781612838, + "ndcg_exponential@10": 0.7830528566422958, + "precision@1": 0.6500607533414338, + "precision@5": 0.1737545565006075, + "precision@10": 0.09216281895504252, + "recall@1": 0.6491494532199271, + "recall@5": 0.8654921020656136, + "recall@10": 0.9177399756986634, + "f1@1": 0.649453219927096, + "f1@5": 0.2892090493548575, + "f1@10": 0.1674177988880298, + "map@1": 0.6500607533414338, + "map@5": 0.7322600243013366, + "map@10": 0.7396808520897221, + "mrr@1": 0.6500607533414338, + "mrr@5": 0.7328270554880519, + "mrr@10": 0.7401239657466876, "num_pred": 1646, "num_data": 1646 }, "ViDoRe_shiftproject": { - "hit@1": 0.95, - "hit@5": 0.98, + "hit@1": 0.92, + "hit@5": 0.99, "hit@10": 0.99, - "ndcg_linear@1": 0.95, - "ndcg_linear@5": 0.9676185950714291, - "ndcg_linear@10": 0.9711806669425094, - "ndcg_exponential@1": 0.95, - "ndcg_exponential@5": 0.9676185950714291, - "ndcg_exponential@10": 0.9711806669425094, - "precision@1": 0.95, - "precision@5": 0.19599999999999995, + "ndcg_linear@1": 0.92, + "ndcg_linear@5": 0.956409951840957, + "ndcg_linear@10": 0.956409951840957, + "ndcg_exponential@1": 0.92, + "ndcg_exponential@5": 0.956409951840957, + "ndcg_exponential@10": 0.956409951840957, + "precision@1": 0.92, + "precision@5": 0.19799999999999998, "precision@10": 0.09899999999999999, - "recall@1": 0.95, - "recall@5": 0.98, + "recall@1": 0.92, + "recall@5": 0.99, "recall@10": 0.99, - "f1@1": 0.95, - "f1@5": 0.32666666666666677, + "f1@1": 0.92, + "f1@5": 0.3300000000000001, "f1@10": 0.18000000000000008, - "map@1": 0.95, - "map@5": 0.9633333333333333, - "map@10": 0.965, - "mrr@1": 0.95, - "mrr@5": 0.9633333333333333, - "mrr@10": 0.965, + "map@1": 0.92, + "map@5": 0.9453333333333334, + "map@10": 0.9453333333333334, + "mrr@1": 0.92, + "mrr@5": 0.9453333333333334, + "mrr@10": 0.9453333333333334, "num_pred": 100, "num_data": 100 }, @@ -1721,478 +1749,422 @@ "num_data": 100 }, "ViDoRe_syntheticDocQA_energy": { - "hit@1": 0.93, + "hit@1": 0.94, "hit@5": 0.97, "hit@10": 0.98, - "ndcg_linear@1": 0.93, - "ndcg_linear@5": 0.9532346581878777, - "ndcg_linear@10": 0.9562449581445175, - "ndcg_exponential@1": 0.93, - "ndcg_exponential@5": 0.9532346581878777, - "ndcg_exponential@10": 0.9562449581445175, - "precision@1": 0.93, + "ndcg_linear@1": 0.94, + "ndcg_linear@5": 0.956925360652163, + "ndcg_linear@10": 0.9600800094200204, + "ndcg_exponential@1": 0.94, + "ndcg_exponential@5": 0.956925360652163, + "ndcg_exponential@10": 0.9600800094200204, + "precision@1": 0.94, "precision@5": 0.19399999999999995, "precision@10": 0.09799999999999998, - "recall@1": 0.93, + "recall@1": 0.94, "recall@5": 0.97, "recall@10": 0.98, - "f1@1": 0.93, + "f1@1": 0.94, "f1@5": 0.3233333333333334, "f1@10": 0.17818181818181827, - "map@1": 0.93, - "map@5": 0.9475, - "map@10": 0.9486111111111112, - "mrr@1": 0.93, - "mrr@5": 0.9475, - "mrr@10": 0.9486111111111112, + "map@1": 0.94, + "map@5": 0.9525, + "map@10": 0.95375, + "mrr@1": 0.94, + "mrr@5": 0.9525, + "mrr@10": 0.95375, "num_pred": 100, "num_data": 100 }, "ViDoRe_syntheticDocQA_government_reports": { - "hit@1": 0.97, + "hit@1": 0.99, "hit@5": 1.0, "hit@10": 1.0, - "ndcg_linear@1": 0.97, - "ndcg_linear@5": 0.9876185950714291, - "ndcg_linear@10": 0.9876185950714291, - "ndcg_exponential@1": 0.97, - "ndcg_exponential@5": 0.9876185950714291, - "ndcg_exponential@10": 0.9876185950714291, - "precision@1": 0.97, + "ndcg_linear@1": 0.99, + "ndcg_linear@5": 0.995, + "ndcg_linear@10": 0.995, + "ndcg_exponential@1": 0.99, + "ndcg_exponential@5": 0.995, + "ndcg_exponential@10": 0.995, + "precision@1": 0.99, "precision@5": 0.19999999999999996, "precision@10": 0.09999999999999998, - "recall@1": 0.97, + "recall@1": 0.99, "recall@5": 1.0, "recall@10": 1.0, - "f1@1": 0.97, + "f1@1": 0.99, "f1@5": 0.3333333333333335, "f1@10": 0.1818181818181819, - "map@1": 0.97, - "map@5": 0.9833333333333333, - "map@10": 0.9833333333333333, - "mrr@1": 0.97, - "mrr@5": 0.9833333333333333, - "mrr@10": 0.9833333333333333, + "map@1": 0.99, + "map@5": 0.9933333333333333, + "map@10": 0.9933333333333333, + "mrr@1": 0.99, + "mrr@5": 0.9933333333333333, + "mrr@10": 0.9933333333333333, "num_pred": 100, "num_data": 100 }, "ViDoRe_syntheticDocQA_healthcare_industry": { - "hit@1": 0.98, + "hit@1": 0.99, "hit@5": 1.0, "hit@10": 1.0, - "ndcg_linear@1": 0.98, - "ndcg_linear@5": 0.9926185950714291, - "ndcg_linear@10": 0.9926185950714291, - "ndcg_exponential@1": 0.98, - "ndcg_exponential@5": 0.9926185950714291, - "ndcg_exponential@10": 0.9926185950714291, - "precision@1": 0.98, + "ndcg_linear@1": 0.99, + "ndcg_linear@5": 0.995, + "ndcg_linear@10": 0.995, + "ndcg_exponential@1": 0.99, + "ndcg_exponential@5": 0.995, + "ndcg_exponential@10": 0.995, + "precision@1": 0.99, "precision@5": 0.19999999999999996, "precision@10": 0.09999999999999998, - "recall@1": 0.98, + "recall@1": 0.99, "recall@5": 1.0, "recall@10": 1.0, - "f1@1": 0.98, + "f1@1": 0.99, "f1@5": 0.3333333333333335, "f1@10": 0.1818181818181819, - "map@1": 0.98, - "map@5": 0.99, - "map@10": 0.99, - "mrr@1": 0.98, - "mrr@5": 0.99, - "mrr@10": 0.99, + "map@1": 0.99, + "map@5": 0.9933333333333333, + "map@10": 0.9933333333333333, + "mrr@1": 0.99, + "mrr@5": 0.9933333333333333, + "mrr@10": 0.9933333333333333, "num_pred": 100, "num_data": 100 }, "ViDoRe_esg_reports_human_labeled_v2": { "hit@1": 0.5961538461538461, "hit@5": 0.7884615384615384, - "hit@10": 0.8846153846153846, + "hit@10": 0.9038461538461539, "ndcg_linear@1": 0.5737179487179488, - "ndcg_linear@5": 0.6336537052325203, - "ndcg_linear@10": 0.6529793585477737, + "ndcg_linear@5": 0.627639422231532, + "ndcg_linear@10": 0.658233467146749, "ndcg_exponential@1": 0.5668498168498168, - "ndcg_exponential@5": 0.6338504365299624, - "ndcg_exponential@10": 0.6525583236140545, + "ndcg_exponential@5": 0.6275512096000645, + "ndcg_exponential@10": 0.6575129304741888, "precision@1": 0.5961538461538461, - "precision@5": 0.23076923076923078, - "precision@10": 0.13653846153846153, + "precision@5": 0.22307692307692306, + "precision@10": 0.13846153846153844, "recall@1": 0.42499999999999993, - "recall@5": 0.6831959706959707, - "recall@10": 0.7332042957042956, + "recall@5": 0.6767857142857143, + "recall@10": 0.7524350649350648, "f1@1": 0.4619329388560158, - "f1@5": 0.31360967463908646, - "f1@10": 0.21265489789245448, + "f1@5": 0.3070737269266682, + "f1@10": 0.216151401388958, "map@1": 0.5961538461538461, - "map@5": 0.5806089743589743, - "map@10": 0.5874552299552299, + "map@5": 0.5756517094017094, + "map@10": 0.5883772675736961, "mrr@1": 0.5961538461538461, - "mrr@5": 0.6778846153846154, - "mrr@10": 0.6906669719169719, + "mrr@5": 0.678525641025641, + "mrr@10": 0.6952152014652015, "num_pred": 52, "num_data": 52 }, "ViDoRe_biomedical_lectures_v2_multilingual": { "hit@1": 0.4546875, - "hit@5": 0.728125, - "hit@10": 0.815625, + "hit@5": 0.734375, + "hit@10": 0.8125, "ndcg_linear@1": 0.4546875, - "ndcg_linear@5": 0.4948824579441539, - "ndcg_linear@10": 0.5296750546342992, + "ndcg_linear@5": 0.4977930552023421, + "ndcg_linear@10": 0.5300759783693756, "ndcg_exponential@1": 0.4546875, - "ndcg_exponential@5": 0.4948824579441539, - "ndcg_exponential@10": 0.5296750546342992, + "ndcg_exponential@5": 0.49779305520234207, + "ndcg_exponential@10": 0.5300759783693756, "precision@1": 0.4546875, - "precision@5": 0.23562500000000003, - "precision@10": 0.15390625, - "recall@1": 0.26338541666666665, - "recall@5": 0.523475905987395, - "recall@10": 0.6315834243670039, - "f1@1": 0.3050258161976912, - "f1@5": 0.28532247873654126, - "f1@10": 0.22208065327918272, + "precision@5": 0.2365625, + "precision@10": 0.15328125, + "recall@1": 0.26260416666666664, + "recall@5": 0.5288144476540616, + "recall@10": 0.6296253954167874, + "f1@1": 0.30450498286435784, + "f1@5": 0.28723163728632484, + "f1@10": 0.2212523947831852, "map@1": 0.4546875, - "map@5": 0.4263871527777778, - "map@10": 0.44231501042257176, + "map@5": 0.4281818576388889, + "map@10": 0.44330076205829555, "mrr@1": 0.4546875, - "mrr@5": 0.56375, - "mrr@10": 0.5754284474206349, + "mrr@5": 0.5658072916666667, + "mrr@10": 0.5762177579365079, "num_pred": 640, "num_data": 640 }, "ViDoRe_economics_reports_v2_multilingual": { - "hit@1": 0.6206896551724138, + "hit@1": 0.6163793103448276, "hit@5": 0.896551724137931, "hit@10": 0.9353448275862069, - "ndcg_linear@1": 0.6206896551724138, - "ndcg_linear@5": 0.5421098675106539, - "ndcg_linear@10": 0.5287811820133248, - "ndcg_exponential@1": 0.6206896551724138, - "ndcg_exponential@5": 0.5421098675106539, - "ndcg_exponential@10": 0.5287811820133248, - "precision@1": 0.6206896551724138, - "precision@5": 0.45431034482758625, - "precision@10": 0.3745689655172414, - "recall@1": 0.10691249388459685, - "recall@5": 0.2925943528645089, - "recall@10": 0.41857708558180107, - "f1@1": 0.14883132087852466, - "f1@5": 0.2819557978845695, - "f1@10": 0.3211854297471819, - "map@1": 0.6206896551724138, - "map@5": 0.42993893678160916, - "map@10": 0.38539058495364936, - "mrr@1": 0.6206896551724138, - "mrr@5": 0.7347701149425288, - "mrr@10": 0.7403188286808976, + "ndcg_linear@1": 0.6163793103448276, + "ndcg_linear@5": 0.5391145954436852, + "ndcg_linear@10": 0.5262705377137538, + "ndcg_exponential@1": 0.6163793103448276, + "ndcg_exponential@5": 0.5391145954436852, + "ndcg_exponential@10": 0.5262705377137538, + "precision@1": 0.6163793103448276, + "precision@5": 0.4525862068965517, + "precision@10": 0.3732758620689655, + "recall@1": 0.10596237096500984, + "recall@5": 0.2923151644207785, + "recall@10": 0.4170151121593348, + "f1@1": 0.14728691764154664, + "f1@5": 0.2810876820342903, + "f1@10": 0.3198371588129757, + "map@1": 0.6163793103448276, + "map@5": 0.426235632183908, + "map@10": 0.38255943217804367, + "mrr@1": 0.6163793103448276, + "mrr@5": 0.7326149425287355, + "mrr@10": 0.738348385331144, "num_pred": 232, "num_data": 232 }, "ViDoRe_esg_reports_v2_multilingual": { - "hit@1": 0.4956140350877193, - "hit@5": 0.8201754385964912, + "hit@1": 0.5087719298245614, + "hit@5": 0.8114035087719298, "hit@10": 0.9078947368421053, - "ndcg_linear@1": 0.4956140350877193, - "ndcg_linear@5": 0.553791630514325, - "ndcg_linear@10": 0.5983952525358399, - "ndcg_exponential@1": 0.4956140350877193, - "ndcg_exponential@5": 0.553791630514325, - "ndcg_exponential@10": 0.5983952525358399, - "precision@1": 0.4956140350877193, - "precision@5": 0.2789473684210526, + "ndcg_linear@1": 0.5087719298245614, + "ndcg_linear@5": 0.554276254545028, + "ndcg_linear@10": 0.5992741885355211, + "ndcg_exponential@1": 0.5087719298245614, + "ndcg_exponential@5": 0.554276254545028, + "ndcg_exponential@10": 0.5992741885355211, + "precision@1": 0.5087719298245614, + "precision@5": 0.27894736842105267, "precision@10": 0.18859649122807023, - "recall@1": 0.2702119883040936, - "recall@5": 0.5824352548036759, - "recall@10": 0.7194914369256475, - "f1@1": 0.31508798219324535, - "f1@5": 0.3248121100141491, - "f1@10": 0.2578734690556428, - "map@1": 0.4956140350877193, - "map@5": 0.4682602339181286, - "map@10": 0.48884156281908475, - "mrr@1": 0.4956140350877193, - "mrr@5": 0.628874269005848, - "mrr@10": 0.6414212614870508, + "recall@1": 0.27039473684210524, + "recall@5": 0.581156015037594, + "recall@10": 0.7197002923976608, + "f1@1": 0.3176318418423681, + "f1@5": 0.3242158789278999, + "f1@10": 0.25786021879797344, + "map@1": 0.5087719298245614, + "map@5": 0.46769249512670563, + "map@10": 0.4879898240972802, + "mrr@1": 0.5087719298245614, + "mrr@5": 0.6352339181286549, + "mrr@10": 0.6496623503202451, "num_pred": 228, "num_data": 228 }, "VisRAG_ArxivQA": { - "hit@1": 0.8259803921568627, - "hit@5": 0.9166666666666666, - "hit@10": 0.9448529411764706, - "ndcg_linear@1": 0.8259803921568627, - "ndcg_linear@5": 0.8734973163142782, - "ndcg_linear@10": 0.8828532837961396, - "ndcg_exponential@1": 0.8259803921568627, - "ndcg_exponential@5": 0.8734973163142782, - "ndcg_exponential@10": 0.8828532837961396, - "precision@1": 0.8259803921568627, - "precision@5": 0.18333333333333332, - "precision@10": 0.09448529411764706, - "recall@1": 0.8259803921568627, - "recall@5": 0.9166666666666666, - "recall@10": 0.9448529411764706, - "f1@1": 0.8259803921568627, - "f1@5": 0.30555555555555564, - "f1@10": 0.17179144385026737, - "map@1": 0.8259803921568627, - "map@5": 0.8592320261437908, - "map@10": 0.86323140367258, - "mrr@1": 0.8259803921568627, - "mrr@5": 0.8592320261437908, - "mrr@10": 0.86323140367258, + "hit@1": 0.8284313725490197, + "hit@5": 0.9203431372549019, + "hit@10": 0.946078431372549, + "ndcg_linear@1": 0.8284313725490197, + "ndcg_linear@5": 0.8768696129768009, + "ndcg_linear@10": 0.8852514511619681, + "ndcg_exponential@1": 0.8284313725490197, + "ndcg_exponential@5": 0.8768696129768009, + "ndcg_exponential@10": 0.8852514511619681, + "precision@1": 0.8284313725490197, + "precision@5": 0.18406862745098038, + "precision@10": 0.0946078431372549, + "recall@1": 0.8284313725490197, + "recall@5": 0.9203431372549019, + "recall@10": 0.946078431372549, + "f1@1": 0.8284313725490197, + "f1@5": 0.306781045751634, + "f1@10": 0.17201426024955438, + "map@1": 0.8284313725490197, + "map@5": 0.8624591503267973, + "map@10": 0.8659503384687207, + "mrr@1": 0.8284313725490197, + "mrr@5": 0.8624591503267973, + "mrr@10": 0.8659503384687207, "num_pred": 816, "num_data": 816 }, "VisRAG_ChartQA": { "hit@1": 0.7619047619047619, - "hit@5": 0.873015873015873, + "hit@5": 0.8571428571428571, "hit@10": 0.9206349206349206, "ndcg_linear@1": 0.7619047619047619, - "ndcg_linear@5": 0.8190027211628073, - "ndcg_linear@10": 0.8349551877854593, + "ndcg_linear@5": 0.8128622004130525, + "ndcg_linear@10": 0.83410567232671, "ndcg_exponential@1": 0.7619047619047619, - "ndcg_exponential@5": 0.8190027211628073, - "ndcg_exponential@10": 0.8349551877854593, + "ndcg_exponential@5": 0.8128622004130525, + "ndcg_exponential@10": 0.83410567232671, "precision@1": 0.7619047619047619, - "precision@5": 0.1746031746031745, + "precision@5": 0.17142857142857135, "precision@10": 0.09206349206349201, "recall@1": 0.7619047619047619, - "recall@5": 0.873015873015873, + "recall@5": 0.8571428571428571, "recall@10": 0.9206349206349206, "f1@1": 0.7619047619047619, - "f1@5": 0.29100529100529093, + "f1@5": 0.28571428571428564, "f1@10": 0.16738816738816742, "map@1": 0.7619047619047619, - "map@5": 0.8013227513227513, - "map@10": 0.8082199546485261, + "map@5": 0.7981481481481482, + "map@10": 0.807312925170068, "mrr@1": 0.7619047619047619, - "mrr@5": 0.8013227513227513, - "mrr@10": 0.8082199546485261, + "mrr@5": 0.7981481481481482, + "mrr@10": 0.807312925170068, "num_pred": 63, "num_data": 63 }, "VisRAG_MP-DocVQA": { - "hit@1": 0.817258883248731, + "hit@1": 0.8155668358714044, "hit@5": 0.9475465313028765, - "hit@10": 0.961082910321489, - "ndcg_linear@1": 0.817258883248731, - "ndcg_linear@5": 0.8921574691348291, - "ndcg_linear@10": 0.8964004804547877, - "ndcg_exponential@1": 0.817258883248731, - "ndcg_exponential@5": 0.8921574691348291, - "ndcg_exponential@10": 0.8964004804547877, - "precision@1": 0.817258883248731, + "hit@10": 0.9593908629441624, + "ndcg_linear@1": 0.8155668358714044, + "ndcg_linear@5": 0.890795131133075, + "ndcg_linear@10": 0.8944841153022184, + "ndcg_exponential@1": 0.8155668358714044, + "ndcg_exponential@5": 0.890795131133075, + "ndcg_exponential@10": 0.8944841153022184, + "precision@1": 0.8155668358714044, "precision@5": 0.1895093062605753, - "precision@10": 0.0961082910321489, - "recall@1": 0.817258883248731, + "precision@10": 0.09593908629441622, + "recall@1": 0.8155668358714044, "recall@5": 0.9475465313028765, - "recall@10": 0.961082910321489, - "f1@1": 0.817258883248731, + "recall@10": 0.9593908629441624, + "f1@1": 0.8155668358714044, "f1@5": 0.3158488437676255, - "f1@10": 0.17474234733117985, - "map@1": 0.817258883248731, - "map@5": 0.873237450648618, - "map@10": 0.8749073402626703, - "mrr@1": 0.817258883248731, - "mrr@5": 0.873237450648618, - "mrr@10": 0.8749073402626703, + "f1@10": 0.17443470235348413, + "map@1": 0.8155668358714044, + "map@5": 0.8714326001128031, + "map@10": 0.8728721832782746, + "mrr@1": 0.8155668358714044, + "mrr@5": 0.8714326001128031, + "mrr@10": 0.8728721832782746, "num_pred": 591, "num_data": 591 }, "VisRAG_SlideVQA": { - "hit@1": 0.9244604316546763, + "hit@1": 0.9280575539568345, "hit@5": 0.9766187050359713, "hit@10": 0.9820143884892086, - "ndcg_linear@1": 0.9244604316546763, - "ndcg_linear@5": 0.9453762073854034, - "ndcg_linear@10": 0.9484324956995065, - "ndcg_exponential@1": 0.9244604316546763, - "ndcg_exponential@5": 0.9453762073854034, - "ndcg_exponential@10": 0.9484324956995065, - "precision@1": 0.9244604316546763, + "ndcg_linear@1": 0.9280575539568345, + "ndcg_linear@5": 0.9472419372386658, + "ndcg_linear@10": 0.9499383064289354, + "ndcg_exponential@1": 0.9280575539568345, + "ndcg_exponential@5": 0.9472419372386658, + "ndcg_exponential@10": 0.9499383064289354, + "precision@1": 0.9280575539568345, "precision@5": 0.24388489208633093, - "precision@10": 0.12320143884892086, - "recall@1": 0.8003597122302158, + "precision@10": 0.12302158273381296, + "recall@1": 0.8039568345323741, "recall@5": 0.9685251798561151, - "recall@10": 0.9766187050359713, - "f1@1": 0.841726618705036, + "recall@10": 0.9757194244604317, + "f1@1": 0.8453237410071942, "f1@5": 0.3825796505652622, - "f1@10": 0.21626335295400048, - "map@1": 0.9244604316546763, - "map@5": 0.9316247002398079, - "map@10": 0.9332980044535798, - "mrr@1": 0.9244604316546763, - "mrr@5": 0.9481414868105514, - "mrr@10": 0.9489658273381295, + "f1@10": 0.21596359276215393, + "map@1": 0.9280575539568345, + "map@5": 0.9343225419664268, + "map@10": 0.9357606771725476, + "mrr@1": 0.9280575539568345, + "mrr@5": 0.9502398081534773, + "mrr@10": 0.951039168665068, "num_pred": 556, "num_data": 556 }, "VisRAG_InfoVQA": { - "hit@1": 0.8704735376044568, - "hit@5": 0.9832869080779945, + "hit@1": 0.8746518105849582, + "hit@5": 0.9805013927576601, "hit@10": 0.9930362116991643, - "ndcg_linear@1": 0.8704735376044568, - "ndcg_linear@5": 0.9352387521932299, - "ndcg_linear@10": 0.9383224079083196, - "ndcg_exponential@1": 0.8704735376044568, - "ndcg_exponential@5": 0.9352387521932299, - "ndcg_exponential@10": 0.9383224079083196, - "precision@1": 0.8704735376044568, - "precision@5": 0.19665738161559887, + "ndcg_linear@1": 0.8746518105849582, + "ndcg_linear@5": 0.9348912982922946, + "ndcg_linear@10": 0.9390169474437507, + "ndcg_exponential@1": 0.8746518105849582, + "ndcg_exponential@5": 0.9348912982922946, + "ndcg_exponential@10": 0.9390169474437507, + "precision@1": 0.8746518105849582, + "precision@5": 0.196100278551532, "precision@10": 0.09930362116991641, - "recall@1": 0.8704735376044568, - "recall@5": 0.9832869080779945, + "recall@1": 0.8746518105849582, + "recall@5": 0.9805013927576601, "recall@10": 0.9930362116991643, - "f1@1": 0.8704735376044568, - "f1@5": 0.3277623026926649, + "f1@1": 0.8746518105849582, + "f1@5": 0.3268337975858868, "f1@10": 0.1805520384907572, - "map@1": 0.8704735376044568, - "map@5": 0.9188254410399258, - "map@10": 0.9200573683512402, - "mrr@1": 0.8704735376044568, - "mrr@5": 0.9188254410399258, - "mrr@10": 0.9200573683512402, + "map@1": 0.8746518105849582, + "map@5": 0.9193593314763232, + "map@10": 0.9211052526860327, + "mrr@1": 0.8746518105849582, + "mrr@5": 0.9193593314763232, + "mrr@10": 0.9211052526860327, "num_pred": 718, "num_data": 718 }, "VisRAG_PlotQA": { - "hit@1": 0.49942062572421786, - "hit@5": 0.7520278099652375, - "hit@10": 0.8389339513325609, - "ndcg_linear@1": 0.49942062572421786, - "ndcg_linear@5": 0.6341927528705769, - "ndcg_linear@10": 0.6624618705964804, - "ndcg_exponential@1": 0.49942062572421786, - "ndcg_exponential@5": 0.6341927528705769, - "ndcg_exponential@10": 0.6624618705964804, - "precision@1": 0.49942062572421786, - "precision@5": 0.1504055619930475, - "precision@10": 0.08389339513325607, - "recall@1": 0.49942062572421786, - "recall@5": 0.7520278099652375, - "recall@10": 0.8389339513325609, - "f1@1": 0.49942062572421786, - "f1@5": 0.2506759366550792, - "f1@10": 0.1525334456968293, - "map@1": 0.49942062572421786, - "map@5": 0.5949787562765547, - "map@10": 0.6067474112085932, - "mrr@1": 0.49942062572421786, - "mrr@5": 0.5949787562765547, - "mrr@10": 0.6067474112085932, + "hit@1": 0.4982618771726535, + "hit@5": 0.7531865585168018, + "hit@10": 0.8377752027809965, + "ndcg_linear@1": 0.4982618771726535, + "ndcg_linear@5": 0.6344460289485018, + "ndcg_linear@10": 0.6619424329905106, + "ndcg_exponential@1": 0.4982618771726535, + "ndcg_exponential@5": 0.6344460289485018, + "ndcg_exponential@10": 0.6619424329905106, + "precision@1": 0.4982618771726535, + "precision@5": 0.15063731170336037, + "precision@10": 0.08377752027809963, + "recall@1": 0.4982618771726535, + "recall@5": 0.7531865585168018, + "recall@10": 0.8377752027809965, + "f1@1": 0.4982618771726535, + "f1@5": 0.25106218617226733, + "f1@10": 0.1523227641419994, + "map@1": 0.4982618771726535, + "map@5": 0.594901506373117, + "map@10": 0.6063455277823759, + "mrr@1": 0.4982618771726535, + "mrr@5": 0.594901506373117, + "mrr@10": 0.6063455277823759, "num_pred": 863, "num_data": 863 }, "ViDoSeek-page": { - "hit@1": 0.02714535901926445, - "hit@5": 0.4290718038528897, - "hit@10": 0.6532399299474606, - "ndcg_linear@1": 0.02714535901926445, - "ndcg_linear@5": 0.23240353892048016, - "ndcg_linear@10": 0.305124886024534, - "ndcg_exponential@1": 0.02714535901926445, - "ndcg_exponential@5": 0.23240353892048016, - "ndcg_exponential@10": 0.305124886024534, - "precision@1": 0.02714535901926445, - "precision@5": 0.08581436077057793, - "precision@10": 0.06532399299474606, - "recall@1": 0.02714535901926445, - "recall@5": 0.4290718038528897, - "recall@10": 0.6532399299474606, - "f1@1": 0.02714535901926445, - "f1@5": 0.1430239346176299, - "f1@10": 0.11877089635408376, - "map@1": 0.02714535901926445, - "map@5": 0.16780502043199066, - "map@10": 0.19794637644900342, - "mrr@1": 0.02714535901926445, - "mrr@5": 0.16780502043199066, - "mrr@10": 0.19794637644900342, - "num_pred": 1142, - "num_data": 1142 - }, - "ViDoSeek-doc": { - "hit@1": 0.9964973730297724, - "hit@5": 0.9982486865148862, - "hit@10": 0.999124343257443, - "ndcg_linear@1": 0.6800934033858728, - "ndcg_linear@5": 0.8387827817558765, - "ndcg_linear@10": 0.8278623787973661, - "ndcg_exponential@1": 0.4540905679259443, - "ndcg_exponential@5": 0.7179848879430044, - "ndcg_exponential@10": 0.7517965160193582, - "precision@1": 0.9964973730297724, - "precision@5": 0.9288966725043785, - "precision@10": 0.8447460595446585, - "recall@1": 0.058761907353168774, - "recall@5": 0.2735768446029065, - "recall@10": 0.48843735028165003, - "f1@1": 0.11022913197970112, - "f1@5": 0.41401147163111984, - "f1@10": 0.6049486379407474, - "map@1": 0.9964973730297724, - "map@5": 0.9212288382953882, - "map@10": 0.8346283571805838, - "mrr@1": 0.9964973730297724, - "mrr@5": 0.9973730297723292, - "mrr@10": 0.9975189725627552, + "hit@1": 0.702276707530648, + "hit@5": 0.9422066549912435, + "hit@10": 0.9833625218914186, + "ndcg_linear@1": 0.702276707530648, + "ndcg_linear@5": 0.835698726940311, + "ndcg_linear@10": 0.8494728827218212, + "ndcg_exponential@1": 0.702276707530648, + "ndcg_exponential@5": 0.835698726940311, + "ndcg_exponential@10": 0.8494728827218212, + "precision@1": 0.702276707530648, + "precision@5": 0.18844133099824867, + "precision@10": 0.09833625218914184, + "recall@1": 0.702276707530648, + "recall@5": 0.9422066549912435, + "recall@10": 0.9833625218914186, + "f1@1": 0.702276707530648, + "f1@5": 0.3140688849970812, + "f1@10": 0.17879318579843978, + "map@1": 0.702276707530648, + "map@5": 0.7997373029772329, + "map@10": 0.805695938620632, + "mrr@1": 0.702276707530648, + "mrr@5": 0.7997373029772329, + "mrr@10": 0.805695938620632, "num_pred": 1142, "num_data": 1142 }, "MMLongBench-page": { - "hit@1": 0.06921241050119331, - "hit@5": 0.32816229116945106, - "hit@10": 0.43556085918854415, - "ndcg_linear@1": 0.06921241050119331, - "ndcg_linear@5": 0.16193698962444822, - "ndcg_linear@10": 0.20158281959554863, - "ndcg_exponential@1": 0.06921241050119331, - "ndcg_exponential@5": 0.16193698962444822, - "ndcg_exponential@10": 0.20158281959554863, - "precision@1": 0.06921241050119331, - "precision@5": 0.07756563245823389, - "precision@10": 0.058233890214797135, - "recall@1": 0.040324184566428, - "recall@5": 0.23706102966246162, - "recall@10": 0.34820150017047385, - "f1@1": 0.048800854409446294, - "f1@5": 0.1101645476522635, - "f1@10": 0.09465332114769648, - "map@1": 0.06921241050119331, - "map@5": 0.11841553964465659, - "map@10": 0.13681524054861935, - "mrr@1": 0.06921241050119331, - "mrr@5": 0.16302704852824182, - "mrr@10": 0.17715081259234006, - "num_pred": 838, - "num_data": 838 - }, - "MMLongBench-doc": { - "hit@1": 0.7326968973747017, - "hit@5": 0.8126491646778043, - "hit@10": 0.837708830548926, - "ndcg_linear@1": 0.5123309466984886, - "ndcg_linear@5": 0.5428858706725166, - "ndcg_linear@10": 0.5275085648914615, - "ndcg_exponential@1": 0.354926696215479, - "ndcg_exponential@5": 0.45441895462567394, - "ndcg_exponential@10": 0.4695562288112078, - "precision@1": 0.7326968973747017, - "precision@5": 0.6021479713603819, - "precision@10": 0.540453460620525, - "recall@1": 0.023809666923602684, - "recall@5": 0.09513770171338518, - "recall@10": 0.16711122344532417, - "f1@1": 0.04565355580911593, - "f1@5": 0.15776265072387466, - "f1@10": 0.2394225821542852, - "map@1": 0.7326968973747017, - "map@5": 0.5741805887032617, - "map@10": 0.4961500044196942, - "mrr@1": 0.7326968973747017, - "mrr@5": 0.7620525059665871, - "mrr@10": 0.7654510929272267, + "hit@1": 0.49761336515513127, + "hit@5": 0.6837708830548926, + "hit@10": 0.7350835322195705, + "ndcg_linear@1": 0.49761336515513127, + "ndcg_linear@5": 0.5532048850775211, + "ndcg_linear@10": 0.5768746006416366, + "ndcg_exponential@1": 0.49761336515513127, + "ndcg_exponential@5": 0.5532048850775211, + "ndcg_exponential@10": 0.5768746006416366, + "precision@1": 0.49761336515513127, + "precision@5": 0.17971360381861579, + "precision@10": 0.10381861575178998, + "recall@1": 0.38775002841231965, + "recall@5": 0.6071982611660416, + "recall@10": 0.6720081827480395, + "f1@1": 0.42017372047204987, + "f1@5": 0.2640387031970635, + "f1@10": 0.17206711863532342, + "map@1": 0.49761336515513127, + "map@5": 0.5116563908777513, + "map@10": 0.5242374279995743, + "mrr@1": 0.49761336515513127, + "mrr@5": 0.572673031026253, + "mrr@10": 0.5799427965299087, "num_pred": 838, "num_data": 838 }