# task

created_at: 2026-05-25 01:10:05.283863
updated_at: 2026-05-26 02:37:09.520421
meta: {}
data:
  {"sample_id": "emb_trial_T20190908_220656_510400_1__2B_k2N3", "source_id": "emb_trial_T20190908_220656_510400_1", "task_domain": "embodiedbench", "paper_domain": "EmbodiedBench", "task_subtask": "look_at_obj_in_light", "task_difficulty": "n/a", "domain_label": "EmbodiedBench / look_at_obj_in_light / N=3", "requirement_count": 2, "scatter_turns": 3, "informality_tier_selected": 0, "constraints_numbered": "1. Walk over to the right side of the wooden desk to your left, then pick up the pencil off of the middle shelf behind the desk\n2. Look down at the lamp on the desk, then turn on the light of the small lamp", "constraints_list": ["Walk over to the right side of the wooden desk to your left, then pick up the pencil off of the middle shelf behind the desk", "Look down at the lamp on the desk, then turn on the light of the small lamp"], "dialogue": "[T1] Could you walk over to the right side of the wooden desk that's to your left? After that, pick up the pencil from the middle shelf behind the desk.\n[T2] Oh and also, once you've got the pencil, look down at the lamp on the desk.\n[T3] Actually, can you turn on the light of the small lamp there?", "in_stage_a": "yes", "audio_t1": "https://huggingface.co/datasets/omniagentbenchspeech/OmniAgentBench/resolve/main/wild_v3/embodiedbench/multi_turn/audio/N3/look_at_obj_in_light/emb_trial_T20190908_220656_510400_1__2B_k2N3_turn1.wav", "audio_t2": "https://huggingface.co/datasets/omniagentbenchspeech/OmniAgentBench/resolve/main/wild_v3/embodiedbench/multi_turn/audio/N3/look_at_obj_in_light/emb_trial_T20190908_220656_510400_1__2B_k2N3_turn2.wav", "audio_t3": "https://huggingface.co/datasets/omniagentbenchspeech/OmniAgentBench/resolve/main/wild_v3/embodiedbench/multi_turn/audio/N3/look_at_obj_in_light/emb_trial_T20190908_220656_510400_1__2B_k2N3_turn3.wav", "audio_concat": "https://huggingface.co/datasets/omniagentbenchspeech/OmniAgentBench/resolve/main/wild_v3/embodiedbench/multi_turn/audio/N3/look_at_obj_in_light/emb_trial_T20190908_220656_510400_1__2B_k2N3_concat.wav", "turn1_text": "Could you walk over to the right side of the wooden desk that's to your left? After that, pick up the pencil from the middle shelf behind the desk.", "turn2_text": "Oh and also, once you've got the pencil, look down at the lamp on the desk.", "turn3_text": "Actually, can you turn on the light of the small lamp there?", "has_turn3": "yes", "clean_text": "Examine a pencil in the light of a small lamp."}


created_at: 2026-05-25 01:10:05.283922
updated_at: 2026-05-26 02:38:30.758490
meta: {}
data:
  {"sample_id": "emb_trial_T20190907_040327_147149_1__2B_k3N3", "source_id": "emb_trial_T20190907_040327_147149_1", "task_domain": "embodiedbench", "paper_domain": "EmbodiedBench", "task_subtask": "pick_cool_then_place_in_recep", "task_difficulty": "n/a", "domain_label": "EmbodiedBench / pick_cool_then_place_in_recep / N=3", "requirement_count": 3, "scatter_turns": 3, "informality_tier_selected": 0, "constraints_numbered": "1. Turn around and go into the kitchen, then pick up the lettuce to the left of the stove\n2. Go to the fridge on your left, then chill the lettuce in the fridge and take it out\n3. Turn right and go to the sink, then put the lettuce in the sink", "constraints_list": ["Turn around and go into the kitchen, then pick up the lettuce to the left of the stove", "Go to the fridge on your left, then chill the lettuce in the fridge and take it out", "Turn right and go to the sink, then put the lettuce in the sink"], "dialogue": "[T1] Could you go into the kitchen and grab the lettuce that's to the left of the stove?\n[T2] Oh and also, head to the fridge on your left, chill the lettuce in there for a bit, and then take it out.\n[T3] One more thing, turn right and go to the sink, then put the lettuce in the sink.", "in_stage_a": "yes", "audio_t1": "https://huggingface.co/datasets/omniagentbenchspeech/OmniAgentBench/resolve/main/wild_v3/embodiedbench/multi_turn/audio/N3/pick_cool_then_place_in_recep/emb_trial_T20190907_040327_147149_1__2B_k3N3_turn1.wav", "audio_t2": "https://huggingface.co/datasets/omniagentbenchspeech/OmniAgentBench/resolve/main/wild_v3/embodiedbench/multi_turn/audio/N3/pick_cool_then_place_in_recep/emb_trial_T20190907_040327_147149_1__2B_k3N3_turn2.wav", "audio_t3": "https://huggingface.co/datasets/omniagentbenchspeech/OmniAgentBench/resolve/main/wild_v3/embodiedbench/multi_turn/audio/N3/pick_cool_then_place_in_recep/emb_trial_T20190907_040327_147149_1__2B_k3N3_turn3.wav", "audio_concat": "https://huggingface.co/datasets/omniagentbenchspeech/OmniAgentBench/resolve/main/wild_v3/embodiedbench/multi_turn/audio/N3/pick_cool_then_place_in_recep/emb_trial_T20190907_040327_147149_1__2B_k3N3_concat.wav", "turn1_text": "Could you go into the kitchen and grab the lettuce that's to the left of the stove?", "turn2_text": "Oh and also, head to the fridge on your left, chill the lettuce in there for a bit, and then take it out.", "turn3_text": "One more thing, turn right and go to the sink, then put the lettuce in the sink.", "has_turn3": "yes", "clean_text": "Put chilled lettuce into the sink."}


created_at: 2026-05-25 01:10:05.283985
updated_at: 2026-05-26 02:40:05.989009
meta: {}
data:
  {"sample_id": "emb_trial_T20190908_214926_337906_1__2B_k3N3", "source_id": "emb_trial_T20190908_214926_337906_1", "task_domain": "embodiedbench", "paper_domain": "EmbodiedBench", "task_subtask": "pick_clean_then_place_in_recep", "task_difficulty": "n/a", "domain_label": "EmbodiedBench / pick_clean_then_place_in_recep / N=3", "requirement_count": 3, "scatter_turns": 3, "informality_tier_selected": 0, "constraints_numbered": "1. Turn around to face the counter, then pick up the bar of soap off of the counter\n2. Turn around and walk towards the door, then turn left and walk forward, turn left again to face the sink, then put the bar of soap in the right sink basin and turn on the water, after a couple seconds turn the water off and remove the now clean bar of soap\n3. Turn around and walk towards the wall, then turn left and walk towards the shower, turn left again to face the counter, then open the second cabinet door from the right and put the clean bar of soap inside the cabinet then close the door", "constraints_list": ["Turn around to face the counter, then pick up the bar of soap off of the counter", "Turn around and walk towards the door, then turn left and walk forward, turn left again to face the sink, then put the bar of soap in the right sink basin and turn on the water, after a couple seconds turn the water off and remove the now clean bar of soap", "Turn around and walk towards the wall, then turn left and walk towards the shower, turn left again to face the counter, then open the second cabinet door from the right and put the clean bar of soap inside the cabinet then close the door"], "dialogue": "[T1] First, turn around to face the counter and pick up the bar of soap that's sitting there.\n[T2] Oh and also, turn around and walk towards the door, then take a left and walk forward. Turn left again to face the sink, put the soap in the right basin, turn on the water for a few seconds, then turn it off and grab the clean soap.\n[T3] Actually, turn around and head towards the wall, then take a left and walk to the shower. Turn left again to face the counter, open the second cabinet door from the right, put the clean soap inside, and close the door.", "in_stage_a": "yes", "audio_t1": "https://huggingface.co/datasets/omniagentbenchspeech/OmniAgentBench/resolve/main/wild_v3/embodiedbench/multi_turn/audio/N3/pick_clean_then_place_in_recep/emb_trial_T20190908_214926_337906_1__2B_k3N3_turn1.wav", "audio_t2": "https://huggingface.co/datasets/omniagentbenchspeech/OmniAgentBench/resolve/main/wild_v3/embodiedbench/multi_turn/audio/N3/pick_clean_then_place_in_recep/emb_trial_T20190908_214926_337906_1__2B_k3N3_turn2.wav", "audio_t3": "https://huggingface.co/datasets/omniagentbenchspeech/OmniAgentBench/resolve/main/wild_v3/embodiedbench/multi_turn/audio/N3/pick_clean_then_place_in_recep/emb_trial_T20190908_214926_337906_1__2B_k3N3_turn3.wav", "audio_concat": "https://huggingface.co/datasets/omniagentbenchspeech/OmniAgentBench/resolve/main/wild_v3/embodiedbench/multi_turn/audio/N3/pick_clean_then_place_in_recep/emb_trial_T20190908_214926_337906_1__2B_k3N3_concat.wav", "turn1_text": "First, turn around to face the counter and pick up the bar of soap that's sitting there.", "turn2_text": "Oh and also, turn around and walk towards the door, then take a left and walk forward. Turn left again to face the sink, put the soap in the right basin, turn on the water for a few seconds, then turn it off and grab the clean soap.", "turn3_text": "Actually, turn around and head towards the wall, then take a left and walk to the shower. Turn left again to face the counter, open the second cabinet door from the right, put the clean soap inside, and close the door.", "has_turn3": "yes", "clean_text": "Put a clean bar of soap in a cabinet."}


created_at: 2026-05-25 01:10:05.284043
updated_at: 2026-05-26 02:41:03.437969
meta: {}
data:
  {"sample_id": "emb_trial_T20190909_103526_341991_1__2B_k3N3", "source_id": "emb_trial_T20190909_103526_341991_1", "task_domain": "embodiedbench", "paper_domain": "EmbodiedBench", "task_subtask": "look_at_obj_in_light", "task_difficulty": "n/a", "domain_label": "EmbodiedBench / look_at_obj_in_light / N=3", "requirement_count": 3, "scatter_turns": 3, "informality_tier_selected": 0, "constraints_numbered": "1. Turn around and walk straight then turn right and walk straight towards the blue chair, then pick up the keys on the chair\n2. Turn right and look up at the lamp\n3. Turn the lamp on", "constraints_list": ["Turn around and walk straight then turn right and walk straight towards the blue chair, then pick up the keys on the chair", "Turn right and look up at the lamp", "Turn the lamp on"], "dialogue": "[T1] I need you to walk straight, then turn right and head towards the blue chair to pick up the keys sitting there.\n[T2] Oh and also, after you've got the keys, turn right and look up at the lamp.\n[T3] Actually, one more thing, can you turn the lamp on once you're looking at it?", "in_stage_a": "yes", "audio_t1": "https://huggingface.co/datasets/omniagentbenchspeech/OmniAgentBench/resolve/main/wild_v3/embodiedbench/multi_turn/audio/N3/look_at_obj_in_light/emb_trial_T20190909_103526_341991_1__2B_k3N3_turn1.wav", "audio_t2": "https://huggingface.co/datasets/omniagentbenchspeech/OmniAgentBench/resolve/main/wild_v3/embodiedbench/multi_turn/audio/N3/look_at_obj_in_light/emb_trial_T20190909_103526_341991_1__2B_k3N3_turn2.wav", "audio_t3": "https://huggingface.co/datasets/omniagentbenchspeech/OmniAgentBench/resolve/main/wild_v3/embodiedbench/multi_turn/audio/N3/look_at_obj_in_light/emb_trial_T20190909_103526_341991_1__2B_k3N3_turn3.wav", "audio_concat": "https://huggingface.co/datasets/omniagentbenchspeech/OmniAgentBench/resolve/main/wild_v3/embodiedbench/multi_turn/audio/N3/look_at_obj_in_light/emb_trial_T20190909_103526_341991_1__2B_k3N3_concat.wav", "turn1_text": "I need you to walk straight, then turn right and head towards the blue chair to pick up the keys sitting there.", "turn2_text": "Oh and also, after you've got the keys, turn right and look up at the lamp.", "turn3_text": "Actually, one more thing, can you turn the lamp on once you're looking at it?", "has_turn3": "yes", "clean_text": "pick up the keys and view them in the lamp light"}


created_at: 2026-05-25 01:10:05.284101
updated_at: 2026-05-26 02:46:38.729149
meta: {}
data:
  {"sample_id": "emb_trial_T20190907_074106_050405_0__2B_k5N3", "source_id": "emb_trial_T20190907_074106_050405_0", "task_domain": "embodiedbench", "paper_domain": "EmbodiedBench", "task_subtask": "pick_clean_then_place_in_recep", "task_difficulty": "n/a", "domain_label": "EmbodiedBench / pick_clean_then_place_in_recep / N=3", "requirement_count": 5, "scatter_turns": 3, "informality_tier_selected": 0, "constraints_numbered": "1. Turn to the right and go to the sink\n2. Pick up the bar of soap from the counter\n3. Move to the right and put the soap in the right sink, then wet the soap with water\n4. Pick up the soap from the sink and bring it to the counter on the left\n5. Put the soap back on the counter", "constraints_list": ["Turn to the right and go to the sink", "Pick up the bar of soap from the counter", "Move to the right and put the soap in the right sink, then wet the soap with water", "Pick up the soap from the sink and bring it to the counter on the left", "Put the soap back on the counter"], "dialogue": "[T1] Hey, could you turn to the right and head over to the sink?\n[T2] Oh and also, grab the bar of soap from the counter, then move to the right, put it in the right sink, and wet it with some water.\n[T3] I forgot to mention, can you pick up the soap from the sink and bring it back to the counter on the left and put it down there?", "in_stage_a": "yes", "audio_t1": "https://huggingface.co/datasets/omniagentbenchspeech/OmniAgentBench/resolve/main/wild_v3/embodiedbench/multi_turn/audio/N3/pick_clean_then_place_in_recep/emb_trial_T20190907_074106_050405_0__2B_k5N3_turn1.wav", "audio_t2": "https://huggingface.co/datasets/omniagentbenchspeech/OmniAgentBench/resolve/main/wild_v3/embodiedbench/multi_turn/audio/N3/pick_clean_then_place_in_recep/emb_trial_T20190907_074106_050405_0__2B_k5N3_turn2.wav", "audio_t3": "https://huggingface.co/datasets/omniagentbenchspeech/OmniAgentBench/resolve/main/wild_v3/embodiedbench/multi_turn/audio/N3/pick_clean_then_place_in_recep/emb_trial_T20190907_074106_050405_0__2B_k5N3_turn3.wav", "audio_concat": "https://huggingface.co/datasets/omniagentbenchspeech/OmniAgentBench/resolve/main/wild_v3/embodiedbench/multi_turn/audio/N3/pick_clean_then_place_in_recep/emb_trial_T20190907_074106_050405_0__2B_k5N3_concat.wav", "turn1_text": "Hey, could you turn to the right and head over to the sink?", "turn2_text": "Oh and also, grab the bar of soap from the counter, then move to the right, put it in the right sink, and wet it with some water.", "turn3_text": "I forgot to mention, can you pick up the soap from the sink and bring it back to the counter on the left and put it down there?", "has_turn3": "yes", "clean_text": "Put a wet soap on the counter."}


created_at: 2026-05-25 01:10:05.284163
updated_at: 2026-05-26 02:48:42.454061
meta: {}
data:
  {"sample_id": "emb_trial_T20190908_021537_611374_2__2B_k3N3", "source_id": "emb_trial_T20190908_021537_611374_2", "task_domain": "embodiedbench", "paper_domain": "EmbodiedBench", "task_subtask": "pick_and_place_simple", "task_difficulty": "n/a", "domain_label": "EmbodiedBench / pick_and_place_simple / N=3", "requirement_count": 3, "scatter_turns": 3, "informality_tier_selected": 0, "constraints_numbered": "1. Turn around towards the sink and the toilet, then pick up the plunger from between the sink and the toilet\n2. Turn towards the sink to the left\n3. Open the cabinet just to the left and place the plunger inside", "constraints_list": ["Turn around towards the sink and the toilet, then pick up the plunger from between the sink and the toilet", "Turn towards the sink to the left", "Open the cabinet just to the left and place the plunger inside"], "dialogue": "[T1] Can you pick up the plunger that's between the sink and the toilet?\n[T2] Oh, and then turn left towards the sink.\n[T3] One more thing, open the cabinet that's just to the left and put the plunger inside.", "in_stage_a": "yes", "audio_t1": "https://huggingface.co/datasets/omniagentbenchspeech/OmniAgentBench/resolve/main/wild_v3/embodiedbench/multi_turn/audio/N3/pick_and_place_simple/emb_trial_T20190908_021537_611374_2__2B_k3N3_turn1.wav", "audio_t2": "https://huggingface.co/datasets/omniagentbenchspeech/OmniAgentBench/resolve/main/wild_v3/embodiedbench/multi_turn/audio/N3/pick_and_place_simple/emb_trial_T20190908_021537_611374_2__2B_k3N3_turn2.wav", "audio_t3": "https://huggingface.co/datasets/omniagentbenchspeech/OmniAgentBench/resolve/main/wild_v3/embodiedbench/multi_turn/audio/N3/pick_and_place_simple/emb_trial_T20190908_021537_611374_2__2B_k3N3_turn3.wav", "audio_concat": "https://huggingface.co/datasets/omniagentbenchspeech/OmniAgentBench/resolve/main/wild_v3/embodiedbench/multi_turn/audio/N3/pick_and_place_simple/emb_trial_T20190908_021537_611374_2__2B_k3N3_concat.wav", "turn1_text": "Can you pick up the plunger that's between the sink and the toilet?", "turn2_text": "Oh, and then turn left towards the sink.", "turn3_text": "One more thing, open the cabinet that's just to the left and put the plunger inside.", "has_turn3": "yes", "clean_text": "Put the plunger under the sink"}


created_at: 2026-05-25 01:10:05.284237
updated_at: 2026-05-26 02:49:44.873120
meta: {}
data:
  {"sample_id": "emb_trial_T20190906_182259_116320_1__2B_k3N3", "source_id": "emb_trial_T20190906_182259_116320_1", "task_domain": "embodiedbench", "paper_domain": "EmbodiedBench", "task_subtask": "pick_heat_then_place_in_recep", "task_difficulty": "n/a", "domain_label": "EmbodiedBench / pick_heat_then_place_in_recep / N=3", "requirement_count": 3, "scatter_turns": 3, "informality_tier_selected": 0, "constraints_numbered": "1. Walk forward and around the kitchen island to get to the sink, then pick up the green apple in the sink\n2. Turn around and go to the microwave, then heat the apple in the microwave\n3. Walk left to stand in front of the fridge, then put the apple on the middle shelf in the middle of the other two food items", "constraints_list": ["Walk forward and around the kitchen island to get to the sink, then pick up the green apple in the sink", "Turn around and go to the microwave, then heat the apple in the microwave", "Walk left to stand in front of the fridge, then put the apple on the middle shelf in the middle of the other two food items"], "dialogue": "[T1] Could you walk forward and around the kitchen island to get to the sink, then pick up the green apple that's in there?\n[T2] Oh and also, once you've got the apple, turn around and head over to the microwave. You'll want to heat the apple in there.\n[T3] One more thing, after heating it, walk left to stand in front of the fridge, and put the apple on the middle shelf right in the middle of the other two food items.", "in_stage_a": "yes", "audio_t1": "https://huggingface.co/datasets/omniagentbenchspeech/OmniAgentBench/resolve/main/wild_v3/embodiedbench/multi_turn/audio/N3/pick_heat_then_place_in_recep/emb_trial_T20190906_182259_116320_1__2B_k3N3_turn1.wav", "audio_t2": "https://huggingface.co/datasets/omniagentbenchspeech/OmniAgentBench/resolve/main/wild_v3/embodiedbench/multi_turn/audio/N3/pick_heat_then_place_in_recep/emb_trial_T20190906_182259_116320_1__2B_k3N3_turn2.wav", "audio_t3": "https://huggingface.co/datasets/omniagentbenchspeech/OmniAgentBench/resolve/main/wild_v3/embodiedbench/multi_turn/audio/N3/pick_heat_then_place_in_recep/emb_trial_T20190906_182259_116320_1__2B_k3N3_turn3.wav", "audio_concat": "https://huggingface.co/datasets/omniagentbenchspeech/OmniAgentBench/resolve/main/wild_v3/embodiedbench/multi_turn/audio/N3/pick_heat_then_place_in_recep/emb_trial_T20190906_182259_116320_1__2B_k3N3_concat.wav", "turn1_text": "Could you walk forward and around the kitchen island to get to the sink, then pick up the green apple that's in there?", "turn2_text": "Oh and also, once you've got the apple, turn around and head over to the microwave. You'll want to heat the apple in there.", "turn3_text": "One more thing, after heating it, walk left to stand in front of the fridge, and put the apple on the middle shelf right in the middle of the other two food items.", "has_turn3": "yes", "clean_text": "Put a microwaved apple in the fridge."}


created_at: 2026-05-25 01:10:05.284300
updated_at: 2026-05-26 02:50:44.838470
meta: {}
data:
  {"sample_id": "emb_trial_T20190909_210238_431966_1__2B_k4N3", "source_id": "emb_trial_T20190909_210238_431966_1", "task_domain": "embodiedbench", "paper_domain": "EmbodiedBench", "task_subtask": "pick_and_place_simple", "task_difficulty": "n/a", "domain_label": "EmbodiedBench / pick_and_place_simple / N=3", "requirement_count": 4, "scatter_turns": 3, "informality_tier_selected": 0, "constraints_numbered": "1. Turn to the right and then the left to the other side of the room, and face the desk on the left side. Look up to the books on the shelf\n2. Pick up the cup from the shelf\n3. Carry the cup and turn to the left and then face the desk on the right side\n4. Place the cup on the short side of the desk, close to you", "constraints_list": ["Turn to the right and then the left to the other side of the room, and face the desk on the left side. Look up to the books on the shelf", "Pick up the cup from the shelf", "Carry the cup and turn to the left and then face the desk on the right side", "Place the cup on the short side of the desk, close to you"], "dialogue": "[T1] Can you turn to the right, then the left, to the other side of the room and face the desk on the left side? Look up at the books on the shelf.\n[T2] Oh and also, pick up the cup from the shelf.\n[T3] Then carry the cup, turn to the left, and face the desk on the right side. Place the cup on the short side of the desk, close to you.", "in_stage_a": "no", "audio_t1": "https://huggingface.co/datasets/omniagentbenchspeech/OmniAgentBench/resolve/main/wild_v3/embodiedbench/multi_turn/audio/N3/pick_and_place_simple/emb_trial_T20190909_210238_431966_1__2B_k4N3_turn1.wav", "audio_t2": "https://huggingface.co/datasets/omniagentbenchspeech/OmniAgentBench/resolve/main/wild_v3/embodiedbench/multi_turn/audio/N3/pick_and_place_simple/emb_trial_T20190909_210238_431966_1__2B_k4N3_turn2.wav", "audio_t3": "https://huggingface.co/datasets/omniagentbenchspeech/OmniAgentBench/resolve/main/wild_v3/embodiedbench/multi_turn/audio/N3/pick_and_place_simple/emb_trial_T20190909_210238_431966_1__2B_k4N3_turn3.wav", "audio_concat": "https://huggingface.co/datasets/omniagentbenchspeech/OmniAgentBench/resolve/main/wild_v3/embodiedbench/multi_turn/audio/N3/pick_and_place_simple/emb_trial_T20190909_210238_431966_1__2B_k4N3_concat.wav", "turn1_text": "Can you turn to the right, then the left, to the other side of the room and face the desk on the left side? Look up at the books on the shelf.", "turn2_text": "Oh and also, pick up the cup from the shelf.", "turn3_text": "Then carry the cup, turn to the left, and face the desk on the right side. Place the cup on the short side of the desk, close to you.", "has_turn3": "yes", "clean_text": "Move a cup from the shelf to the desk."}


created_at: 2026-05-25 01:10:05.284365
updated_at: 2026-05-26 02:58:42.731490
meta: {}
data:
  {"sample_id": "emb_trial_T20190907_161235_786153_2__2B_k2N3", "source_id": "emb_trial_T20190907_161235_786153_2", "task_domain": "embodiedbench", "paper_domain": "EmbodiedBench", "task_subtask": "look_at_obj_in_light", "task_difficulty": "n/a", "domain_label": "EmbodiedBench / look_at_obj_in_light / N=3", "requirement_count": 2, "scatter_turns": 3, "informality_tier_selected": 0, "constraints_numbered": "1. Turn right and walk to face the basketball underneath the foot of the bed, then pick up the basketball from under the bed\n2. Turn left, walk through the room until the desk, turn left at the desk and face it, then turn on the lamp on the desk", "constraints_list": ["Turn right and walk to face the basketball underneath the foot of the bed, then pick up the basketball from under the bed", "Turn left, walk through the room until the desk, turn left at the desk and face it, then turn on the lamp on the desk"], "dialogue": "[T1] Hey, can you turn right and walk to face the basketball that's under the foot of the bed? Then go ahead and pick it up from under there.\n[T2] Oh and also, after that, turn left and walk through the room until you reach the desk.\n[T3] Actually, once you're at the desk, turn left to face it and turn on the lamp that's on the desk.", "in_stage_a": "no", "audio_t1": "https://huggingface.co/datasets/omniagentbenchspeech/OmniAgentBench/resolve/main/wild_v3/embodiedbench/multi_turn/audio/N3/look_at_obj_in_light/emb_trial_T20190907_161235_786153_2__2B_k2N3_turn1.wav", "audio_t2": "https://huggingface.co/datasets/omniagentbenchspeech/OmniAgentBench/resolve/main/wild_v3/embodiedbench/multi_turn/audio/N3/look_at_obj_in_light/emb_trial_T20190907_161235_786153_2__2B_k2N3_turn2.wav", "audio_t3": "https://huggingface.co/datasets/omniagentbenchspeech/OmniAgentBench/resolve/main/wild_v3/embodiedbench/multi_turn/audio/N3/look_at_obj_in_light/emb_trial_T20190907_161235_786153_2__2B_k2N3_turn3.wav", "audio_concat": "https://huggingface.co/datasets/omniagentbenchspeech/OmniAgentBench/resolve/main/wild_v3/embodiedbench/multi_turn/audio/N3/look_at_obj_in_light/emb_trial_T20190907_161235_786153_2__2B_k2N3_concat.wav", "turn1_text": "Hey, can you turn right and walk to face the basketball that's under the foot of the bed? Then go ahead and pick it up from under there.", "turn2_text": "Oh and also, after that, turn left and walk through the room until you reach the desk.", "turn3_text": "Actually, once you're at the desk, turn left to face it and turn on the lamp that's on the desk.", "has_turn3": "yes", "clean_text": "pick up the basketball and turn on the desk lamp in the bedroom"}


created_at: 2026-05-25 01:10:05.284428
updated_at: 2026-05-26 03:00:12.046115
meta: {}
data:
  {"sample_id": "emb_trial_T20190907_173215_875825_1__2B_k5N3", "source_id": "emb_trial_T20190907_173215_875825_1", "task_domain": "embodiedbench", "paper_domain": "EmbodiedBench", "task_subtask": "pick_cool_then_place_in_recep", "task_difficulty": "n/a", "domain_label": "EmbodiedBench / pick_cool_then_place_in_recep / N=3", "requirement_count": 5, "scatter_turns": 3, "informality_tier_selected": 0, "constraints_numbered": "1. Go straight across the room, and then right to go around the island and find the sink on the right, then pick up the egg in the back of the sink\n2. Take the egg and turn right to find the fridge on the right\n3. Place the egg inside the fridge. Then open the door and take out the egg. Shut the fridge door\n4. Take the egg and turn right, then right again at the sink\n5. Place the egg back into the sink", "constraints_list": ["Go straight across the room, and then right to go around the island and find the sink on the right, then pick up the egg in the back of the sink", "Take the egg and turn right to find the fridge on the right", "Place the egg inside the fridge. Then open the door and take out the egg. Shut the fridge door", "Take the egg and turn right, then right again at the sink", "Place the egg back into the sink"], "dialogue": "[T1] I need you to head straight across the room, then turn right to go around the island. You'll find the sink on your right. Go ahead and grab the egg from the back of the sink.\n[T2] Oh, and with the egg in hand, turn right and you'll spot the fridge on your right.\n[T3] Actually, place the egg inside the fridge, then open the door, take the egg out, and close the door. After that, turn right, then right again at the sink, and put the egg back in the sink.", "in_stage_a": "no", "audio_t1": "https://huggingface.co/datasets/omniagentbenchspeech/OmniAgentBench/resolve/main/wild_v3/embodiedbench/multi_turn/audio/N3/pick_cool_then_place_in_recep/emb_trial_T20190907_173215_875825_1__2B_k5N3_turn1.wav", "audio_t2": "https://huggingface.co/datasets/omniagentbenchspeech/OmniAgentBench/resolve/main/wild_v3/embodiedbench/multi_turn/audio/N3/pick_cool_then_place_in_recep/emb_trial_T20190907_173215_875825_1__2B_k5N3_turn2.wav", "audio_t3": "https://huggingface.co/datasets/omniagentbenchspeech/OmniAgentBench/resolve/main/wild_v3/embodiedbench/multi_turn/audio/N3/pick_cool_then_place_in_recep/emb_trial_T20190907_173215_875825_1__2B_k5N3_turn3.wav", "audio_concat": "https://huggingface.co/datasets/omniagentbenchspeech/OmniAgentBench/resolve/main/wild_v3/embodiedbench/multi_turn/audio/N3/pick_cool_then_place_in_recep/emb_trial_T20190907_173215_875825_1__2B_k5N3_concat.wav", "turn1_text": "I need you to head straight across the room, then turn right to go around the island. You'll find the sink on your right. Go ahead and grab the egg from the back of the sink.", "turn2_text": "Oh, and with the egg in hand, turn right and you'll spot the fridge on your right.", "turn3_text": "Actually, place the egg inside the fridge, then open the door, take the egg out, and close the door. After that, turn right, then right again at the sink, and put the egg back in the sink.", "has_turn3": "yes", "clean_text": "Chill an egg and place it in the sink."}


created_at: 2026-05-25 01:10:05.284494
updated_at: 2026-05-26 03:02:18.902684
meta: {}
data:
  {"sample_id": "emb_trial_T20190909_213642_049125_0__2B_k7N3", "source_id": "emb_trial_T20190909_213642_049125_0", "task_domain": "embodiedbench", "paper_domain": "EmbodiedBench", "task_subtask": "pick_cool_then_place_in_recep", "task_difficulty": "n/a", "domain_label": "EmbodiedBench / pick_cool_then_place_in_recep / N=3", "requirement_count": 7, "scatter_turns": 3, "informality_tier_selected": 0, "constraints_numbered": "1. Turn to your left and go to the refrigerator, then turn right and go to the counter top just past the refrigerator, on your left\n2. Pick up the butter knife between the bread and the lettuce, on the counter top\n3. Move over to the left side of the counter, then cut the head of lettuce closest to you, into slices\n4. Move back over to the right side of the counter top, then place the knife back on the counter top, between the lettuce and the bread\n5. Move back over to the left side of the counter top, then pick up on of the lettuce slices\n6. Go to the refrigerator to your left, then place the slice of lettuce to the right of the plate, on the second shelf of the refrigerator, wait a moment for the lettuce to chill, then remove the lettuce from the refrigerator\n7. Turn around and go to the garbage bin on the floor, next to the kitchen island, then place the lettuce in the garbage bin on the floor", "constraints_list": ["Turn to your left and go to the refrigerator, then turn right and go to the counter top just past the refrigerator, on your left", "Pick up the butter knife between the bread and the lettuce, on the counter top", "Move over to the left side of the counter, then cut the head of lettuce closest to you, into slices", "Move back over to the right side of the counter top, then place the knife back on the counter top, between the lettuce and the bread", "Move back over to the left side of the counter top, then pick up on of the lettuce slices", "Go to the refrigerator to your left, then place the slice of lettuce to the right of the plate, on the second shelf of the refrigerator, wait a moment for the lettuce to chill, then remove the lettuce from the refrigerator", "Turn around and go to the garbage bin on the floor, next to the kitchen island, then place the lettuce in the garbage bin on the floor"], "dialogue": "[T1] So, can you start by turning to your left and head over to the refrigerator? Once you're there, turn right and go to the counter top just past the refrigerator, on your left.\n[T2] Oh and also, pick up the butter knife that's between the bread and the lettuce on the counter. Move over to the left side of the counter and slice the head of lettuce that's closest to you. Then, head back to the right side and place the knife back on the counter, right between the lettuce and bread.\n[T3] Actually, move back to the left side again, grab one of the lettuce slices and head to the refrigerator on your left. Put the lettuce slice to the right of the plate on the second shelf, let it chill for a moment, then take it out. Finally, turn around and take it to the garbage bin on the floor next to the kitchen island and toss it in.", "in_stage_a": "no", "audio_t1": "https://huggingface.co/datasets/omniagentbenchspeech/OmniAgentBench/resolve/main/wild_v3/embodiedbench/multi_turn/audio/N3/pick_cool_then_place_in_recep/emb_trial_T20190909_213642_049125_0__2B_k7N3_turn1.wav", "audio_t2": "https://huggingface.co/datasets/omniagentbenchspeech/OmniAgentBench/resolve/main/wild_v3/embodiedbench/multi_turn/audio/N3/pick_cool_then_place_in_recep/emb_trial_T20190909_213642_049125_0__2B_k7N3_turn2.wav", "audio_t3": "https://huggingface.co/datasets/omniagentbenchspeech/OmniAgentBench/resolve/main/wild_v3/embodiedbench/multi_turn/audio/N3/pick_cool_then_place_in_recep/emb_trial_T20190909_213642_049125_0__2B_k7N3_turn3.wav", "audio_concat": "https://huggingface.co/datasets/omniagentbenchspeech/OmniAgentBench/resolve/main/wild_v3/embodiedbench/multi_turn/audio/N3/pick_cool_then_place_in_recep/emb_trial_T20190909_213642_049125_0__2B_k7N3_concat.wav", "turn1_text": "So, can you start by turning to your left and head over to the refrigerator? Once you're there, turn right and go to the counter top just past the refrigerator, on your left.", "turn2_text": "Oh and also, pick up the butter knife that's between the bread and the lettuce on the counter. Move over to the left side of the counter and slice the head of lettuce that's closest to you. Then, head back to the right side and place the knife back on the counter, right between the lettuce and bread.", "turn3_text": "Actually, move back to the left side again, grab one of the lettuce slices and head to the refrigerator on your left. Put the lettuce slice to the right of the plate on the second shelf, let it chill for a moment, then take it out. Finally, turn around and take it to the garbage bin on the floor next to the kitchen island and toss it in.", "has_turn3": "yes", "clean_text": "Put a cold slice of lettuce in the garbage bin."}


created_at: 2026-05-25 01:10:05.284555
updated_at: 2026-05-26 03:03:00.013490
meta: {}
data:
  {"sample_id": "emb_trial_T20190907_144232_616550_2__2B_k5N3", "source_id": "emb_trial_T20190907_144232_616550_2", "task_domain": "embodiedbench", "paper_domain": "EmbodiedBench", "task_subtask": "pick_two_obj_and_place", "task_difficulty": "n/a", "domain_label": "EmbodiedBench / pick_two_obj_and_place / N=3", "requirement_count": 5, "scatter_turns": 3, "informality_tier_selected": 0, "constraints_numbered": "1. Begin walking across the room, then turn right to face the end of the kitchen island\n2. Pick up the metal fork that is on the kitchen island\n3. Turn right and walk to the other side of the kitchen island where the golden statue is resting, then place the metal fork into the green cup that is to the left of the golden statue\n4. Pick up the green cup with the metal fork in it\n5. Turn left and walk over to the sink that is in the middle of the kitchen island, then place the green cup with the metal fork in it into the sink basin", "constraints_list": ["Begin walking across the room, then turn right to face the end of the kitchen island", "Pick up the metal fork that is on the kitchen island", "Turn right and walk to the other side of the kitchen island where the golden statue is resting, then place the metal fork into the green cup that is to the left of the golden statue", "Pick up the green cup with the metal fork in it", "Turn left and walk over to the sink that is in the middle of the kitchen island, then place the green cup with the metal fork in it into the sink basin"], "dialogue": "[T1] I need you to start by walking across the room and then turn right to face the end of the kitchen island.\n[T2] Oh and also, grab the metal fork that's sitting on the kitchen island. Then turn right again and head to the other side where the golden statue is, and put the fork into the green cup to the left of the statue.\n[T3] One more thing, could you pick up that green cup with the fork in it, turn left, and walk over to the sink in the middle of the kitchen island? Just place the cup with the fork into the sink basin there.", "in_stage_a": "no", "audio_t1": "https://huggingface.co/datasets/omniagentbenchspeech/OmniAgentBench/resolve/main/wild_v3/embodiedbench/multi_turn/audio/N3/pick_two_obj_and_place/emb_trial_T20190907_144232_616550_2__2B_k5N3_turn1.wav", "audio_t2": "https://huggingface.co/datasets/omniagentbenchspeech/OmniAgentBench/resolve/main/wild_v3/embodiedbench/multi_turn/audio/N3/pick_two_obj_and_place/emb_trial_T20190907_144232_616550_2__2B_k5N3_turn2.wav", "audio_t3": "https://huggingface.co/datasets/omniagentbenchspeech/OmniAgentBench/resolve/main/wild_v3/embodiedbench/multi_turn/audio/N3/pick_two_obj_and_place/emb_trial_T20190907_144232_616550_2__2B_k5N3_turn3.wav", "audio_concat": "https://huggingface.co/datasets/omniagentbenchspeech/OmniAgentBench/resolve/main/wild_v3/embodiedbench/multi_turn/audio/N3/pick_two_obj_and_place/emb_trial_T20190907_144232_616550_2__2B_k5N3_concat.wav", "turn1_text": "I need you to start by walking across the room and then turn right to face the end of the kitchen island.", "turn2_text": "Oh and also, grab the metal fork that's sitting on the kitchen island. Then turn right again and head to the other side where the golden statue is, and put the fork into the green cup to the left of the statue.", "turn3_text": "One more thing, could you pick up that green cup with the fork in it, turn left, and walk over to the sink in the middle of the kitchen island? Just place the cup with the fork into the sink basin there.", "has_turn3": "yes", "clean_text": "Put a green cup with a metal fork in it into the sink basin."}


# data_manager_view

ordering: ["tasks:completed_at"]
selected_items: {}
data:
  {"title": "Default", "type": "list", "target": "tasks", "hiddenColumns": {"explore": ["tasks:annotations_ids", "tasks:predictions_score", "tasks:predictions_model_versions", "tasks:predictions_results", "tasks:file_upload", "tasks:storage_filename", "tasks:created_at", "tasks:updated_at", "tasks:updated_by", "tasks:avg_lead_time", "tasks:draft_exists", "tasks:annotations_results"], "labeling": ["tasks:data.source_id", "tasks:data.task_domain", "tasks:data.paper_domain", "tasks:data.task_subtask", "tasks:data.task_difficulty", "tasks:data.scatter_turns", "tasks:data.informality_tier_selected", "tasks:data.constraints_list", "tasks:data.dialogue", "tasks:data.in_stage_a", "tasks:data.has_turn3", "tasks:id", "tasks:inner_id", "tasks:completed_at", "tasks:cancelled_annotations", "tasks:total_predictions", "tasks:annotators", "tasks:annotations_results", "tasks:annotations_ids", "tasks:predictions_score", "tasks:predictions_model_versions", "tasks:predictions_results", "tasks:file_upload", "tasks:storage_filename", "tasks:created_at", "tasks:updated_at", "tasks:updated_by", "tasks:avg_lead_time", "tasks:draft_exists"]}, "columnsWidth": {}, "columnsDisplayType": {}, "gridWidth": 4, "gridFitImagesToWidth": false, "semantic_search": [], "agreement_selected": {}}


ordering: ["tasks:id"]
selected_items: {}
data:
  {"title": "Default", "type": "list", "target": "tasks", "hiddenColumns": {"explore": ["tasks:inner_id", "tasks:annotations_results", "tasks:annotations_ids", "tasks:predictions_score", "tasks:predictions_model_versions", "tasks:predictions_results", "tasks:file_upload", "tasks:storage_filename", "tasks:created_at", "tasks:updated_at", "tasks:updated_by", "tasks:avg_lead_time", "tasks:draft_exists"], "labeling": ["tasks:data.source_id", "tasks:data.task_domain", "tasks:data.paper_domain", "tasks:data.task_subtask", "tasks:data.task_difficulty", "tasks:data.scatter_turns", "tasks:data.informality_tier_selected", "tasks:data.constraints_list", "tasks:id", "tasks:inner_id", "tasks:completed_at", "tasks:cancelled_annotations", "tasks:total_predictions", "tasks:annotators", "tasks:annotations_results", "tasks:annotations_ids", "tasks:predictions_score", "tasks:predictions_model_versions", "tasks:predictions_results", "tasks:file_upload", "tasks:storage_filename", "tasks:created_at", "tasks:updated_at", "tasks:updated_by", "tasks:avg_lead_time", "tasks:draft_exists"]}, "columnsWidth": {}, "columnsDisplayType": {"tasks:data.sample_id": "String"}, "gridWidth": 4, "gridFitImagesToWidth": false, "semantic_search": [], "agreement_selected": {}}


ordering: []
selected_items: {}
data:
  {"title": "Default", "type": "list", "target": "tasks", "hiddenColumns": {"explore": ["tasks:inner_id", "tasks:annotations_results", "tasks:annotations_ids", "tasks:predictions_score", "tasks:predictions_model_versions", "tasks:predictions_results", "tasks:file_upload", "tasks:storage_filename", "tasks:created_at", "tasks:updated_at", "tasks:updated_by", "tasks:avg_lead_time", "tasks:draft_exists"], "labeling": ["tasks:data.source_id", "tasks:data.task_domain", "tasks:data.paper_domain", "tasks:data.condition", "tasks:data.noise_env", "tasks:data.task_subtask", "tasks:data.task_difficulty", "tasks:data.scatter_turns", "tasks:data.wav_path", "tasks:data.constraints_list", "tasks:data.step", "tasks:id", "tasks:inner_id", "tasks:completed_at", "tasks:cancelled_annotations", "tasks:total_predictions", "tasks:annotators", "tasks:annotations_results", "tasks:annotations_ids", "tasks:predictions_score", "tasks:predictions_model_versions", "tasks:predictions_results", "tasks:file_upload", "tasks:storage_filename", "tasks:created_at", "tasks:updated_at", "tasks:updated_by", "tasks:avg_lead_time", "tasks:draft_exists"]}, "columnsWidth": {}, "columnsDisplayType": {}, "gridWidth": 4, "gridFitImagesToWidth": false, "semantic_search": [], "agreement_selected": {}}


ordering: []
selected_items: {}
data:
  {"title": "Default", "type": "list", "target": "tasks", "hiddenColumns": {"explore": ["tasks:inner_id", "tasks:annotations_results", "tasks:annotations_ids", "tasks:predictions_score", "tasks:predictions_model_versions", "tasks:predictions_results", "tasks:file_upload", "tasks:storage_filename", "tasks:created_at", "tasks:updated_at", "tasks:updated_by", "tasks:avg_lead_time", "tasks:draft_exists"], "labeling": ["tasks:data.source_id", "tasks:data.task_domain", "tasks:data.paper_domain", "tasks:data.condition", "tasks:data.task_subtask", "tasks:data.task_difficulty", "tasks:data.scatter_turns", "tasks:data.informality_tier_selected", "tasks:data.constraints_list", "tasks:id", "tasks:inner_id", "tasks:completed_at", "tasks:cancelled_annotations", "tasks:total_predictions", "tasks:annotators", "tasks:annotations_results", "tasks:annotations_ids", "tasks:predictions_score", "tasks:predictions_model_versions", "tasks:predictions_results", "tasks:file_upload", "tasks:storage_filename", "tasks:created_at", "tasks:updated_at", "tasks:updated_by", "tasks:avg_lead_time", "tasks:draft_exists"]}, "columnsWidth": {}, "columnsDisplayType": {}, "gridWidth": 4, "gridFitImagesToWidth": false, "semantic_search": [], "agreement_selected": {}}


# project

title: OmniAgentBench multi-turn (Stage A)
model_version: slot_visibility
created_at: 2026-05-18 12:04:21.815973
updated_at: 2026-05-23 18:01:41.362517
sampling: Sequential sampling
token: hcZBI519Pe15ft8ZNXEe1VFDmH3MRrjwR7JQbzgF
description: Stage A text. 150 canonical v3 tasks. Per-slot visibility via predictions.
color: #FFFFFF
skip_queue: REQUEUE_FOR_OTHERS
label_config:
  <!--
    Label Studio labeling interface for the OmniAgentBench Stage A human eval
    of Multi-Turn (Type 2B) Instruction.

    How to use:
      1. Create a new project in Label Studio.
      2. Open Settings -> Labeling Interface, switch to "Code" view, and paste
         this whole file.
      3. Save.
      4. Import the tasks from docs/human_eval/samples/label_studio_tasks.json
         (Settings -> Data Import -> JSON).

    Each task shows:
      - A header chip showing which benchmark / category / N the sample came
        from (ScheduleBench / flight / easy / N=2, GUIBench / Web_Shopping / N=3, ...).
      - The full multi-turn dialogue (user-only turns, with [T1] [T2] ... markers).
      - The numbered list of original constraints from the source task.
      - For each constraint position (1..N), the annotator picks YES / NO / PARTIAL.
      - A 1..5 Likert for naturalness.
      - Optional free-text notes.

    Coverage:
      The same checklist + naturalness form is reused across ScheduleBench
      (mpcc), GUIBench (gui_odyssey), and EmbodiedBench because all three
      canonical v3 source files share the same multi-turn schema (constraints +
      multi-turn text). Per-task task_domain / paper_domain is carried through
      $domain_label for downstream analysis.

    Slot count: 12 covers every sample. Max observed across the canonical
    v3 multi-turn pool is 9, on ScheduleBench.
  -->

  <View>
    <Header value="📝 Multi-turn instruction evaluation (text only)"/>

    <View style="background:#eff6ff; border-left:4px solid #2563eb; padding:12px 16px; border-radius:6px; margin:6px 0 14px 0;">
      <Text name="how_to" value="HOW TO ANNOTATE (30 sec read)&#10;&#10;1. If the task already shows someone's answers, click the + button at the top to start your OWN fresh annotation.&#10;2. The YELLOW box is your reference. It has the original clean instruction and the numbered list of constraints we expect the model to satisfy.&#10;3. The GREY box is the rewritten TEXT that the model would actually receive.&#10;4. Step 1 — count the rows in the yellow list and pick that number.&#10;5. Step 2 — for each constraint, check whether it is expressed in the GREY TEXT:&#10;   - YES — clearly expressed (any wording).&#10;   - PARTIAL — mentioned but vaguer (e.g. 'cheap' instead of 'under ＄700').&#10;   - NO — missing entirely or contradicted.&#10;6. Naturalness — rate 1 (robotic) to 5 (real person talking).&#10;7. Notes — optional, helpful for borderline cases.&#10;&#10;About 1-2 minutes per task."/>
    </View>

    <Style>
      .ls-container { font-size: 16px; line-height: 1.55; }
      .ls-dialogue { background: #f6f8fa; padding: 14px 18px; border-radius: 8px;
                     white-space: pre-wrap; font-family: ui-sans-serif, -apple-system, sans-serif; }
      .ls-constraints { background: #fff8e1; padding: 12px 18px;
                        border-left: 4px solid #f7c948; border-radius: 4px;
                        white-space: pre-wrap; font-family: ui-sans-serif, -apple-system, sans-serif; }
      .ls-howmany { background: #e8f3ff; border-left: 4px solid #2f80ed;
                    padding: 10px 14px; border-radius: 4px; font-weight: 500; }
      .ls-card-yellow { background: #fff8e1; border-left: 4px solid #f7c948;
                        border-radius: 6px; padding: 14px 18px; margin: 8px 0;
                        white-space: normal; font-family: ui-sans-serif, -apple-system, sans-serif; }
      .ls-card-yellow > * + * { margin-top: 6px; }
      .ls-card-label { color: #5b6573; font-size: 13px; font-weight: 600;
                       margin-top: 18px; }
      .ls-card-divider { border: 0; border-top: 1px dashed #e0c66b;
                         margin: 16px 0 8px 0; }
      .meta { color: #5b6573; font-size: 13px; }
      h4 { margin-top: 24px; }
    </Style>

    <View className="ls-container">

      <View style="display:flex; justify-content:space-between; align-items:baseline;">
        <Text name="id" value="$sample_id"/>
        <Text name="meta_subtask" value="$domain_label"/>
      </View>

      <View style="background:#FFF3B0; border:1px solid #E8C547; border-radius:6px; padding:14px 18px; margin:8px 0;">
        <Text name="ref_heading" value="📌 Reference (what we are checking against)" style="font-weight:700; font-size:15px; margin-bottom:8px;"/>
        <Text name="ref_title_clean" value="Original clean instruction:" style="font-weight:600; font-size:13px; color:#7a5b00; margin-top:6px;"/>
        <Text name="clean_text" value="$clean_text" style="font-family:Georgia,serif; font-size:14px; line-height:1.5; color:#1f1f1f; white-space:pre-wrap;"/>
        <Text name="ref_sep" value=" " style="display:block; margin-top:14px; padding-top:10px; border-top:1px dashed #E8C547;"/>
        <Text name="ref_title_constraints" value="Numbered constraints:" style="font-weight:600; font-size:13px; color:#7a5b00;"/>
        <Text name="constraints" value="$constraints_numbered" style="font-family:Menlo,Consolas,monospace; font-size:13px; line-height:1.6; color:#1f1f1f; white-space:pre-wrap;"/>
      </View>
      <Header value="Step 0 - Does the constraint list cover the original instruction?"/>
      <Text name="step0_help" value="Compare the YELLOW box. Do the numbered constraints capture everything in the clean instruction? This is a quick check on our data, not on the dialogue/audio."/>
      <Choices name="reference_covers" toName="dialogue" choice="single" required="true" showInline="true">
        <Choice value="Yes - complete"/>
        <Choice value="Partial - some details missing"/>
        <Choice value="No - major part missing"/>
      </Choices>
      <TextArea name="reference_missing" toName="dialogue" rows="1" maxSubmissions="1" placeholder="(optional) In one line, what was missing?"/>


      <View className="ls-howmany">
        <Text name="howmany" value="This task has $requirement_count constraints — only use slots C1 to C$requirement_count below. Leave the higher-numbered slots untouched."/>
      </View>

      <View>
        <Header value="Multi-turn dialogue"/>
        <Text name="dialogue" value="$dialogue" className="ls-dialogue"/>
      </View>

      
      <Header value="Step 1 - How many constraints does this task have?"/>
      <Text name="numhint" value="Pick the number that matches the constraint list in the yellow box (count the rows)."/>
      <Choices name="num_constraints" toName="dialogue" choice="single" required="true" showInline="true">
        <Choice value="2"/><Choice value="3"/><Choice value="4"/><Choice value="5"/><Choice value="6"/>
        <Choice value="7"/><Choice value="8"/><Choice value="9"/><Choice value="10"/><Choice value="11"/>
        <Choice value="12"/>
      </Choices>

      <Header value="Step 2 - Per-constraint judgement"/>
      <Text name="instr" value="For each constraint in the YELLOW reference box, decide whether it is expressed in the multi-turn TEXT dialogue below. YES = clearly expressed; PARTIAL = mentioned but vague; NO = missing or contradicted. Use only the first $requirement_count slots."/>

      <!-- Per-slot hidden indicators. Each is auto-selected on render to the
           value of data.show_cN ("yes" or "no") via templated <Choice>. The
           visibleWhen on each cN block below reads these indicators and only
           renders the slot when its show flag is "yes". -->
      <View style="display:none">
        <Choices name="_show_c2" toName="dialogue" choice="single"><Choice value="yes"/><Choice value="no"/></Choices>
        <Choices name="_show_c3" toName="dialogue" choice="single"><Choice value="yes"/><Choice value="no"/></Choices>
        <Choices name="_show_c4" toName="dialogue" choice="single"><Choice value="yes"/><Choice value="no"/></Choices>
        <Choices name="_show_c5" toName="dialogue" choice="single"><Choice value="yes"/><Choice value="no"/></Choices>
        <Choices name="_show_c6" toName="dialogue" choice="single"><Choice value="yes"/><Choice value="no"/></Choices>
        <Choices name="_show_c7" toName="dialogue" choice="single"><Choice value="yes"/><Choice value="no"/></Choices>
        <Choices name="_show_c8" toName="dialogue" choice="single"><Choice value="yes"/><Choice value="no"/></Choices>
        <Choices name="_show_c9" toName="dialogue" choice="single"><Choice value="yes"/><Choice value="no"/></Choices>
        <Choices name="_show_c10" toName="dialogue" choice="single"><Choice value="yes"/><Choice value="no"/></Choices>
        <Choices name="_show_c11" toName="dialogue" choice="single"><Choice value="yes"/><Choice value="no"/></Choices>
        <Choices name="_show_c12" toName="dialogue" choice="single"><Choice value="yes"/><Choice value="no"/></Choices>
      </View>

      <View style="display:grid; grid-template-columns: 1fr 1fr; gap: 8px;">
        <!-- c1 always shows; every task has at least 2 constraints. -->
        <Choices name="c1" toName="dialogue" choice="single" required="false">
          <Choice value="C1: YES"/><Choice value="C1: NO"/><Choice value="C1: PARTIAL"/>
        </Choices>
        <View visibleWhen="choice-selected" whenTagName="num_constraints" whenChoiceValue="2,3,4,5,6,7,8,9,10,11,12"><Choices name="c2" toName="dialogue" choice="single" required="false">
            <Choice value="C2: YES"/><Choice value="C2: NO"/><Choice value="C2: PARTIAL"/>
          </Choices></View>
        <View visibleWhen="choice-selected" whenTagName="num_constraints" whenChoiceValue="3,4,5,6,7,8,9,10,11,12"><Choices name="c3" toName="dialogue" choice="single" required="false">
            <Choice value="C3: YES"/><Choice value="C3: NO"/><Choice value="C3: PARTIAL"/>
          </Choices></View>
        <View visibleWhen="choice-selected" whenTagName="num_constraints" whenChoiceValue="4,5,6,7,8,9,10,11,12"><Choices name="c4" toName="dialogue" choice="single" required="false">
            <Choice value="C4: YES"/><Choice value="C4: NO"/><Choice value="C4: PARTIAL"/>
          </Choices></View>
        <View visibleWhen="choice-selected" whenTagName="num_constraints" whenChoiceValue="5,6,7,8,9,10,11,12"><Choices name="c5" toName="dialogue" choice="single" required="false">
            <Choice value="C5: YES"/><Choice value="C5: NO"/><Choice value="C5: PARTIAL"/>
          </Choices></View>
        <View visibleWhen="choice-selected" whenTagName="num_constraints" whenChoiceValue="6,7,8,9,10,11,12"><Choices name="c6" toName="dialogue" choice="single" required="false">
            <Choice value="C6: YES"/><Choice value="C6: NO"/><Choice value="C6: PARTIAL"/>
          </Choices></View>
        <View visibleWhen="choice-selected" whenTagName="num_constraints" whenChoiceValue="7,8,9,10,11,12"><Choices name="c7" toName="dialogue" choice="single" required="false">
            <Choice value="C7: YES"/><Choice value="C7: NO"/><Choice value="C7: PARTIAL"/>
          </Choices></View>
        <View visibleWhen="choice-selected" whenTagName="num_constraints" whenChoiceValue="8,9,10,11,12"><Choices name="c8" toName="dialogue" choice="single" required="false">
            <Choice value="C8: YES"/><Choice value="C8: NO"/><Choice value="C8: PARTIAL"/>
          </Choices></View>
        <View visibleWhen="choice-selected" whenTagName="num_constraints" whenChoiceValue="9,10,11,12"><Choices name="c9" toName="dialogue" choice="single" required="false">
            <Choice value="C9: YES"/><Choice value="C9: NO"/><Choice value="C9: PARTIAL"/>
          </Choices></View>
        <View visibleWhen="choice-selected" whenTagName="num_constraints" whenChoiceValue="10,11,12"><Choices name="c10" toName="dialogue" choice="single" required="false">
            <Choice value="C10: YES"/><Choice value="C10: NO"/><Choice value="C10: PARTIAL"/>
          </Choices></View>
        <View visibleWhen="choice-selected" whenTagName="num_constraints" whenChoiceValue="11,12"><Choices name="c11" toName="dialogue" choice="single" required="false">
            <Choice value="C11: YES"/><Choice value="C11: NO"/><Choice value="C11: PARTIAL"/>
          </Choices></View>
        <View visibleWhen="choice-selected" whenTagName="num_constraints" whenChoiceValue="12"><Choices name="c12" toName="dialogue" choice="single" required="false">
            <Choice value="C12: YES"/><Choice value="C12: NO"/><Choice value="C12: PARTIAL"/>
          </Choices></View>
      </View>

      <Header value="Naturalness"/>
      <Text name="instr_naturalness" value="Does this dialogue read like something a real person would actually say? 1 = sounds completely artificial. 5 = sounds like a normal conversation."/>
      <Rating name="naturalness" toName="dialogue" maxRating="5" required="true"/>

      <Header value="Notes (optional)"/>
      <TextArea name="notes" toName="dialogue" rows="2" maxSubmissions="1" placeholder="Anything odd about this dialogue?"/>

    </View>
  </View>

expert_instruction:
  ## What you're doing

  You'll see 15 items. Each shows the original list of **constraints** (yellow) and a **paraphrased multi-turn dialogue** that should contain all of them (grey).

  For each constraint, click **YES** (clearly expressed), **PARTIAL** (mentioned but weaker), or **NO** (missing). Then rate **naturalness** 1–5. The blue callout on each task tells you how many slots to use.

  ## Decision rules

  - **YES**: the dialogue clearly states the constraint, any wording, any turn. Reordering is fine.
  - **PARTIAL**: weaker than the original — e.g. "cheap" instead of "under ＄700", "early March" instead of "March 5".
  - **NO**: missing or contradicted.
  - When unsure between YES and PARTIAL → pick **PARTIAL**.

  ## Don't

  - Use ChatGPT/AI to answer. We have attention checks.
  - Worry about the dialogue order — only that all constraints appear somewhere.
  - Penalise polite filler, hedges, or extra small talk.

  ## Time

  1–2 minutes per item, ~25 minutes total. Save and resume any time.

data_types:
  {"sample_id": "Text", "domain_label": "Text", "clean_text": "Text", "constraints_numbered": "Text", "requirement_count": "Text", "dialogue": "Text"}

control_weights:
  {"reference_covers": {"overall": 1.0, "type": "Choices", "labels": {"Yes - complete": 1.0, "Partial - some details missing": 1.0, "No - major part missing": 1.0}}, "reference_missing": {"overall": 1.0, "type": "TextArea", "labels": {}}, "num_constraints": {"overall": 1.0, "type": "Choices", "labels": {"2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 1.0, "11": 1.0, "12": 1.0}}, "_show_c2": {"overall": 1.0, "type": "Choices", "labels": {"yes": 1.0, "no": 1.0}}, "_show_c3": {"overall": 1.0, "type": "Choices", "labels": {"yes": 1.0, "no": 1.0}}, "_show_c4": {"overall": 1.0, "type": "Choices", "labels": {"yes": 1.0, "no": 1.0}}, "_show_c5": {"overall": 1.0, "type": "Choices", "labels": {"yes": 1.0, "no": 1.0}}, "_show_c6": {"overall": 1.0, "type": "Choices", "labels": {"yes": 1.0, "no": 1.0}}, "_show_c7": {"overall": 1.0, "type": "Choices", "labels": {"yes": 1.0, "no": 1.0}}, "_show_c8": {"overall": 1.0, "type": "Choices", "labels": {"yes": 1.0, "no": 1.0}}, "_show_c9": {"overall": 1.0, "type": "Choices", "labels": {"yes": 1.0, "no": 1.0}}, "_show_c10": {"overall": 1.0, "type": "Choices", "labels": {"yes": 1.0, "no": 1.0}}, "_show_c11": {"overall": 1.0, "type": "Choices", "labels": {"yes": 1.0, "no": 1.0}}, "_show_c12": {"overall": 1.0, "type": "Choices", "labels": {"yes": 1.0, "no": 1.0}}, "c1": {"overall": 1.0, "type": "Choices", "labels": {"C1: YES": 1.0, "C1: NO": 1.0, "C1: PARTIAL": 1.0}}, "c2": {"overall": 1.0, "type": "Choices", "labels": {"C2: YES": 1.0, "C2: NO": 1.0, "C2: PARTIAL": 1.0}}, "c3": {"overall": 1.0, "type": "Choices", "labels": {"C3: YES": 1.0, "C3: NO": 1.0, "C3: PARTIAL": 1.0}}, "c4": {"overall": 1.0, "type": "Choices", "labels": {"C4: YES": 1.0, "C4: NO": 1.0, "C4: PARTIAL": 1.0}}, "c5": {"overall": 1.0, "type": "Choices", "labels": {"C5: YES": 1.0, "C5: NO": 1.0, "C5: PARTIAL": 1.0}}, "c6": {"overall": 1.0, "type": "Choices", "labels": {"C6: YES": 1.0, "C6: NO": 1.0, "C6: PARTIAL": 1.0}}, "c7": {"overall": 1.0, "type": "Choices", "labels": {"C7: YES": 1.0, "C7: NO": 1.0, "C7: PARTIAL": 1.0}}, "c8": {"overall": 1.0, "type": "Choices", "labels": {"C8: YES": 1.0, "C8: NO": 1.0, "C8: PARTIAL": 1.0}}, "c9": {"overall": 1.0, "type": "Choices", "labels": {"C9: YES": 1.0, "C9: NO": 1.0, "C9: PARTIAL": 1.0}}, "c10": {"overall": 1.0, "type": "Choices", "labels": {"C10: YES": 1.0, "C10: NO": 1.0, "C10: PARTIAL": 1.0}}, "c11": {"overall": 1.0, "type": "Choices", "labels": {"C11: YES": 1.0, "C11: NO": 1.0, "C11: PARTIAL": 1.0}}, "c12": {"overall": 1.0, "type": "Choices", "labels": {"C12: YES": 1.0, "C12: NO": 1.0, "C12: PARTIAL": 1.0}}, "naturalness": {"overall": 1.0, "type": "Rating", "labels": {}}, "notes": {"overall": 1.0, "type": "TextArea", "labels": {}}}

parsed_label_config:
  {"reference_covers": {"type": "Choices", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": ["Yes - complete", "Partial - some details missing", "No - major part missing"], "labels_attrs": {"Yes - complete": {"value": "Yes - complete"}, "Partial - some details missing": {"value": "Partial - some details missing"}, "No - major part missing": {"value": "No - major part missing"}}}, "reference_missing": {"type": "TextArea", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": [], "labels_attrs": {}}, "num_constraints": {"type": "Choices", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": ["2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"], "labels_attrs": {"2": {"value": "2"}, "3": {"value": "3"}, "4": {"value": "4"}, "5": {"value": "5"}, "6": {"value": "6"}, "7": {"value": "7"}, "8": {"value": "8"}, "9": {"value": "9"}, "10": {"value": "10"}, "11": {"value": "11"}, "12": {"value": "12"}}}, "_show_c2": {"type": "Choices", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": ["yes", "no"], "labels_attrs": {"yes": {"value": "yes"}, "no": {"value": "no"}}}, "_show_c3": {"type": "Choices", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": ["yes", "no"], "labels_attrs": {"yes": {"value": "yes"}, "no": {"value": "no"}}}, "_show_c4": {"type": "Choices", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": ["yes", "no"], "labels_attrs": {"yes": {"value": "yes"}, "no": {"value": "no"}}}, "_show_c5": {"type": "Choices", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": ["yes", "no"], "labels_attrs": {"yes": {"value": "yes"}, "no": {"value": "no"}}}, "_show_c6": {"type": "Choices", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": ["yes", "no"], "labels_attrs": {"yes": {"value": "yes"}, "no": {"value": "no"}}}, "_show_c7": {"type": "Choices", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": ["yes", "no"], "labels_attrs": {"yes": {"value": "yes"}, "no": {"value": "no"}}}, "_show_c8": {"type": "Choices", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": ["yes", "no"], "labels_attrs": {"yes": {"value": "yes"}, "no": {"value": "no"}}}, "_show_c9": {"type": "Choices", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": ["yes", "no"], "labels_attrs": {"yes": {"value": "yes"}, "no": {"value": "no"}}}, "_show_c10": {"type": "Choices", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": ["yes", "no"], "labels_attrs": {"yes": {"value": "yes"}, "no": {"value": "no"}}}, "_show_c11": {"type": "Choices", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": ["yes", "no"], "labels_attrs": {"yes": {"value": "yes"}, "no": {"value": "no"}}}, "_show_c12": {"type": "Choices", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": ["yes", "no"], "labels_attrs": {"yes": {"value": "yes"}, "no": {"value": "no"}}}, "c1": {"type": "Choices", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": ["C1: YES", "C1: NO", "C1: PARTIAL"], "labels_attrs": {"C1: YES": {"value": "C1: YES"}, "C1: NO": {"value": "C1: NO"}, "C1: PARTIAL": {"value": "C1: PARTIAL"}}}, "c2": {"type": "Choices", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": ["C2: YES", "C2: NO", "C2: PARTIAL"], "labels_attrs": {"C2: YES": {"value": "C2: YES"}, "C2: NO": {"value": "C2: NO"}, "C2: PARTIAL": {"value": "C2: PARTIAL"}}}, "c3": {"type": "Choices", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": ["C3: YES", "C3: NO", "C3: PARTIAL"], "labels_attrs": {"C3: YES": {"value": "C3: YES"}, "C3: NO": {"value": "C3: NO"}, "C3: PARTIAL": {"value": "C3: PARTIAL"}}}, "c4": {"type": "Choices", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": ["C4: YES", "C4: NO", "C4: PARTIAL"], "labels_attrs": {"C4: YES": {"value": "C4: YES"}, "C4: NO": {"value": "C4: NO"}, "C4: PARTIAL": {"value": "C4: PARTIAL"}}}, "c5": {"type": "Choices", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": ["C5: YES", "C5: NO", "C5: PARTIAL"], "labels_attrs": {"C5: YES": {"value": "C5: YES"}, "C5: NO": {"value": "C5: NO"}, "C5: PARTIAL": {"value": "C5: PARTIAL"}}}, "c6": {"type": "Choices", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": ["C6: YES", "C6: NO", "C6: PARTIAL"], "labels_attrs": {"C6: YES": {"value": "C6: YES"}, "C6: NO": {"value": "C6: NO"}, "C6: PARTIAL": {"value": "C6: PARTIAL"}}}, "c7": {"type": "Choices", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": ["C7: YES", "C7: NO", "C7: PARTIAL"], "labels_attrs": {"C7: YES": {"value": "C7: YES"}, "C7: NO": {"value": "C7: NO"}, "C7: PARTIAL": {"value": "C7: PARTIAL"}}}, "c8": {"type": "Choices", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": ["C8: YES", "C8: NO", "C8: PARTIAL"], "labels_attrs": {"C8: YES": {"value": "C8: YES"}, "C8: NO": {"value": "C8: NO"}, "C8: PARTIAL": {"value": "C8: PARTIAL"}}}, "c9": {"type": "Choices", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": ["C9: YES", "C9: NO", "C9: PARTIAL"], "labels_attrs": {"C9: YES": {"value": "C9: YES"}, "C9: NO": {"value": "C9: NO"}, "C9: PARTIAL": {"value": "C9: PARTIAL"}}}, "c10": {"type": "Choices", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": ["C10: YES", "C10: NO", "C10: PARTIAL"], "labels_attrs": {"C10: YES": {"value": "C10: YES"}, "C10: NO": {"value": "C10: NO"}, "C10: PARTIAL": {"value": "C10: PARTIAL"}}}, "c11": {"type": "Choices", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": ["C11: YES", "C11: NO", "C11: PARTIAL"], "labels_attrs": {"C11: YES": {"value": "C11: YES"}, "C11: NO": {"value": "C11: NO"}, "C11: PARTIAL": {"value": "C11: PARTIAL"}}}, "c12": {"type": "Choices", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": ["C12: YES", "C12: NO", "C12: PARTIAL"], "labels_attrs": {"C12: YES": {"value": "C12: YES"}, "C12: NO": {"value": "C12: NO"}, "C12: PARTIAL": {"value": "C12: PARTIAL"}}}, "naturalness": {"type": "Rating", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": [], "labels_attrs": {}}, "notes": {"type": "TextArea", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": [], "labels_attrs": {}}}


title: OmniAgentBench multi-turn AUDIO (Stage B)
model_version: slot_visibility
created_at: 2026-05-18 12:04:23.479291
updated_at: 2026-05-25 23:11:42.475302
sampling: Sequential sampling
token: IveBUF38rOY6iNTP9gzDlRViSXdc3bsPiAzCoyTy
description: Stage B paired audio+text. 150 N=3 tasks. Per-slot visibility via predictions.
color: #FFFFFF
skip_queue: REQUEUE_FOR_OTHERS
label_config:
  <View>
    <Header value="Multi-turn audio evaluation"/>

    <View style="background:#eff6ff; border-left:4px solid #2563eb; padding:12px 16px; border-radius:6px; margin:6px 0 14px 0;">
      <Text name="how_to" value="HOW TO ANNOTATE (1.5 minutes)&#10;&#10;1. If the task already shows someone's answers, click the + button at the top to start your OWN fresh annotation.&#10;2. The YELLOW box is your reference (clean instruction + numbered constraints).&#10;3. Step 0 - quick check that the numbered constraints match the clean instruction.&#10;4. Listen to the multi-turn audio (concatenated clip).&#10;5. Did you make out the FULL instruction across the turns? Yes / Partial / No.&#10;6. Audio clarity - 1 (hard to understand) to 5 (very clear)."/>
    </View>

    <View>
      <View style="display:flex; justify-content:space-between; align-items:baseline;">
        <Text name="id" value="$sample_id"/>
        <Text name="meta_subtask" value="$domain_label"/>
      </View>

      <View style="background:#FFF3B0; border:1px solid #E8C547; border-radius:6px; padding:14px 18px; margin:8px 0;">
        <Text name="ref_heading" value="Reference (what we are checking against)" style="font-weight:700; font-size:11px; margin-bottom:8px;"/>
        <Text name="ref_title_clean" value="Original clean instruction:" style="font-weight:600; font-size:11px; color:#7a5b00; margin-top:6px;"/>
        <Text name="clean_text" value="$clean_text" style="font-family:Georgia,serif; font-size:12px; line-height:1.5; color:#1f1f1f; white-space:pre-wrap;"/>
        <Text name="ref_sep" value=" " style="display:block; margin-top:14px; padding-top:10px; border-top:1px dashed #E8C547;"/>
        <Text name="ref_title_constraints" value="Numbered constraints:" style="font-weight:600; font-size:11px; color:#7a5b00;"/>
        <Text name="constraints" value="$constraints_numbered" style="font-family:Menlo,Consolas,monospace; font-size:11px; line-height:1.6; color:#1f1f1f; white-space:pre-wrap;"/>
      </View>

      <Header value="Step 0 - Does the constraint list cover the original instruction?"/>
      <Text name="step0_help" value="Compare the YELLOW box. Do the numbered constraints capture everything in the clean instruction? This is a quick check on our data, not on the audio."/>
      <Choices name="reference_covers" toName="aud_concat" choice="single" required="true" showInline="true">
        <Choice value="Yes - complete"/>
        <Choice value="Partial - some details missing"/>
        <Choice value="No - major part missing"/>
      </Choices>
      <TextArea name="reference_missing" toName="aud_concat" rows="1" maxSubmissions="1" placeholder="(optional) In one line, what was missing?"/>

      <Header value="Multi-turn audio (concatenated)"/>
      <Audio name="aud_concat" value="$audio_concat"/>

      <Header value="Could you make out the full instruction across the turns?"/>
      <Text name="instr_makeout" value="Did you hear all the constraints listed in the YELLOW box across the audio turns?"/>
      <Choices name="makeout" toName="aud_concat" choice="single" required="true" showInline="true">
        <Choice value="Yes - heard all"/>
        <Choice value="Partial - some constraints muffled or unclear"/>
        <Choice value="No - lost too much"/>
      </Choices>

      <Header value="Audio clarity"/>
      <Text name="instr_clarity" value="Could you understand the audio? 1 = hard to understand. 5 = very clear."/>
      <Rating name="audio_clarity" toName="aud_concat" maxRating="5" required="true"/>
      
      <View style="display:none">
        <Audio name="aud_t1" value="$audio_t1"/>
        <Audio name="aud_t2" value="$audio_t2"/>
        <Audio name="aud_t3" value="$audio_t3"/>
        <Choices name="num_constraints" toName="aud_concat" choice="single"><Choice value="2"/><Choice value="3"/><Choice value="4"/><Choice value="5"/><Choice value="6"/><Choice value="7"/><Choice value="8"/><Choice value="9"/><Choice value="10"/><Choice value="11"/><Choice value="12"/></Choices>
        <Choices name="_show_c2"  toName="aud_concat" choice="single"><Choice value="yes"/><Choice value="no"/></Choices>
        <Choices name="_show_c3"  toName="aud_concat" choice="single"><Choice value="yes"/><Choice value="no"/></Choices>
        <Choices name="_show_c4"  toName="aud_concat" choice="single"><Choice value="yes"/><Choice value="no"/></Choices>
        <Choices name="_show_c5"  toName="aud_concat" choice="single"><Choice value="yes"/><Choice value="no"/></Choices>
        <Choices name="_show_c6"  toName="aud_concat" choice="single"><Choice value="yes"/><Choice value="no"/></Choices>
        <Choices name="_show_c7"  toName="aud_concat" choice="single"><Choice value="yes"/><Choice value="no"/></Choices>
        <Choices name="_show_c8"  toName="aud_concat" choice="single"><Choice value="yes"/><Choice value="no"/></Choices>
        <Choices name="_show_c9"  toName="aud_concat" choice="single"><Choice value="yes"/><Choice value="no"/></Choices>
        <Choices name="_show_c10" toName="aud_concat" choice="single"><Choice value="yes"/><Choice value="no"/></Choices>
        <Choices name="_show_c11" toName="aud_concat" choice="single"><Choice value="yes"/><Choice value="no"/></Choices>
        <Choices name="_show_c12" toName="aud_concat" choice="single"><Choice value="yes"/><Choice value="no"/></Choices>
        <Choices name="c1"  toName="aud_concat" choice="single"><Choice value="C1: YES"/><Choice value="C1: NO"/><Choice value="C1: PARTIAL"/></Choices>
        <Choices name="c2"  toName="aud_concat" choice="single"><Choice value="C2: YES"/><Choice value="C2: NO"/><Choice value="C2: PARTIAL"/></Choices>
        <Choices name="c3"  toName="aud_concat" choice="single"><Choice value="C3: YES"/><Choice value="C3: NO"/><Choice value="C3: PARTIAL"/></Choices>
        <Choices name="c4"  toName="aud_concat" choice="single"><Choice value="C4: YES"/><Choice value="C4: NO"/><Choice value="C4: PARTIAL"/></Choices>
        <Choices name="c5"  toName="aud_concat" choice="single"><Choice value="C5: YES"/><Choice value="C5: NO"/><Choice value="C5: PARTIAL"/></Choices>
        <Choices name="c6"  toName="aud_concat" choice="single"><Choice value="C6: YES"/><Choice value="C6: NO"/><Choice value="C6: PARTIAL"/></Choices>
        <Choices name="c7"  toName="aud_concat" choice="single"><Choice value="C7: YES"/><Choice value="C7: NO"/><Choice value="C7: PARTIAL"/></Choices>
        <Choices name="c8"  toName="aud_concat" choice="single"><Choice value="C8: YES"/><Choice value="C8: NO"/><Choice value="C8: PARTIAL"/></Choices>
        <Choices name="c9"  toName="aud_concat" choice="single"><Choice value="C9: YES"/><Choice value="C9: NO"/><Choice value="C9: PARTIAL"/></Choices>
        <Choices name="c10" toName="aud_concat" choice="single"><Choice value="C10: YES"/><Choice value="C10: NO"/><Choice value="C10: PARTIAL"/></Choices>
        <Choices name="c11" toName="aud_concat" choice="single"><Choice value="C11: YES"/><Choice value="C11: NO"/><Choice value="C11: PARTIAL"/></Choices>
        <Choices name="c12" toName="aud_concat" choice="single"><Choice value="C12: YES"/><Choice value="C12: NO"/><Choice value="C12: PARTIAL"/></Choices>
        <Choices name="match_t1" toName="aud_t1" choice="single"><Choice value="match"/><Choice value="minor"/><Choice value="major"/></Choices>
        <Choices name="match_t2" toName="aud_t2" choice="single"><Choice value="match"/><Choice value="minor"/><Choice value="major"/></Choices>
        <Choices name="match_t3" toName="aud_t3" choice="single"><Choice value="match"/><Choice value="minor"/><Choice value="major"/></Choices>
        <Rating name="naturalness" toName="aud_concat" maxRating="5"/>
        <TextArea name="notes" toName="aud_concat" rows="2" maxSubmissions="1"/>
      </View>

    </View>
  </View>

expert_instruction:
  ## What you're doing

  You'll see 15 items. Each shows the **original constraints** (yellow), then per-turn **audio + text side-by-side** (T1, T2, T3), then a **concatenated clip** for reference.

  ## What to fill (5 things per item)

  1. **Per-constraint YES / PARTIAL / NO** — judged from what you HEAR. Use only as many slots as the blue callout indicates.
  2. **Per-turn MATCH / MINOR DRIFT / MAJOR MISMATCH** — does the audio say the same thing as the text next to it?
  3. **Audio clarity 1–5** — can you understand the audio without effort?
  4. **Naturalness 1–5** — does it sound like a real person?
  5. **Notes** — strongly recommended whenever you mark DRIFT or MISMATCH; one line on what you heard vs the text.

  ## Listening tips

  - Listen to each turn at least once before scoring; replay 1–2 times max.
  - Use headphones if you have them — laptop speakers smear TTS artefacts.
  - The CONCATENATED clip is for overall flow; the PER-TURN clips are what you score.

  ## Don't

  - Use ChatGPT to listen for you.
  - Penalise correct audio for being slightly TTS-flavoured — only mark DRIFT if a word or clause is actually wrong.

  ## Time

  2–3 minutes per item, ~25–30 minutes total.

data_types:
  {"sample_id": "Text", "domain_label": "Text", "clean_text": "Text", "constraints_numbered": "Text", "audio_concat": "Audio", "audio_t1": "Audio", "audio_t2": "Audio", "audio_t3": "Audio"}

control_weights:
  {"reference_covers": {"overall": 1.0, "type": "Choices", "labels": {"Yes - complete": 1.0, "Partial - some details missing": 1.0, "No - major part missing": 1.0}}, "reference_missing": {"overall": 1.0, "type": "TextArea", "labels": {}}, "makeout": {"overall": 1.0, "type": "Choices", "labels": {"Yes - heard all": 1.0, "Partial - some constraints muffled or unclear": 1.0, "No - lost too much": 1.0}}, "audio_clarity": {"overall": 1.0, "type": "Rating", "labels": {}}, "num_constraints": {"overall": 1.0, "type": "Choices", "labels": {"2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 1.0, "11": 1.0, "12": 1.0}}, "_show_c2": {"overall": 1.0, "type": "Choices", "labels": {"yes": 1.0, "no": 1.0}}, "_show_c3": {"overall": 1.0, "type": "Choices", "labels": {"yes": 1.0, "no": 1.0}}, "_show_c4": {"overall": 1.0, "type": "Choices", "labels": {"yes": 1.0, "no": 1.0}}, "_show_c5": {"overall": 1.0, "type": "Choices", "labels": {"yes": 1.0, "no": 1.0}}, "_show_c6": {"overall": 1.0, "type": "Choices", "labels": {"yes": 1.0, "no": 1.0}}, "_show_c7": {"overall": 1.0, "type": "Choices", "labels": {"yes": 1.0, "no": 1.0}}, "_show_c8": {"overall": 1.0, "type": "Choices", "labels": {"yes": 1.0, "no": 1.0}}, "_show_c9": {"overall": 1.0, "type": "Choices", "labels": {"yes": 1.0, "no": 1.0}}, "_show_c10": {"overall": 1.0, "type": "Choices", "labels": {"yes": 1.0, "no": 1.0}}, "_show_c11": {"overall": 1.0, "type": "Choices", "labels": {"yes": 1.0, "no": 1.0}}, "_show_c12": {"overall": 1.0, "type": "Choices", "labels": {"yes": 1.0, "no": 1.0}}, "c1": {"overall": 1.0, "type": "Choices", "labels": {"C1: YES": 1.0, "C1: NO": 1.0, "C1: PARTIAL": 1.0}}, "c2": {"overall": 1.0, "type": "Choices", "labels": {"C2: YES": 1.0, "C2: NO": 1.0, "C2: PARTIAL": 1.0}}, "c3": {"overall": 1.0, "type": "Choices", "labels": {"C3: YES": 1.0, "C3: NO": 1.0, "C3: PARTIAL": 1.0}}, "c4": {"overall": 1.0, "type": "Choices", "labels": {"C4: YES": 1.0, "C4: NO": 1.0, "C4: PARTIAL": 1.0}}, "c5": {"overall": 1.0, "type": "Choices", "labels": {"C5: YES": 1.0, "C5: NO": 1.0, "C5: PARTIAL": 1.0}}, "c6": {"overall": 1.0, "type": "Choices", "labels": {"C6: YES": 1.0, "C6: NO": 1.0, "C6: PARTIAL": 1.0}}, "c7": {"overall": 1.0, "type": "Choices", "labels": {"C7: YES": 1.0, "C7: NO": 1.0, "C7: PARTIAL": 1.0}}, "c8": {"overall": 1.0, "type": "Choices", "labels": {"C8: YES": 1.0, "C8: NO": 1.0, "C8: PARTIAL": 1.0}}, "c9": {"overall": 1.0, "type": "Choices", "labels": {"C9: YES": 1.0, "C9: NO": 1.0, "C9: PARTIAL": 1.0}}, "c10": {"overall": 1.0, "type": "Choices", "labels": {"C10: YES": 1.0, "C10: NO": 1.0, "C10: PARTIAL": 1.0}}, "c11": {"overall": 1.0, "type": "Choices", "labels": {"C11: YES": 1.0, "C11: NO": 1.0, "C11: PARTIAL": 1.0}}, "c12": {"overall": 1.0, "type": "Choices", "labels": {"C12: YES": 1.0, "C12: NO": 1.0, "C12: PARTIAL": 1.0}}, "match_t1": {"overall": 1.0, "type": "Choices", "labels": {"match": 1.0, "minor": 1.0, "major": 1.0}}, "match_t2": {"overall": 1.0, "type": "Choices", "labels": {"match": 1.0, "minor": 1.0, "major": 1.0}}, "match_t3": {"overall": 1.0, "type": "Choices", "labels": {"match": 1.0, "minor": 1.0, "major": 1.0}}, "naturalness": {"overall": 1.0, "type": "Rating", "labels": {}}, "notes": {"overall": 1.0, "type": "TextArea", "labels": {}}}

parsed_label_config:
  {"reference_covers": {"type": "Choices", "to_name": ["aud_concat"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_concat"}], "labels": ["Yes - complete", "Partial - some details missing", "No - major part missing"], "labels_attrs": {"Yes - complete": {"value": "Yes - complete"}, "Partial - some details missing": {"value": "Partial - some details missing"}, "No - major part missing": {"value": "No - major part missing"}}}, "reference_missing": {"type": "TextArea", "to_name": ["aud_concat"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_concat"}], "labels": [], "labels_attrs": {}}, "makeout": {"type": "Choices", "to_name": ["aud_concat"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_concat"}], "labels": ["Yes - heard all", "Partial - some constraints muffled or unclear", "No - lost too much"], "labels_attrs": {"Yes - heard all": {"value": "Yes - heard all"}, "Partial - some constraints muffled or unclear": {"value": "Partial - some constraints muffled or unclear"}, "No - lost too much": {"value": "No - lost too much"}}}, "audio_clarity": {"type": "Rating", "to_name": ["aud_concat"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_concat"}], "labels": [], "labels_attrs": {}}, "num_constraints": {"type": "Choices", "to_name": ["aud_concat"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_concat"}], "labels": ["2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"], "labels_attrs": {"2": {"value": "2"}, "3": {"value": "3"}, "4": {"value": "4"}, "5": {"value": "5"}, "6": {"value": "6"}, "7": {"value": "7"}, "8": {"value": "8"}, "9": {"value": "9"}, "10": {"value": "10"}, "11": {"value": "11"}, "12": {"value": "12"}}}, "_show_c2": {"type": "Choices", "to_name": ["aud_concat"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_concat"}], "labels": ["yes", "no"], "labels_attrs": {"yes": {"value": "yes"}, "no": {"value": "no"}}}, "_show_c3": {"type": "Choices", "to_name": ["aud_concat"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_concat"}], "labels": ["yes", "no"], "labels_attrs": {"yes": {"value": "yes"}, "no": {"value": "no"}}}, "_show_c4": {"type": "Choices", "to_name": ["aud_concat"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_concat"}], "labels": ["yes", "no"], "labels_attrs": {"yes": {"value": "yes"}, "no": {"value": "no"}}}, "_show_c5": {"type": "Choices", "to_name": ["aud_concat"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_concat"}], "labels": ["yes", "no"], "labels_attrs": {"yes": {"value": "yes"}, "no": {"value": "no"}}}, "_show_c6": {"type": "Choices", "to_name": ["aud_concat"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_concat"}], "labels": ["yes", "no"], "labels_attrs": {"yes": {"value": "yes"}, "no": {"value": "no"}}}, "_show_c7": {"type": "Choices", "to_name": ["aud_concat"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_concat"}], "labels": ["yes", "no"], "labels_attrs": {"yes": {"value": "yes"}, "no": {"value": "no"}}}, "_show_c8": {"type": "Choices", "to_name": ["aud_concat"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_concat"}], "labels": ["yes", "no"], "labels_attrs": {"yes": {"value": "yes"}, "no": {"value": "no"}}}, "_show_c9": {"type": "Choices", "to_name": ["aud_concat"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_concat"}], "labels": ["yes", "no"], "labels_attrs": {"yes": {"value": "yes"}, "no": {"value": "no"}}}, "_show_c10": {"type": "Choices", "to_name": ["aud_concat"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_concat"}], "labels": ["yes", "no"], "labels_attrs": {"yes": {"value": "yes"}, "no": {"value": "no"}}}, "_show_c11": {"type": "Choices", "to_name": ["aud_concat"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_concat"}], "labels": ["yes", "no"], "labels_attrs": {"yes": {"value": "yes"}, "no": {"value": "no"}}}, "_show_c12": {"type": "Choices", "to_name": ["aud_concat"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_concat"}], "labels": ["yes", "no"], "labels_attrs": {"yes": {"value": "yes"}, "no": {"value": "no"}}}, "c1": {"type": "Choices", "to_name": ["aud_concat"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_concat"}], "labels": ["C1: YES", "C1: NO", "C1: PARTIAL"], "labels_attrs": {"C1: YES": {"value": "C1: YES"}, "C1: NO": {"value": "C1: NO"}, "C1: PARTIAL": {"value": "C1: PARTIAL"}}}, "c2": {"type": "Choices", "to_name": ["aud_concat"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_concat"}], "labels": ["C2: YES", "C2: NO", "C2: PARTIAL"], "labels_attrs": {"C2: YES": {"value": "C2: YES"}, "C2: NO": {"value": "C2: NO"}, "C2: PARTIAL": {"value": "C2: PARTIAL"}}}, "c3": {"type": "Choices", "to_name": ["aud_concat"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_concat"}], "labels": ["C3: YES", "C3: NO", "C3: PARTIAL"], "labels_attrs": {"C3: YES": {"value": "C3: YES"}, "C3: NO": {"value": "C3: NO"}, "C3: PARTIAL": {"value": "C3: PARTIAL"}}}, "c4": {"type": "Choices", "to_name": ["aud_concat"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_concat"}], "labels": ["C4: YES", "C4: NO", "C4: PARTIAL"], "labels_attrs": {"C4: YES": {"value": "C4: YES"}, "C4: NO": {"value": "C4: NO"}, "C4: PARTIAL": {"value": "C4: PARTIAL"}}}, "c5": {"type": "Choices", "to_name": ["aud_concat"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_concat"}], "labels": ["C5: YES", "C5: NO", "C5: PARTIAL"], "labels_attrs": {"C5: YES": {"value": "C5: YES"}, "C5: NO": {"value": "C5: NO"}, "C5: PARTIAL": {"value": "C5: PARTIAL"}}}, "c6": {"type": "Choices", "to_name": ["aud_concat"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_concat"}], "labels": ["C6: YES", "C6: NO", "C6: PARTIAL"], "labels_attrs": {"C6: YES": {"value": "C6: YES"}, "C6: NO": {"value": "C6: NO"}, "C6: PARTIAL": {"value": "C6: PARTIAL"}}}, "c7": {"type": "Choices", "to_name": ["aud_concat"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_concat"}], "labels": ["C7: YES", "C7: NO", "C7: PARTIAL"], "labels_attrs": {"C7: YES": {"value": "C7: YES"}, "C7: NO": {"value": "C7: NO"}, "C7: PARTIAL": {"value": "C7: PARTIAL"}}}, "c8": {"type": "Choices", "to_name": ["aud_concat"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_concat"}], "labels": ["C8: YES", "C8: NO", "C8: PARTIAL"], "labels_attrs": {"C8: YES": {"value": "C8: YES"}, "C8: NO": {"value": "C8: NO"}, "C8: PARTIAL": {"value": "C8: PARTIAL"}}}, "c9": {"type": "Choices", "to_name": ["aud_concat"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_concat"}], "labels": ["C9: YES", "C9: NO", "C9: PARTIAL"], "labels_attrs": {"C9: YES": {"value": "C9: YES"}, "C9: NO": {"value": "C9: NO"}, "C9: PARTIAL": {"value": "C9: PARTIAL"}}}, "c10": {"type": "Choices", "to_name": ["aud_concat"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_concat"}], "labels": ["C10: YES", "C10: NO", "C10: PARTIAL"], "labels_attrs": {"C10: YES": {"value": "C10: YES"}, "C10: NO": {"value": "C10: NO"}, "C10: PARTIAL": {"value": "C10: PARTIAL"}}}, "c11": {"type": "Choices", "to_name": ["aud_concat"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_concat"}], "labels": ["C11: YES", "C11: NO", "C11: PARTIAL"], "labels_attrs": {"C11: YES": {"value": "C11: YES"}, "C11: NO": {"value": "C11: NO"}, "C11: PARTIAL": {"value": "C11: PARTIAL"}}}, "c12": {"type": "Choices", "to_name": ["aud_concat"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_concat"}], "labels": ["C12: YES", "C12: NO", "C12: PARTIAL"], "labels_attrs": {"C12: YES": {"value": "C12: YES"}, "C12: NO": {"value": "C12: NO"}, "C12: PARTIAL": {"value": "C12: PARTIAL"}}}, "match_t1": {"type": "Choices", "to_name": ["aud_t1"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_t1"}], "labels": ["match", "minor", "major"], "labels_attrs": {"match": {"value": "match"}, "minor": {"value": "minor"}, "major": {"value": "major"}}}, "match_t2": {"type": "Choices", "to_name": ["aud_t2"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_t2"}], "labels": ["match", "minor", "major"], "labels_attrs": {"match": {"value": "match"}, "minor": {"value": "minor"}, "major": {"value": "major"}}}, "match_t3": {"type": "Choices", "to_name": ["aud_t3"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_t3"}], "labels": ["match", "minor", "major"], "labels_attrs": {"match": {"value": "match"}, "minor": {"value": "minor"}, "major": {"value": "major"}}}, "naturalness": {"type": "Rating", "to_name": ["aud_concat"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_concat"}], "labels": [], "labels_attrs": {}}, "notes": {"type": "TextArea", "to_name": ["aud_concat"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_concat"}], "labels": [], "labels_attrs": {}}}


title: OmniAgentBench Informal text (Stage C)
model_version: slot_visibility
created_at: 2026-05-19 20:14:01.074016
updated_at: 2026-05-23 18:01:42.526839
sampling: Sequential sampling
token: 7x1hntwQ5TJc2PDvwZYEk9lvOJumvzIY86iv0d5T
color: #FFFFFF
skip_queue: REQUEUE_FOR_OTHERS
label_config:
  <!--
    Label Studio labeling interface for the OmniAgentBench Stage A human eval
    of Multi-Turn (Type 2B) Instruction.

    How to use:
      1. Create a new project in Label Studio.
      2. Open Settings -> Labeling Interface, switch to "Code" view, and paste
         this whole file.
      3. Save.
      4. Import the tasks from docs/human_eval/samples/label_studio_tasks.json
         (Settings -> Data Import -> JSON).

    Each task shows:
      - A header chip showing which benchmark / category / N the sample came
        from (ScheduleBench / flight / easy / N=2, GUIBench / Web_Shopping / N=3, ...).
      - The full multi-turn dialogue (user-only turns, with [T1] [T2] ... markers).
      - The numbered list of original constraints from the source task.
      - For each constraint position (1..N), the annotator picks YES / NO / PARTIAL.
      - A 1..5 Likert for naturalness.
      - Optional free-text notes.

    Coverage:
      The same checklist + naturalness form is reused across ScheduleBench
      (mpcc), GUIBench (gui_odyssey), and EmbodiedBench because all three
      canonical v3 source files share the same multi-turn schema (constraints +
      multi-turn text). Per-task task_domain / paper_domain is carried through
      $domain_label for downstream analysis.

    Slot count: 12 covers every sample. Max observed across the canonical
    v3 multi-turn pool is 9, on ScheduleBench.
  -->

  <View>
    <Header value="📝 Multi-turn instruction evaluation (text only)"/>

    <View style="background:#eff6ff; border-left:4px solid #2563eb; padding:12px 16px; border-radius:6px; margin:6px 0 14px 0;">
      <Text name="how_to" value="HOW TO ANNOTATE (30 sec read)&#10;&#10;1. If the task already shows someone's answers, click the + button at the top to start your OWN fresh annotation.&#10;2. The YELLOW box is your reference. It has the original clean instruction and the numbered list of constraints we expect the model to satisfy.&#10;3. The GREY box is the rewritten TEXT that the model would actually receive.&#10;4. Step 1 — count the rows in the yellow list and pick that number.&#10;5. Step 2 — for each constraint, check whether it is expressed in the GREY TEXT:&#10;   - YES — clearly expressed (any wording).&#10;   - PARTIAL — mentioned but vaguer (e.g. 'cheap' instead of 'under ＄700').&#10;   - NO — missing entirely or contradicted.&#10;6. Naturalness — rate 1 (robotic) to 5 (real person talking).&#10;7. Notes — optional, helpful for borderline cases.&#10;&#10;About 1-2 minutes per task."/>
    </View>

    <Style>
      .ls-container { font-size: 16px; line-height: 1.55; }
      .ls-dialogue { background: #f6f8fa; padding: 14px 18px; border-radius: 8px;
                     white-space: pre-wrap; font-family: ui-sans-serif, -apple-system, sans-serif; }
      .ls-constraints { background: #fff8e1; padding: 12px 18px;
                        border-left: 4px solid #f7c948; border-radius: 4px;
                        white-space: pre-wrap; font-family: ui-sans-serif, -apple-system, sans-serif; }
      .ls-howmany { background: #e8f3ff; border-left: 4px solid #2f80ed;
                    padding: 10px 14px; border-radius: 4px; font-weight: 500; }
      .ls-card-yellow { background: #fff8e1; border-left: 4px solid #f7c948;
                        border-radius: 6px; padding: 14px 18px; margin: 8px 0;
                        white-space: normal; font-family: ui-sans-serif, -apple-system, sans-serif; }
      .ls-card-yellow > * + * { margin-top: 6px; }
      .ls-card-label { color: #5b6573; font-size: 13px; font-weight: 600;
                       margin-top: 18px; }
      .ls-card-divider { border: 0; border-top: 1px dashed #e0c66b;
                         margin: 16px 0 8px 0; }
      .meta { color: #5b6573; font-size: 13px; }
      h4 { margin-top: 24px; }
    </Style>

    <View className="ls-container">

      <View style="display:flex; justify-content:space-between; align-items:baseline;">
        <Text name="id" value="$sample_id"/>
        <Text name="meta_subtask" value="$domain_label"/>
      </View>

      <View style="background:#FFF3B0; border:1px solid #E8C547; border-radius:6px; padding:14px 18px; margin:8px 0;">
        <Text name="ref_heading" value="📌 Reference (what we are checking against)" style="font-weight:700; font-size:15px; margin-bottom:8px;"/>
        <Text name="ref_title_clean" value="Original clean instruction:" style="font-weight:600; font-size:13px; color:#7a5b00; margin-top:6px;"/>
        <Text name="clean_text" value="$clean_text" style="font-family:Georgia,serif; font-size:14px; line-height:1.5; color:#1f1f1f; white-space:pre-wrap;"/>
        <Text name="ref_sep" value=" " style="display:block; margin-top:14px; padding-top:10px; border-top:1px dashed #E8C547;"/>
        <Text name="ref_title_constraints" value="Numbered constraints:" style="font-weight:600; font-size:13px; color:#7a5b00;"/>
        <Text name="constraints" value="$constraints_numbered" style="font-family:Menlo,Consolas,monospace; font-size:13px; line-height:1.6; color:#1f1f1f; white-space:pre-wrap;"/>
      </View>
      <Header value="Step 0 - Does the constraint list cover the original instruction?"/>
      <Text name="step0_help" value="Compare the YELLOW box. Do the numbered constraints capture everything in the clean instruction? This is a quick check on our data, not on the dialogue/audio."/>
      <Choices name="reference_covers" toName="dialogue" choice="single" required="true" showInline="true">
        <Choice value="Yes - complete"/>
        <Choice value="Partial - some details missing"/>
        <Choice value="No - major part missing"/>
      </Choices>
      <TextArea name="reference_missing" toName="dialogue" rows="1" maxSubmissions="1" placeholder="(optional) In one line, what was missing?"/>


      <View className="ls-howmany">
        <Text name="howmany" value="This task has $requirement_count constraints — only use slots C1 to C$requirement_count below. Leave the higher-numbered slots untouched."/>
      </View>

      <View>
        <Header value="Multi-turn dialogue"/>
        <Text name="dialogue" value="$dialogue" className="ls-dialogue"/>
      </View>

      
      <Header value="Step 1 - How many constraints does this task have?"/>
      <Text name="numhint" value="Pick the number that matches the constraint list in the yellow box (count the rows)."/>
      <Choices name="num_constraints" toName="dialogue" choice="single" required="true" showInline="true">
        <Choice value="2"/><Choice value="3"/><Choice value="4"/><Choice value="5"/><Choice value="6"/>
        <Choice value="7"/><Choice value="8"/><Choice value="9"/><Choice value="10"/><Choice value="11"/>
        <Choice value="12"/>
      </Choices>

      <Header value="Step 2 - Per-constraint judgement"/>
      <Text name="instr" value="For each constraint in the YELLOW reference box, decide whether it is expressed in the informal TEXT below. YES = clearly expressed; PARTIAL = mentioned but vague; NO = missing or contradicted. Use only the first $requirement_count slots."/>

      <!-- Per-slot hidden indicators. Each is auto-selected on render to the
           value of data.show_cN ("yes" or "no") via templated <Choice>. The
           visibleWhen on each cN block below reads these indicators and only
           renders the slot when its show flag is "yes". -->
      <View style="display:none">
        <Choices name="_show_c2" toName="dialogue" choice="single"><Choice value="yes"/><Choice value="no"/></Choices>
        <Choices name="_show_c3" toName="dialogue" choice="single"><Choice value="yes"/><Choice value="no"/></Choices>
        <Choices name="_show_c4" toName="dialogue" choice="single"><Choice value="yes"/><Choice value="no"/></Choices>
        <Choices name="_show_c5" toName="dialogue" choice="single"><Choice value="yes"/><Choice value="no"/></Choices>
        <Choices name="_show_c6" toName="dialogue" choice="single"><Choice value="yes"/><Choice value="no"/></Choices>
        <Choices name="_show_c7" toName="dialogue" choice="single"><Choice value="yes"/><Choice value="no"/></Choices>
        <Choices name="_show_c8" toName="dialogue" choice="single"><Choice value="yes"/><Choice value="no"/></Choices>
        <Choices name="_show_c9" toName="dialogue" choice="single"><Choice value="yes"/><Choice value="no"/></Choices>
        <Choices name="_show_c10" toName="dialogue" choice="single"><Choice value="yes"/><Choice value="no"/></Choices>
        <Choices name="_show_c11" toName="dialogue" choice="single"><Choice value="yes"/><Choice value="no"/></Choices>
        <Choices name="_show_c12" toName="dialogue" choice="single"><Choice value="yes"/><Choice value="no"/></Choices>
      </View>

      <View style="display:grid; grid-template-columns: 1fr 1fr; gap: 8px;">
        <!-- c1 always shows; every task has at least 2 constraints. -->
        <Choices name="c1" toName="dialogue" choice="single" required="false">
          <Choice value="C1: YES"/><Choice value="C1: NO"/><Choice value="C1: PARTIAL"/>
        </Choices>
        <View visibleWhen="choice-selected" whenTagName="num_constraints" whenChoiceValue="2,3,4,5,6,7,8,9,10,11,12"><Choices name="c2" toName="dialogue" choice="single" required="false">
            <Choice value="C2: YES"/><Choice value="C2: NO"/><Choice value="C2: PARTIAL"/>
          </Choices></View>
        <View visibleWhen="choice-selected" whenTagName="num_constraints" whenChoiceValue="3,4,5,6,7,8,9,10,11,12"><Choices name="c3" toName="dialogue" choice="single" required="false">
            <Choice value="C3: YES"/><Choice value="C3: NO"/><Choice value="C3: PARTIAL"/>
          </Choices></View>
        <View visibleWhen="choice-selected" whenTagName="num_constraints" whenChoiceValue="4,5,6,7,8,9,10,11,12"><Choices name="c4" toName="dialogue" choice="single" required="false">
            <Choice value="C4: YES"/><Choice value="C4: NO"/><Choice value="C4: PARTIAL"/>
          </Choices></View>
        <View visibleWhen="choice-selected" whenTagName="num_constraints" whenChoiceValue="5,6,7,8,9,10,11,12"><Choices name="c5" toName="dialogue" choice="single" required="false">
            <Choice value="C5: YES"/><Choice value="C5: NO"/><Choice value="C5: PARTIAL"/>
          </Choices></View>
        <View visibleWhen="choice-selected" whenTagName="num_constraints" whenChoiceValue="6,7,8,9,10,11,12"><Choices name="c6" toName="dialogue" choice="single" required="false">
            <Choice value="C6: YES"/><Choice value="C6: NO"/><Choice value="C6: PARTIAL"/>
          </Choices></View>
        <View visibleWhen="choice-selected" whenTagName="num_constraints" whenChoiceValue="7,8,9,10,11,12"><Choices name="c7" toName="dialogue" choice="single" required="false">
            <Choice value="C7: YES"/><Choice value="C7: NO"/><Choice value="C7: PARTIAL"/>
          </Choices></View>
        <View visibleWhen="choice-selected" whenTagName="num_constraints" whenChoiceValue="8,9,10,11,12"><Choices name="c8" toName="dialogue" choice="single" required="false">
            <Choice value="C8: YES"/><Choice value="C8: NO"/><Choice value="C8: PARTIAL"/>
          </Choices></View>
        <View visibleWhen="choice-selected" whenTagName="num_constraints" whenChoiceValue="9,10,11,12"><Choices name="c9" toName="dialogue" choice="single" required="false">
            <Choice value="C9: YES"/><Choice value="C9: NO"/><Choice value="C9: PARTIAL"/>
          </Choices></View>
        <View visibleWhen="choice-selected" whenTagName="num_constraints" whenChoiceValue="10,11,12"><Choices name="c10" toName="dialogue" choice="single" required="false">
            <Choice value="C10: YES"/><Choice value="C10: NO"/><Choice value="C10: PARTIAL"/>
          </Choices></View>
        <View visibleWhen="choice-selected" whenTagName="num_constraints" whenChoiceValue="11,12"><Choices name="c11" toName="dialogue" choice="single" required="false">
            <Choice value="C11: YES"/><Choice value="C11: NO"/><Choice value="C11: PARTIAL"/>
          </Choices></View>
        <View visibleWhen="choice-selected" whenTagName="num_constraints" whenChoiceValue="12"><Choices name="c12" toName="dialogue" choice="single" required="false">
            <Choice value="C12: YES"/><Choice value="C12: NO"/><Choice value="C12: PARTIAL"/>
          </Choices></View>
      </View>

      <Header value="Naturalness"/>
      <Text name="instr_naturalness" value="Does this dialogue read like something a real person would actually say? 1 = sounds completely artificial. 5 = sounds like a normal conversation."/>
      <Rating name="naturalness" toName="dialogue" maxRating="5" required="true"/>

      <Header value="Notes (optional)"/>
      <TextArea name="notes" toName="dialogue" rows="2" maxSubmissions="1" placeholder="Anything odd about this dialogue?"/>

    </View>
  </View>

expert_instruction:
  ## What you're doing

  You'll see 15 items. Each shows the **original constraints** (yellow) and a **single-utterance informal paraphrase** (grey) — disfluencies, fillers, self-corrections all included.

  For each constraint, click **YES** (clearly expressed), **PARTIAL** (mentioned but weaker), or **NO** (missing). Then rate **naturalness** 1–5.

  The single paraphrase usually contains ALL the constraints jammed together — you're checking whether the informal style accidentally dropped or muddled any of them.

  ## Decision rules

  - **YES**: the paraphrase clearly states the constraint in any form.
  - **PARTIAL**: it's there but the meaning is weaker (vague pronouns, fillers obscuring a number, etc.).
  - **NO**: missing or contradicted.
  - Borderline → **PARTIAL**.

  ## Time

  1–2 minutes per item, ~25 minutes total.

data_types:
  {"sample_id": "Text", "domain_label": "Text", "clean_text": "Text", "constraints_numbered": "Text", "requirement_count": "Text", "dialogue": "Text"}

control_weights:
  {"reference_covers": {"overall": 1.0, "type": "Choices", "labels": {"Yes - complete": 1.0, "Partial - some details missing": 1.0, "No - major part missing": 1.0}}, "reference_missing": {"overall": 1.0, "type": "TextArea", "labels": {}}, "num_constraints": {"overall": 1.0, "type": "Choices", "labels": {"2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 1.0, "11": 1.0, "12": 1.0}}, "_show_c2": {"overall": 1.0, "type": "Choices", "labels": {"yes": 1.0, "no": 1.0}}, "_show_c3": {"overall": 1.0, "type": "Choices", "labels": {"yes": 1.0, "no": 1.0}}, "_show_c4": {"overall": 1.0, "type": "Choices", "labels": {"yes": 1.0, "no": 1.0}}, "_show_c5": {"overall": 1.0, "type": "Choices", "labels": {"yes": 1.0, "no": 1.0}}, "_show_c6": {"overall": 1.0, "type": "Choices", "labels": {"yes": 1.0, "no": 1.0}}, "_show_c7": {"overall": 1.0, "type": "Choices", "labels": {"yes": 1.0, "no": 1.0}}, "_show_c8": {"overall": 1.0, "type": "Choices", "labels": {"yes": 1.0, "no": 1.0}}, "_show_c9": {"overall": 1.0, "type": "Choices", "labels": {"yes": 1.0, "no": 1.0}}, "_show_c10": {"overall": 1.0, "type": "Choices", "labels": {"yes": 1.0, "no": 1.0}}, "_show_c11": {"overall": 1.0, "type": "Choices", "labels": {"yes": 1.0, "no": 1.0}}, "_show_c12": {"overall": 1.0, "type": "Choices", "labels": {"yes": 1.0, "no": 1.0}}, "c1": {"overall": 1.0, "type": "Choices", "labels": {"C1: YES": 1.0, "C1: NO": 1.0, "C1: PARTIAL": 1.0}}, "c2": {"overall": 1.0, "type": "Choices", "labels": {"C2: YES": 1.0, "C2: NO": 1.0, "C2: PARTIAL": 1.0}}, "c3": {"overall": 1.0, "type": "Choices", "labels": {"C3: YES": 1.0, "C3: NO": 1.0, "C3: PARTIAL": 1.0}}, "c4": {"overall": 1.0, "type": "Choices", "labels": {"C4: YES": 1.0, "C4: NO": 1.0, "C4: PARTIAL": 1.0}}, "c5": {"overall": 1.0, "type": "Choices", "labels": {"C5: YES": 1.0, "C5: NO": 1.0, "C5: PARTIAL": 1.0}}, "c6": {"overall": 1.0, "type": "Choices", "labels": {"C6: YES": 1.0, "C6: NO": 1.0, "C6: PARTIAL": 1.0}}, "c7": {"overall": 1.0, "type": "Choices", "labels": {"C7: YES": 1.0, "C7: NO": 1.0, "C7: PARTIAL": 1.0}}, "c8": {"overall": 1.0, "type": "Choices", "labels": {"C8: YES": 1.0, "C8: NO": 1.0, "C8: PARTIAL": 1.0}}, "c9": {"overall": 1.0, "type": "Choices", "labels": {"C9: YES": 1.0, "C9: NO": 1.0, "C9: PARTIAL": 1.0}}, "c10": {"overall": 1.0, "type": "Choices", "labels": {"C10: YES": 1.0, "C10: NO": 1.0, "C10: PARTIAL": 1.0}}, "c11": {"overall": 1.0, "type": "Choices", "labels": {"C11: YES": 1.0, "C11: NO": 1.0, "C11: PARTIAL": 1.0}}, "c12": {"overall": 1.0, "type": "Choices", "labels": {"C12: YES": 1.0, "C12: NO": 1.0, "C12: PARTIAL": 1.0}}, "naturalness": {"overall": 1.0, "type": "Rating", "labels": {}}, "notes": {"overall": 1.0, "type": "TextArea", "labels": {}}}

description:
  Stage C — text-only validation of the Informal Instruction wild condition. Same form as Stage A but on disfluent single-utterance paraphrases.

parsed_label_config:
  {"reference_covers": {"type": "Choices", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": ["Yes - complete", "Partial - some details missing", "No - major part missing"], "labels_attrs": {"Yes - complete": {"value": "Yes - complete"}, "Partial - some details missing": {"value": "Partial - some details missing"}, "No - major part missing": {"value": "No - major part missing"}}}, "reference_missing": {"type": "TextArea", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": [], "labels_attrs": {}}, "num_constraints": {"type": "Choices", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": ["2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"], "labels_attrs": {"2": {"value": "2"}, "3": {"value": "3"}, "4": {"value": "4"}, "5": {"value": "5"}, "6": {"value": "6"}, "7": {"value": "7"}, "8": {"value": "8"}, "9": {"value": "9"}, "10": {"value": "10"}, "11": {"value": "11"}, "12": {"value": "12"}}}, "_show_c2": {"type": "Choices", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": ["yes", "no"], "labels_attrs": {"yes": {"value": "yes"}, "no": {"value": "no"}}}, "_show_c3": {"type": "Choices", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": ["yes", "no"], "labels_attrs": {"yes": {"value": "yes"}, "no": {"value": "no"}}}, "_show_c4": {"type": "Choices", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": ["yes", "no"], "labels_attrs": {"yes": {"value": "yes"}, "no": {"value": "no"}}}, "_show_c5": {"type": "Choices", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": ["yes", "no"], "labels_attrs": {"yes": {"value": "yes"}, "no": {"value": "no"}}}, "_show_c6": {"type": "Choices", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": ["yes", "no"], "labels_attrs": {"yes": {"value": "yes"}, "no": {"value": "no"}}}, "_show_c7": {"type": "Choices", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": ["yes", "no"], "labels_attrs": {"yes": {"value": "yes"}, "no": {"value": "no"}}}, "_show_c8": {"type": "Choices", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": ["yes", "no"], "labels_attrs": {"yes": {"value": "yes"}, "no": {"value": "no"}}}, "_show_c9": {"type": "Choices", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": ["yes", "no"], "labels_attrs": {"yes": {"value": "yes"}, "no": {"value": "no"}}}, "_show_c10": {"type": "Choices", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": ["yes", "no"], "labels_attrs": {"yes": {"value": "yes"}, "no": {"value": "no"}}}, "_show_c11": {"type": "Choices", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": ["yes", "no"], "labels_attrs": {"yes": {"value": "yes"}, "no": {"value": "no"}}}, "_show_c12": {"type": "Choices", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": ["yes", "no"], "labels_attrs": {"yes": {"value": "yes"}, "no": {"value": "no"}}}, "c1": {"type": "Choices", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": ["C1: YES", "C1: NO", "C1: PARTIAL"], "labels_attrs": {"C1: YES": {"value": "C1: YES"}, "C1: NO": {"value": "C1: NO"}, "C1: PARTIAL": {"value": "C1: PARTIAL"}}}, "c2": {"type": "Choices", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": ["C2: YES", "C2: NO", "C2: PARTIAL"], "labels_attrs": {"C2: YES": {"value": "C2: YES"}, "C2: NO": {"value": "C2: NO"}, "C2: PARTIAL": {"value": "C2: PARTIAL"}}}, "c3": {"type": "Choices", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": ["C3: YES", "C3: NO", "C3: PARTIAL"], "labels_attrs": {"C3: YES": {"value": "C3: YES"}, "C3: NO": {"value": "C3: NO"}, "C3: PARTIAL": {"value": "C3: PARTIAL"}}}, "c4": {"type": "Choices", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": ["C4: YES", "C4: NO", "C4: PARTIAL"], "labels_attrs": {"C4: YES": {"value": "C4: YES"}, "C4: NO": {"value": "C4: NO"}, "C4: PARTIAL": {"value": "C4: PARTIAL"}}}, "c5": {"type": "Choices", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": ["C5: YES", "C5: NO", "C5: PARTIAL"], "labels_attrs": {"C5: YES": {"value": "C5: YES"}, "C5: NO": {"value": "C5: NO"}, "C5: PARTIAL": {"value": "C5: PARTIAL"}}}, "c6": {"type": "Choices", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": ["C6: YES", "C6: NO", "C6: PARTIAL"], "labels_attrs": {"C6: YES": {"value": "C6: YES"}, "C6: NO": {"value": "C6: NO"}, "C6: PARTIAL": {"value": "C6: PARTIAL"}}}, "c7": {"type": "Choices", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": ["C7: YES", "C7: NO", "C7: PARTIAL"], "labels_attrs": {"C7: YES": {"value": "C7: YES"}, "C7: NO": {"value": "C7: NO"}, "C7: PARTIAL": {"value": "C7: PARTIAL"}}}, "c8": {"type": "Choices", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": ["C8: YES", "C8: NO", "C8: PARTIAL"], "labels_attrs": {"C8: YES": {"value": "C8: YES"}, "C8: NO": {"value": "C8: NO"}, "C8: PARTIAL": {"value": "C8: PARTIAL"}}}, "c9": {"type": "Choices", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": ["C9: YES", "C9: NO", "C9: PARTIAL"], "labels_attrs": {"C9: YES": {"value": "C9: YES"}, "C9: NO": {"value": "C9: NO"}, "C9: PARTIAL": {"value": "C9: PARTIAL"}}}, "c10": {"type": "Choices", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": ["C10: YES", "C10: NO", "C10: PARTIAL"], "labels_attrs": {"C10: YES": {"value": "C10: YES"}, "C10: NO": {"value": "C10: NO"}, "C10: PARTIAL": {"value": "C10: PARTIAL"}}}, "c11": {"type": "Choices", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": ["C11: YES", "C11: NO", "C11: PARTIAL"], "labels_attrs": {"C11: YES": {"value": "C11: YES"}, "C11: NO": {"value": "C11: NO"}, "C11: PARTIAL": {"value": "C11: PARTIAL"}}}, "c12": {"type": "Choices", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": ["C12: YES", "C12: NO", "C12: PARTIAL"], "labels_attrs": {"C12: YES": {"value": "C12: YES"}, "C12: NO": {"value": "C12: NO"}, "C12: PARTIAL": {"value": "C12: PARTIAL"}}}, "naturalness": {"type": "Rating", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": [], "labels_attrs": {}}, "notes": {"type": "TextArea", "to_name": ["dialogue"], "inputs": [{"type": "Text", "valueType": null, "value": "dialogue"}], "labels": [], "labels_attrs": {}}}


title: OmniAgentBench Noisy audio (Stage D)
model_version: slot_visibility
created_at: 2026-05-19 20:14:02.681886
updated_at: 2026-05-25 00:55:31.124016
sampling: Sequential sampling
token: lR5sNbI0eufyg4uONH3ETSox9uZh3EA8Fgzn7Hx0
color: #FFFFFF
skip_queue: REQUEUE_FOR_OTHERS
label_config:
  <View>
    <Header value="Noisy instruction evaluation (audio only)"/>

    <View style="background:#eff6ff; border-left:4px solid #2563eb; padding:12px 16px; border-radius:6px; margin:6px 0 14px 0;">
      <Text name="how_to" value="HOW TO ANNOTATE (1 minute)&#10;&#10;1. If the task already shows someone's answers, click the + button at the top to start your OWN fresh annotation.&#10;2. The YELLOW box is your reference (clean instruction + numbered constraints).&#10;3. Step 0 - quick check that the numbered constraints match the clean instruction.&#10;4. Listen to the noisy audio clip.&#10;5. Did you make out the FULL instruction despite the noise? Pick Yes / Partial / No.&#10;6. Audio clarity - 1 (mostly drowned out) to 5 (clear).&#10;7. Naturalness - 1 (robotic) to 5 (real person).&#10;8. Notes - optional, say what the noise specifically masked."/>
    </View>

    <View>

      <View style="display:flex; justify-content:space-between; align-items:baseline;">
        <Text name="id" value="$sample_id"/>
        <Text name="meta_subtask" value="$domain_label"/>
      </View>

      <View style="background:#FFF3B0; border:1px solid #E8C547; border-radius:6px; padding:14px 18px; margin:8px 0;">
        <Text name="ref_heading" value="Reference (what we are checking against)" style="font-weight:700; font-size:15px; margin-bottom:8px;"/>
        <Text name="ref_title_clean" value="Original clean instruction:" style="font-weight:600; font-size:13px; color:#7a5b00; margin-top:6px;"/>
        <Text name="clean_text" value="$clean_text" style="font-family:Georgia,serif; font-size:14px; line-height:1.5; color:#1f1f1f; white-space:pre-wrap;"/>
        <Text name="ref_sep" value=" " style="display:block; margin-top:14px; padding-top:10px; border-top:1px dashed #E8C547;"/>
        <Text name="ref_title_constraints" value="Numbered constraints:" style="font-weight:600; font-size:13px; color:#7a5b00;"/>
        <Text name="constraints" value="$constraints_numbered" style="font-family:Menlo,Consolas,monospace; font-size:13px; line-height:1.6; color:#1f1f1f; white-space:pre-wrap;"/>
      </View>

      <Header value="Step 0 - Does the constraint list cover the original instruction?"/>
      <Text name="step0_help" value="Compare the YELLOW box. Do the numbered constraints capture everything in the clean instruction? This is a quick check on our data, not on the audio."/>
      <Choices name="reference_covers" toName="audio_clip" choice="single" required="true" showInline="true">
        <Choice value="Yes - complete"/>
        <Choice value="Partial - some details missing"/>
        <Choice value="No - major part missing"/>
      </Choices>
      <TextArea name="reference_missing" toName="audio_clip" rows="1" maxSubmissions="1" placeholder="(optional) In one line, what was missing?"/>

      <Header value="Noisy audio clip"/>
      <Audio name="audio_clip" value="$audio_url"/>

      <Header value="Could you make out the full instruction?"/>
      <Text name="instr_makeout" value="Despite the noise, were you able to understand all of the constraints listed in the YELLOW box?"/>
      <Choices name="makeout" toName="audio_clip" choice="single" required="true" showInline="true">
        <Choice value="Yes - heard them all"/>
        <Choice value="Partial - some got muffled"/>
        <Choice value="No - lost too much"/>
      </Choices>

      <Header value="Audio clarity"/>
      <Text name="instr_clarity" value="Could you understand the audio despite the noise? 1 = mostly drowned out. 5 = clear."/>
      <Rating name="audio_clarity" toName="audio_clip" maxRating="5" required="true"/>

      <Header value="Naturalness (spoken delivery)"/>
      <Text name="instr_naturalness" value="Does the speaker sound like a real person? 1 = robotic. 5 = natural."/>
      <Rating name="naturalness" toName="audio_clip" maxRating="5" required="true"/>

      <Header value="Notes (optional, encouraged)"/>
      <TextArea name="notes" toName="audio_clip" rows="2" maxSubmissions="1" placeholder="What did the noise do? e.g. cafe chatter masked the price; static drowned out the place name."/>
      
      <View style="display:none">
        <Choices name="num_constraints" toName="audio_clip" choice="single"><Choice value="2"/><Choice value="3"/><Choice value="4"/><Choice value="5"/><Choice value="6"/><Choice value="7"/><Choice value="8"/><Choice value="9"/><Choice value="10"/><Choice value="11"/><Choice value="12"/></Choices>
        <Choices name="_show_c2"  toName="audio_clip" choice="single"><Choice value="yes"/><Choice value="no"/></Choices>
        <Choices name="_show_c3"  toName="audio_clip" choice="single"><Choice value="yes"/><Choice value="no"/></Choices>
        <Choices name="_show_c4"  toName="audio_clip" choice="single"><Choice value="yes"/><Choice value="no"/></Choices>
        <Choices name="_show_c5"  toName="audio_clip" choice="single"><Choice value="yes"/><Choice value="no"/></Choices>
        <Choices name="_show_c6"  toName="audio_clip" choice="single"><Choice value="yes"/><Choice value="no"/></Choices>
        <Choices name="_show_c7"  toName="audio_clip" choice="single"><Choice value="yes"/><Choice value="no"/></Choices>
        <Choices name="_show_c8"  toName="audio_clip" choice="single"><Choice value="yes"/><Choice value="no"/></Choices>
        <Choices name="_show_c9"  toName="audio_clip" choice="single"><Choice value="yes"/><Choice value="no"/></Choices>
        <Choices name="_show_c10" toName="audio_clip" choice="single"><Choice value="yes"/><Choice value="no"/></Choices>
        <Choices name="_show_c11" toName="audio_clip" choice="single"><Choice value="yes"/><Choice value="no"/></Choices>
        <Choices name="_show_c12" toName="audio_clip" choice="single"><Choice value="yes"/><Choice value="no"/></Choices>
        <Choices name="c1"  toName="audio_clip" choice="single"><Choice value="C1: YES"/><Choice value="C1: NO"/><Choice value="C1: PARTIAL"/></Choices>
        <Choices name="c2"  toName="audio_clip" choice="single"><Choice value="C2: YES"/><Choice value="C2: NO"/><Choice value="C2: PARTIAL"/></Choices>
        <Choices name="c3"  toName="audio_clip" choice="single"><Choice value="C3: YES"/><Choice value="C3: NO"/><Choice value="C3: PARTIAL"/></Choices>
        <Choices name="c4"  toName="audio_clip" choice="single"><Choice value="C4: YES"/><Choice value="C4: NO"/><Choice value="C4: PARTIAL"/></Choices>
        <Choices name="c5"  toName="audio_clip" choice="single"><Choice value="C5: YES"/><Choice value="C5: NO"/><Choice value="C5: PARTIAL"/></Choices>
        <Choices name="c6"  toName="audio_clip" choice="single"><Choice value="C6: YES"/><Choice value="C6: NO"/><Choice value="C6: PARTIAL"/></Choices>
        <Choices name="c7"  toName="audio_clip" choice="single"><Choice value="C7: YES"/><Choice value="C7: NO"/><Choice value="C7: PARTIAL"/></Choices>
        <Choices name="c8"  toName="audio_clip" choice="single"><Choice value="C8: YES"/><Choice value="C8: NO"/><Choice value="C8: PARTIAL"/></Choices>
        <Choices name="c9"  toName="audio_clip" choice="single"><Choice value="C9: YES"/><Choice value="C9: NO"/><Choice value="C9: PARTIAL"/></Choices>
        <Choices name="c10" toName="audio_clip" choice="single"><Choice value="C10: YES"/><Choice value="C10: NO"/><Choice value="C10: PARTIAL"/></Choices>
        <Choices name="c11" toName="audio_clip" choice="single"><Choice value="C11: YES"/><Choice value="C11: NO"/><Choice value="C11: PARTIAL"/></Choices>
        <Choices name="c12" toName="audio_clip" choice="single"><Choice value="C12: YES"/><Choice value="C12: NO"/><Choice value="C12: PARTIAL"/></Choices>
      </View>

    </View>
  </View>

expert_instruction:
  ## What you're doing

  You'll see 15 items. Each plays ONE short audio clip recorded in a noisy environment (coffee shop, call centre, traffic, etc.). Same instruction as a clean task, but with background noise mixed in.

  ## What to fill

  1. **Per-constraint YES / PARTIAL / NO** — from what you can HEAR through the noise. Use only the first N slots (see the blue callout).
  2. **Audio clarity 1–5** — could you understand the audio at all? 1 = mostly drowned out, 5 = clear despite the noise.
  3. **Naturalness 1–5** — does the speaker sound like a real person?
  4. **Notes** — encouraged; one line about what the noise did, e.g. "cafe chatter masked the price", "static drowned the place name".

  ## Listening tips

  - Headphones strongly recommended.
  - Replay 1–2 times max; if you needed 5 replays you're brute-forcing — score CLARITY low instead.
  - The whole instruction is in one short clip; no turns, no per-turn questions.

  ## Time

  1–2 minutes per item.

data_types:
  {"sample_id": "Text", "domain_label": "Text", "clean_text": "Text", "constraints_numbered": "Text", "audio_url": "Audio"}

control_weights:
  {"reference_covers": {"overall": 1.0, "type": "Choices", "labels": {"Yes - complete": 1.0, "Partial - some details missing": 1.0, "No - major part missing": 1.0}}, "reference_missing": {"overall": 1.0, "type": "TextArea", "labels": {}}, "makeout": {"overall": 1.0, "type": "Choices", "labels": {"Yes - heard them all": 1.0, "Partial - some got muffled": 1.0, "No - lost too much": 1.0}}, "audio_clarity": {"overall": 1.0, "type": "Rating", "labels": {}}, "naturalness": {"overall": 1.0, "type": "Rating", "labels": {}}, "notes": {"overall": 1.0, "type": "TextArea", "labels": {}}, "num_constraints": {"overall": 1.0, "type": "Choices", "labels": {"2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 1.0, "11": 1.0, "12": 1.0}}, "_show_c2": {"overall": 1.0, "type": "Choices", "labels": {"yes": 1.0, "no": 1.0}}, "_show_c3": {"overall": 1.0, "type": "Choices", "labels": {"yes": 1.0, "no": 1.0}}, "_show_c4": {"overall": 1.0, "type": "Choices", "labels": {"yes": 1.0, "no": 1.0}}, "_show_c5": {"overall": 1.0, "type": "Choices", "labels": {"yes": 1.0, "no": 1.0}}, "_show_c6": {"overall": 1.0, "type": "Choices", "labels": {"yes": 1.0, "no": 1.0}}, "_show_c7": {"overall": 1.0, "type": "Choices", "labels": {"yes": 1.0, "no": 1.0}}, "_show_c8": {"overall": 1.0, "type": "Choices", "labels": {"yes": 1.0, "no": 1.0}}, "_show_c9": {"overall": 1.0, "type": "Choices", "labels": {"yes": 1.0, "no": 1.0}}, "_show_c10": {"overall": 1.0, "type": "Choices", "labels": {"yes": 1.0, "no": 1.0}}, "_show_c11": {"overall": 1.0, "type": "Choices", "labels": {"yes": 1.0, "no": 1.0}}, "_show_c12": {"overall": 1.0, "type": "Choices", "labels": {"yes": 1.0, "no": 1.0}}, "c1": {"overall": 1.0, "type": "Choices", "labels": {"C1: YES": 1.0, "C1: NO": 1.0, "C1: PARTIAL": 1.0}}, "c2": {"overall": 1.0, "type": "Choices", "labels": {"C2: YES": 1.0, "C2: NO": 1.0, "C2: PARTIAL": 1.0}}, "c3": {"overall": 1.0, "type": "Choices", "labels": {"C3: YES": 1.0, "C3: NO": 1.0, "C3: PARTIAL": 1.0}}, "c4": {"overall": 1.0, "type": "Choices", "labels": {"C4: YES": 1.0, "C4: NO": 1.0, "C4: PARTIAL": 1.0}}, "c5": {"overall": 1.0, "type": "Choices", "labels": {"C5: YES": 1.0, "C5: NO": 1.0, "C5: PARTIAL": 1.0}}, "c6": {"overall": 1.0, "type": "Choices", "labels": {"C6: YES": 1.0, "C6: NO": 1.0, "C6: PARTIAL": 1.0}}, "c7": {"overall": 1.0, "type": "Choices", "labels": {"C7: YES": 1.0, "C7: NO": 1.0, "C7: PARTIAL": 1.0}}, "c8": {"overall": 1.0, "type": "Choices", "labels": {"C8: YES": 1.0, "C8: NO": 1.0, "C8: PARTIAL": 1.0}}, "c9": {"overall": 1.0, "type": "Choices", "labels": {"C9: YES": 1.0, "C9: NO": 1.0, "C9: PARTIAL": 1.0}}, "c10": {"overall": 1.0, "type": "Choices", "labels": {"C10: YES": 1.0, "C10: NO": 1.0, "C10: PARTIAL": 1.0}}, "c11": {"overall": 1.0, "type": "Choices", "labels": {"C11: YES": 1.0, "C11: NO": 1.0, "C11: PARTIAL": 1.0}}, "c12": {"overall": 1.0, "type": "Choices", "labels": {"C12: YES": 1.0, "C12: NO": 1.0, "C12: PARTIAL": 1.0}}}

description:
  Stage D — audio validation under 6 noise environments. Single-clip noisy audio + constraint recall + audio clarity rating.

parsed_label_config:
  {"reference_covers": {"type": "Choices", "to_name": ["audio_clip"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_url"}], "labels": ["Yes - complete", "Partial - some details missing", "No - major part missing"], "labels_attrs": {"Yes - complete": {"value": "Yes - complete"}, "Partial - some details missing": {"value": "Partial - some details missing"}, "No - major part missing": {"value": "No - major part missing"}}}, "reference_missing": {"type": "TextArea", "to_name": ["audio_clip"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_url"}], "labels": [], "labels_attrs": {}}, "makeout": {"type": "Choices", "to_name": ["audio_clip"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_url"}], "labels": ["Yes - heard them all", "Partial - some got muffled", "No - lost too much"], "labels_attrs": {"Yes - heard them all": {"value": "Yes - heard them all"}, "Partial - some got muffled": {"value": "Partial - some got muffled"}, "No - lost too much": {"value": "No - lost too much"}}}, "audio_clarity": {"type": "Rating", "to_name": ["audio_clip"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_url"}], "labels": [], "labels_attrs": {}}, "naturalness": {"type": "Rating", "to_name": ["audio_clip"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_url"}], "labels": [], "labels_attrs": {}}, "notes": {"type": "TextArea", "to_name": ["audio_clip"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_url"}], "labels": [], "labels_attrs": {}}, "num_constraints": {"type": "Choices", "to_name": ["audio_clip"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_url"}], "labels": ["2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"], "labels_attrs": {"2": {"value": "2"}, "3": {"value": "3"}, "4": {"value": "4"}, "5": {"value": "5"}, "6": {"value": "6"}, "7": {"value": "7"}, "8": {"value": "8"}, "9": {"value": "9"}, "10": {"value": "10"}, "11": {"value": "11"}, "12": {"value": "12"}}}, "_show_c2": {"type": "Choices", "to_name": ["audio_clip"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_url"}], "labels": ["yes", "no"], "labels_attrs": {"yes": {"value": "yes"}, "no": {"value": "no"}}}, "_show_c3": {"type": "Choices", "to_name": ["audio_clip"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_url"}], "labels": ["yes", "no"], "labels_attrs": {"yes": {"value": "yes"}, "no": {"value": "no"}}}, "_show_c4": {"type": "Choices", "to_name": ["audio_clip"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_url"}], "labels": ["yes", "no"], "labels_attrs": {"yes": {"value": "yes"}, "no": {"value": "no"}}}, "_show_c5": {"type": "Choices", "to_name": ["audio_clip"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_url"}], "labels": ["yes", "no"], "labels_attrs": {"yes": {"value": "yes"}, "no": {"value": "no"}}}, "_show_c6": {"type": "Choices", "to_name": ["audio_clip"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_url"}], "labels": ["yes", "no"], "labels_attrs": {"yes": {"value": "yes"}, "no": {"value": "no"}}}, "_show_c7": {"type": "Choices", "to_name": ["audio_clip"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_url"}], "labels": ["yes", "no"], "labels_attrs": {"yes": {"value": "yes"}, "no": {"value": "no"}}}, "_show_c8": {"type": "Choices", "to_name": ["audio_clip"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_url"}], "labels": ["yes", "no"], "labels_attrs": {"yes": {"value": "yes"}, "no": {"value": "no"}}}, "_show_c9": {"type": "Choices", "to_name": ["audio_clip"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_url"}], "labels": ["yes", "no"], "labels_attrs": {"yes": {"value": "yes"}, "no": {"value": "no"}}}, "_show_c10": {"type": "Choices", "to_name": ["audio_clip"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_url"}], "labels": ["yes", "no"], "labels_attrs": {"yes": {"value": "yes"}, "no": {"value": "no"}}}, "_show_c11": {"type": "Choices", "to_name": ["audio_clip"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_url"}], "labels": ["yes", "no"], "labels_attrs": {"yes": {"value": "yes"}, "no": {"value": "no"}}}, "_show_c12": {"type": "Choices", "to_name": ["audio_clip"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_url"}], "labels": ["yes", "no"], "labels_attrs": {"yes": {"value": "yes"}, "no": {"value": "no"}}}, "c1": {"type": "Choices", "to_name": ["audio_clip"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_url"}], "labels": ["C1: YES", "C1: NO", "C1: PARTIAL"], "labels_attrs": {"C1: YES": {"value": "C1: YES"}, "C1: NO": {"value": "C1: NO"}, "C1: PARTIAL": {"value": "C1: PARTIAL"}}}, "c2": {"type": "Choices", "to_name": ["audio_clip"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_url"}], "labels": ["C2: YES", "C2: NO", "C2: PARTIAL"], "labels_attrs": {"C2: YES": {"value": "C2: YES"}, "C2: NO": {"value": "C2: NO"}, "C2: PARTIAL": {"value": "C2: PARTIAL"}}}, "c3": {"type": "Choices", "to_name": ["audio_clip"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_url"}], "labels": ["C3: YES", "C3: NO", "C3: PARTIAL"], "labels_attrs": {"C3: YES": {"value": "C3: YES"}, "C3: NO": {"value": "C3: NO"}, "C3: PARTIAL": {"value": "C3: PARTIAL"}}}, "c4": {"type": "Choices", "to_name": ["audio_clip"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_url"}], "labels": ["C4: YES", "C4: NO", "C4: PARTIAL"], "labels_attrs": {"C4: YES": {"value": "C4: YES"}, "C4: NO": {"value": "C4: NO"}, "C4: PARTIAL": {"value": "C4: PARTIAL"}}}, "c5": {"type": "Choices", "to_name": ["audio_clip"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_url"}], "labels": ["C5: YES", "C5: NO", "C5: PARTIAL"], "labels_attrs": {"C5: YES": {"value": "C5: YES"}, "C5: NO": {"value": "C5: NO"}, "C5: PARTIAL": {"value": "C5: PARTIAL"}}}, "c6": {"type": "Choices", "to_name": ["audio_clip"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_url"}], "labels": ["C6: YES", "C6: NO", "C6: PARTIAL"], "labels_attrs": {"C6: YES": {"value": "C6: YES"}, "C6: NO": {"value": "C6: NO"}, "C6: PARTIAL": {"value": "C6: PARTIAL"}}}, "c7": {"type": "Choices", "to_name": ["audio_clip"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_url"}], "labels": ["C7: YES", "C7: NO", "C7: PARTIAL"], "labels_attrs": {"C7: YES": {"value": "C7: YES"}, "C7: NO": {"value": "C7: NO"}, "C7: PARTIAL": {"value": "C7: PARTIAL"}}}, "c8": {"type": "Choices", "to_name": ["audio_clip"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_url"}], "labels": ["C8: YES", "C8: NO", "C8: PARTIAL"], "labels_attrs": {"C8: YES": {"value": "C8: YES"}, "C8: NO": {"value": "C8: NO"}, "C8: PARTIAL": {"value": "C8: PARTIAL"}}}, "c9": {"type": "Choices", "to_name": ["audio_clip"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_url"}], "labels": ["C9: YES", "C9: NO", "C9: PARTIAL"], "labels_attrs": {"C9: YES": {"value": "C9: YES"}, "C9: NO": {"value": "C9: NO"}, "C9: PARTIAL": {"value": "C9: PARTIAL"}}}, "c10": {"type": "Choices", "to_name": ["audio_clip"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_url"}], "labels": ["C10: YES", "C10: NO", "C10: PARTIAL"], "labels_attrs": {"C10: YES": {"value": "C10: YES"}, "C10: NO": {"value": "C10: NO"}, "C10: PARTIAL": {"value": "C10: PARTIAL"}}}, "c11": {"type": "Choices", "to_name": ["audio_clip"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_url"}], "labels": ["C11: YES", "C11: NO", "C11: PARTIAL"], "labels_attrs": {"C11: YES": {"value": "C11: YES"}, "C11: NO": {"value": "C11: NO"}, "C11: PARTIAL": {"value": "C11: PARTIAL"}}}, "c12": {"type": "Choices", "to_name": ["audio_clip"], "inputs": [{"type": "Audio", "valueType": null, "value": "audio_url"}], "labels": ["C12: YES", "C12: NO", "C12: PARTIAL"], "labels_attrs": {"C12: YES": {"value": "C12: YES"}, "C12: NO": {"value": "C12: NO"}, "C12: PARTIAL": {"value": "C12: PARTIAL"}}}}


# token_blacklist_outstandingtoken

created_at: 2026-05-25 04:25:05.395978
expires_at: 2226-04-07 04:25:05
jti: 0f87a262d5904d608ebe6b9e11ef948d
token:
  eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0b2tlbl90eXBlIjoicmVmcmVzaCIsImV4cCI6ODA4Njg4MzEwNSwiaWF0IjoxNzc5NjgzMTA1LCJqdGkiOiIwZjg3YTI2MmQ1OTA0ZDYwOGViZTZiOWUxMWVmOTQ4ZCIsInVzZXJfaWQiOiIzNyJ9


