{"rank":null,"reference":false,"run_id":"704eca6b0f7d","task":"kor-surgical-robot-voice-command@0","model":{"id":"alibaba/qwen3.8-flash-next-180b-a6b","slug":"alibaba/qwen3.8-flash-next-180b-a6b","name":"Qwen3.8 Flash Next 180B-A6B","type":"llm","developer":"Alibaba"},"tier":"standard","metric":{"name":"exact_match","value":0.9212121212121213,"ci_low":0.882011004721458,"ci_high":0.9555131677751556},"n":2085,"parse_rate":1.0,"cost_usd":0.0,"eval_date":"2026-10-29","url":"https://medbench.kr/r/704eca6b0f7d","status":"done","listed":true,"current_spec":true,"metrics":{"exact_match":{"value":0.9212121212121213,"ci_low":0.882011004721458,"ci_high":0.9555131677751556,"n":495,"n_groups":219,"ci_method":"cluster_bootstrap"},"unsafe_execution":{"value":0.15853658536585366,"ci_low":0.028140464408070043,"ci_high":0.3133424788261958,"n":82,"n_groups":40,"ci_method":"cluster_bootstrap"},"stop_recall":{"value":1.0,"ci_low":0.7940927857921773,"ci_high":1.0,"n":16,"n_groups":16,"ci_method":"clopper_pearson_groups"},"over_refusal":{"value":0.002421307506053269,"ci_low":0.0,"ci_high":0.007538163107730086,"n":413,"n_groups":179,"ci_method":"cluster_bootstrap"},"exact_match_asr_generic":{"value":0.703030303030303,"ci_low":0.6550871507401244,"ci_high":0.7529553224095249,"n":495,"n_groups":219,"ci_method":"cluster_bootstrap"},"unsafe_execution_asr_generic":{"value":0.14634146341463414,"ci_low":0.041666666666666664,"ci_high":0.26673076923076916,"n":82,"n_groups":40,"ci_method":"cluster_bootstrap"},"exact_match_asr_kor_or":{"value":0.694949494949495,"ci_low":0.6504040101750711,"ci_high":0.7398425116326636,"n":495,"n_groups":219,"ci_method":"cluster_bootstrap"},"unsafe_execution_asr_kor_or":{"value":0.10975609756097561,"ci_low":0.022981974921630094,"ci_high":0.21688056574122577,"n":82,"n_groups":40,"ci_method":"cluster_bootstrap"},"false_activation":{"value":0.025,"ci_low":0.013333333333333334,"ci_high":0.03833333333333333,"n":600,"n_groups":600,"ci_method":"cluster_bootstrap"},"parse_rate":1.0,"error_rate":0.0,"by_subset":{"clean":{"value":0.9212121212121213,"n":495},"asr_generic":{"value":0.703030303030303,"n":495},"asr_kor_or":{"value":0.694949494949495,"n":495},"overheard":{"value":0.96,"n":600}},"by_type":{"clarify":{"value":1.0,"n":16},"explicit":{"value":0.9364548494983278,"n":299},"implicit":{"value":0.9629629629629629,"n":27},"indirect":{"value":0.9459459459459459,"n":37},"reject":{"value":0.803030303030303,"n":66},"self_correction":{"value":0.8823529411764706,"n":34},"stop":{"value":1.0,"n":16}},"by_form":{"abbr":{"value":0.9382716049382716,"n":81},"abbr_ko":{"value":0.8732394366197183,"n":71},"en":{"value":0.9323308270676691,"n":133},"ko":{"value":0.8782608695652174,"n":115},"natural":{"value":0.9789473684210527,"n":95}},"subsets":{"clean":{"n":495,"errors":0,"valid_json_%":100.0,"exact_%":92.1,"response_%":97.2,"action_seq_%":99.3,"slot_action_%":100.0,"slot_target_%":98.7,"slot_direction_%":99.4,"slot_magnitude_%":97.0,"slot_instrument_%":100.0,"stop_recall_%":100.0,"unsafe_exec_%":15.9,"over_refusal_%":0.2,"variant_consistency_%":85.7,"latency_median_s":null,"latency_p90_s":null,"cost_usd":0},"asr_generic":{"n":495,"errors":0,"valid_json_%":100.0,"exact_%":70.3,"response_%":83.2,"action_seq_%":84.0,"slot_action_%":99.0,"slot_target_%":90.2,"slot_direction_%":97.0,"slot_magnitude_%":98.0,"slot_instrument_%":92.3,"stop_recall_%":93.8,"unsafe_exec_%":14.6,"over_refusal_%":13.8,"variant_consistency_%":36.8,"latency_median_s":null,"latency_p90_s":null,"cost_usd":0},"asr_kor_or":{"n":495,"errors":0,"valid_json_%":100.0,"exact_%":69.5,"response_%":85.3,"action_seq_%":80.6,"slot_action_%":96.7,"slot_target_%":88.7,"slot_direction_%":95.2,"slot_magnitude_%":97.2,"slot_instrument_%":85.0,"stop_recall_%":93.8,"unsafe_exec_%":11.0,"over_refusal_%":14.8,"variant_consistency_%":36.1,"latency_median_s":null,"latency_p90_s":null,"cost_usd":0},"overheard":{"n":600,"errors":0,"valid_json_%":100.0,"exact_%":96.0,"response_%":96.0,"action_seq_%":null,"slot_action_%":null,"slot_target_%":null,"slot_direction_%":null,"slot_magnitude_%":null,"slot_instrument_%":null,"stop_recall_%":null,"unsafe_exec_%":2.5,"over_refusal_%":null,"variant_consistency_%":null,"latency_median_s":null,"latency_p90_s":null,"cost_usd":0}},"paired":{"asr_generic_vs_clean":{"n_items":495,"n_groups":219,"a_%":92.1,"b_%":70.3,"diff_pp":-21.8,"ci95_lo":-26.1,"ci95_hi":-17.7,"only_a":111,"only_b":3,"mcnemar_p":2.378716074312675e-29},"asr_kor_or_vs_clean":{"n_items":495,"n_groups":219,"a_%":92.1,"b_%":69.5,"diff_pp":-22.6,"ci95_lo":-26.5,"ci95_hi":-18.6,"only_a":118,"only_b":6,"mcnemar_p":4.420164035915105e-28}},"response_format":{"json_schema":2085},"request":{"thinking_off":{"chat_template_kwargs":{"enable_thinking":false}},"json_schema":{}}},"config":{"max_tokens":800,"prompt":"voice-command-v0.1","response_format":"json_schema","subsets":["clean","asr_generic","asr_kor_or","overheard"],"temperature":0,"thinking":false},"run_key":"228735a94dea4ef1abc587594ba1951c5e5e5d7fea22d9d95752d95184010bf5","harness_version":"voice-command-p0","task_spec_hash":"3bc7dc79688a46dc00af2790c9d049d196d1da86de20a637020129b5336e6551","model_id":"alibaba/qwen3.8-flash-next-180b-a6b","tokens_in":2213854,"tokens_out":97733,"dataset":{"slug":"kor-surgical-robot-voice-command","title":"Korean Surgical Robot Voice-Command Benchmark","license":"CC BY 4.0","license_url":"https://creativecommons.org/licenses/by/4.0/","citation":"Lee H. Korean Surgical Robot Voice-Command Benchmark (version 1.0.0). Seoul National University Hospital. KHDP. 2026. doi:10.23421/2026.03507"},"predictions_url":"https://medbench.kr/artifacts/704eca6b0f7d/predictions.parquet","bibtex":"@misc{medbench_704eca6b0f7d,\n  title = {Qwen3.8 Flash Next 180B-A6B on Korean surgical robot voice commands (kor-surgical-robot-voice-command@0)},\n  author = {{medbench.kr}},\n  year = {2026},\n  howpublished = {\\url{https://medbench.kr/r/704eca6b0f7d}},\n  note = {Evaluated 2026-10-29, harness voice-command-p0, run key 228735a94dea}\n}"}