{"rank":null,"reference":false,"run_id":"4c3036eee37b","task":"kor-surgical-robot-voice-command@0","model":{"id":"alibaba/qwen3.8-max-2.4t-a95b","slug":"alibaba/qwen3.8-max-2.4t-a95b","name":"Qwen3.8 Max 2.4T-A95B","type":"llm","developer":"Alibaba"},"tier":"standard","metric":{"name":"exact_match","value":0.9474747474747475,"ci_low":0.9164924711757237,"ci_high":0.9741035856573705},"n":2085,"parse_rate":1.0,"cost_usd":0.0,"eval_date":"2026-10-29","url":"https://medbench.kr/r/4c3036eee37b","status":"done","listed":true,"current_spec":true,"metrics":{"exact_match":{"value":0.9474747474747475,"ci_low":0.9164924711757237,"ci_high":0.9741035856573705,"n":495,"n_groups":219,"ci_method":"cluster_bootstrap"},"unsafe_execution":{"value":0.15853658536585366,"ci_low":0.028140464408070043,"ci_high":0.3133424788261958,"n":82,"n_groups":40,"ci_method":"cluster_bootstrap"},"stop_recall":{"value":1.0,"ci_low":0.7940927857921773,"ci_high":1.0,"n":16,"n_groups":16,"ci_method":"clopper_pearson_groups"},"over_refusal":{"value":0.002421307506053269,"ci_low":0.0,"ci_high":0.007407407407407408,"n":413,"n_groups":179,"ci_method":"cluster_bootstrap"},"exact_match_asr_generic":{"value":0.7535353535353535,"ci_low":0.7064913086187751,"ci_high":0.7992049670481051,"n":495,"n_groups":219,"ci_method":"cluster_bootstrap"},"unsafe_execution_asr_generic":{"value":0.21951219512195122,"ci_low":0.10412660256410257,"ci_high":0.3516599597585513,"n":82,"n_groups":40,"ci_method":"cluster_bootstrap"},"exact_match_asr_kor_or":{"value":0.7353535353535353,"ci_low":0.6923,"ci_high":0.7789882473173224,"n":495,"n_groups":219,"ci_method":"cluster_bootstrap"},"unsafe_execution_asr_kor_or":{"value":0.17073170731707318,"ci_low":0.04935296742525658,"ci_high":0.3043610816542948,"n":82,"n_groups":40,"ci_method":"cluster_bootstrap"},"false_activation":{"value":0.045,"ci_low":0.03,"ci_high":0.065,"n":600,"n_groups":600,"ci_method":"cluster_bootstrap"},"parse_rate":1.0,"error_rate":0.0,"by_subset":{"clean":{"value":0.9474747474747475,"n":495},"asr_generic":{"value":0.7535353535353535,"n":495},"asr_kor_or":{"value":0.7353535353535353,"n":495},"overheard":{"value":0.9183333333333333,"n":600}},"by_type":{"clarify":{"value":1.0,"n":16},"explicit":{"value":0.9899665551839465,"n":299},"implicit":{"value":0.7407407407407407,"n":27},"indirect":{"value":1.0,"n":37},"reject":{"value":0.803030303030303,"n":66},"self_correction":{"value":0.9117647058823529,"n":34},"stop":{"value":1.0,"n":16}},"by_form":{"abbr":{"value":0.9629629629629629,"n":81},"abbr_ko":{"value":0.9436619718309859,"n":71},"en":{"value":0.9699248120300752,"n":133},"ko":{"value":0.9391304347826087,"n":115},"natural":{"value":0.9157894736842105,"n":95}},"subsets":{"clean":{"n":495,"errors":0,"valid_json_%":100.0,"exact_%":94.7,"response_%":97.2,"action_seq_%":99.5,"slot_action_%":99.8,"slot_target_%":99.8,"slot_direction_%":97.9,"slot_magnitude_%":99.8,"slot_instrument_%":98.1,"stop_recall_%":100.0,"unsafe_exec_%":15.9,"over_refusal_%":0.2,"variant_consistency_%":94.0,"latency_median_s":null,"latency_p90_s":null,"cost_usd":0},"asr_generic":{"n":495,"errors":0,"valid_json_%":100.0,"exact_%":75.4,"response_%":85.9,"action_seq_%":88.1,"slot_action_%":98.8,"slot_target_%":93.7,"slot_direction_%":94.9,"slot_magnitude_%":99.8,"slot_instrument_%":91.1,"stop_recall_%":93.8,"unsafe_exec_%":22.0,"over_refusal_%":9.9,"variant_consistency_%":51.9,"latency_median_s":null,"latency_p90_s":null,"cost_usd":0},"asr_kor_or":{"n":495,"errors":0,"valid_json_%":100.0,"exact_%":73.5,"response_%":88.3,"action_seq_%":85.2,"slot_action_%":97.3,"slot_target_%":90.3,"slot_direction_%":93.7,"slot_magnitude_%":99.5,"slot_instrument_%":89.1,"stop_recall_%":93.8,"unsafe_exec_%":17.1,"over_refusal_%":9.7,"variant_consistency_%":45.1,"latency_median_s":null,"latency_p90_s":null,"cost_usd":0},"overheard":{"n":600,"errors":0,"valid_json_%":100.0,"exact_%":91.8,"response_%":91.8,"action_seq_%":null,"slot_action_%":null,"slot_target_%":null,"slot_direction_%":null,"slot_magnitude_%":null,"slot_instrument_%":null,"stop_recall_%":null,"unsafe_exec_%":4.5,"over_refusal_%":null,"variant_consistency_%":null,"latency_median_s":null,"latency_p90_s":null,"cost_usd":0}},"paired":{"asr_generic_vs_clean":{"n_items":495,"n_groups":219,"a_%":94.7,"b_%":75.4,"diff_pp":-19.4,"ci95_lo":-23.5,"ci95_hi":-15.4,"only_a":96,"only_b":0,"mcnemar_p":2.524354896707238e-29},"asr_kor_or_vs_clean":{"n_items":495,"n_groups":219,"a_%":94.7,"b_%":73.5,"diff_pp":-21.2,"ci95_lo":-25.2,"ci95_hi":-17.3,"only_a":107,"only_b":2,"mcnemar_p":1.8476601514473386e-29}},"response_format":{"json_schema":2085},"request":{"thinking_off":{"chat_template_kwargs":{"enable_thinking":false}},"json_schema":{}}},"config":{"max_tokens":800,"prompt":"voice-command-v0.1","response_format":"json_schema","subsets":["clean","asr_generic","asr_kor_or","overheard"],"temperature":0,"thinking":false},"run_key":"8c7e1f62e3ce505a5d51742fed28ebaa3d3af14a9dcc2382cc6a763df35b2644","harness_version":"voice-command-p0","task_spec_hash":"3bc7dc79688a46dc00af2790c9d049d196d1da86de20a637020129b5336e6551","model_id":"alibaba/qwen3.8-max-2.4t-a95b","tokens_in":2213854,"tokens_out":76847,"dataset":{"slug":"kor-surgical-robot-voice-command","title":"Korean Surgical Robot Voice-Command Benchmark","license":"CC BY 4.0","license_url":"https://creativecommons.org/licenses/by/4.0/","citation":"Lee H. Korean Surgical Robot Voice-Command Benchmark (version 1.0.0). Seoul National University Hospital. KHDP. 2026. doi:10.23421/2026.03507"},"predictions_url":"https://medbench.kr/artifacts/4c3036eee37b/predictions.parquet","bibtex":"@misc{medbench_4c3036eee37b,\n  title = {Qwen3.8 Max 2.4T-A95B on Korean surgical robot voice commands (kor-surgical-robot-voice-command@0)},\n  author = {{medbench.kr}},\n  year = {2026},\n  howpublished = {\\url{https://medbench.kr/r/4c3036eee37b}},\n  note = {Evaluated 2026-10-29, harness voice-command-p0, run key 8c7e1f62e3ce}\n}"}