{"rank":null,"reference":false,"run_id":"da9bb98f3974","task":"kor-surgical-robot-voice-command@0","model":{"id":"deepseek/deepseek-v4.1-flash-552b-a16b","slug":"deepseek/deepseek-v4.1-flash-552b-a16b","name":"DeepSeek V4.1 Flash 552B-A16B","type":"llm","developer":"DeepSeek"},"tier":"standard","metric":{"name":"exact_match","value":0.9292929292929293,"ci_low":0.8932183266999603,"ci_high":0.9617749839417051},"n":2085,"parse_rate":1.0,"cost_usd":0.0,"eval_date":"2026-10-29","url":"https://medbench.kr/r/da9bb98f3974","status":"done","listed":true,"current_spec":true,"metrics":{"exact_match":{"value":0.9292929292929293,"ci_low":0.8932183266999603,"ci_high":0.9617749839417051,"n":495,"n_groups":219,"ci_method":"cluster_bootstrap"},"unsafe_execution":{"value":0.15853658536585366,"ci_low":0.028140464408070043,"ci_high":0.3133424788261958,"n":82,"n_groups":40,"ci_method":"cluster_bootstrap"},"stop_recall":{"value":1.0,"ci_low":0.7940927857921773,"ci_high":1.0,"n":16,"n_groups":16,"ci_method":"clopper_pearson_groups"},"over_refusal":{"value":0.01937046004842615,"ci_low":0.002518576037070005,"ci_high":0.042870051635111865,"n":413,"n_groups":179,"ci_method":"cluster_bootstrap"},"exact_match_asr_generic":{"value":0.6323232323232323,"ci_low":0.58299496945742,"ci_high":0.6849953144310995,"n":495,"n_groups":219,"ci_method":"cluster_bootstrap"},"unsafe_execution_asr_generic":{"value":0.12195121951219512,"ci_low":0.02896509491733007,"ci_high":0.24452145214521445,"n":82,"n_groups":40,"ci_method":"cluster_bootstrap"},"exact_match_asr_kor_or":{"value":0.6464646464646465,"ci_low":0.5975984725067343,"ci_high":0.6944638114209827,"n":495,"n_groups":219,"ci_method":"cluster_bootstrap"},"unsafe_execution_asr_kor_or":{"value":0.10975609756097561,"ci_low":0.023249131248329325,"ci_high":0.22099208312716473,"n":82,"n_groups":40,"ci_method":"cluster_bootstrap"},"false_activation":{"value":0.021666666666666667,"ci_low":0.01,"ci_high":0.03337499999999996,"n":600,"n_groups":600,"ci_method":"cluster_bootstrap"},"parse_rate":1.0,"error_rate":0.0,"by_subset":{"clean":{"value":0.9292929292929293,"n":495},"asr_generic":{"value":0.6323232323232323,"n":495},"asr_kor_or":{"value":0.6464646464646465,"n":495},"overheard":{"value":0.975,"n":600}},"by_type":{"clarify":{"value":1.0,"n":16},"explicit":{"value":0.9531772575250836,"n":299},"implicit":{"value":0.9629629629629629,"n":27},"indirect":{"value":0.8378378378378378,"n":37},"reject":{"value":0.803030303030303,"n":66},"self_correction":{"value":0.9705882352941176,"n":34},"stop":{"value":1.0,"n":16}},"by_form":{"abbr":{"value":0.8888888888888888,"n":81},"abbr_ko":{"value":0.9295774647887324,"n":71},"en":{"value":0.9398496240601504,"n":133},"ko":{"value":0.9130434782608695,"n":115},"natural":{"value":0.968421052631579,"n":95}},"subsets":{"clean":{"n":495,"errors":0,"valid_json_%":100.0,"exact_%":92.9,"response_%":95.8,"action_seq_%":96.6,"slot_action_%":99.8,"slot_target_%":100.0,"slot_direction_%":99.6,"slot_magnitude_%":98.7,"slot_instrument_%":100.0,"stop_recall_%":100.0,"unsafe_exec_%":15.9,"over_refusal_%":1.9,"variant_consistency_%":88.7,"latency_median_s":null,"latency_p90_s":null,"cost_usd":0},"asr_generic":{"n":495,"errors":0,"valid_json_%":100.0,"exact_%":63.2,"response_%":76.2,"action_seq_%":74.6,"slot_action_%":99.7,"slot_target_%":92.0,"slot_direction_%":95.2,"slot_magnitude_%":98.3,"slot_instrument_%":90.9,"stop_recall_%":93.8,"unsafe_exec_%":12.2,"over_refusal_%":21.3,"variant_consistency_%":30.8,"latency_median_s":null,"latency_p90_s":null,"cost_usd":0},"asr_kor_or":{"n":495,"errors":0,"valid_json_%":100.0,"exact_%":64.6,"response_%":77.4,"action_seq_%":72.6,"slot_action_%":97.5,"slot_target_%":91.0,"slot_direction_%":96.4,"slot_magnitude_%":98.3,"slot_instrument_%":86.7,"stop_recall_%":93.8,"unsafe_exec_%":11.0,"over_refusal_%":23.0,"variant_consistency_%":33.1,"latency_median_s":null,"latency_p90_s":null,"cost_usd":0},"overheard":{"n":600,"errors":0,"valid_json_%":100.0,"exact_%":97.5,"response_%":97.5,"action_seq_%":null,"slot_action_%":null,"slot_target_%":null,"slot_direction_%":null,"slot_magnitude_%":null,"slot_instrument_%":null,"stop_recall_%":null,"unsafe_exec_%":2.2,"over_refusal_%":null,"variant_consistency_%":null,"latency_median_s":null,"latency_p90_s":null,"cost_usd":0}},"paired":{"asr_generic_vs_clean":{"n_items":495,"n_groups":219,"a_%":92.9,"b_%":63.2,"diff_pp":-29.7,"ci95_lo":-34.6,"ci95_hi":-24.7,"only_a":152,"only_b":5,"mcnemar_p":8.434066226919418e-39},"asr_kor_or_vs_clean":{"n_items":495,"n_groups":219,"a_%":92.9,"b_%":64.6,"diff_pp":-28.3,"ci95_lo":-32.8,"ci95_hi":-23.7,"only_a":146,"only_b":6,"mcnemar_p":5.6595265494425486e-36}},"response_format":{"json_schema":2085},"request":{"thinking_off":{"chat_template_kwargs":{"enable_thinking":false}},"json_schema":{}}},"config":{"max_tokens":800,"prompt":"voice-command-v0.1","response_format":"json_schema","subsets":["clean","asr_generic","asr_kor_or","overheard"],"temperature":0,"thinking":false},"run_key":"5ae276e7ec72b46f7c147affa30e0863ec2f57481bb4a66bef3b7f27f71ab004","harness_version":"voice-command-p0","task_spec_hash":"3bc7dc79688a46dc00af2790c9d049d196d1da86de20a637020129b5336e6551","model_id":"deepseek/deepseek-v4.1-flash-552b-a16b","tokens_in":2554034,"tokens_out":90343,"dataset":{"slug":"kor-surgical-robot-voice-command","title":"Korean Surgical Robot Voice-Command Benchmark","license":"CC BY 4.0","license_url":"https://creativecommons.org/licenses/by/4.0/","citation":"Lee H. Korean Surgical Robot Voice-Command Benchmark (version 1.0.0). Seoul National University Hospital. KHDP. 2026. doi:10.23421/2026.03507"},"predictions_url":"https://medbench.kr/artifacts/da9bb98f3974/predictions.parquet","bibtex":"@misc{medbench_da9bb98f3974,\n  title = {DeepSeek V4.1 Flash 552B-A16B on Korean surgical robot voice commands (kor-surgical-robot-voice-command@0)},\n  author = {{medbench.kr}},\n  year = {2026},\n  howpublished = {\\url{https://medbench.kr/r/da9bb98f3974}},\n  note = {Evaluated 2026-10-29, harness voice-command-p0, run key 5ae276e7ec72}\n}"}