{"rank":null,"reference":true,"run_id":"a923656cf11c","task":"kor-surgical-robot-voice-command@0","model":{"id":"medbench/prior","slug":"medbench/prior","name":"Prior (ignores the input)","type":"baseline","developer":"medbench.kr"},"tier":"baseline","metric":{"name":"exact_match","value":0.13333333333333333,"ci_low":0.08751759172666769,"ci_high":0.1916674901185771},"n":2085,"parse_rate":1.0,"cost_usd":0.0,"eval_date":"2026-10-29","url":"https://medbench.kr/r/a923656cf11c","status":"done","listed":true,"current_spec":true,"metrics":{"exact_match":{"value":0.13333333333333333,"ci_low":0.08751759172666769,"ci_high":0.1916674901185771,"n":495,"n_groups":219,"ci_method":"cluster_bootstrap"},"unsafe_execution":{"value":0.0,"ci_low":0.0,"ci_high":0.08809730287880237,"n":82,"n_groups":40,"ci_method":"clopper_pearson_groups"},"stop_recall":{"value":0.0,"ci_low":0.0,"ci_high":0.2059072142078227,"n":16,"n_groups":16,"ci_method":"clopper_pearson_groups"},"over_refusal":{"value":1.0,"ci_low":0.9796026339494179,"ci_high":1.0,"n":413,"n_groups":179,"ci_method":"clopper_pearson_groups"},"exact_match_asr_generic":{"value":0.13333333333333333,"ci_low":0.08751759172666769,"ci_high":0.1916674901185771,"n":495,"n_groups":219,"ci_method":"cluster_bootstrap"},"unsafe_execution_asr_generic":{"value":0.0,"ci_low":0.0,"ci_high":0.08809730287880237,"n":82,"n_groups":40,"ci_method":"clopper_pearson_groups"},"exact_match_asr_kor_or":{"value":0.13333333333333333,"ci_low":0.08751759172666769,"ci_high":0.1916674901185771,"n":495,"n_groups":219,"ci_method":"cluster_bootstrap"},"unsafe_execution_asr_kor_or":{"value":0.0,"ci_low":0.0,"ci_high":0.08809730287880237,"n":82,"n_groups":40,"ci_method":"clopper_pearson_groups"},"false_activation":{"value":0.0,"ci_low":0.0,"ci_high":0.006129271330669258,"n":600,"n_groups":600,"ci_method":"clopper_pearson_groups"},"parse_rate":1.0,"error_rate":0.0,"by_subset":{"clean":{"value":0.13333333333333333,"n":495},"asr_generic":{"value":0.13333333333333333,"n":495},"asr_kor_or":{"value":0.13333333333333333,"n":495},"overheard":{"value":1.0,"n":600}},"by_type":{"clarify":{"value":0.0,"n":16},"explicit":{"value":0.0,"n":299},"implicit":{"value":0.0,"n":27},"indirect":{"value":0.0,"n":37},"reject":{"value":1.0,"n":66},"self_correction":{"value":0.0,"n":34},"stop":{"value":0.0,"n":16}},"by_form":{"abbr":{"value":0.14814814814814814,"n":81},"abbr_ko":{"value":0.14084507042253522,"n":71},"en":{"value":0.14285714285714285,"n":133},"ko":{"value":0.1565217391304348,"n":115},"natural":{"value":0.07368421052631578,"n":95}},"subsets":{"clean":{"n":495,"errors":0,"valid_json_%":100.0,"exact_%":13.3,"response_%":13.3,"action_seq_%":0.0,"slot_action_%":null,"slot_target_%":null,"slot_direction_%":null,"slot_magnitude_%":null,"slot_instrument_%":null,"stop_recall_%":0.0,"unsafe_exec_%":0.0,"over_refusal_%":100.0,"variant_consistency_%":14.3,"latency_median_s":null,"latency_p90_s":null,"cost_usd":0},"asr_generic":{"n":495,"errors":0,"valid_json_%":100.0,"exact_%":13.3,"response_%":13.3,"action_seq_%":0.0,"slot_action_%":null,"slot_target_%":null,"slot_direction_%":null,"slot_magnitude_%":null,"slot_instrument_%":null,"stop_recall_%":0.0,"unsafe_exec_%":0.0,"over_refusal_%":100.0,"variant_consistency_%":14.3,"latency_median_s":null,"latency_p90_s":null,"cost_usd":0},"asr_kor_or":{"n":495,"errors":0,"valid_json_%":100.0,"exact_%":13.3,"response_%":13.3,"action_seq_%":0.0,"slot_action_%":null,"slot_target_%":null,"slot_direction_%":null,"slot_magnitude_%":null,"slot_instrument_%":null,"stop_recall_%":0.0,"unsafe_exec_%":0.0,"over_refusal_%":100.0,"variant_consistency_%":14.3,"latency_median_s":null,"latency_p90_s":null,"cost_usd":0},"overheard":{"n":600,"errors":0,"valid_json_%":100.0,"exact_%":100.0,"response_%":100.0,"action_seq_%":null,"slot_action_%":null,"slot_target_%":null,"slot_direction_%":null,"slot_magnitude_%":null,"slot_instrument_%":null,"stop_recall_%":null,"unsafe_exec_%":0.0,"over_refusal_%":null,"variant_consistency_%":null,"latency_median_s":null,"latency_p90_s":null,"cost_usd":0}},"paired":{"asr_generic_vs_clean":{"n_items":495,"n_groups":219,"a_%":13.3,"b_%":13.3,"diff_pp":0.0,"ci95_lo":0.0,"ci95_hi":0.0,"only_a":0,"only_b":0,"mcnemar_p":1.0},"asr_kor_or_vs_clean":{"n_items":495,"n_groups":219,"a_%":13.3,"b_%":13.3,"diff_pp":0.0,"ci95_lo":0.0,"ci95_hi":0.0,"only_a":0,"only_b":0,"mcnemar_p":1.0}}},"config":{"output":{"actions":[],"message":null,"response":"reject"},"rule":"most frequent (response, actions) of train/train_v0.2.jsonl","subsets":["clean","asr_generic","asr_kor_or","overheard"]},"run_key":"0075c68ad56285efdc41f69bc2af138195fd90b0c2bffbada7d4395d4985341f","harness_version":"p0.1","task_spec_hash":"3bc7dc79688a46dc00af2790c9d049d196d1da86de20a637020129b5336e6551","model_id":"medbench/prior","tokens_in":0,"tokens_out":0,"dataset":{"slug":"kor-surgical-robot-voice-command","title":"Korean Surgical Robot Voice-Command Benchmark","license":"CC BY 4.0","license_url":"https://creativecommons.org/licenses/by/4.0/","citation":"Lee H. Korean Surgical Robot Voice-Command Benchmark (version 1.0.0). Seoul National University Hospital. KHDP. 2026. doi:10.23421/2026.03507"},"predictions_url":"https://medbench.kr/artifacts/a923656cf11c/predictions.parquet","bibtex":"@misc{medbench_a923656cf11c,\n  title = {Prior (ignores the input) on Korean surgical robot voice commands (kor-surgical-robot-voice-command@0)},\n  author = {{medbench.kr}},\n  year = {2026},\n  howpublished = {\\url{https://medbench.kr/r/a923656cf11c}},\n  note = {Evaluated 2026-10-29, harness p0.1, run key 0075c68ad562}\n}"}