{"rank":null,"reference":false,"run_id":"a8444ec73f4e","task":"k-or-speech-asr@0","model":{"id":"alibaba/qwen3-asr-1.7b","slug":"alibaba/qwen3-asr-1.7b","name":"Qwen3-ASR 1.7B","type":"asr","developer":"Alibaba"},"tier":"standard","metric":{"name":"cer","value":0.07257267307157642,"ci_low":0.06846077673945576,"ci_high":0.07644100962199461},"n":8000,"parse_rate":1.0,"cost_usd":5.33336,"eval_date":"2026-10-29","url":"https://medbench.kr/r/a8444ec73f4e","status":"done","listed":true,"current_spec":true,"metrics":{"cer":{"value":0.07257267307157642,"ci_low":0.06846077673945576,"ci_high":0.07644100962199461,"n":8000,"n_groups":2000,"n_missing":0},"wer":{"value":0.2598229608888086,"ci_low":0.25013373720605786,"ci_high":0.26992977433976545,"n":8000,"n_groups":2000},"ser":{"value":0.464375,"ci_low":0.4485,"ci_high":0.479503125,"n":8000,"n_groups":2000},"parse_rate":1.0,"error_rate":0.0,"empty_rate":0.0,"errors":{},"normaliser":"kor-norm-0","by_voice":{"chul":{"value":0.04334126224089463,"ci_low":0.039930021187549965,"ci_high":0.04695123112331224,"wer":0.21335019420106585,"n":2000},"ethan":{"value":0.10080009885391246,"ci_low":0.09465299173860205,"ci_high":0.10657769356865025,"wer":0.2996115978683046,"n":2000},"maia":{"value":0.08411850112755244,"ci_low":0.07848995012870635,"ci_high":0.08916073864450734,"wer":0.27567518742661007,"n":2000},"sohee":{"value":0.06203083006394612,"ci_low":0.05726418228364841,"ci_high":0.06649738902390381,"wer":0.2506548640592539,"n":2000}},"by_cs_style":{"english":{"value":0.2063543003851091,"ci_low":0.15316476048112354,"ci_high":0.2630660493105865,"wer":0.3465447154471545,"n":188},"mixed":{"value":0.11048440610484406,"ci_low":0.09974676230402348,"ci_high":0.12106038616604557,"wer":0.39772277227722774,"n":1604},"none":{"value":0.031212982705520018,"ci_low":0.027927555921948646,"ci_high":0.034805599319672566,"wer":0.14733218588640276,"n":2220},"phonetic_kr":{"value":0.06996477745757285,"ci_low":0.06528918134778772,"ci_high":0.07487823669646189,"wer":0.25189990732159406,"n":3988}},"by_intent":{"airway":{"value":0.09339080459770115,"ci_low":0.0694109464080943,"ci_high":0.12055238856526722,"wer":0.2957393483709273,"n":304},"bleeding_alert":{"value":0.050699300699300696,"ci_low":0.04034652738139207,"ci_high":0.06305423589629913,"wer":0.21565934065934067,"n":384},"chitchat":{"value":0.03223684210526316,"ci_low":0.019519124839442743,"ci_high":0.05085481006391995,"wer":0.10714285714285714,"n":360},"closure_status":{"value":0.07619359058207979,"ci_low":0.06105811819473255,"ci_high":0.09455000216052051,"wer":0.26962809917355374,"n":440},"confirmation":{"value":0.3409090909090909,"ci_low":0.3409090909090909,"ci_high":0.3409090909090909,"wer":0.625,"n":4},"consultation_request":{"value":0.058627450980392154,"ci_low":0.042732055402947146,"ci_high":0.07824372210518427,"wer":0.19942528735632184,"n":232},"drug_administration":{"value":0.13698453608247424,"ci_low":0.12028725266879718,"ci_high":0.1544443106507037,"wer":0.4362244897959184,"n":552},"emergency":{"value":0.06711514392991239,"ci_low":0.052738933356823246,"ci_high":0.08391332488236342,"wer":0.3870967741935484,"n":304},"environment_request":{"value":0.06304034582132564,"ci_low":0.04311263683205776,"ci_high":0.08505427819979502,"wer":0.22821576763485477,"n":188},"equipment_issue":{"value":0.049723756906077346,"ci_low":0.03776791431819592,"ci_high":0.06365091901654917,"wer":0.20019723865877712,"n":272},"equipment_setting":{"value":0.048286180631120786,"ci_low":0.037055656818217965,"ci_high":0.06189681288714766,"wer":0.21181102362204723,"n":472},"handoff":{"value":0.06602675974403723,"ci_low":0.05083022278989592,"ci_high":0.08432792363158392,"wer":0.25398230088495577,"n":372},"imaging_request":{"value":0.07370892018779343,"ci_low":0.04730165300115632,"ci_high":0.10786536836174108,"wer":0.24927953890489912,"n":280},"instrument_request":{"value":0.07118561710398445,"ci_low":0.0610766053167774,"ci_high":0.08043711305813367,"wer":0.22054856115107913,"n":896},"neuromonitoring":{"value":0.08847472150814054,"ci_low":0.06614398112670757,"ci_high":0.11407739628141993,"wer":0.30562659846547313,"n":260},"patient_communication":{"value":0.024671052631578948,"ci_low":0.005680222165474974,"ci_high":0.053036512667660206,"wer":0.10377358490566038,"n":36},"specimen_handling":{"value":0.10297202797202797,"ci_low":0.07522266175577938,"ci_high":0.13354373646622497,"wer":0.2046943231441048,"n":364},"status_update":{"value":0.05229521715228147,"ci_low":0.043531833785345246,"ci_high":0.06109454444341465,"wer":0.198015873015873,"n":964},"surgical_count":{"value":0.06031904287138584,"ci_low":0.04743844743729014,"ci_high":0.0755680492924895,"wer":0.24191176470588235,"n":276},"teaching":{"value":0.01498800959232614,"ci_low":0.009607759100597874,"ci_high":0.02112934551754974,"wer":0.09311512415349887,"n":260},"timeout_checklist":{"value":0.10483870967741936,"ci_low":0.10483870967741936,"ci_high":0.10483870967741936,"wer":0.3,"n":4},"transfusion":{"value":0.06842105263157895,"ci_low":0.0507845905029752,"ci_high":0.08878091089211536,"wer":0.3439655172413793,"n":220},"vital_callout":{"value":0.14469729497638473,"ci_low":0.12400898319271735,"ci_high":0.16479511974826477,"wer":0.5031565656565656,"n":556}},"by_speaker":{"anesthesia_attending":{"value":0.07686357243319268,"ci_low":0.064490906541618,"ci_high":0.0896254088203393,"wer":0.3084795321637427,"n":800},"anesthesia_resident":{"value":0.07969674556213018,"ci_low":0.06930927768821342,"ci_high":0.09002731351889234,"wer":0.28864864864864864,"n":1244},"anesthesiologist":{"value":0.19852941176470587,"ci_low":0.058823529411764705,"ci_high":0.36538461538461536,"wer":0.4782608695652174,"n":12},"assistant":{"value":0.08926673751328375,"ci_low":0.07206397360382104,"ci_high":0.10670828493999225,"wer":0.26640625,"n":500},"circulating_nurse":{"value":0.07524420024420024,"ci_low":0.06253816379628067,"ci_high":0.09021913919697412,"wer":0.28203405017921146,"n":836},"fellow":{"value":0.08294231300027602,"ci_low":0.06862804540227362,"ci_high":0.09707799468728612,"wer":0.25972447325769854,"n":860},"resident":{"value":0.08784871244635194,"ci_low":0.07099824751070774,"ci_high":0.10641364453726473,"wer":0.289576802507837,"n":476},"scrub_nurse":{"value":0.06348939036712697,"ci_low":0.05382533158183595,"ci_high":0.07379457469688733,"wer":0.24495459132189706,"n":804},"surgeon":{"value":0.05801460004112688,"ci_low":0.0518664076069803,"ci_high":0.06385985528507342,"wer":0.21555292726197517,"n":2468}},"by_digits":{"with digits":{"value":0.1308888295742693,"ci_low":0.12172957259759316,"ci_high":0.14119268018274445,"wer":0.46520423600605143,"n":1704},"without digits":{"value":0.055008441193021944,"ci_low":0.05123159149336107,"ci_high":0.0589783687018356,"wer":0.19538388513112614,"n":6296}},"audio_seconds":28672.0,"billed_minutes":8000},"config":{"language":"ko","response_format":"json","temperature":0},"run_key":"73349543b164d916ac44c2a973df1438786e918bce154c52fd4179223c4f9154","harness_version":"asr-p0.1","task_spec_hash":"27b75c766635e4f4f96a4997cf25c8238f344a13fb93ec0c3332df53ca410fec","model_id":"alibaba/qwen3-asr-1.7b","tokens_in":0,"tokens_out":0,"dataset":{"slug":"k-or-speech","title":"Synthetic K-OR Speech Audio","license":"Apache-2.0 with a use restriction","license_url":"https://www.apache.org/licenses/LICENSE-2.0","citation":"Seoul National University Hospital, Department of Anesthesiology and Pain Medicine. Synthetic K-OR Speech Audio: A Synthetic Korean Operating Room Speech Audio Dataset (version 1.0.0). KHDP. 2026. doi:10.23421/2026.0774"},"predictions_url":"https://medbench.kr/artifacts/a8444ec73f4e/predictions.parquet","bibtex":"@misc{medbench_a8444ec73f4e,\n  title = {Qwen3-ASR 1.7B on Korean operating-room speech recognition (k-or-speech-asr@0)},\n  author = {{medbench.kr}},\n  year = {2026},\n  howpublished = {\\url{https://medbench.kr/r/a8444ec73f4e}},\n  note = {Evaluated 2026-10-29, harness asr-p0.1, run key 73349543b164}\n}"}