{"rank":null,"reference":false,"run_id":"324a0e18e940","task":"k-or-speech-asr@0","model":{"id":"nvidia/nemotron-3.5-asr-0.6b","slug":"nvidia/nemotron-3.5-asr-0.6b","name":"Nemotron 3.5 ASR 0.6B","type":"asr","developer":"NVIDIA"},"tier":"standard","metric":{"name":"cer","value":0.13630255475579994,"ci_low":0.1312099792606913,"ci_high":0.14122383603507674},"n":8000,"parse_rate":1.0,"cost_usd":5.33336,"eval_date":"2026-10-29","url":"https://medbench.kr/r/324a0e18e940","status":"done","listed":true,"current_spec":true,"metrics":{"cer":{"value":0.13630255475579994,"ci_low":0.1312099792606913,"ci_high":0.14122383603507674,"n":8000,"n_groups":2000,"n_missing":0},"wer":{"value":0.436658838406648,"ci_low":0.4257039257815965,"ci_high":0.44738071201332474,"n":8000,"n_groups":2000},"ser":{"value":0.720625,"ci_low":0.707,"ci_high":0.734,"n":8000,"n_groups":2000},"parse_rate":1.0,"error_rate":0.0,"empty_rate":0.0015,"errors":{},"normaliser":"kor-norm-0","by_voice":{"chul":{"value":0.08371690710821414,"ci_low":0.07876181315255687,"ci_high":0.08840893526607373,"wer":0.343329419203324,"n":2000},"ethan":{"value":0.17311791418244726,"ci_low":0.16547549646084275,"ci_high":0.18055704918269275,"wer":0.49462559841026105,"n":2000},"maia":{"value":0.17469339841215903,"ci_low":0.16727832030138076,"ci_high":0.18208755172777488,"wer":0.48803179477915276,"n":2000},"sohee":{"value":0.11368199932037935,"ci_low":0.1083299046234854,"ci_high":0.11883269452155433,"wer":0.4206485412338542,"n":2000}},"by_cs_style":{"english":{"value":0.3340821566110398,"ci_low":0.2753247045759303,"ci_high":0.38987938961631946,"wer":0.6209349593495935,"n":188},"mixed":{"value":0.1969475779694758,"ci_low":0.1825696416372162,"ci_high":0.2110056412327302,"wer":0.5554455445544555,"n":1604},"none":{"value":0.08964108031272211,"ci_low":0.08412713543485416,"ci_high":0.0955048795183023,"wer":0.3195352839931153,"n":2220},"phonetic_kr":{"value":0.12239833493435799,"ci_low":0.1169248588178204,"ci_high":0.1281302734826906,"wer":0.43572752548656163,"n":3988}},"by_intent":{"airway":{"value":0.13813628899835798,"ci_low":0.1143433860008628,"ci_high":0.16707176107025506,"wer":0.42293233082706766,"n":304},"bleeding_alert":{"value":0.1250794659885569,"ci_low":0.10853522858375148,"ci_high":0.14384721788407476,"wer":0.38324175824175827,"n":384},"chitchat":{"value":0.09325657894736843,"ci_low":0.0741639188081863,"ci_high":0.11441735722543983,"wer":0.2619047619047619,"n":360},"closure_status":{"value":0.15042511445389142,"ci_low":0.13154317367419358,"ci_high":0.17180566006152131,"wer":0.47882231404958675,"n":440},"confirmation":{"value":0.4318181818181818,"ci_low":0.4318181818181818,"ci_high":0.4318181818181818,"wer":0.75,"n":4},"consultation_request":{"value":0.12333333333333334,"ci_low":0.09964549125222694,"ci_high":0.14933138676956037,"wer":0.3758620689655172,"n":232},"drug_administration":{"value":0.19845360824742267,"ci_low":0.17540457730700992,"ci_high":0.21869207850386968,"wer":0.5914723032069971,"n":552},"emergency":{"value":0.14252190237797246,"ci_low":0.12444444511675638,"ci_high":0.163990364422483,"wer":0.49335863377609107,"n":304},"environment_request":{"value":0.1271613832853026,"ci_low":0.10398376113148246,"ci_high":0.15214279282524132,"wer":0.4087136929460581,"n":188},"equipment_issue":{"value":0.10773480662983426,"ci_low":0.09259115543547737,"ci_high":0.12509236701926232,"wer":0.42011834319526625,"n":272},"equipment_setting":{"value":0.09630032644178455,"ci_low":0.08044848598568247,"ci_high":0.11280632011542477,"wer":0.3358267716535433,"n":472},"handoff":{"value":0.13743455497382198,"ci_low":0.11910312865341051,"ci_high":0.15765937908447636,"wer":0.46592920353982303,"n":372},"imaging_request":{"value":0.12464788732394366,"ci_low":0.09807654111660197,"ci_high":0.15729854637199753,"wer":0.41210374639769454,"n":280},"instrument_request":{"value":0.12933268545513443,"ci_low":0.11635302884677828,"ci_high":0.14170923428461124,"wer":0.4289568345323741,"n":896},"neuromonitoring":{"value":0.1887317909168809,"ci_low":0.15253407656091447,"ci_high":0.23031582023862737,"wer":0.47570332480818417,"n":260},"patient_communication":{"value":0.09210526315789473,"ci_low":0.05717990121580547,"ci_high":0.14287823397075364,"wer":0.29245283018867924,"n":36},"specimen_handling":{"value":0.15996503496503497,"ci_low":0.13362305132629312,"ci_high":0.18884903017810897,"wer":0.4306768558951965,"n":364},"status_update":{"value":0.10520890599230347,"ci_low":0.09326305230169914,"ci_high":0.11730171413588529,"wer":0.39900793650793653,"n":964},"surgical_count":{"value":0.1438185443668993,"ci_low":0.12290310475151114,"ci_high":0.16472866760842778,"wer":0.45808823529411763,"n":276},"teaching":{"value":0.038169464428457235,"ci_low":0.027091876157535538,"ci_high":0.05162472649134194,"wer":0.19977426636568849,"n":260},"timeout_checklist":{"value":0.13709677419354838,"ci_low":0.13709677419354838,"ci_high":0.13709677419354838,"wer":0.475,"n":4},"transfusion":{"value":0.18830409356725147,"ci_low":0.15656403342462358,"ci_high":0.22125801344653073,"wer":0.5422413793103448,"n":220},"vital_callout":{"value":0.2182267067410906,"ci_low":0.1899770899357446,"ci_high":0.24584702814790427,"wer":0.6736111111111112,"n":556}},"by_speaker":{"anesthesia_attending":{"value":0.14845288326300984,"ci_low":0.13420210777148242,"ci_high":0.16353555022105554,"wer":0.4697159565580618,"n":800},"anesthesia_resident":{"value":0.14825258875739644,"ci_low":0.13474450957651835,"ci_high":0.16134481729314848,"wer":0.4751351351351351,"n":1244},"anesthesiologist":{"value":0.3235294117647059,"ci_low":0.11764705882352941,"ci_high":0.47115384615384615,"wer":0.5978260869565217,"n":12},"assistant":{"value":0.15249734325185973,"ci_low":0.13273360229047373,"ci_high":0.17074533696456873,"wer":0.45703125,"n":500},"circulating_nurse":{"value":0.1391941391941392,"ci_low":0.1230940220258507,"ci_high":0.1574206816064554,"wer":0.4778225806451613,"n":836},"fellow":{"value":0.15456803753795198,"ci_low":0.13579505706732276,"ci_high":0.17430585028278026,"wer":0.44266612641815234,"n":860},"resident":{"value":0.15437231759656653,"ci_low":0.13312625906628567,"ci_high":0.1774993527186677,"wer":0.4772727272727273,"n":476},"scrub_nurse":{"value":0.1242000673627484,"ci_low":0.1123128612039719,"ci_high":0.135402309379869,"wer":0.43491422805247226,"n":804},"surgeon":{"value":0.11322743162656797,"ci_low":0.10431217820063474,"ci_high":0.12214341474633726,"wer":0.37603489059727974,"n":2468}},"by_digits":{"with digits":{"value":0.19728413185639931,"ci_low":0.1848131930724527,"ci_high":0.21052145585562831,"wer":0.6430597579425114,"n":1704},"without digits":{"value":0.11793552536377522,"ci_low":0.11287716154719768,"ci_high":0.12329459455831622,"wer":0.3718998457339504,"n":6296}},"audio_seconds":28672.0,"billed_minutes":8000},"config":{"language":"ko","response_format":"json","temperature":0},"run_key":"e31087d323e9f8fa5f435bbeb9678009c0f3239806511832752f0be73bcc2f09","harness_version":"asr-p0.1","task_spec_hash":"27b75c766635e4f4f96a4997cf25c8238f344a13fb93ec0c3332df53ca410fec","model_id":"nvidia/nemotron-3.5-asr-0.6b","tokens_in":0,"tokens_out":0,"dataset":{"slug":"k-or-speech","title":"Synthetic K-OR Speech Audio","license":"Apache-2.0 with a use restriction","license_url":"https://www.apache.org/licenses/LICENSE-2.0","citation":"Seoul National University Hospital, Department of Anesthesiology and Pain Medicine. Synthetic K-OR Speech Audio: A Synthetic Korean Operating Room Speech Audio Dataset (version 1.0.0). KHDP. 2026. doi:10.23421/2026.0774"},"predictions_url":"https://medbench.kr/artifacts/324a0e18e940/predictions.parquet","bibtex":"@misc{medbench_324a0e18e940,\n  title = {Nemotron 3.5 ASR 0.6B on Korean operating-room speech recognition (k-or-speech-asr@0)},\n  author = {{medbench.kr}},\n  year = {2026},\n  howpublished = {\\url{https://medbench.kr/r/324a0e18e940}},\n  note = {Evaluated 2026-10-29, harness asr-p0.1, run key e31087d323e9}\n}"}