{"rank":null,"reference":true,"run_id":"8d826bff8fe5","task":"synthetic-snuh-text2sql@0","model":{"id":"medbench/text2sql-constant-query","slug":"medbench/text2sql-constant-query","name":"Constant query (SELECT 0 for every question)","type":"baseline","developer":"medbench.kr"},"tier":"baseline","metric":{"name":"rs0","value":0.0,"ci_low":0.0,"ci_high":0.0},"n":510,"parse_rate":1.0,"cost_usd":0.0,"eval_date":"2026-10-30","url":"https://medbench.kr/r/8d826bff8fe5","status":"done","listed":true,"current_spec":true,"metrics":{"hard_ex":{"value":0.0,"ci_low":0.0,"ci_high":0.0,"n":166,"n_groups":83,"ci_method":"95% percentile interval of a bootstrap over questions (the Korean and English versions of a question resampled together), 2,000 resamples, seed 0"},"rs0":{"value":0.0,"ci_low":0.0,"ci_high":0.0,"n":202,"n_groups":101,"ci_method":"95% percentile interval of a bootstrap over questions (the Korean and English versions of a question resampled together), 2,000 resamples, seed 0"},"hard_rs5":{"value":-5.0,"ci_low":-5.0,"ci_high":-5.0,"n":202,"n_groups":101,"ci_method":"95% percentile interval of a bootstrap over questions (the Korean and English versions of a question resampled together), 2,000 resamples, seed 0"},"hard_rsN":{"value":-202.0,"ci_low":-202.0,"ci_high":-202.0,"n":202,"n_groups":101,"ci_method":"95% percentile interval of a bootstrap over questions (the Korean and English versions of a question resampled together), 2,000 resamples, seed 0","N":202},"abstention":{"answerable_abstained":{"value":0.0,"n":166},"unanswerable_abstained":{"value":0.0,"n":36}},"easy_ex":{"value":0.0,"ci_low":0.0,"ci_high":0.0,"n":308,"n_groups":154,"ci_method":"95% percentile interval of a bootstrap over questions (the Korean and English versions of a question resampled together), 2,000 resamples, seed 0"},"answers":{"hard":{"sql":202,"abstain":0,"cut_off":0,"format_failure":0,"api_error":0},"easy":{"sql":308,"abstain":0,"cut_off":0,"format_failure":0,"api_error":0}},"cut_off_rate":{"hard":0.0,"easy":0.0,"all":0.0},"format_failure_rate":{"hard":0.0,"easy":0.0,"all":0.0},"api_error_rate":0.0,"parse_rate":1.0,"tiers":{"hard":{"by_language":{"ko":{"value":0.0,"n":83},"en":{"value":0.0,"n":83}},"by_category":{"clinical_terms":{"value":0.0,"n":42},"cohort_definition":{"value":0.0,"n":42},"data_quality":{"value":0.0,"n":40},"relative_time":{"value":0.0,"n":42},"unanswerable":{"value":0.0,"n":36}},"by_difficulty":{"hard":{"value":0.0,"n":166}},"sql_rate":1.0,"execution_error_rate":0.0,"errors":{},"prompt":"text2sql-omop-hard-p0","scorer":"ex-rs-0.1","reliability":{"RS(0)":0.0,"RS(5)":-5.0,"RS(N)":-202.0,"N":202,"by_language":{"ko":{"RS(0)":0.0,"RS(5)":-5.0},"en":{"RS(0)":0.0,"RS(5)":-5.0}}},"n_questions":101,"n_instances":202},"easy":{"by_language":{"ko":{"value":0.0,"n":154},"en":{"value":0.0,"n":154}},"by_category":{"patient_lookup":{"value":0.0,"n":48},"cohort_count":{"value":0.0,"n":54},"aggregation":{"value":0.0,"n":54},"temporal":{"value":0.0,"n":52},"multi_domain_join":{"value":0.0,"n":52},"vocabulary_lookup":{"value":0.0,"n":48}},"by_difficulty":{"easy":{"value":0.0,"n":72},"medium":{"value":0.0,"n":176},"hard":{"value":0.0,"n":60}},"sql_rate":1.0,"execution_error_rate":0.0,"errors":{},"prompt":"text2sql-omop-p0","scorer":"ex-0.1","n_questions":154,"n_instances":308}},"scorer":{"execution":"ex-0.1 (ex-rs-0.1 with abstention)","reliability":"glee4810/ehrsql-2024@f9e1aa0 scoring_utils.reliability_score and penalize","intervals":"95% percentile interval of a bootstrap over questions (the Korean and English versions of a question resampled together), 2,000 resamples, seed 0"},"prompts":{"hard":"text2sql-omop-hard-p0","easy":"text2sql-omop-p0"}},"config":{"prompts":{"easy":"text2sql-omop-p0","hard":"text2sql-omop-hard-p0"},"rule":"SELECT 0 for every item of both tiers","tiers":["hard","easy"]},"run_key":"e181500f494ba47e812b49e6eeb14d5c0f79ac558be6c7f2e42e09992f23c8b1","harness_version":"p0.1","task_spec_hash":"abb0fa4ae8e35e706e34c4862ff8a1c1a3aba1f57c39900eb3ee3d6685f4650c","model_id":"medbench/text2sql-constant-query","tokens_in":0,"tokens_out":0,"dataset":{"slug":"synthetic-snuh","title":"Synthetic-SNUH v4","license":"CC BY-NC 4.0","license_url":"https://creativecommons.org/licenses/by-nc/4.0/","citation":"Lee H-C, Seoul National University Hospital. Synthetic-SNUH v4.0.0: synthetic OMOP CDM and KR Core FHIR records for 10,000 virtual patients (version 4.0.0). KHDP. 2026. doi:10.23421/2026.03183"},"predictions_url":"https://medbench.kr/artifacts/8d826bff8fe5/predictions.parquet","bibtex":"@misc{medbench_8d826bff8fe5,\n  title = {Constant query (SELECT 0 for every question) on Synthetic-SNUH Text-to-SQL (OMOP CDM, Korean and English) (synthetic-snuh-text2sql@0)},\n  author = {{medbench.kr}},\n  year = {2026},\n  howpublished = {\\url{https://medbench.kr/r/8d826bff8fe5}},\n  note = {Evaluated 2026-10-30, harness p0.1, run key e181500f494b}\n}"}