{ "format": "attestql/audit/summary/2", "run_id": "audit-e68aafff-7739-4280-b3dc-d30f0144b6d4", "backend_identity": "SQLite 3.53.4 | file=/private/tmp/attestql-runs/data/minidev/dev_databases/financial/financial.sqlite | size=71294976", "effective_database_role": "file", "session_settings": { "engine": "sqlite", "recorded": { "sqlite_version": "3.53.4", "encoding": "UTF-8", "reverse_unordered_selects": "0", "query_only": "1", "journal_mode": "delete", "data_version": "1", "compile_options": "ATOMIC_INTRINSICS=1,COMPILER=clang-21.0.0,DEFAULT_AUTOVACUUM,DEFAULT_CACHE_SIZE=-2000,DEFAULT_FILE_FORMAT=4,DEFAULT_JOURNAL_SIZE_LIMIT=-1,DEFAULT_MMAP_SIZE=0,DEFAULT_PAGE_SIZE=4096,DEFAULT_PCACHE_INITSZ=20,DEFAULT_RECURSIVE_TRIGGERS,DEFAULT_SECTOR_SIZE=4096,DEFAULT_SYNCHRONOUS=2,DEFAULT_WAL_AUTOCHECKPOINT=1000,DEFAULT_WAL_SYNCHRONOUS=2,DEFAULT_WORKER_THREADS=0,DIRECT_OVERFLOW_READ,ENABLE_API_ARMOR,ENABLE_COLUMN_METADATA,ENABLE_DBSTAT_VTAB,ENABLE_FTS3,ENABLE_FTS3_PARENTHESIS,ENABLE_FTS5,ENABLE_GEOPOLY,ENABLE_MATH_FUNCTIONS,ENABLE_MEMORY_MANAGEMENT,ENABLE_PERCENTILE,ENABLE_PREUPDATE_HOOK,ENABLE_RTREE,ENABLE_SESSION,ENABLE_STAT4,ENABLE_UNLOCK_NOTIFY,MALLOC_SOFT_LIMIT=1024,MAX_ATTACHED=10,MAX_COLUMN=2000,MAX_COMPOUND_SELECT=500,MAX_DEFAULT_PAGE_SIZE=8192,MAX_EXPR_DEPTH=1000,MAX_FUNCTION_ARG=1000,MAX_LENGTH=1000000000,MAX_LIKE_PATTERN_LENGTH=50000,MAX_MMAP_SIZE=0x7fff0000,MAX_PAGE_COUNT=0xfffffffe,MAX_PAGE_SIZE=65536,MAX_SQL_LENGTH=1000000000,MAX_TRIGGER_DEPTH=1000,MAX_VARIABLE_NUMBER=250000,MAX_VDBE_OP=250000000,MAX_WORKER_THREADS=8,MUTEX_PTHREADS,SYSTEM_MALLOC,TEMP_STORE=1,THREADSAFE=1,USE_URI", "collation_list": "RTRIM,NOCASE,BINARY", "case_sensitive_like": "0" } }, "parser": { "validator": "audit:sqlglot-sqlite-parse", "sqlglot": "30.18.0", "dialect": "sqlite" }, "question_set": { "path": "data/questions/mini_dev_sqlite.json", "digest": "sha256:4ba5fa8de55856222f484d380d2ba872b380bf79d825de70478e2120cb0fc43b", "origin": "https://bird-bench.oss-cn-beijing.aliyuncs.com/minidev.zip (sha256 cc48ba16838204e4e214512030cb572eeb5f7bcdd999bae4b9b6ff12ec13b92f, downloaded 2026-09-07), member minidev/MINIDEV/mini_dev_sqlite.json", "date": "2024-06-19", "entries": 500, "audited": 30, "duplicate_ids": [ 137, 138 ], "ids": [ 89, 92, 93, 94, 95, 98, 99, 100, 112, 115, 116, 117, 118, 125, 128, 136, 137, 138, 129, 145, 149, 152, 159, 168, 169, 173, 186, 189, 192, 194 ] }, "predictions": { "path": "data/preds-sqlite/predict_mini_dev_phi-3-medium-128k-instruct-1_sqlite.json", "digest": "sha256:5e1a3a37cb174cbe62ff3bd269b293b6473c99da34f3fe3ac6f186dd5a1a7ff5", "origin": "https://raw.githubusercontent.com/bird-bench/mini_dev/b3d4bcbbae9a96934ad812551eb400c7a3b23c12/llm/exp_result/sql_output_kg/predict_mini_dev_phi-3-medium-128k-instruct-1_sqlite.json", "date": "2024-06-19", "keyed_by": "position", "statements": 500, "positions_unused": [ 487, 488 ] }, "verdicts": { "NOT_EQUAL": 13, "ERROR": 13, "EQUAL": 4 }, "smells": { "ordering-over-numeric-text": 1, "arbitrary-cut": 0, "not-a-function-of-the-data": 0, "float-aggregate-order": 0, "direction-against-question": 0 }, "smells_fired": 1, "credited_but_not_equal": { "total": 0, "by_mechanism": { "multiplicity": 0, "type": 0, "order": 0, "truncation": 0, "other": 0 }, "by_test_suite_ex": { "1": 0, "0": 0 } }, "fixture": { "depth": "counts", "cache": "runs/minidev-sqlite/phi-3-medium-128k-instruct-1/financial/fixture.json", "source": { "path": "data/minidev/dev_databases/financial/financial.sqlite", "digest": "sha256:d15d89cdb068a202b6f2b99342af44dffc1d52545b39ceaf62efdc0ba570101e", "origin": "minidev.zip (sha256 cc48ba16838204e4e214512030cb572eeb5f7bcdd999bae4b9b6ff12ec13b92f), member minidev/MINIDEV/dev_databases, financial/financial.sqlite", "date": "2024-06-13" }, "schema_digest": "sha256:dec3b7590cdaf8df74c4ddbef766cac24a33554a327998fe33ef165b15bdf90a", "row_counts": { "main.account": 4500, "main.card": 892, "main.client": 5369, "main.disp": 5369, "main.district": 77, "main.loan": 682, "main.order": 6471, "main.trans": 1056320 }, "content_digests": {}, "measured_tables": [ "district", "account", "client", "disp", "loan", "trans", "order", "card" ], "missing_tables": [], "unreadable_tables": [], "refused": "" }, "planner_statistics": {}, "shuffle": { "seed": 1, "row_limit": 300000, "scratch_schema": "temp", "prepared": true, "reason": "", "copied": [ "account", "card", "client", "disp", "district", "loan", "order" ], "skipped": { "trans": 1056320 }, "not_reached_by_a_copy": {} }, "settings": { "statement_timeout_seconds": 30, "fail_on_smell": false, "experimental_s2": false, "plan_variant": false, "out": "runs/minidev-sqlite/phi-3-medium-128k-instruct-1/financial", "serialization": "attestql/audit/2", "work_mem": null, "hash_mem_multiplier": null }, "data_as_of": "2026-09-08T05:30:12.599549+00:00", "data_as_of_source": "the instant the run started", "elapsed_seconds": { "fixture": 0.01, "questions": 31.889, "shuffle": 0.028 }, "errors": [ { "question_id": 92, "side": "prediction", "step": "execute", "message": "execute: ambiguous column name: district_id" }, { "question_id": 94, "side": "prediction", "step": "statement", "message": "the text does not parse: Invalid expression / Unexpected token. Line 38, Col: 30. t_id = a.client_id JOIN female_clients fc ON c.client_id = fc.client_id JOIN average_salaries as ON \u001b[4mfc\u001b[0m.district_id = as.district_id JOIN lowest_avg_salary las ON as.avg_salary = las.min_avg_salary JOIN" }, { "question_id": 95, "side": "prediction", "step": "execute", "message": "execute: no such column: account_id" }, { "question_id": 98, "side": "prediction", "step": "execute", "message": "execute: no such column: frequency" }, { "question_id": 116, "side": "prediction", "step": "execute", "message": "execute: no such column: balance" }, { "question_id": 137, "side": "prediction", "step": "execute", "message": "execute: ambiguous column name: account_id" }, { "question_id": 138, "side": "prediction", "step": "execute", "message": "execute: the statement ran past its 30s timeout" }, { "question_id": 149, "side": "prediction", "step": "execute", "message": "execute: no such column: account.type" }, { "question_id": 159, "side": "prediction", "step": "execute", "message": "execute: no such column: account.client_id" }, { "question_id": 169, "side": "prediction", "step": "execute", "message": "execute: no such function: YEAR" }, { "question_id": 186, "side": "prediction", "step": "execute", "message": "execute: no such column: frequency" }, { "question_id": 192, "side": "prediction", "step": "execute", "message": "execute: no such column: account.status" }, { "question_id": 194, "side": "prediction", "step": "execute", "message": "execute: ambiguous column name: client_id" } ], "timed_out": { "gold": [], "prediction": [ 138 ] }, "exit_status": 1 }