R-ORD GOLD-ONLY arbitrary-cut not-a-function-of-the-data
superhero · mini_dev_pg-00000-of-00001 from https://huggingface.co/datasets/birdsql/bird_mini_dev/resolve/f65faf4ae3b638c1fa6df1d3370c8d92c8366301/data/mini_dev_pg-00000-of-00001.json (commit f65faf4a, downloaded 2026-09-08)
Which hero was the fastest?
the hint the set supplies: which hero refers to superhero_name; fastest refers to MAX(attribute_value) WHERE attribute_name = 'Speed';
This question was audited without a prediction beside it, so there is nothing to compare the gold with. The probes below read the gold alone.
SELECT T1.superhero_name FROM superhero AS T1 INNER JOIN hero_attribute AS T2 ON T1.id = T2.hero_id INNER JOIN attribute AS T3 ON T2.attribute_id = T3.id WHERE T3.attribute_name = 'Speed' ORDER BY T2.attribute_value DESC NULLS LAST LIMIT 1sha256:0a763f3ca83468d41b2589495f0a57996a35f4573c68ad1be233e093446566b2
from evidence-gold.json, 1 row
| superhero_nametext |
|---|
| Air-Walker |
A smell is a mechanical reason to read this gold statement again. It is a heuristic: it does not state that the statement is wrong, and a maintainer decides.
this statement orders by a text column holding only numbers, and ordering it as a number gives a different answer, so the gold may be sorting 9.5 above 10
no ORDER BY key resolves to a text column
{
"heuristic": true,
"reason": "no ORDER BY key resolves to a text column",
"keys": [
{
"key": "t2.attribute_value",
"column": "hero_attribute.attribute_value",
"declared_type": "bigint",
"not_applicable": "the column is not declared as text"
}
]
}this statement cuts its result at a LIMIT that does not decide which rows come back, so a different but equally correct statement can return other rows and score zero
from smells.json, 10 rows
| Bizarro | 100 |
| Beyonder | 100 |
| Air-Walker | 100 |
| Ben 10 | 100 |
| Silver Surfer | 100 |
| Spectre | 100 |
| Angela | 100 |
| Stardust | 100 |
| Jack of Hearts | 100 |
| Living Tribunal | 100 |
{
"heuristic": true,
"cut": 1,
"offset": 0,
"distinct_kept": false,
"unbounded_sql": "SELECT t1.superhero_name, t2.attribute_value AS attestql_ordering_key_0 FROM superhero t1 JOIN hero_attribute t2 ON t1.id = t2.hero_id JOIN attribute t3 ON t2.attribute_id = t3.id WHERE t3.attribute_name = 'Speed' ORDER BY t2.attribute_value DESC NULLS LAST",
"unbounded_rows": 623,
"projected_columns": [
"superhero_name"
],
"ordering_key_columns": [
"attestql_ordering_key_0"
],
"ordering_keys": [
{
"key": "t2.attribute_value",
"direction": "desc",
"nulls": "last",
"nulls_first_in_effect": false,
"returned_rows_null_in_this_key": 0,
"fires": false
}
],
"tied_at_the_cut": {
"positions": [
0,
1,
2,
3,
4,
5,
6,
7,
8,
9
],
"tied_rows": 40,
"distinct_projected_answers": 40,
"rows": [
[
{
"type": "str",
"value": "Bizarro"
}
],
[
{
"type": "str",
"value": "Beyonder"
}
],
[
{
"type": "str",
"value": "Air-Walker"
}
],
[
{
"type": "str",
"value": "Ben 10"
}
],
[
{
"type": "str",
"value": "Silver Surfer"
}
],
[
{
"type": "str",
"value": "Spectre"
}
],
[
{
"type": "str",
"value": "Angela"
}
],
[
{
"type": "str",
"value": "Stardust"
}
],
[
{
"type": "str",
"value": "Jack of Hearts"
}
],
[
{
"type": "str",
"value": "Living Tribunal"
}
]
]
},
"case": "tie-at-the-cut"
}rerun over the same rows in another physical order this statement gives another answer, so its result depends on how the rows are stored and not only on the data
from smells.json, 1 row
| One-Above-All |
{
"heuristic": true,
"rule": "R-ORD",
"baseline_result_hash": "sha256:0a763f3ca83468d41b2589495f0a57996a35f4573c68ad1be233e093446566b2",
"baseline_result": {
"columns": [
{
"name": "superhero_name",
"declared_type": "text"
}
],
"row_count": 1,
"truncated": false,
"rows_shown": 1,
"rows": [
[
{
"type": "str",
"value": "Air-Walker"
}
]
],
"result_hash": "sha256:0a763f3ca83468d41b2589495f0a57996a35f4573c68ad1be233e093446566b2"
},
"planner_statistics": {
"attribute": {
"last_analyze": null,
"last_autoanalyze": null,
"n_mod_since_analyze": 6
},
"superhero": {
"last_analyze": null,
"last_autoanalyze": "2026-09-08 05:19:12.518866+00",
"n_mod_since_analyze": 0
},
"hero_attribute": {
"last_analyze": null,
"last_autoanalyze": null,
"n_mod_since_analyze": 3738
}
},
"shuffle": {
"seed": "1",
"row_limit": 300000,
"tables": [
"attribute",
"superhero",
"hero_attribute"
],
"tables_not_shuffled": [],
"tables_skipped_for_size": {
"laptimes": 400524,
"legalities": 427907,
"posthistory": 303155,
"trans": 1056320,
"yearmonth": 383282
},
"tables_not_reached_by_a_copy": {}
},
"shuffled_copies": {
"run": true,
"verdict": "not_equal",
"differs": true,
"result_hash": "sha256:9d5e11be80ae58af70c6b3552619d568094411d8d154e5c0a822c22a333242a1",
"result": {
"columns": [
{
"name": "superhero_name",
"declared_type": "text"
}
],
"row_count": 1,
"truncated": false,
"rows_shown": 1,
"rows": [
[
{
"type": "str",
"value": "One-Above-All"
}
]
],
"result_hash": "sha256:9d5e11be80ae58af70c6b3552619d568094411d8d154e5c0a822c22a333242a1"
}
},
"plan_variant": {
"run": false,
"reason": "the plan variant was not asked for"
}
}SELECT T1.superhero_name FROM superhero AS T1 INNER JOIN hero_attribute AS T2 ON T1.id = T2.hero_id INNER JOIN attribute AS T3 ON T2.attribute_id = T3.id WHERE T3.attribute_name = 'Speed' ORDER BY T2.attribute_value DESC NULLS LAST LIMIT 1result_hash sha256:0a763f3ca83468d41b2589495f0a57996a35f4573c68ad1be233e093446566b2 recomputed from this JSON: match
record_hash sha256:d6ccff7ff9ffe6bd2f2e976bdfb3fe0533eb841a2857c13df936f42336791be4 recomputed from this JSON: match
from evidence-gold.json, 1 row
| superhero_nametext |
|---|
| Air-Walker |
re-run this statement read-only against PostgreSQL 16.15 (Debian 16.15-1.pgdg13+2) on aarch64-unknown-linux-gnu, compiled by gcc (Debian 14.2.0-19) 14.2.0, 64-bit | server=172.17.0.2/32:5432 | database=bird under the session settings and over the data this record's fixture digest names, and compare the two results under R-ORD