{
  "format": "attestql/audit/counterexample/2",
  "question": {
    "question_id": "1035",
    "question_set": "mini_dev_postgresql from https://bird-bench.oss-cn-beijing.aliyuncs.com/minidev.zip (sha256 cc48ba16838204e4e214512030cb572eeb5f7bcdd999bae4b9b6ff12ec13b92f, downloaded 2026-09-07), member minidev/MINIDEV/mini_dev_postgresql.json",
    "question_text": "Give the team_fifa_api_id of teams with more than 50 but less than 60 build-up play speed.",
    "evidence_text": "teams with more than 50 but less than 60 build-up play speed refers to buildUpPlaySpeed >50 AND buildUpPlaySpeed <60; "
  },
  "question_set_version": "sha256:d2731292f20b8d8569cd956dd747ffe1df13cd625076263e38ae9ebcef50b1ab",
  "sources": {
    "gold": {
      "path": "data/questions/mini_dev_postgresql.json",
      "digest": "sha256:d2731292f20b8d8569cd956dd747ffe1df13cd625076263e38ae9ebcef50b1ab",
      "origin": "https://bird-bench.oss-cn-beijing.aliyuncs.com/minidev.zip (sha256 cc48ba16838204e4e214512030cb572eeb5f7bcdd999bae4b9b6ff12ec13b92f, downloaded 2026-09-07), member minidev/MINIDEV/mini_dev_postgresql.json",
      "date": "2024-06-19"
    },
    "second": {
      "path": "data/preds-pg/predict_mini_dev_gpt-35-turbo-instruct_postgresql.json",
      "digest": "sha256:df1edf6aac3c01cf93041bcd7b60378efe6f7359f19add6d2af2345bef092bf9",
      "origin": "https://raw.githubusercontent.com/bird-bench/mini_dev/b3d4bcbbae9a96934ad812551eb400c7a3b23c12/llm/exp_result/sql_output_kg/predict_mini_dev_gpt-35-turbo-instruct_postgresql.json",
      "date": "2024-06-19"
    }
  },
  "replay_rule": "R-SET",
  "verdict": {
    "result": "not_equal",
    "mismatched": [],
    "reading": "NOT_EQUAL states that these two statements disagree on this data under this rule. It does not state which of them is wrong."
  },
  "bird_ex": {
    "value": 1,
    "equal": true,
    "method": "set(second_rows) == set(gold_rows), float4/float8 cells as Python float as psycopg2 returns them, numeric as Decimal",
    "source": "https://github.com/bird-bench/mini_dev/blob/main/evaluation/evaluation_ex.py",
    "gold_rows": 161,
    "second_rows": 356,
    "gold_distinct_rows": 161,
    "second_distinct_rows": 161
  },
  "test_suite_ex": {
    "value": 0,
    "equal": false,
    "method": "result_eq: equal row counts and equal column counts, each row unordered as a quick rejection, then the two equal as a list when the gold text holds ORDER BY and as a multiset otherwise, under some permutation of the columns; DISTINCT is not stripped and re-executed, and the cells are PostgreSQL's as psycopg2 returns them",
    "source": "ruiqi-zhong/test-suite-sql-eval, exec_eval.py, result_eq, at commit 48cb78ec: https://github.com/ruiqi-zhong/test-suite-sql-eval/blob/48cb78ecf7f610620206283846c76751b18a1326/exec_eval.py",
    "order_matters": false,
    "gold_rows": 161,
    "second_rows": 356,
    "gold_columns": 1,
    "second_columns": 1
  },
  "mechanism": {
    "class": "multiplicity",
    "gold_types": [
      "int8"
    ],
    "second_types": [
      "int8"
    ],
    "multiset_equal": false,
    "set_equal": true,
    "order_equal": false,
    "shorter_result_is_a_prefix": false,
    "reading": "The class states what makes these two results unequal under this rule, read off the two results and nothing else. It does not state which of the two statements is wrong."
  },
  "gold": {
    "executed_sql": "SELECT DISTINCT team_fifa_api_id FROM Team_Attributes WHERE buildUpPlaySpeed > 50 AND buildUpPlaySpeed < 60",
    "own_ordering": [],
    "result": {
      "columns": [
        {
          "name": "team_fifa_api_id",
          "declared_type": "int8"
        }
      ],
      "row_count": 161,
      "truncated": false,
      "rows_shown": 25,
      "rows": [
        [
          {
            "type": "int",
            "value": 485
          }
        ],
        [
          {
            "type": "int",
            "value": 110744
          }
        ],
        [
          {
            "type": "int",
            "value": 477
          }
        ],
        [
          {
            "type": "int",
            "value": 71
          }
        ],
        [
          {
            "type": "int",
            "value": 1889
          }
        ],
        [
          {
            "type": "int",
            "value": 68
          }
        ],
        [
          {
            "type": "int",
            "value": 229
          }
        ],
        [
          {
            "type": "int",
            "value": 481
          }
        ],
        [
          {
            "type": "int",
            "value": 52
          }
        ],
        [
          {
            "type": "int",
            "value": 70
          }
        ],
        [
          {
            "type": "int",
            "value": 80
          }
        ],
        [
          {
            "type": "int",
            "value": 1971
          }
        ],
        [
          {
            "type": "int",
            "value": 112225
          }
        ],
        [
          {
            "type": "int",
            "value": 1861
          }
        ],
        [
          {
            "type": "int",
            "value": 378
          }
        ],
        [
          {
            "type": "int",
            "value": 874
          }
        ],
        [
          {
            "type": "int",
            "value": 450
          }
        ],
        [
          {
            "type": "int",
            "value": 10030
          }
        ],
        [
          {
            "type": "int",
            "value": 1799
          }
        ],
        [
          {
            "type": "int",
            "value": 900
          }
        ],
        [
          {
            "type": "int",
            "value": 110745
          }
        ],
        [
          {
            "type": "int",
            "value": 1750
          }
        ],
        [
          {
            "type": "int",
            "value": 462
          }
        ],
        [
          {
            "type": "int",
            "value": 237
          }
        ],
        [
          {
            "type": "int",
            "value": 69
          }
        ]
      ],
      "result_hash": "sha256:1f7a83843d78e06ee35755d20fc3ce45b6c89aaf7eac10e855f720532e328e6b"
    },
    "record": "evidence-gold.json"
  },
  "second": {
    "executed_sql": "SELECT  team_fifa_api_id\nFROM Team_Attributes\nWHERE buildUpPlaySpeed >50 AND buildUpPlaySpeed <60",
    "own_ordering": [],
    "result": {
      "columns": [
        {
          "name": "team_fifa_api_id",
          "declared_type": "int8"
        }
      ],
      "row_count": 356,
      "truncated": false,
      "rows_shown": 25,
      "rows": [
        [
          {
            "type": "int",
            "value": 434
          }
        ],
        [
          {
            "type": "int",
            "value": 77
          }
        ],
        [
          {
            "type": "int",
            "value": 77
          }
        ],
        [
          {
            "type": "int",
            "value": 77
          }
        ],
        [
          {
            "type": "int",
            "value": 614
          }
        ],
        [
          {
            "type": "int",
            "value": 614
          }
        ],
        [
          {
            "type": "int",
            "value": 614
          }
        ],
        [
          {
            "type": "int",
            "value": 614
          }
        ],
        [
          {
            "type": "int",
            "value": 1901
          }
        ],
        [
          {
            "type": "int",
            "value": 1901
          }
        ],
        [
          {
            "type": "int",
            "value": 650
          }
        ],
        [
          {
            "type": "int",
            "value": 650
          }
        ],
        [
          {
            "type": "int",
            "value": 650
          }
        ],
        [
          {
            "type": "int",
            "value": 1861
          }
        ],
        [
          {
            "type": "int",
            "value": 229
          }
        ],
        [
          {
            "type": "int",
            "value": 229
          }
        ],
        [
          {
            "type": "int",
            "value": 229
          }
        ],
        [
          {
            "type": "int",
            "value": 229
          }
        ],
        [
          {
            "type": "int",
            "value": 111989
          }
        ],
        [
          {
            "type": "int",
            "value": 1
          }
        ],
        [
          {
            "type": "int",
            "value": 1
          }
        ],
        [
          {
            "type": "int",
            "value": 240
          }
        ],
        [
          {
            "type": "int",
            "value": 240
          }
        ],
        [
          {
            "type": "int",
            "value": 100409
          }
        ],
        [
          {
            "type": "int",
            "value": 100409
          }
        ]
      ],
      "result_hash": "sha256:1d5fb5460cba8d3b98439e43146df64c792a0e1c0f82b8cf5e922b2b48ba88c2"
    },
    "record": "evidence-second.json"
  },
  "canonical_ordering": [],
  "differing_rows": {
    "in_gold_not_in_second": [],
    "in_gold_not_in_second_total": 0,
    "in_second_not_in_gold": [
      {
        "row": [
          {
            "type": "int",
            "value": 110569
          }
        ],
        "count": 4
      },
      {
        "row": [
          {
            "type": "int",
            "value": 13
          }
        ],
        "count": 4
      },
      {
        "row": [
          {
            "type": "int",
            "value": 15005
          }
        ],
        "count": 4
      },
      {
        "row": [
          {
            "type": "int",
            "value": 1715
          }
        ],
        "count": 4
      },
      {
        "row": [
          {
            "type": "int",
            "value": 242
          }
        ],
        "count": 4
      },
      {
        "row": [
          {
            "type": "int",
            "value": 31
          }
        ],
        "count": 4
      },
      {
        "row": [
          {
            "type": "int",
            "value": 4
          }
        ],
        "count": 4
      },
      {
        "row": [
          {
            "type": "int",
            "value": 462
          }
        ],
        "count": 4
      },
      {
        "row": [
          {
            "type": "int",
            "value": 900
          }
        ],
        "count": 4
      },
      {
        "row": [
          {
            "type": "int",
            "value": 100879
          }
        ],
        "count": 3
      },
      {
        "row": [
          {
            "type": "int",
            "value": 110329
          }
        ],
        "count": 3
      },
      {
        "row": [
          {
            "type": "int",
            "value": 110724
          }
        ],
        "count": 3
      },
      {
        "row": [
          {
            "type": "int",
            "value": 110832
          }
        ],
        "count": 3
      },
      {
        "row": [
          {
            "type": "int",
            "value": 111239
          }
        ],
        "count": 3
      },
      {
        "row": [
          {
            "type": "int",
            "value": 15
          }
        ],
        "count": 3
      },
      {
        "row": [
          {
            "type": "int",
            "value": 165
          }
        ],
        "count": 3
      },
      {
        "row": [
          {
            "type": "int",
            "value": 1832
          }
        ],
        "count": 3
      },
      {
        "row": [
          {
            "type": "int",
            "value": 203
          }
        ],
        "count": 3
      },
      {
        "row": [
          {
            "type": "int",
            "value": 229
          }
        ],
        "count": 3
      },
      {
        "row": [
          {
            "type": "int",
            "value": 232
          }
        ],
        "count": 3
      },
      {
        "row": [
          {
            "type": "int",
            "value": 237
          }
        ],
        "count": 3
      },
      {
        "row": [
          {
            "type": "int",
            "value": 449
          }
        ],
        "count": 3
      },
      {
        "row": [
          {
            "type": "int",
            "value": 450
          }
        ],
        "count": 3
      },
      {
        "row": [
          {
            "type": "int",
            "value": 459
          }
        ],
        "count": 3
      },
      {
        "row": [
          {
            "type": "int",
            "value": 468
          }
        ],
        "count": 3
      }
    ],
    "in_second_not_in_gold_total": 195,
    "rows_shown_per_side": 25
  },
  "result_hashes": {
    "gold": "sha256:1f7a83843d78e06ee35755d20fc3ce45b6c89aaf7eac10e855f720532e328e6b",
    "second": "sha256:1d5fb5460cba8d3b98439e43146df64c792a0e1c0f82b8cf5e922b2b48ba88c2"
  },
  "backend_identity": "PostgreSQL 16.15 (Debian 16.15-1.pgdg13+2) on aarch64-unknown-linux-gnu, compiled by gcc (Debian 14.2.0-19) 14.2.0, 64-bit | server=172.17.0.2/32:5432 | database=bird",
  "fixture": {
    "schema_digest": "sha256:98bc0cba85d167faa0d5bb8e4db8801339559c93568b09cfd7c9a911f2c12123",
    "row_counts": {
      "public.team_attributes": 1458
    },
    "content_digests": {},
    "source_file_sha256": "sha256:31b1da211849d24a57c9af7636da46a5b82fc8a3ca1542bb3ebd8775e9a31cec"
  },
  "run_id": "audit-b6efa569-00a2-4421-b91b-09d107e2dac3"
}