{"run":{"id":"a15-model-comparison-2026-07-24","name":"A1.5 \u00b7 Structured intent model comparison","date":"2026-07-24","status":"complete","dataset":"semantic-intent@1.5","verifier":"exact-query-intent@1","prompt":"baseline-intent-prompt@1","task_count":88,"system_count":6,"splits":{"development":20,"holdout":60,"adversarial":8},"description":"Six complete LLM system configurations evaluated on the same development, held-out, and adversarial semantic tasks."},"systems":[{"id":"qwen3b-current-guardrail","date":"2026-07-24","system":{"name":"qwen3b-current-guardrail","provider":"baseten","model":"Qwen/Qwen2.5-3B-Instruct","correction":"ranking_guardrail"},"question_count":88,"metrics":{"raw_intent_accuracy":0.45454545454545453,"corrected_intent_accuracy":0.7045454545454546,"schema_compliance":1.0,"intervention_precision":0.8214285714285714,"intervention_recall":0.4791666666666667,"p50_latency_ms":3389.3641674999344,"p95_latency_ms":3431.9425029998456,"input_tokens":292603,"output_tokens":10249,"cost_per_1000_questions_usd":0.8005,"raw_error_taxonomy":{"pattern_id":27,"sort":26,"limit":22,"filters":7},"corrected_error_taxonomy":{"limit":19,"filters":7,"pattern_id":5,"sort":9}},"rows":[{"id":"q001","dataset":"development","question":"Top 5 customers by revenue last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":5},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":5},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":5},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":4167.6508330001525,"input_tokens":3324,"output_tokens":116,"error":null},{"id":"q002","dataset":"development","question":"Top 3 regions by revenue last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":3353.524932000255,"input_tokens":3324,"output_tokens":116,"error":null},{"id":"q003","dataset":"development","question":"Top 4 products by revenue last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":4},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":4},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":4},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":3366.4379729998473,"input_tokens":3324,"output_tokens":116,"error":null},{"id":"q004","dataset":"development","question":"Top 2 product categories by revenue last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":2},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":2},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":2},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":3350.3059089998715,"input_tokens":3325,"output_tokens":116,"error":null},{"id":"q005","dataset":"development","question":"Top 5 customers by units sold last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":5},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":5},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":5},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":3310.7642580002903,"input_tokens":3325,"output_tokens":114,"error":null},{"id":"q006","dataset":"development","question":"Top 3 products by quantity last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":3},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":3},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":3},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":3304.532924000341,"input_tokens":3324,"output_tokens":114,"error":null},{"id":"q007","dataset":"development","question":"Top 4 regions by order count last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":4},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":4},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":4},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":3306.681728000058,"input_tokens":3325,"output_tokens":114,"error":null},{"id":"q008","dataset":"development","question":"Revenue by month last quarter","expected_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":3364.855301999796,"input_tokens":3321,"output_tokens":116,"error":null},{"id":"q009","dataset":"development","question":"Order count by month last quarter","expected_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":3334.2752669996116,"input_tokens":3322,"output_tokens":115,"error":null},{"id":"q010","dataset":"development","question":"Units sold by month last quarter","expected_intent":{"pattern_id":"metric_by_time","metric":"quantity","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_time","metric":"quantity","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_time","metric":"quantity","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":3512.722855999982,"input_tokens":3322,"output_tokens":115,"error":null},{"id":"q011","dataset":"development","question":"Monthly revenue for Q2 2026","expected_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":3392.8881159999946,"input_tokens":3326,"output_tokens":116,"error":null},{"id":"q012","dataset":"development","question":"Monthly order volume for April through June 2026","expected_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":3339.9048100000073,"input_tokens":3328,"output_tokens":115,"error":null},{"id":"q013","dataset":"development","question":"Revenue by region last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":true,"raw_mismatch_fields":["pattern_id","sort","limit"],"corrected_mismatch_fields":[],"intervened":true,"latency_ms":3405.8872680002423,"input_tokens":3321,"output_tokens":117,"error":null},{"id":"q014","dataset":"development","question":"Revenue by customer last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":true,"raw_mismatch_fields":["pattern_id","sort"],"corrected_mismatch_fields":[],"intervened":true,"latency_ms":3417.767213999923,"input_tokens":3321,"output_tokens":118,"error":null},{"id":"q015","dataset":"development","question":"Order count by region last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":true,"raw_mismatch_fields":["pattern_id","sort"],"corrected_mismatch_fields":[],"intervened":true,"latency_ms":3364.6924649997345,"input_tokens":3322,"output_tokens":116,"error":null},{"id":"q016","dataset":"development","question":"Units sold by product last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":3391.5043060001153,"input_tokens":3322,"output_tokens":117,"error":null},{"id":"q017","dataset":"development","question":"Revenue by product category last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":true,"raw_mismatch_fields":["pattern_id","sort","limit"],"corrected_mismatch_fields":[],"intervened":true,"latency_ms":3391.4932029997544,"input_tokens":3322,"output_tokens":117,"error":null},{"id":"q018","dataset":"development","question":"Order count by product category last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":true,"raw_mismatch_fields":["pattern_id","sort","limit"],"corrected_mismatch_fields":[],"intervened":true,"latency_ms":3341.169484999682,"input_tokens":3323,"output_tokens":115,"error":null},{"id":"q019","dataset":"development","question":"Revenue by customer in June 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":3419.91809000001,"input_tokens":3326,"output_tokens":118,"error":null},{"id":"q020","dataset":"development","question":"Units sold by region in May 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":3403.047732000232,"input_tokens":3327,"output_tokens":117,"error":null},{"id":"c001_breakdown","dataset":"holdout","question":"Gross sales across regions last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":true,"raw_mismatch_fields":["pattern_id","sort"],"corrected_mismatch_fields":[],"intervened":true,"latency_ms":3424.3675560001066,"input_tokens":3322,"output_tokens":118,"error":null},{"id":"c001_ranking","dataset":"holdout","question":"Regions with the highest gross sales last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":4},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":4},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["limit"],"corrected_mismatch_fields":["limit"],"intervened":false,"latency_ms":3358.4778149997874,"input_tokens":3324,"output_tokens":116,"error":null},{"id":"c002_breakdown","dataset":"holdout","question":"Revenue for each customer in Q2 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":true,"raw_mismatch_fields":["pattern_id"],"corrected_mismatch_fields":[],"intervened":true,"latency_ms":3387.2351320001144,"input_tokens":3328,"output_tokens":117,"error":null},{"id":"c002_ranking","dataset":"holdout","question":"Top 6 customers by revenue in Q2 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":6},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":6},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":6},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":3361.9188980001127,"input_tokens":3330,"output_tokens":116,"error":null},{"id":"c003_breakdown","dataset":"holdout","question":"Order volume broken down by product category last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":true,"raw_mismatch_fields":["pattern_id","sort"],"corrected_mismatch_fields":[],"intervened":true,"latency_ms":3368.3314509999036,"input_tokens":3325,"output_tokens":116,"error":null},{"id":"c003_ranking","dataset":"holdout","question":"Product categories with the most orders last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":3335.7054100001733,"input_tokens":3324,"output_tokens":115,"error":null},{"id":"c004_breakdown","dataset":"holdout","question":"Unit sales by product in June 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":3402.36908199995,"input_tokens":3327,"output_tokens":117,"error":null},{"id":"c004_ranking","dataset":"holdout","question":"Top 4 products by unit sales in June 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":4},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":4},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":4},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":3314.703000999998,"input_tokens":3330,"output_tokens":114,"error":null},{"id":"c005_breakdown","dataset":"holdout","question":"Sales dollars by product category in May 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":3441.8229169996266,"input_tokens":3328,"output_tokens":118,"error":null},{"id":"c005_ranking","dataset":"holdout","question":"Best product categories by sales dollars in May 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":3392.857814999843,"input_tokens":3329,"output_tokens":117,"error":null},{"id":"c006_breakdown","dataset":"holdout","question":"Number of orders for each region in April 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":3396.274605000144,"input_tokens":3329,"output_tokens":117,"error":null},{"id":"c006_ranking","dataset":"holdout","question":"Leading regions by number of orders in April 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-04-30"}],"sort":[{"field":"orders","direction":"desc"}],"limit":100},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-04-30"}],"sort":[{"field":"orders","direction":"desc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["filters","limit"],"corrected_mismatch_fields":["filters","limit"],"intervened":false,"latency_ms":3369.824421999965,"input_tokens":3329,"output_tokens":116,"error":null},{"id":"c007_breakdown","dataset":"holdout","question":"Bookings value by customer last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":true,"raw_mismatch_fields":["pattern_id","sort","limit"],"corrected_mismatch_fields":[],"intervened":true,"latency_ms":3411.5536379999867,"input_tokens":3323,"output_tokens":117,"error":null},{"id":"c007_ranking","dataset":"holdout","question":"Largest customers by bookings value last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":100},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["limit"],"corrected_mismatch_fields":["limit"],"intervened":false,"latency_ms":3425.5093210003906,"input_tokens":3323,"output_tokens":118,"error":null},{"id":"c008_breakdown","dataset":"holdout","question":"Units sold across regions last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":3394.9637840000833,"input_tokens":3322,"output_tokens":117,"error":null},{"id":"c008_ranking","dataset":"holdout","question":"Rank regions by units sold last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":100},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["limit"],"corrected_mismatch_fields":["limit"],"intervened":false,"latency_ms":3367.573519999951,"input_tokens":3323,"output_tokens":116,"error":null},{"id":"c009_breakdown","dataset":"holdout","question":"Revenue grouped by product last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":true,"raw_mismatch_fields":["pattern_id","sort"],"corrected_mismatch_fields":[],"intervened":true,"latency_ms":3418.712730000152,"input_tokens":3322,"output_tokens":118,"error":null},{"id":"c009_ranking","dataset":"holdout","question":"Products ranked by revenue last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":100},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["limit"],"corrected_mismatch_fields":["limit"],"intervened":false,"latency_ms":3422.061501999906,"input_tokens":3322,"output_tokens":118,"error":null},{"id":"c010_breakdown","dataset":"holdout","question":"Orders by customer last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":true,"raw_mismatch_fields":["pattern_id"],"corrected_mismatch_fields":[],"intervened":true,"latency_ms":3374.5357119996697,"input_tokens":3321,"output_tokens":116,"error":null},{"id":"c010_ranking","dataset":"holdout","question":"Customers with the fewest orders last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["limit"],"corrected_mismatch_fields":["limit"],"intervened":false,"latency_ms":3371.4027860000897,"input_tokens":3324,"output_tokens":116,"error":null},{"id":"c011_breakdown","dataset":"holdout","question":"Product category revenue for April through June 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["pattern_id","sort"],"corrected_mismatch_fields":["pattern_id","sort"],"intervened":false,"latency_ms":3368.0029100000866,"input_tokens":3328,"output_tokens":116,"error":null},{"id":"c011_ranking","dataset":"holdout","question":"Bottom 2 product categories by revenue for April through June 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":2},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":2},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":2},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["sort"],"corrected_mismatch_fields":["sort"],"intervened":false,"latency_ms":3344.45129300002,"input_tokens":3332,"output_tokens":115,"error":null},{"id":"c012_breakdown","dataset":"holdout","question":"Customer unit volume in May 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":3407.0572800001173,"input_tokens":3326,"output_tokens":117,"error":null},{"id":"c012_ranking","dataset":"holdout","question":"Which customers had the least unit volume in May 2026?","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"quantity","direction":"asc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"quantity","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"quantity","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["limit"],"corrected_mismatch_fields":["limit"],"intervened":false,"latency_ms":3378.7956529999974,"input_tokens":3331,"output_tokens":116,"error":null},{"id":"c013_breakdown","dataset":"holdout","question":"Order count by product last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":true,"raw_mismatch_fields":["pattern_id","sort"],"corrected_mismatch_fields":[],"intervened":true,"latency_ms":3370.087207999859,"input_tokens":3322,"output_tokens":116,"error":null},{"id":"c013_ranking","dataset":"holdout","question":"Worst products by order count last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["limit"],"corrected_mismatch_fields":["limit"],"intervened":false,"latency_ms":3405.6577330002256,"input_tokens":3323,"output_tokens":116,"error":null},{"id":"c014_breakdown","dataset":"holdout","question":"Quantity by product category in June 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":3401.715293000052,"input_tokens":3327,"output_tokens":117,"error":null},{"id":"c014_ranking","dataset":"holdout","question":"Smallest product categories by quantity in June 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"asc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["limit"],"corrected_mismatch_fields":["limit"],"intervened":false,"latency_ms":3377.1787969999423,"input_tokens":3329,"output_tokens":116,"error":null},{"id":"c015_breakdown","dataset":"holdout","question":"Sales for every product last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":true,"raw_mismatch_fields":["pattern_id","sort"],"corrected_mismatch_fields":[],"intervened":true,"latency_ms":3431.201038999916,"input_tokens":3322,"output_tokens":118,"error":null},{"id":"c015_ranking","dataset":"holdout","question":"First 3 products by sales last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":3379.1883090002557,"input_tokens":3324,"output_tokens":116,"error":null},{"id":"c016_breakdown","dataset":"holdout","question":"Region order volume in Q2 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":true,"raw_mismatch_fields":["pattern_id","sort"],"corrected_mismatch_fields":[],"intervened":true,"latency_ms":3382.1927230001165,"input_tokens":3327,"output_tokens":116,"error":null},{"id":"c016_ranking","dataset":"holdout","question":"Last 2 regions by order volume in Q2 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":2},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-06-30"}],"sort":[{"field":"orders","direction":"desc"}],"limit":2},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-06-30"}],"sort":[{"field":"orders","direction":"desc"}],"limit":2},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["filters","sort"],"corrected_mismatch_fields":["filters","sort"],"intervened":false,"latency_ms":3321.9503010000153,"input_tokens":3331,"output_tokens":114,"error":null},{"id":"c017_breakdown","dataset":"holdout","question":"Show revenue by customer for April 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":3417.196755999612,"input_tokens":3327,"output_tokens":118,"error":null},{"id":"c017_ranking","dataset":"holdout","question":"Show the top 3 customers by revenue for April 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-04-30"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-04-30"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["filters"],"corrected_mismatch_fields":["filters"],"intervened":false,"latency_ms":3372.2742840000137,"input_tokens":3331,"output_tokens":116,"error":null},{"id":"c018_breakdown","dataset":"holdout","question":"Product revenue table for last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":true,"raw_mismatch_fields":["pattern_id","sort"],"corrected_mismatch_fields":[],"intervened":true,"latency_ms":3429.392046999965,"input_tokens":3322,"output_tokens":118,"error":null},{"id":"c018_ranking","dataset":"holdout","question":"Top product revenue table for last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":100},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["limit"],"corrected_mismatch_fields":["limit"],"intervened":false,"latency_ms":3427.1432450000248,"input_tokens":3323,"output_tokens":118,"error":null},{"id":"c019_breakdown","dataset":"holdout","question":"Order count across customers last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":true,"raw_mismatch_fields":["pattern_id","sort"],"corrected_mismatch_fields":[],"intervened":true,"latency_ms":3378.410649000216,"input_tokens":3322,"output_tokens":116,"error":null},{"id":"c019_ranking","dataset":"holdout","question":"Rank customers by order count last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":100},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["limit"],"corrected_mismatch_fields":["limit"],"intervened":false,"latency_ms":3375.1863219999905,"input_tokens":3323,"output_tokens":116,"error":null},{"id":"c020_breakdown","dataset":"holdout","question":"Category unit sales for Q2 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":3400.769344999844,"input_tokens":3327,"output_tokens":117,"error":null},{"id":"c020_ranking","dataset":"holdout","question":"Categories ranked by unit sales for Q2 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":100},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["limit"],"corrected_mismatch_fields":["limit"],"intervened":false,"latency_ms":3382.1331149997604,"input_tokens":3329,"output_tokens":116,"error":null},{"id":"n001","dataset":"holdout","question":"Give me the sales dollars by customer for Q2","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":true,"raw_mismatch_fields":["pattern_id","sort"],"corrected_mismatch_fields":[],"intervened":true,"latency_ms":3435.606972000187,"input_tokens":3326,"output_tokens":118,"error":null},{"id":"n002","dataset":"holdout","question":"Which five regions generated the most revenue last quarter?","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":5},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":5},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":5},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":3378.9711360000183,"input_tokens":3326,"output_tokens":116,"error":null},{"id":"n003","dataset":"holdout","question":"Monthly unit sales for Q2 2026","expected_intent":{"pattern_id":"metric_by_time","metric":"quantity","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_time","metric":"quantity","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_time","metric":"quantity","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":3346.5294909997283,"input_tokens":3327,"output_tokens":115,"error":null},{"id":"n004","dataset":"holdout","question":"Show order volume by region, not ranked, for last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":3429.5656330000384,"input_tokens":3328,"output_tokens":117,"error":null},{"id":"n005","dataset":"holdout","question":"Bottom 3 customers by bookings value last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":3},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":3},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":3},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":3374.4548280001254,"input_tokens":3325,"output_tokens":116,"error":null},{"id":"n006","dataset":"holdout","question":"Most recent order count by region last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":true,"raw_mismatch_fields":["pattern_id","sort"],"corrected_mismatch_fields":[],"intervened":true,"latency_ms":3382.1675409999443,"input_tokens":3324,"output_tokens":116,"error":null},{"id":"n007","dataset":"holdout","question":"High value customer revenue by customer last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":true,"raw_mismatch_fields":["pattern_id","sort","limit"],"corrected_mismatch_fields":[],"intervened":true,"latency_ms":3407.6086029999715,"input_tokens":3324,"output_tokens":117,"error":null},{"id":"n008","dataset":"holdout","question":"Revenue by product, sorted alphabetically, for June 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":3423.630138000135,"input_tokens":3331,"output_tokens":118,"error":null},{"id":"n009","dataset":"holdout","question":"Which products sold the most units last quarter?","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":100},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["limit"],"corrected_mismatch_fields":["limit"],"intervened":false,"latency_ms":3369.5180129998334,"input_tokens":3325,"output_tokens":116,"error":null},{"id":"n010","dataset":"holdout","question":"Trend revenue over the last quarter by month","expected_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":3384.0626880000855,"input_tokens":3324,"output_tokens":116,"error":null},{"id":"n011","dataset":"holdout","question":"Quantity by customer in April 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":3407.6188029998775,"input_tokens":3326,"output_tokens":117,"error":null},{"id":"n012","dataset":"holdout","question":"Top revenue products for April through June","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":3411.6838879999705,"input_tokens":3323,"output_tokens":117,"error":null},{"id":"n013","dataset":"holdout","question":"Category order volume, all categories, Q2 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":true,"raw_mismatch_fields":["pattern_id","sort"],"corrected_mismatch_fields":[],"intervened":true,"latency_ms":3369.639941999594,"input_tokens":3330,"output_tokens":116,"error":null},{"id":"n014","dataset":"holdout","question":"Least revenue customers last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["limit"],"corrected_mismatch_fields":["limit"],"intervened":false,"latency_ms":3431.9425029998456,"input_tokens":3321,"output_tokens":118,"error":null},{"id":"n015","dataset":"holdout","question":"For each product category, how many units were sold in May?","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":3408.4124529999826,"input_tokens":3329,"output_tokens":117,"error":null},{"id":"n016","dataset":"holdout","question":"Rank product categories by revenue in June","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":3398.3727780000663,"input_tokens":3323,"output_tokens":117,"error":null},{"id":"n017","dataset":"holdout","question":"Region sales report for last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":true,"raw_mismatch_fields":["pattern_id","sort"],"corrected_mismatch_fields":[],"intervened":true,"latency_ms":3424.080504000358,"input_tokens":3322,"output_tokens":118,"error":null},{"id":"n018","dataset":"holdout","question":"Largest product categories by unit sales last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":10},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["sort"],"corrected_mismatch_fields":["sort"],"intervened":false,"latency_ms":3353.34812300016,"input_tokens":3324,"output_tokens":115,"error":null},{"id":"n019","dataset":"holdout","question":"Order volume by product for the most recent quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":true,"raw_mismatch_fields":["pattern_id","sort"],"corrected_mismatch_fields":[],"intervened":true,"latency_ms":3370.2355379996334,"input_tokens":3325,"output_tokens":116,"error":null},{"id":"n020","dataset":"holdout","question":"Monthly order count in Q2","expected_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":3352.2004900000866,"input_tokens":3322,"output_tokens":115,"error":null},{"id":"adv001_topline_breakdown","dataset":"adversarial","question":"Top-line revenue by region last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["pattern_id","sort","limit"],"corrected_mismatch_fields":["pattern_id","sort","limit"],"intervened":false,"latency_ms":3406.3473040000645,"input_tokens":3323,"output_tokens":117,"error":null},{"id":"adv002_leaders_ranking","dataset":"adversarial","question":"Revenue leaders by region last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":false,"raw_mismatch_fields":[],"corrected_mismatch_fields":["pattern_id","sort","limit"],"intervened":true,"latency_ms":3424.7522440000466,"input_tokens":3322,"output_tokens":117,"error":null},{"id":"adv003_compare_breakdown","dataset":"adversarial","question":"How do regions compare on revenue last quarter?","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":true,"raw_mismatch_fields":["pattern_id","sort"],"corrected_mismatch_fields":[],"intervened":true,"latency_ms":3423.984689000008,"input_tokens":3325,"output_tokens":118,"error":null},{"id":"adv004_strongest_ranking","dataset":"adversarial","question":"Show the strongest markets by revenue last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["limit"],"corrected_mismatch_fields":["pattern_id","sort","limit"],"intervened":true,"latency_ms":3425.680080999882,"input_tokens":3324,"output_tokens":118,"error":null},{"id":"adv005_first_rows_breakdown","dataset":"adversarial","question":"Show revenue by region, but only the first ten rows","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[],"sort":[{"field":"region","direction":"asc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":10},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["pattern_id","filters"],"corrected_mismatch_fields":["filters","limit"],"intervened":true,"latency_ms":3372.170926999843,"input_tokens":3327,"output_tokens":116,"error":null},{"id":"adv006_top_regions_ranking","dataset":"adversarial","question":"Show the top ten regions by revenue","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["filters"],"corrected_mismatch_fields":["filters"],"intervened":false,"latency_ms":3398.3210910000707,"input_tokens":3323,"output_tokens":117,"error":null},{"id":"adv007_ordered_breakdown","dataset":"adversarial","question":"Break down revenue by region, ordered by revenue descending","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[],"sort":[{"field":"revenue","direction":"desc"}],"limit":100},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["pattern_id","filters"],"corrected_mismatch_fields":["filters","sort"],"intervened":true,"latency_ms":3430.026710999755,"input_tokens":3326,"output_tokens":118,"error":null},{"id":"adv008_greatest_ranking","dataset":"adversarial","question":"Which regions contribute the greatest revenue?","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["filters","limit"],"corrected_mismatch_fields":["pattern_id","filters","sort","limit"],"intervened":true,"latency_ms":3430.1224989999355,"input_tokens":3323,"output_tokens":118,"error":null}]},{"id":"qwen7b-current-guardrail","date":"2026-07-24","system":{"name":"qwen7b-current-guardrail","provider":"baseten","model":"Qwen/Qwen2.5-7B-Instruct","correction":"ranking_guardrail"},"question_count":88,"metrics":{"raw_intent_accuracy":0.8409090909090909,"corrected_intent_accuracy":0.8409090909090909,"schema_compliance":1.0,"intervention_precision":0.25,"intervention_recall":0.07142857142857142,"p50_latency_ms":7049.186274500016,"p95_latency_ms":7136.622975999671,"input_tokens":292603,"output_tokens":10268,"cost_per_1000_questions_usd":1.661,"raw_error_taxonomy":{"limit":9,"pattern_id":4,"sort":4,"filters":4},"corrected_error_taxonomy":{"limit":10,"pattern_id":6,"sort":6,"filters":4}},"rows":[{"id":"q001","dataset":"development","question":"Top 5 customers by revenue last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":5},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":5},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":5},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7516.307336999944,"input_tokens":3324,"output_tokens":116,"error":null},{"id":"q002","dataset":"development","question":"Top 3 regions by revenue last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":6999.841388000277,"input_tokens":3324,"output_tokens":116,"error":null},{"id":"q003","dataset":"development","question":"Top 4 products by revenue last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":4},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":4},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":4},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":6992.70132599986,"input_tokens":3324,"output_tokens":116,"error":null},{"id":"q004","dataset":"development","question":"Top 2 product categories by revenue last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":2},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":2},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":2},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":6992.404654999973,"input_tokens":3325,"output_tokens":116,"error":null},{"id":"q005","dataset":"development","question":"Top 5 customers by units sold last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":5},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":5},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":5},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":6913.371569999981,"input_tokens":3325,"output_tokens":114,"error":null},{"id":"q006","dataset":"development","question":"Top 3 products by quantity last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":3},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":3},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":3},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":6869.229311000254,"input_tokens":3324,"output_tokens":114,"error":null},{"id":"q007","dataset":"development","question":"Top 4 regions by order count last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":4},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":4},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":4},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":6866.807381999934,"input_tokens":3325,"output_tokens":114,"error":null},{"id":"q008","dataset":"development","question":"Revenue by month last quarter","expected_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":6994.946652000181,"input_tokens":3321,"output_tokens":116,"error":null},{"id":"q009","dataset":"development","question":"Order count by month last quarter","expected_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":6935.632692000127,"input_tokens":3322,"output_tokens":115,"error":null},{"id":"q010","dataset":"development","question":"Units sold by month last quarter","expected_intent":{"pattern_id":"metric_by_time","metric":"quantity","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_time","metric":"quantity","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_time","metric":"quantity","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":6941.608517999612,"input_tokens":3322,"output_tokens":115,"error":null},{"id":"q011","dataset":"development","question":"Monthly revenue for Q2 2026","expected_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":6987.9909860001135,"input_tokens":3326,"output_tokens":116,"error":null},{"id":"q012","dataset":"development","question":"Monthly order volume for April through June 2026","expected_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7401.188235000063,"input_tokens":3328,"output_tokens":123,"error":null},{"id":"q013","dataset":"development","question":"Revenue by region last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7123.894250999911,"input_tokens":3321,"output_tokens":118,"error":null},{"id":"q014","dataset":"development","question":"Revenue by customer last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7107.8057519998765,"input_tokens":3321,"output_tokens":118,"error":null},{"id":"q015","dataset":"development","question":"Order count by region last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7045.435708000241,"input_tokens":3322,"output_tokens":117,"error":null},{"id":"q016","dataset":"development","question":"Units sold by product last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7052.183538000008,"input_tokens":3322,"output_tokens":117,"error":null},{"id":"q017","dataset":"development","question":"Revenue by product category last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7109.018300999651,"input_tokens":3322,"output_tokens":118,"error":null},{"id":"q018","dataset":"development","question":"Order count by product category last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7048.152658000163,"input_tokens":3323,"output_tokens":117,"error":null},{"id":"q019","dataset":"development","question":"Revenue by customer in June 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7120.223013999748,"input_tokens":3326,"output_tokens":118,"error":null},{"id":"q020","dataset":"development","question":"Units sold by region in May 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7051.85363999999,"input_tokens":3327,"output_tokens":117,"error":null},{"id":"c001_breakdown","dataset":"holdout","question":"Gross sales across regions last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7123.51132699996,"input_tokens":3322,"output_tokens":118,"error":null},{"id":"c001_ranking","dataset":"holdout","question":"Regions with the highest gross sales last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7055.636072000198,"input_tokens":3324,"output_tokens":117,"error":null},{"id":"c002_breakdown","dataset":"holdout","question":"Revenue for each customer in Q2 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7108.127289000095,"input_tokens":3328,"output_tokens":118,"error":null},{"id":"c002_ranking","dataset":"holdout","question":"Top 6 customers by revenue in Q2 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":6},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":6},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":6},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":6998.407299000064,"input_tokens":3330,"output_tokens":116,"error":null},{"id":"c003_breakdown","dataset":"holdout","question":"Order volume broken down by product category last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7054.311379999945,"input_tokens":3325,"output_tokens":117,"error":null},{"id":"c003_ranking","dataset":"holdout","question":"Product categories with the most orders last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":100},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["limit"],"corrected_mismatch_fields":["limit"],"intervened":false,"latency_ms":7207.491891000245,"input_tokens":3324,"output_tokens":116,"error":null},{"id":"c004_breakdown","dataset":"holdout","question":"Unit sales by product in June 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7049.10464500017,"input_tokens":3327,"output_tokens":117,"error":null},{"id":"c004_ranking","dataset":"holdout","question":"Top 4 products by unit sales in June 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":4},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":4},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":4},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":6891.691789999641,"input_tokens":3330,"output_tokens":114,"error":null},{"id":"c005_breakdown","dataset":"holdout","question":"Sales dollars by product category in May 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7136.622975999671,"input_tokens":3328,"output_tokens":118,"error":null},{"id":"c005_ranking","dataset":"holdout","question":"Best product categories by sales dollars in May 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["pattern_id","sort","limit"],"corrected_mismatch_fields":["pattern_id","sort","limit"],"intervened":false,"latency_ms":7114.387018999878,"input_tokens":3329,"output_tokens":118,"error":null},{"id":"c006_breakdown","dataset":"holdout","question":"Number of orders for each region in April 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7066.760785000042,"input_tokens":3329,"output_tokens":117,"error":null},{"id":"c006_ranking","dataset":"holdout","question":"Leading regions by number of orders in April 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":6946.495080999739,"input_tokens":3329,"output_tokens":115,"error":null},{"id":"c007_breakdown","dataset":"holdout","question":"Bookings value by customer last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7119.267793999825,"input_tokens":3323,"output_tokens":118,"error":null},{"id":"c007_ranking","dataset":"holdout","question":"Largest customers by bookings value last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7052.713949000008,"input_tokens":3323,"output_tokens":117,"error":null},{"id":"c008_breakdown","dataset":"holdout","question":"Units sold across regions last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7060.265248999713,"input_tokens":3322,"output_tokens":117,"error":null},{"id":"c008_ranking","dataset":"holdout","question":"Rank regions by units sold last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":6935.237492000397,"input_tokens":3323,"output_tokens":115,"error":null},{"id":"c009_breakdown","dataset":"holdout","question":"Revenue grouped by product last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7115.02727099969,"input_tokens":3322,"output_tokens":118,"error":null},{"id":"c009_ranking","dataset":"holdout","question":"Products ranked by revenue last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7058.852769999703,"input_tokens":3322,"output_tokens":117,"error":null},{"id":"c010_breakdown","dataset":"holdout","question":"Orders by customer last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7047.111078999933,"input_tokens":3321,"output_tokens":117,"error":null},{"id":"c010_ranking","dataset":"holdout","question":"Customers with the fewest orders last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["limit"],"corrected_mismatch_fields":["limit"],"intervened":false,"latency_ms":6999.563476000276,"input_tokens":3324,"output_tokens":116,"error":null},{"id":"c011_breakdown","dataset":"holdout","question":"Product category revenue for April through June 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7118.444031999843,"input_tokens":3328,"output_tokens":118,"error":null},{"id":"c011_ranking","dataset":"holdout","question":"Bottom 2 product categories by revenue for April through June 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":2},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":2},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":2},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":6998.1779310001,"input_tokens":3332,"output_tokens":116,"error":null},{"id":"c012_breakdown","dataset":"holdout","question":"Customer unit volume in May 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7057.249493000199,"input_tokens":3326,"output_tokens":117,"error":null},{"id":"c012_ranking","dataset":"holdout","question":"Which customers had the least unit volume in May 2026?","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"quantity","direction":"asc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"quantity","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"quantity","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["limit"],"corrected_mismatch_fields":["limit"],"intervened":false,"latency_ms":7001.195548000396,"input_tokens":3331,"output_tokens":116,"error":null},{"id":"c013_breakdown","dataset":"holdout","question":"Order count by product last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7067.285508000168,"input_tokens":3322,"output_tokens":117,"error":null},{"id":"c013_ranking","dataset":"holdout","question":"Worst products by order count last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":6959.262717000001,"input_tokens":3323,"output_tokens":115,"error":null},{"id":"c014_breakdown","dataset":"holdout","question":"Quantity by product category in June 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7048.123731000032,"input_tokens":3327,"output_tokens":117,"error":null},{"id":"c014_ranking","dataset":"holdout","question":"Smallest product categories by quantity in June 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"asc"}],"limit":10},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["pattern_id","sort","limit"],"corrected_mismatch_fields":["pattern_id","sort","limit"],"intervened":false,"latency_ms":7109.933918000024,"input_tokens":3329,"output_tokens":117,"error":null},{"id":"c015_breakdown","dataset":"holdout","question":"Sales for every product last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7111.422494999715,"input_tokens":3322,"output_tokens":118,"error":null},{"id":"c015_ranking","dataset":"holdout","question":"First 3 products by sales last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":6997.831485000006,"input_tokens":3324,"output_tokens":116,"error":null},{"id":"c016_breakdown","dataset":"holdout","question":"Region order volume in Q2 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7066.076382999654,"input_tokens":3327,"output_tokens":117,"error":null},{"id":"c016_ranking","dataset":"holdout","question":"Last 2 regions by order volume in Q2 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":2},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":2},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":2},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["pattern_id","sort"],"corrected_mismatch_fields":["pattern_id","sort"],"intervened":false,"latency_ms":6935.975021000104,"input_tokens":3331,"output_tokens":115,"error":null},{"id":"c017_breakdown","dataset":"holdout","question":"Show revenue by customer for April 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7130.873422999684,"input_tokens":3327,"output_tokens":118,"error":null},{"id":"c017_ranking","dataset":"holdout","question":"Show the top 3 customers by revenue for April 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7002.10359599987,"input_tokens":3331,"output_tokens":116,"error":null},{"id":"c018_breakdown","dataset":"holdout","question":"Product revenue table for last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7114.8302470001,"input_tokens":3322,"output_tokens":118,"error":null},{"id":"c018_ranking","dataset":"holdout","question":"Top product revenue table for last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7058.2657089998975,"input_tokens":3323,"output_tokens":117,"error":null},{"id":"c019_breakdown","dataset":"holdout","question":"Order count across customers last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7046.9367959999545,"input_tokens":3322,"output_tokens":117,"error":null},{"id":"c019_ranking","dataset":"holdout","question":"Rank customers by order count last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":6925.040141999943,"input_tokens":3323,"output_tokens":115,"error":null},{"id":"c020_breakdown","dataset":"holdout","question":"Category unit sales for Q2 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7050.922623000133,"input_tokens":3327,"output_tokens":117,"error":null},{"id":"c020_ranking","dataset":"holdout","question":"Categories ranked by unit sales for Q2 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":6939.87551500004,"input_tokens":3329,"output_tokens":115,"error":null},{"id":"n001","dataset":"holdout","question":"Give me the sales dollars by customer for Q2","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7122.375286999613,"input_tokens":3326,"output_tokens":118,"error":null},{"id":"n002","dataset":"holdout","question":"Which five regions generated the most revenue last quarter?","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":5},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":5},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":5},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":6999.712020000061,"input_tokens":3326,"output_tokens":116,"error":null},{"id":"n003","dataset":"holdout","question":"Monthly unit sales for Q2 2026","expected_intent":{"pattern_id":"metric_by_time","metric":"quantity","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_time","metric":"quantity","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_time","metric":"quantity","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":6942.5829270003305,"input_tokens":3327,"output_tokens":115,"error":null},{"id":"n004","dataset":"holdout","question":"Show order volume by region, not ranked, for last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7111.68277600018,"input_tokens":3328,"output_tokens":117,"error":null},{"id":"n005","dataset":"holdout","question":"Bottom 3 customers by bookings value last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":3},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":3},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":3},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7016.711862000193,"input_tokens":3325,"output_tokens":116,"error":null},{"id":"n006","dataset":"holdout","question":"Most recent order count by region last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7061.034412000026,"input_tokens":3324,"output_tokens":117,"error":null},{"id":"n007","dataset":"holdout","question":"High value customer revenue by customer last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":true,"raw_mismatch_fields":["pattern_id","sort","limit"],"corrected_mismatch_fields":[],"intervened":true,"latency_ms":7061.119168000005,"input_tokens":3324,"output_tokens":117,"error":null},{"id":"n008","dataset":"holdout","question":"Revenue by product, sorted alphabetically, for June 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7106.342949000009,"input_tokens":3331,"output_tokens":118,"error":null},{"id":"n009","dataset":"holdout","question":"Which products sold the most units last quarter?","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7044.7582169999805,"input_tokens":3325,"output_tokens":115,"error":null},{"id":"n010","dataset":"holdout","question":"Trend revenue over the last quarter by month","expected_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7005.92664900023,"input_tokens":3324,"output_tokens":116,"error":null},{"id":"n011","dataset":"holdout","question":"Quantity by customer in April 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7045.678403000238,"input_tokens":3326,"output_tokens":117,"error":null},{"id":"n012","dataset":"holdout","question":"Top revenue products for April through June","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":5},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":5},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["limit"],"corrected_mismatch_fields":["limit"],"intervened":false,"latency_ms":7000.801041000159,"input_tokens":3323,"output_tokens":116,"error":null},{"id":"n013","dataset":"holdout","question":"Category order volume, all categories, Q2 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7049.0204200000335,"input_tokens":3330,"output_tokens":117,"error":null},{"id":"n014","dataset":"holdout","question":"Least revenue customers last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":5},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":5},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["limit"],"corrected_mismatch_fields":["limit"],"intervened":false,"latency_ms":6994.056760000149,"input_tokens":3321,"output_tokens":116,"error":null},{"id":"n015","dataset":"holdout","question":"For each product category, how many units were sold in May?","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7056.890801999998,"input_tokens":3329,"output_tokens":117,"error":null},{"id":"n016","dataset":"holdout","question":"Rank product categories by revenue in June","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7047.8330379996805,"input_tokens":3323,"output_tokens":117,"error":null},{"id":"n017","dataset":"holdout","question":"Region sales report for last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7101.846799000214,"input_tokens":3322,"output_tokens":118,"error":null},{"id":"n018","dataset":"holdout","question":"Largest product categories by unit sales last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":6923.594216000311,"input_tokens":3324,"output_tokens":115,"error":null},{"id":"n019","dataset":"holdout","question":"Order volume by product for the most recent quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7054.933815999902,"input_tokens":3325,"output_tokens":117,"error":null},{"id":"n020","dataset":"holdout","question":"Monthly order count in Q2","expected_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":6929.596500999651,"input_tokens":3322,"output_tokens":115,"error":null},{"id":"adv001_topline_breakdown","dataset":"adversarial","question":"Top-line revenue by region last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7111.09272099975,"input_tokens":3323,"output_tokens":118,"error":null},{"id":"adv002_leaders_ranking","dataset":"adversarial","question":"Revenue leaders by region last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":false,"raw_mismatch_fields":[],"corrected_mismatch_fields":["pattern_id","sort","limit"],"intervened":true,"latency_ms":7075.838209000267,"input_tokens":3322,"output_tokens":117,"error":null},{"id":"adv003_compare_breakdown","dataset":"adversarial","question":"How do regions compare on revenue last quarter?","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7115.549537000334,"input_tokens":3325,"output_tokens":118,"error":null},{"id":"adv004_strongest_ranking","dataset":"adversarial","question":"Show the strongest markets by revenue last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":5},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["limit"],"corrected_mismatch_fields":["pattern_id","sort","limit"],"intervened":true,"latency_ms":6998.4707759999765,"input_tokens":3324,"output_tokens":116,"error":null},{"id":"adv005_first_rows_breakdown","dataset":"adversarial","question":"Show revenue by region, but only the first ten rows","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[],"sort":[{"field":"region","direction":"asc"}],"limit":10},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":10},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":10},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["filters"],"corrected_mismatch_fields":["filters"],"intervened":false,"latency_ms":7049.267903999862,"input_tokens":3327,"output_tokens":117,"error":null},{"id":"adv006_top_regions_ranking","dataset":"adversarial","question":"Show the top ten regions by revenue","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["filters"],"corrected_mismatch_fields":["filters"],"intervened":false,"latency_ms":7040.915110000242,"input_tokens":3323,"output_tokens":117,"error":null},{"id":"adv007_ordered_breakdown","dataset":"adversarial","question":"Break down revenue by region, ordered by revenue descending","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[],"sort":[{"field":"revenue","direction":"desc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["filters"],"corrected_mismatch_fields":["filters"],"intervened":false,"latency_ms":7170.811364000201,"input_tokens":3326,"output_tokens":119,"error":null},{"id":"adv008_greatest_ranking","dataset":"adversarial","question":"Which regions contribute the greatest revenue?","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["filters"],"corrected_mismatch_fields":["pattern_id","filters","sort","limit"],"intervened":true,"latency_ms":7046.165658999598,"input_tokens":3323,"output_tokens":117,"error":null}]},{"id":"claude-haiku","date":"2026-07-24","system":{"name":"claude-haiku","provider":"bedrock","model":"us.anthropic.claude-haiku-4-5-20251001-v1:0","correction":"none"},"question_count":88,"metrics":{"raw_intent_accuracy":0.9545454545454546,"corrected_intent_accuracy":0.9545454545454546,"schema_compliance":1.0,"intervention_precision":null,"intervention_recall":0.0,"p50_latency_ms":1539.8857569998654,"p95_latency_ms":2018.4364820001974,"input_tokens":430827,"output_tokens":7517,"cost_per_1000_questions_usd":5.3229,"raw_error_taxonomy":{"filters":4},"corrected_error_taxonomy":{"filters":4}},"rows":[{"id":"q001","dataset":"development","question":"Top 5 customers by revenue last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":5},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":5},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":5},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":3223.989579999852,"input_tokens":4896,"output_tokens":84,"error":null},{"id":"q002","dataset":"development","question":"Top 3 regions by revenue last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1658.1871690000298,"input_tokens":4896,"output_tokens":84,"error":null},{"id":"q003","dataset":"development","question":"Top 4 products by revenue last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":4},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":4},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":4},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1379.3485179999152,"input_tokens":4896,"output_tokens":84,"error":null},{"id":"q004","dataset":"development","question":"Top 2 product categories by revenue last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":2},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":2},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":2},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1445.0187380002717,"input_tokens":4897,"output_tokens":84,"error":null},{"id":"q005","dataset":"development","question":"Top 5 customers by units sold last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":5},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":5},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":5},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1515.490781999688,"input_tokens":4897,"output_tokens":84,"error":null},{"id":"q006","dataset":"development","question":"Top 3 products by quantity last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":3},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":3},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":3},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1590.5685929997162,"input_tokens":4896,"output_tokens":84,"error":null},{"id":"q007","dataset":"development","question":"Top 4 regions by order count last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":4},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":4},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":4},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1356.6572869999618,"input_tokens":4897,"output_tokens":84,"error":null},{"id":"q008","dataset":"development","question":"Revenue by month last quarter","expected_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1509.7226759999103,"input_tokens":4892,"output_tokens":83,"error":null},{"id":"q009","dataset":"development","question":"Order count by month last quarter","expected_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1377.9072590000396,"input_tokens":4893,"output_tokens":83,"error":null},{"id":"q010","dataset":"development","question":"Units sold by month last quarter","expected_intent":{"pattern_id":"metric_by_time","metric":"quantity","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_time","metric":"quantity","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_time","metric":"quantity","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1538.6285639997368,"input_tokens":4893,"output_tokens":83,"error":null},{"id":"q011","dataset":"development","question":"Monthly revenue for Q2 2026","expected_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1598.0369849999079,"input_tokens":4895,"output_tokens":83,"error":null},{"id":"q012","dataset":"development","question":"Monthly order volume for April through June 2026","expected_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1639.9969530002636,"input_tokens":4897,"output_tokens":83,"error":null},{"id":"q013","dataset":"development","question":"Revenue by region last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1408.6037689999102,"input_tokens":4892,"output_tokens":87,"error":null},{"id":"q014","dataset":"development","question":"Revenue by customer last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1499.7855840001648,"input_tokens":4892,"output_tokens":87,"error":null},{"id":"q015","dataset":"development","question":"Order count by region last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1483.4688400001141,"input_tokens":4893,"output_tokens":87,"error":null},{"id":"q016","dataset":"development","question":"Units sold by product last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1537.9866660000516,"input_tokens":4893,"output_tokens":87,"error":null},{"id":"q017","dataset":"development","question":"Revenue by product category last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2483.6295879999852,"input_tokens":4893,"output_tokens":87,"error":null},{"id":"q018","dataset":"development","question":"Order count by product category last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1489.2570659999365,"input_tokens":4894,"output_tokens":87,"error":null},{"id":"q019","dataset":"development","question":"Revenue by customer in June 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1533.2408150002266,"input_tokens":4895,"output_tokens":87,"error":null},{"id":"q020","dataset":"development","question":"Units sold by region in May 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1645.8229300001221,"input_tokens":4896,"output_tokens":87,"error":null},{"id":"c001_breakdown","dataset":"holdout","question":"Gross sales across regions last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1507.171567000114,"input_tokens":4894,"output_tokens":87,"error":null},{"id":"c001_ranking","dataset":"holdout","question":"Regions with the highest gross sales last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1348.6994869999762,"input_tokens":4896,"output_tokens":84,"error":null},{"id":"c002_breakdown","dataset":"holdout","question":"Revenue for each customer in Q2 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1785.9829509998235,"input_tokens":4897,"output_tokens":87,"error":null},{"id":"c002_ranking","dataset":"holdout","question":"Top 6 customers by revenue in Q2 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":6},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":6},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":6},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1534.6531079999295,"input_tokens":4900,"output_tokens":84,"error":null},{"id":"c003_breakdown","dataset":"holdout","question":"Order volume broken down by product category last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1798.2909369998197,"input_tokens":4896,"output_tokens":87,"error":null},{"id":"c003_ranking","dataset":"holdout","question":"Product categories with the most orders last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1540.341277999687,"input_tokens":4895,"output_tokens":84,"error":null},{"id":"c004_breakdown","dataset":"holdout","question":"Unit sales by product in June 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1566.0283730003357,"input_tokens":4896,"output_tokens":87,"error":null},{"id":"c004_ranking","dataset":"holdout","question":"Top 4 products by unit sales in June 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":4},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":4},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":4},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1923.2662210001763,"input_tokens":4900,"output_tokens":84,"error":null},{"id":"c005_breakdown","dataset":"holdout","question":"Sales dollars by product category in May 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1628.7460460002876,"input_tokens":4897,"output_tokens":87,"error":null},{"id":"c005_ranking","dataset":"holdout","question":"Best product categories by sales dollars in May 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1402.23529900004,"input_tokens":4898,"output_tokens":84,"error":null},{"id":"c006_breakdown","dataset":"holdout","question":"Number of orders for each region in April 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1982.0090719999826,"input_tokens":4898,"output_tokens":87,"error":null},{"id":"c006_ranking","dataset":"holdout","question":"Leading regions by number of orders in April 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1543.7854330002665,"input_tokens":4898,"output_tokens":84,"error":null},{"id":"c007_breakdown","dataset":"holdout","question":"Bookings value by customer last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1440.372319999824,"input_tokens":4895,"output_tokens":87,"error":null},{"id":"c007_ranking","dataset":"holdout","question":"Largest customers by bookings value last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1589.6979359999932,"input_tokens":4896,"output_tokens":84,"error":null},{"id":"c008_breakdown","dataset":"holdout","question":"Units sold across regions last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1566.950080999959,"input_tokens":4893,"output_tokens":87,"error":null},{"id":"c008_ranking","dataset":"holdout","question":"Rank regions by units sold last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1536.8925180000588,"input_tokens":4895,"output_tokens":84,"error":null},{"id":"c009_breakdown","dataset":"holdout","question":"Revenue grouped by product last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1457.1180829998411,"input_tokens":4893,"output_tokens":87,"error":null},{"id":"c009_ranking","dataset":"holdout","question":"Products ranked by revenue last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1514.4512199999554,"input_tokens":4893,"output_tokens":84,"error":null},{"id":"c010_breakdown","dataset":"holdout","question":"Orders by customer last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1789.5114369998737,"input_tokens":4892,"output_tokens":87,"error":null},{"id":"c010_ranking","dataset":"holdout","question":"Customers with the fewest orders last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1528.2910099999754,"input_tokens":4896,"output_tokens":85,"error":null},{"id":"c011_breakdown","dataset":"holdout","question":"Product category revenue for April through June 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1690.226464999796,"input_tokens":4897,"output_tokens":87,"error":null},{"id":"c011_ranking","dataset":"holdout","question":"Bottom 2 product categories by revenue for April through June 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":2},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":2},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":2},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1868.4860029998163,"input_tokens":4902,"output_tokens":85,"error":null},{"id":"c012_breakdown","dataset":"holdout","question":"Customer unit volume in May 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1449.827452000136,"input_tokens":4895,"output_tokens":87,"error":null},{"id":"c012_ranking","dataset":"holdout","question":"Which customers had the least unit volume in May 2026?","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"quantity","direction":"asc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"quantity","direction":"asc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"quantity","direction":"asc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1886.5332680002211,"input_tokens":4900,"output_tokens":85,"error":null},{"id":"c013_breakdown","dataset":"holdout","question":"Order count by product last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1515.1347609998993,"input_tokens":4893,"output_tokens":87,"error":null},{"id":"c013_ranking","dataset":"holdout","question":"Worst products by order count last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1653.872285000034,"input_tokens":4895,"output_tokens":85,"error":null},{"id":"c014_breakdown","dataset":"holdout","question":"Quantity by product category in June 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1647.6359839998622,"input_tokens":4897,"output_tokens":87,"error":null},{"id":"c014_ranking","dataset":"holdout","question":"Smallest product categories by quantity in June 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"asc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"asc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"asc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1460.5014919998212,"input_tokens":4898,"output_tokens":85,"error":null},{"id":"c015_breakdown","dataset":"holdout","question":"Sales for every product last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2018.4364820001974,"input_tokens":4893,"output_tokens":87,"error":null},{"id":"c015_ranking","dataset":"holdout","question":"First 3 products by sales last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1527.0465209996473,"input_tokens":4896,"output_tokens":84,"error":null},{"id":"c016_breakdown","dataset":"holdout","question":"Region order volume in Q2 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1643.4630809999362,"input_tokens":4896,"output_tokens":87,"error":null},{"id":"c016_ranking","dataset":"holdout","question":"Last 2 regions by order volume in Q2 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":2},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":2},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":2},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1394.359015000191,"input_tokens":4901,"output_tokens":85,"error":null},{"id":"c017_breakdown","dataset":"holdout","question":"Show revenue by customer for April 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1556.975747999786,"input_tokens":4896,"output_tokens":87,"error":null},{"id":"c017_ranking","dataset":"holdout","question":"Show the top 3 customers by revenue for April 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1551.9777539998358,"input_tokens":4901,"output_tokens":84,"error":null},{"id":"c018_breakdown","dataset":"holdout","question":"Product revenue table for last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1599.8850900000434,"input_tokens":4893,"output_tokens":87,"error":null},{"id":"c018_ranking","dataset":"holdout","question":"Top product revenue table for last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1570.1276770000732,"input_tokens":4894,"output_tokens":84,"error":null},{"id":"c019_breakdown","dataset":"holdout","question":"Order count across customers last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1752.5291730003119,"input_tokens":4893,"output_tokens":87,"error":null},{"id":"c019_ranking","dataset":"holdout","question":"Rank customers by order count last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2425.0633199999356,"input_tokens":4895,"output_tokens":84,"error":null},{"id":"c020_breakdown","dataset":"holdout","question":"Category unit sales for Q2 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1701.4252290000513,"input_tokens":4896,"output_tokens":87,"error":null},{"id":"c020_ranking","dataset":"holdout","question":"Categories ranked by unit sales for Q2 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1541.6180229999554,"input_tokens":4898,"output_tokens":84,"error":null},{"id":"n001","dataset":"holdout","question":"Give me the sales dollars by customer for Q2","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1686.210857000333,"input_tokens":4897,"output_tokens":87,"error":null},{"id":"n002","dataset":"holdout","question":"Which five regions generated the most revenue last quarter?","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":5},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":5},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":5},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1523.1390040003134,"input_tokens":4897,"output_tokens":84,"error":null},{"id":"n003","dataset":"holdout","question":"Monthly unit sales for Q2 2026","expected_intent":{"pattern_id":"metric_by_time","metric":"quantity","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_time","metric":"quantity","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_time","metric":"quantity","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1730.7165839997651,"input_tokens":4896,"output_tokens":83,"error":null},{"id":"n004","dataset":"holdout","question":"Show order volume by region, not ranked, for last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1767.3446310000145,"input_tokens":4899,"output_tokens":87,"error":null},{"id":"n005","dataset":"holdout","question":"Bottom 3 customers by bookings value last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":3},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":3},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":3},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1795.3394639998805,"input_tokens":4898,"output_tokens":85,"error":null},{"id":"n006","dataset":"holdout","question":"Most recent order count by region last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1830.8592999997018,"input_tokens":4895,"output_tokens":87,"error":null},{"id":"n007","dataset":"holdout","question":"High value customer revenue by customer last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1473.0972269999256,"input_tokens":4895,"output_tokens":87,"error":null},{"id":"n008","dataset":"holdout","question":"Revenue by product, sorted alphabetically, for June 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2562.435223000193,"input_tokens":4900,"output_tokens":87,"error":null},{"id":"n009","dataset":"holdout","question":"Which products sold the most units last quarter?","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1425.7986729999175,"input_tokens":4896,"output_tokens":84,"error":null},{"id":"n010","dataset":"holdout","question":"Trend revenue over the last quarter by month","expected_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1441.1628620000556,"input_tokens":4896,"output_tokens":83,"error":null},{"id":"n011","dataset":"holdout","question":"Quantity by customer in April 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1534.4962529998156,"input_tokens":4896,"output_tokens":87,"error":null},{"id":"n012","dataset":"holdout","question":"Top revenue products for April through June","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1457.1749040001123,"input_tokens":4894,"output_tokens":84,"error":null},{"id":"n013","dataset":"holdout","question":"Category order volume, all categories, Q2 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1578.0324379998092,"input_tokens":4899,"output_tokens":87,"error":null},{"id":"n014","dataset":"holdout","question":"Least revenue customers last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1619.2596330001834,"input_tokens":4893,"output_tokens":85,"error":null},{"id":"n015","dataset":"holdout","question":"For each product category, how many units were sold in May?","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1692.848332000267,"input_tokens":4900,"output_tokens":87,"error":null},{"id":"n016","dataset":"holdout","question":"Rank product categories by revenue in June","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1498.6069510000561,"input_tokens":4895,"output_tokens":84,"error":null},{"id":"n017","dataset":"holdout","question":"Region sales report for last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1546.8942809998225,"input_tokens":4893,"output_tokens":87,"error":null},{"id":"n018","dataset":"holdout","question":"Largest product categories by unit sales last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1519.8415200002273,"input_tokens":4896,"output_tokens":84,"error":null},{"id":"n019","dataset":"holdout","question":"Order volume by product for the most recent quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1436.4649200001622,"input_tokens":4896,"output_tokens":87,"error":null},{"id":"n020","dataset":"holdout","question":"Monthly order count in Q2","expected_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1468.1027710003036,"input_tokens":4893,"output_tokens":83,"error":null},{"id":"adv001_topline_breakdown","dataset":"adversarial","question":"Top-line revenue by region last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1539.4302360000438,"input_tokens":4895,"output_tokens":87,"error":null},{"id":"adv002_leaders_ranking","dataset":"adversarial","question":"Revenue leaders by region last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1434.3860449998829,"input_tokens":4893,"output_tokens":84,"error":null},{"id":"adv003_compare_breakdown","dataset":"adversarial","question":"How do regions compare on revenue last quarter?","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1629.3741280001086,"input_tokens":4896,"output_tokens":87,"error":null},{"id":"adv004_strongest_ranking","dataset":"adversarial","question":"Show the strongest markets by revenue last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1499.8417139995581,"input_tokens":4895,"output_tokens":84,"error":null},{"id":"adv005_first_rows_breakdown","dataset":"adversarial","question":"Show revenue by region, but only the first ten rows","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[],"sort":[{"field":"region","direction":"asc"}],"limit":10},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-01-01","resolved_end_exclusive":"2026-12-31"}],"sort":[{"field":"region","direction":"asc"}],"limit":10},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-01-01","resolved_end_exclusive":"2026-12-31"}],"sort":[{"field":"region","direction":"asc"}],"limit":10},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["filters"],"corrected_mismatch_fields":["filters"],"intervened":false,"latency_ms":1455.2141839999422,"input_tokens":4898,"output_tokens":87,"error":null},{"id":"adv006_top_regions_ranking","dataset":"adversarial","question":"Show the top ten regions by revenue","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-01-01","resolved_end_exclusive":"2026-12-31"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-01-01","resolved_end_exclusive":"2026-12-31"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["filters"],"corrected_mismatch_fields":["filters"],"intervened":false,"latency_ms":1410.1232240000172,"input_tokens":4894,"output_tokens":84,"error":null},{"id":"adv007_ordered_breakdown","dataset":"adversarial","question":"Break down revenue by region, ordered by revenue descending","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[],"sort":[{"field":"revenue","direction":"desc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-01-01","resolved_end_exclusive":"2026-12-31"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-01-01","resolved_end_exclusive":"2026-12-31"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["filters"],"corrected_mismatch_fields":["filters"],"intervened":false,"latency_ms":1471.6431950000697,"input_tokens":4898,"output_tokens":86,"error":null},{"id":"adv008_greatest_ranking","dataset":"adversarial","question":"Which regions contribute the greatest revenue?","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-22"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-22"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["filters"],"corrected_mismatch_fields":["filters"],"intervened":false,"latency_ms":1468.7663180002346,"input_tokens":4894,"output_tokens":84,"error":null}]},{"id":"claude-sonnet","date":"2026-07-24","system":{"name":"claude-sonnet","provider":"bedrock","model":"us.anthropic.claude-sonnet-4-6","correction":"none"},"question_count":88,"metrics":{"raw_intent_accuracy":0.9431818181818182,"corrected_intent_accuracy":0.9431818181818182,"schema_compliance":1.0,"intervention_precision":null,"intervention_recall":0.0,"p50_latency_ms":1982.6321614998506,"p95_latency_ms":2807.9806829996414,"input_tokens":430915,"output_tokens":7514,"cost_per_1000_questions_usd":15.9711,"raw_error_taxonomy":{"pattern_id":1,"sort":1,"limit":1,"filters":4},"corrected_error_taxonomy":{"pattern_id":1,"sort":1,"limit":1,"filters":4}},"rows":[{"id":"q001","dataset":"development","question":"Top 5 customers by revenue last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":5},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":5},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":5},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1875.8058690000325,"input_tokens":4897,"output_tokens":84,"error":null},{"id":"q002","dataset":"development","question":"Top 3 regions by revenue last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1796.656516999974,"input_tokens":4897,"output_tokens":84,"error":null},{"id":"q003","dataset":"development","question":"Top 4 products by revenue last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":4},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":4},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":4},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1792.2887250001622,"input_tokens":4897,"output_tokens":84,"error":null},{"id":"q004","dataset":"development","question":"Top 2 product categories by revenue last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":2},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":2},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":2},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2737.44620799971,"input_tokens":4898,"output_tokens":84,"error":null},{"id":"q005","dataset":"development","question":"Top 5 customers by units sold last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":5},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":5},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":5},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1958.618082999692,"input_tokens":4898,"output_tokens":84,"error":null},{"id":"q006","dataset":"development","question":"Top 3 products by quantity last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":3},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":3},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":3},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1810.3929179997067,"input_tokens":4897,"output_tokens":84,"error":null},{"id":"q007","dataset":"development","question":"Top 4 regions by order count last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":4},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":4},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":4},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1890.5383689998416,"input_tokens":4898,"output_tokens":84,"error":null},{"id":"q008","dataset":"development","question":"Revenue by month last quarter","expected_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1786.8239849999554,"input_tokens":4893,"output_tokens":83,"error":null},{"id":"q009","dataset":"development","question":"Order count by month last quarter","expected_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2004.4055899998057,"input_tokens":4894,"output_tokens":83,"error":null},{"id":"q010","dataset":"development","question":"Units sold by month last quarter","expected_intent":{"pattern_id":"metric_by_time","metric":"quantity","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_time","metric":"quantity","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_time","metric":"quantity","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":3908.8365530001283,"input_tokens":4894,"output_tokens":83,"error":null},{"id":"q011","dataset":"development","question":"Monthly revenue for Q2 2026","expected_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2971.918935999838,"input_tokens":4896,"output_tokens":83,"error":null},{"id":"q012","dataset":"development","question":"Monthly order volume for April through June 2026","expected_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2518.6536299997897,"input_tokens":4898,"output_tokens":83,"error":null},{"id":"q013","dataset":"development","question":"Revenue by region last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2097.0165159997123,"input_tokens":4893,"output_tokens":87,"error":null},{"id":"q014","dataset":"development","question":"Revenue by customer last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2015.7028230000833,"input_tokens":4893,"output_tokens":87,"error":null},{"id":"q015","dataset":"development","question":"Order count by region last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1833.1257180002467,"input_tokens":4894,"output_tokens":87,"error":null},{"id":"q016","dataset":"development","question":"Units sold by product last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2127.65557099965,"input_tokens":4894,"output_tokens":87,"error":null},{"id":"q017","dataset":"development","question":"Revenue by product category last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2884.926193999945,"input_tokens":4894,"output_tokens":87,"error":null},{"id":"q018","dataset":"development","question":"Order count by product category last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2026.1821980002424,"input_tokens":4895,"output_tokens":87,"error":null},{"id":"q019","dataset":"development","question":"Revenue by customer in June 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1840.5843900000036,"input_tokens":4896,"output_tokens":87,"error":null},{"id":"q020","dataset":"development","question":"Units sold by region in May 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1864.3149800000174,"input_tokens":4897,"output_tokens":87,"error":null},{"id":"c001_breakdown","dataset":"holdout","question":"Gross sales across regions last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2398.9604739999777,"input_tokens":4895,"output_tokens":87,"error":null},{"id":"c001_ranking","dataset":"holdout","question":"Regions with the highest gross sales last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1906.9511650000095,"input_tokens":4897,"output_tokens":84,"error":null},{"id":"c002_breakdown","dataset":"holdout","question":"Revenue for each customer in Q2 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1994.0919089999625,"input_tokens":4898,"output_tokens":87,"error":null},{"id":"c002_ranking","dataset":"holdout","question":"Top 6 customers by revenue in Q2 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":6},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":6},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":6},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2090.751009000087,"input_tokens":4901,"output_tokens":84,"error":null},{"id":"c003_breakdown","dataset":"holdout","question":"Order volume broken down by product category last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2097.0382819996303,"input_tokens":4897,"output_tokens":87,"error":null},{"id":"c003_ranking","dataset":"holdout","question":"Product categories with the most orders last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1810.7290660000217,"input_tokens":4896,"output_tokens":84,"error":null},{"id":"c004_breakdown","dataset":"holdout","question":"Unit sales by product in June 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2044.064712999898,"input_tokens":4897,"output_tokens":87,"error":null},{"id":"c004_ranking","dataset":"holdout","question":"Top 4 products by unit sales in June 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":4},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":4},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":4},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1882.1817450002527,"input_tokens":4901,"output_tokens":84,"error":null},{"id":"c005_breakdown","dataset":"holdout","question":"Sales dollars by product category in May 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1961.674265000056,"input_tokens":4898,"output_tokens":87,"error":null},{"id":"c005_ranking","dataset":"holdout","question":"Best product categories by sales dollars in May 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2235.1515350001137,"input_tokens":4899,"output_tokens":84,"error":null},{"id":"c006_breakdown","dataset":"holdout","question":"Number of orders for each region in April 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1746.995608999896,"input_tokens":4899,"output_tokens":87,"error":null},{"id":"c006_ranking","dataset":"holdout","question":"Leading regions by number of orders in April 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1925.8461839999654,"input_tokens":4899,"output_tokens":84,"error":null},{"id":"c007_breakdown","dataset":"holdout","question":"Bookings value by customer last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2017.3347350000768,"input_tokens":4896,"output_tokens":87,"error":null},{"id":"c007_ranking","dataset":"holdout","question":"Largest customers by bookings value last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1925.4987640001673,"input_tokens":4897,"output_tokens":84,"error":null},{"id":"c008_breakdown","dataset":"holdout","question":"Units sold across regions last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1971.7561399997976,"input_tokens":4894,"output_tokens":87,"error":null},{"id":"c008_ranking","dataset":"holdout","question":"Rank regions by units sold last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1961.3770149999254,"input_tokens":4896,"output_tokens":84,"error":null},{"id":"c009_breakdown","dataset":"holdout","question":"Revenue grouped by product last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1991.223720999642,"input_tokens":4894,"output_tokens":87,"error":null},{"id":"c009_ranking","dataset":"holdout","question":"Products ranked by revenue last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1924.0224770001078,"input_tokens":4894,"output_tokens":84,"error":null},{"id":"c010_breakdown","dataset":"holdout","question":"Orders by customer last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2140.5876710000484,"input_tokens":4893,"output_tokens":87,"error":null},{"id":"c010_ranking","dataset":"holdout","question":"Customers with the fewest orders last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2086.311243000182,"input_tokens":4897,"output_tokens":85,"error":null},{"id":"c011_breakdown","dataset":"holdout","question":"Product category revenue for April through June 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2005.3049590001137,"input_tokens":4898,"output_tokens":87,"error":null},{"id":"c011_ranking","dataset":"holdout","question":"Bottom 2 product categories by revenue for April through June 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":2},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":2},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":2},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2111.4417569997386,"input_tokens":4903,"output_tokens":85,"error":null},{"id":"c012_breakdown","dataset":"holdout","question":"Customer unit volume in May 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2091.401510000196,"input_tokens":4896,"output_tokens":87,"error":null},{"id":"c012_ranking","dataset":"holdout","question":"Which customers had the least unit volume in May 2026?","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"quantity","direction":"asc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"quantity","direction":"asc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"quantity","direction":"asc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1940.1087950000147,"input_tokens":4901,"output_tokens":85,"error":null},{"id":"c013_breakdown","dataset":"holdout","question":"Order count by product last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1944.2710429998442,"input_tokens":4894,"output_tokens":87,"error":null},{"id":"c013_ranking","dataset":"holdout","question":"Worst products by order count last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1910.4764999997315,"input_tokens":4896,"output_tokens":85,"error":null},{"id":"c014_breakdown","dataset":"holdout","question":"Quantity by product category in June 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1964.422709000246,"input_tokens":4898,"output_tokens":87,"error":null},{"id":"c014_ranking","dataset":"holdout","question":"Smallest product categories by quantity in June 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"asc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"asc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"asc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1870.6733089998124,"input_tokens":4899,"output_tokens":85,"error":null},{"id":"c015_breakdown","dataset":"holdout","question":"Sales for every product last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2130.19433799991,"input_tokens":4894,"output_tokens":87,"error":null},{"id":"c015_ranking","dataset":"holdout","question":"First 3 products by sales last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2011.481596000067,"input_tokens":4897,"output_tokens":84,"error":null},{"id":"c016_breakdown","dataset":"holdout","question":"Region order volume in Q2 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2023.027003000152,"input_tokens":4897,"output_tokens":87,"error":null},{"id":"c016_ranking","dataset":"holdout","question":"Last 2 regions by order volume in Q2 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":2},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":2},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":2},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2312.896090999857,"input_tokens":4902,"output_tokens":85,"error":null},{"id":"c017_breakdown","dataset":"holdout","question":"Show revenue by customer for April 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2308.7555530000827,"input_tokens":4897,"output_tokens":87,"error":null},{"id":"c017_ranking","dataset":"holdout","question":"Show the top 3 customers by revenue for April 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1896.6332150002927,"input_tokens":4902,"output_tokens":84,"error":null},{"id":"c018_breakdown","dataset":"holdout","question":"Product revenue table for last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2221.7953170002147,"input_tokens":4894,"output_tokens":87,"error":null},{"id":"c018_ranking","dataset":"holdout","question":"Top product revenue table for last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2260.7032589999108,"input_tokens":4895,"output_tokens":84,"error":null},{"id":"c019_breakdown","dataset":"holdout","question":"Order count across customers last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2018.9478329998565,"input_tokens":4894,"output_tokens":87,"error":null},{"id":"c019_ranking","dataset":"holdout","question":"Rank customers by order count last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2109.4307629996365,"input_tokens":4896,"output_tokens":84,"error":null},{"id":"c020_breakdown","dataset":"holdout","question":"Category unit sales for Q2 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1828.838113999609,"input_tokens":4897,"output_tokens":87,"error":null},{"id":"c020_ranking","dataset":"holdout","question":"Categories ranked by unit sales for Q2 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1870.2350250000563,"input_tokens":4899,"output_tokens":84,"error":null},{"id":"n001","dataset":"holdout","question":"Give me the sales dollars by customer for Q2","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1864.3095760003234,"input_tokens":4898,"output_tokens":87,"error":null},{"id":"n002","dataset":"holdout","question":"Which five regions generated the most revenue last quarter?","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":5},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":5},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":5},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1836.7386909999368,"input_tokens":4898,"output_tokens":84,"error":null},{"id":"n003","dataset":"holdout","question":"Monthly unit sales for Q2 2026","expected_intent":{"pattern_id":"metric_by_time","metric":"quantity","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_time","metric":"quantity","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_time","metric":"quantity","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2008.9383540002927,"input_tokens":4897,"output_tokens":83,"error":null},{"id":"n004","dataset":"holdout","question":"Show order volume by region, not ranked, for last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2239.9860220002665,"input_tokens":4900,"output_tokens":87,"error":null},{"id":"n005","dataset":"holdout","question":"Bottom 3 customers by bookings value last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":3},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":3},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":3},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2181.9246820000444,"input_tokens":4899,"output_tokens":85,"error":null},{"id":"n006","dataset":"holdout","question":"Most recent order count by region last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2219.061786000111,"input_tokens":4896,"output_tokens":87,"error":null},{"id":"n007","dataset":"holdout","question":"High value customer revenue by customer last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["pattern_id","sort","limit"],"corrected_mismatch_fields":["pattern_id","sort","limit"],"intervened":false,"latency_ms":1952.828042999954,"input_tokens":4896,"output_tokens":84,"error":null},{"id":"n008","dataset":"holdout","question":"Revenue by product, sorted alphabetically, for June 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1871.5806780001003,"input_tokens":4901,"output_tokens":87,"error":null},{"id":"n009","dataset":"holdout","question":"Which products sold the most units last quarter?","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2065.112104000036,"input_tokens":4897,"output_tokens":84,"error":null},{"id":"n010","dataset":"holdout","question":"Trend revenue over the last quarter by month","expected_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1787.122774999716,"input_tokens":4897,"output_tokens":83,"error":null},{"id":"n011","dataset":"holdout","question":"Quantity by customer in April 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2020.3520639997805,"input_tokens":4897,"output_tokens":87,"error":null},{"id":"n012","dataset":"holdout","question":"Top revenue products for April through June","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1938.9904179997757,"input_tokens":4895,"output_tokens":84,"error":null},{"id":"n013","dataset":"holdout","question":"Category order volume, all categories, Q2 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1842.3953029996483,"input_tokens":4900,"output_tokens":87,"error":null},{"id":"n014","dataset":"holdout","question":"Least revenue customers last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1938.101980999818,"input_tokens":4894,"output_tokens":85,"error":null},{"id":"n015","dataset":"holdout","question":"For each product category, how many units were sold in May?","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2034.1865310001594,"input_tokens":4901,"output_tokens":87,"error":null},{"id":"n016","dataset":"holdout","question":"Rank product categories by revenue in June","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1822.7835170000617,"input_tokens":4896,"output_tokens":84,"error":null},{"id":"n017","dataset":"holdout","question":"Region sales report for last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1870.983305000209,"input_tokens":4894,"output_tokens":87,"error":null},{"id":"n018","dataset":"holdout","question":"Largest product categories by unit sales last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2101.02564600038,"input_tokens":4897,"output_tokens":84,"error":null},{"id":"n019","dataset":"holdout","question":"Order volume by product for the most recent quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1974.0406020000592,"input_tokens":4897,"output_tokens":87,"error":null},{"id":"n020","dataset":"holdout","question":"Monthly order count in Q2","expected_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1833.9459549997628,"input_tokens":4894,"output_tokens":83,"error":null},{"id":"adv001_topline_breakdown","dataset":"adversarial","question":"Top-line revenue by region last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":4718.402485999832,"input_tokens":4896,"output_tokens":87,"error":null},{"id":"adv002_leaders_ranking","dataset":"adversarial","question":"Revenue leaders by region last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1868.1165170000895,"input_tokens":4894,"output_tokens":84,"error":null},{"id":"adv003_compare_breakdown","dataset":"adversarial","question":"How do regions compare on revenue last quarter?","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2261.8817849997868,"input_tokens":4897,"output_tokens":87,"error":null},{"id":"adv004_strongest_ranking","dataset":"adversarial","question":"Show the strongest markets by revenue last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1886.367908000011,"input_tokens":4896,"output_tokens":84,"error":null},{"id":"adv005_first_rows_breakdown","dataset":"adversarial","question":"Show revenue by region, but only the first ten rows","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[],"sort":[{"field":"region","direction":"asc"}],"limit":10},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":10},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":10},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["filters"],"corrected_mismatch_fields":["filters"],"intervened":false,"latency_ms":1807.6910479999242,"input_tokens":4899,"output_tokens":87,"error":null},{"id":"adv006_top_regions_ranking","dataset":"adversarial","question":"Show the top ten regions by revenue","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["filters"],"corrected_mismatch_fields":["filters"],"intervened":false,"latency_ms":1727.7261460003501,"input_tokens":4895,"output_tokens":84,"error":null},{"id":"adv007_ordered_breakdown","dataset":"adversarial","question":"Break down revenue by region, ordered by revenue descending","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[],"sort":[{"field":"revenue","direction":"desc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["filters"],"corrected_mismatch_fields":["filters"],"intervened":false,"latency_ms":2807.9806829996414,"input_tokens":4899,"output_tokens":86,"error":null},{"id":"adv008_greatest_ranking","dataset":"adversarial","question":"Which regions contribute the greatest revenue?","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["filters"],"corrected_mismatch_fields":["filters"],"intervened":false,"latency_ms":2321.8082100001993,"input_tokens":4895,"output_tokens":84,"error":null}]},{"id":"openai-small","date":"2026-07-24","system":{"name":"openai-small","provider":"openai","model":"gpt-5.6-luna","correction":"none"},"question_count":88,"metrics":{"raw_intent_accuracy":0.9318181818181818,"corrected_intent_accuracy":0.9318181818181818,"schema_compliance":1.0,"intervention_precision":null,"intervention_recall":0.0,"p50_latency_ms":1342.2043424998265,"p95_latency_ms":7015.858423000282,"input_tokens":305219,"output_tokens":6749,"cost_per_1000_questions_usd":3.9286,"raw_error_taxonomy":{"limit":3,"pattern_id":4,"sort":5},"corrected_error_taxonomy":{"limit":3,"pattern_id":4,"sort":5}},"rows":[{"id":"q001","dataset":"development","question":"Top 5 customers by revenue last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":5},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":5},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":5},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2139.592582000205,"input_tokens":3468,"output_tokens":79,"error":null},{"id":"q002","dataset":"development","question":"Top 3 regions by revenue last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1338.6043150003388,"input_tokens":3468,"output_tokens":79,"error":null},{"id":"q003","dataset":"development","question":"Top 4 products by revenue last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":4},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":4},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":4},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1578.5974149998765,"input_tokens":3468,"output_tokens":79,"error":null},{"id":"q004","dataset":"development","question":"Top 2 product categories by revenue last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":2},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":2},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":2},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1459.0322850003759,"input_tokens":3469,"output_tokens":79,"error":null},{"id":"q005","dataset":"development","question":"Top 5 customers by units sold last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":5},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":5},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":5},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1440.0123450000137,"input_tokens":3469,"output_tokens":77,"error":null},{"id":"q006","dataset":"development","question":"Top 3 products by quantity last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":3},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":3},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":3},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1464.3956260001687,"input_tokens":3468,"output_tokens":77,"error":null},{"id":"q007","dataset":"development","question":"Top 4 regions by order count last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":4},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":4},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":4},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":16265.531143999851,"input_tokens":3469,"output_tokens":77,"error":null},{"id":"q008","dataset":"development","question":"Revenue by month last quarter","expected_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":8727.256147999924,"input_tokens":3465,"output_tokens":77,"error":null},{"id":"q009","dataset":"development","question":"Order count by month last quarter","expected_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1423.6239010001555,"input_tokens":3466,"output_tokens":76,"error":null},{"id":"q010","dataset":"development","question":"Units sold by month last quarter","expected_intent":{"pattern_id":"metric_by_time","metric":"quantity","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_time","metric":"quantity","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_time","metric":"quantity","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1477.1854989999156,"input_tokens":3466,"output_tokens":76,"error":null},{"id":"q011","dataset":"development","question":"Monthly revenue for Q2 2026","expected_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1302.1585080000477,"input_tokens":3468,"output_tokens":77,"error":null},{"id":"q012","dataset":"development","question":"Monthly order volume for April through June 2026","expected_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1181.0688000000482,"input_tokens":3470,"output_tokens":76,"error":null},{"id":"q013","dataset":"development","question":"Revenue by region last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1292.8291410003112,"input_tokens":3465,"output_tokens":79,"error":null},{"id":"q014","dataset":"development","question":"Revenue by customer last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1386.680689000059,"input_tokens":3465,"output_tokens":79,"error":null},{"id":"q015","dataset":"development","question":"Order count by region last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1411.2825130000601,"input_tokens":3466,"output_tokens":78,"error":null},{"id":"q016","dataset":"development","question":"Units sold by product last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1783.832742999948,"input_tokens":3466,"output_tokens":78,"error":null},{"id":"q017","dataset":"development","question":"Revenue by product category last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1484.5552279998628,"input_tokens":3466,"output_tokens":79,"error":null},{"id":"q018","dataset":"development","question":"Order count by product category last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":16199.599605000003,"input_tokens":3467,"output_tokens":78,"error":null},{"id":"q019","dataset":"development","question":"Revenue by customer in June 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1176.7511669995656,"input_tokens":3468,"output_tokens":79,"error":null},{"id":"q020","dataset":"development","question":"Units sold by region in May 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1192.675836000035,"input_tokens":3469,"output_tokens":78,"error":null},{"id":"c001_breakdown","dataset":"holdout","question":"Gross sales across regions last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1158.6879589999626,"input_tokens":3466,"output_tokens":79,"error":null},{"id":"c001_ranking","dataset":"holdout","question":"Regions with the highest gross sales last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1326.2114909998672,"input_tokens":3468,"output_tokens":79,"error":null},{"id":"c002_breakdown","dataset":"holdout","question":"Revenue for each customer in Q2 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1145.427219999874,"input_tokens":3470,"output_tokens":79,"error":null},{"id":"c002_ranking","dataset":"holdout","question":"Top 6 customers by revenue in Q2 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":6},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":6},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":6},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1318.5539099999914,"input_tokens":3472,"output_tokens":79,"error":null},{"id":"c003_breakdown","dataset":"holdout","question":"Order volume broken down by product category last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1089.7362840000824,"input_tokens":3469,"output_tokens":78,"error":null},{"id":"c003_ranking","dataset":"holdout","question":"Product categories with the most orders last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1360.3684559998328,"input_tokens":3468,"output_tokens":77,"error":null},{"id":"c004_breakdown","dataset":"holdout","question":"Unit sales by product in June 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1226.0741740001322,"input_tokens":3469,"output_tokens":78,"error":null},{"id":"c004_ranking","dataset":"holdout","question":"Top 4 products by unit sales in June 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":4},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":4},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":4},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1356.1446479998267,"input_tokens":3472,"output_tokens":77,"error":null},{"id":"c005_breakdown","dataset":"holdout","question":"Sales dollars by product category in May 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1533.996107999883,"input_tokens":3470,"output_tokens":79,"error":null},{"id":"c005_ranking","dataset":"holdout","question":"Best product categories by sales dollars in May 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1260.0804719995722,"input_tokens":3471,"output_tokens":79,"error":null},{"id":"c006_breakdown","dataset":"holdout","question":"Number of orders for each region in April 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":7015.858423000282,"input_tokens":3471,"output_tokens":78,"error":null},{"id":"c006_ranking","dataset":"holdout","question":"Leading regions by number of orders in April 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1212.4418540006445,"input_tokens":3471,"output_tokens":77,"error":null},{"id":"c007_breakdown","dataset":"holdout","question":"Bookings value by customer last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1241.5623899996717,"input_tokens":3467,"output_tokens":79,"error":null},{"id":"c007_ranking","dataset":"holdout","question":"Largest customers by bookings value last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1156.7245240003103,"input_tokens":3467,"output_tokens":79,"error":null},{"id":"c008_breakdown","dataset":"holdout","question":"Units sold across regions last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1339.3677049998587,"input_tokens":3466,"output_tokens":78,"error":null},{"id":"c008_ranking","dataset":"holdout","question":"Rank regions by units sold last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1371.944153000186,"input_tokens":3467,"output_tokens":77,"error":null},{"id":"c009_breakdown","dataset":"holdout","question":"Revenue grouped by product last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1352.6138440001887,"input_tokens":3466,"output_tokens":79,"error":null},{"id":"c009_ranking","dataset":"holdout","question":"Products ranked by revenue last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1364.3878060001953,"input_tokens":3466,"output_tokens":79,"error":null},{"id":"c010_breakdown","dataset":"holdout","question":"Orders by customer last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1268.6050599995724,"input_tokens":3465,"output_tokens":78,"error":null},{"id":"c010_ranking","dataset":"holdout","question":"Customers with the fewest orders last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1655.6998050000402,"input_tokens":3468,"output_tokens":77,"error":null},{"id":"c011_breakdown","dataset":"holdout","question":"Product category revenue for April through June 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1334.8452129994257,"input_tokens":3470,"output_tokens":79,"error":null},{"id":"c011_ranking","dataset":"holdout","question":"Bottom 2 product categories by revenue for April through June 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":2},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":2},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":2},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1531.4359369995145,"input_tokens":3474,"output_tokens":79,"error":null},{"id":"c012_breakdown","dataset":"holdout","question":"Customer unit volume in May 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1453.7661750000552,"input_tokens":3468,"output_tokens":78,"error":null},{"id":"c012_ranking","dataset":"holdout","question":"Which customers had the least unit volume in May 2026?","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"quantity","direction":"asc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"quantity","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"quantity","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["limit"],"corrected_mismatch_fields":["limit"],"intervened":false,"latency_ms":16194.630257999961,"input_tokens":3473,"output_tokens":77,"error":null},{"id":"c013_breakdown","dataset":"holdout","question":"Order count by product last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1190.518677,"input_tokens":3466,"output_tokens":78,"error":null},{"id":"c013_ranking","dataset":"holdout","question":"Worst products by order count last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1387.3937709995516,"input_tokens":3467,"output_tokens":77,"error":null},{"id":"c014_breakdown","dataset":"holdout","question":"Quantity by product category in June 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1479.5223199998873,"input_tokens":3469,"output_tokens":78,"error":null},{"id":"c014_ranking","dataset":"holdout","question":"Smallest product categories by quantity in June 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"asc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"asc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"asc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1277.7175210003406,"input_tokens":3471,"output_tokens":77,"error":null},{"id":"c015_breakdown","dataset":"holdout","question":"Sales for every product last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1683.2054970000172,"input_tokens":3466,"output_tokens":79,"error":null},{"id":"c015_ranking","dataset":"holdout","question":"First 3 products by sales last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1214.6197520005444,"input_tokens":3468,"output_tokens":79,"error":null},{"id":"c016_breakdown","dataset":"holdout","question":"Region order volume in Q2 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1121.7316070005836,"input_tokens":3469,"output_tokens":78,"error":null},{"id":"c016_ranking","dataset":"holdout","question":"Last 2 regions by order volume in Q2 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":2},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":2},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":2},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1466.7696330006947,"input_tokens":3473,"output_tokens":77,"error":null},{"id":"c017_breakdown","dataset":"holdout","question":"Show revenue by customer for April 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1350.1152959997853,"input_tokens":3469,"output_tokens":79,"error":null},{"id":"c017_ranking","dataset":"holdout","question":"Show the top 3 customers by revenue for April 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1489.772207999522,"input_tokens":3473,"output_tokens":79,"error":null},{"id":"c018_breakdown","dataset":"holdout","question":"Product revenue table for last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1388.6994089998552,"input_tokens":3466,"output_tokens":79,"error":null},{"id":"c018_ranking","dataset":"holdout","question":"Top product revenue table for last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1182.1437910002714,"input_tokens":3467,"output_tokens":79,"error":null},{"id":"c019_breakdown","dataset":"holdout","question":"Order count across customers last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1180.8294720003687,"input_tokens":3466,"output_tokens":78,"error":null},{"id":"c019_ranking","dataset":"holdout","question":"Rank customers by order count last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1501.7954619997909,"input_tokens":3467,"output_tokens":77,"error":null},{"id":"c020_breakdown","dataset":"holdout","question":"Category unit sales for Q2 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1305.489180999757,"input_tokens":3469,"output_tokens":78,"error":null},{"id":"c020_ranking","dataset":"holdout","question":"Categories ranked by unit sales for Q2 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1290.749703000074,"input_tokens":3471,"output_tokens":77,"error":null},{"id":"n001","dataset":"holdout","question":"Give me the sales dollars by customer for Q2","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1161.0222850003993,"input_tokens":3470,"output_tokens":79,"error":null},{"id":"n002","dataset":"holdout","question":"Which five regions generated the most revenue last quarter?","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":5},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":5},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":5},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1305.8982140000808,"input_tokens":3470,"output_tokens":79,"error":null},{"id":"n003","dataset":"holdout","question":"Monthly unit sales for Q2 2026","expected_intent":{"pattern_id":"metric_by_time","metric":"quantity","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_time","metric":"quantity","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_time","metric":"quantity","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1119.5101810008055,"input_tokens":3469,"output_tokens":76,"error":null},{"id":"n004","dataset":"holdout","question":"Show order volume by region, not ranked, for last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":3455.405814000187,"input_tokens":3472,"output_tokens":78,"error":null},{"id":"n005","dataset":"holdout","question":"Bottom 3 customers by bookings value last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":3},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":3},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":3},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2618.731896999634,"input_tokens":3469,"output_tokens":79,"error":null},{"id":"n006","dataset":"holdout","question":"Most recent order count by region last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["pattern_id","sort","limit"],"corrected_mismatch_fields":["pattern_id","sort","limit"],"intervened":false,"latency_ms":1539.9033999992753,"input_tokens":3468,"output_tokens":77,"error":null},{"id":"n007","dataset":"holdout","question":"High value customer revenue by customer last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["pattern_id","sort","limit"],"corrected_mismatch_fields":["pattern_id","sort","limit"],"intervened":false,"latency_ms":1414.9133599994457,"input_tokens":3468,"output_tokens":79,"error":null},{"id":"n008","dataset":"holdout","question":"Revenue by product, sorted alphabetically, for June 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1263.5444300003655,"input_tokens":3473,"output_tokens":79,"error":null},{"id":"n009","dataset":"holdout","question":"Which products sold the most units last quarter?","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1192.186840999966,"input_tokens":3469,"output_tokens":77,"error":null},{"id":"n010","dataset":"holdout","question":"Trend revenue over the last quarter by month","expected_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1112.0738350000465,"input_tokens":3468,"output_tokens":77,"error":null},{"id":"n011","dataset":"holdout","question":"Quantity by customer in April 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1481.2164420000045,"input_tokens":3468,"output_tokens":78,"error":null},{"id":"n012","dataset":"holdout","question":"Top revenue products for April through June","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1345.0409799997942,"input_tokens":3467,"output_tokens":79,"error":null},{"id":"n013","dataset":"holdout","question":"Category order volume, all categories, Q2 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1337.6939640002092,"input_tokens":3472,"output_tokens":78,"error":null},{"id":"n014","dataset":"holdout","question":"Least revenue customers last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1723.0015519999142,"input_tokens":3465,"output_tokens":79,"error":null},{"id":"n015","dataset":"holdout","question":"For each product category, how many units were sold in May?","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1413.1238359996132,"input_tokens":3473,"output_tokens":78,"error":null},{"id":"n016","dataset":"holdout","question":"Rank product categories by revenue in June","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1447.2131169995919,"input_tokens":3467,"output_tokens":79,"error":null},{"id":"n017","dataset":"holdout","question":"Region sales report for last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1140.495059999921,"input_tokens":3466,"output_tokens":79,"error":null},{"id":"n018","dataset":"holdout","question":"Largest product categories by unit sales last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1161.0006769997199,"input_tokens":3468,"output_tokens":77,"error":null},{"id":"n019","dataset":"holdout","question":"Order volume by product for the most recent quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1136.3714279996202,"input_tokens":3469,"output_tokens":78,"error":null},{"id":"n020","dataset":"holdout","question":"Monthly order count in Q2","expected_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1211.3952780000545,"input_tokens":3466,"output_tokens":76,"error":null},{"id":"adv001_topline_breakdown","dataset":"adversarial","question":"Top-line revenue by region last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":100},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["pattern_id","sort"],"corrected_mismatch_fields":["pattern_id","sort"],"intervened":false,"latency_ms":1390.2724930003387,"input_tokens":3467,"output_tokens":79,"error":null},{"id":"adv002_leaders_ranking","dataset":"adversarial","question":"Revenue leaders by region last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1139.8208149994389,"input_tokens":3466,"output_tokens":79,"error":null},{"id":"adv003_compare_breakdown","dataset":"adversarial","question":"How do regions compare on revenue last quarter?","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1307.6644789998682,"input_tokens":3469,"output_tokens":79,"error":null},{"id":"adv004_strongest_ranking","dataset":"adversarial","question":"Show the strongest markets by revenue last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1211.5142319998995,"input_tokens":3468,"output_tokens":79,"error":null},{"id":"adv005_first_rows_breakdown","dataset":"adversarial","question":"Show revenue by region, but only the first ten rows","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[],"sort":[{"field":"region","direction":"asc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["pattern_id","sort"],"corrected_mismatch_fields":["pattern_id","sort"],"intervened":false,"latency_ms":1291.0486829996444,"input_tokens":3471,"output_tokens":47,"error":null},{"id":"adv006_top_regions_ranking","dataset":"adversarial","question":"Show the top ten regions by revenue","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1383.5004879992994,"input_tokens":3467,"output_tokens":47,"error":null},{"id":"adv007_ordered_breakdown","dataset":"adversarial","question":"Break down revenue by region, ordered by revenue descending","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[],"sort":[{"field":"revenue","direction":"desc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["sort"],"corrected_mismatch_fields":["sort"],"intervened":false,"latency_ms":1181.7074460004733,"input_tokens":3470,"output_tokens":47,"error":null},{"id":"adv008_greatest_ranking","dataset":"adversarial","question":"Which regions contribute the greatest revenue?","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1200.515323000218,"input_tokens":3467,"output_tokens":47,"error":null}]},{"id":"openai-strong","date":"2026-07-24","system":{"name":"openai-strong","provider":"openai","model":"gpt-5.6-sol","correction":"none"},"question_count":88,"metrics":{"raw_intent_accuracy":0.9545454545454546,"corrected_intent_accuracy":0.9545454545454546,"schema_compliance":1.0,"intervention_precision":null,"intervention_recall":0.0,"p50_latency_ms":1891.9756190000498,"p95_latency_ms":3356.75517799973,"input_tokens":305219,"output_tokens":6749,"cost_per_1000_questions_usd":19.6428,"raw_error_taxonomy":{"pattern_id":3,"sort":4,"limit":2},"corrected_error_taxonomy":{"pattern_id":3,"sort":4,"limit":2}},"rows":[{"id":"q001","dataset":"development","question":"Top 5 customers by revenue last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":5},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":5},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":5},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2216.5272279999044,"input_tokens":3468,"output_tokens":79,"error":null},{"id":"q002","dataset":"development","question":"Top 3 regions by revenue last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2443.384799999876,"input_tokens":3468,"output_tokens":79,"error":null},{"id":"q003","dataset":"development","question":"Top 4 products by revenue last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":4},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":4},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":4},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2241.2725919998593,"input_tokens":3468,"output_tokens":79,"error":null},{"id":"q004","dataset":"development","question":"Top 2 product categories by revenue last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":2},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":2},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":2},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1665.762851999716,"input_tokens":3469,"output_tokens":79,"error":null},{"id":"q005","dataset":"development","question":"Top 5 customers by units sold last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":5},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":5},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":5},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2896.6156830001637,"input_tokens":3469,"output_tokens":77,"error":null},{"id":"q006","dataset":"development","question":"Top 3 products by quantity last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":3},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":3},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":3},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":3284.77942399968,"input_tokens":3468,"output_tokens":77,"error":null},{"id":"q007","dataset":"development","question":"Top 4 regions by order count last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":4},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":4},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":4},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1951.5764510001645,"input_tokens":3469,"output_tokens":77,"error":null},{"id":"q008","dataset":"development","question":"Revenue by month last quarter","expected_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":18108.16571300029,"input_tokens":3465,"output_tokens":77,"error":null},{"id":"q009","dataset":"development","question":"Order count by month last quarter","expected_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":3767.0646130000023,"input_tokens":3466,"output_tokens":76,"error":null},{"id":"q010","dataset":"development","question":"Units sold by month last quarter","expected_intent":{"pattern_id":"metric_by_time","metric":"quantity","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_time","metric":"quantity","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_time","metric":"quantity","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1661.7304890000923,"input_tokens":3466,"output_tokens":76,"error":null},{"id":"q011","dataset":"development","question":"Monthly revenue for Q2 2026","expected_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1642.3449140002049,"input_tokens":3468,"output_tokens":77,"error":null},{"id":"q012","dataset":"development","question":"Monthly order volume for April through June 2026","expected_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1937.2160020002411,"input_tokens":3470,"output_tokens":76,"error":null},{"id":"q013","dataset":"development","question":"Revenue by region last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":3195.4903369996828,"input_tokens":3465,"output_tokens":79,"error":null},{"id":"q014","dataset":"development","question":"Revenue by customer last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2143.718171999808,"input_tokens":3465,"output_tokens":79,"error":null},{"id":"q015","dataset":"development","question":"Order count by region last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2177.298574999895,"input_tokens":3466,"output_tokens":78,"error":null},{"id":"q016","dataset":"development","question":"Units sold by product last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1919.2737009998382,"input_tokens":3466,"output_tokens":78,"error":null},{"id":"q017","dataset":"development","question":"Revenue by product category last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2402.388421000069,"input_tokens":3466,"output_tokens":79,"error":null},{"id":"q018","dataset":"development","question":"Order count by product category last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1888.828896999712,"input_tokens":3467,"output_tokens":78,"error":null},{"id":"q019","dataset":"development","question":"Revenue by customer in June 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1850.185915000111,"input_tokens":3468,"output_tokens":79,"error":null},{"id":"q020","dataset":"development","question":"Units sold by region in May 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2812.814896999953,"input_tokens":3469,"output_tokens":78,"error":null},{"id":"c001_breakdown","dataset":"holdout","question":"Gross sales across regions last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1909.4825610000044,"input_tokens":3466,"output_tokens":79,"error":null},{"id":"c001_ranking","dataset":"holdout","question":"Regions with the highest gross sales last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1986.3751739999316,"input_tokens":3468,"output_tokens":79,"error":null},{"id":"c002_breakdown","dataset":"holdout","question":"Revenue for each customer in Q2 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1688.3508910000273,"input_tokens":3470,"output_tokens":79,"error":null},{"id":"c002_ranking","dataset":"holdout","question":"Top 6 customers by revenue in Q2 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":6},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":6},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":6},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1526.630600000317,"input_tokens":3472,"output_tokens":79,"error":null},{"id":"c003_breakdown","dataset":"holdout","question":"Order volume broken down by product category last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2817.460769999798,"input_tokens":3469,"output_tokens":78,"error":null},{"id":"c003_ranking","dataset":"holdout","question":"Product categories with the most orders last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2256.3625569996475,"input_tokens":3468,"output_tokens":77,"error":null},{"id":"c004_breakdown","dataset":"holdout","question":"Unit sales by product in June 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1708.7162349998835,"input_tokens":3469,"output_tokens":78,"error":null},{"id":"c004_ranking","dataset":"holdout","question":"Top 4 products by unit sales in June 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":4},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":4},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":4},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1558.1721649996325,"input_tokens":3472,"output_tokens":77,"error":null},{"id":"c005_breakdown","dataset":"holdout","question":"Sales dollars by product category in May 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1627.7566830003707,"input_tokens":3470,"output_tokens":79,"error":null},{"id":"c005_ranking","dataset":"holdout","question":"Best product categories by sales dollars in May 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1718.693066999549,"input_tokens":3471,"output_tokens":79,"error":null},{"id":"c006_breakdown","dataset":"holdout","question":"Number of orders for each region in April 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1847.442362000038,"input_tokens":3471,"output_tokens":78,"error":null},{"id":"c006_ranking","dataset":"holdout","question":"Leading regions by number of orders in April 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1949.9657080004908,"input_tokens":3471,"output_tokens":77,"error":null},{"id":"c007_breakdown","dataset":"holdout","question":"Bookings value by customer last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1736.6026919999058,"input_tokens":3467,"output_tokens":79,"error":null},{"id":"c007_ranking","dataset":"holdout","question":"Largest customers by bookings value last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2414.9832040002366,"input_tokens":3467,"output_tokens":79,"error":null},{"id":"c008_breakdown","dataset":"holdout","question":"Units sold across regions last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2561.951149000379,"input_tokens":3466,"output_tokens":78,"error":null},{"id":"c008_ranking","dataset":"holdout","question":"Rank regions by units sold last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1797.0992650007247,"input_tokens":3467,"output_tokens":77,"error":null},{"id":"c009_breakdown","dataset":"holdout","question":"Revenue grouped by product last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1860.2668659996198,"input_tokens":3466,"output_tokens":79,"error":null},{"id":"c009_ranking","dataset":"holdout","question":"Products ranked by revenue last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1688.1360789993778,"input_tokens":3466,"output_tokens":79,"error":null},{"id":"c010_breakdown","dataset":"holdout","question":"Orders by customer last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1728.7660080000933,"input_tokens":3465,"output_tokens":78,"error":null},{"id":"c010_ranking","dataset":"holdout","question":"Customers with the fewest orders last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2060.5461449995346,"input_tokens":3468,"output_tokens":77,"error":null},{"id":"c011_breakdown","dataset":"holdout","question":"Product category revenue for April through June 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1990.6583080000928,"input_tokens":3470,"output_tokens":79,"error":null},{"id":"c011_ranking","dataset":"holdout","question":"Bottom 2 product categories by revenue for April through June 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":2},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":2},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":2},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":3269.889781000529,"input_tokens":3474,"output_tokens":79,"error":null},{"id":"c012_breakdown","dataset":"holdout","question":"Customer unit volume in May 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":3356.75517799973,"input_tokens":3468,"output_tokens":78,"error":null},{"id":"c012_ranking","dataset":"holdout","question":"Which customers had the least unit volume in May 2026?","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"quantity","direction":"asc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"quantity","direction":"asc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"quantity","direction":"asc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":3768.6033629997837,"input_tokens":3473,"output_tokens":77,"error":null},{"id":"c013_breakdown","dataset":"holdout","question":"Order count by product last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":5505.916833999436,"input_tokens":3466,"output_tokens":78,"error":null},{"id":"c013_ranking","dataset":"holdout","question":"Worst products by order count last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1968.59171899996,"input_tokens":3467,"output_tokens":77,"error":null},{"id":"c014_breakdown","dataset":"holdout","question":"Quantity by product category in June 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1667.8500010002608,"input_tokens":3469,"output_tokens":78,"error":null},{"id":"c014_ranking","dataset":"holdout","question":"Smallest product categories by quantity in June 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"asc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"asc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"asc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2982.947266999872,"input_tokens":3471,"output_tokens":77,"error":null},{"id":"c015_breakdown","dataset":"holdout","question":"Sales for every product last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2052.1167520000745,"input_tokens":3466,"output_tokens":79,"error":null},{"id":"c015_ranking","dataset":"holdout","question":"First 3 products by sales last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2071.495125000183,"input_tokens":3468,"output_tokens":79,"error":null},{"id":"c016_breakdown","dataset":"holdout","question":"Region order volume in Q2 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1604.5954729997902,"input_tokens":3469,"output_tokens":78,"error":null},{"id":"c016_ranking","dataset":"holdout","question":"Last 2 regions by order volume in Q2 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":2},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":2},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"asc"}],"limit":2},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1564.7064040003897,"input_tokens":3473,"output_tokens":77,"error":null},{"id":"c017_breakdown","dataset":"holdout","question":"Show revenue by customer for April 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1768.208991999927,"input_tokens":3469,"output_tokens":79,"error":null},{"id":"c017_ranking","dataset":"holdout","question":"Show the top 3 customers by revenue for April 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":3},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1740.9477699993658,"input_tokens":3473,"output_tokens":79,"error":null},{"id":"c018_breakdown","dataset":"holdout","question":"Product revenue table for last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1756.603222999729,"input_tokens":3466,"output_tokens":79,"error":null},{"id":"c018_ranking","dataset":"holdout","question":"Top product revenue table for last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1870.1324200001181,"input_tokens":3467,"output_tokens":79,"error":null},{"id":"c019_breakdown","dataset":"holdout","question":"Order count across customers last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1746.4909359996454,"input_tokens":3466,"output_tokens":78,"error":null},{"id":"c019_ranking","dataset":"holdout","question":"Rank customers by order count last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2142.251049999686,"input_tokens":3467,"output_tokens":77,"error":null},{"id":"c020_breakdown","dataset":"holdout","question":"Category unit sales for Q2 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1973.894042000211,"input_tokens":3469,"output_tokens":78,"error":null},{"id":"c020_ranking","dataset":"holdout","question":"Categories ranked by unit sales for Q2 2026","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2017.470993999268,"input_tokens":3471,"output_tokens":77,"error":null},{"id":"n001","dataset":"holdout","question":"Give me the sales dollars by customer for Q2","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1895.1223410003877,"input_tokens":3470,"output_tokens":79,"error":null},{"id":"n002","dataset":"holdout","question":"Which five regions generated the most revenue last quarter?","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":5},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":5},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":5},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1627.8046280003764,"input_tokens":3470,"output_tokens":79,"error":null},{"id":"n003","dataset":"holdout","question":"Monthly unit sales for Q2 2026","expected_intent":{"pattern_id":"metric_by_time","metric":"quantity","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_time","metric":"quantity","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_time","metric":"quantity","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1626.2115169993194,"input_tokens":3469,"output_tokens":76,"error":null},{"id":"n004","dataset":"holdout","question":"Show order volume by region, not ranked, for last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1668.298916999447,"input_tokens":3472,"output_tokens":78,"error":null},{"id":"n005","dataset":"holdout","question":"Bottom 3 customers by bookings value last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":3},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":3},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":3},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2082.2689649994572,"input_tokens":3469,"output_tokens":79,"error":null},{"id":"n006","dataset":"holdout","question":"Most recent order count by region last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"orders","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"orders","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["pattern_id","sort","limit"],"corrected_mismatch_fields":["pattern_id","sort","limit"],"intervened":false,"latency_ms":2290.9150349996708,"input_tokens":3468,"output_tokens":77,"error":null},{"id":"n007","dataset":"holdout","question":"High value customer revenue by customer last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1710.3120359997774,"input_tokens":3468,"output_tokens":79,"error":null},{"id":"n008","dataset":"holdout","question":"Revenue by product, sorted alphabetically, for June 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1822.3737820007955,"input_tokens":3473,"output_tokens":79,"error":null},{"id":"n009","dataset":"holdout","question":"Which products sold the most units last quarter?","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1727.7260979999483,"input_tokens":3469,"output_tokens":77,"error":null},{"id":"n010","dataset":"holdout","question":"Trend revenue over the last quarter by month","expected_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_time","metric":"revenue","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1671.9319459998587,"input_tokens":3468,"output_tokens":77,"error":null},{"id":"n011","dataset":"holdout","question":"Quantity by customer in April 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-05-01"}],"sort":[{"field":"customer","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1522.2688370004107,"input_tokens":3468,"output_tokens":78,"error":null},{"id":"n012","dataset":"holdout","question":"Top revenue products for April through June","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1965.3983880007218,"input_tokens":3467,"output_tokens":79,"error":null},{"id":"n013","dataset":"holdout","question":"Category order volume, all categories, Q2 2026","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":3132.4001840002893,"input_tokens":3472,"output_tokens":78,"error":null},{"id":"n014","dataset":"holdout","question":"Least revenue customers last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["customer"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"asc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1880.837139000505,"input_tokens":3465,"output_tokens":79,"error":null},{"id":"n015","dataset":"holdout","question":"For each product category, how many units were sold in May?","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-05-01","resolved_end_exclusive":"2026-06-01"}],"sort":[{"field":"category","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1601.2012790006338,"input_tokens":3473,"output_tokens":78,"error":null},{"id":"n016","dataset":"holdout","question":"Rank product categories by revenue in June","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-06-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2581.35510899956,"input_tokens":3467,"output_tokens":79,"error":null},{"id":"n017","dataset":"holdout","question":"Region sales report for last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":3219.0827429994897,"input_tokens":3466,"output_tokens":79,"error":null},{"id":"n018","dataset":"holdout","question":"Largest product categories by unit sales last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"quantity","dimensions":["category"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"quantity","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1664.9669599992194,"input_tokens":3468,"output_tokens":77,"error":null},{"id":"n019","dataset":"holdout","question":"Order volume by product for the most recent quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"orders","dimensions":["product"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"product","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1814.9031229995671,"input_tokens":3469,"output_tokens":78,"error":null},{"id":"n020","dataset":"holdout","question":"Monthly order count in Q2","expected_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_time","metric":"orders","dimensions":["month"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"month","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2801.440351000565,"input_tokens":3466,"output_tokens":76,"error":null},{"id":"adv001_topline_breakdown","dataset":"adversarial","question":"Top-line revenue by region last quarter","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["pattern_id","sort","limit"],"corrected_mismatch_fields":["pattern_id","sort","limit"],"intervened":false,"latency_ms":1836.4603370000623,"input_tokens":3467,"output_tokens":79,"error":null},{"id":"adv002_leaders_ranking","dataset":"adversarial","question":"Revenue leaders by region last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":2212.4906070002908,"input_tokens":3466,"output_tokens":79,"error":null},{"id":"adv003_compare_breakdown","dataset":"adversarial","question":"How do regions compare on revenue last quarter?","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1782.2884759998487,"input_tokens":3469,"output_tokens":79,"error":null},{"id":"adv004_strongest_ranking","dataset":"adversarial","question":"Show the strongest markets by revenue last quarter","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[{"field":"order_date","operator":"range","resolved_start":"2026-04-01","resolved_end_exclusive":"2026-07-01"}],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1727.9675359995963,"input_tokens":3468,"output_tokens":79,"error":null},{"id":"adv005_first_rows_breakdown","dataset":"adversarial","question":"Show revenue by region, but only the first ten rows","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[],"sort":[{"field":"region","direction":"asc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["pattern_id","sort"],"corrected_mismatch_fields":["pattern_id","sort"],"intervened":false,"latency_ms":1449.5804070002123,"input_tokens":3471,"output_tokens":47,"error":null},{"id":"adv006_top_regions_ranking","dataset":"adversarial","question":"Show the top ten regions by revenue","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1874.795370999891,"input_tokens":3467,"output_tokens":47,"error":null},{"id":"adv007_ordered_breakdown","dataset":"adversarial","question":"Break down revenue by region, ordered by revenue descending","expected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[],"sort":[{"field":"revenue","direction":"desc"}],"limit":100},"raw_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[],"sort":[{"field":"region","direction":"asc"}],"limit":100},"corrected_intent":{"pattern_id":"metric_by_dimension_with_filter","metric":"revenue","dimensions":["region"],"filters":[],"sort":[{"field":"region","direction":"asc"}],"limit":100},"schema_compliant":true,"raw_match":false,"corrected_match":false,"raw_mismatch_fields":["sort"],"corrected_mismatch_fields":["sort"],"intervened":false,"latency_ms":1225.3341340001498,"input_tokens":3470,"output_tokens":47,"error":null},{"id":"adv008_greatest_ranking","dataset":"adversarial","question":"Which regions contribute the greatest revenue?","expected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"raw_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"corrected_intent":{"pattern_id":"top_n_by_metric","metric":"revenue","dimensions":["region"],"filters":[],"sort":[{"field":"revenue","direction":"desc"}],"limit":10},"schema_compliant":true,"raw_match":true,"corrected_match":true,"raw_mismatch_fields":[],"corrected_mismatch_fields":[],"intervened":false,"latency_ms":1333.6465160000444,"input_tokens":3467,"output_tokens":47,"error":null}]}]}