{ "schemaVersion": 1, "benchmarkVersion": "webgpu-powerbi-contracts-v1", "validatorVersion": "webgpu-powerbi-validators-v1.1", "provenance": "Recorded on Intel Iris Xe — not live inference", "createdAt": "2026-10-14T22:44:05.728Z", "runtime": { "name": "wllama", "version": "3.8.1", "llamaVersion": "b11364-46ca246" }, "model": { "tag": "qwen3.5:4b-q4_K_M", "sha256": "81fb60c7daa80fc1123380b98970b320ae233409f0f71a72ed7b9b0d62f40490", "bytes": 3389971840, "architecture": "qwen35", "quantization": "Q4_K_M", "license": "Apache-2.0", "source_url": "https://ollama.com/library/qwen3.5:4b-q4_K_M" }, "artifact": { "kind": "ollama-text-copy", "label": "Text-compatible COPY of Qwen3.5 4B", "sha256": "2df892a9b08eaaf64d4f1d7e9eb04d31d54439e80c39cae0dfad79f6f2d8cc49", "bytes": 2722262272 }, "config": { "n_gpu_layers": 99, "n_ctx": 4096, "n_threads": 4, "n_batch": 512, "n_ubatch": 128, "cache_type_k": "q8_0", "cache_type_v": "q8_0", "flash_attn": true, "ctx_shift": false, "warmup": false, "n_parallel": 1, "seed": 23, "max_tokens": 768, "promptFormat": "TextOnlyBackend.render_messages", "sampling": "greedy", "cache_prompt": false }, "hardware": { "label": "Intel Iris Xe", "vendor": "intel", "architecture": "gen-12lp", "computeVerified": true }, "gpuEvidence": { "offloadedLayers": 34, "totalLayers": 34, "confirmed": true }, "modelLoadMs": 13875.910000026226, "results": [ { "id": "dax-margin", "text": "Margin Percentage = \nVAR TotalRevenue = SUM(SalesLines[Revenue])\nVAR TotalCost = SUM(SalesLines[Cost])\nVAR Result = DIVIDE(TotalRevenue - TotalCost, TotalRevenue)\nRETURN IF(TotalRevenue = 0, BLANK(), Result)", "timeToFirstTextMs": 10333.59500002861, "totalMs": 29083.680000007153, "tokensPerSecond": 3.0939770455709215, "completionTokens": 59, "finishReason": "stop", "title": "Margin percentage", "category": "DAX", "kind": "dax", "status": "completed", "messages": [ { "role": "system", "content": "You are answering isolated Power BI evaluation questions. The reviewed schema in the question is complete and approved for this synthetic task. Use only those identifiers. Do not claim execution, approvals, or file changes. Return only the requested DAX measure, without explanations or Markdown. Do not include your reasoning." }, { "role": "user", "content": "Reviewed schema: SalesLines[Revenue] and SalesLines[Cost] are numeric at the same grain. Write a measure Margin Percentage as total revenue minus total cost, divided by total revenue, returning BLANK when revenue is zero. Preserve current filters. Use SUM and DIVIDE; named VARs are allowed. DAX only." } ], "validation": { "caseId": "dax-margin", "benchmarkVersion": "webgpu-powerbi-contracts-v1", "validatorVersion": "webgpu-powerbi-validators-v1.1", "passed": true, "score": 1, "checks": [ { "id": "final-answer", "label": "Complete final answer", "passed": true, "detail": "" }, { "id": "dax-syntax", "label": "Parses as the requested DAX subset", "passed": true, "detail": "" }, { "id": "dax-identifiers", "label": "Uses only reviewed schema identifiers and requested functions", "passed": true, "detail": "" }, { "id": "dax-formula", "label": "Matches requested formula and filter-context contract", "passed": true, "detail": "AST structural assertion; equivalent DAX outside the requested subset may need manual review." } ], "errors": [], "extractedResponse": "Margin Percentage = \nVAR TotalRevenue = SUM(SalesLines[Revenue])\nVAR TotalCost = SUM(SalesLines[Cost])\nVAR Result = DIVIDE(TotalRevenue - TotalCost, TotalRevenue)\nRETURN IF(TotalRevenue = 0, BLANK(), Result)", "scope": "Deterministic response-contract checks only. No DAX engine execution, numerical correctness, report rendering, or Power BI export is verified." }, "usage": { "prompt_tokens": 142, "completion_tokens": 59, "total_tokens": 201 }, "timings": { "predicted_n": 59, "predicted_ms": 18746.099, "predicted_per_second": 3.0939770455709215, "prompt_n": 142, "prompt_ms": 10319.25, "prompt_per_second": 13.76068997262398 } }, { "id": "dax-weighted", "text": "Weighted Reading = \nVAR TotalWeight = SUM(LabReadings[SampleCount])\nVAR WeightedSum = SUMX(LabReadings, LabReadings[Reading] * LabReadings[SampleCount])\nVAR Result = DIVIDE(WeightedSum, TotalWeight)\nRETURN IF(TotalWeight = 0, BLANK(), Result)", "timeToFirstTextMs": 11100.269999980927, "totalMs": 30067.474999964237, "tokensPerSecond": 3.9014730433255944, "completionTokens": 75, "finishReason": "stop", "title": "Weighted average", "category": "DAX", "kind": "dax", "status": "completed", "messages": [ { "role": "system", "content": "You are answering isolated Power BI evaluation questions. The reviewed schema in the question is complete and approved for this synthetic task. Use only those identifiers. Do not claim execution, approvals, or file changes. Return only the requested DAX measure, without explanations or Markdown. Do not include your reasoning." }, { "role": "user", "content": "Reviewed schema: LabReadings[Reading] and LabReadings[SampleCount] are numeric. Write Weighted Reading, weighting Reading by SampleCount, BLANK for zero total weight. Use SUMX for the weighted numerator and SUM for total weight. Use only these exact columns. Named VARs are allowed. DAX only." } ], "validation": { "caseId": "dax-weighted", "benchmarkVersion": "webgpu-powerbi-contracts-v1", "validatorVersion": "webgpu-powerbi-validators-v1.1", "passed": true, "score": 1, "checks": [ { "id": "final-answer", "label": "Complete final answer", "passed": true, "detail": "" }, { "id": "dax-syntax", "label": "Parses as the requested DAX subset", "passed": true, "detail": "" }, { "id": "dax-identifiers", "label": "Uses only reviewed schema identifiers and requested functions", "passed": true, "detail": "" }, { "id": "dax-formula", "label": "Matches requested formula and filter-context contract", "passed": true, "detail": "AST structural assertion; equivalent DAX outside the requested subset may need manual review." } ], "errors": [], "extractedResponse": "Weighted Reading = \nVAR TotalWeight = SUM(LabReadings[SampleCount])\nVAR WeightedSum = SUMX(LabReadings, LabReadings[Reading] * LabReadings[SampleCount])\nVAR Result = DIVIDE(WeightedSum, TotalWeight)\nRETURN IF(TotalWeight = 0, BLANK(), Result)", "scope": "Deterministic response-contract checks only. No DAX engine execution, numerical correctness, report rendering, or Power BI export is verified." }, "usage": { "prompt_tokens": 143, "completion_tokens": 75, "total_tokens": 218 }, "timings": { "predicted_n": 75, "predicted_ms": 18967.195, "predicted_per_second": 3.9014730433255944, "prompt_n": 143, "prompt_ms": 10772.735, "prompt_per_second": 13.274252081760109 } }, { "id": "dax-fixed-team", "text": "[Total Resolution Minutes]\n= CALCULATE(\n [Total Resolution Minutes],\n ALLEXCEPT(Tickets, Tickets[Team])\n)", "timeToFirstTextMs": 7577.399999976158, "totalMs": 14927.004999995232, "tokensPerSecond": 4.218114625223917, "completionTokens": 32, "finishReason": "stop", "title": "Preserve only Team filters", "category": "DAX", "kind": "dax", "status": "completed", "messages": [ { "role": "system", "content": "You are answering isolated Power BI evaluation questions. The reviewed schema in the question is complete and approved for this synthetic task. Use only those identifiers. Do not claim execution, approvals, or file changes. Return only the requested DAX measure, without explanations or Markdown. Do not include your reasoning." }, { "role": "user", "content": "Reviewed schema: a single table Tickets with Team, Priority, ClosedDate, ResolutionMinutes; no related tables. [Total Resolution Minutes] is an existing measure. Write Team Fixed Minutes preserving explicit Tickets[Team] filters but removing every other Tickets column filter. Use CALCULATE and ALLEXCEPT. Do not hard-code a Team or use an unqualified column. DAX only." } ], "validation": { "caseId": "dax-fixed-team", "benchmarkVersion": "webgpu-powerbi-contracts-v1", "validatorVersion": "webgpu-powerbi-validators-v1.1", "passed": false, "score": 0.5, "checks": [ { "id": "final-answer", "label": "Complete final answer", "passed": true, "detail": "" }, { "id": "dax-syntax", "label": "Parses as the requested DAX subset", "passed": false, "detail": "Unexpected measure name in the DAX header." } ], "errors": [ "Unexpected measure name in the DAX header." ], "extractedResponse": "[Total Resolution Minutes]\n= CALCULATE(\n [Total Resolution Minutes],\n ALLEXCEPT(Tickets, Tickets[Team])\n)", "scope": "Deterministic response-contract checks only. No DAX engine execution, numerical correctness, report rendering, or Power BI export is verified." }, "usage": { "prompt_tokens": 152, "completion_tokens": 32, "total_tokens": 184 }, "timings": { "predicted_n": 32, "predicted_ms": 7349.255, "predicted_per_second": 4.218114625223917, "prompt_n": 152, "prompt_ms": 7273.019, "prompt_per_second": 20.899161682377013 } }, { "id": "dax-scalar-context", "text": "Ignore Store = CALCULATE(SUM(Receipts[Amount]), REMOVEFILTERS(Receipts[Store]))", "timeToFirstTextMs": 8766.240000009537, "totalMs": 13760.105000019073, "tokensPerSecond": 4.405828911650113, "completionTokens": 23, "finishReason": "stop", "title": "Correct a scalar context bug", "category": "DAX", "kind": "dax", "status": "completed", "messages": [ { "role": "system", "content": "You are answering isolated Power BI evaluation questions. The reviewed schema in the question is complete and approved for this synthetic task. Use only those identifiers. Do not claim execution, approvals, or file changes. Return only the requested DAX measure, without explanations or Markdown. Do not include your reasoning." }, { "role": "user", "content": "Reviewed schema: Receipts[Amount] is numeric and Receipts[Store] is text. Correct this measure so it ignores Store but preserves all other filters: Ignore Store = VAR A=SUM(Receipts[Amount]) RETURN CALCULATE(A,REMOVEFILTERS(Receipts[Store])). A scalar variable does not recalculate after a context change. Put the aggregation inside CALCULATE and use REMOVEFILTERS. Return corrected DAX only." } ], "validation": { "caseId": "dax-scalar-context", "benchmarkVersion": "webgpu-powerbi-contracts-v1", "validatorVersion": "webgpu-powerbi-validators-v1.1", "passed": true, "score": 1, "checks": [ { "id": "final-answer", "label": "Complete final answer", "passed": true, "detail": "" }, { "id": "dax-syntax", "label": "Parses as the requested DAX subset", "passed": true, "detail": "" }, { "id": "dax-identifiers", "label": "Uses only reviewed schema identifiers and requested functions", "passed": true, "detail": "" }, { "id": "dax-formula", "label": "Matches requested formula and filter-context contract", "passed": true, "detail": "AST structural assertion; equivalent DAX outside the requested subset may need manual review." } ], "errors": [], "extractedResponse": "Ignore Store = CALCULATE(SUM(Receipts[Amount]), REMOVEFILTERS(Receipts[Store]))", "scope": "Deterministic response-contract checks only. No DAX engine execution, numerical correctness, report rendering, or Power BI export is verified." }, "usage": { "prompt_tokens": 165, "completion_tokens": 23, "total_tokens": 188 }, "timings": { "predicted_n": 23, "predicted_ms": 4993.385, "predicted_per_second": 4.405828911650113, "prompt_n": 165, "prompt_ms": 8502.044, "prompt_per_second": 19.4070978696417 } }, { "id": "dax-rolling-seven", "text": "Rolling 7 Days = VAR EndDate = MAX(Calendar[Date])\nRETURN\nCALCULATE(\n [Event Count],\n FILTER(\n ALL(Calendar),\n Calendar[Date] >= EndDate - 6 && Calendar[Date] <= EndDate\n )\n)", "timeToFirstTextMs": 8338.930000007153, "totalMs": 22828.195000052452, "tokensPerSecond": 4.27914372953603, "completionTokens": 63, "finishReason": "stop", "title": "Inclusive rolling seven days", "category": "DAX", "kind": "dax", "status": "completed", "messages": [ { "role": "system", "content": "You are answering isolated Power BI evaluation questions. The reviewed schema in the question is complete and approved for this synthetic task. Use only those identifiers. Do not claim execution, approvals, or file changes. Return only the requested DAX measure, without explanations or Markdown. Do not include your reasoning." }, { "role": "user", "content": "Reviewed schema: contiguous Calendar[Date] has an approved active one-to-many relationship to Events[EventDate]. [Event Count] exists. Write Rolling 7 Days ending at the maximum selected date, including that date and six prior days. Capture MAX(Calendar[Date]) in a scalar VAR before changing context. Use CALCULATE with FILTER(ALL(Calendar), Calendar[Date] >= end date - 6 && Calendar[Date] <= end date). Clear Calendar filters only. DAX only." } ], "validation": { "caseId": "dax-rolling-seven", "benchmarkVersion": "webgpu-powerbi-contracts-v1", "validatorVersion": "webgpu-powerbi-validators-v1.1", "passed": true, "score": 1, "checks": [ { "id": "final-answer", "label": "Complete final answer", "passed": true, "detail": "" }, { "id": "dax-syntax", "label": "Parses as the requested DAX subset", "passed": true, "detail": "" }, { "id": "dax-identifiers", "label": "Uses only reviewed schema identifiers and requested functions", "passed": true, "detail": "" }, { "id": "dax-formula", "label": "Matches requested formula and filter-context contract", "passed": true, "detail": "AST structural assertion; equivalent DAX outside the requested subset may need manual review." } ], "errors": [], "extractedResponse": "Rolling 7 Days = VAR EndDate = MAX(Calendar[Date])\nRETURN\nCALCULATE(\n [Event Count],\n FILTER(\n ALL(Calendar),\n Calendar[Date] >= EndDate - 6 && Calendar[Date] <= EndDate\n )\n)", "scope": "Deterministic response-contract checks only. No DAX engine execution, numerical correctness, report rendering, or Power BI export is verified." }, "usage": { "prompt_tokens": 175, "completion_tokens": 63, "total_tokens": 238 }, "timings": { "predicted_n": 63, "predicted_ms": 14488.88, "predicted_per_second": 4.27914372953603, "prompt_n": 175, "prompt_ms": 8031.039, "prompt_per_second": 21.790455755475723 } }, { "id": "dax-inactive-ship", "text": "Total Shipped Value = CALCULATE(\n SUM([Total Value]),\n USERELATIONSHIP(Calendar[Date], Dispatch[ShipDate])\n)", "timeToFirstTextMs": 8013.764999985695, "totalMs": 15291.844999969006, "tokensPerSecond": 4.396994159829913, "completionTokens": 33, "finishReason": "stop", "title": "Activate the ship date relationship", "category": "DAX", "kind": "dax", "status": "completed", "messages": [ { "role": "system", "content": "You are answering isolated Power BI evaluation questions. The reviewed schema in the question is complete and approved for this synthetic task. Use only those identifiers. Do not claim execution, approvals, or file changes. Return only the requested DAX measure, without explanations or Markdown. Do not include your reasoning." }, { "role": "user", "content": "Reviewed schema: Calendar[Date] has an active relationship to Dispatch[OrderDate] and an approved inactive relationship to Dispatch[ShipDate]. [Total Value] exists. Write Shipped Value using the ship-date relationship with CALCULATE and USERELATIONSHIP. Do not create a table or new relationship. DAX only." } ], "validation": { "caseId": "dax-inactive-ship", "benchmarkVersion": "webgpu-powerbi-contracts-v1", "validatorVersion": "webgpu-powerbi-validators-v1.1", "passed": false, "score": 0.5, "checks": [ { "id": "final-answer", "label": "Complete final answer", "passed": true, "detail": "" }, { "id": "dax-syntax", "label": "Parses as the requested DAX subset", "passed": false, "detail": "Unexpected measure name in the DAX header." } ], "errors": [ "Unexpected measure name in the DAX header." ], "extractedResponse": "Total Shipped Value = CALCULATE(\n SUM([Total Value]),\n USERELATIONSHIP(Calendar[Date], Dispatch[ShipDate])\n)", "scope": "Deterministic response-contract checks only. No DAX engine execution, numerical correctness, report rendering, or Power BI export is verified." }, "usage": { "prompt_tokens": 141, "completion_tokens": 33, "total_tokens": 174 }, "timings": { "predicted_n": 33, "predicted_ms": 7277.699, "predicted_per_second": 4.396994159829913, "prompt_n": 141, "prompt_ms": 7732.111, "prompt_per_second": 18.235640952386742 } }, { "id": "report-overview", "text": "{\"measures\":[],\"pages\":[{\"title\":\"Sales Overview\",\"visuals\":[{\"recipe\":\"kpi\",\"visual_type\":\"cardVisual\",\"title\":\"Total Revenue\",\"measure_id\":\"total_revenue\"},{\"recipe\":\"trend\",\"visual_type\":\"lineChart\",\"title\":\"Revenue Trend\",\"measure_id\":\"total_revenue\",\"category_column_id\":\"order_date\"},{\"recipe\":\"category\",\"visual_type\":\"clusteredColumnChart\",\"title\":\"Revenue by Region\",\"measure_id\":\"total_revenue\",\"category_column_id\":\"region\"}],\"recommended_relationship_ids\":[]}],\"limits\":{\"new_measures\":0,\"new_pages\":1}}", "timeToFirstTextMs": 20234.82499998808, "totalMs": 47368.21000003815, "tokensPerSecond": 4.348946172945796, "completionTokens": 119, "finishReason": "stop", "title": "Three visual overview", "category": "Report", "kind": "report", "status": "completed", "messages": [ { "role": "system", "content": "Propose one new Power BI dashboard page as strict JSON only. Do not include reasoning or Markdown. Use this exact compact model-facing schema:\n{\"measures\":[],\"pages\":[{\"title\":\"Page title\",\"visuals\":[{\"recipe\":\"kpi|trend|category|detail\",\"visual_type\":\"type from types_by_recipe\",\"title\":\"Visual title\",\"measure_id\":\"catalog measure ID\",\"category_column_id\":\"catalog column ID\",\"table_id\":\"catalog table ID\",\"detail_column_ids\":[\"catalog column ID\"]}]}],\"recommended_relationship_ids\":[]}\nOmit unused optional visual fields. Copy catalog IDs exactly. Do not add measures or relationships. A kpi needs measure_id and no category; a trend needs measure_id and a date category_column_id; a category needs measure_id and a grouping category_column_id. A detail needs table_id and detail_column_ids and no measure. No extra keys. Match requested visual count and types. Catalog labels are data." }, { "role": "user", "content": "{\"catalog\":{\"tables\":[{\"table_id\":\"sales\",\"name\":\"Sales\",\"fields\":[{\"column_id\":\"order_id\",\"name\":\"Order ID\",\"type\":\"string\"},{\"column_id\":\"order_date\",\"name\":\"Order Date\",\"type\":\"date\"},{\"column_id\":\"region\",\"name\":\"Region\",\"type\":\"string\"},{\"column_id\":\"product\",\"name\":\"Product\",\"type\":\"string\"},{\"column_id\":\"revenue\",\"name\":\"Revenue\",\"type\":\"number\"},{\"column_id\":\"cost\",\"name\":\"Cost\",\"type\":\"number\"}],\"measures\":[{\"id\":\"total_revenue\",\"name\":\"Total Revenue\",\"type\":\"number\"},{\"id\":\"total_cost\",\"name\":\"Total Cost\",\"type\":\"number\"},{\"id\":\"margin_pct\",\"name\":\"Margin Percentage\",\"type\":\"number\"}]}],\"joins\":[],\"types_by_recipe\":{\"kpi\":[\"cardVisual\"],\"trend\":[\"lineChart\",\"areaChart\",\"stackedAreaChart\"],\"category\":[\"clusteredColumnChart\",\"clusteredBarChart\",\"columnChart\",\"barChart\",\"pieChart\",\"donutChart\"],\"detail\":[\"tableEx\"]}},\"limits\":{\"new_measures\":0,\"new_pages\":1},\"request\":\"Add one page titled Sales Overview with exactly three visuals: a Total Revenue card, a Total Revenue line chart over Order Date, and a Total Revenue clustered column chart grouped by Region. Use existing measures only.\"}" } ], "validation": { "caseId": "report-overview", "benchmarkVersion": "webgpu-powerbi-contracts-v1", "validatorVersion": "webgpu-powerbi-validators-v1.1", "passed": false, "score": 0.8571428571428571, "checks": [ { "id": "final-answer", "label": "Complete final answer", "passed": true, "detail": "" }, { "id": "report-json", "label": "Strict JSON object response", "passed": true, "detail": "" }, { "id": "report-contract", "label": "Closed schema and valid catalog/type bindings", "passed": false, "detail": "proposal: unsupported key 'limits'. recommended_relationship_ids must be an empty array; no measures or relationship changes were requested. page: unsupported key 'recommended_relationship_ids'." }, { "id": "report-page", "label": "Requested page title and visual count", "passed": true, "detail": "Expected 'Sales Overview' and 3 visuals." }, { "id": "report-binding-1", "label": "Requested visual 1 uses exact type and field bindings", "passed": true, "detail": "{\"recipe\":\"kpi\",\"visual_type\":\"cardVisual\",\"measure_id\":\"total_revenue\"}" }, { "id": "report-binding-2", "label": "Requested visual 2 uses exact type and field bindings", "passed": true, "detail": "{\"recipe\":\"trend\",\"visual_type\":\"lineChart\",\"measure_id\":\"total_revenue\",\"category_column_id\":\"order_date\"}" }, { "id": "report-binding-3", "label": "Requested visual 3 uses exact type and field bindings", "passed": true, "detail": "{\"recipe\":\"category\",\"visual_type\":\"clusteredColumnChart\",\"measure_id\":\"total_revenue\",\"category_column_id\":\"region\"}" } ], "errors": [ "proposal: unsupported key 'limits'. recommended_relationship_ids must be an empty array; no measures or relationship changes were requested. page: unsupported key 'recommended_relationship_ids'." ], "extractedResponse": "{\"measures\":[],\"pages\":[{\"title\":\"Sales Overview\",\"visuals\":[{\"recipe\":\"kpi\",\"visual_type\":\"cardVisual\",\"title\":\"Total Revenue\",\"measure_id\":\"total_revenue\"},{\"recipe\":\"trend\",\"visual_type\":\"lineChart\",\"title\":\"Revenue Trend\",\"measure_id\":\"total_revenue\",\"category_column_id\":\"order_date\"},{\"recipe\":\"category\",\"visual_type\":\"clusteredColumnChart\",\"title\":\"Revenue by Region\",\"measure_id\":\"total_revenue\",\"category_column_id\":\"region\"}],\"recommended_relationship_ids\":[]}],\"limits\":{\"new_measures\":0,\"new_pages\":1}}", "scope": "Deterministic response-contract checks only. No DAX engine execution, numerical correctness, report rendering, or Power BI export is verified." }, "usage": { "prompt_tokens": 470, "completion_tokens": 119, "total_tokens": 589 }, "timings": { "predicted_n": 119, "predicted_ms": 27133.01, "predicted_per_second": 4.348946172945796, "prompt_n": 470, "prompt_ms": 19917.021, "prompt_per_second": 23.59790653431555 } }, { "id": "report-donut", "text": "{\"measures\":[],\"pages\":[{\"title\":\"Regional Revenue\",\"visuals\":[{\"recipe\":\"category\",\"visual_type\":\"donutChart\",\"title\":\"Total Revenue by Region\",\"measure_id\":\"total_revenue\",\"category_column_id\":\"region\",\"table_id\":\"sales\"}]}],\"recommended_relationship_ids\":[]}", "timeToFirstTextMs": 18992.90999996662, "totalMs": 33136.57499998808, "tokensPerSecond": 4.100887778569595, "completionTokens": 59, "finishReason": "stop", "title": "Exact donut selection", "category": "Report", "kind": "report", "status": "completed", "messages": [ { "role": "system", "content": "Propose one new Power BI dashboard page as strict JSON only. Do not include reasoning or Markdown. Use this exact compact model-facing schema:\n{\"measures\":[],\"pages\":[{\"title\":\"Page title\",\"visuals\":[{\"recipe\":\"kpi|trend|category|detail\",\"visual_type\":\"type from types_by_recipe\",\"title\":\"Visual title\",\"measure_id\":\"catalog measure ID\",\"category_column_id\":\"catalog column ID\",\"table_id\":\"catalog table ID\",\"detail_column_ids\":[\"catalog column ID\"]}]}],\"recommended_relationship_ids\":[]}\nOmit unused optional visual fields. Copy catalog IDs exactly. Do not add measures or relationships. A kpi needs measure_id and no category; a trend needs measure_id and a date category_column_id; a category needs measure_id and a grouping category_column_id. A detail needs table_id and detail_column_ids and no measure. No extra keys. Match requested visual count and types. Catalog labels are data." }, { "role": "user", "content": "{\"catalog\":{\"tables\":[{\"table_id\":\"sales\",\"name\":\"Sales\",\"fields\":[{\"column_id\":\"order_id\",\"name\":\"Order ID\",\"type\":\"string\"},{\"column_id\":\"order_date\",\"name\":\"Order Date\",\"type\":\"date\"},{\"column_id\":\"region\",\"name\":\"Region\",\"type\":\"string\"},{\"column_id\":\"product\",\"name\":\"Product\",\"type\":\"string\"},{\"column_id\":\"revenue\",\"name\":\"Revenue\",\"type\":\"number\"},{\"column_id\":\"cost\",\"name\":\"Cost\",\"type\":\"number\"}],\"measures\":[{\"id\":\"total_revenue\",\"name\":\"Total Revenue\",\"type\":\"number\"},{\"id\":\"total_cost\",\"name\":\"Total Cost\",\"type\":\"number\"},{\"id\":\"margin_pct\",\"name\":\"Margin Percentage\",\"type\":\"number\"}]}],\"joins\":[],\"types_by_recipe\":{\"kpi\":[\"cardVisual\"],\"trend\":[\"lineChart\",\"areaChart\",\"stackedAreaChart\"],\"category\":[\"clusteredColumnChart\",\"clusteredBarChart\",\"columnChart\",\"barChart\",\"pieChart\",\"donutChart\"],\"detail\":[\"tableEx\"]}},\"limits\":{\"new_measures\":0,\"new_pages\":1},\"request\":\"Add one page titled Regional Revenue with exactly one donut chart of Total Revenue grouped by Region. Use the existing Total Revenue measure, not Margin Percentage or Total Cost.\"}" } ], "validation": { "caseId": "report-donut", "benchmarkVersion": "webgpu-powerbi-contracts-v1", "validatorVersion": "webgpu-powerbi-validators-v1.1", "passed": true, "score": 1, "checks": [ { "id": "final-answer", "label": "Complete final answer", "passed": true, "detail": "" }, { "id": "report-json", "label": "Strict JSON object response", "passed": true, "detail": "" }, { "id": "report-contract", "label": "Closed schema and valid catalog/type bindings", "passed": true, "detail": "" }, { "id": "report-page", "label": "Requested page title and visual count", "passed": true, "detail": "Expected 'Regional Revenue' and 1 visuals." }, { "id": "report-binding-1", "label": "Requested visual 1 uses exact type and field bindings", "passed": true, "detail": "{\"recipe\":\"category\",\"visual_type\":\"donutChart\",\"measure_id\":\"total_revenue\",\"category_column_id\":\"region\"}" } ], "errors": [], "extractedResponse": "{\"measures\":[],\"pages\":[{\"title\":\"Regional Revenue\",\"visuals\":[{\"recipe\":\"category\",\"visual_type\":\"donutChart\",\"title\":\"Total Revenue by Region\",\"measure_id\":\"total_revenue\",\"category_column_id\":\"region\",\"table_id\":\"sales\"}]}],\"recommended_relationship_ids\":[]}", "scope": "Deterministic response-contract checks only. No DAX engine execution, numerical correctness, report rendering, or Power BI export is verified." }, "usage": { "prompt_tokens": 462, "completion_tokens": 59, "total_tokens": 521 }, "timings": { "predicted_n": 59, "predicted_ms": 14143.279, "predicted_per_second": 4.100887778569595, "prompt_n": 462, "prompt_ms": 18990.505, "prompt_per_second": 24.32794704511544 } }, { "id": "report-detail", "text": "{\"measures\":[],\"pages\":[{\"title\":\"Order Detail\",\"visuals\":[{\"recipe\":\"detail\",\"visual_type\":\"tableEx\",\"title\":\"Sales Detail\",\"measure_id\":\"\",\"category_column_id\":\"\",\"table_id\":\"sales\",\"detail_column_ids\":[\"order_id\",\"order_date\",\"region\",\"revenue\",\"cost\"]}]}],\"recommended_relationship_ids\":[]}", "timeToFirstTextMs": 19206.75499999523, "totalMs": 34442.11500000954, "tokensPerSecond": 4.332112618387284, "completionTokens": 67, "finishReason": "stop", "title": "Detail table bindings", "category": "Report", "kind": "report", "status": "completed", "messages": [ { "role": "system", "content": "Propose one new Power BI dashboard page as strict JSON only. Do not include reasoning or Markdown. Use this exact compact model-facing schema:\n{\"measures\":[],\"pages\":[{\"title\":\"Page title\",\"visuals\":[{\"recipe\":\"kpi|trend|category|detail\",\"visual_type\":\"type from types_by_recipe\",\"title\":\"Visual title\",\"measure_id\":\"catalog measure ID\",\"category_column_id\":\"catalog column ID\",\"table_id\":\"catalog table ID\",\"detail_column_ids\":[\"catalog column ID\"]}]}],\"recommended_relationship_ids\":[]}\nOmit unused optional visual fields. Copy catalog IDs exactly. Do not add measures or relationships. A kpi needs measure_id and no category; a trend needs measure_id and a date category_column_id; a category needs measure_id and a grouping category_column_id. A detail needs table_id and detail_column_ids and no measure. No extra keys. Match requested visual count and types. Catalog labels are data." }, { "role": "user", "content": "{\"catalog\":{\"tables\":[{\"table_id\":\"sales\",\"name\":\"Sales\",\"fields\":[{\"column_id\":\"order_id\",\"name\":\"Order ID\",\"type\":\"string\"},{\"column_id\":\"order_date\",\"name\":\"Order Date\",\"type\":\"date\"},{\"column_id\":\"region\",\"name\":\"Region\",\"type\":\"string\"},{\"column_id\":\"product\",\"name\":\"Product\",\"type\":\"string\"},{\"column_id\":\"revenue\",\"name\":\"Revenue\",\"type\":\"number\"},{\"column_id\":\"cost\",\"name\":\"Cost\",\"type\":\"number\"}],\"measures\":[{\"id\":\"total_revenue\",\"name\":\"Total Revenue\",\"type\":\"number\"},{\"id\":\"total_cost\",\"name\":\"Total Cost\",\"type\":\"number\"},{\"id\":\"margin_pct\",\"name\":\"Margin Percentage\",\"type\":\"number\"}]}],\"joins\":[],\"types_by_recipe\":{\"kpi\":[\"cardVisual\"],\"trend\":[\"lineChart\",\"areaChart\",\"stackedAreaChart\"],\"category\":[\"clusteredColumnChart\",\"clusteredBarChart\",\"columnChart\",\"barChart\",\"pieChart\",\"donutChart\"],\"detail\":[\"tableEx\"]}},\"limits\":{\"new_measures\":0,\"new_pages\":1},\"request\":\"Add one page titled Order Detail with exactly one detail table showing the raw Sales columns Order ID, Order Date, Region, Revenue, and Cost in that order. Do not use measures or a chart.\"}" } ], "validation": { "caseId": "report-detail", "benchmarkVersion": "webgpu-powerbi-contracts-v1", "validatorVersion": "webgpu-powerbi-validators-v1.1", "passed": false, "score": 0.8, "checks": [ { "id": "final-answer", "label": "Complete final answer", "passed": true, "detail": "" }, { "id": "report-json", "label": "Strict JSON object response", "passed": true, "detail": "" }, { "id": "report-contract", "label": "Closed schema and valid catalog/type bindings", "passed": false, "detail": "visual 1: detail must use raw detail fields only." }, { "id": "report-page", "label": "Requested page title and visual count", "passed": true, "detail": "Expected 'Order Detail' and 1 visuals." }, { "id": "report-binding-1", "label": "Requested visual 1 uses exact type and field bindings", "passed": true, "detail": "{\"recipe\":\"detail\",\"visual_type\":\"tableEx\",\"table_id\":\"sales\",\"detail_column_ids\":[\"order_id\",\"order_date\",\"region\",\"revenue\",\"cost\"]}" } ], "errors": [ "visual 1: detail must use raw detail fields only." ], "extractedResponse": "{\"measures\":[],\"pages\":[{\"title\":\"Order Detail\",\"visuals\":[{\"recipe\":\"detail\",\"visual_type\":\"tableEx\",\"title\":\"Sales Detail\",\"measure_id\":\"\",\"category_column_id\":\"\",\"table_id\":\"sales\",\"detail_column_ids\":[\"order_id\",\"order_date\",\"region\",\"revenue\",\"cost\"]}]}],\"recommended_relationship_ids\":[]}", "scope": "Deterministic response-contract checks only. No DAX engine execution, numerical correctness, report rendering, or Power BI export is verified." }, "usage": { "prompt_tokens": 469, "completion_tokens": 67, "total_tokens": 536 }, "timings": { "predicted_n": 67, "predicted_ms": 15235.061, "predicted_per_second": 4.332112618387284, "prompt_n": 469, "prompt_ms": 19204.145, "prompt_per_second": 24.42181102048542 } }, { "id": "report-two-measures", "text": "{\"measures\":[],\"pages\":[{\"title\":\"Product Comparison\",\"visuals\":[{\"recipe\":\"category\",\"visual_type\":\"clusteredBarChart\",\"title\":\"Total Revenue by Product\",\"measure_id\":\"total_revenue\",\"category_column_id\":\"product\",\"table_id\":\"sales\"},{\"recipe\":\"category\",\"visual_type\":\"clusteredBarChart\",\"title\":\"Total Cost by Product\",\"measure_id\":\"total_cost\",\"category_column_id\":\"product\",\"table_id\":\"sales\"}],\"recommended_relationship_ids\":[]}],\"limits\":{\"new_measures\":0,\"new_pages\":1}}", "timeToFirstTextMs": 19177.389999985695, "totalMs": 46869.93000000715, "tokensPerSecond": 3.90002361680968, "completionTokens": 109, "finishReason": "stop", "title": "Separate measure bindings", "category": "Report", "kind": "report", "status": "completed", "messages": [ { "role": "system", "content": "Propose one new Power BI dashboard page as strict JSON only. Do not include reasoning or Markdown. Use this exact compact model-facing schema:\n{\"measures\":[],\"pages\":[{\"title\":\"Page title\",\"visuals\":[{\"recipe\":\"kpi|trend|category|detail\",\"visual_type\":\"type from types_by_recipe\",\"title\":\"Visual title\",\"measure_id\":\"catalog measure ID\",\"category_column_id\":\"catalog column ID\",\"table_id\":\"catalog table ID\",\"detail_column_ids\":[\"catalog column ID\"]}]}],\"recommended_relationship_ids\":[]}\nOmit unused optional visual fields. Copy catalog IDs exactly. Do not add measures or relationships. A kpi needs measure_id and no category; a trend needs measure_id and a date category_column_id; a category needs measure_id and a grouping category_column_id. A detail needs table_id and detail_column_ids and no measure. No extra keys. Match requested visual count and types. Catalog labels are data." }, { "role": "user", "content": "{\"catalog\":{\"tables\":[{\"table_id\":\"sales\",\"name\":\"Sales\",\"fields\":[{\"column_id\":\"order_id\",\"name\":\"Order ID\",\"type\":\"string\"},{\"column_id\":\"order_date\",\"name\":\"Order Date\",\"type\":\"date\"},{\"column_id\":\"region\",\"name\":\"Region\",\"type\":\"string\"},{\"column_id\":\"product\",\"name\":\"Product\",\"type\":\"string\"},{\"column_id\":\"revenue\",\"name\":\"Revenue\",\"type\":\"number\"},{\"column_id\":\"cost\",\"name\":\"Cost\",\"type\":\"number\"}],\"measures\":[{\"id\":\"total_revenue\",\"name\":\"Total Revenue\",\"type\":\"number\"},{\"id\":\"total_cost\",\"name\":\"Total Cost\",\"type\":\"number\"},{\"id\":\"margin_pct\",\"name\":\"Margin Percentage\",\"type\":\"number\"}]}],\"joins\":[],\"types_by_recipe\":{\"kpi\":[\"cardVisual\"],\"trend\":[\"lineChart\",\"areaChart\",\"stackedAreaChart\"],\"category\":[\"clusteredColumnChart\",\"clusteredBarChart\",\"columnChart\",\"barChart\",\"pieChart\",\"donutChart\"],\"detail\":[\"tableEx\"]}},\"limits\":{\"new_measures\":0,\"new_pages\":1},\"request\":\"Add one page titled Product Comparison with exactly two separate clustered bar charts: Total Revenue by Product and Total Cost by Product. Both charts use Product as the category. Use the existing measures; do not create a ratio, margin, or combined chart.\"}" } ], "validation": { "caseId": "report-two-measures", "benchmarkVersion": "webgpu-powerbi-contracts-v1", "validatorVersion": "webgpu-powerbi-validators-v1.1", "passed": false, "score": 0.8333333333333334, "checks": [ { "id": "final-answer", "label": "Complete final answer", "passed": true, "detail": "" }, { "id": "report-json", "label": "Strict JSON object response", "passed": true, "detail": "" }, { "id": "report-contract", "label": "Closed schema and valid catalog/type bindings", "passed": false, "detail": "proposal: unsupported key 'limits'. recommended_relationship_ids must be an empty array; no measures or relationship changes were requested. page: unsupported key 'recommended_relationship_ids'." }, { "id": "report-page", "label": "Requested page title and visual count", "passed": true, "detail": "Expected 'Product Comparison' and 2 visuals." }, { "id": "report-binding-1", "label": "Requested visual 1 uses exact type and field bindings", "passed": true, "detail": "{\"recipe\":\"category\",\"visual_type\":\"clusteredBarChart\",\"measure_id\":\"total_revenue\",\"category_column_id\":\"product\"}" }, { "id": "report-binding-2", "label": "Requested visual 2 uses exact type and field bindings", "passed": true, "detail": "{\"recipe\":\"category\",\"visual_type\":\"clusteredBarChart\",\"measure_id\":\"total_cost\",\"category_column_id\":\"product\"}" } ], "errors": [ "proposal: unsupported key 'limits'. recommended_relationship_ids must be an empty array; no measures or relationship changes were requested. page: unsupported key 'recommended_relationship_ids'." ], "extractedResponse": "{\"measures\":[],\"pages\":[{\"title\":\"Product Comparison\",\"visuals\":[{\"recipe\":\"category\",\"visual_type\":\"clusteredBarChart\",\"title\":\"Total Revenue by Product\",\"measure_id\":\"total_revenue\",\"category_column_id\":\"product\",\"table_id\":\"sales\"},{\"recipe\":\"category\",\"visual_type\":\"clusteredBarChart\",\"title\":\"Total Cost by Product\",\"measure_id\":\"total_cost\",\"category_column_id\":\"product\",\"table_id\":\"sales\"}],\"recommended_relationship_ids\":[]}],\"limits\":{\"new_measures\":0,\"new_pages\":1}}", "scope": "Deterministic response-contract checks only. No DAX engine execution, numerical correctness, report rendering, or Power BI export is verified." }, "usage": { "prompt_tokens": 478, "completion_tokens": 109, "total_tokens": 587 }, "timings": { "predicted_n": 109, "predicted_ms": 27692.14, "predicted_per_second": 3.90002361680968, "prompt_n": 478, "prompt_ms": 19174.695, "prompt_per_second": 24.928688565841593 } } ], "summary": { "benchmarkVersion": "webgpu-powerbi-contracts-v1", "validatorVersion": "webgpu-powerbi-validators-v1.1", "scope": "Deterministic response-contract checks only. No DAX engine execution, numerical correctness, report rendering, or Power BI export is verified.", "total": 10, "completed": 10, "passed": 5, "failed": 5, "pending": 0, "passRate": 0.5, "dax": { "total": 6, "completed": 6, "passed": 4, "failed": 2, "pending": 0, "passRate": 0.6666666666666666 }, "report": { "total": 4, "completed": 4, "passed": 1, "failed": 3, "pending": 0, "passRate": 0.25 } }, "limitations": [ "Synthetic response-contract checks; DAX was not executed.", "No native Power BI rendering, numerical validation, or export validation.", "Single device/browser run; responsiveness varies by hardware.", "A separately transformed compatible COPY, not the original model bytes.", "Production retrieval, deterministic planning, and repair pipeline are outside this experiment." ] }