{"probe":"6x7 = 42","timestamp":1776876585.0124645,"live":6,"unavailable":26,"models":[{"model_id":"eu.amazon.nova-lite-v1:0","name":"Nova Lite","status":"live","answer":"42","correct":true,"expected":"42","latency_ms":654,"tokens":{"input":13,"output":4}},{"model_id":"eu.amazon.nova-micro-v1:0","name":"Nova Micro","status":"live","answer":"42","correct":true,"expected":"42","latency_ms":649,"tokens":{"input":13,"output":4}},{"model_id":"eu.amazon.nova-pro-v1:0","name":"Nova Pro","status":"live","answer":"42","correct":true,"expected":"42","latency_ms":823,"tokens":{"input":13,"output":4}},{"model_id":"eu.anthropic.claude-haiku-4-5-20251001-v1:0","name":"Haiku 4.5","status":"live","answer":"42","correct":true,"expected":"42","latency_ms":1398,"tokens":{"input":20,"output":5}},{"model_id":"eu.anthropic.claude-sonnet-4-20250514-v1:0","name":"Sonnet 4","status":"live","answer":"42","correct":true,"expected":"42","latency_ms":2399,"tokens":{"input":20,"output":5}},{"model_id":"eu.anthropic.claude-sonnet-4-6","name":"Sonnet 4.6","status":"live","answer":"42","correct":true,"expected":"42","latency_ms":1334,"tokens":{"input":20,"output":5}},{"model_id":"cc","name":"cc","status":"unavailable","error":"400 Client Error: Bad Request for url: https://bedrock-runtime.eu-west-3.amazonaws.com/model/cc/converse","answer":null,"correct":false},{"model_id":"charles","name":"charles","status":"unavailable","error":"400 Client Error: Bad Request for url: https://bedrock-runtime.eu-west-3.amazonaws.com/model/charles/converse","answer":null,"correct":false},{"model_id":"charles-architect","name":"charles-architect","status":"unavailable","error":"400 Client Error: Bad Request for url: https://bedrock-runtime.eu-west-3.amazonaws.com/model/charles-architect/converse","answer":null,"correct":false},{"model_id":"charles-auma","name":"charles-auma","status":"unavailable","error":"400 Client Error: Bad Request for url: https://bedrock-runtime.eu-west-3.amazonaws.com/model/charles-auma/converse","answer":null,"correct":false},{"model_id":"charles-json","name":"charles-json","status":"unavailable","error":"400 Client Error: Bad Request for url: https://bedrock-runtime.eu-west-3.amazonaws.com/model/charles-json/converse","answer":null,"correct":false},{"model_id":"charles-matching","name":"charles-matching","status":"unavailable","error":"400 Client Error: Bad Request for url: https://bedrock-runtime.eu-west-3.amazonaws.com/model/charles-matching/converse","answer":null,"correct":false},{"model_id":"charles-monolith","name":"charles-monolith","status":"unavailable","error":"400 Client Error: Bad Request for url: https://bedrock-runtime.eu-west-3.amazonaws.com/model/charles-monolith/converse","answer":null,"correct":false},{"model_id":"charles-science","name":"charles-science","status":"unavailable","error":"400 Client Error: Bad Request for url: https://bedrock-runtime.eu-west-3.amazonaws.com/model/charles-science/converse","answer":null,"correct":false},{"model_id":"charles-selfservice","name":"charles-selfservice","status":"unavailable","error":"400 Client Error: Bad Request for url: https://bedrock-runtime.eu-west-3.amazonaws.com/model/charles-selfservice/converse","answer":null,"correct":false},{"model_id":"concierge","name":"concierge","status":"unavailable","error":"400 Client Error: Bad Request for url: https://bedrock-runtime.eu-west-3.amazonaws.com/model/concierge/converse","answer":null,"correct":false},{"model_id":"concierge-matching","name":"concierge-matching","status":"unavailable","error":"400 Client Error: Bad Request for url: https://bedrock-runtime.eu-west-3.amazonaws.com/model/concierge-matching/converse","answer":null,"correct":false},{"model_id":"concierge-monolith","name":"concierge-monolith","status":"unavailable","error":"400 Client Error: Bad Request for url: https://bedrock-runtime.eu-west-3.amazonaws.com/model/concierge-monolith/converse","answer":null,"correct":false},{"model_id":"concierge-selfservice","name":"concierge-selfservice","status":"unavailable","error":"400 Client Error: Bad Request for url: https://bedrock-runtime.eu-west-3.amazonaws.com/model/concierge-selfservice/converse","answer":null,"correct":false},{"model_id":"concise","name":"concise","status":"unavailable","error":"400 Client Error: Bad Request for url: https://bedrock-runtime.eu-central-1.amazonaws.com/model/concise/converse","answer":null,"correct":false},{"model_id":"eu.anthropic.claude-3-5-haiku-20241022-v1:0","name":"Haiku 3.5","status":"unavailable","error":"400 Client Error: Bad Request for url: https://bedrock-runtime.eu-west-3.amazonaws.com/model/eu.anthropic.claude-3-5-haiku-20241022-v1:0/converse","answer":null,"correct":false},{"model_id":"eu.anthropic.claude-3-5-sonnet-20241022-v2:0","name":"Sonnet 3.5 v2","status":"unavailable","error":"400 Client Error: Bad Request for url: https://bedrock-runtime.eu-west-3.amazonaws.com/model/eu.anthropic.claude-3-5-sonnet-20241022-v2:0/converse","answer":null,"correct":false},{"model_id":"eu.anthropic.claude-opus-4-0-20250514-v1:0","name":"Opus 4","status":"unavailable","error":"400 Client Error: Bad Request for url: https://bedrock-runtime.eu-west-3.amazonaws.com/model/eu.anthropic.claude-opus-4-0-20250514-v1:0/converse","answer":null,"correct":false},{"model_id":"eu.anthropic.claude-opus-4-6","name":"Opus 4.6","status":"unavailable","error":"400 Client Error: Bad Request for url: https://bedrock-runtime.eu-west-3.amazonaws.com/model/eu.anthropic.claude-opus-4-6/converse","answer":null,"correct":false},{"model_id":"eu.meta.llama4-maverick-17b-instruct-v1:0","name":"Llama 4 Maverick","status":"unavailable","error":"400 Client Error: Bad Request for url: https://bedrock-runtime.eu-west-3.amazonaws.com/model/eu.meta.llama4-maverick-17b-instruct-v1:0/converse","answer":null,"correct":false},{"model_id":"eu.meta.llama4-scout-17b-instruct-v1:0","name":"Llama 4 Scout","status":"unavailable","error":"400 Client Error: Bad Request for url: https://bedrock-runtime.eu-west-3.amazonaws.com/model/eu.meta.llama4-scout-17b-instruct-v1:0/converse","answer":null,"correct":false},{"model_id":"eu.mistral.mistral-large-2411-v1:0","name":"Mistral Large","status":"unavailable","error":"400 Client Error: Bad Request for url: https://bedrock-runtime.eu-west-3.amazonaws.com/model/eu.mistral.mistral-large-2411-v1:0/converse","answer":null,"correct":false},{"model_id":"eu.mistral.mistral-small-2503-v1:0","name":"Mistral Small","status":"unavailable","error":"400 Client Error: Bad Request for url: https://bedrock-runtime.eu-west-3.amazonaws.com/model/eu.mistral.mistral-small-2503-v1:0/converse","answer":null,"correct":false},{"model_id":"matching","name":"matching","status":"unavailable","error":"400 Client Error: Bad Request for url: https://bedrock-runtime.eu-west-3.amazonaws.com/model/matching/converse","answer":null,"correct":false},{"model_id":"moncey","name":"moncey","status":"unavailable","error":"400 Client Error: Bad Request for url: https://bedrock-runtime.eu-west-3.amazonaws.com/model/moncey/converse","answer":null,"correct":false},{"model_id":"monolith","name":"monolith","status":"unavailable","error":"400 Client Error: Bad Request for url: https://bedrock-runtime.eu-west-3.amazonaws.com/model/monolith/converse","answer":null,"correct":false},{"model_id":"selfservice","name":"selfservice","status":"unavailable","error":"400 Client Error: Bad Request for url: https://bedrock-runtime.eu-west-3.amazonaws.com/model/selfservice/converse","answer":null,"correct":false}]}