{
  "version": 1,
  "updated": "2026-07-26",
  "source": "local.ai publication pg-20260721-ui-refresh",
  "models": {
    "LiquidAI/LFM2.5-8B-A1B": {"name":"LFM2.5-8B-A1B","quant":"bf16","filesize":"16gb","filesizeGb":15.8,"hardware":[{"id":"rtx5090_32","label":"rtx5090_32","minMemoryGb":32,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve LiquidAI/LFM2.5-8B-A1B --served-model-name LiquidAI/LFM2.5-8B-A1B --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.85 --block-size 16 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 128000 --tool-call-parser lfm2 --mamba-cache-mode align"},"rank":1,"benchmarks":[{"hardwareId":"rtx5090_32","engine":"vllm","decodeTps":419.8,"decodeTps32k":619,"prefillTps":null,"ttftMs":null,"contextTokens":127872,"measuredAt":"2026-07-05","notes":"vllm unknown"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":348.1,"decodeTps32k":924.9,"prefillTps":null,"ttftMs":null,"contextTokens":127872,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":210.2,"decodeTps32k":251.2,"prefillTps":null,"ttftMs":null,"contextTokens":127872,"measuredAt":"2026-07-09","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":55.8,"decodeTps32k":55.4,"prefillTps":null,"ttftMs":null,"contextTokens":127872,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":419.8,"prefillTps":null,"source":"measured"},"params":"8B-A1B","notes":["quality 31.1% (tau2/gaia/gdpval mean)"]},
    "Qwen/Qwen3.6-27B-FP8": {"name":"Qwen3.6-27B-FP8","quant":"fp8","filesize":"29gb","filesizeGb":28.7,"hardware":[{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve Qwen/Qwen3.6-27B-FP8 --served-model-name Qwen/Qwen3.6-27B-FP8 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 262144 --tool-call-parser qwen3_coder --reasoning-parser qwen3 --mamba-cache-mode align --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":4,\"rejection_sample_method\":\"standard\"}'"},"rank":1,"benchmarks":[{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":50.2,"decodeTps32k":50.8,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-18","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":7.8,"decodeTps32k":7.4,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":50.2,"prefillTps":null,"source":"measured"},"params":"27B","notes":["quality 91.5% (tau2/gaia/gdpval mean)"]},
    "Qwen/Qwen3.6-35B-A3B": {"name":"Qwen3.6-35B-A3B","quant":"bf16","filesize":"67gb","filesizeGb":67,"hardware":[{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"gb300","label":"gb300","minMemoryGb":288,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve Qwen/Qwen3.6-35B-A3B --served-model-name Qwen/Qwen3.6-35B-A3B --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --mamba-cache-mode align --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":4,\"moe_backend\":\"triton\",\"rejection_sample_method\":\"standard\"}'"},"rank":1,"benchmarks":[{"hardwareId":"gb300","engine":"vllm","decodeTps":338.9,"decodeTps32k":367.5,"prefillTps":16359.1,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":170.1,"decodeTps32k":208,"prefillTps":5084.2,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-09","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":31,"decodeTps32k":29.7,"prefillTps":4436.1,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":338.9,"prefillTps":16359.1,"source":"measured"},"params":"35B-A3B","notes":["quality 82.2% (tau2/gaia/gdpval mean)"]},
    "exolabs/lfm2-5-8b-a1b-mlx-4bit-dequant-bf16": {"name":"lfm2-5-8b-a1b-mlx-4bit-dequant-bf16","quant":"mlx","filesize":"16gb","filesizeGb":15.8,"hardware":[{"id":"rtx5090_32","label":"rtx5090_32","minMemoryGb":32,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve exolabs/lfm2-5-8b-a1b-mlx-4bit-dequant-bf16 --served-model-name exolabs/lfm2-5-8b-a1b-mlx-4bit-dequant-bf16 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 16 --enable-auto-tool-choice --async-scheduling --dtype bfloat16 --kv-cache-dtype bfloat16 --language-model-only --max-model-len 128000 --tool-call-parser lfm2 --moe-backend triton --attention-backend triton_attn --mamba-cache-mode align"},"rank":1,"benchmarks":[{"hardwareId":"rtx5090_32","engine":"vllm","decodeTps":361.3,"decodeTps32k":709.9,"prefillTps":null,"ttftMs":null,"contextTokens":127872,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":214.3,"decodeTps32k":282.9,"prefillTps":null,"ttftMs":null,"contextTokens":127872,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":56.4,"decodeTps32k":57,"prefillTps":null,"ttftMs":null,"contextTokens":127872,"measuredAt":"2026-07-15","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":361.3,"prefillTps":null,"source":"measured"},"params":null,"notes":["quality 35.7% (tau2/gaia/gdpval mean)"]},
    "leon-se/gemma-4-E2B-it-FP8-Dynamic": {"name":"gemma-4-E2B-it-FP8-Dynamic","quant":"fp8","filesize":"9gb","filesizeGb":8.5,"hardware":[{"id":"rtx4090_24","label":"rtx4090_24","minMemoryGb":24,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtx5090_32","label":"rtx5090_32","minMemoryGb":32,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"gb300","label":"gb300","minMemoryGb":288,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve leon-se/gemma-4-E2B-it-FP8-Dynamic --served-model-name leon-se/gemma-4-E2B-it-FP8-Dynamic --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 16 --max-num-batched-tokens 8192 --attention-backend TRITON_ATTN --async-scheduling --enable-auto-tool-choice --language-model-only --limit-mm-per-prompt '{\"image\": 0, \"audio\": 0}' --max-model-len 131072 --tool-call-parser gemma4 --reasoning-parser gemma4 --kv-cache-dtype fp8 --chat-template /vllm-workspace/examples/tool_chat_template_gemma4.jinja --default-chat-template-kwargs '{\"enable_thinking\": true}'"},"rank":1,"benchmarks":[{"hardwareId":"gb300","engine":"vllm","decodeTps":494.3,"decodeTps32k":546.1,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-21","notes":"vllm 0.25.0"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":281.4,"decodeTps32k":16308.1,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-21","notes":"vllm unknown"},{"hardwareId":"rtx5090_32","engine":"vllm","decodeTps":268.6,"decodeTps32k":327.1,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-09","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":202.4,"decodeTps32k":311.7,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-05","notes":"vllm unknown"},{"hardwareId":"rtx4090_24","engine":"vllm","decodeTps":189.7,"decodeTps32k":208.9,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":66.9,"decodeTps32k":73.8,"prefillTps":null,"ttftMs":null,"contextTokens":126976,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":494.3,"prefillTps":null,"source":"measured"},"params":"2B","notes":["quality 13.9% (tau2/gaia/gdpval mean)"]},
    "nvidia/GLM-5.2-NVFP4": {"name":"GLM-5.2-NVFP4","quant":"nvfp4","filesize":"433gb","filesizeGb":432.9,"hardware":[{"id":"gb300","label":"gb300","minMemoryGb":288,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"gb300x2","label":"gb300x2","minMemoryGb":576,"gpuCount":2,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve nvidia/GLM-5.2-NVFP4 --block-size 128 --trust-remote-code --kv-cache-dtype fp8 --enable-prefix-caching --gpu-memory-utilization 0.90 --kernel-config '{\"enable_flashinfer_autotune\":false}' --max-model-len 1048576 --cpu-offload-gb 267 --cpu-offload-params w13_weight w2_weight"},"rank":1,"benchmarks":[{"hardwareId":"gb300x2","engine":"vllm","decodeTps":41.9,"decodeTps32k":42.5,"prefillTps":null,"ttftMs":null,"contextTokens":749568,"measuredAt":"2026-07-18","notes":"vllm 0.25.0"},{"hardwareId":"gb300","engine":"vllm","decodeTps":28,"decodeTps32k":28.3,"prefillTps":null,"ttftMs":null,"contextTokens":1048448,"measuredAt":"2026-07-15","notes":"vllm unknown"}],"expectSpeed":{"decodeTps":41.9,"prefillTps":null,"source":"measured"},"params":null,"notes":["quality 98.4% (tau2/gaia/gdpval mean)"]},
    "nvidia/Qwen3.6-27B-NVFP4": {"name":"Qwen3.6-27B-NVFP4","quant":"nvfp4","filesize":"20gb","filesizeGb":20.4,"hardware":[{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"gb300","label":"gb300","minMemoryGb":288,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"gb300x2","label":"gb300x2","minMemoryGb":576,"gpuCount":2,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve nvidia/Qwen3.6-27B-NVFP4 --served-model-name nvidia/Qwen3.6-27B-NVFP4 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 262144 --tool-call-parser qwen3_coder --reasoning-parser qwen3 --mamba-cache-mode align --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":4,\"rejection_sample_method\":\"standard\"}'"},"rank":1,"benchmarks":[{"hardwareId":"gb300","engine":"vllm","decodeTps":122.9,"decodeTps32k":125.2,"prefillTps":4033,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb300x2","engine":"vllm","decodeTps":115.8,"decodeTps32k":118.3,"prefillTps":4449.3,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-18","notes":"vllm 0.25.0"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":69.6,"decodeTps32k":71,"prefillTps":4660.3,"ttftMs":null,"contextTokens":73728,"measuredAt":"2026-07-13","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":42.3,"decodeTps32k":42,"prefillTps":3247.1,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-06","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":11.9,"decodeTps32k":11.5,"prefillTps":1149.2,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":122.9,"prefillTps":4660.3,"source":"measured"},"params":"27B","notes":["quality 82.2% (tau2/gaia/gdpval mean)"]},
    "sakamakismile/Qwen3.6-27B-Text-NVFP4-MTP": {"name":"Qwen3.6-27B-Text-NVFP4-MTP","quant":"nvfp4","filesize":"18gb","filesizeGb":18.3,"hardware":[{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve sakamakismile/Qwen3.6-27B-Text-NVFP4-MTP --served-model-name sakamakismile/Qwen3.6-27B-Text-NVFP4-MTP --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --mamba-cache-mode align --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":4,\"rejection_sample_method\":\"standard\"}'"},"rank":1,"benchmarks":[{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":71.7,"decodeTps32k":75.2,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-09","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":12.2,"decodeTps32k":11.4,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":71.7,"prefillTps":null,"source":"measured"},"params":"27B","notes":["quality 82.3% (tau2/gaia/gdpval mean)"]},
    "bottlecapai/ThinkingCap-Qwen3.6-27B": {"name":"ThinkingCap-Qwen3.6-27B","quant":"bf16","filesize":"52gb","filesizeGb":51.7,"hardware":[{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve bottlecapai/ThinkingCap-Qwen3.6-27B --served-model-name bottlecapai/ThinkingCap-Qwen3.6-27B --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --mamba-cache-mode align --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":4,\"rejection_sample_method\":\"standard\"}'"},"rank":2,"benchmarks":[{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":28.5,"decodeTps32k":28.7,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-21","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":4.5,"decodeTps32k":4.3,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":28.5,"prefillTps":null,"source":"measured"},"params":"27B","notes":["quality 90.9% (tau2/gaia/gdpval mean)"]},
    "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4": {"name":"NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4","quant":"nvfp4","filesize":"18gb","filesizeGb":18,"hardware":[{"id":"rtx5090_32","label":"rtx5090_32","minMemoryGb":32,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4 --served-model-name nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 16 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser nemotron_v3 --kv-cache-dtype fp8 --mamba-cache-mode align"},"rank":2,"benchmarks":[{"hardwareId":"rtx5090_32","engine":"vllm","decodeTps":332.4,"decodeTps32k":484.6,"prefillTps":24089.9,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":163.5,"decodeTps32k":204.9,"prefillTps":19443.6,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-18","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":60.8,"decodeTps32k":62.6,"prefillTps":10911.4,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":332.4,"prefillTps":24089.9,"source":"measured"},"params":"30B-A3B","notes":["quality 39.8% (tau2/gaia/gdpval mean)"]},
    "unsloth/Qwen3.6-35B-A3B-NVFP4": {"name":"Qwen3.6-35B-A3B-NVFP4","quant":"nvfp4","filesize":"25gb","filesizeGb":24.7,"hardware":[{"id":"rtx5090_32","label":"rtx5090_32","minMemoryGb":32,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"gb300","label":"gb300","minMemoryGb":288,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve unsloth/Qwen3.6-35B-A3B-NVFP4 --served-model-name unsloth/Qwen3.6-35B-A3B-NVFP4 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --enable-auto-tool-choice --async-scheduling --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --kv-cache-dtype fp8 --attention-backend flashinfer --max-num-batched-tokens 8192 --mamba-cache-mode align"},"rank":2,"benchmarks":[{"hardwareId":"gb300","engine":"vllm","decodeTps":329.3,"decodeTps32k":352.7,"prefillTps":20140.5,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-20","notes":"vllm 0.25.0"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":247.5,"decodeTps32k":258,"prefillTps":21383.2,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-18","notes":"vllm unknown"},{"hardwareId":"rtx5090_32","engine":"vllm","decodeTps":228.1,"decodeTps32k":257.2,"prefillTps":18901.1,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":181,"decodeTps32k":171,"prefillTps":20326.9,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":64,"decodeTps32k":61,"prefillTps":7003.2,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":329.3,"prefillTps":21383.2,"source":"measured"},"params":"35B-A3B","notes":["quality 82.1% (tau2/gaia/gdpval mean)"]},
    "0xSero/Step-3.7-Flash-148B": {"name":"Step-3.7-Flash-148B","quant":"bf16","filesize":"94gb","filesizeGb":94,"hardware":[{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve 0xSero/Step-3.7-Flash-148B --served-model-name 0xSero/Step-3.7-Flash-148B --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 16 --max-num-batched-tokens 8192 --async-scheduling --language-model-only --max-model-len 262144 --quantization modelopt --kv-cache-dtype fp8 --reasoning-parser step3p5 --enable-auto-tool-choice --tool-call-parser step3p5"},"rank":3,"benchmarks":[{"hardwareId":"gb10_121","engine":"vllm","decodeTps":13.8,"decodeTps32k":13.7,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":13.8,"prefillTps":null,"source":"measured"},"params":"148B","notes":["quality 85.4% (tau2/gaia/gdpval mean)"]},
    "Qwen/Qwen3.6-35B-A3B-FP8": {"name":"Qwen3.6-35B-A3B-FP8","quant":"fp8","filesize":"35gb","filesizeGb":34.9,"hardware":[{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"gb300","label":"gb300","minMemoryGb":288,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve Qwen/Qwen3.6-35B-A3B-FP8 --served-model-name Qwen/Qwen3.6-35B-A3B-FP8 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --mamba-cache-mode align --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":4,\"moe_backend\":\"triton\",\"rejection_sample_method\":\"standard\"}'"},"rank":3,"benchmarks":[{"hardwareId":"gb300","engine":"vllm","decodeTps":315,"decodeTps32k":337.3,"prefillTps":13921.8,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":221.2,"decodeTps32k":290.5,"prefillTps":5010.2,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-09","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":50.1,"decodeTps32k":46.9,"prefillTps":6136.3,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":315,"prefillTps":13921.8,"source":"measured"},"params":"35B-A3B","notes":["quality 81.2% (tau2/gaia/gdpval mean)"]},
    "cyankiwi/Qwen3.5-9B-AWQ-4bit": {"name":"Qwen3.5-9B-AWQ-4bit","quant":"awq","filesize":"8gb","filesizeGb":8.4,"hardware":[{"id":"rtx4090_24","label":"rtx4090_24","minMemoryGb":24,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtx5090_32","label":"rtx5090_32","minMemoryGb":32,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"gb300","label":"gb300","minMemoryGb":288,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve cyankiwi/Qwen3.5-9B-AWQ-4bit --served-model-name cyankiwi/Qwen3.5-9B-AWQ-4bit --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 16 --max-num-batched-tokens 8192 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --mamba-cache-mode align"},"rank":3,"benchmarks":[{"hardwareId":"gb300","engine":"vllm","decodeTps":315.4,"decodeTps32k":336.6,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-20","notes":"vllm 0.25.0"},{"hardwareId":"rtx5090_32","engine":"vllm","decodeTps":205.7,"decodeTps32k":203.5,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-08","notes":"vllm unknown"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":202,"decodeTps32k":251.5,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"rtx4090_24","engine":"vllm","decodeTps":126.8,"decodeTps32k":125.5,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-09","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":119.9,"decodeTps32k":113.2,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-09","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":32.6,"decodeTps32k":29.8,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":315.4,"prefillTps":null,"source":"measured"},"params":"9B","notes":["quality 57.1% (tau2/gaia/gdpval mean)"]},
    "nvidia/Qwen3.6-35B-A3B-NVFP4": {"name":"Qwen3.6-35B-A3B-NVFP4","quant":"nvfp4","filesize":"22gb","filesizeGb":21.8,"hardware":[{"id":"rtx5090_32","label":"rtx5090_32","minMemoryGb":32,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"gb300","label":"gb300","minMemoryGb":288,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve nvidia/Qwen3.6-35B-A3B-NVFP4 --served-model-name nvidia/Qwen3.6-35B-A3B-NVFP4 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --async-scheduling --enable-auto-tool-choice --language-model-only --attention-backend flashinfer --quantization modelopt --moe-backend marlin --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --kv-cache-dtype fp8 --mamba-cache-mode align --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\",\"rejection_sample_method\":\"standard\"}'"},"rank":3,"benchmarks":[{"hardwareId":"gb300","engine":"vllm","decodeTps":277.5,"decodeTps32k":306.5,"prefillTps":12732.2,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":259.1,"decodeTps32k":380.7,"prefillTps":16877.5,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-18","notes":"vllm unknown"},{"hardwareId":"rtx5090_32","engine":"vllm","decodeTps":235.6,"decodeTps32k":268.8,"prefillTps":16168.9,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":75.3,"decodeTps32k":71.8,"prefillTps":6780.3,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":277.5,"prefillTps":16877.5,"source":"measured"},"params":"35B-A3B","notes":["quality 81.6% (tau2/gaia/gdpval mean)"]},
    "unsloth/Qwen3.6-27B-NVFP4": {"name":"Qwen3.6-27B-NVFP4","quant":"nvfp4","filesize":"22gb","filesizeGb":21.8,"hardware":[{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve unsloth/Qwen3.6-27B-NVFP4 --served-model-name unsloth/Qwen3.6-27B-NVFP4 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --async-scheduling --enable-auto-tool-choice --language-model-only --attention-backend flashinfer --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --kv-cache-dtype fp8 --mamba-cache-mode align"},"rank":3,"benchmarks":[{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":65.5,"decodeTps32k":69.7,"prefillTps":4492.9,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-18","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":38.2,"decodeTps32k":37.4,"prefillTps":3244.5,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":11.2,"decodeTps32k":10.8,"prefillTps":2295.1,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":65.5,"prefillTps":4492.9,"source":"measured"},"params":"27B","notes":["quality 82.1% (tau2/gaia/gdpval mean)"]},
    "Hcompany/Holo-3.1-35B-A3B-NVFP4": {"name":"Holo-3.1-35B-A3B-NVFP4","quant":"nvfp4","filesize":"22gb","filesizeGb":22.1,"hardware":[{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"gb300","label":"gb300","minMemoryGb":288,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve Hcompany/Holo-3.1-35B-A3B-NVFP4 --served-model-name Hcompany/Holo-3.1-35B-A3B-NVFP4 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --async-scheduling --enable-auto-tool-choice --language-model-only --attention-backend flashinfer --quantization modelopt --moe-backend marlin --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --kv-cache-dtype fp8 --mamba-cache-mode align"},"rank":4,"benchmarks":[{"hardwareId":"gb300","engine":"vllm","decodeTps":287.8,"decodeTps32k":308.2,"prefillTps":14555.6,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":259.3,"decodeTps32k":388.7,"prefillTps":16762,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":74.4,"decodeTps32k":71.2,"prefillTps":6884.8,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":287.8,"prefillTps":16762,"source":"measured"},"params":"35B-A3B","notes":["quality 71.3% (tau2/gaia/gdpval mean)"]},
    "cyankiwi/Qwen3.6-35B-A3B-AWQ-4bit": {"name":"Qwen3.6-35B-A3B-AWQ-4bit","quant":"awq","filesize":"23gb","filesizeGb":23.3,"hardware":[{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"gb300","label":"gb300","minMemoryGb":288,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve cyankiwi/Qwen3.6-35B-A3B-AWQ-4bit --served-model-name cyankiwi/Qwen3.6-35B-A3B-AWQ-4bit --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --max-num-batched-tokens 8192 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --mamba-cache-mode align"},"rank":4,"benchmarks":[{"hardwareId":"gb300","engine":"vllm","decodeTps":266.1,"decodeTps32k":279.1,"prefillTps":7811.1,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-21","notes":"vllm 0.25.0"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":213.2,"decodeTps32k":116.3,"prefillTps":4174.5,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":193.3,"decodeTps32k":245.2,"prefillTps":5166.4,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":36.5,"decodeTps32k":34.7,"prefillTps":6197.7,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":266.1,"prefillTps":7811.1,"source":"measured"},"params":"35B-A3B","notes":["quality 81.1% (tau2/gaia/gdpval mean)"]},
    "exolabs/qwen3-6-27b-optiq-4bit-text-dequant-bf16": {"name":"qwen3-6-27b-optiq-4bit-text-dequant-bf16","quant":"mlx","filesize":"50gb","filesizeGb":50.1,"hardware":[{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve exolabs/qwen3-6-27b-optiq-4bit-text-dequant-bf16 --served-model-name exolabs/qwen3-6-27b-optiq-4bit-text-dequant-bf16 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --max-num-batched-tokens 8192 --enable-auto-tool-choice --async-scheduling --dtype bfloat16 --kv-cache-dtype bfloat16 --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --attention-backend triton_attn --mamba-cache-mode align"},"rank":4,"benchmarks":[{"hardwareId":"gb10_121","engine":"vllm","decodeTps":4.4,"decodeTps32k":4.3,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-18","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":4.4,"prefillTps":null,"source":"measured"},"params":null,"notes":["quality 81.7% (tau2/gaia/gdpval mean)"]},
    "exolabs/qwen3-6-27b-ud-mlx-4bit-text-dequant-bf16-normfix": {"name":"qwen3-6-27b-ud-mlx-4bit-text-dequant-bf16-normfix","quant":"mlx","filesize":"17gb","filesizeGb":17,"hardware":[{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve exolabs/qwen3-6-27b-ud-mlx-4bit-text-dequant-bf16-normfix"},"rank":4,"benchmarks":[{"hardwareId":"gb10_121","engine":"vllm","decodeTps":4.5,"decodeTps32k":4.3,"prefillTps":null,"ttftMs":null,"contextTokens":155904,"measuredAt":"2026-07-15","notes":"vllm unknown"}],"expectSpeed":{"decodeTps":4.5,"prefillTps":null,"source":"measured"},"params":null,"notes":["quality 81.4% (tau2/gaia/gdpval mean)","size estimated from parameter count"]},
    "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4": {"name":"Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4","quant":"nvfp4","filesize":"21gb","filesizeGb":20.9,"hardware":[{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4 --served-model-name nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 16 --async-scheduling --enable-auto-tool-choice --max-model-len 131072 --tool-call-parser qwen3_coder --reasoning-parser nemotron_v3 --kv-cache-dtype fp8"},"rank":4,"benchmarks":[{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":307.9,"decodeTps32k":402.9,"prefillTps":23766.4,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-18","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":55.9,"decodeTps32k":56.8,"prefillTps":9566.3,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":307.9,"prefillTps":23766.4,"source":"measured"},"params":"30B-A3B","notes":["quality 38.9% (tau2/gaia/gdpval mean)"]},
    "openai/gpt-oss-20b": {"name":"gpt-oss-20b","quant":"bf16","filesize":"26gb","filesizeGb":25.6,"hardware":[{"id":"rtx4090_24","label":"rtx4090_24","minMemoryGb":24,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtx5090_32","label":"rtx5090_32","minMemoryGb":32,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve openai/gpt-oss-20b --served-model-name openai/gpt-oss-20b --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 16 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 131072 --tool-call-parser openai --kv-cache-dtype fp8 --max-cudagraph-capture-size 2048 --max-num-batched-tokens 8192 --stream-interval 20"},"rank":4,"benchmarks":[{"hardwareId":"rtx5090_32","engine":"vllm","decodeTps":289.3,"decodeTps32k":304,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-09","notes":"vllm unknown"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":282.2,"decodeTps32k":380.9,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-18","notes":"vllm unknown"},{"hardwareId":"rtx4090_24","engine":"vllm","decodeTps":192.7,"decodeTps32k":263.8,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-09","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":186.5,"decodeTps32k":197.8,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-05","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":46.3,"decodeTps32k":45.7,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":289.3,"prefillTps":null,"source":"measured"},"params":null,"notes":["quality 52.9% (tau2/gaia/gdpval mean)"]},
    "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8": {"name":"NVIDIA-Nemotron-3-Nano-30B-A3B-FP8","quant":"fp8","filesize":"30gb","filesizeGb":30.4,"hardware":[{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 --served-model-name nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 16 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 262144 --tool-call-parser qwen3_coder --reasoning-parser nemotron_v3 --kv-cache-dtype fp8 --mamba-cache-mode align"},"rank":5,"benchmarks":[{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":262.1,"decodeTps32k":405.3,"prefillTps":23306.9,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-18","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":176.8,"decodeTps32k":199.8,"prefillTps":24418.9,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":44.6,"decodeTps32k":45.3,"prefillTps":7831.6,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":262.1,"prefillTps":24418.9,"source":"measured"},"params":"30B-A3B","notes":["quality 42.3% (tau2/gaia/gdpval mean)"]},
    "unsloth/Qwen3.6-35B-A3B-NVFP4-Fast": {"name":"Qwen3.6-35B-A3B-NVFP4-Fast","quant":"nvfp4","filesize":"22gb","filesizeGb":22,"hardware":[{"id":"rtx5090_32","label":"rtx5090_32","minMemoryGb":32,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve unsloth/Qwen3.6-35B-A3B-NVFP4-Fast --served-model-name unsloth/Qwen3.6-35B-A3B-NVFP4-Fast --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --enable-auto-tool-choice --async-scheduling --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --kv-cache-dtype fp8 --attention-backend flashinfer --max-num-batched-tokens 8192 --mamba-cache-mode align"},"rank":5,"benchmarks":[{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":241.6,"decodeTps32k":344.6,"prefillTps":19182.8,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-18","notes":"vllm unknown"},{"hardwareId":"rtx5090_32","engine":"vllm","decodeTps":240.1,"decodeTps32k":276.2,"prefillTps":16637,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":184.2,"decodeTps32k":176.2,"prefillTps":21078.9,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":69.2,"decodeTps32k":66.4,"prefillTps":7197.8,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-18","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":241.6,"prefillTps":21078.9,"source":"measured"},"params":"35B-A3B","notes":["quality 80.9% (tau2/gaia/gdpval mean)"]},
    "InternScience/Agents-A1-FP8": {"name":"Agents-A1-FP8","quant":"fp8","filesize":"35gb","filesizeGb":35.1,"hardware":[{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve InternScience/Agents-A1-FP8 --served-model-name InternScience/Agents-A1-FP8 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --mamba-cache-mode align"},"rank":6,"benchmarks":[{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":196.9,"decodeTps32k":250.2,"prefillTps":4946.6,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":38.1,"decodeTps32k":36.4,"prefillTps":5533.2,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-13","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":196.9,"prefillTps":5533.2,"source":"measured"},"params":null,"notes":["quality 79.4% (tau2/gaia/gdpval mean)"]},
    "RedHatAI/Qwen3.6-35B-A3B-NVFP4": {"name":"Qwen3.6-35B-A3B-NVFP4","quant":"nvfp4","filesize":"23gb","filesizeGb":23.3,"hardware":[{"id":"rtx5090_32","label":"rtx5090_32","minMemoryGb":32,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"gb300","label":"gb300","minMemoryGb":288,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve RedHatAI/Qwen3.6-35B-A3B-NVFP4 --served-model-name RedHatAI/Qwen3.6-35B-A3B-NVFP4 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --kv-cache-dtype fp8 --attention-backend flashinfer --moe-backend flashinfer_cutlass --max-num-batched-tokens 8192 --mamba-cache-mode align"},"rank":6,"benchmarks":[{"hardwareId":"gb300","engine":"vllm","decodeTps":245.9,"decodeTps32k":276.5,"prefillTps":21495.5,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-20","notes":"vllm 0.25.0"},{"hardwareId":"rtx5090_32","engine":"vllm","decodeTps":213.5,"decodeTps32k":347.5,"prefillTps":18559.6,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-09","notes":"vllm unknown"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":209.6,"decodeTps32k":324,"prefillTps":18754.1,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-09","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":40.9,"decodeTps32k":40.2,"prefillTps":6708.6,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":245.9,"prefillTps":21495.5,"source":"measured"},"params":"35B-A3B","notes":["quality 40.8% (tau2/gaia/gdpval mean)"]},
    "RedHatAI/gemma-4-26B-A4B-it-FP8-Dynamic": {"name":"gemma-4-26B-A4B-it-FP8-Dynamic","quant":"fp8","filesize":"27gb","filesizeGb":26.7,"hardware":[{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve RedHatAI/gemma-4-26B-A4B-it-FP8-Dynamic --served-model-name RedHatAI/gemma-4-26B-A4B-it-FP8-Dynamic --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 16 --enable-auto-tool-choice --revision 951346384683e0c783e8acbaf649418d6c77dd4e --tokenizer-revision 951346384683e0c783e8acbaf649418d6c77dd4e --async-scheduling --language-model-only --max-model-len 262144 --reasoning-parser gemma4 --tool-call-parser gemma4 --kv-cache-dtype bfloat16 --chat-template /vllm-workspace/examples/tool_chat_template_gemma4.jinja --default-chat-template-kwargs '{\"enable_thinking\": true}'"},"rank":6,"benchmarks":[{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":201.5,"decodeTps32k":536.7,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-18","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":132.3,"decodeTps32k":134.4,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":38.1,"decodeTps32k":37.4,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":201.5,"prefillTps":null,"source":"measured"},"params":"26B-A4B","notes":["quality 69.3% (tau2/gaia/gdpval mean)"]},
    "cyankiwi/gemma-4-26B-A4B-it-AWQ-4bit": {"name":"gemma-4-26B-A4B-it-AWQ-4bit","quant":"awq","filesize":"16gb","filesizeGb":16,"hardware":[{"id":"rtx5090_32","label":"rtx5090_32","minMemoryGb":32,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve cyankiwi/gemma-4-26B-A4B-it-AWQ-4bit --served-model-name cyankiwi/gemma-4-26B-A4B-it-AWQ-4bit --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.85 --block-size 16 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 131072 --reasoning-parser gemma4 --tool-call-parser gemma4 --chat-template examples/tool_chat_template_gemma4.jinja --default-chat-template-kwargs '{\"enable_thinking\": true}' --speculative-config '{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4,\"rejection_sample_method\":\"standard\"}'"},"rank":6,"benchmarks":[{"hardwareId":"rtx5090_32","engine":"vllm","decodeTps":217.8,"decodeTps32k":214.3,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-08","notes":"vllm unknown"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":209.4,"decodeTps32k":727.6,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":151.9,"decodeTps32k":163.8,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-09","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":42.4,"decodeTps32k":41.1,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":217.8,"prefillTps":null,"source":"measured"},"params":"26B-A4B","notes":["quality 63.8% (tau2/gaia/gdpval mean)"]},
    "deepreinforce-ai/Ornith-1.0-35B-FP8": {"name":"Ornith-1.0-35B-FP8","quant":"fp8","filesize":"35gb","filesizeGb":35.1,"hardware":[{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve deepreinforce-ai/Ornith-1.0-35B-FP8 --served-model-name deepreinforce-ai/Ornith-1.0-35B-FP8 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --mamba-cache-mode align"},"rank":6,"benchmarks":[{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":196.4,"decodeTps32k":249.3,"prefillTps":4944.9,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":36.3,"decodeTps32k":34.4,"prefillTps":5357.6,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":196.4,"prefillTps":5357.6,"source":"measured"},"params":"35B","notes":["quality 80.7% (tau2/gaia/gdpval mean)"]},
    "Jackrong/Qwopus3.6-27B-v2": {"name":"Qwopus3.6-27B-v2","quant":"bf16","filesize":"52gb","filesizeGb":51.7,"hardware":[{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve Jackrong/Qwopus3.6-27B-v2 --served-model-name Jackrong/Qwopus3.6-27B-v2 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.85 --block-size 1024 --revision 66a4ee9d49dbcef3f83528a400ef6bec93684d6b --tokenizer-revision 66a4ee9d49dbcef3f83528a400ef6bec93684d6b --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 262144 --tool-call-parser qwen3_coder --reasoning-parser qwen3 --mamba-cache-mode align"},"rank":7,"benchmarks":[{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":28.5,"decodeTps32k":29.3,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":4.4,"decodeTps32k":4.3,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-14","notes":"vllm 0.1.dev16488+g129a28dc2"}],"expectSpeed":{"decodeTps":28.5,"prefillTps":null,"source":"measured"},"params":"27B","notes":["quality 80% (tau2/gaia/gdpval mean)"]},
    "exolabs/agents-a1-8bit-text-dequant-bf16": {"name":"agents-a1-8bit-text-dequant-bf16","quant":"mlx","filesize":"65gb","filesizeGb":64.6,"hardware":[{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve exolabs/agents-a1-8bit-text-dequant-bf16 --served-model-name exolabs/agents-a1-8bit-text-dequant-bf16 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --enable-auto-tool-choice --async-scheduling --dtype bfloat16 --kv-cache-dtype bfloat16 --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --moe-backend triton --attention-backend triton_attn"},"rank":7,"benchmarks":[{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":178.4,"decodeTps32k":187.3,"prefillTps":5131.3,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-21","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":30.5,"decodeTps32k":28.6,"prefillTps":3251.6,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":178.4,"prefillTps":5131.3,"source":"measured"},"params":null,"notes":["quality 76.2% (tau2/gaia/gdpval mean)"]},
    "exolabs/qwen3-6-35b-a3b-ud-mlx-4bit-text-dequant-bf16-normfix": {"name":"qwen3-6-35b-a3b-ud-mlx-4bit-text-dequant-bf16-normfix","quant":"mlx","filesize":"23gb","filesizeGb":23,"hardware":[{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve exolabs/qwen3-6-35b-a3b-ud-mlx-4bit-text-dequant-bf16-normfix --served-model-name exolabs/qwen3-6-35b-a3b-ud-mlx-4bit-text-dequant-bf16-normfix --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --max-num-batched-tokens 8192 --enable-auto-tool-choice --async-scheduling --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --mamba-cache-mode align"},"rank":7,"benchmarks":[{"hardwareId":"gb10_121","engine":"vllm","decodeTps":29.7,"decodeTps32k":29.1,"prefillTps":2638.9,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":29.7,"prefillTps":2638.9,"source":"measured"},"params":null,"notes":["quality 77.7% (tau2/gaia/gdpval mean)","size estimated from parameter count"]},
    "leon-se/gemma-4-E4B-it-FP8-Dynamic": {"name":"gemma-4-E4B-it-FP8-Dynamic","quant":"fp8","filesize":"12gb","filesizeGb":12.4,"hardware":[{"id":"rtx4090_24","label":"rtx4090_24","minMemoryGb":24,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtx5090_32","label":"rtx5090_32","minMemoryGb":32,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"gb300","label":"gb300","minMemoryGb":288,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve leon-se/gemma-4-E4B-it-FP8-Dynamic --served-model-name leon-se/gemma-4-E4B-it-FP8-Dynamic --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 16 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 131072 --tool-call-parser gemma4 --reasoning-parser gemma4 --kv-cache-dtype fp8 --chat-template /vllm-workspace/examples/tool_chat_template_gemma4.jinja --default-chat-template-kwargs '{\"enable_thinking\": true}' --speculative-config '{\"model\":\"google/gemma-4-E4B-it-assistant\",\"num_speculative_tokens\":4,\"rejection_sample_method\":\"standard\"}'"},"rank":7,"benchmarks":[{"hardwareId":"gb300","engine":"vllm","decodeTps":221.9,"decodeTps32k":191.5,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-18","notes":"vllm 0.25.0"},{"hardwareId":"rtx5090_32","engine":"vllm","decodeTps":212,"decodeTps32k":389.2,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-05","notes":"vllm unknown"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":176.9,"decodeTps32k":551.9,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-21","notes":"vllm unknown"},{"hardwareId":"rtx4090_24","engine":"vllm","decodeTps":122,"decodeTps32k":48.7,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-18","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":117.9,"decodeTps32k":48.5,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-18","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":35,"decodeTps32k":34.9,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-04","notes":"vllm unknown"}],"expectSpeed":{"decodeTps":221.9,"prefillTps":null,"source":"measured"},"params":"4B","notes":["quality 21.1% (tau2/gaia/gdpval mean)"]},
    "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4": {"name":"NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4","quant":"nvfp4","filesize":"75gb","filesizeGb":74.8,"hardware":[{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"gb300","label":"gb300","minMemoryGb":288,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4 --served-model-name nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.90 --block-size 16 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser nemotron_v3 --mamba-cache-mode align --max-num-batched-tokens 8320"},"rank":7,"benchmarks":[{"hardwareId":"gb300","engine":"vllm","decodeTps":196.8,"decodeTps32k":220.8,"prefillTps":17229,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":92.6,"decodeTps32k":99.7,"prefillTps":10404.6,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-13","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":15.7,"decodeTps32k":15.8,"prefillTps":2878.8,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-06","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":196.8,"prefillTps":17229,"source":"measured"},"params":"120B-A12B","notes":["quality 65.1% (tau2/gaia/gdpval mean)"]},
    "openai/gpt-oss-120b": {"name":"gpt-oss-120b","quant":"bf16","filesize":"122gb","filesizeGb":121.5,"hardware":[{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve openai/gpt-oss-120b --served-model-name openai/gpt-oss-120b --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.90 --block-size 16 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 131072 --tool-call-parser openai --kv-cache-dtype fp8 --max-cudagraph-capture-size 2048 --max-num-batched-tokens 8192 --stream-interval 20 --speculative-config '{\"model\":\"nvidia/gpt-oss-120b-Eagle3-v3\",\"method\":\"eagle3\",\"num_speculative_tokens\":4,\"draft_tensor_parallel_size\":1,\"rejection_sample_method\":\"standard\"}'"},"rank":7,"benchmarks":[{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":204.3,"decodeTps32k":226.2,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":33.8,"decodeTps32k":32.5,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":204.3,"prefillTps":null,"source":"measured"},"params":null,"notes":["quality 58.4% (tau2/gaia/gdpval mean)"]},
    "sakamakismile/Qwen3.6-27B-NVFP4": {"name":"Qwen3.6-27B-NVFP4","quant":"nvfp4","filesize":"18gb","filesizeGb":18.4,"hardware":[{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve sakamakismile/Qwen3.6-27B-NVFP4 --served-model-name sakamakismile/Qwen3.6-27B-NVFP4 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 64 --gpu-memory-utilization 0.88 --block-size 1024 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --mamba-cache-mode align"},"rank":7,"benchmarks":[{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":75.2,"decodeTps32k":76.3,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-13","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":12.5,"decodeTps32k":11.8,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":75.2,"prefillTps":null,"source":"measured"},"params":"27B","notes":["quality 77.3% (tau2/gaia/gdpval mean)"]},
    "Qwen/Qwen3.5-122B-A10B-gptq-int4": {"name":"Qwen3.5-122B-A10B-gptq-int4","quant":"gptq","filesize":"73gb","filesizeGb":73.4,"hardware":[{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"gb300","label":"gb300","minMemoryGb":288,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve Qwen/Qwen3.5-122B-A10B-gptq-int4 --served-model-name Qwen/Qwen3.5-122B-A10B-gptq-int4 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.90 --block-size 16 --max-num-batched-tokens 8192 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --mamba-cache-mode align"},"rank":8,"benchmarks":[{"hardwareId":"gb300","engine":"vllm","decodeTps":173.3,"decodeTps32k":178.8,"prefillTps":20441.5,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-20","notes":"vllm 0.25.0"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":89.2,"decodeTps32k":99.3,"prefillTps":11713.2,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-18","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":14.3,"decodeTps32k":13.9,"prefillTps":2461.5,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":173.3,"prefillTps":20441.5,"source":"measured"},"params":"122B-A10B","notes":["quality 72.3% (tau2/gaia/gdpval mean)"]},
    "Qwen/Qwen3.5-9B": {"name":"Qwen3.5-9B","quant":"bf16","filesize":"18gb","filesizeGb":18,"hardware":[{"id":"rtx5090_32","label":"rtx5090_32","minMemoryGb":32,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve Qwen/Qwen3.5-9B --served-model-name Qwen/Qwen3.5-9B --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 16 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --mamba-cache-mode align --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":4,\"rejection_sample_method\":\"standard\"}'"},"rank":9,"benchmarks":[{"hardwareId":"rtx5090_32","engine":"vllm","decodeTps":94.2,"decodeTps32k":93.7,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-09","notes":"vllm unknown"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":82.8,"decodeTps32k":79.5,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-18","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":51.5,"decodeTps32k":50.6,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-05","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":13.4,"decodeTps32k":13,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":94.2,"prefillTps":null,"source":"measured"},"params":"9B","notes":["quality 69.9% (tau2/gaia/gdpval mean)"]},
    "Qwen/Qwen3.6-27B": {"name":"Qwen3.6-27B","quant":"bf16","filesize":"52gb","filesizeGb":51.7,"hardware":[{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve Qwen/Qwen3.6-27B --served-model-name Qwen/Qwen3.6-27B --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --mamba-cache-mode align --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":4,\"rejection_sample_method\":\"standard\"}'"},"rank":9,"benchmarks":[{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":28.7,"decodeTps32k":28.8,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-09","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":10.5,"decodeTps32k":9.7,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-21","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":28.7,"prefillTps":null,"source":"measured"},"params":"27B","notes":["quality 71% (tau2/gaia/gdpval mean)"]},
    "lovedheart/Qwen3.5-9B-FP8": {"name":"Qwen3.5-9B-FP8","quant":"fp8","filesize":"11gb","filesizeGb":11.3,"hardware":[{"id":"rtx4090_24","label":"rtx4090_24","minMemoryGb":24,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtx5090_32","label":"rtx5090_32","minMemoryGb":32,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve lovedheart/Qwen3.5-9B-FP8 --served-model-name lovedheart/Qwen3.5-9B-FP8 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.85 --block-size 16 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 262144 --tool-call-parser qwen3_coder --reasoning-parser qwen3 --mamba-cache-mode align --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":4,\"rejection_sample_method\":\"standard\"}'"},"rank":9,"benchmarks":[{"hardwareId":"rtx4090_24","engine":"vllm","decodeTps":172.9,"decodeTps32k":126.5,"prefillTps":4591.2,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-09","notes":"vllm unknown"},{"hardwareId":"rtx5090_32","engine":"vllm","decodeTps":140.6,"decodeTps32k":141.1,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-09","notes":"vllm unknown"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":139.6,"decodeTps32k":161.8,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":57.2,"decodeTps32k":54.9,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-09","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":21.5,"decodeTps32k":20.4,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":172.9,"prefillTps":4591.2,"source":"measured"},"params":"9B","notes":["quality 61.4% (tau2/gaia/gdpval mean)"]},
    "nvidia/Gemma-4-26B-A4B-NVFP4": {"name":"Gemma-4-26B-A4B-NVFP4","quant":"nvfp4","filesize":"17gb","filesizeGb":17.5,"hardware":[{"id":"rtx5090_32","label":"rtx5090_32","minMemoryGb":32,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve nvidia/Gemma-4-26B-A4B-NVFP4 --served-model-name nvidia/Gemma-4-26B-A4B-NVFP4 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 16 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 131072 --reasoning-parser gemma4 --tool-call-parser gemma4 --quantization modelopt --kv-cache-dtype fp8 --chat-template /vllm-workspace/examples/tool_chat_template_gemma4.jinja --default-chat-template-kwargs '{\"enable_thinking\": true}' --speculative-config '{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"method\":\"mtp\",\"num_speculative_tokens\":4,\"rejection_sample_method\":\"standard\"}'"},"rank":9,"benchmarks":[{"hardwareId":"rtx5090_32","engine":"vllm","decodeTps":156,"decodeTps32k":165.6,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-09","notes":"vllm unknown"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":139.5,"decodeTps32k":173.4,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-18","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":119.8,"decodeTps32k":123.3,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-09","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":28.1,"decodeTps32k":28.1,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-06","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":156,"prefillTps":null,"source":"measured"},"params":"26B-A4B","notes":["quality 69% (tau2/gaia/gdpval mean)"]},
    "RedHatAI/gemma-4-31B-it-FP8-block": {"name":"gemma-4-31B-it-FP8-block","quant":"fp8","filesize":"31gb","filesizeGb":31,"hardware":[{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve RedHatAI/gemma-4-31B-it-FP8-block --served-model-name RedHatAI/gemma-4-31B-it-FP8-block --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 16 --enable-auto-tool-choice --revision e1c5d725be4d3349e85334dc108e901dd99521e4 --tokenizer-revision e1c5d725be4d3349e85334dc108e901dd99521e4 --async-scheduling --language-model-only --max-model-len 262144 --reasoning-parser gemma4 --tool-call-parser gemma4 --kv-cache-dtype bfloat16 --chat-template /vllm-workspace/examples/tool_chat_template_gemma4.jinja --default-chat-template-kwargs '{\"enable_thinking\": true}'"},"rank":10,"benchmarks":[{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":39.9,"decodeTps32k":36.9,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-18","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":7,"decodeTps32k":6.6,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":39.9,"prefillTps":null,"source":"measured"},"params":"31B","notes":["quality 68.6% (tau2/gaia/gdpval mean)"]},
    "cyankiwi/gemma-4-31B-it-AWQ-4bit": {"name":"gemma-4-31B-it-AWQ-4bit","quant":"awq","filesize":"19gb","filesizeGb":19.5,"hardware":[{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve cyankiwi/gemma-4-31B-it-AWQ-4bit --served-model-name cyankiwi/gemma-4-31B-it-AWQ-4bit --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 16 --async-scheduling --enable-auto-tool-choice --language-model-only --limit-mm-per-prompt '{\"image\": 0, \"audio\": 0}' --kv-cache-dtype fp8 --max-model-len 262144 --reasoning-parser gemma4 --tool-call-parser gemma4 --chat-template /vllm-workspace/examples/tool_chat_template_gemma4.jinja --default-chat-template-kwargs '{\"enable_thinking\": true}' --speculative-config '{\"model\":\"google/gemma-4-31B-it-assistant\",\"num_speculative_tokens\":1,\"rejection_sample_method\":\"synthetic\",\"synthetic_acceptance_rates\":[0]}'"},"rank":10,"benchmarks":[{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":63.8,"decodeTps32k":75.7,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":33.1,"decodeTps32k":22.1,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-18","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":10,"decodeTps32k":9.2,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":63.8,"prefillTps":null,"source":"measured"},"params":"31B","notes":["quality 68% (tau2/gaia/gdpval mean)"]},
    "exolabs/gemma-4-26b-a4b-it-4bit-text-dequant-bf16": {"name":"gemma-4-26b-a4b-it-4bit-text-dequant-bf16","quant":"mlx","filesize":"47gb","filesizeGb":47,"hardware":[{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve exolabs/gemma-4-26b-a4b-it-4bit-text-dequant-bf16 --served-model-name exolabs/gemma-4-26b-a4b-it-4bit-text-dequant-bf16 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 16 --enable-auto-tool-choice --async-scheduling --dtype bfloat16 --kv-cache-dtype bfloat16 --language-model-only --max-model-len 262144 --reasoning-parser gemma4 --tool-call-parser gemma4 --chat-template /vllm-workspace/examples/tool_chat_template_gemma4.jinja --default-chat-template-kwargs '{\"enable_thinking\": true}' --speculative-config '{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4,\"rejection_sample_method\":\"standard\"}'"},"rank":10,"benchmarks":[{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":149.6,"decodeTps32k":179.8,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-21","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":24,"decodeTps32k":23,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":149.6,"prefillTps":null,"source":"measured"},"params":null,"notes":["quality 64.8% (tau2/gaia/gdpval mean)"]},
    "google/gemma-4-31B-it": {"name":"gemma-4-31B-it","quant":"bf16","filesize":"58gb","filesizeGb":58.3,"hardware":[{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve google/gemma-4-31B-it"},"rank":10,"benchmarks":[{"hardwareId":"gb10_121","engine":"vllm","decodeTps":3.7,"decodeTps32k":3.6,"prefillTps":null,"ttftMs":null,"contextTokens":98176,"measuredAt":"2026-07-14","notes":"vllm 0.1.dev16488+g129a28dc2"}],"expectSpeed":{"decodeTps":3.7,"prefillTps":null,"source":"measured"},"params":"31B","notes":["quality 70.1% (tau2/gaia/gdpval mean)"]},
    "prithivMLmods/gemma-4-E4B-it-FP8": {"name":"gemma-4-E4B-it-FP8","quant":"fp8","filesize":"12gb","filesizeGb":12.4,"hardware":[{"id":"rtx5090_32","label":"rtx5090_32","minMemoryGb":32,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve prithivMLmods/gemma-4-E4B-it-FP8 --served-model-name prithivMLmods/gemma-4-E4B-it-FP8 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 16 --enable-auto-tool-choice --revision e991fcb6c8b3871071c0ad7f7343dc5e0e7e5639 --tokenizer-revision e991fcb6c8b3871071c0ad7f7343dc5e0e7e5639 --async-scheduling --language-model-only --max-model-len 131072 --reasoning-parser gemma4 --tool-call-parser gemma4 --kv-cache-dtype bfloat16 --chat-template /vllm-workspace/examples/tool_chat_template_gemma4.jinja --default-chat-template-kwargs '{\"enable_thinking\": true}'"},"rank":10,"benchmarks":[{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":170.9,"decodeTps32k":334,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-21","notes":"vllm unknown"},{"hardwareId":"rtx5090_32","engine":"vllm","decodeTps":107.4,"decodeTps32k":102.9,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":104.5,"decodeTps32k":103.4,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":34,"decodeTps32k":32.7,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-15","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":170.9,"prefillTps":null,"source":"measured"},"params":"4B","notes":["quality 22.6% (tau2/gaia/gdpval mean)"]},
    "exolabs/gemma-4-26b-a4b-it-8bit-text-dequant-bf16": {"name":"gemma-4-26b-a4b-it-8bit-text-dequant-bf16","quant":"mlx","filesize":"47gb","filesizeGb":47,"hardware":[{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve exolabs/gemma-4-26b-a4b-it-8bit-text-dequant-bf16 --served-model-name exolabs/gemma-4-26b-a4b-it-8bit-text-dequant-bf16 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 16 --max-num-batched-tokens 8192 --enable-auto-tool-choice --async-scheduling --language-model-only --max-model-len 262144 --reasoning-parser gemma4 --tool-call-parser gemma4 --chat-template /vllm-workspace/examples/tool_chat_template_gemma4.jinja --default-chat-template-kwargs '{\"enable_thinking\": true}'"},"rank":11,"benchmarks":[{"hardwareId":"gb10_121","engine":"vllm","decodeTps":23.7,"decodeTps32k":23.1,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":23.7,"prefillTps":null,"source":"measured"},"params":null,"notes":["quality 66.7% (tau2/gaia/gdpval mean)"]},
    "exolabs/gemma-4-31b-it-4bit-text-dequant-bf16": {"name":"gemma-4-31b-it-4bit-text-dequant-bf16","quant":"mlx","filesize":"57gb","filesizeGb":57.2,"hardware":[{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve exolabs/gemma-4-31b-it-4bit-text-dequant-bf16 --served-model-name exolabs/gemma-4-31b-it-4bit-text-dequant-bf16 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 16 --max-num-batched-tokens 8192 --enable-auto-tool-choice --async-scheduling --language-model-only --max-model-len 262144 --reasoning-parser gemma4 --tool-call-parser gemma4 --chat-template /vllm-workspace/examples/tool_chat_template_gemma4.jinja --default-chat-template-kwargs '{\"enable_thinking\": true}'"},"rank":11,"benchmarks":[{"hardwareId":"gb10_121","engine":"vllm","decodeTps":3.6,"decodeTps32k":3.5,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":3.6,"prefillTps":null,"source":"measured"},"params":null,"notes":["quality 68.6% (tau2/gaia/gdpval mean)"]},
    "google/gemma4-12b-it": {"name":"gemma4-12b-it","quant":"bf16","filesize":"26gb","filesizeGb":26,"hardware":[{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve google/gemma4-12b-it --served-model-name google/gemma4-12b-it --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 16 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 131072 --tool-call-parser gemma4 --reasoning-parser gemma4 --chat-template /vllm-workspace/examples/tool_chat_template_gemma4.jinja --default-chat-template-kwargs '{\"enable_thinking\": true}' --speculative-config '{\"model\":\"google/gemma-4-12B-it-assistant\",\"num_speculative_tokens\":4,\"rejection_sample_method\":\"standard\"}'"},"rank":11,"benchmarks":[{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":41.6,"decodeTps32k":34.4,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-18","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":33.4,"decodeTps32k":36.1,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":8.9,"decodeTps32k":7.8,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-18","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":41.6,"prefillTps":null,"source":"measured"},"params":null,"notes":["quality 63% (tau2/gaia/gdpval mean)","size estimated from parameter count"]},
    "google/gemma4-12b-it-qat-w4a16-ct": {"name":"gemma4-12b-it-qat-w4a16-ct","quant":"bf16","filesize":"26gb","filesizeGb":26,"hardware":[{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve google/gemma4-12b-it-qat-w4a16-ct --served-model-name google/gemma4-12b-it-qat-w4a16-ct --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 16 --max-num-batched-tokens 8192 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 131072 --tool-call-parser gemma4 --reasoning-parser gemma4 --chat-template /vllm-workspace/examples/tool_chat_template_gemma4.jinja --default-chat-template-kwargs '{\"enable_thinking\": true}'"},"rank":11,"benchmarks":[{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":116.4,"decodeTps32k":104.6,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":78.9,"decodeTps32k":82.2,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":25,"decodeTps32k":24,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-18","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":116.4,"prefillTps":null,"source":"measured"},"params":null,"notes":["quality 56.5% (tau2/gaia/gdpval mean)","size estimated from parameter count"]},
    "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4": {"name":"NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4","quant":"nvfp4","filesize":"328gb","filesizeGb":328.1,"hardware":[{"id":"gb300","label":"gb300","minMemoryGb":288,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4 --block-size 128 --trust-remote-code --kv-cache-dtype fp8 --enable-prefix-caching --gpu-memory-utilization 0.90 --kernel-config '{\"enable_flashinfer_autotune\":false}' --max-model-len 262144 --mamba-ssm-cache-dtype float32 --cpu-offload-gb 120 --cpu-offload-params mixer.experts"},"rank":11,"benchmarks":[{"hardwareId":"gb300","engine":"vllm","decodeTps":36.7,"decodeTps32k":37.2,"prefillTps":4592,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-18","notes":"vllm unknown"}],"expectSpeed":{"decodeTps":36.7,"prefillTps":4592,"source":"measured"},"params":"550B-A55B","notes":["quality 63.2% (tau2/gaia/gdpval mean)"]},
    "poolside/Laguna-XS-2.1-NVFP4": {"name":"Laguna-XS-2.1-NVFP4","quant":"nvfp4","filesize":"20gb","filesizeGb":20.1,"hardware":[{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve poolside/Laguna-XS-2.1-NVFP4 --served-model-name poolside/Laguna-XS-2.1-NVFP4 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 16 --async-scheduling --enable-auto-tool-choice --max-model-len 262144 --tool-call-parser poolside_v1 --reasoning-parser poolside_v1 --kv-cache-dtype fp8 --default-chat-template-kwargs '{\"enable_thinking\": true}'"},"rank":11,"benchmarks":[{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":144.5,"decodeTps32k":155.4,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-06","notes":"vllm unknown"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":87.1,"decodeTps32k":95.4,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-18","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":41.8,"decodeTps32k":40.6,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":144.5,"prefillTps":null,"source":"measured"},"params":null,"notes":["quality 3.5% (tau2/gaia/gdpval mean)"]},
    "prism-ml/Ternary-Bonsai-27B-AWQ-4bit": {"name":"Ternary-Bonsai-27B-AWQ-4bit","quant":"awq","filesize":"17gb","filesizeGb":17.4,"hardware":[{"id":"rtx5090_32","label":"rtx5090_32","minMemoryGb":32,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve prism-ml/Ternary-Bonsai-27B-AWQ-4bit --served-model-name prism-ml/Ternary-Bonsai-27B-AWQ-4bit --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --async-scheduling --enable-auto-tool-choice --language-model-only --quantization awq_marlin --max-model-len 131072 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --mamba-cache-mode align"},"rank":11,"benchmarks":[{"hardwareId":"rtx5090_32","engine":"vllm","decodeTps":83.8,"decodeTps32k":80.5,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-18","notes":"vllm unknown"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":82.6,"decodeTps32k":84.7,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-21","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":49.6,"decodeTps32k":45.2,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":13.1,"decodeTps32k":12,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-15","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":83.8,"prefillTps":null,"source":"measured"},"params":"27B","notes":["quality 61.6% (tau2/gaia/gdpval mean)"]},
    "exolabs/gemma-4-12b-it-8bit-text-dequant-bf16": {"name":"gemma-4-12b-it-8bit-text-dequant-bf16","quant":"mlx","filesize":"22gb","filesizeGb":22.2,"hardware":[{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve exolabs/gemma-4-12b-it-8bit-text-dequant-bf16 --served-model-name exolabs/gemma-4-12b-it-8bit-text-dequant-bf16 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 16 --enable-auto-tool-choice --async-scheduling --dtype bfloat16 --kv-cache-dtype bfloat16 --language-model-only --max-model-len 131072 --reasoning-parser gemma4 --tool-call-parser gemma4 --chat-template /vllm-workspace/examples/tool_chat_template_gemma4.jinja --default-chat-template-kwargs '{\"enable_thinking\": true}'"},"rank":12,"benchmarks":[{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":58.5,"decodeTps32k":65.8,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-21","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":34.6,"decodeTps32k":35.1,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":7.4,"decodeTps32k":7.3,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-15","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":58.5,"prefillTps":null,"source":"measured"},"params":null,"notes":["quality 56.4% (tau2/gaia/gdpval mean)"]},
    "exolabs/qwen3-5-9b-mlx-4bit-text-dequant-bf16": {"name":"qwen3-5-9b-mlx-4bit-text-dequant-bf16","quant":"mlx","filesize":"17gb","filesizeGb":16.7,"hardware":[{"id":"rtx5090_32","label":"rtx5090_32","minMemoryGb":32,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve exolabs/qwen3-5-9b-mlx-4bit-text-dequant-bf16 --served-model-name exolabs/qwen3-5-9b-mlx-4bit-text-dequant-bf16 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --enable-auto-tool-choice --async-scheduling --dtype bfloat16 --kv-cache-dtype bfloat16 --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --attention-backend triton_attn --mamba-cache-mode align"},"rank":12,"benchmarks":[{"hardwareId":"rtx5090_32","engine":"vllm","decodeTps":54.7,"decodeTps32k":52.9,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":52,"decodeTps32k":51.4,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":12.6,"decodeTps32k":12.1,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":54.7,"prefillTps":null,"source":"measured"},"params":null,"notes":["quality 59.5% (tau2/gaia/gdpval mean)"]},
    "nvidia/diffusiongemma-26B-A4B-it-NVFP4": {"name":"diffusiongemma-26B-A4B-it-NVFP4","quant":"nvfp4","filesize":"18gb","filesizeGb":17.5,"hardware":[{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve nvidia/diffusiongemma-26B-A4B-it-NVFP4 --served-model-name nvidia/diffusiongemma-26B-A4B-it-NVFP4 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 16 --max-num-batched-tokens 8192 --trust-remote-code --enable-auto-tool-choice --max-model-len 262144 --attention-backend TRITON_ATTN --kv-cache-dtype bfloat16 --tool-call-parser gemma4 --reasoning-parser gemma4 --revision ec4ff3df205028f4e81c954c2227f9312b3ec2ea --tokenizer-revision ec4ff3df205028f4e81c954c2227f9312b3ec2ea --override-generation-config '{\"max_new_tokens\": null}' --default-chat-template-kwargs '{\"enable_thinking\": true}'"},"rank":12,"benchmarks":[{"hardwareId":"gb10_121","engine":"vllm","decodeTps":97.5,"decodeTps32k":75.1,"prefillTps":null,"ttftMs":null,"contextTokens":33280,"measuredAt":"2026-07-20","notes":"vllm 0.25.0"}],"expectSpeed":{"decodeTps":97.5,"prefillTps":null,"source":"measured"},"params":"26B-A4B","notes":["quality 53.9% (tau2/gaia/gdpval mean)"]},
    "exolabs/gpt-oss-20b-mxfp4-q8-dequant-bf16": {"name":"gpt-oss-20b-mxfp4-q8-dequant-bf16","quant":"bf16","filesize":"39gb","filesizeGb":39,"hardware":[{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve exolabs/gpt-oss-20b-mxfp4-q8-dequant-bf16 --served-model-name exolabs/gpt-oss-20b-mxfp4-q8-dequant-bf16 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 16 --enable-auto-tool-choice --async-scheduling --dtype bfloat16 --language-model-only --max-model-len 131072 --tool-call-parser openai --kv-cache-dtype bfloat16 --moe-backend triton --attention-backend triton_attn --max-cudagraph-capture-size 2048 --max-num-batched-tokens 8192 --stream-interval 20"},"rank":13,"benchmarks":[{"hardwareId":"gb10_121","engine":"vllm","decodeTps":28,"decodeTps32k":26.9,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-15","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":28,"prefillTps":null,"source":"measured"},"params":null,"notes":["quality 55.9% (tau2/gaia/gdpval mean)"]},
    "exolabs/ornith-1-0-35b-4bit-text-dequant-bf16": {"name":"ornith-1-0-35b-4bit-text-dequant-bf16","quant":"mlx","filesize":"65gb","filesizeGb":64.6,"hardware":[{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve exolabs/ornith-1-0-35b-4bit-text-dequant-bf16 --served-model-name exolabs/ornith-1-0-35b-4bit-text-dequant-bf16 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --max-num-batched-tokens 8192 --enable-auto-tool-choice --async-scheduling --dtype bfloat16 --kv-cache-dtype bfloat16 --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --moe-backend triton --attention-backend triton_attn"},"rank":13,"benchmarks":[{"hardwareId":"gb10_121","engine":"vllm","decodeTps":30.7,"decodeTps32k":28.9,"prefillTps":4408.1,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-18","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":30.7,"prefillTps":4408.1,"source":"measured"},"params":null,"notes":["quality 47.6% (tau2/gaia/gdpval mean)"]},
    "prism-ml/Bonsai-27B-AWQ-4bit": {"name":"Bonsai-27B-AWQ-4bit","quant":"awq","filesize":"17gb","filesizeGb":17.4,"hardware":[{"id":"rtx5090_32","label":"rtx5090_32","minMemoryGb":32,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve prism-ml/Bonsai-27B-AWQ-4bit --served-model-name prism-ml/Bonsai-27B-AWQ-4bit --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --async-scheduling --enable-auto-tool-choice --language-model-only --quantization awq_marlin --max-model-len 131072 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --mamba-cache-mode align"},"rank":13,"benchmarks":[{"hardwareId":"rtx5090_32","engine":"vllm","decodeTps":84,"decodeTps32k":80.6,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-18","notes":"vllm unknown"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":82.6,"decodeTps32k":84.7,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-21","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":49.6,"decodeTps32k":45.3,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":13.1,"decodeTps32k":12.1,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-15","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":84,"prefillTps":null,"source":"measured"},"params":"27B","notes":["quality 45.7% (tau2/gaia/gdpval mean)"]},
    "Qwen/QwQ-32B": {"name":"QwQ-32B","quant":"bf16","filesize":"61gb","filesizeGb":61,"hardware":[{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve Qwen/QwQ-32B --served-model-name Qwen/QwQ-32B --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 16 --max-num-batched-tokens 8192 --enable-auto-tool-choice --async-scheduling --language-model-only --dtype bfloat16 --kv-cache-dtype bfloat16 --max-model-len 131072 --hf-overrides '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":4.0,\"original_max_position_embeddings\":32768}}' --tool-call-parser hermes --chat-template /vllm-workspace/examples/tool_chat_template_hermes.jinja"},"rank":14,"benchmarks":[{"hardwareId":"gb10_121","engine":"vllm","decodeTps":3.5,"decodeTps32k":3.2,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-15","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":3.5,"prefillTps":null,"source":"measured"},"params":"32B","notes":["quality 21.8% (tau2/gaia/gdpval mean)"]},
    "exolabs/gemma-4-12b-coder-fable5-composer2-5-8bit-text-dequant-bf16": {"name":"gemma-4-12b-coder-fable5-composer2-5-8bit-text-dequant-bf16","quant":"mlx","filesize":"22gb","filesizeGb":22.2,"hardware":[{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve exolabs/gemma-4-12b-coder-fable5-composer2-5-8bit-text-dequant-bf16 --served-model-name exolabs/gemma-4-12b-coder-fable5-composer2-5-8bit-text-dequant-bf16 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 16 --enable-auto-tool-choice --async-scheduling --dtype bfloat16 --kv-cache-dtype bfloat16 --language-model-only --max-model-len 131072 --reasoning-parser gemma4 --tool-call-parser gemma4 --chat-template /vllm-workspace/examples/tool_chat_template_gemma4.jinja --default-chat-template-kwargs '{\"enable_thinking\": true}'"},"rank":14,"benchmarks":[{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":58.5,"decodeTps32k":67.1,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-21","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":34.6,"decodeTps32k":33.6,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":7.4,"decodeTps32k":7.3,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-15","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":58.5,"prefillTps":null,"source":"measured"},"params":null,"notes":["quality 9.1% (tau2/gaia/gdpval mean)"]},
    "Qwen/Qwen3-32B": {"name":"Qwen3-32B","quant":"bf16","filesize":"61gb","filesizeGb":61,"hardware":[{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve Qwen/Qwen3-32B --served-model-name Qwen/Qwen3-32B --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 64 --gpu-memory-utilization 0.88 --block-size 16 --max-num-batched-tokens 8192 --async-scheduling --enable-auto-tool-choice --max-model-len 65536 --tool-call-parser qwen3_coder --reasoning-parser qwen3"},"rank":15,"benchmarks":[{"hardwareId":"gb10_121","engine":"vllm","decodeTps":3.5,"decodeTps32k":3.2,"prefillTps":null,"ttftMs":null,"contextTokens":40960,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":3.5,"prefillTps":null,"source":"measured"},"params":"32B","notes":["quality 15% (tau2/gaia/gdpval mean)"]},
    "Qwen/Qwen2.5-Coder-32B-Instruct": {"name":"Qwen2.5-Coder-32B-Instruct","quant":"bf16","filesize":"61gb","filesizeGb":61,"hardware":[{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve Qwen/Qwen2.5-Coder-32B-Instruct --served-model-name Qwen/Qwen2.5-Coder-32B-Instruct --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 16 --max-num-batched-tokens 8192 --enable-auto-tool-choice --async-scheduling --language-model-only --dtype bfloat16 --kv-cache-dtype bfloat16 --max-model-len 131072 --hf-overrides '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":4.0,\"original_max_position_embeddings\":32768}}' --tool-call-parser hermes"},"rank":16,"benchmarks":[{"hardwareId":"gb10_121","engine":"vllm","decodeTps":3.5,"decodeTps32k":3.2,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-15","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":3.5,"prefillTps":null,"source":"measured"},"params":"32B","notes":["quality 1.5% (tau2/gaia/gdpval mean)"]},
    "0xSero/GLM-5.2-504B": {"name":"GLM-5.2-504B","quant":"bf16","filesize":"288gb","filesizeGb":287.6,"hardware":[{"id":"gb300","label":"gb300","minMemoryGb":288,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"gb300x2","label":"gb300x2","minMemoryGb":576,"gpuCount":2,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve 0xSero/GLM-5.2-504B --trust-remote-code --kv-cache-dtype fp8 --enable-prefix-caching --gpu-memory-utilization 0.90 --kernel-config '{\"enable_flashinfer_autotune\":false}' --max-model-len 1048576 --cpu-offload-gb 160 --cpu-offload-params w13_weight w2_weight"},"rank":17,"benchmarks":[{"hardwareId":"gb300x2","engine":"vllm","decodeTps":41.6,"decodeTps32k":42.3,"prefillTps":null,"ttftMs":null,"contextTokens":1048448,"measuredAt":"2026-07-18","notes":"vllm 0.25.0"},{"hardwareId":"gb300","engine":"vllm","decodeTps":30.4,"decodeTps32k":30.7,"prefillTps":null,"ttftMs":null,"contextTokens":1048448,"measuredAt":"2026-07-18","notes":"vllm 0.25.0"}],"expectSpeed":{"decodeTps":41.6,"prefillTps":null,"source":"measured"},"params":"504B","notes":[]},
    "LiquidAI/LFM2.5-8B-A1B-MLX-4bit": {"name":"LFM2.5-8B-A1B-MLX-4bit","quant":"mlx","filesize":"5gb","filesizeGb":4.5,"hardware":[{"id":"m4_max_36_32c","label":"m4_max_36_32c","minMemoryGb":36,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m4_pro_64_20c","label":"m4_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_pro_64_20c","label":"m5_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_96_60c","label":"m3_ultra_96_60c","minMemoryGb":96,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_max_128_40c","label":"m5_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m4_max_128_40c","label":"m4_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_512_80c","label":"m3_ultra_512_80c","minMemoryGb":512,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"mlx":"mlx_lm.server --model LiquidAI/LFM2.5-8B-A1B-MLX-4bit"},"rank":17,"benchmarks":[{"hardwareId":"m5_max_128_40c","engine":"mlx","decodeTps":223,"decodeTps32k":175.3,"prefillTps":null,"ttftMs":null,"contextTokens":123904,"measuredAt":"2026-07-04","notes":"mlx 0.31.3"},{"hardwareId":"m4_max_128_40c","engine":"mlx","decodeTps":196.1,"decodeTps32k":160.1,"prefillTps":null,"ttftMs":null,"contextTokens":127872,"measuredAt":"2026-07-09","notes":"mlx 0.31.3"},{"hardwareId":"m3_ultra_512_80c","engine":"mlx","decodeTps":177.8,"decodeTps32k":155.6,"prefillTps":null,"ttftMs":null,"contextTokens":127872,"measuredAt":"2026-07-06","notes":"mlx 0.31.3"},{"hardwareId":"m3_ultra_96_60c","engine":"mlx","decodeTps":171.6,"decodeTps32k":138.3,"prefillTps":null,"ttftMs":null,"contextTokens":123904,"measuredAt":"2026-07-04","notes":"mlx 0.31.3"},{"hardwareId":"m5_pro_64_20c","engine":"mlx","decodeTps":152.5,"decodeTps32k":113.9,"prefillTps":null,"ttftMs":null,"contextTokens":123904,"measuredAt":"2026-07-04","notes":"mlx unknown"},{"hardwareId":"m4_pro_64_20c","engine":"mlx","decodeTps":127.4,"decodeTps32k":91.2,"prefillTps":null,"ttftMs":null,"contextTokens":123904,"measuredAt":"2026-07-04","notes":"mlx 0.31.3"},{"hardwareId":"m4_max_36_32c","engine":"mlx","decodeTps":null,"decodeTps32k":null,"prefillTps":null,"ttftMs":null,"contextTokens":null,"measuredAt":null,"notes":"mlx 0.31.3"}],"expectSpeed":{"decodeTps":223,"prefillTps":null,"source":"measured"},"params":"8B-A1B","notes":[]},
    "LiquidAI/LFM2.5-8B-A1B-MLX-8bit": {"name":"LFM2.5-8B-A1B-MLX-8bit","quant":"mlx","filesize":"8gb","filesizeGb":8.4,"hardware":[{"id":"m4_max_36_32c","label":"m4_max_36_32c","minMemoryGb":36,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m4_pro_64_20c","label":"m4_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_pro_64_20c","label":"m5_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_96_60c","label":"m3_ultra_96_60c","minMemoryGb":96,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_max_128_40c","label":"m5_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m4_max_128_40c","label":"m4_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_512_80c","label":"m3_ultra_512_80c","minMemoryGb":512,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"mlx":"mlx_lm.server --model LiquidAI/LFM2.5-8B-A1B-MLX-8bit"},"rank":17,"benchmarks":[{"hardwareId":"m5_max_128_40c","engine":"mlx","decodeTps":171.1,"decodeTps32k":142.1,"prefillTps":null,"ttftMs":null,"contextTokens":123904,"measuredAt":"2026-07-04","notes":"mlx 0.31.3"},{"hardwareId":"m3_ultra_512_80c","engine":"mlx","decodeTps":153.8,"decodeTps32k":136.6,"prefillTps":null,"ttftMs":null,"contextTokens":123904,"measuredAt":"2026-07-05","notes":"mlx unknown"},{"hardwareId":"m4_max_128_40c","engine":"mlx","decodeTps":150.1,"decodeTps32k":127,"prefillTps":null,"ttftMs":null,"contextTokens":127872,"measuredAt":"2026-07-09","notes":"mlx 0.31.3"},{"hardwareId":"m3_ultra_96_60c","engine":"mlx","decodeTps":145.2,"decodeTps32k":120.4,"prefillTps":null,"ttftMs":null,"contextTokens":123904,"measuredAt":"2026-07-04","notes":"mlx 0.31.3"},{"hardwareId":"m5_pro_64_20c","engine":"mlx","decodeTps":109.6,"decodeTps32k":88,"prefillTps":null,"ttftMs":null,"contextTokens":123904,"measuredAt":"2026-07-04","notes":"mlx unknown"},{"hardwareId":"m4_max_36_32c","engine":"mlx","decodeTps":null,"decodeTps32k":null,"prefillTps":null,"ttftMs":null,"contextTokens":null,"measuredAt":null,"notes":"mlx 0.31.3"},{"hardwareId":"m4_pro_64_20c","engine":"mlx","decodeTps":null,"decodeTps32k":null,"prefillTps":null,"ttftMs":null,"contextTokens":null,"measuredAt":null,"notes":"mlx 0.31.3"}],"expectSpeed":{"decodeTps":171.1,"prefillTps":null,"source":"measured"},"params":"8B-A1B","notes":[]},
    "antirez/deepseek-v4-gguf": {"name":"deepseek-v4-gguf","quant":"gguf","filesize":"2271gb","filesizeGb":2271.3,"hardware":[{"id":"m5_max_128_40c","label":"m5_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"llamacpp":"llama-server -m antirez/deepseek-v4-gguf"},"rank":17,"benchmarks":[{"hardwareId":"m5_max_128_40c","engine":"llamacpp","decodeTps":25.3,"decodeTps32k":21,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-21","notes":"llamacpp unknown"}],"expectSpeed":{"decodeTps":25.3,"prefillTps":null,"source":"measured"},"params":null,"notes":[]},
    "cyankiwi/Qwen3.5-9B-AWQ-4bit-bs1024": {"name":"Qwen3.5-9B-AWQ-4bit-bs1024","quant":"awq","filesize":"6gb","filesizeGb":6,"hardware":[{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve cyankiwi/Qwen3.5-9B-AWQ-4bit-bs1024"},"rank":17,"benchmarks":[{"hardwareId":"gb10_121","engine":"vllm","decodeTps":33.2,"decodeTps32k":30.3,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-21","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":33.2,"prefillTps":null,"source":"measured"},"params":"9B","notes":["size estimated from parameter count"]},
    "google/gemma-4-26B-A4B-it": {"name":"gemma-4-26B-A4B-it","quant":"bf16","filesize":"48gb","filesizeGb":48.1,"hardware":[{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve google/gemma-4-26B-A4B-it"},"rank":17,"benchmarks":[{"hardwareId":"gb10_121","engine":"vllm","decodeTps":25.2,"decodeTps32k":24.1,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-14","notes":"vllm 0.1.dev16488+g129a28dc2"}],"expectSpeed":{"decodeTps":25.2,"prefillTps":null,"source":"measured"},"params":"26B-A4B","notes":[]},
    "lovedheart/Qwen3.5-9B-FP8-bs1024": {"name":"Qwen3.5-9B-FP8-bs1024","quant":"fp8","filesize":"10gb","filesizeGb":10,"hardware":[{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve lovedheart/Qwen3.5-9B-FP8-bs1024"},"rank":17,"benchmarks":[{"hardwareId":"gb10_121","engine":"vllm","decodeTps":22.2,"decodeTps32k":20.8,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":22.2,"prefillTps":null,"source":"measured"},"params":"9B","notes":["size estimated from parameter count"]},
    "mlx-community/Agents-A1-8bit": {"name":"Agents-A1-8bit","quant":"mlx","filesize":"35gb","filesizeGb":35.1,"hardware":[{"id":"m4_pro_64_20c","label":"m4_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_pro_64_20c","label":"m5_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_96_60c","label":"m3_ultra_96_60c","minMemoryGb":96,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_max_128_40c","label":"m5_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_512_80c","label":"m3_ultra_512_80c","minMemoryGb":512,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"mlx":"mlx_lm.server --model mlx-community/Agents-A1-8bit"},"rank":17,"benchmarks":[{"hardwareId":"m5_max_128_40c","engine":"mlx","decodeTps":86.3,"decodeTps32k":75.3,"prefillTps":null,"ttftMs":null,"contextTokens":65536,"measuredAt":"2026-07-18","notes":"mlx 0.31.3"},{"hardwareId":"m3_ultra_96_60c","engine":"mlx","decodeTps":71.1,"decodeTps32k":61.4,"prefillTps":null,"ttftMs":null,"contextTokens":65536,"measuredAt":"2026-07-18","notes":"mlx 0.31.3"},{"hardwareId":"m3_ultra_512_80c","engine":"mlx","decodeTps":70.9,"decodeTps32k":62.9,"prefillTps":null,"ttftMs":null,"contextTokens":65536,"measuredAt":"2026-07-18","notes":"mlx 0.31.3"},{"hardwareId":"m5_pro_64_20c","engine":"mlx","decodeTps":56.4,"decodeTps32k":48.7,"prefillTps":null,"ttftMs":null,"contextTokens":65536,"measuredAt":"2026-07-18","notes":"mlx 0.31.3"},{"hardwareId":"m4_pro_64_20c","engine":"mlx","decodeTps":48.4,"decodeTps32k":41.1,"prefillTps":null,"ttftMs":null,"contextTokens":65536,"measuredAt":"2026-07-18","notes":"mlx 0.31.3"}],"expectSpeed":{"decodeTps":86.3,"prefillTps":null,"source":"measured"},"params":null,"notes":[]},
    "mlx-community/NVIDIA-Nemotron-3-Nano-30B-A3B-MLX-8Bit": {"name":"NVIDIA-Nemotron-3-Nano-30B-A3B-MLX-8Bit","quant":"mlx","filesize":"31gb","filesizeGb":31.3,"hardware":[{"id":"m4_pro_64_20c","label":"m4_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_pro_64_20c","label":"m5_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_96_60c","label":"m3_ultra_96_60c","minMemoryGb":96,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_max_128_40c","label":"m5_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m4_max_128_40c","label":"m4_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_512_80c","label":"m3_ultra_512_80c","minMemoryGb":512,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"mlx":"mlx_lm.server --model mlx-community/NVIDIA-Nemotron-3-Nano-30B-A3B-MLX-8Bit"},"rank":17,"benchmarks":[{"hardwareId":"m5_max_128_40c","engine":"mlx","decodeTps":100.5,"decodeTps32k":91.8,"prefillTps":null,"ttftMs":null,"contextTokens":258047,"measuredAt":"2026-07-04","notes":"mlx 0.31.3"},{"hardwareId":"m3_ultra_512_80c","engine":"mlx","decodeTps":95.1,"decodeTps32k":88.1,"prefillTps":null,"ttftMs":null,"contextTokens":258047,"measuredAt":"2026-07-05","notes":"mlx unknown"},{"hardwareId":"m3_ultra_96_60c","engine":"mlx","decodeTps":91.7,"decodeTps32k":83.8,"prefillTps":null,"ttftMs":null,"contextTokens":258047,"measuredAt":"2026-07-04","notes":"mlx 0.31.3"},{"hardwareId":"m4_max_128_40c","engine":"mlx","decodeTps":89.4,"decodeTps32k":79.9,"prefillTps":null,"ttftMs":null,"contextTokens":262015,"measuredAt":"2026-07-09","notes":"mlx 0.31.3"},{"hardwareId":"m5_pro_64_20c","engine":"mlx","decodeTps":59.7,"decodeTps32k":54.6,"prefillTps":null,"ttftMs":null,"contextTokens":258047,"measuredAt":"2026-07-04","notes":"mlx unknown"},{"hardwareId":"m4_pro_64_20c","engine":"mlx","decodeTps":51.7,"decodeTps32k":46.3,"prefillTps":null,"ttftMs":null,"contextTokens":258047,"measuredAt":"2026-07-04","notes":"mlx 0.31.3"}],"expectSpeed":{"decodeTps":100.5,"prefillTps":null,"source":"measured"},"params":"30B-A3B","notes":[]},
    "mlx-community/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4": {"name":"NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4","quant":"nvfp4","filesize":"18gb","filesizeGb":18,"hardware":[{"id":"m4_max_36_32c","label":"m4_max_36_32c","minMemoryGb":36,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m4_pro_64_20c","label":"m4_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_pro_64_20c","label":"m5_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_96_60c","label":"m3_ultra_96_60c","minMemoryGb":96,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_max_128_40c","label":"m5_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m4_max_128_40c","label":"m4_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_512_80c","label":"m3_ultra_512_80c","minMemoryGb":512,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"mlx":"mlx_lm.server --model mlx-community/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4"},"rank":17,"benchmarks":[{"hardwareId":"m5_max_128_40c","engine":"mlx","decodeTps":110.4,"decodeTps32k":100.2,"prefillTps":null,"ttftMs":null,"contextTokens":258047,"measuredAt":"2026-07-04","notes":"mlx 0.31.3"},{"hardwareId":"m3_ultra_512_80c","engine":"mlx","decodeTps":103,"decodeTps32k":94.8,"prefillTps":null,"ttftMs":null,"contextTokens":258047,"measuredAt":"2026-07-05","notes":"mlx unknown"},{"hardwareId":"m3_ultra_96_60c","engine":"mlx","decodeTps":98.3,"decodeTps32k":89.7,"prefillTps":null,"ttftMs":null,"contextTokens":258047,"measuredAt":"2026-07-04","notes":"mlx 0.31.3"},{"hardwareId":"m4_max_128_40c","engine":"mlx","decodeTps":98,"decodeTps32k":87.1,"prefillTps":null,"ttftMs":null,"contextTokens":262015,"measuredAt":"2026-07-09","notes":"mlx 0.31.3"},{"hardwareId":"m5_pro_64_20c","engine":"mlx","decodeTps":67.6,"decodeTps32k":60.7,"prefillTps":null,"ttftMs":null,"contextTokens":258047,"measuredAt":"2026-07-04","notes":"mlx unknown"},{"hardwareId":"m4_pro_64_20c","engine":"mlx","decodeTps":58.1,"decodeTps32k":51.6,"prefillTps":null,"ttftMs":null,"contextTokens":258047,"measuredAt":"2026-07-04","notes":"mlx 0.31.3"},{"hardwareId":"m4_max_36_32c","engine":"mlx","decodeTps":null,"decodeTps32k":null,"prefillTps":null,"ttftMs":null,"contextTokens":null,"measuredAt":null,"notes":"mlx 0.31.3"}],"expectSpeed":{"decodeTps":110.4,"prefillTps":null,"source":"measured"},"params":"30B-A3B","notes":[]},
    "mlx-community/NVIDIA-Nemotron-3-Super-120B-A12B-4bit": {"name":"NVIDIA-Nemotron-3-Super-120B-A12B-4bit","quant":"mlx","filesize":"63gb","filesizeGb":63.3,"hardware":[{"id":"m5_max_128_40c","label":"m5_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_512_80c","label":"m3_ultra_512_80c","minMemoryGb":512,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"mlx":"mlx_lm.server --model mlx-community/NVIDIA-Nemotron-3-Super-120B-A12B-4bit"},"rank":17,"benchmarks":[{"hardwareId":"m5_max_128_40c","engine":"mlx","decodeTps":50.9,"decodeTps32k":48.3,"prefillTps":31.2,"ttftMs":null,"contextTokens":258047,"measuredAt":"2026-07-04","notes":"mlx 0.31.3"},{"hardwareId":"m3_ultra_512_80c","engine":"mlx","decodeTps":50.2,"decodeTps32k":47.7,"prefillTps":20.4,"ttftMs":null,"contextTokens":258047,"measuredAt":"2026-07-05","notes":"mlx unknown"}],"expectSpeed":{"decodeTps":50.9,"prefillTps":31.2,"source":"measured"},"params":"120B-A12B","notes":[]},
    "mlx-community/Nemotron-3-Ultra-550B-A55B": {"name":"Nemotron-3-Ultra-550B-A55B","quant":"mlx","filesize":"336gb","filesizeGb":336,"hardware":[{"id":"m3_ultra_512_80c","label":"m3_ultra_512_80c","minMemoryGb":512,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"mlx":"mlx_lm.server --model mlx-community/Nemotron-3-Ultra-550B-A55B"},"rank":17,"benchmarks":[{"hardwareId":"m3_ultra_512_80c","engine":"mlx","decodeTps":98.6,"decodeTps32k":81,"prefillTps":44.7,"ttftMs":null,"contextTokens":126976,"measuredAt":"2026-07-05","notes":"mlx unknown"}],"expectSpeed":{"decodeTps":98.6,"prefillTps":44.7,"source":"measured"},"params":"550B-A55B","notes":[]},
    "mlx-community/Qwen3.5-122B-A10B-4bit": {"name":"Qwen3.5-122B-A10B-4bit","quant":"mlx","filesize":"65gb","filesizeGb":64.8,"hardware":[{"id":"m3_ultra_96_60c","label":"m3_ultra_96_60c","minMemoryGb":96,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_max_128_40c","label":"m5_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_512_80c","label":"m3_ultra_512_80c","minMemoryGb":512,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"mlx":"mlx_lm.server --model mlx-community/Qwen3.5-122B-A10B-4bit"},"rank":17,"benchmarks":[{"hardwareId":"m5_max_128_40c","engine":"mlx","decodeTps":55.9,"decodeTps32k":48.4,"prefillTps":null,"ttftMs":null,"contextTokens":258047,"measuredAt":"2026-07-04","notes":"mlx 0.31.3"},{"hardwareId":"m3_ultra_512_80c","engine":"mlx","decodeTps":51,"decodeTps32k":44.9,"prefillTps":null,"ttftMs":null,"contextTokens":258047,"measuredAt":"2026-07-05","notes":"mlx 0.31.3"},{"hardwareId":"m3_ultra_96_60c","engine":"mlx","decodeTps":50.2,"decodeTps32k":42.3,"prefillTps":258.9,"ttftMs":null,"contextTokens":69632,"measuredAt":"2026-07-05","notes":"mlx 0.31.3"}],"expectSpeed":{"decodeTps":55.9,"prefillTps":258.9,"source":"measured"},"params":"122B-A10B","notes":[]},
    "mlx-community/Qwen3.5-9B-8bit": {"name":"Qwen3.5-9B-8bit","quant":"mlx","filesize":"10gb","filesizeGb":9.7,"hardware":[{"id":"m4_max_36_32c","label":"m4_max_36_32c","minMemoryGb":36,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m4_pro_64_20c","label":"m4_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_pro_64_20c","label":"m5_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_96_60c","label":"m3_ultra_96_60c","minMemoryGb":96,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_max_128_40c","label":"m5_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m4_max_128_40c","label":"m4_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_512_80c","label":"m3_ultra_512_80c","minMemoryGb":512,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"mlx":"mlx_lm.server --model mlx-community/Qwen3.5-9B-8bit"},"rank":17,"benchmarks":[{"hardwareId":"m3_ultra_96_60c","engine":"mlx","decodeTps":61.5,"decodeTps32k":54.2,"prefillTps":null,"ttftMs":null,"contextTokens":258047,"measuredAt":"2026-07-04","notes":"mlx 0.31.3"},{"hardwareId":"m3_ultra_512_80c","engine":"mlx","decodeTps":59.9,"decodeTps32k":55,"prefillTps":null,"ttftMs":null,"contextTokens":258047,"measuredAt":"2026-07-05","notes":"mlx unknown"},{"hardwareId":"m5_max_128_40c","engine":"mlx","decodeTps":54.9,"decodeTps32k":50.2,"prefillTps":null,"ttftMs":null,"contextTokens":258047,"measuredAt":"2026-07-04","notes":"mlx 0.31.3"},{"hardwareId":"m4_max_128_40c","engine":"mlx","decodeTps":48.3,"decodeTps32k":44.1,"prefillTps":null,"ttftMs":null,"contextTokens":262015,"measuredAt":"2026-07-09","notes":"mlx 0.31.3"},{"hardwareId":"m5_pro_64_20c","engine":"mlx","decodeTps":30,"decodeTps32k":27.4,"prefillTps":null,"ttftMs":null,"contextTokens":258047,"measuredAt":"2026-07-04","notes":"mlx unknown"},{"hardwareId":"m4_pro_64_20c","engine":"mlx","decodeTps":26.2,"decodeTps32k":23.7,"prefillTps":null,"ttftMs":null,"contextTokens":258047,"measuredAt":"2026-07-04","notes":"mlx 0.31.3"},{"hardwareId":"m4_max_36_32c","engine":"mlx","decodeTps":null,"decodeTps32k":null,"prefillTps":null,"ttftMs":null,"contextTokens":null,"measuredAt":null,"notes":"mlx 0.31.3"}],"expectSpeed":{"decodeTps":61.5,"prefillTps":null,"source":"measured"},"params":"9B","notes":[]},
    "mlx-community/Qwen3.5-9B-MLX-4bit": {"name":"Qwen3.5-9B-MLX-4bit","quant":"mlx","filesize":"6gb","filesizeGb":5.5,"hardware":[{"id":"m4_max_36_32c","label":"m4_max_36_32c","minMemoryGb":36,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m4_pro_64_20c","label":"m4_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_pro_64_20c","label":"m5_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_96_60c","label":"m3_ultra_96_60c","minMemoryGb":96,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_max_128_40c","label":"m5_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m4_max_128_40c","label":"m4_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_512_80c","label":"m3_ultra_512_80c","minMemoryGb":512,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"mlx":"mlx_lm.server --model mlx-community/Qwen3.5-9B-MLX-4bit"},"rank":17,"benchmarks":[{"hardwareId":"m3_ultra_512_80c","engine":"mlx","decodeTps":94.1,"decodeTps32k":81.9,"prefillTps":null,"ttftMs":null,"contextTokens":65536,"measuredAt":"2026-07-18","notes":"mlx 0.31.3"},{"hardwareId":"m3_ultra_96_60c","engine":"mlx","decodeTps":90.9,"decodeTps32k":76.1,"prefillTps":null,"ttftMs":null,"contextTokens":65536,"measuredAt":"2026-07-18","notes":"mlx 0.31.3"},{"hardwareId":"m5_max_128_40c","engine":"mlx","decodeTps":89.5,"decodeTps32k":76,"prefillTps":null,"ttftMs":null,"contextTokens":65536,"measuredAt":"2026-07-18","notes":"mlx 0.31.3"},{"hardwareId":"m4_max_128_40c","engine":"mlx","decodeTps":74.8,"decodeTps32k":64,"prefillTps":null,"ttftMs":null,"contextTokens":65536,"measuredAt":"2026-07-19","notes":"mlx 0.31.3"},{"hardwareId":"m4_max_36_32c","engine":"mlx","decodeTps":61.4,"decodeTps32k":53.6,"prefillTps":null,"ttftMs":null,"contextTokens":65536,"measuredAt":"2026-07-18","notes":"mlx 0.31.3"},{"hardwareId":"m5_pro_64_20c","engine":"mlx","decodeTps":51.3,"decodeTps32k":44.8,"prefillTps":null,"ttftMs":null,"contextTokens":65536,"measuredAt":"2026-07-18","notes":"mlx 0.31.3"},{"hardwareId":"m4_pro_64_20c","engine":"mlx","decodeTps":44.5,"decodeTps32k":37.5,"prefillTps":null,"ttftMs":null,"contextTokens":65536,"measuredAt":"2026-07-18","notes":"mlx 0.31.3"}],"expectSpeed":{"decodeTps":94.1,"prefillTps":null,"source":"measured"},"params":"9B","notes":[]},
    "mlx-community/Qwen3.5-9B-OptiQ-4bit": {"name":"Qwen3.5-9B-OptiQ-4bit","quant":"mlx","filesize":"8gb","filesizeGb":7.6,"hardware":[{"id":"m4_max_36_32c","label":"m4_max_36_32c","minMemoryGb":36,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m4_pro_64_20c","label":"m4_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_pro_64_20c","label":"m5_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_96_60c","label":"m3_ultra_96_60c","minMemoryGb":96,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_max_128_40c","label":"m5_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m4_max_128_40c","label":"m4_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_512_80c","label":"m3_ultra_512_80c","minMemoryGb":512,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"mlx":"mlx_lm.server --model mlx-community/Qwen3.5-9B-OptiQ-4bit"},"rank":17,"benchmarks":[{"hardwareId":"m3_ultra_512_80c","engine":"mlx","decodeTps":76,"decodeTps32k":67.9,"prefillTps":null,"ttftMs":null,"contextTokens":258047,"measuredAt":"2026-07-05","notes":"mlx unknown"},{"hardwareId":"m3_ultra_96_60c","engine":"mlx","decodeTps":75.9,"decodeTps32k":65.3,"prefillTps":null,"ttftMs":null,"contextTokens":258047,"measuredAt":"2026-07-04","notes":"mlx 0.31.3"},{"hardwareId":"m5_max_128_40c","engine":"mlx","decodeTps":72.1,"decodeTps32k":63.1,"prefillTps":null,"ttftMs":null,"contextTokens":262015,"measuredAt":"2026-07-05","notes":"mlx 0.31.3"},{"hardwareId":"m4_max_128_40c","engine":"mlx","decodeTps":61.9,"decodeTps32k":54.8,"prefillTps":null,"ttftMs":null,"contextTokens":262015,"measuredAt":"2026-07-09","notes":"mlx 0.31.3"},{"hardwareId":"m5_pro_64_20c","engine":"mlx","decodeTps":40,"decodeTps32k":35.8,"prefillTps":null,"ttftMs":null,"contextTokens":258047,"measuredAt":"2026-07-04","notes":"mlx unknown"},{"hardwareId":"m4_pro_64_20c","engine":"mlx","decodeTps":35,"decodeTps32k":30.8,"prefillTps":null,"ttftMs":null,"contextTokens":258047,"measuredAt":"2026-07-04","notes":"mlx 0.31.3"},{"hardwareId":"m4_max_36_32c","engine":"mlx","decodeTps":null,"decodeTps32k":null,"prefillTps":null,"ttftMs":null,"contextTokens":null,"measuredAt":null,"notes":"mlx 0.31.3"}],"expectSpeed":{"decodeTps":76,"prefillTps":null,"source":"measured"},"params":"9B","notes":[]},
    "mlx-community/Qwen3.6-27B-4bit": {"name":"Qwen3.6-27B-4bit","quant":"mlx","filesize":"15gb","filesizeGb":15,"hardware":[{"id":"m4_max_36_32c","label":"m4_max_36_32c","minMemoryGb":36,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m4_pro_64_20c","label":"m4_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_pro_64_20c","label":"m5_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_96_60c","label":"m3_ultra_96_60c","minMemoryGb":96,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_max_128_40c","label":"m5_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m4_max_128_40c","label":"m4_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_512_80c","label":"m3_ultra_512_80c","minMemoryGb":512,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"mlx":"mlx_lm.server --model mlx-community/Qwen3.6-27B-4bit"},"rank":17,"benchmarks":[{"hardwareId":"m3_ultra_96_60c","engine":"mlx","decodeTps":32.9,"decodeTps32k":28.8,"prefillTps":null,"ttftMs":null,"contextTokens":258047,"measuredAt":"2026-07-04","notes":"mlx 0.31.3"},{"hardwareId":"m3_ultra_512_80c","engine":"mlx","decodeTps":32.2,"decodeTps32k":28.7,"prefillTps":null,"ttftMs":null,"contextTokens":258047,"measuredAt":"2026-07-05","notes":"mlx 0.31.3"},{"hardwareId":"m5_max_128_40c","engine":"mlx","decodeTps":29.7,"decodeTps32k":27.2,"prefillTps":null,"ttftMs":null,"contextTokens":258047,"measuredAt":"2026-07-04","notes":"mlx unknown"},{"hardwareId":"m4_max_128_40c","engine":"mlx","decodeTps":25.8,"decodeTps32k":23.3,"prefillTps":509.1,"ttftMs":null,"contextTokens":32768,"measuredAt":"2026-07-09","notes":"mlx 0.31.3"},{"hardwareId":"m5_pro_64_20c","engine":"mlx","decodeTps":16.7,"decodeTps32k":15,"prefillTps":265.3,"ttftMs":null,"contextTokens":172032,"measuredAt":"2026-07-04","notes":"mlx 0.31.3"},{"hardwareId":"m4_pro_64_20c","engine":"mlx","decodeTps":14.1,"decodeTps32k":12.5,"prefillTps":107.7,"ttftMs":null,"contextTokens":172032,"measuredAt":"2026-07-04","notes":"mlx 0.31.3"},{"hardwareId":"m4_max_36_32c","engine":"mlx","decodeTps":null,"decodeTps32k":null,"prefillTps":null,"ttftMs":null,"contextTokens":null,"measuredAt":null,"notes":"mlx 0.31.3"}],"expectSpeed":{"decodeTps":32.9,"prefillTps":509.1,"source":"measured"},"params":"27B","notes":[]},
    "mlx-community/Qwen3.6-27B-8bit": {"name":"Qwen3.6-27B-8bit","quant":"mlx","filesize":"27gb","filesizeGb":27.5,"hardware":[{"id":"m4_pro_64_20c","label":"m4_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_pro_64_20c","label":"m5_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_96_60c","label":"m3_ultra_96_60c","minMemoryGb":96,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_max_128_40c","label":"m5_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m4_max_128_40c","label":"m4_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_512_80c","label":"m3_ultra_512_80c","minMemoryGb":512,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"mlx":"mlx_lm.server --model mlx-community/Qwen3.6-27B-8bit"},"rank":17,"benchmarks":[{"hardwareId":"m3_ultra_96_60c","engine":"mlx","decodeTps":20.8,"decodeTps32k":19,"prefillTps":null,"ttftMs":null,"contextTokens":225279,"measuredAt":"2026-07-05","notes":"mlx 0.31.3"},{"hardwareId":"m3_ultra_512_80c","engine":"mlx","decodeTps":20.2,"decodeTps32k":18.7,"prefillTps":null,"ttftMs":null,"contextTokens":258047,"measuredAt":"2026-07-05","notes":"mlx 0.31.3"},{"hardwareId":"m5_max_128_40c","engine":"mlx","decodeTps":17,"decodeTps32k":16,"prefillTps":null,"ttftMs":null,"contextTokens":258047,"measuredAt":"2026-07-04","notes":"mlx unknown"},{"hardwareId":"m4_max_128_40c","engine":"mlx","decodeTps":15,"decodeTps32k":14.1,"prefillTps":null,"ttftMs":null,"contextTokens":262015,"measuredAt":"2026-07-09","notes":"mlx 0.31.3"},{"hardwareId":"m5_pro_64_20c","engine":"mlx","decodeTps":9.4,"decodeTps32k":8.9,"prefillTps":171.6,"ttftMs":null,"contextTokens":110592,"measuredAt":"2026-07-04","notes":"mlx 0.31.3"},{"hardwareId":"m4_pro_64_20c","engine":"mlx","decodeTps":8.1,"decodeTps32k":6.9,"prefillTps":67.8,"ttftMs":null,"contextTokens":110592,"measuredAt":"2026-07-04","notes":"mlx 0.31.3"}],"expectSpeed":{"decodeTps":20.8,"prefillTps":171.6,"source":"measured"},"params":"27B","notes":[]},
    "mlx-community/Qwen3.6-27B-OptiQ-4bit": {"name":"Qwen3.6-27B-OptiQ-4bit","quant":"mlx","filesize":"19gb","filesizeGb":18.6,"hardware":[{"id":"m4_max_36_32c","label":"m4_max_36_32c","minMemoryGb":36,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m4_pro_64_20c","label":"m4_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_pro_64_20c","label":"m5_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_96_60c","label":"m3_ultra_96_60c","minMemoryGb":96,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_max_128_40c","label":"m5_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m4_max_128_40c","label":"m4_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"mlx":"mlx_lm.server --model mlx-community/Qwen3.6-27B-OptiQ-4bit"},"rank":17,"benchmarks":[{"hardwareId":"m3_ultra_96_60c","engine":"mlx","decodeTps":28.8,"decodeTps32k":25.6,"prefillTps":null,"ttftMs":null,"contextTokens":65536,"measuredAt":"2026-07-18","notes":"mlx 0.31.3"},{"hardwareId":"m5_max_128_40c","engine":"mlx","decodeTps":25.4,"decodeTps32k":23.7,"prefillTps":null,"ttftMs":null,"contextTokens":65536,"measuredAt":"2026-07-18","notes":"mlx 0.31.3"},{"hardwareId":"m4_max_128_40c","engine":"mlx","decodeTps":22.2,"decodeTps32k":20,"prefillTps":121.6,"ttftMs":null,"contextTokens":65536,"measuredAt":"2026-07-21","notes":"mlx 0.31.3"},{"hardwareId":"m4_max_36_32c","engine":"mlx","decodeTps":17.5,"decodeTps32k":15.8,"prefillTps":143.9,"ttftMs":null,"contextTokens":57344,"measuredAt":"2026-07-18","notes":"mlx 0.31.3"},{"hardwareId":"m5_pro_64_20c","engine":"mlx","decodeTps":14,"decodeTps32k":12.8,"prefillTps":null,"ttftMs":null,"contextTokens":65536,"measuredAt":"2026-07-18","notes":"mlx 0.31.3"},{"hardwareId":"m4_pro_64_20c","engine":"mlx","decodeTps":12.1,"decodeTps32k":10.9,"prefillTps":null,"ttftMs":null,"contextTokens":65536,"measuredAt":"2026-07-18","notes":"mlx 0.31.3"}],"expectSpeed":{"decodeTps":28.8,"prefillTps":143.9,"source":"measured"},"params":"27B","notes":[]},
    "mlx-community/Qwen3.6-35B-A3B-4bit": {"name":"Qwen3.6-35B-A3B-4bit","quant":"mlx","filesize":"19gb","filesizeGb":19,"hardware":[{"id":"m4_max_36_32c","label":"m4_max_36_32c","minMemoryGb":36,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m4_pro_64_20c","label":"m4_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_pro_64_20c","label":"m5_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_96_60c","label":"m3_ultra_96_60c","minMemoryGb":96,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_max_128_40c","label":"m5_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m4_max_128_40c","label":"m4_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_512_80c","label":"m3_ultra_512_80c","minMemoryGb":512,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"mlx":"mlx_lm.server --model mlx-community/Qwen3.6-35B-A3B-4bit"},"rank":17,"benchmarks":[{"hardwareId":"m5_max_128_40c","engine":"mlx","decodeTps":108.7,"decodeTps32k":96.2,"prefillTps":null,"ttftMs":null,"contextTokens":258047,"measuredAt":"2026-07-04","notes":"mlx unknown"},{"hardwareId":"m4_max_128_40c","engine":"mlx","decodeTps":91.4,"decodeTps32k":76.4,"prefillTps":311.4,"ttftMs":null,"contextTokens":123392,"measuredAt":"2026-07-09","notes":"mlx 0.31.3"},{"hardwareId":"m3_ultra_512_80c","engine":"mlx","decodeTps":86.7,"decodeTps32k":75,"prefillTps":null,"ttftMs":null,"contextTokens":258047,"measuredAt":"2026-07-05","notes":"mlx unknown"},{"hardwareId":"m3_ultra_96_60c","engine":"mlx","decodeTps":83.3,"decodeTps32k":69.7,"prefillTps":null,"ttftMs":null,"contextTokens":262015,"measuredAt":"2026-07-05","notes":"mlx 0.31.3"},{"hardwareId":"m5_pro_64_20c","engine":"mlx","decodeTps":79.2,"decodeTps32k":66.5,"prefillTps":null,"ttftMs":null,"contextTokens":258047,"measuredAt":"2026-07-04","notes":"mlx unknown"},{"hardwareId":"m4_max_36_32c","engine":"mlx","decodeTps":null,"decodeTps32k":null,"prefillTps":null,"ttftMs":null,"contextTokens":null,"measuredAt":null,"notes":"mlx 0.31.3"},{"hardwareId":"m4_pro_64_20c","engine":"mlx","decodeTps":null,"decodeTps32k":null,"prefillTps":null,"ttftMs":null,"contextTokens":null,"measuredAt":null,"notes":"mlx 0.31.3"}],"expectSpeed":{"decodeTps":108.7,"prefillTps":311.4,"source":"measured"},"params":"35B-A3B","notes":[]},
    "mlx-community/Qwen3.6-35B-A3B-8bit": {"name":"Qwen3.6-35B-A3B-8bit","quant":"mlx","filesize":"35gb","filesizeGb":35.1,"hardware":[{"id":"m4_pro_64_20c","label":"m4_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_pro_64_20c","label":"m5_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_96_60c","label":"m3_ultra_96_60c","minMemoryGb":96,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_max_128_40c","label":"m5_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m4_max_128_40c","label":"m4_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_512_80c","label":"m3_ultra_512_80c","minMemoryGb":512,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"mlx":"mlx_lm.server --model mlx-community/Qwen3.6-35B-A3B-8bit"},"rank":17,"benchmarks":[{"hardwareId":"m5_max_128_40c","engine":"mlx","decodeTps":86.6,"decodeTps32k":77.1,"prefillTps":null,"ttftMs":null,"contextTokens":258047,"measuredAt":"2026-07-04","notes":"mlx unknown"},{"hardwareId":"m4_max_128_40c","engine":"mlx","decodeTps":72.7,"decodeTps32k":63.2,"prefillTps":null,"ttftMs":null,"contextTokens":262015,"measuredAt":"2026-07-09","notes":"mlx 0.31.3"},{"hardwareId":"m3_ultra_512_80c","engine":"mlx","decodeTps":72,"decodeTps32k":64.1,"prefillTps":null,"ttftMs":null,"contextTokens":258047,"measuredAt":"2026-07-05","notes":"mlx 0.31.3"},{"hardwareId":"m3_ultra_96_60c","engine":"mlx","decodeTps":71,"decodeTps32k":61.4,"prefillTps":null,"ttftMs":null,"contextTokens":262015,"measuredAt":"2026-07-05","notes":"mlx 0.31.3"},{"hardwareId":"m5_pro_64_20c","engine":"mlx","decodeTps":56.7,"decodeTps32k":49.1,"prefillTps":380.2,"ttftMs":null,"contextTokens":163840,"measuredAt":"2026-07-05","notes":"mlx 0.31.3"},{"hardwareId":"m4_pro_64_20c","engine":"mlx","decodeTps":47,"decodeTps32k":40.1,"prefillTps":null,"ttftMs":null,"contextTokens":159744,"measuredAt":"2026-07-04","notes":"mlx 0.31.3"}],"expectSpeed":{"decodeTps":86.6,"prefillTps":380.2,"source":"measured"},"params":"35B-A3B","notes":[]},
    "mlx-community/gemma-4-12b-coder-fable5-composer2.5-8bit": {"name":"gemma-4-12b-coder-fable5-composer2.5-8bit","quant":"mlx","filesize":"12gb","filesizeGb":11.8,"hardware":[{"id":"m4_max_36_32c","label":"m4_max_36_32c","minMemoryGb":36,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m4_pro_64_20c","label":"m4_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_pro_64_20c","label":"m5_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_96_60c","label":"m3_ultra_96_60c","minMemoryGb":96,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_max_128_40c","label":"m5_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_512_80c","label":"m3_ultra_512_80c","minMemoryGb":512,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"mlx":"mlx_lm.server --model mlx-community/gemma-4-12b-coder-fable5-composer2.5-8bit"},"rank":17,"benchmarks":[{"hardwareId":"m3_ultra_96_60c","engine":"mlx","decodeTps":40.5,"decodeTps32k":39,"prefillTps":12.9,"ttftMs":null,"contextTokens":65536,"measuredAt":"2026-07-18","notes":"mlx 0.31.3"},{"hardwareId":"m3_ultra_512_80c","engine":"mlx","decodeTps":39.4,"decodeTps32k":37.9,"prefillTps":16.9,"ttftMs":null,"contextTokens":65536,"measuredAt":"2026-07-18","notes":"mlx 0.31.3"},{"hardwareId":"m5_max_128_40c","engine":"mlx","decodeTps":35.7,"decodeTps32k":34.9,"prefillTps":28,"ttftMs":null,"contextTokens":65536,"measuredAt":"2026-07-18","notes":"mlx 0.31.3"},{"hardwareId":"m4_max_36_32c","engine":"mlx","decodeTps":25,"decodeTps32k":23.9,"prefillTps":8.1,"ttftMs":null,"contextTokens":65536,"measuredAt":"2026-07-18","notes":"mlx 0.31.3"},{"hardwareId":"m5_pro_64_20c","engine":"mlx","decodeTps":19.9,"decodeTps32k":19.4,"prefillTps":16.7,"ttftMs":null,"contextTokens":65536,"measuredAt":"2026-07-18","notes":"mlx 0.31.3"},{"hardwareId":"m4_pro_64_20c","engine":"mlx","decodeTps":17.3,"decodeTps32k":16.5,"prefillTps":4.7,"ttftMs":null,"contextTokens":65536,"measuredAt":"2026-07-18","notes":"mlx 0.31.3"}],"expectSpeed":{"decodeTps":40.5,"prefillTps":28,"source":"measured"},"params":null,"notes":[]},
    "mlx-community/gemma-4-26B-A4B-it-4bit": {"name":"gemma-4-26B-A4B-it-4bit","quant":"mlx","filesize":"14gb","filesizeGb":14.3,"hardware":[{"id":"m4_max_36_32c","label":"m4_max_36_32c","minMemoryGb":36,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m4_pro_64_20c","label":"m4_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_pro_64_20c","label":"m5_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_96_60c","label":"m3_ultra_96_60c","minMemoryGb":96,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_max_128_40c","label":"m5_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m4_max_128_40c","label":"m4_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_512_80c","label":"m3_ultra_512_80c","minMemoryGb":512,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"mlx":"mlx_lm.server --model mlx-community/gemma-4-26B-A4B-it-4bit"},"rank":17,"benchmarks":[{"hardwareId":"m5_max_128_40c","engine":"mlx","decodeTps":102.3,"decodeTps32k":82,"prefillTps":null,"ttftMs":null,"contextTokens":262015,"measuredAt":"2026-07-06","notes":"mlx 0.31.3"},{"hardwareId":"m4_max_128_40c","engine":"mlx","decodeTps":92.2,"decodeTps32k":73,"prefillTps":null,"ttftMs":null,"contextTokens":262015,"measuredAt":"2026-07-09","notes":"mlx 0.31.3"},{"hardwareId":"m3_ultra_512_80c","engine":"mlx","decodeTps":87.8,"decodeTps32k":67.3,"prefillTps":null,"ttftMs":null,"contextTokens":262015,"measuredAt":"2026-07-06","notes":"mlx 0.31.3"},{"hardwareId":"m3_ultra_96_60c","engine":"mlx","decodeTps":83.6,"decodeTps32k":66.3,"prefillTps":null,"ttftMs":null,"contextTokens":262015,"measuredAt":"2026-07-06","notes":"mlx 0.31.3"},{"hardwareId":"m4_max_36_32c","engine":"mlx","decodeTps":79.4,"decodeTps32k":65.1,"prefillTps":623.9,"ttftMs":null,"contextTokens":139264,"measuredAt":"2026-07-06","notes":"mlx 0.31.3"},{"hardwareId":"m5_pro_64_20c","engine":"mlx","decodeTps":67.9,"decodeTps32k":51,"prefillTps":null,"ttftMs":null,"contextTokens":262015,"measuredAt":"2026-07-06","notes":"mlx 0.31.3"},{"hardwareId":"m4_pro_64_20c","engine":"mlx","decodeTps":60,"decodeTps32k":50,"prefillTps":null,"ttftMs":null,"contextTokens":262015,"measuredAt":"2026-07-06","notes":"mlx 0.31.3"}],"expectSpeed":{"decodeTps":102.3,"prefillTps":623.9,"source":"measured"},"params":"26B-A4B","notes":[]},
    "mlx-community/gemma-4-26B-A4B-it-8bit": {"name":"gemma-4-26B-A4B-it-8bit","quant":"mlx","filesize":"26gb","filesizeGb":26,"hardware":[{"id":"m4_max_36_32c","label":"m4_max_36_32c","minMemoryGb":36,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m4_pro_64_20c","label":"m4_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_pro_64_20c","label":"m5_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_96_60c","label":"m3_ultra_96_60c","minMemoryGb":96,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_max_128_40c","label":"m5_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m4_max_128_40c","label":"m4_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_512_80c","label":"m3_ultra_512_80c","minMemoryGb":512,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"mlx":"mlx_lm.server --model mlx-community/gemma-4-26B-A4B-it-8bit"},"rank":17,"benchmarks":[{"hardwareId":"m5_max_128_40c","engine":"mlx","decodeTps":75,"decodeTps32k":63.5,"prefillTps":null,"ttftMs":null,"contextTokens":262015,"measuredAt":"2026-07-06","notes":"mlx 0.31.3"},{"hardwareId":"m3_ultra_512_80c","engine":"mlx","decodeTps":68.9,"decodeTps32k":56.7,"prefillTps":null,"ttftMs":null,"contextTokens":262015,"measuredAt":"2026-07-06","notes":"mlx 0.31.3"},{"hardwareId":"m3_ultra_96_60c","engine":"mlx","decodeTps":68.6,"decodeTps32k":56.4,"prefillTps":null,"ttftMs":null,"contextTokens":262015,"measuredAt":"2026-07-06","notes":"mlx 0.31.3"},{"hardwareId":"m4_max_128_40c","engine":"mlx","decodeTps":65.1,"decodeTps32k":55.7,"prefillTps":null,"ttftMs":null,"contextTokens":262015,"measuredAt":"2026-07-09","notes":"mlx 0.31.3"},{"hardwareId":"m4_max_36_32c","engine":"mlx","decodeTps":55.4,"decodeTps32k":52.9,"prefillTps":2058.9,"ttftMs":null,"contextTokens":24576,"measuredAt":"2026-07-06","notes":"mlx 0.31.3"},{"hardwareId":"m5_pro_64_20c","engine":"mlx","decodeTps":46.4,"decodeTps32k":37.7,"prefillTps":null,"ttftMs":null,"contextTokens":90368,"measuredAt":"2026-07-06","notes":"mlx unknown"},{"hardwareId":"m4_pro_64_20c","engine":"mlx","decodeTps":41.7,"decodeTps32k":36.3,"prefillTps":null,"ttftMs":null,"contextTokens":262015,"measuredAt":"2026-07-06","notes":"mlx 0.31.3"}],"expectSpeed":{"decodeTps":75,"prefillTps":2058.9,"source":"measured"},"params":"26B-A4B","notes":[]},
    "mlx-community/gemma-4-31b-it-4bit": {"name":"gemma-4-31b-it-4bit","quant":"mlx","filesize":"17gb","filesizeGb":17.1,"hardware":[{"id":"m4_max_36_32c","label":"m4_max_36_32c","minMemoryGb":36,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m4_pro_64_20c","label":"m4_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_pro_64_20c","label":"m5_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_96_60c","label":"m3_ultra_96_60c","minMemoryGb":96,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_max_128_40c","label":"m5_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m4_max_128_40c","label":"m4_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_512_80c","label":"m3_ultra_512_80c","minMemoryGb":512,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"mlx":"mlx_lm.server --model mlx-community/gemma-4-31b-it-4bit"},"rank":17,"benchmarks":[{"hardwareId":"m3_ultra_96_60c","engine":"mlx","decodeTps":27.6,"decodeTps32k":22.7,"prefillTps":12.5,"ttftMs":null,"contextTokens":212992,"measuredAt":"2026-07-05","notes":"mlx 0.31.3"},{"hardwareId":"m3_ultra_512_80c","engine":"mlx","decodeTps":27.1,"decodeTps32k":22.6,"prefillTps":3,"ttftMs":null,"contextTokens":258047,"measuredAt":"2026-07-05","notes":"mlx 0.31.3"},{"hardwareId":"m5_max_128_40c","engine":"mlx","decodeTps":24.2,"decodeTps32k":19.4,"prefillTps":null,"ttftMs":null,"contextTokens":258047,"measuredAt":"2026-07-04","notes":"mlx 0.31.3"},{"hardwareId":"m4_max_128_40c","engine":"mlx","decodeTps":21.7,"decodeTps32k":18.5,"prefillTps":86.5,"ttftMs":null,"contextTokens":114816,"measuredAt":"2026-07-09","notes":"mlx 0.31.3"},{"hardwareId":"m5_pro_64_20c","engine":"mlx","decodeTps":13.3,"decodeTps32k":10.4,"prefillTps":null,"ttftMs":null,"contextTokens":118784,"measuredAt":"2026-07-05","notes":"mlx 0.31.3"},{"hardwareId":"m4_pro_64_20c","engine":"mlx","decodeTps":12.1,"decodeTps32k":10.3,"prefillTps":88.6,"ttftMs":null,"contextTokens":122880,"measuredAt":"2026-07-04","notes":"mlx 0.31.3"},{"hardwareId":"m4_max_36_32c","engine":"mlx","decodeTps":null,"decodeTps32k":null,"prefillTps":null,"ttftMs":null,"contextTokens":null,"measuredAt":null,"notes":"mlx 0.31.3"}],"expectSpeed":{"decodeTps":27.6,"prefillTps":88.6,"source":"measured"},"params":null,"notes":[]},
    "mlx-community/gemma-4-31b-it-8bit": {"name":"gemma-4-31b-it-8bit","quant":"mlx","filesize":"31gb","filesizeGb":31.4,"hardware":[{"id":"m4_pro_64_20c","label":"m4_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_pro_64_20c","label":"m5_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_96_60c","label":"m3_ultra_96_60c","minMemoryGb":96,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_max_128_40c","label":"m5_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m4_max_128_40c","label":"m4_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_512_80c","label":"m3_ultra_512_80c","minMemoryGb":512,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"mlx":"mlx_lm.server --model mlx-community/gemma-4-31b-it-8bit"},"rank":17,"benchmarks":[{"hardwareId":"m3_ultra_96_60c","engine":"mlx","decodeTps":17.5,"decodeTps32k":15.4,"prefillTps":163.4,"ttftMs":null,"contextTokens":163840,"measuredAt":"2026-07-05","notes":"mlx 0.31.3"},{"hardwareId":"m3_ultra_512_80c","engine":"mlx","decodeTps":17.2,"decodeTps32k":15.2,"prefillTps":null,"ttftMs":null,"contextTokens":258047,"measuredAt":"2026-07-05","notes":"mlx 0.31.3"},{"hardwareId":"m5_max_128_40c","engine":"mlx","decodeTps":14.5,"decodeTps32k":12.6,"prefillTps":null,"ttftMs":null,"contextTokens":258047,"measuredAt":"2026-07-04","notes":"mlx 0.31.3"},{"hardwareId":"m4_max_128_40c","engine":"mlx","decodeTps":12.7,"decodeTps32k":11.1,"prefillTps":142.3,"ttftMs":null,"contextTokens":67584,"measuredAt":"2026-07-09","notes":"mlx 0.31.3"},{"hardwareId":"m5_pro_64_20c","engine":"mlx","decodeTps":7.8,"decodeTps32k":6.7,"prefillTps":185,"ttftMs":null,"contextTokens":69632,"measuredAt":"2026-07-05","notes":"mlx 0.31.3"},{"hardwareId":"m4_pro_64_20c","engine":"mlx","decodeTps":6.8,"decodeTps32k":6,"prefillTps":67.3,"ttftMs":null,"contextTokens":69632,"measuredAt":"2026-07-04","notes":"mlx 0.31.3"}],"expectSpeed":{"decodeTps":17.5,"prefillTps":185,"source":"measured"},"params":null,"notes":[]},
    "mlx-community/gemma-4-31b-it-bf16": {"name":"gemma-4-31b-it-bf16","quant":"bf16","filesize":"58gb","filesizeGb":58.3,"hardware":[{"id":"m3_ultra_96_60c","label":"m3_ultra_96_60c","minMemoryGb":96,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_max_128_40c","label":"m5_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_512_80c","label":"m3_ultra_512_80c","minMemoryGb":512,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"mlx":"mlx_lm.server --model mlx-community/gemma-4-31b-it-bf16"},"rank":17,"benchmarks":[{"hardwareId":"m3_ultra_96_60c","engine":"mlx","decodeTps":10.6,"decodeTps32k":9.8,"prefillTps":null,"ttftMs":null,"contextTokens":61440,"measuredAt":"2026-07-05","notes":"mlx 0.31.3"},{"hardwareId":"m3_ultra_512_80c","engine":"mlx","decodeTps":9.9,"decodeTps32k":9.2,"prefillTps":null,"ttftMs":null,"contextTokens":258047,"measuredAt":"2026-07-05","notes":"mlx 0.31.3"},{"hardwareId":"m5_max_128_40c","engine":"mlx","decodeTps":7.4,"decodeTps32k":6.9,"prefillTps":267.7,"ttftMs":null,"contextTokens":204800,"measuredAt":"2026-07-04","notes":"mlx 0.31.3"}],"expectSpeed":{"decodeTps":10.6,"prefillTps":267.7,"source":"measured"},"params":null,"notes":[]},
    "mlx-community/gemma-4-e2b-it-4bit": {"name":"gemma-4-e2b-it-4bit","quant":"mlx","filesize":"3gb","filesizeGb":3.3,"hardware":[{"id":"m4_max_36_32c","label":"m4_max_36_32c","minMemoryGb":36,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m4_pro_64_20c","label":"m4_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_pro_64_20c","label":"m5_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_96_60c","label":"m3_ultra_96_60c","minMemoryGb":96,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_max_128_40c","label":"m5_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m4_max_128_40c","label":"m4_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_512_80c","label":"m3_ultra_512_80c","minMemoryGb":512,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"mlx":"mlx_lm.server --model mlx-community/gemma-4-e2b-it-4bit"},"rank":17,"benchmarks":[{"hardwareId":"m5_max_128_40c","engine":"mlx","decodeTps":154.5,"decodeTps32k":141.8,"prefillTps":null,"ttftMs":null,"contextTokens":126976,"measuredAt":"2026-07-04","notes":"mlx 0.31.3"},{"hardwareId":"m4_max_128_40c","engine":"mlx","decodeTps":138.6,"decodeTps32k":118.6,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-09","notes":"mlx 0.31.3"},{"hardwareId":"m3_ultra_512_80c","engine":"mlx","decodeTps":128,"decodeTps32k":116.6,"prefillTps":null,"ttftMs":null,"contextTokens":126976,"measuredAt":"2026-07-05","notes":"mlx 0.31.3"},{"hardwareId":"m3_ultra_96_60c","engine":"mlx","decodeTps":117.3,"decodeTps32k":105.2,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-05","notes":"mlx 0.31.3"},{"hardwareId":"m5_pro_64_20c","engine":"mlx","decodeTps":112.2,"decodeTps32k":100.9,"prefillTps":null,"ttftMs":null,"contextTokens":126976,"measuredAt":"2026-07-04","notes":"mlx 0.31.3"},{"hardwareId":"m4_pro_64_20c","engine":"mlx","decodeTps":97.3,"decodeTps32k":78.5,"prefillTps":null,"ttftMs":null,"contextTokens":126976,"measuredAt":"2026-07-04","notes":"mlx 0.31.3"},{"hardwareId":"m4_max_36_32c","engine":"mlx","decodeTps":null,"decodeTps32k":null,"prefillTps":null,"ttftMs":null,"contextTokens":null,"measuredAt":null,"notes":"mlx 0.31.3"}],"expectSpeed":{"decodeTps":154.5,"prefillTps":null,"source":"measured"},"params":null,"notes":[]},
    "mlx-community/gemma-4-e2b-it-8bit": {"name":"gemma-4-e2b-it-8bit","quant":"mlx","filesize":"5gb","filesizeGb":5.5,"hardware":[{"id":"m4_max_36_32c","label":"m4_max_36_32c","minMemoryGb":36,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m4_pro_64_20c","label":"m4_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_pro_64_20c","label":"m5_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_96_60c","label":"m3_ultra_96_60c","minMemoryGb":96,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_max_128_40c","label":"m5_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m4_max_128_40c","label":"m4_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_512_80c","label":"m3_ultra_512_80c","minMemoryGb":512,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"mlx":"mlx_lm.server --model mlx-community/gemma-4-e2b-it-8bit"},"rank":17,"benchmarks":[{"hardwareId":"m5_max_128_40c","engine":"mlx","decodeTps":119.3,"decodeTps32k":111.5,"prefillTps":null,"ttftMs":null,"contextTokens":126976,"measuredAt":"2026-07-04","notes":"mlx 0.31.3"},{"hardwareId":"m4_max_128_40c","engine":"mlx","decodeTps":107.2,"decodeTps32k":94.1,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-09","notes":"mlx 0.31.3"},{"hardwareId":"m3_ultra_512_80c","engine":"mlx","decodeTps":104.9,"decodeTps32k":97,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-05","notes":"mlx 0.31.3"},{"hardwareId":"m3_ultra_96_60c","engine":"mlx","decodeTps":98.9,"decodeTps32k":90,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-05","notes":"mlx 0.31.3"},{"hardwareId":"m5_pro_64_20c","engine":"mlx","decodeTps":77.7,"decodeTps32k":72.3,"prefillTps":null,"ttftMs":null,"contextTokens":126976,"measuredAt":"2026-07-04","notes":"mlx 0.31.3"},{"hardwareId":"m4_max_36_32c","engine":"mlx","decodeTps":null,"decodeTps32k":null,"prefillTps":null,"ttftMs":null,"contextTokens":null,"measuredAt":null,"notes":"mlx 0.31.3"},{"hardwareId":"m4_pro_64_20c","engine":"mlx","decodeTps":null,"decodeTps32k":null,"prefillTps":null,"ttftMs":null,"contextTokens":null,"measuredAt":null,"notes":"mlx 0.31.3"}],"expectSpeed":{"decodeTps":119.3,"prefillTps":null,"source":"measured"},"params":null,"notes":[]},
    "mlx-community/gemma-4-e4b-it-8bit": {"name":"gemma-4-e4b-it-8bit","quant":"mlx","filesize":"8gb","filesizeGb":8.3,"hardware":[{"id":"m4_max_36_32c","label":"m4_max_36_32c","minMemoryGb":36,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m4_pro_64_20c","label":"m4_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_pro_64_20c","label":"m5_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_96_60c","label":"m3_ultra_96_60c","minMemoryGb":96,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_max_128_40c","label":"m5_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m4_max_128_40c","label":"m4_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_512_80c","label":"m3_ultra_512_80c","minMemoryGb":512,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"mlx":"mlx_lm.server --model mlx-community/gemma-4-e4b-it-8bit"},"rank":17,"benchmarks":[{"hardwareId":"m3_ultra_96_60c","engine":"mlx","decodeTps":179.1,"decodeTps32k":null,"prefillTps":null,"ttftMs":null,"contextTokens":8192,"measuredAt":"2026-07-05","notes":"mlx 0.31.3"},{"hardwareId":"m5_max_128_40c","engine":"mlx","decodeTps":73.8,"decodeTps32k":61.2,"prefillTps":null,"ttftMs":null,"contextTokens":126976,"measuredAt":"2026-07-04","notes":"mlx 0.31.3"},{"hardwareId":"m3_ultra_512_80c","engine":"mlx","decodeTps":66.1,"decodeTps32k":51.3,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-05","notes":"mlx 0.31.3"},{"hardwareId":"m4_max_128_40c","engine":"mlx","decodeTps":62.1,"decodeTps32k":51.3,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-09","notes":"mlx 0.31.3"},{"hardwareId":"m5_pro_64_20c","engine":"mlx","decodeTps":44.5,"decodeTps32k":38.3,"prefillTps":null,"ttftMs":null,"contextTokens":126976,"measuredAt":"2026-07-04","notes":"mlx 0.31.3"},{"hardwareId":"m4_pro_64_20c","engine":"mlx","decodeTps":37.8,"decodeTps32k":32.7,"prefillTps":null,"ttftMs":null,"contextTokens":126976,"measuredAt":"2026-07-04","notes":"mlx 0.31.3"},{"hardwareId":"m4_max_36_32c","engine":"mlx","decodeTps":null,"decodeTps32k":null,"prefillTps":null,"ttftMs":null,"contextTokens":null,"measuredAt":null,"notes":"mlx 0.31.3"}],"expectSpeed":{"decodeTps":179.1,"prefillTps":null,"source":"measured"},"params":null,"notes":[]},
    "mlx-community/gpt-oss-120b-MXFP4-Q8": {"name":"gpt-oss-120b-MXFP4-Q8","quant":"mlx","filesize":"59gb","filesizeGb":59,"hardware":[{"id":"m5_max_128_40c","label":"m5_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m4_max_128_40c","label":"m4_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_512_80c","label":"m3_ultra_512_80c","minMemoryGb":512,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"mlx":"mlx_lm.server --model mlx-community/gpt-oss-120b-MXFP4-Q8"},"rank":17,"benchmarks":[{"hardwareId":"m5_max_128_40c","engine":"mlx","decodeTps":72.2,"decodeTps32k":51.5,"prefillTps":2675.7,"ttftMs":null,"contextTokens":126975,"measuredAt":"2026-07-04","notes":"mlx 0.31.3"},{"hardwareId":"m3_ultra_512_80c","engine":"mlx","decodeTps":68.8,"decodeTps32k":53.8,"prefillTps":1629.6,"ttftMs":null,"contextTokens":126975,"measuredAt":"2026-07-05","notes":"mlx 0.31.3"},{"hardwareId":"m4_max_128_40c","engine":"mlx","decodeTps":64,"decodeTps32k":47.1,"prefillTps":924.9,"ttftMs":null,"contextTokens":130943,"measuredAt":"2026-07-09","notes":"mlx 0.31.3"}],"expectSpeed":{"decodeTps":72.2,"prefillTps":2675.7,"source":"measured"},"params":null,"notes":[]},
    "mlx-community/gpt-oss-20b-MXFP4-Q8": {"name":"gpt-oss-20b-MXFP4-Q8","quant":"mlx","filesize":"11gb","filesizeGb":11.2,"hardware":[{"id":"m4_max_36_32c","label":"m4_max_36_32c","minMemoryGb":36,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m4_pro_64_20c","label":"m4_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_pro_64_20c","label":"m5_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_96_60c","label":"m3_ultra_96_60c","minMemoryGb":96,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_max_128_40c","label":"m5_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m4_max_128_40c","label":"m4_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_512_80c","label":"m3_ultra_512_80c","minMemoryGb":512,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"mlx":"mlx_lm.server --model mlx-community/gpt-oss-20b-MXFP4-Q8"},"rank":17,"benchmarks":[{"hardwareId":"m5_max_128_40c","engine":"mlx","decodeTps":107.4,"decodeTps32k":79.4,"prefillTps":4809,"ttftMs":null,"contextTokens":126975,"measuredAt":"2026-07-04","notes":"mlx 0.31.3"},{"hardwareId":"m3_ultra_512_80c","engine":"mlx","decodeTps":100.7,"decodeTps32k":79.4,"prefillTps":2430.4,"ttftMs":null,"contextTokens":126975,"measuredAt":"2026-07-05","notes":"mlx 0.31.3"},{"hardwareId":"m3_ultra_96_60c","engine":"mlx","decodeTps":95.4,"decodeTps32k":73.3,"prefillTps":1921.1,"ttftMs":null,"contextTokens":130943,"measuredAt":"2026-07-05","notes":"mlx 0.31.3"},{"hardwareId":"m4_max_128_40c","engine":"mlx","decodeTps":94.3,"decodeTps32k":68.3,"prefillTps":1476.3,"ttftMs":null,"contextTokens":130943,"measuredAt":"2026-07-09","notes":"mlx 0.31.3"},{"hardwareId":"m5_pro_64_20c","engine":"mlx","decodeTps":69,"decodeTps32k":48,"prefillTps":2569.5,"ttftMs":null,"contextTokens":126975,"measuredAt":"2026-07-04","notes":"mlx 0.31.3"},{"hardwareId":"m4_pro_64_20c","engine":"mlx","decodeTps":58.3,"decodeTps32k":40.9,"prefillTps":748.5,"ttftMs":null,"contextTokens":130943,"measuredAt":"2026-07-04","notes":"mlx 0.31.3"},{"hardwareId":"m4_max_36_32c","engine":"mlx","decodeTps":null,"decodeTps32k":null,"prefillTps":null,"ttftMs":null,"contextTokens":null,"measuredAt":null,"notes":"mlx 0.31.3"}],"expectSpeed":{"decodeTps":107.4,"prefillTps":4809,"source":"measured"},"params":null,"notes":[]},
    "nvidia/DeepSeek-V4-Flash-NVFP4": {"name":"DeepSeek-V4-Flash-NVFP4","quant":"nvfp4","filesize":"157gb","filesizeGb":156.7,"hardware":[{"id":"gb300","label":"gb300","minMemoryGb":288,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve nvidia/DeepSeek-V4-Flash-NVFP4 --block-size 128 --trust-remote-code --kv-cache-dtype fp8 --enable-prefix-caching --gpu-memory-utilization 0.90 --kernel-config '{\"enable_flashinfer_autotune\":false}' --max-model-len 1048576 --cpu-offload-gb 0"},"rank":17,"benchmarks":[{"hardwareId":"gb300","engine":"vllm","decodeTps":178.9,"decodeTps32k":190.2,"prefillTps":null,"ttftMs":null,"contextTokens":1048448,"measuredAt":"2026-07-15","notes":"vllm unknown"}],"expectSpeed":{"decodeTps":178.9,"prefillTps":null,"source":"measured"},"params":null,"notes":[]},
    "nvidia/Kimi-K2.6-NVFP4": {"name":"Kimi-K2.6-NVFP4","quant":"nvfp4","filesize":"554gb","filesizeGb":554.3,"hardware":[{"id":"gb300","label":"gb300","minMemoryGb":288,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve nvidia/Kimi-K2.6-NVFP4 --trust-remote-code --kv-cache-dtype fp8 --enable-prefix-caching --gpu-memory-utilization 0.90 --kernel-config '{\"enable_flashinfer_autotune\":false}' --max-model-len 262144 --cpu-offload-gb 400 --cpu-offload-params w13_weight w2_weight"},"rank":17,"benchmarks":[{"hardwareId":"gb300","engine":"vllm","decodeTps":25.1,"decodeTps32k":null,"prefillTps":2543.4,"ttftMs":null,"contextTokens":8320,"measuredAt":"2026-07-18","notes":"vllm 0.25.0"}],"expectSpeed":{"decodeTps":25.1,"prefillTps":2543.4,"source":"measured"},"params":null,"notes":[]},
    "nvidia/MiniMax-M3-NVFP4": {"name":"MiniMax-M3-NVFP4","quant":"nvfp4","filesize":"233gb","filesizeGb":232.9,"hardware":[{"id":"gb300","label":"gb300","minMemoryGb":288,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve nvidia/MiniMax-M3-NVFP4 --block-size 128 --trust-remote-code --kv-cache-dtype fp8 --enable-prefix-caching --gpu-memory-utilization 0.90 --kernel-config '{\"enable_flashinfer_autotune\":false}' --skip-mm-profiling --max-model-len 1048576 --cpu-offload-gb 92 --cpu-offload-params w13_weight w2_weight"},"rank":17,"benchmarks":[{"hardwareId":"gb300","engine":"vllm","decodeTps":63.1,"decodeTps32k":65.1,"prefillTps":null,"ttftMs":null,"contextTokens":1048448,"measuredAt":"2026-07-15","notes":"vllm unknown"}],"expectSpeed":{"decodeTps":63.1,"prefillTps":null,"source":"measured"},"params":null,"notes":[]},
    "poolside/Laguna-S-2.1-NVFP4": {"name":"Laguna-S-2.1-NVFP4","quant":"nvfp4","filesize":"67gb","filesizeGb":67,"hardware":[{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve poolside/Laguna-S-2.1-NVFP4 --served-model-name poolside/Laguna-S-2.1-NVFP4 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --block-size 16 --max-num-batched-tokens 8192 --enable-auto-tool-choice --tool-call-parser glm47 --override-generation-config '{\"temperature\":0.7,\"top_p\":0.95,\"min_p\":0.05}' --max-num-seqs 32 --max-model-len 262144 --gpu-memory-utilization 0.85"},"rank":17,"benchmarks":[{"hardwareId":"gb10_121","engine":"vllm","decodeTps":18.6,"decodeTps32k":18.1,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-21","notes":"vllm 0.25.0"}],"expectSpeed":{"decodeTps":18.6,"prefillTps":null,"source":"measured"},"params":null,"notes":[]},
    "prism-ml/Bonsai-27B-gguf": {"name":"Bonsai-27B-Q1_0.gguf","quant":"gguf","filesize":"64gb","filesizeGb":63.6,"hardware":[{"id":"m4_max_36_32c","label":"m4_max_36_32c","minMemoryGb":36,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"m4_pro_64_20c","label":"m4_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_pro_64_20c","label":"m5_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_96_60c","label":"m3_ultra_96_60c","minMemoryGb":96,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"m3_ultra_512_80c","label":"m3_ultra_512_80c","minMemoryGb":512,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"llamacpp":"llama-server -m prism-ml/Bonsai-27B-gguf"},"rank":17,"benchmarks":[{"hardwareId":"rtxpro6000_96","engine":"llamacpp","decodeTps":117.9,"decodeTps32k":106,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-21","notes":"llamacpp unknown"},{"hardwareId":"rtx6000ada_48","engine":"llamacpp","decodeTps":106.8,"decodeTps32k":89.6,"prefillTps":null,"ttftMs":null,"contextTokens":139392,"measuredAt":"2026-07-21","notes":"llamacpp unknown"},{"hardwareId":"m3_ultra_512_80c","engine":"llamacpp","decodeTps":53.3,"decodeTps32k":45.4,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-21","notes":"llamacpp unknown"},{"hardwareId":"m3_ultra_96_60c","engine":"llamacpp","decodeTps":47.9,"decodeTps32k":40.1,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-21","notes":"llamacpp unknown"},{"hardwareId":"m4_max_36_32c","engine":"llamacpp","decodeTps":39.2,"decodeTps32k":31,"prefillTps":null,"ttftMs":null,"contextTokens":89984,"measuredAt":"2026-07-21","notes":"llamacpp unknown"},{"hardwareId":"m5_pro_64_20c","engine":"llamacpp","decodeTps":35.3,"decodeTps32k":26.8,"prefillTps":null,"ttftMs":null,"contextTokens":204672,"measuredAt":"2026-07-21","notes":"llamacpp unknown"},{"hardwareId":"m4_pro_64_20c","engine":"llamacpp","decodeTps":27.4,"decodeTps32k":19.3,"prefillTps":null,"ttftMs":null,"contextTokens":204672,"measuredAt":"2026-07-21","notes":"llamacpp unknown"}],"expectSpeed":{"decodeTps":117.9,"prefillTps":null,"source":"measured"},"params":"27B","notes":[]},
    "prism-ml/Bonsai-27B-unpacked": {"name":"Bonsai-27B-unpacked","quant":"bf16","filesize":"51gb","filesizeGb":51,"hardware":[{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve prism-ml/Bonsai-27B-unpacked --served-model-name prism-ml/Bonsai-27B-unpacked --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 131072 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --mamba-cache-mode align"},"rank":17,"benchmarks":[{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":28.6,"decodeTps32k":28.8,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-21","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":4.4,"decodeTps32k":4.3,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-15","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":28.6,"prefillTps":null,"source":"measured"},"params":"27B","notes":[]},
    "prism-ml/Ternary-Bonsai-27B-gguf": {"name":"Ternary-Bonsai-27B-Q2_0.gguf++prism-b9591","quant":"gguf","filesize":"81gb","filesizeGb":80.6,"hardware":[{"id":"m5_max_128_40c","label":"m5_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"llamacpp":"llama-server -m prism-ml/Ternary-Bonsai-27B-gguf"},"rank":17,"benchmarks":[{"hardwareId":"m5_max_128_40c","engine":"llamacpp","decodeTps":43.5,"decodeTps32k":36,"prefillTps":null,"ttftMs":null,"contextTokens":200704,"measuredAt":"2026-07-21","notes":"llamacpp unknown"}],"expectSpeed":{"decodeTps":43.5,"prefillTps":null,"source":"measured"},"params":"27B","notes":[]},
    "prism-ml/Ternary-Bonsai-27B-mlx-2bit": {"name":"Ternary-Bonsai-27B-mlx-2bit","quant":"mlx","filesize":"8gb","filesizeGb":7.9,"hardware":[{"id":"m4_max_36_32c","label":"m4_max_36_32c","minMemoryGb":36,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m4_pro_64_20c","label":"m4_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_pro_64_20c","label":"m5_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_96_60c","label":"m3_ultra_96_60c","minMemoryGb":96,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m4_max_128_40c","label":"m4_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_max_128_40c","label":"m5_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_512_80c","label":"m3_ultra_512_80c","minMemoryGb":512,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"mlx":"mlx_lm.server --model prism-ml/Ternary-Bonsai-27B-mlx-2bit"},"rank":17,"benchmarks":[{"hardwareId":"m3_ultra_512_80c","engine":"mlx","decodeTps":47,"decodeTps32k":39.5,"prefillTps":null,"ttftMs":null,"contextTokens":65536,"measuredAt":"2026-07-15","notes":"mlx 0.31.3"},{"hardwareId":"m5_max_128_40c","engine":"mlx","decodeTps":46.9,"decodeTps32k":39.9,"prefillTps":null,"ttftMs":null,"contextTokens":65536,"measuredAt":"2026-07-15","notes":"mlx 0.31.3"},{"hardwareId":"m3_ultra_96_60c","engine":"mlx","decodeTps":40.3,"decodeTps32k":34,"prefillTps":null,"ttftMs":null,"contextTokens":65536,"measuredAt":"2026-07-15","notes":"mlx 0.31.3"},{"hardwareId":"m4_max_128_40c","engine":"mlx","decodeTps":38.7,"decodeTps32k":32.7,"prefillTps":null,"ttftMs":null,"contextTokens":65536,"measuredAt":"2026-07-15","notes":"mlx 0.31.3"},{"hardwareId":"m4_max_36_32c","engine":"mlx","decodeTps":32.4,"decodeTps32k":26.7,"prefillTps":null,"ttftMs":null,"contextTokens":65536,"measuredAt":"2026-07-15","notes":"mlx 0.31.3"},{"hardwareId":"m5_pro_64_20c","engine":"mlx","decodeTps":28,"decodeTps32k":23.5,"prefillTps":null,"ttftMs":null,"contextTokens":65536,"measuredAt":"2026-07-15","notes":"mlx 0.31.3"},{"hardwareId":"m4_pro_64_20c","engine":"mlx","decodeTps":22.5,"decodeTps32k":18.6,"prefillTps":null,"ttftMs":null,"contextTokens":65536,"measuredAt":"2026-07-15","notes":"mlx 0.31.3"}],"expectSpeed":{"decodeTps":47,"prefillTps":null,"source":"measured"},"params":"27B","notes":[]},
    "stepfun-ai/Step-3.7-Flash-NVFP4": {"name":"Step-3.7-Flash-NVFP4","quant":"nvfp4","filesize":"120gb","filesizeGb":120.4,"hardware":[{"id":"gb300","label":"gb300","minMemoryGb":288,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve stepfun-ai/Step-3.7-Flash-NVFP4 --block-size 128 --trust-remote-code --kv-cache-dtype fp8 --enable-prefix-caching --gpu-memory-utilization 0.90 --kernel-config '{\"enable_flashinfer_autotune\":false}' --max-model-len 262144 --cpu-offload-gb 0"},"rank":17,"benchmarks":[{"hardwareId":"gb300","engine":"vllm","decodeTps":175.4,"decodeTps32k":193.9,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-21","notes":"vllm unknown"}],"expectSpeed":{"decodeTps":175.4,"prefillTps":null,"source":"measured"},"params":null,"notes":[]},
    "thinkingmachines/Inkling-NVFP4": {"name":"Inkling-NVFP4","quant":"nvfp4","filesize":"551gb","filesizeGb":551.3,"hardware":[{"id":"gb300","label":"gb300","minMemoryGb":288,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve thinkingmachines/Inkling-NVFP4"},"rank":17,"benchmarks":[{"hardwareId":"gb300","engine":"vllm","decodeTps":26.8,"decodeTps32k":27,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-21","notes":"vllm unknown"}],"expectSpeed":{"decodeTps":26.8,"prefillTps":null,"source":"measured"},"params":null,"notes":[]},
    "unsloth/Qwen3.6-27B-MLX-8bit": {"name":"Qwen3.6-27B-MLX-8bit","quant":"mlx","filesize":"32gb","filesizeGb":32.3,"hardware":[{"id":"m4_pro_64_20c","label":"m4_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_pro_64_20c","label":"m5_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_96_60c","label":"m3_ultra_96_60c","minMemoryGb":96,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_max_128_40c","label":"m5_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m4_max_128_40c","label":"m4_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_512_80c","label":"m3_ultra_512_80c","minMemoryGb":512,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"mlx":"mlx_lm.server --model unsloth/Qwen3.6-27B-MLX-8bit"},"rank":17,"benchmarks":[{"hardwareId":"m3_ultra_96_60c","engine":"mlx","decodeTps":18.2,"decodeTps32k":17,"prefillTps":101.5,"ttftMs":null,"contextTokens":204800,"measuredAt":"2026-07-05","notes":"mlx 0.31.3"},{"hardwareId":"m3_ultra_512_80c","engine":"mlx","decodeTps":17.7,"decodeTps32k":16.5,"prefillTps":null,"ttftMs":null,"contextTokens":258047,"measuredAt":"2026-07-05","notes":"mlx unknown"},{"hardwareId":"m5_max_128_40c","engine":"mlx","decodeTps":14.6,"decodeTps32k":14,"prefillTps":null,"ttftMs":null,"contextTokens":258047,"measuredAt":"2026-07-04","notes":"mlx 0.31.3"},{"hardwareId":"m4_max_128_40c","engine":"mlx","decodeTps":13.3,"decodeTps32k":12.2,"prefillTps":null,"ttftMs":null,"contextTokens":262015,"measuredAt":"2026-07-09","notes":"mlx 0.31.3"},{"hardwareId":"m5_pro_64_20c","engine":"mlx","decodeTps":8,"decodeTps32k":7.6,"prefillTps":433,"ttftMs":null,"contextTokens":90112,"measuredAt":"2026-07-04","notes":"mlx 0.31.3"},{"hardwareId":"m4_pro_64_20c","engine":"mlx","decodeTps":6.9,"decodeTps32k":6.5,"prefillTps":78.9,"ttftMs":null,"contextTokens":86016,"measuredAt":"2026-07-04","notes":"mlx 0.31.3"}],"expectSpeed":{"decodeTps":18.2,"prefillTps":433,"source":"measured"},"params":"27B","notes":[]},
    "unsloth/Qwen3.6-27B-UD-MLX-4bit": {"name":"Qwen3.6-27B-UD-MLX-4bit","quant":"mlx","filesize":"24gb","filesizeGb":24.4,"hardware":[{"id":"m4_max_36_32c","label":"m4_max_36_32c","minMemoryGb":36,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m4_pro_64_20c","label":"m4_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_pro_64_20c","label":"m5_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_96_60c","label":"m3_ultra_96_60c","minMemoryGb":96,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_max_128_40c","label":"m5_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m4_max_128_40c","label":"m4_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_512_80c","label":"m3_ultra_512_80c","minMemoryGb":512,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"mlx":"mlx_lm.server --model unsloth/Qwen3.6-27B-UD-MLX-4bit"},"rank":17,"benchmarks":[{"hardwareId":"m3_ultra_96_60c","engine":"mlx","decodeTps":23.2,"decodeTps32k":21.1,"prefillTps":158.8,"ttftMs":null,"contextTokens":241663,"measuredAt":"2026-07-05","notes":"mlx 0.31.3"},{"hardwareId":"m3_ultra_512_80c","engine":"mlx","decodeTps":22.5,"decodeTps32k":20.6,"prefillTps":209.7,"ttftMs":null,"contextTokens":258047,"measuredAt":"2026-07-05","notes":"mlx unknown"},{"hardwareId":"m5_max_128_40c","engine":"mlx","decodeTps":19.6,"decodeTps32k":18.3,"prefillTps":null,"ttftMs":null,"contextTokens":262015,"measuredAt":"2026-07-05","notes":"mlx 0.31.3"},{"hardwareId":"m4_max_128_40c","engine":"mlx","decodeTps":17.3,"decodeTps32k":15.7,"prefillTps":null,"ttftMs":null,"contextTokens":262015,"measuredAt":"2026-07-09","notes":"mlx 0.31.3"},{"hardwareId":"m4_max_36_32c","engine":"mlx","decodeTps":13.5,"decodeTps32k":null,"prefillTps":245.1,"ttftMs":null,"contextTokens":12288,"measuredAt":"2026-07-06","notes":"mlx 0.31.3"},{"hardwareId":"m5_pro_64_20c","engine":"mlx","decodeTps":10.6,"decodeTps32k":9.9,"prefillTps":null,"ttftMs":null,"contextTokens":118784,"measuredAt":"2026-07-04","notes":"mlx 0.31.3"},{"hardwareId":"m4_pro_64_20c","engine":"mlx","decodeTps":9.1,"decodeTps32k":8.4,"prefillTps":2.3,"ttftMs":null,"contextTokens":122880,"measuredAt":"2026-07-04","notes":"mlx 0.31.3"}],"expectSpeed":{"decodeTps":23.2,"prefillTps":245.1,"source":"measured"},"params":"27B","notes":[]},
    "unsloth/Qwen3.6-35B-A3B-GGUF": {"name":"Qwen3.6-35B-A3B-GGUF","quant":"gguf","filesize":"21gb","filesizeGb":20.6,"hardware":[{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true}],"commands":{"llamacpp":"llama-server -m unsloth/Qwen3.6-35B-A3B-GGUF"},"rank":17,"benchmarks":[{"hardwareId":"rtxpro6000_96","engine":"llamacpp","decodeTps":246.8,"decodeTps32k":227.6,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-21","notes":"llamacpp unknown"}],"expectSpeed":{"decodeTps":246.8,"prefillTps":null,"source":"measured"},"params":"35B-A3B","notes":[]},
    "unsloth/Qwen3.6-35B-A3B-MLX-8bit": {"name":"Qwen3.6-35B-A3B-MLX-8bit","quant":"mlx","filesize":"35gb","filesizeGb":35.1,"hardware":[{"id":"m4_pro_64_20c","label":"m4_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_pro_64_20c","label":"m5_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_96_60c","label":"m3_ultra_96_60c","minMemoryGb":96,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_max_128_40c","label":"m5_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m4_max_128_40c","label":"m4_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_512_80c","label":"m3_ultra_512_80c","minMemoryGb":512,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"mlx":"mlx_lm.server --model unsloth/Qwen3.6-35B-A3B-MLX-8bit"},"rank":17,"benchmarks":[{"hardwareId":"m5_max_128_40c","engine":"mlx","decodeTps":86.4,"decodeTps32k":76.2,"prefillTps":null,"ttftMs":null,"contextTokens":262015,"measuredAt":"2026-07-05","notes":"mlx 0.31.3"},{"hardwareId":"m4_max_128_40c","engine":"mlx","decodeTps":72.7,"decodeTps32k":63.2,"prefillTps":null,"ttftMs":null,"contextTokens":262015,"measuredAt":"2026-07-09","notes":"mlx 0.31.3"},{"hardwareId":"m3_ultra_512_80c","engine":"mlx","decodeTps":72.3,"decodeTps32k":64.5,"prefillTps":null,"ttftMs":null,"contextTokens":258047,"measuredAt":"2026-07-05","notes":"mlx 0.31.3"},{"hardwareId":"m3_ultra_96_60c","engine":"mlx","decodeTps":71.3,"decodeTps32k":61.5,"prefillTps":null,"ttftMs":null,"contextTokens":262015,"measuredAt":"2026-07-05","notes":"mlx 0.31.3"},{"hardwareId":"m5_pro_64_20c","engine":"mlx","decodeTps":55.6,"decodeTps32k":48.7,"prefillTps":343.9,"ttftMs":null,"contextTokens":163840,"measuredAt":"2026-07-04","notes":"mlx 0.31.3"},{"hardwareId":"m4_pro_64_20c","engine":"mlx","decodeTps":null,"decodeTps32k":null,"prefillTps":null,"ttftMs":null,"contextTokens":null,"measuredAt":null,"notes":"mlx 0.31.3"}],"expectSpeed":{"decodeTps":86.4,"prefillTps":343.9,"source":"measured"},"params":"35B-A3B","notes":[]},
    "unsloth/Qwen3.6-35B-A3B-UD-MLX-4bit": {"name":"Qwen3.6-35B-A3B-UD-MLX-4bit","quant":"mlx","filesize":"20gb","filesizeGb":20.1,"hardware":[{"id":"m4_max_36_32c","label":"m4_max_36_32c","minMemoryGb":36,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_pro_64_20c","label":"m5_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m4_pro_64_20c","label":"m4_pro_64_20c","minMemoryGb":64,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_96_60c","label":"m3_ultra_96_60c","minMemoryGb":96,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m5_max_128_40c","label":"m5_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m4_max_128_40c","label":"m4_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_512_80c","label":"m3_ultra_512_80c","minMemoryGb":512,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"mlx":"mlx_lm.server --model unsloth/Qwen3.6-35B-A3B-UD-MLX-4bit"},"rank":17,"benchmarks":[{"hardwareId":"m5_max_128_40c","engine":"mlx","decodeTps":94.2,"decodeTps32k":82.9,"prefillTps":null,"ttftMs":null,"contextTokens":262015,"measuredAt":"2026-07-05","notes":"mlx 0.31.3"},{"hardwareId":"m4_max_128_40c","engine":"mlx","decodeTps":78.6,"decodeTps32k":67.9,"prefillTps":null,"ttftMs":null,"contextTokens":262015,"measuredAt":"2026-07-09","notes":"mlx 0.31.3"},{"hardwareId":"m3_ultra_512_80c","engine":"mlx","decodeTps":77,"decodeTps32k":67.9,"prefillTps":null,"ttftMs":null,"contextTokens":258047,"measuredAt":"2026-07-05","notes":"mlx 0.31.3"},{"hardwareId":"m3_ultra_96_60c","engine":"mlx","decodeTps":74.2,"decodeTps32k":64.3,"prefillTps":null,"ttftMs":null,"contextTokens":262015,"measuredAt":"2026-07-05","notes":"mlx 0.31.3"},{"hardwareId":"m5_pro_64_20c","engine":"mlx","decodeTps":63.8,"decodeTps32k":53.7,"prefillTps":null,"ttftMs":null,"contextTokens":258047,"measuredAt":"2026-07-04","notes":"mlx 0.31.3"},{"hardwareId":"m4_pro_64_20c","engine":"mlx","decodeTps":52.9,"decodeTps32k":44.4,"prefillTps":null,"ttftMs":null,"contextTokens":262015,"measuredAt":"2026-07-05","notes":"mlx 0.31.3"},{"hardwareId":"m4_max_36_32c","engine":"mlx","decodeTps":null,"decodeTps32k":null,"prefillTps":null,"ttftMs":null,"contextTokens":null,"measuredAt":null,"notes":"mlx 0.31.3"}],"expectSpeed":{"decodeTps":94.2,"prefillTps":null,"source":"measured"},"params":"35B-A3B","notes":[]},
    "will3509111/Nemotron-3-Super-120B-A12B-MLX-MXFP4": {"name":"Nemotron-3-Super-120B-A12B-MLX-MXFP4","quant":"mlx","filesize":"60gb","filesizeGb":59.8,"hardware":[{"id":"m5_max_128_40c","label":"m5_max_128_40c","minMemoryGb":128,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"m3_ultra_512_80c","label":"m3_ultra_512_80c","minMemoryGb":512,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"mlx":"mlx_lm.server --model will3509111/Nemotron-3-Super-120B-A12B-MLX-MXFP4"},"rank":17,"benchmarks":[{"hardwareId":"m5_max_128_40c","engine":"mlx","decodeTps":52.2,"decodeTps32k":49.7,"prefillTps":31.4,"ttftMs":null,"contextTokens":258047,"measuredAt":"2026-07-04","notes":"mlx 0.31.3"},{"hardwareId":"m3_ultra_512_80c","engine":"mlx","decodeTps":51.3,"decodeTps32k":49.1,"prefillTps":20.1,"ttftMs":null,"contextTokens":258047,"measuredAt":"2026-07-05","notes":"mlx 0.31.3"}],"expectSpeed":{"decodeTps":52.2,"prefillTps":31.4,"source":"measured"},"params":"120B-A12B","notes":[]}
  }
}