{ "version": 1, "updated": "2026-07-26", "source": "local.ai publication pg-20260721-ui-refresh", "models": { "LiquidAI/LFM2.5-8B-A1B": {"name":"LFM2.5-8B-A1B","quant":"bf16","filesize":"16gb","filesizeGb":15.8,"hardware":[{"id":"rtx5090_32","label":"rtx5090_32","minMemoryGb":32,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve LiquidAI/LFM2.5-8B-A1B --served-model-name LiquidAI/LFM2.5-8B-A1B --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.85 --block-size 16 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 128000 --tool-call-parser lfm2 --mamba-cache-mode align"},"rank":1,"benchmarks":[{"hardwareId":"rtx5090_32","engine":"vllm","decodeTps":419.8,"decodeTps32k":619,"prefillTps":null,"ttftMs":null,"contextTokens":127872,"measuredAt":"2026-07-05","notes":"vllm unknown"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":348.1,"decodeTps32k":924.9,"prefillTps":null,"ttftMs":null,"contextTokens":127872,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":210.2,"decodeTps32k":251.2,"prefillTps":null,"ttftMs":null,"contextTokens":127872,"measuredAt":"2026-07-09","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":55.8,"decodeTps32k":55.4,"prefillTps":null,"ttftMs":null,"contextTokens":127872,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":419.8,"prefillTps":null,"source":"measured"},"params":"8B-A1B","notes":["quality 31.1% (tau2/gaia/gdpval mean)"]}, "Qwen/Qwen3.6-27B-FP8": {"name":"Qwen3.6-27B-FP8","quant":"fp8","filesize":"29gb","filesizeGb":28.7,"hardware":[{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve Qwen/Qwen3.6-27B-FP8 --served-model-name Qwen/Qwen3.6-27B-FP8 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 262144 --tool-call-parser qwen3_coder --reasoning-parser qwen3 --mamba-cache-mode align --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":4,\"rejection_sample_method\":\"standard\"}'"},"rank":1,"benchmarks":[{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":50.2,"decodeTps32k":50.8,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-18","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":7.8,"decodeTps32k":7.4,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":50.2,"prefillTps":null,"source":"measured"},"params":"27B","notes":["quality 91.5% (tau2/gaia/gdpval mean)"]}, "Qwen/Qwen3.6-35B-A3B": {"name":"Qwen3.6-35B-A3B","quant":"bf16","filesize":"67gb","filesizeGb":67,"hardware":[{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"gb300","label":"gb300","minMemoryGb":288,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve Qwen/Qwen3.6-35B-A3B --served-model-name Qwen/Qwen3.6-35B-A3B --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --mamba-cache-mode align --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":4,\"moe_backend\":\"triton\",\"rejection_sample_method\":\"standard\"}'"},"rank":1,"benchmarks":[{"hardwareId":"gb300","engine":"vllm","decodeTps":338.9,"decodeTps32k":367.5,"prefillTps":16359.1,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":170.1,"decodeTps32k":208,"prefillTps":5084.2,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-09","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":31,"decodeTps32k":29.7,"prefillTps":4436.1,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":338.9,"prefillTps":16359.1,"source":"measured"},"params":"35B-A3B","notes":["quality 82.2% (tau2/gaia/gdpval mean)"]}, "exolabs/lfm2-5-8b-a1b-mlx-4bit-dequant-bf16": {"name":"lfm2-5-8b-a1b-mlx-4bit-dequant-bf16","quant":"mlx","filesize":"16gb","filesizeGb":15.8,"hardware":[{"id":"rtx5090_32","label":"rtx5090_32","minMemoryGb":32,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve exolabs/lfm2-5-8b-a1b-mlx-4bit-dequant-bf16 --served-model-name exolabs/lfm2-5-8b-a1b-mlx-4bit-dequant-bf16 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 16 --enable-auto-tool-choice --async-scheduling --dtype bfloat16 --kv-cache-dtype bfloat16 --language-model-only --max-model-len 128000 --tool-call-parser lfm2 --moe-backend triton --attention-backend triton_attn --mamba-cache-mode align"},"rank":1,"benchmarks":[{"hardwareId":"rtx5090_32","engine":"vllm","decodeTps":361.3,"decodeTps32k":709.9,"prefillTps":null,"ttftMs":null,"contextTokens":127872,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":214.3,"decodeTps32k":282.9,"prefillTps":null,"ttftMs":null,"contextTokens":127872,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":56.4,"decodeTps32k":57,"prefillTps":null,"ttftMs":null,"contextTokens":127872,"measuredAt":"2026-07-15","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":361.3,"prefillTps":null,"source":"measured"},"params":null,"notes":["quality 35.7% (tau2/gaia/gdpval mean)"]}, "leon-se/gemma-4-E2B-it-FP8-Dynamic": {"name":"gemma-4-E2B-it-FP8-Dynamic","quant":"fp8","filesize":"9gb","filesizeGb":8.5,"hardware":[{"id":"rtx4090_24","label":"rtx4090_24","minMemoryGb":24,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtx5090_32","label":"rtx5090_32","minMemoryGb":32,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"gb300","label":"gb300","minMemoryGb":288,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve leon-se/gemma-4-E2B-it-FP8-Dynamic --served-model-name leon-se/gemma-4-E2B-it-FP8-Dynamic --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 16 --max-num-batched-tokens 8192 --attention-backend TRITON_ATTN --async-scheduling --enable-auto-tool-choice --language-model-only --limit-mm-per-prompt '{\"image\": 0, \"audio\": 0}' --max-model-len 131072 --tool-call-parser gemma4 --reasoning-parser gemma4 --kv-cache-dtype fp8 --chat-template /vllm-workspace/examples/tool_chat_template_gemma4.jinja --default-chat-template-kwargs '{\"enable_thinking\": true}'"},"rank":1,"benchmarks":[{"hardwareId":"gb300","engine":"vllm","decodeTps":494.3,"decodeTps32k":546.1,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-21","notes":"vllm 0.25.0"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":281.4,"decodeTps32k":16308.1,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-21","notes":"vllm unknown"},{"hardwareId":"rtx5090_32","engine":"vllm","decodeTps":268.6,"decodeTps32k":327.1,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-09","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":202.4,"decodeTps32k":311.7,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-05","notes":"vllm unknown"},{"hardwareId":"rtx4090_24","engine":"vllm","decodeTps":189.7,"decodeTps32k":208.9,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":66.9,"decodeTps32k":73.8,"prefillTps":null,"ttftMs":null,"contextTokens":126976,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":494.3,"prefillTps":null,"source":"measured"},"params":"2B","notes":["quality 13.9% (tau2/gaia/gdpval mean)"]}, "nvidia/GLM-5.2-NVFP4": {"name":"GLM-5.2-NVFP4","quant":"nvfp4","filesize":"433gb","filesizeGb":432.9,"hardware":[{"id":"gb300","label":"gb300","minMemoryGb":288,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"gb300x2","label":"gb300x2","minMemoryGb":576,"gpuCount":2,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve nvidia/GLM-5.2-NVFP4 --block-size 128 --trust-remote-code --kv-cache-dtype fp8 --enable-prefix-caching --gpu-memory-utilization 0.90 --kernel-config '{\"enable_flashinfer_autotune\":false}' --max-model-len 1048576 --cpu-offload-gb 267 --cpu-offload-params w13_weight w2_weight"},"rank":1,"benchmarks":[{"hardwareId":"gb300x2","engine":"vllm","decodeTps":41.9,"decodeTps32k":42.5,"prefillTps":null,"ttftMs":null,"contextTokens":749568,"measuredAt":"2026-07-18","notes":"vllm 0.25.0"},{"hardwareId":"gb300","engine":"vllm","decodeTps":28,"decodeTps32k":28.3,"prefillTps":null,"ttftMs":null,"contextTokens":1048448,"measuredAt":"2026-07-15","notes":"vllm unknown"}],"expectSpeed":{"decodeTps":41.9,"prefillTps":null,"source":"measured"},"params":null,"notes":["quality 98.4% (tau2/gaia/gdpval mean)"]}, "nvidia/Qwen3.6-27B-NVFP4": {"name":"Qwen3.6-27B-NVFP4","quant":"nvfp4","filesize":"20gb","filesizeGb":20.4,"hardware":[{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"gb300","label":"gb300","minMemoryGb":288,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"gb300x2","label":"gb300x2","minMemoryGb":576,"gpuCount":2,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve nvidia/Qwen3.6-27B-NVFP4 --served-model-name nvidia/Qwen3.6-27B-NVFP4 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 262144 --tool-call-parser qwen3_coder --reasoning-parser qwen3 --mamba-cache-mode align --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":4,\"rejection_sample_method\":\"standard\"}'"},"rank":1,"benchmarks":[{"hardwareId":"gb300","engine":"vllm","decodeTps":122.9,"decodeTps32k":125.2,"prefillTps":4033,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb300x2","engine":"vllm","decodeTps":115.8,"decodeTps32k":118.3,"prefillTps":4449.3,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-18","notes":"vllm 0.25.0"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":69.6,"decodeTps32k":71,"prefillTps":4660.3,"ttftMs":null,"contextTokens":73728,"measuredAt":"2026-07-13","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":42.3,"decodeTps32k":42,"prefillTps":3247.1,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-06","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":11.9,"decodeTps32k":11.5,"prefillTps":1149.2,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":122.9,"prefillTps":4660.3,"source":"measured"},"params":"27B","notes":["quality 82.2% (tau2/gaia/gdpval mean)"]}, "sakamakismile/Qwen3.6-27B-Text-NVFP4-MTP": {"name":"Qwen3.6-27B-Text-NVFP4-MTP","quant":"nvfp4","filesize":"18gb","filesizeGb":18.3,"hardware":[{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve sakamakismile/Qwen3.6-27B-Text-NVFP4-MTP --served-model-name sakamakismile/Qwen3.6-27B-Text-NVFP4-MTP --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --mamba-cache-mode align --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":4,\"rejection_sample_method\":\"standard\"}'"},"rank":1,"benchmarks":[{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":71.7,"decodeTps32k":75.2,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-09","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":12.2,"decodeTps32k":11.4,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":71.7,"prefillTps":null,"source":"measured"},"params":"27B","notes":["quality 82.3% (tau2/gaia/gdpval mean)"]}, "bottlecapai/ThinkingCap-Qwen3.6-27B": {"name":"ThinkingCap-Qwen3.6-27B","quant":"bf16","filesize":"52gb","filesizeGb":51.7,"hardware":[{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve bottlecapai/ThinkingCap-Qwen3.6-27B --served-model-name bottlecapai/ThinkingCap-Qwen3.6-27B --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --mamba-cache-mode align --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":4,\"rejection_sample_method\":\"standard\"}'"},"rank":2,"benchmarks":[{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":28.5,"decodeTps32k":28.7,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-21","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":4.5,"decodeTps32k":4.3,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":28.5,"prefillTps":null,"source":"measured"},"params":"27B","notes":["quality 90.9% (tau2/gaia/gdpval mean)"]}, "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4": {"name":"NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4","quant":"nvfp4","filesize":"18gb","filesizeGb":18,"hardware":[{"id":"rtx5090_32","label":"rtx5090_32","minMemoryGb":32,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4 --served-model-name nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 16 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser nemotron_v3 --kv-cache-dtype fp8 --mamba-cache-mode align"},"rank":2,"benchmarks":[{"hardwareId":"rtx5090_32","engine":"vllm","decodeTps":332.4,"decodeTps32k":484.6,"prefillTps":24089.9,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":163.5,"decodeTps32k":204.9,"prefillTps":19443.6,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-18","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":60.8,"decodeTps32k":62.6,"prefillTps":10911.4,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":332.4,"prefillTps":24089.9,"source":"measured"},"params":"30B-A3B","notes":["quality 39.8% (tau2/gaia/gdpval mean)"]}, "unsloth/Qwen3.6-35B-A3B-NVFP4": {"name":"Qwen3.6-35B-A3B-NVFP4","quant":"nvfp4","filesize":"25gb","filesizeGb":24.7,"hardware":[{"id":"rtx5090_32","label":"rtx5090_32","minMemoryGb":32,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"gb300","label":"gb300","minMemoryGb":288,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve unsloth/Qwen3.6-35B-A3B-NVFP4 --served-model-name unsloth/Qwen3.6-35B-A3B-NVFP4 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --enable-auto-tool-choice --async-scheduling --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --kv-cache-dtype fp8 --attention-backend flashinfer --max-num-batched-tokens 8192 --mamba-cache-mode align"},"rank":2,"benchmarks":[{"hardwareId":"gb300","engine":"vllm","decodeTps":329.3,"decodeTps32k":352.7,"prefillTps":20140.5,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-20","notes":"vllm 0.25.0"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":247.5,"decodeTps32k":258,"prefillTps":21383.2,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-18","notes":"vllm unknown"},{"hardwareId":"rtx5090_32","engine":"vllm","decodeTps":228.1,"decodeTps32k":257.2,"prefillTps":18901.1,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":181,"decodeTps32k":171,"prefillTps":20326.9,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":64,"decodeTps32k":61,"prefillTps":7003.2,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":329.3,"prefillTps":21383.2,"source":"measured"},"params":"35B-A3B","notes":["quality 82.1% (tau2/gaia/gdpval mean)"]}, "0xSero/Step-3.7-Flash-148B": {"name":"Step-3.7-Flash-148B","quant":"bf16","filesize":"94gb","filesizeGb":94,"hardware":[{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve 0xSero/Step-3.7-Flash-148B --served-model-name 0xSero/Step-3.7-Flash-148B --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 16 --max-num-batched-tokens 8192 --async-scheduling --language-model-only --max-model-len 262144 --quantization modelopt --kv-cache-dtype fp8 --reasoning-parser step3p5 --enable-auto-tool-choice --tool-call-parser step3p5"},"rank":3,"benchmarks":[{"hardwareId":"gb10_121","engine":"vllm","decodeTps":13.8,"decodeTps32k":13.7,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":13.8,"prefillTps":null,"source":"measured"},"params":"148B","notes":["quality 85.4% (tau2/gaia/gdpval mean)"]}, "Qwen/Qwen3.6-35B-A3B-FP8": {"name":"Qwen3.6-35B-A3B-FP8","quant":"fp8","filesize":"35gb","filesizeGb":34.9,"hardware":[{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"gb300","label":"gb300","minMemoryGb":288,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve Qwen/Qwen3.6-35B-A3B-FP8 --served-model-name Qwen/Qwen3.6-35B-A3B-FP8 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --mamba-cache-mode align --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":4,\"moe_backend\":\"triton\",\"rejection_sample_method\":\"standard\"}'"},"rank":3,"benchmarks":[{"hardwareId":"gb300","engine":"vllm","decodeTps":315,"decodeTps32k":337.3,"prefillTps":13921.8,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":221.2,"decodeTps32k":290.5,"prefillTps":5010.2,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-09","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":50.1,"decodeTps32k":46.9,"prefillTps":6136.3,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":315,"prefillTps":13921.8,"source":"measured"},"params":"35B-A3B","notes":["quality 81.2% (tau2/gaia/gdpval mean)"]}, "cyankiwi/Qwen3.5-9B-AWQ-4bit": {"name":"Qwen3.5-9B-AWQ-4bit","quant":"awq","filesize":"8gb","filesizeGb":8.4,"hardware":[{"id":"rtx4090_24","label":"rtx4090_24","minMemoryGb":24,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtx5090_32","label":"rtx5090_32","minMemoryGb":32,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"gb300","label":"gb300","minMemoryGb":288,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve cyankiwi/Qwen3.5-9B-AWQ-4bit --served-model-name cyankiwi/Qwen3.5-9B-AWQ-4bit --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 16 --max-num-batched-tokens 8192 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --mamba-cache-mode align"},"rank":3,"benchmarks":[{"hardwareId":"gb300","engine":"vllm","decodeTps":315.4,"decodeTps32k":336.6,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-20","notes":"vllm 0.25.0"},{"hardwareId":"rtx5090_32","engine":"vllm","decodeTps":205.7,"decodeTps32k":203.5,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-08","notes":"vllm unknown"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":202,"decodeTps32k":251.5,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"rtx4090_24","engine":"vllm","decodeTps":126.8,"decodeTps32k":125.5,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-09","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":119.9,"decodeTps32k":113.2,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-09","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":32.6,"decodeTps32k":29.8,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":315.4,"prefillTps":null,"source":"measured"},"params":"9B","notes":["quality 57.1% (tau2/gaia/gdpval mean)"]}, "nvidia/Qwen3.6-35B-A3B-NVFP4": {"name":"Qwen3.6-35B-A3B-NVFP4","quant":"nvfp4","filesize":"22gb","filesizeGb":21.8,"hardware":[{"id":"rtx5090_32","label":"rtx5090_32","minMemoryGb":32,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"gb300","label":"gb300","minMemoryGb":288,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve nvidia/Qwen3.6-35B-A3B-NVFP4 --served-model-name nvidia/Qwen3.6-35B-A3B-NVFP4 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --async-scheduling --enable-auto-tool-choice --language-model-only --attention-backend flashinfer --quantization modelopt --moe-backend marlin --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --kv-cache-dtype fp8 --mamba-cache-mode align --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\",\"rejection_sample_method\":\"standard\"}'"},"rank":3,"benchmarks":[{"hardwareId":"gb300","engine":"vllm","decodeTps":277.5,"decodeTps32k":306.5,"prefillTps":12732.2,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":259.1,"decodeTps32k":380.7,"prefillTps":16877.5,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-18","notes":"vllm unknown"},{"hardwareId":"rtx5090_32","engine":"vllm","decodeTps":235.6,"decodeTps32k":268.8,"prefillTps":16168.9,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":75.3,"decodeTps32k":71.8,"prefillTps":6780.3,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":277.5,"prefillTps":16877.5,"source":"measured"},"params":"35B-A3B","notes":["quality 81.6% (tau2/gaia/gdpval mean)"]}, "unsloth/Qwen3.6-27B-NVFP4": {"name":"Qwen3.6-27B-NVFP4","quant":"nvfp4","filesize":"22gb","filesizeGb":21.8,"hardware":[{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve unsloth/Qwen3.6-27B-NVFP4 --served-model-name unsloth/Qwen3.6-27B-NVFP4 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --async-scheduling --enable-auto-tool-choice --language-model-only --attention-backend flashinfer --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --kv-cache-dtype fp8 --mamba-cache-mode align"},"rank":3,"benchmarks":[{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":65.5,"decodeTps32k":69.7,"prefillTps":4492.9,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-18","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":38.2,"decodeTps32k":37.4,"prefillTps":3244.5,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":11.2,"decodeTps32k":10.8,"prefillTps":2295.1,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":65.5,"prefillTps":4492.9,"source":"measured"},"params":"27B","notes":["quality 82.1% (tau2/gaia/gdpval mean)"]}, "Hcompany/Holo-3.1-35B-A3B-NVFP4": {"name":"Holo-3.1-35B-A3B-NVFP4","quant":"nvfp4","filesize":"22gb","filesizeGb":22.1,"hardware":[{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"gb300","label":"gb300","minMemoryGb":288,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve Hcompany/Holo-3.1-35B-A3B-NVFP4 --served-model-name Hcompany/Holo-3.1-35B-A3B-NVFP4 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --async-scheduling --enable-auto-tool-choice --language-model-only --attention-backend flashinfer --quantization modelopt --moe-backend marlin --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --kv-cache-dtype fp8 --mamba-cache-mode align"},"rank":4,"benchmarks":[{"hardwareId":"gb300","engine":"vllm","decodeTps":287.8,"decodeTps32k":308.2,"prefillTps":14555.6,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":259.3,"decodeTps32k":388.7,"prefillTps":16762,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":74.4,"decodeTps32k":71.2,"prefillTps":6884.8,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":287.8,"prefillTps":16762,"source":"measured"},"params":"35B-A3B","notes":["quality 71.3% (tau2/gaia/gdpval mean)"]}, "cyankiwi/Qwen3.6-35B-A3B-AWQ-4bit": {"name":"Qwen3.6-35B-A3B-AWQ-4bit","quant":"awq","filesize":"23gb","filesizeGb":23.3,"hardware":[{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"gb300","label":"gb300","minMemoryGb":288,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve cyankiwi/Qwen3.6-35B-A3B-AWQ-4bit --served-model-name cyankiwi/Qwen3.6-35B-A3B-AWQ-4bit --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --max-num-batched-tokens 8192 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --mamba-cache-mode align"},"rank":4,"benchmarks":[{"hardwareId":"gb300","engine":"vllm","decodeTps":266.1,"decodeTps32k":279.1,"prefillTps":7811.1,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-21","notes":"vllm 0.25.0"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":213.2,"decodeTps32k":116.3,"prefillTps":4174.5,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":193.3,"decodeTps32k":245.2,"prefillTps":5166.4,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":36.5,"decodeTps32k":34.7,"prefillTps":6197.7,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":266.1,"prefillTps":7811.1,"source":"measured"},"params":"35B-A3B","notes":["quality 81.1% (tau2/gaia/gdpval mean)"]}, "exolabs/qwen3-6-27b-optiq-4bit-text-dequant-bf16": {"name":"qwen3-6-27b-optiq-4bit-text-dequant-bf16","quant":"mlx","filesize":"50gb","filesizeGb":50.1,"hardware":[{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve exolabs/qwen3-6-27b-optiq-4bit-text-dequant-bf16 --served-model-name exolabs/qwen3-6-27b-optiq-4bit-text-dequant-bf16 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --max-num-batched-tokens 8192 --enable-auto-tool-choice --async-scheduling --dtype bfloat16 --kv-cache-dtype bfloat16 --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --attention-backend triton_attn --mamba-cache-mode align"},"rank":4,"benchmarks":[{"hardwareId":"gb10_121","engine":"vllm","decodeTps":4.4,"decodeTps32k":4.3,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-18","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":4.4,"prefillTps":null,"source":"measured"},"params":null,"notes":["quality 81.7% (tau2/gaia/gdpval mean)"]}, "exolabs/qwen3-6-27b-ud-mlx-4bit-text-dequant-bf16-normfix": {"name":"qwen3-6-27b-ud-mlx-4bit-text-dequant-bf16-normfix","quant":"mlx","filesize":"17gb","filesizeGb":17,"hardware":[{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve exolabs/qwen3-6-27b-ud-mlx-4bit-text-dequant-bf16-normfix"},"rank":4,"benchmarks":[{"hardwareId":"gb10_121","engine":"vllm","decodeTps":4.5,"decodeTps32k":4.3,"prefillTps":null,"ttftMs":null,"contextTokens":155904,"measuredAt":"2026-07-15","notes":"vllm unknown"}],"expectSpeed":{"decodeTps":4.5,"prefillTps":null,"source":"measured"},"params":null,"notes":["quality 81.4% (tau2/gaia/gdpval mean)","size estimated from parameter count"]}, "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4": {"name":"Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4","quant":"nvfp4","filesize":"21gb","filesizeGb":20.9,"hardware":[{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4 --served-model-name nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 16 --async-scheduling --enable-auto-tool-choice --max-model-len 131072 --tool-call-parser qwen3_coder --reasoning-parser nemotron_v3 --kv-cache-dtype fp8"},"rank":4,"benchmarks":[{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":307.9,"decodeTps32k":402.9,"prefillTps":23766.4,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-18","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":55.9,"decodeTps32k":56.8,"prefillTps":9566.3,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":307.9,"prefillTps":23766.4,"source":"measured"},"params":"30B-A3B","notes":["quality 38.9% (tau2/gaia/gdpval mean)"]}, "openai/gpt-oss-20b": {"name":"gpt-oss-20b","quant":"bf16","filesize":"26gb","filesizeGb":25.6,"hardware":[{"id":"rtx4090_24","label":"rtx4090_24","minMemoryGb":24,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtx5090_32","label":"rtx5090_32","minMemoryGb":32,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve openai/gpt-oss-20b --served-model-name openai/gpt-oss-20b --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 16 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 131072 --tool-call-parser openai --kv-cache-dtype fp8 --max-cudagraph-capture-size 2048 --max-num-batched-tokens 8192 --stream-interval 20"},"rank":4,"benchmarks":[{"hardwareId":"rtx5090_32","engine":"vllm","decodeTps":289.3,"decodeTps32k":304,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-09","notes":"vllm unknown"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":282.2,"decodeTps32k":380.9,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-18","notes":"vllm unknown"},{"hardwareId":"rtx4090_24","engine":"vllm","decodeTps":192.7,"decodeTps32k":263.8,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-09","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":186.5,"decodeTps32k":197.8,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-05","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":46.3,"decodeTps32k":45.7,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":289.3,"prefillTps":null,"source":"measured"},"params":null,"notes":["quality 52.9% (tau2/gaia/gdpval mean)"]}, "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8": {"name":"NVIDIA-Nemotron-3-Nano-30B-A3B-FP8","quant":"fp8","filesize":"30gb","filesizeGb":30.4,"hardware":[{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 --served-model-name nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 16 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 262144 --tool-call-parser qwen3_coder --reasoning-parser nemotron_v3 --kv-cache-dtype fp8 --mamba-cache-mode align"},"rank":5,"benchmarks":[{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":262.1,"decodeTps32k":405.3,"prefillTps":23306.9,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-18","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":176.8,"decodeTps32k":199.8,"prefillTps":24418.9,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":44.6,"decodeTps32k":45.3,"prefillTps":7831.6,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":262.1,"prefillTps":24418.9,"source":"measured"},"params":"30B-A3B","notes":["quality 42.3% (tau2/gaia/gdpval mean)"]}, "unsloth/Qwen3.6-35B-A3B-NVFP4-Fast": {"name":"Qwen3.6-35B-A3B-NVFP4-Fast","quant":"nvfp4","filesize":"22gb","filesizeGb":22,"hardware":[{"id":"rtx5090_32","label":"rtx5090_32","minMemoryGb":32,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve unsloth/Qwen3.6-35B-A3B-NVFP4-Fast --served-model-name unsloth/Qwen3.6-35B-A3B-NVFP4-Fast --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --enable-auto-tool-choice --async-scheduling --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --kv-cache-dtype fp8 --attention-backend flashinfer --max-num-batched-tokens 8192 --mamba-cache-mode align"},"rank":5,"benchmarks":[{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":241.6,"decodeTps32k":344.6,"prefillTps":19182.8,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-18","notes":"vllm unknown"},{"hardwareId":"rtx5090_32","engine":"vllm","decodeTps":240.1,"decodeTps32k":276.2,"prefillTps":16637,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":184.2,"decodeTps32k":176.2,"prefillTps":21078.9,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":69.2,"decodeTps32k":66.4,"prefillTps":7197.8,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-18","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":241.6,"prefillTps":21078.9,"source":"measured"},"params":"35B-A3B","notes":["quality 80.9% (tau2/gaia/gdpval mean)"]}, "InternScience/Agents-A1-FP8": {"name":"Agents-A1-FP8","quant":"fp8","filesize":"35gb","filesizeGb":35.1,"hardware":[{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve InternScience/Agents-A1-FP8 --served-model-name InternScience/Agents-A1-FP8 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --mamba-cache-mode align"},"rank":6,"benchmarks":[{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":196.9,"decodeTps32k":250.2,"prefillTps":4946.6,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":38.1,"decodeTps32k":36.4,"prefillTps":5533.2,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-13","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":196.9,"prefillTps":5533.2,"source":"measured"},"params":null,"notes":["quality 79.4% (tau2/gaia/gdpval mean)"]}, "RedHatAI/Qwen3.6-35B-A3B-NVFP4": {"name":"Qwen3.6-35B-A3B-NVFP4","quant":"nvfp4","filesize":"23gb","filesizeGb":23.3,"hardware":[{"id":"rtx5090_32","label":"rtx5090_32","minMemoryGb":32,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"gb300","label":"gb300","minMemoryGb":288,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve RedHatAI/Qwen3.6-35B-A3B-NVFP4 --served-model-name RedHatAI/Qwen3.6-35B-A3B-NVFP4 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --kv-cache-dtype fp8 --attention-backend flashinfer --moe-backend flashinfer_cutlass --max-num-batched-tokens 8192 --mamba-cache-mode align"},"rank":6,"benchmarks":[{"hardwareId":"gb300","engine":"vllm","decodeTps":245.9,"decodeTps32k":276.5,"prefillTps":21495.5,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-20","notes":"vllm 0.25.0"},{"hardwareId":"rtx5090_32","engine":"vllm","decodeTps":213.5,"decodeTps32k":347.5,"prefillTps":18559.6,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-09","notes":"vllm unknown"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":209.6,"decodeTps32k":324,"prefillTps":18754.1,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-09","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":40.9,"decodeTps32k":40.2,"prefillTps":6708.6,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":245.9,"prefillTps":21495.5,"source":"measured"},"params":"35B-A3B","notes":["quality 40.8% (tau2/gaia/gdpval mean)"]}, "RedHatAI/gemma-4-26B-A4B-it-FP8-Dynamic": {"name":"gemma-4-26B-A4B-it-FP8-Dynamic","quant":"fp8","filesize":"27gb","filesizeGb":26.7,"hardware":[{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve RedHatAI/gemma-4-26B-A4B-it-FP8-Dynamic --served-model-name RedHatAI/gemma-4-26B-A4B-it-FP8-Dynamic --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 16 --enable-auto-tool-choice --revision 951346384683e0c783e8acbaf649418d6c77dd4e --tokenizer-revision 951346384683e0c783e8acbaf649418d6c77dd4e --async-scheduling --language-model-only --max-model-len 262144 --reasoning-parser gemma4 --tool-call-parser gemma4 --kv-cache-dtype bfloat16 --chat-template /vllm-workspace/examples/tool_chat_template_gemma4.jinja --default-chat-template-kwargs '{\"enable_thinking\": true}'"},"rank":6,"benchmarks":[{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":201.5,"decodeTps32k":536.7,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-18","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":132.3,"decodeTps32k":134.4,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":38.1,"decodeTps32k":37.4,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":201.5,"prefillTps":null,"source":"measured"},"params":"26B-A4B","notes":["quality 69.3% (tau2/gaia/gdpval mean)"]}, "cyankiwi/gemma-4-26B-A4B-it-AWQ-4bit": {"name":"gemma-4-26B-A4B-it-AWQ-4bit","quant":"awq","filesize":"16gb","filesizeGb":16,"hardware":[{"id":"rtx5090_32","label":"rtx5090_32","minMemoryGb":32,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve cyankiwi/gemma-4-26B-A4B-it-AWQ-4bit --served-model-name cyankiwi/gemma-4-26B-A4B-it-AWQ-4bit --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.85 --block-size 16 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 131072 --reasoning-parser gemma4 --tool-call-parser gemma4 --chat-template examples/tool_chat_template_gemma4.jinja --default-chat-template-kwargs '{\"enable_thinking\": true}' --speculative-config '{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4,\"rejection_sample_method\":\"standard\"}'"},"rank":6,"benchmarks":[{"hardwareId":"rtx5090_32","engine":"vllm","decodeTps":217.8,"decodeTps32k":214.3,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-08","notes":"vllm unknown"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":209.4,"decodeTps32k":727.6,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":151.9,"decodeTps32k":163.8,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-09","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":42.4,"decodeTps32k":41.1,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":217.8,"prefillTps":null,"source":"measured"},"params":"26B-A4B","notes":["quality 63.8% (tau2/gaia/gdpval mean)"]}, "deepreinforce-ai/Ornith-1.0-35B-FP8": {"name":"Ornith-1.0-35B-FP8","quant":"fp8","filesize":"35gb","filesizeGb":35.1,"hardware":[{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve deepreinforce-ai/Ornith-1.0-35B-FP8 --served-model-name deepreinforce-ai/Ornith-1.0-35B-FP8 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --mamba-cache-mode align"},"rank":6,"benchmarks":[{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":196.4,"decodeTps32k":249.3,"prefillTps":4944.9,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":36.3,"decodeTps32k":34.4,"prefillTps":5357.6,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":196.4,"prefillTps":5357.6,"source":"measured"},"params":"35B","notes":["quality 80.7% (tau2/gaia/gdpval mean)"]}, "Jackrong/Qwopus3.6-27B-v2": {"name":"Qwopus3.6-27B-v2","quant":"bf16","filesize":"52gb","filesizeGb":51.7,"hardware":[{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve Jackrong/Qwopus3.6-27B-v2 --served-model-name Jackrong/Qwopus3.6-27B-v2 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.85 --block-size 1024 --revision 66a4ee9d49dbcef3f83528a400ef6bec93684d6b --tokenizer-revision 66a4ee9d49dbcef3f83528a400ef6bec93684d6b --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 262144 --tool-call-parser qwen3_coder --reasoning-parser qwen3 --mamba-cache-mode align"},"rank":7,"benchmarks":[{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":28.5,"decodeTps32k":29.3,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":4.4,"decodeTps32k":4.3,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-14","notes":"vllm 0.1.dev16488+g129a28dc2"}],"expectSpeed":{"decodeTps":28.5,"prefillTps":null,"source":"measured"},"params":"27B","notes":["quality 80% (tau2/gaia/gdpval mean)"]}, "exolabs/agents-a1-8bit-text-dequant-bf16": {"name":"agents-a1-8bit-text-dequant-bf16","quant":"mlx","filesize":"65gb","filesizeGb":64.6,"hardware":[{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve exolabs/agents-a1-8bit-text-dequant-bf16 --served-model-name exolabs/agents-a1-8bit-text-dequant-bf16 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --enable-auto-tool-choice --async-scheduling --dtype bfloat16 --kv-cache-dtype bfloat16 --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --moe-backend triton --attention-backend triton_attn"},"rank":7,"benchmarks":[{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":178.4,"decodeTps32k":187.3,"prefillTps":5131.3,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-21","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":30.5,"decodeTps32k":28.6,"prefillTps":3251.6,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":178.4,"prefillTps":5131.3,"source":"measured"},"params":null,"notes":["quality 76.2% (tau2/gaia/gdpval mean)"]}, "exolabs/qwen3-6-35b-a3b-ud-mlx-4bit-text-dequant-bf16-normfix": {"name":"qwen3-6-35b-a3b-ud-mlx-4bit-text-dequant-bf16-normfix","quant":"mlx","filesize":"23gb","filesizeGb":23,"hardware":[{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve exolabs/qwen3-6-35b-a3b-ud-mlx-4bit-text-dequant-bf16-normfix --served-model-name exolabs/qwen3-6-35b-a3b-ud-mlx-4bit-text-dequant-bf16-normfix --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --max-num-batched-tokens 8192 --enable-auto-tool-choice --async-scheduling --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --mamba-cache-mode align"},"rank":7,"benchmarks":[{"hardwareId":"gb10_121","engine":"vllm","decodeTps":29.7,"decodeTps32k":29.1,"prefillTps":2638.9,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":29.7,"prefillTps":2638.9,"source":"measured"},"params":null,"notes":["quality 77.7% (tau2/gaia/gdpval mean)","size estimated from parameter count"]}, "leon-se/gemma-4-E4B-it-FP8-Dynamic": {"name":"gemma-4-E4B-it-FP8-Dynamic","quant":"fp8","filesize":"12gb","filesizeGb":12.4,"hardware":[{"id":"rtx4090_24","label":"rtx4090_24","minMemoryGb":24,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtx5090_32","label":"rtx5090_32","minMemoryGb":32,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"gb300","label":"gb300","minMemoryGb":288,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve leon-se/gemma-4-E4B-it-FP8-Dynamic --served-model-name leon-se/gemma-4-E4B-it-FP8-Dynamic --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 16 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 131072 --tool-call-parser gemma4 --reasoning-parser gemma4 --kv-cache-dtype fp8 --chat-template /vllm-workspace/examples/tool_chat_template_gemma4.jinja --default-chat-template-kwargs '{\"enable_thinking\": true}' --speculative-config '{\"model\":\"google/gemma-4-E4B-it-assistant\",\"num_speculative_tokens\":4,\"rejection_sample_method\":\"standard\"}'"},"rank":7,"benchmarks":[{"hardwareId":"gb300","engine":"vllm","decodeTps":221.9,"decodeTps32k":191.5,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-18","notes":"vllm 0.25.0"},{"hardwareId":"rtx5090_32","engine":"vllm","decodeTps":212,"decodeTps32k":389.2,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-05","notes":"vllm unknown"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":176.9,"decodeTps32k":551.9,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-21","notes":"vllm unknown"},{"hardwareId":"rtx4090_24","engine":"vllm","decodeTps":122,"decodeTps32k":48.7,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-18","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":117.9,"decodeTps32k":48.5,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-18","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":35,"decodeTps32k":34.9,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-04","notes":"vllm unknown"}],"expectSpeed":{"decodeTps":221.9,"prefillTps":null,"source":"measured"},"params":"4B","notes":["quality 21.1% (tau2/gaia/gdpval mean)"]},Lines 1–38 of 118. Load more
{ "version": 1, "updated": "2026-07-26", "source": "local.ai publication pg-20260721-ui-refresh", "models": { "LiquidAI/LFM2.5-8B-A1B": {"name":"LFM2.5-8B-A1B","quant":"bf16","filesize":"16gb","filesizeGb":15.8,"hardware":[{"id":"rtx5090_32","label":"rtx5090_32","minMemoryGb":32,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve LiquidAI/LFM2.5-8B-A1B --served-model-name LiquidAI/LFM2.5-8B-A1B --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.85 --block-size 16 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 128000 --tool-call-parser lfm2 --mamba-cache-mode align"},"rank":1,"benchmarks":[{"hardwareId":"rtx5090_32","engine":"vllm","decodeTps":419.8,"decodeTps32k":619,"prefillTps":null,"ttftMs":null,"contextTokens":127872,"measuredAt":"2026-07-05","notes":"vllm unknown"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":348.1,"decodeTps32k":924.9,"prefillTps":null,"ttftMs":null,"contextTokens":127872,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":210.2,"decodeTps32k":251.2,"prefillTps":null,"ttftMs":null,"contextTokens":127872,"measuredAt":"2026-07-09","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":55.8,"decodeTps32k":55.4,"prefillTps":null,"ttftMs":null,"contextTokens":127872,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":419.8,"prefillTps":null,"source":"measured"},"params":"8B-A1B","notes":["quality 31.1% (tau2/gaia/gdpval mean)"]}, "Qwen/Qwen3.6-27B-FP8": {"name":"Qwen3.6-27B-FP8","quant":"fp8","filesize":"29gb","filesizeGb":28.7,"hardware":[{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve Qwen/Qwen3.6-27B-FP8 --served-model-name Qwen/Qwen3.6-27B-FP8 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 262144 --tool-call-parser qwen3_coder --reasoning-parser qwen3 --mamba-cache-mode align --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":4,\"rejection_sample_method\":\"standard\"}'"},"rank":1,"benchmarks":[{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":50.2,"decodeTps32k":50.8,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-18","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":7.8,"decodeTps32k":7.4,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":50.2,"prefillTps":null,"source":"measured"},"params":"27B","notes":["quality 91.5% (tau2/gaia/gdpval mean)"]}, "Qwen/Qwen3.6-35B-A3B": {"name":"Qwen3.6-35B-A3B","quant":"bf16","filesize":"67gb","filesizeGb":67,"hardware":[{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"gb300","label":"gb300","minMemoryGb":288,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve Qwen/Qwen3.6-35B-A3B --served-model-name Qwen/Qwen3.6-35B-A3B --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --mamba-cache-mode align --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":4,\"moe_backend\":\"triton\",\"rejection_sample_method\":\"standard\"}'"},"rank":1,"benchmarks":[{"hardwareId":"gb300","engine":"vllm","decodeTps":338.9,"decodeTps32k":367.5,"prefillTps":16359.1,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":170.1,"decodeTps32k":208,"prefillTps":5084.2,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-09","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":31,"decodeTps32k":29.7,"prefillTps":4436.1,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":338.9,"prefillTps":16359.1,"source":"measured"},"params":"35B-A3B","notes":["quality 82.2% (tau2/gaia/gdpval mean)"]}, "exolabs/lfm2-5-8b-a1b-mlx-4bit-dequant-bf16": {"name":"lfm2-5-8b-a1b-mlx-4bit-dequant-bf16","quant":"mlx","filesize":"16gb","filesizeGb":15.8,"hardware":[{"id":"rtx5090_32","label":"rtx5090_32","minMemoryGb":32,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve exolabs/lfm2-5-8b-a1b-mlx-4bit-dequant-bf16 --served-model-name exolabs/lfm2-5-8b-a1b-mlx-4bit-dequant-bf16 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 16 --enable-auto-tool-choice --async-scheduling --dtype bfloat16 --kv-cache-dtype bfloat16 --language-model-only --max-model-len 128000 --tool-call-parser lfm2 --moe-backend triton --attention-backend triton_attn --mamba-cache-mode align"},"rank":1,"benchmarks":[{"hardwareId":"rtx5090_32","engine":"vllm","decodeTps":361.3,"decodeTps32k":709.9,"prefillTps":null,"ttftMs":null,"contextTokens":127872,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":214.3,"decodeTps32k":282.9,"prefillTps":null,"ttftMs":null,"contextTokens":127872,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":56.4,"decodeTps32k":57,"prefillTps":null,"ttftMs":null,"contextTokens":127872,"measuredAt":"2026-07-15","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":361.3,"prefillTps":null,"source":"measured"},"params":null,"notes":["quality 35.7% (tau2/gaia/gdpval mean)"]}, "leon-se/gemma-4-E2B-it-FP8-Dynamic": {"name":"gemma-4-E2B-it-FP8-Dynamic","quant":"fp8","filesize":"9gb","filesizeGb":8.5,"hardware":[{"id":"rtx4090_24","label":"rtx4090_24","minMemoryGb":24,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtx5090_32","label":"rtx5090_32","minMemoryGb":32,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"gb300","label":"gb300","minMemoryGb":288,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve leon-se/gemma-4-E2B-it-FP8-Dynamic --served-model-name leon-se/gemma-4-E2B-it-FP8-Dynamic --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 16 --max-num-batched-tokens 8192 --attention-backend TRITON_ATTN --async-scheduling --enable-auto-tool-choice --language-model-only --limit-mm-per-prompt '{\"image\": 0, \"audio\": 0}' --max-model-len 131072 --tool-call-parser gemma4 --reasoning-parser gemma4 --kv-cache-dtype fp8 --chat-template /vllm-workspace/examples/tool_chat_template_gemma4.jinja --default-chat-template-kwargs '{\"enable_thinking\": true}'"},"rank":1,"benchmarks":[{"hardwareId":"gb300","engine":"vllm","decodeTps":494.3,"decodeTps32k":546.1,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-21","notes":"vllm 0.25.0"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":281.4,"decodeTps32k":16308.1,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-21","notes":"vllm unknown"},{"hardwareId":"rtx5090_32","engine":"vllm","decodeTps":268.6,"decodeTps32k":327.1,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-09","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":202.4,"decodeTps32k":311.7,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-05","notes":"vllm unknown"},{"hardwareId":"rtx4090_24","engine":"vllm","decodeTps":189.7,"decodeTps32k":208.9,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":66.9,"decodeTps32k":73.8,"prefillTps":null,"ttftMs":null,"contextTokens":126976,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":494.3,"prefillTps":null,"source":"measured"},"params":"2B","notes":["quality 13.9% (tau2/gaia/gdpval mean)"]}, "nvidia/GLM-5.2-NVFP4": {"name":"GLM-5.2-NVFP4","quant":"nvfp4","filesize":"433gb","filesizeGb":432.9,"hardware":[{"id":"gb300","label":"gb300","minMemoryGb":288,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"gb300x2","label":"gb300x2","minMemoryGb":576,"gpuCount":2,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve nvidia/GLM-5.2-NVFP4 --block-size 128 --trust-remote-code --kv-cache-dtype fp8 --enable-prefix-caching --gpu-memory-utilization 0.90 --kernel-config '{\"enable_flashinfer_autotune\":false}' --max-model-len 1048576 --cpu-offload-gb 267 --cpu-offload-params w13_weight w2_weight"},"rank":1,"benchmarks":[{"hardwareId":"gb300x2","engine":"vllm","decodeTps":41.9,"decodeTps32k":42.5,"prefillTps":null,"ttftMs":null,"contextTokens":749568,"measuredAt":"2026-07-18","notes":"vllm 0.25.0"},{"hardwareId":"gb300","engine":"vllm","decodeTps":28,"decodeTps32k":28.3,"prefillTps":null,"ttftMs":null,"contextTokens":1048448,"measuredAt":"2026-07-15","notes":"vllm unknown"}],"expectSpeed":{"decodeTps":41.9,"prefillTps":null,"source":"measured"},"params":null,"notes":["quality 98.4% (tau2/gaia/gdpval mean)"]}, "nvidia/Qwen3.6-27B-NVFP4": {"name":"Qwen3.6-27B-NVFP4","quant":"nvfp4","filesize":"20gb","filesizeGb":20.4,"hardware":[{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"gb300","label":"gb300","minMemoryGb":288,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"gb300x2","label":"gb300x2","minMemoryGb":576,"gpuCount":2,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve nvidia/Qwen3.6-27B-NVFP4 --served-model-name nvidia/Qwen3.6-27B-NVFP4 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 262144 --tool-call-parser qwen3_coder --reasoning-parser qwen3 --mamba-cache-mode align --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":4,\"rejection_sample_method\":\"standard\"}'"},"rank":1,"benchmarks":[{"hardwareId":"gb300","engine":"vllm","decodeTps":122.9,"decodeTps32k":125.2,"prefillTps":4033,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb300x2","engine":"vllm","decodeTps":115.8,"decodeTps32k":118.3,"prefillTps":4449.3,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-18","notes":"vllm 0.25.0"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":69.6,"decodeTps32k":71,"prefillTps":4660.3,"ttftMs":null,"contextTokens":73728,"measuredAt":"2026-07-13","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":42.3,"decodeTps32k":42,"prefillTps":3247.1,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-06","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":11.9,"decodeTps32k":11.5,"prefillTps":1149.2,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":122.9,"prefillTps":4660.3,"source":"measured"},"params":"27B","notes":["quality 82.2% (tau2/gaia/gdpval mean)"]}, "sakamakismile/Qwen3.6-27B-Text-NVFP4-MTP": {"name":"Qwen3.6-27B-Text-NVFP4-MTP","quant":"nvfp4","filesize":"18gb","filesizeGb":18.3,"hardware":[{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve sakamakismile/Qwen3.6-27B-Text-NVFP4-MTP --served-model-name sakamakismile/Qwen3.6-27B-Text-NVFP4-MTP --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --mamba-cache-mode align --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":4,\"rejection_sample_method\":\"standard\"}'"},"rank":1,"benchmarks":[{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":71.7,"decodeTps32k":75.2,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-09","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":12.2,"decodeTps32k":11.4,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":71.7,"prefillTps":null,"source":"measured"},"params":"27B","notes":["quality 82.3% (tau2/gaia/gdpval mean)"]}, "bottlecapai/ThinkingCap-Qwen3.6-27B": {"name":"ThinkingCap-Qwen3.6-27B","quant":"bf16","filesize":"52gb","filesizeGb":51.7,"hardware":[{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve bottlecapai/ThinkingCap-Qwen3.6-27B --served-model-name bottlecapai/ThinkingCap-Qwen3.6-27B --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --mamba-cache-mode align --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":4,\"rejection_sample_method\":\"standard\"}'"},"rank":2,"benchmarks":[{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":28.5,"decodeTps32k":28.7,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-21","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":4.5,"decodeTps32k":4.3,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":28.5,"prefillTps":null,"source":"measured"},"params":"27B","notes":["quality 90.9% (tau2/gaia/gdpval mean)"]}, "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4": {"name":"NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4","quant":"nvfp4","filesize":"18gb","filesizeGb":18,"hardware":[{"id":"rtx5090_32","label":"rtx5090_32","minMemoryGb":32,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4 --served-model-name nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 16 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser nemotron_v3 --kv-cache-dtype fp8 --mamba-cache-mode align"},"rank":2,"benchmarks":[{"hardwareId":"rtx5090_32","engine":"vllm","decodeTps":332.4,"decodeTps32k":484.6,"prefillTps":24089.9,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":163.5,"decodeTps32k":204.9,"prefillTps":19443.6,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-18","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":60.8,"decodeTps32k":62.6,"prefillTps":10911.4,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":332.4,"prefillTps":24089.9,"source":"measured"},"params":"30B-A3B","notes":["quality 39.8% (tau2/gaia/gdpval mean)"]}, "unsloth/Qwen3.6-35B-A3B-NVFP4": {"name":"Qwen3.6-35B-A3B-NVFP4","quant":"nvfp4","filesize":"25gb","filesizeGb":24.7,"hardware":[{"id":"rtx5090_32","label":"rtx5090_32","minMemoryGb":32,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"gb300","label":"gb300","minMemoryGb":288,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve unsloth/Qwen3.6-35B-A3B-NVFP4 --served-model-name unsloth/Qwen3.6-35B-A3B-NVFP4 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --enable-auto-tool-choice --async-scheduling --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --kv-cache-dtype fp8 --attention-backend flashinfer --max-num-batched-tokens 8192 --mamba-cache-mode align"},"rank":2,"benchmarks":[{"hardwareId":"gb300","engine":"vllm","decodeTps":329.3,"decodeTps32k":352.7,"prefillTps":20140.5,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-20","notes":"vllm 0.25.0"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":247.5,"decodeTps32k":258,"prefillTps":21383.2,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-18","notes":"vllm unknown"},{"hardwareId":"rtx5090_32","engine":"vllm","decodeTps":228.1,"decodeTps32k":257.2,"prefillTps":18901.1,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":181,"decodeTps32k":171,"prefillTps":20326.9,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":64,"decodeTps32k":61,"prefillTps":7003.2,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":329.3,"prefillTps":21383.2,"source":"measured"},"params":"35B-A3B","notes":["quality 82.1% (tau2/gaia/gdpval mean)"]}, "0xSero/Step-3.7-Flash-148B": {"name":"Step-3.7-Flash-148B","quant":"bf16","filesize":"94gb","filesizeGb":94,"hardware":[{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve 0xSero/Step-3.7-Flash-148B --served-model-name 0xSero/Step-3.7-Flash-148B --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 16 --max-num-batched-tokens 8192 --async-scheduling --language-model-only --max-model-len 262144 --quantization modelopt --kv-cache-dtype fp8 --reasoning-parser step3p5 --enable-auto-tool-choice --tool-call-parser step3p5"},"rank":3,"benchmarks":[{"hardwareId":"gb10_121","engine":"vllm","decodeTps":13.8,"decodeTps32k":13.7,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":13.8,"prefillTps":null,"source":"measured"},"params":"148B","notes":["quality 85.4% (tau2/gaia/gdpval mean)"]}, "Qwen/Qwen3.6-35B-A3B-FP8": {"name":"Qwen3.6-35B-A3B-FP8","quant":"fp8","filesize":"35gb","filesizeGb":34.9,"hardware":[{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"gb300","label":"gb300","minMemoryGb":288,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve Qwen/Qwen3.6-35B-A3B-FP8 --served-model-name Qwen/Qwen3.6-35B-A3B-FP8 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --mamba-cache-mode align --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":4,\"moe_backend\":\"triton\",\"rejection_sample_method\":\"standard\"}'"},"rank":3,"benchmarks":[{"hardwareId":"gb300","engine":"vllm","decodeTps":315,"decodeTps32k":337.3,"prefillTps":13921.8,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":221.2,"decodeTps32k":290.5,"prefillTps":5010.2,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-09","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":50.1,"decodeTps32k":46.9,"prefillTps":6136.3,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":315,"prefillTps":13921.8,"source":"measured"},"params":"35B-A3B","notes":["quality 81.2% (tau2/gaia/gdpval mean)"]}, "cyankiwi/Qwen3.5-9B-AWQ-4bit": {"name":"Qwen3.5-9B-AWQ-4bit","quant":"awq","filesize":"8gb","filesizeGb":8.4,"hardware":[{"id":"rtx4090_24","label":"rtx4090_24","minMemoryGb":24,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtx5090_32","label":"rtx5090_32","minMemoryGb":32,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"gb300","label":"gb300","minMemoryGb":288,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve cyankiwi/Qwen3.5-9B-AWQ-4bit --served-model-name cyankiwi/Qwen3.5-9B-AWQ-4bit --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 16 --max-num-batched-tokens 8192 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --mamba-cache-mode align"},"rank":3,"benchmarks":[{"hardwareId":"gb300","engine":"vllm","decodeTps":315.4,"decodeTps32k":336.6,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-20","notes":"vllm 0.25.0"},{"hardwareId":"rtx5090_32","engine":"vllm","decodeTps":205.7,"decodeTps32k":203.5,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-08","notes":"vllm unknown"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":202,"decodeTps32k":251.5,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"rtx4090_24","engine":"vllm","decodeTps":126.8,"decodeTps32k":125.5,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-09","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":119.9,"decodeTps32k":113.2,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-09","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":32.6,"decodeTps32k":29.8,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":315.4,"prefillTps":null,"source":"measured"},"params":"9B","notes":["quality 57.1% (tau2/gaia/gdpval mean)"]}, "nvidia/Qwen3.6-35B-A3B-NVFP4": {"name":"Qwen3.6-35B-A3B-NVFP4","quant":"nvfp4","filesize":"22gb","filesizeGb":21.8,"hardware":[{"id":"rtx5090_32","label":"rtx5090_32","minMemoryGb":32,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"gb300","label":"gb300","minMemoryGb":288,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve nvidia/Qwen3.6-35B-A3B-NVFP4 --served-model-name nvidia/Qwen3.6-35B-A3B-NVFP4 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --async-scheduling --enable-auto-tool-choice --language-model-only --attention-backend flashinfer --quantization modelopt --moe-backend marlin --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --kv-cache-dtype fp8 --mamba-cache-mode align --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\",\"rejection_sample_method\":\"standard\"}'"},"rank":3,"benchmarks":[{"hardwareId":"gb300","engine":"vllm","decodeTps":277.5,"decodeTps32k":306.5,"prefillTps":12732.2,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":259.1,"decodeTps32k":380.7,"prefillTps":16877.5,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-18","notes":"vllm unknown"},{"hardwareId":"rtx5090_32","engine":"vllm","decodeTps":235.6,"decodeTps32k":268.8,"prefillTps":16168.9,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":75.3,"decodeTps32k":71.8,"prefillTps":6780.3,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":277.5,"prefillTps":16877.5,"source":"measured"},"params":"35B-A3B","notes":["quality 81.6% (tau2/gaia/gdpval mean)"]}, "unsloth/Qwen3.6-27B-NVFP4": {"name":"Qwen3.6-27B-NVFP4","quant":"nvfp4","filesize":"22gb","filesizeGb":21.8,"hardware":[{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve unsloth/Qwen3.6-27B-NVFP4 --served-model-name unsloth/Qwen3.6-27B-NVFP4 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --async-scheduling --enable-auto-tool-choice --language-model-only --attention-backend flashinfer --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --kv-cache-dtype fp8 --mamba-cache-mode align"},"rank":3,"benchmarks":[{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":65.5,"decodeTps32k":69.7,"prefillTps":4492.9,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-18","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":38.2,"decodeTps32k":37.4,"prefillTps":3244.5,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":11.2,"decodeTps32k":10.8,"prefillTps":2295.1,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":65.5,"prefillTps":4492.9,"source":"measured"},"params":"27B","notes":["quality 82.1% (tau2/gaia/gdpval mean)"]}, "Hcompany/Holo-3.1-35B-A3B-NVFP4": {"name":"Holo-3.1-35B-A3B-NVFP4","quant":"nvfp4","filesize":"22gb","filesizeGb":22.1,"hardware":[{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"gb300","label":"gb300","minMemoryGb":288,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve Hcompany/Holo-3.1-35B-A3B-NVFP4 --served-model-name Hcompany/Holo-3.1-35B-A3B-NVFP4 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --async-scheduling --enable-auto-tool-choice --language-model-only --attention-backend flashinfer --quantization modelopt --moe-backend marlin --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --kv-cache-dtype fp8 --mamba-cache-mode align"},"rank":4,"benchmarks":[{"hardwareId":"gb300","engine":"vllm","decodeTps":287.8,"decodeTps32k":308.2,"prefillTps":14555.6,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":259.3,"decodeTps32k":388.7,"prefillTps":16762,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":74.4,"decodeTps32k":71.2,"prefillTps":6884.8,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":287.8,"prefillTps":16762,"source":"measured"},"params":"35B-A3B","notes":["quality 71.3% (tau2/gaia/gdpval mean)"]}, "cyankiwi/Qwen3.6-35B-A3B-AWQ-4bit": {"name":"Qwen3.6-35B-A3B-AWQ-4bit","quant":"awq","filesize":"23gb","filesizeGb":23.3,"hardware":[{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"gb300","label":"gb300","minMemoryGb":288,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve cyankiwi/Qwen3.6-35B-A3B-AWQ-4bit --served-model-name cyankiwi/Qwen3.6-35B-A3B-AWQ-4bit --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --max-num-batched-tokens 8192 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --mamba-cache-mode align"},"rank":4,"benchmarks":[{"hardwareId":"gb300","engine":"vllm","decodeTps":266.1,"decodeTps32k":279.1,"prefillTps":7811.1,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-21","notes":"vllm 0.25.0"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":213.2,"decodeTps32k":116.3,"prefillTps":4174.5,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":193.3,"decodeTps32k":245.2,"prefillTps":5166.4,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":36.5,"decodeTps32k":34.7,"prefillTps":6197.7,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":266.1,"prefillTps":7811.1,"source":"measured"},"params":"35B-A3B","notes":["quality 81.1% (tau2/gaia/gdpval mean)"]}, "exolabs/qwen3-6-27b-optiq-4bit-text-dequant-bf16": {"name":"qwen3-6-27b-optiq-4bit-text-dequant-bf16","quant":"mlx","filesize":"50gb","filesizeGb":50.1,"hardware":[{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve exolabs/qwen3-6-27b-optiq-4bit-text-dequant-bf16 --served-model-name exolabs/qwen3-6-27b-optiq-4bit-text-dequant-bf16 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --max-num-batched-tokens 8192 --enable-auto-tool-choice --async-scheduling --dtype bfloat16 --kv-cache-dtype bfloat16 --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --attention-backend triton_attn --mamba-cache-mode align"},"rank":4,"benchmarks":[{"hardwareId":"gb10_121","engine":"vllm","decodeTps":4.4,"decodeTps32k":4.3,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-18","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":4.4,"prefillTps":null,"source":"measured"},"params":null,"notes":["quality 81.7% (tau2/gaia/gdpval mean)"]}, "exolabs/qwen3-6-27b-ud-mlx-4bit-text-dequant-bf16-normfix": {"name":"qwen3-6-27b-ud-mlx-4bit-text-dequant-bf16-normfix","quant":"mlx","filesize":"17gb","filesizeGb":17,"hardware":[{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve exolabs/qwen3-6-27b-ud-mlx-4bit-text-dequant-bf16-normfix"},"rank":4,"benchmarks":[{"hardwareId":"gb10_121","engine":"vllm","decodeTps":4.5,"decodeTps32k":4.3,"prefillTps":null,"ttftMs":null,"contextTokens":155904,"measuredAt":"2026-07-15","notes":"vllm unknown"}],"expectSpeed":{"decodeTps":4.5,"prefillTps":null,"source":"measured"},"params":null,"notes":["quality 81.4% (tau2/gaia/gdpval mean)","size estimated from parameter count"]}, "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4": {"name":"Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4","quant":"nvfp4","filesize":"21gb","filesizeGb":20.9,"hardware":[{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4 --served-model-name nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 16 --async-scheduling --enable-auto-tool-choice --max-model-len 131072 --tool-call-parser qwen3_coder --reasoning-parser nemotron_v3 --kv-cache-dtype fp8"},"rank":4,"benchmarks":[{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":307.9,"decodeTps32k":402.9,"prefillTps":23766.4,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-18","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":55.9,"decodeTps32k":56.8,"prefillTps":9566.3,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":307.9,"prefillTps":23766.4,"source":"measured"},"params":"30B-A3B","notes":["quality 38.9% (tau2/gaia/gdpval mean)"]}, "openai/gpt-oss-20b": {"name":"gpt-oss-20b","quant":"bf16","filesize":"26gb","filesizeGb":25.6,"hardware":[{"id":"rtx4090_24","label":"rtx4090_24","minMemoryGb":24,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtx5090_32","label":"rtx5090_32","minMemoryGb":32,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve openai/gpt-oss-20b --served-model-name openai/gpt-oss-20b --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 16 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 131072 --tool-call-parser openai --kv-cache-dtype fp8 --max-cudagraph-capture-size 2048 --max-num-batched-tokens 8192 --stream-interval 20"},"rank":4,"benchmarks":[{"hardwareId":"rtx5090_32","engine":"vllm","decodeTps":289.3,"decodeTps32k":304,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-09","notes":"vllm unknown"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":282.2,"decodeTps32k":380.9,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-18","notes":"vllm unknown"},{"hardwareId":"rtx4090_24","engine":"vllm","decodeTps":192.7,"decodeTps32k":263.8,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-09","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":186.5,"decodeTps32k":197.8,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-05","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":46.3,"decodeTps32k":45.7,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":289.3,"prefillTps":null,"source":"measured"},"params":null,"notes":["quality 52.9% (tau2/gaia/gdpval mean)"]}, "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8": {"name":"NVIDIA-Nemotron-3-Nano-30B-A3B-FP8","quant":"fp8","filesize":"30gb","filesizeGb":30.4,"hardware":[{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 --served-model-name nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 16 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 262144 --tool-call-parser qwen3_coder --reasoning-parser nemotron_v3 --kv-cache-dtype fp8 --mamba-cache-mode align"},"rank":5,"benchmarks":[{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":262.1,"decodeTps32k":405.3,"prefillTps":23306.9,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-18","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":176.8,"decodeTps32k":199.8,"prefillTps":24418.9,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":44.6,"decodeTps32k":45.3,"prefillTps":7831.6,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":262.1,"prefillTps":24418.9,"source":"measured"},"params":"30B-A3B","notes":["quality 42.3% (tau2/gaia/gdpval mean)"]}, "unsloth/Qwen3.6-35B-A3B-NVFP4-Fast": {"name":"Qwen3.6-35B-A3B-NVFP4-Fast","quant":"nvfp4","filesize":"22gb","filesizeGb":22,"hardware":[{"id":"rtx5090_32","label":"rtx5090_32","minMemoryGb":32,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve unsloth/Qwen3.6-35B-A3B-NVFP4-Fast --served-model-name unsloth/Qwen3.6-35B-A3B-NVFP4-Fast --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --enable-auto-tool-choice --async-scheduling --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --kv-cache-dtype fp8 --attention-backend flashinfer --max-num-batched-tokens 8192 --mamba-cache-mode align"},"rank":5,"benchmarks":[{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":241.6,"decodeTps32k":344.6,"prefillTps":19182.8,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-18","notes":"vllm unknown"},{"hardwareId":"rtx5090_32","engine":"vllm","decodeTps":240.1,"decodeTps32k":276.2,"prefillTps":16637,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":184.2,"decodeTps32k":176.2,"prefillTps":21078.9,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":69.2,"decodeTps32k":66.4,"prefillTps":7197.8,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-18","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":241.6,"prefillTps":21078.9,"source":"measured"},"params":"35B-A3B","notes":["quality 80.9% (tau2/gaia/gdpval mean)"]}, "InternScience/Agents-A1-FP8": {"name":"Agents-A1-FP8","quant":"fp8","filesize":"35gb","filesizeGb":35.1,"hardware":[{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve InternScience/Agents-A1-FP8 --served-model-name InternScience/Agents-A1-FP8 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --mamba-cache-mode align"},"rank":6,"benchmarks":[{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":196.9,"decodeTps32k":250.2,"prefillTps":4946.6,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":38.1,"decodeTps32k":36.4,"prefillTps":5533.2,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-13","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":196.9,"prefillTps":5533.2,"source":"measured"},"params":null,"notes":["quality 79.4% (tau2/gaia/gdpval mean)"]}, "RedHatAI/Qwen3.6-35B-A3B-NVFP4": {"name":"Qwen3.6-35B-A3B-NVFP4","quant":"nvfp4","filesize":"23gb","filesizeGb":23.3,"hardware":[{"id":"rtx5090_32","label":"rtx5090_32","minMemoryGb":32,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"gb300","label":"gb300","minMemoryGb":288,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve RedHatAI/Qwen3.6-35B-A3B-NVFP4 --served-model-name RedHatAI/Qwen3.6-35B-A3B-NVFP4 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --kv-cache-dtype fp8 --attention-backend flashinfer --moe-backend flashinfer_cutlass --max-num-batched-tokens 8192 --mamba-cache-mode align"},"rank":6,"benchmarks":[{"hardwareId":"gb300","engine":"vllm","decodeTps":245.9,"decodeTps32k":276.5,"prefillTps":21495.5,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-20","notes":"vllm 0.25.0"},{"hardwareId":"rtx5090_32","engine":"vllm","decodeTps":213.5,"decodeTps32k":347.5,"prefillTps":18559.6,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-09","notes":"vllm unknown"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":209.6,"decodeTps32k":324,"prefillTps":18754.1,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-09","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":40.9,"decodeTps32k":40.2,"prefillTps":6708.6,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":245.9,"prefillTps":21495.5,"source":"measured"},"params":"35B-A3B","notes":["quality 40.8% (tau2/gaia/gdpval mean)"]}, "RedHatAI/gemma-4-26B-A4B-it-FP8-Dynamic": {"name":"gemma-4-26B-A4B-it-FP8-Dynamic","quant":"fp8","filesize":"27gb","filesizeGb":26.7,"hardware":[{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve RedHatAI/gemma-4-26B-A4B-it-FP8-Dynamic --served-model-name RedHatAI/gemma-4-26B-A4B-it-FP8-Dynamic --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 16 --enable-auto-tool-choice --revision 951346384683e0c783e8acbaf649418d6c77dd4e --tokenizer-revision 951346384683e0c783e8acbaf649418d6c77dd4e --async-scheduling --language-model-only --max-model-len 262144 --reasoning-parser gemma4 --tool-call-parser gemma4 --kv-cache-dtype bfloat16 --chat-template /vllm-workspace/examples/tool_chat_template_gemma4.jinja --default-chat-template-kwargs '{\"enable_thinking\": true}'"},"rank":6,"benchmarks":[{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":201.5,"decodeTps32k":536.7,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-18","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":132.3,"decodeTps32k":134.4,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":38.1,"decodeTps32k":37.4,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":201.5,"prefillTps":null,"source":"measured"},"params":"26B-A4B","notes":["quality 69.3% (tau2/gaia/gdpval mean)"]}, "cyankiwi/gemma-4-26B-A4B-it-AWQ-4bit": {"name":"gemma-4-26B-A4B-it-AWQ-4bit","quant":"awq","filesize":"16gb","filesizeGb":16,"hardware":[{"id":"rtx5090_32","label":"rtx5090_32","minMemoryGb":32,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve cyankiwi/gemma-4-26B-A4B-it-AWQ-4bit --served-model-name cyankiwi/gemma-4-26B-A4B-it-AWQ-4bit --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.85 --block-size 16 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 131072 --reasoning-parser gemma4 --tool-call-parser gemma4 --chat-template examples/tool_chat_template_gemma4.jinja --default-chat-template-kwargs '{\"enable_thinking\": true}' --speculative-config '{\"model\":\"google/gemma-4-26B-A4B-it-assistant\",\"num_speculative_tokens\":4,\"rejection_sample_method\":\"standard\"}'"},"rank":6,"benchmarks":[{"hardwareId":"rtx5090_32","engine":"vllm","decodeTps":217.8,"decodeTps32k":214.3,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-08","notes":"vllm unknown"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":209.4,"decodeTps32k":727.6,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":151.9,"decodeTps32k":163.8,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-09","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":42.4,"decodeTps32k":41.1,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":217.8,"prefillTps":null,"source":"measured"},"params":"26B-A4B","notes":["quality 63.8% (tau2/gaia/gdpval mean)"]}, "deepreinforce-ai/Ornith-1.0-35B-FP8": {"name":"Ornith-1.0-35B-FP8","quant":"fp8","filesize":"35gb","filesizeGb":35.1,"hardware":[{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve deepreinforce-ai/Ornith-1.0-35B-FP8 --served-model-name deepreinforce-ai/Ornith-1.0-35B-FP8 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --mamba-cache-mode align"},"rank":6,"benchmarks":[{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":196.4,"decodeTps32k":249.3,"prefillTps":4944.9,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":36.3,"decodeTps32k":34.4,"prefillTps":5357.6,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-05","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":196.4,"prefillTps":5357.6,"source":"measured"},"params":"35B","notes":["quality 80.7% (tau2/gaia/gdpval mean)"]}, "Jackrong/Qwopus3.6-27B-v2": {"name":"Qwopus3.6-27B-v2","quant":"bf16","filesize":"52gb","filesizeGb":51.7,"hardware":[{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve Jackrong/Qwopus3.6-27B-v2 --served-model-name Jackrong/Qwopus3.6-27B-v2 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.85 --block-size 1024 --revision 66a4ee9d49dbcef3f83528a400ef6bec93684d6b --tokenizer-revision 66a4ee9d49dbcef3f83528a400ef6bec93684d6b --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 262144 --tool-call-parser qwen3_coder --reasoning-parser qwen3 --mamba-cache-mode align"},"rank":7,"benchmarks":[{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":28.5,"decodeTps32k":29.3,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":4.4,"decodeTps32k":4.3,"prefillTps":null,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-14","notes":"vllm 0.1.dev16488+g129a28dc2"}],"expectSpeed":{"decodeTps":28.5,"prefillTps":null,"source":"measured"},"params":"27B","notes":["quality 80% (tau2/gaia/gdpval mean)"]}, "exolabs/agents-a1-8bit-text-dequant-bf16": {"name":"agents-a1-8bit-text-dequant-bf16","quant":"mlx","filesize":"65gb","filesizeGb":64.6,"hardware":[{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve exolabs/agents-a1-8bit-text-dequant-bf16 --served-model-name exolabs/agents-a1-8bit-text-dequant-bf16 --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --enable-auto-tool-choice --async-scheduling --dtype bfloat16 --kv-cache-dtype bfloat16 --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --moe-backend triton --attention-backend triton_attn"},"rank":7,"benchmarks":[{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":178.4,"decodeTps32k":187.3,"prefillTps":5131.3,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-21","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":30.5,"decodeTps32k":28.6,"prefillTps":3251.6,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":178.4,"prefillTps":5131.3,"source":"measured"},"params":null,"notes":["quality 76.2% (tau2/gaia/gdpval mean)"]}, "exolabs/qwen3-6-35b-a3b-ud-mlx-4bit-text-dequant-bf16-normfix": {"name":"qwen3-6-35b-a3b-ud-mlx-4bit-text-dequant-bf16-normfix","quant":"mlx","filesize":"23gb","filesizeGb":23,"hardware":[{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve exolabs/qwen3-6-35b-a3b-ud-mlx-4bit-text-dequant-bf16-normfix --served-model-name exolabs/qwen3-6-35b-a3b-ud-mlx-4bit-text-dequant-bf16-normfix --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 1024 --max-num-batched-tokens 8192 --enable-auto-tool-choice --async-scheduling --language-model-only --max-model-len 262144 --tool-call-parser qwen3_xml --reasoning-parser qwen3 --mamba-cache-mode align"},"rank":7,"benchmarks":[{"hardwareId":"gb10_121","engine":"vllm","decodeTps":29.7,"decodeTps32k":29.1,"prefillTps":2638.9,"ttftMs":null,"contextTokens":262016,"measuredAt":"2026-07-15","notes":"vllm 0.23.0"}],"expectSpeed":{"decodeTps":29.7,"prefillTps":2638.9,"source":"measured"},"params":null,"notes":["quality 77.7% (tau2/gaia/gdpval mean)","size estimated from parameter count"]}, "leon-se/gemma-4-E4B-it-FP8-Dynamic": {"name":"gemma-4-E4B-it-FP8-Dynamic","quant":"fp8","filesize":"12gb","filesizeGb":12.4,"hardware":[{"id":"rtx4090_24","label":"rtx4090_24","minMemoryGb":24,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtx5090_32","label":"rtx5090_32","minMemoryGb":32,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtx6000ada_48","label":"rtx6000ada_48","minMemoryGb":48,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"rtxpro6000_96","label":"rtxpro6000_96","minMemoryGb":96,"gpuCount":1,"unifiedMemory":false,"tested":true},{"id":"gb10_121","label":"gb10_121","minMemoryGb":121,"gpuCount":1,"unifiedMemory":true,"tested":true},{"id":"gb300","label":"gb300","minMemoryGb":288,"gpuCount":1,"unifiedMemory":true,"tested":true}],"commands":{"vllm":"vllm serve leon-se/gemma-4-E4B-it-FP8-Dynamic --served-model-name leon-se/gemma-4-E4B-it-FP8-Dynamic --host 0.0.0.0 --port 8000 --tensor-parallel-size 1 --pipeline-parallel-size 1 --trust-remote-code --enable-chunked-prefill --enable-prefix-caching --enable-auto-tool-choice --enable-prompt-tokens-details --enable-force-include-usage --enable-request-id-headers --enable-log-requests --max-num-seqs 1 --gpu-memory-utilization 0.88 --block-size 16 --async-scheduling --enable-auto-tool-choice --language-model-only --max-model-len 131072 --tool-call-parser gemma4 --reasoning-parser gemma4 --kv-cache-dtype fp8 --chat-template /vllm-workspace/examples/tool_chat_template_gemma4.jinja --default-chat-template-kwargs '{\"enable_thinking\": true}' --speculative-config '{\"model\":\"google/gemma-4-E4B-it-assistant\",\"num_speculative_tokens\":4,\"rejection_sample_method\":\"standard\"}'"},"rank":7,"benchmarks":[{"hardwareId":"gb300","engine":"vllm","decodeTps":221.9,"decodeTps32k":191.5,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-18","notes":"vllm 0.25.0"},{"hardwareId":"rtx5090_32","engine":"vllm","decodeTps":212,"decodeTps32k":389.2,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-05","notes":"vllm unknown"},{"hardwareId":"rtxpro6000_96","engine":"vllm","decodeTps":176.9,"decodeTps32k":551.9,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-21","notes":"vllm unknown"},{"hardwareId":"rtx4090_24","engine":"vllm","decodeTps":122,"decodeTps32k":48.7,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-18","notes":"vllm unknown"},{"hardwareId":"rtx6000ada_48","engine":"vllm","decodeTps":117.9,"decodeTps32k":48.5,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-18","notes":"vllm unknown"},{"hardwareId":"gb10_121","engine":"vllm","decodeTps":35,"decodeTps32k":34.9,"prefillTps":null,"ttftMs":null,"contextTokens":130944,"measuredAt":"2026-07-04","notes":"vllm unknown"}],"expectSpeed":{"decodeTps":221.9,"prefillTps":null,"source":"measured"},"params":"4B","notes":["quality 21.1% (tau2/gaia/gdpval mean)"]},