# vLLM-Moet on official vLLM v0.24.0 — SM120 (RTX PRO 6000 / RTX 5090)## Produces a serving image = official vllm/vllm-openai:v0.24.0 with:# 1. our patch applied (2-bit MoE experts + FP4 delta/gate + SM120 fixes;# pure-Python — the official compiled artifacts are reused as-is),# 2. DeepGEMM nv-dev a6b593d2 (the release pin 891d57b4 has no family-120# host paths — vLLM issues #47130/#47436),# 3. flashinfer 0.6.14 (0.6.12 predates the SM120 DS4 attention kwargs),# 4. the prebuilt SM120 cubins at /cubit-share.## Build (from the repo root):# DOCKER_BUILDKIT=1 docker build -f Dockerfile.sm120-v024 -t vllm-moet-sm120:v024 .## Serve (official DeepSeek-V4-Flash checkpoint, single 96 GB card):# docker run --rm --gpus '"device=0"' --network host --ipc host --shm-size 64g \# -v /path/to/DeepSeek-V4-Flash:/model:ro \# -e VLLM_MOE_W2=1 -e VLLM_MOE_W2_DELTA_GB=1 \# vllm-moet-sm120:v024 \# --model /model --served-model-name deepseek-v4-flash --trust-remote-code \# --kv-cache-dtype fp8 --block-size 256 --max-model-len 24576 \# --gpu-memory-utilization 0.95 --max-num-batched-tokens 1024 --max-num-seqs 4 \# --tokenizer-mode deepseek_v4 --no-scheduler-reserve-full-isl \# --speculative-config '{"method": "deepseek_mtp", "num_speculative_tokens": 2}' \# --compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' \# --port 8000## Knobs and benchmark methodology: docs/v024-port.md. # ---------------------------------------------------------------- deep_gemm# Built INSIDE the official image itself (it ships nvcc 13.0 + torch 2.11 +# py3.12 for runtime JIT), so the wheel's python/glibc/torch ABI matches the# runtime by construction. A separate nvidia/cuda devel builder does NOT work:# 22.04 = py3.10 (cp310 wheel rejected), 24.04 = glibc 2.38 (runtime has 2.35).# Keep the ref in sync with docs/v024-port.md.FROM vllm/vllm-openai:v0.24.0 AS deepgemm-builderARG DEEPGEMM_REF=a6b593d2826719dcf4892609af7b84ee23aaf32a# git to clone; plus the CUDA dev headers the runtime image lacks:# cusparse/cusolver/cublas (torch's CUDAContextLight.h), nvrtc (DeepGEMM's# JIT compiler.hpp), profiler-api (torch headers on some paths).RUN apt-get update && apt-get install -y --no-install-recommends \ git libcusparse-dev-13-0 libcusolver-dev-13-0 libcublas-dev-13-0 \ cuda-nvrtc-dev-13-0 cuda-profiler-api-13-0 \ && rm -rf /var/lib/apt/lists/*RUN pip install --no-cache-dir --break-system-packages \ pybind11 setuptools wheel packagingRUN git clone https://github.com/deepseek-ai/DeepGEMM.git /deepgemm \ && cd /deepgemm && git checkout "$DEEPGEMM_REF" \ && git submodule update --init --depth 1 third-party/cutlass third-party/fmt \ && python3 setup.py bdist_wheel # ---------------------------------------------------------------- final imageFROM vllm/vllm-openai:v0.24.0 # pip in recent official images may be PEP668-managed; the flag is a no-op# otherwise. git is needed only to apply the patch.RUN apt-get update && apt-get install -y --no-install-recommends git \ && rm -rf /var/lib/apt/lists/* # 1. deep_gemm nv-dev (site-packages wins over the vendored copy in vLLM)COPY --from=deepgemm-builder /deepgemm/dist/*.whl /tmp/RUN pip install --no-cache-dir --no-deps --break-system-packages \ /tmp/deep_gemm-*.whl && rm /tmp/deep_gemm-*.whl # 2. flashinfer 0.6.14 (+ cu130 jit-cache; the 0.6.12 cubin pkg must go —# there is no 0.6.14 build of it and the version check would trip)RUN pip uninstall -y --break-system-packages flashinfer-cubin || trueRUN pip install --no-cache-dir --break-system-packages flashinfer-python==0.6.14 \ && pip install --no-cache-dir --break-system-packages \ --index-url https://flashinfer.ai/whl/cu130 \ "flashinfer-jit-cache==0.6.14+cu130" # 3. our patch, applied to the installed package (pure-Python delta)COPY patch/vllm-moet-v0.24.0.patch /tmp/vllm-moet.patchRUN cd "$(python3 -c 'import vllm, os; print(os.path.dirname(os.path.dirname(vllm.__file__)))')" \ && git apply --verbose /tmp/vllm-moet.patch \ && python3 -c "from vllm.model_executor.layers.quantization.utils import moe_w2_cubit; print('moe_w2 hook OK')" \ && rm /tmp/vllm-moet.patch # 4. SM120 cubins (moe_w2/w4 GEMMs incl. K=6144 for GLM-5.x)COPY kernels/cubins-sm120/ /cubit-share/ENV VLLM_MOE_W2_CUBIT_DIR=/cubit-share
# vLLM-Moet on official vLLM v0.24.0 — SM120 (RTX PRO 6000 / RTX 5090)
#
# Produces a serving image = official vllm/vllm-openai:v0.24.0 with:
# 1. our patch applied (2-bit MoE experts + FP4 delta/gate + SM120 fixes;
# pure-Python — the official compiled artifacts are reused as-is),
# 2. DeepGEMM nv-dev a6b593d2 (the release pin 891d57b4 has no family-120
# host paths — vLLM issues #47130/#47436),
# 3. flashinfer 0.6.14 (0.6.12 predates the SM120 DS4 attention kwargs),
# 4. the prebuilt SM120 cubins at /cubit-share.
#
# Build (from the repo root):
# DOCKER_BUILDKIT=1 docker build -f Dockerfile.sm120-v024 -t vllm-moet-sm120:v024 .
#
# Serve (official DeepSeek-V4-Flash checkpoint, single 96 GB card):
# docker run --rm --gpus '"device=0"' --network host --ipc host --shm-size 64g \
# -v /path/to/DeepSeek-V4-Flash:/model:ro \
# -e VLLM_MOE_W2=1 -e VLLM_MOE_W2_DELTA_GB=1 \
# vllm-moet-sm120:v024 \
# --model /model --served-model-name deepseek-v4-flash --trust-remote-code \
# --kv-cache-dtype fp8 --block-size 256 --max-model-len 24576 \
# --gpu-memory-utilization 0.95 --max-num-batched-tokens 1024 --max-num-seqs 4 \
# --tokenizer-mode deepseek_v4 --no-scheduler-reserve-full-isl \
# --speculative-config '{"method": "deepseek_mtp", "num_speculative_tokens": 2}' \
# --compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' \
# --port 8000
#
# Knobs and benchmark methodology: docs/v024-port.md.
# ---------------------------------------------------------------- deep_gemm
# Built INSIDE the official image itself (it ships nvcc 13.0 + torch 2.11 +
# py3.12 for runtime JIT), so the wheel's python/glibc/torch ABI matches the
# runtime by construction. A separate nvidia/cuda devel builder does NOT work:
# 22.04 = py3.10 (cp310 wheel rejected), 24.04 = glibc 2.38 (runtime has 2.35).
# Keep the ref in sync with docs/v024-port.md.
FROM vllm/vllm-openai:v0.24.0 AS deepgemm-builder
ARG DEEPGEMM_REF=a6b593d2826719dcf4892609af7b84ee23aaf32a
# git to clone; plus the CUDA dev headers the runtime image lacks:
# cusparse/cusolver/cublas (torch's CUDAContextLight.h), nvrtc (DeepGEMM's
# JIT compiler.hpp), profiler-api (torch headers on some paths).
RUN apt-get update && apt-get install -y --no-install-recommends \
git libcusparse-dev-13-0 libcusolver-dev-13-0 libcublas-dev-13-0 \
cuda-nvrtc-dev-13-0 cuda-profiler-api-13-0 \
&& rm -rf /var/lib/apt/lists/*
RUN pip install --no-cache-dir --break-system-packages \
pybind11 setuptools wheel packaging
RUN git clone https://github.com/deepseek-ai/DeepGEMM.git /deepgemm \
&& cd /deepgemm && git checkout "$DEEPGEMM_REF" \
&& git submodule update --init --depth 1 third-party/cutlass third-party/fmt \
&& python3 setup.py bdist_wheel
# ---------------------------------------------------------------- final image
FROM vllm/vllm-openai:v0.24.0
# pip in recent official images may be PEP668-managed; the flag is a no-op
# otherwise. git is needed only to apply the patch.
RUN apt-get update && apt-get install -y --no-install-recommends git \
&& rm -rf /var/lib/apt/lists/*
# 1. deep_gemm nv-dev (site-packages wins over the vendored copy in vLLM)
COPY --from=deepgemm-builder /deepgemm/dist/*.whl /tmp/
RUN pip install --no-cache-dir --no-deps --break-system-packages \
/tmp/deep_gemm-*.whl && rm /tmp/deep_gemm-*.whl
# 2. flashinfer 0.6.14 (+ cu130 jit-cache; the 0.6.12 cubin pkg must go —
# there is no 0.6.14 build of it and the version check would trip)
RUN pip uninstall -y --break-system-packages flashinfer-cubin || true
RUN pip install --no-cache-dir --break-system-packages flashinfer-python==0.6.14 \
&& pip install --no-cache-dir --break-system-packages \
--index-url https://flashinfer.ai/whl/cu130 \
"flashinfer-jit-cache==0.6.14+cu130"
# 3. our patch, applied to the installed package (pure-Python delta)
COPY patch/vllm-moet-v0.24.0.patch /tmp/vllm-moet.patch
RUN cd "$(python3 -c 'import vllm, os; print(os.path.dirname(os.path.dirname(vllm.__file__)))')" \
&& git apply --verbose /tmp/vllm-moet.patch \
&& python3 -c "from vllm.model_executor.layers.quantization.utils import moe_w2_cubit; print('moe_w2 hook OK')" \
&& rm /tmp/vllm-moet.patch
# 4. SM120 cubins (moe_w2/w4 GEMMs incl. K=6144 for GLM-5.x)
COPY kernels/cubins-sm120/ /cubit-share/
ENV VLLM_MOE_W2_CUBIT_DIR=/cubit-share