diff --git a/deploy/docker/patches/vllm/patch_mooncake_mamba_unpack.py b/deploy/docker/patches/vllm/patch_mooncake_mamba_unpack.py new file mode 100644 index 00000000..e03dbb07 --- /dev/null +++ b/deploy/docker/patches/vllm/patch_mooncake_mamba_unpack.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""Mooncake connector: don't assume a Mamba layer has exactly two state tensors. + +register_kv_caches() destructures every MambaSpec layer as a 2-tuple: + + if isinstance(layer_spec, MambaSpec): + conv, _ = cache_or_caches + cache_list = [conv] + +That holds for Mamba2 (conv state, SSM state) and fails for anything else. +Kimi-K3 uses KDA linear attention, whose layers carry a different number of state +tensors, so PD dies during KV registration for every rank at once: + + mooncake_connector.py:1678 in register_kv_caches + conv, _ = cache_or_caches + ValueError: too many values to unpack (expected 2) + +MambaSpec.shapes is already a variable-length tuple of shapes, so the two-tensor +assumption is the connector's, not the spec's. Take the first state tensor — +which is what the original code kept — without constraining how many follow. + +Self-locating and idempotent: re-running is a no-op, and it no-ops if upstream +fixes this, so the patch can stay in place across a base bump. +""" + +import sys +from pathlib import Path + +OLD = """ if isinstance(layer_spec, MambaSpec): + conv, _ = cache_or_caches + cache_list = [conv]""" + +NEW = """ if isinstance(layer_spec, MambaSpec): + # A Mamba-family layer does not necessarily carry exactly two + # state tensors: that is Mamba2's shape (conv, ssm). KDA linear + # attention (Kimi-K3) carries a different number, and `conv, _ =` + # then raises "too many values to unpack" on every rank. Keep the + # first state tensor, as before, without fixing the arity. + cache_list = [cache_or_caches[0]]""" + + +def main() -> int: + import vllm + + target = ( + Path(vllm.__file__).parent + / "distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py" + ) + if not target.is_file(): + print(f"[patch] {target} not found; skipping") + return 0 + src = target.read_text() + if NEW in src: + print("[patch] mooncake mamba unpack: already applied") + return 0 + if OLD not in src: + print("[patch] mooncake mamba unpack: anchor absent (upstream changed?); skipping") + return 0 + target.write_text(src.replace(OLD, NEW)) + print(f"[patch] mooncake mamba unpack: applied to {target}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/deploy/overlay/Dockerfile.payload b/deploy/overlay/Dockerfile.payload index ac39d519..626dc559 100644 --- a/deploy/overlay/Dockerfile.payload +++ b/deploy/overlay/Dockerfile.payload @@ -112,5 +112,11 @@ COPY --from=deps312 /payload/py312 /payload/py312 COPY --from=native312 /native /payload/native COPY --from=native310 /native /payload/native COPY --from=native312 /usr/local/bin/infera-router /payload/bin/infera-router +# Engine patches applied at container start (see infera-exec). These fix vendor +# code we do not otherwise ship — the overlay's whole point is not forking the +# vendor image, and a runtime patch keeps that true where a rebuild would not. +# Each script is self-locating, idempotent, and no-ops once upstream carries the +# fix, so they are safe to leave in place across a base bump. +COPY deploy/docker/patches/vllm/ /payload/patches/vllm/ COPY deploy/overlay/infera-exec /payload/bin/infera-exec CMD ["sh", "-c", "cp -a /payload/. /out/ && echo 'infera overlay payload installed to /out'"] diff --git a/deploy/overlay/infera-exec b/deploy/overlay/infera-exec index e04684aa..8f3a0bc2 100755 --- a/deploy/overlay/infera-exec +++ b/deploy/overlay/infera-exec @@ -110,6 +110,31 @@ if [ -e /host-libionic/libionic.so ]; then fi fi +# --- vendor engine patches -------------------------------------------------- +# Fixes to the vendor's own code, applied here rather than baked into a forked +# image — forking is exactly what this overlay exists to avoid. Each script +# locates its target through the installed package, is idempotent, and no-ops +# when the anchor is absent, so a base that already carries the fix is untouched +# and the patch can stay in place across bumps. +# +# Currently: the Mooncake connector destructures every Mamba-family layer as a +# 2-tuple, which is Mamba2's shape. Kimi-K3's KDA linear attention carries a +# different number of state tensors, so PD died on every rank with "too many +# values to unpack" during KV registration. +# +# Skipped entirely when INFERA_SKIP_ENGINE_PATCHES is set, and a failing patch +# warns rather than blocking startup — a mis-anchored patch must not take down a +# worker that would otherwise serve. +if [ -d "$PAYLOAD_ROOT/patches/vllm" ] && [ -z "${INFERA_SKIP_ENGINE_PATCHES:-}" ]; then + if "$PY" -c 'import vllm' >/dev/null 2>&1; then + for _p in "$PAYLOAD_ROOT"/patches/vllm/*.py; do + [ -e "$_p" ] || continue + (cd / && PYTHONPATH="$PYTREE${PYTHONPATH:+:$PYTHONPATH}" "$PY" "$_p") \ + || echo "infera-exec: patch $(basename "$_p") failed; continuing" >&2 + done + fi +fi + # Python puts the working directory at sys.path[0] — ahead of PYTHONPATH. Vendor # images that already ship infera set WORKDIR to its parent (e.g. /opt/infera), # so the baked-in copy would silently shadow this payload and the overlay would diff --git a/examples/k8s-deployments/glm5.2-sglang.yaml b/examples/k8s-deployments/glm5.2-sglang.yaml index 46866d7c..0702bd7a 100644 --- a/examples/k8s-deployments/glm5.2-sglang.yaml +++ b/examples/k8s-deployments/glm5.2-sglang.yaml @@ -29,7 +29,7 @@ spec: # is an image-tag edit rather than a rebuild of ours. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay:v0.2.2 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: @@ -64,7 +64,7 @@ spec: # is an image-tag edit rather than a rebuild of ours. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay:v0.2.2 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: diff --git a/examples/k8s-deployments/kimi-k3-vllm.yaml b/examples/k8s-deployments/kimi-k3-vllm.yaml index 65bb2d37..084cea42 100644 --- a/examples/k8s-deployments/kimi-k3-vllm.yaml +++ b/examples/k8s-deployments/kimi-k3-vllm.yaml @@ -34,7 +34,7 @@ spec: # is an image-tag edit rather than a rebuild of ours. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay:v0.2.2 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: @@ -69,7 +69,7 @@ spec: # is an image-tag edit rather than a rebuild of ours. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay:v0.2.2 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: diff --git a/examples/k8s-deployments/mixed-kvd-vllm.yaml b/examples/k8s-deployments/mixed-kvd-vllm.yaml index 9f24f516..2b810398 100644 --- a/examples/k8s-deployments/mixed-kvd-vllm.yaml +++ b/examples/k8s-deployments/mixed-kvd-vllm.yaml @@ -58,7 +58,7 @@ spec: # is an image-tag edit rather than a rebuild of ours. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay:v0.2.2 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: @@ -91,7 +91,7 @@ spec: # is an image-tag edit rather than a rebuild of ours. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay:v0.2.2 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: diff --git a/examples/k8s-deployments/mixed-kvd.yaml b/examples/k8s-deployments/mixed-kvd.yaml index 7cd79026..cbf01b83 100644 --- a/examples/k8s-deployments/mixed-kvd.yaml +++ b/examples/k8s-deployments/mixed-kvd.yaml @@ -65,7 +65,7 @@ spec: # is an image-tag edit rather than a rebuild of ours. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay:v0.2.2 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: @@ -98,7 +98,7 @@ spec: # is an image-tag edit rather than a rebuild of ours. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay:v0.2.2 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: diff --git a/examples/k8s-deployments/pd-1p1d-mooncake.yaml b/examples/k8s-deployments/pd-1p1d-mooncake.yaml index fc0ff463..d3c12b05 100644 --- a/examples/k8s-deployments/pd-1p1d-mooncake.yaml +++ b/examples/k8s-deployments/pd-1p1d-mooncake.yaml @@ -37,7 +37,7 @@ spec: # is an image-tag edit rather than a rebuild of ours. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay:v0.2.2 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: @@ -75,7 +75,7 @@ spec: # is an image-tag edit rather than a rebuild of ours. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay:v0.2.2 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: @@ -132,7 +132,7 @@ spec: # is an image-tag edit rather than a rebuild of ours. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay:v0.2.2 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: diff --git a/examples/k8s-deployments/pd-kvd.yaml b/examples/k8s-deployments/pd-kvd.yaml index 7f041ecc..0b84cb66 100644 --- a/examples/k8s-deployments/pd-kvd.yaml +++ b/examples/k8s-deployments/pd-kvd.yaml @@ -81,7 +81,7 @@ spec: # is an image-tag edit rather than a rebuild of ours. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay:v0.2.2 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: @@ -113,7 +113,7 @@ spec: # is an image-tag edit rather than a rebuild of ours. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay:v0.2.2 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: @@ -180,7 +180,7 @@ spec: # is an image-tag edit rather than a rebuild of ours. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay:v0.2.2 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: diff --git a/examples/k8s-deployments/single-node-qwen-sglang.yaml b/examples/k8s-deployments/single-node-qwen-sglang.yaml index d73e25e2..bfc4ad34 100644 --- a/examples/k8s-deployments/single-node-qwen-sglang.yaml +++ b/examples/k8s-deployments/single-node-qwen-sglang.yaml @@ -25,7 +25,7 @@ spec: # is an image-tag edit rather than a rebuild of ours. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay:v0.2.2 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: @@ -60,7 +60,7 @@ spec: # is an image-tag edit rather than a rebuild of ours. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay:v0.2.2 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: diff --git a/examples/k8s-deployments/single-node-qwen-vllm.yaml b/examples/k8s-deployments/single-node-qwen-vllm.yaml index d78bc18b..dbc0811f 100644 --- a/examples/k8s-deployments/single-node-qwen-vllm.yaml +++ b/examples/k8s-deployments/single-node-qwen-vllm.yaml @@ -25,7 +25,7 @@ spec: # is an image-tag edit rather than a rebuild of ours. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay:v0.2.2 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: @@ -60,7 +60,7 @@ spec: # is an image-tag edit rather than a rebuild of ours. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay:v0.2.2 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: diff --git a/examples/recipes/README.md b/examples/recipes/README.md index c6112849..046fe6e6 100644 --- a/examples/recipes/README.md +++ b/examples/recipes/README.md @@ -42,7 +42,7 @@ forking the base for one model broke every other model. Build the overlay before deploying: ```bash -docker build -f deploy/overlay/Dockerfile.payload -t inferaimage/infera-overlay:v0.2.1 . +docker build -f deploy/overlay/Dockerfile.payload -t inferaimage/infera-overlay:v0.2.2 . ``` The build harvests **one native tree per ABI family** — `NATIVE_IMAGE` supplies diff --git a/examples/recipes/glm5.2/mixed-kvd/deploy.yaml b/examples/recipes/glm5.2/mixed-kvd/deploy.yaml index 738cb910..9c5c06d0 100644 --- a/examples/recipes/glm5.2/mixed-kvd/deploy.yaml +++ b/examples/recipes/glm5.2/mixed-kvd/deploy.yaml @@ -1,7 +1,7 @@ # GLM-5.2-MXFP4 — mixed-kvd — Kubernetes recipe # # base image lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x (stock vendor image, unmodified) -# overlay inferaimage/infera-overlay:v0.2.1 (infera + router + kvd + native deps) +# overlay inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 (infera + router + kvd + native deps) # engine infera.engine.sglang TP8 # # Deploy: kubectl apply -f examples/recipes/glm5.2/mixed-kvd/deploy.yaml @@ -45,14 +45,14 @@ spec: # so following an upstream bump costs nothing in this repo. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: - {name: overlay, mountPath: /overlay} containers: - name: main - image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x + image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x@sha256:40e940a0c55b87105c773d8b484616616b3a91662bfa223c48ff721d9793dc8d imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec","python3","-m","infera.server", "--host","0.0.0.0","--port","8000", @@ -79,14 +79,14 @@ spec: # so following an upstream bump costs nothing in this repo. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: - {name: overlay, mountPath: /overlay} containers: - name: main - image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x + image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x@sha256:40e940a0c55b87105c773d8b484616616b3a91662bfa223c48ff721d9793dc8d imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec", "python3","-m","infera.engine.sglang","--host","0.0.0.0","--port","30000", @@ -136,7 +136,7 @@ spec: # infera.kvd out of the shared overlay. The overlay image itself is a # busybox carrying the payload — it has no interpreter to run kvd with. - name: kvd - image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x + image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x@sha256:40e940a0c55b87105c773d8b484616616b3a91662bfa223c48ff721d9793dc8d imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec","python3","-m","infera.kvd", "--socket","/kvd/kvd.sock", diff --git a/examples/recipes/glm5.2/mixed/deploy.yaml b/examples/recipes/glm5.2/mixed/deploy.yaml index 95a38e1e..02d3aaa8 100644 --- a/examples/recipes/glm5.2/mixed/deploy.yaml +++ b/examples/recipes/glm5.2/mixed/deploy.yaml @@ -1,7 +1,7 @@ # GLM-5.2-MXFP4 — mixed — Kubernetes recipe # # base image lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x (stock vendor image, unmodified) -# overlay inferaimage/infera-overlay:v0.2.1 (infera + router + kvd + native deps) +# overlay inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 (infera + router + kvd + native deps) # engine infera.engine.sglang TP8 # # Deploy: kubectl apply -f examples/recipes/glm5.2/mixed/deploy.yaml @@ -34,14 +34,14 @@ spec: # so following an upstream bump costs nothing in this repo. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: - {name: overlay, mountPath: /overlay} containers: - name: main - image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x + image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x@sha256:40e940a0c55b87105c773d8b484616616b3a91662bfa223c48ff721d9793dc8d imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec","python3","-m","infera.server", "--host","0.0.0.0","--port","8000", @@ -68,14 +68,14 @@ spec: # so following an upstream bump costs nothing in this repo. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: - {name: overlay, mountPath: /overlay} containers: - name: main - image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x + image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x@sha256:40e940a0c55b87105c773d8b484616616b3a91662bfa223c48ff721d9793dc8d imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec", "python3","-m","infera.engine.sglang","--host","0.0.0.0","--port","30000", diff --git a/examples/recipes/glm5.2/pd-kvd/deploy.yaml b/examples/recipes/glm5.2/pd-kvd/deploy.yaml index 1b46ef0d..89724b66 100644 --- a/examples/recipes/glm5.2/pd-kvd/deploy.yaml +++ b/examples/recipes/glm5.2/pd-kvd/deploy.yaml @@ -1,7 +1,7 @@ # GLM-5.2-MXFP4 — pd-kvd — Kubernetes recipe # # base image lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x (stock vendor image, unmodified) -# overlay inferaimage/infera-overlay:v0.2.1 (infera + router + kvd + native deps) +# overlay inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 (infera + router + kvd + native deps) # engine infera.engine.sglang TP8 # # Deploy: kubectl apply -f examples/recipes/glm5.2/pd-kvd/deploy.yaml @@ -56,14 +56,14 @@ spec: # so following an upstream bump costs nothing in this repo. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: - {name: overlay, mountPath: /overlay} containers: - name: main - image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x + image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x@sha256:40e940a0c55b87105c773d8b484616616b3a91662bfa223c48ff721d9793dc8d imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec","python3","-m","infera.server", "--host","0.0.0.0","--port","8000", @@ -92,14 +92,14 @@ spec: # so following an upstream bump costs nothing in this repo. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: - {name: overlay, mountPath: /overlay} containers: - name: main - image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x + image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x@sha256:40e940a0c55b87105c773d8b484616616b3a91662bfa223c48ff721d9793dc8d imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec", "python3","-m","infera.engine.sglang","--host","0.0.0.0","--port","30000", @@ -155,7 +155,7 @@ spec: # infera.kvd out of the shared overlay. The overlay image itself is a # busybox carrying the payload — it has no interpreter to run kvd with. - name: kvd - image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x + image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x@sha256:40e940a0c55b87105c773d8b484616616b3a91662bfa223c48ff721d9793dc8d imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec","python3","-m","infera.kvd", "--socket","/kvd/kvd.sock", @@ -190,14 +190,14 @@ spec: # so following an upstream bump costs nothing in this repo. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: - {name: overlay, mountPath: /overlay} containers: - name: main - image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x + image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x@sha256:40e940a0c55b87105c773d8b484616616b3a91662bfa223c48ff721d9793dc8d imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec", "python3","-m","infera.engine.sglang","--host","0.0.0.0","--port","30000", @@ -253,7 +253,7 @@ spec: # infera.kvd out of the shared overlay. The overlay image itself is a # busybox carrying the payload — it has no interpreter to run kvd with. - name: kvd - image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x + image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x@sha256:40e940a0c55b87105c773d8b484616616b3a91662bfa223c48ff721d9793dc8d imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec","python3","-m","infera.kvd", "--socket","/kvd/kvd.sock", diff --git a/examples/recipes/glm5.2/pd/deploy.yaml b/examples/recipes/glm5.2/pd/deploy.yaml index df8ee377..e06b81ed 100644 --- a/examples/recipes/glm5.2/pd/deploy.yaml +++ b/examples/recipes/glm5.2/pd/deploy.yaml @@ -1,7 +1,7 @@ # GLM-5.2-MXFP4 — pd — Kubernetes recipe # # base image lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x (stock vendor image, unmodified) -# overlay inferaimage/infera-overlay:v0.2.1 (infera + router + kvd + native deps) +# overlay inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 (infera + router + kvd + native deps) # engine infera.engine.sglang TP8 # # Deploy: kubectl apply -f examples/recipes/glm5.2/pd/deploy.yaml @@ -36,14 +36,14 @@ spec: # so following an upstream bump costs nothing in this repo. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: - {name: overlay, mountPath: /overlay} containers: - name: main - image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x + image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x@sha256:40e940a0c55b87105c773d8b484616616b3a91662bfa223c48ff721d9793dc8d imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec","python3","-m","infera.server", "--host","0.0.0.0","--port","8000", @@ -72,14 +72,14 @@ spec: # so following an upstream bump costs nothing in this repo. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: - {name: overlay, mountPath: /overlay} containers: - name: main - image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x + image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x@sha256:40e940a0c55b87105c773d8b484616616b3a91662bfa223c48ff721d9793dc8d imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec", "python3","-m","infera.engine.sglang","--host","0.0.0.0","--port","30000", @@ -139,14 +139,14 @@ spec: # so following an upstream bump costs nothing in this repo. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: - {name: overlay, mountPath: /overlay} containers: - name: main - image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x + image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x@sha256:40e940a0c55b87105c773d8b484616616b3a91662bfa223c48ff721d9793dc8d imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec", "python3","-m","infera.engine.sglang","--host","0.0.0.0","--port","30000", diff --git a/examples/recipes/kimi-k3/mixed-kvd/deploy.yaml b/examples/recipes/kimi-k3/mixed-kvd/deploy.yaml index 1100ba3a..90dae334 100644 --- a/examples/recipes/kimi-k3/mixed-kvd/deploy.yaml +++ b/examples/recipes/kimi-k3/mixed-kvd/deploy.yaml @@ -1,7 +1,7 @@ # Kimi-K3 — mixed-kvd — Kubernetes recipe # # base image vllm/vllm-openai-rocm:kimi-k3 (stock vendor image, unmodified) -# overlay inferaimage/infera-overlay:v0.2.1 (infera + router + kvd + native deps) +# overlay inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 (infera + router + kvd + native deps) # engine infera.engine.vllm TP8 # # Deploy: kubectl apply -f examples/recipes/kimi-k3/mixed-kvd/deploy.yaml @@ -44,14 +44,14 @@ spec: # so following an upstream bump costs nothing in this repo. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: - {name: overlay, mountPath: /overlay} containers: - name: main - image: vllm/vllm-openai-rocm:kimi-k3 + image: vllm/vllm-openai-rocm:kimi-k3@sha256:5aa7e626ff73672f5ca7aae46754570488c23d33ca1ac90756a1d2d1a3fe099b imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec","python3","-m","infera.server", "--host","0.0.0.0","--port","8000", @@ -78,14 +78,14 @@ spec: # so following an upstream bump costs nothing in this repo. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: - {name: overlay, mountPath: /overlay} containers: - name: main - image: vllm/vllm-openai-rocm:kimi-k3 + image: vllm/vllm-openai-rocm:kimi-k3@sha256:5aa7e626ff73672f5ca7aae46754570488c23d33ca1ac90756a1d2d1a3fe099b imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec", "python3","-m","infera.engine.vllm","--host","0.0.0.0","--port","30000", @@ -129,7 +129,7 @@ spec: # infera.kvd out of the shared overlay. The overlay image itself is a # busybox carrying the payload — it has no interpreter to run kvd with. - name: kvd - image: vllm/vllm-openai-rocm:kimi-k3 + image: vllm/vllm-openai-rocm:kimi-k3@sha256:5aa7e626ff73672f5ca7aae46754570488c23d33ca1ac90756a1d2d1a3fe099b imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec","python3","-m","infera.kvd", "--socket","/kvd/kvd.sock", diff --git a/examples/recipes/kimi-k3/mixed/deploy.yaml b/examples/recipes/kimi-k3/mixed/deploy.yaml index 627468a0..94506356 100644 --- a/examples/recipes/kimi-k3/mixed/deploy.yaml +++ b/examples/recipes/kimi-k3/mixed/deploy.yaml @@ -1,7 +1,7 @@ # Kimi-K3 — mixed — Kubernetes recipe # # base image vllm/vllm-openai-rocm:kimi-k3 (stock vendor image, unmodified) -# overlay inferaimage/infera-overlay:v0.2.1 (infera + router + kvd + native deps) +# overlay inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 (infera + router + kvd + native deps) # engine infera.engine.vllm TP8 # # Deploy: kubectl apply -f examples/recipes/kimi-k3/mixed/deploy.yaml @@ -42,14 +42,14 @@ spec: # so following an upstream bump costs nothing in this repo. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: - {name: overlay, mountPath: /overlay} containers: - name: main - image: vllm/vllm-openai-rocm:kimi-k3 + image: vllm/vllm-openai-rocm:kimi-k3@sha256:5aa7e626ff73672f5ca7aae46754570488c23d33ca1ac90756a1d2d1a3fe099b imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec","python3","-m","infera.server", "--host","0.0.0.0","--port","8000", @@ -76,14 +76,14 @@ spec: # so following an upstream bump costs nothing in this repo. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: - {name: overlay, mountPath: /overlay} containers: - name: main - image: vllm/vllm-openai-rocm:kimi-k3 + image: vllm/vllm-openai-rocm:kimi-k3@sha256:5aa7e626ff73672f5ca7aae46754570488c23d33ca1ac90756a1d2d1a3fe099b imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec", "python3","-m","infera.engine.vllm","--host","0.0.0.0","--port","30000", diff --git a/examples/recipes/kimi-k3/pd-kvd/deploy.yaml b/examples/recipes/kimi-k3/pd-kvd/deploy.yaml index e85e0c04..32c784c3 100644 --- a/examples/recipes/kimi-k3/pd-kvd/deploy.yaml +++ b/examples/recipes/kimi-k3/pd-kvd/deploy.yaml @@ -1,7 +1,7 @@ # Kimi-K3 — pd-kvd — Kubernetes recipe # # base image vllm/vllm-openai-rocm:kimi-k3 (stock vendor image, unmodified) -# overlay inferaimage/infera-overlay:v0.2.1 (infera + router + kvd + native deps) +# overlay inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 (infera + router + kvd + native deps) # engine infera.engine.vllm TP8 # # Deploy: kubectl apply -f examples/recipes/kimi-k3/pd-kvd/deploy.yaml @@ -56,14 +56,14 @@ spec: # so following an upstream bump costs nothing in this repo. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: - {name: overlay, mountPath: /overlay} containers: - name: main - image: vllm/vllm-openai-rocm:kimi-k3 + image: vllm/vllm-openai-rocm:kimi-k3@sha256:5aa7e626ff73672f5ca7aae46754570488c23d33ca1ac90756a1d2d1a3fe099b imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec","python3","-m","infera.server", "--host","0.0.0.0","--port","8000", @@ -92,14 +92,14 @@ spec: # so following an upstream bump costs nothing in this repo. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: - {name: overlay, mountPath: /overlay} containers: - name: main - image: vllm/vllm-openai-rocm:kimi-k3 + image: vllm/vllm-openai-rocm:kimi-k3@sha256:5aa7e626ff73672f5ca7aae46754570488c23d33ca1ac90756a1d2d1a3fe099b imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec", "python3","-m","infera.engine.vllm","--host","0.0.0.0","--port","30000", @@ -143,7 +143,7 @@ spec: # infera.kvd out of the shared overlay. The overlay image itself is a # busybox carrying the payload — it has no interpreter to run kvd with. - name: kvd - image: vllm/vllm-openai-rocm:kimi-k3 + image: vllm/vllm-openai-rocm:kimi-k3@sha256:5aa7e626ff73672f5ca7aae46754570488c23d33ca1ac90756a1d2d1a3fe099b imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec","python3","-m","infera.kvd", "--socket","/kvd/kvd.sock", @@ -180,14 +180,14 @@ spec: # so following an upstream bump costs nothing in this repo. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: - {name: overlay, mountPath: /overlay} containers: - name: main - image: vllm/vllm-openai-rocm:kimi-k3 + image: vllm/vllm-openai-rocm:kimi-k3@sha256:5aa7e626ff73672f5ca7aae46754570488c23d33ca1ac90756a1d2d1a3fe099b imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec", "python3","-m","infera.engine.vllm","--host","0.0.0.0","--port","30000", @@ -231,7 +231,7 @@ spec: # infera.kvd out of the shared overlay. The overlay image itself is a # busybox carrying the payload — it has no interpreter to run kvd with. - name: kvd - image: vllm/vllm-openai-rocm:kimi-k3 + image: vllm/vllm-openai-rocm:kimi-k3@sha256:5aa7e626ff73672f5ca7aae46754570488c23d33ca1ac90756a1d2d1a3fe099b imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec","python3","-m","infera.kvd", "--socket","/kvd/kvd.sock", diff --git a/examples/recipes/kimi-k3/pd-sglang/deploy.yaml b/examples/recipes/kimi-k3/pd-sglang/deploy.yaml new file mode 100644 index 00000000..59ec20f4 --- /dev/null +++ b/examples/recipes/kimi-k3/pd-sglang/deploy.yaml @@ -0,0 +1,263 @@ +# Kimi-K3 — pd (SGLang) — Kubernetes recipe +# +# The SGLang half of Kimi-K3 PD. The vLLM half is ../pd/deploy.yaml; pick by which +# engine you want, not by which is "the" PD path — they disaggregate differently: +# +# SGLang --disaggregation-mode + a Mooncake bootstrap handshake +# vLLM --kv-transfer-config with a MooncakeConnector +# +# base image lmsysorg/sglang-rocm:rocm720-mi35x-k3-20260727 (the SGLang build +# carrying Kimi-K3 support; the general v0.5.15 tag has none) +# overlay inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 +# +# / +# the model directory on EACH node. They need NOT match: every pod mounts its +# own node's copy, so nodes that name their local NVMe differently +# (/mnt/k3local vs /mnt/shared here) are fine. +# +# Put the weights on LOCAL disk on both nodes. Kimi-K3 loads its 96 shards in +# ~8 min from NVMe and ~95 min from NFS when both PD nodes read the same mount — +# and the second case does not merely run slow, it exceeds infera's ready timeout, +# so the worker restarts mid-load and never finishes. If you must use NFS, raise +# INFERA_ENGINE_READY_TIMEOUT accordingly. +apiVersion: infera.amd.com/v1alpha1 +kind: InferaDeployment +metadata: + name: kimi-k3-pd-sglang + namespace: infera +spec: + backendFramework: sglang + discoveryBackend: kubernetes + nats: + deploy: false + services: + server: + componentType: server + extraPodSpec: + nodeSelector: {kubernetes.io/hostname: } + # The overlay payload is dropped into an emptyDir here; the STOCK vendor + # image below then runs infera out of it, so following an upstream bump + # is an image-tag edit rather than a rebuild of ours. + initContainers: + - name: infera-overlay + image: inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 + imagePullPolicy: IfNotPresent + command: ["sh","-c","cp -a /payload/. /overlay/"] + volumeMounts: + - {name: overlay, mountPath: /overlay} + containers: + - name: main + image: lmsysorg/sglang-rocm:rocm720-mi35x-k3-20260727@sha256:3c01f73fe23aebf4a8853de0899a70b75c2af6c0409d2331353847aac4d3f906 # libionic (ABI 4) baked + imagePullPolicy: IfNotPresent + command: ["/overlay/bin/infera-exec","python3","-m","infera.server","--host","0.0.0.0","--port","8000", + "--router-tokenizer-path","/models/Kimi-K3", + "--request-transport","http","--kv-event-transport","zmq"] + env: + - {name: POD_NAME, valueFrom: {fieldRef: {fieldPath: metadata.name}}} + - {name: POD_NAMESPACE, valueFrom: {fieldRef: {fieldPath: metadata.namespace}}} + - {name: POD_IP, valueFrom: {fieldRef: {fieldPath: status.podIP}}} + # Weight-load time tracks the STORAGE, not the model. Kimi-K3 read its + # 96 shards in 502 s from local NVMe and ~95 min from NFS with both PD + # nodes competing for the same mount. infera's default 1800 s ready + # timeout is generous for the first and impossible for the second — the + # worker kills itself mid-load, restarts, and never finishes. + - {name: INFERA_ENGINE_READY_TIMEOUT, value: "7200"} + resources: {requests: {cpu: "4", memory: 8Gi}, limits: {cpu: "4", memory: 8Gi}} + volumeMounts: [{name: overlay, mountPath: /overlay, readOnly: true}, {name: model, mountPath: /models, readOnly: true}] + volumes: [{name: overlay, emptyDir: {}}, {name: model, hostPath: {path: , type: Directory}}] + prefill: + componentType: worker + role: prefill + replicas: 1 + port: 30000 + skipReadinessProbe: true + extraPodSpec: + nodeSelector: {kubernetes.io/hostname: } + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + # The overlay payload is dropped into an emptyDir here; the STOCK vendor + # image below then runs infera out of it, so following an upstream bump + # is an image-tag edit rather than a rebuild of ours. + initContainers: + - name: infera-overlay + image: inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 + imagePullPolicy: IfNotPresent + command: ["sh","-c","cp -a /payload/. /overlay/"] + volumeMounts: + - {name: overlay, mountPath: /overlay} + containers: + - name: main + image: lmsysorg/sglang-rocm:rocm720-mi35x-k3-20260727@sha256:3c01f73fe23aebf4a8853de0899a70b75c2af6c0409d2331353847aac4d3f906 + imagePullPolicy: IfNotPresent + securityContext: {privileged: true, capabilities: {add: ["IPC_LOCK","SYS_PTRACE"]}} + command: ["/overlay/bin/infera-exec","python3","-m","infera.engine.sglang","--host","0.0.0.0","--port","30000", + "--disaggregation-mode","prefill","--model-path","/models/Kimi-K3", + "--tp-size","8","--attention-backend","aiter","--trust-remote-code", + "--mem-fraction-static","0.97", + # 0.93, not the 0.8 inherited from this manifest's Qwen-0.6B + # ancestor. Kimi-K3 is ~190 GB per rank at TP8, and at 0.8 the + # hybrid Mamba state cache is computed AFTER the weights land: + # Not enough GPU memory for hybrid (mamba/linear-attention) + # state cache. Computed max_mamba_cache_size=-554 + # (total_rest_memory=-60.99 GB) + # i.e. in deficit before any cache is allocated. The engine exits + # and the wrapper reports a bare "-9", which reads as OOM and is + # not — chasing the container memory limit first is a dead end. + # Measured, both knobs the engine itself suggests: + # mem-fraction 0.80 -> total_rest_memory -60.99 GB + # mem-fraction 0.93 -> total_rest_memory -30.39 GB + # so 0.97 plus a capped --max-running-requests (mamba cache is + # 53.57 MB per request) and a 32k context. + "--context-length","32768", + "--max-running-requests","16", + # Pin the hybrid state cache instead of letting it be derived. + # SGLang sizes it from whatever GPU memory is left AFTER the + # weights, and on Kimi-K3 at TP8 that residual is negative, so + # the derivation yields a negative cache and the engine exits: + # 0.80 -> total_rest_memory -60.99 GB + # 0.93 -> -30.39 GB + # 0.97 -> -20.98 GB + # Raising mem-fraction shrinks the gap but cannot close it — + # the residual is the wrong side of zero to begin with. An + # explicit size skips the derivation. 16 requests x 53.57 MB. + "--max-mamba-cache-size","16", + # --language-only is what makes --mem-fraction-static stick. + # Kimi-K3 is multimodal, so SGLang runs adjust_mem_fraction_for_vlm() + # and SCALES the value you passed: + # mem_fraction_static = original * final_overall_factor + # 0.97 became 0.8245, and the engine then failed telling us to + # "raise --mem-fraction-static above 0.901" — advice that cannot + # work, since the raised value is scaled down again. The guard is + # `if model_config.is_multimodal and not self.language_only`. + # Text-only PD does not need the vision tower, so opt out. + "--language-only", + "--kv-event-transport","zmq","--request-transport","http", + "--disaggregation-transfer-backend","mooncake"] + env: + - {name: SGLANG_USE_AITER, value: "1"} + - {name: POD_NAME, valueFrom: {fieldRef: {fieldPath: metadata.name}}} + - {name: POD_NAMESPACE, valueFrom: {fieldRef: {fieldPath: metadata.namespace}}} + - {name: POD_IP, valueFrom: {fieldRef: {fieldPath: status.podIP}}} + # Weight-load time tracks the STORAGE, not the model. Kimi-K3 read its + # 96 shards in 502 s from local NVMe and ~95 min from NFS with both PD + # nodes competing for the same mount. infera's default 1800 s ready + # timeout is generous for the first and impossible for the second — the + # worker kills itself mid-load, restarts, and never finishes. + - {name: INFERA_ENGINE_READY_TIMEOUT, value: "7200"} + resources: {requests: {cpu: "16", memory: 768Gi, amd.com/gpu: 1}, limits: {cpu: "16", memory: 768Gi, amd.com/gpu: 1}} + volumeMounts: + - {name: overlay, mountPath: /overlay, readOnly: true} + - {name: model, mountPath: /models, readOnly: true} + - {name: dshm, mountPath: /dev/shm} + - {name: ib, mountPath: /dev/infiniband} + # The vendor base's libionic must match the host ionic kernel ABI or + # libibverbs rejects every device ("does not support the kernel ABI"), + # which reads as "No RDMA devices found" and sends Mooncake down its + # HIP-IPC path — where cross-node PD cannot work. infera-exec swaps it + # in from here. + - {name: host-libionic, mountPath: /host-libionic/libionic.so, readOnly: true} + volumes: + - {name: overlay, emptyDir: {}} + - {name: model, hostPath: {path: , type: Directory}} + - {name: dshm, emptyDir: {medium: Memory, sizeLimit: 16Gi}} + - {name: ib, hostPath: {path: /dev/infiniband, type: Directory}} + - name: host-libionic + hostPath: {path: /usr/lib/x86_64-linux-gnu/libionic.so.1, type: File} + decode: + componentType: worker + role: decode + replicas: 1 + port: 30000 + skipReadinessProbe: true + extraPodSpec: + nodeSelector: {kubernetes.io/hostname: } + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + # The overlay payload is dropped into an emptyDir here; the STOCK vendor + # image below then runs infera out of it, so following an upstream bump + # is an image-tag edit rather than a rebuild of ours. + initContainers: + - name: infera-overlay + image: inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 + imagePullPolicy: IfNotPresent + command: ["sh","-c","cp -a /payload/. /overlay/"] + volumeMounts: + - {name: overlay, mountPath: /overlay} + containers: + - name: main + image: lmsysorg/sglang-rocm:rocm720-mi35x-k3-20260727@sha256:3c01f73fe23aebf4a8853de0899a70b75c2af6c0409d2331353847aac4d3f906 + imagePullPolicy: IfNotPresent + securityContext: {privileged: true, capabilities: {add: ["IPC_LOCK","SYS_PTRACE"]}} + command: ["/overlay/bin/infera-exec","python3","-m","infera.engine.sglang","--host","0.0.0.0","--port","30000", + "--disaggregation-mode","decode","--model-path","/models/Kimi-K3", + "--tp-size","8","--attention-backend","aiter","--trust-remote-code", + "--mem-fraction-static","0.97", + # 0.93, not the 0.8 inherited from this manifest's Qwen-0.6B + # ancestor. Kimi-K3 is ~190 GB per rank at TP8, and at 0.8 the + # hybrid Mamba state cache is computed AFTER the weights land: + # Not enough GPU memory for hybrid (mamba/linear-attention) + # state cache. Computed max_mamba_cache_size=-554 + # (total_rest_memory=-60.99 GB) + # i.e. in deficit before any cache is allocated. The engine exits + # and the wrapper reports a bare "-9", which reads as OOM and is + # not — chasing the container memory limit first is a dead end. + # Measured, both knobs the engine itself suggests: + # mem-fraction 0.80 -> total_rest_memory -60.99 GB + # mem-fraction 0.93 -> total_rest_memory -30.39 GB + # so 0.97 plus a capped --max-running-requests (mamba cache is + # 53.57 MB per request) and a 32k context. + "--context-length","32768", + "--max-running-requests","16", + # Pin the hybrid state cache instead of letting it be derived. + # SGLang sizes it from whatever GPU memory is left AFTER the + # weights, and on Kimi-K3 at TP8 that residual is negative, so + # the derivation yields a negative cache and the engine exits: + # 0.80 -> total_rest_memory -60.99 GB + # 0.93 -> -30.39 GB + # 0.97 -> -20.98 GB + # Raising mem-fraction shrinks the gap but cannot close it — + # the residual is the wrong side of zero to begin with. An + # explicit size skips the derivation. 16 requests x 53.57 MB. + "--max-mamba-cache-size","16", + # --language-only is what makes --mem-fraction-static stick. + # Kimi-K3 is multimodal, so SGLang runs adjust_mem_fraction_for_vlm() + # and SCALES the value you passed: + # mem_fraction_static = original * final_overall_factor + # 0.97 became 0.8245, and the engine then failed telling us to + # "raise --mem-fraction-static above 0.901" — advice that cannot + # work, since the raised value is scaled down again. The guard is + # `if model_config.is_multimodal and not self.language_only`. + # Text-only PD does not need the vision tower, so opt out. + "--language-only", + "--kv-event-transport","zmq","--request-transport","http", + "--disaggregation-transfer-backend","mooncake"] + env: + - {name: SGLANG_USE_AITER, value: "1"} + - {name: POD_NAME, valueFrom: {fieldRef: {fieldPath: metadata.name}}} + - {name: POD_NAMESPACE, valueFrom: {fieldRef: {fieldPath: metadata.namespace}}} + - {name: POD_IP, valueFrom: {fieldRef: {fieldPath: status.podIP}}} + # Weight-load time tracks the STORAGE, not the model. Kimi-K3 read its + # 96 shards in 502 s from local NVMe and ~95 min from NFS with both PD + # nodes competing for the same mount. infera's default 1800 s ready + # timeout is generous for the first and impossible for the second — the + # worker kills itself mid-load, restarts, and never finishes. + - {name: INFERA_ENGINE_READY_TIMEOUT, value: "7200"} + resources: {requests: {cpu: "16", memory: 768Gi, amd.com/gpu: 1}, limits: {cpu: "16", memory: 768Gi, amd.com/gpu: 1}} + volumeMounts: + - {name: overlay, mountPath: /overlay, readOnly: true} + - {name: model, mountPath: /models, readOnly: true} + - {name: dshm, mountPath: /dev/shm} + - {name: ib, mountPath: /dev/infiniband} + # The vendor base's libionic must match the host ionic kernel ABI or + # libibverbs rejects every device ("does not support the kernel ABI"), + # which reads as "No RDMA devices found" and sends Mooncake down its + # HIP-IPC path — where cross-node PD cannot work. infera-exec swaps it + # in from here. + - {name: host-libionic, mountPath: /host-libionic/libionic.so, readOnly: true} + volumes: + - {name: overlay, emptyDir: {}} + - {name: model, hostPath: {path: , type: Directory}} + - {name: dshm, emptyDir: {medium: Memory, sizeLimit: 16Gi}} + - {name: ib, hostPath: {path: /dev/infiniband, type: Directory}} + - name: host-libionic + hostPath: {path: /usr/lib/x86_64-linux-gnu/libionic.so.1, type: File} diff --git a/examples/recipes/kimi-k3/pd/deploy.yaml b/examples/recipes/kimi-k3/pd/deploy.yaml index f5a4e85c..73002662 100644 --- a/examples/recipes/kimi-k3/pd/deploy.yaml +++ b/examples/recipes/kimi-k3/pd/deploy.yaml @@ -1,7 +1,7 @@ # Kimi-K3 — pd — Kubernetes recipe # # base image vllm/vllm-openai-rocm:kimi-k3 (stock vendor image, unmodified) -# overlay inferaimage/infera-overlay:v0.2.1 (infera + router + kvd + native deps) +# overlay inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 (infera + router + kvd + native deps) # engine infera.engine.vllm TP8 # # Deploy: kubectl apply -f examples/recipes/kimi-k3/pd/deploy.yaml @@ -36,14 +36,14 @@ spec: # so following an upstream bump costs nothing in this repo. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: - {name: overlay, mountPath: /overlay} containers: - name: main - image: vllm/vllm-openai-rocm:kimi-k3 + image: vllm/vllm-openai-rocm:kimi-k3@sha256:5aa7e626ff73672f5ca7aae46754570488c23d33ca1ac90756a1d2d1a3fe099b imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec","python3","-m","infera.server", "--host","0.0.0.0","--port","8000", @@ -67,19 +67,24 @@ spec: port: 30000 extraPodSpec: nodeSelector: {kubernetes.io/hostname: } + # hostNetwork is REQUIRED: the RDMA rails are host interfaces and the + # flannel pod network cannot reach them, so Mooncake has no path to the + # peer while the engine looks healthy. + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet # The overlay payload is dropped into an emptyDir here; the STOCK vendor # image below then runs infera out of it. The vendor image is never forked, # so following an upstream bump costs nothing in this repo. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: - {name: overlay, mountPath: /overlay} containers: - name: main - image: vllm/vllm-openai-rocm:kimi-k3 + image: vllm/vllm-openai-rocm:kimi-k3@sha256:5aa7e626ff73672f5ca7aae46754570488c23d33ca1ac90756a1d2d1a3fe099b imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec", "python3","-m","infera.engine.vllm","--host","0.0.0.0","--port","30000", @@ -91,19 +96,36 @@ spec: "--enable-prefix-caching","--max-num-seqs","128","--max-num-batched-tokens","4096", "--enable-auto-tool-choice","--tool-call-parser","kimi_k3", "--reasoning-parser","kimi_k3", + "--data-parallel-address","$(POD_IP)", + # Mooncake's bootstrap server BINDS 0.0.0.0 (fine) but ADVERTISES + # parallel_config.data_parallel_master_ip, which defaults to + # 127.0.0.1 and is whatever --host implies otherwise. With the + # usual 0.0.0.0 the prefiller registers "http://0.0.0.0:8998" and + # the decoder cannot reach it: + # Failed to connect to bootstrap server http://0.0.0.0:8998 + # Failed to find remote engine_id ... + # while both workers look healthy and the request just hangs to + # a 480 s connector timeout. $(POD_IP) is expanded by kubelet + # from the downward API env below. "--kv-transfer-config","{\"kv_connector\":\"MooncakeConnector\",\"kv_role\":\"kv_producer\"}", "--kv-event-transport","zmq","--request-transport","http"] env: - {name: POD_NAME, valueFrom: {fieldRef: {fieldPath: metadata.name}}} - {name: POD_NAMESPACE, valueFrom: {fieldRef: {fieldPath: metadata.namespace}}} - {name: POD_IP, valueFrom: {fieldRef: {fieldPath: status.podIP}}} + # Load time tracks the STORAGE: Kimi-K3 reads 96 shards in ~8 min from + # local NVMe and ~95 min from NFS. infera's 1800 s default makes the + # worker kill itself mid-load and restart forever on the slow path. + - {name: INFERA_ENGINE_READY_TIMEOUT, value: "7200"} - {name: HF_HUB_OFFLINE, value: "1"} - {name: VLLM_ROCM_USE_AITER, value: "1"} - {name: SAFETENSORS_FAST_GPU, value: "1"} # PD's KV transport is Mooncake, which lives in the overlay's native # tree. Name it so a payload lacking it fails at startup instead of only # warning and quietly serving with no KV transfer at all. - - {name: INFERA_REQUIRE_NATIVE, value: "mooncake"} + - {name: INFERA_REQUIRE_NATIVE, value: "mooncake"} # RDMA needs the device nodes and pinned memory. Without these the + # engine starts, registers, and only fails when KV actually moves. + securityContext: {privileged: true, capabilities: {add: ["IPC_LOCK","SYS_PTRACE"]}} startupProbe: httpGet: {path: /health, port: 30000} periodSeconds: 10 @@ -115,10 +137,17 @@ spec: volumeMounts: - {name: overlay, mountPath: /overlay, readOnly: true} - {name: model, mountPath: /models, readOnly: true} + # The vendor base's libionic must match the host ionic kernel ABI, or + # libibverbs rejects every device ("No RDMA devices found") and Mooncake + # falls back to HIP IPC, which cannot work across nodes. + - {name: host-libionic, mountPath: /host-libionic/libionic.so, readOnly: true} - {name: dshm, mountPath: /dev/shm} volumes: - {name: overlay, emptyDir: {}} - {name: model, persistentVolumeClaim: {claimName: model-cache}} + - name: host-libionic + hostPath: {path: /usr/lib/x86_64-linux-gnu/libionic.so.1, type: File} + - {name: ib, hostPath: {path: /dev/infiniband, type: Directory}} - {name: dshm, emptyDir: {medium: Memory, sizeLimit: 64Gi}} decode: componentType: worker @@ -127,19 +156,24 @@ spec: port: 30000 extraPodSpec: nodeSelector: {kubernetes.io/hostname: } + # hostNetwork is REQUIRED: the RDMA rails are host interfaces and the + # flannel pod network cannot reach them, so Mooncake has no path to the + # peer while the engine looks healthy. + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet # The overlay payload is dropped into an emptyDir here; the STOCK vendor # image below then runs infera out of it. The vendor image is never forked, # so following an upstream bump costs nothing in this repo. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: - {name: overlay, mountPath: /overlay} containers: - name: main - image: vllm/vllm-openai-rocm:kimi-k3 + image: vllm/vllm-openai-rocm:kimi-k3@sha256:5aa7e626ff73672f5ca7aae46754570488c23d33ca1ac90756a1d2d1a3fe099b imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec", "python3","-m","infera.engine.vllm","--host","0.0.0.0","--port","30000", @@ -151,19 +185,36 @@ spec: "--max-num-seqs","128","--max-num-batched-tokens","4096", "--enable-auto-tool-choice","--tool-call-parser","kimi_k3", "--reasoning-parser","kimi_k3", + "--data-parallel-address","$(POD_IP)", + # Mooncake's bootstrap server BINDS 0.0.0.0 (fine) but ADVERTISES + # parallel_config.data_parallel_master_ip, which defaults to + # 127.0.0.1 and is whatever --host implies otherwise. With the + # usual 0.0.0.0 the prefiller registers "http://0.0.0.0:8998" and + # the decoder cannot reach it: + # Failed to connect to bootstrap server http://0.0.0.0:8998 + # Failed to find remote engine_id ... + # while both workers look healthy and the request just hangs to + # a 480 s connector timeout. $(POD_IP) is expanded by kubelet + # from the downward API env below. "--kv-transfer-config","{\"kv_connector\":\"MooncakeConnector\",\"kv_role\":\"kv_consumer\"}", "--kv-event-transport","zmq","--request-transport","http"] env: - {name: POD_NAME, valueFrom: {fieldRef: {fieldPath: metadata.name}}} - {name: POD_NAMESPACE, valueFrom: {fieldRef: {fieldPath: metadata.namespace}}} - {name: POD_IP, valueFrom: {fieldRef: {fieldPath: status.podIP}}} + # Load time tracks the STORAGE: Kimi-K3 reads 96 shards in ~8 min from + # local NVMe and ~95 min from NFS. infera's 1800 s default makes the + # worker kill itself mid-load and restart forever on the slow path. + - {name: INFERA_ENGINE_READY_TIMEOUT, value: "7200"} - {name: HF_HUB_OFFLINE, value: "1"} - {name: VLLM_ROCM_USE_AITER, value: "1"} - {name: SAFETENSORS_FAST_GPU, value: "1"} # PD's KV transport is Mooncake, which lives in the overlay's native # tree. Name it so a payload lacking it fails at startup instead of only # warning and quietly serving with no KV transfer at all. - - {name: INFERA_REQUIRE_NATIVE, value: "mooncake"} + - {name: INFERA_REQUIRE_NATIVE, value: "mooncake"} # RDMA needs the device nodes and pinned memory. Without these the + # engine starts, registers, and only fails when KV actually moves. + securityContext: {privileged: true, capabilities: {add: ["IPC_LOCK","SYS_PTRACE"]}} startupProbe: httpGet: {path: /health, port: 30000} periodSeconds: 10 @@ -175,8 +226,15 @@ spec: volumeMounts: - {name: overlay, mountPath: /overlay, readOnly: true} - {name: model, mountPath: /models, readOnly: true} + # The vendor base's libionic must match the host ionic kernel ABI, or + # libibverbs rejects every device ("No RDMA devices found") and Mooncake + # falls back to HIP IPC, which cannot work across nodes. + - {name: host-libionic, mountPath: /host-libionic/libionic.so, readOnly: true} - {name: dshm, mountPath: /dev/shm} volumes: - {name: overlay, emptyDir: {}} - {name: model, persistentVolumeClaim: {claimName: model-cache}} + - name: host-libionic + hostPath: {path: /usr/lib/x86_64-linux-gnu/libionic.so.1, type: File} + - {name: ib, hostPath: {path: /dev/infiniband, type: Directory}} - {name: dshm, emptyDir: {medium: Memory, sizeLimit: 64Gi}} diff --git a/infera/engine/sglang/args.py b/infera/engine/sglang/args.py index 47ca0ebb..cb28bde6 100644 --- a/infera/engine/sglang/args.py +++ b/infera/engine/sglang/args.py @@ -262,22 +262,31 @@ def parse_sglang_args(argv: list[str] | None = None) -> SglangWorkerArgs: ): remaining.append("--disaggregation-decode-enable-radix-cache") - server_args = ServerArgs.from_cli_args(sglang_parsed) - # infera product default: fp8 KV cache (fp8_e4m3) unless the operator passed # --kv-cache-dtype explicitly. fp8 halves the KV footprint -> ~2x the KV that # fits in VRAM and halves PD KV-transfer + RDMA memory-registration volume # (bf16 hit ionic ibv_reg_mr ENOMEM at high concurrency / long inputs). Small # accuracy cost; opt out with --kv-cache-dtype auto|bf16 or INFERA_DEFAULT_KV_FP8=0. + # + # Applied to the PARSED ARGS, before ServerArgs resolves them. Newer SGLang + # freezes server_args once resolved and raises on assignment: + # + # AttributeError: server_args.kv_cache_dtype assigned after resolution; + # server_args is read-only -- use get_context().override(source, ...) + # + # which took down every worker on the Kimi-K3 build. Setting it pre-resolution + # needs no override API and works on both the old and new SGLang. if os.environ.get("INFERA_DEFAULT_KV_FP8", "1") != "0" and not any( t == "--kv-cache-dtype" or t.startswith("--kv-cache-dtype=") for t in remaining ): - server_args.kv_cache_dtype = "fp8_e4m3" + sglang_parsed.kv_cache_dtype = "fp8_e4m3" logger.info( "infera default: kv_cache_dtype=fp8_e4m3 " "(override with --kv-cache-dtype or INFERA_DEFAULT_KV_FP8=0)" ) + server_args = ServerArgs.from_cli_args(sglang_parsed) + from infera.engine.sglang.hicache_validate import warn_if_hicache_prefetch_disabled warn_if_hicache_prefetch_disabled(server_args) diff --git a/infera/engine/sglang/worker.py b/infera/engine/sglang/worker.py index a75a7263..2fb3f533 100644 --- a/infera/engine/sglang/worker.py +++ b/infera/engine/sglang/worker.py @@ -33,6 +33,14 @@ } +def _ready_timeout() -> float: + """Seconds to wait for the engine's /health, from INFERA_ENGINE_READY_TIMEOUT.""" + try: + return float(os.environ.get("INFERA_ENGINE_READY_TIMEOUT", "1800")) + except ValueError: + return 1800.0 + + class SglangEngine(BaseEngine): """Runs `python -m sglang.launch_server` in a child process. @@ -136,7 +144,12 @@ async def start(self) -> EngineConfig: dp_size=dp_size, ) - async def _wait_ready(self, timeout: float = 1800) -> None: + # Weight-load time tracks the storage, not the model: Kimi-K3 read 96 shards in + # 502 s from local NVMe and ~95 min from NFS with both PD nodes competing for + # the same mount. A hardcoded 1800 s is generous for the first and impossible + # for the second — the worker killed itself mid-load, restarted, and could + # never finish. Tunable via INFERA_ENGINE_READY_TIMEOUT (seconds). + async def _wait_ready(self, timeout: float = _ready_timeout()) -> None: # /health is probed locally; sglang binds on server_args.host, but if # that is 0.0.0.0 we should probe via 127.0.0.1 instead. probe_host = self.server_args.host diff --git a/infera/engine/vllm/__main__.py b/infera/engine/vllm/__main__.py index dce281a3..b8f33c48 100644 --- a/infera/engine/vllm/__main__.py +++ b/infera/engine/vllm/__main__.py @@ -181,6 +181,28 @@ async def main() -> None: args.advertise_host = pod_ip logger.info("k8s discovery: advertising Pod IP %s", pod_ip) + # disagg_meta was already built during arg parsing, when advertise_host + # was still unset — so Mooncake's bootstrap_addr was frozen as the bind + # host. The prefiller then publishes "http://0.0.0.0:8998", the decoder + # reads it from kv_transfer_params and cannot connect: + # + # Failed to connect to bootstrap server http://0.0.0.0:8998 + # Failed to find remote engine_id ... + # + # Both workers stay healthy and the request simply hangs to the + # connector's 480 s timeout, so nothing points at the address. Rebuild + # the address now that the routable host is known. + meta = args.disagg_meta or {} + params = meta.get("params") or {} + addr = params.get("bootstrap_addr") + if addr and "//0.0.0.0:" in addr: + params["bootstrap_addr"] = addr.replace("//0.0.0.0:", f"//{pod_ip}:") + logger.info( + "k8s discovery: bootstrap_addr %s -> %s", + addr, + params["bootstrap_addr"], + ) + logger.info( "parsed args: model=%s host=%s port=%d disagg=%s disagg_meta=%s enable_kv_events=%s", args.model, diff --git a/infera/engine/vllm/worker.py b/infera/engine/vllm/worker.py index ce1024f1..23c10505 100644 --- a/infera/engine/vllm/worker.py +++ b/infera/engine/vllm/worker.py @@ -24,6 +24,14 @@ logger = logging.getLogger(__name__) +def _ready_timeout() -> float: + """Seconds to wait for the engine's /health, from INFERA_ENGINE_READY_TIMEOUT.""" + try: + return float(os.environ.get("INFERA_ENGINE_READY_TIMEOUT", "1800")) + except ValueError: + return 1800.0 + + class VllmEngine(BaseEngine): def __init__( self, @@ -162,7 +170,12 @@ async def stop(self) -> None: except ProcessLookupError: pass - async def _wait_ready(self, timeout: float = 1800) -> None: + # Weight-load time tracks the storage, not the model: Kimi-K3 read 96 shards in + # 502 s from local NVMe and ~95 min from NFS with both PD nodes competing for + # the same mount. A hardcoded 1800 s is generous for the first and impossible + # for the second — the worker killed itself mid-load, restarted, and could + # never finish. Tunable via INFERA_ENGINE_READY_TIMEOUT (seconds). + async def _wait_ready(self, timeout: float = _ready_timeout()) -> None: # 30 min: cold model download + ROCm kernel compile can eat 10+ min. probe_host = "127.0.0.1" if self.host in ("0.0.0.0", "") else self.host url = f"http://{probe_host}:{self.port}/health" diff --git a/manual/examples/k8s_kimi_k3.md b/manual/examples/k8s_kimi_k3.md index 898bdb00..caa12780 100644 --- a/manual/examples/k8s_kimi_k3.md +++ b/manual/examples/k8s_kimi_k3.md @@ -24,7 +24,7 @@ upstream `kimi_k3` vLLM by overriding the base of the standard vLLM image build: ```bash docker pull vllm/vllm-openai-rocm:kimi-k3 # the vLLM carrying kimi_k3 support -docker pull inferaimage/infera-overlay:v0.2.1 # infera, kvd, router, Mooncake, hipFile +docker pull inferaimage/infera-overlay:v0.2.2 # infera, kvd, router, Mooncake, hipFile ``` Nothing is built here. The manifest runs the **stock** vLLM image and mounts infera @@ -81,7 +81,7 @@ PVC read-only. - The engine **image present in the node container runtime**. k3s uses containerd (not docker) — import a local image as a tar, not a stream: ```bash - docker save vllm/vllm-openai-rocm:kimi-k3 inferaimage/infera-overlay:v0.2.1 -o /mnt//img.tar + docker save vllm/vllm-openai-rocm:kimi-k3 inferaimage/infera-overlay:v0.2.2 -o /mnt//img.tar sudo k3s ctr images import /mnt//img.tar ``` diff --git a/manual/recipes/kimi-k3.md b/manual/recipes/kimi-k3.md index 711e505e..2f986dee 100644 --- a/manual/recipes/kimi-k3.md +++ b/manual/recipes/kimi-k3.md @@ -39,7 +39,18 @@ kubectl -n infera get pods -w :::{tab-item} PD :sync: pd -Prefill and decode on separate nodes, KV handed over by Mooncake. +Prefill and decode on separate nodes, KV handed over by Mooncake. **Two engines +disaggregate differently — pick by which engine you want, not by which is "the" +PD path:** + +| Engine | Mechanism | Manifest | Image | +|---|---|---|---| +| vLLM | `--kv-transfer-config` with a `MooncakeConnector` | `pd/deploy.yaml` | `vllm/vllm-openai-rocm:kimi-k3` | +| SGLang | `--disaggregation-mode` + a Mooncake bootstrap handshake | `pd-sglang/deploy.yaml` | `lmsysorg/sglang-rocm:rocm720-mi35x-k3-20260727` | + +The general `lmsysorg/sglang` tags carry **no** Kimi-K3 support — only that dated +`-k3-` build does. Weights must sit on storage both nodes see at the same path; +at ~1.5 TB a per-node copy is usually not an option. This combination declares `INFERA_REQUIRE_NATIVE=mooncake`, so it fails at startup rather than serving quietly without it.