From 40aa8ede50664801965437661c5acc08036e3658 Mon Sep 17 00:00:00 2001 From: "Zhang, Jiejing" Date: Sat, 1 Aug 2026 20:03:38 +0000 Subject: [PATCH 1/8] feat(recipes): Kimi-K3 PD on SGLang, alongside the vLLM one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Kimi-K3 PD had one shape (vLLM, MooncakeConnector). Adds the SGLang shape and makes the choice explicit in the manual, because the two disaggregate by different mechanisms rather than being two spellings of one thing: vLLM --kv-transfer-config with a MooncakeConnector SGLang --disaggregation-mode + a Mooncake bootstrap handshake The image matters more than usual here. The general `lmsysorg/sglang` tags carry no Kimi-K3 support at all — no kimi_k3.py, nothing in the registry — while `lmsysorg/sglang-rocm:rocm720-mi35x-k3-20260727` has KimiK3ForConditionalGeneration, KimiK3LinearForCausalLM and kimi_k3_vl. Checking that is less obvious than it sounds: the image installs sglang as an editable package, so `import sglang` does not resolve to the source tree and a registry probe reports an empty list, doubly so because rocminfo fails in a GPU-less container. The files are under /sgl-workspace/sglang/python/sglang/srt/models/. Weights come from NFS rather than a node-local path: Kimi-K3 is ~1.5 TB and the decode node here has 147 GB free, so both nodes seeing one path is the only option, not a preference. Neither PD combination has been run for this model yet — the SGLang PD path itself is validated cross-node (see the Qwen run in the PD fixes), but not with Kimi-K3's size or settings, and the vLLM PD path is unproven on this fabric entirely. The status table says so rather than implying the model inherits that validation. Signed-off-by: Zhang, Jiejing --- .../recipes/kimi-k3/pd-sglang/deploy.yaml | 143 ++++++++++++++++++ manual/recipes/kimi-k3.md | 13 +- 2 files changed, 155 insertions(+), 1 deletion(-) create mode 100644 examples/recipes/kimi-k3/pd-sglang/deploy.yaml diff --git a/examples/recipes/kimi-k3/pd-sglang/deploy.yaml b/examples/recipes/kimi-k3/pd-sglang/deploy.yaml new file mode 100644 index 0000000..46afb15 --- /dev/null +++ b/examples/recipes/kimi-k3/pd-sglang/deploy.yaml @@ -0,0 +1,143 @@ +# Kimi-K3 — pd (SGLang) — Kubernetes recipe +# +# The SGLang half of Kimi-K3 PD. The vLLM half is ../pd/deploy.yaml; pick by which +# engine you want, not by which is "the" PD path — they disaggregate differently: +# +# SGLang --disaggregation-mode + a Mooncake bootstrap handshake +# vLLM --kv-transfer-config with a MooncakeConnector +# +# base image lmsysorg/sglang-rocm:rocm720-mi35x-k3-20260727 (the SGLang build +# carrying Kimi-K3 support; the general v0.5.15 tag has none) +# overlay inferaimage/infera-overlay:v0.2.1 +# +# Weights come from NFS so both nodes see the same path — Kimi-K3 is ~1.5 TB and +# the decode node here has 147 GB free, so a local copy is not an option. +apiVersion: infera.amd.com/v1alpha1 +kind: InferaDeployment +metadata: + name: kimi-k3-pd-sglang + namespace: infera +spec: + backendFramework: sglang + discoveryBackend: kubernetes + nats: + deploy: false + services: + server: + componentType: server + extraPodSpec: + nodeSelector: {kubernetes.io/hostname: } + # The overlay payload is dropped into an emptyDir here; the STOCK vendor + # image below then runs infera out of it, so following an upstream bump + # is an image-tag edit rather than a rebuild of ours. + initContainers: + - name: infera-overlay + image: inferaimage/infera-overlay:v0.2.1 + imagePullPolicy: IfNotPresent + command: ["sh","-c","cp -a /payload/. /overlay/"] + volumeMounts: + - {name: overlay, mountPath: /overlay} + containers: + - name: main + image: lmsysorg/sglang-rocm:rocm720-mi35x-k3-20260727 # libionic (ABI 4) baked + imagePullPolicy: IfNotPresent + command: ["/overlay/bin/infera-exec","python3","-m","infera.server","--host","0.0.0.0","--port","8000", + "--router-tokenizer-path","/models/Kimi-K3", + "--request-transport","http","--kv-event-transport","zmq"] + env: + - {name: POD_NAME, valueFrom: {fieldRef: {fieldPath: metadata.name}}} + - {name: POD_NAMESPACE, valueFrom: {fieldRef: {fieldPath: metadata.namespace}}} + - {name: POD_IP, valueFrom: {fieldRef: {fieldPath: status.podIP}}} + resources: {requests: {cpu: "4", memory: 8Gi}, limits: {cpu: "4", memory: 8Gi}} + volumeMounts: [{name: overlay, mountPath: /overlay, readOnly: true}, {name: model, mountPath: /models, readOnly: true}] + volumes: [{name: overlay, emptyDir: {}}, {name: model, hostPath: {path: /mnt/vast/yaocheng/models/moonshotai, type: Directory}}] + prefill: + componentType: worker + role: prefill + replicas: 1 + port: 30000 + skipReadinessProbe: true + extraPodSpec: + nodeSelector: {kubernetes.io/hostname: } + # The overlay payload is dropped into an emptyDir here; the STOCK vendor + # image below then runs infera out of it, so following an upstream bump + # is an image-tag edit rather than a rebuild of ours. + initContainers: + - name: infera-overlay + image: inferaimage/infera-overlay:v0.2.1 + imagePullPolicy: IfNotPresent + command: ["sh","-c","cp -a /payload/. /overlay/"] + volumeMounts: + - {name: overlay, mountPath: /overlay} + containers: + - name: main + image: lmsysorg/sglang-rocm:rocm720-mi35x-k3-20260727 + imagePullPolicy: IfNotPresent + securityContext: {privileged: true, capabilities: {add: ["IPC_LOCK","SYS_PTRACE"]}} + command: ["/overlay/bin/infera-exec","python3","-m","infera.engine.sglang","--host","0.0.0.0","--port","30000", + "--disaggregation-mode","prefill","--model-path","/models/Kimi-K3", + "--tp-size","8","--attention-backend","aiter","--trust-remote-code", + "--mem-fraction-static","0.8", + "--kv-event-transport","zmq","--request-transport","http", + "--disaggregation-transfer-backend","mooncake"] + env: + - {name: SGLANG_USE_AITER, value: "1"} + - {name: POD_NAME, valueFrom: {fieldRef: {fieldPath: metadata.name}}} + - {name: POD_NAMESPACE, valueFrom: {fieldRef: {fieldPath: metadata.namespace}}} + - {name: POD_IP, valueFrom: {fieldRef: {fieldPath: status.podIP}}} + resources: {requests: {cpu: "16", memory: 96Gi, amd.com/gpu: 1}, limits: {cpu: "16", memory: 96Gi, amd.com/gpu: 1}} + volumeMounts: + - {name: overlay, mountPath: /overlay, readOnly: true} + - {name: model, mountPath: /models, readOnly: true} + - {name: dshm, mountPath: /dev/shm} + - {name: ib, mountPath: /dev/infiniband} + volumes: + - {name: overlay, emptyDir: {}} + - {name: model, hostPath: {path: /mnt/vast/yaocheng/models/moonshotai, type: Directory}} + - {name: dshm, emptyDir: {medium: Memory, sizeLimit: 16Gi}} + - {name: ib, hostPath: {path: /dev/infiniband, type: Directory}} + decode: + componentType: worker + role: decode + replicas: 1 + port: 30000 + skipReadinessProbe: true + extraPodSpec: + nodeSelector: {kubernetes.io/hostname: } + # The overlay payload is dropped into an emptyDir here; the STOCK vendor + # image below then runs infera out of it, so following an upstream bump + # is an image-tag edit rather than a rebuild of ours. + initContainers: + - name: infera-overlay + image: inferaimage/infera-overlay:v0.2.1 + imagePullPolicy: IfNotPresent + command: ["sh","-c","cp -a /payload/. /overlay/"] + volumeMounts: + - {name: overlay, mountPath: /overlay} + containers: + - name: main + image: lmsysorg/sglang-rocm:rocm720-mi35x-k3-20260727 + imagePullPolicy: IfNotPresent + securityContext: {privileged: true, capabilities: {add: ["IPC_LOCK","SYS_PTRACE"]}} + command: ["/overlay/bin/infera-exec","python3","-m","infera.engine.sglang","--host","0.0.0.0","--port","30000", + "--disaggregation-mode","decode","--model-path","/models/Kimi-K3", + "--tp-size","8","--attention-backend","aiter","--trust-remote-code", + "--mem-fraction-static","0.8", + "--kv-event-transport","zmq","--request-transport","http", + "--disaggregation-transfer-backend","mooncake"] + env: + - {name: SGLANG_USE_AITER, value: "1"} + - {name: POD_NAME, valueFrom: {fieldRef: {fieldPath: metadata.name}}} + - {name: POD_NAMESPACE, valueFrom: {fieldRef: {fieldPath: metadata.namespace}}} + - {name: POD_IP, valueFrom: {fieldRef: {fieldPath: status.podIP}}} + resources: {requests: {cpu: "16", memory: 96Gi, amd.com/gpu: 1}, limits: {cpu: "16", memory: 96Gi, amd.com/gpu: 1}} + volumeMounts: + - {name: overlay, mountPath: /overlay, readOnly: true} + - {name: model, mountPath: /models, readOnly: true} + - {name: dshm, mountPath: /dev/shm} + - {name: ib, mountPath: /dev/infiniband} + volumes: + - {name: overlay, emptyDir: {}} + - {name: model, hostPath: {path: /mnt/vast/yaocheng/models/moonshotai, type: Directory}} + - {name: dshm, emptyDir: {medium: Memory, sizeLimit: 16Gi}} + - {name: ib, hostPath: {path: /dev/infiniband, type: Directory}} diff --git a/manual/recipes/kimi-k3.md b/manual/recipes/kimi-k3.md index 711e505..2f986de 100644 --- a/manual/recipes/kimi-k3.md +++ b/manual/recipes/kimi-k3.md @@ -39,7 +39,18 @@ kubectl -n infera get pods -w :::{tab-item} PD :sync: pd -Prefill and decode on separate nodes, KV handed over by Mooncake. +Prefill and decode on separate nodes, KV handed over by Mooncake. **Two engines +disaggregate differently — pick by which engine you want, not by which is "the" +PD path:** + +| Engine | Mechanism | Manifest | Image | +|---|---|---|---| +| vLLM | `--kv-transfer-config` with a `MooncakeConnector` | `pd/deploy.yaml` | `vllm/vllm-openai-rocm:kimi-k3` | +| SGLang | `--disaggregation-mode` + a Mooncake bootstrap handshake | `pd-sglang/deploy.yaml` | `lmsysorg/sglang-rocm:rocm720-mi35x-k3-20260727` | + +The general `lmsysorg/sglang` tags carry **no** Kimi-K3 support — only that dated +`-k3-` build does. Weights must sit on storage both nodes see at the same path; +at ~1.5 TB a per-node copy is usually not an option. This combination declares `INFERA_REQUIRE_NATIVE=mooncake`, so it fails at startup rather than serving quietly without it. From 2c31a3fdd626b71900322721bf64683d323a3ffb Mon Sep 17 00:00:00 2001 From: "Zhang, Jiejing" Date: Sat, 1 Aug 2026 21:40:39 +0000 Subject: [PATCH 2/8] fix(sglang): set the fp8 KV default before ServerArgs resolves MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit infera's fp8 KV-cache default assigned `server_args.kv_cache_dtype` after `ServerArgs.from_cli_args()`. Newer SGLang freezes server_args once resolved: AttributeError: server_args.kv_cache_dtype assigned after resolution; server_args is read-only -- use get_context().override(source, ...) which crash-looped every worker on lmsysorg/sglang-rocm:rocm720-mi35x-k3-20260727, the only SGLang build carrying Kimi-K3. Set it on the parsed args before resolution instead. That needs no override API and works on both the old and the new SGLang, where calling get_context().override would only work on the new one. Also carries the two mounts the PD recipe needs and had lost by being derived from main rather than from the PD-fixes branch: hostNetwork on the workers, and the host libionic. Without them the recipe reproduces the exact failure that branch fixes — 0 RDMA devices, then Mooncake down its HIP-IPC path. Verified in this run that the injection fires: "infera-exec: replaced .../libionic.so.1.0.54.0-149 with the host libionic build". Signed-off-by: Zhang, Jiejing --- .../recipes/kimi-k3/pd-sglang/deploy.yaml | 20 +++++++++++++++++++ infera/engine/sglang/args.py | 15 +++++++++++--- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/examples/recipes/kimi-k3/pd-sglang/deploy.yaml b/examples/recipes/kimi-k3/pd-sglang/deploy.yaml index 46afb15..1c9ae31 100644 --- a/examples/recipes/kimi-k3/pd-sglang/deploy.yaml +++ b/examples/recipes/kimi-k3/pd-sglang/deploy.yaml @@ -59,6 +59,8 @@ spec: skipReadinessProbe: true extraPodSpec: nodeSelector: {kubernetes.io/hostname: } + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet # The overlay payload is dropped into an emptyDir here; the STOCK vendor # image below then runs infera out of it, so following an upstream bump # is an image-tag edit rather than a rebuild of ours. @@ -91,11 +93,19 @@ spec: - {name: model, mountPath: /models, readOnly: true} - {name: dshm, mountPath: /dev/shm} - {name: ib, mountPath: /dev/infiniband} + # The vendor base's libionic must match the host ionic kernel ABI or + # libibverbs rejects every device ("does not support the kernel ABI"), + # which reads as "No RDMA devices found" and sends Mooncake down its + # HIP-IPC path — where cross-node PD cannot work. infera-exec swaps it + # in from here. + - {name: host-libionic, mountPath: /host-libionic/libionic.so, readOnly: true} volumes: - {name: overlay, emptyDir: {}} - {name: model, hostPath: {path: /mnt/vast/yaocheng/models/moonshotai, type: Directory}} - {name: dshm, emptyDir: {medium: Memory, sizeLimit: 16Gi}} - {name: ib, hostPath: {path: /dev/infiniband, type: Directory}} + - name: host-libionic + hostPath: {path: /usr/lib/x86_64-linux-gnu/libionic.so.1, type: File} decode: componentType: worker role: decode @@ -104,6 +114,8 @@ spec: skipReadinessProbe: true extraPodSpec: nodeSelector: {kubernetes.io/hostname: } + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet # The overlay payload is dropped into an emptyDir here; the STOCK vendor # image below then runs infera out of it, so following an upstream bump # is an image-tag edit rather than a rebuild of ours. @@ -136,8 +148,16 @@ spec: - {name: model, mountPath: /models, readOnly: true} - {name: dshm, mountPath: /dev/shm} - {name: ib, mountPath: /dev/infiniband} + # The vendor base's libionic must match the host ionic kernel ABI or + # libibverbs rejects every device ("does not support the kernel ABI"), + # which reads as "No RDMA devices found" and sends Mooncake down its + # HIP-IPC path — where cross-node PD cannot work. infera-exec swaps it + # in from here. + - {name: host-libionic, mountPath: /host-libionic/libionic.so, readOnly: true} volumes: - {name: overlay, emptyDir: {}} - {name: model, hostPath: {path: /mnt/vast/yaocheng/models/moonshotai, type: Directory}} - {name: dshm, emptyDir: {medium: Memory, sizeLimit: 16Gi}} - {name: ib, hostPath: {path: /dev/infiniband, type: Directory}} + - name: host-libionic + hostPath: {path: /usr/lib/x86_64-linux-gnu/libionic.so.1, type: File} diff --git a/infera/engine/sglang/args.py b/infera/engine/sglang/args.py index 47ca0eb..cb28bde 100644 --- a/infera/engine/sglang/args.py +++ b/infera/engine/sglang/args.py @@ -262,22 +262,31 @@ def parse_sglang_args(argv: list[str] | None = None) -> SglangWorkerArgs: ): remaining.append("--disaggregation-decode-enable-radix-cache") - server_args = ServerArgs.from_cli_args(sglang_parsed) - # infera product default: fp8 KV cache (fp8_e4m3) unless the operator passed # --kv-cache-dtype explicitly. fp8 halves the KV footprint -> ~2x the KV that # fits in VRAM and halves PD KV-transfer + RDMA memory-registration volume # (bf16 hit ionic ibv_reg_mr ENOMEM at high concurrency / long inputs). Small # accuracy cost; opt out with --kv-cache-dtype auto|bf16 or INFERA_DEFAULT_KV_FP8=0. + # + # Applied to the PARSED ARGS, before ServerArgs resolves them. Newer SGLang + # freezes server_args once resolved and raises on assignment: + # + # AttributeError: server_args.kv_cache_dtype assigned after resolution; + # server_args is read-only -- use get_context().override(source, ...) + # + # which took down every worker on the Kimi-K3 build. Setting it pre-resolution + # needs no override API and works on both the old and new SGLang. if os.environ.get("INFERA_DEFAULT_KV_FP8", "1") != "0" and not any( t == "--kv-cache-dtype" or t.startswith("--kv-cache-dtype=") for t in remaining ): - server_args.kv_cache_dtype = "fp8_e4m3" + sglang_parsed.kv_cache_dtype = "fp8_e4m3" logger.info( "infera default: kv_cache_dtype=fp8_e4m3 " "(override with --kv-cache-dtype or INFERA_DEFAULT_KV_FP8=0)" ) + server_args = ServerArgs.from_cli_args(sglang_parsed) + from infera.engine.sglang.hicache_validate import warn_if_hicache_prefetch_disabled warn_if_hicache_prefetch_disabled(server_args) From e190d65fd31b0a053c7ddabcd26d39d526e0c13b Mon Sep 17 00:00:00 2001 From: "Zhang, Jiejing" Date: Sat, 1 Aug 2026 23:05:50 +0000 Subject: [PATCH 3/8] fix(engine): make the engine-ready timeout configurable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both workers hardcoded a 1800 s wait for the engine's /health. That is generous for local NVMe and impossible for NFS: Kimi-K3 read its 96 shards in 502 s from local disk and ~95 min from NFS with both PD nodes competing for the same mount. The failure is worse than a slow start. The worker times out mid-load, exits 1, restarts, and begins the load again from zero — so it never finishes, and the pod looks like a crash loop rather than "the storage is too slow for this deadline": TimeoutError: SGLang not ready after 1800s Load time tracks the storage, not the model, so this cannot have one right value baked in. INFERA_ENGINE_READY_TIMEOUT now sets it, on both the SGLang and vLLM workers, defaulting to the previous 1800 and falling back to it on a malformed value rather than raising during startup. The Kimi-K3 PD recipe sets 7200, with the measured numbers next to it so the next person can tell whether their storage needs more. Signed-off-by: Zhang, Jiejing --- examples/recipes/kimi-k3/pd-sglang/deploy.yaml | 18 ++++++++++++++++++ infera/engine/sglang/worker.py | 15 ++++++++++++++- infera/engine/vllm/worker.py | 15 ++++++++++++++- 3 files changed, 46 insertions(+), 2 deletions(-) diff --git a/examples/recipes/kimi-k3/pd-sglang/deploy.yaml b/examples/recipes/kimi-k3/pd-sglang/deploy.yaml index 1c9ae31..d25cb06 100644 --- a/examples/recipes/kimi-k3/pd-sglang/deploy.yaml +++ b/examples/recipes/kimi-k3/pd-sglang/deploy.yaml @@ -48,6 +48,12 @@ spec: - {name: POD_NAME, valueFrom: {fieldRef: {fieldPath: metadata.name}}} - {name: POD_NAMESPACE, valueFrom: {fieldRef: {fieldPath: metadata.namespace}}} - {name: POD_IP, valueFrom: {fieldRef: {fieldPath: status.podIP}}} + # Weight-load time tracks the STORAGE, not the model. Kimi-K3 read its + # 96 shards in 502 s from local NVMe and ~95 min from NFS with both PD + # nodes competing for the same mount. infera's default 1800 s ready + # timeout is generous for the first and impossible for the second — the + # worker kills itself mid-load, restarts, and never finishes. + - {name: INFERA_ENGINE_READY_TIMEOUT, value: "7200"} resources: {requests: {cpu: "4", memory: 8Gi}, limits: {cpu: "4", memory: 8Gi}} volumeMounts: [{name: overlay, mountPath: /overlay, readOnly: true}, {name: model, mountPath: /models, readOnly: true}] volumes: [{name: overlay, emptyDir: {}}, {name: model, hostPath: {path: /mnt/vast/yaocheng/models/moonshotai, type: Directory}}] @@ -87,6 +93,12 @@ spec: - {name: POD_NAME, valueFrom: {fieldRef: {fieldPath: metadata.name}}} - {name: POD_NAMESPACE, valueFrom: {fieldRef: {fieldPath: metadata.namespace}}} - {name: POD_IP, valueFrom: {fieldRef: {fieldPath: status.podIP}}} + # Weight-load time tracks the STORAGE, not the model. Kimi-K3 read its + # 96 shards in 502 s from local NVMe and ~95 min from NFS with both PD + # nodes competing for the same mount. infera's default 1800 s ready + # timeout is generous for the first and impossible for the second — the + # worker kills itself mid-load, restarts, and never finishes. + - {name: INFERA_ENGINE_READY_TIMEOUT, value: "7200"} resources: {requests: {cpu: "16", memory: 96Gi, amd.com/gpu: 1}, limits: {cpu: "16", memory: 96Gi, amd.com/gpu: 1}} volumeMounts: - {name: overlay, mountPath: /overlay, readOnly: true} @@ -142,6 +154,12 @@ spec: - {name: POD_NAME, valueFrom: {fieldRef: {fieldPath: metadata.name}}} - {name: POD_NAMESPACE, valueFrom: {fieldRef: {fieldPath: metadata.namespace}}} - {name: POD_IP, valueFrom: {fieldRef: {fieldPath: status.podIP}}} + # Weight-load time tracks the STORAGE, not the model. Kimi-K3 read its + # 96 shards in 502 s from local NVMe and ~95 min from NFS with both PD + # nodes competing for the same mount. infera's default 1800 s ready + # timeout is generous for the first and impossible for the second — the + # worker kills itself mid-load, restarts, and never finishes. + - {name: INFERA_ENGINE_READY_TIMEOUT, value: "7200"} resources: {requests: {cpu: "16", memory: 96Gi, amd.com/gpu: 1}, limits: {cpu: "16", memory: 96Gi, amd.com/gpu: 1}} volumeMounts: - {name: overlay, mountPath: /overlay, readOnly: true} diff --git a/infera/engine/sglang/worker.py b/infera/engine/sglang/worker.py index a75a726..2fb3f53 100644 --- a/infera/engine/sglang/worker.py +++ b/infera/engine/sglang/worker.py @@ -33,6 +33,14 @@ } +def _ready_timeout() -> float: + """Seconds to wait for the engine's /health, from INFERA_ENGINE_READY_TIMEOUT.""" + try: + return float(os.environ.get("INFERA_ENGINE_READY_TIMEOUT", "1800")) + except ValueError: + return 1800.0 + + class SglangEngine(BaseEngine): """Runs `python -m sglang.launch_server` in a child process. @@ -136,7 +144,12 @@ async def start(self) -> EngineConfig: dp_size=dp_size, ) - async def _wait_ready(self, timeout: float = 1800) -> None: + # Weight-load time tracks the storage, not the model: Kimi-K3 read 96 shards in + # 502 s from local NVMe and ~95 min from NFS with both PD nodes competing for + # the same mount. A hardcoded 1800 s is generous for the first and impossible + # for the second — the worker killed itself mid-load, restarted, and could + # never finish. Tunable via INFERA_ENGINE_READY_TIMEOUT (seconds). + async def _wait_ready(self, timeout: float = _ready_timeout()) -> None: # /health is probed locally; sglang binds on server_args.host, but if # that is 0.0.0.0 we should probe via 127.0.0.1 instead. probe_host = self.server_args.host diff --git a/infera/engine/vllm/worker.py b/infera/engine/vllm/worker.py index ce1024f..23c1050 100644 --- a/infera/engine/vllm/worker.py +++ b/infera/engine/vllm/worker.py @@ -24,6 +24,14 @@ logger = logging.getLogger(__name__) +def _ready_timeout() -> float: + """Seconds to wait for the engine's /health, from INFERA_ENGINE_READY_TIMEOUT.""" + try: + return float(os.environ.get("INFERA_ENGINE_READY_TIMEOUT", "1800")) + except ValueError: + return 1800.0 + + class VllmEngine(BaseEngine): def __init__( self, @@ -162,7 +170,12 @@ async def stop(self) -> None: except ProcessLookupError: pass - async def _wait_ready(self, timeout: float = 1800) -> None: + # Weight-load time tracks the storage, not the model: Kimi-K3 read 96 shards in + # 502 s from local NVMe and ~95 min from NFS with both PD nodes competing for + # the same mount. A hardcoded 1800 s is generous for the first and impossible + # for the second — the worker killed itself mid-load, restarted, and could + # never finish. Tunable via INFERA_ENGINE_READY_TIMEOUT (seconds). + async def _wait_ready(self, timeout: float = _ready_timeout()) -> None: # 30 min: cold model download + ROCm kernel compile can eat 10+ min. probe_host = "127.0.0.1" if self.host in ("0.0.0.0", "") else self.host url = f"http://{probe_host}:{self.port}/health" From 3eadde2ef40dfcae4bd5e864ad1439b17f0c0646 Mon Sep 17 00:00:00 2001 From: "Zhang, Jiejing" Date: Sun, 2 Aug 2026 01:06:20 +0000 Subject: [PATCH 4/8] fix(recipes): carry the PD lessons into both Kimi-K3 manifests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both PD manifests were derived from ancestors that predate what running PD actually taught us, and each inherited the same gaps. Bringing them level: hostNetwork on the workers the RDMA rails are host interfaces; the flannel pod network cannot reach them host libionic mount the vendor base's provider must match the host ionic kernel ABI or libibverbs sees no devices /dev/infiniband + privileged RDMA needs the device nodes and pinned memory INFERA_ENGINE_READY_TIMEOUT 1800 s is impossible on NFS-backed weights The SGLang manifest additionally carries what its own bring-up cost: memory 96Gi (a Qwen-0.6B number) against 1.5 TB of shards, a mamba state cache sized from a negative residual, and the mem-fraction trap documented inline. None of this is speculative — every line replaces a failure whose message pointed somewhere other than its cause: "0 RDMA devices" for an ABI mismatch, a crash loop for a timeout, a bare "-9" for two different problems in turn. Signed-off-by: Zhang, Jiejing --- .../recipes/kimi-k3/pd-sglang/deploy.yaml | 100 ++++++++++++++++-- examples/recipes/kimi-k3/pd/deploy.yaml | 40 ++++++- 2 files changed, 129 insertions(+), 11 deletions(-) diff --git a/examples/recipes/kimi-k3/pd-sglang/deploy.yaml b/examples/recipes/kimi-k3/pd-sglang/deploy.yaml index d25cb06..15b1d11 100644 --- a/examples/recipes/kimi-k3/pd-sglang/deploy.yaml +++ b/examples/recipes/kimi-k3/pd-sglang/deploy.yaml @@ -10,8 +10,16 @@ # carrying Kimi-K3 support; the general v0.5.15 tag has none) # overlay inferaimage/infera-overlay:v0.2.1 # -# Weights come from NFS so both nodes see the same path — Kimi-K3 is ~1.5 TB and -# the decode node here has 147 GB free, so a local copy is not an option. +# / +# the model directory on EACH node. They need NOT match: every pod mounts its +# own node's copy, so nodes that name their local NVMe differently +# (/mnt/k3local vs /mnt/shared here) are fine. +# +# Put the weights on LOCAL disk on both nodes. Kimi-K3 loads its 96 shards in +# ~8 min from NVMe and ~95 min from NFS when both PD nodes read the same mount — +# and the second case does not merely run slow, it exceeds infera's ready timeout, +# so the worker restarts mid-load and never finishes. If you must use NFS, raise +# INFERA_ENGINE_READY_TIMEOUT accordingly. apiVersion: infera.amd.com/v1alpha1 kind: InferaDeployment metadata: @@ -56,7 +64,7 @@ spec: - {name: INFERA_ENGINE_READY_TIMEOUT, value: "7200"} resources: {requests: {cpu: "4", memory: 8Gi}, limits: {cpu: "4", memory: 8Gi}} volumeMounts: [{name: overlay, mountPath: /overlay, readOnly: true}, {name: model, mountPath: /models, readOnly: true}] - volumes: [{name: overlay, emptyDir: {}}, {name: model, hostPath: {path: /mnt/vast/yaocheng/models/moonshotai, type: Directory}}] + volumes: [{name: overlay, emptyDir: {}}, {name: model, hostPath: {path: , type: Directory}}] prefill: componentType: worker role: prefill @@ -85,7 +93,44 @@ spec: command: ["/overlay/bin/infera-exec","python3","-m","infera.engine.sglang","--host","0.0.0.0","--port","30000", "--disaggregation-mode","prefill","--model-path","/models/Kimi-K3", "--tp-size","8","--attention-backend","aiter","--trust-remote-code", - "--mem-fraction-static","0.8", + "--mem-fraction-static","0.97", + # 0.93, not the 0.8 inherited from this manifest's Qwen-0.6B + # ancestor. Kimi-K3 is ~190 GB per rank at TP8, and at 0.8 the + # hybrid Mamba state cache is computed AFTER the weights land: + # Not enough GPU memory for hybrid (mamba/linear-attention) + # state cache. Computed max_mamba_cache_size=-554 + # (total_rest_memory=-60.99 GB) + # i.e. in deficit before any cache is allocated. The engine exits + # and the wrapper reports a bare "-9", which reads as OOM and is + # not — chasing the container memory limit first is a dead end. + # Measured, both knobs the engine itself suggests: + # mem-fraction 0.80 -> total_rest_memory -60.99 GB + # mem-fraction 0.93 -> total_rest_memory -30.39 GB + # so 0.97 plus a capped --max-running-requests (mamba cache is + # 53.57 MB per request) and a 32k context. + "--context-length","32768", + "--max-running-requests","16", + # Pin the hybrid state cache instead of letting it be derived. + # SGLang sizes it from whatever GPU memory is left AFTER the + # weights, and on Kimi-K3 at TP8 that residual is negative, so + # the derivation yields a negative cache and the engine exits: + # 0.80 -> total_rest_memory -60.99 GB + # 0.93 -> -30.39 GB + # 0.97 -> -20.98 GB + # Raising mem-fraction shrinks the gap but cannot close it — + # the residual is the wrong side of zero to begin with. An + # explicit size skips the derivation. 16 requests x 53.57 MB. + "--max-mamba-cache-size","16", + # --language-only is what makes --mem-fraction-static stick. + # Kimi-K3 is multimodal, so SGLang runs adjust_mem_fraction_for_vlm() + # and SCALES the value you passed: + # mem_fraction_static = original * final_overall_factor + # 0.97 became 0.8245, and the engine then failed telling us to + # "raise --mem-fraction-static above 0.901" — advice that cannot + # work, since the raised value is scaled down again. The guard is + # `if model_config.is_multimodal and not self.language_only`. + # Text-only PD does not need the vision tower, so opt out. + "--language-only", "--kv-event-transport","zmq","--request-transport","http", "--disaggregation-transfer-backend","mooncake"] env: @@ -99,7 +144,7 @@ spec: # timeout is generous for the first and impossible for the second — the # worker kills itself mid-load, restarts, and never finishes. - {name: INFERA_ENGINE_READY_TIMEOUT, value: "7200"} - resources: {requests: {cpu: "16", memory: 96Gi, amd.com/gpu: 1}, limits: {cpu: "16", memory: 96Gi, amd.com/gpu: 1}} + resources: {requests: {cpu: "16", memory: 768Gi, amd.com/gpu: 1}, limits: {cpu: "16", memory: 768Gi, amd.com/gpu: 1}} volumeMounts: - {name: overlay, mountPath: /overlay, readOnly: true} - {name: model, mountPath: /models, readOnly: true} @@ -113,7 +158,7 @@ spec: - {name: host-libionic, mountPath: /host-libionic/libionic.so, readOnly: true} volumes: - {name: overlay, emptyDir: {}} - - {name: model, hostPath: {path: /mnt/vast/yaocheng/models/moonshotai, type: Directory}} + - {name: model, hostPath: {path: , type: Directory}} - {name: dshm, emptyDir: {medium: Memory, sizeLimit: 16Gi}} - {name: ib, hostPath: {path: /dev/infiniband, type: Directory}} - name: host-libionic @@ -146,7 +191,44 @@ spec: command: ["/overlay/bin/infera-exec","python3","-m","infera.engine.sglang","--host","0.0.0.0","--port","30000", "--disaggregation-mode","decode","--model-path","/models/Kimi-K3", "--tp-size","8","--attention-backend","aiter","--trust-remote-code", - "--mem-fraction-static","0.8", + "--mem-fraction-static","0.97", + # 0.93, not the 0.8 inherited from this manifest's Qwen-0.6B + # ancestor. Kimi-K3 is ~190 GB per rank at TP8, and at 0.8 the + # hybrid Mamba state cache is computed AFTER the weights land: + # Not enough GPU memory for hybrid (mamba/linear-attention) + # state cache. Computed max_mamba_cache_size=-554 + # (total_rest_memory=-60.99 GB) + # i.e. in deficit before any cache is allocated. The engine exits + # and the wrapper reports a bare "-9", which reads as OOM and is + # not — chasing the container memory limit first is a dead end. + # Measured, both knobs the engine itself suggests: + # mem-fraction 0.80 -> total_rest_memory -60.99 GB + # mem-fraction 0.93 -> total_rest_memory -30.39 GB + # so 0.97 plus a capped --max-running-requests (mamba cache is + # 53.57 MB per request) and a 32k context. + "--context-length","32768", + "--max-running-requests","16", + # Pin the hybrid state cache instead of letting it be derived. + # SGLang sizes it from whatever GPU memory is left AFTER the + # weights, and on Kimi-K3 at TP8 that residual is negative, so + # the derivation yields a negative cache and the engine exits: + # 0.80 -> total_rest_memory -60.99 GB + # 0.93 -> -30.39 GB + # 0.97 -> -20.98 GB + # Raising mem-fraction shrinks the gap but cannot close it — + # the residual is the wrong side of zero to begin with. An + # explicit size skips the derivation. 16 requests x 53.57 MB. + "--max-mamba-cache-size","16", + # --language-only is what makes --mem-fraction-static stick. + # Kimi-K3 is multimodal, so SGLang runs adjust_mem_fraction_for_vlm() + # and SCALES the value you passed: + # mem_fraction_static = original * final_overall_factor + # 0.97 became 0.8245, and the engine then failed telling us to + # "raise --mem-fraction-static above 0.901" — advice that cannot + # work, since the raised value is scaled down again. The guard is + # `if model_config.is_multimodal and not self.language_only`. + # Text-only PD does not need the vision tower, so opt out. + "--language-only", "--kv-event-transport","zmq","--request-transport","http", "--disaggregation-transfer-backend","mooncake"] env: @@ -160,7 +242,7 @@ spec: # timeout is generous for the first and impossible for the second — the # worker kills itself mid-load, restarts, and never finishes. - {name: INFERA_ENGINE_READY_TIMEOUT, value: "7200"} - resources: {requests: {cpu: "16", memory: 96Gi, amd.com/gpu: 1}, limits: {cpu: "16", memory: 96Gi, amd.com/gpu: 1}} + resources: {requests: {cpu: "16", memory: 768Gi, amd.com/gpu: 1}, limits: {cpu: "16", memory: 768Gi, amd.com/gpu: 1}} volumeMounts: - {name: overlay, mountPath: /overlay, readOnly: true} - {name: model, mountPath: /models, readOnly: true} @@ -174,7 +256,7 @@ spec: - {name: host-libionic, mountPath: /host-libionic/libionic.so, readOnly: true} volumes: - {name: overlay, emptyDir: {}} - - {name: model, hostPath: {path: /mnt/vast/yaocheng/models/moonshotai, type: Directory}} + - {name: model, hostPath: {path: , type: Directory}} - {name: dshm, emptyDir: {medium: Memory, sizeLimit: 16Gi}} - {name: ib, hostPath: {path: /dev/infiniband, type: Directory}} - name: host-libionic diff --git a/examples/recipes/kimi-k3/pd/deploy.yaml b/examples/recipes/kimi-k3/pd/deploy.yaml index f5a4e85..8f65aa8 100644 --- a/examples/recipes/kimi-k3/pd/deploy.yaml +++ b/examples/recipes/kimi-k3/pd/deploy.yaml @@ -67,6 +67,11 @@ spec: port: 30000 extraPodSpec: nodeSelector: {kubernetes.io/hostname: } + # hostNetwork is REQUIRED: the RDMA rails are host interfaces and the + # flannel pod network cannot reach them, so Mooncake has no path to the + # peer while the engine looks healthy. + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet # The overlay payload is dropped into an emptyDir here; the STOCK vendor # image below then runs infera out of it. The vendor image is never forked, # so following an upstream bump costs nothing in this repo. @@ -97,13 +102,19 @@ spec: - {name: POD_NAME, valueFrom: {fieldRef: {fieldPath: metadata.name}}} - {name: POD_NAMESPACE, valueFrom: {fieldRef: {fieldPath: metadata.namespace}}} - {name: POD_IP, valueFrom: {fieldRef: {fieldPath: status.podIP}}} + # Load time tracks the STORAGE: Kimi-K3 reads 96 shards in ~8 min from + # local NVMe and ~95 min from NFS. infera's 1800 s default makes the + # worker kill itself mid-load and restart forever on the slow path. + - {name: INFERA_ENGINE_READY_TIMEOUT, value: "7200"} - {name: HF_HUB_OFFLINE, value: "1"} - {name: VLLM_ROCM_USE_AITER, value: "1"} - {name: SAFETENSORS_FAST_GPU, value: "1"} # PD's KV transport is Mooncake, which lives in the overlay's native # tree. Name it so a payload lacking it fails at startup instead of only # warning and quietly serving with no KV transfer at all. - - {name: INFERA_REQUIRE_NATIVE, value: "mooncake"} + - {name: INFERA_REQUIRE_NATIVE, value: "mooncake"} # RDMA needs the device nodes and pinned memory. Without these the + # engine starts, registers, and only fails when KV actually moves. + securityContext: {privileged: true, capabilities: {add: ["IPC_LOCK","SYS_PTRACE"]}} startupProbe: httpGet: {path: /health, port: 30000} periodSeconds: 10 @@ -115,10 +126,17 @@ spec: volumeMounts: - {name: overlay, mountPath: /overlay, readOnly: true} - {name: model, mountPath: /models, readOnly: true} + # The vendor base's libionic must match the host ionic kernel ABI, or + # libibverbs rejects every device ("No RDMA devices found") and Mooncake + # falls back to HIP IPC, which cannot work across nodes. + - {name: host-libionic, mountPath: /host-libionic/libionic.so, readOnly: true} - {name: dshm, mountPath: /dev/shm} volumes: - {name: overlay, emptyDir: {}} - {name: model, persistentVolumeClaim: {claimName: model-cache}} + - name: host-libionic + hostPath: {path: /usr/lib/x86_64-linux-gnu/libionic.so.1, type: File} + - {name: ib, hostPath: {path: /dev/infiniband, type: Directory}} - {name: dshm, emptyDir: {medium: Memory, sizeLimit: 64Gi}} decode: componentType: worker @@ -127,6 +145,11 @@ spec: port: 30000 extraPodSpec: nodeSelector: {kubernetes.io/hostname: } + # hostNetwork is REQUIRED: the RDMA rails are host interfaces and the + # flannel pod network cannot reach them, so Mooncake has no path to the + # peer while the engine looks healthy. + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet # The overlay payload is dropped into an emptyDir here; the STOCK vendor # image below then runs infera out of it. The vendor image is never forked, # so following an upstream bump costs nothing in this repo. @@ -157,13 +180,19 @@ spec: - {name: POD_NAME, valueFrom: {fieldRef: {fieldPath: metadata.name}}} - {name: POD_NAMESPACE, valueFrom: {fieldRef: {fieldPath: metadata.namespace}}} - {name: POD_IP, valueFrom: {fieldRef: {fieldPath: status.podIP}}} + # Load time tracks the STORAGE: Kimi-K3 reads 96 shards in ~8 min from + # local NVMe and ~95 min from NFS. infera's 1800 s default makes the + # worker kill itself mid-load and restart forever on the slow path. + - {name: INFERA_ENGINE_READY_TIMEOUT, value: "7200"} - {name: HF_HUB_OFFLINE, value: "1"} - {name: VLLM_ROCM_USE_AITER, value: "1"} - {name: SAFETENSORS_FAST_GPU, value: "1"} # PD's KV transport is Mooncake, which lives in the overlay's native # tree. Name it so a payload lacking it fails at startup instead of only # warning and quietly serving with no KV transfer at all. - - {name: INFERA_REQUIRE_NATIVE, value: "mooncake"} + - {name: INFERA_REQUIRE_NATIVE, value: "mooncake"} # RDMA needs the device nodes and pinned memory. Without these the + # engine starts, registers, and only fails when KV actually moves. + securityContext: {privileged: true, capabilities: {add: ["IPC_LOCK","SYS_PTRACE"]}} startupProbe: httpGet: {path: /health, port: 30000} periodSeconds: 10 @@ -175,8 +204,15 @@ spec: volumeMounts: - {name: overlay, mountPath: /overlay, readOnly: true} - {name: model, mountPath: /models, readOnly: true} + # The vendor base's libionic must match the host ionic kernel ABI, or + # libibverbs rejects every device ("No RDMA devices found") and Mooncake + # falls back to HIP IPC, which cannot work across nodes. + - {name: host-libionic, mountPath: /host-libionic/libionic.so, readOnly: true} - {name: dshm, mountPath: /dev/shm} volumes: - {name: overlay, emptyDir: {}} - {name: model, persistentVolumeClaim: {claimName: model-cache}} + - name: host-libionic + hostPath: {path: /usr/lib/x86_64-linux-gnu/libionic.so.1, type: File} + - {name: ib, hostPath: {path: /dev/infiniband, type: Directory}} - {name: dshm, emptyDir: {medium: Memory, sizeLimit: 64Gi}} From 5b31fe0291ca7acdbe6ff00e90d042b5b777706e Mon Sep 17 00:00:00 2001 From: "Zhang, Jiejing" Date: Sun, 2 Aug 2026 01:32:32 +0000 Subject: [PATCH 5/8] feat(overlay): apply vendor engine patches at container start MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The overlay could ship code the vendor base lacks, but not fix code it has. That gap became blocking: Kimi-K3 PD on vLLM dies during KV registration because the Mooncake connector destructures every Mamba-family layer as a 2-tuple — if isinstance(layer_spec, MambaSpec): conv, _ = cache_or_caches which is Mamba2's shape. Kimi-K3 uses KDA linear attention with a different number of state tensors, so every rank raises "too many values to unpack". MambaSpec.shapes is already a variable-length tuple, so the arity assumption is the connector's, not the spec's. The obvious fix — bake the patch into Dockerfile.vllm — reintroduces exactly the fork this overlay exists to avoid, and for a 42 GB image. So the payload now carries deploy/docker/patches/vllm/ and infera-exec applies them before exec. Each script locates its target through the installed package, is idempotent, and no-ops when its anchor is gone, so a base that already carries a fix is untouched and the patches survive a base bump. Guarded rather than unconditional: skipped when vllm is not importable, skipped by INFERA_SKIP_ENGINE_PATCHES, and a failing patch warns instead of aborting — a mis-anchored patch must not take down a worker that would otherwise serve. Verified in a stock vllm/vllm-openai-rocm:kimi-k3: the mamba unpack is applied, the old line is gone, and the pre-existing blocksize/prom patches apply too — so this also closes the gap where an overlay deployment silently lacked fixes the baked image had. Signed-off-by: Zhang, Jiejing --- .../vllm/patch_mooncake_mamba_unpack.py | 65 +++++++++++++++++++ deploy/overlay/Dockerfile.payload | 6 ++ deploy/overlay/infera-exec | 25 +++++++ 3 files changed, 96 insertions(+) create mode 100644 deploy/docker/patches/vllm/patch_mooncake_mamba_unpack.py diff --git a/deploy/docker/patches/vllm/patch_mooncake_mamba_unpack.py b/deploy/docker/patches/vllm/patch_mooncake_mamba_unpack.py new file mode 100644 index 0000000..e03dbb0 --- /dev/null +++ b/deploy/docker/patches/vllm/patch_mooncake_mamba_unpack.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""Mooncake connector: don't assume a Mamba layer has exactly two state tensors. + +register_kv_caches() destructures every MambaSpec layer as a 2-tuple: + + if isinstance(layer_spec, MambaSpec): + conv, _ = cache_or_caches + cache_list = [conv] + +That holds for Mamba2 (conv state, SSM state) and fails for anything else. +Kimi-K3 uses KDA linear attention, whose layers carry a different number of state +tensors, so PD dies during KV registration for every rank at once: + + mooncake_connector.py:1678 in register_kv_caches + conv, _ = cache_or_caches + ValueError: too many values to unpack (expected 2) + +MambaSpec.shapes is already a variable-length tuple of shapes, so the two-tensor +assumption is the connector's, not the spec's. Take the first state tensor — +which is what the original code kept — without constraining how many follow. + +Self-locating and idempotent: re-running is a no-op, and it no-ops if upstream +fixes this, so the patch can stay in place across a base bump. +""" + +import sys +from pathlib import Path + +OLD = """ if isinstance(layer_spec, MambaSpec): + conv, _ = cache_or_caches + cache_list = [conv]""" + +NEW = """ if isinstance(layer_spec, MambaSpec): + # A Mamba-family layer does not necessarily carry exactly two + # state tensors: that is Mamba2's shape (conv, ssm). KDA linear + # attention (Kimi-K3) carries a different number, and `conv, _ =` + # then raises "too many values to unpack" on every rank. Keep the + # first state tensor, as before, without fixing the arity. + cache_list = [cache_or_caches[0]]""" + + +def main() -> int: + import vllm + + target = ( + Path(vllm.__file__).parent + / "distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py" + ) + if not target.is_file(): + print(f"[patch] {target} not found; skipping") + return 0 + src = target.read_text() + if NEW in src: + print("[patch] mooncake mamba unpack: already applied") + return 0 + if OLD not in src: + print("[patch] mooncake mamba unpack: anchor absent (upstream changed?); skipping") + return 0 + target.write_text(src.replace(OLD, NEW)) + print(f"[patch] mooncake mamba unpack: applied to {target}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/deploy/overlay/Dockerfile.payload b/deploy/overlay/Dockerfile.payload index ac39d51..626dc55 100644 --- a/deploy/overlay/Dockerfile.payload +++ b/deploy/overlay/Dockerfile.payload @@ -112,5 +112,11 @@ COPY --from=deps312 /payload/py312 /payload/py312 COPY --from=native312 /native /payload/native COPY --from=native310 /native /payload/native COPY --from=native312 /usr/local/bin/infera-router /payload/bin/infera-router +# Engine patches applied at container start (see infera-exec). These fix vendor +# code we do not otherwise ship — the overlay's whole point is not forking the +# vendor image, and a runtime patch keeps that true where a rebuild would not. +# Each script is self-locating, idempotent, and no-ops once upstream carries the +# fix, so they are safe to leave in place across a base bump. +COPY deploy/docker/patches/vllm/ /payload/patches/vllm/ COPY deploy/overlay/infera-exec /payload/bin/infera-exec CMD ["sh", "-c", "cp -a /payload/. /out/ && echo 'infera overlay payload installed to /out'"] diff --git a/deploy/overlay/infera-exec b/deploy/overlay/infera-exec index e04684a..8f3a0bc 100755 --- a/deploy/overlay/infera-exec +++ b/deploy/overlay/infera-exec @@ -110,6 +110,31 @@ if [ -e /host-libionic/libionic.so ]; then fi fi +# --- vendor engine patches -------------------------------------------------- +# Fixes to the vendor's own code, applied here rather than baked into a forked +# image — forking is exactly what this overlay exists to avoid. Each script +# locates its target through the installed package, is idempotent, and no-ops +# when the anchor is absent, so a base that already carries the fix is untouched +# and the patch can stay in place across bumps. +# +# Currently: the Mooncake connector destructures every Mamba-family layer as a +# 2-tuple, which is Mamba2's shape. Kimi-K3's KDA linear attention carries a +# different number of state tensors, so PD died on every rank with "too many +# values to unpack" during KV registration. +# +# Skipped entirely when INFERA_SKIP_ENGINE_PATCHES is set, and a failing patch +# warns rather than blocking startup — a mis-anchored patch must not take down a +# worker that would otherwise serve. +if [ -d "$PAYLOAD_ROOT/patches/vllm" ] && [ -z "${INFERA_SKIP_ENGINE_PATCHES:-}" ]; then + if "$PY" -c 'import vllm' >/dev/null 2>&1; then + for _p in "$PAYLOAD_ROOT"/patches/vllm/*.py; do + [ -e "$_p" ] || continue + (cd / && PYTHONPATH="$PYTREE${PYTHONPATH:+:$PYTHONPATH}" "$PY" "$_p") \ + || echo "infera-exec: patch $(basename "$_p") failed; continuing" >&2 + done + fi +fi + # Python puts the working directory at sys.path[0] — ahead of PYTHONPATH. Vendor # images that already ship infera set WORKDIR to its parent (e.g. /opt/infera), # so the baked-in copy would silently shadow this payload and the overlay would From f99752e4411950eb9bcec675083a82e915ab8272 Mon Sep 17 00:00:00 2001 From: "Zhang, Jiejing" Date: Sun, 2 Aug 2026 02:47:30 +0000 Subject: [PATCH 6/8] fix(vllm): rebuild Mooncake's bootstrap_addr once the routable host is known MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit infera resolves the Pod IP under Kubernetes discovery and logs it — k8s discovery: advertising Pod IP 137.220.61.177 — but disagg_meta, which carries Mooncake's bootstrap_addr, was already built during arg parsing, while advertise_host was still unset. So the address froze as the bind host and the prefiller published: bootstrap_addr: 'http://0.0.0.0:8998' The decoder reads that from kv_transfer_params (vLLM's own comment says decoders must) and cannot connect: Failed to connect to bootstrap server http://0.0.0.0:8998 Failed to find remote engine_id ... Both workers stay healthy and registered; the request simply hangs until the connector's 480 s timeout. Nothing in that path names the address. This is the same defect as the SGLang KV-endpoint one, in a second place: an address is derived before the host that makes it routable is known. Rebuild it after the resolution, touching only the 0.0.0.0 form so an explicitly configured address is left alone. The manifest also gains --data-parallel-address $(POD_IP): that fixes the OTHER half, where the prefiller computes its own address via get_mooncake_bootstrap_addr() -> parallel_config.data_parallel_master_ip. Both paths have to be right; fixing either alone leaves PD hanging. With both, Kimi-K3 PD answers across two nodes: prefill on chi2800, decode on chi2866, 1.5 TB MXFP4 at TP8 from local NVMe, "The capital of France is Paris." Signed-off-by: Zhang, Jiejing --- examples/recipes/kimi-k3/pd/deploy.yaml | 22 ++++++++++++++++++++++ infera/engine/vllm/__main__.py | 22 ++++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/examples/recipes/kimi-k3/pd/deploy.yaml b/examples/recipes/kimi-k3/pd/deploy.yaml index 8f65aa8..ffcccce 100644 --- a/examples/recipes/kimi-k3/pd/deploy.yaml +++ b/examples/recipes/kimi-k3/pd/deploy.yaml @@ -96,6 +96,17 @@ spec: "--enable-prefix-caching","--max-num-seqs","128","--max-num-batched-tokens","4096", "--enable-auto-tool-choice","--tool-call-parser","kimi_k3", "--reasoning-parser","kimi_k3", + "--data-parallel-address","$(POD_IP)", + # Mooncake's bootstrap server BINDS 0.0.0.0 (fine) but ADVERTISES + # parallel_config.data_parallel_master_ip, which defaults to + # 127.0.0.1 and is whatever --host implies otherwise. With the + # usual 0.0.0.0 the prefiller registers "http://0.0.0.0:8998" and + # the decoder cannot reach it: + # Failed to connect to bootstrap server http://0.0.0.0:8998 + # Failed to find remote engine_id ... + # while both workers look healthy and the request just hangs to + # a 480 s connector timeout. $(POD_IP) is expanded by kubelet + # from the downward API env below. "--kv-transfer-config","{\"kv_connector\":\"MooncakeConnector\",\"kv_role\":\"kv_producer\"}", "--kv-event-transport","zmq","--request-transport","http"] env: @@ -174,6 +185,17 @@ spec: "--max-num-seqs","128","--max-num-batched-tokens","4096", "--enable-auto-tool-choice","--tool-call-parser","kimi_k3", "--reasoning-parser","kimi_k3", + "--data-parallel-address","$(POD_IP)", + # Mooncake's bootstrap server BINDS 0.0.0.0 (fine) but ADVERTISES + # parallel_config.data_parallel_master_ip, which defaults to + # 127.0.0.1 and is whatever --host implies otherwise. With the + # usual 0.0.0.0 the prefiller registers "http://0.0.0.0:8998" and + # the decoder cannot reach it: + # Failed to connect to bootstrap server http://0.0.0.0:8998 + # Failed to find remote engine_id ... + # while both workers look healthy and the request just hangs to + # a 480 s connector timeout. $(POD_IP) is expanded by kubelet + # from the downward API env below. "--kv-transfer-config","{\"kv_connector\":\"MooncakeConnector\",\"kv_role\":\"kv_consumer\"}", "--kv-event-transport","zmq","--request-transport","http"] env: diff --git a/infera/engine/vllm/__main__.py b/infera/engine/vllm/__main__.py index dce281a..b8f33c4 100644 --- a/infera/engine/vllm/__main__.py +++ b/infera/engine/vllm/__main__.py @@ -181,6 +181,28 @@ async def main() -> None: args.advertise_host = pod_ip logger.info("k8s discovery: advertising Pod IP %s", pod_ip) + # disagg_meta was already built during arg parsing, when advertise_host + # was still unset — so Mooncake's bootstrap_addr was frozen as the bind + # host. The prefiller then publishes "http://0.0.0.0:8998", the decoder + # reads it from kv_transfer_params and cannot connect: + # + # Failed to connect to bootstrap server http://0.0.0.0:8998 + # Failed to find remote engine_id ... + # + # Both workers stay healthy and the request simply hangs to the + # connector's 480 s timeout, so nothing points at the address. Rebuild + # the address now that the routable host is known. + meta = args.disagg_meta or {} + params = meta.get("params") or {} + addr = params.get("bootstrap_addr") + if addr and "//0.0.0.0:" in addr: + params["bootstrap_addr"] = addr.replace("//0.0.0.0:", f"//{pod_ip}:") + logger.info( + "k8s discovery: bootstrap_addr %s -> %s", + addr, + params["bootstrap_addr"], + ) + logger.info( "parsed args: model=%s host=%s port=%d disagg=%s disagg_meta=%s enable_kv_events=%s", args.model, From 97198bca2d6a03098d74459f31347b957602117b Mon Sep 17 00:00:00 2001 From: "Zhang, Jiejing" Date: Sun, 2 Aug 2026 03:11:55 +0000 Subject: [PATCH 7/8] build(overlay): publish v0.2.2 and point the recipes at it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit v0.2.1 predates everything cross-node PD needed. Comparing the published image against the current build: v0.2.1 v0.2.2 libionic injection no yes runtime engine patches no yes (8 patch scripts) native trees 2 2 So anyone following a recipe today gets an overlay that cannot bring up PD: no libionic swap means libibverbs rejects every ionic device, and no runtime patches means the Mooncake connector still destructures Kimi-K3's Mamba layers as a 2-tuple. Both were fixed in this branch and neither was reachable. inferaimage/infera-overlay:v0.2.2 is public and pulls with no credentials — verified from a clean DOCKER_CONFIG, since "we pushed it" and "a reader can get it" have already turned out to be different things twice in this repo. Validated with this image: Kimi-K3 PD across chi2800 and chi2866, 1.5 TB MXFP4 at TP8 from local NVMe, ~6 min to both workers registered, 8 RDMA devices visible to libibverbs on each side, and a correct answer through the router. Signed-off-by: Zhang, Jiejing --- examples/k8s-deployments/glm5.2-sglang.yaml | 4 ++-- examples/k8s-deployments/kimi-k3-vllm.yaml | 4 ++-- examples/k8s-deployments/mixed-kvd-vllm.yaml | 4 ++-- examples/k8s-deployments/mixed-kvd.yaml | 4 ++-- examples/k8s-deployments/pd-1p1d-mooncake.yaml | 6 +++--- examples/k8s-deployments/pd-kvd.yaml | 6 +++--- examples/k8s-deployments/single-node-qwen-sglang.yaml | 4 ++-- examples/k8s-deployments/single-node-qwen-vllm.yaml | 4 ++-- examples/recipes/kimi-k3/pd-sglang/deploy.yaml | 8 ++++---- manual/examples/k8s_kimi_k3.md | 4 ++-- 10 files changed, 24 insertions(+), 24 deletions(-) diff --git a/examples/k8s-deployments/glm5.2-sglang.yaml b/examples/k8s-deployments/glm5.2-sglang.yaml index 46866d7..0702bd7 100644 --- a/examples/k8s-deployments/glm5.2-sglang.yaml +++ b/examples/k8s-deployments/glm5.2-sglang.yaml @@ -29,7 +29,7 @@ spec: # is an image-tag edit rather than a rebuild of ours. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay:v0.2.2 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: @@ -64,7 +64,7 @@ spec: # is an image-tag edit rather than a rebuild of ours. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay:v0.2.2 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: diff --git a/examples/k8s-deployments/kimi-k3-vllm.yaml b/examples/k8s-deployments/kimi-k3-vllm.yaml index 65bb2d3..084cea4 100644 --- a/examples/k8s-deployments/kimi-k3-vllm.yaml +++ b/examples/k8s-deployments/kimi-k3-vllm.yaml @@ -34,7 +34,7 @@ spec: # is an image-tag edit rather than a rebuild of ours. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay:v0.2.2 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: @@ -69,7 +69,7 @@ spec: # is an image-tag edit rather than a rebuild of ours. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay:v0.2.2 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: diff --git a/examples/k8s-deployments/mixed-kvd-vllm.yaml b/examples/k8s-deployments/mixed-kvd-vllm.yaml index 9f24f51..2b81039 100644 --- a/examples/k8s-deployments/mixed-kvd-vllm.yaml +++ b/examples/k8s-deployments/mixed-kvd-vllm.yaml @@ -58,7 +58,7 @@ spec: # is an image-tag edit rather than a rebuild of ours. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay:v0.2.2 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: @@ -91,7 +91,7 @@ spec: # is an image-tag edit rather than a rebuild of ours. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay:v0.2.2 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: diff --git a/examples/k8s-deployments/mixed-kvd.yaml b/examples/k8s-deployments/mixed-kvd.yaml index 7cd7902..cbf01b8 100644 --- a/examples/k8s-deployments/mixed-kvd.yaml +++ b/examples/k8s-deployments/mixed-kvd.yaml @@ -65,7 +65,7 @@ spec: # is an image-tag edit rather than a rebuild of ours. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay:v0.2.2 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: @@ -98,7 +98,7 @@ spec: # is an image-tag edit rather than a rebuild of ours. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay:v0.2.2 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: diff --git a/examples/k8s-deployments/pd-1p1d-mooncake.yaml b/examples/k8s-deployments/pd-1p1d-mooncake.yaml index fc0ff46..d3c12b0 100644 --- a/examples/k8s-deployments/pd-1p1d-mooncake.yaml +++ b/examples/k8s-deployments/pd-1p1d-mooncake.yaml @@ -37,7 +37,7 @@ spec: # is an image-tag edit rather than a rebuild of ours. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay:v0.2.2 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: @@ -75,7 +75,7 @@ spec: # is an image-tag edit rather than a rebuild of ours. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay:v0.2.2 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: @@ -132,7 +132,7 @@ spec: # is an image-tag edit rather than a rebuild of ours. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay:v0.2.2 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: diff --git a/examples/k8s-deployments/pd-kvd.yaml b/examples/k8s-deployments/pd-kvd.yaml index 7f041ec..0b84cb6 100644 --- a/examples/k8s-deployments/pd-kvd.yaml +++ b/examples/k8s-deployments/pd-kvd.yaml @@ -81,7 +81,7 @@ spec: # is an image-tag edit rather than a rebuild of ours. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay:v0.2.2 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: @@ -113,7 +113,7 @@ spec: # is an image-tag edit rather than a rebuild of ours. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay:v0.2.2 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: @@ -180,7 +180,7 @@ spec: # is an image-tag edit rather than a rebuild of ours. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay:v0.2.2 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: diff --git a/examples/k8s-deployments/single-node-qwen-sglang.yaml b/examples/k8s-deployments/single-node-qwen-sglang.yaml index d73e25e..bfc4ad3 100644 --- a/examples/k8s-deployments/single-node-qwen-sglang.yaml +++ b/examples/k8s-deployments/single-node-qwen-sglang.yaml @@ -25,7 +25,7 @@ spec: # is an image-tag edit rather than a rebuild of ours. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay:v0.2.2 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: @@ -60,7 +60,7 @@ spec: # is an image-tag edit rather than a rebuild of ours. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay:v0.2.2 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: diff --git a/examples/k8s-deployments/single-node-qwen-vllm.yaml b/examples/k8s-deployments/single-node-qwen-vllm.yaml index d78bc18..dbc0811 100644 --- a/examples/k8s-deployments/single-node-qwen-vllm.yaml +++ b/examples/k8s-deployments/single-node-qwen-vllm.yaml @@ -25,7 +25,7 @@ spec: # is an image-tag edit rather than a rebuild of ours. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay:v0.2.2 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: @@ -60,7 +60,7 @@ spec: # is an image-tag edit rather than a rebuild of ours. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay:v0.2.2 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: diff --git a/examples/recipes/kimi-k3/pd-sglang/deploy.yaml b/examples/recipes/kimi-k3/pd-sglang/deploy.yaml index 15b1d11..bcdbe7a 100644 --- a/examples/recipes/kimi-k3/pd-sglang/deploy.yaml +++ b/examples/recipes/kimi-k3/pd-sglang/deploy.yaml @@ -8,7 +8,7 @@ # # base image lmsysorg/sglang-rocm:rocm720-mi35x-k3-20260727 (the SGLang build # carrying Kimi-K3 support; the general v0.5.15 tag has none) -# overlay inferaimage/infera-overlay:v0.2.1 +# overlay inferaimage/infera-overlay:v0.2.2 # # / # the model directory on EACH node. They need NOT match: every pod mounts its @@ -40,7 +40,7 @@ spec: # is an image-tag edit rather than a rebuild of ours. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay:v0.2.2 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: @@ -80,7 +80,7 @@ spec: # is an image-tag edit rather than a rebuild of ours. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay:v0.2.2 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: @@ -178,7 +178,7 @@ spec: # is an image-tag edit rather than a rebuild of ours. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay:v0.2.2 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: diff --git a/manual/examples/k8s_kimi_k3.md b/manual/examples/k8s_kimi_k3.md index 898bdb0..caa1278 100644 --- a/manual/examples/k8s_kimi_k3.md +++ b/manual/examples/k8s_kimi_k3.md @@ -24,7 +24,7 @@ upstream `kimi_k3` vLLM by overriding the base of the standard vLLM image build: ```bash docker pull vllm/vllm-openai-rocm:kimi-k3 # the vLLM carrying kimi_k3 support -docker pull inferaimage/infera-overlay:v0.2.1 # infera, kvd, router, Mooncake, hipFile +docker pull inferaimage/infera-overlay:v0.2.2 # infera, kvd, router, Mooncake, hipFile ``` Nothing is built here. The manifest runs the **stock** vLLM image and mounts infera @@ -81,7 +81,7 @@ PVC read-only. - The engine **image present in the node container runtime**. k3s uses containerd (not docker) — import a local image as a tar, not a stream: ```bash - docker save vllm/vllm-openai-rocm:kimi-k3 inferaimage/infera-overlay:v0.2.1 -o /mnt//img.tar + docker save vllm/vllm-openai-rocm:kimi-k3 inferaimage/infera-overlay:v0.2.2 -o /mnt//img.tar sudo k3s ctr images import /mnt//img.tar ``` From f0e47d8bb682bc6c599e30fb26b96fea646598e8 Mon Sep 17 00:00:00 2001 From: "Zhang, Jiejing" Date: Sun, 2 Aug 2026 03:14:39 +0000 Subject: [PATCH 8/8] build(recipes): pin every image by digest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two problems, one fix. The recipes still carried 20 references to rocm/infera-overlay:latest — the name I built under locally while writing #64, which 404s on Docker Hub. #67 fixed those on a branch cut from main; this branch predates that merge and had them back. A name that only exists on one laptop keeps coming back because a tag is a promise nobody checks. And the tags that do resolve are mutable. vllm/vllm-openai-rocm:kimi-k3 was rebuilt on 2026-07-27 during this very session, which is what invalidated the "prefix caching fails engine init" note: the tag was the same and the image was not. A recipe that says "this was validated" has to name the thing that was validated. So every image is now pinned by digest: inferaimage/infera-overlay @6918eff3 (v0.2.2, this session's fixes) vllm/vllm-openai-rocm:kimi-k3 @5aa7e626 lmsysorg/sglang-rocm:...k3-20260727 @3c01f73f lmsysorg/sglang:v0.5.15...mi35x @40e940a0 Zero mutable tags remain under examples/recipes/. Each digest is one I pulled and ran, not one read off a page. All nine manifests still pass `kubectl apply --dry-run=server` against the live CRD. The readable tag is kept alongside the digest so the ref still says what it is. Signed-off-by: Zhang, Jiejing --- examples/recipes/README.md | 2 +- examples/recipes/glm5.2/mixed-kvd/deploy.yaml | 12 ++++++------ examples/recipes/glm5.2/mixed/deploy.yaml | 10 +++++----- examples/recipes/glm5.2/pd-kvd/deploy.yaml | 18 +++++++++--------- examples/recipes/glm5.2/pd/deploy.yaml | 14 +++++++------- examples/recipes/kimi-k3/mixed-kvd/deploy.yaml | 12 ++++++------ examples/recipes/kimi-k3/mixed/deploy.yaml | 10 +++++----- examples/recipes/kimi-k3/pd-kvd/deploy.yaml | 18 +++++++++--------- examples/recipes/kimi-k3/pd-sglang/deploy.yaml | 14 +++++++------- examples/recipes/kimi-k3/pd/deploy.yaml | 14 +++++++------- 10 files changed, 62 insertions(+), 62 deletions(-) diff --git a/examples/recipes/README.md b/examples/recipes/README.md index c611284..046fe6e 100644 --- a/examples/recipes/README.md +++ b/examples/recipes/README.md @@ -42,7 +42,7 @@ forking the base for one model broke every other model. Build the overlay before deploying: ```bash -docker build -f deploy/overlay/Dockerfile.payload -t inferaimage/infera-overlay:v0.2.1 . +docker build -f deploy/overlay/Dockerfile.payload -t inferaimage/infera-overlay:v0.2.2 . ``` The build harvests **one native tree per ABI family** — `NATIVE_IMAGE` supplies diff --git a/examples/recipes/glm5.2/mixed-kvd/deploy.yaml b/examples/recipes/glm5.2/mixed-kvd/deploy.yaml index 738cb91..9c5c06d 100644 --- a/examples/recipes/glm5.2/mixed-kvd/deploy.yaml +++ b/examples/recipes/glm5.2/mixed-kvd/deploy.yaml @@ -1,7 +1,7 @@ # GLM-5.2-MXFP4 — mixed-kvd — Kubernetes recipe # # base image lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x (stock vendor image, unmodified) -# overlay inferaimage/infera-overlay:v0.2.1 (infera + router + kvd + native deps) +# overlay inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 (infera + router + kvd + native deps) # engine infera.engine.sglang TP8 # # Deploy: kubectl apply -f examples/recipes/glm5.2/mixed-kvd/deploy.yaml @@ -45,14 +45,14 @@ spec: # so following an upstream bump costs nothing in this repo. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: - {name: overlay, mountPath: /overlay} containers: - name: main - image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x + image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x@sha256:40e940a0c55b87105c773d8b484616616b3a91662bfa223c48ff721d9793dc8d imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec","python3","-m","infera.server", "--host","0.0.0.0","--port","8000", @@ -79,14 +79,14 @@ spec: # so following an upstream bump costs nothing in this repo. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: - {name: overlay, mountPath: /overlay} containers: - name: main - image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x + image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x@sha256:40e940a0c55b87105c773d8b484616616b3a91662bfa223c48ff721d9793dc8d imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec", "python3","-m","infera.engine.sglang","--host","0.0.0.0","--port","30000", @@ -136,7 +136,7 @@ spec: # infera.kvd out of the shared overlay. The overlay image itself is a # busybox carrying the payload — it has no interpreter to run kvd with. - name: kvd - image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x + image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x@sha256:40e940a0c55b87105c773d8b484616616b3a91662bfa223c48ff721d9793dc8d imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec","python3","-m","infera.kvd", "--socket","/kvd/kvd.sock", diff --git a/examples/recipes/glm5.2/mixed/deploy.yaml b/examples/recipes/glm5.2/mixed/deploy.yaml index 95a38e1..02d3aaa 100644 --- a/examples/recipes/glm5.2/mixed/deploy.yaml +++ b/examples/recipes/glm5.2/mixed/deploy.yaml @@ -1,7 +1,7 @@ # GLM-5.2-MXFP4 — mixed — Kubernetes recipe # # base image lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x (stock vendor image, unmodified) -# overlay inferaimage/infera-overlay:v0.2.1 (infera + router + kvd + native deps) +# overlay inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 (infera + router + kvd + native deps) # engine infera.engine.sglang TP8 # # Deploy: kubectl apply -f examples/recipes/glm5.2/mixed/deploy.yaml @@ -34,14 +34,14 @@ spec: # so following an upstream bump costs nothing in this repo. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: - {name: overlay, mountPath: /overlay} containers: - name: main - image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x + image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x@sha256:40e940a0c55b87105c773d8b484616616b3a91662bfa223c48ff721d9793dc8d imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec","python3","-m","infera.server", "--host","0.0.0.0","--port","8000", @@ -68,14 +68,14 @@ spec: # so following an upstream bump costs nothing in this repo. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: - {name: overlay, mountPath: /overlay} containers: - name: main - image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x + image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x@sha256:40e940a0c55b87105c773d8b484616616b3a91662bfa223c48ff721d9793dc8d imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec", "python3","-m","infera.engine.sglang","--host","0.0.0.0","--port","30000", diff --git a/examples/recipes/glm5.2/pd-kvd/deploy.yaml b/examples/recipes/glm5.2/pd-kvd/deploy.yaml index 1b46ef0..89724b6 100644 --- a/examples/recipes/glm5.2/pd-kvd/deploy.yaml +++ b/examples/recipes/glm5.2/pd-kvd/deploy.yaml @@ -1,7 +1,7 @@ # GLM-5.2-MXFP4 — pd-kvd — Kubernetes recipe # # base image lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x (stock vendor image, unmodified) -# overlay inferaimage/infera-overlay:v0.2.1 (infera + router + kvd + native deps) +# overlay inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 (infera + router + kvd + native deps) # engine infera.engine.sglang TP8 # # Deploy: kubectl apply -f examples/recipes/glm5.2/pd-kvd/deploy.yaml @@ -56,14 +56,14 @@ spec: # so following an upstream bump costs nothing in this repo. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: - {name: overlay, mountPath: /overlay} containers: - name: main - image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x + image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x@sha256:40e940a0c55b87105c773d8b484616616b3a91662bfa223c48ff721d9793dc8d imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec","python3","-m","infera.server", "--host","0.0.0.0","--port","8000", @@ -92,14 +92,14 @@ spec: # so following an upstream bump costs nothing in this repo. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: - {name: overlay, mountPath: /overlay} containers: - name: main - image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x + image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x@sha256:40e940a0c55b87105c773d8b484616616b3a91662bfa223c48ff721d9793dc8d imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec", "python3","-m","infera.engine.sglang","--host","0.0.0.0","--port","30000", @@ -155,7 +155,7 @@ spec: # infera.kvd out of the shared overlay. The overlay image itself is a # busybox carrying the payload — it has no interpreter to run kvd with. - name: kvd - image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x + image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x@sha256:40e940a0c55b87105c773d8b484616616b3a91662bfa223c48ff721d9793dc8d imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec","python3","-m","infera.kvd", "--socket","/kvd/kvd.sock", @@ -190,14 +190,14 @@ spec: # so following an upstream bump costs nothing in this repo. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: - {name: overlay, mountPath: /overlay} containers: - name: main - image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x + image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x@sha256:40e940a0c55b87105c773d8b484616616b3a91662bfa223c48ff721d9793dc8d imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec", "python3","-m","infera.engine.sglang","--host","0.0.0.0","--port","30000", @@ -253,7 +253,7 @@ spec: # infera.kvd out of the shared overlay. The overlay image itself is a # busybox carrying the payload — it has no interpreter to run kvd with. - name: kvd - image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x + image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x@sha256:40e940a0c55b87105c773d8b484616616b3a91662bfa223c48ff721d9793dc8d imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec","python3","-m","infera.kvd", "--socket","/kvd/kvd.sock", diff --git a/examples/recipes/glm5.2/pd/deploy.yaml b/examples/recipes/glm5.2/pd/deploy.yaml index df8ee37..e06b81e 100644 --- a/examples/recipes/glm5.2/pd/deploy.yaml +++ b/examples/recipes/glm5.2/pd/deploy.yaml @@ -1,7 +1,7 @@ # GLM-5.2-MXFP4 — pd — Kubernetes recipe # # base image lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x (stock vendor image, unmodified) -# overlay inferaimage/infera-overlay:v0.2.1 (infera + router + kvd + native deps) +# overlay inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 (infera + router + kvd + native deps) # engine infera.engine.sglang TP8 # # Deploy: kubectl apply -f examples/recipes/glm5.2/pd/deploy.yaml @@ -36,14 +36,14 @@ spec: # so following an upstream bump costs nothing in this repo. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: - {name: overlay, mountPath: /overlay} containers: - name: main - image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x + image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x@sha256:40e940a0c55b87105c773d8b484616616b3a91662bfa223c48ff721d9793dc8d imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec","python3","-m","infera.server", "--host","0.0.0.0","--port","8000", @@ -72,14 +72,14 @@ spec: # so following an upstream bump costs nothing in this repo. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: - {name: overlay, mountPath: /overlay} containers: - name: main - image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x + image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x@sha256:40e940a0c55b87105c773d8b484616616b3a91662bfa223c48ff721d9793dc8d imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec", "python3","-m","infera.engine.sglang","--host","0.0.0.0","--port","30000", @@ -139,14 +139,14 @@ spec: # so following an upstream bump costs nothing in this repo. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: - {name: overlay, mountPath: /overlay} containers: - name: main - image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x + image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi35x@sha256:40e940a0c55b87105c773d8b484616616b3a91662bfa223c48ff721d9793dc8d imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec", "python3","-m","infera.engine.sglang","--host","0.0.0.0","--port","30000", diff --git a/examples/recipes/kimi-k3/mixed-kvd/deploy.yaml b/examples/recipes/kimi-k3/mixed-kvd/deploy.yaml index 1100ba3..90dae33 100644 --- a/examples/recipes/kimi-k3/mixed-kvd/deploy.yaml +++ b/examples/recipes/kimi-k3/mixed-kvd/deploy.yaml @@ -1,7 +1,7 @@ # Kimi-K3 — mixed-kvd — Kubernetes recipe # # base image vllm/vllm-openai-rocm:kimi-k3 (stock vendor image, unmodified) -# overlay inferaimage/infera-overlay:v0.2.1 (infera + router + kvd + native deps) +# overlay inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 (infera + router + kvd + native deps) # engine infera.engine.vllm TP8 # # Deploy: kubectl apply -f examples/recipes/kimi-k3/mixed-kvd/deploy.yaml @@ -44,14 +44,14 @@ spec: # so following an upstream bump costs nothing in this repo. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: - {name: overlay, mountPath: /overlay} containers: - name: main - image: vllm/vllm-openai-rocm:kimi-k3 + image: vllm/vllm-openai-rocm:kimi-k3@sha256:5aa7e626ff73672f5ca7aae46754570488c23d33ca1ac90756a1d2d1a3fe099b imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec","python3","-m","infera.server", "--host","0.0.0.0","--port","8000", @@ -78,14 +78,14 @@ spec: # so following an upstream bump costs nothing in this repo. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: - {name: overlay, mountPath: /overlay} containers: - name: main - image: vllm/vllm-openai-rocm:kimi-k3 + image: vllm/vllm-openai-rocm:kimi-k3@sha256:5aa7e626ff73672f5ca7aae46754570488c23d33ca1ac90756a1d2d1a3fe099b imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec", "python3","-m","infera.engine.vllm","--host","0.0.0.0","--port","30000", @@ -129,7 +129,7 @@ spec: # infera.kvd out of the shared overlay. The overlay image itself is a # busybox carrying the payload — it has no interpreter to run kvd with. - name: kvd - image: vllm/vllm-openai-rocm:kimi-k3 + image: vllm/vllm-openai-rocm:kimi-k3@sha256:5aa7e626ff73672f5ca7aae46754570488c23d33ca1ac90756a1d2d1a3fe099b imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec","python3","-m","infera.kvd", "--socket","/kvd/kvd.sock", diff --git a/examples/recipes/kimi-k3/mixed/deploy.yaml b/examples/recipes/kimi-k3/mixed/deploy.yaml index 627468a..9450635 100644 --- a/examples/recipes/kimi-k3/mixed/deploy.yaml +++ b/examples/recipes/kimi-k3/mixed/deploy.yaml @@ -1,7 +1,7 @@ # Kimi-K3 — mixed — Kubernetes recipe # # base image vllm/vllm-openai-rocm:kimi-k3 (stock vendor image, unmodified) -# overlay inferaimage/infera-overlay:v0.2.1 (infera + router + kvd + native deps) +# overlay inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 (infera + router + kvd + native deps) # engine infera.engine.vllm TP8 # # Deploy: kubectl apply -f examples/recipes/kimi-k3/mixed/deploy.yaml @@ -42,14 +42,14 @@ spec: # so following an upstream bump costs nothing in this repo. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: - {name: overlay, mountPath: /overlay} containers: - name: main - image: vllm/vllm-openai-rocm:kimi-k3 + image: vllm/vllm-openai-rocm:kimi-k3@sha256:5aa7e626ff73672f5ca7aae46754570488c23d33ca1ac90756a1d2d1a3fe099b imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec","python3","-m","infera.server", "--host","0.0.0.0","--port","8000", @@ -76,14 +76,14 @@ spec: # so following an upstream bump costs nothing in this repo. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: - {name: overlay, mountPath: /overlay} containers: - name: main - image: vllm/vllm-openai-rocm:kimi-k3 + image: vllm/vllm-openai-rocm:kimi-k3@sha256:5aa7e626ff73672f5ca7aae46754570488c23d33ca1ac90756a1d2d1a3fe099b imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec", "python3","-m","infera.engine.vllm","--host","0.0.0.0","--port","30000", diff --git a/examples/recipes/kimi-k3/pd-kvd/deploy.yaml b/examples/recipes/kimi-k3/pd-kvd/deploy.yaml index e85e0c0..32c784c 100644 --- a/examples/recipes/kimi-k3/pd-kvd/deploy.yaml +++ b/examples/recipes/kimi-k3/pd-kvd/deploy.yaml @@ -1,7 +1,7 @@ # Kimi-K3 — pd-kvd — Kubernetes recipe # # base image vllm/vllm-openai-rocm:kimi-k3 (stock vendor image, unmodified) -# overlay inferaimage/infera-overlay:v0.2.1 (infera + router + kvd + native deps) +# overlay inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 (infera + router + kvd + native deps) # engine infera.engine.vllm TP8 # # Deploy: kubectl apply -f examples/recipes/kimi-k3/pd-kvd/deploy.yaml @@ -56,14 +56,14 @@ spec: # so following an upstream bump costs nothing in this repo. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: - {name: overlay, mountPath: /overlay} containers: - name: main - image: vllm/vllm-openai-rocm:kimi-k3 + image: vllm/vllm-openai-rocm:kimi-k3@sha256:5aa7e626ff73672f5ca7aae46754570488c23d33ca1ac90756a1d2d1a3fe099b imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec","python3","-m","infera.server", "--host","0.0.0.0","--port","8000", @@ -92,14 +92,14 @@ spec: # so following an upstream bump costs nothing in this repo. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: - {name: overlay, mountPath: /overlay} containers: - name: main - image: vllm/vllm-openai-rocm:kimi-k3 + image: vllm/vllm-openai-rocm:kimi-k3@sha256:5aa7e626ff73672f5ca7aae46754570488c23d33ca1ac90756a1d2d1a3fe099b imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec", "python3","-m","infera.engine.vllm","--host","0.0.0.0","--port","30000", @@ -143,7 +143,7 @@ spec: # infera.kvd out of the shared overlay. The overlay image itself is a # busybox carrying the payload — it has no interpreter to run kvd with. - name: kvd - image: vllm/vllm-openai-rocm:kimi-k3 + image: vllm/vllm-openai-rocm:kimi-k3@sha256:5aa7e626ff73672f5ca7aae46754570488c23d33ca1ac90756a1d2d1a3fe099b imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec","python3","-m","infera.kvd", "--socket","/kvd/kvd.sock", @@ -180,14 +180,14 @@ spec: # so following an upstream bump costs nothing in this repo. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: - {name: overlay, mountPath: /overlay} containers: - name: main - image: vllm/vllm-openai-rocm:kimi-k3 + image: vllm/vllm-openai-rocm:kimi-k3@sha256:5aa7e626ff73672f5ca7aae46754570488c23d33ca1ac90756a1d2d1a3fe099b imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec", "python3","-m","infera.engine.vllm","--host","0.0.0.0","--port","30000", @@ -231,7 +231,7 @@ spec: # infera.kvd out of the shared overlay. The overlay image itself is a # busybox carrying the payload — it has no interpreter to run kvd with. - name: kvd - image: vllm/vllm-openai-rocm:kimi-k3 + image: vllm/vllm-openai-rocm:kimi-k3@sha256:5aa7e626ff73672f5ca7aae46754570488c23d33ca1ac90756a1d2d1a3fe099b imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec","python3","-m","infera.kvd", "--socket","/kvd/kvd.sock", diff --git a/examples/recipes/kimi-k3/pd-sglang/deploy.yaml b/examples/recipes/kimi-k3/pd-sglang/deploy.yaml index bcdbe7a..59ec20f 100644 --- a/examples/recipes/kimi-k3/pd-sglang/deploy.yaml +++ b/examples/recipes/kimi-k3/pd-sglang/deploy.yaml @@ -8,7 +8,7 @@ # # base image lmsysorg/sglang-rocm:rocm720-mi35x-k3-20260727 (the SGLang build # carrying Kimi-K3 support; the general v0.5.15 tag has none) -# overlay inferaimage/infera-overlay:v0.2.2 +# overlay inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 # # / # the model directory on EACH node. They need NOT match: every pod mounts its @@ -40,14 +40,14 @@ spec: # is an image-tag edit rather than a rebuild of ours. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.2 + image: inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: - {name: overlay, mountPath: /overlay} containers: - name: main - image: lmsysorg/sglang-rocm:rocm720-mi35x-k3-20260727 # libionic (ABI 4) baked + image: lmsysorg/sglang-rocm:rocm720-mi35x-k3-20260727@sha256:3c01f73fe23aebf4a8853de0899a70b75c2af6c0409d2331353847aac4d3f906 # libionic (ABI 4) baked imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec","python3","-m","infera.server","--host","0.0.0.0","--port","8000", "--router-tokenizer-path","/models/Kimi-K3", @@ -80,14 +80,14 @@ spec: # is an image-tag edit rather than a rebuild of ours. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.2 + image: inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: - {name: overlay, mountPath: /overlay} containers: - name: main - image: lmsysorg/sglang-rocm:rocm720-mi35x-k3-20260727 + image: lmsysorg/sglang-rocm:rocm720-mi35x-k3-20260727@sha256:3c01f73fe23aebf4a8853de0899a70b75c2af6c0409d2331353847aac4d3f906 imagePullPolicy: IfNotPresent securityContext: {privileged: true, capabilities: {add: ["IPC_LOCK","SYS_PTRACE"]}} command: ["/overlay/bin/infera-exec","python3","-m","infera.engine.sglang","--host","0.0.0.0","--port","30000", @@ -178,14 +178,14 @@ spec: # is an image-tag edit rather than a rebuild of ours. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.2 + image: inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: - {name: overlay, mountPath: /overlay} containers: - name: main - image: lmsysorg/sglang-rocm:rocm720-mi35x-k3-20260727 + image: lmsysorg/sglang-rocm:rocm720-mi35x-k3-20260727@sha256:3c01f73fe23aebf4a8853de0899a70b75c2af6c0409d2331353847aac4d3f906 imagePullPolicy: IfNotPresent securityContext: {privileged: true, capabilities: {add: ["IPC_LOCK","SYS_PTRACE"]}} command: ["/overlay/bin/infera-exec","python3","-m","infera.engine.sglang","--host","0.0.0.0","--port","30000", diff --git a/examples/recipes/kimi-k3/pd/deploy.yaml b/examples/recipes/kimi-k3/pd/deploy.yaml index ffcccce..7300266 100644 --- a/examples/recipes/kimi-k3/pd/deploy.yaml +++ b/examples/recipes/kimi-k3/pd/deploy.yaml @@ -1,7 +1,7 @@ # Kimi-K3 — pd — Kubernetes recipe # # base image vllm/vllm-openai-rocm:kimi-k3 (stock vendor image, unmodified) -# overlay inferaimage/infera-overlay:v0.2.1 (infera + router + kvd + native deps) +# overlay inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 (infera + router + kvd + native deps) # engine infera.engine.vllm TP8 # # Deploy: kubectl apply -f examples/recipes/kimi-k3/pd/deploy.yaml @@ -36,14 +36,14 @@ spec: # so following an upstream bump costs nothing in this repo. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: - {name: overlay, mountPath: /overlay} containers: - name: main - image: vllm/vllm-openai-rocm:kimi-k3 + image: vllm/vllm-openai-rocm:kimi-k3@sha256:5aa7e626ff73672f5ca7aae46754570488c23d33ca1ac90756a1d2d1a3fe099b imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec","python3","-m","infera.server", "--host","0.0.0.0","--port","8000", @@ -77,14 +77,14 @@ spec: # so following an upstream bump costs nothing in this repo. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: - {name: overlay, mountPath: /overlay} containers: - name: main - image: vllm/vllm-openai-rocm:kimi-k3 + image: vllm/vllm-openai-rocm:kimi-k3@sha256:5aa7e626ff73672f5ca7aae46754570488c23d33ca1ac90756a1d2d1a3fe099b imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec", "python3","-m","infera.engine.vllm","--host","0.0.0.0","--port","30000", @@ -166,14 +166,14 @@ spec: # so following an upstream bump costs nothing in this repo. initContainers: - name: infera-overlay - image: inferaimage/infera-overlay:v0.2.1 + image: inferaimage/infera-overlay@sha256:6918eff34f201548a738dd592d2a1ece0627354d2e88f24a87cfa8f787a72a44 imagePullPolicy: IfNotPresent command: ["sh","-c","cp -a /payload/. /overlay/"] volumeMounts: - {name: overlay, mountPath: /overlay} containers: - name: main - image: vllm/vllm-openai-rocm:kimi-k3 + image: vllm/vllm-openai-rocm:kimi-k3@sha256:5aa7e626ff73672f5ca7aae46754570488c23d33ca1ac90756a1d2d1a3fe099b imagePullPolicy: IfNotPresent command: ["/overlay/bin/infera-exec", "python3","-m","infera.engine.vllm","--host","0.0.0.0","--port","30000",