Harden Primary decode lifecycle and isolate MLX worker #226

Workflow file for this run

.github/workflows/integration.yaml at c64bd8e

	name: Integration (Mac M4)

	# Self-hosted runner workflow that runs the integration suite under
	# tests/integration/ against real Qwen3-0.6B on Apple Silicon.
	#
	# Trigger model:
	# - Pull-request events touching runtime/model/proto/integration paths.
	# Path filtering is used directly instead of depending on an auto-label:
	# workflows triggered with GITHUB_TOKEN do not recursively trigger the
	# ``labeled`` event, which previously made the first Mac gate skip.
	# - Manual workflow_dispatch for re-runs from the Actions UI.
	#
	# Runner requirements (self-hosted):
	# - macOS 14+ on Apple Silicon (M-series).
	# - Labels: [self-hosted, macOS, ARM64, kakeya-mac-m4].
	# - Pre-warmed HF cache containing Qwen/Qwen3-0.6B at
	# ~/.cache/huggingface/hub/ (avoids 10-minute first-run download).
	# - Python 3.12+ on PATH.
	# - At least 24 GB unified memory and ~50 GB free disk.
	#
	# See docs/ops/mac-m4-runner-setup.md for the one-time runner setup.

	on:
	pull_request:
	# Only run on PR events for branches targeting main.
	types: [opened, synchronize, reopened, labeled]
	branches: [main]
	paths:
	- "inference_engine/**"
	- "kv_cache_proposer/**"
	- "proto/**"
	- "sdks/**"
	- "tests/integration/**"
	- "tests/backends/mlx/**"
	- ".github/workflows/integration.yaml"
	workflow_dispatch: {}

	# Cancel superseded runs on the same PR — saves runner time when
	# the contributor pushes a new commit before the previous run
	# finishes.
	concurrency:
	group: integration-${{ github.ref }}
	cancel-in-progress: true

	jobs:
	integration:
	name: pytest -m integration on Mac M4
	runs-on: [self-hosted, macOS, ARM64, kakeya-mac-m4]
	timeout-minutes: 90
	steps:
	- name: Bootstrap git-lfs before checkout
	# Reused worktrees can retain a git-lfs post-checkout hook. The hook
	# executes inside checkout@v4, so repair PATH before checkout.
	run: \|
	set -euo pipefail
	if [ -x /opt/homebrew/bin/git-lfs ]; then
	echo "/opt/homebrew/bin" >> "$GITHUB_PATH"
	exit 0
	fi
	if [ -x /usr/local/bin/git-lfs ]; then
	echo "/usr/local/bin" >> "$GITHUB_PATH"
	exit 0
	fi
	brew_bin=""
	for candidate in /opt/homebrew/bin/brew /usr/local/bin/brew; do
	if [ -x "$candidate" ]; then brew_bin="$candidate"; break; fi
	done
	if [ -z "$brew_bin" ]; then
	echo "::error::git-lfs is missing and Homebrew is unavailable."
	exit 1
	fi
	"$brew_bin" install git-lfs
	echo "$(dirname "$brew_bin")" >> "$GITHUB_PATH"

	- uses: actions/checkout@v4
	with:
	# Full history so the runner can compare against base for
	# any future rebase-based gating.
	fetch-depth: 0

	- name: Verify host shape
	run: \|
	echo "=== sysctl ==="
	sysctl -n hw.model \|\| true
	sysctl -n hw.memsize \|\| true
	sysctl -n machdep.cpu.brand_string \|\| true
	echo "=== python ==="
	python3 --version
	python3 -c "import platform; print(platform.machine(), platform.platform())"

	- name: Verify Qwen3-0.6B in HF cache
	run: \|
	# Don't download here; the runner is expected to be
	# pre-warmed. If the model isn't cached the test loads
	# would hit HF and exceed the 90-min timeout. Surface a
	# clear error early.
	set -e
	MODEL_DIR="$HOME/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B"
	if [ ! -d "$MODEL_DIR" ]; then
	echo "::error::HF cache miss for Qwen/Qwen3-0.6B."
	echo "::error::Pre-warm the runner: python3 -c 'from transformers import AutoModelForCausalLM, AutoTokenizer; AutoModelForCausalLM.from_pretrained(\"Qwen/Qwen3-0.6B\"); AutoTokenizer.from_pretrained(\"Qwen/Qwen3-0.6B\")'"
	exit 1
	fi
	echo "Found $MODEL_DIR"

	- name: Gate real MLX distributed-prefill continuation equivalence
	env:
	KAKEYA_MAC_VERIFIER_PATH_VAR: ${{ vars.KAKEYA_MAC_VERIFIER_PATH \|\| '' }}
	run: \|
	set -euo pipefail
	default_verifier="$HOME/kakeya-models/gemma-4-26B-A4B-it-mlx-4bit"
	if [ ! -d "$default_verifier" ]; then
	default_verifier="models/gemma-4-26B-A4B-it-mlx-4bit"
	fi
	export KAKEYA_MAC_VERIFIER_PATH="${KAKEYA_MAC_VERIFIER_PATH_VAR:-$default_verifier}"
	if [ ! -d "$KAKEYA_MAC_VERIFIER_PATH" ]; then
	echo "::error::KAKEYA_MAC_VERIFIER_PATH must point to the pre-warmed MLX verifier."
	exit 1
	fi
	PYBIN="$(
	python3 - <<'PY'
	import os, shutil, subprocess
	candidates = [
	os.environ.get("KAKEYA_MAC_PYTHON"),
	os.path.expanduser("~/kakeya-venv/bin/python"),
	os.path.expanduser("~/.venv/bin/python"),
	os.path.expanduser(
	"~/Documents/Kakeya-LLM-Inference-engine-pr109/"
	".venv-mac/bin/python3.13"
	),
	os.path.expanduser(
	"~/Documents/Kakeya-LLM-Inference-engine-pr109/"
	".venv-mac/bin/python"
	),
	shutil.which("python3.13"),
	shutil.which("python3"),
	]
	for candidate in candidates:
	if not candidate or not os.path.isfile(candidate):
	continue
	if subprocess.run(
	[candidate, "-c", "import mlx_lm, torch, pytest"],
	stdout=subprocess.DEVNULL,
	stderr=subprocess.DEVNULL,
	).returncode == 0:
	print(candidate)
	break
	PY
	)"
	test -n "$PYBIN"
	PYTHONPATH=.:sdks/python "$PYBIN" -m pytest \
	-m integration \
	tests/integration/test_prefill_snapshot_mlx_equivalence.py \
	-q

	- name: Install Python dependencies
	run: \|
	# Keep the legacy Qwen/Transformers-4 suite isolated from both
	# Homebrew's PEP-668-managed system Python and the Transformers-5 MLX
	# production venv used by the preceding real-model gate.
	LEGACY_VENV="$HOME/.kakeya/integration-legacy-venv"
	if [ ! -x "$LEGACY_VENV/bin/python" ]; then
	mkdir -p "$(dirname "$LEGACY_VENV")"
	python3 -m venv "$LEGACY_VENV"
	fi
	LEGACY_PY="$LEGACY_VENV/bin/python"
	"$LEGACY_PY" -m pip install --upgrade pip
	# The repo runs via PYTHONPATH (see ci.yaml) — it is NOT a pip package
	# (no setup.py/pyproject.toml), so install runtime deps from
	# requirements.txt rather than an editable `-e .` (which errors with
	# "does not appear to be a Python project").
	"$LEGACY_PY" -m pip install -r requirements.txt
	# The integration suite exercises the legacy dllm-hub Qwen proposer,
	# whose remote modeling file depends on the Transformers 4.x
	# decoder_layer.attention_type API. Keep this runner in the dedicated
	# legacy range; K3/Gemma production paths use requirements.txt's
	# unbounded Transformers 5.x-compatible environment.
	"$LEGACY_PY" -m pip install 'transformers>=4.45,<5.0'
	"$LEGACY_PY" -m pip install pytest pytest-asyncio pytest-timeout coverage
	echo "KAKEYA_INTEGRATION_PY=$LEGACY_PY" >> "$GITHUB_ENV"

	- name: Run integration suite
	env:
	PYTHONPATH: .:sdks/python
	# No HF download in tests; if we hit a cache miss it's a
	# bug or a stale runner.
	HF_HUB_OFFLINE: "1"
	run: \|
	mkdir -p results/platform-tests
	stamp=$(date +%s)
	"$KAKEYA_INTEGRATION_PY" -m pytest \
	-m integration \
	tests/integration/ \
	--junitxml="results/platform-tests/integration-mac-m4-${stamp}.junit.xml" \
	-v
	# Record the artifact path for the upload step below.
	echo "artifact_stamp=${stamp}" >> "$GITHUB_OUTPUT"
	id: pytest_run

	- name: Upload JUnit + log artifacts
	if: always()
	uses: actions/upload-artifact@v4
	with:
	name: integration-mac-m4-${{ steps.pytest_run.outputs.artifact_stamp \|\| github.run_id }}
	path: \|
	results/platform-tests/integration-mac-m4-*.junit.xml
	retention-days: 30

	- name: Surface failure summary
	if: failure()
	run: \|
	# Tail the last few lines of the JUnit so the failure is
	# visible in the action log, not just inside the artifact.
	for f in results/platform-tests/integration-mac-m4-*.junit.xml; do
	echo "=== $f ==="
	python3 - "$f" <<'PY'
	import sys, xml.etree.ElementTree as ET
	r = ET.parse(sys.argv[1]).getroot()
	for tc in r.iter("testcase"):
	for child in tc:
	if child.tag in ("failure", "error"):
	print(f"[{child.tag.upper()}] {tc.get('classname')}::{tc.get('name')}")
	msg = (child.get("message") or "").splitlines()
	if msg:
	print(f" {msg[0][:180]}")
	PY
	done

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Harden Primary decode lifecycle and isolate MLX worker #226

Workflow file

Harden Primary decode lifecycle and isolate MLX worker #226

Uh oh!

Workflow file for this run