integration.yaml

name: Integration (Mac M4)

# Self-hosted runner workflow that runs the integration suite under
# tests/integration/ against real Qwen3-0.6B on Apple Silicon.
#
# Trigger model:
#   - Pull-request events touching runtime/model/proto/integration paths.
#     Path filtering is used directly instead of depending on an auto-label:
#     workflows triggered with GITHUB_TOKEN do not recursively trigger the
#     ``labeled`` event, which previously made the first Mac gate skip.
#   - Manual workflow_dispatch for re-runs from the Actions UI.
#
# Runner requirements (self-hosted):
#   - macOS 14+ on Apple Silicon (M-series).
#   - Labels: [self-hosted, macOS, ARM64, kakeya-mac-m4].
#   - Pre-warmed HF cache containing Qwen/Qwen3-0.6B at
#     ~/.cache/huggingface/hub/ (avoids 10-minute first-run download).
#   - Python 3.12+ on PATH.
#   - At least 24 GB unified memory and ~50 GB free disk.
#
# See docs/ops/mac-m4-runner-setup.md for the one-time runner setup.

on:
  pull_request:
    # Only run on PR events for branches targeting main.
    types: [opened, synchronize, reopened, labeled]
    branches: [main]
    paths:
      - "inference_engine/**"
      - "kv_cache_proposer/**"
      - "proto/**"
      - "sdks/**"
      - "tests/integration/**"
      - "tests/backends/mlx/**"
      - ".github/workflows/integration.yaml"
  workflow_dispatch: {}

# Cancel superseded runs on the same PR — saves runner time when
# the contributor pushes a new commit before the previous run
# finishes.
concurrency:
  group: integration-${{ github.ref }}
  cancel-in-progress: true

jobs:
  integration:
    name: pytest -m integration on Mac M4
    runs-on: [self-hosted, macOS, ARM64, kakeya-mac-m4]
    timeout-minutes: 90
    steps:
      - name: Bootstrap git-lfs before checkout
        # Reused worktrees can retain a git-lfs post-checkout hook. The hook
        # executes inside checkout@v4, so repair PATH before checkout.
        run: |
          set -euo pipefail
          if [ -x /opt/homebrew/bin/git-lfs ]; then
            echo "/opt/homebrew/bin" >> "$GITHUB_PATH"
            exit 0
          fi
          if [ -x /usr/local/bin/git-lfs ]; then
            echo "/usr/local/bin" >> "$GITHUB_PATH"
            exit 0
          fi
          brew_bin=""
          for candidate in /opt/homebrew/bin/brew /usr/local/bin/brew; do
            if [ -x "$candidate" ]; then brew_bin="$candidate"; break; fi
          done
          if [ -z "$brew_bin" ]; then
            echo "::error::git-lfs is missing and Homebrew is unavailable."
            exit 1
          fi
          "$brew_bin" install git-lfs
          echo "$(dirname "$brew_bin")" >> "$GITHUB_PATH"

      - uses: actions/checkout@v4
        with:
          # Full history so the runner can compare against base for
          # any future rebase-based gating.
          fetch-depth: 0

      - name: Verify host shape
        run: |
          echo "=== sysctl ==="
          sysctl -n hw.model || true
          sysctl -n hw.memsize || true
          sysctl -n machdep.cpu.brand_string || true
          echo "=== python ==="
          python3 --version
          python3 -c "import platform; print(platform.machine(), platform.platform())"

      - name: Verify Qwen3-0.6B in HF cache
        run: |
          # Don't download here; the runner is expected to be
          # pre-warmed. If the model isn't cached the test loads
          # would hit HF and exceed the 90-min timeout. Surface a
          # clear error early.
          set -e
          MODEL_DIR="$HOME/.cache/huggingface/hub/models--Qwen--Qwen3-0.6B"
          if [ ! -d "$MODEL_DIR" ]; then
            echo "::error::HF cache miss for Qwen/Qwen3-0.6B."
            echo "::error::Pre-warm the runner: python3 -c 'from transformers import AutoModelForCausalLM, AutoTokenizer; AutoModelForCausalLM.from_pretrained(\"Qwen/Qwen3-0.6B\"); AutoTokenizer.from_pretrained(\"Qwen/Qwen3-0.6B\")'"
            exit 1
          fi
          echo "Found $MODEL_DIR"

      - name: Gate real MLX distributed-prefill continuation equivalence
        env:
          KAKEYA_MAC_VERIFIER_PATH_VAR: ${{ vars.KAKEYA_MAC_VERIFIER_PATH || '' }}
        run: |
          set -euo pipefail
          default_verifier="$HOME/kakeya-models/gemma-4-26B-A4B-it-mlx-4bit"
          if [ ! -d "$default_verifier" ]; then
            default_verifier="models/gemma-4-26B-A4B-it-mlx-4bit"
          fi
          export KAKEYA_MAC_VERIFIER_PATH="${KAKEYA_MAC_VERIFIER_PATH_VAR:-$default_verifier}"
          if [ ! -d "$KAKEYA_MAC_VERIFIER_PATH" ]; then
            echo "::error::KAKEYA_MAC_VERIFIER_PATH must point to the pre-warmed MLX verifier."
            exit 1
          fi
          PYBIN="$(
            python3 - <<'PY'
          import os, shutil, subprocess
          candidates = [
              os.environ.get("KAKEYA_MAC_PYTHON"),
              os.path.expanduser("~/kakeya-venv/bin/python"),
              os.path.expanduser("~/.venv/bin/python"),
              os.path.expanduser(
                  "~/Documents/Kakeya-LLM-Inference-engine-pr109/"
                  ".venv-mac/bin/python3.13"
              ),
              os.path.expanduser(
                  "~/Documents/Kakeya-LLM-Inference-engine-pr109/"
                  ".venv-mac/bin/python"
              ),
              shutil.which("python3.13"),
              shutil.which("python3"),
          ]
          for candidate in candidates:
              if not candidate or not os.path.isfile(candidate):
                  continue
              if subprocess.run(
                  [candidate, "-c", "import mlx_lm, torch, pytest"],
                  stdout=subprocess.DEVNULL,
                  stderr=subprocess.DEVNULL,
              ).returncode == 0:
                  print(candidate)
                  break
          PY
          )"
          test -n "$PYBIN"
          PYTHONPATH=.:sdks/python "$PYBIN" -m pytest \
            -m integration \
            tests/integration/test_prefill_snapshot_mlx_equivalence.py \
            -q

      - name: Install Python dependencies
        run: |
          # Keep the legacy Qwen/Transformers-4 suite isolated from both
          # Homebrew's PEP-668-managed system Python and the Transformers-5 MLX
          # production venv used by the preceding real-model gate.
          LEGACY_VENV="$HOME/.kakeya/integration-legacy-venv"
          if [ ! -x "$LEGACY_VENV/bin/python" ]; then
            mkdir -p "$(dirname "$LEGACY_VENV")"
            python3 -m venv "$LEGACY_VENV"
          fi
          LEGACY_PY="$LEGACY_VENV/bin/python"
          "$LEGACY_PY" -m pip install --upgrade pip
          # The repo runs via PYTHONPATH (see ci.yaml) — it is NOT a pip package
          # (no setup.py/pyproject.toml), so install runtime deps from
          # requirements.txt rather than an editable `-e .` (which errors with
          # "does not appear to be a Python project").
          "$LEGACY_PY" -m pip install -r requirements.txt
          # The integration suite exercises the legacy dllm-hub Qwen proposer,
          # whose remote modeling file depends on the Transformers 4.x
          # decoder_layer.attention_type API. Keep this runner in the dedicated
          # legacy range; K3/Gemma production paths use requirements.txt's
          # unbounded Transformers 5.x-compatible environment.
          "$LEGACY_PY" -m pip install 'transformers>=4.45,<5.0'
          "$LEGACY_PY" -m pip install pytest pytest-asyncio pytest-timeout coverage
          echo "KAKEYA_INTEGRATION_PY=$LEGACY_PY" >> "$GITHUB_ENV"

      - name: Run integration suite
        env:
          PYTHONPATH: .:sdks/python
          # No HF download in tests; if we hit a cache miss it's a
          # bug or a stale runner.
          HF_HUB_OFFLINE: "1"
        run: |
          mkdir -p results/platform-tests
          stamp=$(date +%s)
          "$KAKEYA_INTEGRATION_PY" -m pytest \
            -m integration \
            tests/integration/ \
            --junitxml="results/platform-tests/integration-mac-m4-${stamp}.junit.xml" \
            -v
          # Record the artifact path for the upload step below.
          echo "artifact_stamp=${stamp}" >> "$GITHUB_OUTPUT"
        id: pytest_run

      - name: Upload JUnit + log artifacts
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: integration-mac-m4-${{ steps.pytest_run.outputs.artifact_stamp || github.run_id }}
          path: |
            results/platform-tests/integration-mac-m4-*.junit.xml
          retention-days: 30

      - name: Surface failure summary
        if: failure()
        run: |
          # Tail the last few lines of the JUnit so the failure is
          # visible in the action log, not just inside the artifact.
          for f in results/platform-tests/integration-mac-m4-*.junit.xml; do
            echo "=== $f ==="
            python3 - "$f" <<'PY'
          import sys, xml.etree.ElementTree as ET
          r = ET.parse(sys.argv[1]).getroot()
          for tc in r.iter("testcase"):
              for child in tc:
                  if child.tag in ("failure", "error"):
                      print(f"[{child.tag.upper()}] {tc.get('classname')}::{tc.get('name')}")
                      msg = (child.get("message") or "").splitlines()
                      if msg:
                          print(f"    {msg[0][:180]}")
          PY
          done