diff --git a/deploy/knative/swebench-sandbox-pool.yaml b/deploy/knative/swebench-sandbox-pool.yaml new file mode 100644 index 0000000..2bf56ac --- /dev/null +++ b/deploy/knative/swebench-sandbox-pool.yaml @@ -0,0 +1,168 @@ +# deploy/knative/swebench-sandbox-pool.yaml +# Dedicated SWE-bench sandbox pool (Plan B, Task 4), separate from the default pool +# (deploy/knative/sandbox-pool.yaml). Runs the Task-3 baked OCP x86_64 sandbox image +# (15 conda envs + 8 bare repo mirrors baked into /opt/miniconda3 and /repos in the +# image layers). That image is x86_64-only and lives ONLY in the OCP internal +# registry (no Kind / dev.local build), so — unlike the default pool — this manifest +# is OCP-native directly: the OCP nonroot securityContext + serviceAccountName are +# baked straight into the pod spec instead of a separate overlays/ocp/*.yaml patch. +# N=3 is the PoC size, matching the default pool. +# Each CR produces one pod labeled sh.kagenti.io/sandbox-pool=swebench (distinct from +# the default pool's `default` label so the two pools never collide); Plan C's +# WORKLOAD=swebench provider discovers this pool's Running pods via +# KAGENTI_SANDBOX_POOL_SELECTOR=sh.kagenti.io/sandbox-pool=swebench. +# 50Gi PVC per pod: the shared pod hosts many concurrent large-repo worktrees over the +# pool's lifetime, and while the baked /repos bare mirrors + /opt/miniconda3 envs live +# in the image layers (not the PVC), each leaf's git worktree checkout + build/test +# artifacts under /workspace still need headroom across a busy pod. +apiVersion: agents.x-k8s.io/v1beta1 +kind: Sandbox +metadata: + name: swebench-sandbox-0 + namespace: default + labels: + app: sandbox +spec: + volumeClaimTemplates: + - metadata: + name: workspace + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 50Gi + podTemplate: + metadata: + labels: + sh.kagenti.io/sandbox-pool: swebench # pool discovery label (harness selects on this) + spec: + serviceAccountName: serverless-harness-sandbox + securityContext: + runAsUser: 65532 + runAsNonRoot: true + fsGroup: 65532 + seccompProfile: + type: RuntimeDefault + containers: + - name: sandbox + image: image-registry.openshift-image-registry.svc:5000/default/swebench-sandbox:ff962cb83fe5c624-15of15 + imagePullPolicy: IfNotPresent + command: ["sleep", "infinity"] + workingDir: /workspace + volumeMounts: + - name: workspace + mountPath: /workspace + # Generous-for-pytest PoC sizing: no cpu limit so long SWE-bench test suites are not + # CPU-throttled, matching the repo's emulated-workload pattern; adjust if needed. + resources: + requests: + memory: "512Mi" + cpu: "250m" + limits: + memory: "4Gi" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + +--- +apiVersion: agents.x-k8s.io/v1beta1 +kind: Sandbox +metadata: + name: swebench-sandbox-1 + namespace: default + labels: + app: sandbox +spec: + volumeClaimTemplates: + - metadata: + name: workspace + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 50Gi + podTemplate: + metadata: + labels: + sh.kagenti.io/sandbox-pool: swebench # pool discovery label (harness selects on this) + spec: + serviceAccountName: serverless-harness-sandbox + securityContext: + runAsUser: 65532 + runAsNonRoot: true + fsGroup: 65532 + seccompProfile: + type: RuntimeDefault + containers: + - name: sandbox + image: image-registry.openshift-image-registry.svc:5000/default/swebench-sandbox:ff962cb83fe5c624-15of15 + imagePullPolicy: IfNotPresent + command: ["sleep", "infinity"] + workingDir: /workspace + volumeMounts: + - name: workspace + mountPath: /workspace + # Generous-for-pytest PoC sizing: no cpu limit so long SWE-bench test suites are not + # CPU-throttled, matching the repo's emulated-workload pattern; adjust if needed. + resources: + requests: + memory: "512Mi" + cpu: "250m" + limits: + memory: "4Gi" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + +--- +apiVersion: agents.x-k8s.io/v1beta1 +kind: Sandbox +metadata: + name: swebench-sandbox-2 + namespace: default + labels: + app: sandbox +spec: + volumeClaimTemplates: + - metadata: + name: workspace + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 50Gi + podTemplate: + metadata: + labels: + sh.kagenti.io/sandbox-pool: swebench # pool discovery label (harness selects on this) + spec: + serviceAccountName: serverless-harness-sandbox + securityContext: + runAsUser: 65532 + runAsNonRoot: true + fsGroup: 65532 + seccompProfile: + type: RuntimeDefault + containers: + - name: sandbox + image: image-registry.openshift-image-registry.svc:5000/default/swebench-sandbox:ff962cb83fe5c624-15of15 + imagePullPolicy: IfNotPresent + command: ["sleep", "infinity"] + workingDir: /workspace + volumeMounts: + - name: workspace + mountPath: /workspace + # Generous-for-pytest PoC sizing: no cpu limit so long SWE-bench test suites are not + # CPU-throttled, matching the repo's emulated-workload pattern; adjust if needed. + resources: + requests: + memory: "512Mi" + cpu: "250m" + limits: + memory: "4Gi" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] diff --git a/packages/knative-server/test/swebench-sandbox-pool.test.ts b/packages/knative-server/test/swebench-sandbox-pool.test.ts new file mode 100644 index 0000000..d2bab80 --- /dev/null +++ b/packages/knative-server/test/swebench-sandbox-pool.test.ts @@ -0,0 +1,102 @@ +// Manifest-shape tests for the dedicated `swebench` sandbox pool (Plan B / Task 4a). This pool is +// OCP-only: it runs the Task-3 baked x86_64 image (15 conda envs + 8 bare repo mirrors) from the +// OCP internal registry, separate from the default pool, so Plan C's `WORKLOAD=swebench` provider +// can select it via `KAGENTI_SANDBOX_POOL_SELECTOR=sh.kagenti.io/sandbox-pool=swebench`. This is +// pure file parsing: no cluster, no kustomize binary, runs in the existing `pnpm -r test` CI. +import { describe, it, expect } from 'vitest'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { dirname, resolve } from 'node:path'; +import { parseAllDocuments } from 'yaml'; + +const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); +const DEPLOY = resolve(REPO_ROOT, 'deploy/knative'); + +/** Parse every YAML document in a (possibly multi-doc) manifest file into plain JS objects. */ +function readDocs(path: string): any[] { + return parseAllDocuments(readFileSync(path, 'utf8')).map((d) => d.toJS()); +} + +const SWEBENCH_IMAGE = + 'image-registry.openshift-image-registry.svc:5000/default/swebench-sandbox:ff962cb83fe5c624-15of15'; + +describe('swebench-sandbox-pool manifest', () => { + const SWEBENCH_POOL_PATH = resolve(DEPLOY, 'swebench-sandbox-pool.yaml'); + const docs = readDocs(SWEBENCH_POOL_PATH); + const sandboxes = docs.filter((d) => d?.kind === 'Sandbox'); + + it('defines exactly 3 Sandbox CRs named swebench-sandbox-0/1/2, all in namespace default', () => { + expect(sandboxes).toHaveLength(3); + expect(sandboxes.map((s) => s.apiVersion)).toEqual([ + 'agents.x-k8s.io/v1beta1', + 'agents.x-k8s.io/v1beta1', + 'agents.x-k8s.io/v1beta1', + ]); + expect(sandboxes.map((s) => s.metadata?.name)).toEqual([ + 'swebench-sandbox-0', + 'swebench-sandbox-1', + 'swebench-sandbox-2', + ]); + for (const s of sandboxes) { + expect(s.metadata?.namespace).toBe('default'); + } + }); + + it.each(sandboxes.map((s, i) => [i, s]))('sandbox %s: carries the CR-level app=sandbox label', (_i, sandbox: any) => { + expect(sandbox.metadata?.labels?.app).toBe('sandbox'); + }); + + it.each(sandboxes.map((s, i) => [i, s]))( + 'sandbox %s: podTemplate pool-discovery label is exactly swebench (not default)', + (_i, sandbox: any) => { + const poolLabel = sandbox.spec?.podTemplate?.metadata?.labels?.['sh.kagenti.io/sandbox-pool']; + expect(poolLabel).toBe('swebench'); + expect(poolLabel).not.toBe('default'); + }, + ); + + it.each(sandboxes.map((s, i) => [i, s]))('sandbox %s: container uses the Task-3 baked internal-registry image', (_i, sandbox: any) => { + const containers = sandbox.spec?.podTemplate?.spec?.containers ?? []; + expect(containers[0]?.image).toBe(SWEBENCH_IMAGE); + expect(containers[0]?.imagePullPolicy).toBe('IfNotPresent'); + }); + + it.each(sandboxes.map((s, i) => [i, s]))('sandbox %s: container command is exactly sleep infinity (no apk/startup install)', (_i, sandbox: any) => { + const containers = sandbox.spec?.podTemplate?.spec?.containers ?? []; + expect(containers[0]?.command).toEqual(['sleep', 'infinity']); + }); + + it.each(sandboxes.map((s, i) => [i, s]))('sandbox %s: container workingDir and workspace volumeMount are /workspace', (_i, sandbox: any) => { + const containers = sandbox.spec?.podTemplate?.spec?.containers ?? []; + const container = containers[0] ?? {}; + expect(container.workingDir).toBe('/workspace'); + const workspaceMount = (container.volumeMounts ?? []).find((m: any) => m.name === 'workspace'); + expect(workspaceMount?.mountPath).toBe('/workspace'); + }); + + it.each(sandboxes.map((s, i) => [i, s]))('sandbox %s: volumeClaimTemplate requests a 50Gi RWO PVC named workspace', (_i, sandbox: any) => { + const vct = (sandbox.spec?.volumeClaimTemplates ?? [])[0] ?? {}; + expect(vct.metadata?.name).toBe('workspace'); + expect(vct.spec?.accessModes).toEqual(['ReadWriteOnce']); + expect(vct.spec?.resources?.requests?.storage).toBe('50Gi'); + }); + + it.each(sandboxes.map((s, i) => [i, s]))('sandbox %s: podTemplate uses the serverless-harness-sandbox SA', (_i, sandbox: any) => { + expect(sandbox.spec?.podTemplate?.spec?.serviceAccountName).toBe('serverless-harness-sandbox'); + }); + + it.each(sandboxes.map((s, i) => [i, s]))('sandbox %s: pod-level securityContext is OCP nonroot', (_i, sandbox: any) => { + const podSecurityContext = sandbox.spec?.podTemplate?.spec?.securityContext ?? {}; + expect(podSecurityContext.runAsUser).toBe(65532); + expect(podSecurityContext.runAsNonRoot).toBe(true); + expect(podSecurityContext.fsGroup).toBe(65532); + expect(podSecurityContext.seccompProfile?.type).toBe('RuntimeDefault'); + }); + + it.each(sandboxes.map((s, i) => [i, s]))('sandbox %s: container-level securityContext drops all capabilities', (_i, sandbox: any) => { + const containers = sandbox.spec?.podTemplate?.spec?.containers ?? []; + const containerSecurityContext = containers[0]?.securityContext ?? {}; + expect(containerSecurityContext.allowPrivilegeEscalation).toBe(false); + expect(containerSecurityContext.capabilities?.drop).toContain('ALL'); + }); +});