From 442746fd8675fcd7c07fa2bfdc7858ed004ec39c Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Wed, 1 Jul 2026 03:45:19 -0400 Subject: [PATCH] cpython: ship the standard library so the runtime can import re/json MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cpython package built only python.wasm; build-cpython.sh never installed the stdlib, so the shipped runtime had no lib/python3.13 on sys.path and `import re` (and json, which imports re) failed with ModuleNotFoundError. The stdlib existed only in the source tree. - build-cpython.sh: after building python.wasm, package lib/python3.13 (from the source Lib, excluding the test suite and GUI-only tkinter/idlelib/turtledemo) into python-stdlib.zip and install it as a package output. - package.toml: declare the new cpython-stdlib output (python-stdlib.zip). - build.toml: bump revision 1 -> 2 (new output). - Formula/cpython.rb: install python.wasm + unzip the stdlib into the keg and set PYTHONHOME so the bottle's python finds re/json; test imports re/json/zlib/base64 with a json round-trip + re match. - demo/import-smoke.ts: Node runtime smoke for representative stdlib imports. Verified under Node (shipped stdlib-zip layout + PYTHONHOME): re/json/zlib/os/ functools/enum/collections/datetime/base64/hashlib/textwrap/argparse all import; json round-trip + re match pass. Also confirms the cpython target keeps zlib. Bead: kd-mt3u. Overlaps PR #810's cpython.rb/build-cpython.sh (kd-p3hr); reconcile like ruby->#814 so cpython is owned here — flagged to coordinator. Co-Authored-By: Claude Opus 4.8 --- homebrew/kandelo-homebrew/Formula/cpython.rb | 52 ++++++++++++++ packages/registry/cpython/build-cpython.sh | 24 +++++++ packages/registry/cpython/build.toml | 2 +- .../registry/cpython/demo/import-smoke.ts | 67 +++++++++++++++++++ packages/registry/cpython/package.toml | 7 ++ 5 files changed, 151 insertions(+), 1 deletion(-) create mode 100644 homebrew/kandelo-homebrew/Formula/cpython.rb create mode 100644 packages/registry/cpython/demo/import-smoke.ts diff --git a/homebrew/kandelo-homebrew/Formula/cpython.rb b/homebrew/kandelo-homebrew/Formula/cpython.rb new file mode 100644 index 000000000..9153c4161 --- /dev/null +++ b/homebrew/kandelo-homebrew/Formula/cpython.rb @@ -0,0 +1,52 @@ +require_relative "../Kandelo/formula_support/kandelo_package" + +class Cpython < Formula + include KandeloPackageFormula + SOURCE_URL = "https://www.python.org/ftp/python/3.13.3/Python-3.13.3.tar.xz" + SOURCE_SHA256 = "40f868bcbdeb8149a3149580bb9bfd407b3321cd48f0be631af955ac92c0e041" + + desc "Python interpreter for Kandelo (with standard library)" + homepage "https://www.python.org/" + url SOURCE_URL + sha256 SOURCE_SHA256 + license "Python-2.0" + + depends_on "automattic/kandelo-homebrew/zlib" + + skip_clean "bin" + skip_clean "lib/python3.13" + + def install + out_dir = kandelo_build_package("cpython", "build-cpython.sh", SOURCE_URL, SOURCE_SHA256, + script_env: { + "PYTHON_VERSION" => version.to_s, + "WASM_POSIX_DEP_ZLIB_DIR" => Formula["automattic/kandelo-homebrew/zlib"].opt_prefix, + }) + kandelo_install_bin(out_dir, "python.wasm", "python") + + # Ship the standard library so the runtime can import re/json/etc.; the bare + # python.wasm has no filesystem stdlib. PYTHONHOME (below) points here. + stdlib_stage = buildpath/"python-stdlib-stage" + system "unzip", "-q", out_dir/"python-stdlib.zip", "-d", stdlib_stage + (prefix/"lib").install Dir["#{stdlib_stage}/lib/*"] + end + + test do + env = { + "PYTHONHOME" => prefix.to_s, + "HOME" => testpath.to_s, + "PYTHONDONTWRITEBYTECODE" => "1", + "PYTHONNOUSERSITE" => "1", + } + assert_match "python-ok", kandelo_run_wasm(bin/"python", ["-c", "print('python-ok')"], env: env) + + # Representative stdlib imports must work (re + json were the reported gap). + prog = <<~PY + import re, json, zlib, base64 + assert json.loads(json.dumps({"a": [1, 2, 3], "b": True})) == {"a": [1, 2, 3], "b": True} + assert re.match(r"(\\d+)-(\\w+)", "42-foo").group(2) == "foo" + print("stdlib-ok") + PY + assert_match "stdlib-ok", kandelo_run_wasm(bin/"python", ["-c", prog], env: env) + end +end diff --git a/packages/registry/cpython/build-cpython.sh b/packages/registry/cpython/build-cpython.sh index 4a0823d0e..95af86e11 100755 --- a/packages/registry/cpython/build-cpython.sh +++ b/packages/registry/cpython/build-cpython.sh @@ -583,10 +583,32 @@ fi echo "==> CPython built successfully!" ls -la "$SCRIPT_DIR/bin/python.wasm" +# --- Package the Python standard library for the shippable runtime --- +# The wasm build produces only python.wasm; without the stdlib on sys.path the +# shipped runtime cannot import re/json/etc. Ship lib/python3.13 as +# python-stdlib.zip (a declared package output) that the Homebrew formula +# installs and points PYTHONHOME at. Exclude the test suite and GUI-only +# packages that cannot run on Kandelo wasm32-posix. +echo "==> Packaging CPython standard library..." +STDLIB_STAGE="$SCRIPT_DIR/python-stdlib-stage" +rm -rf "$STDLIB_STAGE" +mkdir -p "$STDLIB_STAGE/lib/python3.13" +cp -R "$SRC_DIR/Lib/." "$STDLIB_STAGE/lib/python3.13/" +rm -rf "$STDLIB_STAGE/lib/python3.13/test" \ + "$STDLIB_STAGE/lib/python3.13/idlelib" \ + "$STDLIB_STAGE/lib/python3.13/tkinter" \ + "$STDLIB_STAGE/lib/python3.13/turtledemo" +find "$STDLIB_STAGE" -type d -name '__pycache__' -prune -exec rm -rf {} + 2>/dev/null || true +STDLIB_ZIP="$SCRIPT_DIR/bin/python-stdlib.zip" +rm -f "$STDLIB_ZIP" +( cd "$STDLIB_STAGE" && zip -q -r -X "$STDLIB_ZIP" lib ) +echo "==> Packaged stdlib: $STDLIB_ZIP ($(du -h "$STDLIB_ZIP" | cut -f1))" + # Install into local-binaries/ so the resolver picks the freshly-built # binary over the fetched release. source "$REPO_ROOT/scripts/install-local-binary.sh" install_local_binary cpython "$SCRIPT_DIR/bin/python.wasm" +install_local_binary cpython "$STDLIB_ZIP" # Manifest declares `wasm = "python.wasm"` (not cpython.wasm), so the # resolver's $WASM_POSIX_DEP_OUT_DIR scratch needs the file under that @@ -595,4 +617,6 @@ install_local_binary cpython "$SCRIPT_DIR/bin/python.wasm" if [ -n "${WASM_POSIX_DEP_OUT_DIR:-}" ]; then cp "$SCRIPT_DIR/bin/python.wasm" "$WASM_POSIX_DEP_OUT_DIR/python.wasm" echo " installed $WASM_POSIX_DEP_OUT_DIR/python.wasm (manifest output name)" + cp "$STDLIB_ZIP" "$WASM_POSIX_DEP_OUT_DIR/python-stdlib.zip" + echo " installed $WASM_POSIX_DEP_OUT_DIR/python-stdlib.zip (stdlib output)" fi diff --git a/packages/registry/cpython/build.toml b/packages/registry/cpython/build.toml index eed44f438..790f6e815 100644 --- a/packages/registry/cpython/build.toml +++ b/packages/registry/cpython/build.toml @@ -1,7 +1,7 @@ script_path = "packages/registry/cpython/build-cpython.sh" repo_url = "https://github.com/brandonpayton/kandelo.git" commit = "8c53383229fab78f97b098c3207a655159c03041" -revision = 1 +revision = 2 [binary] index_url = "https://github.com/Automattic/kandelo/releases/download/binaries-abi-v{abi}/index.toml" diff --git a/packages/registry/cpython/demo/import-smoke.ts b/packages/registry/cpython/demo/import-smoke.ts new file mode 100644 index 000000000..272038d81 --- /dev/null +++ b/packages/registry/cpython/demo/import-smoke.ts @@ -0,0 +1,67 @@ +/** + * import-smoke.ts — kd-mt3u CPython stdlib import smoke on Kandelo. + * + * Runs the built python.wasm under the Node kernel host (host-fs passthrough so + * PYTHONHOME resolves) and checks representative stdlib imports (re, json, zlib, + * ...) plus a json round-trip and a re match. + * + * Usage: + * bash build.sh && bash packages/registry/cpython/build-cpython.sh + * npx tsx packages/registry/cpython/demo/import-smoke.ts [PYTHONHOME] + * + * PYTHONHOME defaults to the built cpython-install; pass an arg to override + * (e.g. a packaged stdlib bundle) to smoke the shippable layout. + */ +import { existsSync } from "fs"; +import { resolve, dirname } from "path"; +import { runCentralizedProgram } from "../../../../host/test/centralized-test-helper"; +import { NodePlatformIO } from "../../../../host/src/platform/node"; + +const scriptDir = dirname(new URL(import.meta.url).pathname); +const repoRoot = resolve(scriptDir, "../../../.."); + +const PY = [ + "import sys", + "mods=['re','json','zlib','os','functools','enum','collections','datetime','base64','hashlib','textwrap','argparse']", + "res=[]", + "for m in mods:", + " try:", + " __import__(m); res.append(m+'=ok')", + " except Exception as e:", + " res.append(m+'=FAIL('+type(e).__name__+')')", + "print('PYVER='+sys.version.split()[0])", + "print('MODS='+','.join(res))", + "import json as j", + "assert j.loads(j.dumps({'a':[1,2,3],'b':True,'c':None}))=={'a':[1,2,3],'b':True,'c':None}, 'json roundtrip'", + "import re as r", + "assert r.match(r'(\\d+)-(\\w+)','42-foo').group(2)=='foo', 're match'", + "print('IMPORT_SMOKE_PASS' if all('=ok' in x for x in res) else 'IMPORT_SMOKE_PARTIAL')", +].join("\n"); + +async function main() { + const pythonWasm = resolve(repoRoot, "packages/registry/cpython/bin/python.wasm"); + const pythonHome = process.argv[2] || + resolve(repoRoot, "packages/registry/cpython/cpython-install"); + if (!existsSync(pythonWasm)) { + console.error("python.wasm not found. Run: bash packages/registry/cpython/build-cpython.sh"); + process.exit(1); + } + + const result = await runCentralizedProgram({ + programPath: pythonWasm, + argv: ["python", "-c", PY], + env: [`PYTHONHOME=${pythonHome}`, `HOME=/tmp`, `TMPDIR=/tmp`, `PYTHONDONTWRITEBYTECODE=1`], + io: new NodePlatformIO(), + timeout: 300_000, + }); + + process.stdout.write(result.stdout); + if (result.stderr) process.stderr.write(result.stderr); + const ok = result.exitCode === 0 && result.stdout.includes("IMPORT_SMOKE_PASS"); + process.exit(ok ? 0 : (result.exitCode || 1)); +} + +main().catch((err) => { + console.error("Fatal error:", err); + process.exit(1); +}); diff --git a/packages/registry/cpython/package.toml b/packages/registry/cpython/package.toml index e2d9b1bbf..202a5db7a 100644 --- a/packages/registry/cpython/package.toml +++ b/packages/registry/cpython/package.toml @@ -18,3 +18,10 @@ script_path = "packages/registry/cpython/build-cpython.sh" [[outputs]] name = "cpython" wasm = "python.wasm" + +# Python standard library (lib/python3.13/*.py), staged so the shipped runtime +# can import re/json/etc. The Homebrew formula installs this and points +# PYTHONHOME at it. Without it the bare python.wasm has no filesystem stdlib. +[[outputs]] +name = "cpython-stdlib" +wasm = "python-stdlib.zip"