Implement watch dog

.github/workflows/ci.yml

@@ -13,7 +13,7 @@ env:
   CARTESI_MACHINE_SHA256_ARM64: 787d823756000cdecd72da8a3494b4c08613087379035959e561bbaef7a220ba
 
 jobs:
-  rust:
+  checks:
     runs-on: ubuntu-latest
     timeout-minutes: 30
 
@@ -30,7 +30,7 @@ jobs:
             libslirp-dev
 
       - name: Install Rust toolchain
-        uses: dtolnay/rust-toolchain@stable
+        uses: dtolnay/rust-toolchain@1.91.1
         with:
           components: rustfmt, clippy
 
@@ -51,13 +51,16 @@ jobs:
       - name: Clippy
         run: cargo clippy --workspace --all-targets --all-features --locked -- -D warnings
 
+      - name: Watchdog Lua tests
+        run: lua watchdog/tests/run.lua
+
       - name: Test
         timeout-minutes: 15
         run: RUN_ANVIL_TESTS=1 cargo test --workspace --all-targets --all-features --locked
 
   canonical-guest:
     runs-on: ubuntu-latest
-    needs: rust
+    needs: checks
     timeout-minutes: 45
 
     steps:
@@ -82,7 +85,7 @@ jobs:
 
   rollups-e2e:
     runs-on: ubuntu-latest
-    needs: rust
+    needs: checks
     timeout-minutes: 60
 
     steps:

.gitignore

@@ -5,5 +5,7 @@ sequencer.db
 sequencer.db-shm
 sequencer.db-wal
 /out/
+examples/canonical-app/out/
 /.DS_Store
+.vscode/
 soljson-latest.js

docs/watchdog/README.md

@@ -0,0 +1,120 @@
+# Watchdog
+
+The watchdog is an off-chain safety process that compares sequencer API state
+against state produced by the canonical Cartesi Machine at an L1 safe block.
+
+## V1 Shape
+
+The implementation lives in `watchdog/` and is intentionally split into small
+Lua modules:
+
+- `http.lua`: HTTP adapter, currently `lua-curl` oriented.
+- `jsonrpc.lua`: JSON-RPC request/response validation.
+- `l1.lua`: partitioned `eth_getLogs` scanning and strict L1 log ordering.
+- `abi.lua`: decoding for the `InputAdded` / `EvmAdvance` envelope.
+- `machine.lua`: narrow adapter boundary for Cartesi Machine bindings.
+- `machine_cli.lua`: `cartesi-machine` CLI adapter for loading snapshot
+  directories, writing raw input files, advancing, and saving a new snapshot
+  directory.
+- `compare.lua`: raw byte comparison.
+- `checkpoint.lua`: manifest-backed checkpoint persistence.
+- `alarm.lua`: webhook alarm delivery.
+- `retry.lua`: bounded retry helper used by the runtime.
+- `runner.lua`: one-shot orchestration across checkpoint load, sequencer poll,
+  L1 fetch, CM replay, raw compare, alarm, and checkpoint write.
+
+The L1 reader follows the Rust partition strategy from
+`sequencer/src/partition.rs`: if an RPC provider rejects a large range, the
+range is split recursively and retried. Lua decodes and validates input
+envelopes, but it does not classify payload tags. Direct input vs batch
+submission remains scheduler logic inside the canonical machine.
+
+`l1.lua` has the `InputAdded(address,uint256,bytes)` event topic baked in and
+filters logs by `topic0 = InputAdded` and `topic1 = app address`, matching the
+Rust reader's app-filtered InputBox scan.
+
+## Runtime Contract
+
+The future sequencer endpoint shape should be generic over the app state bytes,
+even though the toy wallet app will likely use JSON:
+
+```json
+{
+  "safe_block": 123,
+  "state": "{\"balances\":{}}"
+}
+```
+
+`state` must be the exact bytes produced by the bare-metal app serializer
+for the app state anchored at `safe_block`. The watchdog compares those raw
+bytes with the bytes returned by CM inspect. It must not canonicalize both
+values before deciding pass/fail.
+
+The main design gate is safe-state semantics: if the sequencer has already
+applied soft-confirmed transactions beyond L1 safety, `get_state` still needs a
+safe-only state view through snapshotting, replay, or a separate projection.
+
+## Checkpoints
+
+V1 persists only the resulting Cartesi Machine checkpoint, not the fetched L1
+inputs.
+
+```text
+checkpoint_dir/
+  current.json
+  checkpoints/
+    00000000000001234567/
+      snapshot/
+      manifest.json
+```
+
+`manifest.json` records `safe_block`, timestamp, and optionally the CM image
+hash. A new checkpoint directory is written first, then `current.json` is
+atomically replaced to point at it.
+
+When bootstrapping without an existing checkpoint, the operator provides both:
+
+- `WATCHDOG_CM_SNAPSHOT_DIR`
+- `WATCHDOG_CM_SNAPSHOT_SAFE_BLOCK`
+
+## Modes
+
+The default `WATCHDOG_MODE` is `advance`. In this mode the watchdog does not
+poll the sequencer. It:
+
+1. Loads the latest checkpoint, or the bootstrap snapshot directory.
+2. Reads the L1 safe block from the RPC (or `WATCHDOG_TARGET_SAFE_BLOCK` when
+   provided for tests/manual runs).
+3. Fetches and decodes `InputAdded` logs for the block range.
+4. Feeds the raw InputBox input bytes into the CM adapter.
+5. Saves a new snapshot directory and advances `current.json`.
+
+`WATCHDOG_MODE=compare` is reserved for the future state comparison flow once
+CM inspect and the sequencer state endpoint are available.
+
+Useful runtime knobs:
+
+- `WATCHDOG_CM_EXECUTABLE`: Cartesi Machine executable, default
+  `cartesi-machine`.
+- `WATCHDOG_CM_WORK_DIR`: temporary directory for staged input files, default
+  `/tmp`.
+- `WATCHDOG_RETRY_ATTEMPTS`: bounded retry attempts per run, default `3`.
+- `WATCHDOG_RETRY_DELAY_SEC`: delay between retry attempts, default `5`.
+- `WATCHDOG_TARGET_SAFE_BLOCK`: manual/test override for the target safe block.
+
+## Local Tests
+
+Run the pure Lua tests with:
+
+```bash
+just test-watchdog
+```
+
+These cover raw comparison, golden InputAdded ABI decoding, L1 ordering,
+recursive range partitioning, JSON-RPC `eth_getLogs` filter construction,
+config parsing, checkpoint writes, advance-mode runner behavior, the
+fake-backed compare runner, the CLI adapter's input file staging, and retry
+exhaustion/success behavior.
+
+End-to-end comparison tests will be added once CM inspect and the sequencer
+`get_state` endpoint are available.

examples/canonical-app/justfile

@@ -19,13 +19,13 @@ build-dapp: build-dapp-devnet
 
 build-dapp-devnet:
     mkdir -p {{out_dir}}
-    SOURCE_DATE_EPOCH={{source_date_epoch}} CARGO_PROFILE_RELEASE_STRIP=symbols CROSS_CONFIG=Cross.toml DOCKER_DEFAULT_PLATFORM=linux/amd64 cross build --package canonical-app --bin canonical-app-devnet --target riscv64gc-unknown-linux-musl --release
+    SOURCE_DATE_EPOCH={{source_date_epoch}} CARGO_PROFILE_RELEASE_STRIP=symbols CARGO_TARGET_DIR=../../target CROSS_CONFIG=Cross.toml DOCKER_DEFAULT_PLATFORM=linux/amd64 cross build --package canonical-app --bin canonical-app-devnet --target riscv64gc-unknown-linux-musl --release
     cp ../../target/riscv64gc-unknown-linux-musl/release/canonical-app-devnet {{dapp_binary_devnet}}
     cp {{dapp_binary_devnet}} {{dapp_binary}}
 
 build-dapp-sepolia:
     mkdir -p {{out_dir}}
-    SOURCE_DATE_EPOCH={{source_date_epoch}} CARGO_PROFILE_RELEASE_STRIP=symbols CROSS_CONFIG=Cross.toml DOCKER_DEFAULT_PLATFORM=linux/amd64 cross build --package canonical-app --bin canonical-app-sepolia --target riscv64gc-unknown-linux-musl --release
+    SOURCE_DATE_EPOCH={{source_date_epoch}} CARGO_PROFILE_RELEASE_STRIP=symbols CARGO_TARGET_DIR=../../target CROSS_CONFIG=Cross.toml DOCKER_DEFAULT_PLATFORM=linux/amd64 cross build --package canonical-app --bin canonical-app-sepolia --target riscv64gc-unknown-linux-musl --release
     cp ../../target/riscv64gc-unknown-linux-musl/release/canonical-app-sepolia {{dapp_binary_sepolia}}
     cp {{dapp_binary_sepolia}} {{dapp_binary}}
 

justfile

@@ -12,6 +12,9 @@ check-all-targets:
 test:
     cargo test --workspace
 
+test-watchdog:
+    lua watchdog/tests/run.lua
+
 # Run sequencer tests sequentially so partition static config (init) is not shared across parallel tests.
 test-sequencer:
     cargo test -p sequencer --lib -- --test-threads=1

rust-toolchain.toml

@@ -0,0 +1,3 @@
+[toolchain]
+channel = "1.91.1"
+components = ["rustfmt", "clippy"]

watchdog/abi.lua

@@ -0,0 +1,138 @@
+-- (c) Cartesi and individual authors (see AUTHORS)
+-- SPDX-License-Identifier: Apache-2.0 (see LICENSE)
+
+local abi = {}
+
+local WORD_HEX_LEN = 64
+
+local function strip_0x(value)
+    assert(type(value) == "string", "hex value must be a string")
+    if value:sub(1, 2) == "0x" or value:sub(1, 2) == "0X" then
+        return value:sub(3)
+    end
+    return value
+end
+
+local function assert_hex(value)
+    if value:match("^[0-9a-fA-F]*$") == nil then
+        error("invalid hex string")
+    end
+end
+
+local function word_at(hex, index)
+    local start = (index * WORD_HEX_LEN) + 1
+    local word = hex:sub(start, start + WORD_HEX_LEN - 1)
+    if #word ~= WORD_HEX_LEN then
+        error("ABI word out of bounds")
+    end
+    return word
+end
+
+local function uint_word_to_number(word)
+    local value = 0
+    for i = 1, #word do
+        local nibble = tonumber(word:sub(i, i), 16)
+        value = (value * 16) + nibble
+        if value > 9007199254740991 then
+            error("uint value too large for precise Lua number")
+        end
+    end
+    return value
+end
+
+local function uint_word_to_hex(word)
+    local stripped = word:gsub("^0+", "")
+    if stripped == "" then
+        return "0x0"
+    end
+    return "0x" .. stripped:lower()
+end
+
+local function address_from_word(word)
+    if word:sub(1, 24) ~= string.rep("0", 24) then
+        error("address word has non-zero high bytes")
+    end
+    return "0x" .. word:sub(25):lower()
+end
+
+function abi.bytes_from_hex(hex)
+    hex = strip_0x(hex)
+    assert_hex(hex)
+    if (#hex % 2) ~= 0 then
+        error("hex string must have even length")
+    end
+
+    return (hex:gsub("..", function(byte)
+        return string.char(tonumber(byte, 16))
+    end))
+end
+
+function abi.hex_from_bytes(bytes)
+    return (bytes:gsub(".", function(char)
+        return string.format("%02x", char:byte())
+    end))
+end
+
+function abi.decode_single_dynamic_bytes(encoded)
+    local hex = strip_0x(encoded)
+    assert_hex(hex)
+    local offset = uint_word_to_number(word_at(hex, 0))
+    if (offset % 32) ~= 0 then
+        error("dynamic bytes offset is not word-aligned")
+    end
+
+    local offset_words = offset // 32
+    local len = uint_word_to_number(word_at(hex, offset_words))
+    local data_hex_start = ((offset_words + 1) * WORD_HEX_LEN) + 1
+    local data_hex = hex:sub(data_hex_start, data_hex_start + (len * 2) - 1)
+    if #data_hex ~= len * 2 then
+        error("dynamic bytes payload out of bounds")
+    end
+    return abi.bytes_from_hex(data_hex)
+end
+
+function abi.decode_evm_advance_call(encoded)
+    local hex = strip_0x(encoded)
+    assert_hex(hex)
+
+    -- EvmAdvanceCall is calldata, so accept and skip the 4-byte selector.
+    if (#hex % WORD_HEX_LEN) == 8 then
+        hex = hex:sub(9)
+    end
+
+    local payload_offset = uint_word_to_number(word_at(hex, 7))
+    if (payload_offset % 32) ~= 0 then
+        error("payload offset is not word-aligned")
+    end
+
+    local payload_offset_words = payload_offset // 32
+    local payload_len = uint_word_to_number(word_at(hex, payload_offset_words))
+    local payload_hex_start = ((payload_offset_words + 1) * WORD_HEX_LEN) + 1
+    local payload_hex = hex:sub(payload_hex_start, payload_hex_start + (payload_len * 2) - 1)
+    if #payload_hex ~= payload_len * 2 then
+        error("payload out of bounds")
+    end
+
+    return {
+        chain_id_hex = uint_word_to_hex(word_at(hex, 0)),
+        app_contract = address_from_word(word_at(hex, 1)),
+        msg_sender = address_from_word(word_at(hex, 2)),
+        block_number = uint_word_to_number(word_at(hex, 3)),
+        block_timestamp_hex = uint_word_to_hex(word_at(hex, 4)),
+        prev_randao_hex = uint_word_to_hex(word_at(hex, 5)),
+        index_hex = uint_word_to_hex(word_at(hex, 6)),
+        payload = abi.bytes_from_hex(payload_hex),
+    }
+end
+
+function abi.decode_input_added_log(log)
+    if type(log) ~= "table" or type(log.data) ~= "string" then
+        error("log.data is required")
+    end
+    local input = abi.decode_single_dynamic_bytes(log.data)
+    local decoded = abi.decode_evm_advance_call(abi.hex_from_bytes(input))
+    decoded.raw_input = input
+    return decoded
+end
+
+return abi

watchdog/alarm.lua

@@ -0,0 +1,43 @@
+-- (c) Cartesi and individual authors (see AUTHORS)
+-- SPDX-License-Identifier: Apache-2.0 (see LICENSE)
+
+local alarm = {}
+
+local function encode_json_object(fields)
+    local parts = {}
+    for key, value in pairs(fields) do
+        local encoded_value
+        if type(value) == "number" then
+            encoded_value = tostring(value)
+        elseif type(value) == "boolean" then
+            encoded_value = value and "true" or "false"
+        else
+            local string_value = tostring(value)
+                :gsub("\\", "\\\\")
+                :gsub('"', '\\"')
+                :gsub("\n", "\\n")
+            encoded_value = '"' .. string_value .. '"'
+        end
+        table.insert(parts, string.format('"%s":%s', key, encoded_value))
+    end
+    return "{" .. table.concat(parts, ",") .. "}"
+end
+
+function alarm.send_webhook(http, webhook_url, payload)
+    if not webhook_url or webhook_url == "" then
+        return nil, "WATCHDOG_WEBHOOK_URL is not configured"
+    end
+    local body = encode_json_object(payload)
+    local response, err = http:post(webhook_url, body, {
+        ["content-type"] = "application/json",
+    })
+    if not response then
+        return nil, err
+    end
+    if response.status < 200 or response.status >= 300 then
+        return nil, "alarm webhook HTTP " .. tostring(response.status)
+    end
+    return true
+end
+
+return alarm