diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d7f7153..e35c7c9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,6 +71,7 @@ jobs: steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.2.2 with: + fetch-depth: 0 submodules: recursive - name: Install Linux keyring dependencies run: sudo apt-get update && sudo apt-get install -y libdbus-1-dev pkg-config @@ -111,10 +112,10 @@ jobs: working-directory: java env: CHAT2DB_COMMUNITY_CLASSPATH_DIR: "${{ github.workspace }}/target/community-h2-classpath" - CHAT2DB_COMMUNITY_SOURCE_COMMIT: "37a34be858f2566b6b7fcf6c3f64183c1f560853" + CHAT2DB_COMMUNITY_SOURCE_COMMIT: "3cb8af54cad5bd5caa20bb25f10d9b0e4f01931c" run: >- ./mvnw -B -pl compat-runtime - -Dtest='CommunityPluginRegistryTest#realCommunityH2BuildsAndExecutesBoundedDml,CommunityPluginRegistryTest#realCommunityMysqlRejectsBackslashCrossColumnInjection,CommunityPluginRegistryTest#realCommunityMysqlNormalizesBooleanAliasesAndBits,CommunityPluginRegistryTest#realCommunityH2BuildsNamespaceSqlWithoutOpeningJdbc,CommunityPluginRegistryTest#realCommunityMysqlBuildsDatabaseNamespaceSql,CommunityPluginRegistryTest#realCommunityNamespaceMapsUnsupportedAndRejectsOversizedInput,JdbcProtocolLoopTest#communityDmlDispatchDoesNotRequireAJdbcSession,JdbcProtocolLoopTest#communityNamespaceDispatchDoesNotRequireAJdbcSession' + -Dtest='CommunityPluginRegistryTest#realCommunityH2BuildsAndExecutesBoundedDml,CommunityPluginRegistryTest#realCommunityMysqlRejectsBackslashCrossColumnInjection,CommunityPluginRegistryTest#realCommunityMysqlNormalizesBooleanAliasesAndBits,CommunityPluginRegistryTest#realCommunityH2BuildsNamespaceSqlWithoutOpeningJdbc,CommunityPluginRegistryTest#realCommunityMysqlBuildsDatabaseNamespaceSql,CommunityPluginRegistryTest#realCommunityMysqlBuildsBoundedTablePreviewSqlWithoutOpeningJdbc,CommunityPluginRegistryTest#realCommunityNamespaceMapsUnsupportedAndRejectsOversizedInput,JdbcProtocolLoopTest#communityDmlDispatchDoesNotRequireAJdbcSession,JdbcProtocolLoopTest#communityNamespaceDispatchDoesNotRequireAJdbcSession' test - name: Verify real Community H2 SPI vertical slice env: @@ -163,7 +164,7 @@ jobs: runs-on: ubuntu-latest services: mysql: - image: mysql:8.0.30 + image: mysql:8.4.6@sha256:869218921e61d6c3c89820955d63cca42971f0e3e6c1e2792247bbd944ebc6e9 env: MYSQL_ROOT_HOST: "%" MYSQL_ROOT_PASSWORD: chat2db-ci-root @@ -177,15 +178,29 @@ jobs: steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.2.2 with: + fetch-depth: 0 submodules: recursive - - name: Install Linux keyring dependencies - run: sudo apt-get update && sudo apt-get install -y libdbus-1-dev pkg-config + - name: Install Linux keyring and SSH dependencies + run: >- + sudo apt-get update && sudo apt-get install -y + libdbus-1-dev pkg-config openssh-server - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 with: toolchain: stable - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2.8.1 with: shared-key: mysql-integration + - name: Verify native MySQL driver kernel without Java + env: + MYSQL_TEST_HOST: 127.0.0.1 + MYSQL_TEST_PORT: "3306" + MYSQL_TEST_USER: root + MYSQL_TEST_PASSWORD: chat2db-ci-root + MYSQL_TEST_REQUIRED: "1" + run: >- + cargo test -p chat2db-core --lib --locked + live_mysql_console_kernel_preserves_session_results_and_cancellation + -- --ignored - name: Verify native MySQL product vertical without Java env: MYSQL_TEST_HOST: 127.0.0.1 @@ -194,21 +209,120 @@ jobs: MYSQL_TEST_PASSWORD: chat2db-ci-root MYSQL_TEST_REQUIRED: "1" run: cargo test -p chat2db-core --test native_mysql_product --locked + - name: Verify confirmed MySQL writes through the local runtime + env: + MYSQL_TEST_HOST: 127.0.0.1 + MYSQL_TEST_PORT: "3306" + MYSQL_TEST_USER: root + MYSQL_TEST_PASSWORD: chat2db-ci-root + MYSQL_TEST_REQUIRED: "1" + run: >- + cargo test -p chat2db-local --test native_mysql_write_docker + --locked - name: Verify native MySQL Console without Java env: MYSQL_TEST_HOST: 127.0.0.1 MYSQL_TEST_PORT: "3306" MYSQL_TEST_USER: root MYSQL_TEST_PASSWORD: chat2db-ci-root + MYSQL_TEST_REQUIRED: "1" run: >- cargo test -p chat2db-core --test native_mysql_console_docker --locked -- --ignored + - name: Verify native MySQL account lifecycle without Java + env: + MYSQL_TEST_HOST: 127.0.0.1 + MYSQL_TEST_PORT: "3306" + MYSQL_TEST_USER: root + MYSQL_TEST_PASSWORD: chat2db-ci-root + MYSQL_TEST_REQUIRED: "1" + run: cargo test -p chat2db-core --test native_mysql_account_docker --locked + - name: Verify native MySQL schema diff without Java + env: + MYSQL_TEST_HOST: 127.0.0.1 + MYSQL_TEST_PORT: "3306" + MYSQL_TEST_USER: root + MYSQL_TEST_PASSWORD: chat2db-ci-root + MYSQL_TEST_REQUIRED: "1" + run: >- + cargo test -p chat2db-core --test native_mysql_schema_diff_docker + --locked + - name: Verify native MySQL transfer without Java + env: + MYSQL_TEST_HOST: 127.0.0.1 + MYSQL_TEST_PORT: "3306" + MYSQL_TEST_USER: root + MYSQL_TEST_PASSWORD: chat2db-ci-root + MYSQL_TEST_REQUIRED: "1" + run: cargo test -p chat2db-core --test native_mysql_transfer_docker --locked + - name: Verify native MySQL Dashboard refresh without Java + env: + MYSQL_TEST_HOST: 127.0.0.1 + MYSQL_TEST_PORT: "3306" + MYSQL_TEST_USER: root + MYSQL_TEST_PASSWORD: chat2db-ci-root + MYSQL_TEST_REQUIRED: "1" + run: cargo test -p chat2db-core --test native_mysql_dashboard_docker --locked + - name: Start isolated SSH forwarding fixture + run: | + install -d -m 700 "$HOME/.ssh" "$RUNNER_TEMP/chat2db-sshd" + ssh-keygen -q -t ed25519 -N '' -f "$RUNNER_TEMP/chat2db-sshd/client-key" + ssh-keygen -q -t ed25519 -N '' -f "$RUNNER_TEMP/chat2db-sshd/host-key" + install -m 600 "$RUNNER_TEMP/chat2db-sshd/client-key.pub" "$RUNNER_TEMP/chat2db-sshd/authorized_keys" + { + echo 'Port 2222' + echo 'ListenAddress 127.0.0.1' + echo "HostKey $RUNNER_TEMP/chat2db-sshd/host-key" + echo "PidFile $RUNNER_TEMP/chat2db-sshd/sshd.pid" + echo "AuthorizedKeysFile $RUNNER_TEMP/chat2db-sshd/authorized_keys" + echo "AllowUsers $(id -un)" + echo 'AuthenticationMethods publickey' + echo 'PasswordAuthentication no' + echo 'PubkeyAuthentication yes' + echo 'PermitRootLogin no' + echo 'AllowTcpForwarding yes' + echo 'PermitOpen 127.0.0.1:3306' + echo 'StrictModes no' + echo 'UsePAM yes' + } > "$RUNNER_TEMP/chat2db-sshd/sshd_config" + sudo install -d -m 755 /run/sshd + sudo /usr/sbin/sshd -f "$RUNNER_TEMP/chat2db-sshd/sshd_config" -E "$RUNNER_TEMP/chat2db-sshd/sshd.log" + for attempt in {1..20}; do + if ssh-keyscan -p 2222 127.0.0.1 > "$RUNNER_TEMP/chat2db-sshd/known_hosts" 2>/dev/null; then + break + fi + if (( attempt == 20 )); then + cat "$RUNNER_TEMP/chat2db-sshd/sshd.log" >&2 + exit 1 + fi + sleep 1 + done + test -s "$RUNNER_TEMP/chat2db-sshd/known_hosts" + install -m 600 "$RUNNER_TEMP/chat2db-sshd/known_hosts" "$HOME/.ssh/known_hosts" + ssh -i "$RUNNER_TEMP/chat2db-sshd/client-key" -p 2222 \ + -o BatchMode=yes -o StrictHostKeyChecking=yes \ + "$(id -un)@127.0.0.1" true + - name: Verify native MySQL SSH tunnel without Java + env: + CHAT2DB_TEST_MYSQL_HOST: 127.0.0.1 + CHAT2DB_TEST_MYSQL_PORT: "3306" + CHAT2DB_TEST_MYSQL_USER: root + CHAT2DB_TEST_MYSQL_PASSWORD: chat2db-ci-root + CHAT2DB_TEST_SSH_HOST: 127.0.0.1 + CHAT2DB_TEST_SSH_PORT: "2222" + CHAT2DB_TEST_SSH_USER: runner + CHAT2DB_TEST_SSH_PRIVATE_KEY: "${{ runner.temp }}/chat2db-sshd/client-key" + CHAT2DB_TEST_SSH_LOCAL_PORT: "23306" + run: >- + cargo test -p chat2db-core --test native_mysql_ssh_tunnel_docker + --locked -- --ignored - name: Verify native MySQL editable grid and DDL without Java env: MYSQL_TEST_HOST: 127.0.0.1 MYSQL_TEST_PORT: "3306" MYSQL_TEST_USER: root MYSQL_TEST_PASSWORD: chat2db-ci-root + MYSQL_TEST_REQUIRED: "1" run: >- cargo test -p chat2db-web --test native_mysql_editable_ddl_docker --locked -- --ignored @@ -342,7 +456,7 @@ jobs: CHAT2DB_H2_DRIVER_JAR: "${{ github.workspace }}/java/compat-runtime/target/test-drivers/h2-2.3.232.jar" run: >- cargo test -p chat2db-core --features java-integration - --test java_h2_product --locked + --test java_h2_product --locked -- --test-threads=1 - name: Build fixed Community H2 compatibility classpath on Windows if: runner.os == 'Windows' shell: bash diff --git a/.github/workflows/macos-package.yml b/.github/workflows/macos-package.yml index 7f06b63..34797b9 100644 --- a/.github/workflows/macos-package.yml +++ b/.github/workflows/macos-package.yml @@ -1,9 +1,6 @@ name: macOS Package on: - push: - tags: - - "macos-test-*" workflow_dispatch: inputs: publish_authorized_artifact: @@ -27,9 +24,10 @@ jobs: steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.2.2 with: + fetch-depth: 0 submodules: recursive - name: Enforce Object-form artifact authorization - if: ${{ (github.event_name == 'push' || inputs.publish_authorized_artifact) && vars.CHAT2DB_OBJECT_DISTRIBUTION_AUTHORIZED != 'true' }} + if: ${{ inputs.publish_authorized_artifact && vars.CHAT2DB_OBJECT_DISTRIBUTION_AUTHORIZED != 'true' }} run: | echo "Artifact upload requires CHAT2DB_OBJECT_DISTRIBUTION_AUTHORIZED=true" >&2 exit 1 @@ -68,7 +66,7 @@ jobs: - name: Add package manifest to summary run: cat target/macos-package/BUILD-MANIFEST.txt >> "$GITHUB_STEP_SUMMARY" - name: Upload authorized package - if: ${{ (github.event_name == 'push' || inputs.publish_authorized_artifact) && vars.CHAT2DB_OBJECT_DISTRIBUTION_AUTHORIZED == 'true' }} + if: ${{ inputs.publish_authorized_artifact && vars.CHAT2DB_OBJECT_DISTRIBUTION_AUTHORIZED == 'true' }} uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: Chat2DB-Rust-macOS-arm64-${{ github.sha }} diff --git a/Cargo.lock b/Cargo.lock index 43ae5cb..d7d2501 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14,8 +14,18 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" dependencies = [ - "crypto-common", - "generic-array", + "crypto-common 0.1.7", + "generic-array 0.14.7", +] + +[[package]] +name = "aead" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1973cfbc1a2daf9cf550e74e1f088c28e7f7d8c1e1418fb6c9dc5184b7e84c99" +dependencies = [ + "crypto-common 0.2.2", + "inout 0.2.2", ] [[package]] @@ -25,21 +35,48 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ "cfg-if", - "cipher", + "cipher 0.4.4", "cpufeatures 0.2.17", ] +[[package]] +name = "aes" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8eb277bec05f56a0e0591f155a484cbd0f4f07ff2905051a48c72f004f7ed58" +dependencies = [ + "cipher 0.5.2", + "cpubits", + "cpufeatures 0.3.0", + "zeroize", +] + [[package]] name = "aes-gcm" version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" dependencies = [ - "aead", - "aes", - "cipher", - "ctr", - "ghash", + "aead 0.5.2", + "aes 0.8.4", + "cipher 0.4.4", + "ctr 0.9.2", + "ghash 0.5.1", + "subtle", + "zeroize", +] + +[[package]] +name = "aes-gcm" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdf011db2e21ce0d575593d749db5554b47fed37aff429e4dc50bc91ac93a028" +dependencies = [ + "aead 0.6.1", + "aes 0.9.2", + "cipher 0.5.2", + "ctr 0.10.1", + "ghash 0.6.0", "subtle", "zeroize", ] @@ -59,6 +96,12 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "ambient-authority" +version = "0.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9d4ee0d472d1cd2e28c97dfa124b3d8d992e10eb0a035f33f5d12e3a177ba3b" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -124,6 +167,57 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "ar_archive_writer" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73cd58deff2140a0a8eae87e417bd01db68a33e148aa93d1e8cd837e55e312b6" +dependencies = [ + "object", +] + +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + +[[package]] +name = "argon2" +version = "0.6.0-rc.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7af50940b73bf4e16c15c448a2b121c63f2d68e3e54b6a8731673cb4aa0cdff5" +dependencies = [ + "base64ct", + "blake2", + "cpufeatures 0.3.0", + "password-hash", +] + +[[package]] +name = "ashpd" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f3f79755c74fd155000314eb349864caa787c6592eace6c6882dad873d9c39" +dependencies = [ + "enumflags2", + "futures-channel", + "futures-util", + "rand 0.9.5", + "raw-window-handle", + "serde", + "serde_repr", + "tokio", + "url", + "wayland-backend", + "wayland-client", + "wayland-protocols", + "zbus 5.18.0", +] + [[package]] name = "async-broadcast" version = "0.7.2" @@ -148,6 +242,20 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "async-executor" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96bf972d85afc50bf5ab8fe2d54d1586b4e0b46c97c50a0c9e71e2f7bcd812a" +dependencies = [ + "async-task", + "concurrent-queue", + "fastrand", + "futures-lite", + "pin-project-lite", + "slab", +] + [[package]] name = "async-io" version = "2.6.0" @@ -295,6 +403,7 @@ dependencies = [ "matchit", "memchr", "mime", + "multer", "percent-encoding", "pin-project-lite", "serde_core", @@ -328,6 +437,12 @@ dependencies = [ "tracing", ] +[[package]] +name = "base16ct" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd307490d624467aa6f74b0eabb77633d1f758a7b25f12bceb0b22e08d9726f6" + [[package]] name = "base64" version = "0.21.7" @@ -340,6 +455,23 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + +[[package]] +name = "bcrypt-pbkdf" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "144e573728da132683b9488acd528274c790e07fc06ff81ee29f9d8f8b1041e0" +dependencies = [ + "blowfish 0.10.0", + "pbkdf2", + "sha2 0.11.0", +] + [[package]] name = "bit-set" version = "0.8.0" @@ -370,13 +502,32 @@ dependencies = [ "serde_core", ] +[[package]] +name = "blake2" +version = "0.11.0-rc.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "061f1a09225e328e1ffbb378d2d49923c0ca5fee19fb5ac1cc9c1e9d52b93690" +dependencies = [ + "digest 0.11.3", +] + [[package]] name = "block-buffer" version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ - "generic-array", + "generic-array 0.14.7", +] + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", + "zeroize", ] [[package]] @@ -385,7 +536,16 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" dependencies = [ - "generic-array", + "generic-array 0.14.7", +] + +[[package]] +name = "block-padding" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "710f1dd022ef4e93f8a438b4ba958de7f64308434fa6a87104481645cc30068b" +dependencies = [ + "hybrid-array", ] [[package]] @@ -410,6 +570,26 @@ dependencies = [ "piper", ] +[[package]] +name = "blowfish" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e412e2cd0f2b2d93e02543ceae7917b3c70331573df19ee046bcbc35e45e87d7" +dependencies = [ + "byteorder", + "cipher 0.4.4", +] + +[[package]] +name = "blowfish" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62ce3946557b35e71d1bbe07ec385073ce9eda05043f95de134eb578fcf1a298" +dependencies = [ + "byteorder", + "cipher 0.5.2", +] + [[package]] name = "bs58" version = "0.5.1" @@ -489,6 +669,36 @@ dependencies = [ "serde_core", ] +[[package]] +name = "cap-primitives" +version = "4.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdadbd7c002d3a484b35243669abdae85a0ebaded5a61117169dc3400f9a7ff0" +dependencies = [ + "ambient-authority", + "fs-set-times", + "io-extras", + "io-lifetimes 3.0.1", + "ipnet", + "maybe-owned", + "rustix", + "rustix-linux-procfs", + "windows-sys 0.61.2", + "winx", +] + +[[package]] +name = "cap-std" +version = "4.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7281235d6e96d3544ca18bba9049be92f4190f8d923e3caef1b5f66cfa752608" +dependencies = [ + "cap-primitives", + "io-extras", + "io-lifetimes 3.0.1", + "rustix", +] + [[package]] name = "cargo-platform" version = "0.1.9" @@ -528,7 +738,16 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" dependencies = [ - "cipher", + "cipher 0.4.4", +] + +[[package]] +name = "cbc" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2dc9ee5f88d11e0beb842c88b33c8a5cf0d1329c4b19494af42b07dbfe8896" +dependencies = [ + "cipher 0.5.2", ] [[package]] @@ -558,6 +777,17 @@ dependencies = [ "uuid", ] +[[package]] +name = "cfb" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a347dcabdae9c31b0825fd6a8bed285ec9c2acb89c47827126d52fa4f59cece3" +dependencies = [ + "fnv", + "uuid", + "web-time", +] + [[package]] name = "cfg-expr" version = "0.15.8" @@ -587,8 +817,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" dependencies = [ "cfg-if", + "cipher 0.5.2", "cpufeatures 0.3.0", "rand_core 0.10.1", + "zeroize", ] [[package]] @@ -602,7 +834,7 @@ dependencies = [ "reqwest", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "thiserror 2.0.19", "tokio", "tokio-util", @@ -635,19 +867,31 @@ dependencies = [ name = "chat2db-core" version = "0.1.0" dependencies = [ + "aes 0.8.4", "async-trait", "base64 0.22.1", + "blowfish 0.9.1", + "cbc 0.1.2", "chat2db-agent", "chat2db-contract", "chat2db-engine-protocol", "chat2db-java-bridge", "chat2db-storage", + "chrono", + "csv", + "directories", "futures-util", + "hex", "mysql_async", "prost", + "quick-xml 0.37.5", + "russh", "rustix", "serde", "serde_json", + "sha1 0.10.7", + "sha2 0.10.9", + "sqlparser", "tempfile", "thiserror 2.0.19", "tokio", @@ -655,25 +899,33 @@ dependencies = [ "tracing", "url", "uuid", + "xls", + "zip", ] [[package]] name = "chat2db-desktop" version = "0.1.0" dependencies = [ + "cap-std", "chat2db-contract", "chat2db-core", "chat2db-java-bridge", "chat2db-local", "chat2db-web", + "encoding_rs", "serde", "serde_json", "tauri", "tauri-build", + "tauri-plugin-dialog", + "tauri-plugin-opener", "tauri-runtime", "tempfile", "tokio", "tracing", + "trash", + "uuid", ] [[package]] @@ -694,7 +946,7 @@ dependencies = [ "chat2db-engine-protocol", "prost", "rustix", - "sha2", + "sha2 0.10.9", "tempfile", "thiserror 2.0.19", "tokio", @@ -707,6 +959,7 @@ dependencies = [ "base64 0.22.1", "chat2db-contract", "chat2db-core", + "chat2db-java-bridge", "chat2db-local-ipc-windows", "chat2db-storage", "fs2", @@ -714,7 +967,7 @@ dependencies = [ "rustix", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "subtle", "tempfile", "thiserror 2.0.19", @@ -741,9 +994,12 @@ dependencies = [ "chat2db-local", "chat2db-storage", "clap", + "hex", + "rand 0.9.5", "rmcp", "serde", "serde_json", + "sha2 0.10.9", "tempfile", "tokio", "tracing-subscriber", @@ -753,7 +1009,7 @@ dependencies = [ name = "chat2db-storage" version = "0.1.0" dependencies = [ - "aes-gcm", + "aes-gcm 0.10.3", "base64 0.22.1", "chat2db-contract", "chat2db-engine-protocol", @@ -764,7 +1020,7 @@ dependencies = [ "rand 0.9.5", "rusqlite", "serde_json", - "sha2", + "sha2 0.10.9", "tempfile", "thiserror 2.0.19", "uuid", @@ -786,18 +1042,24 @@ dependencies = [ "futures-util", "http-body-util", "mysql_async", + "quick-xml 0.37.5", + "reqwest", "serde", "serde_json", "subtle", "tempfile", "tokio", + "tokio-util", "tower", "tower-http 0.7.0", "tracing", "tracing-subscriber", + "url", "utoipa", "utoipa-axum", "uuid", + "xls", + "zip", ] [[package]] @@ -820,8 +1082,20 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common", - "inout", + "crypto-common 0.1.7", + "inout 0.1.4", +] + +[[package]] +name = "cipher" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c" +dependencies = [ + "block-buffer 0.12.1", + "crypto-common 0.2.2", + "inout 0.2.2", + "zeroize", ] [[package]] @@ -864,6 +1138,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + [[package]] name = "colorchoice" version = "1.0.5" @@ -889,6 +1169,12 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "convert_case" version = "0.4.0" @@ -955,6 +1241,12 @@ dependencies = [ "libc", ] +[[package]] +name = "cpubits" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b85f9c39137c3a891689859392b1bd49812121d0d61c9caf00d46ed5ce06ae" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -1006,17 +1298,55 @@ version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" +[[package]] +name = "crypto-bigint" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a52aa3fcda4e6302a9f48734f234d35d4721b96f8fe07d073f07ce9df4f0271" +dependencies = [ + "cpubits", + "ctutils", + "getrandom 0.4.3", + "hybrid-array", + "num-traits", + "rand_core 0.10.1", + "serdect", + "subtle", + "zeroize", +] + [[package]] name = "crypto-common" version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ - "generic-array", + "generic-array 0.14.7", "rand_core 0.6.4", "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "getrandom 0.4.3", + "hybrid-array", + "rand_core 0.10.1", +] + +[[package]] +name = "crypto-primes" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3633a51a39c69ebbaa4feaa694bd83d241e4093901c84a0963b19d9bb3f0cf8f" +dependencies = [ + "crypto-bigint", + "rand_core 0.10.1", +] + [[package]] name = "cssparser" version = "0.29.6" @@ -1057,6 +1387,27 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] + [[package]] name = "ctor" version = "0.8.0" @@ -1079,7 +1430,54 @@ version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" dependencies = [ - "cipher", + "cipher 0.4.4", +] + +[[package]] +name = "ctr" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baaca1c4b237092596f64d571e9db6ce4109c4ef9742e27590f1709594461f21" +dependencies = [ + "cipher 0.5.2", +] + +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", + "subtle", +] + +[[package]] +name = "curve25519-dalek" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5eed333089e2e1c1ac8c6c0398e5e2497b4c9926ca6d0365ed1e099afa5bc23" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "curve25519-dalek-derive", + "digest 0.11.3", + "fiat-crypto", + "rand_core 0.10.1", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", ] [[package]] @@ -1116,6 +1514,12 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "data-encoding" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" + [[package]] name = "dbus" version = "0.9.12" @@ -1133,15 +1537,37 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "708b509edf7889e53d7efb0ffadd994cc6c2345ccb62f55cfd6b0682165e4fa6" dependencies = [ - "aes", - "block-padding", - "cbc", + "aes 0.8.4", + "block-padding 0.3.3", + "cbc 0.1.2", "dbus", "fastrand", - "hkdf", + "hkdf 0.12.4", "num", "once_cell", - "sha2", + "sha2 0.10.9", + "zeroize", +] + +[[package]] +name = "delegate" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "780eb241654bf097afb00fc5f054a09b687dad862e485fdcf8399bb056565370" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "der" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a69dedd701da44b0536442edf09c81a64b0ab97a7a4a5e3d1971f00027cbc63d" +dependencies = [ + "const-oid", + "pem-rfc7468", "zeroize", ] @@ -1154,6 +1580,17 @@ dependencies = [ "serde_core", ] +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "derive_more" version = "0.99.20" @@ -1188,17 +1625,38 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "des" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "916a94e407b54f9034d71dd748234cd1e516ced6284009906ae246f177eafe5a" +dependencies = [ + "cipher 0.5.2", +] + [[package]] name = "digest" version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "crypto-common", + "block-buffer 0.10.4", + "crypto-common 0.1.7", "subtle", ] +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid", + "crypto-common 0.2.2", + "ctutils", +] + [[package]] name = "directories" version = "6.0.0" @@ -1252,6 +1710,15 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "dlib" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab8ecd87370524b461f8557c119c405552c396ed91fc0a8eec68679eab26f94a" +dependencies = [ + "libloading", +] + [[package]] name = "dlopen2" version = "0.8.2" @@ -1290,6 +1757,12 @@ dependencies = [ "tendril 0.5.1", ] +[[package]] +name = "downcast-rs" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" + [[package]] name = "dpi" version = "0.1.2" @@ -1341,12 +1814,75 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "ecdsa" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0681a4fc24c767085329728d8dfba959af91228aa4610cca4f8ce317ba46ae0" +dependencies = [ + "der", + "digest 0.11.3", + "elliptic-curve", + "rfc6979", + "signature", + "spki", + "zeroize", +] + +[[package]] +name = "ed25519" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29fcf32e6c73d1079f83ab4d782de2d81620346a5f38c6237a86a22f8368980a" +dependencies = [ + "pkcs8", + "signature", +] + +[[package]] +name = "ed25519-dalek" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ebaa1a2bf1290ab3bfe5a7b771d050ebffab2711c19a81691c683a5144a25de" +dependencies = [ + "curve25519-dalek", + "ed25519", + "rand_core 0.10.1", + "serde", + "sha2 0.11.0", + "signature", + "subtle", + "zeroize", +] + [[package]] name = "either" version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" +[[package]] +name = "elliptic-curve" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d65aa39b3a5c1c9c1b745c9a019234bb7a21b77abcb4f4d266d706e2d577d65" +dependencies = [ + "base16ct", + "crypto-bigint", + "crypto-common 0.2.2", + "digest 0.11.3", + "ff", + "group", + "hkdf 0.13.0", + "hybrid-array", + "pem-rfc7468", + "pkcs8", + "rand_core 0.10.1", + "sec1", + "subtle", + "zeroize", +] + [[package]] name = "embed-resource" version = "3.0.11" @@ -1367,12 +1903,33 @@ version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ef6b89e5b37196644d8796de5268852ff179b44e96276cf4290264843743bb7" +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + [[package]] name = "endi" version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66b7e2430c6dff6a955451e2cfc438f09cea1965a9d6f87f7e3b90decc014099" +[[package]] +name = "enum_dispatch" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa18ce2bc66555b3218614519ac839ddb759a7d6720732f979ef8d13be147ecd" +dependencies = [ + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "enumflags2" version = "0.7.12" @@ -1480,6 +2037,22 @@ dependencies = [ "simd-adler32", ] +[[package]] +name = "ff" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1f686ab92a9fb0eaf188f6c6c87b89490baa6fdb0db4544ba4dc47f7942489f" +dependencies = [ + "rand_core 0.10.1", + "subtle", +] + +[[package]] +name = "fiat-crypto" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64cd1e32ddd350061ae6edb1b082d7c54915b5c672c389143b9a63403a109f24" + [[package]] name = "field-offset" version = "0.3.6" @@ -1510,6 +2083,7 @@ checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ "crc32fast", "miniz_oxide", + "zlib-rs", ] [[package]] @@ -1566,6 +2140,17 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs-set-times" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94e7099f6313ecacbe1256e8ff9d617b75d1bcb16a6fddef94866d225a01a14a" +dependencies = [ + "io-lifetimes 2.0.4", + "rustix", + "windows-sys 0.59.0", +] + [[package]] name = "fs2" version = "0.4.3" @@ -1778,6 +2363,17 @@ dependencies = [ "version_check", ] +[[package]] +name = "generic-array" +version = "1.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab4e5aa225bc56696909483320f0ff9b600f1a971b52e07a17d70f3d9b43254b" +dependencies = [ + "generic-array 0.14.7", + "rustversion", + "typenum", +] + [[package]] name = "getrandom" version = "0.1.16" @@ -1835,7 +2431,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" dependencies = [ "opaque-debug", - "polyval", + "polyval 0.6.2", +] + +[[package]] +name = "ghash" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eecf2d5dc9b66b732b97707a0210906b1d30523eb773193ab777c0c84b3e8d5" +dependencies = [ + "polyval 0.7.3", ] [[package]] @@ -1934,6 +2539,17 @@ dependencies = [ "system-deps", ] +[[package]] +name = "group" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fd1a1c7a5206c5b7a3f5a0d7ccd3ff85d0c8f5133d62a02680255b0004af5f4" +dependencies = [ + "ff", + "rand_core 0.10.1", + "subtle", +] + [[package]] name = "gtk" version = "0.18.2" @@ -2045,13 +2661,28 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hex-literal" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e712f64ec3850b98572bffac52e2c6f282b29fe6c5fa6d42334b30be438d95c1" + [[package]] name = "hkdf" version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" dependencies = [ - "hmac", + "hmac 0.12.1", +] + +[[package]] +name = "hkdf" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4aaa26c720c68b866f2c96ef5c1264b3e6f473fe5d4ce61cd44bbe913e553018" +dependencies = [ + "hmac 0.13.0", ] [[package]] @@ -2060,7 +2691,16 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest", + "digest 0.10.7", +] + +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", ] [[package]] @@ -2136,6 +2776,18 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "hybrid-array" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" +dependencies = [ + "ctutils", + "subtle", + "typenum", + "zeroize", +] + [[package]] name = "hyper" version = "1.11.0" @@ -2363,29 +3015,92 @@ dependencies = [ ] [[package]] -name = "infer" -version = "0.19.0" +name = "infer" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a588916bfdfd92e71cacef98a63d9b1f0d74d6599980d11894290e7ddefffcf7" +dependencies = [ + "cfb 0.7.3", +] + +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "block-padding 0.3.3", + "generic-array 0.14.7", +] + +[[package]] +name = "inout" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4250ce6452e92010fdf7268ccc5d14faa80bb12fc741938534c58f16804e03c7" +dependencies = [ + "block-padding 0.4.2", + "hybrid-array", +] + +[[package]] +name = "internal-russh-num-bigint" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae8e22120c32fb4d19ec55fba35015f57095cd95a2e3b732e44457f5915b2ee8" +dependencies = [ + "num-integer", + "num-traits", + "rand 0.10.2", + "rand_core 0.10.1", +] + +[[package]] +name = "io-extras" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20fd6de4ccfcc187e38bc21cfa543cb5a302cb86a8b114eb7f0bf0dc9f8ac00f" +dependencies = [ + "io-lifetimes 3.0.1", + "windows-sys 0.60.2", +] + +[[package]] +name = "io-lifetimes" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06432fb54d3be7964ecd3649233cddf80db2832f47fec34c01f65b3d9d774983" + +[[package]] +name = "io-lifetimes" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f0fb0570afe1fed943c5c3d4102d5358592d8625fda6a0007fdbe65a92fba96" + +[[package]] +name = "ipnet" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a588916bfdfd92e71cacef98a63d9b1f0d74d6599980d11894290e7ddefffcf7" -dependencies = [ - "cfb", -] +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" [[package]] -name = "inout" -version = "0.1.4" +name = "is-docker" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +checksum = "928bae27f42bc99b60d9ac7334e3a21d10ad8f1835a4e12ec3ec0464765ed1b3" dependencies = [ - "block-padding", - "generic-array", + "once_cell", ] [[package]] -name = "ipnet" -version = "2.12.0" +name = "is-wsl" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" +checksum = "173609498df190136aa7dea1a91db051746d339e18476eed5ca40521f02d7aa5" +dependencies = [ + "is-docker", + "once_cell", +] [[package]] name = "is_terminal_polyfill" @@ -2508,6 +3223,26 @@ dependencies = [ "serde_json", ] +[[package]] +name = "keccak" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e24a010dd405bd7ed803e5253182815b41bf2e6a80cc3bfc066658e03a198aa" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", +] + +[[package]] +name = "kem" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01737161ba802849cfd486b5bd209d38ba4943494c249a8126005170c7621edd" +dependencies = [ + "crypto-common 0.2.2", + "rand_core 0.10.1", +] + [[package]] name = "keyboard-types" version = "0.7.0" @@ -2577,6 +3312,16 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link 0.2.1", +] + [[package]] name = "libredox" version = "0.1.18" @@ -2725,6 +3470,18 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" +[[package]] +name = "maybe-owned" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4facc753ae494aeb6e3c22f839b158aebd4f9270f55cd3c79906c45476c47ab4" + +[[package]] +name = "md5" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ebb8d8732c6a6df3d8f032a82911cfc747e00efb95cc46e8d0acd5b5b88570c" + [[package]] name = "memchr" version = "2.8.3" @@ -2783,6 +3540,31 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "ml-kem" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e15f3e5b957493873e396a66914e83e616b6afe335cdef7efe5c6e1216aba66" +dependencies = [ + "hybrid-array", + "kem", + "module-lattice", + "pkcs8", + "rand_core 0.10.1", + "sha3 0.11.0", +] + +[[package]] +name = "module-lattice" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c61b87c9683ab7cb1c6871d261ad5479b6b10ceb52c4352aaca3b5d35a8febe" +dependencies = [ + "ctutils", + "hybrid-array", + "num-traits", +] + [[package]] name = "muda" version = "0.17.2" @@ -2804,6 +3586,23 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "multer" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83e87776546dc87511aa5ee218730c92b666d7264ab6ed41f9d215af9cd5224b" +dependencies = [ + "bytes", + "encoding_rs", + "futures-util", + "http", + "httparse", + "memchr", + "mime", + "spin", + "version_check", +] + [[package]] name = "multimap" version = "0.10.1" @@ -2879,8 +3678,8 @@ dependencies = [ "saturating", "serde", "serde_json", - "sha1", - "sha2", + "sha1 0.10.7", + "sha2 0.10.9", "thiserror 2.0.19", "uuid", ] @@ -2934,6 +3733,18 @@ dependencies = [ "memoffset", ] +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags 2.13.1", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "nodrop" version = "0.1.14" @@ -3271,6 +4082,15 @@ dependencies = [ "objc2-security", ] +[[package]] +name = "object" +version = "0.39.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e5a6c098c7a3b6547378093f5cc30bc54fd361ce711e05293a5cc589562739b" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -3289,6 +4109,17 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" +[[package]] +name = "open" +version = "5.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0b3d059e795d52b8a72fef45658620edd4d9c359b338564aa14391ffa511ed5" +dependencies = [ + "dunce", + "is-wsl", + "libc", +] + [[package]] name = "option-ext" version = "0.2.0" @@ -3305,6 +4136,67 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "p256" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2c9239b2dbc807adbbe147e8cf72ea7450c3a0aabe62cb8e75ff4ec22e1f72a" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primefield", + "primeorder", + "sha2 0.11.0", +] + +[[package]] +name = "p384" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d17b851e6b3e378ab4ecb07fa2ed23f4d15f075735f8fec9fa1e7bdce5f8301f" +dependencies = [ + "ecdsa", + "elliptic-curve", + "fiat-crypto", + "primefield", + "primeorder", + "sha2 0.11.0", +] + +[[package]] +name = "p521" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ad64cc32c2dc466317c12ee5853e61f159f9eab1fe7efade0395dc2e7b43449" +dependencies = [ + "base16ct", + "ecdsa", + "elliptic-curve", + "primefield", + "primeorder", + "sha2 0.11.0", +] + +[[package]] +name = "pageant" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f3a5ae18f65a85c67a77d18d42d3606c07948e3c17c1e5f74852b26589e88a5" +dependencies = [ + "base16ct", + "byteorder", + "bytes", + "delegate", + "futures", + "log", + "rand 0.10.2", + "sha2 0.11.0", + "thiserror 2.0.19", + "tokio", + "windows 0.62.2", + "windows-strings 0.5.1", +] + [[package]] name = "pango" version = "0.18.3" @@ -3359,6 +4251,15 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "password-hash" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aab41826031698d6ffcd9cff78ef56ef998e39dc7e5067cdfebe373842d4723b" +dependencies = [ + "phc", +] + [[package]] name = "paste" version = "1.0.15" @@ -3371,6 +4272,25 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4" +[[package]] +name = "pbkdf2" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112d82ceb8c5bf524d9af484d4e4970c9fd5a0cc15ba14ad93dccd28873b0629" +dependencies = [ + "digest 0.11.3", + "hmac 0.13.0", +] + +[[package]] +name = "pem-rfc7468" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6305423e0e7738146434843d1694d621cce767262b2a86910beab705e4493d9" +dependencies = [ + "base64ct", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -3388,6 +4308,16 @@ dependencies = [ "indexmap 2.14.0", ] +[[package]] +name = "phc" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44dc769b75f93afdddd8c7fa12d685292ddeff1e66f7f0f3a234cf1818afe892" +dependencies = [ + "base64ct", + "ctutils", +] + [[package]] name = "phf" version = "0.8.0" @@ -3578,6 +4508,45 @@ dependencies = [ "futures-io", ] +[[package]] +name = "pkcs1" +version = "0.8.0-rc.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "986d2e952779af96ea048f160fd9194e1751b4faea78bcf3ceb456efe008088e" +dependencies = [ + "der", + "spki", +] + +[[package]] +name = "pkcs5" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63d440a804ec8d6fafbb6b84471e013286658d373248927692ab3366686220ca" +dependencies = [ + "aes 0.9.2", + "aes-gcm 0.11.0", + "cbc 0.2.1", + "der", + "pbkdf2", + "rand_core 0.10.1", + "scrypt", + "sha2 0.11.0", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "451913da69c775a56034ea8d9003d27ee8948e12443eae7c038ba100a4f21cb7" +dependencies = [ + "der", + "pkcs5", + "rand_core 0.10.1", + "spki", +] + [[package]] name = "pkg-config" version = "0.3.33" @@ -3592,7 +4561,7 @@ checksum = "7da1d65da6dd5d1e44199ac0f58712d241c0f439f80adea8924d832384087f85" dependencies = [ "base64 0.22.1", "indexmap 2.14.0", - "quick-xml", + "quick-xml 0.41.0", "serde", "time", ] @@ -3624,6 +4593,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "poly1305" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2d0073b297041425c7c3df6eb4792d598a15323fe63346852b092eca02904c" +dependencies = [ + "cpufeatures 0.3.0", + "universal-hash 0.6.1", + "zeroize", +] + [[package]] name = "polyval" version = "0.6.2" @@ -3633,7 +4613,18 @@ dependencies = [ "cfg-if", "cpufeatures 0.2.17", "opaque-debug", - "universal-hash", + "universal-hash 0.5.1", +] + +[[package]] +name = "polyval" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0fa31d631f2b2cb2a544d0aa321ce847a94764d701ca2becc411138b93d49cd" +dependencies = [ + "cpubits", + "cpufeatures 0.3.0", + "universal-hash 0.6.1", ] [[package]] @@ -3676,6 +4667,33 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "primefield" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c555a6e4eb7d4e158fcb028c835c3b8642206ddc279b5c6b202ef9a8bdb592f4" +dependencies = [ + "crypto-bigint", + "crypto-common 0.2.2", + "ff", + "rand_core 0.10.1", + "subtle", + "zeroize", +] + +[[package]] +name = "primeorder" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c9f42978c78a00e3d68f69fc03e57a234debae69da4020a4fb588fcdcd07b06" +dependencies = [ + "elliptic-curve", + "once_cell", + "primefield", + "serdect", + "wnaf", +] + [[package]] name = "proc-macro-crate" version = "1.3.1" @@ -3870,6 +4888,25 @@ version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95067976aca6421a523e491fce939a3e65249bac4b977adee0ee9771568e8aa3" +[[package]] +name = "psm" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dcd034599e63b970727f70d79e02d62390a4a84f7c6b827c27c46d5ac3fa622" +dependencies = [ + "ar_archive_writer", + "cc", +] + +[[package]] +name = "quick-xml" +version = "0.37.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" +dependencies = [ + "memchr", +] + [[package]] name = "quick-xml" version = "0.41.0" @@ -4098,6 +5135,26 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20675572f6f24e9e76ef639bc5552774ed45f1c30e2951e1e99c59888861c539" +[[package]] +name = "recursive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0786a43debb760f491b1bc0269fe5e84155353c67482b9e60d0cfb596054b43e" +dependencies = [ + "recursive-proc-macro-impl", + "stacker", +] + +[[package]] +name = "recursive-proc-macro-impl" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" +dependencies = [ + "quote", + "syn 2.0.119", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -4203,9 +5260,44 @@ dependencies = [ "url", "wasm-bindgen", "wasm-bindgen-futures", - "wasm-streams", + "wasm-streams", + "web-sys", + "webpki-roots", +] + +[[package]] +name = "rfc6979" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4a459cddafb3fe76b31fd8f1108007566c40301feb64dc7b54656eb7388172b" +dependencies = [ + "crypto-bigint", + "hmac 0.13.0", +] + +[[package]] +name = "rfd" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef2bee61e6cffa4635c72d7d81a84294e28f0930db0ddcb0f66d10244674ebed" +dependencies = [ + "ashpd", + "block2", + "dispatch2", + "glib-sys", + "gobject-sys", + "gtk-sys", + "js-sys", + "log", + "objc2", + "objc2-app-kit", + "objc2-core-foundation", + "objc2-foundation", + "raw-window-handle", + "wasm-bindgen", + "wasm-bindgen-futures", "web-sys", - "webpki-roots", + "windows-sys 0.59.0", ] [[package]] @@ -4239,8 +5331,10 @@ dependencies = [ "serde_json", "thiserror 2.0.19", "tokio", + "tokio-stream", "tokio-util", "tracing", + "url", ] [[package]] @@ -4256,6 +5350,25 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "rsa" +version = "0.10.0-rc.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b2aa4ba0d89f73d1e332df05be0eeab8840351c36ca5654341dfdb57bb3caf" +dependencies = [ + "const-oid", + "crypto-bigint", + "crypto-primes", + "digest 0.11.3", + "pkcs1", + "pkcs8", + "rand_core 0.10.1", + "sha2 0.11.0", + "signature", + "spki", + "zeroize", +] + [[package]] name = "rusqlite" version = "0.37.0" @@ -4270,6 +5383,101 @@ dependencies = [ "smallvec", ] +[[package]] +name = "russh" +version = "0.62.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c230e0ed9cbeb92fbad6c8848985d6df2a1464c0dc247a021abd666e9005e" +dependencies = [ + "aes 0.9.2", + "bitflags 2.13.1", + "block-padding 0.4.2", + "byteorder", + "bytes", + "cbc 0.2.1", + "cipher 0.5.2", + "crypto-bigint", + "ctr 0.10.1", + "curve25519-dalek", + "data-encoding", + "delegate", + "der", + "digest 0.11.3", + "ecdsa", + "ed25519-dalek", + "elliptic-curve", + "enum_dispatch", + "futures", + "generic-array 1.4.4", + "getrandom 0.4.3", + "ghash 0.6.0", + "hex-literal", + "hmac 0.13.0", + "inout 0.2.2", + "internal-russh-num-bigint", + "keccak", + "log", + "md5", + "ml-kem", + "module-lattice", + "num-bigint", + "p256", + "p384", + "p521", + "pageant", + "pbkdf2", + "pkcs1", + "pkcs5", + "pkcs8", + "polyval 0.7.3", + "rand 0.10.2", + "rand_core 0.10.1", + "ring", + "rsa", + "russh-cryptovec", + "russh-util", + "salsa20", + "scrypt", + "sec1", + "sha1 0.11.0", + "sha2 0.11.0", + "sha3 0.12.0", + "signature", + "spki", + "ssh-encoding", + "ssh-key", + "subtle", + "thiserror 2.0.19", + "tokio", + "typenum", + "universal-hash 0.6.1", + "zeroize", +] + +[[package]] +name = "russh-cryptovec" +version = "0.62.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3aec6cb630dbe85d72ffd7bcd95f07e1bd69f9f270ee8adfa1afe443a6331438" +dependencies = [ + "log", + "nix 0.31.3", + "ssh-encoding", + "windows-sys 0.61.2", +] + +[[package]] +name = "russh-util" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "668424a5dde0bcb45b55ba7de8476b93831b4aa2fa6947e145f3b053e22c60b6" +dependencies = [ + "chrono", + "tokio", + "wasm-bindgen", + "wasm-bindgen-futures", +] + [[package]] name = "rustc-hash" version = "2.1.3" @@ -4298,6 +5506,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "rustix-linux-procfs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fc84bf7e9aa16c4f2c758f27412dc9841341e16aa682d9c7ac308fe3ee12056" +dependencies = [ + "once_cell", + "rustix", +] + [[package]] name = "rustls" version = "0.23.42" @@ -4345,6 +5563,16 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "salsa20" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f874456e72520ff1375a06c588eaf074b0f01f9e9e1aada45bd9b7954a6e42c" +dependencies = [ + "cfg-if", + "cipher 0.5.2", +] + [[package]] name = "same-file" version = "1.0.6" @@ -4425,29 +5653,61 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + [[package]] name = "scopeguard" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "scrypt" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d87af57419b594aa23fa95f09f0e06d80d84ba01c26148c43844cad6ff4485f0" +dependencies = [ + "cfg-if", + "pbkdf2", + "salsa20", + "sha2 0.11.0", +] + +[[package]] +name = "sec1" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56d437c2f19203ce5f7122e507831de96f3d2d4d3be5af44a0b0a09d8a80e4d" +dependencies = [ + "base16ct", + "ctutils", + "der", + "hybrid-array", + "subtle", + "zeroize", +] + [[package]] name = "secret-service" version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4d35ad99a181be0a60ffcbe85d680d98f87bdc4d7644ade319b87076b9dbfd4" dependencies = [ - "aes", - "cbc", + "aes 0.8.4", + "cbc 0.1.2", "futures-util", - "generic-array", - "hkdf", + "generic-array 0.14.7", + "hkdf 0.12.4", "num", "once_cell", "rand 0.8.7", "serde", - "sha2", - "zbus", + "sha2 0.10.9", + "zbus 4.4.0", ] [[package]] @@ -4683,6 +5943,16 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "serdect" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66cf8fedced2fcf12406bcb34223dffb92eaf34908ede12fed414c82b7f00b3e" +dependencies = [ + "base16ct", + "serde", +] + [[package]] name = "serialize-to-javascript" version = "0.1.2" @@ -4732,7 +6002,18 @@ checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" dependencies = [ "cfg-if", "cpufeatures 0.2.17", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "sha1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", ] [[package]] @@ -4743,7 +6024,39 @@ checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures 0.2.17", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", +] + +[[package]] +name = "sha3" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be176f1a57ce4e3d31c1a166222d9768de5954f811601fb7ca06fc8203905ce1" +dependencies = [ + "digest 0.11.3", + "keccak", +] + +[[package]] +name = "sha3" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc9bad02c26382724b2d2692c6f179285e4b54eeecd7968f52a50059c3c11759" +dependencies = [ + "digest 0.11.3", + "keccak", + "sponge-cursor", ] [[package]] @@ -4771,6 +6084,16 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d567dcbaf0049cb8ac2608a76cd95ff9e4412e1899d389ee400918ca7537f5" +dependencies = [ + "digest 0.11.3", + "rand_core 0.10.1", +] + [[package]] name = "simd-adler32" version = "0.3.10" @@ -4859,12 +6182,116 @@ dependencies = [ "system-deps", ] +[[package]] +name = "spin" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" + +[[package]] +name = "spki" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d9efca8738c78ee9484207732f728b1ef517bbb1833d6fc0879ca898a522f6f" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "sponge-cursor" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a0219bd7d979d58245a4f41f695e1ac9f8befdffadd7f61f1bae9e39abc6620" + +[[package]] +name = "sqlparser" +version = "0.62.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c6d1b651dc4edf07eead2a0c6c78016ce971bc2c10da5266861b13f25e7cec" +dependencies = [ + "log", + "recursive", +] + +[[package]] +name = "ssh-cipher" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d801accda99469cde6d73da741422610fdf6508a72d9a69d1b55cb241c720597" +dependencies = [ + "aead 0.6.1", + "aes 0.9.2", + "aes-gcm 0.11.0", + "chacha20", + "cipher 0.5.2", + "ctutils", + "des", + "poly1305", + "ssh-encoding", + "zeroize", +] + +[[package]] +name = "ssh-encoding" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b54d0ed0498daf3f78d82e00e28c8eec9d75a067c4cfbcc7a0f7d0f4077749e" +dependencies = [ + "base64ct", + "bytes", + "crypto-bigint", + "ctutils", + "digest 0.11.3", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "ssh-key" +version = "0.7.0-rc.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9a32fae177b74a22aa9c5b01bf7e68b33545be32d9e381e248058d2adc15ce3" +dependencies = [ + "argon2", + "bcrypt-pbkdf", + "ctutils", + "ed25519-dalek", + "hex", + "hmac 0.13.0", + "p256", + "p384", + "p521", + "rand_core 0.10.1", + "rsa", + "sec1", + "sha1 0.11.0", + "sha2 0.11.0", + "signature", + "ssh-cipher", + "ssh-encoding", + "zeroize", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "stacker" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707f49d46706bacf8a2b00d51dace3f9de527c13eec3778f570c411f89e69967" +dependencies = [ + "cc", + "cfg-if", + "libc", + "psm", + "windows-sys 0.61.2", +] + [[package]] name = "static_assertions" version = "1.1.0" @@ -5040,7 +6467,7 @@ dependencies = [ "tao-macros", "unicode-segmentation", "url", - "windows", + "windows 0.61.3", "windows-core 0.61.2", "windows-version", ] @@ -5110,7 +6537,7 @@ dependencies = [ "webkit2gtk", "webview2-com", "window-vibrancy", - "windows", + "windows 0.61.3", ] [[package]] @@ -5151,7 +6578,7 @@ dependencies = [ "semver", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "syn 2.0.119", "tauri-utils", "thiserror 2.0.19", @@ -5175,6 +6602,84 @@ dependencies = [ "tauri-utils", ] +[[package]] +name = "tauri-plugin" +version = "2.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74be5dd4bed9afbd145e5716b5fa2ec28cbc29c34ffa61c258c9273d896c8020" +dependencies = [ + "anyhow", + "glob", + "plist", + "schemars 0.8.22", + "serde", + "serde_json", + "tauri-utils", + "walkdir", +] + +[[package]] +name = "tauri-plugin-dialog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0beee42a4002bc695550599b011728d9dfabf82f767f134754ed6655e434824e" +dependencies = [ + "log", + "raw-window-handle", + "rfd", + "serde", + "serde_json", + "tauri", + "tauri-plugin", + "tauri-plugin-fs", + "thiserror 2.0.19", + "url", +] + +[[package]] +name = "tauri-plugin-fs" +version = "2.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47df422695255ecbe7bac7012440eddaeefd026656171eac9559f5243d3230d9" +dependencies = [ + "anyhow", + "dunce", + "glob", + "percent-encoding", + "schemars 0.8.22", + "serde", + "serde_json", + "serde_repr", + "tauri", + "tauri-plugin", + "tauri-utils", + "thiserror 2.0.19", + "toml 0.9.12+spec-1.1.0", + "url", +] + +[[package]] +name = "tauri-plugin-opener" +version = "2.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66644b71a31ec1a8a52c4a16575edd28cf763c87cf4a7da24c884122b5c77097" +dependencies = [ + "dunce", + "glob", + "objc2-app-kit", + "objc2-foundation", + "open", + "schemars 0.8.22", + "serde", + "serde_json", + "tauri", + "tauri-plugin", + "thiserror 2.0.19", + "url", + "windows 0.61.3", + "zbus 5.18.0", +] + [[package]] name = "tauri-runtime" version = "2.9.2" @@ -5197,7 +6702,7 @@ dependencies = [ "url", "webkit2gtk", "webview2-com", - "windows", + "windows 0.61.3", ] [[package]] @@ -5223,7 +6728,7 @@ dependencies = [ "url", "webkit2gtk", "webview2-com", - "windows", + "windows 0.61.3", "wry", ] @@ -5436,6 +6941,7 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", + "tracing", "windows-sys 0.61.2", ] @@ -5460,6 +6966,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-stream" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.19" @@ -5729,6 +7246,24 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "trash" +version = "5.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7602e0c7d66ec2d92a8c917219fbc7894039efa2063b9064260110828a356f46" +dependencies = [ + "chrono", + "libc", + "log", + "objc2", + "objc2-foundation", + "once_cell", + "percent-encoding", + "scopeguard", + "urlencoding", + "windows 0.56.0", +] + [[package]] name = "try-lock" version = "0.2.5" @@ -5829,10 +7364,20 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "subtle", ] +[[package]] +name = "universal-hash" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4987bdc12753382e0bec4a65c50738ffaabc998b9cdd1f952fb5f39b0048a96" +dependencies = [ + "crypto-common 0.2.2", + "ctutils", +] + [[package]] name = "untrusted" version = "0.9.0" @@ -5852,6 +7397,12 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "urlpattern" version = "0.3.0" @@ -6083,6 +7634,66 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wayland-backend" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "016ccf01d1c58b6f8999612813e17c9b2390f7d70671428869913310f83f54b8" +dependencies = [ + "cc", + "downcast-rs", + "rustix", + "scoped-tls", + "smallvec", + "wayland-sys", +] + +[[package]] +name = "wayland-client" +version = "0.31.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c36a0f861ad76d0901f2800b46321410d9f73f2ea88aac0650d86c32688073" +dependencies = [ + "bitflags 2.13.1", + "rustix", + "wayland-backend", + "wayland-scanner", +] + +[[package]] +name = "wayland-protocols" +version = "0.32.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23d0c813de3daa2ed6520af85a3bd49b0e722a3078506899aa9686fea58dc4b6" +dependencies = [ + "bitflags 2.13.1", + "wayland-backend", + "wayland-client", + "wayland-scanner", +] + +[[package]] +name = "wayland-scanner" +version = "0.31.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "338e30461b3a2b67d70eb30a6d89f8e0c93a833e07d2ae89085cd070c4a00ac0" +dependencies = [ + "proc-macro2", + "quick-xml 0.41.0", + "quote", +] + +[[package]] +name = "wayland-sys" +version = "0.31.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8eab23fefc9e41f8e841df4a9c707e8a8c4ed26e944ef69297184de2785e3be" +dependencies = [ + "dlib", + "log", + "pkg-config", +] + [[package]] name = "web-sys" version = "0.3.103" @@ -6176,10 +7787,10 @@ checksum = "7130243a7a5b33c54a444e54842e6a9e133de08b5ad7b5861cd8ed9a6a5bc96a" dependencies = [ "webview2-com-macros", "webview2-com-sys", - "windows", + "windows 0.61.3", "windows-core 0.61.2", - "windows-implement", - "windows-interface", + "windows-implement 0.60.2", + "windows-interface 0.59.3", ] [[package]] @@ -6200,7 +7811,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "381336cfffd772377d291702245447a5251a2ffa5bad679c99e61bc48bacbf9c" dependencies = [ "thiserror 2.0.19", - "windows", + "windows 0.61.3", "windows-core 0.61.2", ] @@ -6250,17 +7861,39 @@ dependencies = [ "windows-version", ] +[[package]] +name = "windows" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1de69df01bdf1ead2f4ac895dc77c9351aefff65b2f3db429a343f9cbf05e132" +dependencies = [ + "windows-core 0.56.0", + "windows-targets 0.52.6", +] + [[package]] name = "windows" version = "0.61.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" dependencies = [ - "windows-collections", + "windows-collections 0.2.0", "windows-core 0.61.2", - "windows-future", + "windows-future 0.2.1", "windows-link 0.1.3", - "windows-numerics", + "windows-numerics 0.2.0", +] + +[[package]] +name = "windows" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" +dependencies = [ + "windows-collections 0.3.2", + "windows-core 0.62.2", + "windows-future 0.3.2", + "windows-numerics 0.3.1", ] [[package]] @@ -6272,14 +7905,35 @@ dependencies = [ "windows-core 0.61.2", ] +[[package]] +name = "windows-collections" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" +dependencies = [ + "windows-core 0.62.2", +] + +[[package]] +name = "windows-core" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4698e52ed2d08f8658ab0c39512a7c00ee5fe2688c65f8c0a4f06750d729f2a6" +dependencies = [ + "windows-implement 0.56.0", + "windows-interface 0.56.0", + "windows-result 0.1.2", + "windows-targets 0.52.6", +] + [[package]] name = "windows-core" version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" dependencies = [ - "windows-implement", - "windows-interface", + "windows-implement 0.60.2", + "windows-interface 0.59.3", "windows-link 0.1.3", "windows-result 0.3.4", "windows-strings 0.4.2", @@ -6291,8 +7945,8 @@ version = "0.62.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" dependencies = [ - "windows-implement", - "windows-interface", + "windows-implement 0.60.2", + "windows-interface 0.59.3", "windows-link 0.2.1", "windows-result 0.4.1", "windows-strings 0.5.1", @@ -6306,7 +7960,29 @@ checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" dependencies = [ "windows-core 0.61.2", "windows-link 0.1.3", - "windows-threading", + "windows-threading 0.1.0", +] + +[[package]] +name = "windows-future" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" +dependencies = [ + "windows-core 0.62.2", + "windows-link 0.2.1", + "windows-threading 0.2.1", +] + +[[package]] +name = "windows-implement" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6fc35f58ecd95a9b71c4f2329b911016e6bec66b3f2e6a4aad86bd2e99e2f9b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", ] [[package]] @@ -6320,6 +7996,17 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "windows-interface" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08990546bf4edef8f431fa6326e032865f27138718c587dc21bc0265bbcb57cc" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "windows-interface" version = "0.59.3" @@ -6353,6 +8040,25 @@ dependencies = [ "windows-link 0.1.3", ] +[[package]] +name = "windows-numerics" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" +dependencies = [ + "windows-core 0.62.2", + "windows-link 0.2.1", +] + +[[package]] +name = "windows-result" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-result" version = "0.3.4" @@ -6491,6 +8197,15 @@ dependencies = [ "windows-link 0.1.3", ] +[[package]] +name = "windows-threading" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" +dependencies = [ + "windows-link 0.2.1", +] + [[package]] name = "windows-version" version = "0.1.7" @@ -6672,12 +8387,33 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "winx" +version = "0.36.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f3fd376f71958b862e7afb20cfe5a22830e1963462f3a17f49d82a6c1d1f42d" +dependencies = [ + "bitflags 2.13.1", + "windows-sys 0.59.0", +] + [[package]] name = "wit-bindgen" version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" +[[package]] +name = "wnaf" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab12e7090f27e2ffd9322651492942d50c2926094af30601e1964337db39daf1" +dependencies = [ + "ff", + "group", + "hybrid-array", +] + [[package]] name = "writeable" version = "0.6.3" @@ -6714,7 +8450,7 @@ dependencies = [ "once_cell", "percent-encoding", "raw-window-handle", - "sha2", + "sha2 0.10.9", "soup3", "tao-macros", "thiserror 2.0.19", @@ -6722,7 +8458,7 @@ dependencies = [ "webkit2gtk", "webkit2gtk-sys", "webview2-com", - "windows", + "windows 0.61.3", "windows-core 0.61.2", "windows-version", ] @@ -6737,6 +8473,23 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "xls" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c27cbcc90bd3ef2bb39712816a3a15c1aa33917c6093b0e1306e3bc3e2cafc9" +dependencies = [ + "anyhow", + "cfb 0.14.0", + "chrono", + "csv", + "encoding_rs", + "quick-xml 0.37.5", + "regex", + "thiserror 2.0.19", + "zip", +] + [[package]] name = "yoke" version = "0.8.3" @@ -6776,20 +8529,56 @@ dependencies = [ "futures-sink", "futures-util", "hex", - "nix", + "nix 0.29.0", "ordered-stream", "rand 0.8.7", "serde", "serde_repr", - "sha1", + "sha1 0.10.7", "static_assertions", "tracing", "uds_windows", "windows-sys 0.52.0", "xdg-home", - "zbus_macros", - "zbus_names", - "zvariant", + "zbus_macros 4.4.0", + "zbus_names 3.0.0", + "zvariant 4.2.0", +] + +[[package]] +name = "zbus" +version = "5.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe18fb60dc696039e738717b76eaea21e7a4489bbb1885020b43c94236d7e98a" +dependencies = [ + "async-broadcast", + "async-executor", + "async-io", + "async-lock", + "async-process", + "async-recursion", + "async-task", + "async-trait", + "blocking", + "enumflags2", + "event-listener", + "futures-core", + "futures-lite", + "hex", + "libc", + "ordered-stream", + "rustix", + "serde", + "serde_repr", + "tokio", + "tracing", + "uds_windows", + "uuid", + "windows-sys 0.61.2", + "winnow 1.0.4", + "zbus_macros 5.18.0", + "zbus_names 4.3.4", + "zvariant 5.13.1", ] [[package]] @@ -6802,7 +8591,22 @@ dependencies = [ "proc-macro2", "quote", "syn 2.0.119", - "zvariant_utils", + "zvariant_utils 2.1.0", +] + +[[package]] +name = "zbus_macros" +version = "5.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe96480bed92df2b442a1a30df364e12d08eed03aeb061f2b8dc6afb2be91119" +dependencies = [ + "proc-macro-crate 3.5.0", + "proc-macro2", + "quote", + "syn 2.0.119", + "zbus_names 4.3.4", + "zvariant 5.13.1", + "zvariant_utils 3.5.0", ] [[package]] @@ -6813,7 +8617,18 @@ checksum = "4b9b1fef7d021261cc16cba64c351d291b715febe0fa10dc3a443ac5a5022e6c" dependencies = [ "serde", "static_assertions", - "zvariant", + "zvariant 4.2.0", +] + +[[package]] +name = "zbus_names" +version = "4.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8bf88b4a3ff53e883001e0e0115b297a9d53c31b9c1edd2bfdd853e3428624e" +dependencies = [ + "serde", + "winnow 1.0.4", + "zvariant 5.13.1", ] [[package]] @@ -6910,12 +8725,44 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "zip" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa8cd6af31c3b31c6631b8f483848b91589021b28fffe50adada48d4f4d2ed1" +dependencies = [ + "arbitrary", + "crc32fast", + "flate2", + "indexmap 2.14.0", + "memchr", + "zopfli", +] + +[[package]] +name = "zlib-rs" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b142a20ec14a91d5bc708c1dc21b080c550113d8aa77afa29635673a65dd02c5" + [[package]] name = "zmij" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] + [[package]] name = "zvariant" version = "4.2.0" @@ -6926,7 +8773,22 @@ dependencies = [ "enumflags2", "serde", "static_assertions", - "zvariant_derive", + "zvariant_derive 4.2.0", +] + +[[package]] +name = "zvariant" +version = "5.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee2a0bcd2a907786a456fff45aaaaf54c9ba5f50b71ae9ec1a4edd200c94911" +dependencies = [ + "endi", + "enumflags2", + "serde", + "url", + "winnow 1.0.4", + "zvariant_derive 5.13.1", + "zvariant_utils 3.5.0", ] [[package]] @@ -6939,7 +8801,20 @@ dependencies = [ "proc-macro2", "quote", "syn 2.0.119", - "zvariant_utils", + "zvariant_utils 2.1.0", +] + +[[package]] +name = "zvariant_derive" +version = "5.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38a708216a18780796770bfe3f4739c7c83a3e8f789b755534bbbc06e4e23e12" +dependencies = [ + "proc-macro-crate 3.5.0", + "proc-macro2", + "quote", + "syn 2.0.119", + "zvariant_utils 3.5.0", ] [[package]] @@ -6952,3 +8827,16 @@ dependencies = [ "quote", "syn 2.0.119", ] + +[[package]] +name = "zvariant_utils" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90cb9383f9b45290407a1258b202d3f8f01db719eb60b4e4055c6375af4fc7c7" +dependencies = [ + "proc-macro2", + "quote", + "serde", + "syn 2.0.119", + "winnow 1.0.4", +] diff --git a/Cargo.toml b/Cargo.toml index 8ec4305..04cdc2c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,35 +25,46 @@ repository = "https://github.com/OtterMind/Chat2DB-Rust" [workspace.dependencies] aes-gcm = { version = "0.10.3", features = ["std", "zeroize"] } +aes = "0.8.4" async-trait = "0.1" -axum = "0.8.9" +axum = { version = "0.8.9", features = ["multipart"] } base64 = "0.22.1" +blowfish = "0.9.1" bytes = "1" chrono = "0.4.45" +cbc = "0.1.2" clap = { version = "4.5", features = ["derive"] } +csv = "1.4.0" directories = "6.0" eventsource-stream = "0.2.3" fs2 = "0.4" futures-util = "0.3" http-body-util = "0.1" +hex = "0.4.3" keyring = { version = "3.6.3", default-features = false } mysql_async = { version = "=0.37.0", default-features = false, features = ["default-rustls-ring"] } prost = "0.14" +quick-xml = "0.37.5" rand = "0.9.2" reqwest = { version = "0.12.24", default-features = false, features = ["json", "rustls-tls", "stream"] } -rmcp = { version = "=2.2.0", default-features = false, features = ["server", "macros", "transport-io"] } +rmcp = { version = "=2.2.0", default-features = false, features = ["server", "macros", "transport-io", "elicitation", "schemars"] } rusqlite = { version = "0.37", features = ["bundled"] } +russh = { version = "0.62.5", default-features = false, features = ["ring", "rsa"] } rustix = { version = "1", features = ["fs", "process"] } serde = { version = "1", features = ["derive"] } serde_json = "1" +sha1 = "0.10.6" sha2 = "0.10" +sqlparser = "0.62.0" subtle = "2.6" tauri = { version = "=2.8.5", default-features = false } tauri-build = "=2.4.1" +tauri-plugin-dialog = "=2.4.0" +tauri-plugin-opener = "=2.2.7" thiserror = "2" -tokio = { version = "1", features = ["io-std", "io-util", "macros", "net", "process", "rt-multi-thread", "signal", "sync", "time"] } +tokio = { version = "1", features = ["fs", "io-std", "io-util", "macros", "net", "process", "rt-multi-thread", "signal", "sync", "time"] } tokio-stream = { version = "0.1", features = ["sync"] } -tokio-util = { version = "0.7.16", features = ["rt"] } +tokio-util = { version = "0.7.16", features = ["io", "rt"] } tower = { version = "0.5", features = ["util"] } tower-http = { version = "0.7", features = ["fs", "trace"] } tracing = "0.1" @@ -62,7 +73,9 @@ utoipa = { version = "5.5.0", features = ["axum_extras"] } utoipa-axum = "0.2.0" url = "2" uuid = { version = "1.18", features = ["v4"] } +xls = { version = "0.1.0", default-features = false } zeroize = "1.8" +zip = { version = "4", default-features = false, features = ["deflate"] } [workspace.lints.rust] unsafe_code = "forbid" diff --git a/Makefile b/Makefile index 877b725..9481d63 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,9 @@ .PHONY: verify rust rust-process-tests java ipc-integration jdbc-h2-integration \ community-h2-classpath community-h2-reproducibility community-java-h2-integration \ community-h2-integration \ - community-product-h2-integration product-h2-integration mysql-driver-pack \ - native-mysql-integration community-product-mysql-integration \ + community-product-h2-integration product-h2-integration mysql-driver-pack h2-driver-pack \ + native-mysql-integration native-mysql-direct-integration native-mysql-ssh-integration \ + community-product-mysql-integration \ frontend-deps frontend-source frontend desktop generate-contracts check-contracts \ macos-runtime macos-package-java macos-package macos-package-verify @@ -44,7 +45,7 @@ community-h2-reproducibility: community-java-h2-integration: java community-h2-classpath cd java && \ CHAT2DB_COMMUNITY_CLASSPATH_DIR="$(COMMUNITY_CLASSPATH_DIR)" \ - CHAT2DB_COMMUNITY_SOURCE_COMMIT="37a34be858f2566b6b7fcf6c3f64183c1f560853" \ + CHAT2DB_COMMUNITY_SOURCE_COMMIT="3cb8af54cad5bd5caa20bb25f10d9b0e4f01931c" \ ./mvnw -B -pl compat-runtime \ -Dtest='CommunityPluginRegistryTest#realCommunityH2BuildsAndExecutesBoundedDml,CommunityPluginRegistryTest#realCommunityMysqlRejectsBackslashCrossColumnInjection,CommunityPluginRegistryTest#realCommunityMysqlNormalizesBooleanAliasesAndBits,CommunityPluginRegistryTest#realCommunityH2BuildsNamespaceSqlWithoutOpeningJdbc,CommunityPluginRegistryTest#realCommunityMysqlBuildsDatabaseNamespaceSql,CommunityPluginRegistryTest#realCommunityMysqlBuildsBoundedTablePreviewSqlWithoutOpeningJdbc,CommunityPluginRegistryTest#realCommunityNamespaceMapsUnsupportedAndRejectsOversizedInput,JdbcProtocolLoopTest#communityDmlDispatchDoesNotRequireAJdbcSession,JdbcProtocolLoopTest#communityNamespaceDispatchDoesNotRequireAJdbcSession' \ test @@ -61,7 +62,12 @@ product-h2-integration: java mysql-driver-pack: ./scripts/prepare-mysql-driver-pack.sh "$(MYSQL_DRIVER_PACK_DIR)" -native-mysql-integration: +h2-driver-pack: + ./scripts/prepare-h2-driver-pack.sh "$(MYSQL_DRIVER_PACK_DIR)" + +native-mysql-integration: native-mysql-direct-integration native-mysql-ssh-integration + +native-mysql-direct-integration: @test -n "$(MYSQL_TEST_USER)" || (echo "MYSQL_TEST_USER is required" >&2; exit 1) @test -n "$(MYSQL_TEST_PASSWORD)" || (echo "MYSQL_TEST_PASSWORD is required" >&2; exit 1) @MYSQL_TEST_HOST="$(MYSQL_TEST_HOST)" \ @@ -69,18 +75,77 @@ native-mysql-integration: MYSQL_TEST_USER="$(MYSQL_TEST_USER)" \ MYSQL_TEST_PASSWORD="$(MYSQL_TEST_PASSWORD)" \ MYSQL_TEST_REQUIRED="1" \ + cargo test -p chat2db-core --lib --locked \ + live_mysql_console_kernel_preserves_session_results_and_cancellation -- --ignored + @MYSQL_TEST_HOST="$(MYSQL_TEST_HOST)" \ + MYSQL_TEST_PORT="$(MYSQL_TEST_PORT)" \ + MYSQL_TEST_USER="$(MYSQL_TEST_USER)" \ + MYSQL_TEST_PASSWORD="$(MYSQL_TEST_PASSWORD)" \ + MYSQL_TEST_REQUIRED="1" \ cargo test -p chat2db-core --test native_mysql_product --locked @MYSQL_TEST_HOST="$(MYSQL_TEST_HOST)" \ MYSQL_TEST_PORT="$(MYSQL_TEST_PORT)" \ MYSQL_TEST_USER="$(MYSQL_TEST_USER)" \ MYSQL_TEST_PASSWORD="$(MYSQL_TEST_PASSWORD)" \ + MYSQL_TEST_REQUIRED="1" \ + cargo test -p chat2db-local --test native_mysql_write_docker --locked + @MYSQL_TEST_HOST="$(MYSQL_TEST_HOST)" \ + MYSQL_TEST_PORT="$(MYSQL_TEST_PORT)" \ + MYSQL_TEST_USER="$(MYSQL_TEST_USER)" \ + MYSQL_TEST_PASSWORD="$(MYSQL_TEST_PASSWORD)" \ cargo test -p chat2db-core --test native_mysql_console_docker --locked -- --ignored @MYSQL_TEST_HOST="$(MYSQL_TEST_HOST)" \ MYSQL_TEST_PORT="$(MYSQL_TEST_PORT)" \ MYSQL_TEST_USER="$(MYSQL_TEST_USER)" \ MYSQL_TEST_PASSWORD="$(MYSQL_TEST_PASSWORD)" \ + MYSQL_TEST_REQUIRED="1" \ + cargo test -p chat2db-core --test native_mysql_account_docker --locked + @MYSQL_TEST_HOST="$(MYSQL_TEST_HOST)" \ + MYSQL_TEST_PORT="$(MYSQL_TEST_PORT)" \ + MYSQL_TEST_USER="$(MYSQL_TEST_USER)" \ + MYSQL_TEST_PASSWORD="$(MYSQL_TEST_PASSWORD)" \ + MYSQL_TEST_REQUIRED="1" \ + cargo test -p chat2db-core --test native_mysql_schema_diff_docker --locked + @MYSQL_TEST_HOST="$(MYSQL_TEST_HOST)" \ + MYSQL_TEST_PORT="$(MYSQL_TEST_PORT)" \ + MYSQL_TEST_USER="$(MYSQL_TEST_USER)" \ + MYSQL_TEST_PASSWORD="$(MYSQL_TEST_PASSWORD)" \ + MYSQL_TEST_REQUIRED="1" \ + cargo test -p chat2db-core --test native_mysql_transfer_docker --locked + @MYSQL_TEST_HOST="$(MYSQL_TEST_HOST)" \ + MYSQL_TEST_PORT="$(MYSQL_TEST_PORT)" \ + MYSQL_TEST_USER="$(MYSQL_TEST_USER)" \ + MYSQL_TEST_PASSWORD="$(MYSQL_TEST_PASSWORD)" \ + MYSQL_TEST_REQUIRED="1" \ + cargo test -p chat2db-core --test native_mysql_dashboard_docker --locked + @MYSQL_TEST_HOST="$(MYSQL_TEST_HOST)" \ + MYSQL_TEST_PORT="$(MYSQL_TEST_PORT)" \ + MYSQL_TEST_USER="$(MYSQL_TEST_USER)" \ + MYSQL_TEST_PASSWORD="$(MYSQL_TEST_PASSWORD)" \ + MYSQL_TEST_REQUIRED="1" \ cargo test -p chat2db-web --test native_mysql_editable_ddl_docker --locked -- --ignored +native-mysql-ssh-integration: + @test -n "$(CHAT2DB_TEST_SSH_HOST)" || (echo "CHAT2DB_TEST_SSH_HOST is required" >&2; exit 1) + @test -n "$(CHAT2DB_TEST_SSH_PORT)" || (echo "CHAT2DB_TEST_SSH_PORT is required" >&2; exit 1) + @test -n "$(CHAT2DB_TEST_SSH_USER)" || (echo "CHAT2DB_TEST_SSH_USER is required" >&2; exit 1) + @test -n "$(CHAT2DB_TEST_SSH_LOCAL_PORT)" || (echo "CHAT2DB_TEST_SSH_LOCAL_PORT is required" >&2; exit 1) + @test -n "$(CHAT2DB_TEST_SSH_PASSWORD)$(CHAT2DB_TEST_SSH_PRIVATE_KEY)" || (echo "CHAT2DB_TEST_SSH_PASSWORD or CHAT2DB_TEST_SSH_PRIVATE_KEY is required" >&2; exit 1) + @test -n "$(MYSQL_TEST_USER)" || (echo "MYSQL_TEST_USER is required" >&2; exit 1) + @test -n "$(MYSQL_TEST_PASSWORD)" || (echo "MYSQL_TEST_PASSWORD is required" >&2; exit 1) + @CHAT2DB_TEST_MYSQL_HOST="$(MYSQL_TEST_HOST)" \ + CHAT2DB_TEST_MYSQL_PORT="$(MYSQL_TEST_PORT)" \ + CHAT2DB_TEST_MYSQL_USER="$(MYSQL_TEST_USER)" \ + CHAT2DB_TEST_MYSQL_PASSWORD="$(MYSQL_TEST_PASSWORD)" \ + CHAT2DB_TEST_SSH_HOST="$(CHAT2DB_TEST_SSH_HOST)" \ + CHAT2DB_TEST_SSH_PORT="$(CHAT2DB_TEST_SSH_PORT)" \ + CHAT2DB_TEST_SSH_USER="$(CHAT2DB_TEST_SSH_USER)" \ + CHAT2DB_TEST_SSH_PASSWORD="$(CHAT2DB_TEST_SSH_PASSWORD)" \ + CHAT2DB_TEST_SSH_PRIVATE_KEY="$(CHAT2DB_TEST_SSH_PRIVATE_KEY)" \ + CHAT2DB_TEST_SSH_PRIVATE_KEY_PASSPHRASE="$(CHAT2DB_TEST_SSH_PRIVATE_KEY_PASSPHRASE)" \ + CHAT2DB_TEST_SSH_LOCAL_PORT="$(CHAT2DB_TEST_SSH_LOCAL_PORT)" \ + cargo test -p chat2db-core --test native_mysql_ssh_tunnel_docker --locked -- --ignored + community-product-mysql-integration: java community-h2-classpath mysql-driver-pack @test -n "$(MYSQL_TEST_USER)" || (echo "MYSQL_TEST_USER is required" >&2; exit 1) @test -n "$(MYSQL_TEST_PASSWORD)" || (echo "MYSQL_TEST_PASSWORD is required" >&2; exit 1) @@ -120,7 +185,7 @@ macos-runtime: macos-package-java: community-h2-classpath $(MAKE) java -macos-package: macos-package-java mysql-driver-pack frontend macos-runtime +macos-package: macos-package-java mysql-driver-pack h2-driver-pack frontend macos-runtime ./scripts/build-macos-package.sh macos-package-verify: diff --git a/README.md b/README.md index 7c0d23a..3ffb97d 100644 --- a/README.md +++ b/README.md @@ -27,8 +27,8 @@ git submodule update --init --recursive ## Current state The repository has completed Stages 1 through 6, the first thirteen -independently buildable Stage 7 slices, and a native end-user Community -Console compatibility slice: +independently buildable Stage 7 slices, and the complete MySQL workbench +surface reached by the pinned Community frontend: - canonical Rust API contracts; - a transport-neutral Rust application service root; @@ -61,10 +61,11 @@ Console compatibility slice: Tauri 2 commands/channels; - a checked-in OpenAPI contract with generated TypeScript types and drift verification; and -- the pinned original Community Umi/React layout and components, served by +- the original Community Umi/React layout, components, and styles, served by Axum over historical HTTP routes on Web and bridged from `window.javaQuery` - to Tauri IPC on desktop without a replacement UI or style fork; the pinned - source carries one CSP-safe callback-cloning compatibility fix; + to Tauri IPC on desktop without a replacement UI or style fork; a locked, + reviewable host-adapter patch supplies CSP-safe callbacks plus Web/Desktop + file upload and download transport compatibility; - a provider-neutral bounded agent loop with direct OpenAI, Anthropic, and Gemini adapters, durable sessions/messages/runs/permissions, and atomic context compaction; @@ -77,9 +78,12 @@ Console compatibility slice: frontend HTTP/Tauri observers; - an authenticated owner-only local attachment started by both product hosts, plus a JSON CLI for datasource discovery, forced-read-only query lifecycle, - cancellation, and retained-result paging; and -- an `rmcp` 2.2 stdio server with five bounded datasource/query tools backed by - that same running `Application`; + cancellation, retained-result paging, and explicitly confirmed MySQL writes; + and +- an `rmcp` 2.2 stdio server with six bounded datasource/query tools, including + a MySQL write tool backed by that same running `Application`; writes require + protocol-level Form elicitation from the trusted client, while model-visible + tool arguments expose neither `confirm` nor an approval token; - strict local JDBC driver-pack discovery, hash verification, immutable Core/Axum/Tauri inventory, and repeatable per-generation preload; and - a fixed Community 5.3.0 compatibility classpath that discovers real @@ -111,7 +115,14 @@ Console compatibility slice: while desktop preserves the original JCEF correlation envelope through one `legacy_request` Tauri command; and - forced-read-only table preview that ignores caller SQL, generates a bounded - SELECT through the selected Community plugin, and pages retained results. + SELECT through the selected Community plugin, and pages retained results; +- complete native MySQL datasource lifecycle, SSH tunneling, portability, + metadata, editable DML and DDL, views and routines, transfer tasks, account + administration, schema diff, pins, ER layout, workspace persistence, and + SQLite-backed Dashboard/Chart CRUD with native read-only chart refresh; and +- the pinned Community AI workspace routes plus confirmed Agent, CLI, and MCP + writes, with explicit approval, read-only enforcement, single-statement + validation, and conservative unknown-outcome handling. Runtime-tested: yes. The Stage 7M product vertical passed against a real local MySQL 8.4 server on 2026-07-27, including plugin-built qualified table SQL, @@ -129,8 +140,8 @@ the restored SQL successfully. On 2026-07-29 commits `81301c3`, `4199862`, and vertical covering native connection, database/schema/table discovery, preview, typed Console SELECT, row truncation, active-query cancellation, retained paging, and proof after every operation that Java remained dormant. -The complete repository `make verify` gate and the explicit real-MySQL -`native-mysql-integration` target also passed after the final compatibility fix. +The complete repository `make verify` gate and the explicit real-MySQL direct +and SSH integration targets also passed after the final compatibility fix. The metadata parity increment adds a real MySQL fixture with a foreign key, composite index, view, function, procedure, and trigger; its Core product test, Axum queries, and desktop dispatch contracts pass while Java remains dormant. @@ -142,64 +153,43 @@ default. The native Console integration additionally passes against MySQL 8.4 for DDL/DML, `DELIMITER` procedures, multi-results, transactions, error continuation, cancellation, a 6 MiB `LONGTEXT`, `single`, `EXPLAIN`, `pageSizeAll`, and datasource read-only protection while Java remains dormant. - -Stage 6 is complete. Web and desktop own the product runtime and publish its -owner-only local endpoint; CLI and MCP attach to that host and never contact -Java directly. The current MCP surface is deliberately read-only and does not -accept JDBC bind parameters. A complete end-user Agent workspace and -CLI-started headless host remain follow-on product work. Stage 7A implements -strict local driver packs. Stage 7B pins Community source, loads its runtime in -an isolated Java classloader, and proves one real H2 SPI/ANTLR vertical slice. -Stage 7C composes those four operations into the product Core and both delivery -transports. Stage 7D adds database, table, column, and index metadata through a -separate capability. Stage 7E adds views, imported and exported foreign keys, -and primary keys through another capability. Stage 7F adds functions, -function parameters, procedures, procedure parameters, and triggers through the -same Core/Axum/Tauri/frontend boundary. Stage 7G connects all 20 fixed Community -operations to the shared React workbench through a three-pane object explorer, -partial long-tail metadata, lazy detail views, schema SQL generation, and -explicit SQL analysis. Stage 7H adds a separately negotiated SQL-validation -capability and an explicit editor Validate action without opening a JDBC -session. Stage 7I adds separately negotiated SQL formatting through the retained -Community formatter dependency, preserves Community's dialect mapping and -fallback behavior, and replaces editor SQL only while the originating SQL, -datasource, and database type are still current. Stage 7J adds the separately -negotiated `community.sql-completion.v1` capability, calls the real Community -completion service against the existing read-only JDBC session, and exposes -bounded suggestions through Core, Axum, Tauri, and the shared React editor. -Stage 7K adds the separately negotiated `community.dml-builder.v1` capability, -calls the selected plugin's real DML, value, and identifier processors without -opening a JDBC session, and exposes typed INSERT/UPDATE generation through the -same product boundary and table detail UI. -Stage 7L adds `community.namespace-builder.v1`, retains the old CREATE SCHEMA -contract, and exposes a closed database/schema DDL union through Core, Axum, -Tauri, and the shared frontend. Stage 7M adds `community.dql-builder.v1` at tag -`225`: Java uses the selected plugin to quote the database/schema/table name and -build a row-limited SELECT without opening JDBC, then Rust validates that SQL and -executes it through the existing forced-read-only query and retained-result -path. The fixed 149-JAR classpath keeps H2 and MySQL; PostgreSQL and other -dialects do not block the MySQL preview. Agent, CLI, and MCP MySQL conformance -remain outside this milestone. The current MySQL connection, object metadata, -preview, and Console data plane dispatch to `mysql_async` before Java lease -acquisition; Community parser, formatter, completion, and builders remain -Java-backed. +The Dashboard/Chart integration additionally passed against MySQL 8.4 with a +selected database, a 200-row response cap, SELECT CTE support, response-only +refreshed metadata with Community primary-key/auto-increment/nullability/default +and comment headers, rejected writes/multi-statements/locking reads/server-file +output, `CHART` operation history, fixture cleanup, and Java dormant. +The complete `rtk make verify` gate then passed with the Dashboard/Chart +increment included. + +Stage 6 and the Stage 7A-7M foundations are complete. Web and desktop own the +product runtime and publish its owner-only local endpoint; CLI and MCP attach to +that host and never contact Java directly. The pinned Community frontend's +complete MySQL workbench surface is mapped through the shared Axum/Tauri legacy +dispatcher. Native MySQL connections, metadata, Console, mutations, transfer, +class generation, accounts, schema diff, chart refresh, and workspace operations +remain in Rust and do not acquire a Java lease. Dashboard and chart documents +remain in SQLite. Community parser, formatter, completion, SQL-builder, and +exact plugin compatibility operations remain Java-backed and start the +supervised process only on demand. The Console compatibility path uses SQLite migrations 3 and 4 for saved Consoles and durable execution history. Historical `/api/operation/saved/*`, `/api/operation/log/*`, `/api/rdb/dml/execute`, `/execute_ddl`, and large-cell routes share the same native Core execution. Desktop `sql-execute` and `sql-cancel` keep active cancellation handles and emit row payloads exactly once -through Tauri. Native bind parameters and remaining edge-case Community result -shapes are not implemented. +through Tauri. Native typed SELECT bind parameters use the MySQL prepared +protocol; the pinned Community write request has no bind-parameter field. The Stage 5 and Stage 7G through Stage 7M custom React workbench was an intermediate implementation and is no longer the product frontend. Commit `928e62c5d775d0e81d95db7fee186db756834a72` deletes that replacement UI and its styles. Current builds export the original Community frontend from the -pinned submodule; backend capabilities not yet mapped to its historical API -remain internal rather than requiring a redesigned page. -Signing, downloading, updating, rollback, the remaining compatibility estate, -and full per-dialect compatibility remain Stage 7 work. +pinned submodule plus the locked host-transport compatibility patch. Every +historical API used by its MySQL workbench is mapped; +cloud account, login, payment, subscription, invitation, notification, and +Enterprise-only features are outside this database milestone. Signing, +downloading, updating, rollback, and full compatibility for other database +dialects remain follow-on work. ## Architecture diff --git a/apps/chat2db-cli/src/main.rs b/apps/chat2db-cli/src/main.rs index 3a6ea2e..f524d87 100644 --- a/apps/chat2db-cli/src/main.rs +++ b/apps/chat2db-cli/src/main.rs @@ -1,6 +1,9 @@ use std::{env, ffi::OsString, path::PathBuf}; -use chat2db_contract::{QueryLimits, ResultPageRequest, StartQueryRequest}; +use chat2db_contract::{ + DatabaseWriteState, ExecuteDatabaseWriteRequest, QueryLimits, ResultPageRequest, + StartQueryRequest, +}; use chat2db_local::LocalClient; use clap::{Parser, Subcommand}; @@ -27,6 +30,11 @@ enum Command { #[command(subcommand)] command: QueryCommand, }, + /// Execute one explicitly confirmed `MySQL` write statement. + Write { + #[command(subcommand)] + command: WriteCommand, + }, /// Read one bounded page from a retained query result. Result { result_id: String, @@ -60,10 +68,25 @@ enum QueryCommand { Cancel { operation_id: String }, } +#[derive(Debug, Subcommand)] +enum WriteCommand { + /// Execute exactly one write. Only `not_started` is safe to retry after correction. + Execute { + #[arg(long)] + datasource_id: String, + #[arg(long)] + sql: String, + /// Explicitly confirm that this statement may change the database. + #[arg(long)] + confirm_write: bool, + }, +} + #[tokio::main] async fn main() -> Result<(), Box> { let cli = Cli::parse(); let client = local_client(cli.data_dir)?; + let mut command_succeeded = true; let output = match cli.command { Command::Status => serde_json::to_value(client.health().await?)?, @@ -98,6 +121,23 @@ async fn main() -> Result<(), Box> { serde_json::to_value(client.cancel_operation(operation_id).await?)? } }, + Command::Write { command } => match command { + WriteCommand::Execute { + datasource_id, + sql, + confirm_write, + } => { + let result = client + .execute_database_write(ExecuteDatabaseWriteRequest { + datasource_id, + sql, + confirmed: confirm_write, + }) + .await; + command_succeeded = result.state == DatabaseWriteState::Succeeded; + serde_json::to_value(result)? + } + }, Command::Result { result_id, offset, @@ -117,6 +157,9 @@ async fn main() -> Result<(), Box> { )?, }; println!("{}", serde_json::to_string_pretty(&output)?); + if !command_succeeded { + return Err(std::io::Error::other("database write did not succeed").into()); + } Ok(()) } @@ -147,7 +190,7 @@ mod tests { use clap::Parser; - use super::{Cli, Command, QueryCommand, attachment_data_dir}; + use super::{Cli, Command, QueryCommand, WriteCommand, attachment_data_dir}; #[test] fn parses_status_command() { @@ -201,6 +244,50 @@ mod tests { assert!(matches!(cli.command, Command::Result { .. })); } + #[test] + fn database_write_requires_an_explicit_confirmation_flag_value() { + let confirmed = Cli::try_parse_from([ + "chat2db", + "write", + "execute", + "--datasource-id", + "datasource-1", + "--sql", + "UPDATE items SET label = 'changed' WHERE id = 1", + "--confirm-write", + ]) + .expect("confirmed write must parse"); + assert!(matches!( + confirmed.command, + Command::Write { + command: WriteCommand::Execute { + confirm_write: true, + .. + } + } + )); + + let unconfirmed = Cli::try_parse_from([ + "chat2db", + "write", + "execute", + "--datasource-id", + "datasource-1", + "--sql", + "DELETE FROM items WHERE id = 1", + ]) + .expect("unconfirmed write parses so the runtime can fail closed"); + assert!(matches!( + unconfirmed.command, + Command::Write { + command: WriteCommand::Execute { + confirm_write: false, + .. + } + } + )); + } + #[test] fn rejects_empty_data_directory_sources() { assert!(attachment_data_dir(Some(PathBuf::new()), None).is_err()); diff --git a/apps/chat2db-desktop/Cargo.toml b/apps/chat2db-desktop/Cargo.toml index 2520237..fd026a3 100644 --- a/apps/chat2db-desktop/Cargo.toml +++ b/apps/chat2db-desktop/Cargo.toml @@ -16,17 +16,23 @@ path = "src/main.rs" tauri-build.workspace = true [dependencies] +cap-std = "4.0.2" chat2db-contract = { path = "../../crates/chat2db-contract" } chat2db-core = { path = "../../crates/chat2db-core" } chat2db-java-bridge = { path = "../../crates/chat2db-java-bridge" } chat2db-local = { path = "../../crates/chat2db-local" } chat2db-web = { path = "../chat2db-web" } +encoding_rs = "0.8.35" serde.workspace = true serde_json.workspace = true tauri = { workspace = true, features = ["wry"] } tauri-runtime = "=2.9.2" +tauri-plugin-dialog.workspace = true +tauri-plugin-opener.workspace = true tokio.workspace = true tracing.workspace = true +trash = "5.2.6" +uuid.workspace = true [dev-dependencies] tempfile = "3" diff --git a/apps/chat2db-desktop/src/legacy_files.rs b/apps/chat2db-desktop/src/legacy_files.rs new file mode 100644 index 0000000..b332d97 --- /dev/null +++ b/apps/chat2db-desktop/src/legacy_files.rs @@ -0,0 +1,1354 @@ +use std::{ + collections::HashMap, + ffi::{OsStr, OsString}, + fs, + io::{self, Read, Write}, + path::{Component, Path, PathBuf}, + process::Command, + sync::{Arc, Mutex}, +}; + +use cap_std::{ + ambient_authority, + fs::{Dir, OpenOptions}, +}; +use encoding_rs::Encoding; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +const MAX_REGISTERED_ROOTS: usize = 64; +const MAX_RELATIVE_PATH_BYTES: usize = 4 * 1024; +const MAX_PATH_COMPONENTS: usize = 128; +const MAX_CHILDREN: usize = 1_000; +const MAX_FILE_NAME_BYTES: usize = 240; +const MAX_FILE_TYPE_BYTES: usize = 32; +const MAX_TEXT_FILE_BYTES: u64 = 16 * 1024 * 1024; +const MAX_TEXT_CONTENT_BYTES: usize = 16 * 1024 * 1024; + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +#[allow(clippy::struct_field_names)] +pub(crate) struct LegacySaveFileRequest { + pub file_name: String, + pub file_content: String, + pub file_type: String, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct LegacyUpdateFileRequest { + pub file_path: String, + pub file_content: String, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct LegacyReadFileRequest { + pub path: String, + #[serde(default)] + pub charsets: Option, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct LegacyOpenSqlDirectoryRequest { + pub path: String, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct LegacySqlDirectoryPathRequest { + pub root_token: String, + #[serde(default)] + pub relative_path: String, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct LegacyCreateSqlDirectoryChildRequest { + pub root_token: String, + #[serde(default)] + pub parent_relative_path: String, + pub name: String, + #[serde(rename = "type")] + pub node_type: String, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct LegacySaveSqlDirectoryFileRequest { + pub root_token: String, + #[serde(default)] + pub parent_relative_path: String, + pub name: String, + pub content: String, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct LegacyRenameSqlDirectoryChildRequest { + pub root_token: String, + pub relative_path: String, + pub name: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct LegacySavedFile { + pub path: String, + pub size: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +#[allow(clippy::struct_excessive_bools)] +pub(crate) struct LegacySqlTreeNode { + pub key: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub root_token: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub root_path: Option, + pub name: String, + pub path: String, + pub relative_path: String, + #[serde(rename = "type")] + pub node_type: String, + pub disabled: bool, + pub sql_file: bool, + pub text_file: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub file_extension: Option, + pub has_children: bool, + pub loaded: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub children: Option>, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct LegacyCreateSqlDirectoryChildResponse { + pub created_node: LegacySqlTreeNode, + pub children: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct LegacyRenameSqlDirectoryChildResponse { + pub renamed_node: LegacySqlTreeNode, + pub parent_relative_path: String, + pub children: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct LegacyDeleteSqlDirectoryChildResponse { + pub parent_relative_path: String, + pub children: Vec, +} + +#[derive(Default)] +pub(crate) struct LegacySqlDirectoryRegistry { + roots: Mutex>>, +} + +struct LegacySqlRoot { + token: String, + canonical_path: PathBuf, + directory: Dir, + operations: Mutex<()>, +} + +impl LegacySqlDirectoryRegistry { + pub(crate) fn register_root(&self, path: &Path) -> Result { + let canonical_path = fs::canonicalize(path) + .map_err(|_| "The selected SQL directory is not available".to_owned())?; + let metadata = fs::symlink_metadata(&canonical_path) + .map_err(|_| "The selected SQL directory is not available".to_owned())?; + if !metadata.is_dir() { + return Err("The selected SQL path is not a directory".to_owned()); + } + let directory = Dir::open_ambient_dir(&canonical_path, ambient_authority()) + .map_err(|_| "The selected SQL directory could not be opened safely".to_owned())?; + let token = Uuid::new_v4().to_string(); + let root = Arc::new(LegacySqlRoot { + token: token.clone(), + canonical_path, + directory, + operations: Mutex::new(()), + }); + let mut roots = lock(&self.roots)?; + if roots.len() >= MAX_REGISTERED_ROOTS { + return Err("Too many SQL directories are open".to_owned()); + } + roots.insert(token, Arc::clone(&root)); + drop(roots); + root_node(&root) + } + + pub(crate) fn list_children( + &self, + request: &LegacySqlDirectoryPathRequest, + ) -> Result, String> { + let root = self.root(&request.root_token)?; + let _operation = lock(&root.operations)?; + list_children_locked(&root, &request.relative_path) + } + + pub(crate) fn create_child( + &self, + request: &LegacyCreateSqlDirectoryChildRequest, + ) -> Result { + let root = self.root(&request.root_token)?; + let _operation = lock(&root.operations)?; + let parent = existing_relative_path(&root, &request.parent_relative_path)?; + require_directory(&root, &parent)?; + let is_directory = match request.node_type.as_str() { + "directory" => true, + "file" => false, + _ => return Err("SQL directory child type must be file or directory".to_owned()), + }; + let name = if is_directory { + validate_leaf_name(&request.name)?.to_owned() + } else { + normalize_text_file_name(&request.name, "sql")? + }; + let target = parent.join(&name); + require_missing(&root, &target)?; + if is_directory { + root.directory + .create_dir(&target) + .map_err(|_| "The SQL directory could not be created".to_owned())?; + } else { + atomic_write_in_root(&root, &target, b"", false)?; + } + sync_root_directory(&root)?; + Ok(LegacyCreateSqlDirectoryChildResponse { + created_node: tree_node(&root, &target)?, + children: list_children_relative(&root, &parent)?, + }) + } + + pub(crate) fn save_file( + &self, + request: &LegacySaveSqlDirectoryFileRequest, + ) -> Result { + validate_content_size(&request.content)?; + let root = self.root(&request.root_token)?; + let _operation = lock(&root.operations)?; + let parent = existing_relative_path(&root, &request.parent_relative_path)?; + require_directory(&root, &parent)?; + let name = normalize_text_file_name(&request.name, "sql")?; + let target = available_file_path(&root, &parent, &name)?; + atomic_write_in_root(&root, &target, request.content.as_bytes(), false)?; + sync_root_directory(&root)?; + Ok(LegacyCreateSqlDirectoryChildResponse { + created_node: tree_node(&root, &target)?, + children: list_children_relative(&root, &parent)?, + }) + } + + pub(crate) fn rename_child( + &self, + request: &LegacyRenameSqlDirectoryChildRequest, + ) -> Result { + let root = self.root(&request.root_token)?; + let _operation = lock(&root.operations)?; + let source = existing_relative_path(&root, &request.relative_path)?; + if source.as_os_str().is_empty() { + return Err("The selected SQL directory root cannot be renamed".to_owned()); + } + let metadata = root + .directory + .symlink_metadata(&source) + .map_err(|_| "The selected SQL path is not available".to_owned())?; + let name = if metadata.is_dir() { + validate_leaf_name(&request.name)?.to_owned() + } else if metadata.is_file() { + let fallback = file_extension( + source + .file_name() + .and_then(OsStr::to_str) + .ok_or_else(|| "The SQL file name is not valid UTF-8".to_owned())?, + ); + normalize_text_file_name( + &request.name, + if fallback.is_empty() { + "sql" + } else { + &fallback + }, + )? + } else { + return Err("The selected SQL path is not available".to_owned()); + }; + let parent = source.parent().map_or_else(PathBuf::new, Path::to_path_buf); + let target = parent.join(name); + if target != source { + require_missing(&root, &target)?; + root.directory + .rename(&source, &root.directory, &target) + .map_err(|_| "The SQL file or directory could not be renamed".to_owned())?; + sync_root_directory(&root)?; + } + Ok(LegacyRenameSqlDirectoryChildResponse { + renamed_node: tree_node(&root, &target)?, + parent_relative_path: path_text(&parent)?, + children: list_children_relative(&root, &parent)?, + }) + } + + pub(crate) fn delete_child( + &self, + request: &LegacySqlDirectoryPathRequest, + ) -> Result { + self.delete_child_with(request, |path| { + trash::delete(path) + .map_err(|_| "The SQL file or directory could not be moved to Trash".to_owned()) + }) + } + + fn delete_child_with( + &self, + request: &LegacySqlDirectoryPathRequest, + move_to_trash: F, + ) -> Result + where + F: FnOnce(&Path) -> Result<(), String>, + { + let root = self.root(&request.root_token)?; + let _operation = lock(&root.operations)?; + let target = existing_relative_path(&root, &request.relative_path)?; + if target.as_os_str().is_empty() { + return Err("The selected SQL directory root cannot be deleted".to_owned()); + } + let metadata = root + .directory + .symlink_metadata(&target) + .map_err(|_| "The selected SQL path is not available".to_owned())?; + if !metadata.is_dir() && !metadata.is_file() { + return Err("The selected SQL path is not available".to_owned()); + } + if metadata.is_file() { + let name = target + .file_name() + .and_then(OsStr::to_str) + .ok_or_else(|| "The SQL file name is not valid UTF-8".to_owned())?; + if !is_supported_text_file(name) { + return Err("Only supported text files can be deleted".to_owned()); + } + } + let parent = target.parent().map_or_else(PathBuf::new, Path::to_path_buf); + let original_name = target + .file_name() + .ok_or_else(|| "The selected SQL path is not available".to_owned())?; + let staging = parent.join(format!(".chat2db-trash-{}", Uuid::new_v4())); + root.directory + .create_dir(&staging) + .map_err(|_| "The SQL Trash staging directory could not be created".to_owned())?; + let staged_target = staging.join(original_name); + if let Err(error) = root + .directory + .rename(&target, &root.directory, &staged_target) + { + let _ = root.directory.remove_dir(&staging); + return Err(format!( + "The SQL path could not be staged for Trash: {error}" + )); + } + sync_root_directory(&root)?; + let staged_absolute = root.canonical_path.join(&staged_target); + if let Err(error) = move_to_trash(&staged_absolute) { + let _ = root + .directory + .rename(&staged_target, &root.directory, &target); + let _ = root.directory.remove_dir(&staging); + let _ = sync_root_directory(&root); + return Err(error); + } + root.directory + .remove_dir(&staging) + .map_err(|_| "The SQL Trash staging directory could not be removed".to_owned())?; + sync_root_directory(&root)?; + Ok(LegacyDeleteSqlDirectoryChildResponse { + parent_relative_path: path_text(&parent)?, + children: list_children_relative(&root, &parent)?, + }) + } + + pub(crate) fn terminal_directory( + &self, + request: &LegacySqlDirectoryPathRequest, + ) -> Result { + let root = self.root(&request.root_token)?; + let _operation = lock(&root.operations)?; + let target = existing_relative_path(&root, &request.relative_path)?; + let metadata = if target.as_os_str().is_empty() { + root.directory + .dir_metadata() + .map_err(|_| "The selected SQL directory is not available".to_owned())? + } else { + root.directory + .symlink_metadata(&target) + .map_err(|_| "The selected SQL path is not available".to_owned())? + }; + let directory = if metadata.is_dir() { + target + } else if metadata.is_file() { + target.parent().map_or_else(PathBuf::new, Path::to_path_buf) + } else { + return Err("The selected SQL path is not available".to_owned()); + }; + Ok(root.canonical_path.join(directory)) + } + + fn root(&self, token: &str) -> Result, String> { + if token.is_empty() || Uuid::parse_str(token).is_err() { + return Err("The SQL directory root token is invalid".to_owned()); + } + lock(&self.roots)? + .get(token) + .cloned() + .ok_or_else(|| "The SQL directory root is not available".to_owned()) + } +} + +pub(crate) fn save_dialog_file_name(request: &LegacySaveFileRequest) -> Result { + validate_content_size(&request.file_content)?; + let file_type = normalize_file_type(&request.file_type)?; + let name = validate_leaf_name(&request.file_name)?; + if file_extension(name).eq_ignore_ascii_case(&file_type) { + Ok(name.to_owned()) + } else if file_extension(name).is_empty() { + let completed = format!("{name}.{file_type}"); + validate_leaf_name(&completed)?; + Ok(completed) + } else { + Err("The save file name does not match the requested file type".to_owned()) + } +} + +pub(crate) fn save_dialog_file_type(request: &LegacySaveFileRequest) -> Result { + normalize_file_type(&request.file_type) +} + +pub(crate) fn save_text_file( + selected_path: &Path, + request: &LegacySaveFileRequest, +) -> Result { + validate_content_size(&request.file_content)?; + let file_type = normalize_file_type(&request.file_type)?; + let selected_name = selected_path + .file_name() + .and_then(OsStr::to_str) + .ok_or_else(|| "The selected save file name is not valid UTF-8".to_owned())?; + validate_leaf_name(selected_name)?; + if !file_extension(selected_name).eq_ignore_ascii_case(&file_type) { + return Err("The selected save file does not match the requested file type".to_owned()); + } + atomic_write_absolute(selected_path, request.file_content.as_bytes(), false)?; + Ok(LegacySavedFile { + path: absolute_path_text(selected_path)?, + size: u64::try_from(request.file_content.len()).unwrap_or(u64::MAX), + }) +} + +pub(crate) fn update_text_file(request: &LegacyUpdateFileRequest) -> Result { + validate_content_size(&request.file_content)?; + let path = validated_absolute_text_path(&request.file_path)?; + atomic_write_absolute(&path, request.file_content.as_bytes(), true)?; + Ok(true) +} + +pub(crate) fn read_text_file(request: &LegacyReadFileRequest) -> Result { + let path = validated_absolute_text_path(&request.path)?; + let parent = path + .parent() + .ok_or_else(|| "The selected text file has no parent directory".to_owned())?; + let file_name = path + .file_name() + .ok_or_else(|| "The selected text file name is invalid".to_owned())?; + let canonical_parent = fs::canonicalize(parent) + .map_err(|_| "The selected text file directory is not available".to_owned())?; + let directory = Dir::open_ambient_dir(canonical_parent, ambient_authority()) + .map_err(|_| "The selected text file directory could not be opened safely".to_owned())?; + let metadata = directory + .symlink_metadata(file_name) + .map_err(|_| "The selected text file is not available".to_owned())?; + if metadata.file_type().is_symlink() || !metadata.is_file() { + return Err("The selected text path is not a regular file".to_owned()); + } + if metadata.len() > MAX_TEXT_FILE_BYTES { + return Err("The selected text file exceeds the 16 MiB limit".to_owned()); + } + let mut file = directory + .open(file_name) + .map_err(|_| "The selected text file could not be opened safely".to_owned())?; + let mut bytes = Vec::with_capacity(usize::try_from(metadata.len()).unwrap_or_default()); + Read::by_ref(&mut file) + .take(MAX_TEXT_FILE_BYTES + 1) + .read_to_end(&mut bytes) + .map_err(|_| "The selected text file could not be read".to_owned())?; + if bytes.len() > MAX_TEXT_CONTENT_BYTES { + return Err("The selected text file exceeds the 16 MiB limit".to_owned()); + } + decode_text(&bytes, request.charsets.as_deref()) +} + +pub(crate) fn open_terminal(directory: &Path) -> Result<(), String> { + if !directory.is_absolute() || !directory.is_dir() { + return Err("The terminal working directory is not available".to_owned()); + } + spawn_terminal(directory) +} + +fn root_node(root: &Arc) -> Result { + let _operation = lock(&root.operations)?; + let children = list_children_relative(root, Path::new(""))?; + let name = root + .canonical_path + .file_name() + .and_then(OsStr::to_str) + .map_or_else( + || path_text(&root.canonical_path), + |name| Ok(name.to_owned()), + )?; + Ok(LegacySqlTreeNode { + key: format!("{}:", root.token), + root_token: Some(root.token.clone()), + root_path: Some(path_text(&root.canonical_path)?), + name, + path: path_text(&root.canonical_path)?, + relative_path: String::new(), + node_type: "directory".to_owned(), + disabled: false, + sql_file: false, + text_file: false, + file_extension: None, + has_children: true, + loaded: true, + children: Some(children), + }) +} + +fn list_children_locked( + root: &LegacySqlRoot, + relative_path: &str, +) -> Result, String> { + let relative = existing_relative_path(root, relative_path)?; + require_directory(root, &relative)?; + list_children_relative(root, &relative) +} + +fn list_children_relative( + root: &LegacySqlRoot, + relative: &Path, +) -> Result, String> { + let directory_path = if relative.as_os_str().is_empty() { + Path::new(".") + } else { + relative + }; + let entries = root + .directory + .read_dir(directory_path) + .map_err(|_| "The SQL directory children could not be read".to_owned())?; + let mut children = Vec::new(); + for entry in entries { + let entry = entry.map_err(|_| "A SQL directory child could not be read".to_owned())?; + let file_type = entry + .file_type() + .map_err(|_| "A SQL directory child could not be inspected".to_owned())?; + if file_type.is_symlink() { + continue; + } + let Ok(name) = entry.file_name().into_string() else { + continue; + }; + if !file_type.is_dir() && (!file_type.is_file() || !is_supported_text_file(&name)) { + continue; + } + children.push((file_type.is_dir(), name)); + if children.len() > MAX_CHILDREN { + break; + } + } + children.sort_by(|left, right| { + right + .0 + .cmp(&left.0) + .then_with(|| left.1.to_lowercase().cmp(&right.1.to_lowercase())) + .then_with(|| left.1.cmp(&right.1)) + }); + let overflow = children.len() > MAX_CHILDREN; + children.truncate(MAX_CHILDREN); + let mut nodes = children + .into_iter() + .map(|(_, name)| tree_node(root, &relative.join(name))) + .collect::, _>>()?; + if overflow { + nodes.push(LegacySqlTreeNode { + key: format!("{}:overflow:{}", root.token, path_text(relative)?), + root_token: None, + root_path: None, + name: format!("Only first {MAX_CHILDREN} entries are shown"), + path: String::new(), + relative_path: path_text(relative)?, + node_type: "file".to_owned(), + disabled: true, + sql_file: false, + text_file: false, + file_extension: None, + has_children: false, + loaded: true, + children: None, + }); + } + Ok(nodes) +} + +fn tree_node(root: &LegacySqlRoot, relative: &Path) -> Result { + ensure_no_symlink_components(root, relative, false)?; + let metadata = root + .directory + .symlink_metadata(relative) + .map_err(|_| "The SQL path is not available".to_owned())?; + if metadata.file_type().is_symlink() { + return Err("Symbolic links are not supported in SQL directories".to_owned()); + } + let name = relative + .file_name() + .and_then(OsStr::to_str) + .ok_or_else(|| "The SQL path name is not valid UTF-8".to_owned())?; + let is_directory = metadata.is_dir(); + let is_file = metadata.is_file(); + if !is_directory && !is_file { + return Err("The SQL path is not a file or directory".to_owned()); + } + let extension = if is_file { + file_extension(name) + } else { + String::new() + }; + let relative_text = path_text(relative)?; + Ok(LegacySqlTreeNode { + key: format!("{}:{relative_text}", root.token), + root_token: Some(root.token.clone()), + root_path: None, + name: name.to_owned(), + path: path_text(&root.canonical_path.join(relative))?, + relative_path: relative_text, + node_type: if is_directory { "directory" } else { "file" }.to_owned(), + disabled: false, + sql_file: is_file && extension == "sql", + text_file: is_file && is_supported_text_file(name), + file_extension: Some(extension), + has_children: is_directory, + loaded: !is_directory, + children: None, + }) +} + +fn existing_relative_path(root: &LegacySqlRoot, value: &str) -> Result { + let relative = validate_relative_path(value)?; + ensure_no_symlink_components(root, &relative, false)?; + if relative.as_os_str().is_empty() { + return Ok(relative); + } + let canonical = root + .directory + .canonicalize(&relative) + .map_err(|_| "The SQL path is not available".to_owned())?; + validate_relative_path_os(&canonical)?; + Ok(canonical) +} + +fn validate_relative_path(value: &str) -> Result { + if value.len() > MAX_RELATIVE_PATH_BYTES { + return Err("The SQL relative path is too long".to_owned()); + } + validate_relative_path_os(Path::new(value)) +} + +fn validate_relative_path_os(path: &Path) -> Result { + if path.is_absolute() { + return Err("The SQL path must be relative to its registered root".to_owned()); + } + let mut normalized = PathBuf::new(); + let mut count = 0_usize; + for component in path.components() { + count = count.saturating_add(1); + if count > MAX_PATH_COMPONENTS { + return Err("The SQL relative path has too many components".to_owned()); + } + match component { + Component::Normal(value) => normalized.push(value), + Component::CurDir if path.as_os_str().is_empty() => {} + Component::CurDir => { + return Err("The SQL relative path cannot contain dot components".to_owned()); + } + Component::ParentDir | Component::RootDir | Component::Prefix(_) => { + return Err("The SQL relative path cannot escape its registered root".to_owned()); + } + } + } + Ok(normalized) +} + +fn ensure_no_symlink_components( + root: &LegacySqlRoot, + relative: &Path, + allow_missing_last: bool, +) -> Result<(), String> { + let mut current = PathBuf::new(); + let components = relative.components().collect::>(); + for (index, component) in components.iter().enumerate() { + let Component::Normal(value) = component else { + return Err("The SQL relative path is invalid".to_owned()); + }; + current.push(value); + match root.directory.symlink_metadata(¤t) { + Ok(metadata) if metadata.file_type().is_symlink() => { + return Err("Symbolic links are not supported in SQL directories".to_owned()); + } + Ok(_) => {} + Err(error) + if allow_missing_last + && index + 1 == components.len() + && error.kind() == io::ErrorKind::NotFound => {} + Err(_) => return Err("The SQL path is not available".to_owned()), + } + } + Ok(()) +} + +fn require_directory(root: &LegacySqlRoot, relative: &Path) -> Result<(), String> { + let metadata = if relative.as_os_str().is_empty() { + root.directory.dir_metadata() + } else { + root.directory.symlink_metadata(relative) + } + .map_err(|_| "The SQL directory is not available".to_owned())?; + if metadata.file_type().is_symlink() || !metadata.is_dir() { + return Err("The SQL path is not a directory".to_owned()); + } + Ok(()) +} + +fn require_missing(root: &LegacySqlRoot, relative: &Path) -> Result<(), String> { + ensure_no_symlink_components(root, relative, true)?; + match root.directory.symlink_metadata(relative) { + Ok(_) => Err("The SQL file or directory already exists".to_owned()), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), + Err(_) => Err("The SQL target could not be inspected".to_owned()), + } +} + +fn available_file_path(root: &LegacySqlRoot, parent: &Path, name: &str) -> Result { + let extension = Path::new(name) + .extension() + .and_then(OsStr::to_str) + .unwrap_or_default(); + let stem = Path::new(name) + .file_stem() + .and_then(OsStr::to_str) + .ok_or_else(|| "The SQL file name is invalid".to_owned())?; + for index in 0..=MAX_CHILDREN { + let candidate = if index == 0 { + name.to_owned() + } else { + format!("{stem}-{index}.{extension}") + }; + let target = parent.join(candidate); + match root.directory.symlink_metadata(&target) { + Ok(_) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(target), + Err(_) => return Err("The SQL target could not be inspected".to_owned()), + } + } + Err("No available SQL file name remains in this directory".to_owned()) +} + +fn atomic_write_in_root( + root: &LegacySqlRoot, + target: &Path, + contents: &[u8], + replace: bool, +) -> Result<(), String> { + if contents.len() > MAX_TEXT_CONTENT_BYTES { + return Err("The text content exceeds the 16 MiB limit".to_owned()); + } + let parent = target.parent().map_or_else(PathBuf::new, Path::to_path_buf); + require_directory(root, &parent)?; + if !replace { + require_missing(root, target)?; + } + let temporary = parent.join(format!(".chat2db-write-{}.tmp", Uuid::new_v4())); + let mut file = root + .directory + .open_with(&temporary, OpenOptions::new().write(true).create_new(true)) + .map_err(|_| "The temporary SQL file could not be created".to_owned())?; + let write_result = (|| { + file.write_all(contents)?; + file.sync_all()?; + if replace { + let metadata = root.directory.symlink_metadata(target)?; + if metadata.file_type().is_symlink() || !metadata.is_file() { + return Err(io::Error::other("target is not a regular file")); + } + } else if root.directory.symlink_metadata(target).is_ok() { + return Err(io::Error::new( + io::ErrorKind::AlreadyExists, + "target already exists", + )); + } + root.directory.rename(&temporary, &root.directory, target) + })(); + if let Err(error) = write_result { + let _ = root.directory.remove_file(&temporary); + return Err(format!( + "The SQL file could not be written atomically: {error}" + )); + } + sync_root_directory(root) +} + +fn atomic_write_absolute( + path: &Path, + contents: &[u8], + require_existing: bool, +) -> Result<(), String> { + if contents.len() > MAX_TEXT_CONTENT_BYTES { + return Err("The text content exceeds the 16 MiB limit".to_owned()); + } + if !path.is_absolute() { + return Err("The selected file path must be absolute".to_owned()); + } + let parent = path + .parent() + .ok_or_else(|| "The selected file path has no parent directory".to_owned())?; + let file_name = path + .file_name() + .ok_or_else(|| "The selected file name is invalid".to_owned())?; + let canonical_parent = fs::canonicalize(parent) + .map_err(|_| "The selected file directory is not available".to_owned())?; + let directory = Dir::open_ambient_dir(canonical_parent, ambient_authority()) + .map_err(|_| "The selected file directory could not be opened safely".to_owned())?; + let existing_permissions = match directory.symlink_metadata(file_name) { + Ok(metadata) if metadata.file_type().is_symlink() || !metadata.is_file() => { + return Err("The selected file target is not a regular file".to_owned()); + } + Ok(metadata) => Some(metadata.permissions()), + Err(error) if error.kind() == io::ErrorKind::NotFound && !require_existing => None, + Err(error) if error.kind() == io::ErrorKind::NotFound => { + return Err("The selected file does not exist".to_owned()); + } + Err(_) => return Err("The selected file target could not be inspected".to_owned()), + }; + let temporary_name = OsString::from(format!(".chat2db-write-{}.tmp", Uuid::new_v4())); + let mut file = directory + .open_with( + &temporary_name, + OpenOptions::new().write(true).create_new(true), + ) + .map_err(|_| "The temporary file could not be created".to_owned())?; + let write_result = (|| { + file.write_all(contents)?; + if let Some(permissions) = existing_permissions { + directory.set_permissions(&temporary_name, permissions)?; + } + file.sync_all()?; + if require_existing { + let metadata = directory.symlink_metadata(file_name)?; + if metadata.file_type().is_symlink() || !metadata.is_file() { + return Err(io::Error::other("target is not a regular file")); + } + } + directory.rename(&temporary_name, &directory, file_name) + })(); + if let Err(error) = write_result { + let _ = directory.remove_file(&temporary_name); + return Err(format!("The file could not be written atomically: {error}")); + } + sync_directory(&directory) + .map_err(|_| "The file directory could not be synchronized".to_owned()) +} + +fn sync_root_directory(root: &LegacySqlRoot) -> Result<(), String> { + sync_directory(&root.directory) + .map_err(|_| "The SQL directory could not be synchronized".to_owned()) +} + +fn sync_directory(directory: &Dir) -> io::Result<()> { + #[cfg(not(windows))] + let sync_result = directory.open(".")?.sync_all(); + #[cfg(windows)] + let sync_result = directory.try_clone()?.into_std_file().sync_all(); + match sync_result { + Ok(()) => Ok(()), + #[cfg(windows)] + Err(error) + if matches!( + error.kind(), + io::ErrorKind::InvalidInput + | io::ErrorKind::PermissionDenied + | io::ErrorKind::Unsupported + ) => + { + Ok(()) + } + Err(error) => Err(error), + } +} + +fn validated_absolute_text_path(value: &str) -> Result { + let path = PathBuf::from(value.trim()); + if value.trim().is_empty() || !path.is_absolute() { + return Err("The selected text file path must be absolute".to_owned()); + } + let file_name = path + .file_name() + .and_then(OsStr::to_str) + .ok_or_else(|| "The selected text file name is not valid UTF-8".to_owned())?; + validate_leaf_name(file_name)?; + if !is_supported_text_file(file_name) { + return Err("Only supported text files can be opened or updated".to_owned()); + } + let metadata = fs::symlink_metadata(&path) + .map_err(|_| "The selected text file is not available".to_owned())?; + if metadata.file_type().is_symlink() || !metadata.is_file() { + return Err("The selected text path is not a regular file".to_owned()); + } + Ok(path) +} + +fn decode_text(bytes: &[u8], charset: Option<&str>) -> Result { + let label = charset + .map(str::trim) + .filter(|value| !value.is_empty()) + .unwrap_or("utf-8"); + let encoding = Encoding::for_label(label.as_bytes()) + .ok_or_else(|| "The requested text charset is not supported".to_owned())?; + let (decoded, _, had_errors) = encoding.decode(bytes); + if had_errors { + return Err(format!( + "The selected text file is not valid {}", + encoding.name() + )); + } + let decoded = decoded.strip_prefix('\u{feff}').unwrap_or(decoded.as_ref()); + Ok(decoded.to_owned()) +} + +fn normalize_file_type(value: &str) -> Result { + let value = value.trim().trim_start_matches('.').to_ascii_lowercase(); + if value.is_empty() + || value.len() > MAX_FILE_TYPE_BYTES + || !value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-')) + { + return Err("The requested file type is invalid".to_owned()); + } + Ok(value) +} + +fn normalize_text_file_name(value: &str, fallback_extension: &str) -> Result { + let value = validate_leaf_name(value)?; + if is_supported_text_file(value) { + return Ok(value.to_owned()); + } + if !file_extension(value).is_empty() { + return Err("Only supported text file extensions are allowed".to_owned()); + } + let completed = format!("{value}.{fallback_extension}"); + validate_leaf_name(&completed)?; + Ok(completed) +} + +fn validate_leaf_name(value: &str) -> Result<&str, String> { + let value = value.trim(); + if value.is_empty() + || matches!(value, "." | "..") + || value.len() > MAX_FILE_NAME_BYTES + || value.ends_with(['.', ' ']) + || value.chars().any(|character| { + character.is_control() + || matches!( + character, + '<' | '>' | ':' | '"' | '/' | '\\' | '|' | '?' | '*' + ) + }) + { + return Err("The file or directory name is invalid".to_owned()); + } + let stem = value + .split('.') + .next() + .unwrap_or_default() + .to_ascii_uppercase(); + if matches!(stem.as_str(), "CON" | "PRN" | "AUX" | "NUL") + || stem.strip_prefix("COM").is_some_and(|suffix| { + matches!(suffix, "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9") + }) + || stem.strip_prefix("LPT").is_some_and(|suffix| { + matches!(suffix, "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9") + }) + { + return Err("The file or directory name is reserved".to_owned()); + } + Ok(value) +} + +fn validate_content_size(value: &str) -> Result<(), String> { + if value.len() > MAX_TEXT_CONTENT_BYTES { + return Err("The text content exceeds the 16 MiB limit".to_owned()); + } + Ok(()) +} + +fn is_supported_text_file(name: &str) -> bool { + matches!( + file_extension(name).as_str(), + "sql" + | "txt" + | "md" + | "markdown" + | "json" + | "jsonl" + | "yaml" + | "yml" + | "csv" + | "tsv" + | "xml" + | "log" + | "env" + | "ini" + | "conf" + | "config" + | "properties" + | "toml" + ) +} + +fn file_extension(name: &str) -> String { + name.rsplit_once('.') + .filter(|(stem, extension)| !stem.is_empty() || !extension.is_empty()) + .map_or_else(String::new, |(_, extension)| extension.to_ascii_lowercase()) +} + +fn absolute_path_text(path: &Path) -> Result { + let absolute = if path.is_absolute() { + path.to_path_buf() + } else { + env_current_dir()?.join(path) + }; + path_text(&absolute) +} + +fn env_current_dir() -> Result { + std::env::current_dir().map_err(|_| "The current directory is not available".to_owned()) +} + +fn path_text(path: &Path) -> Result { + path.to_str() + .map(str::to_owned) + .ok_or_else(|| "The local path is not valid UTF-8".to_owned()) +} + +fn lock(mutex: &Mutex) -> Result, String> { + mutex + .lock() + .map_err(|_| "The desktop file registry is unavailable".to_owned()) +} + +#[cfg(target_os = "macos")] +fn spawn_terminal(directory: &Path) -> Result<(), String> { + Command::new("/usr/bin/open") + .arg("-a") + .arg("Terminal") + .arg("--") + .arg(directory) + .spawn() + .map(|_| ()) + .map_err(|_| "The terminal could not be opened".to_owned()) +} + +#[cfg(target_os = "windows")] +fn spawn_terminal(directory: &Path) -> Result<(), String> { + Command::new("cmd.exe") + .current_dir(directory) + .spawn() + .map(|_| ()) + .map_err(|_| "The terminal could not be opened".to_owned()) +} + +#[cfg(target_os = "linux")] +fn spawn_terminal(directory: &Path) -> Result<(), String> { + let candidates: [(&str, &[&str]); 4] = [ + ("x-terminal-emulator", &["--working-directory"]), + ("gnome-terminal", &["--working-directory"]), + ("konsole", &["--workdir"]), + ("xfce4-terminal", &["--working-directory"]), + ]; + for (program, arguments) in candidates { + let mut command = Command::new(program); + command.args(arguments).arg(directory); + match command.spawn() { + Ok(_) => return Ok(()), + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(_) => return Err("The terminal could not be opened".to_owned()), + } + } + Err("No supported terminal application is installed".to_owned()) +} + +#[cfg(not(any(target_os = "macos", target_os = "windows", target_os = "linux")))] +fn spawn_terminal(_directory: &Path) -> Result<(), String> { + Err("Opening a terminal is not supported on this platform".to_owned()) +} + +#[cfg(test)] +mod tests { + use std::fs; + + use super::{ + LegacyCreateSqlDirectoryChildRequest, LegacyReadFileRequest, + LegacyRenameSqlDirectoryChildRequest, LegacySaveFileRequest, + LegacySaveSqlDirectoryFileRequest, LegacySqlDirectoryPathRequest, + LegacySqlDirectoryRegistry, LegacyUpdateFileRequest, read_text_file, save_dialog_file_name, + save_text_file, update_text_file, + }; + + fn registry_fixture() -> (tempfile::TempDir, LegacySqlDirectoryRegistry, String) { + let directory = tempfile::tempdir().expect("temporary SQL root"); + fs::write(directory.path().join("inventory.sql"), "SELECT 1;").expect("SQL fixture"); + fs::create_dir(directory.path().join("nested")).expect("nested fixture"); + let registry = LegacySqlDirectoryRegistry::default(); + let root = registry + .register_root(directory.path()) + .expect("root registration"); + let token = root.root_token.expect("root token"); + (directory, registry, token) + } + + #[test] + fn sql_root_and_children_match_the_retained_frontend_shape() { + let (directory, registry, token) = registry_fixture(); + let root = registry + .register_root(directory.path()) + .expect("second root registration"); + let value = serde_json::to_value(root).expect("root serialization"); + assert_eq!(value["type"], "directory"); + assert_eq!(value["relativePath"], ""); + assert_eq!(value["loaded"], true); + assert!(value["rootToken"].as_str().is_some()); + assert!(value["rootPath"].as_str().is_some()); + assert!(value["children"].as_array().is_some()); + + let children = registry + .list_children(&LegacySqlDirectoryPathRequest { + root_token: token, + relative_path: String::new(), + }) + .expect("children"); + assert_eq!(children[0].name, "nested"); + assert_eq!(children[0].node_type, "directory"); + assert_eq!(children[1].name, "inventory.sql"); + assert!(children[1].sql_file); + } + + #[test] + fn sql_root_tokens_and_relative_paths_fail_closed() { + let (directory, registry, token) = registry_fixture(); + let unknown = LegacySqlDirectoryPathRequest { + root_token: uuid::Uuid::new_v4().to_string(), + relative_path: String::new(), + }; + assert!(registry.list_children(&unknown).is_err()); + for path in ["../outside", "/tmp", "nested/../outside", "./nested"] { + assert!( + registry + .list_children(&LegacySqlDirectoryPathRequest { + root_token: token.clone(), + relative_path: path.to_owned(), + }) + .is_err(), + "path must fail: {path}" + ); + } + assert!(directory.path().join("inventory.sql").is_file()); + } + + #[cfg(unix)] + #[test] + fn sql_directory_operations_reject_symbolic_links() { + use std::os::unix::fs::symlink; + + let (directory, registry, token) = registry_fixture(); + let outside = tempfile::tempdir().expect("outside directory"); + symlink(outside.path(), directory.path().join("outside-link")).expect("directory link"); + symlink( + directory.path().join("inventory.sql"), + directory.path().join("file-link.sql"), + ) + .expect("file link"); + + let children = registry + .list_children(&LegacySqlDirectoryPathRequest { + root_token: token.clone(), + relative_path: String::new(), + }) + .expect("children"); + assert!(!children.iter().any(|node| node.name.contains("link"))); + assert!( + registry + .list_children(&LegacySqlDirectoryPathRequest { + root_token: token, + relative_path: "outside-link".to_owned(), + }) + .is_err() + ); + } + + #[test] + fn create_save_and_rename_return_exact_tree_refresh_shapes() { + let (directory, registry, token) = registry_fixture(); + let created = registry + .create_child(&LegacyCreateSqlDirectoryChildRequest { + root_token: token.clone(), + parent_relative_path: "nested".to_owned(), + name: "query".to_owned(), + node_type: "file".to_owned(), + }) + .expect("create file"); + assert_eq!(created.created_node.name, "query.sql"); + assert!(directory.path().join("nested/query.sql").is_file()); + + let saved = registry + .save_file(&LegacySaveSqlDirectoryFileRequest { + root_token: token.clone(), + parent_relative_path: "nested".to_owned(), + name: "query.sql".to_owned(), + content: "SELECT 2;".to_owned(), + }) + .expect("save collision"); + assert_eq!(saved.created_node.name, "query-1.sql"); + assert_eq!( + fs::read_to_string(directory.path().join("nested/query-1.sql")).expect("saved content"), + "SELECT 2;" + ); + + let renamed = registry + .rename_child(&LegacyRenameSqlDirectoryChildRequest { + root_token: token, + relative_path: "nested/query-1.sql".to_owned(), + name: "renamed".to_owned(), + }) + .expect("rename file"); + assert_eq!(renamed.parent_relative_path, "nested"); + assert_eq!(renamed.renamed_node.name, "renamed.sql"); + assert!(directory.path().join("nested/renamed.sql").is_file()); + assert!(!directory.path().join("nested/query-1.sql").exists()); + } + + #[test] + fn delete_stages_safely_and_returns_refreshed_children() { + let (directory, registry, token) = registry_fixture(); + let request = LegacySqlDirectoryPathRequest { + root_token: token, + relative_path: "inventory.sql".to_owned(), + }; + let deleted = registry + .delete_child_with(&request, |staged| { + assert_eq!( + staged.file_name().and_then(std::ffi::OsStr::to_str), + Some("inventory.sql") + ); + fs::remove_file(staged).map_err(|error| error.to_string()) + }) + .expect("delete fixture"); + assert_eq!(deleted.parent_relative_path, ""); + assert!(!directory.path().join("inventory.sql").exists()); + assert!( + !deleted + .children + .iter() + .any(|node| node.name == "inventory.sql") + ); + assert!( + fs::read_dir(directory.path()) + .expect("root entries") + .all(|entry| !entry + .expect("entry") + .file_name() + .to_string_lossy() + .starts_with(".chat2db-trash-")) + ); + } + + #[test] + fn absolute_file_writes_are_atomic_bounded_and_utf8_explicit() { + let directory = tempfile::tempdir().expect("temporary directory"); + let path = directory.path().join("query.sql"); + fs::write(&path, "SELECT 1;").expect("file fixture"); + update_text_file(&LegacyUpdateFileRequest { + file_path: path.to_string_lossy().into_owned(), + file_content: "SELECT 2;".to_owned(), + }) + .expect("atomic update"); + assert_eq!( + fs::read_to_string(&path).expect("updated file"), + "SELECT 2;" + ); + assert!( + fs::read_dir(directory.path()) + .expect("directory entries") + .all(|entry| !entry + .expect("entry") + .file_name() + .to_string_lossy() + .starts_with(".chat2db-write-")) + ); + + let read = read_text_file(&LegacyReadFileRequest { + path: path.to_string_lossy().into_owned(), + charsets: Some("UTF-8".to_owned()), + }) + .expect("UTF-8 read"); + assert_eq!(read, "SELECT 2;"); + fs::write(&path, [0xff, 0xfe, 0xfd]).expect("invalid UTF-8 fixture"); + assert!( + read_text_file(&LegacyReadFileRequest { + path: path.to_string_lossy().into_owned(), + charsets: None, + }) + .is_err() + ); + } + + #[test] + fn save_file_contract_normalizes_names_and_reports_size() { + let request = LegacySaveFileRequest { + file_name: "connections".to_owned(), + file_content: "{}".to_owned(), + file_type: ".json".to_owned(), + }; + assert_eq!( + save_dialog_file_name(&request).expect("dialog name"), + "connections.json" + ); + let directory = tempfile::tempdir().expect("temporary directory"); + let path = directory.path().join("connections.json"); + let saved = save_text_file(&path, &request).expect("saved file"); + assert_eq!(saved.path, path.to_string_lossy()); + assert_eq!(saved.size, 2); + assert_eq!(fs::read_to_string(path).expect("saved content"), "{}"); + assert!( + save_dialog_file_name(&LegacySaveFileRequest { + file_name: "../secret".to_owned(), + file_content: String::new(), + file_type: "sql".to_owned(), + }) + .is_err() + ); + } +} diff --git a/apps/chat2db-desktop/src/lib.rs b/apps/chat2db-desktop/src/lib.rs index a103014..69fe6b6 100644 --- a/apps/chat2db-desktop/src/lib.rs +++ b/apps/chat2db-desktop/src/lib.rs @@ -1,5 +1,7 @@ //! Tauri IPC delivery adapter for the `Chat2DB` desktop product. +mod legacy_files; + use std::{ collections::HashMap, env, @@ -44,7 +46,15 @@ use chat2db_core::{ }; use chat2db_java_bridge::{BridgeError, EngineCommand, EngineConfig}; use chat2db_local::{LocalError, LocalServer}; +use legacy_files::{ + LegacyCreateSqlDirectoryChildRequest, LegacyOpenSqlDirectoryRequest, LegacyReadFileRequest, + LegacyRenameSqlDirectoryChildRequest, LegacySaveFileRequest, LegacySaveSqlDirectoryFileRequest, + LegacySqlDirectoryPathRequest, LegacySqlDirectoryRegistry, LegacyUpdateFileRequest, + open_terminal, read_text_file, save_dialog_file_name, save_dialog_file_type, save_text_file, + update_text_file, +}; use tauri::{Emitter, State, WebviewWindow, ipc::Channel}; +use tauri_plugin_dialog::{DialogExt, FilePath}; use tokio::sync::{Mutex, oneshot}; const DATA_DIR_ENV: &str = "CHAT2DB_DATA_DIR"; @@ -115,6 +125,7 @@ struct DesktopState { application: Application, local_server: Mutex>, runtime_host: Mutex>, + legacy_sql_directories: LegacySqlDirectoryRegistry, legacy_sql_cancellations: LegacySqlCancellationRegistry, subscriptions: SubscriptionRegistry, next_legacy_execution_id: AtomicU64, @@ -243,6 +254,7 @@ impl DesktopState { application, local_server: Mutex::new(Some(local_server)), runtime_host: Mutex::new(Some(runtime_host)), + legacy_sql_directories: LegacySqlDirectoryRegistry::default(), legacy_sql_cancellations: LegacySqlCancellationRegistry::default(), subscriptions: SubscriptionRegistry::default(), next_legacy_execution_id: AtomicU64::new(1), @@ -384,6 +396,7 @@ pub fn run() -> Result { )?); let managed_state = Arc::clone(&state); let application = tauri::Builder::default() + .plugin(tauri_plugin_dialog::init()) .manage(managed_state) .invoke_handler(tauri::generate_handler![ health, @@ -617,12 +630,122 @@ async fn legacy_request( window: WebviewWindow, request: String, ) -> Result { + if let Some(response) = legacy_ai_stream_request_for(state.inner(), &window, &request).await? { + return Ok(response); + } if let Some(response) = legacy_client_command_for(state.inner(), &window, &request).await? { return Ok(response); } legacy_request_for(&state.application, &request).await } +async fn legacy_ai_stream_request_for( + state: &Arc, + window: &WebviewWindow, + request: &str, +) -> Result, String> { + let value: serde_json::Value = serde_json::from_str(request) + .map_err(|_| "Community desktop request must be valid JSON".to_owned())?; + let request = value + .as_object() + .ok_or_else(|| "Community desktop request must be a JSON object".to_owned())?; + let method = legacy_request_string(request, "method")?; + let request_url = legacy_request_string(request, "requestUrl")?; + let path = request_url + .split('?') + .next() + .unwrap_or(request_url.as_str()); + if !method.eq_ignore_ascii_case("post") || path != "/api/v3/ai/chat/stream" { + return Ok(None); + } + let request_uuid = legacy_request_string(request, "uuid")?; + let chat_request = decode_client_message::( + request.get("message"), + )?; + let started = chat2db_web::legacy_ai::start_chat_run(&state.application, chat_request) + .await + .map_err(|error| format!("{}: {}", error.code, error.message))?; + let run_id = started.run_id.clone(); + let session_id = started.session_id.clone(); + let application = state.application.clone(); + let task_window = window.clone(); + tauri::async_runtime::spawn(async move { + forward_legacy_ai_stream(application, task_window, request_uuid, started).await; + }); + Ok(Some(client_command_response(&serde_json::json!({ + "runId": run_id, + "sessionId": session_id, + })))) +} + +async fn forward_legacy_ai_stream( + application: Application, + window: WebviewWindow, + request_uuid: String, + mut started: chat2db_web::legacy_ai::LegacyAiStartedRun, +) { + let session_chunk = chat2db_web::legacy_ai::LegacyAiStreamChunk { + event_type: "session".to_owned(), + message_type: "session".to_owned(), + content: None, + name: None, + arguments: None, + session_id: Some(started.session_id.clone()), + ts: Some(unix_epoch_millis()), + id: None, + error_code: None, + error_message: None, + }; + if emit_legacy_ai_event(&window, &request_uuid, &session_chunk).is_err() { + let _ = application.cancel_agent_run(&started.run_id).await; + return; + } + while let Some((chunk, terminal)) = chat2db_web::legacy_ai::next_stream_chunk( + &application, + &mut started.subscription, + &started.session_id, + ) + .await + { + if emit_legacy_ai_event(&window, &request_uuid, &chunk).is_err() { + let _ = application.cancel_agent_run(&started.run_id).await; + return; + } + if terminal { + return; + } + } +} + +fn emit_legacy_ai_event( + window: &WebviewWindow, + request_uuid: &str, + chunk: &chat2db_web::legacy_ai::LegacyAiStreamChunk, +) -> Result<(), tauri::Error> { + window.emit( + COMMUNITY_JAVA_MESSAGE_EVENT, + legacy_ai_push_message(request_uuid, chunk), + ) +} + +fn legacy_ai_push_message( + request_uuid: &str, + chunk: &chat2db_web::legacy_ai::LegacyAiStreamChunk, +) -> serde_json::Value { + let data = serde_json::to_string(chunk).unwrap_or_else(|_| { + r#"{"type":"error","messageType":"error","content":"AI event serialization failed"}"# + .to_owned() + }); + serde_json::json!({ + "uuid": request_uuid, + "actionType": "ai_sse_message", + "message": { + "event": chunk.event_name(), + "data": data, + }, + }) +} + #[allow(clippy::too_many_lines)] async fn legacy_client_command_for( state: &Arc, @@ -643,6 +766,132 @@ async fn legacy_client_command_for( "handle-java-message-is-ready" => { Ok(Some(client_command_response(&serde_json::json!(true)))) } + "select-directory" => { + let selected = window + .dialog() + .file() + .set_parent(window) + .set_title("Select Directory") + .blocking_pick_folder() + .map(legacy_file_path) + .transpose()?; + Ok(Some(client_command_response(&serde_json::json!(selected)))) + } + "select-file" => { + let selection = + decode_client_message::(request.get("message"))?; + let selected = select_legacy_files(window, &selection)?; + Ok(Some(client_command_response(&serde_json::json!(selected)))) + } + "reveal-in-explorer" => { + let reveal = + decode_client_message::(request.get("message"))?; + let path = PathBuf::from(reveal.path.trim()); + if reveal.path.trim().is_empty() { + return Err("reveal-in-explorer requires a non-empty path".to_owned()); + } + tauri_plugin_opener::reveal_item_in_dir(path) + .map_err(|_| "The selected path could not be revealed".to_owned())?; + Ok(Some(serde_json::json!({ "success": true }).to_string())) + } + "save-file" => { + let save = decode_client_message::(request.get("message"))?; + let file_name = save_dialog_file_name(&save)?; + let file_type = save_dialog_file_type(&save)?; + let selected = window + .dialog() + .file() + .set_parent(window) + .set_title("Save File") + .set_file_name(&file_name) + .add_filter("Selected File", &[file_type.as_str()]) + .blocking_save_file() + .map(legacy_file_path) + .transpose()?; + let saved = selected + .as_deref() + .map(|path| save_text_file(path, &save)) + .transpose()?; + Ok(Some(client_command_response(&serde_json::json!(saved)))) + } + "update-file-content" => { + let update = decode_client_message::(request.get("message"))?; + let updated = update_text_file(&update)?; + Ok(Some(client_command_response(&serde_json::json!(updated)))) + } + "read-file" => { + let read = decode_client_message::(request.get("message"))?; + let content = read_text_file(&read)?; + Ok(Some(client_command_response(&serde_json::json!(content)))) + } + "select-sql-directory" => { + let selected = window + .dialog() + .file() + .set_parent(window) + .set_title("Select SQL Directory") + .blocking_pick_folder() + .map(legacy_file_path) + .transpose()?; + let root = selected + .as_deref() + .map(|path| state.legacy_sql_directories.register_root(path)) + .transpose()?; + Ok(Some(client_command_response(&serde_json::json!(root)))) + } + "open-sql-directory" => { + let open = + decode_client_message::(request.get("message"))?; + let root = if open.path.trim().is_empty() { + None + } else { + Some( + state + .legacy_sql_directories + .register_root(Path::new(open.path.trim()))?, + ) + }; + Ok(Some(client_command_response(&serde_json::json!(root)))) + } + "get-sql-directory-children" => { + let path = + decode_client_message::(request.get("message"))?; + let children = state.legacy_sql_directories.list_children(&path)?; + Ok(Some(client_command_response(&serde_json::json!(children)))) + } + "create-sql-directory-child" => { + let create = decode_client_message::( + request.get("message"), + )?; + let response = state.legacy_sql_directories.create_child(&create)?; + Ok(Some(client_command_response(&serde_json::json!(response)))) + } + "save-sql-directory-file" => { + let save = + decode_client_message::(request.get("message"))?; + let response = state.legacy_sql_directories.save_file(&save)?; + Ok(Some(client_command_response(&serde_json::json!(response)))) + } + "rename-sql-directory-child" => { + let rename = decode_client_message::( + request.get("message"), + )?; + let response = state.legacy_sql_directories.rename_child(&rename)?; + Ok(Some(client_command_response(&serde_json::json!(response)))) + } + "delete-sql-directory-child" => { + let path = + decode_client_message::(request.get("message"))?; + let response = state.legacy_sql_directories.delete_child(&path)?; + Ok(Some(client_command_response(&serde_json::json!(response)))) + } + "open-sql-directory-terminal" => { + let path = + decode_client_message::(request.get("message"))?; + let directory = state.legacy_sql_directories.terminal_directory(&path)?; + open_terminal(&directory)?; + Ok(Some(client_command_response(&serde_json::json!(true)))) + } "sql-execute" => { let request_uuid = legacy_request_string(request, "uuid")?; let sql_request = decode_client_message::( @@ -668,14 +917,14 @@ async fn legacy_client_command_for( let task_window = window.clone(); let task_execution_id = execution_id.clone(); tauri::async_runtime::spawn(async move { - forward_native_mysql_sql_execution( + Box::pin(forward_native_mysql_sql_execution( Arc::clone(&task_state), task_window, request_uuid, task_execution_id.clone(), sql_request, cancellation, - ) + )) .await; task_state .legacy_sql_cancellations @@ -739,6 +988,124 @@ async fn legacy_client_command_for( } } +#[derive(Debug, Default, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +struct LegacySelectFileRequest { + #[serde(default)] + file_type_list: Vec, + #[serde(default)] + file_size: Option, + #[serde(default)] + multiple: bool, +} + +#[derive(Debug, serde::Deserialize)] +struct LegacyRevealInExplorerRequest { + path: String, +} + +#[derive(Debug, PartialEq, Eq, serde::Serialize)] +#[serde(rename_all = "camelCase")] +struct LegacySelectedFile { + file_name: String, + file_path: String, +} + +fn select_legacy_files( + window: &WebviewWindow, + request: &LegacySelectFileRequest, +) -> Result>, String> { + let mut dialog = window + .dialog() + .file() + .set_parent(window) + .set_title("Select File"); + let extensions = legacy_file_extensions(&request.file_type_list)?; + if !extensions.is_empty() { + let extensions = extensions.iter().map(String::as_str).collect::>(); + dialog = dialog.add_filter("Selected Files", &extensions); + } + + let selected = if request.multiple { + dialog.blocking_pick_files() + } else { + dialog.blocking_pick_file().map(|path| vec![path]) + }; + let Some(selected) = selected else { + return Ok(None); + }; + + selected + .into_iter() + .map(|path| legacy_selected_file(path, request.file_size)) + .collect::, _>>() + .map(Some) +} + +fn legacy_file_extensions(file_types: &[String]) -> Result, String> { + if file_types.len() > 64 { + return Err("select-file accepts at most 64 file extensions".to_owned()); + } + let mut extensions = Vec::with_capacity(file_types.len()); + for file_type in file_types { + let extension = file_type + .trim() + .trim_start_matches('*') + .trim_start_matches('.'); + if extension.is_empty() + || extension.len() > 64 + || !extension + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-')) + { + return Err("select-file contains an invalid file extension".to_owned()); + } + if !extensions.iter().any(|existing| existing == extension) { + extensions.push(extension.to_owned()); + } + } + Ok(extensions) +} + +fn legacy_selected_file( + path: FilePath, + maximum_size_mb: Option, +) -> Result { + let path = legacy_file_path(path)?; + let metadata = + fs::metadata(&path).map_err(|_| "The selected file is no longer available".to_owned())?; + if !metadata.is_file() { + return Err("The selected path is not a regular file".to_owned()); + } + if let Some(maximum_size_mb) = maximum_size_mb.filter(|value| *value > 0) { + let maximum_size = maximum_size_mb + .checked_mul(1024 * 1024) + .ok_or_else(|| "select-file contains an invalid file size limit".to_owned())?; + if metadata.len() > maximum_size { + return Err(format!( + "The selected file exceeds the {maximum_size_mb} MB size limit" + )); + } + } + let file_name = path + .file_name() + .and_then(OsStr::to_str) + .filter(|name| !name.is_empty()) + .ok_or_else(|| "The selected file name cannot be represented as UTF-8".to_owned())?; + let file_path = path + .to_str() + .ok_or_else(|| "The selected file path cannot be represented as UTF-8".to_owned())?; + Ok(LegacySelectedFile { + file_name: file_name.to_owned(), + file_path: file_path.to_owned(), + }) +} + +fn legacy_file_path(path: FilePath) -> Result { + path.into_path() + .map_err(|_| "Community desktop file commands require a local filesystem path".to_owned()) +} + fn decode_client_message( message: Option<&serde_json::Value>, ) -> Result { @@ -785,13 +1152,13 @@ async fn forward_native_mysql_sql_execution( return; } - let results = chat2db_web::legacy::execute_mysql_sql( + let results = Box::pin(chat2db_web::legacy::execute_mysql_sql( &state.application, &request, cancellation.clone(), &execution_id, "SQL_EDITOR_JCEF", - ) + )) .await; if cancellation.is_cancelled() { let _ = emit_legacy_sql_event( @@ -1290,15 +1657,23 @@ async fn legacy_request_for(application: &Application, request: &str) -> Result< .cloned() .unwrap_or(serde_json::Value::Null); - let response = chat2db_web::legacy::dispatch( - application, - chat2db_web::legacy::LegacyDispatchRequest { - request_url: request_url.clone(), - method: method.clone(), - message, - }, - ) - .await; + let response = + match chat2db_web::legacy_ai::dispatch(application, &method, &request_url, message.clone()) + .await + { + Some(response) => response, + None => { + chat2db_web::legacy::dispatch_desktop( + application, + chat2db_web::legacy::LegacyDispatchRequest { + request_url: request_url.clone(), + method: method.clone(), + message, + }, + ) + .await + } + }; Ok(serde_json::json!({ "uuid": uuid, @@ -2194,11 +2569,12 @@ mod tests { use super::{ BUNDLED_COMMUNITY_CLASSPATH, BUNDLED_DRIVER_PACKS, BUNDLED_JAVA_BIN, - BUNDLED_JAVA_ENGINE_JAR, BundledRuntimeResources, DesktopError, + BUNDLED_JAVA_ENGINE_JAR, BundledRuntimeResources, DesktopError, FilePath, LegacySqlCancellationRegistry, RuntimeResourceOverrides, SubscriptionRegistry, agent_stream_message, build_community_dml_for, build_community_namespace_sql_for, client_command_response, complete_community_sql_for, decode_client_message, - format_community_sql_for, legacy_request_for, legacy_sql_push_message, + format_community_sql_for, legacy_ai_push_message, legacy_file_extensions, + legacy_request_for, legacy_selected_file, legacy_sql_push_message, legacy_sql_rowless_payload, operation_stream_message, parse_after_sequence, resolve_runtime_resource_paths, start_community_table_preview_for, validate_community_sql_for, validate_java_engine_jar, validate_optional_os_env, @@ -2253,6 +2629,86 @@ mod tests { assert_eq!(response["data"]["executionId"], "operation-1"); } + #[test] + fn community_file_command_extensions_are_bounded_and_normalized() { + assert_eq!( + legacy_file_extensions(&[ + ".sql".to_owned(), + "*.csv".to_owned(), + "sql".to_owned(), + "tar.gz".to_owned(), + ]) + .expect("supported extensions must normalize"), + ["sql", "csv", "tar.gz"] + ); + assert!(legacy_file_extensions(&["../pem".to_owned()]).is_err()); + assert!(legacy_file_extensions(&[String::new()]).is_err()); + assert!(legacy_file_extensions(&vec!["sql".to_owned(); 65]).is_err()); + } + + #[test] + fn community_selected_files_match_the_jcef_shape_and_enforce_size() { + let directory = tempfile::tempdir().expect("temporary directory"); + let selected_path = directory.path().join("inventory.sql"); + fs::write(&selected_path, "SELECT 1;").expect("selected file fixture"); + + let selected = legacy_selected_file(FilePath::from(selected_path.clone()), Some(1)) + .expect("small selected file must pass"); + assert_eq!(selected.file_name, "inventory.sql"); + assert_eq!(selected.file_path, selected_path.to_string_lossy()); + + let response: serde_json::Value = serde_json::from_str(&client_command_response( + &serde_json::to_value([selected]).expect("selected file must serialize"), + )) + .expect("client-command response must serialize"); + assert_eq!(response["data"][0]["fileName"], "inventory.sql"); + assert_eq!( + response["data"][0]["filePath"].as_str(), + selected_path.to_str() + ); + + let oversized_path = directory.path().join("oversized.csv"); + let oversized = File::create(&oversized_path).expect("oversized fixture"); + oversized + .set_len(1024 * 1024 + 1) + .expect("oversized fixture length"); + let error = legacy_selected_file(FilePath::from(oversized_path), Some(1)) + .expect_err("oversized selected file must fail"); + assert!(error.contains("exceeds the 1 MB size limit")); + + let error = legacy_selected_file(FilePath::from(directory.path().to_path_buf()), None) + .expect_err("directories must not pass as files"); + assert_eq!(error, "The selected path is not a regular file"); + } + + #[test] + fn legacy_ai_push_message_matches_the_retained_desktop_event_shape() { + let envelope = AgentEventEnvelope { + run_id: "run-1".to_owned(), + sequence: "2".to_owned(), + occurred_at_ms: "1700000000000".to_owned(), + event: AgentEvent::TextDelta { + delta: "hello".to_owned(), + }, + }; + let chunk = chat2db_web::legacy_ai::project_agent_event(&envelope, "session-1") + .expect("text delta must project"); + let payload = legacy_ai_push_message("request-1", &chunk); + + assert_eq!(payload["uuid"], "request-1"); + assert_eq!(payload["actionType"], "ai_sse_message"); + assert_eq!(payload["message"]["event"], "answer"); + let data: serde_json::Value = serde_json::from_str( + payload["message"]["data"] + .as_str() + .expect("desktop SSE data must be a JSON string"), + ) + .expect("desktop SSE data must decode"); + assert_eq!(data["type"], "answer"); + assert_eq!(data["messageType"], "answer"); + assert_eq!(data["content"], "hello"); + } + #[test] fn community_sql_push_message_matches_the_existing_event_bus_contract() { let message = legacy_sql_push_message( @@ -2347,6 +2803,36 @@ mod tests { assert!(response["message"]["errorMessage"].is_null()); } + #[tokio::test] + async fn legacy_request_dispatches_dashboard_routes_through_the_generic_envelope() { + let response = legacy_request_for( + &Application::new(), + r#"{ + "actionType":"execute", + "uuid":"dashboard-request-1", + "requestUrl":"/api/dashboard/list?pageNo=1&pageSize=20", + "method":"get", + "message":{"pageNo":1,"pageSize":20,"searchKey":""} + }"#, + ) + .await + .expect("dashboard request must be serialized"); + let response: serde_json::Value = + serde_json::from_str(&response).expect("legacy response must be JSON"); + + assert_eq!(response["uuid"], "dashboard-request-1"); + assert_eq!(response["actionType"], "execute"); + assert_eq!( + response["requestUrl"], + "/api/dashboard/list?pageNo=1&pageSize=20" + ); + assert_eq!(response["method"], "get"); + assert!(response["param"].is_null()); + assert_eq!(response["message"]["success"], false); + assert_eq!(response["message"]["errorCode"], "storage_unavailable"); + assert!(response["message"]["data"].is_null()); + } + #[tokio::test] async fn legacy_request_keeps_correlation_fields_for_dispatch_failures() { let response = legacy_request_for( diff --git a/apps/chat2db-mcp/Cargo.toml b/apps/chat2db-mcp/Cargo.toml index a4f5a1f..c15b61c 100644 --- a/apps/chat2db-mcp/Cargo.toml +++ b/apps/chat2db-mcp/Cargo.toml @@ -16,15 +16,19 @@ path = "src/main.rs" chat2db-contract = { path = "../../crates/chat2db-contract" } chat2db-local = { path = "../../crates/chat2db-local" } clap.workspace = true +hex.workspace = true +rand.workspace = true rmcp.workspace = true serde.workspace = true serde_json.workspace = true +sha2.workspace = true tokio.workspace = true tracing-subscriber.workspace = true [dev-dependencies] chat2db-core = { path = "../../crates/chat2db-core" } chat2db-storage = { path = "../../crates/chat2db-storage" } +rmcp = { workspace = true, features = ["client"] } tempfile = "3" [lints] diff --git a/apps/chat2db-mcp/src/lib.rs b/apps/chat2db-mcp/src/lib.rs index 08881f3..f66db9d 100644 --- a/apps/chat2db-mcp/src/lib.rs +++ b/apps/chat2db-mcp/src/lib.rs @@ -1,14 +1,28 @@ //! Bounded MCP tools attached to the running local `Chat2DB` product host. -use chat2db_contract::{ApiError, QueryLimits, ResultPageRequest, StartQueryRequest}; +use std::{ + collections::HashMap, + fmt, + sync::{Arc, Mutex}, + time::{Duration, Instant}, +}; + +use chat2db_contract::{ + ApiError, DatabaseWriteResult, DatabaseWriteState, ExecuteDatabaseWriteRequest, QueryLimits, + ResultPageRequest, StartQueryRequest, +}; use chat2db_local::{LocalClient, LocalError}; +use rand::RngCore as _; use rmcp::{ - ServerHandler, + Peer, RoleServer, ServerHandler, handler::server::wrapper::Parameters, model::{CallToolResult, Implementation, ServerCapabilities, ServerInfo}, - schemars, tool, tool_handler, tool_router, + schemars, + service::ElicitationMode, + tool, tool_handler, tool_router, }; use serde::{Deserialize, Serialize}; +use sha2::{Digest as _, Sha256}; const MAX_QUERY_ROWS: u64 = 10_000; const MAX_QUERY_RESULT_BYTES: u64 = 16 * 1024 * 1024; @@ -19,17 +33,260 @@ const DEFAULT_PAGE_ROWS: u64 = 100; const DEFAULT_PAGE_BYTES: u64 = 256 * 1024; const MAX_PAGE_ROWS: u64 = 1_000; const MAX_PAGE_BYTES: u64 = 512 * 1024; +const WRITE_APPROVAL_TTL: Duration = Duration::from_secs(5 * 60); +const WRITE_ELICITATION_TIMEOUT: Duration = Duration::from_secs(2 * 60); +const MAX_PENDING_WRITE_APPROVALS: usize = 256; +const MAX_ELICITATION_DATASOURCE_CHARS: usize = 128; +const MAX_ELICITATION_SQL_CHARS: usize = 512; + +/// One externally authorized, exact database-write capability. +pub struct DatabaseWriteApproval { + approval_id: String, + datasource_id: String, + sql_sha256: String, +} + +impl DatabaseWriteApproval { + /// Returns an opaque host-side receipt id that is never accepted by MCP tool arguments. + #[must_use] + pub fn approval_id(&self) -> &str { + &self.approval_id + } + + /// Returns the datasource id bound to this capability. + #[must_use] + pub fn datasource_id(&self) -> &str { + &self.datasource_id + } + + /// Returns the lowercase SHA-256 digest of the exact approved SQL bytes. + #[must_use] + pub fn sql_sha256(&self) -> &str { + &self.sql_sha256 + } +} + +impl fmt::Debug for DatabaseWriteApproval { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("DatabaseWriteApproval") + .field("approval_id", &"[REDACTED]") + .field("datasource_id", &self.datasource_id) + .field("sql_sha256", &self.sql_sha256) + .finish() + } +} + +/// Trusted host-side authority for minting database-write capabilities. +/// +/// This handle is deliberately not exposed as an MCP tool. Product UI or CLI code +/// may hold it after completing an approval interaction, while the model-facing +/// server receives only the consuming half of the registry. +#[derive(Clone)] +pub struct WriteApprovalAuthority { + registry: WriteApprovalRegistry, +} + +impl WriteApprovalAuthority { + /// Authorizes one exact datasource and SQL byte sequence for one use. + /// + /// # Errors + /// + /// Returns an error for empty bindings, an unavailable registry, or a full + /// pending-approval queue. + pub fn approve_database_write( + &self, + datasource_id: &str, + sql: &str, + ) -> Result> { + self.registry.issue(datasource_id, sql) + } +} + +impl fmt::Debug for WriteApprovalAuthority { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("WriteApprovalAuthority") + .finish_non_exhaustive() + } +} + +#[derive(Clone, Default)] +struct WriteApprovalRegistry { + pending: Arc>>, +} + +struct WriteApprovalBinding { + datasource_id: String, + sql_sha256: [u8; 32], + expires_at: Instant, +} + +impl fmt::Debug for WriteApprovalRegistry { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("WriteApprovalRegistry") + .finish_non_exhaustive() + } +} + +impl WriteApprovalRegistry { + fn issue( + &self, + datasource_id: &str, + sql: &str, + ) -> Result> { + if datasource_id.trim().is_empty() || sql.trim().is_empty() { + return Err(Box::new(ApiError::new( + "invalid_database_write_approval", + "Database write approvals require a datasource and SQL", + ))); + } + let now = Instant::now(); + let mut pending = self.pending.lock().map_err(|_| { + Box::new(ApiError::new( + "database_write_approval_unavailable", + "Database write approval is unavailable", + )) + })?; + pending.retain(|_, binding| binding.expires_at > now); + if pending.len() >= MAX_PENDING_WRITE_APPROVALS { + return Err(Box::new(ApiError::new( + "database_write_approval_capacity_reached", + "Too many database write approvals are pending", + ))); + } + + let sql_sha256 = sha256(sql.as_bytes()); + loop { + let mut token_bytes = [0_u8; 32]; + rand::rng().fill_bytes(&mut token_bytes); + let token = hex::encode(token_bytes); + let token_sha256 = sha256(token.as_bytes()); + if pending.contains_key(&token_sha256) { + continue; + } + pending.insert( + token_sha256, + WriteApprovalBinding { + datasource_id: datasource_id.to_owned(), + sql_sha256, + expires_at: now + WRITE_APPROVAL_TTL, + }, + ); + return Ok(DatabaseWriteApproval { + approval_id: token, + datasource_id: datasource_id.to_owned(), + sql_sha256: hex::encode(sql_sha256), + }); + } + } + + fn consume(&self, token: &str, datasource_id: &str, sql: &str) -> bool { + if token.len() != 64 || !token.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return false; + } + let token_sha256 = sha256(token.as_bytes()); + let Ok(mut pending) = self.pending.lock() else { + return false; + }; + let Some(binding) = pending.remove(&token_sha256) else { + return false; + }; + binding.expires_at > Instant::now() + && binding.datasource_id == datasource_id + && binding.sql_sha256 == sha256(sql.as_bytes()) + } + + fn consume_matching(&self, datasource_id: &str, sql: &str) -> bool { + let now = Instant::now(); + let sql_sha256 = sha256(sql.as_bytes()); + let Ok(mut pending) = self.pending.lock() else { + return false; + }; + pending.retain(|_, binding| binding.expires_at > now); + let matching_token = pending.iter().find_map(|(token, binding)| { + (binding.datasource_id == datasource_id && binding.sql_sha256 == sql_sha256) + .then_some(*token) + }); + matching_token + .and_then(|token| pending.remove(&token)) + .is_some() + } +} + +fn sha256(value: &[u8]) -> [u8; 32] { + Sha256::digest(value).into() +} /// MCP service backed by the same application instance used by Web and desktop. #[derive(Debug, Clone)] pub struct McpServer { local: LocalClient, + write_approvals: WriteApprovalRegistry, } impl McpServer { #[must_use] - pub const fn new(local: LocalClient) -> Self { - Self { local } + pub fn new(local: LocalClient) -> Self { + Self { + local, + write_approvals: WriteApprovalRegistry::default(), + } + } + + /// Builds a server plus a separate trusted handle that can approve exact writes. + #[must_use] + pub fn with_write_approval_authority(local: LocalClient) -> (Self, WriteApprovalAuthority) { + let registry = WriteApprovalRegistry::default(); + let authority = WriteApprovalAuthority { + registry: registry.clone(), + }; + ( + Self { + local, + write_approvals: registry, + }, + authority, + ) + } + + async fn obtain_external_write_approval( + &self, + client: &Peer, + input: &WriteDatabaseInput, + ) -> bool { + if self + .write_approvals + .consume_matching(&input.datasource_id, &input.sql) + { + return true; + } + if !client + .supported_elicitation_modes() + .contains(&ElicitationMode::Form) + { + return false; + } + + let response = client + .elicit_with_timeout::( + database_write_elicitation_message(&input.datasource_id, &input.sql), + Some(WRITE_ELICITATION_TIMEOUT), + ) + .await; + if !matches!( + response, + Ok(Some(DatabaseWriteApprovalForm { confirm: true })) + ) { + return false; + } + + let Ok(approval) = self.write_approvals.issue(&input.datasource_id, &input.sql) else { + return false; + }; + self.write_approvals + .consume(approval.approval_id(), &input.datasource_id, &input.sql) } } @@ -84,6 +341,36 @@ impl McpServer { } } + /// Execute one externally approved database write without automatic retries. + #[tool( + name = "execute_database_write", + description = "Request trusted-host approval through MCP elicitation, then execute exactly one MySQL write. Approval is bound to the exact datasource and SQL digest and cannot be supplied by tool arguments. Clients without form elicitation fail closed. Only not_started is safe to retry after correction; never retry failed or unknown blindly.", + annotations( + read_only_hint = false, + destructive_hint = true, + idempotent_hint = false, + open_world_hint = false + ) + )] + async fn execute_database_write( + &self, + Parameters(input): Parameters, + client: Peer, + ) -> CallToolResult { + if !self.obtain_external_write_approval(&client, &input).await { + return database_write_approval_required(); + } + let result = self + .local + .execute_database_write(ExecuteDatabaseWriteRequest { + datasource_id: input.datasource_id, + sql: input.sql, + confirmed: true, + }) + .await; + structured_write_result(&result) + } + /// Inspect the current lifecycle state of one query operation. #[tool( name = "inspect_query_operation", @@ -162,7 +449,7 @@ impl ServerHandler for McpServer { env!("CARGO_PKG_VERSION"), )) .with_instructions( - "Use asynchronous read-only queries, poll operations, and page retained results.", + "Use asynchronous read-only queries by default. Database writes trigger trusted-host form elicitation showing the datasource, exact SQL SHA-256, and a bounded SQL preview. Tool arguments cannot approve writes, and clients without elicitation fail closed. Only not_started is safe to retry after correction; never retry failed or unknown blindly.", ) } } @@ -185,6 +472,42 @@ struct QueryDatabaseInput { result_ttl_seconds: Option, } +#[derive(Debug, Deserialize, schemars::JsonSchema)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct WriteDatabaseInput { + /// Opaque datasource id returned by `list_datasources`. + datasource_id: String, + /// Exactly one `MySQL` DML, DDL, grant, or routine statement. + sql: String, +} + +#[derive(Debug, Deserialize, schemars::JsonSchema)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct DatabaseWriteApprovalForm { + /// Set true only after the human has inspected the datasource, SQL digest, and preview. + confirm: bool, +} + +rmcp::elicit_safe!(DatabaseWriteApprovalForm); + +fn database_write_elicitation_message(datasource_id: &str, sql: &str) -> String { + format!( + "Approve one database write requested by an untrusted model.\nDatasource ID: {}\nExact SQL SHA-256: {}\nBounded SQL preview: {}\nSet confirm to true only after checking all three values.", + bounded_elicitation_text(datasource_id, MAX_ELICITATION_DATASOURCE_CHARS), + hex::encode(sha256(sql.as_bytes())), + bounded_elicitation_text(sql, MAX_ELICITATION_SQL_CHARS), + ) +} + +fn bounded_elicitation_text(value: &str, maximum_chars: usize) -> String { + let mut escaped = value.chars().flat_map(char::escape_default); + let mut output = escaped.by_ref().take(maximum_chars).collect::(); + if escaped.next().is_some() { + output.push_str("..."); + } + output +} + #[derive(Debug, Deserialize, schemars::JsonSchema)] #[serde(rename_all = "camelCase", deny_unknown_fields)] struct OperationInput { @@ -248,6 +571,26 @@ fn structured_success(value: &impl Serialize) -> CallToolResult { } } +fn structured_write_result(value: &DatabaseWriteResult) -> CallToolResult { + let succeeded = value.state == DatabaseWriteState::Succeeded; + match serde_json::to_value(value) { + Ok(value) if succeeded => CallToolResult::structured(value), + Ok(value) => CallToolResult::structured_error(value), + Err(_) => encoding_failure(), + } +} + +fn database_write_approval_required() -> CallToolResult { + structured_write_result(&DatabaseWriteResult { + state: DatabaseWriteState::NotStarted, + affected_rows: None, + error: Some(ApiError::new( + "database_write_approval_required", + "Explicit trusted-host approval bound to this exact datasource and SQL is required", + )), + }) +} + fn structured_local_error(error: LocalError) -> CallToolResult { structured_api_error(safe_local_error(error)) } @@ -301,17 +644,30 @@ fn encoding_failure() -> CallToolResult { #[cfg(test)] mod tests { - use std::sync::Arc; + use std::{ + collections::VecDeque, + sync::{Arc, Mutex}, + }; use chat2db_contract::CreateDatasourceRequest; use chat2db_core::Application; use chat2db_local::LocalServer; use chat2db_storage::{SecretRef, SecretValue, SecretVault, SecretVaultError, Storage}; - use rmcp::ServerHandler as _; - use rmcp::handler::server::wrapper::Parameters; + use rmcp::{ + ClientHandler, RoleClient, ServerHandler as _, ServiceExt as _, + handler::server::wrapper::Parameters, + model::{ + CallToolRequestParams, ClientCapabilities, ClientInfo, ElicitRequestParams, + ElicitResult, ElicitationAction, + }, + service::RequestContext, + }; use tempfile::TempDir; - use super::{McpServer, OperationInput, QueryDatabaseInput, ResultPageInput}; + use super::{ + McpServer, OperationInput, QueryDatabaseInput, ResultPageInput, WriteApprovalAuthority, + WriteDatabaseInput, + }; struct EmptyVault; @@ -337,6 +693,93 @@ mod tests { } } + #[derive(Clone)] + struct ApprovalClient { + decisions: Arc>>, + messages: Arc>>, + } + + #[derive(Clone, Copy)] + enum ApprovalDecision { + Confirm(bool), + Decline, + Cancel, + InvalidContent, + ProtocolError, + } + + impl ApprovalClient { + fn new(decisions: impl IntoIterator) -> Self { + Self::with_decisions(decisions.into_iter().map(ApprovalDecision::Confirm)) + } + + fn with_decisions(decisions: impl IntoIterator) -> Self { + Self { + decisions: Arc::new(Mutex::new(decisions.into_iter().collect())), + messages: Arc::new(Mutex::new(Vec::new())), + } + } + + fn messages(&self) -> Vec { + self.messages.lock().expect("messages lock").clone() + } + } + + impl ClientHandler for ApprovalClient { + fn get_info(&self) -> ClientInfo { + let mut info = ClientInfo::default(); + info.capabilities = ClientCapabilities::builder().enable_elicitation().build(); + info + } + + async fn create_elicitation( + &self, + request: ElicitRequestParams, + _context: RequestContext, + ) -> Result { + let ElicitRequestParams::FormElicitationParams { + message, + requested_schema, + .. + } = request + else { + return Err(rmcp::ErrorData::invalid_params( + "database write approval requires form elicitation", + None, + )); + }; + assert!(requested_schema.properties.contains_key("confirm")); + assert!( + requested_schema + .required + .as_ref() + .is_some_and(|required| required.iter().any(|field| field == "confirm")) + ); + self.messages.lock().expect("messages lock").push(message); + let decision = self + .decisions + .lock() + .expect("decisions lock") + .pop_front() + .unwrap_or(ApprovalDecision::Cancel); + if matches!(decision, ApprovalDecision::ProtocolError) { + return Err(rmcp::ErrorData::internal_error( + "elicitation transport failed", + None, + )); + } + Ok(match decision { + ApprovalDecision::Confirm(confirm) => ElicitResult::new(ElicitationAction::Accept) + .with_content(serde_json::json!({ "confirm": confirm })), + ApprovalDecision::Decline => ElicitResult::new(ElicitationAction::Decline), + ApprovalDecision::Cancel => ElicitResult::new(ElicitationAction::Cancel), + ApprovalDecision::InvalidContent => ElicitResult::new(ElicitationAction::Accept) + .with_content(serde_json::json!({ "confirm": "not-a-boolean" })), + ApprovalDecision::ProtocolError => unreachable!("returned above"), + }) + } + } + struct Fixture { server: LocalServer, application: Application, @@ -346,18 +789,39 @@ mod tests { impl Fixture { fn new() -> Self { + Self::build(false).0 + } + + fn with_write_approval_authority() -> (Self, WriteApprovalAuthority) { + let (fixture, authority) = Self::build(true); + ( + fixture, + authority.expect("write approval authority must be constructed"), + ) + } + + fn build(with_authority: bool) -> (Self, Option) { let directory = TempDir::new().expect("temp dir"); let storage = Storage::open(directory.path(), Arc::new(EmptyVault)).expect("storage opens"); let application = Application::with_storage(storage); let server = LocalServer::start(application.clone()).expect("local server starts"); - let mcp = McpServer::new(chat2db_local::LocalClient::new(directory.path())); - Self { - server, - application, - mcp, - _directory: directory, - } + let local = chat2db_local::LocalClient::new(directory.path()); + let (mcp, authority) = if with_authority { + let (mcp, authority) = McpServer::with_write_approval_authority(local); + (mcp, Some(authority)) + } else { + (McpServer::new(local), None) + }; + ( + Self { + server, + application, + mcp, + _directory: directory, + }, + authority, + ) } async fn shutdown(mut self) { @@ -493,6 +957,236 @@ mod tests { } } + #[test] + fn model_confirmation_boolean_cannot_authorize_a_write() { + let old_input = serde_json::json!({ + "datasourceId": "datasource-1", + "sql": "UPDATE items SET label = 'changed'", + "confirm": true + }); + assert!(serde_json::from_value::(old_input).is_err()); + + let forged_token = serde_json::json!({ + "datasourceId": "datasource-1", + "sql": "UPDATE items SET label = 'changed'", + "approvalToken": "model-controlled-token" + }); + assert!(serde_json::from_value::(forged_token).is_err()); + } + + #[test] + fn external_write_approvals_are_exact_single_use_capabilities() { + let (server, authority) = + McpServer::with_write_approval_authority(chat2db_local::LocalClient::new("unused")); + let datasource_id = "datasource-1"; + let sql = "UPDATE items SET label = 'changed'"; + + let changed_sql_approval = authority + .approve_database_write(datasource_id, sql) + .expect("trusted host can approve a write"); + assert_eq!(changed_sql_approval.datasource_id(), datasource_id); + assert_eq!(changed_sql_approval.approval_id().len(), 64); + assert_eq!( + changed_sql_approval.sql_sha256(), + hex::encode(super::sha256(sql.as_bytes())) + ); + let approval_debug = format!("{changed_sql_approval:?}"); + assert!(!approval_debug.contains(changed_sql_approval.approval_id())); + assert!(approval_debug.contains("[REDACTED]")); + + assert!(!server.write_approvals.consume( + changed_sql_approval.approval_id(), + datasource_id, + &format!("{sql} ") + )); + assert!(!server.write_approvals.consume( + changed_sql_approval.approval_id(), + datasource_id, + sql + )); + + let datasource_approval = authority + .approve_database_write(datasource_id, sql) + .expect("trusted host can approve a second write"); + assert!(!server.write_approvals.consume( + datasource_approval.approval_id(), + "datasource-2", + sql + )); + + let valid_approval = authority + .approve_database_write(datasource_id, sql) + .expect("trusted host can approve an exact write"); + assert!( + server + .write_approvals + .consume(valid_approval.approval_id(), datasource_id, sql) + ); + assert!( + !server + .write_approvals + .consume(valid_approval.approval_id(), datasource_id, sql) + ); + + let _concurrent_approval = authority + .approve_database_write(datasource_id, sql) + .expect("trusted host can approve a concurrent write"); + let first_registry = server.write_approvals.clone(); + let second_registry = first_registry.clone(); + let barrier = Arc::new(std::sync::Barrier::new(3)); + let first_barrier = barrier.clone(); + let second_barrier = barrier.clone(); + let (first, second) = std::thread::scope(|scope| { + let first = scope.spawn(move || { + first_barrier.wait(); + first_registry.consume_matching(datasource_id, sql) + }); + let second = scope.spawn(move || { + second_barrier.wait(); + second_registry.consume_matching(datasource_id, sql) + }); + barrier.wait(); + ( + first.join().expect("first approval consumer"), + second.join().expect("second approval consumer"), + ) + }); + assert_eq!( + usize::from(first) + usize::from(second), + 1, + "atomic consumption must allow exactly one concurrent caller past approval" + ); + } + + #[tokio::test] + async fn stdio_without_elicitation_fails_closed_but_accepts_host_preapproval() { + let (fixture, authority) = Fixture::with_write_approval_authority(); + let (server_transport, client_transport) = tokio::io::duplex(16 * 1024); + let mcp = fixture.mcp.clone(); + let server_task = tokio::spawn(async move { + mcp.serve(server_transport) + .await + .expect("MCP server handshake") + .waiting() + .await + .expect("MCP server shutdown"); + }); + let client = ().serve(client_transport).await.expect("MCP client handshake"); + + let read = client + .call_tool(CallToolRequestParams::new("list_datasources")) + .await + .expect("read tool call"); + assert_eq!(read.is_error, Some(false)); + + let datasource_id = "datasource-1"; + let sql = "UPDATE items SET label = 'changed'"; + let unapproved = call_write_tool(&client, datasource_id, sql).await; + assert_write_error_code(&unapproved, "database_write_approval_required"); + + let _approval = authority + .approve_database_write(datasource_id, sql) + .expect("trusted embedding host can preapprove"); + let preapproved = call_write_tool(&client, datasource_id, sql).await; + assert_ne!( + write_error_code(&preapproved), + "database_write_approval_required" + ); + let replay = call_write_tool(&client, datasource_id, sql).await; + assert_write_error_code(&replay, "database_write_approval_required"); + + client.cancel().await.expect("MCP client cancellation"); + server_task.await.expect("MCP server task"); + fixture.shutdown().await; + } + + #[tokio::test] + async fn stdio_elicitation_requires_explicit_true_for_the_exact_write() { + let fixture = Fixture::new(); + let approval_client = ApprovalClient::new([false, true]); + let observed_client = approval_client.clone(); + let (server_transport, client_transport) = tokio::io::duplex(16 * 1024); + let mcp = fixture.mcp.clone(); + let server_task = tokio::spawn(async move { + mcp.serve(server_transport) + .await + .expect("MCP server handshake") + .waiting() + .await + .expect("MCP server shutdown"); + }); + let client = approval_client + .serve(client_transport) + .await + .expect("elicitation client handshake"); + + let datasource_id = "datasource-1"; + let sql = "UPDATE items SET label = 'changed'"; + let declined = call_write_tool(&client, datasource_id, sql).await; + assert_write_error_code(&declined, "database_write_approval_required"); + let approved = call_write_tool(&client, datasource_id, sql).await; + assert_ne!( + write_error_code(&approved), + "database_write_approval_required", + "accepted elicitation with confirm=true must cross the approval boundary" + ); + + let messages = observed_client.messages(); + assert_eq!(messages.len(), 2); + for message in messages { + assert!(message.contains(datasource_id)); + assert!(message.contains(&hex::encode(super::sha256(sql.as_bytes())))); + assert!(message.contains(&super::bounded_elicitation_text( + sql, + super::MAX_ELICITATION_SQL_CHARS, + ))); + } + + client.cancel().await.expect("MCP client cancellation"); + server_task.await.expect("MCP server task"); + fixture.shutdown().await; + } + + #[tokio::test] + async fn stdio_elicitation_decline_cancel_invalid_content_and_errors_fail_closed() { + let fixture = Fixture::new(); + let approval_client = ApprovalClient::with_decisions([ + ApprovalDecision::Confirm(false), + ApprovalDecision::Decline, + ApprovalDecision::Cancel, + ApprovalDecision::InvalidContent, + ApprovalDecision::ProtocolError, + ]); + let (server_transport, client_transport) = tokio::io::duplex(16 * 1024); + let mcp = fixture.mcp.clone(); + let server_task = tokio::spawn(async move { + mcp.serve(server_transport) + .await + .expect("MCP server handshake") + .waiting() + .await + .expect("MCP server shutdown"); + }); + let client = approval_client + .serve(client_transport) + .await + .expect("elicitation client handshake"); + + for _ in 0..5 { + let result = call_write_tool( + &client, + "datasource-1", + "UPDATE items SET label = 'changed'", + ) + .await; + assert_write_error_code(&result, "database_write_approval_required"); + } + + client.cancel().await.expect("MCP client cancellation"); + server_task.await.expect("MCP server task"); + fixture.shutdown().await; + } + #[test] fn publishes_stable_tool_names_and_safety_annotations() { let info = McpServer::new(chat2db_local::LocalClient::new("unused")).get_info(); @@ -508,6 +1202,7 @@ mod tests { .collect::>(), [ "cancel_database_query", + "execute_database_write", "inspect_query_operation", "inspect_query_result", "list_datasources", @@ -517,11 +1212,17 @@ mod tests { for tool in &tools { let annotations = tool.annotations.as_ref().expect("annotations exist"); - assert_eq!(annotations.destructive_hint, Some(false)); + assert_eq!( + annotations.destructive_hint, + Some(tool.name == "execute_database_write") + ); assert_eq!(annotations.open_world_hint, Some(false)); assert_eq!( annotations.read_only_hint, - Some(tool.name != "cancel_database_query") + Some(!matches!( + tool.name.as_ref(), + "cancel_database_query" | "execute_database_write" + )) ); } let cancellation = tools @@ -550,6 +1251,47 @@ mod tests { properties["resultTtlSeconds"]["maximum"], serde_json::json!(super::MAX_QUERY_RESULT_TTL_SECONDS) ); + + let write = tools + .iter() + .find(|tool| tool.name == "execute_database_write") + .unwrap(); + assert_eq!( + write.annotations.as_ref().unwrap().idempotent_hint, + Some(false) + ); + let required = write.input_schema["required"].as_array().unwrap(); + assert!(required.iter().any(|field| field == "datasourceId")); + assert!(required.iter().any(|field| field == "sql")); + assert!(!required.iter().any(|field| field == "confirm")); + assert!(!required.iter().any(|field| field == "approvalToken")); + assert!(write.input_schema["properties"].get("confirm").is_none()); + assert!( + write.input_schema["properties"] + .get("approvalToken") + .is_none() + ); + } + + async fn call_write_tool( + client: &rmcp::service::RunningService, + datasource_id: &str, + sql: &str, + ) -> rmcp::model::CallToolResult { + client + .call_tool( + CallToolRequestParams::new("execute_database_write").with_arguments( + serde_json::json!({ + "datasourceId": datasource_id, + "sql": sql + }) + .as_object() + .expect("write arguments must be an object") + .clone(), + ), + ) + .await + .expect("write tool call") } fn assert_error_code(result: &rmcp::model::CallToolResult, expected: &str) { @@ -559,4 +1301,15 @@ mod tests { expected ); } + + fn write_error_code(result: &rmcp::model::CallToolResult) -> &str { + result.structured_content.as_ref().unwrap()["error"]["code"] + .as_str() + .expect("write error code must be a string") + } + + fn assert_write_error_code(result: &rmcp::model::CallToolResult, expected: &str) { + assert_eq!(result.is_error, Some(true)); + assert_eq!(write_error_code(result), expected); + } } diff --git a/apps/chat2db-web/Cargo.toml b/apps/chat2db-web/Cargo.toml index 4c859e8..5943fa1 100644 --- a/apps/chat2db-web/Cargo.toml +++ b/apps/chat2db-web/Cargo.toml @@ -19,15 +19,20 @@ chat2db-local = { path = "../../crates/chat2db-local" } chat2db-storage = { path = "../../crates/chat2db-storage" } chrono.workspace = true futures-util.workspace = true +quick-xml.workspace = true +reqwest.workspace = true serde.workspace = true serde_json.workspace = true subtle.workspace = true tokio.workspace = true +tokio-util.workspace = true tower-http.workspace = true tracing.workspace = true tracing-subscriber.workspace = true utoipa.workspace = true utoipa-axum.workspace = true +xls.workspace = true +zip.workspace = true [dev-dependencies] http-body-util.workspace = true @@ -36,6 +41,7 @@ serde.workspace = true serde_json.workspace = true tempfile = "3" tower.workspace = true +url.workspace = true uuid.workspace = true [lints] diff --git a/apps/chat2db-web/src/api.rs b/apps/chat2db-web/src/api.rs index 4287c39..4563d97 100644 --- a/apps/chat2db-web/src/api.rs +++ b/apps/chat2db-web/src/api.rs @@ -310,6 +310,7 @@ pub(crate) fn router(application: Application) -> Router { router .merge(crate::legacy::routes()) + .merge(crate::legacy_ai::routes()) .layer(Extension(Arc::new(document))) .with_state(application) } diff --git a/apps/chat2db-web/src/legacy.rs b/apps/chat2db-web/src/legacy.rs index eb59888..480211c 100644 --- a/apps/chat2db-web/src/legacy.rs +++ b/apps/chat2db-web/src/legacy.rs @@ -5,17 +5,18 @@ //! response translations without duplicating datasource or JDBC behavior. use std::{ - collections::{BTreeMap, HashSet}, + collections::{BTreeMap, HashMap, HashSet}, fs, future::Future, + io::Write as _, path::PathBuf, time::{Duration, Instant, SystemTime, UNIX_EPOCH}, }; use axum::{ Json, Router, - body::Body, - extract::{Query, State}, + body::{Body, Bytes}, + extract::{DefaultBodyLimit, FromRequest as _, Multipart, Query, State}, http::{StatusCode, header}, middleware, response::{IntoResponse, Response}, @@ -23,21 +24,41 @@ use axum::{ }; use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD}; use chat2db_contract::{ - ApiError, ColumnNullability, CommunityFunction, CommunityProcedure, - CommunityRoutineInvocationPreview, CommunityTable, CommunityTableColumn, CommunityTableIndex, - CommunityTableIndexColumn, CommunityTrigger, Datasource, DatasourceConnection, - DatasourceConnectionProperty, DatasourceSecretChange, GetCommunityFunctionRequest, - GetCommunityProcedureRequest, GetCommunityTriggerRequest, JdbcDriver, JdbcValue, JdbcValueType, - ListCommunityColumnsRequest, ListCommunityDatabasesRequest, ListCommunityFunctionsRequest, - ListCommunityIndexesRequest, ListCommunityProceduresRequest, ListCommunitySchemasRequest, - ListCommunityTablesRequest, ListCommunityTriggersRequest, ListCommunityViewsRequest, - OperationEvent, PreviewCommunityRoutineInvocationRequest, QueryAccepted, QueryLimits, - ResultColumn, ResultMetadata, ResultPageRequest, StartCommunityTablePreviewRequest, - StartQueryRequest, UpdateDatasourceRequest, + ApiError, CloneDatasourceRequest, ColumnNullability, CommunityAccount, CommunityAccountAction, + CommunityAccountCapability, CommunityAccountCommandRequest, CommunityAccountExecution, + CommunityAccountGrantsRequest, CommunityAccountPreview, CommunityAccountPrivilegeScope, + CommunityChart, CommunityChartDetailQuery, CommunityDashboard, CommunityDashboardListQuery, + CommunityDashboardPage, CommunityDatabase, CommunityDatasourceExport, + CommunityDatasourceFileImportRequest, CommunityDatasourceFileImportResult, + CommunityDatasourceImportFormat, CommunityErModel, CommunityErPositionRequest, + CommunityErQueryRequest, CommunityFunction, CommunityPinnedTableRequest, CommunityProcedure, + CommunityRoutineInvocationPreview, CommunityRoutineMigrationExecution, + CommunityRoutineMigrationRequest, CommunitySchema, CommunitySchemaDiffRequest, + CommunitySqlCompletionActiveSnippetSlot, CommunityTable, CommunityTableColumn, + CommunityTableIndex, CommunityTableIndexColumn, CommunityTrigger, CompleteCommunitySqlRequest, + CreateCommunityChartRequest, CreateCommunityDashboardRequest, CreateWorkspaceNamespaceRequest, + DatasourceConnection, DatasourceConnectionProperty, DatasourceEditProjection, + DatasourceSecretChange, DmlExportFormat, DmlExportRequest, DmlExportSize, + ExportCommunityDatasourcesRequest, FormatCommunitySqlRequest, GenerateMysqlClassRequest, + GetCommunityFunctionRequest, GetCommunityProcedureRequest, GetCommunityTriggerRequest, + ImportFileRequest, JdbcDriver, JdbcValue, JdbcValueType, ListCommunityColumnsRequest, + ListCommunityDatabasesRequest, ListCommunityFunctionsRequest, ListCommunityIndexesRequest, + ListCommunityProceduresRequest, ListCommunitySchemasRequest, ListCommunityTablesRequest, + ListCommunityTriggersRequest, ListCommunityViewsRequest, MoveWorkspaceNodeRequest, + NativeDriverAction, OperationEvent, OtherFileExportRequest, + PreviewCommunityRoutineInvocationRequest, QueryAccepted, QueryLimits, ResultColumn, + ResultMetadata, ResultPageRequest, SqlFileExportRequest, SshAuthentication, + SshAuthenticationType, SshHostKeyVerification, SshTunnelConfig, + StartCommunityTablePreviewRequest, StartQueryRequest, TabularImportEncoding, + TransferFileFormat, TransferSqlScope, TransferTask, TransferTaskKind, TransferTaskStatus, + UpdateCommunityChartRequest, UpdateCommunityDashboardRequest, UpdateDatasourceRequest, + UpdateWorkspaceNamespaceRequest, ValidateCommunitySqlRequest, WorkspaceNodeKind, + WorkspaceNodeRef, WorkspaceTreeNode, }; use chat2db_core::{ - AppError, Application, LargeValueChunk, LargeValueEncoding, LargeValuePreview, LargeValueType, - MysqlConsoleCancellation, MysqlConsoleRequest, MysqlConsoleResult, + AppError, AppErrorKind, Application, LargeValueChunk, LargeValueEncoding, LargeValuePreview, + LargeValueType, MysqlConsoleCancellation, MysqlConsoleRequest, MysqlConsoleResult, + TransferArtifactDownload, mysql_ddl::{ MysqlColumnAlter, MysqlColumnDefinition, MysqlColumnPosition, MysqlDatabaseDefinition, MysqlIndexAlter, MysqlIndexColumn, MysqlIndexDefinition, MysqlIndexKind, MysqlIndexMethod, @@ -60,6 +81,7 @@ use chat2db_storage::{ }; use chrono::{Local, TimeZone as _}; use serde::{Deserialize, Deserializer, Serialize, de::DeserializeOwned}; +use tokio_util::io::ReaderStream; const DEFAULT_PAGE_NO: u32 = 1; const DEFAULT_PAGE_SIZE: u32 = 20; @@ -71,6 +93,10 @@ const MAX_SQL_ROWS: u32 = 10_000; const LARGE_VALUE_CHUNK_SIZE: u32 = 256 * 1024; const LARGE_VALUE_FALLBACK_PREVIEW_BYTES: usize = 64 * 1024; const RESULT_PAGE_MAX_BYTES: u64 = 8 * 1024 * 1024; +const MAX_LEGACY_DATASOURCE_IMPORT_BYTES: usize = 16 * 1024 * 1024; +const MAX_LEGACY_MULTIPART_BYTES: usize = MAX_LEGACY_DATASOURCE_IMPORT_BYTES + 1024 * 1024; +const LEGACY_IMPORT_UPLOAD_TTL_MS: i64 = 24 * 60 * 60 * 1_000; +const LEGACY_IMPORT_CLEANUP_POLL: Duration = Duration::from_millis(250); const LARGE_VALUE_PREVIEW_PREFIX: &str = "CHAT2DB_LARGE_VALUE_PREVIEW:"; const PREVIEW_TIMEOUT: Duration = Duration::from_secs(30); const SQL_EXECUTION_TIMEOUT: Duration = Duration::from_secs(60); @@ -260,10 +286,14 @@ pub struct LegacyDatasourceRequest { pub extend_info: Vec, #[serde(default)] pub read_only: bool, + #[serde(default)] + pub ssh: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] +// These independent flags are fixed fields in Community's historical datasource DTO. +#[allow(clippy::struct_excessive_bools)] pub struct LegacyDatasourceResponse { pub id: String, pub alias: String, @@ -272,6 +302,9 @@ pub struct LegacyDatasourceResponse { pub url: String, pub user: String, pub password: String, + pub read_only: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub ssh: Option, pub environment: LegacyEnvironment, pub environment_id: u64, pub extend_info: Vec, @@ -301,6 +334,133 @@ pub struct LegacyPage { pub has_next_page: bool, } +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacyImportFileRequest { + pub data_source_id: LegacyIdentifier, + #[serde(default)] + pub database_name: String, + #[serde(default)] + pub schema_name: String, + #[serde(default)] + pub table_name: String, + #[serde(default)] + pub file_name: String, + #[serde(default)] + pub import_type: String, + #[serde(default = "default_true")] + pub contains_header: bool, + #[serde(default)] + pub tabular_encoding: TabularImportEncoding, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacySqlFileExportRequest { + pub data_source_id: LegacyIdentifier, + #[serde(default)] + pub database_name: String, + #[serde(default)] + pub schema_name: String, + #[serde(default)] + pub table_name: String, + #[serde(default)] + pub table_names: Vec, + #[serde(default)] + pub scope: String, + #[serde(default)] + pub contain_data: bool, + #[serde(default)] + pub export_path: String, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacyOtherFileExportRequest { + pub data_source_id: LegacyIdentifier, + #[serde(default)] + pub database_name: String, + #[serde(default)] + pub schema_name: String, + #[serde(default)] + pub table_name: String, + #[serde(default)] + pub table_names: Vec, + #[serde(default)] + pub export_type: String, + #[serde(default = "default_true")] + pub contains_header: bool, + #[serde(default)] + pub export_path: String, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacyDmlExportRequest { + pub data_source_id: LegacyIdentifier, + #[serde(default)] + pub database_name: String, + #[serde(default)] + pub schema_name: String, + #[serde(default)] + pub sql: String, + #[serde(default)] + pub original_sql: String, + #[serde(default)] + pub result_set_id: Option, + #[serde(default)] + pub export_size: String, + #[serde(default)] + pub export_type: String, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacyGenerateClassRequest { + pub data_source_id: LegacyIdentifier, + #[serde(default)] + pub database_name: String, + #[serde(default)] + pub schema_name: String, + #[serde(default)] + pub table_name: String, + #[serde(default)] + pub export_path: String, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacyTaskListQuery { + #[serde(default = "default_page_no")] + pub page_no: u32, + #[serde(default = "default_page_size")] + pub page_size: u32, + #[serde(default)] + pub task_status: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacyTransferTask { + pub id: i64, + pub gmt_create: u64, + pub gmt_modified: u64, + pub data_source_id: String, + pub database_name: String, + pub schema_name: String, + pub table_name: Option, + pub task_type: String, + pub task_status: String, + pub task_progress: String, + pub progress: String, + pub progress_desc: String, + pub current_progress: String, + pub task_name: String, + pub download_url: String, + pub info_log: String, + pub error_log: String, +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct LegacyNamespaceNode { @@ -308,10 +468,47 @@ pub struct LegacyNamespaceNode { #[serde(rename = "type")] pub node_type: String, pub name: String, - pub data: LegacyDatasourceResponse, + pub data: serde_json::Value, pub children: Vec, } +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacyProgressResponse { + pub message: String, + pub count: usize, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacyDatasourceFileUploadRequest { + pub file: serde_json::Value, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacyDatagripUploadRequest { + #[serde(default)] + pub text: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacyDatasourceUploadResponse { + pub result: String, + pub count: u32, +} + +struct LegacyMultipartFile { + file_name: String, + content: Vec, +} + +struct LegacyMysqlMultipartUpload { + request: LegacyImportFileRequest, + upload: LegacyMultipartFile, +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct LegacyDatabase { @@ -328,6 +525,20 @@ pub struct LegacySchema { pub system: bool, } +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacyMetaDatabase { + pub name: String, + pub schemas: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacyMetaSchemaResponse { + pub databases: Vec, + pub schemas: Vec, +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct LegacyTable { @@ -907,6 +1118,28 @@ pub struct LegacyPageQuery { pub search_key: String, } +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacyCommunityDashboardUpdateRequest { + pub id: i64, + #[serde(flatten)] + pub update: UpdateCommunityDashboardRequest, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacyCommunityChartUpdateRequest { + pub id: i64, + #[serde(flatten)] + pub update: UpdateCommunityChartRequest, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacyCommunityEntityQuery { + pub id: i64, +} + #[derive(Debug, Clone, Default, Deserialize)] #[serde(rename_all = "camelCase")] pub struct LegacySavedConsoleCreateRequest { @@ -1119,57 +1352,111 @@ pub struct LegacyDriverQuery { #[derive(Debug, Clone, Deserialize)] #[serde(rename_all = "camelCase")] -pub struct LegacyTableDdlExampleQuery { +pub struct LegacyDriverMutationRequest { + #[serde(default)] pub db_type: String, + #[serde(default)] + pub jdbc_driver_class: String, + #[serde(default)] + pub jdbc_driver: Vec, } #[derive(Debug, Clone, Deserialize)] #[serde(rename_all = "camelCase")] -pub struct LegacyMetadataQuery { - pub data_source_id: LegacyIdentifier, - #[serde(default)] - pub database_name: String, - #[serde(default)] - pub schema_name: String, +pub struct LegacyCloneDatasourceRequest { + pub id: LegacyIdentifier, #[serde(default)] - pub database_type: String, + pub name: Option, } #[derive(Debug, Clone, Deserialize)] #[serde(rename_all = "camelCase")] -pub struct LegacyTableListQuery { +pub struct LegacyConsoleConnectQuery { pub data_source_id: LegacyIdentifier, #[serde(default)] + pub console_id: Option, + #[serde(default)] pub database_name: String, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacySshTestRequest { + #[serde(default, rename = "use")] + pub enabled: bool, #[serde(default)] - pub schema_name: String, + pub host_name: String, #[serde(default)] - pub database_type: String, - #[serde(default = "default_page_no")] - pub page_no: u32, - #[serde(default = "default_page_size")] - pub page_size: u32, + pub port: String, #[serde(default)] - pub search_key: String, + pub user_name: String, + #[serde(default)] + pub local_port: String, + #[serde(default)] + pub authentication_type: String, + #[serde(default)] + pub password: String, + #[serde(default)] + pub key_file: String, + #[serde(default)] + pub passphrase: String, + #[serde(default)] + pub r_host: String, + #[serde(default)] + pub r_port: String, } #[derive(Debug, Clone, Deserialize)] #[serde(rename_all = "camelCase")] -pub struct LegacyTableDetailQuery { - pub data_source_id: LegacyIdentifier, +pub struct LegacyNamespaceRequest { #[serde(default)] - pub database_name: String, + pub id: Option, #[serde(default)] - pub schema_name: String, + pub name: String, #[serde(default)] - pub database_type: String, + pub parent_id: Option, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacyWorkspaceNodeRef { + pub id: LegacyIdentifier, + #[serde(rename = "type")] + pub node_type: String, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacyWorkspaceMoveRequest { + pub drag_node: LegacyWorkspaceNodeRef, + pub drop_to_node: LegacyWorkspaceNodeRef, + pub drop_position: i8, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacyDatasourceAssignmentRequest { + pub data_source_id: LegacyIdentifier, #[serde(default)] - pub table_name: String, + pub namespace_id: Option, +} + +#[derive(Debug, Clone, Default, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacyDatasourceExportRequest { + #[serde(default)] + pub datasource_ids: Option>, } #[derive(Debug, Clone, Deserialize)] #[serde(rename_all = "camelCase")] -pub struct LegacyFunctionDetailQuery { +pub struct LegacyTableDdlExampleQuery { + pub db_type: String, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacyMetadataQuery { pub data_source_id: LegacyIdentifier, #[serde(default)] pub database_name: String, @@ -1177,65 +1464,270 @@ pub struct LegacyFunctionDetailQuery { pub schema_name: String, #[serde(default)] pub database_type: String, - #[serde(default)] - pub function_name: String, } #[derive(Debug, Clone, Deserialize)] #[serde(rename_all = "camelCase")] -pub struct LegacyProcedureDetailQuery { +pub struct LegacyErPositionRequest { pub data_source_id: LegacyIdentifier, #[serde(default)] pub database_name: String, #[serde(default)] pub schema_name: String, #[serde(default)] - pub database_type: String, - #[serde(default)] - pub procedure_name: String, + pub position: String, } -/// Invocation-preview payload used by Community's routine dialog. #[derive(Debug, Clone, Deserialize)] #[serde(rename_all = "camelCase")] -pub struct LegacyRoutineInvocationRequest { +pub struct LegacyAccountQuery { pub data_source_id: LegacyIdentifier, - #[serde(default, deserialize_with = "deserialize_string_or_default")] - pub database_name: String, - #[serde(default, deserialize_with = "deserialize_string_or_default")] - pub schema_name: String, - #[serde(default, deserialize_with = "deserialize_string_or_default")] - pub database_type: String, - #[serde(default, deserialize_with = "deserialize_string_or_default")] - pub routine_type: String, - #[serde(default, deserialize_with = "deserialize_string_or_default")] - pub routine_name: String, + #[serde(default)] + pub user: String, + #[serde(default)] + pub host: String, } -#[derive(Debug, Clone, Deserialize)] +#[derive(Clone, Deserialize)] #[serde(rename_all = "camelCase")] -pub struct LegacyTriggerDetailQuery { +pub struct LegacyAccountCommandRequest { pub data_source_id: LegacyIdentifier, #[serde(default)] - pub database_name: String, + pub user: String, #[serde(default)] - pub schema_name: String, + pub host: String, + pub action_type: CommunityAccountAction, #[serde(default)] - pub database_type: String, + pub scope: Option, #[serde(default)] - pub trigger_name: String, + pub database_name: Option, + #[serde(default)] + pub table_name: Option, + #[serde(default)] + pub privileges: Vec, + #[serde(default)] + pub grant_option: bool, + #[serde(default)] + pub password: Option, + #[serde(default)] + pub preview_token: Option, +} + +impl LegacyAccountCommandRequest { + fn core_request(&self) -> CommunityAccountCommandRequest { + CommunityAccountCommandRequest { + datasource_id: self.data_source_id.as_string(), + user: self.user.clone(), + host: self.host.clone(), + action_type: self.action_type, + scope: self.scope, + database_name: self.database_name.clone(), + table_name: self.table_name.clone(), + privileges: self.privileges.clone(), + grant_option: self.grant_option, + password: self.password.clone(), + preview_token: self.preview_token.clone(), + } + } } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Deserialize)] #[serde(rename_all = "camelCase")] -pub struct LegacyTablePreviewRequest { - pub data_source_id: LegacyIdentifier, - #[serde(default, deserialize_with = "deserialize_string_or_default")] - pub database_name: String, - #[serde(default, deserialize_with = "deserialize_string_or_default")] - pub schema_name: String, - #[serde(default, deserialize_with = "deserialize_string_or_default")] - pub database_type: String, +pub struct LegacySqlUtilityRequest { + #[serde(default, deserialize_with = "deserialize_string_or_default")] + pub sql: String, + #[serde( + default, + alias = "databaseType", + deserialize_with = "deserialize_string_or_default" + )] + pub db_type: String, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacySqlParserRequest { + pub data_source_id: LegacyIdentifier, + #[serde(default)] + pub console_id: Option, + #[serde(default, deserialize_with = "deserialize_string_or_default")] + pub database_name: String, + #[serde(default, deserialize_with = "deserialize_string_or_default")] + pub schema_name: String, + #[serde(default, deserialize_with = "deserialize_string_or_default")] + pub sql: String, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacySqlCompletionActiveSnippetSlot { + #[serde( + default, + rename = "type", + deserialize_with = "deserialize_string_or_default" + )] + pub slot_type: String, + #[serde(default)] + pub replace_start: Option, + #[serde(default)] + pub replace_end: Option, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacySqlCompletionRequest { + pub data_source_id: LegacyIdentifier, + #[serde(default)] + pub console_id: Option, + #[serde(default, deserialize_with = "deserialize_string_or_default")] + pub database_name: String, + #[serde(default, deserialize_with = "deserialize_string_or_default")] + pub schema_name: String, + #[serde(default, deserialize_with = "deserialize_string_or_default")] + pub sql: String, + #[serde(default, deserialize_with = "deserialize_string_or_default")] + pub before_sql: String, + #[serde(default, deserialize_with = "deserialize_string_or_default")] + pub after_sql: String, + #[serde(default)] + pub cursor: Option, + #[serde(default)] + pub need_full_name: bool, + #[serde(default, deserialize_with = "deserialize_string_or_default")] + pub keyword_case: String, + #[serde(default)] + pub active_snippet_slot: Option, +} + +#[derive(Debug, Clone, Default, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct LegacySqlIdentifier { + pub name: String, + pub alias: String, + #[serde(rename = "type")] + pub identifier_type: String, + pub identifier_database: String, + pub identifier_schema: String, + pub identifier_table: String, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacySqlHoverRequest { + pub data_source_id: LegacyIdentifier, + #[serde(default)] + pub console_id: Option, + #[serde(default, deserialize_with = "deserialize_string_or_default")] + pub database_name: String, + #[serde(default, deserialize_with = "deserialize_string_or_default")] + pub schema_name: String, + #[serde(default)] + pub hover_identifier: LegacySqlIdentifier, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacyTableListQuery { + pub data_source_id: LegacyIdentifier, + #[serde(default)] + pub database_name: String, + #[serde(default)] + pub schema_name: String, + #[serde(default)] + pub database_type: String, + #[serde(default = "default_page_no")] + pub page_no: u32, + #[serde(default = "default_page_size")] + pub page_size: u32, + #[serde(default)] + pub search_key: String, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacyTableDetailQuery { + pub data_source_id: LegacyIdentifier, + #[serde(default)] + pub database_name: String, + #[serde(default)] + pub schema_name: String, + #[serde(default)] + pub database_type: String, + #[serde(default)] + pub table_name: String, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacyFunctionDetailQuery { + pub data_source_id: LegacyIdentifier, + #[serde(default)] + pub database_name: String, + #[serde(default)] + pub schema_name: String, + #[serde(default)] + pub database_type: String, + #[serde(default)] + pub function_name: String, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacyProcedureDetailQuery { + pub data_source_id: LegacyIdentifier, + #[serde(default)] + pub database_name: String, + #[serde(default)] + pub schema_name: String, + #[serde(default)] + pub database_type: String, + #[serde(default)] + pub procedure_name: String, +} + +/// Invocation-preview payload used by Community's routine dialog. +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacyRoutineInvocationRequest { + pub data_source_id: LegacyIdentifier, + #[serde(default, deserialize_with = "deserialize_string_or_default")] + pub database_name: String, + #[serde(default, deserialize_with = "deserialize_string_or_default")] + pub schema_name: String, + #[serde(default, deserialize_with = "deserialize_string_or_default")] + pub database_type: String, + #[serde(default, deserialize_with = "deserialize_string_or_default")] + pub routine_type: String, + #[serde(default, deserialize_with = "deserialize_string_or_default")] + pub routine_name: String, + #[serde(default, deserialize_with = "deserialize_string_or_default")] + pub ddl: String, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacyTriggerDetailQuery { + pub data_source_id: LegacyIdentifier, + #[serde(default)] + pub database_name: String, + #[serde(default)] + pub schema_name: String, + #[serde(default)] + pub database_type: String, + #[serde(default)] + pub trigger_name: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacyTablePreviewRequest { + pub data_source_id: LegacyIdentifier, + #[serde(default, deserialize_with = "deserialize_string_or_default")] + pub database_name: String, + #[serde(default, deserialize_with = "deserialize_string_or_default")] + pub schema_name: String, + #[serde(default, deserialize_with = "deserialize_string_or_default")] + pub database_type: String, #[serde(default, deserialize_with = "deserialize_string_or_default")] pub table_name: String, #[serde(default = "default_page_no")] @@ -1365,6 +1857,10 @@ fn default_page_size() -> u32 { DEFAULT_PAGE_SIZE } +const fn default_true() -> bool { + true +} + fn default_preview_page_size() -> u32 { DEFAULT_PREVIEW_PAGE_SIZE } @@ -1454,18 +1950,121 @@ pub fn drivers(application: &Application, requested_type: &str) -> LegacyDriverR } } +pub(crate) async fn list_community_dashboards( + application: &Application, + query: CommunityDashboardListQuery, +) -> LegacyResult { + application + .list_community_dashboards(query) + .await + .map_err(LegacyFailure::from) +} + +pub(crate) async fn get_community_dashboard( + application: &Application, + id: i64, +) -> LegacyResult> { + application + .get_community_dashboard(id) + .await + .map_err(LegacyFailure::from) +} + +pub(crate) async fn create_community_dashboard( + application: &Application, + request: CreateCommunityDashboardRequest, +) -> LegacyResult { + application + .create_community_dashboard(request) + .await + .map_err(LegacyFailure::from) +} + +pub(crate) async fn update_community_dashboard( + application: &Application, + request: LegacyCommunityDashboardUpdateRequest, +) -> LegacyResult<()> { + application + .update_community_dashboard(request.id, request.update) + .await + .map_err(LegacyFailure::from) +} + +pub(crate) async fn delete_community_dashboard( + application: &Application, + id: i64, +) -> LegacyResult { + application + .delete_community_dashboard(id) + .await + .map(|_| "success".to_owned()) + .map_err(LegacyFailure::from) +} + +pub(crate) async fn get_community_chart( + application: &Application, + id: i64, +) -> LegacyResult> { + application + .get_community_chart(id) + .await + .map_err(LegacyFailure::from) +} + +pub(crate) async fn get_community_chart_detail( + application: &Application, + query: CommunityChartDetailQuery, +) -> LegacyResult> { + application + .get_community_chart_detail(query.chart_id, query.refresh) + .await + .map_err(LegacyFailure::from) +} + +pub(crate) async fn create_community_chart( + application: &Application, + request: CreateCommunityChartRequest, +) -> LegacyResult { + application + .create_community_chart(request) + .await + .map_err(LegacyFailure::from) +} + +pub(crate) async fn update_community_chart( + application: &Application, + request: LegacyCommunityChartUpdateRequest, +) -> LegacyResult<()> { + application + .update_community_chart(request.id, request.update) + .await + .map_err(LegacyFailure::from) +} + +pub(crate) async fn delete_community_chart( + application: &Application, + id: i64, +) -> LegacyResult { + application + .delete_community_chart(id) + .await + .map(|_| "success".to_owned()) + .map_err(LegacyFailure::from) +} + /// Lists and paginates datasource records using the old page DTO. pub(crate) async fn list_datasources( application: &Application, query: &LegacyPageQuery, ) -> LegacyResult> { - let mut items: Vec = application - .list_datasources() - .await? - .items - .into_iter() - .map(|datasource| datasource_response(application, datasource)) - .collect(); + let datasources = application.list_datasources().await?; + let mut items = Vec::with_capacity(datasources.items.len()); + for datasource in datasources.items { + let projection = application + .get_datasource_edit_projection(&datasource.id) + .await?; + items.push(datasource_response(application, projection)); + } if !query.search_key.trim().is_empty() { let needle = query.search_key.to_lowercase(); items.retain(|item| item.alias.to_lowercase().contains(&needle)); @@ -1478,8 +2077,10 @@ pub(crate) async fn get_datasource( application: &Application, id: &LegacyIdentifier, ) -> LegacyResult { - let datasource = application.get_datasource(&id.as_string()).await?; - Ok(datasource_response(application, datasource)) + let projection = application + .get_datasource_edit_projection(&id.as_string()) + .await?; + Ok(datasource_response(application, projection)) } /// Creates a datasource while keeping JDBC material inside Core's vault path. @@ -1497,7 +2098,17 @@ pub(crate) async fn create_datasource( connection: Some(connection), }) .await?; - Ok(datasource_response(application, datasource)) + let projection = application + .get_datasource_edit_projection(&datasource.id) + .await?; + let mut response = datasource_response(application, projection); + // Create responses never echo request connection material. The edit/list APIs provide the + // separately sanitized projection after the caller refreshes its datasource state. + response.url.clear(); + response.user.clear(); + response.extend_info.clear(); + response.ssh = None; + Ok(response) } /// Tests a saved or unsaved datasource without persisting an unsaved request. @@ -1557,7 +2168,7 @@ pub(crate) async fn update_datasource( request.alias.trim().to_owned() }; let datasource = application - .update_datasource( + .update_datasource_preserving_secrets( &id, UpdateDatasourceRequest { expected_revision: existing.revision, @@ -1567,7 +2178,10 @@ pub(crate) async fn update_datasource( }, ) .await?; - Ok(datasource_response(application, datasource)) + let projection = application + .get_datasource_edit_projection(&datasource.id) + .await?; + Ok(datasource_response(application, projection)) } /// Deletes a datasource using the latest revision hidden by the old API. @@ -1583,2082 +2197,3798 @@ pub(crate) async fn delete_datasource( Ok(()) } -pub(crate) async fn create_saved_console( +pub(crate) async fn clone_datasource( application: &Application, - request: &LegacySavedConsoleCreateRequest, -) -> LegacyResult { - let storage = legacy_storage(application)?; - let input = CreateSavedConsole { - id: request.id, - name: request.name.clone(), - data_source_id: request - .data_source_id - .as_ref() - .map(LegacyIdentifier::as_string), - data_source_name: request.data_source_name.clone(), - database_name: request.database_name.clone(), - schema_name: request.schema_name.clone(), - database_type: request.database_type.clone(), - ddl: request.ddl.clone(), - status: default_if_blank(&request.status, "DRAFT"), - // Community always opens a newly created Console. - tab_opened: "y".to_owned(), - operation_type: default_if_blank(&request.operation_type, "console"), - }; - legacy_storage_call(move || storage.create_saved_console(input)) - .await - .map(|record| record.id) + request: &LegacyCloneDatasourceRequest, +) -> LegacyResult { + Ok(application + .clone_datasource(CloneDatasourceRequest { + id: request.id.as_string(), + name: request.name.clone(), + }) + .await? + .id) } -pub(crate) async fn get_saved_console( +pub(crate) async fn connect_datasource( application: &Application, - id: i64, -) -> LegacyResult> { - let storage = legacy_storage(application)?; - legacy_storage_call(move || storage.get_saved_console(id)) - .await - .map(|record| record.map(saved_console_response)) + id: &LegacyIdentifier, +) -> LegacyResult> { + Ok(application + .connect_datasource_compatibility(&id.as_string(), "") + .await? + .databases + .into_iter() + .map(|database| LegacyDatabase { + name: database.name, + description: database.comment, + count: 0, + system: database.system, + }) + .collect()) } -pub(crate) async fn list_saved_consoles( +pub(crate) async fn close_datasource( application: &Application, - query: &LegacySavedConsoleListQuery, -) -> LegacyResult> { - let storage = legacy_storage(application)?; - let storage_query = SavedConsoleListQuery { - data_source_id: query - .data_source_id - .as_ref() - .map(LegacyIdentifier::as_string), - database_name: query.database_name.clone(), - schema_name: query.schema_name.clone(), - status: query.status.clone(), - tab_opened: query.tab_opened.clone(), - operation_type: query.operation_type.clone(), - search_key: query.search_key.clone(), - page_no: query.page_no, - page_size: query.page_size, - order_by_desc: query.order_by_desc, - }; - legacy_storage_call(move || storage.list_saved_consoles(&storage_query)) - .await - .map(|page| { - let total = usize::try_from(page.total).unwrap_or(usize::MAX); - let data = page - .records - .into_iter() - .map(saved_console_response) - .collect::>(); - LegacyPage { - has_next_page: u64::from(page.page_no) * u64::from(page.page_size) < page.total, - data, - page_no: page.page_no, - page_size: page.page_size, - total, - } - }) + id: &LegacyIdentifier, +) -> LegacyResult<()> { + application + .close_datasource_compatibility(&id.as_string()) + .await?; + Ok(()) } -pub(crate) async fn update_saved_console( +pub(crate) async fn connect_console( application: &Application, - request: LegacySavedConsoleUpdateRequest, + request: &LegacyConsoleConnectQuery, ) -> LegacyResult<()> { - let storage = legacy_storage(application)?; - let input = UpdateSavedConsole { - name: required_string_patch(request.name), - data_source_id: identifier_patch(request.data_source_id), - data_source_name: nullable_string_patch(request.data_source_name), - database_name: nullable_string_patch(request.database_name), - schema_name: nullable_string_patch(request.schema_name), - database_type: nullable_string_patch(request.database_type), - ddl: required_string_patch(request.ddl), - status: required_string_patch(request.status), - tab_opened: required_string_patch(request.tab_opened), - operation_type: required_string_patch(request.operation_type), + application + .connect_console_compatibility(&request.data_source_id.as_string()) + .await?; + Ok(()) +} + +pub(crate) fn native_driver_action( + application: &Application, + database_type: &str, + action: NativeDriverAction, +) -> LegacyResult<()> { + let database_type = if database_type.trim().is_empty() { + "MYSQL" + } else { + database_type }; - legacy_storage_call(move || storage.update_saved_console(request.id, input)) - .await - .map(|_| ()) + application.native_driver_compatibility(database_type, action)?; + Ok(()) } -pub(crate) async fn delete_saved_console(application: &Application, id: i64) -> LegacyResult<()> { - let storage = legacy_storage(application)?; - legacy_storage_call(move || storage.delete_saved_console(id)) - .await - .map(|_| ()) +pub(crate) async fn test_legacy_ssh( + application: &Application, + request: &LegacySshTestRequest, +) -> LegacyResult { + application + .test_ssh_connection(legacy_ssh_config(request)?) + .await?; + Ok(true) } -pub(crate) async fn create_operation_log( +pub(crate) async fn export_legacy_datasources( application: &Application, - request: &LegacyOperationLogCreateRequest, -) -> LegacyResult { - if request.ddl.trim().is_empty() { - return Err(LegacyFailure::invalid( - "invalid_operation_log", - "ddl is required", - )); - } - let operation_rows = request - .operation_rows - .map(i64::try_from) - .transpose() - .map_err(|_| { - LegacyFailure::invalid( - "invalid_operation_log", - "operationRows is outside the supported range", - ) - })?; - let use_time = request - .use_time - .map(i64::try_from) - .transpose() - .map_err(|_| { - LegacyFailure::invalid( - "invalid_operation_log", - "useTime is outside the supported range", - ) - })?; - let data_source_id = request - .data_source_id - .as_ref() - .map(LegacyIdentifier::as_string); - let storage = legacy_storage(application)?; - let input = CreateOperationLog { - name: non_blank(&request.name), - connectable: request - .connectable - .or_else(|| data_source_id.as_ref().map(|id| !id.trim().is_empty())), - data_source_id, - data_source_name: request.data_source_name.clone(), - database_name: request.database_name.clone(), - database_type: request.database_type.clone(), - ddl: request.ddl.clone(), - status: request - .status - .as_deref() - .map(str::trim) - .filter(|value| !value.is_empty()) - .unwrap_or("SUCCESS") - .to_owned(), - operation_rows, - use_time, - extend_info: request.extend_info.clone(), - schema_name: request.schema_name.clone(), - organization_id: request.organization_id, - user_name: request.user_name.clone(), - more: request.more, - operation_type: "SQL_EXECUTE".to_owned(), - }; - legacy_storage_call(move || storage.create_operation_log(input)) - .await - .map(|record| record.id) + request: &LegacyDatasourceExportRequest, +) -> LegacyResult { + let document = application + .export_community_datasources(ExportCommunityDatasourcesRequest { + datasource_ids: request + .datasource_ids + .clone() + .unwrap_or_default() + .into_iter() + .map(|id| id.as_string()) + .collect(), + }) + .await?; + Ok(LegacyProgressResponse { + count: document.datasources.len(), + message: serde_json::to_string(&document).map_err(|_| LegacyFailure { + code: "internal_error".to_owned(), + message: "The datasource export could not be encoded".to_owned(), + })?, + }) } -pub(crate) async fn get_operation_log( +pub(crate) async fn import_legacy_datasources( application: &Application, - id: i64, -) -> LegacyResult { - let storage = legacy_storage(application)?; - let record = legacy_storage_call(move || storage.get_operation_log(id)) - .await? - .ok_or_else(|| { - LegacyFailure::invalid( - "operation_log_not_found", - "The operation log does not exist", - ) - })?; - Ok(operation_log_response(record, false)) + document: CommunityDatasourceExport, +) -> LegacyResult { + let imported = application.import_community_datasources(document).await?; + Ok(LegacyProgressResponse { + count: usize::try_from(imported.count).unwrap_or(usize::MAX), + message: "success".to_owned(), + }) } -pub(crate) async fn list_operation_logs( +pub(crate) async fn import_installed_legacy_datasources( application: &Application, - query: &LegacyOperationLogListQuery, -) -> LegacyResult> { - let storage = legacy_storage(application)?; - let storage_query = OperationLogListQuery { - data_source_id: query - .data_source_id - .as_ref() - .map(LegacyIdentifier::as_string), - database_name: query.database_name.clone(), - schema_name: query.schema_name.clone(), - operation_type: query.operation_type.clone(), - search_key: non_blank(&query.search_key), - page_no: query.page_no, - page_size: query.page_size, +) -> LegacyResult { + let outcome = application.import_legacy_community_datasources().await?; + Ok(LegacyProgressResponse { + message: "success".to_owned(), + count: usize::try_from(outcome.imported).unwrap_or(usize::MAX), + }) +} + +pub(crate) async fn import_legacy_datasource_file_path( + application: &Application, + request: &LegacyDatasourceFileUploadRequest, + format: Option, +) -> LegacyResult { + let path = legacy_datasource_upload_path(&request.file)?; + let metadata = tokio::fs::metadata(&path) + .await + .map_err(|_| invalid_legacy_datasource_upload())?; + if !metadata.is_file() + || metadata.len() > u64::try_from(MAX_LEGACY_DATASOURCE_IMPORT_BYTES).unwrap_or(u64::MAX) + { + return Err(invalid_legacy_datasource_upload()); + } + let inferred = match format { + Some(format) => format, + None => legacy_datasource_import_format( + path.extension() + .and_then(|extension| extension.to_str()) + .unwrap_or_default(), + )?, }; - legacy_storage_call(move || storage.list_operation_logs(&storage_query)) + let content = tokio::fs::read(path) .await - .map(|page| { - let total = usize::try_from(page.total).unwrap_or(usize::MAX); - LegacyPage { - data: page - .records - .into_iter() - .map(|record| operation_log_response(record, true)) - .collect(), - page_no: page.page_no, - page_size: page.page_size, - total, - has_next_page: u64::from(page.page_no) * u64::from(page.page_size) < page.total, - } - }) + .map_err(|_| invalid_legacy_datasource_upload())?; + import_legacy_datasource_content(application, inferred, content).await } -/// Builds the flat namespace tree used when no custom grouping exists. -pub(crate) async fn namespace_tree( +pub(crate) async fn import_legacy_datagrip_text( application: &Application, -) -> LegacyResult> { - Ok(application - .list_datasources() - .await? - .items - .into_iter() - .map(|datasource| { - let response = datasource_response(application, datasource); - LegacyNamespaceNode { - id: response.id.clone(), - node_type: "DATA_SOURCE".to_owned(), - name: response.alias.clone(), - data: response, - children: Vec::new(), - } - }) - .collect()) + request: &LegacyDatagripUploadRequest, +) -> LegacyResult { + import_legacy_datasource_content( + application, + CommunityDatasourceImportFormat::DatagripText, + request.text.as_bytes().to_vec(), + ) + .await } -/// Lists databases through the retained Community metadata implementation. -pub(crate) async fn list_databases( +async fn import_legacy_datasource_content( application: &Application, - query: &LegacyMetadataQuery, -) -> LegacyResult> { - let datasource_id = query.data_source_id.as_string(); - let database_type = - resolve_database_type(application, &datasource_id, &query.database_type).await?; - Ok(application - .list_community_databases(ListCommunityDatabasesRequest { - datasource_id, - database_type, - }) - .await? - .items - .into_iter() - .map(|database| LegacyDatabase { - name: database.name, - description: database.comment, - count: 0, - system: database.system, - }) - .collect()) + format: CommunityDatasourceImportFormat, + content: Vec, +) -> LegacyResult { + if content.len() > MAX_LEGACY_DATASOURCE_IMPORT_BYTES { + return Err(invalid_legacy_datasource_upload()); + } + let result = application + .import_community_datasource_file(CommunityDatasourceFileImportRequest { format, content }) + .await?; + Ok(legacy_datasource_upload_response(&result)) } -/// Lists schemas through the retained Community metadata implementation. -pub(crate) async fn list_schemas( - application: &Application, - query: &LegacyMetadataQuery, -) -> LegacyResult> { - let datasource_id = query.data_source_id.as_string(); - let database_type = - resolve_database_type(application, &datasource_id, &query.database_type).await?; - Ok(application - .list_community_schemas(ListCommunitySchemasRequest { - datasource_id, - database_type, - database_name: query.database_name.clone(), - }) - .await? - .items - .into_iter() - .map(|schema| LegacySchema { - name: schema.name, - system: schema.system, - }) - .collect()) +fn legacy_datasource_upload_response( + result: &CommunityDatasourceFileImportResult, +) -> LegacyDatasourceUploadResponse { + LegacyDatasourceUploadResponse { + result: if result.skipped == 0 { + String::new() + } else { + format!( + "Imported {} MySQL connection(s); skipped {} unsupported connection(s).", + result.count, result.skipped + ) + }, + count: result.count, + } } -/// Lists and paginates tables through Community metadata. -pub(crate) async fn list_tables( - application: &Application, - query: &LegacyTableListQuery, -) -> LegacyResult> { - validate_metadata_page(query)?; - let datasource_id = query.data_source_id.as_string(); - let database_type = - resolve_database_type(application, &datasource_id, &query.database_type).await?; - let mut items: Vec = application - .list_community_tables(ListCommunityTablesRequest { - datasource_id, - database_type, - database_name: query.database_name.clone(), - schema_name: query.schema_name.clone(), - // Community's MySQL metadata implementation requires an empty - // pattern to enumerate all tables. - table_name_pattern: String::new(), - }) - .await? - .items - .into_iter() - .map(table_response) - .collect(); - items.retain(|item| table_matches_search(item, &query.search_key)); - Ok(paginate(items, query.page_no, query.page_size)) +fn legacy_datasource_upload_path(value: &serde_json::Value) -> LegacyResult { + let path = match value { + serde_json::Value::String(path) => Some(path.as_str()), + serde_json::Value::Array(paths) => paths.iter().find_map(serde_json::Value::as_str), + serde_json::Value::Object(file) => file + .get("path") + .or_else(|| file.get("filePath")) + .and_then(serde_json::Value::as_str), + _ => None, + } + .filter(|path| !path.trim().is_empty()) + .ok_or_else(invalid_legacy_datasource_upload)?; + Ok(PathBuf::from(path)) } -/// Lists the compact table projection used by autocomplete and table pickers. -pub(crate) async fn list_simple_tables( - application: &Application, - query: &LegacyTableListQuery, -) -> LegacyResult> { - validate_metadata_page(query)?; - let datasource_id = query.data_source_id.as_string(); - let database_type = - resolve_database_type(application, &datasource_id, &query.database_type).await?; - Ok(application - .list_community_tables(ListCommunityTablesRequest { - datasource_id, - database_type, - database_name: query.database_name.clone(), - schema_name: query.schema_name.clone(), - table_name_pattern: String::new(), - }) - .await? - .items - .into_iter() - .map(simple_table_response) - .collect()) +fn legacy_datasource_import_format( + extension: &str, +) -> LegacyResult { + match extension.trim().to_ascii_lowercase().as_str() { + "ncx" => Ok(CommunityDatasourceImportFormat::NavicatNcx), + "dbp" => Ok(CommunityDatasourceImportFormat::DbeaverDbp), + "json" => Ok(CommunityDatasourceImportFormat::Chat2dbJson), + _ => Err(LegacyFailure::invalid( + "unsupported_datasource_import_format", + "The datasource import file type is not supported", + )), + } } -/// Lists table or view columns in the historical `ColumnResponse` shape. -pub(crate) async fn list_columns( - application: &Application, - query: &LegacyTableDetailQuery, -) -> LegacyResult> { - let datasource_id = query.data_source_id.as_string(); - let database_type = - resolve_database_type(application, &datasource_id, &query.database_type).await?; - Ok(application - .list_community_columns(ListCommunityColumnsRequest { - datasource_id, - database_type, - database_name: query.database_name.clone(), - schema_name: query.schema_name.clone(), - table_name: query.table_name.clone(), - }) - .await? - .items - .into_iter() - .map(column_response) - .collect()) +fn invalid_legacy_datasource_upload() -> LegacyFailure { + LegacyFailure::invalid( + "invalid_datasource_import_file", + "The datasource import file is invalid", + ) } -/// Lists table indexes in the historical `IndexResponse` shape. -pub(crate) async fn list_indexes( - application: &Application, - query: &LegacyTableDetailQuery, -) -> LegacyResult> { - let datasource_id = query.data_source_id.as_string(); - let database_type = - resolve_database_type(application, &datasource_id, &query.database_type).await?; - Ok(application - .list_community_indexes(ListCommunityIndexesRequest { - datasource_id, - database_type, - database_name: query.database_name.clone(), - schema_name: query.schema_name.clone(), - table_name: query.table_name.clone(), - }) - .await? - .items - .into_iter() - .map(index_response) - .collect()) +async fn read_legacy_multipart_file(mut multipart: Multipart) -> LegacyResult { + while let Some(field) = multipart + .next_field() + .await + .map_err(|_| invalid_legacy_datasource_upload())? + { + if field.name() != Some("file") { + continue; + } + let file_name = field.file_name().unwrap_or_default().to_owned(); + let content = field + .bytes() + .await + .map_err(|_| invalid_legacy_datasource_upload())? + .to_vec(); + if content.len() > MAX_LEGACY_DATASOURCE_IMPORT_BYTES { + return Err(invalid_legacy_datasource_upload()); + } + return Ok(LegacyMultipartFile { file_name, content }); + } + Err(invalid_legacy_datasource_upload()) } -/// Community's historical key endpoint is an alias of its index metadata. -pub(crate) async fn list_keys( +async fn import_legacy_multipart_datasource( application: &Application, - query: &LegacyTableDetailQuery, -) -> LegacyResult> { - list_indexes(application, query).await + multipart: Multipart, + format: Option, +) -> LegacyResult { + let upload = read_legacy_multipart_file(multipart).await?; + let format = match format { + Some(format) => format, + None => legacy_datasource_import_format( + PathBuf::from(upload.file_name) + .extension() + .and_then(|extension| extension.to_str()) + .unwrap_or_default(), + )?, + }; + import_legacy_datasource_content(application, format, upload.content).await } -/// Returns the native `SHOW CREATE TABLE` result used by Community's export action. -pub(crate) async fn export_table_ddl( - application: &Application, - query: &LegacyTableDetailQuery, -) -> LegacyResult { - let datasource_id = query.data_source_id.as_string(); - resolve_mysql_database_type(application, &datasource_id, &query.database_type).await?; - Ok(application - .table_ddl( - &datasource_id, - &query.database_name, - &query.schema_name, - &query.table_name, - ) - .await?) +fn invalid_legacy_mysql_upload() -> LegacyFailure { + LegacyFailure::invalid( + "invalid_import_upload", + "The Web import file upload is invalid", + ) } -/// Preserves Community `MySQL`'s null create/alter example configuration. -pub(crate) fn mysql_table_ddl_example( - query: &LegacyTableDdlExampleQuery, -) -> LegacyResult> { - if normalize_database_type(&query.db_type) != "MYSQL" { - return Err(LegacyFailure { - code: "unsupported_database_type".to_owned(), - message: "This Community compatibility route currently supports MySQL only".to_owned(), - }); - } - Ok(None) +fn web_import_upload_required() -> LegacyFailure { + LegacyFailure::invalid( + "web_import_upload_required", + "Web imports require a multipart file upload", + ) } -/// Lists views in the same page wrapper used by the retained tree. -pub(crate) async fn list_views( - application: &Application, - query: &LegacyTableListQuery, -) -> LegacyResult> { - validate_metadata_page(query)?; - let datasource_id = query.data_source_id.as_string(); - let database_type = - resolve_database_type(application, &datasource_id, &query.database_type).await?; - let items = application - .list_community_views(ListCommunityViewsRequest { - datasource_id, - database_type, - database_name: query.database_name.clone(), - schema_name: query.schema_name.clone(), - view_name_pattern: String::new(), - }) - .await? - .items - .into_iter() - .map(table_response) - .collect(); - Ok(full_page(items)) +fn desktop_file_operation_required() -> LegacyFailure { + LegacyFailure::invalid( + "desktop_file_operation_required", + "Local file paths are available only through the Desktop IPC adapter", + ) } -/// Reads one view, including its DDL, through the exact Core detail path. -pub(crate) async fn get_view( - application: &Application, - query: &LegacyTableDetailQuery, -) -> LegacyResult { - let datasource_id = query.data_source_id.as_string(); - let database_type = - resolve_database_type(application, &datasource_id, &query.database_type).await?; - Ok(table_response( - application - .get_community_view(ListCommunityViewsRequest { - datasource_id, - database_type, - database_name: query.database_name.clone(), - schema_name: query.schema_name.clone(), - view_name_pattern: query.table_name.clone(), - }) - .await?, - )) +fn reject_web_export_path(path: &str) -> LegacyResult<()> { + if path.trim().is_empty() { + Ok(()) + } else { + Err(desktop_file_operation_required()) + } } -/// Reads the full table projection required by Community's table editor. -pub(crate) async fn get_editable_table( - application: &Application, - query: &LegacyTableDetailQuery, -) -> LegacyResult { - if query.table_name.trim().is_empty() { - return Err(LegacyFailure::invalid( - "invalid_table_query", - "tableName is required", - )); +async fn read_legacy_mysql_multipart( + mut multipart: Multipart, +) -> LegacyResult { + let mut data_source_id = None; + let mut database_name = String::new(); + let mut schema_name = String::new(); + let mut table_name = String::new(); + let mut import_type = String::new(); + let mut contains_header = true; + let mut tabular_encoding = TabularImportEncoding::Plain; + let mut upload = None; + + while let Some(field) = multipart + .next_field() + .await + .map_err(|_| invalid_legacy_mysql_upload())? + { + let name = field.name().unwrap_or_default().to_owned(); + if name == "file" { + if upload.is_some() { + return Err(invalid_legacy_mysql_upload()); + } + let file_name = field.file_name().unwrap_or_default().to_owned(); + let content = field + .bytes() + .await + .map_err(|_| invalid_legacy_mysql_upload())? + .to_vec(); + if content.len() > MAX_LEGACY_DATASOURCE_IMPORT_BYTES { + return Err(invalid_legacy_mysql_upload()); + } + upload = Some(LegacyMultipartFile { file_name, content }); + continue; + } + + let value = field + .text() + .await + .map_err(|_| invalid_legacy_mysql_upload())?; + match name.as_str() { + "dataSourceId" => data_source_id = non_blank(&value), + "databaseName" => database_name = value, + "schemaName" => schema_name = value, + "tableName" => table_name = value, + "importType" => import_type = value, + "tabularEncoding" => tabular_encoding = legacy_tabular_import_encoding(&value)?, + "containsHeader" => { + contains_header = match value.trim().to_ascii_lowercase().as_str() { + "true" | "1" => true, + "false" | "0" => false, + _ => return Err(invalid_legacy_mysql_upload()), + }; + } + _ => {} + } } - let datasource_id = query.data_source_id.as_string(); - let database_type = - resolve_mysql_database_type(application, &datasource_id, &query.database_type).await?; - let tables = application - .list_community_tables(ListCommunityTablesRequest { - datasource_id: datasource_id.clone(), - database_type: database_type.clone(), - database_name: query.database_name.clone(), - schema_name: query.schema_name.clone(), - table_name_pattern: query.table_name.clone(), - }) - .await? - .items; - let table = tables - .into_iter() - .find(|table| table.name.eq_ignore_ascii_case(&query.table_name)) - .ok_or_else(|| LegacyFailure { - code: "table_not_found".to_owned(), - message: format!("Table {} does not exist", query.table_name), - })?; - let (columns, indexes) = tokio::try_join!( - application.list_community_columns(ListCommunityColumnsRequest { - datasource_id: datasource_id.clone(), - database_type: database_type.clone(), - database_name: query.database_name.clone(), - schema_name: query.schema_name.clone(), - table_name: query.table_name.clone(), - }), - application.list_community_indexes(ListCommunityIndexesRequest { - datasource_id, - database_type: database_type.clone(), - database_name: query.database_name.clone(), - schema_name: query.schema_name.clone(), - table_name: query.table_name.clone(), - }), - )?; - Ok(editable_table_response( - table, - columns.items, - indexes.items, - database_type, - )) + + let data_source_id = data_source_id.ok_or_else(invalid_legacy_mysql_upload)?; + let upload = upload.ok_or_else(invalid_legacy_mysql_upload)?; + Ok(LegacyMysqlMultipartUpload { + request: LegacyImportFileRequest { + data_source_id: LegacyIdentifier::Text(data_source_id), + database_name, + schema_name, + table_name, + file_name: upload.file_name.clone(), + import_type, + contains_header, + tabular_encoding, + }, + upload, + }) } -/// Reads the full view projection used by Community's retained view editor. -pub(crate) async fn get_editable_view( +async fn import_legacy_mysql_multipart( application: &Application, - query: &LegacyTableDetailQuery, -) -> LegacyResult { - if query.table_name.trim().is_empty() { - return Err(LegacyFailure::invalid( - "invalid_view_query", - "tableName is required", - )); + multipart: Multipart, + sql_file_route: bool, +) -> LegacyResult { + let LegacyMysqlMultipartUpload { request, upload } = + read_legacy_mysql_multipart(multipart).await?; + let format = if sql_file_route { + TransferFileFormat::Sql + } else { + legacy_transfer_format(&request.import_type, "importType")? + }; + let (media_type, format_name) = match format { + TransferFileFormat::Csv => ("text/csv", "CSV"), + TransferFileFormat::Xls => ("application/vnd.ms-excel", "XLS"), + TransferFileFormat::Xlsx => ( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "XLSX", + ), + TransferFileFormat::Sql => ("application/sql; charset=utf-8", "SQL"), + }; + let expires_at_ms = i64::try_from(unix_epoch_millis()) + .unwrap_or(i64::MAX) + .saturating_add(LEGACY_IMPORT_UPLOAD_TTL_MS); + let storage = legacy_storage(application)?; + let cleanup_storage = storage.clone(); + let file_name = upload.file_name; + let extension = format.extension().to_owned(); + let staged = legacy_storage_call(move || { + let mut writer = storage.begin_transfer_artifact( + None, + &file_name, + media_type, + format_name, + &extension, + Some(expires_at_ms), + )?; + writer.write_all(&upload.content).map_err(|_| { + StorageError::InvalidTransfer("uploaded import file could not be stored") + })?; + let artifact = writer.finish()?; + let resolved = match storage.resolve_transfer_artifact(&artifact.id) { + Ok(resolved) => resolved, + Err(error) => { + let _ = storage.delete_temporary_transfer_artifact(&artifact.id); + return Err(error); + } + }; + Ok((artifact.id, resolved)) + }) + .await?; + let (artifact_id, resolved) = staged; + let Some(file_path) = resolved.path.to_str().map(str::to_owned) else { + drop(resolved.file); + cleanup_legacy_import_upload(cleanup_storage, artifact_id).await; + return Err(invalid_legacy_mysql_upload()); + }; + drop(resolved.file); + let table_name = non_blank(&request.table_name); + let accepted = match application + .import_mysql_file(ImportFileRequest { + datasource_id: request.data_source_id.as_string(), + database_name: request.database_name, + schema_name: request.schema_name, + table_name: if format == TransferFileFormat::Sql { + None + } else { + table_name + }, + file_path, + format, + contains_header: !sql_file_route && request.contains_header, + tabular_encoding: request.tabular_encoding, + }) + .await + { + Ok(accepted) => accepted, + Err(error) => { + cleanup_legacy_import_upload(cleanup_storage, artifact_id).await; + return Err(error.into()); + } + }; + schedule_legacy_import_upload_cleanup( + application.clone(), + cleanup_storage, + accepted.task_id, + artifact_id, + ); + Ok(accepted.task_id) +} + +async fn cleanup_legacy_import_upload(storage: Storage, artifact_id: String) { + let cleanup = tokio::task::spawn_blocking(move || { + storage.delete_temporary_transfer_artifact(&artifact_id) + }) + .await; + match cleanup { + Ok(Ok(_)) => {} + Ok(Err(error)) => tracing::warn!(%error, "temporary Web import artifact cleanup failed"), + Err(error) => tracing::warn!(%error, "temporary Web import artifact cleanup task failed"), } - let datasource_id = query.data_source_id.as_string(); - let database_type = - resolve_mysql_database_type(application, &datasource_id, &query.database_type).await?; - let (view, columns) = tokio::try_join!( - application.get_community_view(ListCommunityViewsRequest { - datasource_id: datasource_id.clone(), - database_type: database_type.clone(), - database_name: query.database_name.clone(), - schema_name: query.schema_name.clone(), - view_name_pattern: query.table_name.clone(), - }), - application.list_community_columns(ListCommunityColumnsRequest { - datasource_id, - database_type: database_type.clone(), - database_name: query.database_name.clone(), - schema_name: query.schema_name.clone(), - table_name: query.table_name.clone(), - }), - )?; - Ok(editable_table_response( - view, - columns.items, - Vec::new(), - database_type, - )) } -/// Returns the `MySQL` type and option inventory used by the retained table editor. -pub(crate) async fn table_editor_meta( - application: &Application, - query: &LegacyMetadataQuery, -) -> LegacyResult { - let datasource_id = query.data_source_id.as_string(); - resolve_mysql_database_type(application, &datasource_id, &query.database_type).await?; - Ok(mysql_table_editor_meta()) +fn schedule_legacy_import_upload_cleanup( + application: Application, + storage: Storage, + task_id: i64, + artifact_id: String, +) { + tokio::spawn(async move { + loop { + match application.transfer_task(task_id).await { + Ok(task) + if matches!( + task.status, + TransferTaskStatus::Succeeded + | TransferTaskStatus::Failed + | TransferTaskStatus::Cancelled + | TransferTaskStatus::Interrupted + ) => + { + break; + } + Ok(_) => tokio::time::sleep(LEGACY_IMPORT_CLEANUP_POLL).await, + Err(error) if error.kind() == AppErrorKind::NotFound => break, + Err(error) => { + tracing::warn!(%error, task_id, "temporary Web import cleanup is waiting for task state"); + tokio::time::sleep(LEGACY_IMPORT_CLEANUP_POLL).await; + } + } + } + cleanup_legacy_import_upload(storage, artifact_id).await; + }); } -/// Builds a `MySQL` script for Community result-grid create, update, and delete operations. -pub(crate) async fn build_grid_update_sql( +async fn import_legacy_mysql_http( application: &Application, - request: &LegacyGridUpdateRequest, -) -> LegacyResult { - let datasource_id = request.data_source_id.as_string(); - resolve_mysql_database_type(application, &datasource_id, &request.database_type).await?; - if request.table_name.trim().is_empty() { - return Err(LegacyFailure::invalid( - "invalid_mysql_result_grid", - "tableName is required", - )); + request: axum::extract::Request, + sql_file_route: bool, +) -> LegacyResult { + let is_multipart = request + .headers() + .get(header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| { + value + .to_ascii_lowercase() + .starts_with("multipart/form-data") + }); + if !is_multipart { + return Err(web_import_upload_required()); } - let headers = request - .header_list - .iter() - .map(mysql_grid_header) - .collect::>>()?; - let operations = request - .operations - .iter() - .map(mysql_grid_operation) - .collect::>>()?; - reject_legacy_partial_large_values(&operations)?; - Ok(build_mysql_result_grid_script( - &mysql_qualified_name( - &request.database_name, - &request.schema_name, - &request.table_name, - ), - &headers, - &operations, - )?) + let multipart = Multipart::from_request(request, application) + .await + .map_err(|_| invalid_legacy_mysql_upload())?; + import_legacy_mysql_multipart(application, multipart, sql_file_route).await } -/// Builds Community's copy-as-INSERT, copy-as-UPDATE, or copy-as-WHERE SQL. -pub(crate) async fn build_grid_copy_sql( +/// Starts a desktop-only import from a path returned by the platform file picker. +/// HTTP handlers must stage uploaded bytes through [`import_legacy_mysql_multipart`]. +pub(crate) async fn import_legacy_mysql_desktop_file( application: &Application, - request: &LegacyGridUpdateRequest, -) -> LegacyResult { - let datasource_id = request.data_source_id.as_string(); - resolve_mysql_database_type(application, &datasource_id, &request.database_type).await?; - let headers = request - .header_list - .iter() - .map(mysql_grid_header) - .collect::>>()?; - let operations = request - .operations - .iter() - .map(mysql_grid_copy_operation) - .collect::>>()?; - Ok(build_mysql_result_grid_copy_sql( - &mysql_qualified_name( - &request.database_name, - &request.schema_name, - &required_name(&request.table_name, "tableName")?, - ), - &headers, - &operations, - )?) + request: &LegacyImportFileRequest, + sql_file_route: bool, +) -> LegacyResult { + let format = if sql_file_route { + TransferFileFormat::Sql + } else { + legacy_transfer_format(&request.import_type, "importType")? + }; + let table_name = non_blank(&request.table_name); + let accepted = application + .import_mysql_file(ImportFileRequest { + datasource_id: request.data_source_id.as_string(), + database_name: request.database_name.clone(), + schema_name: request.schema_name.clone(), + table_name: if format == TransferFileFormat::Sql { + None + } else { + table_name + }, + file_path: request.file_name.clone(), + format, + contains_header: !sql_file_route && request.contains_header, + tabular_encoding: request.tabular_encoding, + }) + .await?; + Ok(accepted.task_id) } -/// Builds Community's clipboard SQL `IN` list for result cells or external text. -pub(crate) async fn build_grid_in_values( +pub(crate) async fn export_legacy_mysql_sql_file( application: &Application, - request: &LegacyGridUpdateRequest, -) -> LegacyResult { - let datasource_id = request.data_source_id.as_string(); - resolve_mysql_database_type(application, &datasource_id, &request.database_type).await?; - match request.source_type.trim().to_ascii_uppercase().as_str() { - "EXTERNAL_TEXT" => Ok(build_mysql_external_in_values(&request.external_values)?), - "RESULT_SET" => { - let headers = request - .header_list - .iter() - .map(mysql_grid_header) - .collect::>>()?; - reject_unsupported_copy_cells(&request.operations)?; - let operations = request - .operations - .iter() - .map(mysql_grid_copy_operation) - .collect::>>()?; - Ok(build_mysql_result_grid_in_values(&headers, &operations)?) - } - _ => Err(LegacyFailure::invalid( - "invalid_mysql_result_grid", - "sourceType must be RESULT_SET or EXTERNAL_TEXT", - )), + request: &LegacySqlFileExportRequest, +) -> LegacyResult { + let mut table_names: Vec = request + .table_names + .iter() + .filter_map(|name| non_blank(name)) + .collect(); + if table_names.is_empty() + && let Some(table_name) = non_blank(&request.table_name) + { + table_names.push(table_name); } -} - -/// Builds CREATE or ALTER TABLE statements in the historical `{ sql }[]` shape. -pub(crate) async fn build_table_modify_sql( - application: &Application, - request: &LegacyTableModifyRequest, -) -> LegacyResult> { - let datasource_id = request.data_source_id.as_string(); - resolve_mysql_database_type(application, &datasource_id, &request.database_type).await?; - let sql = if let Some(old_table) = request.old_table.as_ref() { - let reordered_columns = mysql_reordered_column_names(old_table, &request.new_table); - if !reordered_columns.is_empty() { - application - .validate_native_mysql_column_reorder( - &datasource_id, - &first_non_blank(&request.database_name, &old_table.database_name), - &required_name(&old_table.name, "oldTable.name")?, - &reordered_columns, - ) - .await?; + let scope = match request.scope.trim().to_ascii_uppercase().as_str() { + "" | "ALL" => TransferSqlScope::All, + "SCHEMA" => TransferSqlScope::Schema, + "TABLE" => TransferSqlScope::Table, + _ => { + return Err(LegacyFailure::invalid( + "invalid_export_scope", + "scope must be ALL, SCHEMA, or TABLE", + )); } - build_mysql_alter_table(&mysql_table_alter( - old_table, - &request.new_table, - &request.database_name, - &request.schema_name, - )?)? - } else { - build_mysql_create_table(&mysql_table_definition( - &request.new_table, - &request.database_name, - &request.schema_name, - )?)? }; - Ok(vec![LegacySqlResponse { sql }]) + let accepted = application + .export_mysql_sql_file(SqlFileExportRequest { + datasource_id: request.data_source_id.as_string(), + database_name: request.database_name.clone(), + schema_name: request.schema_name.clone(), + table_names, + scope, + export_path: non_blank(&request.export_path), + }) + .await?; + Ok(accepted.task_id) } -/// Builds a CREATE DATABASE preview without executing it. -pub(crate) async fn build_create_database_sql( +pub(crate) async fn export_legacy_mysql_other_file( application: &Application, - request: &LegacyDatabaseDefinitionRequest, -) -> LegacyResult { - let datasource_id = request.data_source_id.as_string(); - resolve_mysql_database_type(application, &datasource_id, &request.database_type).await?; - let name = first_non_blank(&request.name, &request.database_name); - Ok(LegacySqlResponse { - sql: build_mysql_create_database(&MysqlDatabaseDefinition { - name, - if_not_exists: false, - charset: non_blank(&request.charset), - collation: non_blank(&request.collation), - })?, - }) + request: &LegacyOtherFileExportRequest, +) -> LegacyResult { + let mut table_names: Vec = request + .table_names + .iter() + .filter_map(|name| non_blank(name)) + .collect(); + if table_names.is_empty() + && let Some(table_name) = non_blank(&request.table_name) + { + table_names.push(table_name); + } + let accepted = application + .export_mysql_other_file(OtherFileExportRequest { + datasource_id: request.data_source_id.as_string(), + database_name: request.database_name.clone(), + schema_name: request.schema_name.clone(), + table_names, + format: legacy_transfer_format(&request.export_type, "exportType")?, + contains_header: request.contains_header, + export_path: non_blank(&request.export_path), + }) + .await?; + Ok(accepted.task_id) } -/// `MySQL` treats Community schemas as database aliases. -pub(crate) async fn build_create_schema_sql( +pub(crate) async fn list_legacy_transfer_tasks( application: &Application, - request: &LegacySchemaDefinitionRequest, -) -> LegacyResult { - let datasource_id = request.data_source_id.as_string(); - resolve_mysql_database_type(application, &datasource_id, &request.database_type).await?; - let name = first_non_blank(&request.name, &request.schema_name); - Ok(LegacySqlResponse { - sql: build_mysql_create_schema(&MysqlDatabaseDefinition { - name, - if_not_exists: false, - charset: None, - collation: None, - })?, + query: &LegacyTaskListQuery, +) -> LegacyResult> { + let statuses = legacy_transfer_status_filter(&query.task_status)?; + let page = application + .list_transfer_tasks_by_statuses(query.page_no, query.page_size, &statuses) + .await?; + Ok(LegacyPage { + data: page + .items + .into_iter() + .map(legacy_transfer_task_for_web) + .collect(), + page_no: page.page_no, + page_size: page.page_size, + total: usize::try_from(page.total).unwrap_or(usize::MAX), + has_next_page: u64::from(page.page_no).saturating_mul(u64::from(page.page_size)) + < page.total, }) } -pub(crate) async fn prepare_database_delete( +pub(crate) async fn get_legacy_transfer_task( application: &Application, - request: &LegacyDeleteObjectRequest, -) -> LegacyResult { - let datasource_id = request.data_source_id.as_string(); - let database_type = - resolve_mysql_database_type(application, &datasource_id, &request.database_type).await?; - let confirm_name = required_name(&request.database_name, "databaseName")?; - Ok(LegacyDeletePrepareResponse { - sql_preview: build_mysql_drop_database(&confirm_name, false)?, - confirm_name, - object_type: "DATABASE".to_owned(), - db_type: database_type, - }) + id: &LegacyIdentifier, +) -> LegacyResult { + let task = application + .transfer_task(legacy_transfer_task_id(id)?) + .await?; + Ok(legacy_transfer_task_for_web(task)) } -pub(crate) async fn prepare_schema_delete( +async fn list_legacy_transfer_tasks_for_desktop( application: &Application, - request: &LegacyDeleteObjectRequest, -) -> LegacyResult { - let datasource_id = request.data_source_id.as_string(); - let database_type = - resolve_mysql_database_type(application, &datasource_id, &request.database_type).await?; - let target = first_non_blank(&request.schema_name, &request.database_name); - let confirm_name = required_name(&target, "schemaName")?; - Ok(LegacyDeletePrepareResponse { - sql_preview: build_mysql_drop_schema(&confirm_name, false)?, - confirm_name, - object_type: "SCHEMA".to_owned(), - db_type: database_type, + query: &LegacyTaskListQuery, +) -> LegacyResult> { + let statuses = legacy_transfer_status_filter(&query.task_status)?; + let page = application + .list_transfer_tasks_by_statuses(query.page_no, query.page_size, &statuses) + .await?; + let mut data = Vec::with_capacity(page.items.len()); + for task in page.items { + data.push(legacy_transfer_task_for_desktop(application, task).await?); + } + Ok(LegacyPage { + data, + page_no: page.page_no, + page_size: page.page_size, + total: usize::try_from(page.total).unwrap_or(usize::MAX), + has_next_page: u64::from(page.page_no).saturating_mul(u64::from(page.page_size)) + < page.total, }) } -pub(crate) async fn execute_database_delete( +async fn get_legacy_transfer_task_for_desktop( application: &Application, - request: &LegacyDeleteObjectRequest, -) -> LegacyResult<()> { - let prepared = prepare_database_delete(application, request).await?; - validate_delete_confirmation(&prepared.confirm_name, &request.confirm_name)?; - execute_generated_action( - application, - request.data_source_id.clone(), - &prepared.confirm_name, - "", - "", - prepared.sql_preview, - ) - .await + id: &LegacyIdentifier, +) -> LegacyResult { + let task = application + .transfer_task(legacy_transfer_task_id(id)?) + .await?; + legacy_transfer_task_for_desktop(application, task).await } -pub(crate) async fn execute_schema_delete( +pub(crate) async fn stop_legacy_transfer_task( application: &Application, - request: &LegacyDeleteObjectRequest, + id: &LegacyIdentifier, ) -> LegacyResult<()> { - let prepared = prepare_schema_delete(application, request).await?; - validate_delete_confirmation(&prepared.confirm_name, &request.confirm_name)?; - execute_generated_action( - application, - request.data_source_id.clone(), - &prepared.confirm_name, - "", - "", - prepared.sql_preview, - ) - .await + application + .stop_transfer_task(legacy_transfer_task_id(id)?) + .await?; + Ok(()) } -pub(crate) async fn drop_table( +pub(crate) async fn legacy_transfer_task_download( application: &Application, - request: &LegacyTableOperationRequest, -) -> LegacyResult<()> { - let datasource_id = request.data_source_id.as_string(); - resolve_mysql_database_type(application, &datasource_id, &request.database_type).await?; - let sql = build_mysql_drop_table( - &mysql_qualified_name( - &request.database_name, - &request.schema_name, - &request.table_name, - ), - false, - )?; - execute_generated_action( - application, - request.data_source_id.clone(), - &request.database_name, - &request.schema_name, - &request.table_name, - sql, - ) - .await + id: &LegacyIdentifier, +) -> LegacyResult { + Ok(application + .transfer_task_artifact_download(legacy_transfer_task_id(id)?) + .await?) } -pub(crate) async fn truncate_table( +pub(crate) async fn export_legacy_mysql_dml( application: &Application, - request: &LegacyTableOperationRequest, -) -> LegacyResult<()> { - let datasource_id = request.data_source_id.as_string(); - resolve_mysql_database_type(application, &datasource_id, &request.database_type).await?; - let sql = build_mysql_truncate_table(&mysql_qualified_name( - &request.database_name, - &request.schema_name, - &request.table_name, - ))?; - execute_generated_action( - application, - request.data_source_id.clone(), - &request.database_name, - &request.schema_name, - &request.table_name, - sql, - ) - .await + request: &LegacyDmlExportRequest, +) -> LegacyResult { + let export_size = match request.export_size.trim().to_ascii_uppercase().as_str() { + "CURRENT_PAGE" => DmlExportSize::CurrentPage, + "" | "ALL" => DmlExportSize::All, + _ => { + return Err(LegacyFailure::invalid( + "invalid_export_size", + "exportSize must be CURRENT_PAGE or ALL", + )); + } + }; + let format = match request.export_type.trim().to_ascii_uppercase().as_str() { + "CSV" => DmlExportFormat::Csv, + "EXCEL" | "XLSX" => DmlExportFormat::Xlsx, + "INSERT" => DmlExportFormat::Insert, + _ => { + return Err(LegacyFailure::invalid( + "invalid_export_type", + "exportType must be CSV, EXCEL, XLSX, or INSERT", + )); + } + }; + let original_sql = if request.original_sql.trim().is_empty() { + request.sql.clone() + } else { + request.original_sql.clone() + }; + let artifact = application + .export_mysql_dml(DmlExportRequest { + datasource_id: request.data_source_id.as_string(), + database_name: request.database_name.clone(), + schema_name: request.schema_name.clone(), + sql: request.sql.clone(), + original_sql, + result_set_id: request.result_set_id, + export_size, + format, + }) + .await?; + Ok(application.transfer_artifact_download(&artifact.id).await?) } -pub(crate) async fn copy_table( +pub(crate) async fn generate_legacy_mysql_classes( application: &Application, - request: &LegacyTableCopyRequest, + request: &LegacyGenerateClassRequest, ) -> LegacyResult<()> { - let datasource_id = request.data_source_id.as_string(); - resolve_mysql_database_type(application, &datasource_id, &request.database_type).await?; - let new_name = if request.new_name.trim().is_empty() { - format!("{}_copy", request.table_name.trim()) - } else { - request.new_name.trim().to_owned() - }; - let statements = build_mysql_copy_table(&MysqlTableCopy { - source: mysql_qualified_name( - &request.database_name, - &request.schema_name, - &request.table_name, - ), - target: mysql_qualified_name(&request.database_name, &request.schema_name, &new_name), - if_not_exists: false, - copy_data: request.copy_data, - })?; - for sql in statements { - execute_generated_action( - application, - request.data_source_id.clone(), - &request.database_name, - &request.schema_name, - &new_name, - sql, - ) + application + .generate_mysql_classes(GenerateMysqlClassRequest { + datasource_id: request.data_source_id.as_string(), + database_name: request.database_name.clone(), + schema_name: request.schema_name.clone(), + table_name: request.table_name.clone(), + export_path: request.export_path.clone(), + }) .await?; - } Ok(()) } -pub(crate) async fn build_view_modify_sql( +pub(crate) async fn generate_legacy_mysql_class_archive( application: &Application, - request: &LegacyViewOperationRequest, -) -> LegacyResult { - let datasource_id = request.data_source_id.as_string(); - resolve_mysql_database_type(application, &datasource_id, &request.database_type).await?; - Ok(build_mysql_create_view(&mysql_view_definition(request)?)?) + request: &LegacyGenerateClassRequest, +) -> LegacyResult { + reject_web_export_path(&request.export_path)?; + let artifact = application + .generate_mysql_class_archive(GenerateMysqlClassRequest { + datasource_id: request.data_source_id.as_string(), + database_name: request.database_name.clone(), + schema_name: request.schema_name.clone(), + table_name: request.table_name.clone(), + export_path: String::new(), + }) + .await?; + Ok(application.transfer_artifact_download(&artifact.id).await?) +} + +fn legacy_transfer_format(value: &str, field: &'static str) -> LegacyResult { + match value.trim().to_ascii_uppercase().as_str() { + "CSV" => Ok(TransferFileFormat::Csv), + "XLS" => Ok(TransferFileFormat::Xls), + "XLSX" | "EXCEL" => Ok(TransferFileFormat::Xlsx), + "SQL" => Ok(TransferFileFormat::Sql), + _ => Err(LegacyFailure { + code: "invalid_transfer_format".to_owned(), + message: format!("{field} must be CSV, XLS, XLSX, or SQL"), + }), + } } -pub(crate) async fn drop_view( - application: &Application, - request: &LegacyViewOperationRequest, -) -> LegacyResult<()> { - let datasource_id = request.data_source_id.as_string(); - resolve_mysql_database_type(application, &datasource_id, &request.database_type).await?; - let view_name = first_non_blank(&request.view_name, &request.table_name); - let sql = build_mysql_drop_view( - &mysql_qualified_name(&request.database_name, &request.schema_name, &view_name), - false, - )?; - execute_generated_action( - application, - request.data_source_id.clone(), - &request.database_name, - &request.schema_name, - &view_name, - sql, - ) - .await +fn legacy_tabular_import_encoding(value: &str) -> LegacyResult { + match value.trim().to_ascii_uppercase().as_str() { + "" | "PLAIN" => Ok(TabularImportEncoding::Plain), + "CHAT2DB_V1" => Ok(TabularImportEncoding::Chat2dbV1), + _ => Err(LegacyFailure::invalid( + "invalid_tabular_import_encoding", + "tabularEncoding must be PLAIN or CHAT2DB_V1", + )), + } } -pub(crate) async fn view_editor_meta( +fn legacy_transfer_status_filter(value: &str) -> LegacyResult> { + match value.trim().to_ascii_uppercase().as_str() { + "" => Ok(Vec::new()), + "INIT" => Ok(vec![TransferTaskStatus::Queued]), + "PROCESSING" | "RUNNING" => Ok(vec![TransferTaskStatus::Running]), + "FINISHED" => Ok(vec![TransferTaskStatus::Succeeded]), + "ERROR" => Ok(vec![ + TransferTaskStatus::Failed, + TransferTaskStatus::Interrupted, + ]), + "STOP" => Ok(vec![TransferTaskStatus::Cancelled]), + _ => Err(LegacyFailure::invalid( + "invalid_task_status", + "taskStatus is not supported", + )), + } +} + +fn legacy_transfer_task_id(id: &LegacyIdentifier) -> LegacyResult { + let id = id + .as_string() + .parse::() + .map_err(|_| LegacyFailure::invalid("invalid_task_id", "id must be a task number"))?; + if id <= 0 { + return Err(LegacyFailure::invalid( + "invalid_task_id", + "id must be a positive task number", + )); + } + Ok(id) +} + +fn legacy_transfer_task_for_web(task: TransferTask) -> LegacyTransferTask { + let download_url = task.artifact_id.as_ref().map_or_else(String::new, |_| { + format!("/api/task/download?id={}", task.id) + }); + legacy_transfer_task(task, download_url) +} + +async fn legacy_transfer_task_for_desktop( application: &Application, - request: &LegacyViewOperationRequest, -) -> LegacyResult { - let datasource_id = request.data_source_id.as_string(); - resolve_mysql_database_type(application, &datasource_id, &request.database_type).await?; - let sql = "select * from table_name".to_owned(); - let preview_name = if request.database_name.trim().is_empty() { - "`undefined`".to_owned() + task: TransferTask, +) -> LegacyResult { + let download_url = if task.artifact_id.is_some() { + application + .transfer_task_artifact_download(task.id) + .await? + .path + .to_str() + .map(str::to_owned) + .ok_or_else(|| LegacyFailure { + code: "invalid_transfer_artifact_path".to_owned(), + message: "The transfer artifact path cannot be represented as UTF-8".to_owned(), + })? } else { - format!("`{}`.`undefined`", request.database_name.replace('`', "``")) + String::new() }; - let preview_sql = format!("create view {preview_name} AS \n{sql};"); - Ok(LegacyViewMetaResponse { - configurations: mysql_view_configurations(), - preview_sql, - sql, - }) + Ok(legacy_transfer_task(task, download_url)) } -fn mysql_view_configurations() -> Vec { - vec![ - serde_json::json!({ - "labelName": "算法", - "name": "algorithm", - "inputType": "select", - "defaultValue": "3", - "required": false, - "multiple": false, - "display": null, - "selects": [ - { "label": "UNDEFINED", "value": 0 }, - { "label": "MERGE", "value": 1 }, - { "label": "TEMPTABLE", "value": 2 }, - { "label": null, "value": 3 } - ] - }), - serde_json::json!({ - "labelName": "检查选项", - "name": "checkOption", - "inputType": "select", - "defaultValue": "2", - "required": false, - "multiple": false, - "display": null, - "selects": [ - { "label": "CASCADED", "value": 0 }, - { "label": "LOCAL", "value": 1 }, - { "label": null, "value": 2 } - ] - }), - serde_json::json!({ - "labelName": "SQL 安全性", - "name": "security", - "inputType": "select", - "defaultValue": "2", - "required": false, - "multiple": false, - "display": null, - "selects": [ - { "label": "DEFINER", "value": 0 }, - { "label": "INVOKER", "value": 1 }, - { "label": null, "value": 2 } - ] - }), - serde_json::json!({ - "labelName": "视图名称", - "name": "viewName", - "inputType": "input", - "defaultValue": null, - "required": false, - "multiple": false, - "display": null, - "selects": null - }), - serde_json::json!({ - "labelName": "定义者", - "name": "definer", - "inputType": "input", - "defaultValue": null, - "required": false, - "multiple": false, - "display": null, - "selects": null - }), - serde_json::json!({ - "labelName": "use or replace", - "name": "useOrReplace", - "inputType": "checkbox", - "defaultValue": "false", - "required": false, - "multiple": false, - "display": null, - "selects": null - }), - ] +fn legacy_transfer_task(task: TransferTask, download_url: String) -> LegacyTransferTask { + let task_status = match task.status { + TransferTaskStatus::Queued => "INIT", + TransferTaskStatus::Running => "RUNNING", + TransferTaskStatus::Succeeded => "FINISHED", + TransferTaskStatus::Failed | TransferTaskStatus::Interrupted => "ERROR", + TransferTaskStatus::Cancelled => "STOP", + }; + let task_type = match task.kind { + TransferTaskKind::ImportFile => "UPLOAD_TABLE_DATA", + TransferTaskKind::ExportSql | TransferTaskKind::ExportFile => "DOWNLOAD_TABLE_STRUCTURE", + }; + let task_progress = transfer_progress_percent(&task); + LegacyTransferTask { + id: task.id, + gmt_create: task.created_at_ms.parse().unwrap_or_default(), + gmt_modified: task.updated_at_ms.parse().unwrap_or_default(), + data_source_id: task.datasource_id, + database_name: task.database_name, + schema_name: task.schema_name, + table_name: task.table_name, + task_type: task_type.to_owned(), + task_status: task_status.to_owned(), + task_progress, + progress: task.progress_current, + current_progress: task.progress_description.clone(), + progress_desc: task.progress_description, + task_name: task.task_name, + download_url, + info_log: task.info_log, + error_log: task.error_log, + } +} + +fn transfer_progress_percent(task: &TransferTask) -> String { + if task.status == TransferTaskStatus::Succeeded { + return "100".to_owned(); + } + let current = task.progress_current.parse::().unwrap_or_default(); + task.progress_total + .as_deref() + .and_then(|total| total.parse::().ok()) + .filter(|total| *total > 0) + .map_or_else( + || "0".to_owned(), + |total| { + current + .saturating_mul(100) + .checked_div(total) + .unwrap_or_default() + .min(100) + .to_string() + }, + ) } -/// Lists stored functions in the historical paged metadata shape. -pub(crate) async fn list_functions( +pub(crate) async fn create_saved_console( application: &Application, - query: &LegacyTableListQuery, -) -> LegacyResult> { - validate_metadata_page(query)?; - let datasource_id = query.data_source_id.as_string(); - let database_type = - resolve_database_type(application, &datasource_id, &query.database_type).await?; - let items = application - .list_community_functions(ListCommunityFunctionsRequest { - datasource_id, - database_type, - database_name: query.database_name.clone(), - schema_name: query.schema_name.clone(), - }) - .await? - .items - .into_iter() - .map(function_response) - .collect(); - Ok(full_page(items)) + request: &LegacySavedConsoleCreateRequest, +) -> LegacyResult { + let storage = legacy_storage(application)?; + let input = CreateSavedConsole { + id: request.id, + name: request.name.clone(), + data_source_id: request + .data_source_id + .as_ref() + .map(LegacyIdentifier::as_string), + data_source_name: request.data_source_name.clone(), + database_name: request.database_name.clone(), + schema_name: request.schema_name.clone(), + database_type: request.database_type.clone(), + ddl: request.ddl.clone(), + status: default_if_blank(&request.status, "DRAFT"), + // Community always opens a newly created Console. + tab_opened: "y".to_owned(), + operation_type: default_if_blank(&request.operation_type, "console"), + }; + legacy_storage_call(move || storage.create_saved_console(input)) + .await + .map(|record| record.id) } -/// Reads one stored function in the historical metadata shape. -pub(crate) async fn get_function( +pub(crate) async fn get_saved_console( application: &Application, - query: &LegacyFunctionDetailQuery, -) -> LegacyResult { - let datasource_id = query.data_source_id.as_string(); - let database_type = - resolve_database_type(application, &datasource_id, &query.database_type).await?; - Ok(function_response( - application - .get_community_function(GetCommunityFunctionRequest { - datasource_id, - database_type, - database_name: query.database_name.clone(), - schema_name: query.schema_name.clone(), - function_name: query.function_name.clone(), - }) - .await?, - )) + id: i64, +) -> LegacyResult> { + let storage = legacy_storage(application)?; + legacy_storage_call(move || storage.get_saved_console(id)) + .await + .map(|record| record.map(saved_console_response)) } -/// Lists stored procedures in the historical paged metadata shape. -pub(crate) async fn list_procedures( +pub(crate) async fn list_saved_consoles( application: &Application, - query: &LegacyTableListQuery, -) -> LegacyResult> { - validate_metadata_page(query)?; - let datasource_id = query.data_source_id.as_string(); - let database_type = - resolve_database_type(application, &datasource_id, &query.database_type).await?; - let items = application - .list_community_procedures(ListCommunityProceduresRequest { - datasource_id, - database_type, - database_name: query.database_name.clone(), - schema_name: query.schema_name.clone(), + query: &LegacySavedConsoleListQuery, +) -> LegacyResult> { + let storage = legacy_storage(application)?; + let storage_query = SavedConsoleListQuery { + data_source_id: query + .data_source_id + .as_ref() + .map(LegacyIdentifier::as_string), + database_name: query.database_name.clone(), + schema_name: query.schema_name.clone(), + status: query.status.clone(), + tab_opened: query.tab_opened.clone(), + operation_type: query.operation_type.clone(), + search_key: query.search_key.clone(), + page_no: query.page_no, + page_size: query.page_size, + order_by_desc: query.order_by_desc, + }; + legacy_storage_call(move || storage.list_saved_consoles(&storage_query)) + .await + .map(|page| { + let total = usize::try_from(page.total).unwrap_or(usize::MAX); + let data = page + .records + .into_iter() + .map(saved_console_response) + .collect::>(); + LegacyPage { + has_next_page: u64::from(page.page_no) * u64::from(page.page_size) < page.total, + data, + page_no: page.page_no, + page_size: page.page_size, + total, + } }) - .await? - .items - .into_iter() - .map(procedure_response) - .collect(); - Ok(full_page(items)) } -/// Reads one stored procedure in the historical metadata shape. -pub(crate) async fn get_procedure( +pub(crate) async fn update_saved_console( application: &Application, - query: &LegacyProcedureDetailQuery, -) -> LegacyResult { - let datasource_id = query.data_source_id.as_string(); - let database_type = - resolve_database_type(application, &datasource_id, &query.database_type).await?; - Ok(procedure_response( - application - .get_community_procedure(GetCommunityProcedureRequest { - datasource_id, - database_type, - database_name: query.database_name.clone(), - schema_name: query.schema_name.clone(), - procedure_name: query.procedure_name.clone(), - }) - .await?, - )) + request: LegacySavedConsoleUpdateRequest, +) -> LegacyResult<()> { + let storage = legacy_storage(application)?; + let input = UpdateSavedConsole { + name: required_string_patch(request.name), + data_source_id: identifier_patch(request.data_source_id), + data_source_name: nullable_string_patch(request.data_source_name), + database_name: nullable_string_patch(request.database_name), + schema_name: nullable_string_patch(request.schema_name), + database_type: nullable_string_patch(request.database_type), + ddl: required_string_patch(request.ddl), + status: required_string_patch(request.status), + tab_opened: required_string_patch(request.tab_opened), + operation_type: required_string_patch(request.operation_type), + }; + legacy_storage_call(move || storage.update_saved_console(request.id, input)) + .await + .map(|_| ()) } -/// Renders the SQL shown by Community's routine invocation dialog. -pub(crate) async fn preview_routine_invocation( +pub(crate) async fn delete_saved_console(application: &Application, id: i64) -> LegacyResult<()> { + let storage = legacy_storage(application)?; + legacy_storage_call(move || storage.delete_saved_console(id)) + .await + .map(|_| ()) +} + +pub(crate) async fn create_operation_log( application: &Application, - request: &LegacyRoutineInvocationRequest, -) -> LegacyResult { - let datasource_id = request.data_source_id.as_string(); - let database_type = - resolve_database_type(application, &datasource_id, &request.database_type).await?; - Ok(application - .preview_community_routine_invocation(PreviewCommunityRoutineInvocationRequest { - datasource_id, - database_type, - database_name: request.database_name.clone(), - schema_name: request.schema_name.clone(), - routine_type: request.routine_type.clone(), - routine_name: request.routine_name.clone(), - }) - .await?) + request: &LegacyOperationLogCreateRequest, +) -> LegacyResult { + if request.ddl.trim().is_empty() { + return Err(LegacyFailure::invalid( + "invalid_operation_log", + "ddl is required", + )); + } + let operation_rows = request + .operation_rows + .map(i64::try_from) + .transpose() + .map_err(|_| { + LegacyFailure::invalid( + "invalid_operation_log", + "operationRows is outside the supported range", + ) + })?; + let use_time = request + .use_time + .map(i64::try_from) + .transpose() + .map_err(|_| { + LegacyFailure::invalid( + "invalid_operation_log", + "useTime is outside the supported range", + ) + })?; + let data_source_id = request + .data_source_id + .as_ref() + .map(LegacyIdentifier::as_string); + let storage = legacy_storage(application)?; + let input = CreateOperationLog { + name: non_blank(&request.name), + connectable: request + .connectable + .or_else(|| data_source_id.as_ref().map(|id| !id.trim().is_empty())), + data_source_id, + data_source_name: request.data_source_name.clone(), + database_name: request.database_name.clone(), + database_type: request.database_type.clone(), + ddl: request.ddl.clone(), + status: request + .status + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + .unwrap_or("SUCCESS") + .to_owned(), + operation_rows, + use_time, + extend_info: request.extend_info.clone(), + schema_name: request.schema_name.clone(), + organization_id: request.organization_id, + user_name: request.user_name.clone(), + more: request.more, + operation_type: "SQL_EXECUTE".to_owned(), + }; + legacy_storage_call(move || storage.create_operation_log(input)) + .await + .map(|record| record.id) } -/// Lists triggers in the historical paged metadata shape. -pub(crate) async fn list_triggers( +pub(crate) async fn get_operation_log( application: &Application, - query: &LegacyTableListQuery, -) -> LegacyResult> { - validate_metadata_page(query)?; - let datasource_id = query.data_source_id.as_string(); - let database_type = - resolve_database_type(application, &datasource_id, &query.database_type).await?; - let items = application - .list_community_triggers(ListCommunityTriggersRequest { - datasource_id, - database_type, - database_name: query.database_name.clone(), - schema_name: query.schema_name.clone(), - }) + id: i64, +) -> LegacyResult { + let storage = legacy_storage(application)?; + let record = legacy_storage_call(move || storage.get_operation_log(id)) .await? - .items - .into_iter() - .map(trigger_response) - .collect(); - Ok(full_page(items)) + .ok_or_else(|| { + LegacyFailure::invalid( + "operation_log_not_found", + "The operation log does not exist", + ) + })?; + Ok(operation_log_response(record, false)) } -/// Reads one trigger in the historical metadata shape. -pub(crate) async fn get_trigger( +pub(crate) async fn list_operation_logs( application: &Application, - query: &LegacyTriggerDetailQuery, -) -> LegacyResult { - let datasource_id = query.data_source_id.as_string(); - let database_type = - resolve_database_type(application, &datasource_id, &query.database_type).await?; - Ok(trigger_response( - application - .get_community_trigger(GetCommunityTriggerRequest { - datasource_id, - database_type, - database_name: query.database_name.clone(), - schema_name: query.schema_name.clone(), - trigger_name: query.trigger_name.clone(), - }) - .await?, - )) + query: &LegacyOperationLogListQuery, +) -> LegacyResult> { + let storage = legacy_storage(application)?; + let storage_query = OperationLogListQuery { + data_source_id: query + .data_source_id + .as_ref() + .map(LegacyIdentifier::as_string), + database_name: query.database_name.clone(), + schema_name: query.schema_name.clone(), + operation_type: query.operation_type.clone(), + search_key: non_blank(&query.search_key), + page_no: query.page_no, + page_size: query.page_size, + }; + legacy_storage_call(move || storage.list_operation_logs(&storage_query)) + .await + .map(|page| { + let total = usize::try_from(page.total).unwrap_or(usize::MAX); + LegacyPage { + data: page + .records + .into_iter() + .map(|record| operation_log_response(record, true)) + .collect(), + page_no: page.page_no, + page_size: page.page_size, + total, + has_next_page: u64::from(page.page_no) * u64::from(page.page_size) < page.total, + } + }) } -/// Runs a table preview through the generated-SQL, forced-read-only Core path -/// and waits for its retained result so the old synchronous frontend can use it. -#[allow(clippy::too_many_lines)] -pub(crate) async fn preview_table( +/// Builds the persisted Community namespace tree without exposing connection secrets. +pub(crate) async fn namespace_tree( application: &Application, - request: &LegacyTablePreviewRequest, -) -> LegacyResult> { - if request.table_name.trim().is_empty() { - return Err(LegacyFailure::invalid( - "invalid_table_preview_request", - "tableName is required", - )); +) -> LegacyResult> { + let listed = application.list_datasources().await?; + let mut datasources = HashMap::with_capacity(listed.items.len()); + for datasource in listed.items { + let projection = application + .get_datasource_edit_projection(&datasource.id) + .await?; + datasources.insert(datasource.id, projection); } - if request.page_no == 0 || request.page_size == 0 { - return Err(LegacyFailure::invalid( - "invalid_table_preview_request", - "pageNo and pageSize must be positive", - )); + let tree = application.workspace_tree().await?; + let nodes = tree + .items + .into_iter() + .map(|node| legacy_workspace_node(application, node, &mut datasources)) + .collect::>>()?; + if !datasources.is_empty() { + return Err(LegacyFailure { + code: "workspace_tree_incomplete".to_owned(), + message: "The datasource workspace tree is incomplete".to_owned(), + }); } - let offset = request - .page_no - .saturating_sub(1) - .checked_mul(request.page_size) - .ok_or_else(|| { - LegacyFailure::invalid( - "invalid_table_preview_request", - "requested page is outside the preview window", - ) - })?; - let row_limit = offset.checked_add(request.page_size).ok_or_else(|| { - LegacyFailure::invalid( - "invalid_table_preview_request", - "requested page is outside the preview window", - ) - })?; - if offset >= MAX_PREVIEW_ROWS || row_limit > MAX_PREVIEW_ROWS { - return Err(LegacyFailure::invalid( - "invalid_table_preview_request", - "table preview is limited to the first 1000 rows", - )); + Ok(nodes) +} + +fn legacy_workspace_node( + application: &Application, + node: WorkspaceTreeNode, + datasources: &mut HashMap, +) -> LegacyResult { + let WorkspaceTreeNode { + id, + node_type, + name, + children, + .. + } = node; + let children = children + .into_iter() + .map(|child| legacy_workspace_node(application, child, datasources)) + .collect::>>()?; + match node_type { + WorkspaceNodeKind::Namespace => Ok(LegacyNamespaceNode { + id: id.clone(), + node_type: "NAMESPACE".to_owned(), + name: name.clone(), + data: serde_json::json!({ + "id": id, + "name": name, + "dataSources": [] + }), + children, + }), + WorkspaceNodeKind::DataSource => { + let datasource = datasources.remove(&id).ok_or_else(|| LegacyFailure { + code: "workspace_datasource_not_found".to_owned(), + message: "A datasource referenced by the workspace tree does not exist".to_owned(), + })?; + let response = datasource_response(application, datasource); + Ok(LegacyNamespaceNode { + id, + node_type: "DATA_SOURCE".to_owned(), + name, + data: serialize_data(response)?, + children, + }) + } } +} - let datasource_id = request.data_source_id.as_string(); - let database_type = - resolve_database_type(application, &datasource_id, &request.database_type).await?; - let editable_columns = application - .list_community_columns(ListCommunityColumnsRequest { - datasource_id: datasource_id.clone(), - database_type: database_type.clone(), - database_name: request.database_name.clone(), - schema_name: request.schema_name.clone(), - table_name: request.table_name.clone(), +pub(crate) async fn create_namespace( + application: &Application, + request: &LegacyNamespaceRequest, +) -> LegacyResult { + Ok(application + .create_workspace_namespace(CreateWorkspaceNamespaceRequest { + name: request.name.clone(), + parent_id: request.parent_id.as_ref().map(LegacyIdentifier::as_string), }) .await? - .items; - let accepted = application - .start_community_table_preview(StartCommunityTablePreviewRequest { - datasource_id, - database_type, - database_name: request.database_name.clone(), - schema_name: request.schema_name.clone(), - table_name: request.table_name.clone(), - row_limit: Some(row_limit), + .id) +} + +pub(crate) async fn update_namespace( + application: &Application, + request: &LegacyNamespaceRequest, +) -> LegacyResult<()> { + let id = request + .id + .as_ref() + .ok_or_else(|| LegacyFailure::invalid("invalid_workspace_operation", "id is required"))?; + application + .update_workspace_namespace(UpdateWorkspaceNamespaceRequest { + id: id.as_string(), + name: request.name.clone(), }) .await?; + Ok(()) +} - let preview_result = tokio::time::timeout( - PREVIEW_TIMEOUT, - wait_for_sql_execution(application, &accepted.operation_id), - ) - .await; - let Ok(preview_result) = preview_result else { - application.cancel_operation(&accepted.operation_id).await; - return Err(LegacyFailure::invalid( - "table_preview_timeout", - "The table preview did not finish in time", - )); - }; - let metadata = preview_result?; - let page = application - .result_page( - &metadata.id, - ResultPageRequest { - offset: offset.to_string(), - max_rows: request.page_size.to_string(), - max_bytes: RESULT_PAGE_MAX_BYTES.to_string(), - }, - ) +pub(crate) async fn delete_namespace( + application: &Application, + request: &LegacyNamespaceRequest, +) -> LegacyResult<()> { + let id = request + .id + .as_ref() + .ok_or_else(|| LegacyFailure::invalid("invalid_workspace_operation", "id is required"))?; + application + .delete_workspace_namespace(&id.as_string()) .await?; - let has_next_page = page.has_more - || page.metadata.truncated_by_max_rows - || page.metadata.truncated_by_max_result_bytes; - let mut headers: Vec = page.columns.iter().map(result_header).collect(); - enrich_headers_from_columns(&mut headers, &editable_columns); - let large_value_owner = application.create_large_value_owner(); - let mut data_list: Vec> = page - .rows - .into_iter() - .map(|row| { - row.values - .into_iter() - .zip(page.columns.iter()) - .map(|(value, column)| result_cell(application, &large_value_owner, value, column)) - .collect() + Ok(()) +} + +pub(crate) async fn move_namespace_node( + application: &Application, + request: LegacyWorkspaceMoveRequest, +) -> LegacyResult<()> { + application + .move_workspace_node(MoveWorkspaceNodeRequest { + drag_node: legacy_workspace_node_ref(&request.drag_node)?, + drop_to_node: legacy_workspace_node_ref(&request.drop_to_node)?, + drop_position: request.drop_position, }) - .collect(); - prepend_synthetic_row_numbers(&mut headers, &mut data_list, u64::from(offset)); - Ok(vec![LegacyManageResult { - data_list, - header_list: headers, - description: "Query executed successfully".to_owned(), - message: String::new(), - sql: accepted.sql.clone(), - original_sql: accepted.sql, - success: true, - duration: 0, - update_count: 0, - can_edit: true, - table_name: request.table_name.clone(), - sql_type: "SELECT".to_owned(), - refresh_targets: Vec::new(), - page_no: request.page_no, - page_size: request.page_size, - fuzzy_total: page.metadata.row_count, - has_next_page, - execute_sql_params: LegacySqlExecuteRequest::from(request), - extra: serde_json::json!({}), - comment: None, - result_set_id: None, - statement_sequence: Some(1), - execution_metrics: None, - execution_context: Some(LegacyExecutionContext { - database_name: (!request.database_name.is_empty()) - .then(|| request.database_name.clone()), - schema_name: (!request.schema_name.is_empty()).then(|| request.schema_name.clone()), - }), - }]) + .await?; + Ok(()) } -/// Starts one Community Console query through Core and returns the opaque -/// operation id used by both HTTP and desktop transports. -/// -/// # Errors -/// -/// Returns request validation, datasource, storage, or engine failures before -/// the operation is accepted. -pub async fn start_sql_execution( +pub(crate) async fn assign_datasource_namespace( application: &Application, - request: &LegacySqlExecuteRequest, -) -> LegacyResult { - let (datasource_id, row_limit) = validate_sql_execute_request(request)?; + request: &LegacyDatasourceAssignmentRequest, +) -> LegacyResult<()> { application - .start_query(StartQueryRequest { - datasource_id, + .assign_datasource_namespace(chat2db_contract::AssignDatasourceNamespaceRequest { + datasource_id: request.data_source_id.as_string(), + namespace_id: request + .namespace_id + .as_ref() + .map(LegacyIdentifier::as_string), + }) + .await?; + Ok(()) +} + +fn legacy_workspace_node_ref(reference: &LegacyWorkspaceNodeRef) -> LegacyResult { + let node_type = match reference.node_type.trim().to_ascii_uppercase().as_str() { + "NAMESPACE" => WorkspaceNodeKind::Namespace, + "DATA_SOURCE" | "DATASOURCE" => WorkspaceNodeKind::DataSource, + _ => { + return Err(LegacyFailure::invalid( + "invalid_workspace_operation", + "workspace node type must be NAMESPACE or DATA_SOURCE", + )); + } + }; + Ok(WorkspaceNodeRef { + id: reference.id.as_string(), + node_type, + }) +} + +pub(crate) async fn format_legacy_sql( + application: &Application, + request: &LegacySqlUtilityRequest, +) -> LegacyResult { + let database_type = legacy_mysql_utility_database_type(&request.db_type)?; + Ok(application + .format_community_sql(FormatCommunitySqlRequest { + database_type, sql: request.sql.clone(), - parameters: Vec::new(), - limits: QueryLimits { - max_rows: row_limit.to_string(), - max_result_bytes: RESULT_PAGE_MAX_BYTES.to_string(), - batch_rows: request.page_size.min(512), - batch_bytes: 1024 * 1024, - result_ttl_seconds: 60, - }, }) - .await - .map_err(Into::into) + .await? + .sql) } -/// Waits for a Core query terminal event without translating away its error -/// code or message. Desktop streaming can subscribe independently and use -/// this as the final retained-result barrier. -/// -/// # Errors -/// -/// Returns subscription, database execution, or cancellation failures. -pub async fn wait_for_sql_execution( +pub(crate) async fn validate_legacy_select( application: &Application, - operation_id: &str, -) -> LegacyResult { - let mut subscription = application.subscribe_operation(operation_id, None).await?; - while let Some(envelope) = subscription.next_event().await? { - match envelope.event { - OperationEvent::Completed { result } => return Ok(result), - OperationEvent::Failed { error } => return Err(LegacyFailure::from_api(error)), - OperationEvent::Cancelled { .. } => { - return Err(LegacyFailure::invalid( - "sql_execution_cancelled", - "The SQL execution was cancelled", - )); - } - OperationEvent::Started | OperationEvent::Progress { .. } => {} - } - } - Err(LegacyFailure::invalid( - "sql_execution_incomplete", - "The SQL execution ended without a result", - )) + request: &LegacySqlUtilityRequest, +) -> LegacyResult { + let database_type = legacy_mysql_utility_database_type(&request.db_type)?; + Ok(application + .parse_community_sql(chat2db_contract::ParseCommunitySqlRequest { + database_type, + sql: request.sql.clone(), + }) + .await? + .is_select) } -/// Reads and translates a retained Core result into Community's historical -/// result-grid shape. This is shared by synchronous HTTP and desktop IPC. -/// -/// # Errors -/// -/// Returns invalid paging or retained-result read failures. -pub async fn read_sql_result( +pub(crate) async fn parse_legacy_sql( application: &Application, - request: &LegacySqlExecuteRequest, - metadata: &ResultMetadata, - duration: u64, -) -> LegacyResult { - let (offset, _) = sql_page_window(request)?; - let page = application - .result_page( - &metadata.id, - ResultPageRequest { - offset: offset.to_string(), - max_rows: request.page_size.to_string(), - max_bytes: RESULT_PAGE_MAX_BYTES.to_string(), - }, - ) + request: &LegacySqlParserRequest, +) -> LegacyResult { + let datasource_id = request.data_source_id.as_string(); + let database_type = resolve_mysql_database_type(application, &datasource_id, "").await?; + let validation = application + .validate_community_sql(ValidateCommunitySqlRequest { + database_type, + sql: request.sql.clone(), + }) .await?; - let has_next_page = page.has_more - || page.metadata.truncated_by_max_rows - || page.metadata.truncated_by_max_result_bytes; - let fuzzy_total = - if page.metadata.truncated_by_max_rows || page.metadata.truncated_by_max_result_bytes { - format!("{}+", page.metadata.row_count) - } else { - page.metadata.row_count.clone() - }; - let header_list = page.columns.iter().map(result_header).collect(); - let large_value_owner = application.create_large_value_owner(); - let data_list = page - .rows + + let mut search_from = 0; + let statements = validation + .statements + .iter() + .map(|statement| { + let (start, end) = locate_statement(&request.sql, &statement.sql, search_from); + search_from = end; + let (start_row, start_column) = utf16_line_column(&request.sql, start); + let (end_row, end_column) = utf16_line_column(&request.sql, end); + serde_json::json!({ + "sql": statement.sql, + "sqlStartRowNum": start_row, + "sqlStartColNum": start_column, + "sqlEndRowNum": end_row, + "sqlEndColNum": end_column, + "type": statement.kind, + "statementType": statement.statement_type, + "comment": "", + "identifiers": [], + "tableColumns": [], + "insertValueMappings": [] + }) + }) + .collect::>(); + let marks = validation + .diagnostics .into_iter() - .map(|row| { - row.values - .into_iter() - .zip(page.columns.iter()) - .map(|(value, column)| result_cell(application, &large_value_owner, value, column)) - .collect() + .map(|diagnostic| { + serde_json::json!({ + "endLineNum": diagnostic.end_line, + "startLineNum": diagnostic.start_line, + "startColNum": diagnostic.start_column, + "endColNum": diagnostic.end_column, + "message": diagnostic.message, + "type": "error" + }) }) - .collect(); - Ok(LegacyManageResult { - data_list, - header_list, - description: "Query executed successfully".to_owned(), - message: String::new(), - sql: request.sql.clone(), - original_sql: request.sql.clone(), - success: true, - duration, - update_count: 0, - can_edit: false, - table_name: request.table_name.clone(), - sql_type: legacy_sql_type(&request.sql).to_owned(), - refresh_targets: Vec::new(), - page_no: request.page_no, - page_size: request.page_size, - fuzzy_total, - has_next_page, - execute_sql_params: request.clone(), - extra: serde_json::json!({}), - comment: None, - result_set_id: request.result_set_id, - statement_sequence: Some(1), - execution_metrics: None, - execution_context: Some(LegacyExecutionContext { - database_name: (!request.database_name.is_empty()) - .then(|| request.database_name.clone()), - schema_name: (!request.schema_name.is_empty()).then(|| request.schema_name.clone()), - }), - }) + .collect::>(); + Ok(serde_json::json!({ + "sqlStatementList": statements, + "markMessageList": marks + })) } -/// Executes the synchronous Community web contract while retaining Core's -/// asynchronous operation and result-store lifecycle internally. -/// -/// # Errors -/// -/// Returns request validation or failures that occur before Core accepts the -/// query. Failures after acceptance are returned as Community result items. -pub async fn execute_sql( +pub(crate) async fn complete_legacy_sql( application: &Application, - request: &LegacySqlExecuteRequest, -) -> LegacyResult> { - let _ = validate_sql_execute_request(request)?; - if uses_native_mysql_console(application, request).await? { - let execution_id = application.create_large_value_owner(); - return execute_mysql_sql( - application, - request, - MysqlConsoleCancellation::new(), - &execution_id, - "SQL_EDITOR_HTTP", - ) - .await; + request: &LegacySqlCompletionRequest, +) -> LegacyResult { + let datasource_id = request.data_source_id.as_string(); + let database_type = resolve_mysql_database_type(application, &datasource_id, "").await?; + let sql = if request.sql.is_empty() { + format!("{}{}", request.before_sql, request.after_sql) + } else { + request.sql.clone() + }; + let cursor_utf16 = request + .cursor + .unwrap_or_else(|| utf16_len(&request.before_sql)); + let keyword_case = match request.keyword_case.trim().to_ascii_uppercase().as_str() { + "LOWER" => "LOWER", + _ => "UPPER", } - let started_at = Instant::now(); - let accepted = start_sql_execution(application, request).await?; - let terminal = tokio::time::timeout( - SQL_EXECUTION_TIMEOUT, - wait_for_sql_execution(application, &accepted.operation_id), - ) - .await; - let duration = elapsed_millis(started_at); - match terminal { - Ok(Ok(metadata)) => read_sql_result(application, request, &metadata, duration) - .await - .map(|result| vec![result]), - Ok(Err(error)) => Ok(vec![sql_failure_result(request, &error, duration)]), - Err(_) => { - application.cancel_operation(&accepted.operation_id).await; - Ok(vec![sql_failure_result( - request, - &LegacyFailure::invalid( - "sql_execution_timeout", - "The SQL execution did not finish in time", - ), - duration, - )]) + .to_owned(); + let active_snippet_slot = request.active_snippet_slot.as_ref().and_then(|slot| { + match (slot.replace_start, slot.replace_end) { + (Some(replace_start_utf16), Some(replace_end_utf16)) + if !slot.slot_type.trim().is_empty() => + { + Some(CommunitySqlCompletionActiveSnippetSlot { + r#type: slot.slot_type.clone(), + replace_start_utf16, + replace_end_utf16, + }) + } + _ => None, + } + }); + let completion = application + .complete_community_sql(CompleteCommunitySqlRequest { + datasource_id, + database_type, + database_name: request.database_name.clone(), + schema_name: request.schema_name.clone(), + sql, + cursor_utf16, + min_prefix_length: 0, + need_full_name: request.need_full_name, + keyword_case, + active_snippet_slot, + }) + .await?; + let mut value = serde_json::to_value(completion).map_err(|_| LegacyFailure { + code: "internal_error".to_owned(), + message: "The SQL completion response could not be encoded".to_owned(), + })?; + if let Some(response) = value.as_object_mut() { + rename_json_field(response, "replaceStartUtf16", "replaceStart"); + rename_json_field(response, "replaceEndUtf16", "replaceEnd"); + if let Some(candidates) = response + .get_mut("candidates") + .and_then(serde_json::Value::as_array_mut) + { + for candidate in candidates { + if let Some(candidate) = candidate.as_object_mut() { + rename_json_field(candidate, "replaceStartUtf16", "replaceStart"); + rename_json_field(candidate, "replaceEndUtf16", "replaceEnd"); + } + } } } + Ok(value) } -/// Executes the Community single-result DDL contract. -/// -/// # Errors -/// -/// Returns request, datasource, execution, or missing-result failures. -pub async fn execute_ddl( +pub(crate) async fn legacy_sql_keywords( application: &Application, - request: &LegacySqlExecuteRequest, -) -> LegacyResult { - execute_sql(application, request) + request: &LegacyMetadataQuery, +) -> LegacyResult { + let datasource_id = request.data_source_id.as_string(); + let database_type = + resolve_mysql_database_type(application, &datasource_id, &request.database_type).await?; + let datasource_name = application.get_datasource(&datasource_id).await?.name; + let databases = application + .list_community_databases(ListCommunityDatabasesRequest { + datasource_id: datasource_id.clone(), + database_type: database_type.clone(), + }) .await? - .into_iter() - .next() - .ok_or_else(|| { - LegacyFailure::invalid( - "sql_execution_incomplete", - "The SQL execution ended without a result", - ) + .items; + + let database_name = request.database_name.trim(); + if database_name.is_empty() { + return Ok(legacy_sql_keyword_payload( + &datasource_name, + databases, + Vec::new(), + Vec::new(), + Vec::new(), + Vec::new(), + Vec::new(), + )); + } + + let schemas = application + .list_community_schemas(ListCommunitySchemasRequest { + datasource_id: datasource_id.clone(), + database_type: database_type.clone(), + database_name: database_name.to_owned(), + }) + .await? + .items; + let tables = application + .list_community_tables(ListCommunityTablesRequest { + datasource_id: datasource_id.clone(), + database_type: database_type.clone(), + database_name: database_name.to_owned(), + schema_name: request.schema_name.clone(), + table_name_pattern: String::new(), + }) + .await? + .items; + let views = application + .list_community_views(ListCommunityViewsRequest { + datasource_id: datasource_id.clone(), + database_type: database_type.clone(), + database_name: database_name.to_owned(), + schema_name: request.schema_name.clone(), + view_name_pattern: String::new(), }) + .await? + .items; + let functions = application + .list_community_functions(ListCommunityFunctionsRequest { + datasource_id: datasource_id.clone(), + database_type: database_type.clone(), + database_name: database_name.to_owned(), + schema_name: request.schema_name.clone(), + }) + .await? + .items; + let procedures = application + .list_community_procedures(ListCommunityProceduresRequest { + datasource_id, + database_type, + database_name: database_name.to_owned(), + schema_name: request.schema_name.clone(), + }) + .await? + .items; + + Ok(legacy_sql_keyword_payload( + &datasource_name, + databases, + schemas, + tables, + views, + functions, + procedures, + )) } -/// Counts the rows produced by one `MySQL` query for Community's total-row control. -pub(crate) async fn count_mysql_rows( +fn legacy_sql_keyword_payload( + datasource_name: &str, + databases: Vec, + schemas: Vec, + tables: Vec, + views: Vec, + functions: Vec, + procedures: Vec, +) -> serde_json::Value { + serde_json::json!({ + "databases": databases.into_iter().map(|database| serde_json::json!({ + "datasourceName": datasource_name, + "databaseName": database.name, + "insertText": database.name + })).collect::>(), + "schemas": schemas.into_iter().map(|schema| serde_json::json!({ + "datasourceName": datasource_name, + "databaseName": schema.database_name, + "schemaName": schema.name, + "insertText": schema.name + })).collect::>(), + "tables": tables.into_iter().map(|table| serde_json::json!({ + "datasourceName": datasource_name, + "databaseName": table.database_name, + "schemaName": table.schema_name, + "tableName": table.name, + "tableAlias": "", + "comment": table.comment, + "insertText": table.name + })).collect::>(), + "views": views.into_iter().map(|view| serde_json::json!({ + "datasourceName": datasource_name, + "databaseName": view.database_name, + "schemaName": view.schema_name, + "viewName": view.name, + "insertText": view.name + })).collect::>(), + "functions": functions.into_iter().map(|function| serde_json::json!({ + "datasourceName": datasource_name, + "databaseName": function.database_name, + "schemaName": function.schema_name, + "functionName": function.name, + "returnType": "", + "parameters": [], + "insertText": function.name + })).collect::>(), + "procedures": procedures.into_iter().map(|procedure| serde_json::json!({ + "datasourceName": datasource_name, + "databaseName": procedure.database_name, + "schemaName": procedure.schema_name, + "procedureName": procedure.name, + "returnType": "", + "parameters": [], + "insertText": procedure.name + })).collect::>() + }) +} + +pub(crate) async fn legacy_sql_hover( application: &Application, - request: &LegacySqlExecuteRequest, -) -> LegacyResult { - let (datasource_id, _) = validate_sql_execute_request(request)?; - if !uses_native_mysql_console(application, request).await? { - return Err(LegacyFailure::invalid( - "unsupported_database_type", - "The historical count route currently supports native MySQL only", - )); - } - if request.sql.trim().is_empty() { - return Ok(0); + request: &LegacySqlHoverRequest, +) -> LegacyResult> { + let datasource_id = request.data_source_id.as_string(); + let database_type = resolve_mysql_database_type(application, &datasource_id, "").await?; + let datasource_name = application.get_datasource(&datasource_id).await?.name; + let database_name = if request + .hover_identifier + .identifier_database + .trim() + .is_empty() + { + request.database_name.clone() + } else { + request.hover_identifier.identifier_database.clone() + }; + let schema_name = if request.hover_identifier.identifier_schema.trim().is_empty() { + request.schema_name.clone() + } else { + request.hover_identifier.identifier_schema.clone() + }; + let table_name = if request.hover_identifier.identifier_table.trim().is_empty() { + request.hover_identifier.name.clone() + } else { + request.hover_identifier.identifier_table.clone() + }; + if table_name.trim().is_empty() { + return Ok(Vec::new()); } - let count_sql = build_mysql_count_query(&request.sql)?; - let results = application - .execute_mysql_console( - MysqlConsoleRequest { - datasource_id, - database_name: request.database_name.clone(), - sql: count_sql, - page_no: 1, - page_size: 1, - result_set_id: None, - single: true, - page_size_all: false, - explain: false, - error_continue: false, - }, - MysqlConsoleCancellation::new(), - ) - .await?; - let result = results.into_iter().next().ok_or_else(|| { - LegacyFailure::invalid( - "mysql_count_failed", - "The MySQL count query returned no result", - ) - })?; - if !result.success { - return Err(LegacyFailure { - code: "mysql_count_failed".to_owned(), - message: result.error.map_or(result.message, |error| error.message), - }); + + let columns = application + .list_community_columns(ListCommunityColumnsRequest { + datasource_id: datasource_id.clone(), + database_type: database_type.clone(), + database_name: database_name.clone(), + schema_name: schema_name.clone(), + table_name: table_name.clone(), + }) + .await? + .items; + if let Some(column) = columns.into_iter().find(|column| { + column + .name + .eq_ignore_ascii_case(&request.hover_identifier.name) + }) { + return Ok(vec![serde_json::json!({ + "databaseName": database_name, + "schemaName": schema_name, + "tableName": table_name, + "datasourceName": datasource_name, + "viewName": "", + "triggerName": "", + "ddl": "", + "comment": column.comment, + "dataType": column.column_type, + "columnName": column.name + })]); } - let value = result - .rows - .first() - .and_then(|row| row.values.first()) - .ok_or_else(|| { - LegacyFailure::invalid( - "mysql_count_failed", - "The MySQL count query returned no value", - ) - })?; - let (JdbcValue::SignedInteger { value } - | JdbcValue::UnsignedInteger { value } - | JdbcValue::Decimal { value } - | JdbcValue::Text { value }) = value + + let tables = application + .list_community_tables(ListCommunityTablesRequest { + datasource_id: datasource_id.clone(), + database_type, + database_name: database_name.clone(), + schema_name: schema_name.clone(), + table_name_pattern: table_name.clone(), + }) + .await? + .items; + let Some(table) = tables + .into_iter() + .find(|table| table.name.eq_ignore_ascii_case(&table_name)) else { - return Err(LegacyFailure::invalid( - "mysql_count_failed", - "The MySQL count query returned a non-integer value", - )); + return Ok(Vec::new()); }; - value.parse::().map_err(|_| { - LegacyFailure::invalid( - "mysql_count_failed", - "The MySQL count query returned an invalid integer", - ) - }) -} - -/// Executes the native `MySQL` Console contract with a caller-owned cancellation -/// source. Desktop keeps this source by execution id while HTTP uses it for the -/// synchronous timeout boundary. -/// -/// # Errors -/// -/// Returns validation or datasource failures that occur before execution. -pub async fn execute_mysql_sql( - application: &Application, - request: &LegacySqlExecuteRequest, - cancellation: MysqlConsoleCancellation, - execution_id: &str, - history_source: &str, -) -> LegacyResult> { - let (datasource_id, _) = validate_sql_execute_request(request)?; - let editable_columns = if request.table_name.trim().is_empty() { - None + let ddl = application + .table_ddl(&datasource_id, &database_name, &schema_name, &table_name) + .await + .unwrap_or_default(); + Ok(vec![serde_json::json!({ + "databaseName": database_name, + "schemaName": schema_name, + "tableName": table.name, + "datasourceName": datasource_name, + "viewName": "", + "triggerName": "", + "ddl": ddl, + "comment": table.comment, + "dataType": "", + "columnName": "" + })]) +} + +fn legacy_mysql_utility_database_type(database_type: &str) -> LegacyResult { + let database_type = normalize_database_type(database_type); + let database_type = if database_type.is_empty() { + "MYSQL".to_owned() } else { - let database_type = - resolve_database_type(application, &datasource_id, &request.database_type).await?; - application - .list_community_columns(ListCommunityColumnsRequest { - datasource_id: datasource_id.clone(), - database_type, - database_name: request.database_name.clone(), - schema_name: request.schema_name.clone(), - table_name: request.table_name.clone(), - }) - .await - .ok() - .map(|columns| columns.items) + database_type }; + if database_type == "MYSQL" { + Ok(database_type) + } else { + Err(LegacyFailure { + code: "unsupported_database_type".to_owned(), + message: "This Community compatibility route currently supports MySQL only".to_owned(), + }) + } +} - let execution = application.execute_mysql_console( - MysqlConsoleRequest { - datasource_id, - database_name: request.database_name.clone(), - sql: request.sql.clone(), - page_no: request.page_no, - page_size: request.page_size, - result_set_id: request.result_set_id, - single: request.single, - page_size_all: request.page_size_all, - explain: request.explain, - error_continue: request.error_continue.unwrap_or(true), - }, - cancellation.clone(), - ); - let large_value_owner = application.create_large_value_owner(); - tokio::pin!(execution); - let timed_out = tokio::select! { - result = &mut execution => Some(result), - () = tokio::time::sleep(SQL_EXECUTION_TIMEOUT) => None, - }; - let results = match timed_out { - Some(Ok(results)) => results - .into_iter() - .map(|result| { - mysql_console_result( - application, - &large_value_owner, - request, - result, - editable_columns.as_deref(), - ) - }) - .collect(), - Some(Err(error)) => { - let error = LegacyFailure::from(error); - vec![sql_failure_result(request, &error, 0)] - } - None => { - let _ = cancellation.cancel(Some("The SQL execution timed out".to_owned())); - if tokio::time::timeout(SQL_CANCELLATION_GRACE, &mut execution) - .await - .is_err() - { - tracing::warn!( - "MySQL Console execution did not finish during the timeout cleanup window" - ); - } - vec![sql_failure_result( - request, - &LegacyFailure::invalid( - "sql_execution_timeout", - "The SQL execution did not finish in time", - ), - u64::try_from(SQL_EXECUTION_TIMEOUT.as_millis()).unwrap_or(u64::MAX), - )] +fn locate_statement(sql: &str, statement: &str, search_from: usize) -> (usize, usize) { + let needle = statement.trim(); + if needle.is_empty() { + return (search_from.min(sql.len()), search_from.min(sql.len())); + } + let search_from = search_from.min(sql.len()); + if let Some(relative) = sql[search_from..].find(needle) { + let start = search_from + relative; + return (start, start + needle.len()); + } + if let Some(start) = sql.find(needle) { + return (start, start + needle.len()); + } + (0, sql.len()) +} + +fn utf16_line_column(value: &str, byte_offset: usize) -> (u32, u32) { + let mut line = 1_u32; + let mut column = 1_u32; + for character in value[..byte_offset.min(value.len())].chars() { + if character == '\n' { + line = line.saturating_add(1); + column = 1; + } else { + column = + column.saturating_add(u32::try_from(character.len_utf16()).unwrap_or(u32::MAX)); } - }; - record_mysql_console_history_best_effort( - application, - request, - &results, - execution_id, - history_source, - ) - .await; - Ok(results) + } + (line, column) } -async fn record_mysql_console_history_best_effort( - application: &Application, - request: &LegacySqlExecuteRequest, - results: &[LegacyManageResult], - execution_id: &str, - history_source: &str, +fn utf16_len(value: &str) -> u32 { + u32::try_from(value.encode_utf16().count()).unwrap_or(u32::MAX) +} + +fn rename_json_field( + object: &mut serde_json::Map, + old: &str, + new: &str, ) { - let Some(storage) = application.storage().cloned() else { - return; - }; - let mut statements = BTreeMap::>::new(); - for (index, result) in results.iter().enumerate() { - let fallback = u32::try_from(index).unwrap_or(u32::MAX).saturating_add(1); - statements - .entry(result.statement_sequence.unwrap_or(fallback)) - .or_default() - .push(result); - } - for statement_results in statements.into_values() { - let Some(first) = statement_results.first().copied() else { - continue; - }; - let operation_rows = statement_results - .iter() - .map(|result| result.update_count) - .fold(0_u64, u64::saturating_add); - let use_time = statement_results - .iter() - .map(|result| result.duration) - .fold(0_u64, u64::saturating_add); - let message = statement_results - .iter() - .find(|result| !result.success && !result.message.trim().is_empty()) - .map_or("", |result| result.message.as_str()); - let extend_info = serde_json::to_string(&serde_json::json!({ - "source": history_source, - "sqlType": first.sql_type, - "executionId": execution_id, - "statementSequence": first.statement_sequence.unwrap_or(1), - "message": message, - })) - .ok(); - let input = CreateOperationLog { - name: None, - data_source_id: Some(request.data_source_id.as_string()), - data_source_name: non_blank(&request.data_source_name), - connectable: Some(true), - database_name: non_blank(&request.database_name), - database_type: Some(default_if_blank(&request.database_type, "MYSQL")), - ddl: first.original_sql.clone(), - status: mysql_console_history_status(statement_results.as_slice()).to_owned(), - operation_rows: i64::try_from(operation_rows).ok(), - use_time: i64::try_from(use_time).ok(), - extend_info, - schema_name: non_blank(&request.schema_name), - organization_id: None, - user_name: None, - more: first.original_sql.chars().count() > 200, - operation_type: "SQL_EXECUTE".to_owned(), - }; - let result = tokio::task::spawn_blocking({ - let storage = storage.clone(); - move || storage.create_operation_log(input) - }) - .await; - match result { - Ok(Ok(_)) => {} - Ok(Err(error)) => tracing::warn!(%error, "MySQL Console history write failed"), - Err(error) => tracing::warn!(%error, "MySQL Console history task failed"), - } + if let Some(value) = object.remove(old) { + object.insert(new.to_owned(), value); } } -fn mysql_console_history_status(results: &[&LegacyManageResult]) -> &'static str { - if results.iter().any(|result| { - result - .extra - .get("messages") - .and_then(serde_json::Value::as_array) - .into_iter() - .flatten() - .filter_map(|message| message.get("errorCode")) - .filter_map(serde_json::Value::as_str) - .any(|code| matches!(code, "mysql_console_cancelled" | "sql_execution_cancelled")) - }) { - "cancelled" - } else if results.iter().all(|result| result.success) { - "success" - } else { - "fail" +fn pinned_table_request(query: &LegacyTableDetailQuery) -> CommunityPinnedTableRequest { + CommunityPinnedTableRequest { + data_source_id: query.data_source_id.as_string(), + database_name: query.database_name.clone(), + schema_name: query.schema_name.clone(), + table_name: query.table_name.clone(), } } -/// Reads one bounded retained large-cell chunk in the requested display format. -/// -/// # Errors -/// -/// Returns validation, expired-token, range, decoding, or unsupported-format failures. -pub fn read_large_cell_value( +pub(crate) async fn add_table_pin( application: &Application, - request: &LegacyLargeCellValueRequest, -) -> LegacyResult { - if request.large_value_id.trim().is_empty() { - return Err(LegacyFailure::invalid( - "invalid_large_cell_value_request", - "largeValueId is required", - )); - } - let encoded = matches!( - request.format.trim().to_ascii_lowercase().as_str(), - "base64" | "hex" - ); - let chunk = if encoded { - application.read_large_value_encoded_chunk( - &request.large_value_id, - request.offset, - request.limit, - ) - } else { - application.read_large_value_chunk(&request.large_value_id, request.offset, request.limit) - } - .map_err(LegacyFailure::from)?; - format_large_value_chunk(chunk, &request.format) + request: &LegacyTableDetailQuery, +) -> LegacyResult<()> { + Ok(application + .pin_community_mysql_table(pinned_table_request(request)) + .await?) } -/// Writes one complete retained large-cell value to a unique temporary download path. -/// -/// # Errors -/// -/// Returns validation, expired-token, decoding, task, directory, or file-write failures. -pub async fn download_large_cell_value_to_path( +pub(crate) async fn delete_table_pin( application: &Application, - request: &LegacyLargeCellDownloadRequest, -) -> LegacyResult { - let application = application.clone(); - let request = request.clone(); - tokio::task::spawn_blocking(move || { - let download = prepare_large_cell_download(&application, &request)?; - let directory = std::env::temp_dir().join("chat2db").join("downloads"); - fs::create_dir_all(&directory).map_err(|_| LegacyFailure { - code: "large_cell_download_failed".to_owned(), - message: "The large cell download directory could not be created".to_owned(), - })?; - let path = unique_large_cell_download_path( - &directory, - &request.large_value_id, - download.extension, - ); - fs::write(&path, download.bytes).map_err(|_| LegacyFailure { - code: "large_cell_download_failed".to_owned(), - message: "The large cell value could not be written to disk".to_owned(), - })?; - Ok(path.to_string_lossy().into_owned()) - }) - .await - .map_err(|_| LegacyFailure { - code: "large_cell_download_failed".to_owned(), - message: "The large cell download task did not finish".to_owned(), - })? + request: &LegacyTableDetailQuery, +) -> LegacyResult<()> { + Ok(application + .unpin_community_mysql_table(pinned_table_request(request)) + .await?) } -struct PreparedLargeCellDownload { - bytes: Vec, - content_type: &'static str, - extension: &'static str, +pub(crate) async fn list_table_pins( + application: &Application, + request: &LegacyTableDetailQuery, +) -> LegacyResult> { + Ok(application + .list_community_mysql_pinned_tables(pinned_table_request(request)) + .await? + .items) } -fn prepare_large_cell_download( +pub(crate) async fn get_er_info( application: &Application, - request: &LegacyLargeCellDownloadRequest, -) -> LegacyResult { - if request.large_value_id.trim().is_empty() { - return Err(LegacyFailure::invalid( - "invalid_large_cell_value_request", - "largeValueId is required", - )); - } - let (raw, value_type) = read_complete_large_value(application, &request.large_value_id)?; - match request.format.trim().to_ascii_lowercase().as_str() { - "" | "raw" => Ok(PreparedLargeCellDownload { - bytes: raw, - content_type: if value_type == LargeValueType::Text { - "text/plain; charset=utf-8" - } else { - "application/octet-stream" - }, - extension: if value_type == LargeValueType::Text { - "txt" - } else { - "bin" - }, - }), - "text" => Ok(PreparedLargeCellDownload { - bytes: String::from_utf8_lossy(&raw).into_owned().into_bytes(), - content_type: "text/plain; charset=utf-8", - extension: "txt", - }), - "hex" => Ok(PreparedLargeCellDownload { - bytes: encode_hex(&raw).into_bytes(), - content_type: "text/plain; charset=utf-8", - extension: "hex", - }), - _ => Err(LegacyFailure::invalid( - "invalid_large_cell_value_request", - "format must be raw, text, or hex", - )), - } + query: &LegacyMetadataQuery, +) -> LegacyResult { + Ok(application + .community_mysql_er_model(CommunityErQueryRequest { + data_source_id: query.data_source_id.as_string(), + database_name: query.database_name.clone(), + schema_name: query.schema_name.clone(), + }) + .await?) } -fn read_complete_large_value( +pub(crate) async fn save_er_position( application: &Application, - large_value_id: &str, -) -> LegacyResult<(Vec, LargeValueType)> { - let mut offset = 0_u64; - let mut output = Vec::new(); - let mut value_type = None; - loop { - let chunk = application - .read_large_value_chunk(large_value_id, offset, LARGE_VALUE_CHUNK_SIZE) - .map_err(LegacyFailure::from)?; - value_type.get_or_insert(chunk.display_mode); - output.extend(decode_large_value_chunk(&chunk)?); - if chunk.eof { - return Ok((output, value_type.unwrap_or(LargeValueType::Binary))); - } - if chunk.next_offset <= offset { - return Err(LegacyFailure { - code: "large_cell_download_failed".to_owned(), - message: "The large cell value did not advance while downloading".to_owned(), - }); - } - offset = chunk.next_offset; - } + request: &LegacyErPositionRequest, +) -> LegacyResult<()> { + Ok(application + .save_community_mysql_er_position(CommunityErPositionRequest { + data_source_id: request.data_source_id.as_string(), + database_name: request.database_name.clone(), + schema_name: request.schema_name.clone(), + position: request.position.clone(), + }) + .await?) } -fn format_large_value_chunk( - chunk: LargeValueChunk, - format: &str, -) -> LegacyResult { - let normalized = format.trim().to_ascii_lowercase(); - let (value, encoding) = match normalized.as_str() { - "" | "auto" => ( - chunk.value.clone(), - match chunk.encoding { - LargeValueEncoding::Utf8 => "utf-8", - LargeValueEncoding::Base64 => "base64", - }, - ), - "base64" => ( - BASE64_STANDARD.encode(decode_large_value_chunk(&chunk)?), - "base64", - ), - "text" => ( - String::from_utf8_lossy(&decode_large_value_chunk(&chunk)?).into_owned(), - "utf-8", - ), - "hex" => (encode_hex(&decode_large_value_chunk(&chunk)?), "hex"), - _ => { - return Err(LegacyFailure::invalid( - "invalid_large_cell_value_request", - "format must be text, hex, base64, or auto", - )); - } - }; - Ok(LegacyLargeCellChunk { - value, - offset: chunk.offset, - next_offset: chunk.next_offset, - eof: chunk.eof, - size_bytes: chunk.size_bytes, - size_chars: chunk.size_chars, - encoding: encoding.to_owned(), - content_type: chunk.content_type, - display_mode: chunk.display_mode, - }) +pub(crate) async fn account_capability( + application: &Application, + query: &LegacyAccountQuery, +) -> LegacyResult { + Ok(application + .mysql_account_capability(&query.data_source_id.as_string()) + .await?) } -fn decode_large_value_chunk(chunk: &LargeValueChunk) -> LegacyResult> { - match chunk.encoding { - LargeValueEncoding::Utf8 => Ok(chunk.value.as_bytes().to_vec()), - LargeValueEncoding::Base64 => { - BASE64_STANDARD - .decode(chunk.value.as_bytes()) - .map_err(|_| LegacyFailure { - code: "large_cell_value_invalid".to_owned(), - message: "The retained binary value could not be decoded".to_owned(), - }) - } - } +pub(crate) async fn list_accounts( + application: &Application, + query: &LegacyAccountQuery, +) -> LegacyResult> { + Ok(application + .list_mysql_accounts(&query.data_source_id.as_string()) + .await? + .items) } -fn encode_hex(bytes: &[u8]) -> String { - const HEX: &[u8; 16] = b"0123456789ABCDEF"; - let mut output = String::with_capacity(bytes.len().saturating_mul(2)); - for byte in bytes { - output.push(char::from(HEX[usize::from(byte >> 4)])); - output.push(char::from(HEX[usize::from(byte & 0x0f)])); - } - output +pub(crate) async fn account_grants( + application: &Application, + query: &LegacyAccountQuery, +) -> LegacyResult> { + Ok(application + .mysql_account_grants(&CommunityAccountGrantsRequest { + datasource_id: query.data_source_id.as_string(), + user: query.user.clone(), + host: query.host.clone(), + }) + .await? + .items) } -fn unique_large_cell_download_path( - directory: &std::path::Path, - large_value_id: &str, - extension: &str, -) -> PathBuf { - let token_fragment = large_value_id - .chars() - .filter(char::is_ascii_alphanumeric) - .take(12) - .collect::(); - directory.join(format!( - "chat2db-cell-{}-{token_fragment}.{extension}", - unix_epoch_millis() - )) +pub(crate) fn preview_account( + application: &Application, + request: &LegacyAccountCommandRequest, +) -> LegacyResult { + Ok(application.preview_mysql_account(&request.core_request())?) } -/// Reports whether the request resolves to the native `MySQL` driver. -/// -/// # Errors -/// -/// Returns a datasource lookup failure when the requested datasource is absent. -pub async fn uses_native_mysql_console( +pub(crate) async fn execute_account( application: &Application, - request: &LegacySqlExecuteRequest, -) -> LegacyResult { - if request.database_type.eq_ignore_ascii_case("mysql") { - return Ok(true); - } - let datasource = application - .get_datasource(&request.data_source_id.as_string()) - .await?; - Ok(datasource.driver_id.eq_ignore_ascii_case("mysql") - || datasource.driver_id.to_ascii_lowercase().contains("mysql")) + request: &LegacyAccountCommandRequest, +) -> LegacyResult { + Ok(application + .execute_mysql_account(&request.core_request()) + .await?) } -fn mysql_console_result( +pub(crate) async fn preview_schema_diff( application: &Application, - large_value_owner: &str, - request: &LegacySqlExecuteRequest, - result: MysqlConsoleResult, - editable_columns: Option<&[CommunityTableColumn]>, -) -> LegacyManageResult { - let MysqlConsoleResult { - statement_sequence, - result_set_id, - sql, - success, - message, - update_count, - columns, - rows, - row_count, - has_more, - duration_ms, - error, - } = result; - let mut header_list: Vec<_> = columns.iter().map(result_header).collect(); - let can_edit = editable_columns.is_some_and(|editable_columns| { - enrich_direct_table_headers(&mut header_list, editable_columns, &request.table_name) - }); - let mut data_list: Vec> = rows + request: &CommunitySchemaDiffRequest, +) -> LegacyResult { + Ok(application + .preview_mysql_schema_diff(request) + .await? + .into_inner()) +} + +/// Lists databases through the retained Community metadata implementation. +pub(crate) async fn list_databases( + application: &Application, + query: &LegacyMetadataQuery, +) -> LegacyResult> { + let datasource_id = query.data_source_id.as_string(); + let database_type = + resolve_database_type(application, &datasource_id, &query.database_type).await?; + Ok(application + .list_community_databases(ListCommunityDatabasesRequest { + datasource_id, + database_type, + }) + .await? + .items .into_iter() - .map(|row| { - row.values - .into_iter() - .zip(columns.iter()) - .map(|(value, column)| result_cell(application, large_value_owner, value, column)) - .collect() + .map(|database| LegacyDatabase { + name: database.name, + description: database.comment, + count: 0, + system: database.system, + }) + .collect()) +} + +/// Lists schemas through the retained Community metadata implementation. +pub(crate) async fn list_schemas( + application: &Application, + query: &LegacyMetadataQuery, +) -> LegacyResult> { + let datasource_id = query.data_source_id.as_string(); + let database_type = + resolve_database_type(application, &datasource_id, &query.database_type).await?; + Ok(application + .list_community_schemas(ListCommunitySchemasRequest { + datasource_id, + database_type, + database_name: query.database_name.clone(), + }) + .await? + .items + .into_iter() + .map(|schema| LegacySchema { + name: schema.name, + system: schema.system, + }) + .collect()) +} + +pub(crate) async fn database_schema_list( + application: &Application, + query: &LegacyMetadataQuery, +) -> LegacyResult { + let databases = list_databases(application, query) + .await? + .into_iter() + .map(|database| LegacyMetaDatabase { + name: database.name, + // Native MySQL exposes databases as catalogs and has no second schema layer. + schemas: Vec::new(), }) .collect(); - if can_edit { - let offset = u64::from(request.page_no.saturating_sub(1)) - .saturating_mul(u64::from(request.page_size)); - prepend_synthetic_row_numbers(&mut header_list, &mut data_list, offset); - } - let sql_type = legacy_sql_type(&sql).to_owned(); - let extra = error.map_or_else( - || serde_json::json!({}), + Ok(LegacyMetaSchemaResponse { + databases, + schemas: Vec::new(), + }) +} + +/// Lists and paginates tables through Community metadata. +pub(crate) async fn list_tables( + application: &Application, + query: &LegacyTableListQuery, +) -> LegacyResult> { + validate_metadata_page(query)?; + let datasource_id = query.data_source_id.as_string(); + let database_type = + resolve_database_type(application, &datasource_id, &query.database_type).await?; + let pinned: HashSet = application + .list_community_mysql_pinned_tables(CommunityPinnedTableRequest { + data_source_id: datasource_id.clone(), + database_name: query.database_name.clone(), + schema_name: query.schema_name.clone(), + table_name: String::new(), + }) + .await? + .items + .into_iter() + .collect(); + let mut items: Vec = application + .list_community_tables(ListCommunityTablesRequest { + datasource_id, + database_type, + database_name: query.database_name.clone(), + schema_name: query.schema_name.clone(), + // Community's MySQL metadata implementation requires an empty + // pattern to enumerate all tables. + table_name_pattern: String::new(), + }) + .await? + .items + .into_iter() + .map(|table| { + let mut response = table_response(table); + response.pinned = pinned.contains(&response.name); + response + }) + .collect(); + items.retain(|item| table_matches_search(item, &query.search_key)); + Ok(paginate(items, query.page_no, query.page_size)) +} + +/// Lists the compact table projection used by autocomplete and table pickers. +pub(crate) async fn list_simple_tables( + application: &Application, + query: &LegacyTableListQuery, +) -> LegacyResult> { + validate_metadata_page(query)?; + let datasource_id = query.data_source_id.as_string(); + let database_type = + resolve_database_type(application, &datasource_id, &query.database_type).await?; + Ok(application + .list_community_tables(ListCommunityTablesRequest { + datasource_id, + database_type, + database_name: query.database_name.clone(), + schema_name: query.schema_name.clone(), + table_name_pattern: String::new(), + }) + .await? + .items + .into_iter() + .map(simple_table_response) + .collect()) +} + +/// Lists table or view columns in the historical `ColumnResponse` shape. +pub(crate) async fn list_columns( + application: &Application, + query: &LegacyTableDetailQuery, +) -> LegacyResult> { + let datasource_id = query.data_source_id.as_string(); + let database_type = + resolve_database_type(application, &datasource_id, &query.database_type).await?; + Ok(application + .list_community_columns(ListCommunityColumnsRequest { + datasource_id, + database_type, + database_name: query.database_name.clone(), + schema_name: query.schema_name.clone(), + table_name: query.table_name.clone(), + }) + .await? + .items + .into_iter() + .map(column_response) + .collect()) +} + +/// Lists table indexes in the historical `IndexResponse` shape. +pub(crate) async fn list_indexes( + application: &Application, + query: &LegacyTableDetailQuery, +) -> LegacyResult> { + let datasource_id = query.data_source_id.as_string(); + let database_type = + resolve_database_type(application, &datasource_id, &query.database_type).await?; + Ok(application + .list_community_indexes(ListCommunityIndexesRequest { + datasource_id, + database_type, + database_name: query.database_name.clone(), + schema_name: query.schema_name.clone(), + table_name: query.table_name.clone(), + }) + .await? + .items + .into_iter() + .map(index_response) + .collect()) +} + +/// Community's historical key endpoint is an alias of its index metadata. +pub(crate) async fn list_keys( + application: &Application, + query: &LegacyTableDetailQuery, +) -> LegacyResult> { + list_indexes(application, query).await +} + +/// Returns the native `SHOW CREATE TABLE` result used by Community's export action. +pub(crate) async fn export_table_ddl( + application: &Application, + query: &LegacyTableDetailQuery, +) -> LegacyResult { + let datasource_id = query.data_source_id.as_string(); + resolve_mysql_database_type(application, &datasource_id, &query.database_type).await?; + Ok(application + .table_ddl( + &datasource_id, + &query.database_name, + &query.schema_name, + &query.table_name, + ) + .await?) +} + +/// Preserves Community `MySQL`'s null create/alter example configuration. +pub(crate) fn mysql_table_ddl_example( + query: &LegacyTableDdlExampleQuery, +) -> LegacyResult> { + if normalize_database_type(&query.db_type) != "MYSQL" { + return Err(LegacyFailure { + code: "unsupported_database_type".to_owned(), + message: "This Community compatibility route currently supports MySQL only".to_owned(), + }); + } + Ok(None) +} + +/// Lists views in the same page wrapper used by the retained tree. +pub(crate) async fn list_views( + application: &Application, + query: &LegacyTableListQuery, +) -> LegacyResult> { + validate_metadata_page(query)?; + let datasource_id = query.data_source_id.as_string(); + let database_type = + resolve_database_type(application, &datasource_id, &query.database_type).await?; + let items = application + .list_community_views(ListCommunityViewsRequest { + datasource_id, + database_type, + database_name: query.database_name.clone(), + schema_name: query.schema_name.clone(), + view_name_pattern: String::new(), + }) + .await? + .items + .into_iter() + .map(table_response) + .collect(); + Ok(full_page(items)) +} + +/// Reads one view, including its DDL, through the exact Core detail path. +pub(crate) async fn get_view( + application: &Application, + query: &LegacyTableDetailQuery, +) -> LegacyResult { + let datasource_id = query.data_source_id.as_string(); + let database_type = + resolve_database_type(application, &datasource_id, &query.database_type).await?; + Ok(table_response( + application + .get_community_view(ListCommunityViewsRequest { + datasource_id, + database_type, + database_name: query.database_name.clone(), + schema_name: query.schema_name.clone(), + view_name_pattern: query.table_name.clone(), + }) + .await?, + )) +} + +/// Reads the full table projection required by Community's table editor. +pub(crate) async fn get_editable_table( + application: &Application, + query: &LegacyTableDetailQuery, +) -> LegacyResult { + if query.table_name.trim().is_empty() { + return Err(LegacyFailure::invalid( + "invalid_table_query", + "tableName is required", + )); + } + let datasource_id = query.data_source_id.as_string(); + let database_type = + resolve_mysql_database_type(application, &datasource_id, &query.database_type).await?; + let tables = application + .list_community_tables(ListCommunityTablesRequest { + datasource_id: datasource_id.clone(), + database_type: database_type.clone(), + database_name: query.database_name.clone(), + schema_name: query.schema_name.clone(), + table_name_pattern: query.table_name.clone(), + }) + .await? + .items; + let table = tables + .into_iter() + .find(|table| table.name.eq_ignore_ascii_case(&query.table_name)) + .ok_or_else(|| LegacyFailure { + code: "table_not_found".to_owned(), + message: format!("Table {} does not exist", query.table_name), + })?; + let (columns, indexes) = tokio::try_join!( + application.list_community_columns(ListCommunityColumnsRequest { + datasource_id: datasource_id.clone(), + database_type: database_type.clone(), + database_name: query.database_name.clone(), + schema_name: query.schema_name.clone(), + table_name: query.table_name.clone(), + }), + application.list_community_indexes(ListCommunityIndexesRequest { + datasource_id, + database_type: database_type.clone(), + database_name: query.database_name.clone(), + schema_name: query.schema_name.clone(), + table_name: query.table_name.clone(), + }), + )?; + Ok(editable_table_response( + table, + columns.items, + indexes.items, + database_type, + )) +} + +/// Reads the full view projection used by Community's retained view editor. +pub(crate) async fn get_editable_view( + application: &Application, + query: &LegacyTableDetailQuery, +) -> LegacyResult { + if query.table_name.trim().is_empty() { + return Err(LegacyFailure::invalid( + "invalid_view_query", + "tableName is required", + )); + } + let datasource_id = query.data_source_id.as_string(); + let database_type = + resolve_mysql_database_type(application, &datasource_id, &query.database_type).await?; + let (view, columns) = tokio::try_join!( + application.get_community_view(ListCommunityViewsRequest { + datasource_id: datasource_id.clone(), + database_type: database_type.clone(), + database_name: query.database_name.clone(), + schema_name: query.schema_name.clone(), + view_name_pattern: query.table_name.clone(), + }), + application.list_community_columns(ListCommunityColumnsRequest { + datasource_id, + database_type: database_type.clone(), + database_name: query.database_name.clone(), + schema_name: query.schema_name.clone(), + table_name: query.table_name.clone(), + }), + )?; + Ok(editable_table_response( + view, + columns.items, + Vec::new(), + database_type, + )) +} + +/// Returns the `MySQL` type and option inventory used by the retained table editor. +pub(crate) async fn table_editor_meta( + application: &Application, + query: &LegacyMetadataQuery, +) -> LegacyResult { + let datasource_id = query.data_source_id.as_string(); + resolve_mysql_database_type(application, &datasource_id, &query.database_type).await?; + Ok(mysql_table_editor_meta()) +} + +/// Builds a `MySQL` script for Community result-grid create, update, and delete operations. +pub(crate) async fn build_grid_update_sql( + application: &Application, + request: &LegacyGridUpdateRequest, +) -> LegacyResult { + let datasource_id = request.data_source_id.as_string(); + resolve_mysql_database_type(application, &datasource_id, &request.database_type).await?; + if request.table_name.trim().is_empty() { + return Err(LegacyFailure::invalid( + "invalid_mysql_result_grid", + "tableName is required", + )); + } + let headers = request + .header_list + .iter() + .map(mysql_grid_header) + .collect::>>()?; + let operations = request + .operations + .iter() + .map(mysql_grid_operation) + .collect::>>()?; + reject_legacy_partial_large_values(&operations)?; + Ok(build_mysql_result_grid_script( + &mysql_qualified_name( + &request.database_name, + &request.schema_name, + &request.table_name, + ), + &headers, + &operations, + )?) +} + +/// Builds Community's copy-as-INSERT, copy-as-UPDATE, or copy-as-WHERE SQL. +pub(crate) async fn build_grid_copy_sql( + application: &Application, + request: &LegacyGridUpdateRequest, +) -> LegacyResult { + let datasource_id = request.data_source_id.as_string(); + resolve_mysql_database_type(application, &datasource_id, &request.database_type).await?; + let headers = request + .header_list + .iter() + .map(mysql_grid_header) + .collect::>>()?; + let operations = request + .operations + .iter() + .map(mysql_grid_copy_operation) + .collect::>>()?; + Ok(build_mysql_result_grid_copy_sql( + &mysql_qualified_name( + &request.database_name, + &request.schema_name, + &required_name(&request.table_name, "tableName")?, + ), + &headers, + &operations, + )?) +} + +/// Builds Community's clipboard SQL `IN` list for result cells or external text. +pub(crate) async fn build_grid_in_values( + application: &Application, + request: &LegacyGridUpdateRequest, +) -> LegacyResult { + let datasource_id = request.data_source_id.as_string(); + resolve_mysql_database_type(application, &datasource_id, &request.database_type).await?; + match request.source_type.trim().to_ascii_uppercase().as_str() { + "EXTERNAL_TEXT" => Ok(build_mysql_external_in_values(&request.external_values)?), + "RESULT_SET" => { + let headers = request + .header_list + .iter() + .map(mysql_grid_header) + .collect::>>()?; + reject_unsupported_copy_cells(&request.operations)?; + let operations = request + .operations + .iter() + .map(mysql_grid_copy_operation) + .collect::>>()?; + Ok(build_mysql_result_grid_in_values(&headers, &operations)?) + } + _ => Err(LegacyFailure::invalid( + "invalid_mysql_result_grid", + "sourceType must be RESULT_SET or EXTERNAL_TEXT", + )), + } +} + +/// Builds CREATE or ALTER TABLE statements in the historical `{ sql }[]` shape. +pub(crate) async fn build_table_modify_sql( + application: &Application, + request: &LegacyTableModifyRequest, +) -> LegacyResult> { + let datasource_id = request.data_source_id.as_string(); + resolve_mysql_database_type(application, &datasource_id, &request.database_type).await?; + let sql = if let Some(old_table) = request.old_table.as_ref() { + let reordered_columns = mysql_reordered_column_names(old_table, &request.new_table); + if !reordered_columns.is_empty() { + application + .validate_native_mysql_column_reorder( + &datasource_id, + &first_non_blank(&request.database_name, &old_table.database_name), + &required_name(&old_table.name, "oldTable.name")?, + &reordered_columns, + ) + .await?; + } + build_mysql_alter_table(&mysql_table_alter( + old_table, + &request.new_table, + &request.database_name, + &request.schema_name, + )?)? + } else { + build_mysql_create_table(&mysql_table_definition( + &request.new_table, + &request.database_name, + &request.schema_name, + )?)? + }; + Ok(vec![LegacySqlResponse { sql }]) +} + +/// Builds a CREATE DATABASE preview without executing it. +pub(crate) async fn build_create_database_sql( + application: &Application, + request: &LegacyDatabaseDefinitionRequest, +) -> LegacyResult { + let datasource_id = request.data_source_id.as_string(); + resolve_mysql_database_type(application, &datasource_id, &request.database_type).await?; + let name = first_non_blank(&request.name, &request.database_name); + Ok(LegacySqlResponse { + sql: build_mysql_create_database(&MysqlDatabaseDefinition { + name, + if_not_exists: false, + charset: non_blank(&request.charset), + collation: non_blank(&request.collation), + })?, + }) +} + +/// `MySQL` treats Community schemas as database aliases. +pub(crate) async fn build_create_schema_sql( + application: &Application, + request: &LegacySchemaDefinitionRequest, +) -> LegacyResult { + let datasource_id = request.data_source_id.as_string(); + resolve_mysql_database_type(application, &datasource_id, &request.database_type).await?; + let name = first_non_blank(&request.name, &request.schema_name); + Ok(LegacySqlResponse { + sql: build_mysql_create_schema(&MysqlDatabaseDefinition { + name, + if_not_exists: false, + charset: None, + collation: None, + })?, + }) +} + +pub(crate) async fn prepare_database_delete( + application: &Application, + request: &LegacyDeleteObjectRequest, +) -> LegacyResult { + let datasource_id = request.data_source_id.as_string(); + let database_type = + resolve_mysql_database_type(application, &datasource_id, &request.database_type).await?; + let confirm_name = required_name(&request.database_name, "databaseName")?; + Ok(LegacyDeletePrepareResponse { + sql_preview: build_mysql_drop_database(&confirm_name, false)?, + confirm_name, + object_type: "DATABASE".to_owned(), + db_type: database_type, + }) +} + +pub(crate) async fn prepare_schema_delete( + application: &Application, + request: &LegacyDeleteObjectRequest, +) -> LegacyResult { + let datasource_id = request.data_source_id.as_string(); + let database_type = + resolve_mysql_database_type(application, &datasource_id, &request.database_type).await?; + let target = first_non_blank(&request.schema_name, &request.database_name); + let confirm_name = required_name(&target, "schemaName")?; + Ok(LegacyDeletePrepareResponse { + sql_preview: build_mysql_drop_schema(&confirm_name, false)?, + confirm_name, + object_type: "SCHEMA".to_owned(), + db_type: database_type, + }) +} + +pub(crate) async fn execute_database_delete( + application: &Application, + request: &LegacyDeleteObjectRequest, +) -> LegacyResult<()> { + let prepared = prepare_database_delete(application, request).await?; + validate_delete_confirmation(&prepared.confirm_name, &request.confirm_name)?; + Box::pin(execute_generated_action( + application, + request.data_source_id.clone(), + &prepared.confirm_name, + "", + "", + prepared.sql_preview, + )) + .await +} + +pub(crate) async fn execute_schema_delete( + application: &Application, + request: &LegacyDeleteObjectRequest, +) -> LegacyResult<()> { + let prepared = prepare_schema_delete(application, request).await?; + validate_delete_confirmation(&prepared.confirm_name, &request.confirm_name)?; + Box::pin(execute_generated_action( + application, + request.data_source_id.clone(), + &prepared.confirm_name, + "", + "", + prepared.sql_preview, + )) + .await +} + +pub(crate) async fn drop_table( + application: &Application, + request: &LegacyTableOperationRequest, +) -> LegacyResult<()> { + let datasource_id = request.data_source_id.as_string(); + resolve_mysql_database_type(application, &datasource_id, &request.database_type).await?; + let sql = build_mysql_drop_table( + &mysql_qualified_name( + &request.database_name, + &request.schema_name, + &request.table_name, + ), + false, + )?; + Box::pin(execute_generated_action( + application, + request.data_source_id.clone(), + &request.database_name, + &request.schema_name, + &request.table_name, + sql, + )) + .await +} + +pub(crate) async fn truncate_table( + application: &Application, + request: &LegacyTableOperationRequest, +) -> LegacyResult<()> { + let datasource_id = request.data_source_id.as_string(); + resolve_mysql_database_type(application, &datasource_id, &request.database_type).await?; + let sql = build_mysql_truncate_table(&mysql_qualified_name( + &request.database_name, + &request.schema_name, + &request.table_name, + ))?; + Box::pin(execute_generated_action( + application, + request.data_source_id.clone(), + &request.database_name, + &request.schema_name, + &request.table_name, + sql, + )) + .await +} + +pub(crate) async fn copy_table( + application: &Application, + request: &LegacyTableCopyRequest, +) -> LegacyResult<()> { + let datasource_id = request.data_source_id.as_string(); + resolve_mysql_database_type(application, &datasource_id, &request.database_type).await?; + let new_name = if request.new_name.trim().is_empty() { + format!("{}_copy", request.table_name.trim()) + } else { + request.new_name.trim().to_owned() + }; + let statements = build_mysql_copy_table(&MysqlTableCopy { + source: mysql_qualified_name( + &request.database_name, + &request.schema_name, + &request.table_name, + ), + target: mysql_qualified_name(&request.database_name, &request.schema_name, &new_name), + if_not_exists: false, + copy_data: request.copy_data, + })?; + for sql in statements { + Box::pin(execute_generated_action( + application, + request.data_source_id.clone(), + &request.database_name, + &request.schema_name, + &new_name, + sql, + )) + .await?; + } + Ok(()) +} + +pub(crate) async fn build_view_modify_sql( + application: &Application, + request: &LegacyViewOperationRequest, +) -> LegacyResult { + let datasource_id = request.data_source_id.as_string(); + resolve_mysql_database_type(application, &datasource_id, &request.database_type).await?; + Ok(build_mysql_create_view(&mysql_view_definition(request)?)?) +} + +pub(crate) async fn drop_view( + application: &Application, + request: &LegacyViewOperationRequest, +) -> LegacyResult<()> { + let datasource_id = request.data_source_id.as_string(); + resolve_mysql_database_type(application, &datasource_id, &request.database_type).await?; + let view_name = first_non_blank(&request.view_name, &request.table_name); + let sql = build_mysql_drop_view( + &mysql_qualified_name(&request.database_name, &request.schema_name, &view_name), + false, + )?; + Box::pin(execute_generated_action( + application, + request.data_source_id.clone(), + &request.database_name, + &request.schema_name, + &view_name, + sql, + )) + .await +} + +pub(crate) async fn view_editor_meta( + application: &Application, + request: &LegacyViewOperationRequest, +) -> LegacyResult { + let datasource_id = request.data_source_id.as_string(); + resolve_mysql_database_type(application, &datasource_id, &request.database_type).await?; + let sql = "select * from table_name".to_owned(); + let preview_name = if request.database_name.trim().is_empty() { + "`undefined`".to_owned() + } else { + format!("`{}`.`undefined`", request.database_name.replace('`', "``")) + }; + let preview_sql = format!("create view {preview_name} AS \n{sql};"); + Ok(LegacyViewMetaResponse { + configurations: mysql_view_configurations(), + preview_sql, + sql, + }) +} + +fn mysql_view_configurations() -> Vec { + vec![ + serde_json::json!({ + "labelName": "算法", + "name": "algorithm", + "inputType": "select", + "defaultValue": "3", + "required": false, + "multiple": false, + "display": null, + "selects": [ + { "label": "UNDEFINED", "value": 0 }, + { "label": "MERGE", "value": 1 }, + { "label": "TEMPTABLE", "value": 2 }, + { "label": null, "value": 3 } + ] + }), + serde_json::json!({ + "labelName": "检查选项", + "name": "checkOption", + "inputType": "select", + "defaultValue": "2", + "required": false, + "multiple": false, + "display": null, + "selects": [ + { "label": "CASCADED", "value": 0 }, + { "label": "LOCAL", "value": 1 }, + { "label": null, "value": 2 } + ] + }), + serde_json::json!({ + "labelName": "SQL 安全性", + "name": "security", + "inputType": "select", + "defaultValue": "2", + "required": false, + "multiple": false, + "display": null, + "selects": [ + { "label": "DEFINER", "value": 0 }, + { "label": "INVOKER", "value": 1 }, + { "label": null, "value": 2 } + ] + }), + serde_json::json!({ + "labelName": "视图名称", + "name": "viewName", + "inputType": "input", + "defaultValue": null, + "required": false, + "multiple": false, + "display": null, + "selects": null + }), + serde_json::json!({ + "labelName": "定义者", + "name": "definer", + "inputType": "input", + "defaultValue": null, + "required": false, + "multiple": false, + "display": null, + "selects": null + }), + serde_json::json!({ + "labelName": "use or replace", + "name": "useOrReplace", + "inputType": "checkbox", + "defaultValue": "false", + "required": false, + "multiple": false, + "display": null, + "selects": null + }), + ] +} + +/// Lists stored functions in the historical paged metadata shape. +pub(crate) async fn list_functions( + application: &Application, + query: &LegacyTableListQuery, +) -> LegacyResult> { + validate_metadata_page(query)?; + let datasource_id = query.data_source_id.as_string(); + let database_type = + resolve_database_type(application, &datasource_id, &query.database_type).await?; + let items = application + .list_community_functions(ListCommunityFunctionsRequest { + datasource_id, + database_type, + database_name: query.database_name.clone(), + schema_name: query.schema_name.clone(), + }) + .await? + .items + .into_iter() + .map(function_response) + .collect(); + Ok(full_page(items)) +} + +/// Reads one stored function in the historical metadata shape. +pub(crate) async fn get_function( + application: &Application, + query: &LegacyFunctionDetailQuery, +) -> LegacyResult { + let datasource_id = query.data_source_id.as_string(); + let database_type = + resolve_database_type(application, &datasource_id, &query.database_type).await?; + Ok(function_response( + application + .get_community_function(GetCommunityFunctionRequest { + datasource_id, + database_type, + database_name: query.database_name.clone(), + schema_name: query.schema_name.clone(), + function_name: query.function_name.clone(), + }) + .await?, + )) +} + +/// Lists stored procedures in the historical paged metadata shape. +pub(crate) async fn list_procedures( + application: &Application, + query: &LegacyTableListQuery, +) -> LegacyResult> { + validate_metadata_page(query)?; + let datasource_id = query.data_source_id.as_string(); + let database_type = + resolve_database_type(application, &datasource_id, &query.database_type).await?; + let items = application + .list_community_procedures(ListCommunityProceduresRequest { + datasource_id, + database_type, + database_name: query.database_name.clone(), + schema_name: query.schema_name.clone(), + }) + .await? + .items + .into_iter() + .map(procedure_response) + .collect(); + Ok(full_page(items)) +} + +/// Reads one stored procedure in the historical metadata shape. +pub(crate) async fn get_procedure( + application: &Application, + query: &LegacyProcedureDetailQuery, +) -> LegacyResult { + let datasource_id = query.data_source_id.as_string(); + let database_type = + resolve_database_type(application, &datasource_id, &query.database_type).await?; + Ok(procedure_response( + application + .get_community_procedure(GetCommunityProcedureRequest { + datasource_id, + database_type, + database_name: query.database_name.clone(), + schema_name: query.schema_name.clone(), + procedure_name: query.procedure_name.clone(), + }) + .await?, + )) +} + +/// Renders the SQL shown by Community's routine invocation dialog. +pub(crate) async fn preview_routine_invocation( + application: &Application, + request: &LegacyRoutineInvocationRequest, +) -> LegacyResult { + let datasource_id = request.data_source_id.as_string(); + let database_type = + resolve_database_type(application, &datasource_id, &request.database_type).await?; + Ok(application + .preview_community_routine_invocation(PreviewCommunityRoutineInvocationRequest { + datasource_id, + database_type, + database_name: request.database_name.clone(), + schema_name: request.schema_name.clone(), + routine_type: request.routine_type.clone(), + routine_name: request.routine_name.clone(), + }) + .await?) +} + +pub(crate) async fn preview_routine_migration( + application: &Application, + request: &LegacyRoutineInvocationRequest, +) -> LegacyResult { + let migration = routine_migration_request(application, request).await?; + Ok(application.preview_community_routine_migration(&migration)?) +} + +pub(crate) async fn execute_routine_migration( + application: &Application, + request: &LegacyRoutineInvocationRequest, +) -> LegacyResult { + let migration = routine_migration_request(application, request).await?; + Ok(application + .execute_community_routine_migration(migration) + .await?) +} + +async fn routine_migration_request( + application: &Application, + request: &LegacyRoutineInvocationRequest, +) -> LegacyResult { + let datasource_id = request.data_source_id.as_string(); + let database_type = + resolve_database_type(application, &datasource_id, &request.database_type).await?; + Ok(CommunityRoutineMigrationRequest { + datasource_id, + database_type, + database_name: request.database_name.clone(), + schema_name: request.schema_name.clone(), + routine_type: request.routine_type.clone(), + routine_name: request.routine_name.clone(), + ddl: request.ddl.clone(), + }) +} + +/// Lists triggers in the historical paged metadata shape. +pub(crate) async fn list_triggers( + application: &Application, + query: &LegacyTableListQuery, +) -> LegacyResult> { + validate_metadata_page(query)?; + let datasource_id = query.data_source_id.as_string(); + let database_type = + resolve_database_type(application, &datasource_id, &query.database_type).await?; + let items = application + .list_community_triggers(ListCommunityTriggersRequest { + datasource_id, + database_type, + database_name: query.database_name.clone(), + schema_name: query.schema_name.clone(), + }) + .await? + .items + .into_iter() + .map(trigger_response) + .collect(); + Ok(full_page(items)) +} + +/// Reads one trigger in the historical metadata shape. +pub(crate) async fn get_trigger( + application: &Application, + query: &LegacyTriggerDetailQuery, +) -> LegacyResult { + let datasource_id = query.data_source_id.as_string(); + let database_type = + resolve_database_type(application, &datasource_id, &query.database_type).await?; + Ok(trigger_response( + application + .get_community_trigger(GetCommunityTriggerRequest { + datasource_id, + database_type, + database_name: query.database_name.clone(), + schema_name: query.schema_name.clone(), + trigger_name: query.trigger_name.clone(), + }) + .await?, + )) +} + +/// Runs a table preview through the generated-SQL, forced-read-only Core path +/// and waits for its retained result so the old synchronous frontend can use it. +#[allow(clippy::too_many_lines)] +pub(crate) async fn preview_table( + application: &Application, + request: &LegacyTablePreviewRequest, +) -> LegacyResult> { + if request.table_name.trim().is_empty() { + return Err(LegacyFailure::invalid( + "invalid_table_preview_request", + "tableName is required", + )); + } + if request.page_no == 0 || request.page_size == 0 { + return Err(LegacyFailure::invalid( + "invalid_table_preview_request", + "pageNo and pageSize must be positive", + )); + } + let offset = request + .page_no + .saturating_sub(1) + .checked_mul(request.page_size) + .ok_or_else(|| { + LegacyFailure::invalid( + "invalid_table_preview_request", + "requested page is outside the preview window", + ) + })?; + let row_limit = offset.checked_add(request.page_size).ok_or_else(|| { + LegacyFailure::invalid( + "invalid_table_preview_request", + "requested page is outside the preview window", + ) + })?; + if offset >= MAX_PREVIEW_ROWS || row_limit > MAX_PREVIEW_ROWS { + return Err(LegacyFailure::invalid( + "invalid_table_preview_request", + "table preview is limited to the first 1000 rows", + )); + } + + let datasource_id = request.data_source_id.as_string(); + let database_type = + resolve_database_type(application, &datasource_id, &request.database_type).await?; + let editable_columns = application + .list_community_columns(ListCommunityColumnsRequest { + datasource_id: datasource_id.clone(), + database_type: database_type.clone(), + database_name: request.database_name.clone(), + schema_name: request.schema_name.clone(), + table_name: request.table_name.clone(), + }) + .await? + .items; + let accepted = application + .start_community_table_preview(StartCommunityTablePreviewRequest { + datasource_id, + database_type, + database_name: request.database_name.clone(), + schema_name: request.schema_name.clone(), + table_name: request.table_name.clone(), + row_limit: Some(row_limit), + }) + .await?; + + let preview_result = tokio::time::timeout( + PREVIEW_TIMEOUT, + wait_for_sql_execution(application, &accepted.operation_id), + ) + .await; + let Ok(preview_result) = preview_result else { + application.cancel_operation(&accepted.operation_id).await; + return Err(LegacyFailure::invalid( + "table_preview_timeout", + "The table preview did not finish in time", + )); + }; + let metadata = preview_result?; + let page = application + .result_page( + &metadata.id, + ResultPageRequest { + offset: offset.to_string(), + max_rows: request.page_size.to_string(), + max_bytes: RESULT_PAGE_MAX_BYTES.to_string(), + }, + ) + .await?; + let has_next_page = page.has_more + || page.metadata.truncated_by_max_rows + || page.metadata.truncated_by_max_result_bytes; + let mut headers: Vec = page.columns.iter().map(result_header).collect(); + enrich_headers_from_columns(&mut headers, &editable_columns); + let large_value_owner = application.create_large_value_owner(); + let mut data_list: Vec> = page + .rows + .into_iter() + .map(|row| { + row.values + .into_iter() + .zip(page.columns.iter()) + .map(|(value, column)| result_cell(application, &large_value_owner, value, column)) + .collect() + }) + .collect(); + prepend_synthetic_row_numbers(&mut headers, &mut data_list, u64::from(offset)); + Ok(vec![LegacyManageResult { + data_list, + header_list: headers, + description: "Query executed successfully".to_owned(), + message: String::new(), + sql: accepted.sql.clone(), + original_sql: accepted.sql, + success: true, + duration: 0, + update_count: 0, + can_edit: true, + table_name: request.table_name.clone(), + sql_type: "SELECT".to_owned(), + refresh_targets: Vec::new(), + page_no: request.page_no, + page_size: request.page_size, + fuzzy_total: page.metadata.row_count, + has_next_page, + execute_sql_params: LegacySqlExecuteRequest::from(request), + extra: serde_json::json!({}), + comment: None, + result_set_id: None, + statement_sequence: Some(1), + execution_metrics: None, + execution_context: Some(LegacyExecutionContext { + database_name: (!request.database_name.is_empty()) + .then(|| request.database_name.clone()), + schema_name: (!request.schema_name.is_empty()).then(|| request.schema_name.clone()), + }), + }]) +} + +/// Starts one Community Console query through Core and returns the opaque +/// operation id used by both HTTP and desktop transports. +/// +/// # Errors +/// +/// Returns request validation, datasource, storage, or engine failures before +/// the operation is accepted. +pub async fn start_sql_execution( + application: &Application, + request: &LegacySqlExecuteRequest, +) -> LegacyResult { + let (datasource_id, row_limit) = validate_sql_execute_request(request)?; + application + .start_query(StartQueryRequest { + datasource_id, + sql: request.sql.clone(), + parameters: Vec::new(), + limits: QueryLimits { + max_rows: row_limit.to_string(), + max_result_bytes: RESULT_PAGE_MAX_BYTES.to_string(), + batch_rows: request.page_size.min(512), + batch_bytes: 1024 * 1024, + result_ttl_seconds: 60, + }, + }) + .await + .map_err(Into::into) +} + +/// Waits for a Core query terminal event without translating away its error +/// code or message. Desktop streaming can subscribe independently and use +/// this as the final retained-result barrier. +/// +/// # Errors +/// +/// Returns subscription, database execution, or cancellation failures. +pub async fn wait_for_sql_execution( + application: &Application, + operation_id: &str, +) -> LegacyResult { + let mut subscription = application.subscribe_operation(operation_id, None).await?; + while let Some(envelope) = subscription.next_event().await? { + match envelope.event { + OperationEvent::Completed { result } => return Ok(result), + OperationEvent::Failed { error } => return Err(LegacyFailure::from_api(error)), + OperationEvent::Cancelled { .. } => { + return Err(LegacyFailure::invalid( + "sql_execution_cancelled", + "The SQL execution was cancelled", + )); + } + OperationEvent::Started | OperationEvent::Progress { .. } => {} + } + } + Err(LegacyFailure::invalid( + "sql_execution_incomplete", + "The SQL execution ended without a result", + )) +} + +/// Reads and translates a retained Core result into Community's historical +/// result-grid shape. This is shared by synchronous HTTP and desktop IPC. +/// +/// # Errors +/// +/// Returns invalid paging or retained-result read failures. +pub async fn read_sql_result( + application: &Application, + request: &LegacySqlExecuteRequest, + metadata: &ResultMetadata, + duration: u64, +) -> LegacyResult { + let (offset, _) = sql_page_window(request)?; + let page = application + .result_page( + &metadata.id, + ResultPageRequest { + offset: offset.to_string(), + max_rows: request.page_size.to_string(), + max_bytes: RESULT_PAGE_MAX_BYTES.to_string(), + }, + ) + .await?; + let has_next_page = page.has_more + || page.metadata.truncated_by_max_rows + || page.metadata.truncated_by_max_result_bytes; + let fuzzy_total = + if page.metadata.truncated_by_max_rows || page.metadata.truncated_by_max_result_bytes { + format!("{}+", page.metadata.row_count) + } else { + page.metadata.row_count.clone() + }; + let header_list = page.columns.iter().map(result_header).collect(); + let large_value_owner = application.create_large_value_owner(); + let data_list = page + .rows + .into_iter() + .map(|row| { + row.values + .into_iter() + .zip(page.columns.iter()) + .map(|(value, column)| result_cell(application, &large_value_owner, value, column)) + .collect() + }) + .collect(); + Ok(LegacyManageResult { + data_list, + header_list, + description: "Query executed successfully".to_owned(), + message: String::new(), + sql: request.sql.clone(), + original_sql: request.sql.clone(), + success: true, + duration, + update_count: 0, + can_edit: false, + table_name: request.table_name.clone(), + sql_type: legacy_sql_type(&request.sql).to_owned(), + refresh_targets: Vec::new(), + page_no: request.page_no, + page_size: request.page_size, + fuzzy_total, + has_next_page, + execute_sql_params: request.clone(), + extra: serde_json::json!({}), + comment: None, + result_set_id: request.result_set_id, + statement_sequence: Some(1), + execution_metrics: None, + execution_context: Some(LegacyExecutionContext { + database_name: (!request.database_name.is_empty()) + .then(|| request.database_name.clone()), + schema_name: (!request.schema_name.is_empty()).then(|| request.schema_name.clone()), + }), + }) +} + +/// Executes the synchronous Community web contract while retaining Core's +/// asynchronous operation and result-store lifecycle internally. +/// +/// # Errors +/// +/// Returns request validation or failures that occur before Core accepts the +/// query. Failures after acceptance are returned as Community result items. +pub async fn execute_sql( + application: &Application, + request: &LegacySqlExecuteRequest, +) -> LegacyResult> { + let _ = validate_sql_execute_request(request)?; + if uses_native_mysql_console(application, request).await? { + let execution_id = application.create_large_value_owner(); + return Box::pin(execute_mysql_sql( + application, + request, + MysqlConsoleCancellation::new(), + &execution_id, + "SQL_EDITOR_HTTP", + )) + .await; + } + let started_at = Instant::now(); + let accepted = start_sql_execution(application, request).await?; + let terminal = tokio::time::timeout( + SQL_EXECUTION_TIMEOUT, + wait_for_sql_execution(application, &accepted.operation_id), + ) + .await; + let duration = elapsed_millis(started_at); + match terminal { + Ok(Ok(metadata)) => read_sql_result(application, request, &metadata, duration) + .await + .map(|result| vec![result]), + Ok(Err(error)) => Ok(vec![sql_failure_result(request, &error, duration)]), + Err(_) => { + application.cancel_operation(&accepted.operation_id).await; + Ok(vec![sql_failure_result( + request, + &LegacyFailure::invalid( + "sql_execution_timeout", + "The SQL execution did not finish in time", + ), + duration, + )]) + } + } +} + +/// Executes the Community single-result DDL contract. +/// +/// # Errors +/// +/// Returns request, datasource, execution, or missing-result failures. +pub async fn execute_ddl( + application: &Application, + request: &LegacySqlExecuteRequest, +) -> LegacyResult { + Box::pin(execute_sql(application, request)) + .await? + .into_iter() + .next() + .ok_or_else(|| { + LegacyFailure::invalid( + "sql_execution_incomplete", + "The SQL execution ended without a result", + ) + }) +} + +/// Counts the rows produced by one `MySQL` query for Community's total-row control. +pub(crate) async fn count_mysql_rows( + application: &Application, + request: &LegacySqlExecuteRequest, +) -> LegacyResult { + let (datasource_id, _) = validate_sql_execute_request(request)?; + if !uses_native_mysql_console(application, request).await? { + return Err(LegacyFailure::invalid( + "unsupported_database_type", + "The historical count route currently supports native MySQL only", + )); + } + if request.sql.trim().is_empty() { + return Ok(0); + } + let count_sql = build_mysql_count_query(&request.sql)?; + let results = application + .execute_mysql_console( + MysqlConsoleRequest { + datasource_id, + database_name: request.database_name.clone(), + sql: count_sql, + page_no: 1, + page_size: 1, + result_set_id: None, + single: true, + page_size_all: false, + explain: false, + error_continue: false, + }, + MysqlConsoleCancellation::new(), + ) + .await?; + let result = results.into_iter().next().ok_or_else(|| { + LegacyFailure::invalid( + "mysql_count_failed", + "The MySQL count query returned no result", + ) + })?; + if !result.success { + return Err(LegacyFailure { + code: "mysql_count_failed".to_owned(), + message: result.error.map_or(result.message, |error| error.message), + }); + } + let value = result + .rows + .first() + .and_then(|row| row.values.first()) + .ok_or_else(|| { + LegacyFailure::invalid( + "mysql_count_failed", + "The MySQL count query returned no value", + ) + })?; + let (JdbcValue::SignedInteger { value } + | JdbcValue::UnsignedInteger { value } + | JdbcValue::Decimal { value } + | JdbcValue::Text { value }) = value + else { + return Err(LegacyFailure::invalid( + "mysql_count_failed", + "The MySQL count query returned a non-integer value", + )); + }; + value.parse::().map_err(|_| { + LegacyFailure::invalid( + "mysql_count_failed", + "The MySQL count query returned an invalid integer", + ) + }) +} + +/// Executes the native `MySQL` Console contract with a caller-owned cancellation +/// source. Desktop keeps this source by execution id while HTTP uses it for the +/// synchronous timeout boundary. +/// +/// # Errors +/// +/// Returns validation or datasource failures that occur before execution. +pub async fn execute_mysql_sql( + application: &Application, + request: &LegacySqlExecuteRequest, + cancellation: MysqlConsoleCancellation, + execution_id: &str, + history_source: &str, +) -> LegacyResult> { + let (datasource_id, _) = validate_sql_execute_request(request)?; + let editable_columns = if request.table_name.trim().is_empty() { + None + } else { + let database_type = + resolve_database_type(application, &datasource_id, &request.database_type).await?; + application + .list_community_columns(ListCommunityColumnsRequest { + datasource_id: datasource_id.clone(), + database_type, + database_name: request.database_name.clone(), + schema_name: request.schema_name.clone(), + table_name: request.table_name.clone(), + }) + .await + .ok() + .map(|columns| columns.items) + }; + + let execution = application.execute_mysql_console( + MysqlConsoleRequest { + datasource_id, + database_name: request.database_name.clone(), + sql: request.sql.clone(), + page_no: request.page_no, + page_size: request.page_size, + result_set_id: request.result_set_id, + single: request.single, + page_size_all: request.page_size_all, + explain: request.explain, + error_continue: request.error_continue.unwrap_or(true), + }, + cancellation.clone(), + ); + let large_value_owner = application.create_large_value_owner(); + tokio::pin!(execution); + let timed_out = tokio::select! { + result = &mut execution => Some(result), + () = tokio::time::sleep(SQL_EXECUTION_TIMEOUT) => None, + }; + let results = match timed_out { + Some(Ok(results)) => results + .into_iter() + .map(|result| { + mysql_console_result( + application, + &large_value_owner, + request, + result, + editable_columns.as_deref(), + ) + }) + .collect(), + Some(Err(error)) => { + let error = LegacyFailure::from(error); + vec![sql_failure_result(request, &error, 0)] + } + None => { + let _ = cancellation.cancel(Some("The SQL execution timed out".to_owned())); + if tokio::time::timeout(SQL_CANCELLATION_GRACE, &mut execution) + .await + .is_err() + { + tracing::warn!( + "MySQL Console execution did not finish during the timeout cleanup window" + ); + } + vec![sql_failure_result( + request, + &LegacyFailure::invalid( + "sql_execution_timeout", + "The SQL execution did not finish in time", + ), + u64::try_from(SQL_EXECUTION_TIMEOUT.as_millis()).unwrap_or(u64::MAX), + )] + } + }; + record_mysql_console_history_best_effort( + application, + request, + &results, + execution_id, + history_source, + ) + .await; + Ok(results) +} + +async fn record_mysql_console_history_best_effort( + application: &Application, + request: &LegacySqlExecuteRequest, + results: &[LegacyManageResult], + execution_id: &str, + history_source: &str, +) { + let Some(storage) = application.storage().cloned() else { + return; + }; + let mut statements = BTreeMap::>::new(); + for (index, result) in results.iter().enumerate() { + let fallback = u32::try_from(index).unwrap_or(u32::MAX).saturating_add(1); + statements + .entry(result.statement_sequence.unwrap_or(fallback)) + .or_default() + .push(result); + } + for statement_results in statements.into_values() { + let Some(first) = statement_results.first().copied() else { + continue; + }; + let operation_rows = statement_results + .iter() + .map(|result| result.update_count) + .fold(0_u64, u64::saturating_add); + let use_time = statement_results + .iter() + .map(|result| result.duration) + .fold(0_u64, u64::saturating_add); + let message = statement_results + .iter() + .find(|result| !result.success && !result.message.trim().is_empty()) + .map_or("", |result| result.message.as_str()); + let extend_info = serde_json::to_string(&serde_json::json!({ + "source": history_source, + "sqlType": first.sql_type, + "executionId": execution_id, + "statementSequence": first.statement_sequence.unwrap_or(1), + "message": message, + })) + .ok(); + let input = CreateOperationLog { + name: None, + data_source_id: Some(request.data_source_id.as_string()), + data_source_name: non_blank(&request.data_source_name), + connectable: Some(true), + database_name: non_blank(&request.database_name), + database_type: Some(default_if_blank(&request.database_type, "MYSQL")), + ddl: first.original_sql.clone(), + status: mysql_console_history_status(statement_results.as_slice()).to_owned(), + operation_rows: i64::try_from(operation_rows).ok(), + use_time: i64::try_from(use_time).ok(), + extend_info, + schema_name: non_blank(&request.schema_name), + organization_id: None, + user_name: None, + more: first.original_sql.chars().count() > 200, + operation_type: "SQL_EXECUTE".to_owned(), + }; + let result = tokio::task::spawn_blocking({ + let storage = storage.clone(); + move || storage.create_operation_log(input) + }) + .await; + match result { + Ok(Ok(_)) => {} + Ok(Err(error)) => tracing::warn!(%error, "MySQL Console history write failed"), + Err(error) => tracing::warn!(%error, "MySQL Console history task failed"), + } + } +} + +fn mysql_console_history_status(results: &[&LegacyManageResult]) -> &'static str { + if results.iter().any(|result| { + result + .extra + .get("messages") + .and_then(serde_json::Value::as_array) + .into_iter() + .flatten() + .filter_map(|message| message.get("errorCode")) + .filter_map(serde_json::Value::as_str) + .any(|code| matches!(code, "mysql_console_cancelled" | "sql_execution_cancelled")) + }) { + "cancelled" + } else if results.iter().all(|result| result.success) { + "success" + } else { + "fail" + } +} + +/// Reads one bounded retained large-cell chunk in the requested display format. +/// +/// # Errors +/// +/// Returns validation, expired-token, range, decoding, or unsupported-format failures. +pub fn read_large_cell_value( + application: &Application, + request: &LegacyLargeCellValueRequest, +) -> LegacyResult { + if request.large_value_id.trim().is_empty() { + return Err(LegacyFailure::invalid( + "invalid_large_cell_value_request", + "largeValueId is required", + )); + } + let encoded = matches!( + request.format.trim().to_ascii_lowercase().as_str(), + "base64" | "hex" + ); + let chunk = if encoded { + application.read_large_value_encoded_chunk( + &request.large_value_id, + request.offset, + request.limit, + ) + } else { + application.read_large_value_chunk(&request.large_value_id, request.offset, request.limit) + } + .map_err(LegacyFailure::from)?; + format_large_value_chunk(chunk, &request.format) +} + +/// Writes one complete retained large-cell value to a unique temporary download path. +/// +/// # Errors +/// +/// Returns validation, expired-token, decoding, task, directory, or file-write failures. +pub async fn download_large_cell_value_to_path( + application: &Application, + request: &LegacyLargeCellDownloadRequest, +) -> LegacyResult { + let application = application.clone(); + let request = request.clone(); + tokio::task::spawn_blocking(move || { + let download = prepare_large_cell_download(&application, &request)?; + let directory = std::env::temp_dir().join("chat2db").join("downloads"); + fs::create_dir_all(&directory).map_err(|_| LegacyFailure { + code: "large_cell_download_failed".to_owned(), + message: "The large cell download directory could not be created".to_owned(), + })?; + let path = unique_large_cell_download_path( + &directory, + &request.large_value_id, + download.extension, + ); + fs::write(&path, download.bytes).map_err(|_| LegacyFailure { + code: "large_cell_download_failed".to_owned(), + message: "The large cell value could not be written to disk".to_owned(), + })?; + Ok(path.to_string_lossy().into_owned()) + }) + .await + .map_err(|_| LegacyFailure { + code: "large_cell_download_failed".to_owned(), + message: "The large cell download task did not finish".to_owned(), + })? +} + +struct PreparedLargeCellDownload { + bytes: Vec, + content_type: &'static str, + extension: &'static str, +} + +fn prepare_large_cell_download( + application: &Application, + request: &LegacyLargeCellDownloadRequest, +) -> LegacyResult { + if request.large_value_id.trim().is_empty() { + return Err(LegacyFailure::invalid( + "invalid_large_cell_value_request", + "largeValueId is required", + )); + } + let (raw, value_type) = read_complete_large_value(application, &request.large_value_id)?; + match request.format.trim().to_ascii_lowercase().as_str() { + "" | "raw" => Ok(PreparedLargeCellDownload { + bytes: raw, + content_type: if value_type == LargeValueType::Text { + "text/plain; charset=utf-8" + } else { + "application/octet-stream" + }, + extension: if value_type == LargeValueType::Text { + "txt" + } else { + "bin" + }, + }), + "text" => Ok(PreparedLargeCellDownload { + bytes: String::from_utf8_lossy(&raw).into_owned().into_bytes(), + content_type: "text/plain; charset=utf-8", + extension: "txt", + }), + "hex" => Ok(PreparedLargeCellDownload { + bytes: encode_hex(&raw).into_bytes(), + content_type: "text/plain; charset=utf-8", + extension: "hex", + }), + _ => Err(LegacyFailure::invalid( + "invalid_large_cell_value_request", + "format must be raw, text, or hex", + )), + } +} + +fn read_complete_large_value( + application: &Application, + large_value_id: &str, +) -> LegacyResult<(Vec, LargeValueType)> { + let mut offset = 0_u64; + let mut output = Vec::new(); + let mut value_type = None; + loop { + let chunk = application + .read_large_value_chunk(large_value_id, offset, LARGE_VALUE_CHUNK_SIZE) + .map_err(LegacyFailure::from)?; + value_type.get_or_insert(chunk.display_mode); + output.extend(decode_large_value_chunk(&chunk)?); + if chunk.eof { + return Ok((output, value_type.unwrap_or(LargeValueType::Binary))); + } + if chunk.next_offset <= offset { + return Err(LegacyFailure { + code: "large_cell_download_failed".to_owned(), + message: "The large cell value did not advance while downloading".to_owned(), + }); + } + offset = chunk.next_offset; + } +} + +fn format_large_value_chunk( + chunk: LargeValueChunk, + format: &str, +) -> LegacyResult { + let normalized = format.trim().to_ascii_lowercase(); + let (value, encoding) = match normalized.as_str() { + "" | "auto" => ( + chunk.value.clone(), + match chunk.encoding { + LargeValueEncoding::Utf8 => "utf-8", + LargeValueEncoding::Base64 => "base64", + }, + ), + "base64" => ( + BASE64_STANDARD.encode(decode_large_value_chunk(&chunk)?), + "base64", + ), + "text" => ( + String::from_utf8_lossy(&decode_large_value_chunk(&chunk)?).into_owned(), + "utf-8", + ), + "hex" => (encode_hex(&decode_large_value_chunk(&chunk)?), "hex"), + _ => { + return Err(LegacyFailure::invalid( + "invalid_large_cell_value_request", + "format must be text, hex, base64, or auto", + )); + } + }; + Ok(LegacyLargeCellChunk { + value, + offset: chunk.offset, + next_offset: chunk.next_offset, + eof: chunk.eof, + size_bytes: chunk.size_bytes, + size_chars: chunk.size_chars, + encoding: encoding.to_owned(), + content_type: chunk.content_type, + display_mode: chunk.display_mode, + }) +} + +fn decode_large_value_chunk(chunk: &LargeValueChunk) -> LegacyResult> { + match chunk.encoding { + LargeValueEncoding::Utf8 => Ok(chunk.value.as_bytes().to_vec()), + LargeValueEncoding::Base64 => { + BASE64_STANDARD + .decode(chunk.value.as_bytes()) + .map_err(|_| LegacyFailure { + code: "large_cell_value_invalid".to_owned(), + message: "The retained binary value could not be decoded".to_owned(), + }) + } + } +} + +fn encode_hex(bytes: &[u8]) -> String { + const HEX: &[u8; 16] = b"0123456789ABCDEF"; + let mut output = String::with_capacity(bytes.len().saturating_mul(2)); + for byte in bytes { + output.push(char::from(HEX[usize::from(byte >> 4)])); + output.push(char::from(HEX[usize::from(byte & 0x0f)])); + } + output +} + +fn unique_large_cell_download_path( + directory: &std::path::Path, + large_value_id: &str, + extension: &str, +) -> PathBuf { + let token_fragment = large_value_id + .chars() + .filter(char::is_ascii_alphanumeric) + .take(12) + .collect::(); + directory.join(format!( + "chat2db-cell-{}-{token_fragment}.{extension}", + unix_epoch_millis() + )) +} + +/// Reports whether the request resolves to the native `MySQL` driver. +/// +/// # Errors +/// +/// Returns a datasource lookup failure when the requested datasource is absent. +pub async fn uses_native_mysql_console( + application: &Application, + request: &LegacySqlExecuteRequest, +) -> LegacyResult { + if request.database_type.eq_ignore_ascii_case("mysql") { + return Ok(true); + } + let datasource = application + .get_datasource(&request.data_source_id.as_string()) + .await?; + Ok(datasource.driver_id.eq_ignore_ascii_case("mysql") + || datasource.driver_id.to_ascii_lowercase().contains("mysql")) +} + +fn mysql_console_result( + application: &Application, + large_value_owner: &str, + request: &LegacySqlExecuteRequest, + result: MysqlConsoleResult, + editable_columns: Option<&[CommunityTableColumn]>, +) -> LegacyManageResult { + let MysqlConsoleResult { + statement_sequence, + result_set_id, + sql, + success, + message, + update_count, + columns, + rows, + row_count, + has_more, + duration_ms, + error, + } = result; + let mut header_list: Vec<_> = columns.iter().map(result_header).collect(); + let can_edit = editable_columns.is_some_and(|editable_columns| { + enrich_direct_table_headers(&mut header_list, editable_columns, &request.table_name) + }); + let mut data_list: Vec> = rows + .into_iter() + .map(|row| { + row.values + .into_iter() + .zip(columns.iter()) + .map(|(value, column)| result_cell(application, large_value_owner, value, column)) + .collect() + }) + .collect(); + if can_edit { + let offset = u64::from(request.page_no.saturating_sub(1)) + .saturating_mul(u64::from(request.page_size)); + prepend_synthetic_row_numbers(&mut header_list, &mut data_list, offset); + } + let sql_type = legacy_sql_type(&sql).to_owned(); + let extra = error.map_or_else( + || serde_json::json!({}), |error| { serde_json::json!({ "messages": [{ @@ -4045,7 +6375,7 @@ fn storage_failure(error: StorageError) -> LegacyFailure { fn datasource_response( application: &Application, - datasource: Datasource, + datasource: DatasourceEditProjection, ) -> LegacyDatasourceResponse { let driver = application .list_drivers() @@ -4073,13 +6403,39 @@ fn datasource_response( id: datasource.id, alias: datasource.name, database_type, - // Core deliberately never exposes stored connection material. - url: String::new(), - user: String::new(), + url: datasource.jdbc_url, + user: datasource.username.unwrap_or_default(), + // Passwords and sensitive properties never leave Core's vault boundary. password: String::new(), + read_only: datasource.read_only, + ssh: datasource.ssh.map(|ssh| LegacySshTestRequest { + enabled: true, + host_name: ssh.host_name, + port: ssh.port.to_string(), + user_name: ssh.user_name, + local_port: ssh + .local_port + .map_or_else(String::new, |port| port.to_string()), + authentication_type: match ssh.authentication_type { + SshAuthenticationType::Password => "password".to_owned(), + SshAuthenticationType::PrivateKey => "keyFile".to_owned(), + }, + password: String::new(), + key_file: ssh.key_file.unwrap_or_default(), + passphrase: String::new(), + r_host: String::new(), + r_port: String::new(), + }), environment: default_environment(), environment_id: 1, - extend_info: Vec::new(), + extend_info: datasource + .properties + .into_iter() + .map(|property| LegacyConnectionPropertyResponse { + key: property.key, + value: property.value, + }) + .collect(), driver_config, storage_type: "LOCAL".to_owned(), space_id: 0, @@ -4661,6 +7017,36 @@ fn datasource_connection(request: &LegacyDatasourceRequest) -> LegacyResult LegacyResult { + let authentication = if request.authentication_type.eq_ignore_ascii_case("keyFile") + || (!request.key_file.trim().is_empty() + && !request.authentication_type.eq_ignore_ascii_case("password")) + { + SshAuthentication::PrivateKey { + key_file: request.key_file.clone(), + passphrase: non_blank(&request.passphrase), + } + } else { + SshAuthentication::Password { + password: request.password.clone(), + } + }; + Ok(SshTunnelConfig { + host_name: request.host_name.clone(), + port: parse_legacy_port(&request.port, 22, "SSH port")?, + user_name: request.user_name.clone(), + authentication, + host_key_verification: SshHostKeyVerification::KnownHosts, + local_port: optional_legacy_port(&request.local_port, "SSH local port")?, }) } @@ -5448,7 +7834,7 @@ async fn execute_generated_action( table_name: &str, sql: String, ) -> LegacyResult<()> { - let result = execute_ddl( + let result = Box::pin(execute_ddl( application, &LegacySqlExecuteRequest { data_source_id, @@ -5468,7 +7854,7 @@ async fn execute_generated_action( error_continue: Some(false), explain: false, }, - ) + )) .await?; if result.success { Ok(()) @@ -5507,6 +7893,30 @@ fn required_name(value: &str, field: &'static str) -> LegacyResult { } } +fn parse_legacy_port(value: &str, default: u16, field: &'static str) -> LegacyResult { + let value = value.trim(); + if value.is_empty() { + return Ok(default); + } + value + .parse::() + .ok() + .filter(|port| *port > 0) + .ok_or_else(|| LegacyFailure { + code: "invalid_legacy_request".to_owned(), + message: format!("{field} must be an integer between 1 and 65535"), + }) +} + +fn optional_legacy_port(value: &str, field: &'static str) -> LegacyResult> { + let value = value.trim(); + if value.is_empty() || value == "0" { + Ok(None) + } else { + parse_legacy_port(value, 0, field).map(Some) + } +} + fn first_non_blank(primary: &str, fallback: &str) -> String { if primary.trim().is_empty() { fallback.trim().to_owned() @@ -5671,21 +8081,55 @@ fn full_page(items: Vec) -> LegacyPage { } } -/// Dispatches a historical Community request without depending on Axum. -/// -/// Tauri IPC can pass its `requestUrl`, `method`, and `message` fields here and -/// return the resulting JSON value unchanged. +/// Dispatches a historical Community request without granting local-file access. pub fn dispatch( application: &Application, request: LegacyDispatchRequest, ) -> impl Future + Send + '_ { - Box::pin(dispatch_inner(application, request)) + Box::pin(dispatch_inner(application, request, false)) +} + +/// Dispatches historical Community requests with desktop-specific file paths. +pub fn dispatch_desktop( + application: &Application, + request: LegacyDispatchRequest, +) -> impl Future + Send + '_ { + Box::pin(dispatch_desktop_inner(application, request)) +} + +async fn dispatch_desktop_inner( + application: &Application, + request: LegacyDispatchRequest, +) -> serde_json::Value { + let path = request + .request_url + .split('?') + .next() + .unwrap_or(request.request_url.as_str()); + let method = request.method.to_ascii_lowercase(); + let result = match (method.as_str(), path) { + ("get", "/api/task/list") => match decode::(request.message) { + Ok(query) => { + serialized(list_legacy_transfer_tasks_for_desktop(application, &query).await) + } + Err(error) => Err(error), + }, + ("get", "/api/task/get") => match decode::(request.message) { + Ok(query) => { + serialized(get_legacy_transfer_task_for_desktop(application, &query.id).await) + } + Err(error) => Err(error), + }, + _ => return Box::pin(dispatch_inner(application, request, true)).await, + }; + envelope_value(result) } #[allow(clippy::too_many_lines)] async fn dispatch_inner( application: &Application, request: LegacyDispatchRequest, + desktop_paths: bool, ) -> serde_json::Value { let path = request .request_url @@ -5698,29 +8142,220 @@ async fn dispatch_inner( ("get", "/api/system") => Ok(serde_json::json!({ "systemUuid": "chat2db-rust-community" })), + ("get", "/api/dashboard/list") => { + match decode::(request.message) { + Ok(query) => serialized(list_community_dashboards(application, query).await), + Err(error) => Err(error), + } + } + ("get", "/api/dashboard") => match decode::(request.message) { + Ok(query) => serialized(get_community_dashboard(application, query.id).await), + Err(error) => Err(error), + }, + ("post", "/api/dashboard/create") => { + match decode::(request.message) { + Ok(body) => serialized(create_community_dashboard(application, body).await), + Err(error) => Err(error), + } + } + ("post", "/api/dashboard/update") => { + match decode::(request.message) { + Ok(body) => serialized(update_community_dashboard(application, body).await), + Err(error) => Err(error), + } + } + ("delete", "/api/dashboard") => { + match decode::(request.message) { + Ok(query) => serialized(delete_community_dashboard(application, query.id).await), + Err(error) => Err(error), + } + } + ("get", "/api/v1/chart") => match decode::(request.message) { + Ok(query) => serialized(get_community_chart(application, query.id).await), + Err(error) => Err(error), + }, + ("get", "/api/chart/detail") => { + match decode::(request.message) { + Ok(query) => serialized(get_community_chart_detail(application, query).await), + Err(error) => Err(error), + } + } + ("post", "/api/v1/chart/create") => { + match decode::(request.message) { + Ok(body) => serialized(create_community_chart(application, body).await), + Err(error) => Err(error), + } + } + ("post", "/api/v1/chart/update") => { + match decode::(request.message) { + Ok(body) => serialized(update_community_chart(application, body).await), + Err(error) => Err(error), + } + } + ("delete", "/api/chart") => match decode::(request.message) { + Ok(query) => serialized(delete_community_chart(application, query.id).await), + Err(error) => Err(error), + }, ("get", "/api/common/environment/list_all") => serialized(Ok(environments())), ("get", "/api/jdbc/driver/list") => decode(request.message) .map(|query: LegacyDriverQuery| drivers(application, &query.db_type)) .and_then(serialize_data), + ("get", "/api/jdbc/driver/download") => { + match decode::(request.message) { + Ok(query) => serialized(native_driver_action( + application, + &query.db_type, + NativeDriverAction::Download, + )), + Err(error) => Err(error), + } + } + ("post", "/api/jdbc/driver/save") => { + match decode::(request.message) { + Ok(body) => serialized(native_driver_action( + application, + &body.db_type, + NativeDriverAction::Save, + )), + Err(error) => Err(error), + } + } + ("delete", "/api/jdbc/driver/delete") => { + match decode::(request.message) { + Ok(body) => serialized(native_driver_action( + application, + &body.db_type, + NativeDriverAction::Delete, + )), + Err(error) => Err(error), + } + } ("get", "/api/connection/datasource/list") => { match decode::(request.message) { Ok(query) => serialized(list_datasources(application, &query).await), Err(error) => Err(error), } } - ("get", "/api/connection/datasource") => match decode::(request.message) { - Ok(query) => serialized(get_datasource(application, &query.id).await), - Err(error) => Err(error), - }, - ("post", "/api/connection/datasource/create") => { - match decode::(request.message) { - Ok(body) => serialized(create_datasource(application, &body).await), + ("get", "/api/connection/datasource") => match decode::(request.message) { + Ok(query) => serialized(get_datasource(application, &query.id).await), + Err(error) => Err(error), + }, + ("post", "/api/connection/datasource/create") => { + match decode::(request.message) { + Ok(body) => serialized(create_datasource(application, &body).await), + Err(error) => Err(error), + } + } + ("post", "/api/connection/datasource/pre_connect") => { + match decode::(request.message) { + Ok(body) => serialized(pre_connect(application, &body).await), + Err(error) => Err(error), + } + } + ("post", "/api/connection/ssh/pre_connect") => { + match decode::(request.message) { + Ok(body) => serialized(test_legacy_ssh(application, &body).await), + Err(error) => Err(error), + } + } + ("post", "/api/connection/datasource/clone") => { + match decode::(request.message) { + Ok(body) => serialized(clone_datasource(application, &body).await), + Err(error) => Err(error), + } + } + ("get", "/api/connection/datasource/connect") => { + match decode::(request.message) { + Ok(query) => serialized(connect_datasource(application, &query.id).await), + Err(error) => Err(error), + } + } + ("get" | "post", "/api/connection/datasource/close") | ("get", "/api/connection/close") => { + match decode::(request.message) { + Ok(query) => serialized(close_datasource(application, &query.id).await), + Err(error) => Err(error), + } + } + ("get", "/api/connection/console/connect") => { + match decode::(request.message) { + Ok(query) => serialized(connect_console(application, &query).await), + Err(error) => Err(error), + } + } + ("post", "/api/connection/datasource/export") => { + match decode::(request.message) { + Ok(body) => serialized(export_legacy_datasources(application, &body).await), + Err(error) => Err(error), + } + } + ("get" | "post", "/api/connection/datasource/import_community") => { + if request.message.get("schemaVersion").is_some() { + match decode::(request.message) { + Ok(document) => { + serialized(import_legacy_datasources(application, document).await) + } + Err(error) => Err(error), + } + } else if desktop_paths { + serialized(import_installed_legacy_datasources(application).await) + } else { + Err(desktop_file_operation_required()) + } + } + ("get" | "post", "/api/converter/upload") => { + match decode::(request.message) { + Ok(body) if desktop_paths => { + serialized(import_legacy_datasource_file_path(application, &body, None).await) + } + Ok(_) => Err(desktop_file_operation_required()), Err(error) => Err(error), } } - ("post", "/api/connection/datasource/pre_connect") => { - match decode::(request.message) { - Ok(body) => serialized(pre_connect(application, &body).await), + ("post", "/api/converter/ncx/upload") => { + match decode::(request.message) { + Ok(body) if desktop_paths => serialized( + import_legacy_datasource_file_path( + application, + &body, + Some(CommunityDatasourceImportFormat::NavicatNcx), + ) + .await, + ), + Ok(_) => Err(desktop_file_operation_required()), + Err(error) => Err(error), + } + } + ("post", "/api/converter/dbp/upload") => { + match decode::(request.message) { + Ok(body) if desktop_paths => serialized( + import_legacy_datasource_file_path( + application, + &body, + Some(CommunityDatasourceImportFormat::DbeaverDbp), + ) + .await, + ), + Ok(_) => Err(desktop_file_operation_required()), + Err(error) => Err(error), + } + } + ("post", "/api/converter/chat2db/upload") => { + match decode::(request.message) { + Ok(body) if desktop_paths => serialized( + import_legacy_datasource_file_path( + application, + &body, + Some(CommunityDatasourceImportFormat::Chat2dbJson), + ) + .await, + ), + Ok(_) => Err(desktop_file_operation_required()), + Err(error) => Err(error), + } + } + ("post", "/api/converter/datagrip/upload") => { + match decode::(request.message) { + Ok(body) => serialized(import_legacy_datagrip_text(application, &body).await), Err(error) => Err(error), } } @@ -5736,6 +8371,97 @@ async fn dispatch_inner( Err(error) => Err(error), } } + ("post", "/api/import/sql_file") => { + match decode::(request.message) { + Ok(body) if desktop_paths => { + serialized(import_legacy_mysql_desktop_file(application, &body, true).await) + } + Ok(_) => Err(desktop_file_operation_required()), + Err(error) => Err(error), + } + } + ("post", "/api/import/other_file") => { + match decode::(request.message) { + Ok(body) if desktop_paths => { + serialized(import_legacy_mysql_desktop_file(application, &body, false).await) + } + Ok(_) => Err(desktop_file_operation_required()), + Err(error) => Err(error), + } + } + ("post", "/api/export/sql_file") => { + match decode::(request.message) { + Ok(body) if desktop_paths || body.export_path.trim().is_empty() => { + serialized(export_legacy_mysql_sql_file(application, &body).await) + } + Ok(_) => Err(desktop_file_operation_required()), + Err(error) => Err(error), + } + } + ("post", "/api/export/other_file") => { + match decode::(request.message) { + Ok(body) if desktop_paths || body.export_path.trim().is_empty() => { + serialized(export_legacy_mysql_other_file(application, &body).await) + } + Ok(_) => Err(desktop_file_operation_required()), + Err(error) => Err(error), + } + } + ("get", "/api/task/list") => match decode::(request.message) { + Ok(query) => serialized(list_legacy_transfer_tasks(application, &query).await), + Err(error) => Err(error), + }, + ("get", "/api/task/get") => match decode::(request.message) { + Ok(query) => serialized(get_legacy_transfer_task(application, &query.id).await), + Err(error) => Err(error), + }, + ("get", "/api/task/stop") => match decode::(request.message) { + Ok(query) => serialized(stop_legacy_transfer_task(application, &query.id).await), + Err(error) => Err(error), + }, + ("get", "/api/task/download") => match decode::(request.message) { + Ok(query) if desktop_paths => serialized( + legacy_transfer_task_download(application, &query.id) + .await + .map(|download| download.path.to_string_lossy().into_owned()), + ), + Ok(_) => Err(desktop_file_operation_required()), + Err(error) => Err(error), + }, + ("get", "/api/sql/format") => match decode::(request.message) { + Ok(query) => serialized(format_legacy_sql(application, &query).await), + Err(error) => Err(error), + }, + ("get", "/api/sql/valid_select") => { + match decode::(request.message) { + Ok(query) => serialized(validate_legacy_select(application, &query).await), + Err(error) => Err(error), + } + } + ("get", "/api/sql_parser/get_keywords") => { + match decode::(request.message) { + Ok(query) => serialized(legacy_sql_keywords(application, &query).await), + Err(error) => Err(error), + } + } + ("post", "/api/sql_parser/context/parser" | "/api/sql_parser/context/quick_parser") => { + match decode::(request.message) { + Ok(body) => serialized(parse_legacy_sql(application, &body).await), + Err(error) => Err(error), + } + } + ("post", "/api/sql_parser/context/tip") => { + match decode::(request.message) { + Ok(body) => serialized(complete_legacy_sql(application, &body).await), + Err(error) => Err(error), + } + } + ("post", "/api/sql_parser/context/hover") => { + match decode::(request.message) { + Ok(body) => serialized(legacy_sql_hover(application, &body).await), + Err(error) => Err(error), + } + } ("post", "/api/operation/saved/create") => { match decode::(request.message) { Ok(body) => serialized(create_saved_console(application, &body).await), @@ -5788,14 +8514,106 @@ async fn dispatch_inner( Err(error) => Err(error), }, ("get", "/api/namespaces/tree_list") => serialized(namespace_tree(application).await), - ("get", "/api/rdb/database/list") => match decode::(request.message) { - Ok(query) => serialized(list_databases(application, &query).await), + ("post", "/api/namespaces/create") => { + match decode::(request.message) { + Ok(body) => serialized(create_namespace(application, &body).await), + Err(error) => Err(error), + } + } + ("post", "/api/namespaces/update") => { + match decode::(request.message) { + Ok(body) => serialized(update_namespace(application, &body).await), + Err(error) => Err(error), + } + } + ("post", "/api/namespaces/delete") => { + match decode::(request.message) { + Ok(body) => serialized(delete_namespace(application, &body).await), + Err(error) => Err(error), + } + } + ("post", "/api/namespaces/update_position") => { + match decode::(request.message) { + Ok(body) => serialized(move_namespace_node(application, body).await), + Err(error) => Err(error), + } + } + ("post", "/api/namespaces/update_data_source_position") => { + match decode::(request.message) { + Ok(body) => serialized(assign_datasource_namespace(application, &body).await), + Err(error) => Err(error), + } + } + ("post", "/api/pin/table/add") => match decode::(request.message) { + Ok(body) => serialized(add_table_pin(application, &body).await), + Err(error) => Err(error), + }, + ("post", "/api/pin/table/delete") => { + match decode::(request.message) { + Ok(body) => serialized(delete_table_pin(application, &body).await), + Err(error) => Err(error), + } + } + ("get", "/api/pin/table/list") => match decode::(request.message) { + Ok(query) => serialized(list_table_pins(application, &query).await), + Err(error) => Err(error), + }, + ("get", "/api/er/get_info") => match decode::(request.message) { + Ok(query) => serialized(get_er_info(application, &query).await), Err(error) => Err(error), }, - ("get", "/api/rdb/schema/list") => match decode::(request.message) { - Ok(query) => serialized(list_schemas(application, &query).await), + ("post", "/api/er/save_position") => { + match decode::(request.message) { + Ok(body) => serialized(save_er_position(application, &body).await), + Err(error) => Err(error), + } + } + ("get", "/api/rdb/account/capability") => { + match decode::(request.message) { + Ok(query) => serialized(account_capability(application, &query).await), + Err(error) => Err(error), + } + } + ("get", "/api/rdb/account/list") => match decode::(request.message) { + Ok(query) => serialized(list_accounts(application, &query).await), + Err(error) => Err(error), + }, + ("get", "/api/rdb/account/grants") => match decode::(request.message) { + Ok(query) => serialized(account_grants(application, &query).await), + Err(error) => Err(error), + }, + ("post", "/api/rdb/account/preview") => { + match decode::(request.message) { + Ok(body) => serialized(preview_account(application, &body)), + Err(error) => Err(error), + } + } + ("post", "/api/rdb/account/execute") => { + match decode::(request.message) { + Ok(body) => serialized(execute_account(application, &body).await), + Err(error) => Err(error), + } + } + ("post", "/api/diff/sql") => match decode::(request.message) { + Ok(body) => serialized(preview_schema_diff(application, &body).await), Err(error) => Err(error), }, + ("get", "/api/rdb/database/list") => match decode::(request.message) { + Ok(query) => serialized(list_databases(application, &query).await), + Err(error) => Err(error), + }, + ("get", "/api/rdb/schema/list" | "/api/rdb/ddl/schema_list") => { + match decode::(request.message) { + Ok(query) => serialized(list_schemas(application, &query).await), + Err(error) => Err(error), + } + } + ("get", "/api/rdb/ddl/database_schema_list") => { + match decode::(request.message) { + Ok(query) => serialized(database_schema_list(application, &query).await), + Err(error) => Err(error), + } + } ("get", "/api/rdb/table/table_meta") => { match decode::(request.message) { Ok(query) => serialized(table_editor_meta(application, &query).await), @@ -5903,6 +8721,18 @@ async fn dispatch_inner( Err(error) => Err(error), } } + ("post", "/api/rdb/routine/preview_migration") => { + match decode::(request.message) { + Ok(body) => serialized(preview_routine_migration(application, &body).await), + Err(error) => Err(error), + } + } + ("post", "/api/rdb/routine/execute_migration") => { + match decode::(request.message) { + Ok(body) => serialized(execute_routine_migration(application, &body).await), + Err(error) => Err(error), + } + } ("get", "/api/rdb/trigger/list") => match decode::(request.message) { Ok(query) => serialized(list_triggers(application, &query).await), Err(error) => Err(error), @@ -5921,16 +8751,17 @@ async fn dispatch_inner( } ("post" | "put", "/api/rdb/dml/execute") => { match decode::(request.message) { - Ok(body) => serialized(execute_sql(application, &body).await), - Err(error) => Err(error), - } - } - ("post" | "put", "/api/rdb/dml/execute_ddl" | "/api/rdb/dml/execute_update") => { - match decode::(request.message) { - Ok(body) => serialized(execute_ddl(application, &body).await), + Ok(body) => serialized(Box::pin(execute_sql(application, &body)).await), Err(error) => Err(error), } } + ( + "post" | "put", + "/api/rdb/dml/execute_ddl" | "/api/rdb/dml/execute_update" | "/api/rdb/ddl/execute", + ) => match decode::(request.message) { + Ok(body) => serialized(Box::pin(execute_ddl(application, &body)).await), + Err(error) => Err(error), + }, ("post" | "put", "/api/rdb/dml/get_update_sql") => { match decode::(request.message) { Ok(body) => serialized(build_grid_update_sql(application, &body).await), @@ -5955,6 +8786,26 @@ async fn dispatch_inner( Err(error) => Err(error), } } + ("post", "/api/rdb/dml/export") => { + match decode::(request.message) { + Ok(body) if desktop_paths => serialized( + export_legacy_mysql_dml(application, &body) + .await + .map(|download| download.path.to_string_lossy().into_owned()), + ), + Ok(_) => Err(desktop_file_operation_required()), + Err(error) => Err(error), + } + } + ("post", "/api/rdb/table/generate/class") => { + match decode::(request.message) { + Ok(body) if desktop_paths => { + serialized(generate_legacy_mysql_classes(application, &body).await) + } + Ok(_) => Err(desktop_file_operation_required()), + Err(error) => Err(error), + } + } ("post", "/api/rdb/table/modify/sql") => { match decode::(request.message) { Ok(body) => serialized(build_table_modify_sql(application, &body).await), @@ -6033,9 +8884,12 @@ async fn dispatch_inner( Err(error) => Err(error), } } - ("post", "/api/rdb/cell/download_path") => { + ("post", "/api/rdb/cell/download" | "/api/rdb/cell/download_path") => { match decode::(request.message) { - Ok(body) => serialized(download_large_cell_value_to_path(application, &body).await), + Ok(body) if desktop_paths => { + serialized(download_large_cell_value_to_path(application, &body).await) + } + Ok(_) => Err(desktop_file_operation_required()), Err(error) => Err(error), } } @@ -6057,13 +8911,53 @@ async fn dispatch_inner( const LEGACY_PATHS: &[&str] = &[ "/api/system", + "/api/dashboard/list", + "/api/dashboard", + "/api/dashboard/create", + "/api/dashboard/update", + "/api/v1/chart", + "/api/chart/detail", + "/api/v1/chart/create", + "/api/v1/chart/update", + "/api/chart", "/api/common/environment/list_all", "/api/jdbc/driver/list", + "/api/jdbc/driver/download", + "/api/jdbc/driver/save", + "/api/jdbc/driver/delete", "/api/connection/datasource/list", "/api/connection/datasource", "/api/connection/datasource/create", "/api/connection/datasource/pre_connect", "/api/connection/datasource/update", + "/api/connection/datasource/clone", + "/api/connection/datasource/connect", + "/api/connection/datasource/close", + "/api/connection/datasource/export", + "/api/connection/datasource/import_community", + "/api/converter/upload", + "/api/converter/ncx/upload", + "/api/converter/dbp/upload", + "/api/converter/chat2db/upload", + "/api/converter/datagrip/upload", + "/api/connection/ssh/pre_connect", + "/api/connection/close", + "/api/connection/console/connect", + "/api/import/sql_file", + "/api/import/other_file", + "/api/export/sql_file", + "/api/export/other_file", + "/api/task/list", + "/api/task/get", + "/api/task/stop", + "/api/task/download", + "/api/sql/format", + "/api/sql/valid_select", + "/api/sql_parser/get_keywords", + "/api/sql_parser/context/parser", + "/api/sql_parser/context/quick_parser", + "/api/sql_parser/context/tip", + "/api/sql_parser/context/hover", "/api/operation/saved/create", "/api/operation/saved/list", "/api/operation/saved", @@ -6072,16 +8966,35 @@ const LEGACY_PATHS: &[&str] = &[ "/api/operation/log/list", "/api/operation/log", "/api/namespaces/tree_list", + "/api/namespaces/create", + "/api/namespaces/update", + "/api/namespaces/delete", + "/api/namespaces/update_position", + "/api/namespaces/update_data_source_position", + "/api/pin/table/add", + "/api/pin/table/delete", + "/api/pin/table/list", + "/api/er/get_info", + "/api/er/save_position", + "/api/rdb/account/capability", + "/api/rdb/account/list", + "/api/rdb/account/grants", + "/api/rdb/account/preview", + "/api/rdb/account/execute", + "/api/diff/sql", "/api/rdb/database/list", "/api/rdb/database/create_database_sql", "/api/rdb/schema/list", "/api/rdb/schema/create_schema_sql", + "/api/rdb/ddl/schema_list", + "/api/rdb/ddl/database_schema_list", "/api/rdb/table/list", "/api/rdb/table/table_meta", "/api/rdb/table/query", "/api/rdb/table/export", "/api/rdb/table/create/example", "/api/rdb/table/update/example", + "/api/rdb/table/generate/class", "/api/rdb/table/modify/sql", "/api/rdb/table/truncate", "/api/rdb/table/copy", @@ -6112,15 +9025,19 @@ const LEGACY_PATHS: &[&str] = &[ "/api/rdb/procedure/list", "/api/rdb/procedure/detail", "/api/rdb/routine/preview_invocation", + "/api/rdb/routine/preview_migration", + "/api/rdb/routine/execute_migration", "/api/rdb/trigger/list", "/api/rdb/trigger/detail", "/api/rdb/dml/execute", + "/api/rdb/ddl/execute", "/api/rdb/dml/execute_ddl", "/api/rdb/dml/execute_update", "/api/rdb/dml/get_update_sql", "/api/rdb/dml/copy_update_sql", "/api/rdb/dml/copy_in_values_sql", "/api/rdb/dml/count", + "/api/rdb/dml/export", "/api/rdb/dml/execute_table", "/api/rdb/cell/value", "/api/rdb/cell/download", @@ -6193,8 +9110,26 @@ fn counted_envelope_value(result: LegacyResult) -> serde_json pub(crate) fn routes() -> Router { Router::new() .route("/api/system", get(system_handler)) + .route("/api/dashboard/list", get(dashboard_list_handler)) + .route( + "/api/dashboard", + get(dashboard_get_handler).delete(dashboard_delete_handler), + ) + .route("/api/dashboard/create", post(dashboard_create_handler)) + .route("/api/dashboard/update", post(dashboard_update_handler)) + .route("/api/v1/chart", get(chart_get_handler)) + .route("/api/chart/detail", get(chart_detail_handler)) + .route("/api/v1/chart/create", post(chart_create_handler)) + .route("/api/v1/chart/update", post(chart_update_handler)) + .route("/api/chart", axum::routing::delete(chart_delete_handler)) .route("/api/common/environment/list_all", get(environment_handler)) .route("/api/jdbc/driver/list", get(driver_handler)) + .route("/api/jdbc/driver/download", get(driver_download_handler)) + .route("/api/jdbc/driver/save", post(driver_save_handler)) + .route( + "/api/jdbc/driver/delete", + axum::routing::delete(driver_delete_handler), + ) .route( "/api/connection/datasource/list", get(list_datasources_handler), @@ -6211,10 +9146,98 @@ pub(crate) fn routes() -> Router { "/api/connection/datasource/pre_connect", post(pre_connect_handler), ) + .route( + "/api/connection/ssh/pre_connect", + post(ssh_pre_connect_handler), + ) + .route( + "/api/connection/datasource/clone", + post(clone_datasource_handler), + ) + .route( + "/api/connection/datasource/connect", + get(connect_datasource_handler), + ) + .route( + "/api/connection/datasource/close", + get(close_datasource_handler).post(close_datasource_body_handler), + ) + .route("/api/connection/close", get(close_datasource_handler)) + .route( + "/api/connection/console/connect", + get(connect_console_handler), + ) + .route( + "/api/connection/datasource/export", + post(export_datasources_handler), + ) + .route( + "/api/connection/datasource/import_community", + get(import_community_handler).post(import_community_handler), + ) + .route( + "/api/converter/upload", + get(converter_upload_handler) + .post(converter_upload_handler) + .layer(DefaultBodyLimit::max(MAX_LEGACY_MULTIPART_BYTES)), + ) + .route( + "/api/converter/ncx/upload", + post(converter_ncx_upload_handler) + .layer(DefaultBodyLimit::max(MAX_LEGACY_MULTIPART_BYTES)), + ) + .route( + "/api/converter/dbp/upload", + post(converter_dbp_upload_handler) + .layer(DefaultBodyLimit::max(MAX_LEGACY_MULTIPART_BYTES)), + ) + .route( + "/api/converter/chat2db/upload", + post(converter_chat2db_upload_handler) + .layer(DefaultBodyLimit::max(MAX_LEGACY_MULTIPART_BYTES)), + ) + .route( + "/api/converter/datagrip/upload", + post(converter_datagrip_upload_handler), + ) .route( "/api/connection/datasource/update", post(update_datasource_handler).put(update_datasource_handler), ) + .route( + "/api/import/sql_file", + post(import_sql_file_handler).layer(DefaultBodyLimit::max(MAX_LEGACY_MULTIPART_BYTES)), + ) + .route( + "/api/import/other_file", + post(import_other_file_handler) + .layer(DefaultBodyLimit::max(MAX_LEGACY_MULTIPART_BYTES)), + ) + .route("/api/export/sql_file", post(export_sql_file_handler)) + .route("/api/export/other_file", post(export_other_file_handler)) + .route("/api/task/list", get(transfer_task_list_handler)) + .route("/api/task/get", get(transfer_task_get_handler)) + .route("/api/task/stop", get(transfer_task_stop_handler)) + .route("/api/task/download", get(transfer_task_download_handler)) + .route("/api/sql/format", get(sql_format_handler)) + .route("/api/sql/valid_select", get(sql_valid_select_handler)) + .route( + "/api/sql_parser/get_keywords", + get(sql_parser_keywords_handler), + ) + .route( + "/api/sql_parser/context/parser", + post(sql_parser_context_handler), + ) + .route( + "/api/sql_parser/context/quick_parser", + post(sql_parser_context_handler), + ) + .route("/api/sql_parser/context/tip", post(sql_parser_tip_handler)) + .route( + "/api/sql_parser/context/hover", + post(sql_parser_hover_handler), + ) .route( "/api/operation/saved/create", post(create_saved_console_handler), @@ -6238,12 +9261,42 @@ pub(crate) fn routes() -> Router { .route("/api/operation/log/list", get(list_operation_logs_handler)) .route("/api/operation/log", get(get_operation_log_handler)) .route("/api/namespaces/tree_list", get(namespace_tree_handler)) + .route("/api/namespaces/create", post(namespace_create_handler)) + .route("/api/namespaces/update", post(namespace_update_handler)) + .route("/api/namespaces/delete", post(namespace_delete_handler)) + .route( + "/api/namespaces/update_position", + post(namespace_move_handler), + ) + .route( + "/api/namespaces/update_data_source_position", + post(datasource_namespace_assignment_handler), + ) + .route("/api/pin/table/add", post(table_pin_add_handler)) + .route("/api/pin/table/delete", post(table_pin_delete_handler)) + .route("/api/pin/table/list", get(table_pin_list_handler)) + .route("/api/er/get_info", get(er_info_handler)) + .route("/api/er/save_position", post(er_position_save_handler)) + .route( + "/api/rdb/account/capability", + get(account_capability_handler), + ) + .route("/api/rdb/account/list", get(account_list_handler)) + .route("/api/rdb/account/grants", get(account_grants_handler)) + .route("/api/rdb/account/preview", post(account_preview_handler)) + .route("/api/rdb/account/execute", post(account_execute_handler)) + .route("/api/diff/sql", post(schema_diff_handler)) .route("/api/rdb/database/list", get(database_list_handler)) .route( "/api/rdb/database/create_database_sql", post(create_database_sql_handler), ) .route("/api/rdb/schema/list", get(schema_list_handler)) + .route("/api/rdb/ddl/schema_list", get(schema_list_handler)) + .route( + "/api/rdb/ddl/database_schema_list", + get(database_schema_list_handler), + ) .route( "/api/rdb/schema/create_schema_sql", post(create_schema_sql_handler), @@ -6252,6 +9305,10 @@ pub(crate) fn routes() -> Router { .route("/api/rdb/table/table_meta", get(table_meta_handler)) .route("/api/rdb/table/query", get(table_query_handler)) .route("/api/rdb/table/export", get(table_ddl_export_handler)) + .route( + "/api/rdb/table/generate/class", + post(generate_class_handler), + ) .route( "/api/rdb/table/create/example", get(table_ddl_example_handler), @@ -6311,6 +9368,14 @@ pub(crate) fn routes() -> Router { "/api/rdb/routine/preview_invocation", post(routine_invocation_preview_handler), ) + .route( + "/api/rdb/routine/preview_migration", + post(routine_migration_preview_handler), + ) + .route( + "/api/rdb/routine/execute_migration", + post(routine_migration_execute_handler), + ) .route("/api/rdb/trigger/list", get(trigger_list_handler)) .route("/api/rdb/trigger/detail", get(trigger_detail_handler)) .route( @@ -6321,6 +9386,7 @@ pub(crate) fn routes() -> Router { "/api/rdb/dml/execute_ddl", post(sql_execute_ddl_handler).put(sql_execute_ddl_handler), ) + .route("/api/rdb/ddl/execute", post(sql_execute_ddl_handler)) .route( "/api/rdb/dml/execute_update", post(sql_execute_ddl_handler).put(sql_execute_ddl_handler), @@ -6341,16 +9407,13 @@ pub(crate) fn routes() -> Router { "/api/rdb/dml/count", post(sql_count_handler).put(sql_count_handler), ) + .route("/api/rdb/dml/export", post(dml_export_handler)) .route( "/api/rdb/dml/execute_table", post(table_preview_handler).put(table_preview_handler), ) .route("/api/rdb/cell/value", post(large_cell_value_handler)) .route("/api/rdb/cell/download", post(large_cell_download_handler)) - .route( - "/api/rdb/cell/download_path", - post(large_cell_download_path_handler), - ) .layer(middleware::map_response(legacy_bad_request_envelope)) } @@ -6364,7 +9427,7 @@ fn envelope(result: LegacyResult) -> Json> { async fn legacy_bad_request_envelope(response: Response) -> Response { if !matches!( response.status(), - StatusCode::BAD_REQUEST | StatusCode::UNPROCESSABLE_ENTITY + StatusCode::BAD_REQUEST | StatusCode::PAYLOAD_TOO_LARGE | StatusCode::UNPROCESSABLE_ENTITY ) { return response; } @@ -6391,15 +9454,118 @@ async fn system_handler() -> Json> { }))) } +async fn dashboard_list_handler( + State(application): State, + Query(query): Query, +) -> Json> { + envelope(list_community_dashboards(&application, query).await) +} + +async fn dashboard_get_handler( + State(application): State, + Query(query): Query, +) -> Json>> { + envelope(get_community_dashboard(&application, query.id).await) +} + +async fn dashboard_create_handler( + State(application): State, + Json(request): Json, +) -> Json> { + envelope(create_community_dashboard(&application, request).await) +} + +async fn dashboard_update_handler( + State(application): State, + Json(request): Json, +) -> Json> { + envelope(update_community_dashboard(&application, request).await) +} + +async fn dashboard_delete_handler( + State(application): State, + Query(query): Query, +) -> Json> { + envelope(delete_community_dashboard(&application, query.id).await) +} + +async fn chart_get_handler( + State(application): State, + Query(query): Query, +) -> Json>> { + envelope(get_community_chart(&application, query.id).await) +} + +async fn chart_detail_handler( + State(application): State, + Query(query): Query, +) -> Json>> { + envelope(get_community_chart_detail(&application, query).await) +} + +async fn chart_create_handler( + State(application): State, + Json(request): Json, +) -> Json> { + envelope(create_community_chart(&application, request).await) +} + +async fn chart_update_handler( + State(application): State, + Json(request): Json, +) -> Json> { + envelope(update_community_chart(&application, request).await) +} + +async fn chart_delete_handler( + State(application): State, + Query(query): Query, +) -> Json> { + envelope(delete_community_chart(&application, query.id).await) +} + async fn environment_handler() -> Json>> { envelope(Ok(environments())) } -async fn driver_handler( +async fn driver_handler( + State(application): State, + Query(query): Query, +) -> Json> { + envelope(Ok(drivers(&application, &query.db_type))) +} + +async fn driver_download_handler( + State(application): State, + Query(query): Query, +) -> Json> { + envelope(native_driver_action( + &application, + &query.db_type, + NativeDriverAction::Download, + )) +} + +async fn driver_save_handler( + State(application): State, + Json(request): Json, +) -> Json> { + envelope(native_driver_action( + &application, + &request.db_type, + NativeDriverAction::Save, + )) +} + +async fn driver_delete_handler( State(application): State, - Query(query): Query, -) -> Json> { - envelope(Ok(drivers(&application, &query.db_type))) + Json(request): Json, +) -> Json> { + envelope(native_driver_action( + &application, + &request.db_type, + NativeDriverAction::Delete, + )) } async fn list_datasources_handler( @@ -6430,6 +9596,127 @@ async fn pre_connect_handler( envelope(pre_connect(&application, &request).await) } +async fn ssh_pre_connect_handler( + State(application): State, + Json(request): Json, +) -> Json> { + envelope(test_legacy_ssh(&application, &request).await) +} + +async fn clone_datasource_handler( + State(application): State, + Json(request): Json, +) -> Json> { + envelope(clone_datasource(&application, &request).await) +} + +async fn connect_datasource_handler( + State(application): State, + Query(query): Query, +) -> Json>> { + envelope(connect_datasource(&application, &query.id).await) +} + +async fn close_datasource_handler( + State(application): State, + Query(query): Query, +) -> Json> { + envelope(close_datasource(&application, &query.id).await) +} + +async fn close_datasource_body_handler( + State(application): State, + Json(request): Json, +) -> Json> { + envelope(close_datasource(&application, &request.id).await) +} + +async fn connect_console_handler( + State(application): State, + Query(query): Query, +) -> Json> { + envelope(connect_console(&application, &query).await) +} + +async fn export_datasources_handler( + State(application): State, + Json(request): Json, +) -> Json> { + envelope(export_legacy_datasources(&application, &request).await) +} + +async fn import_community_handler( + State(application): State, + body: Bytes, +) -> Json> { + if body.is_empty() { + return envelope(Err(desktop_file_operation_required())); + } + let Ok(document) = serde_json::from_slice::(&body) else { + return envelope(Err(LegacyFailure::invalid( + "invalid_legacy_request", + "The Community datasource import document is invalid", + ))); + }; + envelope(import_legacy_datasources(&application, document).await) +} + +async fn converter_upload_handler( + State(application): State, + multipart: Multipart, +) -> Json> { + envelope(import_legacy_multipart_datasource(&application, multipart, None).await) +} + +async fn converter_ncx_upload_handler( + State(application): State, + multipart: Multipart, +) -> Json> { + envelope( + import_legacy_multipart_datasource( + &application, + multipart, + Some(CommunityDatasourceImportFormat::NavicatNcx), + ) + .await, + ) +} + +async fn converter_dbp_upload_handler( + State(application): State, + multipart: Multipart, +) -> Json> { + envelope( + import_legacy_multipart_datasource( + &application, + multipart, + Some(CommunityDatasourceImportFormat::DbeaverDbp), + ) + .await, + ) +} + +async fn converter_chat2db_upload_handler( + State(application): State, + multipart: Multipart, +) -> Json> { + envelope( + import_legacy_multipart_datasource( + &application, + multipart, + Some(CommunityDatasourceImportFormat::Chat2dbJson), + ) + .await, + ) +} + +async fn converter_datagrip_upload_handler( + State(application): State, + Json(request): Json, +) -> Json> { + envelope(import_legacy_datagrip_text(&application, &request).await) +} + async fn update_datasource_handler( State(application): State, Json(request): Json, @@ -6444,6 +9731,161 @@ async fn delete_datasource_handler( envelope(delete_datasource(&application, &query.id).await) } +async fn import_sql_file_handler( + State(application): State, + request: axum::extract::Request, +) -> Json> { + envelope(import_legacy_mysql_http(&application, request, true).await) +} + +async fn import_other_file_handler( + State(application): State, + request: axum::extract::Request, +) -> Json> { + envelope(import_legacy_mysql_http(&application, request, false).await) +} + +async fn export_sql_file_handler( + State(application): State, + Json(request): Json, +) -> Json> { + if let Err(error) = reject_web_export_path(&request.export_path) { + return envelope(Err(error)); + } + envelope(export_legacy_mysql_sql_file(&application, &request).await) +} + +async fn export_other_file_handler( + State(application): State, + Json(request): Json, +) -> Json> { + if let Err(error) = reject_web_export_path(&request.export_path) { + return envelope(Err(error)); + } + envelope(export_legacy_mysql_other_file(&application, &request).await) +} + +async fn transfer_task_list_handler( + State(application): State, + Query(query): Query, +) -> Json>> { + envelope(list_legacy_transfer_tasks(&application, &query).await) +} + +async fn transfer_task_get_handler( + State(application): State, + Query(query): Query, +) -> Json> { + envelope(get_legacy_transfer_task(&application, &query.id).await) +} + +async fn transfer_task_stop_handler( + State(application): State, + Query(query): Query, +) -> Json> { + envelope(stop_legacy_transfer_task(&application, &query.id).await) +} + +async fn transfer_task_download_handler( + State(application): State, + Query(query): Query, +) -> Response { + transfer_attachment_response(legacy_transfer_task_download(&application, &query.id).await) +} + +async fn dml_export_handler( + State(application): State, + Json(request): Json, +) -> Response { + transfer_attachment_response(export_legacy_mysql_dml(&application, &request).await) +} + +async fn generate_class_handler( + State(application): State, + Json(request): Json, +) -> Response { + transfer_attachment_response(generate_legacy_mysql_class_archive(&application, &request).await) +} + +fn transfer_attachment_response(result: LegacyResult) -> Response { + let download = match result { + Ok(download) => download, + Err(error) => return Json(LegacyEnvelope::<()>::failure(error)).into_response(), + }; + let file = tokio::fs::File::from_std(download.file); + let file_name = safe_attachment_filename(&download.artifact.file_name); + Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, download.artifact.media_type) + .header( + header::CONTENT_DISPOSITION, + format!("attachment; filename=\"{file_name}\""), + ) + .header(header::CONTENT_LENGTH, download.artifact.byte_count) + .body(Body::from_stream(ReaderStream::new(file))) + .unwrap_or_else(|_| StatusCode::INTERNAL_SERVER_ERROR.into_response()) +} + +fn safe_attachment_filename(value: &str) -> String { + let value: String = value + .chars() + .map(|character| { + if character.is_ascii_alphanumeric() || matches!(character, '.' | '-' | '_') { + character + } else { + '_' + } + }) + .collect(); + if value.is_empty() { + "chat2db-export".to_owned() + } else { + value + } +} + +async fn sql_format_handler( + State(application): State, + Query(query): Query, +) -> Json> { + envelope(format_legacy_sql(&application, &query).await) +} + +async fn sql_valid_select_handler( + State(application): State, + Query(query): Query, +) -> Json> { + envelope(validate_legacy_select(&application, &query).await) +} + +async fn sql_parser_keywords_handler( + State(application): State, + Query(query): Query, +) -> Json> { + envelope(legacy_sql_keywords(&application, &query).await) +} + +async fn sql_parser_context_handler( + State(application): State, + Json(request): Json, +) -> Json> { + envelope(parse_legacy_sql(&application, &request).await) +} + +async fn sql_parser_tip_handler( + State(application): State, + Json(request): Json, +) -> Json> { + envelope(complete_legacy_sql(&application, &request).await) +} + +async fn sql_parser_hover_handler( + State(application): State, + Json(request): Json, +) -> Json>> { + envelope(legacy_sql_hover(&application, &request).await) +} + async fn create_saved_console_handler( State(application): State, Json(request): Json, @@ -6515,6 +9957,118 @@ async fn namespace_tree_handler( envelope(namespace_tree(&application).await) } +async fn namespace_create_handler( + State(application): State, + Json(request): Json, +) -> Json> { + envelope(create_namespace(&application, &request).await) +} + +async fn namespace_update_handler( + State(application): State, + Json(request): Json, +) -> Json> { + envelope(update_namespace(&application, &request).await) +} + +async fn namespace_delete_handler( + State(application): State, + Json(request): Json, +) -> Json> { + envelope(delete_namespace(&application, &request).await) +} + +async fn namespace_move_handler( + State(application): State, + Json(request): Json, +) -> Json> { + envelope(move_namespace_node(&application, request).await) +} + +async fn datasource_namespace_assignment_handler( + State(application): State, + Json(request): Json, +) -> Json> { + envelope(assign_datasource_namespace(&application, &request).await) +} + +async fn table_pin_add_handler( + State(application): State, + Json(request): Json, +) -> Json> { + envelope(add_table_pin(&application, &request).await) +} + +async fn table_pin_delete_handler( + State(application): State, + Json(request): Json, +) -> Json> { + envelope(delete_table_pin(&application, &request).await) +} + +async fn table_pin_list_handler( + State(application): State, + Query(query): Query, +) -> Json>> { + envelope(list_table_pins(&application, &query).await) +} + +async fn er_info_handler( + State(application): State, + Query(query): Query, +) -> Json> { + envelope(get_er_info(&application, &query).await) +} + +async fn er_position_save_handler( + State(application): State, + Json(request): Json, +) -> Json> { + envelope(save_er_position(&application, &request).await) +} + +async fn account_capability_handler( + State(application): State, + Query(query): Query, +) -> Json> { + envelope(account_capability(&application, &query).await) +} + +async fn account_list_handler( + State(application): State, + Query(query): Query, +) -> Json>> { + envelope(list_accounts(&application, &query).await) +} + +async fn account_grants_handler( + State(application): State, + Query(query): Query, +) -> Json>> { + envelope(account_grants(&application, &query).await) +} + +async fn account_preview_handler( + State(application): State, + Json(request): Json, +) -> Json> { + envelope(preview_account(&application, &request)) +} + +async fn account_execute_handler( + State(application): State, + Json(request): Json, +) -> Json> { + envelope(execute_account(&application, &request).await) +} + +async fn schema_diff_handler( + State(application): State, + Json(request): Json, +) -> Json> { + envelope(preview_schema_diff(&application, &request).await) +} + async fn database_list_handler( State(application): State, Query(query): Query, @@ -6536,6 +10090,13 @@ async fn schema_list_handler( envelope(list_schemas(&application, &query).await) } +async fn database_schema_list_handler( + State(application): State, + Query(query): Query, +) -> Json> { + envelope(database_schema_list(&application, &query).await) +} + async fn create_schema_sql_handler( State(application): State, Json(request): Json, @@ -6766,6 +10327,20 @@ async fn routine_invocation_preview_handler( envelope(preview_routine_invocation(&application, &request).await) } +async fn routine_migration_preview_handler( + State(application): State, + Json(request): Json, +) -> Json> { + envelope(preview_routine_migration(&application, &request).await) +} + +async fn routine_migration_execute_handler( + State(application): State, + Json(request): Json, +) -> Json> { + envelope(execute_routine_migration(&application, &request).await) +} + async fn trigger_list_handler( State(application): State, Query(query): Query, @@ -6791,14 +10366,14 @@ async fn sql_execute_handler( State(application): State, Json(request): Json, ) -> Json>> { - envelope(execute_sql(&application, &request).await) + envelope(Box::pin(execute_sql(&application, &request)).await) } async fn sql_execute_ddl_handler( State(application): State, Json(request): Json, ) -> Json> { - envelope(execute_ddl(&application, &request).await) + envelope(Box::pin(execute_ddl(&application, &request)).await) } async fn grid_update_sql_handler( @@ -6836,13 +10411,6 @@ async fn large_cell_value_handler( envelope(read_large_cell_value(&application, &request)) } -async fn large_cell_download_path_handler( - State(application): State, - Json(request): Json, -) -> Json> { - envelope(download_large_cell_value_to_path(&application, &request).await) -} - async fn large_cell_download_handler( State(application): State, Json(request): Json, @@ -6874,15 +10442,130 @@ async fn large_cell_download_handler( #[cfg(test)] mod tests { + use std::{io::Write as _, sync::Arc}; + use axum::{ body::Body, http::{Request, StatusCode}, }; + use chat2db_storage::{ + CreateTransferTask, SecretRef, SecretValue, SecretVault, SecretVaultError, + StoredTransferTaskKind, + }; use http_body_util::BodyExt as _; use tower::ServiceExt as _; use super::*; + #[derive(Debug)] + struct EmptyVault; + + impl SecretVault for EmptyVault { + fn probe(&self) -> Result<(), SecretVaultError> { + Ok(()) + } + + fn create( + &self, + _reference: &SecretRef, + _value: &SecretValue, + ) -> Result<(), SecretVaultError> { + Ok(()) + } + + fn get(&self, _reference: &SecretRef) -> Result, SecretVaultError> { + Ok(None) + } + + fn delete(&self, _reference: &SecretRef) -> Result<(), SecretVaultError> { + Ok(()) + } + } + + #[test] + fn community_ssh_payload_maps_both_auth_modes_and_never_projects_secrets() { + let password_request: LegacyDatasourceRequest = serde_json::from_value(serde_json::json!({ + "alias": "MySQL SSH", + "url": "jdbc:mysql://db.internal:3306/app", + "type": "MYSQL", + "ssh": { + "use": true, + "hostName": "bastion.internal", + "port": "22", + "userName": "developer", + "localPort": "33060", + "authenticationType": "password", + "password": "sentinel-ssh-password" + } + })) + .expect("Community SSH request decodes"); + let password = datasource_connection(&password_request) + .expect("password SSH connection maps") + .ssh + .expect("SSH enabled"); + assert!(matches!( + password.authentication, + SshAuthentication::Password { password } if password == "sentinel-ssh-password" + )); + assert_eq!(password.local_port, Some(33060)); + + let private_key_request: LegacyDatasourceRequest = + serde_json::from_value(serde_json::json!({ + "alias": "MySQL SSH key", + "url": "jdbc:mysql://db.internal:3306/app", + "type": "MYSQL", + "ssh": { + "use": true, + "hostName": "bastion.internal", + "port": "2222", + "userName": "developer", + "authenticationType": "keyFile", + "keyFile": "/keys/id_ed25519", + "passphrase": "sentinel-passphrase" + } + })) + .expect("Community key request decodes"); + let private_key = datasource_connection(&private_key_request) + .expect("private-key SSH connection maps") + .ssh + .expect("SSH enabled"); + assert!(matches!( + private_key.authentication, + SshAuthentication::PrivateKey { key_file, passphrase: Some(passphrase) } + if key_file == "/keys/id_ed25519" && passphrase == "sentinel-passphrase" + )); + + let response = datasource_response( + &Application::new(), + DatasourceEditProjection { + id: "datasource-1".to_owned(), + name: "MySQL SSH".to_owned(), + driver_id: "mysql".to_owned(), + jdbc_url: "jdbc:mysql://db.internal:3306/app".to_owned(), + username: Some("db-user".to_owned()), + properties: Vec::new(), + read_only: false, + ssh: Some(chat2db_contract::SshTunnelEditProjection { + host_name: "bastion.internal".to_owned(), + port: 2222, + user_name: "developer".to_owned(), + local_port: None, + authentication_type: SshAuthenticationType::PrivateKey, + key_file: Some("/keys/id_ed25519".to_owned()), + host_key_verification: SshHostKeyVerification::KnownHosts, + }), + has_secret: true, + revision: "1".to_owned(), + }, + ); + let json = serde_json::to_string(&response).expect("response serializes"); + assert!(json.contains("bastion.internal")); + assert!(json.contains("keyFile")); + assert!(!json.contains("sentinel-ssh-password")); + assert!(!json.contains("sentinel-passphrase")); + assert_eq!(response.ssh.expect("SSH projection exists").password, ""); + } + const REQUIRED_METADATA_PATHS: &[&str] = &[ "/api/rdb/table/table_list", "/api/rdb/table/column_list", @@ -6902,7 +10585,34 @@ mod tests { "/api/rdb/trigger/detail", ]; + const REQUIRED_DASHBOARD_PATHS: &[(&str, &str)] = &[ + ("get", "/api/dashboard/list"), + ("get", "/api/dashboard"), + ("post", "/api/dashboard/create"), + ("post", "/api/dashboard/update"), + ("delete", "/api/dashboard"), + ("get", "/api/v1/chart"), + ("get", "/api/chart/detail"), + ("post", "/api/v1/chart/create"), + ("post", "/api/v1/chart/update"), + ("delete", "/api/chart"), + ]; + const REQUIRED_EDITABLE_PATHS: &[(&str, &str)] = &[ + ("post", "/api/pin/table/add"), + ("post", "/api/pin/table/delete"), + ("get", "/api/pin/table/list"), + ("get", "/api/er/get_info"), + ("post", "/api/er/save_position"), + ("get", "/api/rdb/account/capability"), + ("get", "/api/rdb/account/list"), + ("get", "/api/rdb/account/grants"), + ("post", "/api/rdb/account/preview"), + ("post", "/api/rdb/account/execute"), + ("post", "/api/diff/sql"), + ("get", "/api/rdb/ddl/schema_list"), + ("get", "/api/rdb/ddl/database_schema_list"), + ("post", "/api/rdb/ddl/execute"), ("get", "/api/rdb/table/table_meta"), ("get", "/api/rdb/table/query"), ("get", "/api/rdb/table/export"), @@ -6926,6 +10636,8 @@ mod tests { ("post", "/api/rdb/view/modify/sql"), ("post", "/api/rdb/view/drop"), ("post", "/api/rdb/routine/preview_invocation"), + ("post", "/api/rdb/routine/preview_migration"), + ("post", "/api/rdb/routine/execute_migration"), ("post", "/api/rdb/dml/get_update_sql"), ("put", "/api/rdb/dml/get_update_sql"), ("post", "/api/rdb/dml/copy_update_sql"), @@ -6936,6 +10648,55 @@ mod tests { ("put", "/api/rdb/dml/count"), ("post", "/api/rdb/dml/execute_update"), ("put", "/api/rdb/dml/execute_update"), + ("post", "/api/rdb/cell/download"), + ]; + + const REQUIRED_SQL_COMPATIBILITY_PATHS: &[(&str, &str)] = &[ + ("get", "/api/sql/format"), + ("get", "/api/sql/valid_select"), + ("get", "/api/sql_parser/get_keywords"), + ("post", "/api/sql_parser/context/parser"), + ("post", "/api/sql_parser/context/quick_parser"), + ("post", "/api/sql_parser/context/tip"), + ("post", "/api/sql_parser/context/hover"), + ]; + + const REQUIRED_WORKSPACE_PATHS: &[(&str, &str)] = &[ + ("get", "/api/jdbc/driver/download"), + ("post", "/api/jdbc/driver/save"), + ("delete", "/api/jdbc/driver/delete"), + ("post", "/api/connection/ssh/pre_connect"), + ("post", "/api/connection/datasource/clone"), + ("get", "/api/connection/datasource/connect"), + ("post", "/api/connection/datasource/close"), + ("get", "/api/connection/close"), + ("get", "/api/connection/console/connect"), + ("post", "/api/connection/datasource/export"), + ("get", "/api/connection/datasource/import_community"), + ("get", "/api/converter/upload"), + ("post", "/api/converter/upload"), + ("post", "/api/converter/ncx/upload"), + ("post", "/api/converter/dbp/upload"), + ("post", "/api/converter/chat2db/upload"), + ("post", "/api/converter/datagrip/upload"), + ("post", "/api/namespaces/create"), + ("post", "/api/namespaces/update"), + ("post", "/api/namespaces/delete"), + ("post", "/api/namespaces/update_position"), + ("post", "/api/namespaces/update_data_source_position"), + ]; + + const REQUIRED_TRANSFER_PATHS: &[(&str, &str)] = &[ + ("post", "/api/import/sql_file"), + ("post", "/api/import/other_file"), + ("post", "/api/export/sql_file"), + ("post", "/api/export/other_file"), + ("get", "/api/task/list"), + ("get", "/api/task/get"), + ("get", "/api/task/stop"), + ("get", "/api/task/download"), + ("post", "/api/rdb/dml/export"), + ("post", "/api/rdb/table/generate/class"), ]; fn metadata_message(path: &str) -> serde_json::Value { @@ -7026,537 +10787,1199 @@ mod tests { assert_eq!(chunk.encoding, "base64"); assert_eq!(chunk.display_mode, LargeValueType::Text); - let next = read_large_cell_value( - &application, - &LegacyLargeCellValueRequest { - large_value_id: large_value_id.clone(), - offset: chunk.next_offset, - limit: 128, - format: "base64".to_owned(), - }, - ) - .expect("second large text chunk must load"); - let next_decoded = BASE64_STANDARD - .decode(next.value) - .expect("second chunk must use frontend-compatible base64"); - assert_eq!(next_decoded, value.as_bytes()[128..256]); - assert_eq!(next.offset, 128); - assert_eq!(next.next_offset, 256); + let next = read_large_cell_value( + &application, + &LegacyLargeCellValueRequest { + large_value_id: large_value_id.clone(), + offset: chunk.next_offset, + limit: 128, + format: "base64".to_owned(), + }, + ) + .expect("second large text chunk must load"); + let next_decoded = BASE64_STANDARD + .decode(next.value) + .expect("second chunk must use frontend-compatible base64"); + assert_eq!(next_decoded, value.as_bytes()[128..256]); + assert_eq!(next.offset, 128); + assert_eq!(next.next_offset, 256); + + let path = download_large_cell_value_to_path( + &application, + &LegacyLargeCellDownloadRequest { + large_value_id, + format: "text".to_owned(), + }, + ) + .await + .expect("large text must download to a local temporary file"); + assert_eq!( + std::fs::read_to_string(&path).expect("download must be readable"), + value + ); + std::fs::remove_file(path).expect("temporary test download must be removed"); + } + + #[test] + fn mysql_console_history_distinguishes_cancellation_from_failure() { + let request = serde_json::from_value(serde_json::json!({ + "dataSourceId": "datasource-1", + "databaseType": "MYSQL", + "sql": "SELECT SLEEP(30)", + "pageNo": 1, + "pageSize": 200 + })) + .expect("legacy SQL request must deserialize"); + let cancelled = sql_failure_result( + &request, + &LegacyFailure { + code: "mysql_console_cancelled".to_owned(), + message: "The SQL execution was cancelled".to_owned(), + }, + 12, + ); + assert_eq!(mysql_console_history_status(&[&cancelled]), "cancelled"); + + let failed = sql_failure_result( + &request, + &LegacyFailure { + code: "database.query_failed".to_owned(), + message: "Query failed".to_owned(), + }, + 12, + ); + assert_eq!(mysql_console_history_status(&[&failed]), "fail"); + + let mut succeeded = failed; + succeeded.success = true; + succeeded.extra = serde_json::json!({}); + assert_eq!(mysql_console_history_status(&[&succeeded]), "success"); + } + + #[test] + fn counted_metadata_envelope_keeps_total_at_the_top_level() { + let body = counted_envelope_value(Ok(serde_json::json!([ + { "name": "id" }, + { "name": "created_at" } + ]))); + + assert_eq!(body["success"], true); + assert_eq!(body["total"], 2); + assert_eq!(body["data"][0]["name"], "id"); + assert!(body["data"].get("total").is_none()); + } + + #[test] + fn editable_preview_prepends_page_aware_row_numbers_and_column_metadata() { + let mut headers = vec![LegacyResultHeader { + data_type: "NUMERIC".to_owned(), + name: "id".to_owned(), + column_name: "id".to_owned(), + column_type: "BIGINT".to_owned(), + table_name: Some("items".to_owned()), + database_name: Some("inventory".to_owned()), + schema_name: None, + primary_key: false, + comment: None, + default_value: None, + auto_increment: None, + nullable: true, + column_size: None, + decimal_digits: None, + editor_type: None, + }]; + let metadata = vec![CommunityTableColumn { + database_name: "inventory".to_owned(), + table_name: "items".to_owned(), + name: "id".to_owned(), + column_type: "BIGINT".to_owned(), + default_value: Some("0".to_owned()), + auto_increment: Some(true), + comment: "primary id".to_owned(), + primary_key: Some(true), + column_size: Some(20), + decimal_digits: Some(0), + nullable: Some(0), + ..CommunityTableColumn::default() + }]; + assert!(enrich_direct_table_headers( + &mut headers, + &metadata, + "items" + )); + let mut rows = vec![vec![LegacyResultCell { + value: Some("7".to_owned()), + large_value: false, + large_value_id: None, + value_type: "UNKNOWN".to_owned(), + sql_type: -5, + column_type: "BIGINT".to_owned(), + size_bytes: None, + size_chars: None, + loaded_bytes: None, + loaded_chars: None, + truncated: false, + unsupported_reason: None, + }]]; + prepend_synthetic_row_numbers(&mut headers, &mut rows, 20); + + assert_eq!(headers[0].name, "CHAT2DB_ROW_NUMBER"); + assert_eq!(headers[0].data_type, "CHAT2DB_ROW_NUMBER"); + assert_eq!(rows[0][0].value.as_deref(), Some("21")); + assert_eq!(rows[0][1].value.as_deref(), Some("7")); + assert!(headers[1].primary_key); + assert_eq!(headers[1].default_value.as_deref(), Some("0")); + assert_eq!(headers[1].auto_increment, Some(1)); + assert!(!headers[1].nullable); + assert_eq!(headers[1].comment.as_deref(), Some("primary id")); + assert_eq!(headers[1].editor_type.as_deref(), Some("TEXT")); + } + + #[tokio::test] + async fn grid_update_mapping_builds_sql_and_rejects_partial_large_values() { + let request: LegacyGridUpdateRequest = serde_json::from_value(serde_json::json!({ + "dataSourceId": "mysql-local", + "databaseType": "MYSQL", + "databaseName": "inventory", + "schemaName": "", + "tableName": "items", + "headerList": [ + { "name": "CHAT2DB_ROW_NUMBER", "columnType": "BIGINT" }, + { "name": "id", "columnType": "BIGINT", "primaryKey": true, "autoIncrement": 1 }, + { "name": "label", "columnType": "VARCHAR" } + ], + "operations": [{ + "type": "UPDATE", + "dataList": ["1", "7", "new label"], + "oldDataList": ["1", "7", "old label"] + }] + })) + .expect("frontend grid request must deserialize"); + let sql = build_grid_update_sql(&Application::new(), &request) + .await + .expect("grid SQL must build without opening a datasource"); + assert_eq!( + sql, + "UPDATE `inventory`.`items` SET `label` = 'new label' WHERE `id` = 7;" + ); + + let mut partial = request; + partial.operations[0].old_data_list[2] = + Some("CHAT2DB_LARGE_VALUE_PREVIEW:PARTIAL".to_owned()); + let error = build_grid_update_sql(&Application::new(), &partial) + .await + .expect_err("partial large-value previews must never enter DML"); + assert_eq!(error.code, "mysql_partial_large_value_rejected"); + } + + #[test] + fn table_alter_derives_primary_key_changes_from_editor_columns() { + let old_table = LegacyEditableTable { + name: "items".to_owned(), + database_name: "inventory".to_owned(), + column_list: vec![ + LegacyColumn { + name: "id".to_owned(), + column_type: "BIGINT".to_owned(), + primary_key: Some(true), + primary_key_order: 1, + nullable: Some(0), + ..LegacyColumn::default() + }, + LegacyColumn { + name: "code".to_owned(), + column_type: "VARCHAR".to_owned(), + column_size: Some(64), + nullable: Some(0), + ..LegacyColumn::default() + }, + ], + index_list: vec![LegacyIndex { + name: "PRIMARY".to_owned(), + index_type: "Primary".to_owned(), + column_list: vec![LegacyIndexColumn { + column_name: "id".to_owned(), + ..LegacyIndexColumn::default() + }], + ..LegacyIndex::default() + }], + ..LegacyEditableTable::default() + }; + let mut new_table = old_table.clone(); + new_table.column_list[0].primary_key = Some(false); + new_table.column_list[0].primary_key_order = 0; + new_table.column_list[0].edit_status = Some("MODIFY".to_owned()); + new_table.column_list[1].primary_key = Some(true); + new_table.column_list[1].primary_key_order = 1; + new_table.column_list[1].edit_status = Some("MODIFY".to_owned()); + + let alter = mysql_table_alter(&old_table, &new_table, "inventory", "") + .expect("column primary-key edits must normalize"); + let sql = build_mysql_alter_table(&alter).expect("primary-key ALTER must build"); + + assert_eq!(sql.matches("DROP PRIMARY KEY").count(), 1); + assert_eq!( + sql.matches("ADD PRIMARY KEY (`code`) USING BTREE").count(), + 1 + ); + } + + #[test] + fn table_editor_metadata_keeps_only_type_appropriate_dimensions() { + let definition = |column_type: &str, column_size, decimal_digits| { + mysql_column_definition(&LegacyColumn { + name: "value".to_owned(), + column_type: column_type.to_owned(), + column_size, + decimal_digits, + ..LegacyColumn::default() + }) + .expect("metadata column must normalize") + }; + + let varchar = definition("VARCHAR", Some(128), Some(0)); + assert_eq!((varchar.length, varchar.scale), (Some(128), None)); + let text = definition("TEXT", Some(65_535), Some(0)); + assert_eq!((text.length, text.scale), (None, None)); + let decimal = definition("DECIMAL", Some(12), Some(3)); + assert_eq!((decimal.length, decimal.scale), (Some(12), Some(3))); + let timestamp = definition("TIMESTAMP", Some(26), Some(6)); + assert_eq!((timestamp.length, timestamp.scale), (Some(6), None)); + } + + #[test] + fn table_editor_preserves_mysql_enum_and_set_values() { + let column = column_response(CommunityTableColumn { + name: "state".to_owned(), + column_type: "ENUM".to_owned(), + extent: "('','draft','needs,review','O''Reilly','close)later')".to_owned(), + ..CommunityTableColumn::default() + }); - let path = download_large_cell_value_to_path( - &application, - &LegacyLargeCellDownloadRequest { - large_value_id, - format: "text".to_owned(), - }, - ) - .await - .expect("large text must download to a local temporary file"); assert_eq!( - std::fs::read_to_string(&path).expect("download must be readable"), - value + column.value, + "'','draft','needs,review','O''Reilly','close)later'" + ); + let definition = mysql_column_definition(&column) + .expect("the retained enum definition must normalize for DDL"); + assert_eq!( + definition.enum_values, + vec!["", "draft", "needs,review", "O'Reilly", "close)later"] ); - std::fs::remove_file(path).expect("temporary test download must be removed"); } #[test] - fn mysql_console_history_distinguishes_cancellation_from_failure() { - let request = serde_json::from_value(serde_json::json!({ - "dataSourceId": "datasource-1", + fn table_editor_accepts_null_heavy_frontend_rows() { + let request: LegacyTableModifyRequest = serde_json::from_str( + r#"{ + "dataSourceId": "mysql-local", "databaseType": "MYSQL", - "sql": "SELECT SLEEP(30)", - "pageNo": 1, - "pageSize": 200 - })) - .expect("legacy SQL request must deserialize"); - let cancelled = sql_failure_result( - &request, - &LegacyFailure { - code: "mysql_console_cancelled".to_owned(), - message: "The SQL execution was cancelled".to_owned(), - }, - 12, - ); - assert_eq!(mysql_console_history_status(&[&cancelled]), "cancelled"); - - let failed = sql_failure_result( - &request, - &LegacyFailure { - code: "database.query_failed".to_owned(), - message: "Query failed".to_owned(), - }, - 12, - ); - assert_eq!(mysql_console_history_status(&[&failed]), "fail"); + "databaseName": "inventory", + "newTable": { + "name": "items", + "comment": null, + "schemaName": null, + "type": null, + "dbType": null, + "ddl": null, + "engine": null, + "charset": null, + "collate": null, + "partition": null, + "tablespace": null, + "createTime": null, + "updateTime": null, + "columnList": [{ + "oldName": null, + "name": "state", + "tableName": null, + "columnType": "VARCHAR", + "dataType": null, + "defaultValue": null, + "autoIncrement": null, + "comment": null, + "primaryKey": null, + "primaryKeyName": null, + "primaryKeyOrder": null, + "schemaName": null, + "databaseName": null, + "typeName": null, + "columnSize": 32, + "bufferLength": null, + "decimalDigits": null, + "numPrecRadix": null, + "nullableInt": null, + "sqlDataType": null, + "sqlDatetimeSub": null, + "charOctetLength": null, + "ordinalPosition": null, + "nullable": 1, + "generatedColumn": null, + "extent": null, + "charSetName": null, + "collationName": null, + "value": null, + "unit": null, + "defaultConstraintName": null, + "editStatus": "ADD" + }], + "indexList": [{ + "name": "", + "type": null, + "comment": null, + "schemaName": null, + "databaseName": null, + "method": null, + "foreignSchemaName": null, + "foreignTableName": null, + "foreignColumnNamelist": null, + "columnList": [{ + "indexName": null, + "tableName": null, + "type": null, + "comment": null, + "columnName": "state", + "collation": null, + "schemaName": null, + "databaseName": null, + "indexQualifier": null, + "ascOrDesc": null, + "filterCondition": null + }], + "editStatus": "ADD" + }] + } + }"#, + ) + .expect("the retained Community editor payload must accept explicit nulls"); - let mut succeeded = failed; - succeeded.success = true; - succeeded.extra = serde_json::json!({}); - assert_eq!(mysql_console_history_status(&[&succeeded]), "success"); + let column = &request.new_table.column_list[0]; + assert_eq!(column.primary_key_order, 0); + assert!(column.comment.is_empty()); + assert!(column.char_set_name.is_empty()); + let index = &request.new_table.index_list[0]; + assert!(index.index_type.is_empty()); + assert!(index.comment.is_empty()); + assert!(index.column_list[0].index_name.is_empty()); } #[test] - fn counted_metadata_envelope_keeps_total_at_the_top_level() { - let body = counted_envelope_value(Ok(serde_json::json!([ - { "name": "id" }, - { "name": "created_at" } - ]))); + fn result_grid_in_values_accepts_the_frontend_operation_name() { + let operation: LegacyGridOperationRequest = serde_json::from_value(serde_json::json!({ + "type": "IN_VALUES", + "dataList": ["1", "active"], + "selectCols": [1] + })) + .expect("the IN-values operation must deserialize"); - assert_eq!(body["success"], true); - assert_eq!(body["total"], 2); - assert_eq!(body["data"][0]["name"], "id"); - assert!(body["data"].get("total").is_none()); + let operation = mysql_grid_copy_operation(&operation) + .expect("the retained frontend operation name must normalize"); + assert_eq!( + operation.operation_type, + MysqlResultGridCopyOperationType::Where + ); } #[test] - fn editable_preview_prepends_page_aware_row_numbers_and_column_metadata() { - let mut headers = vec![LegacyResultHeader { - data_type: "NUMERIC".to_owned(), - name: "id".to_owned(), - column_name: "id".to_owned(), - column_type: "BIGINT".to_owned(), - table_name: Some("items".to_owned()), - database_name: Some("inventory".to_owned()), - schema_name: None, - primary_key: false, - comment: None, - default_value: None, - auto_increment: None, - nullable: true, - column_size: None, - decimal_digits: None, - editor_type: None, - }]; - let metadata = vec![CommunityTableColumn { - database_name: "inventory".to_owned(), - table_name: "items".to_owned(), - name: "id".to_owned(), - column_type: "BIGINT".to_owned(), - default_value: Some("0".to_owned()), - auto_increment: Some(true), - comment: "primary id".to_owned(), - primary_key: Some(true), - column_size: Some(20), - decimal_digits: Some(0), - nullable: Some(0), - ..CommunityTableColumn::default() - }]; - assert!(enrich_direct_table_headers( - &mut headers, - &metadata, - "items" - )); - let mut rows = vec![vec![LegacyResultCell { - value: Some("7".to_owned()), - large_value: false, - large_value_id: None, - value_type: "UNKNOWN".to_owned(), - sql_type: -5, - column_type: "BIGINT".to_owned(), - size_bytes: None, - size_chars: None, - loaded_bytes: None, - loaded_chars: None, - truncated: false, - unsupported_reason: None, - }]]; - prepend_synthetic_row_numbers(&mut headers, &mut rows, 20); + fn table_alter_detects_column_order_from_array_position() { + let column = |name: &str| LegacyColumn { + old_name: Some(name.to_owned()), + name: name.to_owned(), + column_type: "INT".to_owned(), + nullable: Some(1), + ..LegacyColumn::default() + }; + let old_table = LegacyEditableTable { + name: "items".to_owned(), + database_name: "inventory".to_owned(), + column_list: vec![column("a"), column("b"), column("c")], + ..LegacyEditableTable::default() + }; + let mut new_table = old_table.clone(); + new_table.column_list = vec![ + old_table.column_list[2].clone(), + old_table.column_list[0].clone(), + old_table.column_list[1].clone(), + ]; - assert_eq!(headers[0].name, "CHAT2DB_ROW_NUMBER"); - assert_eq!(headers[0].data_type, "CHAT2DB_ROW_NUMBER"); - assert_eq!(rows[0][0].value.as_deref(), Some("21")); - assert_eq!(rows[0][1].value.as_deref(), Some("7")); - assert!(headers[1].primary_key); - assert_eq!(headers[1].default_value.as_deref(), Some("0")); - assert_eq!(headers[1].auto_increment, Some(1)); - assert!(!headers[1].nullable); - assert_eq!(headers[1].comment.as_deref(), Some("primary id")); - assert_eq!(headers[1].editor_type.as_deref(), Some("TEXT")); + assert_eq!( + mysql_reordered_column_names(&old_table, &new_table), + ["c", "a"] + ); + + let alter = mysql_table_alter(&old_table, &new_table, "inventory", "") + .expect("a drag-only reorder must normalize"); + let sql = build_mysql_alter_table(&alter).expect("a drag-only reorder must build"); + + assert_eq!(sql.matches("MODIFY COLUMN").count(), 2); + assert!(sql.contains("MODIFY COLUMN `c` INT NULL FIRST")); + assert!(sql.contains("MODIFY COLUMN `a` INT NULL AFTER `c`")); } #[tokio::test] - async fn grid_update_mapping_builds_sql_and_rejects_partial_large_values() { - let request: LegacyGridUpdateRequest = serde_json::from_value(serde_json::json!({ + async fn view_meta_returns_the_community_creation_template() { + let request: LegacyViewOperationRequest = serde_json::from_value(serde_json::json!({ "dataSourceId": "mysql-local", "databaseType": "MYSQL", "databaseName": "inventory", - "schemaName": "", - "tableName": "items", - "headerList": [ - { "name": "CHAT2DB_ROW_NUMBER", "columnType": "BIGINT" }, - { "name": "id", "columnType": "BIGINT", "primaryKey": true, "autoIncrement": 1 }, - { "name": "label", "columnType": "VARCHAR" } - ], - "operations": [{ - "type": "UPDATE", - "dataList": ["1", "7", "new label"], - "oldDataList": ["1", "7", "old label"] - }] + "schemaName": "ignored_schema", + "viewName": "" })) - .expect("frontend grid request must deserialize"); - let sql = build_grid_update_sql(&Application::new(), &request) + .expect("view metadata request must deserialize"); + let metadata = view_editor_meta(&Application::new(), &request) .await - .expect("grid SQL must build without opening a datasource"); + .expect("view metadata must not require an existing view"); + + assert_eq!(metadata.sql, "select * from table_name"); + assert_eq!(metadata.configurations.len(), 6); assert_eq!( - sql, - "UPDATE `inventory`.`items` SET `label` = 'new label' WHERE `id` = 7;" + metadata + .configurations + .iter() + .map(|configuration| configuration["name"].as_str().unwrap_or_default()) + .collect::>(), + vec![ + "algorithm", + "checkOption", + "security", + "viewName", + "definer", + "useOrReplace" + ] ); + assert!(metadata.preview_sql.contains("`inventory`.`undefined`")); + assert!(!metadata.preview_sql.contains("ignored_schema")); + } - let mut partial = request; - partial.operations[0].old_data_list[2] = - Some("CHAT2DB_LARGE_VALUE_PREVIEW:PARTIAL".to_owned()); - let error = build_grid_update_sql(&Application::new(), &partial) + #[tokio::test] + async fn table_and_view_editor_payloads_map_to_core_builders() { + let table_request: LegacyTableModifyRequest = serde_json::from_value(serde_json::json!({ + "dataSourceId": "mysql-local", + "databaseType": "MYSQL", + "databaseName": "inventory", + "newTable": { + "name": "items", + "comment": "stock", + "engine": "InnoDB", + "charset": "utf8mb4", + "columnList": [ + { + "name": "id", + "columnType": "BIGINT", + "nullable": 0, + "autoIncrement": true, + "primaryKey": true + }, + { + "name": "label", + "columnType": "VARCHAR", + "columnSize": 255, + "nullable": 0 + } + ], + "indexList": [] + } + })) + .expect("table editor request must deserialize"); + let table_sql = build_table_modify_sql(&Application::new(), &table_request) .await - .expect_err("partial large-value previews must never enter DML"); - assert_eq!(error.code, "mysql_partial_large_value_rejected"); + .expect("table SQL must build"); + assert_eq!(table_sql.len(), 1); + assert!( + table_sql[0] + .sql + .starts_with("CREATE TABLE `inventory`.`items`") + ); + assert!(table_sql[0].sql.contains("PRIMARY KEY (`id`) USING BTREE")); + assert!(table_sql[0].sql.contains("`label` VARCHAR(255) NOT NULL")); + + let view_request: LegacyViewOperationRequest = serde_json::from_value(serde_json::json!({ + "dataSourceId": "mysql-local", + "databaseType": "MYSQL", + "databaseName": "inventory", + "viewName": "active_items", + "viewBody": "SELECT id FROM items WHERE active = 1", + "useOrReplace": true, + "algorithm": "MERGE", + "definer": "reporter@localhost", + "security": "INVOKER", + "checkOption": "LOCAL" + })) + .expect("view editor request must deserialize"); + let view_sql = build_view_modify_sql(&Application::new(), &view_request) + .await + .expect("view SQL must build"); + assert!(view_sql.starts_with("CREATE OR REPLACE ALGORITHM = MERGE")); + assert!(view_sql.contains("DEFINER = 'reporter'@'localhost'")); + assert!(view_sql.ends_with("WITH LOCAL CHECK OPTION")); } - #[test] - fn table_alter_derives_primary_key_changes_from_editor_columns() { - let old_table = LegacyEditableTable { - name: "items".to_owned(), - database_name: "inventory".to_owned(), - column_list: vec![ - LegacyColumn { - name: "id".to_owned(), - column_type: "BIGINT".to_owned(), - primary_key: Some(true), - primary_key_order: 1, - nullable: Some(0), - ..LegacyColumn::default() - }, - LegacyColumn { - name: "code".to_owned(), - column_type: "VARCHAR".to_owned(), - column_size: Some(64), - nullable: Some(0), - ..LegacyColumn::default() + #[tokio::test] + async fn dashboard_paths_are_registered_for_dispatch_and_axum() { + let application = Application::new(); + let router = routes().with_state(application.clone()); + for (method, path) in REQUIRED_DASHBOARD_PATHS { + assert!(LEGACY_PATHS.contains(path), "missing dispatch path: {path}"); + let response = dispatch( + &application, + LegacyDispatchRequest { + request_url: (*path).to_owned(), + method: (*method).to_owned(), + message: serde_json::Value::Null, }, - ], - index_list: vec![LegacyIndex { - name: "PRIMARY".to_owned(), - index_type: "Primary".to_owned(), - column_list: vec![LegacyIndexColumn { - column_name: "id".to_owned(), - ..LegacyIndexColumn::default() - }], - ..LegacyIndex::default() - }], - ..LegacyEditableTable::default() + ) + .await; + assert_ne!( + response["errorCode"], "route_not_found", + "missing desktop dispatch branch: {method} {path}" + ); + + let http_method = method + .to_ascii_uppercase() + .parse::() + .expect("method must be valid"); + let mut builder = Request::builder().method(http_method).uri(*path); + let body = if matches!(*method, "get" | "delete") { + Body::empty() + } else { + builder = builder.header("content-type", "application/json"); + Body::from("null") + }; + let response = router + .clone() + .oneshot(builder.body(body).expect("request must build")) + .await + .expect("router must respond"); + assert_eq!( + response.status(), + StatusCode::OK, + "missing Axum route: {method} {path}" + ); + } + } + + async fn dashboard_http_json( + router: &Router, + method: &str, + uri: &str, + payload: Option, + ) -> serde_json::Value { + let method = method + .parse::() + .expect("HTTP method must parse"); + let mut builder = Request::builder().method(method).uri(uri); + let body = match payload { + Some(payload) => { + builder = builder.header("content-type", "application/json"); + Body::from(serde_json::to_vec(&payload).expect("payload must encode")) + } + None => Body::empty(), }; - let mut new_table = old_table.clone(); - new_table.column_list[0].primary_key = Some(false); - new_table.column_list[0].primary_key_order = 0; - new_table.column_list[0].edit_status = Some("MODIFY".to_owned()); - new_table.column_list[1].primary_key = Some(true); - new_table.column_list[1].primary_key_order = 1; - new_table.column_list[1].edit_status = Some("MODIFY".to_owned()); + let response = router + .clone() + .oneshot(builder.body(body).expect("request must build")) + .await + .expect("router must respond"); + assert_eq!(response.status(), StatusCode::OK, "{uri} must use HTTP 200"); + let body = response + .into_body() + .collect() + .await + .expect("response body must collect") + .to_bytes(); + serde_json::from_slice(&body).expect("response body must be JSON") + } - let alter = mysql_table_alter(&old_table, &new_table, "inventory", "") - .expect("column primary-key edits must normalize"); - let sql = build_mysql_alter_table(&alter).expect("primary-key ALTER must build"); + #[tokio::test] + async fn dashboard_http_routes_preserve_community_crud_and_envelopes() { + let directory = tempfile::TempDir::new().expect("temporary directory"); + let storage = Storage::open(directory.path(), Arc::new(EmptyVault)).expect("storage opens"); + let router = routes().with_state(Application::with_storage(storage)); + + let created = dashboard_http_json( + &router, + "POST", + "/api/dashboard/create", + Some(serde_json::json!({ + "name": "Operations Board", + "description": "before update", + "chartIds": [] + })), + ) + .await; + assert_eq!(created["success"], true); + assert!(created["errorCode"].is_null()); + assert!(created["errorMessage"].is_null()); + let dashboard_id = created["data"] + .as_i64() + .expect("dashboard id must be numeric"); + + let updated = dashboard_http_json( + &router, + "POST", + "/api/dashboard/update", + Some(serde_json::json!({ + "id": dashboard_id, + "description": "after update", + "chartIds": [17] + })), + ) + .await; + assert_eq!(updated["success"], true); + assert!(updated["data"].is_null()); - assert_eq!(sql.matches("DROP PRIMARY KEY").count(), 1); - assert_eq!( - sql.matches("ADD PRIMARY KEY (`code`) USING BTREE").count(), - 1 - ); + let dashboard = dashboard_http_json( + &router, + "GET", + &format!("/api/dashboard?id={dashboard_id}"), + None, + ) + .await; + assert_eq!(dashboard["data"]["description"], "after update"); + assert_eq!(dashboard["data"]["chartIds"][0], 17); + + let list = dashboard_http_json( + &router, + "GET", + "/api/dashboard/list?pageNo=1&pageSize=20&searchKey=operations", + None, + ) + .await; + assert_eq!(list["success"], true); + assert_eq!(list["data"]["total"], 1); + assert_eq!(list["data"]["data"][0]["id"], dashboard_id); + assert_eq!(list["data"]["hasNextPage"], false); + + let deleted = dashboard_http_json( + &router, + "DELETE", + &format!("/api/dashboard?id={dashboard_id}"), + None, + ) + .await; + assert_eq!(deleted["success"], true); + assert_eq!(deleted["data"], "success"); + let missing = dashboard_http_json( + &router, + "GET", + &format!("/api/dashboard?id={dashboard_id}"), + None, + ) + .await; + assert_eq!(missing["success"], true); + assert!(missing["data"].is_null()); } - #[test] - fn table_editor_metadata_keeps_only_type_appropriate_dimensions() { - let definition = |column_type: &str, column_size, decimal_digits| { - mysql_column_definition(&LegacyColumn { - name: "value".to_owned(), - column_type: column_type.to_owned(), - column_size, - decimal_digits, - ..LegacyColumn::default() - }) - .expect("metadata column must normalize") - }; + #[tokio::test] + async fn chart_http_routes_preserve_community_crud_and_envelopes() { + let directory = tempfile::TempDir::new().expect("temporary directory"); + let storage = Storage::open(directory.path(), Arc::new(EmptyVault)).expect("storage opens"); + let router = routes().with_state(Application::with_storage(storage)); + + let created = dashboard_http_json( + &router, + "POST", + "/api/v1/chart/create", + Some(serde_json::json!({ + "name": "Revenue", + "chartSchema": {"type": "bar", "title": "Revenue"}, + "metaData": {"dataList": [["42"]]}, + "databaseInfo": {"sql": "SELECT 42"}, + "refreshType": "MANUAL" + })), + ) + .await; + assert_eq!(created["success"], true); + assert!(created["errorCode"].is_null()); + let chart_id = created["data"].as_i64().expect("chart id must be numeric"); + + let chart = dashboard_http_json( + &router, + "GET", + &format!("/api/v1/chart?id={chart_id}"), + None, + ) + .await; + assert_eq!(chart["data"]["name"], "Revenue"); + assert_eq!(chart["data"]["chartSchema"]["type"], "bar"); - let varchar = definition("VARCHAR", Some(128), Some(0)); - assert_eq!((varchar.length, varchar.scale), (Some(128), None)); - let text = definition("TEXT", Some(65_535), Some(0)); - assert_eq!((text.length, text.scale), (None, None)); - let decimal = definition("DECIMAL", Some(12), Some(3)); - assert_eq!((decimal.length, decimal.scale), (Some(12), Some(3))); - let timestamp = definition("TIMESTAMP", Some(26), Some(6)); - assert_eq!((timestamp.length, timestamp.scale), (Some(6), None)); - } + let detail = dashboard_http_json( + &router, + "GET", + &format!("/api/chart/detail?chartId={chart_id}&refresh=false"), + None, + ) + .await; + assert_eq!(detail["success"], true); + assert_eq!(detail["data"]["metaData"]["dataList"][0][0], "42"); + + let updated = dashboard_http_json( + &router, + "POST", + "/api/v1/chart/update", + Some(serde_json::json!({ + "id": chart_id, + "name": "Revenue Updated", + "description": "after update" + })), + ) + .await; + assert_eq!(updated["success"], true); + assert!(updated["data"].is_null()); - #[test] - fn table_editor_preserves_mysql_enum_and_set_values() { - let column = column_response(CommunityTableColumn { - name: "state".to_owned(), - column_type: "ENUM".to_owned(), - extent: "('','draft','needs,review','O''Reilly','close)later')".to_owned(), - ..CommunityTableColumn::default() - }); + let updated_chart = dashboard_http_json( + &router, + "GET", + &format!("/api/v1/chart?id={chart_id}"), + None, + ) + .await; + assert_eq!(updated_chart["data"]["name"], "Revenue Updated"); - assert_eq!( - column.value, - "'','draft','needs,review','O''Reilly','close)later'" - ); - let definition = mysql_column_definition(&column) - .expect("the retained enum definition must normalize for DDL"); - assert_eq!( - definition.enum_values, - vec!["", "draft", "needs,review", "O'Reilly", "close)later"] - ); + let deleted = dashboard_http_json( + &router, + "DELETE", + &format!("/api/chart?id={chart_id}"), + None, + ) + .await; + assert_eq!(deleted["success"], true); + assert_eq!(deleted["data"], "success"); + let missing = dashboard_http_json( + &router, + "GET", + &format!("/api/v1/chart?id={chart_id}"), + None, + ) + .await; + assert_eq!(missing["success"], true); + assert!(missing["data"].is_null()); } - #[test] - fn table_editor_accepts_null_heavy_frontend_rows() { - let request: LegacyTableModifyRequest = serde_json::from_str( - r#"{ - "dataSourceId": "mysql-local", - "databaseType": "MYSQL", - "databaseName": "inventory", - "newTable": { - "name": "items", - "comment": null, - "schemaName": null, - "type": null, - "dbType": null, - "ddl": null, - "engine": null, - "charset": null, - "collate": null, - "partition": null, - "tablespace": null, - "createTime": null, - "updateTime": null, - "columnList": [{ - "oldName": null, - "name": "state", - "tableName": null, - "columnType": "VARCHAR", - "dataType": null, - "defaultValue": null, - "autoIncrement": null, - "comment": null, - "primaryKey": null, - "primaryKeyName": null, - "primaryKeyOrder": null, - "schemaName": null, - "databaseName": null, - "typeName": null, - "columnSize": 32, - "bufferLength": null, - "decimalDigits": null, - "numPrecRadix": null, - "nullableInt": null, - "sqlDataType": null, - "sqlDatetimeSub": null, - "charOctetLength": null, - "ordinalPosition": null, - "nullable": 1, - "generatedColumn": null, - "extent": null, - "charSetName": null, - "collationName": null, - "value": null, - "unit": null, - "defaultConstraintName": null, - "editStatus": "ADD" - }], - "indexList": [{ - "name": "", - "type": null, - "comment": null, - "schemaName": null, - "databaseName": null, - "method": null, - "foreignSchemaName": null, - "foreignTableName": null, - "foreignColumnNamelist": null, - "columnList": [{ - "indexName": null, - "tableName": null, - "type": null, - "comment": null, - "columnName": "state", - "collation": null, - "schemaName": null, - "databaseName": null, - "indexQualifier": null, - "ascOrDesc": null, - "filterCondition": null - }], - "editStatus": "ADD" - }] - } - }"#, - ) - .expect("the retained Community editor payload must accept explicit nulls"); + #[tokio::test] + async fn editable_paths_are_registered_for_dispatch_and_axum() { + let router = routes().with_state(Application::new()); + for (method, path) in REQUIRED_EDITABLE_PATHS { + assert!(LEGACY_PATHS.contains(path), "missing dispatch path: {path}"); + let response = dispatch( + &Application::new(), + LegacyDispatchRequest { + request_url: (*path).to_owned(), + method: (*method).to_owned(), + message: serde_json::Value::Null, + }, + ) + .await; + assert_eq!( + response["errorCode"], "invalid_legacy_request", + "missing desktop dispatch branch: {method} {path}" + ); - let column = &request.new_table.column_list[0]; - assert_eq!(column.primary_key_order, 0); - assert!(column.comment.is_empty()); - assert!(column.char_set_name.is_empty()); - let index = &request.new_table.index_list[0]; - assert!(index.index_type.is_empty()); - assert!(index.comment.is_empty()); - assert!(index.column_list[0].index_name.is_empty()); + let http_method = method + .to_ascii_uppercase() + .parse::() + .expect("method must be valid"); + let mut builder = Request::builder().method(http_method).uri(*path); + let body = if *method == "get" { + Body::empty() + } else { + builder = builder.header("content-type", "application/json"); + Body::from("null") + }; + let response = router + .clone() + .oneshot(builder.body(body).expect("request must build")) + .await + .expect("router must respond"); + assert_eq!( + response.status(), + StatusCode::OK, + "missing Axum route: {method} {path}" + ); + } } - #[test] - fn result_grid_in_values_accepts_the_frontend_operation_name() { - let operation: LegacyGridOperationRequest = serde_json::from_value(serde_json::json!({ - "type": "IN_VALUES", - "dataList": ["1", "active"], - "selectCols": [1] - })) - .expect("the IN-values operation must deserialize"); + #[tokio::test] + async fn transfer_paths_are_registered_for_dispatch_and_axum() { + let application = Application::new(); + let router = routes().with_state(application.clone()); + for (method, path) in REQUIRED_TRANSFER_PATHS { + assert!(LEGACY_PATHS.contains(path), "missing dispatch path: {path}"); + let response = dispatch( + &application, + LegacyDispatchRequest { + request_url: (*path).to_owned(), + method: (*method).to_owned(), + message: serde_json::Value::Null, + }, + ) + .await; + assert_ne!( + response["errorCode"], "route_not_found", + "missing desktop dispatch branch: {method} {path}" + ); - let operation = mysql_grid_copy_operation(&operation) - .expect("the retained frontend operation name must normalize"); - assert_eq!( - operation.operation_type, - MysqlResultGridCopyOperationType::Where - ); + let http_method = method + .to_ascii_uppercase() + .parse::() + .expect("method must be valid"); + let mut builder = Request::builder().method(http_method).uri(*path); + let body = if *method == "get" { + Body::empty() + } else { + builder = builder.header("content-type", "application/json"); + Body::from("null") + }; + let response = router + .clone() + .oneshot(builder.body(body).expect("request must build")) + .await + .expect("router must respond"); + assert_eq!( + response.status(), + StatusCode::OK, + "missing Axum route: {method} {path}" + ); + } } #[test] - fn table_alter_detects_column_order_from_array_position() { - let column = |name: &str| LegacyColumn { - old_name: Some(name.to_owned()), - name: name.to_owned(), - column_type: "INT".to_owned(), - nullable: Some(1), - ..LegacyColumn::default() - }; - let old_table = LegacyEditableTable { - name: "items".to_owned(), + fn transfer_projection_matches_the_retained_task_drawer() { + let task = TransferTask { + id: 42, + datasource_id: "mysql-local".to_owned(), database_name: "inventory".to_owned(), - column_list: vec![column("a"), column("b"), column("c")], - ..LegacyEditableTable::default() + schema_name: String::new(), + table_name: Some("items".to_owned()), + kind: TransferTaskKind::ExportFile, + status: TransferTaskStatus::Succeeded, + task_name: "Export items".to_owned(), + progress_current: "9".to_owned(), + progress_total: Some("10".to_owned()), + progress_description: "Completed".to_owned(), + info_log: "done".to_owned(), + error_log: String::new(), + artifact_id: Some("artifact-42".to_owned()), + cancel_requested: false, + created_at_ms: "1700000000000".to_owned(), + updated_at_ms: "1700000000100".to_owned(), + finished_at_ms: Some("1700000000100".to_owned()), }; - let mut new_table = old_table.clone(); - new_table.column_list = vec![ - old_table.column_list[2].clone(), - old_table.column_list[0].clone(), - old_table.column_list[1].clone(), - ]; + let projected = legacy_transfer_task_for_web(task.clone()); + assert_eq!(projected.task_type, "DOWNLOAD_TABLE_STRUCTURE"); + assert_eq!(projected.task_status, "FINISHED"); + assert_eq!(projected.task_progress, "100"); + assert_eq!(projected.progress, "9"); + assert_eq!(projected.download_url, "/api/task/download?id=42"); + assert_eq!(projected.gmt_create, 1_700_000_000_000); + + let projected = legacy_transfer_task(task, "/tmp/export-items.csv".to_owned()); + assert_eq!(projected.download_url, "/tmp/export-items.csv"); + } + + #[test] + fn transfer_compatibility_parses_frontend_aliases_and_filters() { assert_eq!( - mysql_reordered_column_names(&old_table, &new_table), - ["c", "a"] + legacy_transfer_format("EXCEL", "exportType").expect("Excel must map"), + TransferFileFormat::Xlsx ); + assert_eq!( + legacy_transfer_status_filter("ERROR").expect("ERROR must map"), + vec![TransferTaskStatus::Failed, TransferTaskStatus::Interrupted] + ); + assert_eq!(safe_attachment_filename("report\r\n.csv"), "report__.csv"); - let alter = mysql_table_alter(&old_table, &new_table, "inventory", "") - .expect("a drag-only reorder must normalize"); - let sql = build_mysql_alter_table(&alter).expect("a drag-only reorder must build"); - - assert_eq!(sql.matches("MODIFY COLUMN").count(), 2); - assert!(sql.contains("MODIFY COLUMN `c` INT NULL FIRST")); - assert!(sql.contains("MODIFY COLUMN `a` INT NULL AFTER `c`")); + let request: LegacyImportFileRequest = serde_json::from_value(serde_json::json!({ + "dataSourceId": 7, + "databaseName": "inventory", + "tableName": "items", + "fileName": "/tmp/items.csv", + "importType": "CSV" + })) + .expect("numeric datasource ids must remain compatible"); + assert_eq!(request.data_source_id.as_string(), "7"); + assert!(request.contains_header); + assert_eq!(request.tabular_encoding, TabularImportEncoding::Plain); + assert_eq!( + legacy_tabular_import_encoding("CHAT2DB_V1").expect("v1 encoding maps"), + TabularImportEncoding::Chat2dbV1 + ); } #[tokio::test] - async fn view_meta_returns_the_community_creation_template() { - let request: LegacyViewOperationRequest = serde_json::from_value(serde_json::json!({ + async fn http_import_rejects_server_paths_without_disclosing_them() { + let directory = tempfile::TempDir::new().expect("temporary directory"); + let secret_path = directory.path().join("server-secret.sql"); + let secret = "sentinel-server-only-secret"; + fs::write(&secret_path, secret).expect("secret fixture writes"); + let payload = serde_json::json!({ "dataSourceId": "mysql-local", - "databaseType": "MYSQL", "databaseName": "inventory", - "schemaName": "ignored_schema", - "viewName": "" - })) - .expect("view metadata request must deserialize"); - let metadata = view_editor_meta(&Application::new(), &request) - .await - .expect("view metadata must not require an existing view"); + "tableName": "items", + "fileName": secret_path.to_string_lossy(), + "importType": "CSV" + }); + for path in ["/api/import/sql_file", "/api/import/other_file"] { + let response = routes() + .with_state(Application::new()) + .oneshot( + Request::builder() + .method("POST") + .uri(path) + .header("content-type", "application/json") + .body(Body::from( + serde_json::to_vec(&payload).expect("request encodes"), + )) + .expect("request builds"), + ) + .await + .expect("router responds"); + assert_eq!(response.status(), StatusCode::OK); + let body = response + .into_body() + .collect() + .await + .expect("response collects") + .to_bytes(); + let rendered = String::from_utf8(body.to_vec()).expect("response is UTF-8"); + let envelope: serde_json::Value = + serde_json::from_slice(&body).expect("response is JSON"); + assert_eq!(envelope["success"], false); + assert_eq!(envelope["errorCode"], "web_import_upload_required"); + assert!(!rendered.contains(secret)); + assert!(!rendered.contains(&secret_path.to_string_lossy().into_owned())); + } + assert_eq!( + fs::read_to_string(secret_path).expect("secret reads"), + secret + ); + } - assert_eq!(metadata.sql, "select * from table_name"); - assert_eq!(metadata.configurations.len(), 6); + #[tokio::test] + async fn failed_http_multipart_import_removes_the_staged_artifact() { + let directory = tempfile::TempDir::new().expect("temporary directory"); + let storage = Storage::open(directory.path(), Arc::new(EmptyVault)).expect("storage opens"); + let application = Application::with_storage(storage); + let boundary = "chat2db-failed-import-cleanup"; + let multipart = format!( + "--{boundary}\r\nContent-Disposition: form-data; name=\"dataSourceId\"\r\n\r\nmissing\r\n\ + --{boundary}\r\nContent-Disposition: form-data; name=\"databaseName\"\r\n\r\ninventory\r\n\ + --{boundary}\r\nContent-Disposition: form-data; name=\"tableName\"\r\n\r\nitems\r\n\ + --{boundary}\r\nContent-Disposition: form-data; name=\"importType\"\r\n\r\nCSV\r\n\ + --{boundary}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"upload.csv\"\r\n\ + Content-Type: text/csv\r\n\r\nid,name\n1,alpha\n\r\n--{boundary}--\r\n" + ); + let response = routes() + .with_state(application) + .oneshot( + Request::builder() + .method("POST") + .uri("/api/import/other_file") + .header( + "content-type", + format!("multipart/form-data; boundary={boundary}"), + ) + .body(Body::from(multipart)) + .expect("request builds"), + ) + .await + .expect("router responds"); + assert_eq!(response.status(), StatusCode::OK); + let body = response + .into_body() + .collect() + .await + .expect("response collects") + .to_bytes(); + let envelope: serde_json::Value = serde_json::from_slice(&body).expect("response is JSON"); + assert_eq!(envelope["success"], false); assert_eq!( - metadata - .configurations - .iter() - .map(|configuration| configuration["name"].as_str().unwrap_or_default()) - .collect::>(), - vec![ - "algorithm", - "checkOption", - "security", - "viewName", - "definer", - "useOrReplace" - ] + fs::read_dir(directory.path().join("artifacts")) + .expect("artifact directory reads") + .count(), + 0, + "a rejected upload must not leave a managed artifact" ); - assert!(metadata.preview_sql.contains("`inventory`.`undefined`")); - assert!(!metadata.preview_sql.contains("ignored_schema")); } #[tokio::test] - async fn table_and_view_editor_payloads_map_to_core_builders() { - let table_request: LegacyTableModifyRequest = serde_json::from_value(serde_json::json!({ - "dataSourceId": "mysql-local", - "databaseType": "MYSQL", - "databaseName": "inventory", - "newTable": { - "name": "items", - "comment": "stock", - "engine": "InnoDB", - "charset": "utf8mb4", - "columnList": [ - { - "name": "id", - "columnType": "BIGINT", - "nullable": 0, - "autoIncrement": true, - "primaryKey": true - }, - { - "name": "label", - "columnType": "VARCHAR", - "columnSize": 255, - "nullable": 0 - } - ], - "indexList": [] + async fn completed_import_task_removes_its_staged_artifact() { + let directory = tempfile::TempDir::new().expect("temporary directory"); + let storage = Storage::open(directory.path(), Arc::new(EmptyVault)).expect("storage opens"); + let application = Application::with_storage(storage.clone()); + let mut writer = storage + .begin_transfer_artifact(None, "upload.csv", "text/csv", "CSV", "csv", None) + .expect("artifact begins"); + writer.write_all(b"id\n1\n").expect("artifact writes"); + let artifact = writer.finish().expect("artifact finishes"); + let path = storage + .resolve_transfer_artifact(&artifact.id) + .expect("artifact resolves") + .path; + let task = storage + .create_transfer_task(&CreateTransferTask { + datasource_id: "mysql-local".to_owned(), + database_name: "inventory".to_owned(), + schema_name: String::new(), + table_name: Some("items".to_owned()), + kind: StoredTransferTaskKind::ImportFile, + task_name: "Import upload.csv".to_owned(), + }) + .expect("task creates"); + storage.start_transfer_task(task.id).expect("task starts"); + schedule_legacy_import_upload_cleanup(application, storage.clone(), task.id, artifact.id); + storage + .complete_transfer_task(task.id, "done") + .expect("task completes"); + + for _ in 0..40 { + if !path.exists() { + break; } - })) - .expect("table editor request must deserialize"); - let table_sql = build_table_modify_sql(&Application::new(), &table_request) + tokio::time::sleep(Duration::from_millis(50)).await; + } + assert!(!path.exists(), "terminal imports must release staged files"); + } + + #[tokio::test] + async fn managed_zip_attachment_streams_exact_download_headers() { + let directory = tempfile::TempDir::new().expect("temporary directory"); + let storage = Storage::open(directory.path(), Arc::new(EmptyVault)).expect("storage opens"); + let application = Application::with_storage(storage.clone()); + let archive = b"PK\x03\x04generated-class-archive"; + let mut writer = storage + .begin_transfer_artifact( + None, + "items mybatis.zip", + "application/zip", + "ZIP", + "zip", + None, + ) + .expect("artifact begins"); + writer.write_all(archive).expect("artifact writes"); + let artifact = writer.finish().expect("artifact finishes"); + let download = application + .transfer_artifact_download(&artifact.id) .await - .expect("table SQL must build"); - assert_eq!(table_sql.len(), 1); - assert!( - table_sql[0] - .sql - .starts_with("CREATE TABLE `inventory`.`items`") + .expect("artifact resolves"); + + let response = transfer_attachment_response(Ok(download)); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response.headers()[header::CONTENT_TYPE], "application/zip"); + assert_eq!( + response.headers()[header::CONTENT_DISPOSITION], + "attachment; filename=\"items_mybatis.zip\"" ); - assert!(table_sql[0].sql.contains("PRIMARY KEY (`id`) USING BTREE")); - assert!(table_sql[0].sql.contains("`label` VARCHAR(255) NOT NULL")); + assert_eq!( + response.headers()[header::CONTENT_LENGTH], + archive.len().to_string() + ); + let body = response + .into_body() + .collect() + .await + .expect("response streams") + .to_bytes(); + assert_eq!(body.as_ref(), archive); + } - let view_request: LegacyViewOperationRequest = serde_json::from_value(serde_json::json!({ - "dataSourceId": "mysql-local", - "databaseType": "MYSQL", - "databaseName": "inventory", - "viewName": "active_items", - "viewBody": "SELECT id FROM items WHERE active = 1", - "useOrReplace": true, - "algorithm": "MERGE", - "definer": "reporter@localhost", - "security": "INVOKER", - "checkOption": "LOCAL" - })) - .expect("view editor request must deserialize"); - let view_sql = build_view_modify_sql(&Application::new(), &view_request) + #[tokio::test] + async fn http_never_accepts_or_returns_server_local_paths() { + let directory = tempfile::TempDir::new().expect("temporary directory"); + let export_path = directory.path().join("server-output"); + let router = routes().with_state(Application::new()); + for (path, payload) in [ + ( + "/api/export/sql_file", + serde_json::json!({ + "dataSourceId": "mysql-local", + "databaseName": "inventory", + "exportPath": export_path + }), + ), + ( + "/api/export/other_file", + serde_json::json!({ + "dataSourceId": "mysql-local", + "databaseName": "inventory", + "tableNames": ["items"], + "exportType": "CSV", + "exportPath": export_path + }), + ), + ( + "/api/rdb/table/generate/class", + serde_json::json!({ + "dataSourceId": "mysql-local", + "databaseName": "inventory", + "tableName": "items", + "exportPath": export_path + }), + ), + ] { + let response = router + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri(path) + .header("content-type", "application/json") + .body(Body::from( + serde_json::to_vec(&payload).expect("payload encodes"), + )) + .expect("request builds"), + ) + .await + .expect("router responds"); + let body = response + .into_body() + .collect() + .await + .expect("response collects") + .to_bytes(); + let envelope: serde_json::Value = + serde_json::from_slice(&body).expect("response is JSON"); + assert_eq!(envelope["errorCode"], "desktop_file_operation_required"); + } + assert!(!export_path.exists()); + + let local_export = LegacyDispatchRequest { + request_url: "/api/export/sql_file".to_owned(), + method: "post".to_owned(), + message: serde_json::json!({ + "dataSourceId": "mysql-local", + "databaseName": "inventory", + "exportPath": export_path + }), + }; + let generic = dispatch(&Application::new(), local_export.clone()).await; + assert_eq!(generic["errorCode"], "desktop_file_operation_required"); + let desktop = dispatch_desktop(&Application::new(), local_export).await; + assert_ne!( + desktop["errorCode"], "desktop_file_operation_required", + "Desktop IPC must retain local-path behavior" + ); + + let response = router + .oneshot( + Request::builder() + .method("POST") + .uri("/api/rdb/cell/download_path") + .header("content-type", "application/json") + .body(Body::from("{}")) + .expect("request builds"), + ) .await - .expect("view SQL must build"); - assert!(view_sql.starts_with("CREATE OR REPLACE ALGORITHM = MERGE")); - assert!(view_sql.contains("DEFINER = 'reporter'@'localhost'")); - assert!(view_sql.ends_with("WITH LOCAL CHECK OPTION")); + .expect("router responds"); + assert_eq!(response.status(), StatusCode::NOT_FOUND); } #[tokio::test] - async fn editable_paths_are_registered_for_dispatch_and_axum() { + async fn sql_compatibility_paths_are_registered_for_dispatch_and_axum() { let router = routes().with_state(Application::new()); - for (method, path) in REQUIRED_EDITABLE_PATHS { + for (method, path) in REQUIRED_SQL_COMPATIBILITY_PATHS { assert!(LEGACY_PATHS.contains(path), "missing dispatch path: {path}"); let response = dispatch( &Application::new(), @@ -7596,6 +12019,149 @@ mod tests { } } + #[tokio::test] + async fn workspace_paths_are_registered_for_dispatch_and_axum() { + let application = Application::new(); + let router = routes().with_state(application.clone()); + for (method, path) in REQUIRED_WORKSPACE_PATHS { + assert!(LEGACY_PATHS.contains(path), "missing dispatch path: {path}"); + let desktop = dispatch( + &application, + LegacyDispatchRequest { + request_url: (*path).to_owned(), + method: (*method).to_owned(), + message: serde_json::Value::Null, + }, + ) + .await; + assert_ne!( + desktop["errorCode"], "route_not_found", + "missing desktop dispatch branch: {method} {path}" + ); + + let http_method = method + .to_ascii_uppercase() + .parse::() + .expect("method must be valid"); + let mut builder = Request::builder().method(http_method).uri(*path); + let body = if *method == "get" { + Body::empty() + } else { + builder = builder.header("content-type", "application/json"); + Body::from("null") + }; + let response = router + .clone() + .oneshot(builder.body(body).expect("request must build")) + .await + .expect("router must respond"); + assert_eq!( + response.status(), + StatusCode::OK, + "missing Axum route: {method} {path}" + ); + } + } + + #[tokio::test] + async fn parameterless_community_import_is_desktop_only_but_documents_remain_portable() { + let directory = tempfile::tempdir().expect("temporary storage"); + let storage = Storage::open(directory.path(), Arc::new(EmptyVault)).expect("storage opens"); + let application = Application::with_storage(storage); + + let dispatched = dispatch( + &application, + LegacyDispatchRequest { + request_url: "/api/connection/datasource/import_community".to_owned(), + method: "get".to_owned(), + message: serde_json::Value::Null, + }, + ) + .await; + assert_eq!(dispatched["success"], false, "{dispatched}"); + assert_eq!( + dispatched["errorCode"], "desktop_file_operation_required", + "{dispatched}" + ); + + let router = routes().with_state(application); + let response = router + .clone() + .oneshot( + Request::builder() + .uri("/api/connection/datasource/import_community") + .body(Body::empty()) + .expect("request builds"), + ) + .await + .expect("route responds"); + let body: serde_json::Value = serde_json::from_slice( + &response + .into_body() + .collect() + .await + .expect("body collects") + .to_bytes(), + ) + .expect("response decodes"); + assert_eq!(body["success"], false, "{body}"); + assert_eq!(body["errorCode"], "desktop_file_operation_required"); + + let document = serde_json::json!({ + "schemaVersion": 1, + "exportedAtMs": "0", + "datasources": [] + }); + let response = router + .oneshot( + Request::builder() + .method("POST") + .uri("/api/connection/datasource/import_community") + .header("content-type", "application/json") + .body(Body::from(document.to_string())) + .expect("request builds"), + ) + .await + .expect("route responds"); + let body: serde_json::Value = serde_json::from_slice( + &response + .into_body() + .collect() + .await + .expect("body collects") + .to_bytes(), + ) + .expect("response decodes"); + assert_eq!(body["success"], true, "{body}"); + assert_eq!(body["data"]["count"], 0, "{body}"); + } + + #[test] + fn legacy_sql_helpers_preserve_utf16_positions_and_frontend_field_names() { + assert_eq!( + legacy_mysql_utility_database_type("mysql"), + Ok("MYSQL".to_owned()) + ); + assert!(legacy_mysql_utility_database_type("postgresql").is_err()); + + let sql = "SELECT '🙂';\nSELECT 2"; + let second = sql.find("SELECT 2").expect("second statement must exist"); + assert_eq!(utf16_line_column(sql, second), (2, 1)); + assert_eq!(utf16_len("a🙂"), 3); + assert_eq!(locate_statement(sql, "SELECT 2", 0), (second, sql.len())); + + let mut candidate = serde_json::Map::from_iter([ + ("replaceStartUtf16".to_owned(), serde_json::json!(2)), + ("replaceEndUtf16".to_owned(), serde_json::json!(4)), + ]); + rename_json_field(&mut candidate, "replaceStartUtf16", "replaceStart"); + rename_json_field(&mut candidate, "replaceEndUtf16", "replaceEnd"); + assert_eq!(candidate["replaceStart"], 2); + assert_eq!(candidate["replaceEnd"], 4); + assert!(!candidate.contains_key("replaceStartUtf16")); + assert!(!candidate.contains_key("replaceEndUtf16")); + } + #[test] fn routine_invocation_payload_accepts_numeric_and_text_datasource_ids() { let numeric: LegacyRoutineInvocationRequest = serde_json::from_value(serde_json::json!({ diff --git a/apps/chat2db-web/src/legacy_ai.rs b/apps/chat2db-web/src/legacy_ai.rs new file mode 100644 index 0000000..a7e9387 --- /dev/null +++ b/apps/chat2db-web/src/legacy_ai.rs @@ -0,0 +1,2455 @@ +//! Compatibility facade for the retained Community AI workbench. + +use std::{ + collections::BTreeMap, + convert::Infallible, + io::{Cursor, Read as _}, + path::{Path, PathBuf}, + time::Duration, +}; + +use axum::{ + Json, Router, + extract::{DefaultBodyLimit, Multipart, Query, State}, + response::{IntoResponse, Response, sse::Event, sse::KeepAlive, sse::Sse}, + routing::{get, post}, +}; +use chat2db_contract::{ + AgentEvent, AgentEventEnvelope, AgentMessage, AgentMessageContent, AgentMessageRole, + AgentPermissionDecision, AgentToolOutput, CreateAgentSessionRequest, + CreateProviderProfileRequest, DecideAgentPermissionRequest, ProviderCredentials, ProviderKind, + ProviderProfile, ProviderSecretChange, SqlPermissionMode, StartAgentRunRequest, + UpdateAgentSessionRequest, UpdateProviderProfileRequest, +}; +use chat2db_core::{AgentRunSubscription, Application}; +use chrono::{TimeZone as _, Utc}; +use futures_util::{Stream, stream}; +use quick_xml::{Reader as XmlReader, events::Event as XmlEvent}; +use serde::{Deserialize, Serialize}; +use zip::ZipArchive; + +const DEFAULT_CONTEXT_WINDOW_TOKENS: &str = "128000"; +const DEFAULT_MAX_OUTPUT_TOKENS: &str = "4096"; +const LEGACY_MESSAGE_PAGE_SIZE: &str = "512"; +const SSE_KEEP_ALIVE_SECONDS: u64 = 15; +const MAX_ATTACHMENT_FILE_BYTES: usize = 32 * 1024 * 1024; +const MAX_ATTACHMENT_CONTENT_CHARS: usize = 12_000; +const MAX_ATTACHMENT_CONTEXT_CHARS: usize = 24_000; +const MAX_SHEET_ROWS: usize = 100; +const MAX_SHEET_COLUMNS: usize = 20; + +/// Community's historical AI chat request. +#[derive(Debug, Clone, Default, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacyAiChatRequest { + #[serde(default)] + pub input: String, + #[serde(default)] + pub session_id: Option, + #[serde(default)] + pub data_source_id: Option, + #[serde(default)] + pub database_name: Option, + #[serde(default)] + pub schema_name: Option, + #[serde(default)] + pub system_prompt: Option, + #[serde(default)] + pub model_config_id: Option, + #[serde(default)] + pub provider: Option, + #[serde(default)] + pub model: Option, + #[serde(default)] + pub api_key: Option, + #[serde(default)] + pub base_url: Option, + #[serde(default)] + pub project_id: Option, + #[serde(default)] + pub location: Option, + #[serde(default)] + pub temperature: Option, + #[serde(default)] + pub max_tokens: Option, + #[serde(default)] + pub attachments: Vec, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacyAiAttachment { + pub file_name: String, + pub file_type: String, + pub content_category: String, + pub content: String, + #[serde(default)] + pub content_length: Option, + #[serde(default)] + pub truncated: Option, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacyAiLocalAttachmentRequest { + pub file_path: String, + #[serde(default)] + pub file_name: Option, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacyAiModelConfigRequest { + #[serde(default)] + pub id: Option, + #[serde(default)] + pub name: String, + pub provider: String, + pub model: String, + #[serde(default)] + pub api_key: Option, + #[serde(default)] + pub base_url: Option, + #[serde(default)] + pub project_id: Option, + #[serde(default)] + pub location: Option, + #[serde(default)] + pub temperature: Option, + #[serde(default)] + pub max_tokens: Option, + #[serde(default)] + pub enabled: Option, + #[serde(default)] + pub default_config: Option, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacyAiModelConfigDeleteRequest { + pub id: String, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacyAiModelConfig { + pub id: String, + pub name: String, + pub provider: String, + pub model: String, + pub base_url: String, + pub max_tokens: u64, + pub enabled: bool, + pub default_config: bool, + pub has_api_key: bool, + pub api_key_masked: String, + pub gmt_modified: String, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacyAiModelConfigTestResult { + pub success: bool, + pub message: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub status_code: Option, + pub endpoint: String, +} + +/// Legacy datasource ids can be either numeric or opaque Rust ids. +#[derive(Debug, Clone, Deserialize)] +#[serde(untagged)] +pub enum LegacyAiIdentifier { + Text(String), + Unsigned(u64), + Signed(i64), +} + +impl LegacyAiIdentifier { + fn into_string(self) -> String { + match self { + Self::Text(value) => value, + Self::Unsigned(value) => value.to_string(), + Self::Signed(value) => value.to_string(), + } + } +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacyAiSessionDeleteRequest { + pub id: String, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +struct LegacyAiMessagesQuery { + session_id: String, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacyAiSession { + pub id: String, + pub title: String, + pub gmt_create: String, + pub gmt_modified: String, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacyAiMessage { + pub id: String, + pub session_id: String, + pub role: String, + pub content: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub reasoning_content: Option, + pub gmt_create: String, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacyAiModelOption { + pub value: String, + pub label: String, + pub provider: String, + pub model: String, + pub model_config_id: String, + pub custom_option: bool, + pub default_option: bool, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacyAiModelCatalogItem { + pub provider: String, + pub models: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LegacyAiStreamChunk { + #[serde(rename = "type")] + pub event_type: String, + pub message_type: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub content: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub name: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub arguments: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub session_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub ts: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub error_code: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub error_message: Option, +} + +impl LegacyAiStreamChunk { + #[must_use] + pub fn event_name(&self) -> &str { + &self.event_type + } + + fn session(session_id: String) -> Self { + Self::new("session", current_epoch_millis()).with_session(session_id) + } + + fn answer(content: String, ts: u64) -> Self { + Self::new("answer", ts).with_content(content) + } + + fn done(session_id: String, ts: u64) -> Self { + Self::new("done", ts) + .with_content("[DONE]".to_owned()) + .with_session(session_id) + } + + fn error(code: impl Into, message: impl Into, ts: u64) -> Self { + let code = code.into(); + let message = message.into(); + Self { + error_code: Some(code), + error_message: Some(message.clone()), + ..Self::new("error", ts).with_content(message) + } + } + + fn new(event_type: &str, ts: u64) -> Self { + Self { + event_type: event_type.to_owned(), + message_type: event_type.to_owned(), + content: None, + name: None, + arguments: None, + session_id: None, + ts: Some(ts), + id: None, + error_code: None, + error_message: None, + } + } + + fn with_content(mut self, content: String) -> Self { + self.content = Some(content); + self + } + + fn with_session(mut self, session_id: String) -> Self { + self.session_id = Some(session_id); + self + } +} + +/// A started compatibility run with replay-safe subscription already attached. +pub struct LegacyAiStartedRun { + pub run_id: String, + pub session_id: String, + pub subscription: AgentRunSubscription, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LegacyAiFailure { + pub code: String, + pub message: String, +} + +impl LegacyAiFailure { + fn invalid(code: &str, message: &str) -> Self { + Self { + code: code.to_owned(), + message: message.to_owned(), + } + } +} + +impl From for LegacyAiFailure { + fn from(error: chat2db_core::AppError) -> Self { + let error = error.api_error(); + Self { + code: error.code, + message: error.message, + } + } +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct LegacyAiEnvelope { + success: bool, + data: Option, + error_code: Option, + error_message: Option, +} + +impl LegacyAiEnvelope { + fn success(data: T) -> Self { + Self { + success: true, + data: Some(data), + error_code: None, + error_message: None, + } + } + + fn failure(error: LegacyAiFailure) -> Self { + Self { + success: false, + data: None, + error_code: Some(error.code), + error_message: Some(error.message), + } + } +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct LegacyAiListEnvelope { + success: bool, + data: Option>, + total: Option, + error_code: Option, + error_message: Option, +} + +impl LegacyAiListEnvelope { + fn success(data: Vec) -> Self { + let total = data.len(); + Self { + success: true, + data: Some(data), + total: Some(total), + error_code: None, + error_message: None, + } + } + + fn failure(error: LegacyAiFailure) -> Self { + Self { + success: false, + data: None, + total: None, + error_code: Some(error.code), + error_message: Some(error.message), + } + } +} + +/// Creates or reuses a Community chat session and starts a read-only run. +/// +/// # Errors +/// +/// Returns request validation, provider-profile, storage, or run-start failures. +pub async fn start_chat_run( + application: &Application, + request: LegacyAiChatRequest, +) -> Result { + let input = request.input.trim(); + if input.is_empty() { + return Err(LegacyAiFailure::invalid( + "invalid_ai_request", + "input must not be empty", + )); + } + let message = build_chat_message(input, &request); + + let requested_datasource = request + .data_source_id + .clone() + .map(LegacyAiIdentifier::into_string) + .filter(|value| !value.trim().is_empty()); + let session = if let Some(session_id) = nonempty(request.session_id.as_deref()) { + let current = application.get_agent_session(session_id).await?; + let provider_id = if has_explicit_provider_selection(&request) { + resolve_provider_profile(application, &request).await?.id + } else { + current.provider_id.clone() + }; + let datasource_id = requested_datasource.or_else(|| current.datasource_id.clone()); + if provider_id != current.provider_id || datasource_id != current.datasource_id { + application + .update_agent_session( + ¤t.id, + UpdateAgentSessionRequest { + expected_revision: current.revision, + title: current.title, + provider_id, + datasource_id, + }, + ) + .await? + } else { + current + } + } else { + let provider = resolve_provider_profile(application, &request).await?; + application + .create_agent_session(CreateAgentSessionRequest { + title: bounded_title(input), + provider_id: provider.id, + datasource_id: requested_datasource, + system_prompt: nonempty_owned(request.system_prompt), + }) + .await? + }; + + let accepted = application + .start_agent_run(StartAgentRunRequest { + session_id: session.id.clone(), + message, + sql_permission_mode: SqlPermissionMode::ReadOnly, + }) + .await?; + let subscription = application + .subscribe_agent_run(&accepted.run_id, None) + .await?; + Ok(LegacyAiStartedRun { + run_id: accepted.run_id, + session_id: session.id, + subscription, + }) +} + +/// Returns the next frontend-visible compatibility event. +pub async fn next_stream_chunk( + application: &Application, + subscription: &mut AgentRunSubscription, + session_id: &str, +) -> Option<(LegacyAiStreamChunk, bool)> { + loop { + let envelope = match subscription.next_event().await { + Ok(Some(envelope)) => envelope, + Ok(None) => return None, + Err(error) => { + let error = error.api_error(); + return Some(( + LegacyAiStreamChunk::error(error.code, error.message, current_epoch_millis()), + true, + )); + } + }; + if let AgentEvent::PermissionRequested { permission } = &envelope.event { + let denial = application + .decide_agent_permission( + &permission.permission_id, + DecideAgentPermissionRequest { + run_id: permission.run_id.clone(), + tool_call_id: permission.tool_call_id.clone(), + decision: AgentPermissionDecision::Deny, + arguments_sha256: permission.arguments_sha256.clone(), + }, + ) + .await; + let message = if denial.is_ok() { + "The Community compatibility facade rejected a write permission request" + } else { + "The Community compatibility facade could not safely resolve a write permission request" + }; + return Some(( + LegacyAiStreamChunk::error( + "agent_write_permission_denied", + message, + event_timestamp(&envelope), + ), + true, + )); + } + if let Some(projected) = project_agent_event(&envelope, session_id) { + let terminal = matches!( + envelope.event, + AgentEvent::Completed { .. } + | AgentEvent::Failed { .. } + | AgentEvent::Cancelled { .. } + | AgentEvent::ToolFailed { .. } + ); + return Some((projected, terminal)); + } + } +} + +/// Projects a canonical event without exposing private model reasoning. +#[must_use] +pub fn project_agent_event( + envelope: &AgentEventEnvelope, + session_id: &str, +) -> Option { + let ts = event_timestamp(envelope); + match &envelope.event { + AgentEvent::Started + | AgentEvent::PermissionResolved { .. } + | AgentEvent::ContextCompacted { .. } + | AgentEvent::Usage { .. } => None, + AgentEvent::TextDelta { delta } => Some(LegacyAiStreamChunk::answer(delta.clone(), ts)), + AgentEvent::ToolStarted { + tool_call_id, + name, + arguments_sha256, + } => Some(LegacyAiStreamChunk { + name: Some(name.clone()), + arguments: Some(serde_json::json!({ "argumentsSha256": arguments_sha256 }).to_string()), + id: Some(tool_call_id.clone()), + ..LegacyAiStreamChunk::new("tool_call", ts) + }), + AgentEvent::ToolCompleted { + tool_call_id, + name, + output, + } => Some(LegacyAiStreamChunk { + content: Some(tool_output_json(output)), + name: Some(name.clone()), + id: Some(tool_call_id.clone()), + ..LegacyAiStreamChunk::new("tool_result", ts) + }), + AgentEvent::ToolFailed { error, .. } | AgentEvent::Failed { error } => Some( + LegacyAiStreamChunk::error(error.code.clone(), error.message.clone(), ts), + ), + AgentEvent::PermissionRequested { .. } => Some(LegacyAiStreamChunk::error( + "agent_write_permission_denied", + "The Community compatibility facade rejected a write permission request", + ts, + )), + AgentEvent::Completed { .. } => Some(LegacyAiStreamChunk::done(session_id.to_owned(), ts)), + AgentEvent::Cancelled { reason } => Some(LegacyAiStreamChunk::error( + "agent_run_cancelled", + reason + .clone() + .unwrap_or_else(|| "The AI run was cancelled".to_owned()), + ts, + )), + } +} + +/// Lists durable sessions in the shape consumed by the retained frontend. +/// +/// # Errors +/// +/// Returns storage failures while loading the durable session catalog. +pub async fn list_sessions( + application: &Application, +) -> Result, LegacyAiFailure> { + Ok(application + .list_agent_sessions() + .await? + .items + .into_iter() + .map(|session| LegacyAiSession { + id: session.id, + title: session.title, + gmt_create: legacy_timestamp(&session.created_at_ms), + gmt_modified: legacy_timestamp(&session.updated_at_ms), + }) + .collect()) +} + +/// Lists the complete visible transcript for one Community session. +/// +/// # Errors +/// +/// Returns validation, storage, or transcript pagination failures. +pub async fn list_messages( + application: &Application, + session_id: &str, +) -> Result, LegacyAiFailure> { + let session_id = session_id.trim(); + if session_id.is_empty() { + return Err(LegacyAiFailure::invalid( + "invalid_ai_request", + "sessionId must not be empty", + )); + } + let mut start_ordinal = "0".to_owned(); + let mut messages = Vec::new(); + loop { + let page = application + .list_agent_messages(session_id, &start_ordinal, LEGACY_MESSAGE_PAGE_SIZE) + .await?; + let next_ordinal = page + .items + .last() + .and_then(|message| message.ordinal.parse::().ok()) + .and_then(|ordinal| ordinal.checked_add(1)); + messages.extend(page.items.into_iter().filter_map(project_message)); + if !page.has_more { + break; + } + let Some(next_ordinal) = next_ordinal else { + return Err(LegacyAiFailure::invalid( + "agent_message_ordinal_invalid", + "The AI transcript contains an invalid message ordinal", + )); + }; + start_ordinal = next_ordinal.to_string(); + } + Ok(messages) +} + +/// Deletes one durable Community AI session. +/// +/// # Errors +/// +/// Returns lookup, revision, or storage failures. +pub async fn delete_session(application: &Application, id: &str) -> Result<(), LegacyAiFailure> { + let session = application.get_agent_session(id.trim()).await?; + application + .delete_agent_session(&session.id, &session.revision) + .await?; + Ok(()) +} + +/// Lists provider profiles that have usable credentials as frontend model options. +/// +/// # Errors +/// +/// Returns storage failures while loading provider profiles. +pub async fn model_options( + application: &Application, +) -> Result, LegacyAiFailure> { + Ok(application + .list_provider_profiles() + .await? + .items + .into_iter() + .filter(|profile| profile.has_secret) + .enumerate() + .map(|(index, profile)| LegacyAiModelOption { + value: format!("config:{}", profile.id), + label: profile.name, + provider: legacy_provider_name(profile.kind).to_owned(), + model: profile.model, + model_config_id: profile.id, + custom_option: true, + default_option: index == 0, + }) + .collect()) +} + +/// Builds the provider/model catalog represented by saved profiles. +/// +/// # Errors +/// +/// Returns storage failures while loading provider profiles. +pub async fn model_catalog( + application: &Application, +) -> Result, LegacyAiFailure> { + let mut catalog = BTreeMap::>::new(); + for profile in application.list_provider_profiles().await?.items { + let models = catalog + .entry(legacy_provider_name(profile.kind).to_owned()) + .or_default(); + if !models.contains(&profile.model) { + models.push(profile.model); + } + } + Ok(catalog + .into_iter() + .map(|(provider, models)| LegacyAiModelCatalogItem { provider, models }) + .collect()) +} + +/// Lists secret-free model configurations for the retained settings UI. +/// +/// # Errors +/// +/// Returns storage failures while loading provider profiles. +pub async fn list_model_configs( + application: &Application, +) -> Result, LegacyAiFailure> { + Ok(application + .list_provider_profiles() + .await? + .items + .into_iter() + .enumerate() + .map(|(index, profile)| model_config_projection(profile, index == 0)) + .collect()) +} + +/// Creates or updates one model configuration without exposing its credential. +/// +/// # Errors +/// +/// Returns validation, revision, vault, or storage failures. +pub async fn save_model_config( + application: &Application, + request: LegacyAiModelConfigRequest, +) -> Result { + let kind = parse_provider_kind(&request.provider)?; + let model = nonempty(Some(&request.model)) + .ok_or_else(|| LegacyAiFailure::invalid("invalid_ai_model", "model must not be empty"))?; + let name = nonempty(Some(&request.name)).unwrap_or(model).to_owned(); + let max_output_tokens = request.max_tokens.unwrap_or(4096); + if max_output_tokens == 0 { + return Err(LegacyAiFailure::invalid( + "invalid_ai_model", + "maxTokens must be greater than zero", + )); + } + + let profile = if let Some(id) = nonempty(request.id.as_deref()) { + let current = application.get_provider_profile(id).await?; + let base_url = nonempty(request.base_url.as_deref()) + .map(ToOwned::to_owned) + .unwrap_or(current.base_url); + let secret_change = match nonempty(request.api_key.as_deref()) { + Some(api_key) => ProviderSecretChange::Replace { + credentials: ProviderCredentials { + api_key: api_key.to_owned(), + }, + }, + None => ProviderSecretChange::Keep, + }; + application + .update_provider_profile( + id, + UpdateProviderProfileRequest { + expected_revision: current.revision, + name, + kind, + base_url, + model: model.to_owned(), + context_window_tokens: current.context_window_tokens, + max_output_tokens: max_output_tokens.to_string(), + secret_change, + }, + ) + .await? + } else { + let api_key = nonempty(request.api_key.as_deref()).ok_or_else(|| { + LegacyAiFailure::invalid( + "provider_credentials_missing", + "API Key is required when creating an AI model configuration", + ) + })?; + application + .create_provider_profile(CreateProviderProfileRequest { + name, + kind, + base_url: nonempty(request.base_url.as_deref()) + .map_or_else(|| default_base_url(kind).to_owned(), ToOwned::to_owned), + model: model.to_owned(), + context_window_tokens: DEFAULT_CONTEXT_WINDOW_TOKENS.to_owned(), + max_output_tokens: max_output_tokens.to_string(), + credentials: Some(ProviderCredentials { + api_key: api_key.to_owned(), + }), + }) + .await? + }; + let default_config = request.default_config.unwrap_or(false) + || application.list_provider_profiles().await?.items.len() == 1; + Ok(model_config_projection(profile, default_config)) +} + +/// Deletes one saved model configuration. +/// +/// # Errors +/// +/// Returns lookup, revision, vault, or storage failures. +pub async fn delete_model_config( + application: &Application, + id: &str, +) -> Result<(), LegacyAiFailure> { + let profile = application.get_provider_profile(id.trim()).await?; + application + .delete_provider_profile(&profile.id, &profile.revision) + .await?; + Ok(()) +} + +pub async fn test_model_config( + request: &LegacyAiModelConfigRequest, +) -> LegacyAiModelConfigTestResult { + let kind = match parse_provider_kind(&request.provider) { + Ok(kind) => kind, + Err(error) => { + return LegacyAiModelConfigTestResult { + success: false, + message: error.message, + status_code: None, + endpoint: String::new(), + }; + } + }; + if kind != ProviderKind::OpenAiCompatible { + return LegacyAiModelConfigTestResult { + success: false, + message: "Connection test currently supports OpenAI-compatible models only.".to_owned(), + status_code: None, + endpoint: String::new(), + }; + } + let base_url = nonempty(request.base_url.as_deref()).unwrap_or_else(|| default_base_url(kind)); + let endpoint = format!("{}/chat/completions", normalized_url(base_url)); + let Some(api_key) = nonempty(request.api_key.as_deref()) else { + return LegacyAiModelConfigTestResult { + success: false, + message: "API Key is required for the connection test.".to_owned(), + status_code: None, + endpoint, + }; + }; + let Some(model) = nonempty(Some(&request.model)) else { + return LegacyAiModelConfigTestResult { + success: false, + message: "model must not be empty".to_owned(), + status_code: None, + endpoint, + }; + }; + let client = match reqwest::Client::builder() + .connect_timeout(Duration::from_secs(10)) + .timeout(Duration::from_secs(20)) + .build() + { + Ok(client) => client, + Err(error) => { + return LegacyAiModelConfigTestResult { + success: false, + message: bounded_error_message(&error.to_string()), + status_code: None, + endpoint, + }; + } + }; + let mut payload = serde_json::json!({ + "model": model, + "messages": [{"role": "user", "content": "ping"}], + "max_tokens": 1, + }); + if let Some(temperature) = request.temperature.filter(|value| value.is_finite()) { + payload["temperature"] = serde_json::json!(temperature); + } + match client + .post(&endpoint) + .bearer_auth(api_key) + .json(&payload) + .send() + .await + { + Ok(response) if response.status().is_success() => LegacyAiModelConfigTestResult { + success: true, + message: "Connection test passed".to_owned(), + status_code: Some(response.status().as_u16()), + endpoint, + }, + Ok(response) => { + let status_code = response.status().as_u16(); + let message = response + .text() + .await + .unwrap_or_else(|error| error.to_string()); + LegacyAiModelConfigTestResult { + success: false, + message: bounded_error_message(&message), + status_code: Some(status_code), + endpoint, + } + } + Err(error) => LegacyAiModelConfigTestResult { + success: false, + message: bounded_error_message(&error.to_string()), + status_code: error.status().map(|status| status.as_u16()), + endpoint, + }, + } +} + +/// Reads and parses one attachment selected by its desktop-local path. +/// +/// # Errors +/// +/// Returns path, size, I/O, unsupported-format, or document parsing failures. +pub async fn parse_local_attachment( + request: LegacyAiLocalAttachmentRequest, +) -> Result { + let file_path = nonempty(Some(&request.file_path)).ok_or_else(|| { + LegacyAiFailure::invalid("ai_attachment_path_required", "filePath must not be empty") + })?; + let path = PathBuf::from(file_path); + let metadata = tokio::fs::metadata(&path).await.map_err(|_| { + LegacyAiFailure::invalid( + "ai_attachment_file_not_found", + "The selected attachment does not exist", + ) + })?; + if !metadata.is_file() { + return Err(LegacyAiFailure::invalid( + "ai_attachment_file_not_found", + "The selected attachment is not a file", + )); + } + if metadata.len() > MAX_ATTACHMENT_FILE_BYTES as u64 { + return Err(LegacyAiFailure::invalid( + "ai_attachment_too_large", + "Attachments are limited to 32 MiB", + )); + } + let bytes = tokio::fs::read(&path).await.map_err(|_| { + LegacyAiFailure::invalid( + "ai_attachment_read_failed", + "The selected attachment could not be read", + ) + })?; + let file_name = nonempty(request.file_name.as_deref()) + .map(ToOwned::to_owned) + .or_else(|| { + path.file_name() + .and_then(|name| name.to_str()) + .map(ToOwned::to_owned) + }) + .unwrap_or_else(|| "attachment".to_owned()); + parse_attachment_bytes(&file_name, &bytes) +} + +/// Parses bounded attachment bytes into the historical frontend shape. +/// +/// # Errors +/// +/// Returns size, unsupported-format, empty-content, or document parsing failures. +pub fn parse_attachment_bytes( + file_name: &str, + bytes: &[u8], +) -> Result { + if bytes.len() > MAX_ATTACHMENT_FILE_BYTES { + return Err(LegacyAiFailure::invalid( + "ai_attachment_too_large", + "Attachments are limited to 32 MiB", + )); + } + let extension = Path::new(file_name) + .extension() + .and_then(|value| value.to_str()) + .map(str::to_ascii_lowercase) + .ok_or_else(|| { + LegacyAiFailure::invalid( + "ai_attachment_unsupported", + "The attachment must have a supported file extension", + ) + })?; + let raw_content = match extension.as_str() { + "md" | "txt" | "json" => String::from_utf8_lossy(bytes).into_owned(), + "csv" => parse_csv_attachment(bytes), + "docx" => parse_docx_attachment(bytes)?, + "xls" | "xlsx" => parse_workbook_attachment(bytes, &extension)?, + "pdf" | "doc" => extract_binary_text(bytes), + _ => { + return Err(LegacyAiFailure::invalid( + "ai_attachment_unsupported", + "Supported attachments are PDF, DOC, DOCX, MD, TXT, JSON, CSV, XLS, and XLSX", + )); + } + }; + let normalized = normalize_attachment_text(&raw_content); + if normalized.is_empty() { + return Err(LegacyAiFailure::invalid( + "ai_attachment_empty", + "The attachment does not contain readable text", + )); + } + let content_length = normalized.chars().count(); + let truncated = content_length > MAX_ATTACHMENT_CONTENT_CHARS; + let content = normalized + .chars() + .take(MAX_ATTACHMENT_CONTENT_CHARS) + .collect(); + Ok(LegacyAiAttachment { + file_name: file_name.to_owned(), + file_type: extension.clone(), + content_category: if matches!(extension.as_str(), "csv" | "xls" | "xlsx") { + "TABULAR" + } else { + "DOCUMENT" + } + .to_owned(), + content, + content_length: Some(content_length), + truncated: Some(truncated), + }) +} + +/// Handles non-streaming legacy AI requests for the Tauri `javaQuery` bridge. +#[allow(clippy::too_many_lines)] +pub async fn dispatch( + application: &Application, + method: &str, + request_url: &str, + message: serde_json::Value, +) -> Option { + let path = request_url.split('?').next().unwrap_or(request_url); + let method = method.to_ascii_lowercase(); + match (method.as_str(), path) { + ("get", "/api/v3/ai/chat/history/sessions") => Some( + serde_json::to_value(match list_sessions(application).await { + Ok(data) => LegacyAiListEnvelope::success(data), + Err(error) => LegacyAiListEnvelope::failure(error), + }) + .unwrap_or_else(|_| internal_failure_value()), + ), + ("get", "/api/v3/ai/chat/history/messages") => { + let session_id = message + .get("sessionId") + .and_then(serde_json::Value::as_str) + .or_else(|| query_value(request_url, "sessionId")); + let result = match session_id { + Some(session_id) => list_messages(application, session_id).await, + None => Err(LegacyAiFailure::invalid( + "invalid_ai_request", + "sessionId must not be empty", + )), + }; + Some( + serde_json::to_value(match result { + Ok(data) => LegacyAiListEnvelope::success(data), + Err(error) => LegacyAiListEnvelope::failure(error), + }) + .unwrap_or_else(|_| internal_failure_value()), + ) + } + ("post", "/api/v3/ai/chat/history/session/delete") => { + let request = serde_json::from_value::(message); + let result = match request { + Ok(request) => delete_session(application, &request.id).await, + Err(_) => Err(LegacyAiFailure::invalid( + "invalid_ai_request", + "id must not be empty", + )), + }; + Some( + serde_json::to_value(match result { + Ok(()) => LegacyAiEnvelope::success(()), + Err(error) => LegacyAiEnvelope::failure(error), + }) + .unwrap_or_else(|_| internal_failure_value()), + ) + } + ("get", "/api/v3/ai/model/options") => Some( + serde_json::to_value(match model_options(application).await { + Ok(data) => LegacyAiEnvelope::success(data), + Err(error) => LegacyAiEnvelope::failure(error), + }) + .unwrap_or_else(|_| internal_failure_value()), + ), + ("get", "/api/v3/ai/model/list") => Some( + serde_json::to_value(match model_catalog(application).await { + Ok(data) => LegacyAiEnvelope::success(data), + Err(error) => LegacyAiEnvelope::failure(error), + }) + .unwrap_or_else(|_| internal_failure_value()), + ), + ("get", "/api/v3/ai/model/config/list") => Some( + serde_json::to_value(match list_model_configs(application).await { + Ok(data) => LegacyAiEnvelope::success(data), + Err(error) => LegacyAiEnvelope::failure(error), + }) + .unwrap_or_else(|_| internal_failure_value()), + ), + ("post", "/api/v3/ai/model/config/save") => { + let result = match serde_json::from_value::(message) { + Ok(request) => save_model_config(application, request).await, + Err(_) => Err(LegacyAiFailure::invalid( + "invalid_ai_request", + "The AI model configuration is invalid", + )), + }; + Some( + serde_json::to_value(match result { + Ok(data) => LegacyAiEnvelope::success(data), + Err(error) => LegacyAiEnvelope::failure(error), + }) + .unwrap_or_else(|_| internal_failure_value()), + ) + } + ("post", "/api/v3/ai/model/config/delete") => { + let result = match serde_json::from_value::(message) { + Ok(request) => delete_model_config(application, &request.id).await, + Err(_) => Err(LegacyAiFailure::invalid( + "invalid_ai_request", + "id must not be empty", + )), + }; + Some( + serde_json::to_value(match result { + Ok(()) => LegacyAiEnvelope::success(()), + Err(error) => LegacyAiEnvelope::failure(error), + }) + .unwrap_or_else(|_| internal_failure_value()), + ) + } + ("post", "/api/v3/ai/model/config/test") => { + let result = match serde_json::from_value::(message) { + Ok(request) => LegacyAiEnvelope::success(test_model_config(&request).await), + Err(_) => LegacyAiEnvelope::failure(LegacyAiFailure::invalid( + "invalid_ai_request", + "The AI model configuration is invalid", + )), + }; + Some(serde_json::to_value(result).unwrap_or_else(|_| internal_failure_value())) + } + ("post", "/api/v3/ai/chat/attachment/parse/local") => { + let result = match serde_json::from_value::(message) { + Ok(request) => parse_local_attachment(request).await, + Err(_) => Err(LegacyAiFailure::invalid( + "invalid_ai_request", + "filePath must not be empty", + )), + }; + Some( + serde_json::to_value(match result { + Ok(data) => LegacyAiEnvelope::success(data), + Err(error) => LegacyAiEnvelope::failure(error), + }) + .unwrap_or_else(|_| internal_failure_value()), + ) + } + _ => None, + } +} + +pub(crate) fn routes() -> Router { + Router::new() + .route("/api/v3/ai/chat/stream", post(chat_stream_handler)) + .route( + "/api/v3/ai/chat/history/sessions", + get(list_sessions_handler), + ) + .route( + "/api/v3/ai/chat/history/messages", + get(list_messages_handler), + ) + .route( + "/api/v3/ai/chat/history/session/delete", + post(delete_session_handler), + ) + .route("/api/v3/ai/model/options", get(model_options_handler)) + .route("/api/v3/ai/model/list", get(model_catalog_handler)) + .route( + "/api/v3/ai/model/config/list", + get(list_model_configs_handler), + ) + .route( + "/api/v3/ai/model/config/save", + post(save_model_config_handler), + ) + .route( + "/api/v3/ai/model/config/delete", + post(delete_model_config_handler), + ) + .route( + "/api/v3/ai/model/config/test", + post(test_model_config_handler), + ) + .route( + "/api/v3/ai/chat/attachment/parse/upload", + post(parse_uploaded_attachment_handler) + .layer(DefaultBodyLimit::max(MAX_ATTACHMENT_FILE_BYTES)), + ) +} + +async fn chat_stream_handler( + State(application): State, + Json(request): Json, +) -> Response { + let started = match start_chat_run(&application, request).await { + Ok(started) => started, + Err(error) => return Json(LegacyAiEnvelope::<()>::failure(error)).into_response(), + }; + let events = legacy_ai_stream(application, started); + Sse::new(events) + .keep_alive( + KeepAlive::new() + .interval(Duration::from_secs(SSE_KEEP_ALIVE_SECONDS)) + .text("keep-alive"), + ) + .into_response() +} + +fn legacy_ai_stream( + application: Application, + started: LegacyAiStartedRun, +) -> impl Stream> { + struct StateData { + application: Application, + subscription: AgentRunSubscription, + session_id: String, + initial: Option, + finished: bool, + } + + stream::unfold( + StateData { + application, + subscription: started.subscription, + session_id: started.session_id.clone(), + initial: Some(LegacyAiStreamChunk::session(started.session_id)), + finished: false, + }, + |mut state| async move { + if state.finished { + return None; + } + if let Some(chunk) = state.initial.take() { + return Some((Ok(sse_event(&chunk)), state)); + } + let (chunk, terminal) = next_stream_chunk( + &state.application, + &mut state.subscription, + &state.session_id, + ) + .await?; + state.finished = terminal; + Some((Ok(sse_event(&chunk)), state)) + }, + ) +} + +fn sse_event(chunk: &LegacyAiStreamChunk) -> Event { + let data = serde_json::to_string(chunk).unwrap_or_else(|_| { + r#"{"type":"error","messageType":"error","content":"AI event serialization failed"}"# + .to_owned() + }); + Event::default().event(chunk.event_name()).data(data) +} + +async fn list_sessions_handler(State(application): State) -> Json { + Json( + serde_json::to_value(match list_sessions(&application).await { + Ok(data) => LegacyAiListEnvelope::success(data), + Err(error) => LegacyAiListEnvelope::failure(error), + }) + .unwrap_or_else(|_| internal_failure_value()), + ) +} + +async fn list_messages_handler( + State(application): State, + Query(query): Query, +) -> Json { + Json( + serde_json::to_value(match list_messages(&application, &query.session_id).await { + Ok(data) => LegacyAiListEnvelope::success(data), + Err(error) => LegacyAiListEnvelope::failure(error), + }) + .unwrap_or_else(|_| internal_failure_value()), + ) +} + +async fn delete_session_handler( + State(application): State, + Json(request): Json, +) -> Json { + Json( + serde_json::to_value(match delete_session(&application, &request.id).await { + Ok(()) => LegacyAiEnvelope::success(()), + Err(error) => LegacyAiEnvelope::failure(error), + }) + .unwrap_or_else(|_| internal_failure_value()), + ) +} + +async fn model_options_handler(State(application): State) -> Json { + Json( + serde_json::to_value(match model_options(&application).await { + Ok(data) => LegacyAiEnvelope::success(data), + Err(error) => LegacyAiEnvelope::failure(error), + }) + .unwrap_or_else(|_| internal_failure_value()), + ) +} + +async fn model_catalog_handler(State(application): State) -> Json { + Json( + serde_json::to_value(match model_catalog(&application).await { + Ok(data) => LegacyAiEnvelope::success(data), + Err(error) => LegacyAiEnvelope::failure(error), + }) + .unwrap_or_else(|_| internal_failure_value()), + ) +} + +async fn list_model_configs_handler( + State(application): State, +) -> Json { + Json( + serde_json::to_value(match list_model_configs(&application).await { + Ok(data) => LegacyAiEnvelope::success(data), + Err(error) => LegacyAiEnvelope::failure(error), + }) + .unwrap_or_else(|_| internal_failure_value()), + ) +} + +async fn save_model_config_handler( + State(application): State, + Json(request): Json, +) -> Json { + Json( + serde_json::to_value(match save_model_config(&application, request).await { + Ok(data) => LegacyAiEnvelope::success(data), + Err(error) => LegacyAiEnvelope::failure(error), + }) + .unwrap_or_else(|_| internal_failure_value()), + ) +} + +async fn delete_model_config_handler( + State(application): State, + Json(request): Json, +) -> Json { + Json( + serde_json::to_value(match delete_model_config(&application, &request.id).await { + Ok(()) => LegacyAiEnvelope::success(()), + Err(error) => LegacyAiEnvelope::failure(error), + }) + .unwrap_or_else(|_| internal_failure_value()), + ) +} + +async fn test_model_config_handler( + Json(request): Json, +) -> Json> { + Json(LegacyAiEnvelope::success(test_model_config(&request).await)) +} + +async fn parse_uploaded_attachment_handler(mut multipart: Multipart) -> Json { + let result = async { + while let Some(field) = multipart.next_field().await.map_err(|_| { + LegacyAiFailure::invalid( + "ai_attachment_upload_invalid", + "The attachment upload is invalid", + ) + })? { + if field.name() != Some("file") { + continue; + } + let file_name = field + .file_name() + .map_or_else(|| "attachment".to_owned(), ToOwned::to_owned); + let bytes = field.bytes().await.map_err(|_| { + LegacyAiFailure::invalid( + "ai_attachment_upload_invalid", + "The uploaded attachment could not be read", + ) + })?; + return parse_attachment_bytes(&file_name, &bytes); + } + Err(LegacyAiFailure::invalid( + "ai_attachment_upload_invalid", + "The multipart request must include a file field", + )) + } + .await; + Json( + serde_json::to_value(match result { + Ok(data) => LegacyAiEnvelope::success(data), + Err(error) => LegacyAiEnvelope::failure(error), + }) + .unwrap_or_else(|_| internal_failure_value()), + ) +} + +#[allow(clippy::too_many_lines)] +async fn resolve_provider_profile( + application: &Application, + request: &LegacyAiChatRequest, +) -> Result { + let profiles = application.list_provider_profiles().await?.items; + if let Some(profile_id) = nonempty(request.model_config_id.as_deref()) { + let profile_id = profile_id.strip_prefix("config:").unwrap_or(profile_id); + let profile = profiles + .into_iter() + .find(|profile| profile.id == profile_id) + .ok_or_else(|| { + LegacyAiFailure::invalid( + "provider_not_found", + "The selected AI model configuration does not exist", + ) + })?; + if !profile.has_secret { + return Err(LegacyAiFailure::invalid( + "provider_credentials_missing", + "The selected AI model configuration does not have an API key", + )); + } + return Ok(profile); + } + + let requested_kind = request + .provider + .as_deref() + .map(parse_provider_kind) + .transpose()?; + let requested_model = nonempty(request.model.as_deref()); + let requested_base_url = nonempty(request.base_url.as_deref()); + let mut matching = profiles.into_iter().find(|profile| { + profile.has_secret + && requested_kind.is_none_or(|kind| profile.kind == kind) + && requested_model.is_none_or(|model| profile.model == model) + && requested_base_url.is_none_or(|base_url| { + normalized_url(&profile.base_url) == normalized_url(base_url) + }) + }); + + if let Some(api_key) = nonempty(request.api_key.as_deref()) { + if let Some(profile) = matching.take() { + return application + .update_provider_profile( + &profile.id, + UpdateProviderProfileRequest { + expected_revision: profile.revision, + name: profile.name, + kind: profile.kind, + base_url: profile.base_url, + model: profile.model, + context_window_tokens: profile.context_window_tokens, + max_output_tokens: request + .max_tokens + .map_or(profile.max_output_tokens, |tokens| tokens.to_string()), + secret_change: ProviderSecretChange::Replace { + credentials: ProviderCredentials { + api_key: api_key.to_owned(), + }, + }, + }, + ) + .await + .map_err(Into::into); + } + let kind = requested_kind.ok_or_else(|| { + LegacyAiFailure::invalid( + "invalid_ai_provider", + "provider is required when creating an AI model configuration", + ) + })?; + let model = requested_model.ok_or_else(|| { + LegacyAiFailure::invalid( + "invalid_ai_model", + "model is required when creating an AI model configuration", + ) + })?; + let base_url = + requested_base_url.map_or_else(|| default_base_url(kind).to_owned(), ToOwned::to_owned); + return application + .create_provider_profile(CreateProviderProfileRequest { + name: format!("Community {} {model}", legacy_provider_name(kind)), + kind, + base_url, + model: model.to_owned(), + context_window_tokens: DEFAULT_CONTEXT_WINDOW_TOKENS.to_owned(), + max_output_tokens: request.max_tokens.map_or_else( + || DEFAULT_MAX_OUTPUT_TOKENS.to_owned(), + |tokens| tokens.to_string(), + ), + credentials: Some(ProviderCredentials { + api_key: api_key.to_owned(), + }), + }) + .await + .map_err(Into::into); + } + + matching.ok_or_else(|| { + LegacyAiFailure::invalid( + "provider_credentials_missing", + "Configure an AI provider with an API key before starting a chat", + ) + }) +} + +fn project_message(message: AgentMessage) -> Option { + let (role, content, reasoning_content) = match message.role { + AgentMessageRole::User => ("user", text_content(&message.content), None), + AgentMessageRole::Assistant => ( + "assistant", + text_content(&message.content), + trace_content(&message.content), + ), + AgentMessageRole::Tool => ("assistant", String::new(), trace_content(&message.content)), + AgentMessageRole::System | AgentMessageRole::Summary => return None, + }; + Some(LegacyAiMessage { + id: message.id, + session_id: message.session_id, + role: role.to_owned(), + content, + reasoning_content, + gmt_create: legacy_timestamp(&message.created_at_ms), + }) +} + +fn model_config_projection(profile: ProviderProfile, default_config: bool) -> LegacyAiModelConfig { + LegacyAiModelConfig { + id: profile.id, + name: profile.name, + provider: legacy_provider_name(profile.kind).to_owned(), + model: profile.model, + base_url: profile.base_url, + max_tokens: profile.max_output_tokens.parse().unwrap_or(4096), + enabled: true, + default_config, + has_api_key: profile.has_secret, + api_key_masked: if profile.has_secret { "****" } else { "" }.to_owned(), + gmt_modified: legacy_timestamp(&profile.updated_at_ms), + } +} + +fn build_chat_message(input: &str, request: &LegacyAiChatRequest) -> String { + let mut message = input.to_owned(); + let database = nonempty(request.database_name.as_deref()); + let schema = nonempty(request.schema_name.as_deref()); + if database.is_some() || schema.is_some() { + message.push_str("\n\nCurrent database context:\n"); + if let Some(database) = database { + message.push_str("- Database: "); + message.push_str(database); + message.push('\n'); + } + if let Some(schema) = schema { + message.push_str("- Schema: "); + message.push_str(schema); + message.push('\n'); + } + } + let mut remaining = MAX_ATTACHMENT_CONTEXT_CHARS; + for (index, attachment) in request.attachments.iter().enumerate() { + if remaining == 0 { + break; + } + let normalized = normalize_attachment_text(&attachment.content); + if normalized.is_empty() { + continue; + } + let content = normalized.chars().take(remaining).collect::(); + let used = content.chars().count(); + remaining = remaining.saturating_sub(used); + message.push_str("\n\n### Attachment "); + message.push_str(&(index + 1).to_string()); + message.push_str("\n- File name: "); + message.push_str(&attachment.file_name); + message.push_str("\n- File type: "); + message.push_str(&attachment.file_type); + message.push_str("\n- Content category: "); + message.push_str(&attachment.content_category); + message.push_str("\n- Truncated: "); + message.push_str( + if attachment.truncated.unwrap_or(false) || used < normalized.chars().count() { + "true" + } else { + "false" + }, + ); + message.push_str("\n```text\n"); + message.push_str(&content); + message.push_str("\n```"); + } + message +} + +fn parse_csv_attachment(bytes: &[u8]) -> String { + let text = String::from_utf8_lossy(bytes); + let lines = text.lines().collect::>(); + let limit = lines.len().min(MAX_SHEET_ROWS + 1); + let mut output = String::from("[CSV]\n"); + for line in &lines[..limit] { + output.push_str(line); + output.push('\n'); + } + if lines.len() > limit { + output.push_str("... omitted "); + output.push_str(&(lines.len() - limit).to_string()); + output.push_str(" more rows"); + } + output +} + +fn parse_docx_attachment(bytes: &[u8]) -> Result { + let mut archive = ZipArchive::new(Cursor::new(bytes)).map_err(|_| { + LegacyAiFailure::invalid( + "ai_attachment_parse_failed", + "The DOCX attachment is invalid", + ) + })?; + let mut document = archive.by_name("word/document.xml").map_err(|_| { + LegacyAiFailure::invalid( + "ai_attachment_parse_failed", + "The DOCX attachment does not contain a document body", + ) + })?; + let mut xml = String::new(); + document.read_to_string(&mut xml).map_err(|_| { + LegacyAiFailure::invalid( + "ai_attachment_parse_failed", + "The DOCX attachment could not be read", + ) + })?; + let mut reader = XmlReader::from_str(&xml); + reader.config_mut().trim_text(false); + let mut output = String::new(); + loop { + match reader.read_event() { + Ok(XmlEvent::Text(text)) => { + let text = text.unescape().map_err(|_| { + LegacyAiFailure::invalid( + "ai_attachment_parse_failed", + "The DOCX attachment contains invalid text", + ) + })?; + output.push_str(&text); + } + Ok(XmlEvent::End(element)) => match xml_local_name(element.name().as_ref()) { + b"p" | b"tr" => output.push('\n'), + b"tc" => output.push('\t'), + _ => {} + }, + Ok(XmlEvent::Eof) => break, + Ok(_) => {} + Err(_) => { + return Err(LegacyAiFailure::invalid( + "ai_attachment_parse_failed", + "The DOCX attachment contains invalid XML", + )); + } + } + } + Ok(output) +} + +fn parse_workbook_attachment(bytes: &[u8], extension: &str) -> Result { + let cursor = Cursor::new(bytes.to_vec()); + let workbook = if extension == "xls" { + xls::core::xls::read(cursor) + } else { + xls::core::xlsx::read(cursor) + } + .map_err(|_| { + LegacyAiFailure::invalid( + "ai_attachment_parse_failed", + "The spreadsheet attachment is invalid", + ) + })?; + let mut output = String::new(); + for (sheet_index, sheet) in workbook.sheets.iter().enumerate() { + output.push_str("[Sheet] "); + output.push_str(&sheet.name); + output.push('\n'); + let (rows, columns) = sheet.dimensions(); + let row_limit = usize::try_from(rows) + .unwrap_or(usize::MAX) + .min(MAX_SHEET_ROWS); + let column_limit = usize::try_from(columns) + .unwrap_or(usize::MAX) + .min(MAX_SHEET_COLUMNS); + for row in 0..row_limit { + output.push_str("Row "); + output.push_str(&(row + 1).to_string()); + output.push_str(": "); + for column in 0..column_limit { + if column > 0 { + output.push_str(" | "); + } + output.push_str(&workbook.display_cell( + sheet_index, + u32::try_from(row).unwrap_or(u32::MAX), + u32::try_from(column).unwrap_or(u32::MAX), + )); + } + if usize::try_from(columns).unwrap_or(usize::MAX) > MAX_SHEET_COLUMNS { + output.push_str(" | ..."); + } + output.push('\n'); + } + if usize::try_from(rows).unwrap_or(usize::MAX) > MAX_SHEET_ROWS { + output.push_str("... omitted "); + output.push_str( + &(usize::try_from(rows).unwrap_or(usize::MAX) - MAX_SHEET_ROWS).to_string(), + ); + output.push_str(" more rows\n"); + } + output.push('\n'); + } + Ok(output) +} + +fn extract_binary_text(bytes: &[u8]) -> String { + let mut output = String::new(); + let mut ascii = Vec::new(); + for &byte in bytes { + if byte == b'\n' || byte == b'\r' || byte == b'\t' || (0x20..=0x7e).contains(&byte) { + ascii.push(byte); + } else { + if ascii.len() >= 4 { + output.push_str(&String::from_utf8_lossy(&ascii)); + output.push('\n'); + } + ascii.clear(); + } + } + if ascii.len() >= 4 { + output.push_str(&String::from_utf8_lossy(&ascii)); + } + let utf16 = bytes + .chunks_exact(2) + .map(|pair| u16::from_le_bytes([pair[0], pair[1]])) + .collect::>(); + let decoded = String::from_utf16_lossy(&utf16); + let readable = decoded + .split(|character: char| character.is_control() && !matches!(character, '\n' | '\r' | '\t')) + .filter(|part| part.chars().count() >= 4) + .collect::>() + .join("\n"); + if !readable.is_empty() { + output.push('\n'); + output.push_str(&readable); + } + output +} + +fn normalize_attachment_text(content: &str) -> String { + let mut output = String::with_capacity(content.len()); + let mut previous_newline = false; + let mut consecutive_newlines = 0_u8; + for character in content.replace("\r\n", "\n").replace('\r', "\n").chars() { + let character = match character { + '\0' => continue, + '\t' | '\u{000b}' | '\u{000c}' => ' ', + other => other, + }; + if character == '\n' { + consecutive_newlines = consecutive_newlines.saturating_add(1); + if consecutive_newlines <= 2 { + output.push(character); + } + previous_newline = true; + } else { + consecutive_newlines = 0; + if !(character == ' ' && previous_newline) { + output.push(character); + } + previous_newline = false; + } + } + output.trim().to_owned() +} + +fn xml_local_name(name: &[u8]) -> &[u8] { + name.rsplit(|byte| *byte == b':').next().unwrap_or(name) +} + +fn bounded_error_message(message: &str) -> String { + message.chars().take(2_000).collect() +} + +fn text_content(content: &[AgentMessageContent]) -> String { + content + .iter() + .filter_map(|block| match block { + AgentMessageContent::Text { text } => Some(text.as_str()), + AgentMessageContent::ToolCalls { .. } | AgentMessageContent::ToolResult { .. } => None, + }) + .collect::>() + .join("") +} + +fn trace_content(content: &[AgentMessageContent]) -> Option { + let events = content + .iter() + .flat_map(|block| match block { + AgentMessageContent::ToolCalls { calls } => calls + .iter() + .map(|call| { + serde_json::json!({ + "type": "tool_call", + "messageType": "tool_call", + "id": call.id, + "name": call.name, + "arguments": call.arguments_json, + }) + }) + .collect::>(), + AgentMessageContent::ToolResult { + tool_call_id, + name, + output, + } => vec![serde_json::json!({ + "type": "tool_result", + "messageType": "tool_result", + "id": tool_call_id, + "name": name, + "content": tool_output_json(output), + })], + AgentMessageContent::Text { .. } => Vec::new(), + }) + .collect::>(); + (!events.is_empty()).then(|| serde_json::Value::Array(events).to_string()) +} + +fn tool_output_json(output: &AgentToolOutput) -> String { + serde_json::to_string(output).unwrap_or_else(|_| { + r#"{"type":"text","content":"Tool result unavailable","truncated":true}"#.to_owned() + }) +} + +fn has_explicit_provider_selection(request: &LegacyAiChatRequest) -> bool { + [ + request.model_config_id.as_deref(), + request.provider.as_deref(), + request.model.as_deref(), + request.api_key.as_deref(), + request.base_url.as_deref(), + ] + .into_iter() + .any(|value| nonempty(value).is_some()) +} + +fn parse_provider_kind(value: &str) -> Result { + match value.trim().to_ascii_uppercase().as_str() { + "OPENAI" | "OPEN_AI" | "OPENAI_COMPATIBLE" | "OPEN_AI_COMPATIBLE" => { + Ok(ProviderKind::OpenAiCompatible) + } + "CLAUDE" | "ANTHROPIC" => Ok(ProviderKind::Anthropic), + "GEMINI" | "GOOGLE" => Ok(ProviderKind::Gemini), + _ => Err(LegacyAiFailure::invalid( + "invalid_ai_provider", + "provider must be OPENAI, CLAUDE, or GEMINI", + )), + } +} + +fn legacy_provider_name(kind: ProviderKind) -> &'static str { + match kind { + ProviderKind::OpenAiCompatible => "OPENAI", + ProviderKind::Anthropic => "CLAUDE", + ProviderKind::Gemini => "GEMINI", + } +} + +fn default_base_url(kind: ProviderKind) -> &'static str { + match kind { + ProviderKind::OpenAiCompatible => "https://api.openai.com/v1", + ProviderKind::Anthropic => "https://api.anthropic.com/v1", + ProviderKind::Gemini => "https://generativelanguage.googleapis.com/v1beta", + } +} + +fn bounded_title(input: &str) -> String { + let title = input.chars().take(80).collect::(); + if title.is_empty() { + "New chat".to_owned() + } else { + title + } +} + +fn nonempty(value: Option<&str>) -> Option<&str> { + value.map(str::trim).filter(|value| !value.is_empty()) +} + +fn nonempty_owned(value: Option) -> Option { + value + .map(|value| value.trim().to_owned()) + .filter(|value| !value.is_empty()) +} + +fn normalized_url(value: &str) -> &str { + value.trim().trim_end_matches('/') +} + +fn event_timestamp(envelope: &AgentEventEnvelope) -> u64 { + envelope + .occurred_at_ms + .parse() + .unwrap_or_else(|_| current_epoch_millis()) +} + +fn current_epoch_millis() -> u64 { + u64::try_from(Utc::now().timestamp_millis()).unwrap_or(0) +} + +fn legacy_timestamp(epoch_millis: &str) -> String { + epoch_millis + .parse::() + .ok() + .and_then(|millis| Utc.timestamp_millis_opt(millis).single()) + .map_or_else(|| epoch_millis.to_owned(), |time| time.to_rfc3339()) +} + +fn query_value<'a>(request_url: &'a str, name: &str) -> Option<&'a str> { + request_url + .split_once('?') + .map(|(_, query)| query) + .into_iter() + .flat_map(|query| query.split('&')) + .find_map(|pair| { + let (key, value) = pair.split_once('=')?; + (key == name).then_some(value) + }) +} + +fn internal_failure_value() -> serde_json::Value { + serde_json::json!({ + "success": false, + "data": null, + "errorCode": "internal_error", + "errorMessage": "The operation could not be completed" + }) +} + +#[cfg(test)] +mod tests { + use std::{ + collections::HashMap, + sync::{Arc, Mutex}, + time::Duration, + }; + + use axum::{ + Router, + body::Body, + http::{Method, Request, StatusCode, header}, + response::IntoResponse, + routing::post, + }; + use chat2db_contract::{ + AgentEvent, AgentEventEnvelope, AgentPermissionRequest, ApiError, + CreateProviderProfileRequest, ProviderKind, + }; + use chat2db_core::Application; + use chat2db_storage::{SecretRef, SecretValue, SecretVault, SecretVaultError, Storage}; + use http_body_util::BodyExt as _; + use tempfile::TempDir; + use tokio::{net::TcpListener, task::JoinHandle, time::timeout}; + use tower::ServiceExt as _; + + use super::{LegacyAiChatRequest, dispatch, project_agent_event, routes, start_chat_run}; + + const MOCK_OPENAI_STREAM: &str = concat!( + "data: {\"choices\":[{\"index\":0,\"delta\":{\"content\":\"hello from mock\"},\"finish_reason\":null}]}\n\n", + "data: {\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\n", + "data: [DONE]\n\n", + ); + + #[derive(Default)] + struct MemoryVault { + values: Mutex>>, + } + + impl SecretVault for MemoryVault { + fn probe(&self) -> Result<(), SecretVaultError> { + Ok(()) + } + + fn create( + &self, + reference: &SecretRef, + value: &SecretValue, + ) -> Result<(), SecretVaultError> { + self.values.lock().expect("vault lock").insert( + reference.as_str().to_owned(), + value.expose_secret().to_vec(), + ); + Ok(()) + } + + fn get(&self, reference: &SecretRef) -> Result, SecretVaultError> { + Ok(self + .values + .lock() + .expect("vault lock") + .get(reference.as_str()) + .cloned() + .map(SecretValue::new)) + } + + fn delete(&self, reference: &SecretRef) -> Result<(), SecretVaultError> { + self.values + .lock() + .expect("vault lock") + .remove(reference.as_str()); + Ok(()) + } + } + + struct TestApplication { + _directory: TempDir, + application: Application, + } + + fn test_application() -> TestApplication { + let directory = TempDir::new().expect("temporary application directory"); + let storage = Storage::open(directory.path(), Arc::new(MemoryVault::default())) + .expect("test storage must open"); + TestApplication { + _directory: directory, + application: Application::with_storage(storage), + } + } + + fn json_request(method: Method, uri: &str, body: &serde_json::Value) -> Request { + Request::builder() + .method(method) + .uri(uri) + .header(header::CONTENT_TYPE, "application/json") + .body(Body::from( + serde_json::to_vec(body).expect("request JSON must serialize"), + )) + .expect("request must build") + } + + fn empty_request(method: Method, uri: &str) -> Request { + Request::builder() + .method(method) + .uri(uri) + .body(Body::empty()) + .expect("request must build") + } + + async fn response_bytes(response: axum::response::Response) -> Vec { + response + .into_body() + .collect() + .await + .expect("response body must collect") + .to_bytes() + .to_vec() + } + + async fn response_json(response: axum::response::Response) -> serde_json::Value { + serde_json::from_slice(&response_bytes(response).await) + .expect("response body must contain JSON") + } + + async fn mock_openai_response() -> impl IntoResponse { + ( + [(header::CONTENT_TYPE, "text/event-stream")], + MOCK_OPENAI_STREAM, + ) + } + + async fn spawn_mock_openai() -> (String, JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("mock provider must bind"); + let address = listener.local_addr().expect("mock provider address"); + let router = Router::new().route("/v1/chat/completions", post(mock_openai_response)); + let server = tokio::spawn(async move { + axum::serve(listener, router) + .await + .expect("mock provider must serve"); + }); + (format!("http://{address}/v1"), server) + } + + fn sse_payloads(body: &str) -> Vec { + body.lines() + .filter_map(|line| line.strip_prefix("data:")) + .map(str::trim) + .filter(|data| !data.is_empty()) + .map(|data| serde_json::from_str(data).expect("legacy SSE data must be JSON")) + .collect() + } + + fn envelope(event: AgentEvent) -> AgentEventEnvelope { + AgentEventEnvelope { + run_id: "run-1".to_owned(), + sequence: "1".to_owned(), + occurred_at_ms: "1700000000000".to_owned(), + event, + } + } + + #[test] + fn event_projection_matches_the_retained_frontend_and_never_fakes_reasoning() { + let answer = project_agent_event( + &envelope(AgentEvent::TextDelta { + delta: "hello".to_owned(), + }), + "session-1", + ) + .expect("text delta projects"); + assert_eq!(answer.event_type, "answer"); + assert_eq!(answer.content.as_deref(), Some("hello")); + + let done = project_agent_event( + &envelope(AgentEvent::Completed { + message_id: "message-1".to_owned(), + }), + "session-1", + ) + .expect("completion projects"); + assert_eq!(done.event_type, "done"); + assert_eq!(done.session_id.as_deref(), Some("session-1")); + + let failed = project_agent_event( + &envelope(AgentEvent::Failed { + error: ApiError::new("provider_failed", "provider failed"), + }), + "session-1", + ) + .expect("failure projects"); + assert_eq!(failed.event_type, "error"); + assert_eq!(failed.error_code.as_deref(), Some("provider_failed")); + + let denied = project_agent_event( + &envelope(AgentEvent::PermissionRequested { + permission: AgentPermissionRequest { + permission_id: "permission-1".to_owned(), + run_id: "run-1".to_owned(), + tool_call_id: "tool-1".to_owned(), + tool_name: "execute_sql".to_owned(), + arguments_sha256: "0".repeat(64), + summary: "write data".to_owned(), + requested_at_ms: "1700000000000".to_owned(), + expires_at_ms: "1700000060000".to_owned(), + }, + }), + "session-1", + ) + .expect("permission request projects to denial"); + assert_eq!(denied.event_type, "error"); + assert_eq!( + denied.error_code.as_deref(), + Some("agent_write_permission_denied") + ); + + assert!(project_agent_event(&envelope(AgentEvent::Started), "session-1").is_none()); + } + + #[test] + fn legacy_chat_request_defaults_to_no_write_permission_surface() { + let request: LegacyAiChatRequest = serde_json::from_value(serde_json::json!({ + "input": "drop the table", + "enableTools": true, + "provider": "OPENAI", + "model": "model-1", + "apiKey": "secret" + })) + .expect("legacy request decodes"); + assert_eq!(request.input, "drop the table"); + } + + #[tokio::test] + async fn web_stream_emits_session_answer_done_and_persists_history() { + let fixture = test_application(); + let (base_url, server) = spawn_mock_openai().await; + let application = routes().with_state(fixture.application); + let response = application + .clone() + .oneshot(json_request( + Method::POST, + "/api/v3/ai/chat/stream", + &serde_json::json!({ + "input": "Say hello", + "provider": "OPENAI", + "model": "mock-model", + "apiKey": "sentinel-api-key", + "baseUrl": base_url + }), + )) + .await + .expect("AI stream route must respond"); + assert_eq!(response.status(), StatusCode::OK); + assert!( + response.headers()[header::CONTENT_TYPE] + .to_str() + .expect("content type must be ASCII") + .starts_with("text/event-stream") + ); + let body = timeout(Duration::from_secs(3), response_bytes(response)) + .await + .expect("terminal legacy SSE must close"); + let body = String::from_utf8(body).expect("legacy SSE must be UTF-8"); + let payloads = sse_payloads(&body); + assert_eq!( + payloads + .iter() + .map(|payload| payload["type"].as_str().expect("event type")) + .collect::>(), + ["session", "answer", "done"] + ); + assert_eq!(payloads[1]["content"], "hello from mock"); + let session_id = payloads[0]["sessionId"] + .as_str() + .expect("session event must carry an id"); + + let history = application + .oneshot(empty_request( + Method::GET, + &format!("/api/v3/ai/chat/history/messages?sessionId={session_id}"), + )) + .await + .expect("history route must respond"); + let history = response_json(history).await; + assert_eq!(history["success"], true); + assert_eq!(history["total"], 2); + assert_eq!(history["data"][0]["role"], "user"); + assert_eq!(history["data"][0]["content"], "Say hello"); + assert_eq!(history["data"][1]["role"], "assistant"); + assert_eq!(history["data"][1]["content"], "hello from mock"); + + server.abort(); + let _ = server.await; + } + + #[tokio::test] + #[allow(clippy::too_many_lines)] + async fn model_config_routes_cover_crud_options_test_and_secret_retention() { + let fixture = test_application(); + let (base_url, server) = spawn_mock_openai().await; + let application = routes().with_state(fixture.application); + let create_payload = serde_json::json!({ + "name": "Mock OpenAI", + "provider": "OPENAI", + "model": "mock-model", + "apiKey": "sentinel-api-key", + "baseUrl": base_url, + "maxTokens": 64, + "defaultConfig": true + }); + let created = application + .clone() + .oneshot(json_request( + Method::POST, + "/api/v3/ai/model/config/save", + &create_payload, + )) + .await + .expect("model config save must respond"); + let created = response_json(created).await; + assert_eq!(created["success"], true); + assert_eq!(created["data"]["hasApiKey"], true); + assert!(!created.to_string().contains("sentinel-api-key")); + let id = created["data"]["id"] + .as_str() + .expect("saved config id") + .to_owned(); + + let listed = application + .clone() + .oneshot(empty_request(Method::GET, "/api/v3/ai/model/config/list")) + .await + .expect("model config list must respond"); + let listed = response_json(listed).await; + assert_eq!(listed["success"], true); + assert_eq!(listed["data"][0]["id"], id); + assert_eq!(listed["data"][0]["apiKeyMasked"], "****"); + + let options = application + .clone() + .oneshot(empty_request(Method::GET, "/api/v3/ai/model/options")) + .await + .expect("model options must respond"); + let options = response_json(options).await; + assert_eq!(options["success"], true); + assert_eq!(options["data"][0]["value"], format!("config:{id}")); + assert_eq!(options["data"][0]["defaultOption"], true); + + let tested = application + .clone() + .oneshot(json_request( + Method::POST, + "/api/v3/ai/model/config/test", + &create_payload, + )) + .await + .expect("model config test must respond"); + let tested = response_json(tested).await; + assert_eq!(tested["success"], true); + assert_eq!(tested["data"]["success"], true); + assert_eq!(tested["data"]["statusCode"], 200); + + let updated = application + .clone() + .oneshot(json_request( + Method::POST, + "/api/v3/ai/model/config/save", + &serde_json::json!({ + "id": id, + "name": "Updated Mock", + "provider": "OPENAI", + "model": "mock-model", + "baseUrl": base_url, + "maxTokens": 128 + }), + )) + .await + .expect("model config update must respond"); + let updated = response_json(updated).await; + assert_eq!(updated["success"], true); + assert_eq!(updated["data"]["name"], "Updated Mock"); + assert_eq!(updated["data"]["hasApiKey"], true); + + let deleted = application + .clone() + .oneshot(json_request( + Method::POST, + "/api/v3/ai/model/config/delete", + &serde_json::json!({ "id": id }), + )) + .await + .expect("model config delete must respond"); + assert_eq!(response_json(deleted).await["success"], true); + let listed = application + .oneshot(empty_request(Method::GET, "/api/v3/ai/model/config/list")) + .await + .expect("empty model config list must respond"); + assert_eq!(response_json(listed).await["data"], serde_json::json!([])); + + server.abort(); + let _ = server.await; + } + + #[tokio::test] + async fn selected_model_config_without_a_secret_is_rejected_before_start() { + let fixture = test_application(); + let profile = fixture + .application + .create_provider_profile(CreateProviderProfileRequest { + name: "Missing secret".to_owned(), + kind: ProviderKind::OpenAiCompatible, + base_url: "https://provider.example/v1".to_owned(), + model: "mock-model".to_owned(), + context_window_tokens: "4096".to_owned(), + max_output_tokens: "1024".to_owned(), + credentials: None, + }) + .await + .expect("secret-free provider profile must be created"); + let result = start_chat_run( + &fixture.application, + LegacyAiChatRequest { + input: "hello".to_owned(), + model_config_id: Some(profile.id), + ..LegacyAiChatRequest::default() + }, + ) + .await; + let Err(error) = result else { + panic!("secret-free model config must not start a run"); + }; + assert_eq!(error.code, "provider_credentials_missing"); + } + + #[tokio::test] + async fn attachment_paths_are_desktop_only_and_http_uses_uploads() { + let directory = TempDir::new().expect("attachment directory"); + let path = directory.path().join("notes.txt"); + std::fs::write(&path, "first line\nsecond line").expect("local attachment must write"); + let application = routes().with_state(Application::new()); + let local = dispatch( + &Application::new(), + "post", + "/api/v3/ai/chat/attachment/parse/local", + serde_json::json!({ "filePath": path, "fileName": "notes.txt" }), + ) + .await + .expect("Desktop attachment dispatch must handle local paths"); + assert_eq!(local["success"], true); + assert_eq!(local["data"]["fileType"], "txt"); + assert_eq!(local["data"]["contentCategory"], "DOCUMENT"); + assert_eq!(local["data"]["content"], "first line\nsecond line"); + + let local_http = application + .clone() + .oneshot(json_request( + Method::POST, + "/api/v3/ai/chat/attachment/parse/local", + &serde_json::json!({ "filePath": path, "fileName": "notes.txt" }), + )) + .await + .expect("local attachment route must respond"); + assert_eq!(local_http.status(), StatusCode::NOT_FOUND); + + let boundary = "chat2db-test-boundary"; + let multipart = format!( + "--{boundary}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"rows.csv\"\r\nContent-Type: text/csv\r\n\r\nname,value\r\nalpha,1\r\n--{boundary}--\r\n" + ); + let upload_request = Request::builder() + .method(Method::POST) + .uri("/api/v3/ai/chat/attachment/parse/upload") + .header( + header::CONTENT_TYPE, + format!("multipart/form-data; boundary={boundary}"), + ) + .body(Body::from(multipart)) + .expect("multipart request must build"); + let uploaded = application + .oneshot(upload_request) + .await + .expect("upload attachment route must respond"); + let uploaded = response_json(uploaded).await; + assert_eq!(uploaded["success"], true); + assert_eq!(uploaded["data"]["fileType"], "csv"); + assert_eq!(uploaded["data"]["contentCategory"], "TABULAR"); + assert!( + uploaded["data"]["content"] + .as_str() + .expect("CSV content") + .contains("alpha") + ); + } +} diff --git a/apps/chat2db-web/src/lib.rs b/apps/chat2db-web/src/lib.rs index 0ab9886..7bbf5b7 100644 --- a/apps/chat2db-web/src/lib.rs +++ b/apps/chat2db-web/src/lib.rs @@ -4,6 +4,7 @@ mod api; mod error; mod extract; pub mod legacy; +pub mod legacy_ai; use std::{ error::Error, @@ -173,9 +174,10 @@ async fn authorize(State(policy): State, request: Request, next: N #[cfg(test)] mod tests { use std::{ + collections::HashMap, fs, net::{IpAddr, Ipv4Addr, SocketAddr}, - sync::Arc, + sync::{Arc, Mutex}, time::Duration, }; @@ -236,7 +238,11 @@ mod tests { assert_eq!(response.status(), StatusCode::OK); let inventory: JdbcDriverList = response_json(response).await; - assert!(inventory.items.is_empty()); + assert_eq!(inventory.items.len(), 1); + let mysql = &inventory.items[0]; + assert_eq!(mysql.driver_id, "mysql"); + assert_eq!(mysql.driver_class, "rust:mysql_async"); + assert_eq!(mysql.artifact_count, 0); } #[tokio::test] @@ -296,8 +302,8 @@ mod tests { #[allow(clippy::too_many_lines)] async fn legacy_datasource_routes_cover_crud_without_echoing_connection_secrets() { let directory = TempDir::new().expect("temp directory"); - let storage = - Storage::open(directory.path(), Arc::new(TestVault)).expect("test storage must open"); + let storage = Storage::open(directory.path(), Arc::new(TestVault::default())) + .expect("test storage must open"); let application = router(Application::with_storage(storage)); let create_response = application @@ -411,8 +417,8 @@ mod tests { #[allow(clippy::too_many_lines)] async fn legacy_saved_console_routes_cover_create_list_get_update_and_delete() { let directory = TempDir::new().expect("temp directory"); - let storage = - Storage::open(directory.path(), Arc::new(TestVault)).expect("test storage must open"); + let storage = Storage::open(directory.path(), Arc::new(TestVault::default())) + .expect("test storage must open"); let application = router(Application::with_storage(storage)); let created_response = application @@ -551,8 +557,8 @@ mod tests { assert_eq!(empty["errorCode"], "invalid_sql_execute_request"); let directory = TempDir::new().expect("temp directory"); - let storage = - Storage::open(directory.path(), Arc::new(TestVault)).expect("test storage must open"); + let storage = Storage::open(directory.path(), Arc::new(TestVault::default())) + .expect("test storage must open"); let unavailable_response = router(Application::with_storage(storage.clone())) .oneshot(json_request( Method::POST, @@ -1398,8 +1404,8 @@ mod tests { const API_KEY: &str = "provider-secret-sentinel"; let directory = TempDir::new().expect("temp directory"); - let storage = - Storage::open(directory.path(), Arc::new(TestVault)).expect("test storage must open"); + let storage = Storage::open(directory.path(), Arc::new(TestVault::default())) + .expect("test storage must open"); let application = router(Application::with_storage(storage)); let create_response = application @@ -1530,8 +1536,8 @@ mod tests { #[allow(clippy::too_many_lines)] async fn agent_session_routes_cover_lifecycle_and_message_pagination() { let directory = TempDir::new().expect("temp directory"); - let storage = - Storage::open(directory.path(), Arc::new(TestVault)).expect("test storage must open"); + let storage = Storage::open(directory.path(), Arc::new(TestVault::default())) + .expect("test storage must open"); let application = router(Application::with_storage(storage.clone())); let provider_response = application @@ -1718,8 +1724,8 @@ mod tests { #[allow(clippy::too_many_lines)] async fn agent_run_routes_cover_acceptance_snapshot_replay_and_cancellation() { let directory = TempDir::new().expect("temp directory"); - let storage = - Storage::open(directory.path(), Arc::new(TestVault)).expect("test storage must open"); + let storage = Storage::open(directory.path(), Arc::new(TestVault::default())) + .expect("test storage must open"); let application = router(Application::with_storage(storage)); let provider_response = application @@ -1872,8 +1878,8 @@ mod tests { #[tokio::test] async fn datasource_routes_cover_the_storage_lifecycle_without_echoing_secrets() { let directory = TempDir::new().expect("temp directory"); - let storage = - Storage::open(directory.path(), Arc::new(TestVault)).expect("test storage must open"); + let storage = Storage::open(directory.path(), Arc::new(TestVault::default())) + .expect("test storage must open"); let application = router(Application::with_storage(storage)); let create_response = application @@ -2171,8 +2177,10 @@ mod tests { String::from_utf8(body.to_vec()).expect("response body must be UTF-8") } - #[derive(Debug)] - struct TestVault; + #[derive(Debug, Default)] + struct TestVault { + values: Mutex>>, + } impl SecretVault for TestVault { fn probe(&self) -> Result<(), SecretVaultError> { @@ -2181,17 +2189,31 @@ mod tests { fn create( &self, - _reference: &SecretRef, - _value: &SecretValue, + reference: &SecretRef, + value: &SecretValue, ) -> Result<(), SecretVaultError> { + self.values.lock().expect("test vault lock").insert( + reference.as_str().to_owned(), + value.expose_secret().to_vec(), + ); Ok(()) } - fn get(&self, _reference: &SecretRef) -> Result, SecretVaultError> { - Ok(None) + fn get(&self, reference: &SecretRef) -> Result, SecretVaultError> { + Ok(self + .values + .lock() + .expect("test vault lock") + .get(reference.as_str()) + .cloned() + .map(SecretValue::new)) } - fn delete(&self, _reference: &SecretRef) -> Result<(), SecretVaultError> { + fn delete(&self, reference: &SecretRef) -> Result<(), SecretVaultError> { + self.values + .lock() + .expect("test vault lock") + .remove(reference.as_str()); Ok(()) } } diff --git a/apps/chat2db-web/tests/native_mysql_editable_ddl_docker.rs b/apps/chat2db-web/tests/native_mysql_editable_ddl_docker.rs index a446917..7323e53 100644 --- a/apps/chat2db-web/tests/native_mysql_editable_ddl_docker.rs +++ b/apps/chat2db-web/tests/native_mysql_editable_ddl_docker.rs @@ -1,4 +1,10 @@ -use std::panic::AssertUnwindSafe; +use std::{ + fs, + io::{Cursor, Read as _}, + panic::AssertUnwindSafe, + path::Path, + time::Duration, +}; use axum::{ Router, @@ -35,11 +41,18 @@ struct MysqlTestConfig { impl MysqlTestConfig { fn from_environment() -> Option { + let required = std::env::var("MYSQL_TEST_REQUIRED") + .ok() + .is_some_and(|value| matches!(value.as_str(), "1" | "true" | "TRUE")); let configured = REQUIRED_MYSQL_ENV .iter() .filter(|name| std::env::var_os(name).is_some()) .count(); if configured == 0 { + assert!( + !required, + "MYSQL_TEST_REQUIRED is enabled but the MySQL endpoint is absent" + ); eprintln!("skipping editable MySQL Web test; MYSQL_TEST_* variables are absent"); return None; } @@ -91,6 +104,7 @@ impl MysqlTestConfig { }, ], read_only: false, + ssh: None, } } } @@ -159,6 +173,15 @@ async fn verify_product_vertical(config: &MysqlTestConfig, database_name: &str) let datasource = create_datasource(&application, config, Some(database_name), "MySQL editable").await; + verify_workspace_routes( + &router, + &application, + config, + &datasource, + database_name, + directory.path(), + ) + .await; verify_routine_invocation_preview(&router, &application, &datasource, database_name).await; let editor_meta = get( &router, @@ -214,6 +237,9 @@ async fn verify_product_vertical(config: &MysqlTestConfig, database_name: &str) table_sql[0]["sql"].as_str().expect("table SQL"), ) .await; + verify_pin_and_er_routes(&router, &application, &datasource, database_name).await; + verify_account_routes(&router, &application, &datasource).await; + verify_schema_diff_routes(&router, &application, &datasource, database_name).await; let export_message = json!({ "dataSourceId": datasource, @@ -491,6 +517,15 @@ async fn verify_product_vertical(config: &MysqlTestConfig, database_name: &str) ) .await; assert_eq!(count, 1); + verify_transfer_routes( + &router, + &application, + config, + &datasource, + database_name, + directory.path(), + ) + .await; let updated_preview = preview(&router, &datasource, database_name, "items").await; let updated_values = cell_values(&updated_preview["dataList"][0]); @@ -966,97 +1001,1430 @@ async fn verify_product_vertical(config: &MysqlTestConfig, database_name: &str) .expect("native-only Web runtime must shut down"); } -async fn verify_routine_invocation_preview( +async fn verify_pin_and_er_routes( router: &Router, application: &Application, datasource_id: &str, database_name: &str, ) { - execute_ddl( + let table = object_request(datasource_id, database_name, "items"); + post(router, "/api/pin/table/add", table.clone()).await; + post(router, "/api/pin/table/add", table.clone()).await; + + let list_path = format!( + "/api/pin/table/list?dataSourceId={datasource_id}&databaseName={database_name}&schemaName=" + ); + assert_eq!(get(router, &list_path).await, json!(["items"])); + let desktop_pins = chat2db_web::legacy::dispatch( + application, + chat2db_web::legacy::LegacyDispatchRequest { + request_url: "/api/pin/table/list".to_owned(), + method: "get".to_owned(), + message: json!({ + "dataSourceId": datasource_id, + "databaseName": database_name, + "schemaName": "" + }), + }, + ) + .await; + assert_eq!(desktop_pins["success"], true); + assert_eq!(desktop_pins["data"], json!(["items"])); + + assert_table_is_pinned(router, datasource_id, database_name).await; + + let er_path = format!( + "/api/er/get_info?dataSourceId={datasource_id}&databaseName={database_name}&schemaName=" + ); + let er = get(router, &er_path).await; + assert!(er["position"].is_null()); + let items = er["tables"] + .as_array() + .and_then(|tables| tables.iter().find(|table| table["name"] == "items")) + .expect("items table must be present in ER metadata"); + assert!(items["columnList"].as_array().is_some_and(|columns| { + columns + .iter() + .any(|column| column["name"] == "id" && column["primaryKey"] == true) + })); + let desktop_er = chat2db_web::legacy::dispatch( + application, + chat2db_web::legacy::LegacyDispatchRequest { + request_url: "/api/er/get_info".to_owned(), + method: "get".to_owned(), + message: json!({ + "dataSourceId": datasource_id, + "databaseName": database_name, + "schemaName": "" + }), + }, + ) + .await; + assert_eq!(desktop_er["success"], true); + assert_eq!(desktop_er["data"], er); + + post( router, - datasource_id, - database_name, - "", - "CREATE FUNCTION `routine``add`(input_value INT) RETURNS INT \ - DETERMINISTIC NO SQL RETURN input_value + 1;", + "/api/er/save_position", + json!({ + "dataSourceId": datasource_id, + "databaseName": database_name, + "schemaName": "", + "position": "{\"version\":1}" + }), ) .await; - execute_ddl( + let desktop_save = chat2db_web::legacy::dispatch( + application, + chat2db_web::legacy::LegacyDispatchRequest { + request_url: "/api/er/save_position".to_owned(), + method: "post".to_owned(), + message: json!({ + "dataSourceId": datasource_id, + "databaseName": database_name, + "schemaName": "", + "position": "{\"version\":2}" + }), + }, + ) + .await; + assert_eq!(desktop_save["success"], true); + assert_eq!(get(router, &er_path).await["position"], "{\"version\":2}"); + + let desktop_delete = chat2db_web::legacy::dispatch( + application, + chat2db_web::legacy::LegacyDispatchRequest { + request_url: "/api/pin/table/delete".to_owned(), + method: "post".to_owned(), + message: table, + }, + ) + .await; + assert_eq!(desktop_delete["success"], true); + assert_eq!(get(router, &list_path).await, json!([])); + assert_java_dormant(application); +} + +async fn assert_table_is_pinned(router: &Router, datasource_id: &str, database_name: &str) { + let tables = get( router, - datasource_id, - database_name, - "", - "CREATE PROCEDURE routine_mix(\ - IN input_value INT, OUT output_text VARCHAR(32), INOUT running_total BIGINT\ - ) BEGIN \ - SET output_text = CONCAT('v', input_value); \ - SET running_total = running_total + input_value + 7; \ - END;", + &format!( + "/api/rdb/table/list?dataSourceId={datasource_id}&databaseType=MYSQL&databaseName={database_name}&schemaName=&pageNo=1&pageSize=20&searchKey=" + ), ) .await; + assert!(tables["data"].as_array().is_some_and(|tables| { + tables + .iter() + .any(|table| table["name"] == "items" && table["pinned"] == true) + })); +} - let function_request = json!({ - "dataSourceId": datasource_id, - "databaseName": database_name, - "schemaName": null, - "routineType": "FUNCTION", - "routineName": "`routine``add`" - }); - let function_preview = post( +async fn verify_account_routes(router: &Router, application: &Application, datasource_id: &str) { + let capability = get( router, - "/api/rdb/routine/preview_invocation", - function_request.clone(), + &format!("/api/rdb/account/capability?dataSourceId={datasource_id}"), ) .await; + assert_eq!(capability["dbType"], "MYSQL"); + assert_eq!(capability["accountListReadable"], true); assert_eq!( - function_preview["sql"], - "set @input_value = 0;\n\nselect `routine``add`(\n @input_value\n);" + capability["editablePrivileges"].as_array().map(Vec::len), + Some(14) ); - assert_desktop_routine_preview_matches(application, function_request, &function_preview).await; - let function_results = execute_console_sql( + assert!( + get( + router, + &format!("/api/rdb/account/list?dataSourceId={datasource_id}"), + ) + .await + .as_array() + .is_some_and(|accounts| !accounts.is_empty()) + ); + + let current_account = capability["currentUser"] + .as_str() + .expect("current MySQL account must be reported"); + let (current_user, current_host) = current_account + .split_once('@') + .expect("current MySQL account must contain a host"); + let grants_query = url::form_urlencoded::Serializer::new(String::new()) + .append_pair("dataSourceId", datasource_id) + .append_pair("user", current_user) + .append_pair("host", current_host) + .finish(); + assert!( + get(router, &format!("/api/rdb/account/grants?{grants_query}")) + .await + .as_array() + .is_some_and(|grants| !grants.is_empty()) + ); + + let missing_user = format!("c2d_missing_{}", &Uuid::new_v4().simple().to_string()[..10]); + let password = "MustNotLeak'\\Password"; + let create_preview = post( router, - datasource_id, - database_name, - function_preview["sql"] + "/api/rdb/account/preview", + json!({ + "dataSourceId": datasource_id, + "user": missing_user, + "host": "%", + "actionType": "CREATE_USER", + "password": password + }), + ) + .await; + assert!( + create_preview["sql"] .as_str() - .expect("function preview SQL"), + .is_some_and(|sql| { sql.contains("******") && !sql.contains(password) }) + ); + + let mut drop_request = json!({ + "dataSourceId": datasource_id, + "user": missing_user, + "host": "%", + "actionType": "DROP_USER" + }); + let drop_preview = post(router, "/api/rdb/account/preview", drop_request.clone()).await; + drop_request + .as_object_mut() + .expect("account request object") + .insert( + "previewToken".to_owned(), + drop_preview["previewToken"].clone(), + ); + let failed_execution = post(router, "/api/rdb/account/execute", drop_request.clone()).await; + assert_eq!(failed_execution["success"], false); + assert_eq!( + failed_execution["failureCode"], + "mysql.account.executeFailed" + ); + verify_account_token_replay_and_desktop( + application, + datasource_id, + &missing_user, + drop_request, + password, ) .await; - assert_eq!(last_console_values(&function_results), json!(["1"])); + assert_java_dormant(application); +} - let procedure_request = json!({ +async fn verify_account_token_replay_and_desktop( + application: &Application, + datasource_id: &str, + missing_user: &str, + drop_request: Value, + password: &str, +) { + let replay = chat2db_web::legacy::dispatch( + application, + chat2db_web::legacy::LegacyDispatchRequest { + request_url: "/api/rdb/account/execute".to_owned(), + method: "post".to_owned(), + message: drop_request.clone(), + }, + ) + .await; + assert_eq!(replay["success"], false); + assert_eq!(replay["errorCode"], "mysql.account.previewTokenMismatch"); + + let mut desktop_request = json!({ "dataSourceId": datasource_id, - "databaseType": "MYSQL", - "databaseName": database_name, - "schemaName": "", - "routineType": "PROCEDURE", - "routineName": "routine_mix" + "user": missing_user, + "host": "%", + "actionType": "DROP_USER" }); - let procedure_preview = post( - router, - "/api/rdb/routine/preview_invocation", - procedure_request.clone(), + let desktop_preview = chat2db_web::legacy::dispatch( + application, + chat2db_web::legacy::LegacyDispatchRequest { + request_url: "/api/rdb/account/preview".to_owned(), + method: "post".to_owned(), + message: desktop_request.clone(), + }, + ) + .await; + assert_eq!(desktop_preview["success"], true); + desktop_request + .as_object_mut() + .expect("desktop account request object") + .insert( + "previewToken".to_owned(), + desktop_preview["data"]["previewToken"].clone(), + ); + let desktop_execution = chat2db_web::legacy::dispatch( + application, + chat2db_web::legacy::LegacyDispatchRequest { + request_url: "/api/rdb/account/execute".to_owned(), + method: "post".to_owned(), + message: desktop_request, + }, ) .await; + assert_eq!(desktop_execution["success"], true); + assert_eq!(desktop_execution["data"]["success"], false); assert_eq!( - procedure_preview["sql"], - "set @input_value = 0;\nset @running_total = 0;\n\n\ - call routine_mix(\n @input_value,\n @output_text,\n @running_total\n);\n\ - select @output_text, @running_total;" + desktop_execution["data"]["failureCode"], + "mysql.account.executeFailed" ); - assert_desktop_routine_preview_matches(application, procedure_request, &procedure_preview) - .await; - let procedure_results = execute_console_sql( + assert!(!desktop_execution.to_string().contains(password)); +} + +async fn verify_schema_diff_routes( + router: &Router, + application: &Application, + datasource_id: &str, + database_name: &str, +) { + let request = json!({ + "source": { + "dataSourceId": datasource_id, + "databaseName": database_name, + "schemaName": "" + }, + "target": { + "dataSourceId": datasource_id, + "databaseName": database_name, + "schemaName": "" + } + }); + let http_sql = post(router, "/api/diff/sql", request.clone()).await; + assert_eq!(http_sql, json!("-- No differences. ")); + + let desktop = chat2db_web::legacy::dispatch( + application, + chat2db_web::legacy::LegacyDispatchRequest { + request_url: "/api/diff/sql".to_owned(), + method: "post".to_owned(), + message: request, + }, + ) + .await; + assert_eq!( + desktop["success"], true, + "desktop schema diff failed: {desktop}" + ); + assert_eq!(desktop["data"], http_sql); + assert_java_dormant(application); +} + +#[allow(clippy::too_many_lines)] +async fn verify_transfer_routes( + router: &Router, + application: &Application, + config: &MysqlTestConfig, + datasource_id: &str, + database_name: &str, + directory: &Path, +) { + execute_ddl( router, datasource_id, database_name, - procedure_preview["sql"] + "transfer_items", + "CREATE TABLE `transfer_items` (`id` INT PRIMARY KEY, `label` VARCHAR(64) NOT NULL)", + ) + .await; + + let csv_task = multipart_mysql_import( + router, + "/api/import/other_file", + &[ + ("dataSourceId", datasource_id), + ("databaseName", database_name), + ("schemaName", ""), + ("tableName", "transfer_items"), + ("importType", "CSV"), + ("containsHeader", "true"), + ], + "transfer-items.csv", + b"id,label\n1,alpha\n2,beta\n", + ) + .await + .as_i64() + .expect("CSV import task id"); + let csv_task = wait_for_transfer_task(router, csv_task).await; + assert_eq!(csv_task["taskType"], "UPLOAD_TABLE_DATA"); + assert_eq!(csv_task["taskStatus"], "FINISHED"); + assert_eq!(csv_task["taskProgress"], "100"); + + let sql_task = multipart_mysql_import( + router, + "/api/import/sql_file", + &[ + ("dataSourceId", datasource_id), + ("databaseName", database_name), + ("schemaName", ""), + ], + "transfer-items.sql", + b"INSERT INTO `transfer_items` (`id`, `label`) VALUES (3, 'gamma');\n", + ) + .await + .as_i64() + .expect("SQL import task id"); + wait_for_transfer_task(router, sql_task).await; + assert_eq!( + scalar_count(config, database_name, "transfer_items").await, + 3 + ); + + let desktop_sql_path = directory.join("desktop-transfer-items.sql"); + fs::write( + &desktop_sql_path, + "INSERT INTO `transfer_items` (`id`, `label`) VALUES (4, 'desktop');\n", + ) + .expect("desktop SQL import fixture must write"); + let desktop_import = chat2db_web::legacy::dispatch_desktop( + application, + chat2db_web::legacy::LegacyDispatchRequest { + request_url: "/api/import/sql_file".to_owned(), + method: "post".to_owned(), + message: json!({ + "dataSourceId": datasource_id, + "databaseName": database_name, + "schemaName": "", + "fileName": desktop_sql_path.to_string_lossy() + }), + }, + ) + .await; + assert_eq!(desktop_import["success"], true, "{desktop_import}"); + let desktop_task = desktop_import["data"] + .as_i64() + .expect("desktop SQL import task id"); + wait_for_transfer_task(router, desktop_task).await; + assert_eq!( + scalar_count(config, database_name, "transfer_items").await, + 4 + ); + + let sql_export_task = post( + router, + "/api/export/sql_file", + json!({ + "dataSourceId": datasource_id, + "databaseName": database_name, + "schemaName": "", + "tableNames": ["transfer_items"], + "scope": "ALL", + "containData": true, + "exportPath": "" + }), + ) + .await + .as_i64() + .expect("SQL export task id"); + let sql_export = wait_for_transfer_task(router, sql_export_task).await; + assert_eq!(sql_export["taskType"], "DOWNLOAD_TABLE_STRUCTURE"); + assert!( + sql_export["downloadUrl"] .as_str() - .expect("procedure preview SQL"), + .is_some_and(|url| url.ends_with(&format!("id={sql_export_task}"))) + ); + let sql_dump = download_attachment( + router, + Method::GET, + &format!("/api/task/download?id={sql_export_task}"), + None, ) .await; - assert_eq!(last_console_values(&procedure_results), json!(["v0", "7"])); - assert_java_dormant(application); + let sql_dump = String::from_utf8(sql_dump).expect("SQL task download must be UTF-8"); + assert!(sql_dump.contains("CREATE TABLE")); + assert!(sql_dump.contains("INSERT INTO")); + assert!(sql_dump.contains("transfer_items")); + + let desktop_download = chat2db_web::legacy::dispatch_desktop( + application, + chat2db_web::legacy::LegacyDispatchRequest { + request_url: "/api/task/download".to_owned(), + method: "get".to_owned(), + message: json!({"id": sql_export_task}), + }, + ) + .await; + assert_eq!(desktop_download["success"], true, "{desktop_download}"); + assert!( + desktop_download["data"] + .as_str() + .is_some_and(|path| Path::new(path).is_file()) + ); + + let csv_export_task = post( + router, + "/api/export/other_file", + json!({ + "dataSourceId": datasource_id, + "databaseName": database_name, + "schemaName": "", + "tableNames": ["transfer_items"], + "exportType": "CSV", + "containsHeader": true, + "exportPath": "" + }), + ) + .await + .as_i64() + .expect("CSV export task id"); + wait_for_transfer_task(router, csv_export_task).await; + let csv_export = download_attachment( + router, + Method::GET, + &format!("/api/task/download?id={csv_export_task}"), + None, + ) + .await; + let csv_export = String::from_utf8(csv_export).expect("CSV task download must be UTF-8"); + assert!(csv_export.starts_with("id,label")); + assert!(csv_export.contains("3,gamma")); + + let finished = get( + router, + "/api/task/list?pageNo=1&pageSize=20&taskStatus=FINISHED", + ) + .await; + assert!(finished["total"].as_u64().is_some_and(|total| total >= 4)); + assert!(finished["data"].as_array().is_some_and(|tasks| { + tasks + .iter() + .any(|task| task["id"] == sql_export_task && task["taskProgress"] == "100") + })); + let desktop_finished = chat2db_web::legacy::dispatch_desktop( + application, + chat2db_web::legacy::LegacyDispatchRequest { + request_url: "/api/task/list".to_owned(), + method: "get".to_owned(), + message: json!({ + "pageNo": 1, + "pageSize": 20, + "taskStatus": "FINISHED" + }), + }, + ) + .await; + assert_eq!(desktop_finished["success"], true, "{desktop_finished}"); + assert!( + desktop_finished["data"]["data"] + .as_array() + .is_some_and(|tasks| tasks.iter().any(|task| { + task["id"] == sql_export_task + && task["downloadUrl"] + .as_str() + .is_some_and(|path| Path::new(path).is_file()) + })) + ); + + let desktop_task = chat2db_web::legacy::dispatch_desktop( + application, + chat2db_web::legacy::LegacyDispatchRequest { + request_url: "/api/task/get".to_owned(), + method: "get".to_owned(), + message: json!({"id": sql_export_task}), + }, + ) + .await; + assert_eq!(desktop_task["success"], true, "{desktop_task}"); + assert!( + desktop_task["data"]["downloadUrl"] + .as_str() + .is_some_and(|path| Path::new(path).is_file()) + ); + assert!( + get(router, &format!("/api/task/stop?id={csv_export_task}"),) + .await + .is_null() + ); + + let dml_request = json!({ + "dataSourceId": datasource_id, + "databaseName": database_name, + "schemaName": "", + "sql": "SELECT id, label FROM transfer_items ORDER BY id", + "originalSql": "SELECT id, label FROM transfer_items ORDER BY id", + "resultSetId": 0, + "exportSize": "ALL", + "exportType": "CSV" + }); + let dml_csv = download_attachment( + router, + Method::POST, + "/api/rdb/dml/export", + Some(dml_request.clone()), + ) + .await; + let dml_csv = String::from_utf8(dml_csv).expect("DML CSV must be UTF-8"); + assert!(dml_csv.starts_with("id,label")); + assert!(dml_csv.contains("2,beta")); + let desktop_dml = chat2db_web::legacy::dispatch_desktop( + application, + chat2db_web::legacy::LegacyDispatchRequest { + request_url: "/api/rdb/dml/export".to_owned(), + method: "post".to_owned(), + message: dml_request, + }, + ) + .await; + assert_eq!(desktop_dml["success"], true, "{desktop_dml}"); + assert!( + desktop_dml["data"] + .as_str() + .is_some_and(|path| Path::new(path).is_file()) + ); + + let class_request = json!({ + "dataSourceId": datasource_id, + "databaseName": database_name, + "schemaName": "", + "tableName": "transfer_items", + "exportPath": "" + }); + let archive = download_attachment( + router, + Method::POST, + "/api/rdb/table/generate/class", + Some(class_request.clone()), + ) + .await; + let mut archive = zip::ZipArchive::new(Cursor::new(archive)).expect("class ZIP must open"); + for file_name in [ + "TransferItemsDO.java", + "TransferItemsMapper.java", + "TransferItemsMapper.xml", + ] { + let mut entry = archive + .by_name(&format!("transfer_items/{file_name}")) + .expect("generated class entry must exist"); + let mut contents = String::new(); + entry + .read_to_string(&mut contents) + .expect("generated class entry must be UTF-8"); + assert!(!contents.is_empty()); + } + + let generated = directory.join("generated-classes"); + let mut desktop_class_request = class_request; + desktop_class_request["exportPath"] = json!(generated.to_string_lossy()); + let desktop_generated = chat2db_web::legacy::dispatch_desktop( + application, + chat2db_web::legacy::LegacyDispatchRequest { + request_url: "/api/rdb/table/generate/class".to_owned(), + method: "post".to_owned(), + message: desktop_class_request, + }, + ) + .await; + assert_eq!(desktop_generated["success"], true, "{desktop_generated}"); + let generated_table = generated.join("transfer_items"); + assert!(generated_table.join("TransferItemsDO.java").is_file()); + assert!(generated_table.join("TransferItemsMapper.java").is_file()); + assert!(generated_table.join("TransferItemsMapper.xml").is_file()); + assert_java_dormant(application); +} + +async fn wait_for_transfer_task(router: &Router, task_id: i64) -> Value { + for _ in 0..300 { + let task = get(router, &format!("/api/task/get?id={task_id}")).await; + match task["taskStatus"].as_str() { + Some("FINISHED") => return task, + Some("ERROR" | "STOP") => panic!("transfer task did not succeed: {task}"), + _ => tokio::time::sleep(Duration::from_millis(50)).await, + } + } + panic!("transfer task {task_id} did not finish before timeout") +} + +async fn download_attachment( + router: &Router, + method: Method, + path: &str, + payload: Option, +) -> Vec { + let builder = Request::builder().method(method).uri(path); + let (builder, body) = if let Some(payload) = payload { + ( + builder.header("content-type", "application/json"), + Body::from(serde_json::to_vec(&payload).expect("payload must encode")), + ) + } else { + (builder, Body::empty()) + }; + let response = router + .clone() + .oneshot(builder.body(body).expect("request must build")) + .await + .expect("attachment route must respond"); + assert_eq!(response.status(), StatusCode::OK, "{path}"); + assert!( + response + .headers() + .get("content-disposition") + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value.starts_with("attachment;")), + "{path} did not return an attachment" + ); + response + .into_body() + .collect() + .await + .expect("attachment body must collect") + .to_bytes() + .to_vec() +} + +async fn verify_converter_routes( + router: &Router, + application: &Application, + config: &MysqlTestConfig, + database_name: &str, + directory: &Path, +) { + let jdbc_url = mysql_test_jdbc_url(config, database_name); + + let desktop_file = directory.join("desktop-chat2db-import.json"); + fs::write( + &desktop_file, + serde_json::to_vec(&json!([{ + "alias": "Desktop imported MySQL", + "type": "MYSQL", + "url": jdbc_url.as_str(), + "user": config.user.as_str(), + "password": config.password.as_str(), + "extendInfo": [{"key": "connectionTimeZone", "value": "LOCAL"}] + }])) + .expect("desktop import JSON must encode"), + ) + .expect("desktop import JSON must write"); + let desktop = chat2db_web::legacy::dispatch_desktop( + application, + chat2db_web::legacy::LegacyDispatchRequest { + request_url: "/api/converter/chat2db/upload".to_owned(), + method: "post".to_owned(), + message: json!({"file": [desktop_file.to_string_lossy()]}), + }, + ) + .await; + assert_eq!(desktop["success"], true, "desktop import failed: {desktop}"); + assert_eq!(desktop["data"]["count"], 1); + + let web_document = serde_json::to_vec(&json!([{ + "alias": "Web imported MySQL", + "type": "MYSQL", + "url": jdbc_url.as_str(), + "user": config.user.as_str(), + "password": "must-not-be-imported" + }])) + .expect("Web import JSON must encode"); + let web = multipart_upload( + router, + "/api/converter/upload", + "connections.json", + &web_document, + ) + .await; + assert_eq!(web["count"], 1); + + let datagrip = post( + router, + "/api/converter/datagrip/upload", + json!({ + "text": format!( + "#DataSourceSettings#\n#BEGIN#\n\ + \n\ + \n\ + {jdbc_url}\n\ + {}\n\ + \n#END#\n", + config.user + ) + }), + ) + .await; + assert_eq!(datagrip["count"], 1); + + let list = get( + router, + "/api/connection/datasource/list?pageNo=1&pageSize=20", + ) + .await; + assert_password_fields_empty(&list); + for alias in [ + "Desktop imported MySQL", + "Web imported MySQL", + "DataGrip imported MySQL", + ] { + let imported = list["data"] + .as_array() + .and_then(|items| items.iter().find(|item| item["alias"] == alias)) + .unwrap_or_else(|| panic!("missing imported datasource {alias}: {list}")); + assert_eq!(imported["password"], ""); + assert!( + imported["url"] + .as_str() + .is_some_and(|url| url.contains(database_name)) + ); + let id = imported["id"].as_str().expect("imported datasource id"); + let deleted = request( + router, + Method::DELETE, + &format!("/api/connection/datasource?id={id}"), + None, + ) + .await; + assert_eq!(deleted["success"], true, "import cleanup failed: {deleted}"); + } + assert_java_dormant(application); +} + +fn mysql_test_jdbc_url(config: &MysqlTestConfig, database_name: &str) -> String { + let host = if config.host.contains(':') + && !(config.host.starts_with('[') && config.host.ends_with(']')) + { + format!("[{}]", config.host) + } else { + config.host.clone() + }; + format!( + "jdbc:mysql://{host}:{}/{database_name}?useSSL=false", + config.port + ) +} + +async fn multipart_mysql_import( + router: &Router, + path: &str, + fields: &[(&str, &str)], + file_name: &str, + content: &[u8], +) -> Value { + let boundary = "chat2db-rust-mysql-import-boundary"; + let mut body = Vec::new(); + for (name, value) in fields { + body.extend_from_slice( + format!( + "--{boundary}\r\nContent-Disposition: form-data; name=\"{name}\"\r\n\r\n{value}\r\n" + ) + .as_bytes(), + ); + } + body.extend_from_slice( + format!( + "--{boundary}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"{file_name}\"\r\n\ + Content-Type: application/octet-stream\r\n\r\n" + ) + .as_bytes(), + ); + body.extend_from_slice(content); + body.extend_from_slice(format!("\r\n--{boundary}--\r\n").as_bytes()); + let response = router + .clone() + .oneshot( + Request::builder() + .method(Method::POST) + .uri(path) + .header( + "content-type", + format!("multipart/form-data; boundary={boundary}"), + ) + .body(Body::from(body)) + .expect("multipart import request must build"), + ) + .await + .expect("multipart import route must respond"); + assert_eq!(response.status(), StatusCode::OK, "{path}"); + let envelope: Value = serde_json::from_slice( + &response + .into_body() + .collect() + .await + .expect("multipart import response must collect") + .to_bytes(), + ) + .expect("multipart import response must be JSON"); + assert_eq!(envelope["success"], true, "multipart failed: {envelope}"); + envelope["data"].clone() +} + +async fn multipart_upload(router: &Router, path: &str, file_name: &str, content: &[u8]) -> Value { + let boundary = "chat2db-rust-product-boundary"; + let mut body = format!( + "--{boundary}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"{file_name}\"\r\n\ + Content-Type: application/octet-stream\r\n\r\n" + ) + .into_bytes(); + body.extend_from_slice(content); + body.extend_from_slice(format!("\r\n--{boundary}--\r\n").as_bytes()); + let response = router + .clone() + .oneshot( + Request::builder() + .method(Method::POST) + .uri(path) + .header( + "content-type", + format!("multipart/form-data; boundary={boundary}"), + ) + .body(Body::from(body)) + .expect("multipart request must build"), + ) + .await + .expect("multipart route must respond"); + assert_eq!(response.status(), StatusCode::OK, "{path}"); + let envelope: Value = serde_json::from_slice( + &response + .into_body() + .collect() + .await + .expect("multipart response must collect") + .to_bytes(), + ) + .expect("multipart response must be JSON"); + assert_eq!(envelope["success"], true, "multipart failed: {envelope}"); + envelope["data"].clone() +} + +async fn verify_workspace_routes( + router: &Router, + application: &Application, + config: &MysqlTestConfig, + datasource_id: &str, + database_name: &str, + directory: &Path, +) { + verify_native_driver_and_connection_routes(router, application, datasource_id, database_name) + .await; + verify_datasource_edit_route(router, config, datasource_id, database_name).await; + verify_converter_routes(router, application, config, database_name, directory).await; + let clone_id = clone_and_export_datasource(router, application, datasource_id).await; + verify_namespace_routes( + router, + application, + config, + datasource_id, + database_name, + &clone_id, + ) + .await; + close_and_delete_clone(router, application, datasource_id, &clone_id).await; + assert_java_dormant(application); +} + +async fn verify_native_driver_and_connection_routes( + router: &Router, + application: &Application, + datasource_id: &str, + database_name: &str, +) { + assert!( + get(router, "/api/jdbc/driver/download?dbType=MYSQL") + .await + .is_null() + ); + for (path, method) in [ + ("/api/jdbc/driver/save", "post"), + ("/api/jdbc/driver/delete", "delete"), + ] { + let response = chat2db_web::legacy::dispatch( + application, + chat2db_web::legacy::LegacyDispatchRequest { + request_url: path.to_owned(), + method: method.to_owned(), + message: json!({ + "dbType": "MYSQL", + "jdbcDriverClass": "rust:mysql_async", + "jdbcDriver": [] + }), + }, + ) + .await; + assert_eq!( + response["success"], true, + "native driver route failed: {response}" + ); + } + + let databases = get( + router, + &format!("/api/connection/datasource/connect?id={datasource_id}"), + ) + .await; + assert!(databases.as_array().is_some_and(|databases| { + databases + .iter() + .any(|database| database["name"] == database_name) + })); + let console = chat2db_web::legacy::dispatch( + application, + chat2db_web::legacy::LegacyDispatchRequest { + request_url: "/api/connection/console/connect".to_owned(), + method: "get".to_owned(), + message: json!({ + "consoleId": 1, + "dataSourceId": datasource_id, + "databaseName": database_name + }), + }, + ) + .await; + assert_eq!( + console["success"], true, + "desktop Console connect failed: {console}" + ); +} + +async fn verify_datasource_edit_route( + router: &Router, + config: &MysqlTestConfig, + datasource_id: &str, + database_name: &str, +) { + let detail = get( + router, + &format!("/api/connection/datasource?id={datasource_id}"), + ) + .await; + assert_eq!(detail["user"], config.user.as_str()); + assert_eq!(detail["password"], ""); + assert_eq!(detail["readOnly"], false); + assert!( + detail["url"] + .as_str() + .is_some_and(|url| url.contains(database_name) && !url.contains(&config.password)) + ); + let updated = post( + router, + "/api/connection/datasource/update", + json!({ + "id": datasource_id, + "alias": "MySQL editable updated", + "type": "MYSQL", + "url": detail["url"], + "user": detail["user"], + "password": "", + "readOnly": false, + "extendInfo": [{"key": "connectionTimeZone", "value": "LOCAL"}] + }), + ) + .await; + assert_eq!(updated["alias"], "MySQL editable updated"); + assert_eq!(updated["password"], ""); + assert_eq!(updated["extendInfo"][0]["key"], "connectionTimeZone"); + assert_eq!(updated["extendInfo"][0]["value"], "LOCAL"); + assert!( + get( + router, + &format!("/api/connection/datasource/connect?id={datasource_id}"), + ) + .await + .as_array() + .is_some_and(|databases| databases + .iter() + .any(|database| database["name"] == database_name)), + "empty-password edit must preserve the stored MySQL password" + ); +} + +async fn clone_and_export_datasource( + router: &Router, + application: &Application, + datasource_id: &str, +) -> String { + let clone_id = post( + router, + "/api/connection/datasource/clone", + json!({"id": datasource_id, "name": "MySQL editable clone"}), + ) + .await + .as_str() + .expect("clone id") + .to_owned(); + let clone_connect = chat2db_web::legacy::dispatch( + application, + chat2db_web::legacy::LegacyDispatchRequest { + request_url: "/api/connection/datasource/connect".to_owned(), + method: "get".to_owned(), + message: json!({"id": clone_id}), + }, + ) + .await; + assert_eq!( + clone_connect["success"], true, + "cloned datasource cannot connect" + ); + + let exported = post( + router, + "/api/connection/datasource/export", + json!({"datasourceIds": [datasource_id]}), + ) + .await; + assert_eq!(exported["count"], 1); + let export_message = exported["message"].as_str().expect("export document"); + let export_document: Value = serde_json::from_str(export_message).expect("export JSON"); + assert_eq!(export_document["schemaVersion"], 1); + assert_eq!(export_document["datasources"][0]["sourceId"], datasource_id); + let exported_connection = &export_document["datasources"][0]["connection"]; + assert!( + exported_connection["jdbcUrl"] + .as_str() + .is_some_and(|url| !url.contains('@')) + ); + assert!( + exported_connection["properties"] + .as_array() + .is_some_and(|properties| { + properties.iter().all(|property| { + property["key"].as_str().is_some_and(|key| { + !matches!( + key.to_ascii_lowercase().as_str(), + "password" | "passwd" | "pwd" | "token" | "secret" + ) + }) + }) + }) + ); + clone_id +} + +async fn verify_namespace_routes( + router: &Router, + application: &Application, + config: &MysqlTestConfig, + datasource_id: &str, + database_name: &str, + clone_id: &str, +) { + let namespace_id = post( + router, + "/api/namespaces/create", + json!({"name": "Integration"}), + ) + .await + .as_str() + .expect("namespace id") + .to_owned(); + post( + router, + "/api/namespaces/update", + json!({"id": namespace_id, "name": "Integration renamed"}), + ) + .await; + post( + router, + "/api/namespaces/update_position", + json!({ + "dragNode": {"id": datasource_id, "type": "DATA_SOURCE"}, + "dropToNode": {"id": namespace_id, "type": "NAMESPACE"}, + "dropPosition": 2 + }), + ) + .await; + let desktop_move = chat2db_web::legacy::dispatch( + application, + chat2db_web::legacy::LegacyDispatchRequest { + request_url: "/api/namespaces/update_position".to_owned(), + method: "post".to_owned(), + message: json!({ + "dragNode": {"id": clone_id, "type": "DATA_SOURCE"}, + "dropToNode": {"id": namespace_id, "type": "NAMESPACE"}, + "dropPosition": 2 + }), + }, + ) + .await; + assert_eq!( + desktop_move["success"], true, + "desktop namespace move failed" + ); + + let tree = get(router, "/api/namespaces/tree_list").await; + assert_password_fields_empty(&tree); + let namespace = tree + .as_array() + .and_then(|nodes| nodes.iter().find(|node| node["id"] == namespace_id)) + .expect("namespace in tree"); + assert_eq!(namespace["data"]["name"], "Integration renamed"); + assert_eq!(namespace["children"][0]["id"], datasource_id); + assert_eq!(namespace["children"][1]["id"], clone_id); + assert_eq!(namespace["children"][0]["data"]["password"], ""); + assert_eq!( + namespace["children"][0]["data"]["user"], + config.user.as_str() + ); + assert!( + namespace["children"][0]["data"]["url"] + .as_str() + .is_some_and(|url| url.contains(database_name)) + ); + + post( + router, + "/api/namespaces/delete", + json!({"id": namespace_id}), + ) + .await; + let promoted = get(router, "/api/namespaces/tree_list").await; + assert!(promoted.as_array().is_some_and(|nodes| { + nodes.iter().any(|node| node["id"] == datasource_id) + && nodes.iter().any(|node| node["id"] == clone_id) + })); +} + +async fn close_and_delete_clone( + router: &Router, + application: &Application, + datasource_id: &str, + clone_id: &str, +) { + let desktop_close = chat2db_web::legacy::dispatch( + application, + chat2db_web::legacy::LegacyDispatchRequest { + request_url: "/api/connection/close".to_owned(), + method: "get".to_owned(), + message: json!({"id": datasource_id}), + }, + ) + .await; + assert_eq!(desktop_close["success"], true); + assert!( + post( + router, + "/api/connection/datasource/close", + json!({"id": clone_id}) + ) + .await + .is_null() + ); + let deleted = request( + router, + Method::DELETE, + &format!("/api/connection/datasource?id={clone_id}"), + None, + ) + .await; + assert_eq!(deleted["success"], true, "clone cleanup failed: {deleted}"); +} + +fn assert_password_fields_empty(value: &Value) { + match value { + Value::Object(object) => { + for (key, value) in object { + if key.eq_ignore_ascii_case("password") { + assert!( + value.is_null() || value.as_str() == Some(""), + "password field leaked in response" + ); + } else { + assert_password_fields_empty(value); + } + } + } + Value::Array(values) => { + for value in values { + assert_password_fields_empty(value); + } + } + _ => {} + } +} + +async fn verify_routine_invocation_preview( + router: &Router, + application: &Application, + datasource_id: &str, + database_name: &str, +) { + install_test_routines(router, datasource_id, database_name).await; + verify_function_invocation_preview(router, application, datasource_id, database_name).await; + verify_procedure_invocation_preview(router, application, datasource_id, database_name).await; + + let migration_request = json!({ + "dataSourceId": datasource_id, + "databaseType": "MYSQL", + "databaseName": database_name, + "schemaName": "", + "routineType": "FUNCTION", + "routineName": "`routine``add`", + "ddl": format!( + "CREATE FUNCTION `{database_name}`.`routine``add`(input_value INT) RETURNS INT \ + DETERMINISTIC NO SQL RETURN input_value + 2" + ) + }); + let migration_preview = post( + router, + "/api/rdb/routine/preview_migration", + migration_request.clone(), + ) + .await; + assert!( + migration_preview["sql"] + .as_str() + .expect("migration preview SQL") + .starts_with(&format!( + "DROP FUNCTION IF EXISTS `{database_name}`.`routine``add`;" + )) + ); + let desktop_preview = chat2db_web::legacy::dispatch( + application, + chat2db_web::legacy::LegacyDispatchRequest { + request_url: "/api/rdb/routine/preview_migration".to_owned(), + method: "post".to_owned(), + message: migration_request.clone(), + }, + ) + .await; + assert_eq!(desktop_preview["success"], true); + assert_eq!(desktop_preview["data"], migration_preview); + + let migrated = chat2db_web::legacy::dispatch( + application, + chat2db_web::legacy::LegacyDispatchRequest { + request_url: "/api/rdb/routine/execute_migration".to_owned(), + method: "post".to_owned(), + message: migration_request.clone(), + }, + ) + .await; + assert_eq!(migrated["success"], true, "desktop migration failed"); + assert_eq!(migrated["data"]["success"], true, "{migrated}"); + let migrated_results = execute_console_sql( + router, + datasource_id, + database_name, + "SELECT `routine``add`(0)", + ) + .await; + assert_eq!(last_console_values(&migrated_results), json!(["2"])); + + let mut failed_migration_request = migration_request.clone(); + failed_migration_request + .as_object_mut() + .expect("migration request object") + .insert( + "ddl".to_owned(), + json!(format!( + "CREATE FUNCTION `{database_name}`.`routine``add`(input_value INT) RETURNS INT RETURN" + )), + ); + let failed = post( + router, + "/api/rdb/routine/execute_migration", + failed_migration_request, + ) + .await; + assert_eq!(failed["success"], false); + assert_eq!(failed["failureStage"], "APPLY"); + assert_eq!(failed["restoreAttempted"], true); + assert_eq!(failed["restoreSucceeded"], true); + let restored_results = execute_console_sql( + router, + datasource_id, + database_name, + "SELECT `routine``add`(0)", + ) + .await; + assert_eq!(last_console_values(&restored_results), json!(["2"])); + assert_java_dormant(application); +} + +async fn install_test_routines(router: &Router, datasource_id: &str, database_name: &str) { + execute_ddl( + router, + datasource_id, + database_name, + "", + "CREATE FUNCTION `routine``add`(input_value INT) RETURNS INT \ + DETERMINISTIC NO SQL RETURN input_value + 1;", + ) + .await; + execute_ddl( + router, + datasource_id, + database_name, + "", + "CREATE PROCEDURE routine_mix(\ + IN input_value INT, OUT output_text VARCHAR(32), INOUT running_total BIGINT\ + ) BEGIN \ + SET output_text = CONCAT('v', input_value); \ + SET running_total = running_total + input_value + 7; \ + END;", + ) + .await; +} + +async fn verify_function_invocation_preview( + router: &Router, + application: &Application, + datasource_id: &str, + database_name: &str, +) { + let function_request = json!({ + "dataSourceId": datasource_id, + "databaseName": database_name, + "schemaName": null, + "routineType": "FUNCTION", + "routineName": "`routine``add`" + }); + let function_preview = post( + router, + "/api/rdb/routine/preview_invocation", + function_request.clone(), + ) + .await; + assert_eq!( + function_preview["sql"], + "set @input_value = 0;\n\nselect `routine``add`(\n @input_value\n);" + ); + assert_desktop_routine_preview_matches(application, function_request, &function_preview).await; + let function_results = execute_console_sql( + router, + datasource_id, + database_name, + function_preview["sql"] + .as_str() + .expect("function preview SQL"), + ) + .await; + assert_eq!(last_console_values(&function_results), json!(["1"])); +} + +async fn verify_procedure_invocation_preview( + router: &Router, + application: &Application, + datasource_id: &str, + database_name: &str, +) { + let procedure_request = json!({ + "dataSourceId": datasource_id, + "databaseType": "MYSQL", + "databaseName": database_name, + "schemaName": "", + "routineType": "PROCEDURE", + "routineName": "routine_mix" + }); + let procedure_preview = post( + router, + "/api/rdb/routine/preview_invocation", + procedure_request.clone(), + ) + .await; + assert_eq!( + procedure_preview["sql"], + "set @input_value = 0;\nset @running_total = 0;\n\n\ + call routine_mix(\n @input_value,\n @output_text,\n @running_total\n);\n\ + select @output_text, @running_total;" + ); + assert_desktop_routine_preview_matches(application, procedure_request, &procedure_preview) + .await; + let procedure_results = execute_console_sql( + router, + datasource_id, + database_name, + procedure_preview["sql"] + .as_str() + .expect("procedure preview SQL"), + ) + .await; + assert_eq!(last_console_values(&procedure_results), json!(["v0", "7"])); } async fn assert_desktop_routine_preview_matches( diff --git a/apps/frontend/src/backend/community.test.ts b/apps/frontend/src/backend/community.test.ts index c828f25..4e7509f 100644 --- a/apps/frontend/src/backend/community.test.ts +++ b/apps/frontend/src/backend/community.test.ts @@ -184,7 +184,7 @@ const completeSqlRequest = { } satisfies CompleteCommunitySqlRequest; const catalog = { - sourceCommit: '37a34be858f2566b6b7fcf6c3f64183c1f560853', + sourceCommit: '3cb8af54cad5bd5caa20bb25f10d9b0e4f01931c', plugins: [{ databaseType: 'H2', name: 'H2', diff --git a/apps/frontend/src/generated/contract.ts b/apps/frontend/src/generated/contract.ts index 97bfb51..f42da1d 100644 --- a/apps/frontend/src/generated/contract.ts +++ b/apps/frontend/src/generated/contract.ts @@ -1880,6 +1880,7 @@ export interface components { properties: components["schemas"]["DatasourceConnectionProperty"][]; /** @description Whether sessions opened from this descriptor must be read-only. */ readOnly: boolean; + ssh?: null | components["schemas"]["SshTunnelConfig"]; }; /** @description One JDBC connection property supplied by the user. */ DatasourceConnectionProperty: { @@ -2456,6 +2457,46 @@ export interface components { * @enum {string} */ SqlPermissionMode: "read_only" | "ask_before_write"; + /** @description SSH user authentication material accepted only at a connection boundary. */ + SshAuthentication: { + /** @description SSH password, never returned or logged. */ + password: string; + /** @enum {string} */ + type: "password"; + } | { + /** @description User-selected local private-key path. */ + key_file: string; + /** @description Optional encrypted-key passphrase, never returned or logged. */ + passphrase?: string | null; + /** @enum {string} */ + type: "private_key"; + }; + /** + * @description SSH server host-key verification policy. + * @enum {string} + */ + SshHostKeyVerification: "known_hosts"; + /** @description Complete ephemeral SSH connection descriptor. */ + SshTunnelConfig: { + /** @description Password or private-key authentication. */ + authentication: components["schemas"]["SshAuthentication"]; + /** @description Server host-key verification policy. */ + hostKeyVerification?: components["schemas"]["SshHostKeyVerification"]; + /** @description SSH server hostname or IP address. */ + hostName: string; + /** + * Format: int32 + * @description Preferred loopback listener port, or an OS-assigned port when absent/zero. + */ + localPort?: number | null; + /** + * Format: int32 + * @description SSH server port. + */ + port: number; + /** @description SSH username. */ + userName: string; + }; /** @description Request to start one bounded agent run in an existing session. */ StartAgentRunRequest: { /** @description New user message. */ diff --git a/contracts/openapi/chat2db-v1.json b/contracts/openapi/chat2db-v1.json index 04b5221..2428721 100644 --- a/contracts/openapi/chat2db-v1.json +++ b/contracts/openapi/chat2db-v1.json @@ -7247,6 +7247,17 @@ "readOnly": { "type": "boolean", "description": "Whether sessions opened from this descriptor must be read-only." + }, + "ssh": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/SshTunnelConfig", + "description": "Optional SSH local-forward settings stored with the encrypted connection descriptor." + } + ] } } }, @@ -8957,6 +8968,108 @@ "ask_before_write" ] }, + "SshAuthentication": { + "oneOf": [ + { + "type": "object", + "description": "Password authentication.", + "required": [ + "password", + "type" + ], + "properties": { + "password": { + "type": "string", + "description": "SSH password, never returned or logged." + }, + "type": { + "type": "string", + "enum": [ + "password" + ] + } + } + }, + { + "type": "object", + "description": "OpenSSH-compatible private-key authentication.", + "required": [ + "key_file", + "type" + ], + "properties": { + "key_file": { + "type": "string", + "description": "User-selected local private-key path." + }, + "passphrase": { + "type": [ + "string", + "null" + ], + "description": "Optional encrypted-key passphrase, never returned or logged." + }, + "type": { + "type": "string", + "enum": [ + "private_key" + ] + } + } + } + ], + "description": "SSH user authentication material accepted only at a connection boundary." + }, + "SshHostKeyVerification": { + "type": "string", + "description": "SSH server host-key verification policy.", + "enum": [ + "known_hosts" + ] + }, + "SshTunnelConfig": { + "type": "object", + "description": "Complete ephemeral SSH connection descriptor.", + "required": [ + "hostName", + "port", + "userName", + "authentication" + ], + "properties": { + "authentication": { + "$ref": "#/components/schemas/SshAuthentication", + "description": "Password or private-key authentication." + }, + "hostKeyVerification": { + "$ref": "#/components/schemas/SshHostKeyVerification", + "description": "Server host-key verification policy." + }, + "hostName": { + "type": "string", + "description": "SSH server hostname or IP address." + }, + "localPort": { + "type": [ + "integer", + "null" + ], + "format": "int32", + "description": "Preferred loopback listener port, or an OS-assigned port when absent/zero.", + "minimum": 0 + }, + "port": { + "type": "integer", + "format": "int32", + "description": "SSH server port.", + "minimum": 0 + }, + "userName": { + "type": "string", + "description": "SSH username." + } + } + }, "StartAgentRunRequest": { "type": "object", "description": "Request to start one bounded agent run in an existing session.", diff --git a/crates/chat2db-contract/Cargo.toml b/crates/chat2db-contract/Cargo.toml index bb302df..eb0a64d 100644 --- a/crates/chat2db-contract/Cargo.toml +++ b/crates/chat2db-contract/Cargo.toml @@ -10,10 +10,10 @@ repository.workspace = true [dependencies] serde.workspace = true +serde_json.workspace = true utoipa.workspace = true [dev-dependencies] -serde_json.workspace = true [lints] workspace = true diff --git a/crates/chat2db-contract/src/community.rs b/crates/chat2db-contract/src/community.rs index 6694f1c..36ba20e 100644 --- a/crates/chat2db-contract/src/community.rs +++ b/crates/chat2db-contract/src/community.rs @@ -543,6 +543,32 @@ pub struct CommunityRoutineInvocationPreview { pub sql: String, } +/// Request to preview or execute replacement of one Community routine. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct CommunityRoutineMigrationRequest { + pub datasource_id: String, + pub database_type: String, + pub database_name: String, + pub schema_name: String, + pub routine_type: String, + pub routine_name: String, + pub ddl: String, +} + +/// Result of a compensating `MySQL` routine replacement. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct CommunityRoutineMigrationExecution { + pub success: bool, + pub message: String, + pub sql: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub failure_stage: Option, + pub restore_attempted: bool, + pub restore_succeeded: bool, +} + /// Secret-free Community trigger metadata. #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, ToSchema)] #[serde(rename_all = "camelCase")] diff --git a/crates/chat2db-contract/src/community_account.rs b/crates/chat2db-contract/src/community_account.rs new file mode 100644 index 0000000..dbbb5ca --- /dev/null +++ b/crates/chat2db-contract/src/community_account.rs @@ -0,0 +1,334 @@ +use std::fmt::{Debug, Formatter}; + +use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; + +/// `MySQL` account action exposed by the retained Community account UI. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub enum CommunityAccountAction { + CreateUser, + AlterPassword, + LockAccount, + UnlockAccount, + DropUser, + GrantPrivilege, + RevokePrivilege, +} + +/// `MySQL` privilege scope exposed by the retained Community account UI. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub enum CommunityAccountPrivilegeScope { + Global, + Database, + Table, +} + +/// `MySQL` privilege accepted by Community account grant and revoke operations. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub enum CommunityMysqlPrivilege { + Select, + Insert, + Update, + Delete, + Create, + Drop, + Alter, + Index, + References, + Execute, + ShowView, + Trigger, + Event, + CreateTemporaryTables, +} + +impl CommunityMysqlPrivilege { + /// Complete privilege allowlist in the order presented by Community. + pub const ALL: [Self; 14] = [ + Self::Select, + Self::Insert, + Self::Update, + Self::Delete, + Self::Create, + Self::Drop, + Self::Alter, + Self::Index, + Self::References, + Self::Execute, + Self::ShowView, + Self::Trigger, + Self::Event, + Self::CreateTemporaryTables, + ]; + + /// Community wire value used by capability responses and command payloads. + #[must_use] + pub const fn wire_name(self) -> &'static str { + match self { + Self::Select => "SELECT", + Self::Insert => "INSERT", + Self::Update => "UPDATE", + Self::Delete => "DELETE", + Self::Create => "CREATE", + Self::Drop => "DROP", + Self::Alter => "ALTER", + Self::Index => "INDEX", + Self::References => "REFERENCES", + Self::Execute => "EXECUTE", + Self::ShowView => "SHOW_VIEW", + Self::Trigger => "TRIGGER", + Self::Event => "EVENT", + Self::CreateTemporaryTables => "CREATE_TEMPORARY_TABLES", + } + } +} + +/// Request for grants belonging to one `MySQL` account. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct CommunityAccountGrantsRequest { + #[serde(rename = "dataSourceId", alias = "datasourceId")] + pub datasource_id: String, + pub user: String, + pub host: String, +} + +/// Preview or execution request for one `MySQL` account operation. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct CommunityAccountCommandRequest { + #[serde(rename = "dataSourceId", alias = "datasourceId")] + pub datasource_id: String, + pub user: String, + pub host: String, + pub action_type: CommunityAccountAction, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub scope: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub database_name: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub table_name: Option, + #[serde(default)] + pub privileges: Vec, + #[serde(default)] + pub grant_option: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schema(write_only)] + pub password: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub preview_token: Option, +} + +impl Debug for CommunityAccountCommandRequest { + fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("CommunityAccountCommandRequest") + .field("datasource_id", &self.datasource_id) + .field("user", &self.user) + .field("host", &self.host) + .field("action_type", &self.action_type) + .field("scope", &self.scope) + .field("database_name", &self.database_name) + .field("table_name", &self.table_name) + .field("privileges", &self.privileges) + .field("grant_option", &self.grant_option) + .field("password", &self.password.as_ref().map(|_| "[REDACTED]")) + .field( + "preview_token", + &self.preview_token.as_ref().map(|_| "[REDACTED]"), + ) + .finish() + } +} + +/// `MySQL` server and permission capabilities for account administration. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct CommunityAccountCapability { + pub db_type: String, + pub product_name: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub product_version: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub current_user: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub connection_user: Option, + pub account_list_readable: bool, + pub account_lock_supported: bool, + pub editable_privileges: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub message: Option, +} + +/// One `MySQL` account projected for the retained Community account tree. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct CommunityAccount { + pub user: String, + pub host: String, + pub display_name: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub authentication_plugin: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub locked: Option, +} + +/// Stable `MySQL` account collection. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct CommunityAccountList { + pub items: Vec, +} + +/// Stable `SHOW GRANTS` collection for one `MySQL` account. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct CommunityAccountGrantList { + pub items: Vec, +} + +/// Masked SQL preview and authorization token for one account operation. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct CommunityAccountPreview { + pub action_type: CommunityAccountAction, + pub sql: String, + pub preview_token: String, +} + +impl Debug for CommunityAccountPreview { + fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("CommunityAccountPreview") + .field("action_type", &self.action_type) + .field("sql", &"[REDACTED]") + .field("preview_token", &"[REDACTED]") + .finish() + } +} + +/// Result of executing one preview-authorized `MySQL` account operation. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct CommunityAccountExecution { + pub action_type: CommunityAccountAction, + pub sql: String, + pub success: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub message: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub failure_code: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_code: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sql_state: Option, +} + +impl Debug for CommunityAccountExecution { + fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("CommunityAccountExecution") + .field("action_type", &self.action_type) + .field("sql", &"[REDACTED]") + .field("success", &self.success) + .field("message", &self.message.as_ref().map(|_| "[REDACTED]")) + .field("failure_code", &self.failure_code) + .field("error_code", &self.error_code) + .field("sql_state", &self.sql_state) + .finish() + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::{ + CommunityAccountAction, CommunityAccountCommandRequest, CommunityAccountExecution, + CommunityAccountPreview, CommunityAccountPrivilegeScope, CommunityMysqlPrivilege, + }; + + #[test] + fn command_wire_shape_matches_the_retained_frontend() { + let request = command("pa'ss\\word"); + + assert_eq!( + serde_json::to_value(&request).expect("account command must serialize"), + json!({ + "dataSourceId": "42", + "user": "reader", + "host": "%", + "actionType": "GRANT_PRIVILEGE", + "scope": "TABLE", + "databaseName": "inventory", + "tableName": "orders", + "privileges": ["SELECT", "SHOW_VIEW"], + "grantOption": true, + "password": "pa'ss\\word", + "previewToken": "sensitive-preview-token-value" + }) + ); + } + + #[test] + fn debug_output_never_exposes_password_tokens_or_sql() { + let password = "plain-secret"; + let command_debug = format!("{:?}", command(password)); + assert!(!command_debug.contains(password)); + assert!(!command_debug.contains("sensitive-preview-token-value")); + assert!(command_debug.contains("[REDACTED]")); + + let preview = CommunityAccountPreview { + action_type: CommunityAccountAction::CreateUser, + sql: "CREATE USER 'reader'@'%' IDENTIFIED BY 'plain-secret'".to_owned(), + preview_token: "sensitive-preview-token-value".to_owned(), + }; + let preview_debug = format!("{preview:?}"); + assert!(!preview_debug.contains("CREATE USER")); + assert!(!preview_debug.contains("plain-secret")); + assert!(!preview_debug.contains("sensitive-preview-token-value")); + + let execution = CommunityAccountExecution { + action_type: CommunityAccountAction::CreateUser, + sql: preview.sql, + success: false, + message: Some("near plain-secret".to_owned()), + failure_code: Some("mysql.account.executeFailed".to_owned()), + error_code: Some(1064), + sql_state: Some("42000".to_owned()), + }; + let execution_debug = format!("{execution:?}"); + assert!(!execution_debug.contains("CREATE USER")); + assert!(!execution_debug.contains("plain-secret")); + } + + #[test] + fn privilege_allowlist_is_complete_and_stable() { + assert_eq!(CommunityMysqlPrivilege::ALL.len(), 14); + assert_eq!(CommunityMysqlPrivilege::ALL[0].wire_name(), "SELECT"); + assert_eq!(CommunityMysqlPrivilege::ALL[10].wire_name(), "SHOW_VIEW"); + assert_eq!( + CommunityMysqlPrivilege::ALL[13].wire_name(), + "CREATE_TEMPORARY_TABLES" + ); + } + + fn command(password: &str) -> CommunityAccountCommandRequest { + CommunityAccountCommandRequest { + datasource_id: "42".to_owned(), + user: "reader".to_owned(), + host: "%".to_owned(), + action_type: CommunityAccountAction::GrantPrivilege, + scope: Some(CommunityAccountPrivilegeScope::Table), + database_name: Some("inventory".to_owned()), + table_name: Some("orders".to_owned()), + privileges: vec!["SELECT".to_owned(), "SHOW_VIEW".to_owned()], + grant_option: true, + password: Some(password.to_owned()), + preview_token: Some("sensitive-preview-token-value".to_owned()), + } + } +} diff --git a/crates/chat2db-contract/src/community_dashboard.rs b/crates/chat2db-contract/src/community_dashboard.rs new file mode 100644 index 0000000..1e533c2 --- /dev/null +++ b/crates/chat2db-contract/src/community_dashboard.rs @@ -0,0 +1,225 @@ +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use utoipa::ToSchema; + +/// One Community dashboard persisted in the local workspace. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct CommunityDashboard { + pub id: i64, + pub gmt_create: i64, + pub gmt_modified: i64, + pub name: Option, + pub description: Option, + pub data_source_collection_id: Option, + #[serde(default)] + pub chart_ids: Vec, + pub schema: Option, + pub refresh_type: Option, + pub refresh_cycle: Option, + pub user_id: Option, +} + +/// Community-compatible dashboard list query. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct CommunityDashboardListQuery { + #[serde(default = "default_page_no")] + pub page_no: u32, + #[serde(default = "default_page_size")] + pub page_size: u32, + #[serde(default)] + pub search_key: Option, +} + +impl Default for CommunityDashboardListQuery { + fn default() -> Self { + Self { + page_no: default_page_no(), + page_size: default_page_size(), + search_key: None, + } + } +} + +/// One stable page of Community dashboards. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct CommunityDashboardPage { + pub data: Vec, + pub total: u64, + pub page_no: u32, + pub page_size: u32, + pub has_next_page: bool, +} + +/// Fields accepted when a Community dashboard is created. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct CreateCommunityDashboardRequest { + pub name: Option, + pub description: Option, + pub data_source_collection_id: Option, + #[serde(default)] + pub chart_ids: Vec, + pub schema: Option, + pub refresh_type: Option, + pub refresh_cycle: Option, + pub user_id: Option, +} + +/// Non-null partial Community dashboard update. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct UpdateCommunityDashboardRequest { + pub name: Option, + pub description: Option, + pub data_source_collection_id: Option, + pub chart_ids: Option>, + pub schema: Option, + pub refresh_type: Option, + pub refresh_cycle: Option, + pub user_id: Option, +} + +/// One Community chart persisted in the local workspace. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct CommunityChart { + pub id: i64, + pub gmt_create: i64, + pub gmt_modified: i64, + pub name: Option, + pub description: Option, + pub schema: Option, + pub data_source_id: Option, + pub data_source_name: Option, + pub schema_name: Option, + pub r#type: Option, + pub database_name: Option, + pub ddl: Option, + pub deleted: Option, + pub user_id: Option, + pub chart_schema: Option, + pub meta_data: Option, + pub database_info: Option, + pub refresh_type: Option, + pub refresh_cycle: Option, +} + +/// Fields accepted when a Community chart is created. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct CreateCommunityChartRequest { + pub name: Option, + pub description: Option, + pub schema: Option, + pub data_source_id: Option, + pub data_source_name: Option, + pub schema_name: Option, + pub r#type: Option, + pub database_name: Option, + pub ddl: Option, + pub deleted: Option, + pub user_id: Option, + pub chart_schema: Option, + pub meta_data: Option, + pub database_info: Option, + pub refresh_type: Option, + pub refresh_cycle: Option, +} + +/// Non-null partial Community chart update. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct UpdateCommunityChartRequest { + pub name: Option, + pub description: Option, + pub schema: Option, + pub data_source_id: Option, + pub data_source_name: Option, + pub schema_name: Option, + pub r#type: Option, + pub database_name: Option, + pub ddl: Option, + pub deleted: Option, + pub user_id: Option, + pub chart_schema: Option, + pub meta_data: Option, + pub database_info: Option, + pub refresh_type: Option, + pub refresh_cycle: Option, +} + +/// Community chart-detail query, including the optional SQL refresh switch. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct CommunityChartDetailQuery { + pub chart_id: i64, + #[serde(default)] + pub refresh: bool, +} + +const fn default_page_no() -> u32 { + 1 +} + +const fn default_page_size() -> u32 { + 20 +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::{CommunityChart, CommunityDashboardListQuery, CreateCommunityDashboardRequest}; + + #[test] + fn dashboard_contract_uses_community_camel_case_and_defaults() { + let query: CommunityDashboardListQuery = + serde_json::from_value(json!({})).expect("default query decodes"); + assert_eq!(query.page_no, 1); + assert_eq!(query.page_size, 20); + + let request: CreateCommunityDashboardRequest = serde_json::from_value(json!({ + "name": "Sales", + "refreshCycle": {"unit": "seconds", "value": 30} + })) + .expect("dashboard request decodes"); + assert!(request.chart_ids.is_empty()); + let encoded = serde_json::to_value(request).expect("dashboard request encodes"); + assert_eq!(encoded["name"], "Sales"); + assert_eq!(encoded["refreshCycle"]["value"], 30); + assert!(encoded.get("chart_ids").is_none()); + } + + #[test] + fn chart_json_fields_round_trip_without_stringification() { + let source = json!({ + "id": 9, + "gmtCreate": 10, + "gmtModified": 11, + "name": "Revenue", + "description": null, + "schema": null, + "dataSourceId": 12, + "dataSourceName": "MySQL", + "schemaName": null, + "type": "BAR", + "databaseName": "analytics", + "ddl": "select 1", + "deleted": "N", + "userId": null, + "chartSchema": {"title": "Revenue", "series": [1, 2]}, + "metaData": {"dataList": [{"amount": 42}]}, + "databaseInfo": {"sql": "select 1"}, + "refreshType": "MANUAL", + "refreshCycle": {"cron": "0 * * * *"} + }); + let chart: CommunityChart = serde_json::from_value(source.clone()).expect("chart decodes"); + assert_eq!( + serde_json::to_value(chart).expect("chart re-encodes"), + source + ); + } +} diff --git a/crates/chat2db-contract/src/community_diff.rs b/crates/chat2db-contract/src/community_diff.rs new file mode 100644 index 0000000..c008baa --- /dev/null +++ b/crates/chat2db-contract/src/community_diff.rs @@ -0,0 +1,128 @@ +use std::fmt; + +use serde::{Deserialize, Deserializer, Serialize, de::Visitor}; +use utoipa::ToSchema; + +/// One source or target namespace selected by the retained Community schema-sync UI. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct CommunitySchemaDiffEndpoint { + #[serde( + default, + rename = "dataSourceId", + alias = "datasourceId", + deserialize_with = "deserialize_datasource_id" + )] + pub datasource_id: String, + #[serde(default)] + pub database_name: String, + #[serde(default)] + pub schema_name: String, +} + +/// Historical `/api/diff/sql` request, directed from the desired source to the target. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct CommunitySchemaDiffRequest { + pub source: CommunitySchemaDiffEndpoint, + pub target: CommunitySchemaDiffEndpoint, +} + +/// SQL preview returned by the historical endpoint as a JSON string. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(transparent)] +pub struct CommunitySchemaDiffSql(pub String); + +impl CommunitySchemaDiffSql { + #[must_use] + pub fn new(sql: impl Into) -> Self { + Self(sql.into()) + } + + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } + + #[must_use] + pub fn into_inner(self) -> String { + self.0 + } +} + +fn deserialize_datasource_id<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + struct DatasourceIdVisitor; + + impl Visitor<'_> for DatasourceIdVisitor { + type Value = String; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a string or integer datasource id") + } + + fn visit_str(self, value: &str) -> Result { + Ok(value.to_owned()) + } + + fn visit_string(self, value: String) -> Result { + Ok(value) + } + + fn visit_u64(self, value: u64) -> Result { + Ok(value.to_string()) + } + + fn visit_i64(self, value: i64) -> Result { + Ok(value.to_string()) + } + + fn visit_unit(self) -> Result { + Ok(String::new()) + } + + fn visit_none(self) -> Result { + Ok(String::new()) + } + } + + deserializer.deserialize_any(DatasourceIdVisitor) +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::{CommunitySchemaDiffRequest, CommunitySchemaDiffSql}; + + #[test] + fn historical_numeric_datasource_ids_deserialize_to_canonical_strings() { + let request: CommunitySchemaDiffRequest = serde_json::from_value(json!({ + "source": { + "dataSourceId": 42, + "databaseName": "source_db", + "schemaName": "" + }, + "target": { + "dataSourceId": "target-uuid", + "databaseName": "target_db" + } + })) + .expect("historical schema diff request must deserialize"); + + assert_eq!(request.source.datasource_id, "42"); + assert_eq!(request.target.datasource_id, "target-uuid"); + assert_eq!(request.target.schema_name, ""); + } + + #[test] + fn schema_diff_sql_retains_the_historical_json_string_shape() { + assert_eq!( + serde_json::to_value(CommunitySchemaDiffSql::new("ALTER TABLE `t` ADD `c` INT;")) + .expect("schema diff SQL must serialize"), + json!("ALTER TABLE `t` ADD `c` INT;") + ); + } +} diff --git a/crates/chat2db-contract/src/datasource.rs b/crates/chat2db-contract/src/datasource.rs index 83eb8e2..1323be7 100644 --- a/crates/chat2db-contract/src/datasource.rs +++ b/crates/chat2db-contract/src/datasource.rs @@ -3,6 +3,8 @@ use std::fmt::{Debug, Formatter}; use serde::{Deserialize, Serialize}; use utoipa::ToSchema; +use crate::SshTunnelConfig; + /// Complete connection descriptor accepted only at a secret-handling boundary. #[derive(Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] #[serde(rename_all = "camelCase")] @@ -13,6 +15,9 @@ pub struct DatasourceConnection { pub properties: Vec, /// Whether sessions opened from this descriptor must be read-only. pub read_only: bool, + /// Optional SSH local-forward settings stored with the encrypted connection descriptor. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ssh: Option, } impl Debug for DatasourceConnection { @@ -22,6 +27,7 @@ impl Debug for DatasourceConnection { .field("jdbc_url", &"[REDACTED]") .field("properties", &self.properties) .field("read_only", &self.read_only) + .field("ssh_configured", &self.ssh.is_some()) .finish() } } @@ -161,6 +167,7 @@ mod tests { sensitive: true, }], read_only: true, + ssh: None, }, }; @@ -186,6 +193,7 @@ mod tests { sensitive: true, }], read_only: false, + ssh: None, }), }; diff --git a/crates/chat2db-contract/src/datasource_compatibility.rs b/crates/chat2db-contract/src/datasource_compatibility.rs new file mode 100644 index 0000000..9a7243a --- /dev/null +++ b/crates/chat2db-contract/src/datasource_compatibility.rs @@ -0,0 +1,207 @@ +use std::fmt::{Debug, Formatter}; + +use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; + +use crate::{CommunityDatabase, Datasource, SshTunnelEditProjection}; + +/// Request to clone one datasource under a new opaque id and vault reference. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct CloneDatasourceRequest { + /// Source datasource id. + pub id: String, + /// Optional replacement name. The source name plus ` Copy` is used when absent. + #[serde(skip_serializing_if = "Option::is_none")] + pub name: Option, +} + +/// Request to export selected datasources, or every datasource when the list is empty. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct ExportCommunityDatasourcesRequest { + /// Selected opaque datasource ids. Empty selects all datasources. + #[serde(default)] + pub datasource_ids: Vec, +} + +/// Portable non-sensitive connection property. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct PortableDatasourceProperty { + /// Driver property name. + pub key: String, + /// Non-sensitive value. Import rejects credential-like keys. + pub value: String, +} + +impl Debug for PortableDatasourceProperty { + fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("PortableDatasourceProperty") + .field("key", &self.key) + .field("value", &"[REDACTED]") + .finish() + } +} + +/// Credential-free connection descriptor suitable for an explicit export file. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct PortableDatasourceConnection { + /// JDBC URL with userinfo and sensitive query parameters removed. + pub jdbc_url: String, + /// Ordered non-sensitive connection properties. + pub properties: Vec, + /// Whether the imported datasource should remain read-only. + pub read_only: bool, + /// Optional non-secret SSH endpoint and authentication mode. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ssh: Option, +} + +impl Debug for PortableDatasourceConnection { + fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("PortableDatasourceConnection") + .field("jdbc_url", &"[REDACTED]") + .field("properties", &self.properties) + .field("read_only", &self.read_only) + .field("ssh", &self.ssh) + .finish() + } +} + +/// One datasource in the portable Community JSON document. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct PortableCommunityDatasource { + /// Original id retained for traceability only. Import never updates this id. + #[serde(skip_serializing_if = "Option::is_none")] + pub source_id: Option, + /// User-visible datasource name. + #[serde(alias = "alias")] + pub name: String, + /// Rust/native or compatibility driver identity. + pub driver_id: String, + /// Optional credential-free connection descriptor. + #[serde(skip_serializing_if = "Option::is_none")] + pub connection: Option, +} + +/// Versioned, secret-safe Community datasource export document. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct CommunityDatasourceExport { + /// Document schema version. Version `1` is the only currently accepted value. + pub schema_version: u32, + /// Export time as Unix epoch milliseconds encoded as a decimal integer. + pub exported_at_ms: String, + /// Selected datasource definitions. + pub datasources: Vec, +} + +/// Result of importing a portable Community datasource document. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct CommunityDatasourceImportResult { + /// Number of new datasource records created. + pub count: u32, + /// Secret-free metadata for the new records. + pub created: Vec, +} + +/// Connection ownership model exposed to the retained Community client. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum DatasourceSessionMode { + /// Every operation owns and closes its database connection. + Ephemeral, +} + +/// Result of Community's datasource `connect` compatibility operation. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct DatasourceConnectResult { + /// Opaque datasource id that was connected and inspected. + pub datasource_id: String, + /// Explicit connection ownership model. + pub session_mode: DatasourceSessionMode, + /// Databases returned by the real ephemeral connection. + pub databases: Vec, +} + +/// Result of Community's Console `connect` compatibility operation. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct ConsoleConnectResult { + /// Opaque datasource id whose connection was verified. + pub datasource_id: String, + /// Explicit connection ownership model. + pub session_mode: DatasourceSessionMode, + /// True only after a real database connection and ping succeeded. + pub verified: bool, +} + +/// Result of explicit datasource or generic connection close. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct DatasourceCloseResult { + /// Opaque datasource id accepted by the close operation. + pub datasource_id: String, + /// Explicit connection ownership model. + pub session_mode: DatasourceSessionMode, + /// Always zero while connections are operation-scoped and already closed. + pub closed_connections: u32, +} + +/// Frontend-requested mutation of a driver artifact. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum NativeDriverAction { + /// Download an advertised driver. + Download, + /// Save selected custom artifacts. + Save, + /// Delete selected custom artifacts. + Delete, +} + +/// Explicit response for driver actions satisfied by a native Rust implementation. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct NativeDriverCompatibility { + /// Normalized Community database type. + pub database_type: String, + /// Stable Rust driver identity. + pub driver_id: String, + /// Requested frontend action. + pub action: NativeDriverAction, + /// Native implementation name. + pub implementation: String, + /// Always false: native `MySQL` never needs a Java JAR. + pub artifact_required: bool, + /// Always false: the native implementation is immutable at runtime. + pub changed: bool, +} + +#[cfg(test)] +mod tests { + use super::{PortableDatasourceConnection, PortableDatasourceProperty}; + + #[test] + fn portable_connection_debug_redacts_values() { + let connection = PortableDatasourceConnection { + jdbc_url: "jdbc:mysql://localhost/demo".to_owned(), + properties: vec![PortableDatasourceProperty { + key: "user".to_owned(), + value: "sentinel-user".to_owned(), + }], + read_only: false, + ssh: None, + }; + let debug = format!("{connection:?}"); + assert!(!debug.contains("sentinel-user")); + assert!(!debug.contains("jdbc:mysql")); + } +} diff --git a/crates/chat2db-contract/src/datasource_converter.rs b/crates/chat2db-contract/src/datasource_converter.rs new file mode 100644 index 0000000..7da3974 --- /dev/null +++ b/crates/chat2db-contract/src/datasource_converter.rs @@ -0,0 +1,72 @@ +use std::fmt::{Debug, Formatter}; + +use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; + +use crate::Datasource; + +/// Community datasource file formats accepted by the compatibility importer. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum CommunityDatasourceImportFormat { + /// `Chat2DB`'s portable object or legacy datasource array JSON. + Chat2dbJson, + /// Navicat connection export XML (`.ncx`). + NavicatNcx, + /// `DBeaver` project export ZIP (`.dbp`). + DbeaverDbp, + /// `DataGrip` clipboard datasource settings text. + DatagripText, +} + +/// Raw Community datasource file submitted to Core for validation and import. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct CommunityDatasourceFileImportRequest { + /// Explicit format selected by the delivery adapter. + pub format: CommunityDatasourceImportFormat, + /// Raw file bytes. Core applies its own size limits before parsing. + pub content: Vec, +} + +impl Debug for CommunityDatasourceFileImportRequest { + fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("CommunityDatasourceFileImportRequest") + .field("format", &self.format) + .field( + "content", + &format_args!("[REDACTED; {} bytes]", self.content.len()), + ) + .finish() + } +} + +/// Result of importing one third-party or legacy datasource file. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct CommunityDatasourceFileImportResult { + /// Number of new datasource records created. + pub count: u32, + /// Secret-free metadata for the new records. + pub created: Vec, + /// Entries intentionally ignored because they are not supported native `MySQL` definitions. + pub skipped: u32, +} + +#[cfg(test)] +mod tests { + use super::{CommunityDatasourceFileImportRequest, CommunityDatasourceImportFormat}; + + #[test] + fn import_request_debug_never_exposes_file_contents() { + let request = CommunityDatasourceFileImportRequest { + format: CommunityDatasourceImportFormat::DbeaverDbp, + content: b"sentinel-password-and-token".to_vec(), + }; + + let debug = format!("{request:?}"); + assert!(!debug.contains("sentinel-password")); + assert!(debug.contains("27 bytes")); + } +} diff --git a/crates/chat2db-contract/src/datasource_edit.rs b/crates/chat2db-contract/src/datasource_edit.rs new file mode 100644 index 0000000..9aad373 --- /dev/null +++ b/crates/chat2db-contract/src/datasource_edit.rs @@ -0,0 +1,93 @@ +use std::fmt::{Debug, Formatter}; + +use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; + +use crate::{PortableDatasourceProperty, SshTunnelEditProjection}; + +/// Secret-safe datasource details used to populate an edit form. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct DatasourceEditProjection { + /// Opaque datasource id. + pub id: String, + /// User-visible datasource name. + pub name: String, + /// Rust/native or compatibility driver identity. + pub driver_id: String, + /// JDBC URL with userinfo, fragments, and sensitive query values removed. + pub jdbc_url: String, + /// Non-secret database username, when one is configured. + #[serde(skip_serializing_if = "Option::is_none")] + pub username: Option, + /// Ordered non-sensitive properties, excluding the separately projected username. + pub properties: Vec, + /// Whether sessions opened from this datasource must be read-only. + pub read_only: bool, + /// Optional non-secret SSH settings used to populate the retained connection form. + #[serde(skip_serializing_if = "Option::is_none")] + pub ssh: Option, + /// Whether a complete connection descriptor exists in the vault. + pub has_secret: bool, + /// Monotonic revision encoded as a decimal integer. + pub revision: String, +} + +impl Debug for DatasourceEditProjection { + fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("DatasourceEditProjection") + .field("id", &self.id) + .field("name", &self.name) + .field("driver_id", &self.driver_id) + .field("jdbc_url", &"[REDACTED]") + .field("username", &self.username.as_ref().map(|_| "[REDACTED]")) + .field("properties", &self.properties) + .field("read_only", &self.read_only) + .field("ssh", &self.ssh) + .field("has_secret", &self.has_secret) + .field("revision", &self.revision) + .finish() + } +} + +#[cfg(test)] +mod tests { + use super::DatasourceEditProjection; + use crate::PortableDatasourceProperty; + + #[test] + fn edit_projection_serializes_only_explicitly_safe_connection_fields() { + let projection = DatasourceEditProjection { + id: "datasource-1".to_owned(), + name: "Production".to_owned(), + driver_id: "mysql".to_owned(), + jdbc_url: "jdbc:mysql://localhost/demo?useSSL=false".to_owned(), + username: Some("sentinel-user".to_owned()), + properties: vec![PortableDatasourceProperty { + key: "connectionTimeZone".to_owned(), + value: "sentinel-zone".to_owned(), + }], + read_only: false, + ssh: None, + has_secret: true, + revision: "3".to_owned(), + }; + + let json = serde_json::to_string(&projection).expect("projection serializes"); + assert!(json.contains("sentinel-user")); + assert!(json.contains("sentinel-zone")); + for forbidden in ["password", "token", "credential", "privateKey"] { + assert!( + !json + .to_ascii_lowercase() + .contains(&forbidden.to_ascii_lowercase()) + ); + } + + let debug = format!("{projection:?}"); + assert!(!debug.contains("sentinel-user")); + assert!(!debug.contains("sentinel-zone")); + assert!(!debug.contains("jdbc:mysql")); + } +} diff --git a/crates/chat2db-contract/src/lib.rs b/crates/chat2db-contract/src/lib.rs index b7b9fd4..5b0e122 100644 --- a/crates/chat2db-contract/src/lib.rs +++ b/crates/chat2db-contract/src/lib.rs @@ -2,13 +2,23 @@ pub mod agent; pub mod community; +pub mod community_account; +pub mod community_dashboard; +pub mod community_diff; pub mod datasource; +pub mod datasource_compatibility; +pub mod datasource_converter; +pub mod datasource_edit; pub mod driver; pub mod error; +pub mod mysql_workspace; pub mod operation; pub mod query; pub mod result; +pub mod ssh; pub mod system; +pub mod transfer; +pub mod workspace; pub use agent::{ AgentEvent, AgentEventEnvelope, AgentMessage, AgentMessageContent, AgentMessageList, @@ -31,7 +41,8 @@ pub use community::{ CommunityParsedStatement, CommunityPlugin, CommunityPluginBehavior, CommunityPluginCatalog, CommunityPluginServices, CommunityPrimaryKey, CommunityPrimaryKeyList, CommunityProcedure, CommunityProcedureList, CommunityProcedureParameter, CommunityProcedureParameterList, - CommunityRoutineInvocationPreview, CommunitySchema, CommunitySchemaList, CommunitySqlAnalysis, + CommunityRoutineInvocationPreview, CommunityRoutineMigrationExecution, + CommunityRoutineMigrationRequest, CommunitySchema, CommunitySchemaList, CommunitySqlAnalysis, CommunitySqlCompletion, CommunitySqlCompletionActiveSnippetSlot, CommunitySqlCompletionCandidate, CommunitySqlCompletionEditorHint, CommunitySqlCompletionEditorHintItem, CommunitySqlCompletionRange, CommunitySqlDiagnostic, @@ -46,22 +57,71 @@ pub use community::{ ParseCommunitySqlRequest, PreviewCommunityRoutineInvocationRequest, StartCommunityTablePreviewRequest, ValidateCommunitySqlRequest, }; +pub use community_account::{ + CommunityAccount, CommunityAccountAction, CommunityAccountCapability, + CommunityAccountCommandRequest, CommunityAccountExecution, CommunityAccountGrantList, + CommunityAccountGrantsRequest, CommunityAccountList, CommunityAccountPreview, + CommunityAccountPrivilegeScope, CommunityMysqlPrivilege, +}; +pub use community_dashboard::{ + CommunityChart, CommunityChartDetailQuery, CommunityDashboard, CommunityDashboardListQuery, + CommunityDashboardPage, CreateCommunityChartRequest, CreateCommunityDashboardRequest, + UpdateCommunityChartRequest, UpdateCommunityDashboardRequest, +}; +pub use community_diff::{ + CommunitySchemaDiffEndpoint, CommunitySchemaDiffRequest, CommunitySchemaDiffSql, +}; pub use datasource::{ CreateDatasourceRequest, Datasource, DatasourceConnection, DatasourceConnectionProperty, DatasourceList, DatasourceSecretChange, UpdateDatasourceRequest, }; +pub use datasource_compatibility::{ + CloneDatasourceRequest, CommunityDatasourceExport, CommunityDatasourceImportResult, + ConsoleConnectResult, DatasourceCloseResult, DatasourceConnectResult, DatasourceSessionMode, + ExportCommunityDatasourcesRequest, NativeDriverAction, NativeDriverCompatibility, + PortableCommunityDatasource, PortableDatasourceConnection, PortableDatasourceProperty, +}; +pub use datasource_converter::{ + CommunityDatasourceFileImportRequest, CommunityDatasourceFileImportResult, + CommunityDatasourceImportFormat, +}; +pub use datasource_edit::DatasourceEditProjection; pub use driver::{JdbcDriver, JdbcDriverList}; pub use error::{ApiError, ApiErrorDetails}; +pub use mysql_workspace::{ + CommunityErColumn, CommunityErForeignKey, CommunityErModel, CommunityErPositionRequest, + CommunityErQueryRequest, CommunityErTable, CommunityPinnedTableList, + CommunityPinnedTableRequest, +}; pub use operation::{ CancelDisposition, CancelOperationResponse, OperationEvent, OperationEventEnvelope, OperationSnapshot, OperationStatus, OperationStreamMessage, OperationSubscriptionAccepted, }; -pub use query::{JdbcValue, QueryAccepted, QueryLimits, QueryParameter, StartQueryRequest}; +pub use query::{ + DatabaseWriteResult, DatabaseWriteState, ExecuteDatabaseWriteRequest, JdbcValue, QueryAccepted, + QueryLimits, QueryParameter, StartQueryRequest, +}; pub use result::{ ColumnNullability, JdbcValueType, ResultColumn, ResultMetadata, ResultPage, ResultPageRequest, ResultRow, }; +pub use ssh::{ + SshAuthentication, SshAuthenticationType, SshConnectionTestResult, + SshDatasourcePreConnectRequest, SshDatasourcePreConnectResult, SshHostKeyVerification, + SshTunnelConfig, SshTunnelEditProjection, +}; pub use system::{ComponentHealth, ComponentState, HealthResponse, ProductInfo, RuntimeStatus}; +pub use transfer::{ + DmlExportFormat, DmlExportRequest, DmlExportSize, GenerateMysqlClassRequest, + GeneratedMysqlClassSet, ImportFileRequest, OtherFileExportRequest, SqlFileExportRequest, + TabularImportEncoding, TransferArtifact, TransferFileFormat, TransferSqlScope, TransferTask, + TransferTaskAccepted, TransferTaskKind, TransferTaskPage, TransferTaskStatus, +}; +pub use workspace::{ + AssignDatasourceNamespaceRequest, CreateWorkspaceNamespaceRequest, MoveWorkspaceNodeRequest, + UpdateWorkspaceNamespaceRequest, WorkspaceDatasourceGroup, WorkspaceDatasourceList, + WorkspaceNamespace, WorkspaceNodeKind, WorkspaceNodeRef, WorkspaceTree, WorkspaceTreeNode, +}; #[cfg(test)] mod tests { @@ -88,8 +148,9 @@ mod tests { CommunityTableList, CommunityTablePreviewAccepted, CommunityViewList, CompleteCommunitySqlRequest, ComponentHealth, ComponentState, ContextCompactionStrategy, CreateAgentSessionRequest, CreateDatasourceRequest, CreateProviderProfileRequest, - Datasource, DatasourceConnection, DatasourceConnectionProperty, DatasourceList, - DatasourceSecretChange, DecideAgentPermissionRequest, FormatCommunitySqlRequest, + DatabaseWriteResult, DatabaseWriteState, Datasource, DatasourceConnection, + DatasourceConnectionProperty, DatasourceList, DatasourceSecretChange, + DecideAgentPermissionRequest, ExecuteDatabaseWriteRequest, FormatCommunitySqlRequest, HealthResponse, JdbcDriver, JdbcDriverList, JdbcValue, JdbcValueType, ListCommunityColumnsRequest, ListCommunityDatabasesRequest, ListCommunityIndexesRequest, ListCommunitySchemasRequest, ListCommunityTableKeysRequest, ListCommunityTablesRequest, @@ -208,6 +269,9 @@ mod tests { ProviderProfileList, ProviderSecretChange, QueryAccepted, + DatabaseWriteResult, + DatabaseWriteState, + ExecuteDatabaseWriteRequest, QueryLimits, QueryParameter, ResultColumn, @@ -266,6 +330,9 @@ mod tests { "OperationEventEnvelope", "OperationStreamMessage", "ResultPage", + "DatabaseWriteResult", + "DatabaseWriteState", + "ExecuteDatabaseWriteRequest", "StartQueryRequest", "StartCommunityTablePreviewRequest", ] { diff --git a/crates/chat2db-contract/src/mysql_workspace.rs b/crates/chat2db-contract/src/mysql_workspace.rs new file mode 100644 index 0000000..e5e1c16 --- /dev/null +++ b/crates/chat2db-contract/src/mysql_workspace.rs @@ -0,0 +1,119 @@ +use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; + +/// Community-compatible locator used by `MySQL` table pin operations. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct CommunityPinnedTableRequest { + pub data_source_id: String, + #[serde(default)] + pub database_name: String, + #[serde(default)] + pub schema_name: String, + #[serde(default)] + pub table_name: String, +} + +/// Stable pinned-table-name collection for one `MySQL` database/schema scope. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct CommunityPinnedTableList { + pub items: Vec, +} + +/// Community-compatible request for one `MySQL` ER model. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct CommunityErQueryRequest { + pub data_source_id: String, + #[serde(default)] + pub database_name: String, + #[serde(default)] + pub schema_name: String, +} + +/// Community-compatible request that persists an ER canvas layout. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct CommunityErPositionRequest { + pub data_source_id: String, + #[serde(default)] + pub database_name: String, + #[serde(default)] + pub schema_name: String, + #[serde(default)] + pub position: String, +} + +/// Column projection consumed by the retained Community ER canvas. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct CommunityErColumn { + pub name: String, + pub column_type: String, + pub primary_key: bool, + pub comment: String, +} + +/// Foreign-key edge projection consumed by the retained Community ER canvas. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct CommunityErForeignKey { + pub pk_table_name: String, + pub pk_column_name: String, + pub fk_table_name: String, + pub fk_column_name: String, +} + +/// One table node in the retained Community ER canvas. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct CommunityErTable { + pub name: String, + pub comment: String, + pub column_list: Vec, + pub foreign_key_list: Vec, +} + +/// Complete `MySQL` ER metadata plus the caller's last persisted canvas layout. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct CommunityErModel { + pub tables: Vec, + pub position: Option, +} + +#[cfg(test)] +mod tests { + use super::{CommunityErColumn, CommunityErForeignKey, CommunityErModel, CommunityErTable}; + + #[test] + fn er_model_uses_the_exact_retained_frontend_field_names() { + let model = CommunityErModel { + tables: vec![CommunityErTable { + name: "orders".to_owned(), + comment: String::new(), + column_list: vec![CommunityErColumn { + name: "id".to_owned(), + column_type: "BIGINT".to_owned(), + primary_key: true, + comment: String::new(), + }], + foreign_key_list: vec![CommunityErForeignKey { + pk_table_name: "users".to_owned(), + pk_column_name: "id".to_owned(), + fk_table_name: "orders".to_owned(), + fk_column_name: "user_id".to_owned(), + }], + }], + position: None, + }; + let value = serde_json::to_value(model).expect("ER model serializes"); + assert_eq!(value["tables"][0]["columnList"][0]["primaryKey"], true); + assert_eq!( + value["tables"][0]["foreignKeyList"][0]["pkTableName"], + "users" + ); + assert!(value["position"].is_null()); + } +} diff --git a/crates/chat2db-contract/src/query.rs b/crates/chat2db-contract/src/query.rs index 701dc18..dbf834e 100644 --- a/crates/chat2db-contract/src/query.rs +++ b/crates/chat2db-contract/src/query.rs @@ -1,6 +1,8 @@ use serde::{Deserialize, Serialize}; use utoipa::ToSchema; +use crate::error::ApiError; + /// Lossless JSON representation of one JDBC scalar value. /// /// Numeric values that could lose precision in JavaScript are decimal strings. @@ -141,9 +143,49 @@ pub struct QueryAccepted { pub operation_id: String, } +/// Explicit, single-statement database write requested by a local automation surface. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct ExecuteDatabaseWriteRequest { + /// Opaque datasource id. + pub datasource_id: String, + /// Exactly one SQL write statement. + pub sql: String, + /// Must be true at the trusted product boundary before any write is dispatched. + pub confirmed: bool, +} + +/// Whether a database write completed and whether retrying it can be safe. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum DatabaseWriteState { + /// The database confirmed successful completion. + Succeeded, + /// No SQL was dispatched to the database, so a corrected request may be retried. + NotStarted, + /// The database confirmed that the dispatched statement failed. + Failed, + /// SQL may have reached the database, but its outcome cannot be determined. Never retry blindly. + Unknown, +} + +/// Structured result for an explicitly confirmed database write. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct DatabaseWriteResult { + /// Execution state with explicit unknown-outcome semantics. + pub state: DatabaseWriteState, + /// Server-reported affected rows when the write succeeded. + #[serde(skip_serializing_if = "Option::is_none")] + pub affected_rows: Option, + /// Safe failure details without SQL text, parameters, or credentials. + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, +} + #[cfg(test)] mod tests { - use super::JdbcValue; + use super::{DatabaseWriteResult, DatabaseWriteState, JdbcValue}; #[test] fn every_jdbc_value_variant_round_trips_losslessly() { @@ -208,6 +250,23 @@ mod tests { } } + #[test] + fn database_write_state_is_explicit_and_affected_rows_remain_lossless() { + let result = DatabaseWriteResult { + state: DatabaseWriteState::Succeeded, + affected_rows: Some("9007199254740993".to_owned()), + error: None, + }; + let json = serde_json::to_value(&result).expect("write result must serialize"); + assert_eq!(json["state"], "succeeded"); + assert_eq!(json["affectedRows"], "9007199254740993"); + assert_eq!( + serde_json::from_value::(json) + .expect("write result must deserialize"), + result + ); + } + #[test] fn binary_and_non_finite_floats_are_json_strings() { let binary = serde_json::to_value(JdbcValue::Binary { diff --git a/crates/chat2db-contract/src/ssh.rs b/crates/chat2db-contract/src/ssh.rs new file mode 100644 index 0000000..f8c17b9 --- /dev/null +++ b/crates/chat2db-contract/src/ssh.rs @@ -0,0 +1,177 @@ +use std::fmt::{Debug, Formatter}; + +use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; + +use crate::DatasourceConnection; + +/// SSH user authentication material accepted only at a connection boundary. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum SshAuthentication { + /// Password authentication. + Password { + /// SSH password, never returned or logged. + password: String, + }, + /// OpenSSH-compatible private-key authentication. + PrivateKey { + /// User-selected local private-key path. + key_file: String, + /// Optional encrypted-key passphrase, never returned or logged. + #[serde(skip_serializing_if = "Option::is_none")] + passphrase: Option, + }, +} + +/// Non-secret SSH authentication mode used by edit and export projections. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum SshAuthenticationType { + /// Password authentication. The password itself is never projected. + Password, + /// Private-key authentication. The key passphrase is never projected. + PrivateKey, +} + +impl Debug for SshAuthentication { + fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Self::Password { .. } => formatter.write_str("Password([REDACTED])"), + Self::PrivateKey { key_file, .. } => formatter + .debug_struct("PrivateKey") + .field("key_file", key_file) + .field("passphrase", &"[REDACTED]") + .finish(), + } + } +} + +/// SSH server host-key verification policy. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum SshHostKeyVerification { + /// Require a matching entry in the user's standard OpenSSH `known_hosts` file. + #[default] + KnownHosts, +} + +/// Complete ephemeral SSH connection descriptor. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct SshTunnelConfig { + /// SSH server hostname or IP address. + pub host_name: String, + /// SSH server port. + pub port: u16, + /// SSH username. + pub user_name: String, + /// Password or private-key authentication. + pub authentication: SshAuthentication, + /// Server host-key verification policy. + #[serde(default)] + pub host_key_verification: SshHostKeyVerification, + /// Preferred loopback listener port, or an OS-assigned port when absent/zero. + #[serde(skip_serializing_if = "Option::is_none")] + pub local_port: Option, +} + +/// Secret-free SSH settings returned to datasource edit surfaces. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct SshTunnelEditProjection { + /// SSH server hostname or IP address. + pub host_name: String, + /// SSH server port. + pub port: u16, + /// SSH username. + pub user_name: String, + /// Preferred loopback listener port, when configured. + #[serde(skip_serializing_if = "Option::is_none")] + pub local_port: Option, + /// Authentication mode without its password or passphrase. + pub authentication_type: SshAuthenticationType, + /// Selected local private-key path for private-key authentication. + #[serde(skip_serializing_if = "Option::is_none")] + pub key_file: Option, + /// Host keys are always checked against OpenSSH `known_hosts`. + pub host_key_verification: SshHostKeyVerification, +} + +impl Debug for SshTunnelConfig { + fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("SshTunnelConfig") + .field("host_name", &self.host_name) + .field("port", &self.port) + .field("user_name", &self.user_name) + .field("authentication", &self.authentication) + .field("host_key_verification", &self.host_key_verification) + .field("local_port", &self.local_port) + .finish() + } +} + +/// Unsaved datasource connection test with optional SSH local forwarding. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct SshDatasourcePreConnectRequest { + /// Native or compatibility driver identity. + pub driver_id: String, + /// Database connection descriptor whose target is forwarded through SSH. + pub connection: DatasourceConnection, + /// Optional SSH tunnel. Absence performs a direct database test. + #[serde(skip_serializing_if = "Option::is_none")] + pub ssh: Option, +} + +/// Successful standalone SSH authentication result. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct SshConnectionTestResult { + /// True only after transport, key verification, and user authentication succeeded. + pub verified: bool, + /// Host-key policy used by the successful connection. + pub host_key_verification: SshHostKeyVerification, +} + +/// Successful database pre-connect result after optional SSH forwarding. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct SshDatasourcePreConnectResult { + /// True only after a real database open/ping/close cycle succeeded. + pub verified: bool, + /// Loopback port used for the ephemeral tunnel, absent for a direct connection. + #[serde(skip_serializing_if = "Option::is_none")] + pub local_port: Option, +} + +#[cfg(test)] +mod tests { + use super::{SshAuthentication, SshHostKeyVerification, SshTunnelConfig}; + + #[test] + fn ssh_debug_output_redacts_passwords_and_passphrases() { + for authentication in [ + SshAuthentication::Password { + password: "sentinel-password".to_owned(), + }, + SshAuthentication::PrivateKey { + key_file: "/tmp/id_ed25519".to_owned(), + passphrase: Some("sentinel-passphrase".to_owned()), + }, + ] { + let config = SshTunnelConfig { + host_name: "ssh.example.test".to_owned(), + port: 22, + user_name: "developer".to_owned(), + authentication, + host_key_verification: SshHostKeyVerification::KnownHosts, + local_port: None, + }; + let debug = format!("{config:?}"); + assert!(!debug.contains("sentinel-password")); + assert!(!debug.contains("sentinel-passphrase")); + } + } +} diff --git a/crates/chat2db-contract/src/transfer.rs b/crates/chat2db-contract/src/transfer.rs new file mode 100644 index 0000000..83b75be --- /dev/null +++ b/crates/chat2db-contract/src/transfer.rs @@ -0,0 +1,255 @@ +//! Transport-neutral import, export, task, and generated-code contracts. + +use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; + +/// Supported tabular or SQL file formats. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "UPPERCASE")] +pub enum TransferFileFormat { + Csv, + Xls, + Xlsx, + Sql, +} + +/// Controls whether tabular import cells are interpreted as ordinary text or +/// as `Chat2DB`'s lossless NULL/binary transfer envelope. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub enum TabularImportEncoding { + /// Preserve every present cell as ordinary external-file text. + #[default] + Plain, + /// Decode cells emitted by `Chat2DB` tabular exports. + Chat2dbV1, +} + +impl TransferFileFormat { + #[must_use] + pub const fn extension(self) -> &'static str { + match self { + Self::Csv => "csv", + Self::Xls => "xls", + Self::Xlsx => "xlsx", + Self::Sql => "sql", + } + } +} + +/// Community SQL export scope. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "UPPERCASE")] +pub enum TransferSqlScope { + /// Export object definitions and table data. + All, + /// Export object definitions only. + Schema, + /// Export table data only. + Table, +} + +/// Durable transfer operation category. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub enum TransferTaskKind { + ImportFile, + ExportSql, + ExportFile, +} + +/// Durable transfer lifecycle state. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub enum TransferTaskStatus { + Queued, + Running, + Succeeded, + Failed, + Cancelled, + Interrupted, +} + +/// Starts an asynchronous file import into one table or executes a SQL file. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct ImportFileRequest { + pub datasource_id: String, + pub database_name: String, + #[serde(default)] + pub schema_name: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub table_name: Option, + pub file_path: String, + pub format: TransferFileFormat, + #[serde(default = "default_true")] + pub contains_header: bool, + #[serde(default)] + pub tabular_encoding: TabularImportEncoding, +} + +/// Starts an asynchronous SQL dump export. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct SqlFileExportRequest { + pub datasource_id: String, + pub database_name: String, + #[serde(default)] + pub schema_name: String, + #[serde(default)] + pub table_names: Vec, + pub scope: TransferSqlScope, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub export_path: Option, +} + +/// Starts an asynchronous CSV, XLS, XLSX, or SQL table export. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct OtherFileExportRequest { + pub datasource_id: String, + pub database_name: String, + #[serde(default)] + pub schema_name: String, + pub table_names: Vec, + pub format: TransferFileFormat, + #[serde(default = "default_true")] + pub contains_header: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub export_path: Option, +} + +/// DML result export window. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub enum DmlExportSize { + CurrentPage, + All, +} + +/// DML result export encoding. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "UPPERCASE")] +pub enum DmlExportFormat { + Csv, + Xlsx, + Insert, +} + +/// Synchronously exports one selected result set into a managed artifact. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct DmlExportRequest { + pub datasource_id: String, + pub database_name: String, + #[serde(default)] + pub schema_name: String, + #[serde(default)] + pub sql: String, + pub original_sql: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub result_set_id: Option, + pub export_size: DmlExportSize, + pub format: DmlExportFormat, +} + +/// Generates Java entity, Mapper, and Mapper XML files for one table. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct GenerateMysqlClassRequest { + pub datasource_id: String, + pub database_name: String, + #[serde(default)] + pub schema_name: String, + pub table_name: String, + pub export_path: String, +} + +/// Asynchronous transfer acceptance. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct TransferTaskAccepted { + pub task_id: i64, +} + +/// Durable task projection shared by HTTP and Desktop adapters. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct TransferTask { + pub id: i64, + pub datasource_id: String, + pub database_name: String, + pub schema_name: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub table_name: Option, + pub kind: TransferTaskKind, + pub status: TransferTaskStatus, + pub task_name: String, + pub progress_current: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub progress_total: Option, + pub progress_description: String, + pub info_log: String, + pub error_log: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub artifact_id: Option, + pub cancel_requested: bool, + pub created_at_ms: String, + pub updated_at_ms: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub finished_at_ms: Option, +} + +/// Bounded task list. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct TransferTaskPage { + pub items: Vec, + pub total: u64, + pub page_no: u32, + pub page_size: u32, +} + +/// Secret-free artifact metadata. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct TransferArtifact { + pub id: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub task_id: Option, + pub file_name: String, + pub media_type: String, + pub format: String, + pub byte_count: String, + pub sha256: String, + pub created_at_ms: String, +} + +/// Files emitted by Java class generation. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct GeneratedMysqlClassSet { + pub output_directory: String, + pub files: Vec, +} + +const fn default_true() -> bool { + true +} + +#[cfg(test)] +mod tests { + use super::{TransferFileFormat, TransferTaskStatus}; + + #[test] + fn community_facing_enums_keep_stable_uppercase_values() { + assert_eq!( + serde_json::to_string(&TransferFileFormat::Xlsx).expect("format serializes"), + "\"XLSX\"" + ); + assert_eq!( + serde_json::to_string(&TransferTaskStatus::Interrupted).expect("status serializes"), + "\"INTERRUPTED\"" + ); + } +} diff --git a/crates/chat2db-contract/src/workspace.rs b/crates/chat2db-contract/src/workspace.rs new file mode 100644 index 0000000..12d86f2 --- /dev/null +++ b/crates/chat2db-contract/src/workspace.rs @@ -0,0 +1,150 @@ +use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; + +/// Stable kind discriminator for one node in the local datasource workspace tree. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub enum WorkspaceNodeKind { + /// A user-created grouping node. + Namespace, + /// A persisted datasource. + DataSource, +} + +/// Disambiguated reference to a workspace node. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct WorkspaceNodeRef { + /// Namespace decimal id or opaque datasource id. + pub id: String, + /// Node category used to resolve the id safely. + #[serde(rename = "type")] + pub node_type: WorkspaceNodeKind, +} + +/// Secret-free node returned by the local workspace tree. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct WorkspaceTreeNode { + /// Namespace decimal id or opaque datasource id. + pub id: String, + /// Node category. + #[serde(rename = "type")] + pub node_type: WorkspaceNodeKind, + /// Current display name. + pub name: String, + /// Opaque datasource id for datasource nodes only. + #[serde(skip_serializing_if = "Option::is_none")] + pub datasource_id: Option, + /// Decimal namespace id for namespace nodes only. + #[serde(skip_serializing_if = "Option::is_none")] + pub namespace_id: Option, + /// Ordered direct children. Datasource nodes always return an empty list. + pub children: Vec, +} + +/// Ordered root nodes of the local datasource workspace. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct WorkspaceTree { + /// Root namespace and datasource nodes. + pub items: Vec, +} + +/// Request to create one root or child namespace. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct CreateWorkspaceNamespaceRequest { + /// User-visible namespace name. + pub name: String, + /// Optional decimal id of the parent namespace. + #[serde(skip_serializing_if = "Option::is_none")] + pub parent_id: Option, +} + +/// Request to rename one namespace. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct UpdateWorkspaceNamespaceRequest { + /// Decimal namespace id. + pub id: String, + /// Replacement display name. + pub name: String, +} + +/// Persisted namespace metadata. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct WorkspaceNamespace { + /// Decimal namespace id. + pub id: String, + /// User-visible namespace name. + pub name: String, + /// Optional decimal id of the parent namespace. + #[serde(skip_serializing_if = "Option::is_none")] + pub parent_id: Option, +} + +/// Community-compatible drag-and-drop operation. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct MoveWorkspaceNodeRequest { + /// Node being moved. + pub drag_node: WorkspaceNodeRef, + /// Node receiving or anchoring the move. + pub drop_to_node: WorkspaceNodeRef, + /// `0` inserts as first child, `2` as last child, `-1` before, and `1` after the target. + pub drop_position: i8, +} + +/// Explicit datasource-to-namespace assignment used by the compatibility API. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct AssignDatasourceNamespaceRequest { + /// Opaque datasource id. + pub datasource_id: String, + /// Destination namespace, or `None` for the root. + #[serde(skip_serializing_if = "Option::is_none")] + pub namespace_id: Option, +} + +/// Datasource ids directly assigned to one namespace or to the root. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct WorkspaceDatasourceGroup { + /// Namespace decimal id, or `None` for root datasources. + #[serde(skip_serializing_if = "Option::is_none")] + pub namespace_id: Option, + /// Ordered datasource ids directly owned by the namespace. + pub datasource_ids: Vec, +} + +/// Stable namespace-to-datasource mapping used by the retained Community client. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct WorkspaceDatasourceList { + /// Root and namespace groups in stable tree order. + pub groups: Vec, +} + +#[cfg(test)] +mod tests { + use super::{WorkspaceNodeKind, WorkspaceTreeNode}; + + #[test] + fn workspace_nodes_are_disambiguated_and_secret_free() { + let node = WorkspaceTreeNode { + id: "datasource-1".to_owned(), + node_type: WorkspaceNodeKind::DataSource, + name: "Local MySQL".to_owned(), + datasource_id: Some("datasource-1".to_owned()), + namespace_id: None, + children: Vec::new(), + }; + let json = serde_json::to_value(node).expect("node serializes"); + assert_eq!(json["type"], "DATA_SOURCE"); + for forbidden in ["password", "jdbcUrl", "properties", "secretRef"] { + assert!(!json.to_string().contains(forbidden)); + } + } +} diff --git a/crates/chat2db-core/Cargo.toml b/crates/chat2db-core/Cargo.toml index 9ff4ca0..0336994 100644 --- a/crates/chat2db-core/Cargo.toml +++ b/crates/chat2db-core/Cargo.toml @@ -10,16 +10,28 @@ repository.workspace = true [dependencies] async-trait.workspace = true +aes.workspace = true base64.workspace = true +blowfish.workspace = true +cbc.workspace = true chat2db-agent = { path = "../chat2db-agent" } chat2db-contract = { path = "../chat2db-contract" } chat2db-engine-protocol = { path = "../chat2db-engine-protocol" } chat2db-java-bridge = { path = "../chat2db-java-bridge" } chat2db-storage = { path = "../chat2db-storage" } +chrono.workspace = true +csv.workspace = true +directories.workspace = true +hex.workspace = true mysql_async.workspace = true prost.workspace = true +quick-xml.workspace = true +russh.workspace = true serde.workspace = true serde_json.workspace = true +sha1.workspace = true +sha2.workspace = true +sqlparser.workspace = true rustix.workspace = true tempfile = "3" thiserror.workspace = true @@ -28,6 +40,8 @@ tokio-util.workspace = true tracing.workspace = true url.workspace = true uuid.workspace = true +xls.workspace = true +zip.workspace = true [dev-dependencies] futures-util.workspace = true diff --git a/crates/chat2db-core/src/agent/sql_tools.rs b/crates/chat2db-core/src/agent/sql_tools.rs index 47fb0db..693bffa 100644 --- a/crates/chat2db-core/src/agent/sql_tools.rs +++ b/crates/chat2db-core/src/agent/sql_tools.rs @@ -762,7 +762,6 @@ fn tool_error(error: ApiError, outcome: ExecutionOutcome) -> ToolExecutionError fn database_write_error(error: &DatabaseWriteError) -> ToolExecutionError { let outcome = match error.outcome { DatabaseWriteOutcome::NotStarted => ExecutionOutcome::NotStarted, - DatabaseWriteOutcome::Failed => ExecutionOutcome::Failed, DatabaseWriteOutcome::Unknown => ExecutionOutcome::Unknown, }; tool_error(error.error.api_error(), outcome) @@ -1366,7 +1365,6 @@ mod tests { DatabaseWriteOutcome::NotStarted, ExecutionOutcome::NotStarted, ), - (DatabaseWriteOutcome::Failed, ExecutionOutcome::Failed), (DatabaseWriteOutcome::Unknown, ExecutionOutcome::Unknown), ] { let error = DatabaseWriteError { diff --git a/crates/chat2db-core/src/community.rs b/crates/chat2db-core/src/community.rs index 243eeb4..1a9f38c 100644 --- a/crates/chat2db-core/src/community.rs +++ b/crates/chat2db-core/src/community.rs @@ -11,7 +11,8 @@ use chat2db_contract::{ CommunityParsedStatement, CommunityPlugin, CommunityPluginBehavior, CommunityPluginCatalog, CommunityPluginServices, CommunityPrimaryKey, CommunityPrimaryKeyList, CommunityProcedure, CommunityProcedureList, CommunityProcedureParameter, CommunityProcedureParameterList, - CommunityRoutineInvocationPreview, CommunitySchema, CommunitySchemaList, CommunitySqlAnalysis, + CommunityRoutineInvocationPreview, CommunityRoutineMigrationExecution, + CommunityRoutineMigrationRequest, CommunitySchema, CommunitySchemaList, CommunitySqlAnalysis, CommunitySqlCompletion, CommunitySqlCompletionActiveSnippetSlot, CommunitySqlCompletionCandidate, CommunitySqlCompletionEditorHint, CommunitySqlCompletionEditorHintItem, CommunitySqlCompletionRange, CommunitySqlDiagnostic, @@ -958,6 +959,45 @@ impl Application { native_mysql::preview_routine_invocation(self, request).await } + /// Previews the compensating `MySQL` routine-replacement script. + /// + /// # Errors + /// + /// Returns validation errors for unsupported database types or malformed + /// routine migration input. + pub fn preview_community_routine_migration( + &self, + request: &CommunityRoutineMigrationRequest, + ) -> Result { + if !native_mysql::is_mysql_database_type(&request.database_type) { + return Err(AppError::invalid( + "invalid_community_routine_migration_request", + "routine migration supports only MySQL", + )); + } + native_mysql::preview_routine_migration(request) + } + + /// Replaces one `MySQL` routine and restores its before-image when apply fails. + /// + /// # Errors + /// + /// Returns validation, datasource, connection, or cleanup errors. SQL apply + /// and compensation failures are returned as a successful product result + /// with `success = false`, matching the Community frontend contract. + pub async fn execute_community_routine_migration( + &self, + request: CommunityRoutineMigrationRequest, + ) -> Result { + if !native_mysql::is_mysql_database_type(&request.database_type) { + return Err(AppError::invalid( + "invalid_community_routine_migration_request", + "routine migration supports only MySQL", + )); + } + native_mysql::execute_routine_migration(self, request).await + } + /// Lists triggers through Community metadata using a forced read-only session. /// /// # Errors diff --git a/crates/chat2db-core/src/datasource_compatibility.rs b/crates/chat2db-core/src/datasource_compatibility.rs new file mode 100644 index 0000000..01ee5b3 --- /dev/null +++ b/crates/chat2db-core/src/datasource_compatibility.rs @@ -0,0 +1,668 @@ +use std::collections::HashSet; + +use chat2db_contract::{ + CloneDatasourceRequest, CommunityDatasourceExport, CommunityDatasourceImportResult, + ConsoleConnectResult, CreateDatasourceRequest, DatasourceCloseResult, DatasourceConnectResult, + DatasourceConnection, DatasourceConnectionProperty, DatasourceSessionMode, + ExportCommunityDatasourcesRequest, JdbcDriver, ListCommunityDatabasesRequest, + NativeDriverAction, NativeDriverCompatibility, PortableCommunityDatasource, + PortableDatasourceConnection, PortableDatasourceProperty, SshAuthentication, + SshAuthenticationType, SshTunnelConfig, +}; +use chat2db_storage::{CreateDatasource, SecretValue, StorageError}; +use url::Url; + +use crate::{ + AppError, Application, convert, datasource_edit::project_ssh, + datasource_session::resolve_datasource_connection, now_millis, storage_call, +}; + +const COMMUNITY_DATASOURCE_DOCUMENT_VERSION: u32 = 1; +const MAX_TRANSFER_DATASOURCES: usize = 1_000; + +impl Application { + /// Clones datasource metadata and installs a separately referenced copy of its vault secret. + /// + /// # Errors + /// + /// Returns validation, datasource, vault, availability, or storage failures. + pub async fn clone_datasource( + &self, + request: CloneDatasourceRequest, + ) -> Result { + if request.id.trim().is_empty() { + return Err(AppError::invalid( + "invalid_datasource_clone", + "datasource id cannot be empty", + )); + } + let storage = self.require_storage()?; + let source_id = request.id; + let requested_name = request.name; + let record = storage_call(move || { + let (source, secret) = storage.get_datasource_with_secret(&source_id)?; + let name = requested_name + .filter(|name| !name.trim().is_empty()) + .unwrap_or_else(|| copy_name(&source.name)); + let copied_secret = secret + .as_ref() + .map(|value| SecretValue::new(value.expose_secret().to_vec())); + storage.create_datasource( + CreateDatasource { + name, + driver_id: source.driver_id, + }, + copied_secret, + ) + }) + .await?; + Ok(convert::datasource(record)) + } + + /// Exports selected datasource definitions without passwords or sensitive properties. + /// + /// # Errors + /// + /// Returns validation, datasource, vault, availability, or storage failures. + pub async fn export_community_datasources( + &self, + request: ExportCommunityDatasourcesRequest, + ) -> Result { + if request.datasource_ids.len() > MAX_TRANSFER_DATASOURCES { + return Err(AppError::invalid( + "datasource_export_limit_exceeded", + "at most 1000 datasources can be exported at once", + )); + } + let storage = self.require_storage()?; + let ids = if request.datasource_ids.is_empty() { + let storage = storage.clone(); + storage_call(move || { + storage.list_datasources().map(|records| { + records + .into_iter() + .map(|record| record.id) + .collect::>() + }) + }) + .await? + } else { + unique_non_empty_ids(request.datasource_ids)? + }; + + let mut datasources = Vec::with_capacity(ids.len()); + for id in ids { + let storage = storage.clone(); + let (record, connection) = storage_call(move || { + let (record, secret) = storage.get_datasource_with_secret(&id)?; + let connection = secret + .as_ref() + .map(|secret| { + serde_json::from_slice::(secret.expose_secret()) + .map_err(|_| { + StorageError::InvalidDatasource( + "stored datasource connection descriptor is invalid", + ) + }) + }) + .transpose()?; + Ok((record, connection)) + }) + .await?; + let connection = connection + .map(|connection| portable_connection(&connection)) + .transpose()?; + datasources.push(PortableCommunityDatasource { + source_id: Some(record.id), + name: record.name, + driver_id: record.driver_id, + connection, + }); + } + Ok(CommunityDatasourceExport { + schema_version: COMMUNITY_DATASOURCE_DOCUMENT_VERSION, + exported_at_ms: now_millis()?.to_string(), + datasources, + }) + } + + /// Imports a secret-safe Community document as new datasource records only. + /// + /// Imported source ids are intentionally ignored. Existing datasource metadata and vault + /// references are never updated by this operation. + /// + /// # Errors + /// + /// Returns validation, driver, vault, availability, or storage failures. + pub async fn import_community_datasources( + &self, + document: CommunityDatasourceExport, + ) -> Result { + if document.schema_version != COMMUNITY_DATASOURCE_DOCUMENT_VERSION { + return Err(AppError::invalid( + "unsupported_datasource_import_version", + "the datasource import document version is not supported", + )); + } + if document.datasources.len() > MAX_TRANSFER_DATASOURCES { + return Err(AppError::invalid( + "datasource_import_limit_exceeded", + "at most 1000 datasources can be imported at once", + )); + } + + let mut prepared = Vec::with_capacity(document.datasources.len()); + for datasource in document.datasources { + self.require_managed_driver(&datasource.driver_id)?; + let connection = datasource.connection.map(imported_connection).transpose()?; + prepared.push(CreateDatasourceRequest { + name: datasource.name, + driver_id: datasource.driver_id, + connection, + }); + } + + let mut created = Vec::with_capacity(prepared.len()); + for request in prepared { + created.push(self.create_datasource(request).await?); + } + let count = u32::try_from(created.len()).map_err(|_| AppError::internal())?; + Ok(CommunityDatasourceImportResult { count, created }) + } + + /// Opens a real ephemeral metadata connection and returns its database list. + /// + /// # Errors + /// + /// Returns datasource, storage, driver, engine, or database failures. + pub async fn connect_datasource_compatibility( + &self, + datasource_id: &str, + database_type: &str, + ) -> Result { + let database_type = if database_type.trim().is_empty() { + let datasource = self.get_datasource(datasource_id).await?; + if self.is_native_mysql_driver(&datasource.driver_id) { + "MYSQL".to_owned() + } else { + datasource.driver_id + } + } else { + database_type.trim().to_owned() + }; + let databases = self + .list_community_databases(ListCommunityDatabasesRequest { + datasource_id: datasource_id.to_owned(), + database_type, + }) + .await?; + Ok(DatasourceConnectResult { + datasource_id: datasource_id.to_owned(), + session_mode: DatasourceSessionMode::Ephemeral, + databases: databases.items, + }) + } + + /// Verifies a Console datasource using a real open/ping/close cycle. + /// + /// # Errors + /// + /// Returns datasource, storage, driver, engine, or database failures. + pub async fn connect_console_compatibility( + &self, + datasource_id: &str, + ) -> Result { + let storage = self.require_storage()?; + let resolved = resolve_datasource_connection(&storage, datasource_id).await?; + self.test_datasource_connection(&resolved.driver_id, resolved.connection) + .await?; + Ok(ConsoleConnectResult { + datasource_id: datasource_id.to_owned(), + session_mode: DatasourceSessionMode::Ephemeral, + verified: true, + }) + } + + /// Acknowledges close after verifying the datasource exists. + /// + /// Connections are operation-scoped, so successful operations have already disconnected and + /// there is no retained pool generation to drain. + /// + /// # Errors + /// + /// Returns datasource-not-found, availability, or storage failures. + pub async fn close_datasource_compatibility( + &self, + datasource_id: &str, + ) -> Result { + self.get_datasource(datasource_id).await?; + Ok(DatasourceCloseResult { + datasource_id: datasource_id.to_owned(), + session_mode: DatasourceSessionMode::Ephemeral, + closed_connections: 0, + }) + } + + /// Returns an explicit no-JAR result for `MySQL` driver mutations handled by `mysql_async`. + /// + /// # Errors + /// + /// Returns invalid-request for non-MySQL database types. + pub fn native_driver_compatibility( + &self, + database_type: &str, + action: NativeDriverAction, + ) -> Result { + if !database_type.trim().eq_ignore_ascii_case("mysql") { + return Err(AppError::invalid( + "native_driver_not_available", + "the requested database type is not implemented by a native Rust driver", + )); + } + Ok(NativeDriverCompatibility { + database_type: "MYSQL".to_owned(), + driver_id: "mysql".to_owned(), + action, + implementation: "mysql_async".to_owned(), + artifact_required: false, + changed: false, + }) + } +} + +pub(crate) fn native_mysql_driver() -> JdbcDriver { + JdbcDriver { + pack_id: "native:mysql_async".to_owned(), + name: "MySQL (native Rust)".to_owned(), + version: "native".to_owned(), + driver_id: "mysql".to_owned(), + driver_class: "rust:mysql_async".to_owned(), + artifact_count: 0, + artifact_bytes: "0".to_owned(), + } +} + +fn copy_name(name: &str) -> String { + let candidate = format!("{name} Copy"); + if candidate.len() <= 512 { + candidate + } else { + name.to_owned() + } +} + +fn unique_non_empty_ids(ids: Vec) -> Result, AppError> { + let mut seen = HashSet::with_capacity(ids.len()); + let mut unique = Vec::with_capacity(ids.len()); + for id in ids { + if id.trim().is_empty() { + return Err(AppError::invalid( + "invalid_datasource_export", + "datasource ids cannot be empty", + )); + } + if seen.insert(id.clone()) { + unique.push(id); + } + } + Ok(unique) +} + +fn portable_connection( + connection: &DatasourceConnection, +) -> Result { + Ok(PortableDatasourceConnection { + jdbc_url: sanitize_jdbc_url(&connection.jdbc_url)?, + properties: connection + .properties + .iter() + .filter(|property| !property.sensitive && !is_sensitive_key(&property.key)) + .map(|property| PortableDatasourceProperty { + key: property.key.clone(), + value: property.value.clone(), + }) + .collect(), + read_only: connection.read_only, + ssh: connection.ssh.as_ref().map(project_ssh), + }) +} + +fn imported_connection( + connection: PortableDatasourceConnection, +) -> Result { + if connection.jdbc_url.trim().is_empty() { + return Err(AppError::invalid( + "invalid_datasource_import", + "portable JDBC URL cannot be empty", + )); + } + if connection + .properties + .iter() + .any(|property| is_sensitive_key(&property.key)) + { + return Err(AppError::invalid( + "unsafe_datasource_import", + "portable datasource properties cannot contain credentials", + )); + } + Ok(DatasourceConnection { + jdbc_url: sanitize_jdbc_url(&connection.jdbc_url)?, + properties: connection + .properties + .into_iter() + .map(|property| DatasourceConnectionProperty { + key: property.key, + value: property.value, + sensitive: false, + }) + .collect(), + read_only: connection.read_only, + ssh: connection.ssh.map(imported_ssh), + }) +} + +fn imported_ssh(ssh: chat2db_contract::SshTunnelEditProjection) -> SshTunnelConfig { + let authentication = match ssh.authentication_type { + SshAuthenticationType::Password => SshAuthentication::Password { + password: String::new(), + }, + SshAuthenticationType::PrivateKey => SshAuthentication::PrivateKey { + key_file: ssh.key_file.unwrap_or_default(), + passphrase: None, + }, + }; + SshTunnelConfig { + host_name: ssh.host_name, + port: ssh.port, + user_name: ssh.user_name, + authentication, + host_key_verification: ssh.host_key_verification, + local_port: ssh.local_port, + } +} + +fn sanitize_jdbc_url(jdbc_url: &str) -> Result { + let jdbc_url = jdbc_url.trim(); + let (prefix, raw_url) = jdbc_url + .strip_prefix("jdbc:") + .map_or(("", jdbc_url), |url| ("jdbc:", url)); + let mut parsed = Url::parse(raw_url).map_err(|_| { + AppError::invalid( + "unsafe_datasource_export", + "the datasource URL cannot be exported safely", + ) + })?; + parsed.set_username("").map_err(|()| AppError::internal())?; + parsed + .set_password(None) + .map_err(|()| AppError::internal())?; + parsed.set_fragment(None); + let retained_query = parsed + .query_pairs() + .filter(|(key, _)| !is_sensitive_key(key)) + .map(|(key, value)| (key.into_owned(), value.into_owned())) + .collect::>(); + { + let mut query = parsed.query_pairs_mut(); + query.clear(); + query.extend_pairs(retained_query); + } + Ok(format!("{prefix}{parsed}")) +} + +fn is_sensitive_key(key: &str) -> bool { + let key = key.trim().to_ascii_lowercase(); + key.contains("password") + || key.contains("passwd") + || key.contains("secret") + || key.contains("token") + || key.contains("credential") + || key.contains("privatekey") + || key.contains("passphrase") +} + +#[cfg(test)] +mod tests { + use std::{ + collections::HashMap, + sync::{Arc, Mutex}, + }; + + use chat2db_contract::{ + CreateDatasourceRequest, DatasourceConnection, DatasourceConnectionProperty, + ExportCommunityDatasourcesRequest, NativeDriverAction, SshAuthentication, + SshAuthenticationType, SshHostKeyVerification, SshTunnelConfig, + }; + use chat2db_storage::{SecretRef, SecretValue, SecretVault, SecretVaultError, Storage}; + use tempfile::TempDir; + + use super::CloneDatasourceRequest; + use crate::Application; + + #[derive(Debug, Default)] + struct MemoryVault { + values: Mutex>>, + } + + impl SecretVault for MemoryVault { + fn probe(&self) -> Result<(), SecretVaultError> { + Ok(()) + } + + fn create( + &self, + reference: &SecretRef, + value: &SecretValue, + ) -> Result<(), SecretVaultError> { + self.values.lock().expect("vault lock").insert( + reference.as_str().to_owned(), + value.expose_secret().to_vec(), + ); + Ok(()) + } + + fn get(&self, reference: &SecretRef) -> Result, SecretVaultError> { + Ok(self + .values + .lock() + .expect("vault lock") + .get(reference.as_str()) + .cloned() + .map(SecretValue::new)) + } + + fn delete(&self, reference: &SecretRef) -> Result<(), SecretVaultError> { + self.values + .lock() + .expect("vault lock") + .remove(reference.as_str()); + Ok(()) + } + } + + fn application() -> (TempDir, Application) { + let directory = TempDir::new().expect("temp dir"); + let storage = Storage::open(directory.path(), Arc::new(MemoryVault::default())) + .expect("storage opens"); + (directory, Application::with_storage(storage)) + } + + fn connection() -> DatasourceConnection { + DatasourceConnection { + jdbc_url: + "jdbc:mysql://url-user:url-password@localhost:3306/demo?token=hidden&useSSL=false" + .to_owned(), + properties: vec![ + DatasourceConnectionProperty { + key: "user".to_owned(), + value: "root".to_owned(), + sensitive: false, + }, + DatasourceConnectionProperty { + key: "password".to_owned(), + value: "sentinel-password".to_owned(), + sensitive: true, + }, + ], + read_only: false, + ssh: Some(SshTunnelConfig { + host_name: "bastion.internal".to_owned(), + port: 22, + user_name: "ssh-user".to_owned(), + authentication: SshAuthentication::Password { + password: "sentinel-ssh-password".to_owned(), + }, + host_key_verification: SshHostKeyVerification::KnownHosts, + local_port: None, + }), + } + } + + #[tokio::test] + async fn export_import_never_overwrites_or_serializes_credentials() { + let (_directory, application) = application(); + let existing = application + .create_datasource(CreateDatasourceRequest { + name: "Existing".to_owned(), + driver_id: "mysql".to_owned(), + connection: Some(connection()), + }) + .await + .expect("datasource creates"); + let mut document = application + .export_community_datasources(ExportCommunityDatasourcesRequest { + datasource_ids: vec![existing.id.clone()], + }) + .await + .expect("datasource exports"); + let json = serde_json::to_string(&document).expect("document serializes"); + for forbidden in [ + "sentinel-password", + "sentinel-ssh-password", + "url-password", + "hidden", + ] { + assert!(!json.contains(forbidden), "export leaked {forbidden}"); + } + let exported_ssh = document.datasources[0] + .connection + .as_ref() + .and_then(|connection| connection.ssh.as_ref()) + .expect("SSH metadata exports"); + assert_eq!(exported_ssh.host_name, "bastion.internal"); + assert_eq!( + exported_ssh.authentication_type, + SshAuthenticationType::Password + ); + + document.datasources[0].source_id = Some(existing.id.clone()); + let imported = application + .import_community_datasources(document) + .await + .expect("datasource imports"); + assert_eq!(imported.count, 1); + assert_ne!(imported.created[0].id, existing.id); + + let storage = application.storage().expect("storage configured"); + let (_, secret) = storage + .get_datasource_with_secret(&existing.id) + .expect("existing secret resolves"); + let secret = secret.expect("existing secret remains"); + let existing_connection: DatasourceConnection = + serde_json::from_slice(secret.expose_secret()).expect("connection decodes"); + assert!(existing_connection.properties.iter().any(|property| { + property.key == "password" && property.value == "sentinel-password" + })); + assert!(matches!( + existing_connection + .ssh + .expect("existing SSH config remains") + .authentication, + SshAuthentication::Password { password } if password == "sentinel-ssh-password" + )); + let (_, imported_secret) = storage + .get_datasource_with_secret(&imported.created[0].id) + .expect("imported secret resolves"); + let imported_connection: DatasourceConnection = serde_json::from_slice( + imported_secret + .expect("imported descriptor exists") + .expose_secret(), + ) + .expect("imported descriptor decodes"); + assert!(matches!( + imported_connection + .ssh + .expect("SSH metadata imports") + .authentication, + SshAuthentication::Password { password } if password.is_empty() + )); + } + + #[tokio::test] + async fn clone_uses_a_distinct_vault_reference_and_close_is_stateless() { + let (_directory, application) = application(); + let original = application + .create_datasource(CreateDatasourceRequest { + name: "Source".to_owned(), + driver_id: "mysql".to_owned(), + connection: Some(connection()), + }) + .await + .expect("source creates"); + let duplicate = application + .clone_datasource(CloneDatasourceRequest { + id: original.id.clone(), + name: None, + }) + .await + .expect("datasource clones"); + let storage = application.storage().expect("storage configured"); + let original_record = storage + .get_datasource(&original.id) + .expect("source reads") + .expect("source exists"); + let duplicate_record = storage + .get_datasource(&duplicate.id) + .expect("clone reads") + .expect("clone exists"); + assert_ne!(original_record.secret_ref, duplicate_record.secret_ref); + let (_, duplicate_secret) = storage + .get_datasource_with_secret(&duplicate.id) + .expect("clone secret resolves"); + let duplicate_connection: DatasourceConnection = serde_json::from_slice( + duplicate_secret + .expect("clone descriptor exists") + .expose_secret(), + ) + .expect("clone descriptor decodes"); + assert!(matches!( + duplicate_connection + .ssh + .expect("clone retains SSH") + .authentication, + SshAuthentication::Password { password } if password == "sentinel-ssh-password" + )); + let closed = application + .close_datasource_compatibility(&original.id) + .await + .expect("close succeeds"); + assert_eq!(closed.closed_connections, 0); + } + + #[test] + fn native_mysql_inventory_and_mutations_never_require_a_jar() { + let application = Application::new(); + let drivers = application.list_drivers(); + assert!(drivers.items.iter().any(|driver| { + driver.driver_id == "mysql" + && driver.driver_class == "rust:mysql_async" + && driver.artifact_count == 0 + })); + let compatibility = application + .native_driver_compatibility("MYSQL", NativeDriverAction::Download) + .expect("native compatibility resolves"); + assert!(!compatibility.artifact_required); + assert!(!compatibility.changed); + } +} diff --git a/crates/chat2db-core/src/datasource_converter.rs b/crates/chat2db-core/src/datasource_converter.rs new file mode 100644 index 0000000..aea3cc7 --- /dev/null +++ b/crates/chat2db-core/src/datasource_converter.rs @@ -0,0 +1,1179 @@ +//! Strict importers for Community datasource exchange formats. + +use std::{ + collections::{HashMap, HashSet}, + io::{Cursor, Read}, +}; + +use aes::Aes128; +use blowfish::{ + Blowfish, + cipher::{BlockDecrypt, BlockEncrypt, KeyInit, generic_array::GenericArray}, +}; +use cbc::cipher::{BlockDecryptMut, KeyIvInit, block_padding::Pkcs7}; +use chat2db_contract::{ + CommunityDatasourceFileImportRequest, CommunityDatasourceFileImportResult, + CommunityDatasourceImportFormat, CreateDatasourceRequest, DatasourceConnection, + DatasourceConnectionProperty, +}; +use quick_xml::{Reader, events::Event}; +use serde_json::Value; +use sha1::{Digest, Sha1}; +use url::Url; +use zip::ZipArchive; + +use crate::{AppError, Application, datasource_edit::is_sensitive_key}; + +const MAX_IMPORT_BYTES: usize = 16 * 1024 * 1024; +const MAX_DATASOURCES: usize = 1_000; +const MAX_ZIP_ENTRIES: usize = 256; +const MAX_ZIP_ENTRY_BYTES: u64 = 8 * 1024 * 1024; +const MAX_ZIP_EXPANDED_BYTES: u64 = 32 * 1024 * 1024; +const MAX_XML_DEPTH: usize = 64; +const MAX_NAME_BYTES: usize = 512; +const MAX_URL_BYTES: usize = 8 * 1024; +const MAX_USERNAME_BYTES: usize = 1_024; +const MAX_PROPERTIES: usize = 128; +const MAX_PROPERTY_KEY_BYTES: usize = 255; +const MAX_PROPERTY_VALUE_BYTES: usize = 8 * 1024; + +type Aes128CbcDec = cbc::Decryptor; + +struct ParsedImport { + datasources: Vec, + skipped: usize, +} + +struct PreparedDatasource { + name: String, + connection: Option, +} + +impl Application { + /// Parses and imports one Community datasource exchange file. + /// + /// Every entry is parsed and validated before the first datasource record is created. Only + /// native `MySQL` entries are accepted; unsupported database types are counted as skipped. + /// + /// # Errors + /// + /// Returns format, size, validation, driver, vault, or durable-storage failures. + pub async fn import_community_datasource_file( + &self, + request: CommunityDatasourceFileImportRequest, + ) -> Result { + if request.content.is_empty() { + return Err(invalid_file()); + } + if request.content.len() > MAX_IMPORT_BYTES { + return Err(import_limit("the datasource import file is too large")); + } + self.require_managed_driver("mysql")?; + let parsed = tokio::task::spawn_blocking(move || parse_import(&request)) + .await + .map_err(|_| AppError::internal())??; + + let mut created = Vec::with_capacity(parsed.datasources.len()); + for datasource in parsed.datasources { + created.push( + self.create_datasource(CreateDatasourceRequest { + name: datasource.name, + driver_id: "mysql".to_owned(), + connection: datasource.connection, + }) + .await?, + ); + } + Ok(CommunityDatasourceFileImportResult { + count: u32::try_from(created.len()).map_err(|_| AppError::internal())?, + created, + skipped: u32::try_from(parsed.skipped).map_err(|_| AppError::internal())?, + }) + } +} + +fn parse_import(request: &CommunityDatasourceFileImportRequest) -> Result { + let mut parsed = match request.format { + CommunityDatasourceImportFormat::Chat2dbJson => parse_chat2db_json(&request.content)?, + CommunityDatasourceImportFormat::NavicatNcx => parse_navicat_ncx(&request.content)?, + CommunityDatasourceImportFormat::DbeaverDbp => parse_dbeaver_dbp(&request.content)?, + CommunityDatasourceImportFormat::DatagripText => parse_datagrip_text(&request.content)?, + }; + if parsed.datasources.len() > MAX_DATASOURCES { + return Err(import_limit( + "at most 1000 datasources can be imported at once", + )); + } + for (index, datasource) in parsed.datasources.iter_mut().enumerate() { + validate_datasource(datasource, index)?; + } + Ok(parsed) +} + +fn validate_datasource(datasource: &mut PreparedDatasource, index: usize) -> Result<(), AppError> { + datasource.name = datasource.name.trim().to_owned(); + if datasource.name.is_empty() { + datasource.name = format!("Imported MySQL {}", index + 1); + } + if datasource.name.len() > MAX_NAME_BYTES || datasource.name.contains('\0') { + return Err(invalid_file()); + } + let Some(connection) = datasource.connection.as_mut() else { + return Ok(()); + }; + connection.jdbc_url = connection.jdbc_url.trim().to_owned(); + if connection.jdbc_url.is_empty() + || connection.jdbc_url.len() > MAX_URL_BYTES + || connection.jdbc_url.contains('\0') + || !is_mysql_url(&connection.jdbc_url) + { + return Err(invalid_file()); + } + if connection.properties.len() > MAX_PROPERTIES { + return Err(import_limit( + "a datasource contains too many connection properties", + )); + } + let mut keys = HashSet::with_capacity(connection.properties.len()); + for property in &mut connection.properties { + property.key = property.key.trim().to_owned(); + if property.key.is_empty() + || property.key.len() > MAX_PROPERTY_KEY_BYTES + || property.value.len() > MAX_PROPERTY_VALUE_BYTES + || property.key.contains('\0') + || property.value.contains('\0') + || !keys.insert(property.key.to_ascii_lowercase()) + { + return Err(invalid_file()); + } + if is_sensitive_key(&property.key) { + property.sensitive = true; + } + if is_username_key(&property.key) && property.value.len() > MAX_USERNAME_BYTES { + return Err(invalid_file()); + } + } + Ok(()) +} + +fn invalid_file() -> AppError { + AppError::invalid( + "invalid_datasource_import_file", + "The datasource import file is invalid", + ) +} + +fn import_limit(message: &'static str) -> AppError { + AppError::invalid("datasource_import_limit_exceeded", message) +} + +fn is_mysql_url(jdbc_url: &str) -> bool { + jdbc_url + .strip_prefix("jdbc:") + .and_then(|raw_url| Url::parse(raw_url).ok()) + .is_some_and(|url| url.scheme().eq_ignore_ascii_case("mysql")) +} + +fn is_username_key(key: &str) -> bool { + matches!( + key.trim().to_ascii_lowercase().as_str(), + "user" | "username" | "user_name" + ) +} + +fn push_property( + properties: &mut Vec, + key: impl Into, + value: impl Into, + sensitive: bool, +) { + let key = key.into(); + let value = value.into(); + if key.trim().is_empty() || value.is_empty() { + return; + } + if let Some(existing) = properties + .iter_mut() + .find(|property| property.key.eq_ignore_ascii_case(&key)) + { + existing.value = value; + existing.sensitive |= sensitive || is_sensitive_key(&key); + return; + } + properties.push(DatasourceConnectionProperty { + sensitive: sensitive || is_sensitive_key(&key), + key, + value, + }); +} + +fn connection( + jdbc_url: String, + username: Option, + password: Option, + mut properties: Vec, + read_only: bool, +) -> DatasourceConnection { + if let Some(username) = username { + push_property(&mut properties, "user", username, false); + } + if let Some(password) = password { + push_property(&mut properties, "password", password, true); + } + DatasourceConnection { + jdbc_url, + properties, + read_only, + ssh: None, + } +} + +fn value_string(value: &Value, key: &str) -> Option { + value.get(key).and_then(Value::as_str).map(str::to_owned) +} + +fn indicates_mysql(value: &Value, jdbc_url: Option<&str>) -> bool { + jdbc_url.is_some_and(is_mysql_url) + || ["type", "driver", "jdbc", "driverId", "provider"] + .iter() + .filter_map(|key| value.get(*key).and_then(Value::as_str)) + .any(|candidate| candidate.to_ascii_lowercase().contains("mysql")) + || value + .get("driverConfig") + .and_then(Value::as_object) + .is_some_and(|driver| { + driver + .values() + .filter_map(Value::as_str) + .any(|candidate| candidate.to_ascii_lowercase().contains("mysql")) + }) +} + +fn parse_chat2db_json(content: &[u8]) -> Result { + let value: Value = serde_json::from_slice(content).map_err(|_| invalid_file())?; + let items = match &value { + Value::Array(items) => items, + Value::Object(object) => object + .get("datasources") + .and_then(Value::as_array) + .ok_or_else(invalid_file)?, + _ => return Err(invalid_file()), + }; + if items.len() > MAX_DATASOURCES { + return Err(import_limit( + "at most 1000 datasources can be imported at once", + )); + } + + let mut datasources = Vec::with_capacity(items.len()); + let mut skipped = 0; + for item in items { + let portable_connection = item.get("connection").and_then(Value::as_object); + let jdbc_url = portable_connection + .and_then(|object| object.get("jdbcUrl")) + .and_then(Value::as_str) + .or_else(|| item.get("url").and_then(Value::as_str)); + if !indicates_mysql(item, jdbc_url) { + skipped += 1; + continue; + } + let name = value_string(item, "name") + .or_else(|| value_string(item, "alias")) + .unwrap_or_default(); + let Some(jdbc_url) = jdbc_url.map(str::to_owned) else { + datasources.push(PreparedDatasource { + name, + connection: None, + }); + continue; + }; + let mut properties = Vec::new(); + let property_values = portable_connection + .and_then(|object| object.get("properties")) + .and_then(Value::as_array) + .or_else(|| item.get("extendInfo").and_then(Value::as_array)); + if let Some(property_values) = property_values { + for property in property_values { + let Some(key) = property.get("key").and_then(Value::as_str) else { + continue; + }; + if is_sensitive_key(key) { + continue; + } + let value = property + .get("value") + .and_then(Value::as_str) + .unwrap_or_default(); + push_property(&mut properties, key, value, false); + } + } + let username = value_string(item, "user"); + let read_only = portable_connection + .and_then(|object| object.get("readOnly")) + .and_then(Value::as_bool) + .or_else(|| item.get("readOnly").and_then(Value::as_bool)) + .unwrap_or(false); + // Legacy Chat2DB exports intentionally do not restore the exported password. + datasources.push(PreparedDatasource { + name, + connection: Some(connection(jdbc_url, username, None, properties, read_only)), + }); + } + Ok(ParsedImport { + datasources, + skipped, + }) +} + +fn parse_navicat_ncx(content: &[u8]) -> Result { + let mut reader = Reader::from_reader(content); + reader.config_mut().trim_text(true); + let mut version = None; + let mut raw_connections = Vec::new(); + let mut depth = 0_usize; + loop { + match reader.read_event().map_err(|_| invalid_file())? { + Event::Start(element) => { + depth = depth.checked_add(1).ok_or_else(invalid_file)?; + if depth > MAX_XML_DEPTH { + return Err(import_limit("the datasource XML is nested too deeply")); + } + collect_navicat_element(&element, &mut version, &mut raw_connections)?; + } + Event::Empty(element) => { + collect_navicat_element(&element, &mut version, &mut raw_connections)?; + } + Event::End(_) => depth = depth.saturating_sub(1), + Event::DocType(_) => return Err(invalid_file()), + Event::Eof => break, + _ => {} + } + } + let version = version.ok_or_else(invalid_file)?; + if raw_connections.len() > MAX_DATASOURCES { + return Err(import_limit( + "at most 1000 datasources can be imported at once", + )); + } + + let mut datasources = Vec::with_capacity(raw_connections.len()); + let mut skipped = 0; + for attributes in raw_connections { + let connection_type = attribute(&attributes, "ConnType").unwrap_or_default(); + if !connection_type.to_ascii_lowercase().contains("mysql") || navicat_uses_ssh(&attributes) + { + skipped += 1; + continue; + } + let jdbc_url = navicat_mysql_url(&attributes)?; + let encrypted_password = attribute(&attributes, "Password").unwrap_or_default(); + let password = (!encrypted_password.is_empty()) + .then(|| decrypt_navicat_password(version, encrypted_password)) + .transpose()?; + datasources.push(PreparedDatasource { + name: attribute(&attributes, "ConnectionName") + .unwrap_or_default() + .to_owned(), + connection: Some(connection( + jdbc_url, + attribute(&attributes, "UserName").map(str::to_owned), + password, + Vec::new(), + false, + )), + }); + } + Ok(ParsedImport { + datasources, + skipped, + }) +} + +fn collect_navicat_element( + element: &quick_xml::events::BytesStart<'_>, + version: &mut Option, + connections: &mut Vec>, +) -> Result<(), AppError> { + let name = element.name(); + if xml_local_name(name.as_ref()).eq_ignore_ascii_case(b"Connections") { + let attributes = xml_attributes(element)?; + let raw_version = attribute(&attributes, "Ver").ok_or_else(invalid_file)?; + let parsed = raw_version.parse::().map_err(|_| invalid_file())?; + if !parsed.is_finite() || parsed <= 0.0 { + return Err(invalid_file()); + } + *version = Some(parsed); + } else if xml_local_name(name.as_ref()).eq_ignore_ascii_case(b"Connection") { + connections.push(xml_attributes(element)?); + } + Ok(()) +} + +fn xml_attributes( + element: &quick_xml::events::BytesStart<'_>, +) -> Result, AppError> { + let mut values = HashMap::new(); + for raw_attribute in element.attributes().with_checks(true) { + let raw_attribute = raw_attribute.map_err(|_| invalid_file())?; + let key = std::str::from_utf8(raw_attribute.key.as_ref()) + .map_err(|_| invalid_file())? + .to_owned(); + let value = raw_attribute + .unescape_value() + .map_err(|_| invalid_file())? + .into_owned(); + if values.insert(key, value).is_some() { + return Err(invalid_file()); + } + } + Ok(values) +} + +fn attribute<'a>(attributes: &'a HashMap, key: &str) -> Option<&'a str> { + attributes + .iter() + .find(|(candidate, _)| candidate.eq_ignore_ascii_case(key)) + .map(|(_, value)| value.as_str()) +} + +fn navicat_uses_ssh(attributes: &HashMap) -> bool { + attribute(attributes, "SSH").is_some_and(|value| { + !value.is_empty() && !value.eq_ignore_ascii_case("false") && value != "0" + }) +} + +fn navicat_mysql_url(attributes: &HashMap) -> Result { + if let Some(candidate) = ["URL", "Url", "ConnectionString"] + .iter() + .find_map(|key| attribute(attributes, key)) + && is_mysql_url(candidate) + { + return Ok(candidate.to_owned()); + } + let host = attribute(attributes, "Host").ok_or_else(invalid_file)?; + let port = attribute(attributes, "Port") + .filter(|value| !value.is_empty()) + .unwrap_or("3306"); + let mut parsed = Url::parse(&format!("mysql://{host}:{port}")).map_err(|_| invalid_file())?; + if let Some(database) = ["Database", "DatabaseName", "InitialDatabase"] + .iter() + .find_map(|key| attribute(attributes, key)) + .filter(|value| !value.is_empty()) + { + parsed.set_path(database); + } + Ok(format!("jdbc:{parsed}")) +} + +fn decrypt_navicat_password(version: f64, ciphertext: &str) -> Result { + if version <= 1.1 { + decrypt_navicat_11(ciphertext) + } else { + decrypt_navicat_12(ciphertext) + } +} + +fn decrypt_navicat_12(ciphertext: &str) -> Result { + let mut bytes = hex::decode(ciphertext).map_err(|_| invalid_file())?; + let plaintext = Aes128CbcDec::new_from_slices(b"libcckeylibcckey", b"libcciv libcciv ") + .map_err(|_| invalid_file())? + .decrypt_padded_mut::(&mut bytes) + .map_err(|_| invalid_file())?; + std::str::from_utf8(plaintext) + .map(str::to_owned) + .map_err(|_| invalid_file()) +} + +#[allow(deprecated)] +fn decrypt_navicat_11(ciphertext: &str) -> Result { + let input = hex::decode(ciphertext).map_err(|_| invalid_file())?; + let key = Sha1::digest(b"3DC5CA39"); + let cipher: Blowfish = Blowfish::new_from_slice(&key).map_err(|_| invalid_file())?; + let mut iv = GenericArray::clone_from_slice(&[0xff_u8; 8]); + cipher.encrypt_block(&mut iv); + let mut chaining_value = <[u8; 8]>::from(iv); + let mut output = vec![0_u8; input.len()]; + + let full_blocks = input.len() / 8; + for block_index in 0..full_blocks { + let offset = block_index * 8; + let ciphertext_block = &input[offset..offset + 8]; + let mut block = GenericArray::clone_from_slice(ciphertext_block); + cipher.decrypt_block(&mut block); + for index in 0..8 { + output[offset + index] = block[index] ^ chaining_value[index]; + chaining_value[index] ^= ciphertext_block[index]; + } + } + let remaining = input.len() % 8; + if remaining != 0 { + let offset = full_blocks * 8; + let mut block = GenericArray::clone_from_slice(&chaining_value); + cipher.encrypt_block(&mut block); + for index in 0..remaining { + output[offset + index] = input[offset + index] ^ block[index]; + } + } + String::from_utf8(output).map_err(|_| invalid_file()) +} + +fn xml_local_name(name: &[u8]) -> &[u8] { + name.rsplit(|byte| *byte == b':').next().unwrap_or(name) +} + +fn parse_dbeaver_dbp(content: &[u8]) -> Result { + let files = read_dbeaver_archive(content)?; + let mut datasource_paths = files + .keys() + .filter(|path| path.ends_with("/data-sources.json") || *path == "data-sources.json") + .cloned() + .collect::>(); + datasource_paths.sort(); + if datasource_paths.is_empty() { + return Err(invalid_file()); + } + + let mut datasources = Vec::new(); + let mut skipped = 0; + let mut seen_entries = 0_usize; + for path in datasource_paths { + let document: Value = serde_json::from_slice(files.get(&path).ok_or_else(invalid_file)?) + .map_err(|_| invalid_file())?; + let credentials_path = path + .strip_suffix("data-sources.json") + .map(|prefix| format!("{prefix}credentials-config.json")) + .ok_or_else(invalid_file)?; + let credentials = files + .get(&credentials_path) + .map(|bytes| parse_dbeaver_credentials(bytes)) + .transpose()?; + let connections = document + .get("connections") + .and_then(Value::as_object) + .ok_or_else(invalid_file)?; + let mut ids = connections.keys().cloned().collect::>(); + ids.sort(); + for id in ids { + seen_entries = seen_entries + .checked_add(1) + .ok_or_else(|| import_limit("at most 1000 datasources can be imported at once"))?; + if seen_entries > MAX_DATASOURCES { + return Err(import_limit( + "at most 1000 datasources can be imported at once", + )); + } + let raw = connections.get(&id).ok_or_else(invalid_file)?; + let configuration = raw + .get("configuration") + .and_then(Value::as_object) + .ok_or_else(invalid_file)?; + let configured_url = configuration.get("url").and_then(Value::as_str); + if !dbeaver_is_mysql(&document, raw, configured_url) { + skipped += 1; + continue; + } + let jdbc_url = match configured_url { + Some(url) if is_mysql_url(url) => url.to_owned(), + Some(_) => { + skipped += 1; + continue; + } + None => dbeaver_mysql_url(configuration)?, + }; + let credential = credentials + .as_ref() + .and_then(|document| document.get(&id)) + .and_then(|value| value.get("#connection")); + let username = credential.and_then(|value| value_string(value, "user")); + let password = credential.and_then(|value| value_string(value, "password")); + let properties = dbeaver_properties(configuration)?; + let read_only = configuration + .get("read-only") + .or_else(|| configuration.get("readOnly")) + .and_then(Value::as_bool) + .unwrap_or(false); + datasources.push(PreparedDatasource { + name: value_string(raw, "name").unwrap_or_default(), + connection: Some(connection( + jdbc_url, username, password, properties, read_only, + )), + }); + if datasources.len() > MAX_DATASOURCES { + return Err(import_limit( + "at most 1000 datasources can be imported at once", + )); + } + } + } + Ok(ParsedImport { + datasources, + skipped, + }) +} + +fn read_dbeaver_archive(content: &[u8]) -> Result>, AppError> { + let mut archive = ZipArchive::new(Cursor::new(content)).map_err(|_| invalid_file())?; + if archive.len() > MAX_ZIP_ENTRIES { + return Err(import_limit( + "the DBeaver archive contains too many entries", + )); + } + let mut expanded_bytes = 0_u64; + let mut files = HashMap::new(); + for index in 0..archive.len() { + let mut entry = archive.by_index(index).map_err(|_| invalid_file())?; + let path = entry.enclosed_name().ok_or_else(invalid_file)?; + let path = path.to_string_lossy().replace('\\', "/"); + if entry.is_dir() { + continue; + } + if entry.size() > MAX_ZIP_ENTRY_BYTES { + return Err(import_limit("a DBeaver archive entry is too large")); + } + expanded_bytes = expanded_bytes + .checked_add(entry.size()) + .ok_or_else(|| import_limit("the DBeaver archive is too large"))?; + if expanded_bytes > MAX_ZIP_EXPANDED_BYTES { + return Err(import_limit("the DBeaver archive expands beyond its limit")); + } + let relevant = path.ends_with("/data-sources.json") + || path == "data-sources.json" + || path.ends_with("/credentials-config.json") + || path == "credentials-config.json"; + if !relevant { + continue; + } + let mut bytes = Vec::with_capacity(usize::try_from(entry.size()).unwrap_or(0)); + entry + .by_ref() + .take(MAX_ZIP_ENTRY_BYTES + 1) + .read_to_end(&mut bytes) + .map_err(|_| invalid_file())?; + if u64::try_from(bytes.len()).map_err(|_| invalid_file())? > MAX_ZIP_ENTRY_BYTES { + return Err(import_limit("a DBeaver archive entry is too large")); + } + if files.insert(path, bytes).is_some() { + return Err(invalid_file()); + } + } + Ok(files) +} + +fn parse_dbeaver_credentials(bytes: &[u8]) -> Result { + let plaintext = if bytes + .iter() + .copied() + .find(|byte| !byte.is_ascii_whitespace()) + == Some(b'{') + { + bytes.to_vec() + } else { + decrypt_dbeaver_credentials(bytes)? + }; + serde_json::from_slice(&plaintext).map_err(|_| invalid_file()) +} + +fn decrypt_dbeaver_credentials(bytes: &[u8]) -> Result, AppError> { + const KEY: [u8; 16] = [ + 0xba, 0xbb, 0x4a, 0x9f, 0x77, 0x4a, 0xb8, 0x53, 0xc9, 0x6c, 0x2d, 0x65, 0x3d, 0xfe, 0x54, + 0x4a, + ]; + if bytes.len() < 32 || !(bytes.len() - 16).is_multiple_of(16) { + return Err(invalid_file()); + } + let (iv, ciphertext) = bytes.split_at(16); + let mut ciphertext = ciphertext.to_vec(); + let plaintext = Aes128CbcDec::new_from_slices(&KEY, iv) + .map_err(|_| invalid_file())? + .decrypt_padded_mut::(&mut ciphertext) + .map_err(|_| invalid_file())?; + Ok(plaintext.to_vec()) +} + +fn dbeaver_is_mysql(document: &Value, connection: &Value, jdbc_url: Option<&str>) -> bool { + if jdbc_url.is_some_and(is_mysql_url) { + return true; + } + let provider = connection + .get("provider") + .and_then(Value::as_str) + .unwrap_or_default(); + if provider.to_ascii_lowercase().contains("mysql") { + return true; + } + if !provider.eq_ignore_ascii_case("generic") { + return false; + } + let driver_id = connection + .get("driver") + .and_then(Value::as_str) + .unwrap_or_default(); + document + .get("drivers") + .and_then(|drivers| drivers.get("generic")) + .and_then(|drivers| drivers.get(driver_id)) + .is_some_and(|driver| { + driver + .get("name") + .and_then(Value::as_str) + .unwrap_or_default() + .to_ascii_lowercase() + .contains("mysql") + }) +} + +fn dbeaver_mysql_url(configuration: &serde_json::Map) -> Result { + let host = configuration + .get("host") + .and_then(Value::as_str) + .ok_or_else(invalid_file)?; + let port = configuration + .get("port") + .and_then(Value::as_str) + .filter(|port| !port.is_empty()) + .unwrap_or("3306"); + let mut url = Url::parse(&format!("mysql://{host}:{port}")).map_err(|_| invalid_file())?; + if let Some(database) = configuration + .get("database") + .and_then(Value::as_str) + .filter(|database| !database.is_empty()) + { + url.set_path(database); + } + Ok(format!("jdbc:{url}")) +} + +fn dbeaver_properties( + configuration: &serde_json::Map, +) -> Result, AppError> { + let Some(properties) = configuration.get("properties") else { + return Ok(Vec::new()); + }; + let properties = properties.as_object().ok_or_else(invalid_file)?; + let mut output = Vec::with_capacity(properties.len()); + for (key, value) in properties { + let value = match value { + Value::String(value) => value.clone(), + Value::Bool(value) => value.to_string(), + Value::Number(value) => value.to_string(), + Value::Null => continue, + Value::Array(_) | Value::Object(_) => return Err(invalid_file()), + }; + push_property(&mut output, key, value, is_sensitive_key(key)); + } + Ok(output) +} + +fn parse_datagrip_text(content: &[u8]) -> Result { + let text = std::str::from_utf8(content).map_err(|_| invalid_file())?; + let text = text.strip_prefix('\u{feff}').unwrap_or(text); + let mut lines = text.lines(); + if lines.next().map(str::trim_end) != Some("#DataSourceSettings#") { + return Err(invalid_file()); + } + let mut blocks = Vec::new(); + let mut current = None::; + for line in lines { + match line.trim() { + "#BEGIN#" => { + if let Some(block) = current.take() + && !block.trim().is_empty() + { + blocks.push(block); + } + current = Some(String::new()); + } + "#END#" => { + if let Some(block) = current.take() + && !block.trim().is_empty() + { + blocks.push(block); + } + } + _ => { + if let Some(block) = current.as_mut() { + block.push_str(line); + block.push('\n'); + } + } + } + } + if let Some(block) = current + && !block.trim().is_empty() + { + blocks.push(block); + } + if blocks.is_empty() || blocks.len() > MAX_DATASOURCES { + return Err(invalid_file()); + } + + let mut datasources = Vec::with_capacity(blocks.len()); + let mut skipped = 0; + for block in blocks { + match parse_datagrip_datasource(&block)? { + Some(datasource) => datasources.push(datasource), + None => skipped += 1, + } + } + Ok(ParsedImport { + datasources, + skipped, + }) +} + +fn parse_datagrip_datasource(xml: &str) -> Result, AppError> { + let mut reader = Reader::from_str(xml); + reader.config_mut().trim_text(true); + let mut name = String::new(); + let mut dbms = String::new(); + let mut jdbc_url = String::new(); + let mut username = String::new(); + let mut active_text = None::>; + let mut depth = 0_usize; + loop { + match reader.read_event().map_err(|_| invalid_file())? { + Event::Start(element) => { + depth = depth.checked_add(1).ok_or_else(invalid_file)?; + if depth > MAX_XML_DEPTH { + return Err(import_limit("the datasource XML is nested too deeply")); + } + let local_name = xml_local_name(element.name().as_ref()).to_vec(); + if depth == 1 { + name = xml_attribute(&element, "name")?.unwrap_or_default(); + } + if local_name.eq_ignore_ascii_case(b"database-info") { + dbms = xml_attribute(&element, "dbms")?.unwrap_or_default(); + } + if [b"jdbc-url".as_slice(), b"user-name".as_slice()] + .iter() + .any(|candidate| local_name.eq_ignore_ascii_case(candidate)) + { + active_text = Some(local_name); + } + } + Event::Empty(element) + if xml_local_name(element.name().as_ref()) + .eq_ignore_ascii_case(b"database-info") => + { + dbms = xml_attribute(&element, "dbms")?.unwrap_or_default(); + } + Event::Text(text) => { + let value = text.unescape().map_err(|_| invalid_file())?; + match active_text.as_deref() { + Some(tag) if tag.eq_ignore_ascii_case(b"jdbc-url") => { + jdbc_url.push_str(&value); + } + Some(tag) if tag.eq_ignore_ascii_case(b"user-name") => { + username.push_str(&value); + } + _ => {} + } + } + Event::CData(text) => { + let value = std::str::from_utf8(text.as_ref()).map_err(|_| invalid_file())?; + match active_text.as_deref() { + Some(tag) if tag.eq_ignore_ascii_case(b"jdbc-url") => { + jdbc_url.push_str(value); + } + Some(tag) if tag.eq_ignore_ascii_case(b"user-name") => { + username.push_str(value); + } + _ => {} + } + } + Event::End(element) => { + if active_text.as_deref().is_some_and(|tag| { + xml_local_name(element.name().as_ref()).eq_ignore_ascii_case(tag) + }) { + active_text = None; + } + depth = depth.saturating_sub(1); + } + Event::DocType(_) => return Err(invalid_file()), + Event::Eof => break, + _ => {} + } + } + if !is_mysql_url(jdbc_url.trim()) + || (!dbms.is_empty() && !dbms.to_ascii_lowercase().contains("mysql")) + { + return Ok(None); + } + Ok(Some(PreparedDatasource { + name, + connection: Some(connection( + jdbc_url.trim().to_owned(), + (!username.is_empty()).then_some(username), + None, + Vec::new(), + false, + )), + })) +} + +fn xml_attribute( + element: &quick_xml::events::BytesStart<'_>, + name: &str, +) -> Result, AppError> { + for raw_attribute in element.attributes().with_checks(true) { + let raw_attribute = raw_attribute.map_err(|_| invalid_file())?; + if xml_local_name(raw_attribute.key.as_ref()).eq_ignore_ascii_case(name.as_bytes()) { + return raw_attribute + .unescape_value() + .map(|value| Some(value.into_owned())) + .map_err(|_| invalid_file()); + } + } + Ok(None) +} + +#[cfg(test)] +mod tests { + use std::io::{Cursor, Write}; + + use cbc::cipher::{BlockEncryptMut, KeyIvInit, block_padding::Pkcs7}; + use chat2db_contract::{CommunityDatasourceFileImportRequest, CommunityDatasourceImportFormat}; + use zip::{CompressionMethod, ZipWriter, write::SimpleFileOptions}; + + use super::{decrypt_navicat_11, parse_import}; + + type Aes128CbcEnc = cbc::Encryptor; + + fn parse(format: CommunityDatasourceImportFormat, content: Vec) -> super::ParsedImport { + parse_import(&CommunityDatasourceFileImportRequest { format, content }) + .expect("fixture parses") + } + + #[test] + fn chat2db_legacy_json_import_discards_passwords_and_sensitive_properties() { + let content = br#"[ + { + "id": 9, + "spaceId": 4, + "alias": "Legacy MySQL", + "type": "MYSQL", + "url": "jdbc:mysql://localhost:3306/demo", + "user": "root", + "password": "sentinel-password", + "extendInfo": [ + {"key": "useSSL", "value": "false"}, + {"key": "apiToken", "value": "sentinel-token"} + ] + }, + {"alias": "PostgreSQL", "type": "POSTGRESQL", "url": "jdbc:postgresql://localhost/db"} + ]"#; + let parsed = parse( + CommunityDatasourceImportFormat::Chat2dbJson, + content.to_vec(), + ); + assert_eq!(parsed.datasources.len(), 1); + assert_eq!(parsed.skipped, 1); + let connection = parsed.datasources[0] + .connection + .as_ref() + .expect("connection exists"); + assert!( + connection + .properties + .iter() + .any(|property| { property.key == "user" && property.value == "root" }) + ); + assert!( + connection + .properties + .iter() + .any(|property| { property.key == "useSSL" && property.value == "false" }) + ); + assert!(!connection.properties.iter().any(|property| { + property.value.contains("sentinel") || property.key.eq_ignore_ascii_case("password") + })); + } + + #[test] + fn navicat_11_and_12_passwords_are_compatible_with_the_java_algorithms() { + let plaintext_11 = b"secret11"; + let encrypted_11 = encrypt_navicat_11(plaintext_11); + assert_eq!( + decrypt_navicat_11(&hex::encode_upper(encrypted_11)).expect("v11 decrypts"), + "secret11" + ); + + let encrypted_12 = encrypt_aes_cbc(b"libcckeylibcckey", b"libcciv libcciv ", b"secret12"); + let ncx = format!( + r#""#, + hex::encode_upper(encrypted_12) + ); + let parsed = parse( + CommunityDatasourceImportFormat::NavicatNcx, + ncx.into_bytes(), + ); + let connection = parsed.datasources[0] + .connection + .as_ref() + .expect("connection exists"); + assert!(connection.properties.iter().any(|property| { + property.key == "password" && property.value == "secret12" && property.sensitive + })); + } + + #[test] + fn dbeaver_dbp_reads_encrypted_credentials_with_bounded_zip_entries() { + let data_sources = br#"{ + "connections": { + "mysql-1": { + "provider": "mysql", + "driver": "mysql8", + "name": "DBeaver MySQL", + "configuration": { + "host": "localhost", + "port": "3306", + "database": "demo", + "url": "jdbc:mysql://localhost:3306/demo", + "properties": {"useSSL": false} + } + }, + "pg-1": { + "provider": "postgresql", + "name": "PostgreSQL", + "configuration": {"url": "jdbc:postgresql://localhost/demo"} + } + } + }"#; + let credentials = + br##"{"mysql-1":{"#connection":{"user":"root","password":"dbeaver-secret"}}}"##; + let encrypted_credentials = encrypt_dbeaver_credentials(credentials); + let archive = zip_fixture(&[ + ( + "projects/demo/.dbeaver/data-sources.json", + data_sources.as_slice(), + ), + ( + "projects/demo/.dbeaver/credentials-config.json", + &encrypted_credentials, + ), + ]); + let parsed = parse(CommunityDatasourceImportFormat::DbeaverDbp, archive); + assert_eq!(parsed.datasources.len(), 1); + assert_eq!(parsed.skipped, 1); + let connection = parsed.datasources[0] + .connection + .as_ref() + .expect("connection exists"); + assert!(connection.properties.iter().any(|property| { + property.key == "password" && property.value == "dbeaver-secret" && property.sensitive + })); + } + + #[test] + fn datagrip_text_accepts_mysql_and_rejects_doctype() { + let text = br#"#DataSourceSettings# +#BEGIN# + + + jdbc:mysql://localhost:3306/demo + root + com.mysql.cj.jdbc.Driver + +#END# +"#; + let parsed = parse(CommunityDatasourceImportFormat::DatagripText, text.to_vec()); + assert_eq!(parsed.datasources.len(), 1); + let connection = parsed.datasources[0] + .connection + .as_ref() + .expect("connection exists"); + assert!( + connection + .properties + .iter() + .any(|property| { property.key == "user" && property.value == "root" }) + ); + + let malicious = br#"#DataSourceSettings# +#BEGIN# +]> +jdbc:mysql://localhost/demo +#END# +"#; + let result = parse_import(&CommunityDatasourceFileImportRequest { + format: CommunityDatasourceImportFormat::DatagripText, + content: malicious.to_vec(), + }); + assert!(result.is_err()); + } + + #[allow(deprecated)] + fn encrypt_navicat_11(input: &[u8]) -> Vec { + use blowfish::{ + Blowfish, + cipher::{BlockEncrypt, KeyInit, generic_array::GenericArray}, + }; + use sha1::{Digest, Sha1}; + + let key = Sha1::digest(b"3DC5CA39"); + let cipher: Blowfish = Blowfish::new_from_slice(&key).expect("valid key"); + let mut iv = GenericArray::clone_from_slice(&[0xff_u8; 8]); + cipher.encrypt_block(&mut iv); + let mut chaining_value = <[u8; 8]>::from(iv); + let mut output = vec![0_u8; input.len()]; + let full_blocks = input.len() / 8; + for block_index in 0..full_blocks { + let offset = block_index * 8; + let mut block = [0_u8; 8]; + for index in 0..8 { + block[index] = input[offset + index] ^ chaining_value[index]; + } + let mut encrypted = GenericArray::clone_from_slice(&block); + cipher.encrypt_block(&mut encrypted); + for index in 0..8 { + output[offset + index] = encrypted[index]; + chaining_value[index] ^= encrypted[index]; + } + } + let remaining = input.len() % 8; + if remaining != 0 { + let offset = full_blocks * 8; + let mut encrypted = GenericArray::clone_from_slice(&chaining_value); + cipher.encrypt_block(&mut encrypted); + for index in 0..remaining { + output[offset + index] = input[offset + index] ^ encrypted[index]; + } + } + output + } + + fn encrypt_aes_cbc(key: &[u8], iv: &[u8], plaintext: &[u8]) -> Vec { + let padded_len = (plaintext.len() / 16 + 1) * 16; + let mut buffer = vec![0_u8; padded_len]; + buffer[..plaintext.len()].copy_from_slice(plaintext); + Aes128CbcEnc::new_from_slices(key, iv) + .expect("valid key and IV") + .encrypt_padded_mut::(&mut buffer, plaintext.len()) + .expect("padding succeeds") + .to_vec() + } + + fn encrypt_dbeaver_credentials(plaintext: &[u8]) -> Vec { + const KEY: [u8; 16] = [ + 0xba, 0xbb, 0x4a, 0x9f, 0x77, 0x4a, 0xb8, 0x53, 0xc9, 0x6c, 0x2d, 0x65, 0x3d, 0xfe, + 0x54, 0x4a, + ]; + let iv = [0x24_u8; 16]; + let mut output = iv.to_vec(); + output.extend(encrypt_aes_cbc(&KEY, &iv, plaintext)); + output + } + + fn zip_fixture(entries: &[(&str, &[u8])]) -> Vec { + let cursor = Cursor::new(Vec::new()); + let mut writer = ZipWriter::new(cursor); + let options = SimpleFileOptions::default().compression_method(CompressionMethod::Deflated); + for (path, content) in entries { + writer + .start_file(path, options) + .expect("fixture entry starts"); + writer.write_all(content).expect("fixture entry writes"); + } + writer.finish().expect("fixture zip finishes").into_inner() + } +} diff --git a/crates/chat2db-core/src/datasource_edit.rs b/crates/chat2db-core/src/datasource_edit.rs new file mode 100644 index 0000000..6278871 --- /dev/null +++ b/crates/chat2db-core/src/datasource_edit.rs @@ -0,0 +1,710 @@ +use std::collections::HashSet; + +use chat2db_contract::{ + DatasourceConnection, DatasourceConnectionProperty, DatasourceEditProjection, + DatasourceSecretChange, PortableDatasourceProperty, SshAuthentication, SshAuthenticationType, + SshTunnelConfig, SshTunnelEditProjection, UpdateDatasourceRequest, +}; +use chat2db_storage::StorageError; +use url::Url; + +use crate::{AppError, Application, storage_call}; + +struct ProjectedConnection { + jdbc_url: String, + username: Option, + properties: Vec, + read_only: bool, + ssh: Option, +} + +impl Application { + /// Returns the connection fields required by an edit form without exposing credentials. + /// + /// # Errors + /// + /// Returns datasource, vault, persisted-descriptor, or URL validation failures. + pub async fn get_datasource_edit_projection( + &self, + id: &str, + ) -> Result { + let storage = self.require_storage()?; + let id = id.to_owned(); + let (record, connection) = storage_call(move || { + let (record, secret) = storage.get_datasource_with_secret(&id)?; + let connection = secret + .as_ref() + .map(|secret| decode_connection(secret.expose_secret())) + .transpose()?; + Ok((record, connection)) + }) + .await?; + + let has_secret = connection.is_some(); + let projected = connection.map_or_else( + || { + Ok(ProjectedConnection { + jdbc_url: String::new(), + username: None, + properties: Vec::new(), + read_only: false, + ssh: None, + }) + }, + |connection| project_connection(&connection), + )?; + Ok(DatasourceEditProjection { + id: record.id, + name: record.name, + driver_id: record.driver_id, + jdbc_url: projected.jdbc_url, + username: projected.username, + properties: projected.properties, + read_only: projected.read_only, + ssh: projected.ssh, + has_secret, + revision: record.revision.to_string(), + }) + } + + /// Applies an edit-form replacement while retaining omitted or blank sensitive values. + /// + /// The ordinary `update_datasource` method retains strict full-replacement semantics. This + /// compatibility entry point is intentionally separate because the retained Community form + /// never receives stored passwords and submits an empty password when it was not changed. + /// + /// # Errors + /// + /// Returns validation, datasource, revision-conflict, vault, or storage failures. + pub async fn update_datasource_preserving_secrets( + &self, + id: &str, + request: UpdateDatasourceRequest, + ) -> Result { + let UpdateDatasourceRequest { + expected_revision, + name, + driver_id, + secret_change, + } = request; + let DatasourceSecretChange::Replace { + connection: incoming, + } = secret_change + else { + return self + .update_datasource( + id, + UpdateDatasourceRequest { + expected_revision, + name, + driver_id, + secret_change, + }, + ) + .await; + }; + + let expected = expected_revision.parse::().map_err(|_| { + AppError::invalid( + "invalid_numeric_value", + "expectedRevision must be an unsigned decimal integer", + ) + })?; + let storage = self.require_storage()?; + let datasource_id = id.to_owned(); + let old_connection = storage_call(move || { + let (record, secret) = storage.get_datasource_with_secret(&datasource_id)?; + if record.revision != expected { + return Err(StorageError::RevisionConflict { + id: datasource_id, + expected, + actual: Some(record.revision), + }); + } + secret + .as_ref() + .map(|secret| decode_connection(secret.expose_secret())) + .transpose() + }) + .await?; + let connection = match old_connection { + Some(old) => merge_preserved_secrets(old, incoming)?, + None => incoming, + }; + + self.update_datasource( + id, + UpdateDatasourceRequest { + expected_revision, + name, + driver_id, + secret_change: DatasourceSecretChange::Replace { connection }, + }, + ) + .await + } +} + +fn decode_connection(bytes: &[u8]) -> Result { + serde_json::from_slice(bytes).map_err(|_| { + StorageError::InvalidDatasource("stored datasource connection descriptor is invalid") + }) +} + +fn project_connection(connection: &DatasourceConnection) -> Result { + let url_username = jdbc_url_username(&connection.jdbc_url)?; + let username = connection + .properties + .iter() + .find(|property| is_username_key(&property.key)) + .map(|property| property.value.clone()) + .filter(|value| !value.is_empty()) + .or(url_username); + let properties = connection + .properties + .iter() + .filter(|property| { + !property.sensitive + && !is_sensitive_key(&property.key) + && !is_username_key(&property.key) + }) + .map(|property| PortableDatasourceProperty { + key: property.key.clone(), + value: property.value.clone(), + }) + .collect(); + Ok(ProjectedConnection { + jdbc_url: sanitize_jdbc_url(&connection.jdbc_url)?, + username, + properties, + read_only: connection.read_only, + ssh: connection.ssh.as_ref().map(project_ssh), + }) +} + +pub(crate) fn project_ssh(config: &SshTunnelConfig) -> SshTunnelEditProjection { + let (authentication_type, key_file) = match &config.authentication { + SshAuthentication::Password { .. } => (SshAuthenticationType::Password, None), + SshAuthentication::PrivateKey { key_file, .. } => { + (SshAuthenticationType::PrivateKey, Some(key_file.clone())) + } + }; + SshTunnelEditProjection { + host_name: config.host_name.clone(), + port: config.port, + user_name: config.user_name.clone(), + local_port: config.local_port, + authentication_type, + key_file, + host_key_verification: config.host_key_verification, + } +} + +fn merge_preserved_secrets( + mut old: DatasourceConnection, + mut incoming: DatasourceConnection, +) -> Result { + move_url_userinfo_to_properties(&mut old)?; + move_url_userinfo_to_properties(&mut incoming)?; + incoming.jdbc_url = merge_sensitive_query(&old.jdbc_url, &incoming.jdbc_url)?; + incoming.ssh = merge_preserved_ssh(old.ssh.take(), incoming.ssh.take())?; + + for property in &mut incoming.properties { + if is_sensitive_key(&property.key) { + property.sensitive = true; + } + } + for old_property in old + .properties + .into_iter() + .filter(|property| property.sensitive || is_sensitive_key(&property.key)) + { + match incoming + .properties + .iter_mut() + .find(|property| property.key.eq_ignore_ascii_case(&old_property.key)) + { + Some(property) if property.value.trim().is_empty() => { + property.value = old_property.value; + property.sensitive = true; + } + Some(_) => {} + None => incoming.properties.push(DatasourceConnectionProperty { + sensitive: true, + ..old_property + }), + } + } + Ok(incoming) +} + +fn merge_preserved_ssh( + old: Option, + incoming: Option, +) -> Result, AppError> { + let Some(mut incoming) = incoming else { + return Ok(None); + }; + match &mut incoming.authentication { + SshAuthentication::Password { password } if password.is_empty() => { + let Some(SshTunnelConfig { + authentication: SshAuthentication::Password { password: old }, + .. + }) = old + else { + return Err(AppError::invalid( + "missing_ssh_password", + "SSH password is required when password authentication is selected", + )); + }; + *password = old; + } + SshAuthentication::PrivateKey { + key_file, + passphrase, + } => { + if let Some(SshTunnelConfig { + authentication: + SshAuthentication::PrivateKey { + key_file: old_key_file, + passphrase: old_passphrase, + }, + .. + }) = old + { + if key_file.trim().is_empty() { + *key_file = old_key_file; + } + if passphrase.as_deref().is_none_or(str::is_empty) { + *passphrase = old_passphrase; + } + } + if key_file.trim().is_empty() { + return Err(AppError::invalid( + "missing_ssh_private_key", + "SSH private-key path is required when private-key authentication is selected", + )); + } + } + SshAuthentication::Password { .. } => {} + } + Ok(Some(incoming)) +} + +fn move_url_userinfo_to_properties(connection: &mut DatasourceConnection) -> Result<(), AppError> { + let (prefix, mut parsed) = parse_jdbc_url(&connection.jdbc_url, "invalid_datasource_url")?; + let username = parsed.username().to_owned(); + let password = parsed.password().map(str::to_owned); + parsed.set_username("").map_err(|()| AppError::internal())?; + parsed + .set_password(None) + .map_err(|()| AppError::internal())?; + connection.jdbc_url = format!("{prefix}{parsed}"); + if !username.is_empty() + && !connection + .properties + .iter() + .any(|property| is_username_key(&property.key)) + { + connection.properties.push(DatasourceConnectionProperty { + key: "user".to_owned(), + value: username, + sensitive: false, + }); + } + if let Some(password) = password + && !connection + .properties + .iter() + .any(|property| property.key.eq_ignore_ascii_case("password")) + { + connection.properties.push(DatasourceConnectionProperty { + key: "password".to_owned(), + value: password, + sensitive: true, + }); + } + Ok(()) +} + +fn merge_sensitive_query(old_url: &str, incoming_url: &str) -> Result { + let (_, old) = parse_jdbc_url(old_url, "invalid_datasource_url")?; + let (prefix, mut incoming) = parse_jdbc_url(incoming_url, "invalid_datasource_url")?; + let mut incoming_pairs = incoming + .query_pairs() + .map(|(key, value)| (key.into_owned(), value.into_owned())) + .collect::>(); + let mut incoming_keys = incoming_pairs + .iter() + .map(|(key, _)| key.to_ascii_lowercase()) + .collect::>(); + for (key, value) in old.query_pairs() { + let normalized = key.to_ascii_lowercase(); + if is_sensitive_key(&key) && incoming_keys.insert(normalized) { + incoming_pairs.push((key.into_owned(), value.into_owned())); + } + } + { + let mut query = incoming.query_pairs_mut(); + query.clear(); + query.extend_pairs(incoming_pairs); + } + Ok(format!("{prefix}{incoming}")) +} + +pub(crate) fn sanitize_jdbc_url(jdbc_url: &str) -> Result { + let (prefix, mut parsed) = parse_jdbc_url(jdbc_url, "unsafe_datasource_projection")?; + parsed.set_username("").map_err(|()| AppError::internal())?; + parsed + .set_password(None) + .map_err(|()| AppError::internal())?; + parsed.set_fragment(None); + let retained_query = parsed + .query_pairs() + .filter(|(key, _)| !is_sensitive_key(key)) + .map(|(key, value)| (key.into_owned(), value.into_owned())) + .collect::>(); + { + let mut query = parsed.query_pairs_mut(); + query.clear(); + query.extend_pairs(retained_query); + } + Ok(format!("{prefix}{parsed}")) +} + +fn jdbc_url_username(jdbc_url: &str) -> Result, AppError> { + let (_, parsed) = parse_jdbc_url(jdbc_url, "unsafe_datasource_projection")?; + Ok((!parsed.username().is_empty()).then(|| parsed.username().to_owned())) +} + +fn parse_jdbc_url(jdbc_url: &str, code: &'static str) -> Result<(&'static str, Url), AppError> { + let jdbc_url = jdbc_url.trim(); + let (prefix, raw_url) = jdbc_url + .strip_prefix("jdbc:") + .map_or(("", jdbc_url), |url| ("jdbc:", url)); + let parsed = Url::parse(raw_url).map_err(|_| { + AppError::invalid(code, "the datasource JDBC URL cannot be processed safely") + })?; + Ok((prefix, parsed)) +} + +pub(crate) fn is_sensitive_key(key: &str) -> bool { + let key = key.trim().to_ascii_lowercase(); + key.contains("password") + || key.contains("passwd") + || key == "pwd" + || key.contains("secret") + || key.contains("token") + || key.contains("credential") + || key.contains("privatekey") + || key.contains("private_key") + || key.contains("passphrase") + || key.contains("apikey") + || key.contains("api_key") + || key.contains("api-key") + || key.contains("accesskey") + || key.contains("access_key") +} + +fn is_username_key(key: &str) -> bool { + matches!( + key.trim().to_ascii_lowercase().as_str(), + "user" | "username" | "user_name" + ) +} + +#[cfg(test)] +mod tests { + use std::{ + collections::HashMap, + sync::{Arc, Mutex}, + }; + + use chat2db_contract::{ + CreateDatasourceRequest, DatasourceConnection, DatasourceConnectionProperty, + DatasourceSecretChange, SshAuthentication, SshAuthenticationType, SshHostKeyVerification, + SshTunnelConfig, UpdateDatasourceRequest, + }; + use chat2db_storage::{SecretRef, SecretValue, SecretVault, SecretVaultError, Storage}; + use tempfile::TempDir; + + use crate::Application; + + #[derive(Debug, Default)] + struct MemoryVault { + values: Mutex>>, + } + + impl SecretVault for MemoryVault { + fn probe(&self) -> Result<(), SecretVaultError> { + Ok(()) + } + + fn create( + &self, + reference: &SecretRef, + value: &SecretValue, + ) -> Result<(), SecretVaultError> { + self.values.lock().expect("vault lock").insert( + reference.as_str().to_owned(), + value.expose_secret().to_vec(), + ); + Ok(()) + } + + fn get(&self, reference: &SecretRef) -> Result, SecretVaultError> { + Ok(self + .values + .lock() + .expect("vault lock") + .get(reference.as_str()) + .cloned() + .map(SecretValue::new)) + } + + fn delete(&self, reference: &SecretRef) -> Result<(), SecretVaultError> { + self.values + .lock() + .expect("vault lock") + .remove(reference.as_str()); + Ok(()) + } + } + + fn application() -> (TempDir, Application) { + let directory = TempDir::new().expect("temp dir"); + let storage = Storage::open(directory.path(), Arc::new(MemoryVault::default())) + .expect("storage opens"); + (directory, Application::with_storage(storage)) + } + + fn property(key: &str, value: &str, sensitive: bool) -> DatasourceConnectionProperty { + DatasourceConnectionProperty { + key: key.to_owned(), + value: value.to_owned(), + sensitive, + } + } + + #[tokio::test] + #[allow(clippy::too_many_lines)] + async fn edit_projection_redacts_credentials_and_compat_update_keeps_them() { + let (_directory, application) = application(); + let created = application + .create_datasource(CreateDatasourceRequest { + name: "Original".to_owned(), + driver_id: "mysql".to_owned(), + connection: Some(DatasourceConnection { + jdbc_url: "jdbc:mysql://url-user:url-password@localhost:3306/old?token=url-token&useSSL=false".to_owned(), + properties: vec![ + property("user", "root", false), + property("password", "stored-password", true), + property("connectionTimeZone", "UTC", false), + property("apiToken", "property-token", false), + ], + read_only: false, + ssh: Some(SshTunnelConfig { + host_name: "ssh-old.internal".to_owned(), + port: 22, + user_name: "ssh-user".to_owned(), + authentication: SshAuthentication::Password { + password: "stored-ssh-password".to_owned(), + }, + host_key_verification: SshHostKeyVerification::KnownHosts, + local_port: None, + }), + }), + }) + .await + .expect("datasource creates"); + + let projection = application + .get_datasource_edit_projection(&created.id) + .await + .expect("projection loads"); + assert_eq!(projection.username.as_deref(), Some("root")); + assert_eq!( + projection.jdbc_url, + "jdbc:mysql://localhost:3306/old?useSSL=false" + ); + assert_eq!(projection.properties.len(), 1); + let ssh = projection.ssh.as_ref().expect("SSH projection exists"); + assert_eq!(ssh.host_name, "ssh-old.internal"); + assert_eq!(ssh.authentication_type, SshAuthenticationType::Password); + let serialized = serde_json::to_string(&projection).expect("projection serializes"); + for forbidden in [ + "url-user", + "url-password", + "url-token", + "stored-password", + "property-token", + "stored-ssh-password", + ] { + assert!( + !serialized.contains(forbidden), + "projection leaked {forbidden}" + ); + } + + let updated = application + .update_datasource_preserving_secrets( + &created.id, + UpdateDatasourceRequest { + expected_revision: projection.revision, + name: "Updated".to_owned(), + driver_id: "mysql".to_owned(), + secret_change: DatasourceSecretChange::Replace { + connection: DatasourceConnection { + jdbc_url: "jdbc:mysql://localhost:3306/new?useSSL=true".to_owned(), + properties: vec![ + property("user", "new-user", false), + property("connectionTimeZone", "Asia/Shanghai", false), + ], + read_only: true, + ssh: Some(SshTunnelConfig { + host_name: "ssh-new.internal".to_owned(), + port: 2222, + user_name: "new-ssh-user".to_owned(), + authentication: SshAuthentication::Password { + password: String::new(), + }, + host_key_verification: SshHostKeyVerification::KnownHosts, + local_port: Some(33060), + }), + }, + }, + }, + ) + .await + .expect("compatibility update succeeds"); + assert_eq!(updated.revision, "2"); + + let storage = application.storage().expect("storage configured"); + let (_, secret) = storage + .get_datasource_with_secret(&created.id) + .expect("stored secret loads"); + let connection: DatasourceConnection = + serde_json::from_slice(secret.expect("stored descriptor exists").expose_secret()) + .expect("stored descriptor decodes"); + assert!(connection.jdbc_url.contains("/new")); + assert!(connection.jdbc_url.contains("token=url-token")); + assert!( + connection + .properties + .iter() + .any(|property| { property.key == "user" && property.value == "new-user" }) + ); + assert!( + connection.properties.iter().any(|property| { + property.key == "password" && property.value == "stored-password" + }) + ); + assert!( + connection.properties.iter().any(|property| { + property.key == "apiToken" && property.value == "property-token" + }) + ); + let ssh = connection.ssh.expect("SSH config remains installed"); + assert_eq!(ssh.host_name, "ssh-new.internal"); + assert_eq!(ssh.port, 2222); + assert!(matches!( + ssh.authentication, + SshAuthentication::Password { password } if password == "stored-ssh-password" + )); + let stale = application + .update_datasource_preserving_secrets( + &created.id, + UpdateDatasourceRequest { + expected_revision: "1".to_owned(), + name: "Stale".to_owned(), + driver_id: "mysql".to_owned(), + secret_change: DatasourceSecretChange::Keep, + }, + ) + .await + .expect_err("SSH updates must not bypass datasource revision CAS"); + assert_eq!(stale.api_error().code, "revision_conflict"); + } + + #[tokio::test] + async fn private_key_projection_and_blank_passphrase_update_are_secret_safe() { + let (_directory, application) = application(); + let created = application + .create_datasource(CreateDatasourceRequest { + name: "Private key".to_owned(), + driver_id: "mysql".to_owned(), + connection: Some(DatasourceConnection { + jdbc_url: "jdbc:mysql://db.internal:3306/app".to_owned(), + properties: Vec::new(), + read_only: false, + ssh: Some(SshTunnelConfig { + host_name: "bastion.internal".to_owned(), + port: 22, + user_name: "developer".to_owned(), + authentication: SshAuthentication::PrivateKey { + key_file: "/keys/id_ed25519".to_owned(), + passphrase: Some("stored-passphrase".to_owned()), + }, + host_key_verification: SshHostKeyVerification::KnownHosts, + local_port: None, + }), + }), + }) + .await + .expect("datasource creates"); + let projection = application + .get_datasource_edit_projection(&created.id) + .await + .expect("projection loads"); + let ssh = projection.ssh.as_ref().expect("SSH projection exists"); + assert_eq!(ssh.authentication_type, SshAuthenticationType::PrivateKey); + assert_eq!(ssh.key_file.as_deref(), Some("/keys/id_ed25519")); + assert!( + !serde_json::to_string(&projection) + .expect("projection serializes") + .contains("stored-passphrase") + ); + + application + .update_datasource_preserving_secrets( + &created.id, + UpdateDatasourceRequest { + expected_revision: projection.revision, + name: "Private key updated".to_owned(), + driver_id: "mysql".to_owned(), + secret_change: DatasourceSecretChange::Replace { + connection: DatasourceConnection { + jdbc_url: "jdbc:mysql://db.internal:3306/app".to_owned(), + properties: Vec::new(), + read_only: false, + ssh: Some(SshTunnelConfig { + host_name: "bastion.internal".to_owned(), + port: 22, + user_name: "developer".to_owned(), + authentication: SshAuthentication::PrivateKey { + key_file: "/keys/id_ed25519".to_owned(), + passphrase: None, + }, + host_key_verification: SshHostKeyVerification::KnownHosts, + local_port: None, + }), + }, + }, + }, + ) + .await + .expect("blank passphrase retains the stored value"); + let storage = application.storage().expect("storage configured"); + let (_, secret) = storage + .get_datasource_with_secret(&created.id) + .expect("stored secret loads"); + let connection: DatasourceConnection = + serde_json::from_slice(secret.expect("descriptor exists").expose_secret()) + .expect("descriptor decodes"); + assert!(matches!( + connection.ssh.expect("SSH remains").authentication, + SshAuthentication::PrivateKey { passphrase: Some(value), .. } + if value == "stored-passphrase" + )); + } +} diff --git a/crates/chat2db-core/src/datasource_session.rs b/crates/chat2db-core/src/datasource_session.rs index 741ce6d..de2c4c9 100644 --- a/crates/chat2db-core/src/datasource_session.rs +++ b/crates/chat2db-core/src/datasource_session.rs @@ -5,6 +5,8 @@ use chat2db_storage::Storage; use crate::{AppError, AppErrorKind}; pub(crate) struct ResolvedDatasourceConnection { + pub(crate) datasource_id: String, + pub(crate) datasource_revision: u64, pub(crate) driver_id: String, pub(crate) datasource_name: String, pub(crate) connection: DatasourceConnection, @@ -43,6 +45,8 @@ pub(crate) async fn resolve_datasource_connection( ) })?; Ok(ResolvedDatasourceConnection { + datasource_id: datasource.id, + datasource_revision: datasource.revision, driver_id: datasource.driver_id, datasource_name: datasource.name, connection, @@ -69,6 +73,7 @@ fn session_config( driver_id, datasource_name: _, connection, + .. } = resolved; let read_only = match read_only { SessionReadOnly::Configured => connection.read_only, @@ -118,6 +123,8 @@ mod tests { fn resolved(read_only: bool) -> ResolvedDatasourceConnection { ResolvedDatasourceConnection { + datasource_id: "datasource-1".to_owned(), + datasource_revision: 1, driver_id: "driver-1".to_owned(), datasource_name: "Local H2".to_owned(), connection: DatasourceConnection { @@ -128,6 +135,7 @@ mod tests { sensitive: false, }], read_only, + ssh: None, }, } } diff --git a/crates/chat2db-core/src/error.rs b/crates/chat2db-core/src/error.rs index 9c46be7..6f65b7f 100644 --- a/crates/chat2db-core/src/error.rs +++ b/crates/chat2db-core/src/error.rs @@ -269,6 +269,42 @@ impl From for AppError { StorageError::InvalidDatasource(message) => { Self::invalid("invalid_datasource", message) } + StorageError::WorkspaceNamespaceNotFound(id) => Self::not_found( + "workspace_namespace_not_found", + format!("Workspace namespace {id} does not exist"), + ), + StorageError::WorkspaceNodeNotFound(id) => Self::not_found( + "workspace_node_not_found", + format!("Workspace node {id} does not exist"), + ), + StorageError::InvalidWorkspace(message) => { + Self::invalid("invalid_workspace_operation", message) + } + StorageError::CommunityDashboardNotFound(id) => Self::not_found( + "community_dashboard_not_found", + format!("Community dashboard {id} does not exist"), + ), + StorageError::InvalidCommunityDashboard(message) => { + Self::invalid("invalid_community_dashboard", message) + } + StorageError::CommunityChartNotFound(id) => Self::not_found( + "community_chart_not_found", + format!("Community chart {id} does not exist"), + ), + StorageError::InvalidCommunityChart(message) => { + Self::invalid("invalid_community_chart", message) + } + StorageError::TransferTaskNotFound(id) => Self::not_found( + "transfer_task_not_found", + format!("Transfer task {id} does not exist"), + ), + StorageError::TransferArtifactNotFound(id) => Self::not_found( + "transfer_artifact_not_found", + format!("Transfer artifact {id} does not exist or expired"), + ), + StorageError::InvalidTransfer(message) => { + Self::invalid("invalid_transfer_operation", message) + } StorageError::SavedConsoleNotFound(id) => Self::not_found( "saved_console_not_found", format!("Saved Console {id} does not exist"), diff --git a/crates/chat2db-core/src/legacy_community_import.rs b/crates/chat2db-core/src/legacy_community_import.rs new file mode 100644 index 0000000..511d5fc --- /dev/null +++ b/crates/chat2db-core/src/legacy_community_import.rs @@ -0,0 +1,638 @@ +use std::{ + collections::HashMap, + fs::{self, File, OpenOptions}, + io::{self, Read as _, Write as _}, + path::{Path, PathBuf}, +}; + +use chat2db_contract::{ + CommunityDatasourceExport, DatasourceConnection, DatasourceConnectionProperty, + PortableCommunityDatasource, PortableDatasourceConnection, PortableDatasourceProperty, +}; +use chat2db_java_bridge::{JdbcRow, JdbcValue, QueryEvent, QueryOptions, QueryRequest}; +use directories::BaseDirs; +use tempfile::{Builder as TempDirBuilder, TempDir}; +use url::Url; + +use crate::{ + AppError, Application, + datasource_edit::sanitize_jdbc_url, + datasource_session::{ResolvedDatasourceConnection, SessionReadOnly, open_datasource_session}, + now_millis, +}; + +const H2_DRIVER_CLASS: &str = "org.h2.Driver"; +const H2_MIGRATION_DRIVER_PACK_ID: &str = "h2-legacy-migration"; +const MAX_LEGACY_DATABASE_BYTES: u64 = 2 * 1024 * 1024 * 1024; +const MAX_LEGACY_DATASOURCES: usize = 1_000; +const MAX_LEGACY_QUERY_BYTES: u64 = 16 * 1024 * 1024; + +/// Summary of one Desktop-only migration from the pre-Community `Chat2DB` H2 store. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct LegacyCommunityImportOutcome { + pub database_found: bool, + pub imported: u32, + pub skipped_unsupported: u32, + pub password_fields_omitted: u32, + pub other_sensitive_fields_omitted: u32, +} + +struct LegacyDatabaseSnapshot { + _directory: TempDir, + jdbc_base_path: PathBuf, +} + +struct LegacyRows { + labels: HashMap, + rows: Vec, +} + +impl Application { + /// Imports native `MySQL` datasource definitions from the old `~/.chat2db` H2 database. + /// + /// The old database is copied to a private snapshot before Java starts. Passwords and other + /// sensitive legacy fields are intentionally omitted and must be supplied again by the user. + /// + /// # Errors + /// + /// Returns discovery, snapshot, H2 driver, engine, query, validation, or storage failures. + pub async fn import_legacy_community_datasources( + &self, + ) -> Result { + let Some(base_dirs) = BaseDirs::new() else { + return Err(AppError::unavailable( + "legacy_community_home_unavailable", + "The legacy Chat2DB home directory could not be resolved", + )); + }; + let database_file = require_legacy_database(base_dirs.home_dir())?; + self.import_legacy_community_datasources_from_file(&database_file) + .await + } + + #[doc(hidden)] + pub async fn import_legacy_community_datasources_from_file( + &self, + database_file: &Path, + ) -> Result { + let snapshot = tokio::task::spawn_blocking({ + let database_file = database_file.to_path_buf(); + move || snapshot_legacy_database(&database_file) + }) + .await + .map_err(|_| AppError::internal())??; + let h2_driver = self + .list_drivers() + .items + .into_iter() + .find(|driver| driver.pack_id == H2_MIGRATION_DRIVER_PACK_ID) + .ok_or_else(|| { + AppError::unavailable( + "legacy_community_h2_driver_missing", + "The bundled H2 migration driver is not installed", + ) + })?; + if h2_driver.driver_class != H2_DRIVER_CLASS { + return Err(AppError::unavailable( + "legacy_community_h2_driver_invalid", + "The bundled H2 migration driver has an unexpected driver class", + )); + } + let jdbc_url = snapshot_jdbc_url(&snapshot.jdbc_base_path)?; + let engine = self.require_engine().await?; + let session = open_datasource_session( + &engine, + ResolvedDatasourceConnection { + datasource_id: "legacy-community-import".to_owned(), + datasource_revision: 0, + driver_id: h2_driver.driver_id, + datasource_name: "Legacy Chat2DB migration snapshot".to_owned(), + connection: DatasourceConnection { + jdbc_url, + properties: vec![DatasourceConnectionProperty { + key: "user".to_owned(), + value: "sa".to_owned(), + sensitive: false, + }], + read_only: true, + ssh: None, + }, + }, + SessionReadOnly::Forced, + ) + .await?; + let queried = query_legacy_datasources(&session).await; + let closed = session.close().await.map_err(AppError::from); + let rows = match (queried, closed) { + (Ok(rows), Ok(())) => rows, + (Err(error), _) | (Ok(_), Err(error)) => return Err(error), + }; + drop(snapshot); + + let (mut outcome, datasources) = convert_legacy_rows(rows)?; + let imported = self + .import_community_datasources(CommunityDatasourceExport { + schema_version: 1, + exported_at_ms: now_millis()?.to_string(), + datasources, + }) + .await?; + if imported.count == 0 { + return Err(AppError::unavailable( + "legacy_community_mysql_not_imported", + "No legacy MySQL datasource was imported", + )); + } + outcome.imported = imported.count; + Ok(outcome) + } +} + +fn convert_legacy_rows( + rows: LegacyRows, +) -> Result< + ( + LegacyCommunityImportOutcome, + Vec, + ), + AppError, +> { + let mut outcome = LegacyCommunityImportOutcome { + database_found: true, + ..LegacyCommunityImportOutcome::default() + }; + let mut datasources = Vec::new(); + for row in rows.rows { + if !text_field(&rows.labels, &row, "type")? + .is_some_and(|value| value.trim().eq_ignore_ascii_case("mysql")) + { + outcome.skipped_unsupported = outcome.skipped_unsupported.saturating_add(1); + continue; + } + if nonempty_field(&rows.labels, &row, "password")? { + outcome.password_fields_omitted = outcome.password_fields_omitted.saturating_add(1); + } + if ["ssh", "ssl", "driver_config", "extend_info"] + .into_iter() + .any(|field| nonempty_field(&rows.labels, &row, field).unwrap_or(true)) + { + outcome.other_sensitive_fields_omitted = + outcome.other_sensitive_fields_omitted.saturating_add(1); + } + datasources.push(portable_mysql_datasource(&rows.labels, &row)?); + } + if datasources.is_empty() { + return Err(AppError::not_found( + "legacy_community_mysql_not_found", + "The legacy Chat2DB database contains no compatible MySQL datasources", + )); + } + Ok((outcome, datasources)) +} + +fn require_legacy_database(home: &Path) -> Result { + find_legacy_database(home)?.ok_or_else(|| { + AppError::not_found( + "legacy_community_database_not_found", + "The legacy Chat2DB datasource database does not exist", + ) + }) +} + +fn find_legacy_database(home: &Path) -> Result, AppError> { + let candidate = home.join(".chat2db").join("db").join("chat2db.mv.db"); + match fs::symlink_metadata(&candidate) { + Ok(metadata) if metadata.file_type().is_symlink() || !metadata.is_file() => { + Err(AppError::invalid( + "unsafe_legacy_community_database", + "The legacy Chat2DB database path is not a regular file", + )) + } + Ok(_) => Ok(Some(candidate)), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(None), + Err(error) => Err(snapshot_error(&error)), + } +} + +fn snapshot_legacy_database(source_path: &Path) -> Result { + let mut source = + open_regular_file_no_follow(source_path).map_err(|error| snapshot_error(&error))?; + let source_length = source + .metadata() + .map_err(|error| snapshot_error(&error))? + .len(); + if source_length > MAX_LEGACY_DATABASE_BYTES { + return Err(AppError::invalid( + "legacy_community_database_too_large", + "The legacy Chat2DB database exceeds the migration size limit", + )); + } + let directory = TempDirBuilder::new() + .prefix("chat2db-legacy-import-") + .tempdir() + .map_err(|error| snapshot_error(&error))?; + let source_name = source_path + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| { + AppError::invalid( + "invalid_legacy_community_database", + "The legacy Chat2DB database filename is invalid", + ) + })?; + if !source_name.ends_with(".mv.db") { + return Err(AppError::invalid( + "legacy_community_database_format_unsupported", + "Only the Community H2 MVStore database format can be migrated", + )); + } + let snapshot_path = directory.path().join("chat2db.mv.db"); + let mut output = OpenOptions::new() + .create_new(true) + .write(true) + .open(&snapshot_path) + .map_err(|error| snapshot_error(&error))?; + let copied = io::copy( + &mut std::io::Read::by_ref(&mut source).take(MAX_LEGACY_DATABASE_BYTES.saturating_add(1)), + &mut output, + ) + .map_err(|error| snapshot_error(&error))?; + if copied != source_length || copied > MAX_LEGACY_DATABASE_BYTES { + return Err(AppError::unavailable( + "legacy_community_snapshot_changed", + "The legacy Chat2DB database changed while its migration snapshot was created", + )); + } + output.flush().map_err(|error| snapshot_error(&error))?; + output.sync_all().map_err(|error| snapshot_error(&error))?; + Ok(LegacyDatabaseSnapshot { + jdbc_base_path: directory.path().join("chat2db"), + _directory: directory, + }) +} + +fn open_regular_file_no_follow(path: &Path) -> io::Result { + let mut options = OpenOptions::new(); + options.read(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt as _; + + let flags = i32::try_from(rustix::fs::OFlags::NOFOLLOW.bits()) + .map_err(|_| io::Error::other("legacy database open flags are not representable"))?; + options.custom_flags(flags); + } + #[cfg(windows)] + { + use std::os::windows::fs::OpenOptionsExt as _; + + const FILE_FLAG_OPEN_REPARSE_POINT: u32 = 0x0020_0000; + options.custom_flags(FILE_FLAG_OPEN_REPARSE_POINT); + } + let file = options.open(path)?; + if !file.metadata()?.is_file() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "legacy database path is not a regular file", + )); + } + Ok(file) +} + +fn snapshot_jdbc_url(base_path: &Path) -> Result { + let path = base_path.to_str().ok_or_else(|| { + AppError::invalid( + "invalid_legacy_community_database", + "The migration snapshot path is not valid UTF-8", + ) + })?; + if path.contains([';', '\0']) { + return Err(AppError::invalid( + "invalid_legacy_community_database", + "The migration snapshot path cannot be represented as an H2 URL", + )); + } + Ok(format!( + "jdbc:h2:file:{};ACCESS_MODE_DATA=r;IFEXISTS=TRUE;MODE=MYSQL;FILE_LOCK=NO", + path.replace('\\', "/") + )) +} + +async fn query_legacy_datasources( + session: &chat2db_java_bridge::Session, +) -> Result { + let mut stream = session + .execute_query(QueryRequest { + sql: "SELECT * FROM DATA_SOURCE".to_owned(), + parameters: Vec::new(), + transaction_id: None, + options: QueryOptions { + max_rows: u64::try_from(MAX_LEGACY_DATASOURCES + 1) + .map_err(|_| AppError::internal())?, + target_batch_rows: 250, + target_batch_bytes: 256 * 1024, + initial_batch_credits: 8, + max_result_bytes: MAX_LEGACY_QUERY_BYTES, + }, + }) + .await + .map_err(|error| import_query_error(&error))?; + let mut labels = None; + let mut rows = Vec::new(); + let mut completed = false; + while let Some(event) = stream + .next_event() + .await + .map_err(|error| import_query_error(&error))? + { + match event { + QueryEvent::Started(started) => { + if labels.is_some() { + return Err(AppError::internal()); + } + labels = Some( + started + .columns + .into_iter() + .enumerate() + .map(|(index, column)| (column.label.to_ascii_lowercase(), index)) + .collect(), + ); + } + QueryEvent::Batch(batch) => rows.extend(batch.rows), + QueryEvent::Completed(result) => { + if result.truncated_by_max_rows + || result.truncated_by_max_result_bytes + || rows.len() > MAX_LEGACY_DATASOURCES + { + return Err(AppError::invalid( + "legacy_community_datasource_limit_exceeded", + "The legacy Chat2DB database contains too many datasource records", + )); + } + completed = true; + break; + } + } + } + if !completed { + return Err(AppError::unavailable( + "legacy_community_import_failed", + "The legacy Chat2DB datasource query ended unexpectedly", + )); + } + Ok(LegacyRows { + labels: labels.ok_or_else(AppError::internal)?, + rows, + }) +} + +fn portable_mysql_datasource( + labels: &HashMap, + row: &JdbcRow, +) -> Result { + let source_id = text_field(labels, row, "id")?; + let host = text_field(labels, row, "host")?.unwrap_or_default(); + let name = text_field(labels, row, "alias")? + .filter(|name| !name.trim().is_empty()) + .unwrap_or_else(|| { + if host.trim().is_empty() { + "Imported MySQL".to_owned() + } else { + format!("@{}", host.trim()) + } + }); + let jdbc_url = legacy_mysql_url(labels, row, &host)?; + let properties = text_field(labels, row, "user_name")? + .filter(|user| !user.trim().is_empty()) + .map(|user| { + vec![PortableDatasourceProperty { + key: "user".to_owned(), + value: user, + }] + }) + .unwrap_or_default(); + Ok(PortableCommunityDatasource { + source_id, + name, + driver_id: "mysql".to_owned(), + connection: Some(PortableDatasourceConnection { + jdbc_url, + properties, + read_only: false, + ssh: None, + }), + }) +} + +fn legacy_mysql_url( + labels: &HashMap, + row: &JdbcRow, + host: &str, +) -> Result { + let configured = text_field(labels, row, "url")? + .filter(|value| !value.trim().is_empty()) + .or(text_field(labels, row, "jdbc")?.filter(|value| !value.trim().is_empty())); + if let Some(configured) = configured { + let configured = configured.trim(); + if configured.to_ascii_lowercase().starts_with("jdbc:mysql://") { + return sanitize_jdbc_url(configured); + } + if configured.to_ascii_lowercase().starts_with("mysql://") { + return sanitize_jdbc_url(&format!("jdbc:{configured}")); + } + return Err(AppError::invalid( + "invalid_legacy_mysql_url", + "A legacy MySQL datasource contains a non-MySQL URL", + )); + } + if host.trim().is_empty() { + return Err(AppError::invalid( + "invalid_legacy_mysql_url", + "A legacy MySQL datasource has neither a URL nor a host", + )); + } + let port = text_field(labels, row, "port")? + .filter(|value| !value.trim().is_empty()) + .map(|value| value.parse::()) + .transpose() + .map_err(|_| { + AppError::invalid( + "invalid_legacy_mysql_url", + "A legacy MySQL datasource has an invalid port", + ) + })? + .unwrap_or(3306); + let database = text_field(labels, row, "service_name")?.unwrap_or_default(); + let mut url = Url::parse("mysql://localhost").map_err(|_| AppError::internal())?; + url.set_host(Some(host.trim())).map_err(|_| { + AppError::invalid( + "invalid_legacy_mysql_url", + "A legacy MySQL datasource has an invalid host", + ) + })?; + url.set_port(Some(port)) + .map_err(|()| AppError::internal())?; + if !database.trim().is_empty() { + url.set_path(&format!("/{}", database.trim().trim_start_matches('/'))); + } + Ok(format!("jdbc:{url}")) +} + +fn nonempty_field( + labels: &HashMap, + row: &JdbcRow, + field: &str, +) -> Result { + Ok(text_field(labels, row, field)?.is_some_and(|value| !value.trim().is_empty())) +} + +fn text_field( + labels: &HashMap, + row: &JdbcRow, + field: &str, +) -> Result, AppError> { + let Some(index) = labels.get(field).copied() else { + return Ok(None); + }; + let value = row.values.get(index).ok_or_else(AppError::internal)?; + match value { + JdbcValue::Null => Ok(None), + JdbcValue::Boolean(value) => Ok(Some(value.to_string())), + JdbcValue::SignedInteger(value) => Ok(Some(value.to_string())), + JdbcValue::UnsignedInteger(value) => Ok(Some(value.to_string())), + JdbcValue::Float32(value) => Ok(Some(value.to_string())), + JdbcValue::Float64(value) => Ok(Some(value.to_string())), + JdbcValue::Decimal(value) + | JdbcValue::Text(value) + | JdbcValue::Date(value) + | JdbcValue::Time(value) + | JdbcValue::Timestamp(value) + | JdbcValue::TimestampWithTimeZone(value) + | JdbcValue::Json(value) + | JdbcValue::Uuid(value) => Ok(Some(value.clone())), + JdbcValue::Opaque { display_value, .. } => Ok(Some(display_value.clone())), + JdbcValue::Binary(_) => Err(AppError::invalid( + "invalid_legacy_community_datasource", + format!("Legacy datasource field {field} is not text"), + )), + } +} + +fn snapshot_error(error: &io::Error) -> AppError { + tracing::warn!(%error, "legacy Chat2DB database snapshot failed"); + AppError::unavailable( + "legacy_community_snapshot_failed", + "The legacy Chat2DB database could not be copied for migration", + ) +} + +fn import_query_error(error: &chat2db_java_bridge::BridgeError) -> AppError { + tracing::warn!(%error, "legacy Chat2DB datasource query failed"); + AppError::unavailable( + "legacy_community_import_failed", + "The legacy Chat2DB datasource database could not be read", + ) +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use chat2db_java_bridge::{JdbcRow, JdbcValue}; + use tempfile::TempDir; + + use super::{ + LegacyRows, convert_legacy_rows, find_legacy_database, portable_mysql_datasource, + require_legacy_database, + }; + + #[cfg(any(unix, windows))] + #[test] + fn discovery_is_scoped_to_an_isolated_home_and_rejects_symlinks() { + let home = TempDir::new().expect("temporary home"); + assert!( + find_legacy_database(home.path()) + .expect("missing legacy database is valid") + .is_none() + ); + let database_dir = home.path().join(".chat2db/db"); + std::fs::create_dir_all(&database_dir).expect("legacy database directory creates"); + let database = database_dir.join("chat2db.mv.db"); + std::fs::write(&database, b"snapshot").expect("legacy database fixture writes"); + assert_eq!( + find_legacy_database(home.path()).expect("legacy database discovers"), + Some(database.clone()) + ); + + std::fs::remove_file(&database).expect("regular fixture removes"); + let target = home.path().join("outside.mv.db"); + std::fs::write(&target, b"snapshot").expect("symlink target writes"); + #[cfg(unix)] + std::os::unix::fs::symlink(&target, &database).expect("file symlink creates"); + #[cfg(windows)] + std::os::windows::fs::symlink_file(&target, &database).expect("file symlink creates"); + let error = find_legacy_database(home.path()).expect_err("symlink must be rejected"); + assert_eq!(error.api_error().code, "unsafe_legacy_community_database"); + } + + #[test] + fn missing_legacy_database_is_an_explicit_error() { + let home = TempDir::new().expect("temporary home"); + let error = require_legacy_database(home.path()).expect_err("missing database must fail"); + assert_eq!( + error.api_error().code, + "legacy_community_database_not_found" + ); + } + + #[test] + fn legacy_database_without_mysql_is_an_explicit_error() { + let labels = HashMap::from([("type".to_owned(), 0), ("alias".to_owned(), 1)]); + let rows = LegacyRows { + labels, + rows: vec![JdbcRow { + values: vec![ + JdbcValue::Text("POSTGRESQL".to_owned()), + JdbcValue::Text("Legacy PostgreSQL".to_owned()), + ], + }], + }; + let error = convert_legacy_rows(rows).expect_err("non-MySQL database must fail"); + assert_eq!(error.api_error().code, "legacy_community_mysql_not_found"); + } + + #[test] + fn legacy_mysql_conversion_omits_passwords_and_keeps_username() { + let fields = ["id", "alias", "type", "url", "user_name", "password"]; + let labels = fields + .into_iter() + .enumerate() + .map(|(index, field)| (field.to_owned(), index)) + .collect::>(); + let row = JdbcRow { + values: vec![ + JdbcValue::SignedInteger(7), + JdbcValue::Text("Local MySQL".to_owned()), + JdbcValue::Text("MYSQL".to_owned()), + JdbcValue::Text( + "jdbc:mysql://embedded:must-not-migrate@127.0.0.1:3306/demo?useSSL=true&password=must-not-migrate" + .to_owned(), + ), + JdbcValue::Text("developer".to_owned()), + JdbcValue::Text("must-not-migrate".to_owned()), + ], + }; + let converted = portable_mysql_datasource(&labels, &row).expect("row converts"); + let connection = converted.connection.as_ref().expect("connection exists"); + assert_eq!(connection.properties.len(), 1); + assert_eq!(connection.properties[0].key, "user"); + assert_eq!(connection.properties[0].value, "developer"); + assert_eq!( + connection.jdbc_url, + "jdbc:mysql://127.0.0.1:3306/demo?useSSL=true" + ); + let json = serde_json::to_string(&converted).expect("datasource serializes"); + assert!(!json.contains("must-not-migrate")); + } +} diff --git a/crates/chat2db-core/src/lib.rs b/crates/chat2db-core/src/lib.rs index e748164..9ce9f5c 100644 --- a/crates/chat2db-core/src/lib.rs +++ b/crates/chat2db-core/src/lib.rs @@ -3,15 +3,26 @@ mod agent; mod community; mod convert; +mod datasource_compatibility; +mod datasource_converter; +mod datasource_edit; mod datasource_session; mod driver_pack; mod engine_manager; mod error; mod large_value; +mod legacy_community_import; +mod mysql_account; +mod mysql_dashboard; pub mod mysql_ddl; +mod mysql_schema_diff; +mod mysql_workspace; mod native_mysql; mod operation; mod query; +mod ssh; +mod transfer; +mod workspace; use std::{ collections::{HashMap, HashSet}, @@ -47,8 +58,10 @@ pub use large_value::{ LargeValueChunk, LargeValueEncoding, LargeValueError, LargeValuePreview, LargeValueStoreStats, LargeValueType, }; +pub use legacy_community_import::LegacyCommunityImportOutcome; pub use operation::OperationSubscription; pub use query::{MysqlConsoleCancellation, MysqlConsoleRequest, MysqlConsoleResult}; +pub use transfer::TransferArtifactDownload; use engine_manager::{ DEFAULT_ENGINE_IDLE_TIMEOUT, EngineManagerOwner, EngineManagerStatus, EngineProvider, @@ -74,6 +87,8 @@ pub(crate) struct ApplicationInner { large_values: large_value::LargeValueStore, agent_runs: AgentRunHub, operations: OperationHub, + transfer_tasks: transfer::TransferTaskHub, + account_previews: mysql_account::AccountPreviewRegistry, accepting_work: Mutex, shutdown_agent_run_ids: Mutex>, tasks: Mutex>>, @@ -250,6 +265,8 @@ impl Application { large_values: large_value::LargeValueStore::default(), agent_runs: AgentRunHub::new(), operations: OperationHub::new(), + transfer_tasks: transfer::TransferTaskHub::new(), + account_previews: mysql_account::AccountPreviewRegistry::default(), accepting_work: Mutex::new(true), shutdown_agent_run_ids: Mutex::new(Vec::new()), tasks: Mutex::new(HashMap::new()), @@ -465,9 +482,14 @@ impl Application { /// Returns the immutable driver inventory loaded during host startup. #[must_use] pub fn list_drivers(&self) -> JdbcDriverList { - JdbcDriverList { - items: self.inner.drivers.clone(), + let mut items = self.inner.drivers.clone(); + if !items + .iter() + .any(|driver| driver.driver_id.eq_ignore_ascii_case("mysql")) + { + items.push(datasource_compatibility::native_mysql_driver()); } + JdbcDriverList { items } } /// Opens and immediately closes an ephemeral JDBC session without @@ -496,6 +518,8 @@ impl Application { let session = datasource_session::open_datasource_session( &engine, datasource_session::ResolvedDatasourceConnection { + datasource_id: "connection-test".to_owned(), + datasource_revision: 0, driver_id: driver_id.to_owned(), datasource_name: "Connection test".to_owned(), connection, @@ -733,12 +757,14 @@ impl Application { self.persist_agent_shutdown_cancellations(&agent_run_ids) .await; self.inner.agent_runs.cancel_all().await; + self.begin_transfer_shutdown().await; } async fn join_tasks(&self) { let query_tasks = self.join_query_tasks(); let agent_tasks = self.inner.agent_runs.join_tasks(TASK_SHUTDOWN_TIMEOUT); - let ((), mut agent_run_ids) = tokio::join!(query_tasks, agent_tasks); + let transfer_tasks = self.join_transfer_tasks(TASK_SHUTDOWN_TIMEOUT); + let ((), mut agent_run_ids, ()) = tokio::join!(query_tasks, agent_tasks, transfer_tasks); agent_run_ids.extend(std::mem::take( &mut *self.inner.shutdown_agent_run_ids.lock().await, )); @@ -1119,11 +1145,15 @@ pub(crate) fn now_millis() -> Result { #[cfg(test)] mod tests { - use std::sync::Arc; + use std::{ + collections::HashMap, + sync::{Arc, Mutex}, + }; use chat2db_contract::{ - ComponentState, CreateDatasourceRequest, DatasourceSecretChange, RuntimeStatus, - UpdateDatasourceRequest, + ComponentState, CreateDatasourceRequest, DatabaseWriteState, DatasourceConnection, + DatasourceConnectionProperty, DatasourceSecretChange, ExecuteDatabaseWriteRequest, + RuntimeStatus, UpdateDatasourceRequest, }; use chat2db_storage::{SecretRef, SecretValue, SecretVault, SecretVaultError, Storage}; use tempfile::TempDir; @@ -1176,6 +1206,48 @@ mod tests { } } + #[derive(Debug, Default)] + struct RoundTripVault(Mutex>>); + + impl SecretVault for RoundTripVault { + fn probe(&self) -> Result<(), SecretVaultError> { + Ok(()) + } + + fn create( + &self, + reference: &SecretRef, + value: &SecretValue, + ) -> Result<(), SecretVaultError> { + self.0 + .lock() + .map_err(|_| SecretVaultError::Backend)? + .insert( + reference.as_str().to_owned(), + value.expose_secret().to_vec(), + ); + Ok(()) + } + + fn get(&self, reference: &SecretRef) -> Result, SecretVaultError> { + Ok(self + .0 + .lock() + .map_err(|_| SecretVaultError::Backend)? + .get(reference.as_str()) + .cloned() + .map(SecretValue::new)) + } + + fn delete(&self, reference: &SecretRef) -> Result<(), SecretVaultError> { + self.0 + .lock() + .map_err(|_| SecretVaultError::Backend)? + .remove(reference.as_str()); + Ok(()) + } + } + #[test] fn composed_storage_is_ready_without_enabling_the_database_engine() { let directory = TempDir::new().expect("temp dir"); @@ -1205,6 +1277,55 @@ mod tests { assert!(application.storage().is_some()); } + #[tokio::test] + async fn confirmed_write_rejects_non_mysql_before_engine_start() { + let directory = TempDir::new().expect("temp dir"); + let storage = Storage::open(directory.path(), Arc::new(RoundTripVault::default())) + .expect("local storage must open"); + let application = Application::with_storage(storage); + let datasource = application + .create_datasource(CreateDatasourceRequest { + name: "Local H2".to_owned(), + driver_id: "h2".to_owned(), + connection: Some(DatasourceConnection { + jdbc_url: "jdbc:h2:mem:write-boundary".to_owned(), + properties: vec![DatasourceConnectionProperty { + key: "user".to_owned(), + value: "sa".to_owned(), + sensitive: false, + }], + read_only: false, + ssh: None, + }), + }) + .await + .expect("unmanaged storage accepts a legacy H2 datasource"); + + let result = application + .execute_confirmed_database_write(ExecuteDatabaseWriteRequest { + datasource_id: datasource.id, + sql: "UPDATE items SET label = 'blocked'".to_owned(), + confirmed: true, + }) + .await; + + assert_eq!(result.state, DatabaseWriteState::NotStarted); + assert_eq!( + result.error.as_ref().map(|error| error.code.as_str()), + Some("mysql_driver_mismatch") + ); + assert_eq!( + application + .health() + .components + .iter() + .find(|component| component.id == "database-engine") + .expect("database engine health") + .state, + ComponentState::Disabled + ); + } + #[tokio::test] async fn managed_inventory_rejects_unknown_driver_changes_but_preserves_stale_ids() { let directory = TempDir::new().expect("temp dir"); diff --git a/crates/chat2db-core/src/mysql_account.rs b/crates/chat2db-core/src/mysql_account.rs new file mode 100644 index 0000000..eb69d20 --- /dev/null +++ b/crates/chat2db-core/src/mysql_account.rs @@ -0,0 +1,1005 @@ +use std::{ + collections::HashMap, + future::Future, + sync::Mutex as StdMutex, + time::{Duration, Instant}, +}; + +use chat2db_contract::{ + ApiError, CommunityAccount, CommunityAccountAction, CommunityAccountCapability, + CommunityAccountCommandRequest, CommunityAccountExecution, CommunityAccountGrantList, + CommunityAccountGrantsRequest, CommunityAccountList, CommunityAccountPreview, + CommunityAccountPrivilegeScope, CommunityMysqlPrivilege, DatasourceConnection, +}; +use mysql_async::{Conn, Error as MysqlError, prelude::Queryable}; +use sha2::{Digest, Sha256}; +use uuid::Uuid; + +use crate::{ + AppError, AppErrorKind, Application, + native_mysql::{finish_connection, open_resolved_connection, resolve_native_connection}, +}; + +const ACCOUNT_QUERY_TIMEOUT: Duration = Duration::from_secs(30); +const ACCOUNT_EXECUTE_FAILED: &str = "mysql.account.executeFailed"; +const ACCOUNT_EXECUTE_OUTCOME_UNKNOWN: &str = "mysql.account.outcomeUnknown"; +const ACCOUNT_LIST_UNAVAILABLE: &str = "mysql.account.listUnavailable"; +const ACCOUNT_GRANTS_UNAVAILABLE: &str = "mysql.account.grantsUnavailable"; +const ACCOUNT_PREVIEW_TOKEN_MISMATCH: &str = "mysql.account.previewTokenMismatch"; +const ACCOUNT_PREVIEW_UNAVAILABLE: &str = "mysql.account.previewUnavailable"; +const MASKED_PASSWORD_LITERAL: &str = "'******'"; +const ACCOUNT_PREVIEW_TTL: Duration = Duration::from_secs(5 * 60); +const MAX_PENDING_ACCOUNT_PREVIEWS: usize = 256; + +const SELECT_CURRENT_ACCOUNT: &str = "SELECT VERSION(), CURRENT_USER()"; +const PROBE_ACCOUNT_LIST: &str = "SELECT User, Host FROM mysql.user LIMIT 1"; +const PROBE_ACCOUNT_LOCK: &str = "SELECT account_locked FROM mysql.user LIMIT 1"; +const SELECT_ACCOUNTS: &str = "SELECT User, Host, plugin FROM mysql.user ORDER BY User, Host"; +const SELECT_ACCOUNTS_WITH_LOCK: &str = + "SELECT User, Host, plugin, account_locked FROM mysql.user ORDER BY User, Host"; + +enum AccountQueryFailure { + Timeout, + Mysql(MysqlError), +} + +#[derive(Default)] +pub(crate) struct AccountPreviewRegistry { + pending: StdMutex>, +} + +struct AccountPreviewBinding { + datasource_id: String, + sql_sha256: [u8; 32], + expires_at: Instant, +} + +impl AccountPreviewRegistry { + fn issue(&self, datasource_id: &str, sql: &str) -> Result { + let now = Instant::now(); + let mut pending = self.pending.lock().map_err(|_| { + AppError::unavailable( + ACCOUNT_PREVIEW_UNAVAILABLE, + "MySQL account preview authorization is unavailable", + ) + })?; + pending.retain(|_, binding| binding.expires_at > now); + if pending.len() >= MAX_PENDING_ACCOUNT_PREVIEWS { + return Err(AppError::unavailable( + ACCOUNT_PREVIEW_UNAVAILABLE, + "Too many MySQL account previews are pending", + )); + } + + let sql_sha256 = sha256(sql.as_bytes()); + loop { + let token = format!("{}{}", Uuid::new_v4().simple(), Uuid::new_v4().simple()); + let token_sha256 = sha256(token.as_bytes()); + if pending.contains_key(&token_sha256) { + continue; + } + pending.insert( + token_sha256, + AccountPreviewBinding { + datasource_id: datasource_id.to_owned(), + sql_sha256, + expires_at: now + ACCOUNT_PREVIEW_TTL, + }, + ); + return Ok(token); + } + } + + fn consume(&self, token: &str, datasource_id: &str, sql: &str) -> bool { + if token.len() != 64 || !token.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return false; + } + let token_sha256 = sha256(token.as_bytes()); + let Ok(mut pending) = self.pending.lock() else { + return false; + }; + let Some(binding) = pending.remove(&token_sha256) else { + return false; + }; + binding.expires_at > Instant::now() + && binding.datasource_id == datasource_id + && binding.sql_sha256 == sha256(sql.as_bytes()) + } +} + +impl Application { + /// Returns `MySQL` account-administration capability for one datasource. + /// + /// # Errors + /// + /// Returns datasource, secret, driver, connection, or cleanup errors. + pub async fn mysql_account_capability( + &self, + datasource_id: &str, + ) -> Result { + let resolved = resolve_native_connection(self, datasource_id).await?; + let connection_user = configured_connection_user(&resolved.connection); + let mut conn = open_resolved_connection(&resolved).await?; + + let account_list_readable = match timed_query(conn.query_drop(PROBE_ACCOUNT_LIST)).await { + Ok(()) => true, + Err(AccountQueryFailure::Mysql(_)) => false, + Err(AccountQueryFailure::Timeout) => { + return finish_connection( + conn, + Ok(capability_with_message( + connection_user, + false, + false, + "The MySQL account capability query timed out", + )), + ) + .await; + } + }; + let account_lock_supported = match timed_query(conn.query_drop(PROBE_ACCOUNT_LOCK)).await { + Ok(()) => true, + Err(AccountQueryFailure::Mysql(_)) => false, + Err(AccountQueryFailure::Timeout) => { + return finish_connection( + conn, + Ok(capability_with_message( + connection_user, + account_list_readable, + false, + "The MySQL account capability query timed out", + )), + ) + .await; + } + }; + + let (product_version, current_user, message) = match timed_query( + conn.query_first::<(String, String), _>(SELECT_CURRENT_ACCOUNT), + ) + .await + { + Ok(Some((version, current_user))) => (Some(version), Some(current_user), None), + Ok(None) => (None, None, None), + Err(AccountQueryFailure::Timeout) => ( + None, + None, + Some("The MySQL account capability query timed out".to_owned()), + ), + Err(AccountQueryFailure::Mysql(error)) => { + (None, None, Some(safe_query_message(&error))) + } + }; + finish_connection( + conn, + Ok(CommunityAccountCapability { + db_type: "MYSQL".to_owned(), + product_name: "MySQL".to_owned(), + product_version, + current_user, + connection_user, + account_list_readable, + account_lock_supported, + editable_privileges: CommunityMysqlPrivilege::ALL + .into_iter() + .map(|privilege| privilege.wire_name().to_owned()) + .collect(), + message, + }), + ) + .await + } + + /// Lists `MySQL` accounts in stable user and host order. + /// + /// # Errors + /// + /// Returns datasource, connection, permission, query, or cleanup errors. + pub async fn list_mysql_accounts( + &self, + datasource_id: &str, + ) -> Result { + let resolved = resolve_native_connection(self, datasource_id).await?; + let mut conn = open_resolved_connection(&resolved).await?; + let with_lock = timed_query( + conn.query::<(String, String, Option, Option), _>( + SELECT_ACCOUNTS_WITH_LOCK, + ), + ) + .await; + let result = match with_lock { + Ok(rows) => Ok(CommunityAccountList { + items: rows + .into_iter() + .map(|(user, host, plugin, locked)| { + account(user, host, plugin, locked.as_deref()) + }) + .collect(), + }), + Err(AccountQueryFailure::Timeout) => Err(account_query_unavailable( + ACCOUNT_LIST_UNAVAILABLE, + "The MySQL account list query timed out", + )), + Err(AccountQueryFailure::Mysql(_)) => { + match timed_query( + conn.query::<(String, String, Option), _>(SELECT_ACCOUNTS), + ) + .await + { + Ok(rows) => Ok(CommunityAccountList { + items: rows + .into_iter() + .map(|(user, host, plugin)| account(user, host, plugin, None)) + .collect(), + }), + Err(_) => Err(account_query_unavailable( + ACCOUNT_LIST_UNAVAILABLE, + "The MySQL account list is unavailable", + )), + } + } + }; + finish_connection(conn, result).await + } + + /// Returns `SHOW GRANTS` rows for one `MySQL` account. + /// + /// # Errors + /// + /// Returns validation, datasource, connection, permission, query, or cleanup errors. + pub async fn mysql_account_grants( + &self, + request: &CommunityAccountGrantsRequest, + ) -> Result { + let account = account_literal(&request.user, &request.host)?; + let resolved = resolve_native_connection(self, &request.datasource_id).await?; + let mut conn = open_resolved_connection(&resolved).await?; + let sql = format!("SHOW GRANTS FOR {account}"); + let result = match timed_query(query_account_grants(&mut conn, &sql)).await { + Ok(items) => Ok(CommunityAccountGrantList { items }), + Err(_) => Err(account_query_unavailable( + ACCOUNT_GRANTS_UNAVAILABLE, + "The MySQL grants are unavailable", + )), + }; + finish_connection(conn, result).await + } + + /// Builds a masked account-operation preview without opening `MySQL` or starting Java. + /// + /// # Errors + /// + /// Returns a field-specific account validation error. + pub fn preview_mysql_account( + &self, + request: &CommunityAccountCommandRequest, + ) -> Result { + preview_account(&self.inner.account_previews, request) + } + + /// Executes one preview-authorized `MySQL` account operation through `mysql_async`. + /// + /// SQL execution errors are returned in [`CommunityAccountExecution`]. Datasource, + /// validation, preview-token, connection, read-only, and cleanup failures remain errors. + /// + /// # Errors + /// + /// Returns validation, token, datasource, connection, read-only, or cleanup errors. + pub async fn execute_mysql_account( + &self, + request: &CommunityAccountCommandRequest, + ) -> Result { + let execution_sql = build_account_sql(request, false)?; + let preview_sql = build_account_sql(request, true)?; + let supplied_token = request.preview_token.as_deref().unwrap_or_default(); + if !self.inner.account_previews.consume( + supplied_token, + &request.datasource_id, + &execution_sql, + ) { + return Err(AppError::new( + AppErrorKind::Conflict, + ApiError::new( + ACCOUNT_PREVIEW_TOKEN_MISMATCH, + "The MySQL account preview token does not match this operation", + ), + )); + } + + let resolved = resolve_native_connection(self, &request.datasource_id).await?; + if resolved.connection.read_only { + return Err(AppError::new( + AppErrorKind::Conflict, + ApiError::new( + "datasource_read_only", + "The datasource connection is configured as read-only", + ), + )); + } + let mut conn = open_resolved_connection(&resolved).await?; + let query_result = timed_query(execute_account_sql(&mut conn, &execution_sql)).await; + drop(execution_sql); + + let response = match query_result { + Ok(()) => CommunityAccountExecution { + action_type: request.action_type, + sql: preview_sql.clone(), + success: true, + message: Some("OK".to_owned()), + failure_code: None, + error_code: None, + sql_state: None, + }, + Err(AccountQueryFailure::Mysql(MysqlError::Server(server))) => { + CommunityAccountExecution { + action_type: request.action_type, + sql: preview_sql.clone(), + success: false, + message: Some(redact_password( + &server.message, + request.password.as_deref(), + )), + failure_code: Some(ACCOUNT_EXECUTE_FAILED.to_owned()), + error_code: Some(server.code), + sql_state: Some(server.state), + } + } + Err(AccountQueryFailure::Mysql(_)) => account_outcome_unknown( + request.action_type, + preview_sql.clone(), + "The MySQL connection ended after dispatch; the account-operation outcome is unknown and must not be retried blindly".to_owned(), + ), + Err(AccountQueryFailure::Timeout) => account_outcome_unknown( + request.action_type, + preview_sql, + "The MySQL account operation timed out after dispatch; its outcome is unknown and must not be retried blindly".to_owned(), + ), + }; + if let Err(error) = finish_connection(conn, Ok(())).await { + tracing::warn!( + error = %error, + "native MySQL account connection cleanup failed after a settled operation" + ); + } + Ok(response) + } +} + +fn account_outcome_unknown( + action_type: CommunityAccountAction, + sql: String, + message: String, +) -> CommunityAccountExecution { + CommunityAccountExecution { + action_type, + sql, + success: false, + message: Some(message), + failure_code: Some(ACCOUNT_EXECUTE_OUTCOME_UNKNOWN.to_owned()), + error_code: None, + sql_state: None, + } +} + +fn preview_account( + registry: &AccountPreviewRegistry, + request: &CommunityAccountCommandRequest, +) -> Result { + let execution_sql = build_account_sql(request, false)?; + let sql = build_account_sql(request, true)?; + let preview_token = registry.issue(&request.datasource_id, &execution_sql)?; + drop(execution_sql); + Ok(CommunityAccountPreview { + action_type: request.action_type, + sql, + preview_token, + }) +} + +fn build_account_sql( + request: &CommunityAccountCommandRequest, + mask_sensitive: bool, +) -> Result { + let account = account_literal(&request.user, &request.host)?; + match request.action_type { + CommunityAccountAction::CreateUser => Ok(format!( + "CREATE USER {account} IDENTIFIED BY {}", + password_literal(request, mask_sensitive)? + )), + CommunityAccountAction::AlterPassword => Ok(format!( + "ALTER USER {account} IDENTIFIED BY {}", + password_literal(request, mask_sensitive)? + )), + CommunityAccountAction::LockAccount => Ok(format!("ALTER USER {account} ACCOUNT LOCK")), + CommunityAccountAction::UnlockAccount => Ok(format!("ALTER USER {account} ACCOUNT UNLOCK")), + CommunityAccountAction::DropUser => Ok(format!("DROP USER {account}")), + CommunityAccountAction::GrantPrivilege => { + let privileges = privilege_list(&request.privileges)?; + let scope = privilege_scope(request)?; + let grant_option = if request.grant_option { + " WITH GRANT OPTION" + } else { + "" + }; + Ok(format!( + "GRANT {privileges} ON {scope} TO {account}{grant_option}" + )) + } + CommunityAccountAction::RevokePrivilege => { + let privileges = privilege_list(&request.privileges)?; + let scope = privilege_scope(request)?; + Ok(format!("REVOKE {privileges} ON {scope} FROM {account}")) + } + } +} + +fn password_literal( + request: &CommunityAccountCommandRequest, + mask_sensitive: bool, +) -> Result { + let password = request.password.as_deref().filter(|value| !is_blank(value)); + if password.is_none() { + return Err(account_validation_error( + "mysql.account.passwordRequired", + "A non-blank password is required", + )); + } + if mask_sensitive { + Ok(MASKED_PASSWORD_LITERAL.to_owned()) + } else { + Ok(string_literal(password.unwrap_or_default())) + } +} + +fn account_literal(user: &str, host: &str) -> Result { + validate_account_part( + user, + "mysql.account.userRequired", + "A non-blank MySQL account user is required", + )?; + validate_account_part( + host, + "mysql.account.hostRequired", + "A non-blank MySQL account host is required", + )?; + Ok(format!("{}@{}", string_literal(user), string_literal(host))) +} + +fn validate_account_part( + value: &str, + code: &'static str, + message: &'static str, +) -> Result<(), AppError> { + if is_blank(value) { + return Err(account_validation_error(code, message)); + } + if value.contains('\0') { + return Err(account_validation_error( + "mysql.account.invalidAccountName", + "MySQL account user and host names cannot contain NUL", + )); + } + Ok(()) +} + +fn privilege_scope(request: &CommunityAccountCommandRequest) -> Result { + match request.scope { + Some(CommunityAccountPrivilegeScope::Global) => Ok("*.*".to_owned()), + Some(CommunityAccountPrivilegeScope::Database) => { + let database = required_identifier( + request.database_name.as_deref(), + "mysql.account.databaseRequired", + "A database name is required for database privileges", + )?; + Ok(format!("{database}.*")) + } + Some(CommunityAccountPrivilegeScope::Table) => { + let database = required_identifier( + request.database_name.as_deref(), + "mysql.account.databaseRequired", + "A database name is required for table privileges", + )?; + let table = required_identifier( + request.table_name.as_deref(), + "mysql.account.tableRequired", + "A table name is required for table privileges", + )?; + Ok(format!("{database}.{table}")) + } + None => Err(account_validation_error( + "mysql.account.scopeRequired", + "A privilege scope is required", + )), + } +} + +fn required_identifier( + value: Option<&str>, + code: &'static str, + message: &'static str, +) -> Result { + let value = value.filter(|value| !is_blank(value)); + match value { + Some(value) => Ok(identifier(value)), + None => Err(account_validation_error(code, message)), + } +} + +fn identifier(value: &str) -> String { + format!("`{}`", value.replace('`', "``")) +} + +fn string_literal(value: &str) -> String { + let mut literal = String::with_capacity(value.len() + 2); + literal.push('\''); + for character in value.chars() { + match character { + '\'' => literal.push_str("''"), + _ => literal.push(character), + } + } + literal.push('\''); + literal +} + +async fn query_account_grants(conn: &mut Conn, sql: &str) -> Result, MysqlError> { + enforce_mode_independent_account_literals(conn).await?; + conn.query(sql).await +} + +async fn execute_account_sql(conn: &mut Conn, sql: &str) -> Result<(), MysqlError> { + enforce_mode_independent_account_literals(conn).await?; + conn.query_drop(sql).await +} + +async fn enforce_mode_independent_account_literals(conn: &mut Conn) -> Result<(), MysqlError> { + let current = conn + .query_first::("SELECT @@SESSION.sql_mode") + .await? + .unwrap_or_default(); + let Some(required) = sql_mode_with_no_backslash_escapes(¤t) else { + return Ok(()); + }; + conn.exec_drop("SET SESSION sql_mode = ?", (required,)) + .await +} + +fn sql_mode_with_no_backslash_escapes(current: &str) -> Option { + if current + .split(',') + .any(|mode| mode.trim().eq_ignore_ascii_case("NO_BACKSLASH_ESCAPES")) + { + return None; + } + let current = current.trim(); + Some(if current.is_empty() { + "NO_BACKSLASH_ESCAPES".to_owned() + } else { + format!("{current},NO_BACKSLASH_ESCAPES") + }) +} + +fn privilege_list(privileges: &[String]) -> Result { + if privileges.is_empty() { + return Err(account_validation_error( + "mysql.account.privilegeRequired", + "At least one MySQL privilege is required", + )); + } + let mut accepted = Vec::new(); + for privilege in privileges { + let privilege = parse_privilege(privilege)?; + if !accepted.contains(&privilege) { + accepted.push(privilege); + } + } + if accepted.is_empty() { + return Err(account_validation_error( + "mysql.account.privilegeRequired", + "At least one MySQL privilege is required", + )); + } + Ok(accepted + .into_iter() + .map(privilege_sql_name) + .collect::>() + .join(", ")) +} + +fn parse_privilege(value: &str) -> Result { + match value.trim().to_ascii_uppercase().as_str() { + "SELECT" => Ok(CommunityMysqlPrivilege::Select), + "INSERT" => Ok(CommunityMysqlPrivilege::Insert), + "UPDATE" => Ok(CommunityMysqlPrivilege::Update), + "DELETE" => Ok(CommunityMysqlPrivilege::Delete), + "CREATE" => Ok(CommunityMysqlPrivilege::Create), + "DROP" => Ok(CommunityMysqlPrivilege::Drop), + "ALTER" => Ok(CommunityMysqlPrivilege::Alter), + "INDEX" => Ok(CommunityMysqlPrivilege::Index), + "REFERENCES" => Ok(CommunityMysqlPrivilege::References), + "EXECUTE" => Ok(CommunityMysqlPrivilege::Execute), + "SHOW_VIEW" => Ok(CommunityMysqlPrivilege::ShowView), + "TRIGGER" => Ok(CommunityMysqlPrivilege::Trigger), + "EVENT" => Ok(CommunityMysqlPrivilege::Event), + "CREATE_TEMPORARY_TABLES" => Ok(CommunityMysqlPrivilege::CreateTemporaryTables), + _ => Err(account_validation_error( + "mysql.account.privilegeUnsupported", + "The requested MySQL privilege is not supported", + )), + } +} + +const fn privilege_sql_name(privilege: CommunityMysqlPrivilege) -> &'static str { + match privilege { + CommunityMysqlPrivilege::ShowView => "SHOW VIEW", + CommunityMysqlPrivilege::CreateTemporaryTables => "CREATE TEMPORARY TABLES", + other => other.wire_name(), + } +} + +fn sha256(value: &[u8]) -> [u8; 32] { + Sha256::digest(value).into() +} + +fn account( + user: String, + host: String, + authentication_plugin: Option, + locked: Option<&str>, +) -> CommunityAccount { + CommunityAccount { + display_name: format!("{user}@{host}"), + user, + host, + authentication_plugin, + locked: locked + .and_then(|value| (!value.trim().is_empty()).then(|| value.eq_ignore_ascii_case("Y"))), + } +} + +fn configured_connection_user(connection: &DatasourceConnection) -> Option { + connection + .properties + .iter() + .find(|property| { + property.key.eq_ignore_ascii_case("user") + || property.key.eq_ignore_ascii_case("username") + }) + .map(|property| property.value.clone()) + .filter(|value| !value.trim().is_empty()) +} + +fn capability_with_message( + connection_user: Option, + account_list_readable: bool, + account_lock_supported: bool, + message: &str, +) -> CommunityAccountCapability { + CommunityAccountCapability { + db_type: "MYSQL".to_owned(), + product_name: "MySQL".to_owned(), + product_version: None, + current_user: None, + connection_user, + account_list_readable, + account_lock_supported, + editable_privileges: CommunityMysqlPrivilege::ALL + .into_iter() + .map(|privilege| privilege.wire_name().to_owned()) + .collect(), + message: Some(message.to_owned()), + } +} + +async fn timed_query( + future: impl Future>, +) -> Result { + tokio::time::timeout(ACCOUNT_QUERY_TIMEOUT, future) + .await + .map_err(|_| AccountQueryFailure::Timeout)? + .map_err(AccountQueryFailure::Mysql) +} + +fn safe_query_message(error: &MysqlError) -> String { + match error { + MysqlError::Server(server) => server.message.clone(), + _ => "The MySQL connection ended before the capability query completed".to_owned(), + } +} + +fn redact_password(message: &str, password: Option<&str>) -> String { + let Some(password) = password.filter(|password| !password.is_empty()) else { + return message.to_owned(); + }; + let literal = string_literal(password); + message + .replace(&literal, "'[REDACTED]'") + .replace(password, "[REDACTED]") +} + +fn account_query_unavailable(code: &'static str, message: &'static str) -> AppError { + AppError::new(AppErrorKind::InvalidRequest, ApiError::new(code, message)) +} + +fn account_validation_error(code: &'static str, message: &'static str) -> AppError { + AppError::invalid(code, message) +} + +fn is_blank(value: &str) -> bool { + value.trim().is_empty() +} + +#[cfg(test)] +mod tests { + use chat2db_contract::{ + CommunityAccountAction, CommunityAccountCommandRequest, CommunityAccountPrivilegeScope, + }; + + use crate::Application; + + use super::{ + AccountPreviewRegistry, account_outcome_unknown, build_account_sql, preview_account, + redact_password, sql_mode_with_no_backslash_escapes, string_literal, + }; + + #[test] + fn every_account_action_matches_community_sql() { + assert_eq!( + sql(CommunityAccountAction::CreateUser), + "CREATE USER 'reader'@'%' IDENTIFIED BY 'pa''ss\\word'" + ); + assert_eq!( + sql(CommunityAccountAction::AlterPassword), + "ALTER USER 'reader'@'%' IDENTIFIED BY 'pa''ss\\word'" + ); + assert_eq!( + sql(CommunityAccountAction::LockAccount), + "ALTER USER 'reader'@'%' ACCOUNT LOCK" + ); + assert_eq!( + sql(CommunityAccountAction::UnlockAccount), + "ALTER USER 'reader'@'%' ACCOUNT UNLOCK" + ); + assert_eq!( + sql(CommunityAccountAction::DropUser), + "DROP USER 'reader'@'%'" + ); + assert_eq!( + sql(CommunityAccountAction::GrantPrivilege), + "GRANT SELECT, SHOW VIEW, CREATE TEMPORARY TABLES ON `odd``db`.`order``item` TO 'reader'@'%' WITH GRANT OPTION" + ); + assert_eq!( + sql(CommunityAccountAction::RevokePrivilege), + "REVOKE SELECT, SHOW VIEW, CREATE TEMPORARY TABLES ON `odd``db`.`order``item` FROM 'reader'@'%'" + ); + } + + #[test] + fn scopes_and_account_literals_match_community_escaping() { + let mut request = command(CommunityAccountAction::GrantPrivilege); + request.user = "o'brien\\ops".to_owned(); + request.host = "local'host".to_owned(); + request.scope = Some(CommunityAccountPrivilegeScope::Global); + assert_eq!( + build_account_sql(&request, false).expect("global grant"), + "GRANT SELECT, SHOW VIEW, CREATE TEMPORARY TABLES ON *.* TO 'o''brien\\ops'@'local''host' WITH GRANT OPTION" + ); + + request.scope = Some(CommunityAccountPrivilegeScope::Database); + assert_eq!( + build_account_sql(&request, false).expect("database grant"), + "GRANT SELECT, SHOW VIEW, CREATE TEMPORARY TABLES ON `odd``db`.* TO 'o''brien\\ops'@'local''host' WITH GRANT OPTION" + ); + } + + #[test] + fn account_literal_mode_is_stable_from_default_and_no_backslash_modes() { + assert_eq!( + string_literal("o'brien\\ops"), + "'o''brien\\ops'", + "backslashes must remain data while quotes use SQL-standard doubling" + ); + assert_eq!( + sql_mode_with_no_backslash_escapes("STRICT_TRANS_TABLES"), + Some("STRICT_TRANS_TABLES,NO_BACKSLASH_ESCAPES".to_owned()) + ); + assert_eq!( + sql_mode_with_no_backslash_escapes(""), + Some("NO_BACKSLASH_ESCAPES".to_owned()) + ); + assert_eq!( + sql_mode_with_no_backslash_escapes("STRICT_TRANS_TABLES,NO_BACKSLASH_ESCAPES"), + None + ); + assert_eq!( + sql_mode_with_no_backslash_escapes("no_backslash_escapes"), + None + ); + } + + #[test] + fn preview_masks_password_and_issues_an_opaque_token() { + let registry = AccountPreviewRegistry::default(); + let request = command(CommunityAccountAction::CreateUser); + let preview = preview_account(®istry, &request).expect("valid account preview"); + + assert_eq!( + preview.sql, + "CREATE USER 'reader'@'%' IDENTIFIED BY '******'" + ); + assert_eq!(preview.preview_token.len(), 64); + assert!( + preview + .preview_token + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + ); + assert_ne!( + preview.preview_token, + preview_account(®istry, &request) + .expect("repeated preview") + .preview_token, + "repeating identical input must not produce a caller-computable token" + ); + assert_ne!( + preview.preview_token, + preview_account(®istry, &command_with_password("different")) + .expect("second preview") + .preview_token + ); + } + + #[test] + fn preview_tokens_are_datasource_bound_exact_and_single_use() { + let registry = AccountPreviewRegistry::default(); + let request = command(CommunityAccountAction::DropUser); + let sql = build_account_sql(&request, false).expect("account SQL"); + + let wrong_datasource = + preview_account(®istry, &request).expect("wrong-datasource preview"); + assert!(!registry.consume(&wrong_datasource.preview_token, "other-datasource", &sql)); + assert!(!registry.consume( + &wrong_datasource.preview_token, + &request.datasource_id, + &sql + )); + + let wrong_sql = preview_account(®istry, &request).expect("wrong-SQL preview"); + assert!(!registry.consume( + &wrong_sql.preview_token, + &request.datasource_id, + &format!("{sql} ") + )); + assert!(!registry.consume(&wrong_sql.preview_token, &request.datasource_id, &sql)); + + let valid = preview_account(®istry, &request).expect("valid preview"); + assert!(registry.consume(&valid.preview_token, &request.datasource_id, &sql)); + assert!(!registry.consume(&valid.preview_token, &request.datasource_id, &sql)); + } + + #[test] + fn duplicate_privileges_are_removed_in_first_seen_order() { + let mut request = command(CommunityAccountAction::GrantPrivilege); + request.privileges = vec![ + "select".to_owned(), + " SELECT ".to_owned(), + "update".to_owned(), + ]; + assert_eq!( + build_account_sql(&request, false).expect("deduplicated grant"), + "GRANT SELECT, UPDATE ON `odd``db`.`order``item` TO 'reader'@'%' WITH GRANT OPTION" + ); + } + + #[test] + fn invalid_fields_return_community_error_codes() { + let mut request = command(CommunityAccountAction::CreateUser); + request.user.clear(); + assert_code(&request, "mysql.account.userRequired"); + + request.user = "reader\0hidden".to_owned(); + assert_code(&request, "mysql.account.invalidAccountName"); + + request.user = "reader".to_owned(); + request.password = Some(" ".to_owned()); + assert_code(&request, "mysql.account.passwordRequired"); + + request = command(CommunityAccountAction::GrantPrivilege); + request.scope = None; + assert_code(&request, "mysql.account.scopeRequired"); + + request.scope = Some(CommunityAccountPrivilegeScope::Database); + request.database_name = None; + assert_code(&request, "mysql.account.databaseRequired"); + + request.scope = Some(CommunityAccountPrivilegeScope::Table); + request.database_name = Some("inventory".to_owned()); + request.table_name = None; + assert_code(&request, "mysql.account.tableRequired"); + + request.table_name = Some("orders".to_owned()); + request.privileges = vec!["ROLE_ADMIN".to_owned()]; + assert_code(&request, "mysql.account.privilegeUnsupported"); + } + + #[tokio::test] + async fn token_mismatch_is_rejected_before_storage_or_mysql_access() { + let mut request = command(CommunityAccountAction::DropUser); + request.preview_token = Some("not-the-preview-token".to_owned()); + + let error = Application::new() + .execute_mysql_account(&request) + .await + .expect_err("token mismatch must fail before datasource resolution"); + assert_eq!(error.api_error().code, "mysql.account.previewTokenMismatch"); + } + + #[test] + fn interrupted_account_writes_are_explicitly_non_retryable_unknown_outcomes() { + let result = account_outcome_unknown( + CommunityAccountAction::AlterPassword, + "ALTER USER 'reader'@'%' IDENTIFIED BY '******'".to_owned(), + "The outcome is unknown and must not be retried blindly".to_owned(), + ); + + assert!(!result.success); + assert_eq!( + result.failure_code.as_deref(), + Some("mysql.account.outcomeUnknown") + ); + assert!( + result + .message + .as_deref() + .is_some_and(|message| message.contains("must not be retried blindly")) + ); + } + + #[test] + fn server_messages_cannot_echo_the_password() { + let password = "pa'ss\\word"; + let message = format!( + "syntax near {} containing {password}", + string_literal(password) + ); + let redacted = redact_password(&message, Some(password)); + assert!(!redacted.contains(password)); + assert!(!redacted.contains(&string_literal(password))); + assert!(redacted.contains("[REDACTED]")); + } + + fn sql(action: CommunityAccountAction) -> String { + build_account_sql(&command(action), false).expect("account SQL") + } + + fn command(action_type: CommunityAccountAction) -> CommunityAccountCommandRequest { + CommunityAccountCommandRequest { + datasource_id: "42".to_owned(), + user: "reader".to_owned(), + host: "%".to_owned(), + action_type, + scope: Some(CommunityAccountPrivilegeScope::Table), + database_name: Some("odd`db".to_owned()), + table_name: Some("order`item".to_owned()), + privileges: vec![ + "SELECT".to_owned(), + "SHOW_VIEW".to_owned(), + "CREATE_TEMPORARY_TABLES".to_owned(), + ], + grant_option: true, + password: Some("pa'ss\\word".to_owned()), + preview_token: None, + } + } + + fn command_with_password(password: &str) -> CommunityAccountCommandRequest { + let mut request = command(CommunityAccountAction::CreateUser); + request.password = Some(password.to_owned()); + request + } + + fn assert_code(request: &CommunityAccountCommandRequest, expected: &str) { + let error = build_account_sql(request, false).expect_err("request must be invalid"); + assert_eq!(error.api_error().code, expected); + } +} diff --git a/crates/chat2db-core/src/mysql_dashboard.rs b/crates/chat2db-core/src/mysql_dashboard.rs new file mode 100644 index 0000000..b248816 --- /dev/null +++ b/crates/chat2db-core/src/mysql_dashboard.rs @@ -0,0 +1,755 @@ +use std::collections::HashMap; + +use chat2db_contract::{ + ApiError, CommunityChart, CommunityDashboard, CommunityDashboardListQuery, + CommunityDashboardPage, CommunityTableColumn, CreateCommunityChartRequest, + CreateCommunityDashboardRequest, JdbcValue, ListCommunityColumnsRequest, ResultColumn, + UpdateCommunityChartRequest, UpdateCommunityDashboardRequest, +}; +use chat2db_storage::CreateOperationLog; +use serde_json::{Map, Value, json}; +use sqlparser::{ + ast::{Expr, SelectItem, SetExpr, Statement, TableFactor}, + dialect::MySqlDialect, + parser::Parser, +}; + +use crate::{ + AppError, AppErrorKind, Application, MysqlConsoleCancellation, MysqlConsoleRequest, + MysqlConsoleResult, now_millis, storage_call, +}; + +const CHART_PAGE_SIZE: u32 = 200; +const MAX_CHART_METADATA_BYTES: usize = 8 * 1024 * 1024; + +impl Application { + /// Lists durable Community dashboards using the historical stable paging contract. + /// + /// # Errors + /// + /// Returns validation, availability, or durable-storage failures. + pub async fn list_community_dashboards( + &self, + query: CommunityDashboardListQuery, + ) -> Result { + let storage = self.require_storage()?; + storage_call(move || storage.list_community_dashboards(&query)).await + } + + /// Returns one dashboard or `None` when its id is absent. + /// + /// # Errors + /// + /// Returns validation, availability, or durable-storage failures. + pub async fn get_community_dashboard( + &self, + id: i64, + ) -> Result, AppError> { + let storage = self.require_storage()?; + storage_call(move || storage.get_community_dashboard(id)).await + } + + /// Creates one durable Community dashboard and returns its numeric id. + /// + /// # Errors + /// + /// Returns validation, availability, or durable-storage failures. + pub async fn create_community_dashboard( + &self, + request: CreateCommunityDashboardRequest, + ) -> Result { + let storage = self.require_storage()?; + storage_call(move || storage.create_community_dashboard(request)).await + } + + /// Applies Community's non-null partial dashboard update. + /// + /// # Errors + /// + /// Returns validation, not-found, availability, or durable-storage failures. + pub async fn update_community_dashboard( + &self, + id: i64, + request: UpdateCommunityDashboardRequest, + ) -> Result<(), AppError> { + let storage = self.require_storage()?; + storage_call(move || storage.update_community_dashboard(id, request)).await + } + + /// Deletes a dashboard and its chart relations. + /// + /// # Errors + /// + /// Returns validation, availability, or durable-storage failures. + pub async fn delete_community_dashboard(&self, id: i64) -> Result { + let storage = self.require_storage()?; + storage_call(move || storage.delete_community_dashboard(id)).await + } + + /// Returns one durable Community chart without executing its SQL. + /// + /// # Errors + /// + /// Returns validation, availability, or durable-storage failures. + pub async fn get_community_chart(&self, id: i64) -> Result, AppError> { + let storage = self.require_storage()?; + storage_call(move || storage.get_community_chart(id)).await + } + + /// Returns a detached chart copy, optionally refreshing its result through native `MySQL`. + /// + /// # Errors + /// + /// Returns validation, datasource, `MySQL`, result-limit, or durable-storage failures. + pub async fn get_community_chart_detail( + &self, + id: i64, + refresh: bool, + ) -> Result, AppError> { + let Some(mut chart) = self.get_community_chart(id).await? else { + return Ok(None); + }; + if !refresh { + return Ok(Some(chart)); + } + let Some(context) = chart_refresh_context(&chart) else { + return Ok(Some(chart)); + }; + + let execution = self + .execute_mysql_read_console( + MysqlConsoleRequest { + datasource_id: context.datasource_id.clone(), + database_name: context.database_name.clone().unwrap_or_default(), + sql: context.sql.clone(), + page_no: 1, + page_size: CHART_PAGE_SIZE, + result_set_id: None, + single: true, + page_size_all: false, + explain: false, + error_continue: false, + }, + MysqlConsoleCancellation::new(), + ) + .await; + + let result = match execution { + Ok(results) => { + let Some(result) = results.into_iter().next() else { + let error = AppError::unavailable( + "chart_query_incomplete", + "The chart query completed without a result", + ); + self.record_chart_history(&chart, &context, None, Some(&error)) + .await; + return Err(error); + }; + if result.success { + result + } else { + let error = AppError::invalid( + "chart_query_failed", + result + .error + .as_ref() + .map_or_else(|| result.message.clone(), |error| error.message.clone()), + ); + self.record_chart_history(&chart, &context, Some(&result), Some(&error)) + .await; + return Err(error); + } + } + Err(error) => { + self.record_chart_history(&chart, &context, None, Some(&error)) + .await; + return Err(error); + } + }; + let header_metadata = self.chart_header_metadata(&context).await; + chart.meta_data = Some(chart_metadata(&result, header_metadata.as_ref())?); + self.record_chart_history(&chart, &context, Some(&result), None) + .await; + Ok(Some(chart)) + } + + /// Creates one durable Community chart and returns its numeric id. + /// + /// # Errors + /// + /// Returns validation, availability, or durable-storage failures. + pub async fn create_community_chart( + &self, + request: CreateCommunityChartRequest, + ) -> Result { + let storage = self.require_storage()?; + storage_call(move || storage.create_community_chart(request)).await + } + + /// Applies Community's non-null partial chart update. + /// + /// # Errors + /// + /// Returns validation, not-found, availability, or durable-storage failures. + pub async fn update_community_chart( + &self, + id: i64, + request: UpdateCommunityChartRequest, + ) -> Result<(), AppError> { + let storage = self.require_storage()?; + storage_call(move || storage.update_community_chart(id, request)).await + } + + /// Deletes one durable Community chart. + /// + /// # Errors + /// + /// Returns validation, availability, or durable-storage failures. + pub async fn delete_community_chart(&self, id: i64) -> Result { + let storage = self.require_storage()?; + storage_call(move || storage.delete_community_chart(id)).await + } + + async fn record_chart_history( + &self, + chart: &CommunityChart, + context: &ChartRefreshContext, + result: Option<&MysqlConsoleResult>, + error: Option<&AppError>, + ) { + let Some(storage) = self.storage().cloned() else { + return; + }; + let extend_info = serde_json::to_string(&json!({ + "source": "CHART", + "chartId": chart.id, + "consoleId": context.console_id, + "message": error.map(|error| error.api_error().message), + })) + .ok(); + let input = CreateOperationLog { + name: chart.name.clone(), + data_source_id: Some(context.datasource_id.clone()), + data_source_name: chart.data_source_name.clone(), + connectable: Some(true), + database_name: context.database_name.clone(), + database_type: Some("MYSQL".to_owned()), + ddl: context.sql.clone(), + status: if error.is_none() { "success" } else { "fail" }.to_owned(), + operation_rows: result.and_then(|result| i64::try_from(result.row_count).ok()), + use_time: result.and_then(|result| i64::try_from(result.duration_ms).ok()), + extend_info, + schema_name: context.schema_name.clone(), + organization_id: None, + user_name: None, + more: context.sql.chars().count() > 200, + operation_type: "SQL_EXECUTE".to_owned(), + }; + let write = tokio::task::spawn_blocking(move || storage.create_operation_log(input)).await; + match write { + Ok(Ok(_)) => {} + Ok(Err(error)) => tracing::warn!(%error, "chart query history write failed"), + Err(error) => tracing::warn!(%error, "chart query history task failed"), + } + } + + async fn chart_header_metadata( + &self, + context: &ChartRefreshContext, + ) -> Option> { + let table = chart_editable_table(&context.sql)?; + let database_name = table + .database_name + .or_else(|| context.database_name.clone())?; + let schema_name = context + .schema_name + .clone() + .unwrap_or_else(|| database_name.clone()); + let columns = match self + .list_community_columns(ListCommunityColumnsRequest { + datasource_id: context.datasource_id.clone(), + database_type: "MYSQL".to_owned(), + database_name, + schema_name, + table_name: table.table_name, + }) + .await + { + Ok(columns) => columns, + Err(error) => { + tracing::warn!(%error, "chart header metadata enhancement failed"); + return None; + } + }; + Some( + columns + .items + .into_iter() + .map(|column| (column.name.to_ascii_lowercase(), column)) + .collect(), + ) + } +} + +#[derive(Debug)] +struct ChartRefreshContext { + datasource_id: String, + database_name: Option, + schema_name: Option, + sql: String, + console_id: String, +} + +#[derive(Debug, PartialEq, Eq)] +struct ChartEditableTable { + database_name: Option, + table_name: String, +} + +fn chart_refresh_context(chart: &CommunityChart) -> Option { + let database_info = json_object(chart.database_info.as_ref()?)?; + let datasource_id = json_identifier(database_info.get("dataSourceId")?)?; + let sql = database_info.get("sql")?.as_str()?.trim(); + if sql.is_empty() { + return None; + } + let database_name = json_non_blank(database_info.get("databaseName")); + let schema_name = json_non_blank(database_info.get("schemaName")); + let console_id = database_info + .get("consoleId") + .and_then(json_identifier) + .unwrap_or_else(|| now_millis().unwrap_or_default().to_string()); + Some(ChartRefreshContext { + datasource_id, + database_name, + schema_name, + sql: sql.to_owned(), + console_id, + }) +} + +fn json_object(value: &Value) -> Option<&Map> { + match value { + Value::Object(object) => Some(object), + _ => None, + } +} + +fn json_identifier(value: &Value) -> Option { + match value { + Value::String(value) if !value.trim().is_empty() => Some(value.clone()), + Value::Number(value) => Some(value.to_string()), + _ => None, + } +} + +fn json_non_blank(value: Option<&Value>) -> Option { + value + .and_then(Value::as_str) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(ToOwned::to_owned) +} + +fn chart_editable_table(sql: &str) -> Option { + let statements = Parser::parse_sql(&MySqlDialect {}, sql).ok()?; + let [Statement::Query(query)] = statements.as_slice() else { + return None; + }; + if query.with.is_some() { + return None; + } + let SetExpr::Select(select) = query.body.as_ref() else { + return None; + }; + let [from] = select.from.as_slice() else { + return None; + }; + if !from.joins.is_empty() || select.projection.iter().any(non_editable_projection) { + return None; + } + let TableFactor::Table { name, args, .. } = &from.relation else { + return None; + }; + if args.is_some() { + return None; + } + let identifiers = name + .0 + .iter() + .filter_map(|part| part.as_ident()) + .map(|identifier| identifier.value.clone()) + .collect::>(); + let table_name = identifiers.last()?.clone(); + let database_name = identifiers + .len() + .checked_sub(2) + .and_then(|index| identifiers.get(index).cloned()); + Some(ChartEditableTable { + database_name, + table_name, + }) +} + +fn non_editable_projection(item: &SelectItem) -> bool { + match item { + SelectItem::ExprWithAlias { .. } | SelectItem::ExprWithAliases { .. } => true, + SelectItem::UnnamedExpr(Expr::Function(function)) => function + .name + .0 + .last() + .and_then(|part| part.as_ident()) + .is_some_and(|identifier| identifier.value.eq_ignore_ascii_case("count")), + SelectItem::UnnamedExpr(_) + | SelectItem::QualifiedWildcard(_, _) + | SelectItem::Wildcard(_) => false, + } +} + +fn chart_metadata( + result: &MysqlConsoleResult, + header_metadata: Option<&HashMap>, +) -> Result { + let metadata = json!({ + "dataList": result + .rows + .iter() + .map(|row| row.values.iter().map(chart_value).collect::>()) + .collect::>(), + "headerList": result + .columns + .iter() + .map(|column| { + let metadata = header_metadata + .and_then(|columns| columns.get(&column.name.to_ascii_lowercase())); + chart_header(column, metadata) + }) + .collect::>(), + }); + let encoded_bytes = serde_json::to_vec(&metadata) + .map_err(|_| AppError::internal())? + .len(); + if encoded_bytes > MAX_CHART_METADATA_BYTES { + return Err(AppError::new( + AppErrorKind::ResourceExhausted, + ApiError::new( + "chart_result_too_large", + "The chart result exceeds the 8 MiB response limit", + ), + )); + } + Ok(metadata) +} + +fn chart_header(column: &ResultColumn, metadata: Option<&CommunityTableColumn>) -> Value { + let column_type = metadata.map_or(column.jdbc_type_name.as_str(), |column| { + column.column_type.as_str() + }); + json!({ + "dataType": chart_data_type(column), + "name": column.label, + "columnName": column.name, + "columnType": column_type, + "tableName": column.table_name, + "databaseName": column.catalog_name, + "schemaName": column.schema_name, + "primaryKey": metadata.and_then(|column| column.primary_key), + "comment": metadata.map(|column| column.comment.as_str()), + "defaultValue": metadata.and_then(|column| column.default_value.as_deref()), + "autoIncrement": metadata + .and_then(|column| column.auto_increment) + .map_or(0, i32::from), + "nullable": metadata.and_then(|column| column.nullable), + "columnSize": metadata.and_then(|column| column.column_size), + "decimalDigits": metadata.and_then(|column| column.decimal_digits), + "editorType": chart_editor_type(column_type, column.jdbc_type), + }) +} + +fn chart_value(value: &JdbcValue) -> Value { + match value { + JdbcValue::Null => Value::Null, + JdbcValue::Boolean { value } => Value::String(value.to_string()), + JdbcValue::SignedInteger { value } + | JdbcValue::UnsignedInteger { value } + | JdbcValue::Float32 { value } + | JdbcValue::Float64 { value } + | JdbcValue::Decimal { value } + | JdbcValue::Text { value } + | JdbcValue::Binary { value } + | JdbcValue::Date { value } + | JdbcValue::Time { value } + | JdbcValue::Timestamp { value } + | JdbcValue::TimestampWithTimeZone { value } + | JdbcValue::Json { value } + | JdbcValue::Uuid { value } => Value::String(value.clone()), + JdbcValue::Opaque { display_value, .. } => Value::String(display_value.clone()), + } +} + +fn chart_data_type(column: &ResultColumn) -> &'static str { + let type_name = column.jdbc_type_name.to_ascii_uppercase(); + let jdbc_type = match column.jdbc_type { + 12 | 1111 if type_name == "BLOB" => 2004, + 12 | 1111 if type_name == "CLOB" => 2005, + 12 | 1111 if type_name == "NCLOB" => 2011, + -7 if type_name == "TINYINT" => -6, + value => value, + }; + match jdbc_type { + 16 => "BOOLEAN", + 1 | 12 | -9 | -1 | -16 => "STRING", + -5 | 3 | 8 | 6 | 4 | 2 | 7 | 5 => "NUMERIC", + -7 => "BIT", + -6 if type_name.contains("BOOL") => "BOOLEAN", + -6 => "NUMERIC", + 91 | 92 | 93 | 2013 | 2014 => "DATETIME", + -4..=-2 => "BINARY", + 2004 | 2005 | 2011 | 2009 => "CONTENT", + 2002 => "STRUCT", + 2003 => "ARRAY", + -8 => "ROWID", + 2006 => "REFERENCE", + 1111 => "OBJECT", + _ => "UNKNOWN", + } +} + +fn chart_editor_type(type_name: &str, jdbc_type: i32) -> &'static str { + let normalized = type_name + .split(['(', ' ']) + .next() + .unwrap_or_default() + .to_ascii_uppercase(); + match normalized.as_str() { + "DATE" => "DATE", + "TIME" => "TIME", + "DATETIME" => "DATETIME", + "TIMESTAMP" => "TIMESTAMP", + _ => match jdbc_type { + 91 => "DATE", + 92 => "TIME", + 93 => "TIMESTAMP", + _ => "TEXT", + }, + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use chat2db_contract::{ + ColumnNullability, CommunityTableColumn, JdbcValue, JdbcValueType, ResultColumn, ResultRow, + }; + use serde_json::json; + + use super::{chart_data_type, chart_editable_table, chart_metadata, chart_refresh_context}; + + #[test] + fn chart_context_accepts_numeric_community_ids_and_selected_database() { + let chart = chat2db_contract::CommunityChart { + id: 7, + gmt_create: 1, + gmt_modified: 1, + name: None, + description: None, + schema: None, + data_source_id: Some(42), + data_source_name: None, + schema_name: None, + r#type: None, + database_name: None, + ddl: None, + deleted: None, + user_id: None, + chart_schema: None, + meta_data: None, + database_info: Some(json!({ + "dataSourceId": 42, + "databaseName": "analytics", + "schemaName": "analytics", + "consoleId": "9007199254740993", + "sql": "SELECT 1" + })), + refresh_type: None, + refresh_cycle: None, + }; + let context = chart_refresh_context(&chart).expect("chart context"); + assert_eq!(context.datasource_id, "42"); + assert_eq!(context.database_name.as_deref(), Some("analytics")); + assert_eq!(context.console_id, "9007199254740993"); + } + + #[test] + fn chart_context_never_falls_back_to_stale_top_level_names() { + let chart = chat2db_contract::CommunityChart { + id: 8, + gmt_create: 1, + gmt_modified: 1, + name: None, + description: None, + schema: None, + data_source_id: Some(42), + data_source_name: None, + database_name: Some("stale_database".to_owned()), + schema_name: Some("stale_schema".to_owned()), + r#type: None, + ddl: None, + deleted: None, + user_id: None, + chart_schema: None, + meta_data: None, + database_info: Some(json!({ + "dataSourceId": 42, + "sql": "SELECT 1" + })), + refresh_type: None, + refresh_cycle: None, + }; + let context = chart_refresh_context(&chart).expect("chart context"); + assert_eq!(context.database_name, None); + assert_eq!(context.schema_name, None); + } + + #[test] + fn chart_metadata_matches_community_display_shape() { + let result = crate::MysqlConsoleResult { + statement_sequence: 1, + result_set_id: Some(1), + sql: "SELECT amount, note".to_owned(), + success: true, + message: String::new(), + update_count: 0, + columns: vec![ResultColumn { + ordinal: 1, + label: "amount".to_owned(), + name: "amount".to_owned(), + jdbc_type: 3, + jdbc_type_name: "DECIMAL".to_owned(), + value_type: JdbcValueType::Decimal, + nullability: ColumnNullability::Nullable, + precision: Some(10), + scale: Some(2), + display_size: Some(12), + signed: Some(true), + catalog_name: Some("analytics".to_owned()), + schema_name: None, + table_name: Some("metrics".to_owned()), + }], + rows: vec![ResultRow { + values: vec![JdbcValue::Decimal { + value: "42.50".to_owned(), + }], + }], + row_count: 1, + has_more: false, + duration_ms: 3, + error: None, + }; + let header_metadata = HashMap::from([( + "amount".to_owned(), + CommunityTableColumn { + name: "amount".to_owned(), + column_type: "DECIMAL".to_owned(), + auto_increment: Some(false), + comment: "Invoice amount".to_owned(), + primary_key: Some(false), + column_size: Some(10), + decimal_digits: Some(2), + nullable: Some(1), + ..CommunityTableColumn::default() + }, + )]); + let metadata = chart_metadata(&result, Some(&header_metadata)).expect("chart metadata"); + assert_eq!(metadata["dataList"], json!([["42.50"]])); + assert_eq!(metadata["headerList"][0]["name"], "amount"); + assert_eq!(metadata["headerList"][0]["dataType"], "NUMERIC"); + assert_eq!(metadata["headerList"][0]["nullable"], 1); + assert_eq!(metadata["headerList"][0]["autoIncrement"], 0); + assert_eq!(metadata["headerList"][0]["primaryKey"], false); + assert_eq!(metadata["headerList"][0]["comment"], "Invoice amount"); + assert_eq!(metadata["headerList"][0]["editorType"], "TEXT"); + } + + #[test] + fn chart_jdbc_type_projection_matches_community() { + let mut column = result_column(-7, "BIT", JdbcValueType::Boolean); + assert_eq!(chart_data_type(&column), "BIT"); + + column.jdbc_type = -1; + column.jdbc_type_name = "JSON".to_owned(); + column.value_type = JdbcValueType::Json; + assert_eq!(chart_data_type(&column), "STRING"); + + column.jdbc_type = 93; + column.jdbc_type_name = "DATETIME".to_owned(); + assert_eq!( + super::chart_editor_type(&column.jdbc_type_name, column.jdbc_type), + "DATETIME" + ); + + let result = crate::MysqlConsoleResult { + statement_sequence: 1, + result_set_id: Some(1), + sql: "SELECT CAST('2024-01-02' AS DATETIME)".to_owned(), + success: true, + message: String::new(), + update_count: 0, + columns: vec![column], + rows: Vec::new(), + row_count: 0, + has_more: false, + duration_ms: 1, + error: None, + }; + let metadata = chart_metadata(&result, None).expect("basic chart metadata"); + assert_eq!(metadata["headerList"][0]["primaryKey"], json!(null)); + assert_eq!(metadata["headerList"][0]["nullable"], json!(null)); + assert_eq!(metadata["headerList"][0]["autoIncrement"], 0); + assert_eq!(metadata["headerList"][0]["editorType"], "DATETIME"); + } + + #[test] + fn chart_header_enhancement_is_limited_to_simple_editable_tables() { + let table = chart_editable_table("SELECT id, label FROM analytics.metrics") + .expect("simple table query"); + assert_eq!(table.database_name.as_deref(), Some("analytics")); + assert_eq!(table.table_name, "metrics"); + assert!(chart_editable_table("SELECT id AS value FROM metrics").is_none()); + assert!(chart_editable_table("SELECT COUNT(id) FROM metrics").is_none()); + assert!( + chart_editable_table("WITH cte AS (SELECT id FROM metrics) SELECT id FROM cte") + .is_none() + ); + assert!( + chart_editable_table( + "SELECT metrics.id FROM metrics JOIN tags ON tags.id = metrics.id" + ) + .is_none() + ); + } + + fn result_column( + jdbc_type: i32, + jdbc_type_name: &str, + value_type: JdbcValueType, + ) -> ResultColumn { + ResultColumn { + ordinal: 1, + label: "value".to_owned(), + name: "value".to_owned(), + jdbc_type, + jdbc_type_name: jdbc_type_name.to_owned(), + value_type, + nullability: ColumnNullability::Nullable, + precision: None, + scale: None, + display_size: None, + signed: None, + catalog_name: None, + schema_name: None, + table_name: None, + } + } +} diff --git a/crates/chat2db-core/src/mysql_schema_diff.rs b/crates/chat2db-core/src/mysql_schema_diff.rs new file mode 100644 index 0000000..8d2698f --- /dev/null +++ b/crates/chat2db-core/src/mysql_schema_diff.rs @@ -0,0 +1,1899 @@ +use std::{ + collections::{BTreeMap, BTreeSet}, + time::Duration, +}; + +use chat2db_contract::{ + ApiError, CommunitySchemaDiffEndpoint, CommunitySchemaDiffRequest, CommunitySchemaDiffSql, +}; +use mysql_async::{Conn, Error as MysqlError, prelude::Queryable}; + +use crate::{ + AppError, AppErrorKind, Application, + native_mysql::{finish_connection, open_resolved_connection, resolve_native_connection}, +}; + +const SCHEMA_DIFF_TIMEOUT: Duration = Duration::from_secs(30); +const MAX_SCHEMA_DIFF_OBJECTS: usize = 2_048; +const MAX_TABLE_DDL_BYTES: usize = 4 * 1024 * 1024; +const MAX_VIEW_DEFINITION_BYTES: usize = 4 * 1024 * 1024; +const MAX_SCHEMA_SNAPSHOT_BYTES: usize = 64 * 1024 * 1024; +const MAX_SCHEMA_DIFF_SQL_BYTES: usize = 16 * 1024 * 1024; +const NO_DIFFERENCES_SQL: &str = "-- No differences. "; + +#[derive(Debug, Default)] +struct SchemaSnapshot { + database_name: String, + lower_case_table_names: u8, + tables: BTreeMap, + views: BTreeMap, +} + +#[derive(Debug)] +struct ViewSnapshot { + definition: String, +} + +#[derive(Debug)] +struct TableSnapshot { + create_sql_without_foreign_keys: String, + columns: Vec, + indexes: Vec, + foreign_keys: Vec, + options: TableOptions, +} + +#[derive(Debug)] +struct ColumnDefinition { + name: String, + sql: String, + comparison_sql: String, +} + +#[derive(Debug)] +struct IndexDefinition { + name: String, + sql: String, + primary: bool, +} + +#[derive(Debug)] +struct ForeignKeyDefinition { + name: String, + sql: String, + referenced_table: String, +} + +/// Pinned Community parity intentionally compares only these existing-table options. +/// +/// CHECK constraints, partition definitions, and additional `MySQL` table options are preserved +/// when a missing table is created, but the Community schema-diff contract does not alter them on +/// an existing table. The SHOW CREATE `AUTO_INCREMENT=N` next counter is runtime state and is +/// deliberately excluded from both comparison and generated CREATE statements. +#[derive(Debug, Default)] +struct TableOptions { + engine: Option, + charset: Option, + collation: Option, + comment: Option, +} + +#[derive(Debug, Default)] +struct ExistingTableDiff { + foreign_key_drops: Vec, + table_changes: Vec, + foreign_key_adds: Vec, +} + +impl Application { + /// Previews SQL that changes the target `MySQL` namespace to match the source. + /// + /// This method only reads metadata. Generated SQL is never executed automatically. + /// + /// # Errors + /// + /// Returns validation, datasource, connection, metadata, parse, resource-limit, or cleanup + /// errors. + pub async fn preview_mysql_schema_diff( + &self, + request: &CommunitySchemaDiffRequest, + ) -> Result { + validate_endpoint(&request.source, "source")?; + validate_endpoint(&request.target, "target")?; + + let source = load_endpoint_snapshot(self, &request.source).await?; + let target = load_endpoint_snapshot(self, &request.target).await?; + build_schema_diff(&source, &target).map(CommunitySchemaDiffSql::new) + } +} + +fn validate_endpoint( + endpoint: &CommunitySchemaDiffEndpoint, + role: &'static str, +) -> Result<(), AppError> { + if endpoint.datasource_id.trim().is_empty() { + return Err(invalid_schema_diff(format!( + "The {role} datasource id is required" + ))); + } + if endpoint.database_name.trim().is_empty() { + return Err(invalid_schema_diff(format!( + "The {role} database name is required" + ))); + } + if endpoint.database_name.contains('\0') { + return Err(invalid_schema_diff(format!( + "The {role} database name cannot contain NUL" + ))); + } + Ok(()) +} + +async fn load_endpoint_snapshot( + application: &Application, + endpoint: &CommunitySchemaDiffEndpoint, +) -> Result { + let resolved = resolve_native_connection(application, &endpoint.datasource_id).await?; + let mut conn = open_resolved_connection(&resolved).await?; + let result = tokio::time::timeout( + SCHEMA_DIFF_TIMEOUT, + load_schema_snapshot(&mut conn, &endpoint.database_name), + ) + .await + .map_err(|_| { + AppError::unavailable( + "mysql_schema_diff_timeout", + "The MySQL schema diff metadata query timed out", + ) + })?; + finish_connection(conn, result).await +} + +#[allow(clippy::too_many_lines)] +async fn load_schema_snapshot( + conn: &mut Conn, + database_name: &str, +) -> Result { + let database_exists = conn + .exec_first::( + "SELECT 1 FROM information_schema.SCHEMATA WHERE SCHEMA_NAME = ? LIMIT 1", + (database_name,), + ) + .await + .map_err(schema_diff_query_error)?; + if database_exists != Some(1) { + return Err(AppError::not_found( + "mysql_schema_diff_database_not_found", + "The selected MySQL database does not exist", + )); + } + + let lower_case_table_names = conn + .query_first::("SELECT @@lower_case_table_names") + .await + .map_err(schema_diff_query_error)? + .ok_or_else(malformed_show_create)?; + + let table_query = format!( + "SELECT TABLE_NAME FROM information_schema.TABLES \ + WHERE TABLE_SCHEMA = ? AND TABLE_TYPE = 'BASE TABLE' \ + ORDER BY TABLE_NAME LIMIT {}", + MAX_SCHEMA_DIFF_OBJECTS + 1 + ); + let table_names = conn + .exec::(table_query, (database_name,)) + .await + .map_err(schema_diff_query_error)?; + if table_names.len() > MAX_SCHEMA_DIFF_OBJECTS { + return Err(schema_diff_resource_limit( + "The selected MySQL database contains too many tables to diff safely", + )); + } + + let view_query = format!( + "SELECT TABLE_NAME, VIEW_DEFINITION FROM information_schema.VIEWS \ + WHERE TABLE_SCHEMA = ? ORDER BY TABLE_NAME LIMIT {}", + MAX_SCHEMA_DIFF_OBJECTS + 1 + ); + let view_rows = conn + .exec::<(String, Option), _, _>(view_query, (database_name,)) + .await + .map_err(schema_diff_query_error)?; + if table_names.len().saturating_add(view_rows.len()) > MAX_SCHEMA_DIFF_OBJECTS { + return Err(schema_diff_resource_limit( + "The selected MySQL database contains too many objects to diff safely", + )); + } + + let mut snapshot = SchemaSnapshot { + database_name: database_name.to_owned(), + lower_case_table_names, + ..SchemaSnapshot::default() + }; + let mut snapshot_bytes = 0_usize; + for table_name in table_names { + let qualified_name = format!( + "{}.{}", + quote_identifier(database_name), + quote_identifier(&table_name) + ); + let row = conn + .query_first::<(String, String), _>(format!("SHOW CREATE TABLE {qualified_name}")) + .await + .map_err(schema_diff_query_error)? + .ok_or_else(malformed_show_create)?; + let ddl = row.1; + if ddl.len() > MAX_TABLE_DDL_BYTES { + return Err(schema_diff_resource_limit( + "A MySQL table definition is too large to diff safely", + )); + } + snapshot_bytes = snapshot_bytes.checked_add(ddl.len()).ok_or_else(|| { + schema_diff_resource_limit("The MySQL schema snapshot is too large to diff safely") + })?; + if snapshot_bytes > MAX_SCHEMA_SNAPSHOT_BYTES { + return Err(schema_diff_resource_limit( + "The MySQL schema snapshot is too large to diff safely", + )); + } + snapshot + .tables + .insert(table_name.clone(), parse_table_snapshot(&table_name, &ddl)?); + } + for (view_name, definition) in view_rows { + let definition = definition.ok_or_else(malformed_view_definition)?; + if definition.len() > MAX_VIEW_DEFINITION_BYTES { + return Err(schema_diff_resource_limit( + "A MySQL view definition is too large to diff safely", + )); + } + snapshot_bytes = snapshot_bytes + .checked_add(definition.len()) + .ok_or_else(|| { + schema_diff_resource_limit("The MySQL schema snapshot is too large to diff safely") + })?; + if snapshot_bytes > MAX_SCHEMA_SNAPSHOT_BYTES { + return Err(schema_diff_resource_limit( + "The MySQL schema snapshot is too large to diff safely", + )); + } + snapshot.views.insert( + view_name, + ViewSnapshot { + definition: definition.trim().to_owned(), + }, + ); + } + Ok(snapshot) +} + +fn parse_table_snapshot(table_name: &str, ddl: &str) -> Result { + let (prefix, body, suffix) = create_table_parts(ddl).ok_or_else(malformed_show_create)?; + let mut columns: Vec = Vec::new(); + let mut indexes: Vec = Vec::new(); + let mut foreign_keys: Vec = Vec::new(); + let mut definitions_without_foreign_keys = Vec::new(); + + for definition in split_top_level_definitions(body) { + let definition = definition.trim(); + if definition.is_empty() { + continue; + } + if definition.starts_with('`') { + let (name, _) = + parse_quoted_identifier(definition).ok_or_else(malformed_show_create)?; + if columns + .iter() + .any(|column| names_equal(&column.name, &name)) + { + return Err(malformed_show_create()); + } + columns.push(ColumnDefinition { + name, + sql: definition.to_owned(), + comparison_sql: canonicalize_column_definition(definition), + }); + definitions_without_foreign_keys.push(definition); + continue; + } + if let Some(foreign_key) = parse_foreign_key_definition(definition)? { + if foreign_keys + .iter() + .any(|existing| names_equal(&existing.name, &foreign_key.name)) + { + return Err(malformed_show_create()); + } + foreign_keys.push(foreign_key); + continue; + } + if let Some(index) = parse_index_definition(definition)? { + if indexes + .iter() + .any(|existing| names_equal(&existing.name, &index.name)) + { + return Err(malformed_show_create()); + } + indexes.push(index); + } + // Pinned Community parity preserves CHECK and other table-level clauses for new tables, + // but does not claim to diff them on an existing table. + definitions_without_foreign_keys.push(definition); + } + + if columns.is_empty() { + return Err(AppError::unavailable( + "mysql_schema_diff_metadata_invalid", + format!("MySQL returned an invalid definition for table {table_name}"), + )); + } + let suffix_without_runtime_counter = strip_table_option(suffix, "AUTO_INCREMENT"); + let create_sql_without_foreign_keys = format!( + "{}\n {}\n{}", + prefix.trim_end(), + definitions_without_foreign_keys.join(",\n "), + suffix_without_runtime_counter + .trim_start() + .trim_end_matches(';') + ); + Ok(TableSnapshot { + create_sql_without_foreign_keys, + columns, + indexes, + foreign_keys, + options: parse_table_options(suffix)?, + }) +} + +fn parse_foreign_key_definition( + definition: &str, +) -> Result, AppError> { + let Some(rest) = strip_ascii_prefix(definition, "CONSTRAINT") else { + return Ok(None); + }; + let rest = rest.trim_start(); + let (name, consumed) = parse_quoted_identifier(rest).ok_or_else(malformed_show_create)?; + if !has_ascii_prefix(rest[consumed..].trim_start(), "FOREIGN KEY") { + return Ok(None); + } + let referenced_table = parse_referenced_table(definition).ok_or_else(malformed_show_create)?; + Ok(Some(ForeignKeyDefinition { + name, + sql: definition.to_owned(), + referenced_table, + })) +} + +fn parse_referenced_table(definition: &str) -> Option { + const REFERENCES: &str = "REFERENCES"; + let start = find_unquoted_keyword(definition, REFERENCES)? + REFERENCES.len(); + let rest = definition.get(start..)?.trim_start(); + let (first, consumed) = parse_quoted_identifier(rest)?; + let after_first = rest.get(consumed..)?.trim_start(); + let Some(after_dot) = after_first.strip_prefix('.') else { + return Some(first); + }; + parse_quoted_identifier(after_dot.trim_start()).map(|(table, _)| table) +} + +fn parse_index_definition(definition: &str) -> Result, AppError> { + if has_ascii_prefix(definition, "PRIMARY KEY") { + return Ok(Some(IndexDefinition { + name: "PRIMARY".to_owned(), + sql: definition.to_owned(), + primary: true, + })); + } + for prefix in ["UNIQUE KEY", "FULLTEXT KEY", "SPATIAL KEY", "KEY"] { + let Some(rest) = strip_ascii_prefix(definition, prefix) else { + continue; + }; + let (name, _) = + parse_quoted_identifier(rest.trim_start()).ok_or_else(malformed_show_create)?; + return Ok(Some(IndexDefinition { + name, + sql: definition.to_owned(), + primary: false, + })); + } + Ok(None) +} + +fn parse_table_options(suffix: &str) -> Result { + let engine = parse_identifier_table_option(suffix, "ENGINE")?; + let charset = parse_identifier_table_option(suffix, "CHARSET")?; + let collation = parse_identifier_table_option(suffix, "COLLATE")?; + let comment = table_option_value(suffix, "COMMENT") + .map(|value| { + if quoted_value_end(value) == Some(value.len()) { + Ok(value.to_owned()) + } else { + Err(malformed_show_create()) + } + }) + .transpose()?; + Ok(TableOptions { + engine, + charset, + collation, + comment, + }) +} + +fn strip_table_option(suffix: &str, name: &str) -> String { + let Some(name_start) = find_unquoted_keyword(suffix, name) else { + return suffix.to_owned(); + }; + let bytes = suffix.as_bytes(); + let mut value_start = skip_ascii_whitespace(bytes, name_start + name.len()); + if bytes.get(value_start) == Some(&b'=') { + value_start = skip_ascii_whitespace(bytes, value_start + 1); + } + let value_end = if matches!(bytes.get(value_start), Some(b'\'' | b'"')) { + suffix + .get(value_start..) + .and_then(quoted_value_end) + .map_or(value_start, |length| value_start + length) + } else { + let mut end = value_start; + while bytes + .get(end) + .is_some_and(|byte| !byte.is_ascii_whitespace() && *byte != b',') + { + end += 1; + } + end + }; + if value_end == value_start { + return suffix.to_owned(); + } + let before = suffix[..name_start].trim_end(); + let after = suffix[value_end..].trim_start(); + match (before.is_empty(), after.is_empty()) { + (true, _) => after.to_owned(), + (_, true) => before.to_owned(), + (false, false) => format!("{before} {after}"), + } +} + +fn parse_identifier_table_option(suffix: &str, name: &str) -> Result, AppError> { + table_option_value(suffix, name) + .map(|value| { + if value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'_') + { + Ok(value.to_owned()) + } else { + Err(malformed_show_create()) + } + }) + .transpose() +} + +fn table_option_value<'a>(suffix: &'a str, name: &str) -> Option<&'a str> { + let name_start = find_unquoted_keyword(suffix, name)?; + let bytes = suffix.as_bytes(); + let mut value_start = skip_ascii_whitespace(bytes, name_start + name.len()); + if bytes.get(value_start) == Some(&b'=') { + value_start = skip_ascii_whitespace(bytes, value_start + 1); + } + if matches!(bytes.get(value_start), Some(b'\'' | b'"')) { + let value = suffix.get(value_start..)?; + let value_end = quoted_value_end(value)?; + return value.get(..value_end); + } + let mut value_end = value_start; + while bytes + .get(value_end) + .is_some_and(|byte| !byte.is_ascii_whitespace() && *byte != b',') + { + value_end += 1; + } + (value_end > value_start).then(|| &suffix[value_start..value_end]) +} + +fn find_unquoted_keyword(value: &str, keyword: &str) -> Option { + let bytes = value.as_bytes(); + let mut quote = None; + let mut index = 0_usize; + while index < bytes.len() { + if let Some(delimiter) = quote { + if bytes[index] == b'\\' && matches!(delimiter, b'\'' | b'"') { + index = (index + 2).min(bytes.len()); + continue; + } + if bytes[index] == delimiter { + if bytes.get(index + 1) == Some(&delimiter) { + index += 2; + continue; + } + quote = None; + } + index += 1; + continue; + } + if matches!(bytes[index], b'\'' | b'"' | b'`') { + quote = Some(bytes[index]); + index += 1; + continue; + } + let end = index.checked_add(keyword.len())?; + let matches = value + .get(index..end) + .is_some_and(|candidate| candidate.eq_ignore_ascii_case(keyword)); + let boundary_before = index == 0 + || bytes + .get(index - 1) + .is_none_or(|byte| !byte.is_ascii_alphanumeric() && *byte != b'_'); + let boundary_after = bytes + .get(end) + .is_none_or(|byte| !byte.is_ascii_alphanumeric() && *byte != b'_'); + if matches && boundary_before && boundary_after { + return Some(index); + } + index += 1; + } + None +} + +fn quoted_value_end(value: &str) -> Option { + let bytes = value.as_bytes(); + let delimiter = *bytes.first()?; + if !matches!(delimiter, b'\'' | b'"') { + return None; + } + let mut index = 1_usize; + while index < bytes.len() { + if bytes[index] == b'\\' { + index = (index + 2).min(bytes.len()); + continue; + } + if bytes[index] == delimiter { + if bytes.get(index + 1) == Some(&delimiter) { + index += 2; + continue; + } + return Some(index + 1); + } + index += 1; + } + None +} + +#[allow(clippy::too_many_lines)] +fn build_schema_diff(source: &SchemaSnapshot, target: &SchemaSnapshot) -> Result { + validate_case_only_object_conflicts(source, target)?; + let source_view_order = topologically_order_views(source)?; + let target_view_order = topologically_order_views(target)?; + let mut view_drops = Vec::new(); + let mut foreign_key_drops = Vec::new(); + let mut table_changes = Vec::new(); + let mut foreign_key_adds = Vec::new(); + let mut view_upserts = Vec::new(); + let mut changed_tables = BTreeSet::new(); + let mut existing_diffs = BTreeMap::new(); + + for view_name in target_view_order.iter().rev() { + if !source.views.contains_key(view_name) { + view_drops.push(format!( + "DROP VIEW {}", + qualified_object(&target.database_name, view_name) + )); + } + } + + for (table_name, source_table) in &source.tables { + match target.tables.get(table_name) { + None => { + changed_tables.insert(table_name.clone()); + table_changes.push(qualify_create_table_sql( + &source_table.create_sql_without_foreign_keys, + &target.database_name, + table_name, + )?); + foreign_key_adds.extend(add_all_foreign_keys( + table_name, + source_table, + &source.database_name, + &target.database_name, + )); + } + Some(target_table) => { + let diff = diff_existing_table( + table_name, + source_table, + target_table, + &source.database_name, + &target.database_name, + ); + if !diff.table_changes.is_empty() { + changed_tables.insert(table_name.clone()); + } + existing_diffs.insert(table_name.clone(), diff); + } + } + } + for (table_name, target_table) in &target.tables { + if !source.tables.contains_key(table_name) { + changed_tables.insert(table_name.clone()); + foreign_key_drops.extend(drop_all_foreign_keys( + table_name, + target_table, + &target.database_name, + )); + table_changes.push(format!( + "DROP TABLE {}", + qualified_object(&target.database_name, table_name) + )); + } + } + + for (table_name, diff) in &mut existing_diffs { + let source_table = source + .tables + .get(table_name) + .expect("existing source table was collected above"); + let target_table = target + .tables + .get(table_name) + .expect("existing target table was collected above"); + let owner_changed = changed_tables.contains(table_name); + for foreign_key in &target_table.foreign_keys { + if owner_changed || changed_tables.contains(&foreign_key.referenced_table) { + push_unique( + &mut diff.foreign_key_drops, + format!( + "ALTER TABLE {} DROP FOREIGN KEY {}", + qualified_object(&target.database_name, table_name), + quote_identifier(&foreign_key.name) + ), + ); + } + } + for foreign_key in &source_table.foreign_keys { + if owner_changed || changed_tables.contains(&foreign_key.referenced_table) { + push_unique( + &mut diff.foreign_key_adds, + format!( + "ALTER TABLE {} ADD {}", + qualified_object(&target.database_name, table_name), + retarget_foreign_key_sql( + &foreign_key.sql, + &source.database_name, + &target.database_name + ) + ), + ); + } + } + } + for diff in existing_diffs.into_values() { + foreign_key_drops.extend(diff.foreign_key_drops); + table_changes.extend(diff.table_changes); + foreign_key_adds.extend(diff.foreign_key_adds); + } + + for view_name in source_view_order { + let source_view = source + .views + .get(&view_name) + .expect("the view dependency order only contains source views"); + let target_definition = rewrite_qualified_catalog( + &source_view.definition, + &source.database_name, + &target.database_name, + ); + let statement = match target.views.get(&view_name) { + None => Some(format!( + "CREATE VIEW {} AS {target_definition}", + qualified_object(&target.database_name, &view_name) + )), + Some(target_view) if target_view.definition.trim() != target_definition => { + Some(format!( + "CREATE OR REPLACE VIEW {} AS {target_definition}", + qualified_object(&target.database_name, &view_name) + )) + } + Some(_) => None, + }; + if let Some(statement) = statement { + view_upserts.push(statement); + } + } + + let statements = view_drops + .into_iter() + .chain(foreign_key_drops) + .chain(table_changes) + .chain(foreign_key_adds) + .chain(view_upserts) + .collect::>(); + render_statements(&statements) +} + +fn validate_case_only_object_conflicts( + source: &SchemaSnapshot, + target: &SchemaSnapshot, +) -> Result<(), AppError> { + if target.lower_case_table_names == 0 { + return Ok(()); + } + let source_names = source + .tables + .keys() + .chain(source.views.keys()) + .collect::>(); + let target_names = target + .tables + .keys() + .chain(target.views.keys()) + .collect::>(); + let source_has_collision = source_names.iter().enumerate().any(|(index, left)| { + source_names[index + 1..] + .iter() + .any(|right| *left != *right && names_equal(left, right)) + }); + let source_target_conflict = source_names.iter().any(|source_name| { + target_names.iter().any(|target_name| { + *source_name != *target_name && names_equal(source_name, target_name) + }) + }); + if source_has_collision || source_target_conflict { + return Err(AppError::invalid( + "mysql_schema_diff_case_conflict", + "The target MySQL server uses case-insensitive object names and the schema contains a case-only name conflict", + )); + } + Ok(()) +} + +fn topologically_order_views(snapshot: &SchemaSnapshot) -> Result, AppError> { + let mut dependency_count = snapshot + .views + .keys() + .map(|name| (name.clone(), 0_usize)) + .collect::>(); + let mut dependents = BTreeMap::>::new(); + for (view_name, view) in &snapshot.views { + for dependency in qualified_catalog_objects(&view.definition, &snapshot.database_name) { + let dependency = snapshot.views.keys().find(|candidate| { + *candidate == &dependency + || (snapshot.lower_case_table_names != 0 && names_equal(candidate, &dependency)) + }); + let Some(dependency) = dependency else { + continue; + }; + if dependents + .entry(dependency.clone()) + .or_default() + .insert(view_name.clone()) + { + *dependency_count + .get_mut(view_name) + .expect("every source view has a dependency counter") += 1; + } + } + } + + let mut ready = dependency_count + .iter() + .filter_map(|(name, count)| (*count == 0).then_some(name.clone())) + .collect::>(); + let mut ordered = Vec::with_capacity(snapshot.views.len()); + while let Some(view_name) = ready.pop_first() { + ordered.push(view_name.clone()); + if let Some(children) = dependents.get(&view_name) { + for child in children { + let count = dependency_count + .get_mut(child) + .expect("every dependent view has a dependency counter"); + *count -= 1; + if *count == 0 { + ready.insert(child.clone()); + } + } + } + } + if ordered.len() != snapshot.views.len() { + return Err(AppError::invalid( + "mysql_schema_diff_view_dependency_cycle", + "The MySQL schema contains a view dependency cycle that cannot be migrated safely", + )); + } + Ok(ordered) +} + +fn qualified_catalog_objects(definition: &str, catalog: &str) -> BTreeSet { + let bytes = definition.as_bytes(); + let mut objects = BTreeSet::new(); + let mut index = 0_usize; + while index < bytes.len() { + if matches!(bytes[index], b'\'' | b'"') { + let remainder = &definition[index..]; + let end = quoted_value_end(remainder).unwrap_or(remainder.len()); + index += end; + continue; + } + if bytes[index] != b'`' { + index += 1; + continue; + } + let Some((identifier, consumed)) = parse_quoted_identifier(&definition[index..]) else { + index += 1; + continue; + }; + let token_end = index + consumed; + let dot = skip_ascii_whitespace(bytes, token_end); + let object_start = skip_ascii_whitespace(bytes, dot.saturating_add(1)); + if identifier.eq_ignore_ascii_case(catalog) + && bytes.get(dot) == Some(&b'.') + && bytes.get(object_start) == Some(&b'`') + && let Some((object, object_length)) = + parse_quoted_identifier(&definition[object_start..]) + { + objects.insert(object); + index = object_start + object_length; + continue; + } + index = token_end; + } + objects +} + +fn qualify_create_table_sql( + create_sql: &str, + target_database: &str, + table_name: &str, +) -> Result { + let Some(rest) = strip_ascii_prefix(create_sql.trim(), "CREATE TABLE") else { + return Err(malformed_show_create()); + }; + let rest = rest.trim_start(); + let (parsed_name, consumed) = + parse_quoted_identifier(rest).ok_or_else(malformed_show_create)?; + if !names_equal(&parsed_name, table_name) { + return Err(malformed_show_create()); + } + Ok(format!( + "CREATE TABLE {}{}", + qualified_object(target_database, table_name), + &rest[consumed..] + )) +} + +fn rewrite_qualified_catalog(definition: &str, source: &str, target: &str) -> String { + if source.eq_ignore_ascii_case(target) { + return definition.trim().to_owned(); + } + let bytes = definition.as_bytes(); + let mut output = Vec::with_capacity(definition.len().saturating_add(target.len())); + let mut index = 0_usize; + while index < bytes.len() { + if matches!(bytes[index], b'\'' | b'"') { + let remainder = &definition[index..]; + let end = quoted_value_end(remainder).unwrap_or(remainder.len()); + output.extend_from_slice(&bytes[index..index + end]); + index += end; + continue; + } + if bytes[index] != b'`' { + output.push(bytes[index]); + index += 1; + continue; + } + let Some((identifier, consumed)) = parse_quoted_identifier(&definition[index..]) else { + output.push(bytes[index]); + index += 1; + continue; + }; + let token_end = index + consumed; + let dot = skip_ascii_whitespace(bytes, token_end); + if identifier.eq_ignore_ascii_case(source) && bytes.get(dot) == Some(&b'.') { + output.extend_from_slice(quote_identifier(target).as_bytes()); + } else { + output.extend_from_slice(&bytes[index..token_end]); + } + index = token_end; + } + String::from_utf8(output) + .expect("rewriting a valid UTF-8 MySQL view definition must preserve UTF-8") + .trim() + .to_owned() +} + +fn retarget_foreign_key_sql(definition: &str, source: &str, target: &str) -> String { + const REFERENCES: &str = "REFERENCES"; + let Some(table_start) = find_unquoted_keyword(definition, REFERENCES) + .map(|start| skip_ascii_whitespace(definition.as_bytes(), start + REFERENCES.len())) + else { + return definition.trim().to_owned(); + }; + let Some((_, consumed)) = parse_quoted_identifier(&definition[table_start..]) else { + return definition.trim().to_owned(); + }; + let after_first = skip_ascii_whitespace(definition.as_bytes(), table_start + consumed); + if definition.as_bytes().get(after_first) == Some(&b'.') { + return rewrite_qualified_catalog(definition, source, target); + } + + format!( + "{}{}.{}", + &definition[..table_start], + quote_identifier(target), + &definition[table_start..] + ) + .trim() + .to_owned() +} + +fn canonicalize_foreign_key_sql(definition: &str, local_catalog: &str) -> String { + const REFERENCES: &str = "REFERENCES"; + let Some(table_start) = find_unquoted_keyword(definition, REFERENCES) + .map(|start| skip_ascii_whitespace(definition.as_bytes(), start + REFERENCES.len())) + else { + return definition.trim().to_owned(); + }; + let Some((catalog, consumed)) = parse_quoted_identifier(&definition[table_start..]) else { + return definition.trim().to_owned(); + }; + let dot = skip_ascii_whitespace(definition.as_bytes(), table_start + consumed); + if !catalog.eq_ignore_ascii_case(local_catalog) || definition.as_bytes().get(dot) != Some(&b'.') + { + return definition.trim().to_owned(); + } + let object_start = skip_ascii_whitespace(definition.as_bytes(), dot + 1); + if parse_quoted_identifier(&definition[object_start..]).is_none() { + return definition.trim().to_owned(); + } + format!( + "{}{}", + &definition[..table_start], + &definition[object_start..] + ) + .trim() + .to_owned() +} + +fn push_unique(values: &mut Vec, value: String) { + if !values.iter().any(|existing| existing == &value) { + values.push(value); + } +} + +#[allow(clippy::too_many_lines)] +fn diff_existing_table( + table_name: &str, + source: &TableSnapshot, + target: &TableSnapshot, + source_database: &str, + target_database: &str, +) -> ExistingTableDiff { + let qualified_table = qualified_object(target_database, table_name); + let mut diff = ExistingTableDiff::default(); + + for target_foreign_key in &target.foreign_keys { + let source_foreign_key = find_foreign_key(&source.foreign_keys, &target_foreign_key.name); + let source_sql = source_foreign_key + .map(|foreign_key| canonicalize_foreign_key_sql(&foreign_key.sql, source_database)); + let target_sql = canonicalize_foreign_key_sql(&target_foreign_key.sql, target_database); + if source_sql.as_deref() != Some(target_sql.as_str()) { + diff.foreign_key_drops.push(format!( + "ALTER TABLE {qualified_table} DROP FOREIGN KEY {}", + quote_identifier(&target_foreign_key.name) + )); + } + } + + for target_index in target.indexes.iter().filter(|index| !index.primary) { + let source_index = find_index(&source.indexes, &target_index.name); + if source_index.is_none_or(|index| index.sql != target_index.sql) { + diff.table_changes.push(format!( + "ALTER TABLE {qualified_table} DROP INDEX {}", + quote_identifier(&target_index.name) + )); + } + } + + let mut current_order = target + .columns + .iter() + .filter(|column| find_column(&source.columns, &column.name).is_some()) + .map(|column| column.name.clone()) + .collect::>(); + for (desired_position, source_column) in source.columns.iter().enumerate() { + let target_column = find_column(&target.columns, &source_column.name); + let current_position = current_order + .iter() + .position(|name| names_equal(name, &source_column.name)); + let position_changed = current_position != Some(desired_position); + let position = column_position(&source.columns, desired_position); + + match target_column { + None => diff.table_changes.push(format!( + "ALTER TABLE {qualified_table} ADD COLUMN {}{position}", + source_column.sql + )), + Some(target_column) + if target_column.comparison_sql != source_column.comparison_sql + || position_changed => + { + diff.table_changes.push(format!( + "ALTER TABLE {qualified_table} MODIFY COLUMN {}{position}", + source_column.sql + )); + } + Some(_) => {} + } + + if let Some(position) = current_position { + current_order.remove(position); + } + current_order.insert(desired_position, source_column.name.clone()); + } + + for source_index in source.indexes.iter().filter(|index| !index.primary) { + let target_index = find_index(&target.indexes, &source_index.name); + if target_index.is_none_or(|index| index.sql != source_index.sql) { + diff.table_changes.push(format!( + "ALTER TABLE {qualified_table} ADD {}", + source_index.sql + )); + } + } + + let source_primary = source.indexes.iter().find(|index| index.primary); + let target_primary = target.indexes.iter().find(|index| index.primary); + match (source_primary, target_primary) { + (Some(source_primary), Some(target_primary)) + if source_primary.sql != target_primary.sql => + { + diff.table_changes.push(format!( + "ALTER TABLE {qualified_table} DROP PRIMARY KEY, ADD {}", + source_primary.sql + )); + } + (Some(source_primary), None) => diff.table_changes.push(format!( + "ALTER TABLE {qualified_table} ADD {}", + source_primary.sql + )), + (None, Some(_)) => diff + .table_changes + .push(format!("ALTER TABLE {qualified_table} DROP PRIMARY KEY")), + _ => {} + } + + for target_column in &target.columns { + if find_column(&source.columns, &target_column.name).is_none() { + diff.table_changes.push(format!( + "ALTER TABLE {qualified_table} DROP COLUMN {}", + quote_identifier(&target_column.name) + )); + } + } + + if let Some(statement) = diff_table_options( + target_database, + table_name, + &source.options, + &target.options, + ) { + diff.table_changes.push(statement); + } + for source_foreign_key in &source.foreign_keys { + let target_foreign_key = find_foreign_key(&target.foreign_keys, &source_foreign_key.name); + let source_comparison_sql = + canonicalize_foreign_key_sql(&source_foreign_key.sql, source_database); + let target_matches = target_foreign_key.is_some_and(|foreign_key| { + canonicalize_foreign_key_sql(&foreign_key.sql, target_database) == source_comparison_sql + }); + if !target_matches { + let source_sql = + retarget_foreign_key_sql(&source_foreign_key.sql, source_database, target_database); + diff.foreign_key_adds + .push(format!("ALTER TABLE {qualified_table} ADD {source_sql}")); + } + } + diff +} + +fn drop_all_foreign_keys( + table_name: &str, + table: &TableSnapshot, + target_database: &str, +) -> Vec { + let qualified_table = qualified_object(target_database, table_name); + table + .foreign_keys + .iter() + .map(|foreign_key| { + format!( + "ALTER TABLE {qualified_table} DROP FOREIGN KEY {}", + quote_identifier(&foreign_key.name) + ) + }) + .collect() +} + +fn add_all_foreign_keys( + table_name: &str, + table: &TableSnapshot, + source_database: &str, + target_database: &str, +) -> Vec { + let qualified_table = qualified_object(target_database, table_name); + table + .foreign_keys + .iter() + .map(|foreign_key| { + let source_sql = + retarget_foreign_key_sql(&foreign_key.sql, source_database, target_database); + format!("ALTER TABLE {qualified_table} ADD {source_sql}") + }) + .collect() +} + +fn diff_table_options( + target_database: &str, + table_name: &str, + source: &TableOptions, + target: &TableOptions, +) -> Option { + let mut clauses = Vec::new(); + if !option_eq_ignore_ascii_case(source.engine.as_deref(), target.engine.as_deref()) { + clauses.push(format!("ENGINE={}", source.engine.as_deref()?)); + } + if !option_eq_ignore_ascii_case(source.charset.as_deref(), target.charset.as_deref()) + || !option_eq_ignore_ascii_case(source.collation.as_deref(), target.collation.as_deref()) + { + if let Some(charset) = source.charset.as_deref() { + clauses.push(format!("DEFAULT CHARACTER SET={charset}")); + } + if let Some(collation) = source.collation.as_deref() { + clauses.push(format!("COLLATE={collation}")); + } + } + if source.comment != target.comment { + clauses.push(format!( + "COMMENT={}", + source.comment.as_deref().unwrap_or("''") + )); + } + (!clauses.is_empty()).then(|| { + format!( + "ALTER TABLE {} {}", + qualified_object(target_database, table_name), + clauses.join(", ") + ) + }) +} + +fn option_eq_ignore_ascii_case(left: Option<&str>, right: Option<&str>) -> bool { + match (left, right) { + (Some(left), Some(right)) => left.eq_ignore_ascii_case(right), + (None, None) => true, + _ => false, + } +} + +fn render_statements(statements: &[String]) -> Result { + if statements.is_empty() { + return Ok(NO_DIFFERENCES_SQL.to_owned()); + } + let required_bytes = statements.iter().try_fold(0_usize, |total, statement| { + total.checked_add(statement.trim_end_matches(';').len() + 3) + }); + let Some(required_bytes) = required_bytes else { + return Err(schema_diff_resource_limit( + "The generated MySQL schema diff is too large", + )); + }; + if required_bytes > MAX_SCHEMA_DIFF_SQL_BYTES { + return Err(schema_diff_resource_limit( + "The generated MySQL schema diff is too large", + )); + } + let mut sql = String::with_capacity(required_bytes); + for statement in statements { + sql.push_str(statement.trim_end_matches(';')); + sql.push_str(";\n\n"); + } + Ok(sql) +} + +fn column_position(columns: &[ColumnDefinition], position: usize) -> String { + if position == 0 { + " FIRST".to_owned() + } else { + format!(" AFTER {}", quote_identifier(&columns[position - 1].name)) + } +} + +fn find_column<'a>(columns: &'a [ColumnDefinition], name: &str) -> Option<&'a ColumnDefinition> { + columns + .iter() + .find(|column| names_equal(&column.name, name)) +} + +fn find_index<'a>(indexes: &'a [IndexDefinition], name: &str) -> Option<&'a IndexDefinition> { + indexes.iter().find(|index| names_equal(&index.name, name)) +} + +fn find_foreign_key<'a>( + foreign_keys: &'a [ForeignKeyDefinition], + name: &str, +) -> Option<&'a ForeignKeyDefinition> { + foreign_keys + .iter() + .find(|foreign_key| names_equal(&foreign_key.name, name)) +} + +fn names_equal(left: &str, right: &str) -> bool { + left.eq_ignore_ascii_case(right) +} + +fn quote_identifier(value: &str) -> String { + format!("`{}`", value.replace('`', "``")) +} + +fn qualified_object(database_name: &str, object_name: &str) -> String { + format!( + "{}.{}", + quote_identifier(database_name), + quote_identifier(object_name) + ) +} + +fn canonicalize_column_definition(definition: &str) -> String { + let bytes = definition.as_bytes(); + let mut canonical = String::with_capacity(definition.len()); + let mut copied_until = 0_usize; + let mut quote = None; + let mut index = 0_usize; + while index < bytes.len() { + if let Some(delimiter) = quote { + if matches!(delimiter, b'\'' | b'"') && bytes[index] == b'\\' { + index = (index + 2).min(bytes.len()); + continue; + } + if bytes[index] == delimiter { + if bytes.get(index + 1) == Some(&delimiter) { + index += 2; + continue; + } + quote = None; + } + index += 1; + continue; + } + if matches!(bytes[index], b'\'' | b'"' | b'`') { + quote = Some(bytes[index]); + index += 1; + continue; + } + let Some(after_character) = keyword_end(definition, index, "CHARACTER") else { + index += 1; + continue; + }; + let after_character = skip_ascii_whitespace(bytes, after_character); + let Some(after_set) = keyword_end(definition, after_character, "SET") else { + index += 1; + continue; + }; + let charset_start = skip_ascii_whitespace(bytes, after_set); + let mut charset_end = charset_start; + while bytes + .get(charset_end) + .is_some_and(|byte| byte.is_ascii_alphanumeric() || *byte == b'_') + { + charset_end += 1; + } + if charset_end == charset_start { + index += 1; + continue; + } + let collate_start = skip_ascii_whitespace(bytes, charset_end); + if keyword_end(definition, collate_start, "COLLATE").is_none() { + index += 1; + continue; + } + canonical.push_str(&definition[copied_until..index]); + copied_until = collate_start; + index = collate_start; + } + canonical.push_str(&definition[copied_until..]); + canonical +} + +fn keyword_end(value: &str, start: usize, keyword: &str) -> Option { + let end = start.checked_add(keyword.len())?; + let candidate = value.get(start..end)?; + if !candidate.eq_ignore_ascii_case(keyword) { + return None; + } + let bytes = value.as_bytes(); + let boundary_before = start == 0 + || bytes + .get(start - 1) + .is_none_or(|byte| !byte.is_ascii_alphanumeric() && *byte != b'_'); + let boundary_after = bytes + .get(end) + .is_none_or(|byte| !byte.is_ascii_alphanumeric() && *byte != b'_'); + (boundary_before && boundary_after).then_some(end) +} + +fn skip_ascii_whitespace(bytes: &[u8], mut index: usize) -> usize { + while bytes.get(index).is_some_and(u8::is_ascii_whitespace) { + index += 1; + } + index +} + +fn parse_quoted_identifier(input: &str) -> Option<(String, usize)> { + let mut characters = input.char_indices().peekable(); + if characters.next()?.1 != '`' { + return None; + } + let mut value = String::new(); + while let Some((index, character)) = characters.next() { + if character != '`' { + value.push(character); + continue; + } + if characters.peek().is_some_and(|(_, next)| *next == '`') { + characters.next(); + value.push('`'); + continue; + } + return Some((value, index + character.len_utf8())); + } + None +} + +fn has_ascii_prefix(value: &str, prefix: &str) -> bool { + value + .get(..prefix.len()) + .is_some_and(|candidate| candidate.eq_ignore_ascii_case(prefix)) +} + +fn strip_ascii_prefix<'a>(value: &'a str, prefix: &str) -> Option<&'a str> { + has_ascii_prefix(value, prefix).then(|| &value[prefix.len()..]) +} + +fn create_table_parts(ddl: &str) -> Option<(&str, &str, &str)> { + let bytes = ddl.as_bytes(); + let mut quote = None; + let mut open = None; + let mut depth = 0_usize; + let mut index = 0_usize; + while index < bytes.len() { + if let Some(delimiter) = quote { + if matches!(delimiter, b'\'' | b'"') && bytes[index] == b'\\' { + index = (index + 2).min(bytes.len()); + continue; + } + if bytes[index] == delimiter { + if bytes.get(index + 1) == Some(&delimiter) { + index += 2; + continue; + } + quote = None; + } + index += 1; + continue; + } + match bytes[index] { + b'\'' | b'"' | b'`' => quote = Some(bytes[index]), + b'(' => { + depth += 1; + if open.is_none() { + open = Some(index + 1); + } + } + b')' if depth > 0 => { + depth -= 1; + if depth == 0 { + return open.map(|start| (&ddl[..start], &ddl[start..index], &ddl[index..])); + } + } + _ => {} + } + index += 1; + } + None +} + +fn split_top_level_definitions(body: &str) -> Vec<&str> { + let bytes = body.as_bytes(); + let mut definitions = Vec::new(); + let mut quote = None; + let mut depth = 0_usize; + let mut start = 0_usize; + let mut index = 0_usize; + while index < bytes.len() { + if let Some(delimiter) = quote { + if matches!(delimiter, b'\'' | b'"') && bytes[index] == b'\\' { + index = (index + 2).min(bytes.len()); + continue; + } + if bytes[index] == delimiter { + if bytes.get(index + 1) == Some(&delimiter) { + index += 2; + continue; + } + quote = None; + } + index += 1; + continue; + } + match bytes[index] { + b'\'' | b'"' | b'`' => quote = Some(bytes[index]), + b'(' => depth += 1, + b')' if depth > 0 => depth -= 1, + b',' if depth == 0 => { + definitions.push(&body[start..index]); + start = index + 1; + } + _ => {} + } + index += 1; + } + definitions.push(&body[start..]); + definitions +} + +fn schema_diff_query_error(_error: MysqlError) -> AppError { + AppError::unavailable( + "mysql_schema_diff_query_failed", + "The MySQL schema diff metadata could not be loaded", + ) +} + +fn malformed_show_create() -> AppError { + AppError::unavailable( + "mysql_schema_diff_metadata_invalid", + "MySQL returned a table definition that could not be compared safely", + ) +} + +fn malformed_view_definition() -> AppError { + AppError::unavailable( + "mysql_schema_diff_metadata_invalid", + "MySQL returned a view definition that could not be compared safely", + ) +} + +fn invalid_schema_diff(message: impl Into) -> AppError { + AppError::invalid("invalid_community_schema_diff_request", message) +} + +fn schema_diff_resource_limit(message: &'static str) -> AppError { + AppError::new( + AppErrorKind::ResourceExhausted, + ApiError::new("mysql_schema_diff_resource_limit", message), + ) +} + +#[cfg(test)] +mod tests { + use chat2db_contract::{CommunitySchemaDiffEndpoint, CommunitySchemaDiffRequest}; + + use crate::Application; + + use super::{ + NO_DIFFERENCES_SQL, SchemaSnapshot, ViewSnapshot, build_schema_diff, + canonicalize_column_definition, parse_table_snapshot, rewrite_qualified_catalog, + }; + + fn snapshot(database_name: &str) -> SchemaSnapshot { + SchemaSnapshot { + database_name: database_name.to_owned(), + ..SchemaSnapshot::default() + } + } + + #[test] + fn no_difference_uses_the_pinned_community_comment() { + let mut snapshot = snapshot("same_db"); + snapshot.tables.insert( + "items".to_owned(), + parse_table_snapshot( + "items", + "CREATE TABLE `items` (\n `id` bigint NOT NULL,\n PRIMARY KEY (`id`)\n) ENGINE=InnoDB", + ) + .expect("table ddl"), + ); + + assert_eq!( + build_schema_diff(&snapshot, &snapshot).expect("no diff"), + NO_DIFFERENCES_SQL + ); + } + + #[test] + fn table_column_order_and_index_changes_generate_target_migration_sql() { + let mut source = snapshot("source_db"); + source.tables.insert( + "added".to_owned(), + parse_table_snapshot( + "added", + "CREATE TABLE `added` (\n `id` bigint NOT NULL,\n PRIMARY KEY (`id`)\n) ENGINE=InnoDB", + ) + .expect("added table"), + ); + source.tables.insert( + "changed".to_owned(), + parse_table_snapshot( + "changed", + "CREATE TABLE `changed` (\n `title` varchar(100) NOT NULL,\n `id` bigint NOT NULL,\n `new_col` enum('a','b,c') DEFAULT NULL,\n PRIMARY KEY (`id`),\n KEY `idx_title` (`title`)\n) ENGINE=InnoDB", + ) + .expect("source changed table"), + ); + + let mut target = snapshot("target_db"); + target.tables.insert( + "changed".to_owned(), + parse_table_snapshot( + "changed", + "CREATE TABLE `changed` (\n `id` int NOT NULL,\n `old_col` varchar(10) DEFAULT NULL,\n `title` varchar(20) DEFAULT NULL,\n PRIMARY KEY (`id`),\n KEY `idx_old` (`old_col`)\n) ENGINE=InnoDB", + ) + .expect("target changed table"), + ); + target.tables.insert( + "removed".to_owned(), + parse_table_snapshot( + "removed", + "CREATE TABLE `removed` (\n `id` bigint NOT NULL\n) ENGINE=InnoDB", + ) + .expect("removed table"), + ); + + let sql = build_schema_diff(&source, &target).expect("schema diff"); + assert!(sql.contains("CREATE TABLE `target_db`.`added`")); + assert!(sql.contains("ALTER TABLE `target_db`.`changed` DROP INDEX `idx_old`;")); + assert!(sql.contains("ALTER TABLE `target_db`.`changed` DROP COLUMN `old_col`;")); + assert!(sql.contains("MODIFY COLUMN `title` varchar(100) NOT NULL FIRST;")); + assert!(sql.contains("MODIFY COLUMN `id` bigint NOT NULL AFTER `title`;")); + assert!(sql.contains("ADD COLUMN `new_col` enum('a','b,c') DEFAULT NULL AFTER `id`;")); + assert!(sql.contains("ALTER TABLE `target_db`.`changed` ADD KEY `idx_title` (`title`);")); + assert!(sql.contains("DROP TABLE `target_db`.`removed`;")); + } + + #[test] + fn foreign_keys_and_table_options_generate_ordered_migration_sql() { + let mut source = snapshot("source_db"); + source.tables.insert( + "relations".to_owned(), + parse_table_snapshot( + "relations", + "CREATE TABLE `relations` (\n `id` bigint NOT NULL AUTO_INCREMENT,\n `parent_id` bigint NOT NULL,\n PRIMARY KEY (`id`),\n KEY `idx_parent` (`parent_id`),\n CONSTRAINT `fk_parent` FOREIGN KEY (`parent_id`) REFERENCES `parents` (`id`) ON DELETE CASCADE\n) ENGINE=InnoDB AUTO_INCREMENT=42 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='source option'", + ) + .expect("source relation table"), + ); + + let mut target = snapshot("target_db"); + target.tables.insert( + "relations".to_owned(), + parse_table_snapshot( + "relations", + "CREATE TABLE `relations` (\n `id` bigint NOT NULL AUTO_INCREMENT,\n `parent_id` bigint NOT NULL,\n PRIMARY KEY (`id`),\n KEY `idx_parent` (`parent_id`),\n CONSTRAINT `fk_parent_old` FOREIGN KEY (`parent_id`) REFERENCES `parents` (`id`)\n) ENGINE=MyISAM AUTO_INCREMENT=7 DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='target option'", + ) + .expect("target relation table"), + ); + + let sql = build_schema_diff(&source, &target).expect("schema diff"); + let drop_position = sql + .find("DROP FOREIGN KEY `fk_parent_old`") + .expect("old foreign key drop"); + let options_position = sql + .find("ENGINE=InnoDB, DEFAULT CHARACTER SET=utf8mb4, COLLATE=utf8mb4_unicode_ci, COMMENT='source option'") + .expect("table option changes"); + let add_position = sql + .find("ADD CONSTRAINT `fk_parent` FOREIGN KEY") + .expect("new foreign key add"); + assert!(drop_position < options_position); + assert!(options_position < add_position); + } + + #[test] + fn new_tables_create_before_their_foreign_keys_are_added() { + let mut source = snapshot("source_db"); + source.tables.insert( + "children".to_owned(), + parse_table_snapshot( + "children", + "CREATE TABLE `children` (\n `id` bigint NOT NULL,\n `parent_id` bigint NOT NULL,\n PRIMARY KEY (`id`),\n CONSTRAINT `fk_child_parent` FOREIGN KEY (`parent_id`) REFERENCES `parents` (`id`)\n) ENGINE=InnoDB", + ) + .expect("child table"), + ); + + let sql = build_schema_diff(&source, &snapshot("target_db")).expect("schema diff"); + let create_end = sql.find("ENGINE=InnoDB;").expect("table creation"); + let add_position = sql + .find("ALTER TABLE `target_db`.`children` ADD CONSTRAINT `fk_child_parent`") + .expect("foreign key add"); + assert!(create_end < add_position); + assert!(!sql[..create_end].contains("CONSTRAINT `fk_child_parent`")); + } + + #[test] + fn referenced_table_changes_temporarily_rebuild_unchanged_foreign_keys() { + let child_ddl = "CREATE TABLE `children` (\n `id` bigint NOT NULL,\n `parent_id` bigint NOT NULL,\n PRIMARY KEY (`id`),\n KEY `idx_parent` (`parent_id`),\n CONSTRAINT `fk_child_parent` FOREIGN KEY (`parent_id`) REFERENCES `parents` (`id`)\n) ENGINE=InnoDB"; + let mut source = snapshot("source_db"); + source.tables.insert( + "parents".to_owned(), + parse_table_snapshot( + "parents", + "CREATE TABLE `parents` (\n `id` bigint NOT NULL,\n PRIMARY KEY (`id`)\n) ENGINE=InnoDB", + ) + .expect("source parent"), + ); + source.tables.insert( + "children".to_owned(), + parse_table_snapshot("children", child_ddl).expect("source child"), + ); + + let mut target = snapshot("target_db"); + target.tables.insert( + "parents".to_owned(), + parse_table_snapshot( + "parents", + "CREATE TABLE `parents` (\n `id` int NOT NULL,\n PRIMARY KEY (`id`)\n) ENGINE=InnoDB", + ) + .expect("target parent"), + ); + target.tables.insert( + "children".to_owned(), + parse_table_snapshot("children", child_ddl).expect("target child"), + ); + + let sql = build_schema_diff(&source, &target).expect("schema diff"); + let drop_position = sql + .find("ALTER TABLE `target_db`.`children` DROP FOREIGN KEY `fk_child_parent`") + .expect("foreign key drop"); + let parent_position = sql + .find("ALTER TABLE `target_db`.`parents` MODIFY COLUMN `id` bigint NOT NULL FIRST") + .expect("parent modification"); + let add_position = sql + .find("ALTER TABLE `target_db`.`children` ADD CONSTRAINT `fk_child_parent`") + .expect("foreign key restoration"); + assert!(drop_position < parent_position); + assert!(parent_position < add_position); + } + + #[test] + fn views_are_dropped_created_replaced_and_retargeted() { + let mut source = SchemaSnapshot { + database_name: "source_db".to_owned(), + ..SchemaSnapshot::default() + }; + source.views.insert( + "added_view".to_owned(), + ViewSnapshot { + definition: "select `source_db`.`items`.`id` AS `id` from `source_db`.`items`" + .to_owned(), + }, + ); + source.views.insert( + "changed_view".to_owned(), + ViewSnapshot { + definition: + "select `source_db`.`items`.`new_value` AS `new_value` from `source_db`.`items`" + .to_owned(), + }, + ); + + let mut target = SchemaSnapshot { + database_name: "target_db".to_owned(), + ..SchemaSnapshot::default() + }; + target.views.insert( + "changed_view".to_owned(), + ViewSnapshot { + definition: + "select `target_db`.`items`.`old_value` AS `old_value` from `target_db`.`items`" + .to_owned(), + }, + ); + target.views.insert( + "removed_view".to_owned(), + ViewSnapshot { + definition: "select 1 AS `value`".to_owned(), + }, + ); + + let sql = build_schema_diff(&source, &target).expect("view diff"); + assert!(sql.contains("DROP VIEW `target_db`.`removed_view`;")); + assert!(sql.contains( + "CREATE VIEW `target_db`.`added_view` AS select `target_db`.`items`.`id` AS `id` from `target_db`.`items`;" + )); + assert!(sql.contains( + "CREATE OR REPLACE VIEW `target_db`.`changed_view` AS select `target_db`.`items`.`new_value` AS `new_value` from `target_db`.`items`;" + )); + assert!(!sql.contains("`source_db`.")); + assert_eq!( + rewrite_qualified_catalog( + "select '`source_db`.`items`' AS `literal`, `source_db`.`items`.`id` from `source_db`.`items`", + "source_db", + "target_db" + ), + "select '`source_db`.`items`' AS `literal`, `target_db`.`items`.`id` from `target_db`.`items`" + ); + } + + #[test] + fn schema_qualified_foreign_keys_are_retargeted_with_their_owner() { + let mut source = snapshot("source_db"); + source.tables.insert( + "children".to_owned(), + parse_table_snapshot( + "children", + "CREATE TABLE `children` (\n `id` bigint NOT NULL,\n `parent_id` bigint NOT NULL,\n PRIMARY KEY (`id`),\n CONSTRAINT `fk_parent` FOREIGN KEY (`parent_id`) REFERENCES `source_db`.`parents` (`id`)\n) ENGINE=InnoDB", + ) + .expect("qualified foreign key"), + ); + + let sql = build_schema_diff(&source, &snapshot("target_db")).expect("schema diff"); + assert!(sql.contains("ALTER TABLE `target_db`.`children` ADD CONSTRAINT `fk_parent`")); + assert!(sql.contains("REFERENCES `target_db`.`parents` (`id`)")); + assert!(!sql.contains("`source_db`.")); + } + + #[test] + fn views_are_topologically_ordered_and_cycles_fail_closed() { + let mut source = snapshot("source_db"); + source.views.insert( + "a_child".to_owned(), + ViewSnapshot { + definition: "select * from `source_db`.`z_parent`".to_owned(), + }, + ); + source.views.insert( + "z_parent".to_owned(), + ViewSnapshot { + definition: "select 1 AS `value`".to_owned(), + }, + ); + let sql = build_schema_diff(&source, &snapshot("target_db")).expect("ordered views"); + let parent = sql + .find("CREATE VIEW `target_db`.`z_parent`") + .expect("parent view"); + let child = sql + .find("CREATE VIEW `target_db`.`a_child`") + .expect("child view"); + assert!(parent < child); + + source.views.get_mut("z_parent").expect("parent").definition = + "select * from `source_db`.`a_child`".to_owned(); + let error = build_schema_diff(&source, &snapshot("target_db")) + .expect_err("view dependency cycle must fail closed"); + assert_eq!( + error.api_error().code, + "mysql_schema_diff_view_dependency_cycle" + ); + } + + #[test] + fn runtime_auto_increment_counter_is_not_schema_state() { + let source_table = parse_table_snapshot( + "items", + "CREATE TABLE `items` (\n `id` bigint NOT NULL AUTO_INCREMENT,\n PRIMARY KEY (`id`)\n) ENGINE=InnoDB AUTO_INCREMENT=42", + ) + .expect("source table"); + assert!( + !source_table + .create_sql_without_foreign_keys + .contains("AUTO_INCREMENT=42") + ); + let target_table = parse_table_snapshot( + "items", + "CREATE TABLE `items` (\n `id` bigint NOT NULL AUTO_INCREMENT,\n PRIMARY KEY (`id`)\n) ENGINE=InnoDB AUTO_INCREMENT=7", + ) + .expect("target table"); + let mut source = snapshot("source_db"); + source.tables.insert("items".to_owned(), source_table); + let mut target = snapshot("target_db"); + target.tables.insert("items".to_owned(), target_table); + assert_eq!( + build_schema_diff(&source, &target).expect("runtime counters are ignored"), + NO_DIFFERENCES_SQL + ); + } + + #[test] + fn primary_key_replacement_uses_one_alter_statement() { + let mut source = snapshot("source_db"); + source.tables.insert( + "items".to_owned(), + parse_table_snapshot( + "items", + "CREATE TABLE `items` (\n `id` bigint NOT NULL AUTO_INCREMENT,\n `code` bigint NOT NULL,\n PRIMARY KEY (`code`),\n KEY `idx_id` (`id`)\n) ENGINE=InnoDB", + ) + .expect("source table"), + ); + let mut target = snapshot("target_db"); + target.tables.insert( + "items".to_owned(), + parse_table_snapshot( + "items", + "CREATE TABLE `items` (\n `id` bigint NOT NULL AUTO_INCREMENT,\n `code` bigint NOT NULL,\n PRIMARY KEY (`id`)\n) ENGINE=InnoDB", + ) + .expect("target table"), + ); + + let sql = build_schema_diff(&source, &target).expect("primary key diff"); + assert!(sql.contains( + "ALTER TABLE `target_db`.`items` DROP PRIMARY KEY, ADD PRIMARY KEY (`code`);" + )); + assert!(!sql.contains("DROP PRIMARY KEY;")); + } + + #[test] + fn case_only_object_conflicts_fail_closed_on_case_insensitive_targets() { + let table = || { + parse_table_snapshot( + "items", + "CREATE TABLE `items` (\n `id` bigint NOT NULL,\n PRIMARY KEY (`id`)\n) ENGINE=InnoDB", + ) + .expect("table") + }; + let mut source = snapshot("source_db"); + source.tables.insert("Items".to_owned(), table()); + let mut target = snapshot("target_db"); + target.lower_case_table_names = 1; + target.tables.insert("items".to_owned(), table()); + + let error = build_schema_diff(&source, &target) + .expect_err("case-only conflict must not produce destructive SQL"); + assert_eq!(error.api_error().code, "mysql_schema_diff_case_conflict"); + } + + #[test] + fn show_create_parser_keeps_generated_and_functional_index_commas_intact() { + let table = parse_table_snapshot( + "expressions", + "CREATE TABLE `expressions` (\n `id` bigint NOT NULL,\n `label` varchar(20) GENERATED ALWAYS AS (concat('a,b',`id`)) STORED,\n KEY `idx_expr` ((lower(`label`)))\n) ENGINE=InnoDB", + ) + .expect("complex table ddl"); + + assert_eq!(table.columns.len(), 2); + assert_eq!(table.indexes.len(), 1); + assert!(table.columns[1].sql.contains("concat('a,b',`id`)")); + assert!(table.indexes[0].sql.contains("(lower(`label`))")); + } + + #[test] + fn redundant_character_set_before_collation_has_one_stable_comparison_form() { + assert_eq!( + canonicalize_column_definition( + "`label` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL" + ), + canonicalize_column_definition( + "`label` varchar(64) COLLATE utf8mb4_unicode_ci NOT NULL" + ) + ); + assert_ne!( + canonicalize_column_definition( + "`label` varchar(64) COLLATE utf8mb4_unicode_ci NOT NULL" + ), + canonicalize_column_definition( + "`label` varchar(64) COLLATE utf8mb4_0900_ai_ci NOT NULL" + ) + ); + } + + #[tokio::test] + async fn invalid_request_fails_before_storage_or_java_access() { + let error = Application::new() + .preview_mysql_schema_diff(&CommunitySchemaDiffRequest { + source: CommunitySchemaDiffEndpoint::default(), + target: CommunitySchemaDiffEndpoint::default(), + }) + .await + .expect_err("missing source endpoint must fail"); + + assert_eq!( + error.api_error().code, + "invalid_community_schema_diff_request" + ); + } +} diff --git a/crates/chat2db-core/src/mysql_workspace.rs b/crates/chat2db-core/src/mysql_workspace.rs new file mode 100644 index 0000000..53300d8 --- /dev/null +++ b/crates/chat2db-core/src/mysql_workspace.rs @@ -0,0 +1,125 @@ +use chat2db_contract::{ + CommunityErModel, CommunityErPositionRequest, CommunityErQueryRequest, + CommunityPinnedTableList, CommunityPinnedTableRequest, +}; + +use crate::{AppError, Application, native_mysql, storage_call}; + +impl Application { + /// Pins one `MySQL` table in the local workspace. + /// + /// # Errors + /// + /// Returns datasource, validation, availability, or storage failures. + pub async fn pin_community_mysql_table( + &self, + request: CommunityPinnedTableRequest, + ) -> Result<(), AppError> { + self.get_datasource(&request.data_source_id).await?; + let storage = self.require_storage()?; + storage_call(move || { + storage.pin_mysql_table( + &request.data_source_id, + &request.database_name, + &request.schema_name, + &request.table_name, + ) + }) + .await + } + + /// Removes one pinned `MySQL` table from the local workspace. + /// + /// # Errors + /// + /// Returns datasource, validation, availability, or storage failures. + pub async fn unpin_community_mysql_table( + &self, + request: CommunityPinnedTableRequest, + ) -> Result<(), AppError> { + self.get_datasource(&request.data_source_id).await?; + let storage = self.require_storage()?; + storage_call(move || { + storage.unpin_mysql_table( + &request.data_source_id, + &request.database_name, + &request.schema_name, + &request.table_name, + ) + }) + .await + } + + /// Lists pinned `MySQL` table names for one database/schema scope. + /// + /// # Errors + /// + /// Returns datasource, validation, availability, or storage failures. + pub async fn list_community_mysql_pinned_tables( + &self, + request: CommunityPinnedTableRequest, + ) -> Result { + self.get_datasource(&request.data_source_id).await?; + let storage = self.require_storage()?; + storage_call(move || { + storage + .list_mysql_pinned_tables( + &request.data_source_id, + &request.database_name, + &request.schema_name, + ) + .map(|items| CommunityPinnedTableList { items }) + }) + .await + } + + /// Loads native `MySQL` ER metadata and the last persisted canvas layout. + /// + /// # Errors + /// + /// Returns datasource, `MySQL` metadata, validation, availability, or storage failures. + pub async fn community_mysql_er_model( + &self, + request: CommunityErQueryRequest, + ) -> Result { + let tables = native_mysql::load_er_tables( + self, + &request.data_source_id, + &request.database_name, + &request.schema_name, + ) + .await?; + let storage = self.require_storage()?; + let position = storage_call(move || { + storage.mysql_er_position( + &request.data_source_id, + &request.database_name, + &request.schema_name, + ) + }) + .await?; + Ok(CommunityErModel { tables, position }) + } + + /// Persists the Community ER canvas layout using a true upsert. + /// + /// # Errors + /// + /// Returns datasource, validation, availability, or storage failures. + pub async fn save_community_mysql_er_position( + &self, + request: CommunityErPositionRequest, + ) -> Result<(), AppError> { + self.get_datasource(&request.data_source_id).await?; + let storage = self.require_storage()?; + storage_call(move || { + storage.save_mysql_er_position( + &request.data_source_id, + &request.database_name, + &request.schema_name, + &request.position, + ) + }) + .await + } +} diff --git a/crates/chat2db-core/src/native_mysql.rs b/crates/chat2db-core/src/native_mysql.rs index 28ab7db..daf423f 100644 --- a/crates/chat2db-core/src/native_mysql.rs +++ b/crates/chat2db-core/src/native_mysql.rs @@ -1,33 +1,38 @@ use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD}; use chat2db_contract::{ - ApiError, CommunityDatabase, CommunityDatabaseList, CommunityForeignKey, - CommunityForeignKeyList, CommunityFunction, CommunityFunctionList, CommunityFunctionParameter, - CommunityFunctionParameterList, CommunityPrimaryKey, CommunityPrimaryKeyList, - CommunityProcedure, CommunityProcedureList, CommunityProcedureParameter, - CommunityProcedureParameterList, CommunityRoutineInvocationPreview, CommunitySchemaList, - CommunityTable, CommunityTableColumn, CommunityTableColumnList, CommunityTableIndex, - CommunityTableIndexColumn, CommunityTableIndexList, CommunityTableList, - CommunityTablePreviewAccepted, CommunityTrigger, CommunityTriggerList, CommunityViewList, - DatasourceConnection, JdbcValue, JdbcValueType, PreviewCommunityRoutineInvocationRequest, - QueryLimits, ResultColumn, ResultMetadata, ResultRow, StartCommunityTablePreviewRequest, - StartQueryRequest, + ApiError, CommunityDatabase, CommunityDatabaseList, CommunityErColumn, CommunityErForeignKey, + CommunityErTable, CommunityForeignKey, CommunityForeignKeyList, CommunityFunction, + CommunityFunctionList, CommunityFunctionParameter, CommunityFunctionParameterList, + CommunityPrimaryKey, CommunityPrimaryKeyList, CommunityProcedure, CommunityProcedureList, + CommunityProcedureParameter, CommunityProcedureParameterList, + CommunityRoutineInvocationPreview, CommunityRoutineMigrationExecution, + CommunityRoutineMigrationRequest, CommunitySchemaList, CommunityTable, CommunityTableColumn, + CommunityTableColumnList, CommunityTableIndex, CommunityTableIndexColumn, + CommunityTableIndexList, CommunityTableList, CommunityTablePreviewAccepted, CommunityTrigger, + CommunityTriggerList, CommunityViewList, DatasourceConnection, JdbcValue, JdbcValueType, + PreviewCommunityRoutineInvocationRequest, QueryLimits, ResultColumn, ResultMetadata, ResultRow, + StartCommunityTablePreviewRequest, StartQueryRequest, }; use chat2db_engine_protocol::wire; -use chat2db_java_bridge::QueryOptions; +use chat2db_java_bridge::{JdbcParameter, JdbcValue as BridgeJdbcValue, QueryOptions}; use chat2db_storage::Storage; +use chrono::{DateTime, Datelike, NaiveDate, NaiveDateTime, Timelike, Utc}; use mysql_async::{ - Column, Conn, Error as MysqlError, Opts, OptsBuilder, Row, SslOpts, Value, + Column, Conn, DriverError, Error as MysqlError, Opts, OptsBuilder, Params, Row, SslOpts, Value, consts::{ColumnFlags, ColumnType}, prelude::{FromRow, FromValue, Queryable}, }; use prost::Message; +use sqlparser::{ast::Statement, dialect::MySqlDialect, parser::Parser}; use std::{ collections::HashMap, future::Future, mem::size_of, + ops::{Deref, DerefMut}, time::{Duration, Instant}, }; use tokio::sync::watch; +use tokio_util::sync::CancellationToken; use url::Url; use crate::{ @@ -35,8 +40,10 @@ use crate::{ datasource_session::{ResolvedDatasourceConnection, resolve_datasource_connection}, operation::CancellationRequest, query::{ - MysqlConsoleRequest, MysqlConsoleResult, PreparedQuery, QueryTaskError, RetainedWriter, + DatabaseWriteError, MysqlConsoleRequest, MysqlConsoleResult, PreparedQuery, QueryTaskError, + RetainedWriter, }, + ssh::{SshTunnel, SshTunnelIdentity, mysql_target, rewrite_mysql_target}, }; const MYSQL_SCHEME: &str = "mysql://"; @@ -52,6 +59,7 @@ const MAX_RESULT_BYTES: u64 = wire::JdbcResultByteLimit::MaxResultBytes as u64; const MAX_BATCH_ROWS: u32 = wire::JdbcProtocolLimit::MaxBatchRows as u32; const MAX_BATCH_BYTES: u32 = wire::JdbcProtocolLimit::MaxBatchBytes as u32; const MAX_COLUMNS: usize = wire::JdbcProtocolLimit::MaxColumns as usize; +const MAX_PARAMETERS: usize = wire::JdbcProtocolLimit::MaxParameters as usize; const MAX_SQL_BYTES: usize = wire::JdbcProtocolLimit::MaxSqlBytes as usize; const MAX_SCALAR_BYTES: usize = wire::JdbcProtocolLimit::MaxScalarBytes as usize; const MAX_CONSOLE_VALUE_BYTES: usize = 32 * 1024 * 1024; @@ -59,6 +67,48 @@ const MAX_CONSOLE_RESULT_BYTES: u64 = DEFAULT_RESULT_BYTES; const MAX_CONSOLE_STATEMENTS: usize = 1_000; const MAX_IDENTIFIER_BYTES: usize = 256; const MAX_CONSOLE_PAGE_SIZE: u32 = 10_000; +const ER_TABLE_QUERY: &str = "SELECT TABLE_SCHEMA, TABLE_NAME, TABLE_TYPE, \ + COALESCE(TABLE_COMMENT, ''), COALESCE(ENGINE, ''), \ + COALESCE(TABLE_COLLATION, ''), CAST(AUTO_INCREMENT AS CHAR), \ + CAST(TABLE_ROWS AS CHAR), CAST(DATA_LENGTH AS CHAR), \ + DATE_FORMAT(CREATE_TIME, '%Y-%m-%dT%H:%i:%s'), \ + DATE_FORMAT(UPDATE_TIME, '%Y-%m-%dT%H:%i:%s') \ + FROM information_schema.TABLES \ + WHERE TABLE_SCHEMA = ? AND TABLE_TYPE = 'BASE TABLE' \ + ORDER BY TABLE_NAME"; +const ER_COLUMN_QUERY: &str = "SELECT c.TABLE_NAME AS table_name, c.COLUMN_NAME AS name, \ + c.DATA_TYPE AS data_type, c.COLUMN_DEFAULT AS default_value, \ + COALESCE(c.EXTRA, '') AS extra, \ + COALESCE(c.COLUMN_COMMENT, '') AS comment, \ + COALESCE(c.COLUMN_KEY, '') AS column_key, \ + c.IS_NULLABLE AS is_nullable, \ + c.ORDINAL_POSITION AS ordinal_position, \ + c.NUMERIC_SCALE AS numeric_scale, \ + c.COLUMN_TYPE AS column_definition, \ + c.CHARACTER_SET_NAME AS charset, c.COLLATION_NAME AS collation, \ + CAST(COALESCE(pk.SEQ_IN_INDEX, 0) AS SIGNED) AS primary_key_order \ + FROM information_schema.COLUMNS AS c \ + LEFT JOIN information_schema.STATISTICS AS pk \ + ON pk.TABLE_SCHEMA = c.TABLE_SCHEMA \ + AND pk.TABLE_NAME = c.TABLE_NAME \ + AND pk.COLUMN_NAME = c.COLUMN_NAME \ + AND pk.INDEX_NAME = 'PRIMARY' \ + WHERE c.TABLE_SCHEMA = ? \ + ORDER BY c.TABLE_NAME, c.ORDINAL_POSITION"; +const ER_FOREIGN_KEY_QUERY: &str = "SELECT kcu.REFERENCED_TABLE_SCHEMA, \ + kcu.REFERENCED_TABLE_NAME, kcu.REFERENCED_COLUMN_NAME, \ + kcu.TABLE_SCHEMA, kcu.TABLE_NAME, kcu.COLUMN_NAME, \ + kcu.ORDINAL_POSITION, rc.UPDATE_RULE, rc.DELETE_RULE, \ + kcu.CONSTRAINT_NAME, rc.UNIQUE_CONSTRAINT_NAME \ + FROM information_schema.KEY_COLUMN_USAGE kcu \ + JOIN information_schema.REFERENTIAL_CONSTRAINTS rc \ + ON rc.CONSTRAINT_SCHEMA = kcu.CONSTRAINT_SCHEMA \ + AND rc.CONSTRAINT_NAME = kcu.CONSTRAINT_NAME \ + AND rc.TABLE_NAME = kcu.TABLE_NAME \ + WHERE kcu.TABLE_SCHEMA = ? \ + AND kcu.REFERENCED_TABLE_NAME IS NOT NULL \ + ORDER BY kcu.TABLE_NAME, kcu.CONSTRAINT_NAME, \ + kcu.ORDINAL_POSITION"; type TableRow = ( String, String, @@ -89,6 +139,24 @@ struct ColumnRow { collation: Option, primary_key_order: i32, } +#[derive(FromRow)] +#[mysql(crate_name = "mysql_async")] +struct ErColumnRow { + table_name: String, + name: String, + data_type: String, + default_value: Option, + extra: String, + comment: String, + column_key: String, + is_nullable: String, + ordinal_position: i32, + numeric_scale: Option, + column_definition: String, + charset: Option, + collation: Option, + primary_key_order: i32, +} type IndexRow = ( String, String, @@ -162,21 +230,81 @@ struct RoutineInvocationParameter { ordinal_position: i32, } +#[derive(Debug, Clone, PartialEq, Eq)] +struct RoutineMigrationPlan { + routine_type: MysqlRoutineType, + database_name: String, + routine_name: String, + drop_sql: String, + create_sql: String, + preview_sql: String, +} + #[derive(Debug, PartialEq, Eq)] enum SqlToken { Word(String), Semicolon, } +pub(crate) struct ManagedMysqlConnection { + connection: Option, + tunnel: Option, +} + +impl ManagedMysqlConnection { + fn new(connection: Conn, tunnel: Option) -> Self { + Self { + connection: Some(connection), + tunnel, + } + } + + fn local_tunnel_port(&self) -> Option { + self.tunnel.as_ref().map(SshTunnel::local_port) + } +} + +impl Deref for ManagedMysqlConnection { + type Target = Conn; + + fn deref(&self) -> &Self::Target { + self.connection + .as_ref() + .expect("managed MySQL connection must exist until cleanup") + } +} + +impl DerefMut for ManagedMysqlConnection { + fn deref_mut(&mut self) -> &mut Self::Target { + self.connection + .as_mut() + .expect("managed MySQL connection must exist until cleanup") + } +} + +struct PreparedMysqlConnection { + options: Opts, + tunnel: Option, +} + pub(crate) fn is_mysql_database_type(database_type: &str) -> bool { database_type.trim().eq_ignore_ascii_case("mysql") } pub(crate) async fn test_connection(connection: &DatasourceConnection) -> Result<(), AppError> { + test_connection_with_local_port(connection) + .await + .map(|_| ()) +} + +pub(crate) async fn test_connection_with_local_port( + connection: &DatasourceConnection, +) -> Result, AppError> { let mut conn = open_connection(connection).await?; + let local_port = conn.local_tunnel_port(); let result = conn.ping().await.map_err(mysql_connection_error); - let close = conn.disconnect().await.map_err(mysql_connection_error); - result.and(close) + finish_connection(conn, result).await?; + Ok(local_port) } pub(crate) async fn list_databases( @@ -184,7 +312,7 @@ pub(crate) async fn list_databases( datasource_id: &str, ) -> Result { let resolved = resolve_native_connection(application, datasource_id).await?; - let mut conn = open_connection(&resolved.connection).await?; + let mut conn = open_resolved_connection(&resolved).await?; let result = metadata_query(conn.query::<(String, String, String), _>( "SELECT SCHEMA_NAME, DEFAULT_CHARACTER_SET_NAME, DEFAULT_COLLATION_NAME \ FROM information_schema.SCHEMATA ORDER BY SCHEMA_NAME", @@ -210,7 +338,7 @@ pub(crate) async fn list_schemas( datasource_id: &str, ) -> Result { let resolved = resolve_native_connection(application, datasource_id).await?; - let conn = open_connection(&resolved.connection).await?; + let conn = open_resolved_connection(&resolved).await?; finish_connection(conn, Ok(CommunitySchemaList::default())).await } @@ -227,7 +355,7 @@ pub(crate) async fn list_tables( )); } let resolved = resolve_native_connection(application, datasource_id).await?; - let mut conn = open_connection(&resolved.connection).await?; + let mut conn = open_resolved_connection(&resolved).await?; let query = "SELECT TABLE_SCHEMA, TABLE_NAME, TABLE_TYPE, COALESCE(TABLE_COMMENT, ''), \ COALESCE(ENGINE, ''), COALESCE(TABLE_COLLATION, ''), \ CAST(AUTO_INCREMENT AS CHAR), CAST(TABLE_ROWS AS CHAR), \ @@ -294,7 +422,7 @@ pub(crate) async fn list_columns( validate_metadata_identifier(database_name, "databaseName")?; validate_metadata_identifier(table_name, "tableName")?; let resolved = resolve_native_connection(application, datasource_id).await?; - let mut conn = open_connection(&resolved.connection).await?; + let mut conn = open_resolved_connection(&resolved).await?; let query = "SELECT c.COLUMN_NAME AS name, c.DATA_TYPE AS data_type, \ c.COLUMN_DEFAULT AS default_value, COALESCE(c.EXTRA, '') AS extra, \ COALESCE(c.COLUMN_COMMENT, '') AS comment, \ @@ -321,6 +449,110 @@ pub(crate) async fn list_columns( finish_connection(conn, result).await } +pub(crate) async fn load_er_tables( + application: &Application, + datasource_id: &str, + database_name: &str, + schema_name: &str, +) -> Result, AppError> { + validate_metadata_identifier(database_name, "databaseName")?; + let resolved = resolve_native_connection(application, datasource_id).await?; + let mut conn = open_resolved_connection(&resolved).await?; + let result = async { + let table_rows = metadata_query( + conn.exec::(ER_TABLE_QUERY, (database_name.to_owned(),)), + ) + .await?; + let column_rows = metadata_query( + conn.exec::(ER_COLUMN_QUERY, (database_name.to_owned(),)), + ) + .await?; + let foreign_key_rows = metadata_query( + conn.exec::(ER_FOREIGN_KEY_QUERY, (database_name.to_owned(),)), + ) + .await?; + + let mut columns_by_table = HashMap::>::new(); + for row in column_rows { + let ErColumnRow { + table_name, + name, + data_type, + default_value, + extra, + comment, + column_key, + is_nullable, + ordinal_position, + numeric_scale, + column_definition, + charset, + collation, + primary_key_order, + } = row; + let column = community_column( + database_name, + schema_name, + &table_name, + ColumnRow { + name, + data_type, + default_value, + extra, + comment, + column_key, + is_nullable, + ordinal_position, + numeric_scale, + column_definition, + charset, + collation, + primary_key_order, + }, + ); + columns_by_table + .entry(table_name) + .or_default() + .push(CommunityErColumn { + name: column.name, + column_type: column.column_type, + primary_key: column.primary_key.unwrap_or(false), + comment: column.comment, + }); + } + + let mut foreign_keys_by_table = HashMap::>::new(); + for row in foreign_key_rows { + let table_name = row.4.clone(); + foreign_keys_by_table + .entry(table_name) + .or_default() + .push(CommunityErForeignKey { + pk_table_name: row.1, + pk_column_name: row.2, + fk_table_name: row.4, + fk_column_name: row.5, + }); + } + + Ok(table_rows + .into_iter() + .map(|row| { + let table = community_table(row, schema_name); + let name = table.name; + CommunityErTable { + comment: table.comment, + column_list: columns_by_table.remove(&name).unwrap_or_default(), + foreign_key_list: foreign_keys_by_table.remove(&name).unwrap_or_default(), + name, + } + }) + .collect()) + } + .await; + finish_connection(conn, result).await +} + pub(crate) async fn validate_column_reorder( application: &Application, datasource_id: &str, @@ -334,7 +566,7 @@ pub(crate) async fn validate_column_reorder( validate_metadata_identifier(database_name, "databaseName")?; validate_metadata_identifier(table_name, "tableName")?; let resolved = resolve_native_connection(application, datasource_id).await?; - let mut conn = open_connection(&resolved.connection).await?; + let mut conn = open_resolved_connection(&resolved).await?; let query = "SELECT COLUMN_NAME, COLUMN_TYPE, COALESCE(EXTRA, ''), \ COALESCE(GENERATION_EXPRESSION, '') \ FROM information_schema.COLUMNS \ @@ -381,7 +613,7 @@ pub(crate) async fn list_indexes( validate_metadata_identifier(database_name, "databaseName")?; validate_metadata_identifier(table_name, "tableName")?; let resolved = resolve_native_connection(application, datasource_id).await?; - let mut conn = open_connection(&resolved.connection).await?; + let mut conn = open_resolved_connection(&resolved).await?; let query = "SELECT TABLE_SCHEMA, TABLE_NAME, NON_UNIQUE, INDEX_SCHEMA, INDEX_NAME, \ SEQ_IN_INDEX, COLUMN_NAME, COLLATION, CARDINALITY, SUB_PART, \ INDEX_TYPE, COALESCE(INDEX_COMMENT, '') \ @@ -407,7 +639,7 @@ pub(crate) async fn list_views( ) -> Result { validate_metadata_identifier(database_name, "databaseName")?; let resolved = resolve_native_connection(application, datasource_id).await?; - let mut conn = open_connection(&resolved.connection).await?; + let mut conn = open_resolved_connection(&resolved).await?; let query = "SELECT TABLE_SCHEMA, TABLE_NAME, TABLE_TYPE, COALESCE(TABLE_COMMENT, ''), \ COALESCE(ENGINE, ''), COALESCE(TABLE_COLLATION, ''), \ CAST(AUTO_INCREMENT AS CHAR), CAST(TABLE_ROWS AS CHAR), \ @@ -443,7 +675,7 @@ pub(crate) async fn get_view( let qualified_name = qualified_identifier(database_name, "databaseName", view_name, "viewName")?; let resolved = resolve_native_connection(application, datasource_id).await?; - let mut conn = open_connection(&resolved.connection).await?; + let mut conn = open_resolved_connection(&resolved).await?; let result = metadata_query(conn.query_first::(format!("SHOW CREATE VIEW {qualified_name}"))) .await @@ -475,7 +707,7 @@ pub(crate) async fn table_ddl( let qualified_name = qualified_identifier(database_name, "databaseName", table_name, "tableName")?; let resolved = resolve_native_connection(application, datasource_id).await?; - let mut conn = open_connection(&resolved.connection).await?; + let mut conn = open_resolved_connection(&resolved).await?; let result = metadata_query(conn.query_first::(format!("SHOW CREATE TABLE {qualified_name}"))) .await @@ -496,7 +728,7 @@ pub(crate) async fn list_imported_keys( validate_metadata_identifier(database_name, "databaseName")?; validate_metadata_identifier(table_name, "tableName")?; let resolved = resolve_native_connection(application, datasource_id).await?; - let mut conn = open_connection(&resolved.connection).await?; + let mut conn = open_resolved_connection(&resolved).await?; let query = "SELECT kcu.REFERENCED_TABLE_SCHEMA, kcu.REFERENCED_TABLE_NAME, \ kcu.REFERENCED_COLUMN_NAME, kcu.TABLE_SCHEMA, kcu.TABLE_NAME, \ kcu.COLUMN_NAME, kcu.ORDINAL_POSITION, rc.UPDATE_RULE, rc.DELETE_RULE, \ @@ -528,7 +760,7 @@ pub(crate) async fn list_exported_keys( validate_metadata_identifier(database_name, "databaseName")?; validate_metadata_identifier(table_name, "tableName")?; let resolved = resolve_native_connection(application, datasource_id).await?; - let mut conn = open_connection(&resolved.connection).await?; + let mut conn = open_resolved_connection(&resolved).await?; let query = "SELECT kcu.REFERENCED_TABLE_SCHEMA, kcu.REFERENCED_TABLE_NAME, \ kcu.REFERENCED_COLUMN_NAME, kcu.TABLE_SCHEMA, kcu.TABLE_NAME, \ kcu.COLUMN_NAME, kcu.ORDINAL_POSITION, rc.UPDATE_RULE, rc.DELETE_RULE, \ @@ -561,7 +793,7 @@ pub(crate) async fn list_primary_keys( validate_metadata_identifier(database_name, "databaseName")?; validate_metadata_identifier(table_name, "tableName")?; let resolved = resolve_native_connection(application, datasource_id).await?; - let mut conn = open_connection(&resolved.connection).await?; + let mut conn = open_resolved_connection(&resolved).await?; let query = "SELECT TABLE_SCHEMA, TABLE_NAME, COLUMN_NAME, CONSTRAINT_NAME \ FROM information_schema.KEY_COLUMN_USAGE \ WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? AND CONSTRAINT_NAME = 'PRIMARY' \ @@ -596,7 +828,7 @@ pub(crate) async fn list_functions( ) -> Result { validate_metadata_identifier(database_name, "databaseName")?; let resolved = resolve_native_connection(application, datasource_id).await?; - let mut conn = open_connection(&resolved.connection).await?; + let mut conn = open_resolved_connection(&resolved).await?; let query = "SELECT ROUTINE_SCHEMA, ROUTINE_NAME, SPECIFIC_NAME, \ COALESCE(ROUTINE_COMMENT, '') \ FROM information_schema.ROUTINES \ @@ -637,7 +869,7 @@ pub(crate) async fn get_function( let qualified_name = qualified_identifier(database_name, "databaseName", function_name, "functionName")?; let resolved = resolve_native_connection(application, datasource_id).await?; - let mut conn = open_connection(&resolved.connection).await?; + let mut conn = open_resolved_connection(&resolved).await?; let result = async { let metadata = metadata_query(conn.exec_first::<(String, String, String, String), _, _>( "SELECT ROUTINE_SCHEMA, ROUTINE_NAME, SPECIFIC_NAME, \ @@ -678,7 +910,7 @@ pub(crate) async fn list_function_parameters( validate_metadata_identifier(database_name, "databaseName")?; validate_metadata_identifier(function_name, "functionName")?; let resolved = resolve_native_connection(application, datasource_id).await?; - let mut conn = open_connection(&resolved.connection).await?; + let mut conn = open_resolved_connection(&resolved).await?; let query = "SELECT SPECIFIC_SCHEMA, SPECIFIC_NAME, ORDINAL_POSITION, PARAMETER_MODE, \ PARAMETER_NAME, DATA_TYPE, DTD_IDENTIFIER, CHARACTER_MAXIMUM_LENGTH, \ CHARACTER_OCTET_LENGTH, NUMERIC_PRECISION, NUMERIC_SCALE, \ @@ -707,7 +939,7 @@ pub(crate) async fn list_procedures( ) -> Result { validate_metadata_identifier(database_name, "databaseName")?; let resolved = resolve_native_connection(application, datasource_id).await?; - let mut conn = open_connection(&resolved.connection).await?; + let mut conn = open_resolved_connection(&resolved).await?; let query = "SELECT ROUTINE_SCHEMA, ROUTINE_NAME, SPECIFIC_NAME, \ COALESCE(ROUTINE_COMMENT, '') \ FROM information_schema.ROUTINES \ @@ -752,7 +984,7 @@ pub(crate) async fn get_procedure( "procedureName", )?; let resolved = resolve_native_connection(application, datasource_id).await?; - let mut conn = open_connection(&resolved.connection).await?; + let mut conn = open_resolved_connection(&resolved).await?; let result = async { let metadata = metadata_query(conn.exec_first::<(String, String, String, String), _, _>( "SELECT ROUTINE_SCHEMA, ROUTINE_NAME, SPECIFIC_NAME, \ @@ -792,7 +1024,7 @@ pub(crate) async fn list_procedure_parameters( validate_metadata_identifier(database_name, "databaseName")?; validate_metadata_identifier(procedure_name, "procedureName")?; let resolved = resolve_native_connection(application, datasource_id).await?; - let mut conn = open_connection(&resolved.connection).await?; + let mut conn = open_resolved_connection(&resolved).await?; let query = "SELECT SPECIFIC_SCHEMA, SPECIFIC_NAME, ORDINAL_POSITION, PARAMETER_MODE, \ PARAMETER_NAME, DATA_TYPE, DTD_IDENTIFIER, CHARACTER_MAXIMUM_LENGTH, \ CHARACTER_OCTET_LENGTH, NUMERIC_PRECISION, NUMERIC_SCALE, \ @@ -824,7 +1056,7 @@ pub(crate) async fn preview_routine_invocation( validate_metadata_identifier(&routine_name, "routineName")?; validate_metadata_identifier(&database_name, "databaseName")?; let resolved = resolve_native_connection(application, &request.datasource_id).await?; - let mut conn = open_connection(&resolved.connection).await?; + let mut conn = open_resolved_connection(&resolved).await?; let query = "SELECT ORDINAL_POSITION, PARAMETER_MODE, PARAMETER_NAME, DATA_TYPE \ FROM information_schema.PARAMETERS \ WHERE SPECIFIC_SCHEMA = ? AND SPECIFIC_NAME = ? AND ROUTINE_TYPE = ? \ @@ -847,6 +1079,218 @@ pub(crate) async fn preview_routine_invocation( finish_connection(conn, result).await } +pub(crate) fn preview_routine_migration( + request: &CommunityRoutineMigrationRequest, +) -> Result { + let plan = routine_migration_plan(request)?; + Ok(CommunityRoutineInvocationPreview { + sql: plan.preview_sql, + }) +} + +pub(crate) async fn execute_routine_migration( + application: &Application, + request: CommunityRoutineMigrationRequest, +) -> Result { + let plan = routine_migration_plan(&request)?; + let resolved = resolve_native_connection(application, &request.datasource_id).await?; + let mut conn = open_resolved_connection(&resolved).await?; + let result = execute_routine_migration_with_connection(&mut conn, &plan).await; + finish_connection(conn, Ok(result)).await +} + +async fn execute_routine_migration_with_connection( + conn: &mut Conn, + plan: &RoutineMigrationPlan, +) -> CommunityRoutineMigrationExecution { + let selected_database = quote_identifier(&plan.database_name, "databaseName") + .expect("validated migration database name must remain valid"); + if let Err(error) = metadata_query(conn.query_drop(format!("USE {selected_database}"))).await { + return routine_migration_failure( + plan, + migration_error(&error), + "BEFORE_IMAGE", + false, + false, + ); + } + + let previous = match capture_previous_routine(conn, plan).await { + Ok(previous) => previous, + Err(error) => { + return routine_migration_failure( + plan, + format!( + "Routine migration was rejected because the existing routine definition could not be captured before DROP: {}", + migration_error(&error) + ), + "BEFORE_IMAGE", + false, + false, + ); + } + }; + + if let Err(error) = metadata_query(conn.query_drop(&plan.drop_sql)).await { + return routine_migration_failure( + plan, + format!( + "Routine migration failed before the previous routine was dropped. Original error: {}", + migration_error(&error) + ), + "DROP", + false, + false, + ); + } + + match metadata_query(conn.query_drop(&plan.create_sql)).await { + Ok(()) => CommunityRoutineMigrationExecution { + success: true, + message: "Statement executed successfully".to_owned(), + sql: plan.preview_sql.clone(), + failure_stage: None, + restore_attempted: false, + restore_succeeded: false, + }, + Err(create_error) => { + let Some(previous) = previous else { + return routine_migration_failure( + plan, + format!( + "Routine migration failed. No previous routine definition existed. Original error: {}", + migration_error(&create_error) + ), + "APPLY", + false, + false, + ); + }; + let restore = async { + metadata_query(conn.query_drop(&plan.drop_sql)).await?; + metadata_query(conn.query_drop(previous)).await + } + .await; + match restore { + Ok(()) => routine_migration_failure( + plan, + format!( + "Routine migration failed. The previous routine definition was restored. Original error: {}", + migration_error(&create_error) + ), + "APPLY", + true, + true, + ), + Err(restore_error) => routine_migration_failure( + plan, + format!( + "Routine migration failed after the previous routine was dropped, and automatic restore failed. Original error: {}; restore error: {}", + migration_error(&create_error), + migration_error(&restore_error) + ), + "APPLY", + true, + false, + ), + } + } + } +} + +async fn capture_previous_routine( + conn: &mut Conn, + plan: &RoutineMigrationPlan, +) -> Result, AppError> { + let exists = metadata_query(conn.exec_first::( + "SELECT 1 FROM information_schema.ROUTINES \ + WHERE ROUTINE_SCHEMA = ? AND ROUTINE_NAME = ? AND ROUTINE_TYPE = ? LIMIT 1", + ( + plan.database_name.clone(), + plan.routine_name.clone(), + plan.routine_type.as_str(), + ), + )) + .await? + .is_some(); + if !exists { + return Ok(None); + } + let qualified_name = qualified_identifier( + &plan.database_name, + "databaseName", + &plan.routine_name, + "routineName", + )?; + let row = metadata_query(conn.query_first::(format!( + "SHOW CREATE {} {qualified_name}", + plan.routine_type.as_str() + ))) + .await? + .ok_or_else(AppError::internal)?; + let mut ddl = row_string_at(&row, 2)?; + ensure_sql_terminated(&mut ddl); + Ok(Some(ddl)) +} + +fn routine_migration_plan( + request: &CommunityRoutineMigrationRequest, +) -> Result { + let routine_type = normalize_mysql_routine_type(&request.routine_type)?; + let database_name = request.database_name.trim().to_owned(); + let routine_name = mysql_routine_lookup_name(request.routine_name.trim()); + validate_metadata_identifier(&database_name, "databaseName")?; + validate_metadata_identifier(&routine_name, "routineName")?; + let ddl = request.ddl.trim(); + if ddl.is_empty() || ddl.len() > MAX_SQL_BYTES || ddl.contains('\0') { + return Err(AppError::invalid( + "invalid_community_routine_migration_request", + "ddl is invalid", + )); + } + let qualified_name = + qualified_identifier(&database_name, "databaseName", &routine_name, "routineName")?; + let drop_sql = format!("DROP {} IF EXISTS {qualified_name}", routine_type.as_str()); + let mut create_sql = ddl.to_owned(); + ensure_sql_terminated(&mut create_sql); + let preview_sql = format!("{drop_sql};\n\n{create_sql}"); + Ok(RoutineMigrationPlan { + routine_type, + database_name, + routine_name, + drop_sql, + create_sql, + preview_sql, + }) +} + +fn ensure_sql_terminated(sql: &mut String) { + if !sql.trim_end().ends_with(';') { + sql.push(';'); + } +} + +fn routine_migration_failure( + plan: &RoutineMigrationPlan, + message: String, + failure_stage: &str, + restore_attempted: bool, + restore_succeeded: bool, +) -> CommunityRoutineMigrationExecution { + CommunityRoutineMigrationExecution { + success: false, + message, + sql: plan.preview_sql.clone(), + failure_stage: Some(failure_stage.to_owned()), + restore_attempted, + restore_succeeded, + } +} + +fn migration_error(error: &AppError) -> String { + error.api_error().message +} + pub(crate) async fn list_triggers( application: &Application, datasource_id: &str, @@ -855,7 +1299,7 @@ pub(crate) async fn list_triggers( ) -> Result { validate_metadata_identifier(database_name, "databaseName")?; let resolved = resolve_native_connection(application, datasource_id).await?; - let mut conn = open_connection(&resolved.connection).await?; + let mut conn = open_resolved_connection(&resolved).await?; let query = "SELECT TRIGGER_SCHEMA, TRIGGER_NAME, EVENT_MANIPULATION \ FROM information_schema.TRIGGERS \ WHERE TRIGGER_SCHEMA = ? ORDER BY TRIGGER_NAME"; @@ -892,7 +1336,7 @@ pub(crate) async fn get_trigger( let qualified_name = qualified_identifier(database_name, "databaseName", trigger_name, "triggerName")?; let resolved = resolve_native_connection(application, datasource_id).await?; - let mut conn = open_connection(&resolved.connection).await?; + let mut conn = open_resolved_connection(&resolved).await?; let result = async { let metadata = metadata_query(conn.exec_first::<(String, String, String), _, _>( "SELECT TRIGGER_SCHEMA, TRIGGER_NAME, EVENT_MANIPULATION \ @@ -933,18 +1377,216 @@ pub(crate) fn validate_query(query: &PreparedQuery) -> Result<(), AppError> { format!("SQL cannot exceed {MAX_SQL_BYTES} UTF-8 bytes"), )); } - if !query.parameters.is_empty() { + let _ = mysql_query_parameters(&query.parameters)?; + validate_read_sql(&query.sql)?; + validate_query_options(query.options) +} + +fn mysql_query_parameters(parameters: &[JdbcParameter]) -> Result { + if parameters.is_empty() { + return Ok(Params::Empty); + } + if parameters.len() > MAX_PARAMETERS { return Err(AppError::invalid( - "invalid_query_request", - "Native MySQL SELECT does not accept parameters yet", + "invalid_query_parameter_count", + format!("MySQL queries accept at most {MAX_PARAMETERS} parameters"), )); } - validate_read_sql(&query.sql)?; - validate_query_options(query.options) + + let mut ordered = parameters.iter().collect::>(); + ordered.sort_unstable_by_key(|parameter| parameter.position); + let mut values = Vec::with_capacity(ordered.len()); + for (index, parameter) in ordered.into_iter().enumerate() { + let expected = u32::try_from(index + 1).map_err(|_| AppError::internal())?; + if parameter.position != expected { + return Err(AppError::invalid( + "invalid_query_parameter", + "MySQL parameter positions must be unique and contiguous from 1", + )); + } + values.push(mysql_query_value(¶meter.value)?); + } + Ok(Params::Positional(values)) +} + +fn mysql_query_value(value: &BridgeJdbcValue) -> Result { + match value { + BridgeJdbcValue::Null => Ok(Value::NULL), + BridgeJdbcValue::Boolean(value) => Ok(Value::Int(i64::from(*value))), + BridgeJdbcValue::SignedInteger(value) => Ok(Value::Int(*value)), + BridgeJdbcValue::UnsignedInteger(value) => Ok(Value::UInt(*value)), + BridgeJdbcValue::Float32(value) => Ok(Value::Float(*value)), + BridgeJdbcValue::Float64(value) => Ok(Value::Double(*value)), + BridgeJdbcValue::Decimal(value) => { + validate_mysql_decimal(value)?; + mysql_query_bytes(value.as_bytes(), "decimal") + } + BridgeJdbcValue::Text(value) => mysql_query_bytes(value.as_bytes(), "text"), + BridgeJdbcValue::Binary(value) => mysql_query_bytes(value, "binary"), + BridgeJdbcValue::Date(value) => mysql_date_parameter(value), + BridgeJdbcValue::Time(value) => mysql_time_parameter(value), + BridgeJdbcValue::Timestamp(value) => mysql_timestamp_parameter(value), + BridgeJdbcValue::TimestampWithTimeZone(value) => { + mysql_timestamp_with_time_zone_parameter(value) + } + BridgeJdbcValue::Json(value) => mysql_query_bytes(value.as_bytes(), "JSON"), + BridgeJdbcValue::Uuid(value) => mysql_query_bytes(value.as_bytes(), "UUID"), + BridgeJdbcValue::Opaque { .. } => Err(AppError::invalid( + "invalid_query_parameter", + "Opaque JDBC values cannot be MySQL query parameters", + )), + } +} + +fn mysql_query_bytes(value: &[u8], label: &str) -> Result { + if value.len() > MAX_SCALAR_BYTES { + return Err(AppError::invalid( + "invalid_query_parameter", + format!("The MySQL {label} parameter exceeds {MAX_SCALAR_BYTES} bytes"), + )); + } + Ok(Value::Bytes(value.to_vec())) +} + +fn validate_mysql_decimal(value: &str) -> Result<(), AppError> { + let unsigned = value.strip_prefix(['+', '-']).unwrap_or(value); + let mut digits = 0_usize; + let mut decimal_points = 0_u8; + for byte in unsigned.bytes() { + if byte.is_ascii_digit() { + digits += 1; + } else if byte == b'.' { + decimal_points += 1; + } else { + return Err(mysql_temporal_parameter_error("decimal")); + } + } + if digits == 0 || decimal_points > 1 { + return Err(mysql_temporal_parameter_error("decimal")); + } + Ok(()) +} + +fn mysql_date_parameter(value: &str) -> Result { + mysql_query_bytes(value.as_bytes(), "date")?; + let date = NaiveDate::parse_from_str(value, "%Y-%m-%d") + .map_err(|_| mysql_temporal_parameter_error("date"))?; + let datetime = date + .and_hms_opt(0, 0, 0) + .ok_or_else(|| mysql_temporal_parameter_error("date"))?; + mysql_datetime_value(datetime, "date") +} + +fn mysql_time_parameter(value: &str) -> Result { + mysql_query_bytes(value.as_bytes(), "time")?; + let (negative, unsigned) = value + .strip_prefix('-') + .map_or((false, value), |value| (true, value)); + let unsigned = unsigned.strip_prefix('+').unwrap_or(unsigned); + let mut parts = unsigned.split(':'); + let hours = parse_mysql_time_part::(parts.next(), "time")?; + let minutes = parse_mysql_time_part::(parts.next(), "time")?; + let seconds = parts + .next() + .ok_or_else(|| mysql_temporal_parameter_error("time"))?; + if parts.next().is_some() || hours > 838 || minutes > 59 { + return Err(mysql_temporal_parameter_error("time")); + } + let (seconds, micros) = parse_mysql_seconds(seconds)?; + Ok(Value::Time( + negative, + hours / 24, + u8::try_from(hours % 24).map_err(|_| AppError::internal())?, + minutes, + seconds, + micros, + )) +} + +fn parse_mysql_time_part(value: Option<&str>, label: &str) -> Result +where + T: std::str::FromStr, +{ + value + .filter(|value| !value.is_empty()) + .ok_or_else(|| mysql_temporal_parameter_error(label))? + .parse::() + .map_err(|_| mysql_temporal_parameter_error(label)) +} + +fn parse_mysql_seconds(value: &str) -> Result<(u8, u32), AppError> { + let (seconds, fraction) = value + .split_once('.') + .map_or((value, None), |(seconds, fraction)| { + (seconds, Some(fraction)) + }); + let seconds = parse_mysql_time_part::(Some(seconds), "time")?; + if seconds > 59 { + return Err(mysql_temporal_parameter_error("time")); + } + let Some(fraction) = fraction else { + return Ok((seconds, 0)); + }; + if fraction.is_empty() + || fraction.len() > 6 + || !fraction.bytes().all(|byte| byte.is_ascii_digit()) + { + return Err(mysql_temporal_parameter_error("time")); + } + let parsed = fraction + .parse::() + .map_err(|_| mysql_temporal_parameter_error("time"))?; + let padding = u32::try_from(6 - fraction.len()).map_err(|_| AppError::internal())?; + Ok((seconds, parsed * 10_u32.pow(padding))) +} + +fn mysql_timestamp_parameter(value: &str) -> Result { + mysql_query_bytes(value.as_bytes(), "timestamp")?; + let datetime = ["%Y-%m-%dT%H:%M:%S%.f", "%Y-%m-%d %H:%M:%S%.f"] + .into_iter() + .find_map(|format| NaiveDateTime::parse_from_str(value, format).ok()) + .ok_or_else(|| mysql_temporal_parameter_error("timestamp"))?; + mysql_datetime_value(datetime, "timestamp") +} + +fn mysql_timestamp_with_time_zone_parameter(value: &str) -> Result { + mysql_query_bytes(value.as_bytes(), "timestamp with time zone")?; + let datetime = DateTime::parse_from_rfc3339(value) + .map_err(|_| mysql_temporal_parameter_error("timestamp with time zone"))? + .with_timezone(&Utc) + .naive_utc(); + mysql_datetime_value(datetime, "timestamp with time zone") +} + +fn mysql_datetime_value(datetime: NaiveDateTime, label: &str) -> Result { + let year = u16::try_from(datetime.year()) + .ok() + .filter(|year| *year > 0 && *year <= 9_999) + .ok_or_else(|| mysql_temporal_parameter_error(label))?; + Ok(Value::Date( + year, + u8::try_from(datetime.month()).map_err(|_| AppError::internal())?, + u8::try_from(datetime.day()).map_err(|_| AppError::internal())?, + u8::try_from(datetime.hour()).map_err(|_| AppError::internal())?, + u8::try_from(datetime.minute()).map_err(|_| AppError::internal())?, + u8::try_from(datetime.second()).map_err(|_| AppError::internal())?, + datetime.nanosecond() / 1_000, + )) +} + +fn mysql_temporal_parameter_error(label: &str) -> AppError { + AppError::invalid( + "invalid_query_parameter", + format!("The MySQL {label} parameter is invalid"), + ) } fn validate_read_sql(sql: &str) -> Result<(), AppError> { - let tokens = sql_tokens(sql)?; + let tokens = read_policy_tokens( + sql, + "mysql_native_query_unsupported", + "Native MySQL read queries do not accept executable comments", + )?; if !matches!(tokens.first(), Some(SqlToken::Word(keyword)) if keyword == "SELECT") { return Err(AppError::invalid( "mysql_native_query_unsupported", @@ -986,9 +1628,31 @@ fn validate_read_sql(sql: &str) -> Result<(), AppError> { Ok(()) } +struct SqlLexemes { + tokens: Vec, + executable_comment: bool, +} + fn sql_tokens(sql: &str) -> Result, AppError> { + Ok(sql_lexemes(sql)?.tokens) +} + +fn read_policy_tokens( + sql: &str, + error_code: &'static str, + error_message: &'static str, +) -> Result, AppError> { + let lexemes = sql_lexemes(sql)?; + if lexemes.executable_comment { + return Err(AppError::invalid(error_code, error_message)); + } + Ok(lexemes.tokens) +} + +fn sql_lexemes(sql: &str) -> Result { let bytes = sql.as_bytes(); let mut tokens = Vec::new(); + let mut executable_comment = false; let mut index = 0; while index < bytes.len() { match bytes[index] { @@ -1000,6 +1664,11 @@ fn sql_tokens(sql: &str) -> Result, AppError> { skip_line_comment(bytes, &mut index); } b'/' if bytes.get(index + 1) == Some(&b'*') => { + executable_comment |= bytes.get(index + 2) == Some(&b'!') + || (bytes + .get(index + 2) + .is_some_and(|byte| matches!(byte, b'm' | b'M')) + && bytes.get(index + 3) == Some(&b'!')); index += 2; let mut terminated = false; while index + 1 < bytes.len() { @@ -1054,7 +1723,10 @@ fn sql_tokens(sql: &str) -> Result, AppError> { _ => index += 1, } } - Ok(tokens) + Ok(SqlLexemes { + tokens, + executable_comment, + }) } fn skip_line_comment(bytes: &[u8], index: &mut usize) { @@ -1113,35 +1785,24 @@ pub(crate) async fn execute_console( application: &Application, request: MysqlConsoleRequest, mut cancellation: watch::Receiver, + force_read_only: bool, ) -> Result, AppError> { - let (page_offset, page_end) = validate_console_request(&request)?; - let mut statements = if request.single { - vec![request.sql.trim().to_owned()] - } else { - split_mysql_script(&request.sql)? - }; - if statements.is_empty() { - return Err(AppError::invalid( - "invalid_mysql_console_request", - "sql must contain at least one MySQL statement", - )); - } - if request.explain { - for statement in &mut statements { - *statement = format!("EXPLAIN {statement}"); - } - } + let (statements, page_offset, page_end) = prepare_console_statements(&request)?; let initial_cancellation = { cancellation.borrow().clone() }; if let CancellationRequest::Requested { reason } = initial_cancellation { return Err(mysql_console_cancelled(reason)); } + if force_read_only { + validate_forced_read_console(&statements)?; + } let resolved = resolve_native_connection(application, &request.datasource_id).await?; - if resolved.connection.read_only { + if resolved.connection.read_only && !force_read_only { validate_read_only_console(&statements)?; } - let options = connection_opts(&resolved.connection)?; - let mut conn = match open_query_connection(options.clone(), &mut cancellation).await { + let prepared = prepare_resolved_connection(&resolved).await?; + let options = prepared.options.clone(); + let mut conn = match open_query_connection(prepared, &mut cancellation).await { Ok(conn) => conn, Err(QueryTaskError::Cancelled(reason)) => return Err(mysql_console_cancelled(reason)), Err(QueryTaskError::Failed(error)) => return Err(error), @@ -1162,6 +1823,10 @@ pub(crate) async fn execute_console( return finish_console_error(conn, options, connection_id, error).await; } } + if force_read_only && let Err(error) = start_read_only_transaction(&mut conn).await { + disconnect_quietly(conn).await; + return Err(error); + } let mut results = Vec::new(); let mut retained_result_bytes = 0_u64; @@ -1205,10 +1870,144 @@ pub(crate) async fn execute_console( } } - disconnect_connection(conn).await?; + if force_read_only { + finish_read_only_connection_quietly(conn).await; + } else { + disconnect_connection(conn).await?; + } Ok(results) } +fn prepare_console_statements( + request: &MysqlConsoleRequest, +) -> Result<(Vec, u64, u64), AppError> { + let (page_offset, page_end) = validate_console_request(request)?; + let mut statements = if request.single { + vec![request.sql.trim().to_owned()] + } else { + split_mysql_script(&request.sql)? + }; + if statements.is_empty() { + return Err(AppError::invalid( + "invalid_mysql_console_request", + "sql must contain at least one MySQL statement", + )); + } + if request.explain { + for statement in &mut statements { + *statement = format!("EXPLAIN {statement}"); + } + } + Ok((statements, page_offset, page_end)) +} + +pub(crate) async fn execute_update( + resolved: ResolvedDatasourceConnection, + sql: String, + cancellation: CancellationToken, +) -> Result { + if cancellation.is_cancelled() { + return Err(DatabaseWriteError::not_started(AppError::new( + AppErrorKind::Conflict, + ApiError::new( + "database_write_cancelled", + "The database write was cancelled before dispatch", + ), + ))); + } + let sql = validate_single_write_sql(&sql).map_err(DatabaseWriteError::not_started)?; + if resolved.connection.read_only { + return Err(DatabaseWriteError::not_started(AppError::new( + AppErrorKind::Conflict, + ApiError::new( + "datasource_read_only", + "The datasource connection is configured as read-only", + ), + ))); + } + + let prepared = prepare_resolved_connection(&resolved) + .await + .map_err(DatabaseWriteError::not_started)?; + let options = prepared.options.clone(); + let open = open_prepared_connection(prepared); + tokio::pin!(open); + let mut conn = tokio::select! { + biased; + () = cancellation.cancelled() => { + return Err(DatabaseWriteError::not_started(AppError::new( + AppErrorKind::Conflict, + ApiError::new( + "database_write_cancelled", + "The database write was cancelled before dispatch", + ), + ))); + } + result = &mut open => result.map_err(DatabaseWriteError::not_started)?, + }; + if cancellation.is_cancelled() { + disconnect_quietly(conn).await; + return Err(DatabaseWriteError::not_started(AppError::new( + AppErrorKind::Conflict, + ApiError::new( + "database_write_cancelled", + "The database write was cancelled before dispatch", + ), + ))); + } + + let connection_id = conn.id(); + let result = { + // Prepared statements are a second boundary against multi-statement + // execution even if a future parser regression accepts a script. + let query = conn.exec_drop(sql, ()); + tokio::pin!(query); + tokio::select! { + biased; + () = cancellation.cancelled() => None, + result = &mut query => Some(result), + } + }; + let Some(result) = result else { + terminate_connection_quietly(options, connection_id).await; + drop(conn); + return Err(DatabaseWriteError::unknown(AppError::new( + AppErrorKind::Unavailable, + ApiError::new( + "database_write_outcome_unknown", + "The database write was interrupted after dispatch; do not retry it blindly", + ), + ))); + }; + + match result { + Ok(()) => { + let affected_rows = conn.affected_rows(); + disconnect_quietly(conn).await; + Ok(affected_rows) + } + Err(error @ MysqlError::Server(_)) => { + disconnect_quietly(conn).await; + tracing::warn!( + error = %error, + "MySQL rejected a dispatched write whose partial effects cannot be excluded" + ); + Err(DatabaseWriteError::unknown(AppError::new( + AppErrorKind::Unavailable, + ApiError::new( + "database_write_outcome_unknown", + "MySQL reported an error after write dispatch; partial effects cannot be excluded, so do not retry it blindly", + ), + ))) + } + Err(error) => { + terminate_connection_quietly(options, connection_id).await; + drop(conn); + Err(DatabaseWriteError::unknown(mysql_query_error(error))) + } + } +} + #[allow(clippy::too_many_arguments, clippy::too_many_lines)] async fn execute_console_statement( conn: &mut Conn, @@ -1415,16 +2214,19 @@ async fn execute_console_control( } async fn finish_console_error( - conn: Conn, + conn: ManagedMysqlConnection, options: Opts, connection_id: u32, error: ConsoleExecutionError, ) -> Result { - drop(conn); match error { - ConsoleExecutionError::Cancelled(reason) => Err(mysql_console_cancelled(reason)), + ConsoleExecutionError::Cancelled(reason) => { + drop(conn); + Err(mysql_console_cancelled(reason)) + } ConsoleExecutionError::Fatal(error) => { terminate_connection_quietly(options, connection_id).await; + drop(conn); Err(error) } } @@ -1503,7 +2305,11 @@ fn validate_console_request(request: &MysqlConsoleRequest) -> Result<(u64, u64), fn validate_read_only_console(statements: &[String]) -> Result<(), AppError> { for statement in statements { - let tokens = sql_tokens(statement)?; + let tokens = read_policy_tokens( + statement, + "datasource_read_only", + "Read-only datasource connections do not accept executable comments", + )?; let words = tokens .iter() .filter_map(|token| match token { @@ -1533,6 +2339,145 @@ fn validate_read_only_console(statements: &[String]) -> Result<(), AppError> { Ok(()) } +fn validate_forced_read_console(statements: &[String]) -> Result<(), AppError> { + let [statement] = statements else { + return Err(AppError::invalid( + "chart_query_must_be_read_only", + "Chart refresh accepts exactly one MySQL SELECT statement", + )); + }; + let tokens = read_policy_tokens( + statement, + "chart_query_must_be_read_only", + "Chart refresh SQL must not use MySQL executable comments", + )?; + let parsed = Parser::parse_sql(&MySqlDialect {}, statement).map_err(|_| { + AppError::invalid( + "chart_query_must_be_read_only", + "Chart refresh SQL must be one valid MySQL SELECT statement", + ) + })?; + if !matches!(parsed.as_slice(), [Statement::Query(_)]) { + return Err(AppError::invalid( + "chart_query_must_be_read_only", + "Chart refresh accepts exactly one MySQL SELECT statement", + )); + } + + if !matches!( + tokens.first(), + Some(SqlToken::Word(keyword)) if keyword == "SELECT" || keyword == "WITH" + ) { + return Err(AppError::invalid( + "chart_query_must_be_read_only", + "Chart refresh accepts SELECT statements and SELECT CTEs only", + )); + } + let words = tokens + .iter() + .filter_map(|token| match token { + SqlToken::Word(word) => Some(word.as_str()), + SqlToken::Semicolon => None, + }) + .collect::>(); + let mutating = words.iter().any(|word| { + matches!( + *word, + "INSERT" + | "UPDATE" + | "DELETE" + | "REPLACE" + | "CREATE" + | "ALTER" + | "DROP" + | "TRUNCATE" + | "RENAME" + | "CALL" + | "GRANT" + | "REVOKE" + | "LOAD" + ) + }); + let unsafe_select = words.windows(2).any(|window| { + matches!( + window, + ["INTO", "OUTFILE" | "DUMPFILE"] | ["FOR", "UPDATE" | "SHARE"] + ) + }) || words + .windows(4) + .any(|window| matches!(window, ["LOCK", "IN", "SHARE", "MODE"])); + if mutating || unsafe_select { + return Err(AppError::invalid( + "chart_query_must_be_read_only", + "Chart refresh SQL must not write data, lock rows, or write server files", + )); + } + Ok(()) +} + +fn validate_single_write_sql(sql: &str) -> Result { + if contains_delimiter_directive(sql) { + return Err(AppError::invalid( + "invalid_database_write", + "DELIMITER is not accepted by the confirmed MySQL write surface", + )); + } + let mut statements = split_mysql_script(sql)?; + if statements.len() != 1 { + return Err(AppError::invalid( + "invalid_database_write", + "Exactly one MySQL write statement is required", + )); + } + let statement = statements.pop().expect("length checked above"); + let tokens = sql_tokens(&statement)?; + let first_word = tokens.iter().find_map(|token| match token { + SqlToken::Word(word) => Some(word.as_str()), + SqlToken::Semicolon => None, + }); + if !matches!( + first_word, + Some( + "INSERT" + | "UPDATE" + | "DELETE" + | "REPLACE" + | "CREATE" + | "ALTER" + | "DROP" + | "TRUNCATE" + | "RENAME" + | "GRANT" + | "REVOKE" + | "ANALYZE" + | "OPTIMIZE" + | "REPAIR" + | "CALL" + ) + ) { + return Err(AppError::invalid( + "database_write_statement_required", + "The confirmed MySQL write surface accepts one DML, DDL, grant, or routine statement", + )); + } + Ok(statement) +} + +fn contains_delimiter_directive(sql: &str) -> bool { + const KEYWORD: &str = "delimiter"; + + sql.lines().any(|line| { + let line = line.trim_start(); + line.get(..KEYWORD.len()).is_some_and(|prefix| { + prefix.eq_ignore_ascii_case(KEYWORD) + && line + .as_bytes() + .get(KEYWORD.len()) + .is_none_or(u8::is_ascii_whitespace) + }) + }) +} + fn mysql_console_cancelled(reason: Option) -> AppError { AppError::new( AppErrorKind::Conflict, @@ -1931,7 +2876,7 @@ async fn cancel_console_connection(options: Opts, connection_id: u32) -> Result< kill_console_target(&mut control, format!("KILL QUERY {connection_id}")).await; let connection_cancel = kill_console_target(&mut control, format!("KILL CONNECTION {connection_id}")).await; - disconnect_quietly(control).await; + disconnect_raw_quietly(control).await; match (query_cancel, connection_cancel) { (_, Ok(())) => Ok(()), (Ok(()), Err(error)) => Err(error), @@ -1971,15 +2916,17 @@ pub(crate) async fn execute_query_task( return Err(QueryTaskError::Cancelled(reason)); } - let options = connection_opts(&resolved.connection)?; - let mut conn = open_query_connection(options.clone(), &mut cancellation).await?; + let parameters = mysql_query_parameters(&query.parameters)?; + let prepared = prepare_resolved_connection(&resolved).await?; + let options = prepared.options.clone(); + let mut conn = open_query_connection(prepared, &mut cancellation).await?; let connection_id = conn.id(); if let Err(error) = start_read_only_transaction(&mut conn).await { disconnect_quietly(conn).await; return Err(error.into()); } let query_result = { - let query_future = conn.exec_iter(query.sql, ()); + let query_future = conn.exec_iter(query.sql, parameters); tokio::pin!(query_future); let mut cancellation_open = true; loop { @@ -3433,7 +4380,7 @@ fn validate_query_options(options: QueryOptions) -> Result<(), AppError> { Ok(()) } -fn quote_identifier(value: &str, field: &str) -> Result { +pub(crate) fn quote_identifier(value: &str, field: &str) -> Result { if value.trim().is_empty() || value.len() > MAX_IDENTIFIER_BYTES || value.contains('\0') { return Err(AppError::invalid( "invalid_community_table_preview_request", @@ -3459,15 +4406,15 @@ async fn terminate_connection(options: Opts, connection_id: u32) -> Result<(), A }; match result { Ok(()) => { - disconnect_quietly(control).await; + disconnect_raw_quietly(control).await; Ok(()) } Err(MysqlError::Server(server)) if server.code == 1094 => { - disconnect_quietly(control).await; + disconnect_raw_quietly(control).await; Ok(()) } Err(error) => { - disconnect_quietly(control).await; + disconnect_raw_quietly(control).await; Err(mysql_query_error(error)) } } @@ -3480,10 +4427,10 @@ async fn terminate_connection_quietly(options: Opts, connection_id: u32) { } async fn open_query_connection( - options: Opts, + prepared: PreparedMysqlConnection, cancellation: &mut watch::Receiver, -) -> Result { - let open = open_connection_with_opts(options); +) -> Result { + let open = open_prepared_connection(prepared); tokio::pin!(open); let mut cancellation_open = true; loop { @@ -3519,7 +4466,7 @@ async fn start_read_only_transaction(conn: &mut Conn) -> Result<(), AppError> { .map_err(mysql_query_error) } -async fn finish_read_only_connection_quietly(mut conn: Conn) { +async fn finish_read_only_connection_quietly(mut conn: ManagedMysqlConnection) { let rollback = tokio::time::timeout(CONTROL_TIMEOUT, conn.query_drop("ROLLBACK")).await; match rollback { Ok(Ok(())) => {} @@ -3532,7 +4479,28 @@ async fn finish_read_only_connection_quietly(mut conn: Conn) { disconnect_quietly(conn).await; } -async fn disconnect_connection(conn: Conn) -> Result<(), AppError> { +async fn disconnect_connection(mut conn: ManagedMysqlConnection) -> Result<(), AppError> { + let connection = conn + .connection + .take() + .expect("managed MySQL connection must exist until cleanup"); + let database_result = disconnect_raw_connection(connection).await; + let tunnel_result = match conn.tunnel.take() { + Some(tunnel) => tunnel.close().await, + None => Ok(()), + }; + match database_result { + Ok(()) => tunnel_result, + Err(error) => { + if let Err(tunnel_error) = tunnel_result { + tracing::warn!(error = %tunnel_error, "SSH tunnel cleanup failed after MySQL disconnect failure"); + } + Err(error) + } + } +} + +async fn disconnect_raw_connection(conn: Conn) -> Result<(), AppError> { tokio::time::timeout(DISCONNECT_TIMEOUT, conn.disconnect()) .await .map_err(|_| { @@ -3544,12 +4512,18 @@ async fn disconnect_connection(conn: Conn) -> Result<(), AppError> { .map_err(mysql_connection_error) } -async fn disconnect_quietly(conn: Conn) { +async fn disconnect_quietly(conn: ManagedMysqlConnection) { if let Err(error) = disconnect_connection(conn).await { tracing::warn!(error = %error, "native MySQL connection cleanup failed"); } } +async fn disconnect_raw_quietly(conn: Conn) { + if let Err(error) = disconnect_raw_connection(conn).await { + tracing::warn!(error = %error, "native MySQL control connection cleanup failed"); + } +} + fn resource_error(code: impl Into, message: impl Into) -> AppError { AppError::new( AppErrorKind::ResourceExhausted, @@ -3567,7 +4541,7 @@ fn result_decode_error() -> AppError { ) } -async fn resolve_native_connection( +pub(crate) async fn resolve_native_connection( application: &Application, datasource_id: &str, ) -> Result { @@ -3582,9 +4556,81 @@ async fn resolve_native_connection( Ok(resolved) } -async fn open_connection(connection: &DatasourceConnection) -> Result { - let opts = connection_opts(connection)?; - open_connection_with_opts(opts).await +pub(crate) async fn open_connection( + connection: &DatasourceConnection, +) -> Result { + open_prepared_connection(prepare_connection(connection).await?).await +} + +pub(crate) async fn open_resolved_connection( + resolved: &ResolvedDatasourceConnection, +) -> Result { + open_prepared_connection(prepare_resolved_connection(resolved).await?).await +} + +async fn prepare_connection( + connection: &DatasourceConnection, +) -> Result { + prepare_connection_with_identity(connection, SshTunnelIdentity::Ephemeral).await +} + +async fn prepare_resolved_connection( + resolved: &ResolvedDatasourceConnection, +) -> Result { + prepare_connection_with_identity( + &resolved.connection, + SshTunnelIdentity::Datasource { + datasource_id: &resolved.datasource_id, + revision: resolved.datasource_revision, + }, + ) + .await +} + +async fn prepare_connection_with_identity( + connection: &DatasourceConnection, + identity: SshTunnelIdentity<'_>, +) -> Result { + let Some(ssh) = connection.ssh.as_ref() else { + return Ok(PreparedMysqlConnection { + options: connection_opts(connection)?, + tunnel: None, + }); + }; + let (target_host, target_port) = mysql_target(&connection.jdbc_url)?; + let tunnel = SshTunnel::open(identity, ssh, target_host, target_port).await?; + let mut forwarded = connection.clone(); + forwarded.jdbc_url = rewrite_mysql_target(&forwarded.jdbc_url, tunnel.local_port())?; + forwarded.ssh = None; + let options = match connection_opts(&forwarded) { + Ok(options) => options, + Err(error) => { + if tunnel.close().await.is_err() { + tracing::warn!("SSH tunnel cleanup failed after MySQL option validation failure"); + } + return Err(error); + } + }; + Ok(PreparedMysqlConnection { + options, + tunnel: Some(tunnel), + }) +} + +async fn open_prepared_connection( + mut prepared: PreparedMysqlConnection, +) -> Result { + match open_connection_with_opts(prepared.options).await { + Ok(connection) => Ok(ManagedMysqlConnection::new(connection, prepared.tunnel)), + Err(error) => { + if let Some(tunnel) = prepared.tunnel.take() + && tunnel.close().await.is_err() + { + tracing::warn!("SSH tunnel cleanup failed after MySQL connection failure"); + } + Err(error) + } + } } async fn open_connection_with_opts(opts: Opts) -> Result { @@ -3614,7 +4660,10 @@ where .map_err(mysql_query_error) } -async fn finish_connection(conn: Conn, result: Result) -> Result { +pub(crate) async fn finish_connection( + conn: ManagedMysqlConnection, + result: Result, +) -> Result { let close = disconnect_connection(conn).await; match result { Ok(value) => close.map(|()| value), @@ -3759,6 +4808,20 @@ fn mysql_connection_error(error: MysqlError) -> AppError { fn mysql_query_error(error: MysqlError) -> AppError { match error { + MysqlError::Driver(DriverError::StmtParamsMismatch { required, supplied }) => { + AppError::invalid( + "invalid_query_parameter_count", + format!( + "The MySQL statement expects {required} parameters but {supplied} were supplied" + ), + ) + } + MysqlError::Driver(DriverError::StmtParamsNumberExceedsLimit { supplied }) => { + AppError::invalid( + "invalid_query_parameter_count", + format!("The MySQL statement cannot accept {supplied} parameters"), + ) + } MysqlError::Server(server) => AppError::new( AppErrorKind::InvalidRequest, ApiError::new("mysql_query_failed", server.message), @@ -3773,7 +4836,8 @@ fn mysql_query_error(error: MysqlError) -> AppError { #[cfg(test)] mod tests { use chat2db_contract::{ - DatasourceConnection, DatasourceConnectionProperty, JdbcValue, ResultRow, + CommunityRoutineMigrationRequest, DatasourceConnection, DatasourceConnectionProperty, + JdbcValue, ResultRow, }; use mysql_async::{Conn, Opts}; use tokio::sync::watch; @@ -3788,8 +4852,9 @@ mod tests { mysql_routine_lookup_name, normalize_mysql_routine_type, normalize_table_type, open_connection_with_opts, qualified_identifier, quote_identifier, render_routine_invocation_preview, reserve_console_result_bytes, - routine_invocation_parameter, split_mysql_script, validate_console_request, - validate_read_only_console, validate_read_sql, + routine_invocation_parameter, routine_migration_plan, split_mysql_script, + validate_console_request, validate_forced_read_console, validate_read_only_console, + validate_read_sql, validate_single_write_sql, }; use super::{MysqlRoutineType, RoutineInvocationParameter}; use crate::{MysqlConsoleRequest, operation::CancellationRequest}; @@ -3885,6 +4950,7 @@ mod tests { }, ], read_only: false, + ssh: None, }) .expect("live MySQL options should build"); let mut conn = open_connection_with_opts(options.clone()) @@ -4125,6 +5191,7 @@ mod tests { }, ], read_only: false, + ssh: None, }) .expect("JDBC URL should convert"); @@ -4272,6 +5339,44 @@ mod tests { } } + #[test] + fn mysql_routine_migration_preview_is_qualified_and_terminated() { + let plan = routine_migration_plan(&CommunityRoutineMigrationRequest { + datasource_id: "mysql-local".to_owned(), + database_type: "MYSQL".to_owned(), + database_name: "inventory".to_owned(), + schema_name: String::new(), + routine_type: " function ".to_owned(), + routine_name: "`odd``name`".to_owned(), + ddl: "CREATE FUNCTION `odd``name`() RETURNS INT RETURN 2".to_owned(), + }) + .expect("valid migration must render"); + + assert_eq!(plan.routine_name, "odd`name"); + assert_eq!( + plan.preview_sql, + "DROP FUNCTION IF EXISTS `inventory`.`odd``name`;\n\nCREATE FUNCTION `odd``name`() RETURNS INT RETURN 2;" + ); + } + + #[test] + fn mysql_routine_migration_rejects_missing_ddl() { + let error = routine_migration_plan(&CommunityRoutineMigrationRequest { + datasource_id: "mysql-local".to_owned(), + database_type: "MYSQL".to_owned(), + database_name: "inventory".to_owned(), + schema_name: String::new(), + routine_type: "PROCEDURE".to_owned(), + routine_name: "refresh_items".to_owned(), + ddl: " ".to_owned(), + }) + .expect_err("empty ddl must fail"); + assert_eq!( + error.api_error().code, + "invalid_community_routine_migration_request" + ); + } + #[test] fn explicit_properties_override_url_values_and_ssl_modes_are_mapped() { let opts = connection_opts(&DatasourceConnection { @@ -4290,6 +5395,7 @@ mod tests { }, ], read_only: false, + ssh: None, }) .expect("native URL should convert"); @@ -4310,6 +5416,7 @@ mod tests { jdbc_url: jdbc_url.to_owned(), properties: Vec::new(), read_only: false, + ssh: None, }) .expect_err("non-MySQL URLs must fail"); assert_eq!(error.api_error().code, "invalid_mysql_connection"); @@ -4346,6 +5453,43 @@ mod tests { } } + #[test] + fn confirmed_write_policy_accepts_one_write_and_rejects_reads_or_scripts() { + for sql in [ + "INSERT INTO items(label) VALUES ('new')", + "UPDATE items SET label = 'changed' WHERE id = 1", + "DELETE FROM items WHERE id = 1", + "CREATE TABLE created_by_cli(id BIGINT PRIMARY KEY)", + "ALTER TABLE items ADD COLUMN note TEXT", + "GRANT SELECT ON app.* TO 'reader'@'localhost'", + "CALL mutating_procedure()", + ] { + validate_single_write_sql(sql) + .unwrap_or_else(|error| panic!("{sql} should be accepted: {error}")); + } + for sql in [ + "SELECT 1", + "SHOW TABLES", + "START TRANSACTION", + "UPDATE items SET label = 'one'; DELETE FROM items WHERE id = 2", + ] { + assert!( + validate_single_write_sql(sql).is_err(), + "{sql} should be rejected" + ); + } + + for sql in [ + "DELIMITER $$\nCREATE PROCEDURE mutate_item()\nBEGIN\n UPDATE items SET label = 'changed' WHERE id = 1;\nEND$$\nDELIMITER ;", + "DELIMITER $$\nUPDATE items SET label = 'changed'; DELETE FROM items$$\nDELIMITER ;", + " delimiter //\nDELETE FROM items//", + ] { + let error = validate_single_write_sql(sql) + .expect_err("confirmed writes must reject client delimiter directives"); + assert_eq!(error.api_error().code, "invalid_database_write"); + } + } + #[test] fn console_read_only_policy_allows_inspection_and_rejects_writes() { for sql in [ @@ -4374,6 +5518,32 @@ mod tests { } } + #[test] + fn chart_refresh_policy_accepts_select_ctes_and_rejects_side_effects() { + for sql in [ + "SELECT 1", + "SELECT '/*! FOR SHARE */' AS harmless_text", + "WITH values_cte AS (SELECT 1 AS value) SELECT value FROM values_cte", + ] { + validate_forced_read_console(&[sql.to_owned()]) + .unwrap_or_else(|error| panic!("{sql} should be chart-safe: {error}")); + } + for statements in [ + vec!["UPDATE items SET label = 'changed'".to_owned()], + vec!["SELECT * FROM items FOR UPDATE".to_owned()], + vec!["SELECT * FROM items FOR SHARE".to_owned()], + vec!["SELECT 1 /*! INTO OUTFILE '/tmp/chart' */".to_owned()], + vec!["SELECT * FROM items /*M! FOR SHARE */".to_owned()], + vec!["SELECT 1 INTO OUTFILE '/tmp/chart'".to_owned()], + vec!["SELECT 1".to_owned(), "SELECT 2".to_owned()], + vec!["WITH ids AS (SELECT 1) UPDATE items SET label = 'changed'".to_owned()], + ] { + let error = validate_forced_read_console(&statements) + .expect_err("chart refresh must fail closed on non-read-only SQL"); + assert_eq!(error.api_error().code, "chart_query_must_be_read_only"); + } + } + #[test] fn console_page_size_all_uses_the_bounded_complete_window() { let mut request = MysqlConsoleRequest { diff --git a/crates/chat2db-core/src/query.rs b/crates/chat2db-core/src/query.rs index cf3428d..65b61e1 100644 --- a/crates/chat2db-core/src/query.rs +++ b/crates/chat2db-core/src/query.rs @@ -1,12 +1,13 @@ use std::time::Duration; use chat2db_contract::{ - ApiError, QueryAccepted, QueryLimits, ResultColumn, ResultRow, StartQueryRequest, + ApiError, DatabaseWriteResult, DatabaseWriteState, ExecuteDatabaseWriteRequest, QueryAccepted, + QueryLimits, ResultColumn, ResultRow, StartQueryRequest, }; use chat2db_engine_protocol::wire; use chat2db_java_bridge::{ - BridgeError, CancelDisposition as BridgeCancelDisposition, ConnectionProperty, JdbcParameter, - QueryEvent, QueryOptions, QueryRequest, QueryStream, SessionConfig, UpdateRequest, + BridgeError, CancelDisposition as BridgeCancelDisposition, JdbcParameter, QueryEvent, + QueryOptions, QueryRequest, QueryStream, }; use chat2db_storage::{ResultWriter, Storage, StorageError}; use tokio::sync::{oneshot, watch}; @@ -156,7 +157,6 @@ enum QueryBackend { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum DatabaseWriteOutcome { NotStarted, - Failed, Unknown, } @@ -193,7 +193,15 @@ impl Application { request: MysqlConsoleRequest, cancellation: MysqlConsoleCancellation, ) -> Result, AppError> { - crate::native_mysql::execute_console(self, request, cancellation.subscribe()).await + crate::native_mysql::execute_console(self, request, cancellation.subscribe(), false).await + } + + pub(crate) async fn execute_mysql_read_console( + &self, + request: MysqlConsoleRequest, + cancellation: MysqlConsoleCancellation, + ) -> Result, AppError> { + crate::native_mysql::execute_console(self, request, cancellation.subscribe(), true).await } /// Accepts a query for asynchronous execution and returns its operation id. @@ -221,6 +229,39 @@ impl Application { self.start_prepared_query(prepared).await } + /// Executes one explicitly confirmed database write and always reports whether + /// the statement was dispatched and whether retrying it can be safe. + #[must_use] + pub async fn execute_confirmed_database_write( + &self, + request: ExecuteDatabaseWriteRequest, + ) -> DatabaseWriteResult { + if !request.confirmed { + return database_write_result( + DatabaseWriteState::NotStarted, + None, + Some(ApiError::new( + "database_write_confirmation_required", + "Database writes require explicit confirmation", + )), + ); + } + + match self + .execute_agent_update(request.datasource_id, request.sql, CancellationToken::new()) + .await + { + Ok(affected_rows) => { + database_write_result(DatabaseWriteState::Succeeded, Some(affected_rows), None) + } + Err(error) => database_write_result( + database_write_state(error.outcome), + None, + Some(error.error.api_error()), + ), + } + } + pub(crate) async fn start_agent_read_query( &self, datasource_id: String, @@ -548,18 +589,11 @@ impl Application { let storage = self .require_storage() .map_err(DatabaseWriteError::not_started)?; - let engine = self - .require_engine() - .await - .map_err(DatabaseWriteError::not_started)?; - let ResolvedDatasourceConnection { - driver_id, - datasource_name: _, - connection, - } = resolve_datasource_connection(&storage, &datasource_id) + validate_database_write_sql(&sql).map_err(DatabaseWriteError::not_started)?; + let resolved = resolve_datasource_connection(&storage, &datasource_id) .await .map_err(DatabaseWriteError::not_started)?; - if connection.read_only { + if resolved.connection.read_only { return Err(DatabaseWriteError::not_started(AppError::new( AppErrorKind::Conflict, ApiError::new( @@ -568,75 +602,76 @@ impl Application { ), ))); } - let driver = engine - .driver_client() - .map_err(AppError::from) - .map_err(DatabaseWriteError::not_started)?; - let session = driver - .open_session(SessionConfig { - driver_id, - jdbc_url: connection.jdbc_url, - properties: connection - .properties - .into_iter() - .map(|property| ConnectionProperty { - key: property.key, - value: property.value, - sensitive: property.sensitive, - }) - .collect(), - read_only: false, - }) - .await - .map_err(|error| DatabaseWriteError::from_bridge(error, false))?; - if cancellation.is_cancelled() { - let _ = session.close().await; - return Err(DatabaseWriteError::not_started(AppError::new( - AppErrorKind::Conflict, - ApiError::new( - "agent_tool_cancelled", - "The database write was cancelled before dispatch", - ), - ))); + if self.is_native_mysql_driver(&resolved.driver_id) { + return crate::native_mysql::execute_update(resolved, sql, cancellation).await; } - - let result = session - .execute_update(UpdateRequest { - sql, - parameters: Vec::new(), - transaction_id: None, - }) - .await; - let close_result = session.close().await; - if let Err(close_error) = close_result { - tracing::warn!( - error = %close_error, - "database write session cleanup failed after the outcome was determined" - ); - } - result - .map(|completed| completed.affected_rows) - .map_err(|error| DatabaseWriteError::from_bridge(error, true)) + Err(DatabaseWriteError::not_started(AppError::invalid( + "mysql_driver_mismatch", + "Confirmed database writes require a native MySQL datasource", + ))) } } impl DatabaseWriteError { - fn not_started(error: AppError) -> Self { + pub(crate) fn not_started(error: AppError) -> Self { Self { error, outcome: DatabaseWriteOutcome::NotStarted, } } - fn from_bridge(error: BridgeError, dispatched: bool) -> Self { - let outcome = database_write_outcome(&error, dispatched); + pub(crate) fn unknown(error: AppError) -> Self { Self { - error: error.into(), - outcome, + error, + outcome: DatabaseWriteOutcome::Unknown, } } } +fn database_write_result( + state: DatabaseWriteState, + affected_rows: Option, + error: Option, +) -> DatabaseWriteResult { + DatabaseWriteResult { + state, + affected_rows: affected_rows.map(|value| value.to_string()), + error, + } +} + +const fn database_write_state(outcome: DatabaseWriteOutcome) -> DatabaseWriteState { + match outcome { + DatabaseWriteOutcome::NotStarted => DatabaseWriteState::NotStarted, + DatabaseWriteOutcome::Unknown => DatabaseWriteState::Unknown, + } +} + +fn validate_database_write_sql(sql: &str) -> Result<(), AppError> { + if sql.trim().is_empty() { + return Err(AppError::invalid( + "invalid_database_write", + "SQL cannot be empty", + )); + } + if sql.len() > wire::JdbcProtocolLimit::MaxSqlBytes as usize { + return Err(AppError::invalid( + "invalid_database_write", + format!( + "SQL exceeds the {} byte database-write limit", + wire::JdbcProtocolLimit::MaxSqlBytes as usize + ), + )); + } + if sql.contains('\0') { + return Err(AppError::invalid( + "invalid_database_write", + "SQL contains an invalid NUL byte", + )); + } + Ok(()) +} + impl TryFrom for PreparedQuery { type Error = AppError; @@ -736,40 +771,6 @@ impl RetainedWriter { } } -fn database_write_outcome(error: &BridgeError, dispatched: bool) -> DatabaseWriteOutcome { - use chat2db_engine_protocol::wire::OperationOutcome; - use chat2db_java_bridge::DeliveryOutcome; - - match error { - BridgeError::Remote(remote) => match remote.outcome { - OperationOutcome::NotApplicable | OperationOutcome::NotStarted => { - DatabaseWriteOutcome::NotStarted - } - OperationOutcome::KnownFailed => DatabaseWriteOutcome::Failed, - OperationOutcome::Unknown | OperationOutcome::Unspecified => { - DatabaseWriteOutcome::Unknown - } - }, - BridgeError::CommandChannelClosed { outcome } - | BridgeError::RequestTimeout { outcome, .. } - | BridgeError::ProcessUnavailable { outcome, .. } => match outcome { - DeliveryOutcome::NotSent => DatabaseWriteOutcome::NotStarted, - DeliveryOutcome::Unknown => DatabaseWriteOutcome::Unknown, - }, - BridgeError::Protocol(_) - | BridgeError::UnexpectedResponse(_) - | BridgeError::Frame(_) - | BridgeError::SupervisorTask(_) - | BridgeError::ShutdownTimeout - if dispatched => - { - DatabaseWriteOutcome::Unknown - } - _ if dispatched => DatabaseWriteOutcome::Failed, - _ => DatabaseWriteOutcome::NotStarted, - } -} - fn parse_u64(value: &str, field: &str) -> Result { value.parse().map_err(|_| { AppError::invalid( diff --git a/crates/chat2db-core/src/ssh.rs b/crates/chat2db-core/src/ssh.rs new file mode 100644 index 0000000..aa5c894 --- /dev/null +++ b/crates/chat2db-core/src/ssh.rs @@ -0,0 +1,911 @@ +use std::{ + collections::HashMap, + future::Future, + net::Ipv4Addr, + sync::{Arc, OnceLock, Weak}, + time::Duration, +}; + +use chat2db_contract::{ + SshAuthentication, SshConnectionTestResult, SshDatasourcePreConnectRequest, + SshDatasourcePreConnectResult, SshHostKeyVerification, SshTunnelConfig, +}; +use russh::{ + Disconnect, + client::{self, Config, Handle}, + keys::{self, key::PrivateKeyWithHashAlg, ssh_key}, +}; +use sha2::{Digest, Sha256}; +use tokio::{ + io::copy_bidirectional, + net::{TcpListener, TcpStream}, + sync::{Mutex, oneshot}, + task::{JoinHandle, JoinSet}, +}; +use url::Url; + +use crate::{AppError, Application, native_mysql}; + +const SSH_CONNECT_TIMEOUT: Duration = Duration::from_secs(15); +const SSH_AUTH_TIMEOUT: Duration = Duration::from_secs(15); +const SSH_DISCONNECT_TIMEOUT: Duration = Duration::from_secs(5); +const SSH_CHANNEL_OPEN_TIMEOUT: Duration = Duration::from_secs(15); +const SSH_FORWARD_IDLE_TIMEOUT: Duration = Duration::from_secs(300); +const SSH_SESSION_IDLE_TIMEOUT: Duration = Duration::from_secs(600); +const SSH_KEEPALIVE_INTERVAL: Duration = Duration::from_secs(30); +const MAX_SSH_FORWARD_CONNECTIONS: usize = 16; +const MAX_SSH_HOST_BYTES: usize = 255; +const MAX_SSH_USER_BYTES: usize = 255; +const MAX_SSH_SECRET_BYTES: usize = 64 * 1024; +const MAX_SSH_KEY_PATH_BYTES: usize = 4 * 1024; +const MYSQL_DEFAULT_PORT: u16 = 3_306; + +#[derive(Clone, Copy)] +pub(crate) enum SshTunnelIdentity<'a> { + Datasource { + datasource_id: &'a str, + revision: u64, + }, + Ephemeral, +} + +#[derive(Clone, PartialEq, Eq, Hash)] +struct SshTunnelKey([u8; 32]); + +impl SshTunnelKey { + fn new( + identity: SshTunnelIdentity<'_>, + config: &SshTunnelConfig, + target_host: &str, + target_port: u16, + ) -> Self { + let mut digest = Sha256::new(); + digest.update(b"chat2db-ssh-tunnel-v1"); + match identity { + SshTunnelIdentity::Datasource { + datasource_id, + revision, + } => { + digest.update([1]); + hash_component(&mut digest, datasource_id.as_bytes()); + digest.update(revision.to_be_bytes()); + } + SshTunnelIdentity::Ephemeral => digest.update([0]), + } + hash_component(&mut digest, config.host_name.as_bytes()); + digest.update(config.port.to_be_bytes()); + hash_component(&mut digest, config.user_name.as_bytes()); + digest.update([match config.host_key_verification { + SshHostKeyVerification::KnownHosts => 0, + }]); + match config.local_port { + Some(port) => { + digest.update([1]); + digest.update(port.to_be_bytes()); + } + None => digest.update([0]), + } + match &config.authentication { + SshAuthentication::Password { password } => { + digest.update([0]); + hash_component(&mut digest, password.as_bytes()); + } + SshAuthentication::PrivateKey { + key_file, + passphrase, + } => { + digest.update([1]); + hash_component(&mut digest, key_file.as_bytes()); + match passphrase { + Some(passphrase) => { + digest.update([1]); + hash_component(&mut digest, passphrase.as_bytes()); + } + None => digest.update([0]), + } + } + } + hash_component(&mut digest, target_host.as_bytes()); + digest.update(target_port.to_be_bytes()); + Self(digest.finalize().into()) + } +} + +fn hash_component(digest: &mut Sha256, value: &[u8]) { + digest.update(value.len().to_be_bytes()); + digest.update(value); +} + +#[derive(Default)] +struct SshTunnelRegistry { + entries: Mutex>>, +} + +struct SshTunnelEntry { + current: Mutex>, +} + +impl SshTunnelRegistry { + async fn acquire_with( + &self, + key: SshTunnelKey, + opener: F, + ) -> Result + where + F: FnOnce() -> Fut, + Fut: Future>, + { + let entry = { + let mut entries = self.entries.lock().await; + entries.retain(|_, entry| entry.strong_count() > 0); + if let Some(entry) = entries.get(&key).and_then(Weak::upgrade) { + entry + } else { + let entry = Arc::new(SshTunnelEntry { + current: Mutex::new(Weak::new()), + }); + entries.insert(key, Arc::downgrade(&entry)); + entry + } + }; + + let mut current = entry.current.lock().await; + if let Some(inner) = current.upgrade() { + if inner.is_active() { + drop(current); + return Ok(SshTunnel { inner, entry }); + } + drop(inner); + *current = Weak::new(); + } + let inner = Arc::new(opener().await?); + *current = Arc::downgrade(&inner); + drop(current); + Ok(SshTunnel { inner, entry }) + } +} + +fn ssh_tunnel_registry() -> &'static SshTunnelRegistry { + static REGISTRY: OnceLock = OnceLock::new(); + REGISTRY.get_or_init(SshTunnelRegistry::default) +} + +impl Application { + /// Tests SSH transport, host-key verification, and authentication without opening a tunnel. + /// + /// # Errors + /// + /// Returns validation or a secret-safe SSH availability failure. + pub async fn test_ssh_connection( + &self, + config: SshTunnelConfig, + ) -> Result { + let verification = config.host_key_verification; + let mut session = connect_authenticated(&config).await?; + disconnect(&mut session).await?; + Ok(SshConnectionTestResult { + verified: true, + host_key_verification: verification, + }) + } + + /// Tests an unsaved datasource directly or through an ephemeral SSH local forward. + /// + /// Only native `MySQL` uses this tunnel path. The listener binds to loopback, the database URL + /// is rewritten in memory, and the SSH session is closed before the method returns. + /// + /// # Errors + /// + /// Returns validation, driver, SSH, or database failures without exposing credentials. + pub async fn test_datasource_connection_with_ssh( + &self, + request: SshDatasourcePreConnectRequest, + ) -> Result { + let Some(ssh) = request.ssh else { + self.test_datasource_connection(&request.driver_id, request.connection) + .await?; + return Ok(SshDatasourcePreConnectResult { + verified: true, + local_port: None, + }); + }; + self.require_managed_driver(&request.driver_id)?; + if !self.is_native_mysql_driver(&request.driver_id) { + return Err(AppError::invalid( + "ssh_driver_not_supported", + "SSH forwarding is currently implemented for native MySQL only", + )); + } + let mut forwarded = request.connection; + forwarded.ssh = Some(ssh); + let local_port = native_mysql::test_connection_with_local_port(&forwarded) + .await? + .ok_or_else(AppError::internal)?; + Ok(SshDatasourcePreConnectResult { + verified: true, + local_port: Some(local_port), + }) + } +} + +struct HostKeyHandler { + host: String, + port: u16, +} + +impl client::Handler for HostKeyHandler { + type Error = russh::Error; + + async fn check_server_key( + &mut self, + server_public_key: &ssh_key::PublicKey, + ) -> Result { + keys::check_known_hosts(&self.host, self.port, server_public_key).map_err(Into::into) + } +} + +async fn connect_authenticated( + config: &SshTunnelConfig, +) -> Result, AppError> { + validate_ssh_config(config)?; + let client_config = Arc::new(Config { + nodelay: true, + inactivity_timeout: Some(SSH_SESSION_IDLE_TIMEOUT), + keepalive_interval: Some(SSH_KEEPALIVE_INTERVAL), + keepalive_max: 3, + ..Config::default() + }); + let handler = HostKeyHandler { + host: config.host_name.clone(), + port: config.port, + }; + let mut session = tokio::time::timeout( + SSH_CONNECT_TIMEOUT, + client::connect( + client_config, + (config.host_name.as_str(), config.port), + handler, + ), + ) + .await + .map_err(|_| ssh_unavailable())? + .map_err(|_| ssh_unavailable())?; + + let authenticated = match &config.authentication { + SshAuthentication::Password { password } => tokio::time::timeout( + SSH_AUTH_TIMEOUT, + session.authenticate_password(config.user_name.clone(), password.clone()), + ) + .await + .map_err(|_| ssh_unavailable())? + .map_err(|_| ssh_unavailable())? + .success(), + SshAuthentication::PrivateKey { + key_file, + passphrase, + } => { + let key_file = key_file.clone(); + let passphrase = passphrase.clone(); + let private_key = tokio::task::spawn_blocking(move || { + keys::load_secret_key(key_file, passphrase.as_deref()) + }) + .await + .map_err(|_| AppError::internal())? + .map_err(|_| { + AppError::invalid( + "ssh_private_key_invalid", + "The selected SSH private key could not be loaded", + ) + })?; + let hash = tokio::time::timeout(SSH_AUTH_TIMEOUT, session.best_supported_rsa_hash()) + .await + .map_err(|_| ssh_unavailable())? + .map_err(|_| ssh_unavailable())? + .flatten(); + tokio::time::timeout( + SSH_AUTH_TIMEOUT, + session.authenticate_publickey( + config.user_name.clone(), + PrivateKeyWithHashAlg::new(Arc::new(private_key), hash), + ), + ) + .await + .map_err(|_| ssh_unavailable())? + .map_err(|_| ssh_unavailable())? + .success() + } + }; + if !authenticated { + let _ = disconnect(&mut session).await; + return Err(AppError::unavailable( + "ssh_authentication_failed", + "SSH authentication was rejected", + )); + } + Ok(session) +} + +pub(crate) struct SshTunnel { + inner: Arc, + entry: Arc, +} + +struct SshTunnelInner { + local_port: u16, + shutdown: Option>, + task: Option>>, +} + +impl SshTunnel { + pub(crate) async fn open( + identity: SshTunnelIdentity<'_>, + config: &SshTunnelConfig, + target_host: String, + target_port: u16, + ) -> Result { + let key = SshTunnelKey::new(identity, config, &target_host, target_port); + ssh_tunnel_registry() + .acquire_with(key, || async move { + SshTunnelInner::open(config, target_host, target_port).await + }) + .await + } + + pub(crate) fn local_port(&self) -> u16 { + self.inner.local_port + } + + pub(crate) async fn close(self) -> Result<(), AppError> { + let Self { inner, entry } = self; + let mut current = entry.current.lock().await; + let own_tunnel = Arc::downgrade(&inner); + match Arc::try_unwrap(inner) { + Ok(inner) => { + if current.ptr_eq(&own_tunnel) { + *current = Weak::new(); + } + let result = inner.close().await; + drop(current); + result + } + Err(inner) => { + drop(inner); + drop(current); + Ok(()) + } + } + } +} + +impl SshTunnelInner { + fn is_active(&self) -> bool { + self.task.as_ref().is_some_and(|task| !task.is_finished()) + } + + async fn open( + config: &SshTunnelConfig, + target_host: String, + target_port: u16, + ) -> Result { + validate_ssh_config(config)?; + let requested_port = config.local_port.unwrap_or(0); + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, requested_port)) + .await + .map_err(|_| { + AppError::unavailable( + "ssh_tunnel_bind_failed", + "The SSH loopback tunnel port is unavailable", + ) + })?; + let local_port = listener.local_addr().map_err(|_| ssh_unavailable())?.port(); + let session = connect_authenticated(config).await?; + let (shutdown, receiver) = oneshot::channel(); + let task = tokio::spawn(run_tunnel( + session, + listener, + target_host, + target_port, + receiver, + )); + Ok(Self { + local_port, + shutdown: Some(shutdown), + task: Some(task), + }) + } + + async fn close(mut self) -> Result<(), AppError> { + if let Some(shutdown) = self.shutdown.take() { + let _ = shutdown.send(()); + } + let Some(task) = self.task.take() else { + return Ok(()); + }; + task.await.map_err(|_| AppError::internal())? + } +} + +impl Drop for SshTunnelInner { + fn drop(&mut self) { + if let Some(shutdown) = self.shutdown.take() { + let _ = shutdown.send(()); + } + if let Some(task) = self.task.take() { + task.abort(); + } + } +} + +async fn run_tunnel( + mut session: Handle, + listener: TcpListener, + target_host: String, + target_port: u16, + mut shutdown: oneshot::Receiver<()>, +) -> Result<(), AppError> { + let mut transfers = JoinSet::new(); + loop { + tokio::select! { + _ = &mut shutdown => break, + accepted = listener.accept(), if transfers.len() < MAX_SSH_FORWARD_CONNECTIONS => { + let (socket, origin) = accepted.map_err(|_| ssh_unavailable())?; + match tokio::time::timeout( + SSH_CHANNEL_OPEN_TIMEOUT, + session.channel_open_direct_tcpip( + target_host.clone(), + u32::from(target_port), + origin.ip().to_string(), + u32::from(origin.port()), + ), + ).await + { + Ok(Ok(channel)) => { + transfers.spawn(forward_connection(socket, channel.into_stream())); + } + Ok(Err(_)) | Err(_) => { + tracing::warn!("SSH server rejected a direct-tcpip channel"); + } + } + } + Some(_) = transfers.join_next(), if !transfers.is_empty() => {} + } + } + disconnect(&mut session).await?; + transfers.abort_all(); + while transfers.join_next().await.is_some() {} + Ok(()) +} + +async fn forward_connection(mut socket: TcpStream, mut channel: S) +where + S: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin, +{ + match tokio::time::timeout( + SSH_FORWARD_IDLE_TIMEOUT, + copy_bidirectional(&mut socket, &mut channel), + ) + .await + { + Ok(Ok(_)) => {} + Ok(Err(_)) => tracing::debug!("SSH forwarded connection closed with an I/O error"), + Err(_) => tracing::debug!("SSH forwarded connection exceeded its idle lifetime"), + } +} + +async fn disconnect(session: &mut Handle) -> Result<(), AppError> { + tokio::time::timeout( + SSH_DISCONNECT_TIMEOUT, + session.disconnect(Disconnect::ByApplication, "", "English"), + ) + .await + .map_err(|_| ssh_unavailable())? + .map_err(|_| ssh_unavailable()) +} + +fn validate_ssh_config(config: &SshTunnelConfig) -> Result<(), AppError> { + if config.host_name.trim().is_empty() || config.host_name.len() > MAX_SSH_HOST_BYTES { + return Err(AppError::invalid( + "invalid_ssh_config", + "SSH hostname must be non-empty and at most 255 UTF-8 bytes", + )); + } + if config.port == 0 { + return Err(AppError::invalid( + "invalid_ssh_config", + "SSH port must be greater than zero", + )); + } + if config.user_name.trim().is_empty() || config.user_name.len() > MAX_SSH_USER_BYTES { + return Err(AppError::invalid( + "invalid_ssh_config", + "SSH username must be non-empty and at most 255 UTF-8 bytes", + )); + } + if config.host_key_verification != SshHostKeyVerification::KnownHosts { + return Err(AppError::invalid( + "invalid_ssh_host_key_policy", + "SSH host keys must be verified through the user's OpenSSH known_hosts file", + )); + } + match &config.authentication { + SshAuthentication::Password { password } + if password.is_empty() || password.len() > MAX_SSH_SECRET_BYTES => + { + Err(AppError::invalid( + "invalid_ssh_config", + "SSH password must be non-empty and at most 65536 UTF-8 bytes", + )) + } + SshAuthentication::PrivateKey { + key_file, + passphrase, + } if key_file.trim().is_empty() + || key_file.len() > MAX_SSH_KEY_PATH_BYTES + || passphrase + .as_ref() + .is_some_and(|value| value.len() > MAX_SSH_SECRET_BYTES) => + { + Err(AppError::invalid( + "invalid_ssh_config", + "SSH private-key settings exceed their allowed size", + )) + } + _ => Ok(()), + } +} + +pub(crate) fn mysql_target(jdbc_url: &str) -> Result<(String, u16), AppError> { + let parsed = parse_mysql_url(jdbc_url)?; + let host = parsed + .host_str() + .filter(|host| !host.is_empty()) + .ok_or_else(invalid_mysql_ssh_url)? + .to_owned(); + Ok((host, parsed.port().unwrap_or(MYSQL_DEFAULT_PORT))) +} + +pub(crate) fn rewrite_mysql_target(jdbc_url: &str, local_port: u16) -> Result { + let has_jdbc_prefix = jdbc_url.trim().starts_with("jdbc:"); + let mut parsed = parse_mysql_url(jdbc_url)?; + parsed + .set_host(Some("127.0.0.1")) + .map_err(|_| invalid_mysql_ssh_url())?; + parsed + .set_port(Some(local_port)) + .map_err(|()| invalid_mysql_ssh_url())?; + let prefix = if has_jdbc_prefix { "jdbc:" } else { "" }; + Ok(format!("{prefix}{parsed}")) +} + +fn parse_mysql_url(jdbc_url: &str) -> Result { + let raw = jdbc_url + .trim() + .strip_prefix("jdbc:") + .unwrap_or(jdbc_url.trim()); + let parsed = Url::parse(raw).map_err(|_| invalid_mysql_ssh_url())?; + if parsed.scheme() != "mysql" { + return Err(invalid_mysql_ssh_url()); + } + Ok(parsed) +} + +fn invalid_mysql_ssh_url() -> AppError { + AppError::invalid( + "invalid_mysql_ssh_url", + "The MySQL URL does not contain a valid tunnel target", + ) +} + +fn ssh_unavailable() -> AppError { + AppError::unavailable( + "ssh_connection_failed", + "The SSH connection or tunnel could not be established", + ) +} + +#[cfg(test)] +mod tests { + use std::{ + net::Ipv4Addr, + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, + }; + + use chat2db_contract::{SshAuthentication, SshHostKeyVerification, SshTunnelConfig}; + use tokio::{net::TcpListener, sync::oneshot, task::JoinSet}; + + use super::{ + AppError, SshTunnelIdentity, SshTunnelInner, SshTunnelKey, SshTunnelRegistry, mysql_target, + rewrite_mysql_target, validate_ssh_config, + }; + + #[test] + fn mysql_target_rewrite_preserves_database_and_query() { + let url = "jdbc:mysql://db.internal:3307/example?useSSL=true"; + assert_eq!( + mysql_target(url).expect("target parses"), + ("db.internal".to_owned(), 3307) + ); + let rewritten = rewrite_mysql_target(url, 41_223).expect("URL rewrites"); + assert_eq!( + rewritten, + "jdbc:mysql://127.0.0.1:41223/example?useSSL=true" + ); + } + + #[test] + fn ssh_config_requires_endpoint_user_and_authentication_material() { + let mut config = SshTunnelConfig { + host_name: "ssh.internal".to_owned(), + port: 22, + user_name: "developer".to_owned(), + authentication: SshAuthentication::Password { + password: "secret".to_owned(), + }, + host_key_verification: SshHostKeyVerification::KnownHosts, + local_port: None, + }; + assert!(validate_ssh_config(&config).is_ok()); + config.port = 0; + assert!(validate_ssh_config(&config).is_err()); + config.port = 22; + config.authentication = SshAuthentication::Password { + password: String::new(), + }; + assert!(validate_ssh_config(&config).is_err()); + } + + #[test] + fn tunnel_key_is_scoped_by_datasource_revision_config_and_target() { + let config = ssh_config("first-secret", Some(43_210)); + let base = SshTunnelKey::new( + SshTunnelIdentity::Datasource { + datasource_id: "datasource-1", + revision: 7, + }, + &config, + "mysql.internal", + 3_306, + ); + let same = SshTunnelKey::new( + SshTunnelIdentity::Datasource { + datasource_id: "datasource-1", + revision: 7, + }, + &config, + "mysql.internal", + 3_306, + ); + assert!(base == same); + + let changed_secret = ssh_config("second-secret", Some(43_210)); + for changed in [ + SshTunnelKey::new( + SshTunnelIdentity::Datasource { + datasource_id: "datasource-2", + revision: 7, + }, + &config, + "mysql.internal", + 3_306, + ), + SshTunnelKey::new( + SshTunnelIdentity::Datasource { + datasource_id: "datasource-1", + revision: 8, + }, + &config, + "mysql.internal", + 3_306, + ), + SshTunnelKey::new( + SshTunnelIdentity::Datasource { + datasource_id: "datasource-1", + revision: 7, + }, + &changed_secret, + "mysql.internal", + 3_306, + ), + SshTunnelKey::new( + SshTunnelIdentity::Datasource { + datasource_id: "datasource-1", + revision: 7, + }, + &config, + "other.internal", + 3_306, + ), + ] { + assert!(base != changed); + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn concurrent_leases_share_one_fixed_listener_until_the_last_close() { + let port = unused_loopback_port().await; + let registry = Arc::new(SshTunnelRegistry::default()); + let opens = Arc::new(AtomicUsize::new(0)); + let shutdowns = Arc::new(AtomicUsize::new(0)); + let mut tasks = JoinSet::new(); + + for _ in 0..16 { + let registry = Arc::clone(®istry); + let opens = Arc::clone(&opens); + let shutdowns = Arc::clone(&shutdowns); + tasks.spawn(async move { + registry + .acquire_with(SshTunnelKey([7; 32]), || async move { + opens.fetch_add(1, Ordering::SeqCst); + fake_tunnel_inner(port, shutdowns).await + }) + .await + }); + } + + let mut leases = Vec::new(); + while let Some(result) = tasks.join_next().await { + leases.push(result.expect("lease task joins").expect("lease opens")); + } + assert_eq!(opens.load(Ordering::SeqCst), 1); + assert!(leases.iter().all(|lease| lease.local_port() == port)); + assert!( + TcpListener::bind((Ipv4Addr::LOCALHOST, port)) + .await + .is_err() + ); + + while leases.len() > 1 { + leases + .pop() + .expect("lease exists") + .close() + .await + .expect("lease closes"); + assert_eq!(shutdowns.load(Ordering::SeqCst), 0); + assert!( + TcpListener::bind((Ipv4Addr::LOCALHOST, port)) + .await + .is_err() + ); + } + leases + .pop() + .expect("last lease exists") + .close() + .await + .expect("last lease closes"); + assert_eq!(shutdowns.load(Ordering::SeqCst), 1); + let rebound = TcpListener::bind((Ipv4Addr::LOCALHOST, port)) + .await + .expect("fixed port is released after the final lease"); + drop(rebound); + } + + #[tokio::test] + async fn changed_scope_never_reuses_an_old_fixed_port_tunnel() { + let port = unused_loopback_port().await; + let registry = SshTunnelRegistry::default(); + let shutdowns = Arc::new(AtomicUsize::new(0)); + let old = registry + .acquire_with(SshTunnelKey([1; 32]), || { + fake_tunnel_inner(port, Arc::clone(&shutdowns)) + }) + .await + .expect("old tunnel opens"); + + let replacement = registry + .acquire_with(SshTunnelKey([2; 32]), || { + fake_tunnel_inner(port, Arc::clone(&shutdowns)) + }) + .await; + assert!( + replacement.is_err(), + "a changed scope must not reuse the old listener" + ); + assert_eq!(shutdowns.load(Ordering::SeqCst), 0); + + old.close().await.expect("old tunnel closes"); + let replacement = registry + .acquire_with(SshTunnelKey([2; 32]), || { + fake_tunnel_inner(port, Arc::clone(&shutdowns)) + }) + .await + .expect("replacement uses the configured port after old leases close"); + assert_eq!(replacement.local_port(), port); + replacement.close().await.expect("replacement closes"); + assert_eq!(shutdowns.load(Ordering::SeqCst), 2); + } + + #[tokio::test] + async fn failed_open_leaves_no_registry_entry_or_listener() { + let port = unused_loopback_port().await; + let registry = SshTunnelRegistry::default(); + let attempts = Arc::new(AtomicUsize::new(0)); + let shutdowns = Arc::new(AtomicUsize::new(0)); + + let result = registry + .acquire_with(SshTunnelKey([9; 32]), || { + let attempts = Arc::clone(&attempts); + async move { + attempts.fetch_add(1, Ordering::SeqCst); + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, port)) + .await + .map_err(|_| AppError::internal())?; + drop(listener); + Err(AppError::unavailable( + "test_tunnel_open_failed", + "test tunnel failed", + )) + } + }) + .await; + assert!(result.is_err()); + + let lease = registry + .acquire_with(SshTunnelKey([9; 32]), || { + attempts.fetch_add(1, Ordering::SeqCst); + fake_tunnel_inner(port, Arc::clone(&shutdowns)) + }) + .await + .expect("a failed open does not poison the key or retain its listener"); + assert_eq!(attempts.load(Ordering::SeqCst), 2); + lease.close().await.expect("retry lease closes"); + assert_eq!(shutdowns.load(Ordering::SeqCst), 1); + } + + fn ssh_config(password: &str, local_port: Option) -> SshTunnelConfig { + SshTunnelConfig { + host_name: "ssh.internal".to_owned(), + port: 22, + user_name: "developer".to_owned(), + authentication: SshAuthentication::Password { + password: password.to_owned(), + }, + host_key_verification: SshHostKeyVerification::KnownHosts, + local_port, + } + } + + async fn unused_loopback_port() -> u16 { + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)) + .await + .expect("an ephemeral port is available"); + listener + .local_addr() + .expect("listener has an address") + .port() + } + + async fn fake_tunnel_inner( + port: u16, + shutdowns: Arc, + ) -> Result { + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, port)) + .await + .map_err(|_| { + AppError::unavailable( + "ssh_tunnel_bind_failed", + "The SSH loopback tunnel port is unavailable", + ) + })?; + let local_port = listener + .local_addr() + .map_err(|_| AppError::internal())? + .port(); + let (shutdown, receiver) = oneshot::channel(); + let task = tokio::spawn(async move { + let _listener = listener; + let _ = receiver.await; + shutdowns.fetch_add(1, Ordering::SeqCst); + Ok(()) + }); + Ok(SshTunnelInner { + local_port, + shutdown: Some(shutdown), + task: Some(task), + }) + } +} diff --git a/crates/chat2db-core/src/transfer/class_generation.rs b/crates/chat2db-core/src/transfer/class_generation.rs new file mode 100644 index 0000000..f54d5cf --- /dev/null +++ b/crates/chat2db-core/src/transfer/class_generation.rs @@ -0,0 +1,527 @@ +use std::{ + collections::BTreeSet, + fs::{self, File, OpenOptions}, + io::{Seek, Write}, + path::{Path, PathBuf}, +}; + +use chat2db_contract::{CommunityTableColumn, GenerateMysqlClassRequest, GeneratedMysqlClassSet}; +use chat2db_storage::TransferArtifactRecord; +use uuid::Uuid; +use zip::{CompressionMethod, ZipWriter, write::SimpleFileOptions}; + +use crate::{AppError, Application, native_mysql, now_millis}; + +const CLASS_ARCHIVE_TTL_MS: i64 = 24 * 60 * 60 * 1_000; +const MAX_CLASS_ARCHIVE_BYTES: u64 = 16 * 1024 * 1024; + +struct RenderedClassSet { + directory_name: String, + files: Vec<(String, String)>, +} + +pub(super) async fn generate( + application: &Application, + request: GenerateMysqlClassRequest, +) -> Result { + validate_desktop_request(&request)?; + let rendered = render_request(application, &request).await?; + + tokio::task::spawn_blocking(move || write_class_set(&request.export_path, rendered)) + .await + .map_err(|_| AppError::internal())? +} + +pub(super) async fn generate_archive( + application: &Application, + request: GenerateMysqlClassRequest, +) -> Result { + if !request.export_path.trim().is_empty() { + return Err(AppError::invalid( + "invalid_class_export_path", + "Web class generation does not accept exportPath", + )); + } + validate_table_name(&request.table_name)?; + let rendered = render_request(application, &request).await?; + let storage = application.require_storage()?; + let file_name = format!( + "{}-mybatis.zip", + safe_archive_component(&request.table_name) + ); + let expires_at_ms = now_millis()?.saturating_add(CLASS_ARCHIVE_TTL_MS); + tokio::task::spawn_blocking(move || { + let mut writer = storage + .begin_transfer_artifact( + None, + &file_name, + "application/zip", + "ZIP", + "zip", + Some(expires_at_ms), + ) + .map_err(AppError::from)?; + write_class_archive(writer.file_mut(), &rendered)?; + let byte_count = writer.file_mut().metadata().map_err(file_error)?.len(); + if byte_count > MAX_CLASS_ARCHIVE_BYTES { + return Err(AppError::invalid( + "class_archive_limit_exceeded", + "The generated class archive is too large", + )); + } + writer.finish().map_err(AppError::from) + }) + .await + .map_err(|_| AppError::internal())? +} + +async fn render_request( + application: &Application, + request: &GenerateMysqlClassRequest, +) -> Result { + let columns = native_mysql::list_columns( + application, + &request.datasource_id, + &request.database_name, + &request.schema_name, + &request.table_name, + ) + .await? + .items; + if columns.is_empty() { + return Err(AppError::not_found( + "mysql_table_not_found", + "The selected MySQL table does not exist", + )); + } + render_class_set(&request.table_name, &columns) +} + +fn validate_desktop_request(request: &GenerateMysqlClassRequest) -> Result<(), AppError> { + if request.export_path.trim().is_empty() || request.export_path.contains('\0') { + return Err(AppError::invalid( + "invalid_class_export_path", + "exportPath must be a local directory", + )); + } + validate_table_name(&request.table_name) +} + +fn validate_table_name(table_name: &str) -> Result<(), AppError> { + if table_name.trim().is_empty() + || table_name.len() > 256 + || table_name.contains(['/', '\\', '\0']) + || matches!(table_name, "." | "..") + { + return Err(AppError::invalid( + "invalid_mysql_table_name", + "tableName cannot be used as an output directory", + )); + } + Ok(()) +} + +fn write_class_set( + export_path: &str, + rendered: RenderedClassSet, +) -> Result { + let base = PathBuf::from(export_path); + fs::create_dir_all(&base).map_err(file_error)?; + let base = fs::canonicalize(&base).map_err(file_error)?; + let output = base.join(&rendered.directory_name); + fs::create_dir_all(&output).map_err(file_error)?; + let output = fs::canonicalize(&output).map_err(file_error)?; + if !output.starts_with(&base) { + return Err(AppError::invalid( + "invalid_class_export_path", + "The generated output directory escaped exportPath", + )); + } + + let mut written = Vec::with_capacity(rendered.files.len()); + for (name, contents) in rendered.files { + let path = output.join(name); + atomic_write(&path, contents.as_bytes())?; + written.push(path.to_string_lossy().into_owned()); + } + Ok(GeneratedMysqlClassSet { + output_directory: output.to_string_lossy().into_owned(), + files: written, + }) +} + +fn render_class_set( + table_name: &str, + columns: &[CommunityTableColumn], +) -> Result { + let class_name = format!("{}DO", upper_camel(table_name)); + let entity_name = format!("{class_name}.java"); + let mapper_name = format!("{}Mapper.java", upper_camel(table_name)); + let xml_name = format!("{}Mapper.xml", upper_camel(table_name)); + let files = vec![ + (entity_name, render_entity(&class_name, table_name, columns)), + (mapper_name, render_mapper(&class_name, table_name)), + (xml_name, render_mapper_xml(table_name)), + ]; + let total_bytes = files.iter().try_fold(0_u64, |total, (_, contents)| { + total.checked_add(u64::try_from(contents.len()).ok()?) + }); + if total_bytes.is_none_or(|total| total > MAX_CLASS_ARCHIVE_BYTES) { + return Err(AppError::invalid( + "class_archive_limit_exceeded", + "The generated class files are too large", + )); + } + Ok(RenderedClassSet { + directory_name: table_name.to_owned(), + files, + }) +} + +fn write_class_archive( + output: W, + rendered: &RenderedClassSet, +) -> Result<(), AppError> { + let directory = safe_archive_component(&rendered.directory_name); + let mut zip = ZipWriter::new(output); + let options = SimpleFileOptions::default().compression_method(CompressionMethod::Deflated); + for (name, contents) in &rendered.files { + zip.start_file(format!("{directory}/{name}"), options) + .map_err(|error| zip_error(&error))?; + zip.write_all(contents.as_bytes()).map_err(file_error)?; + } + zip.finish().map_err(|error| zip_error(&error))?; + Ok(()) +} + +fn safe_archive_component(value: &str) -> String { + let value: String = value + .chars() + .map(|character| { + if character.is_ascii_alphanumeric() || matches!(character, '.' | '-' | '_') { + character + } else { + '_' + } + }) + .collect(); + if value.is_empty() { + "generated".to_owned() + } else { + value + } +} + +fn render_entity(class_name: &str, table_name: &str, columns: &[CommunityTableColumn]) -> String { + let mut imports = BTreeSet::from([ + "com.baomidou.mybatisplus.annotation.TableField", + "com.baomidou.mybatisplus.annotation.TableName", + "lombok.Data", + ]); + if columns + .iter() + .any(|column| column.primary_key == Some(true)) + { + imports.insert("com.baomidou.mybatisplus.annotation.TableId"); + } + for column in columns { + match java_type(&column.column_type) { + "BigDecimal" => { + imports.insert("java.math.BigDecimal"); + } + "LocalDate" => { + imports.insert("java.time.LocalDate"); + } + "LocalDateTime" => { + imports.insert("java.time.LocalDateTime"); + } + "LocalTime" => { + imports.insert("java.time.LocalTime"); + } + _ => {} + } + } + + let mut output = String::from("package com.my.entity;\n\n"); + for import in imports { + output.push_str("import "); + output.push_str(import); + output.push_str(";\n"); + } + output.push_str("\n@Data\n@TableName(\""); + output.push_str(&java_string(table_name)); + output.push_str("\")\npublic class "); + output.push_str(class_name); + output.push_str(" {\n"); + + let mut table_id_written = false; + for column in columns { + if column.comment.trim().is_empty() { + output.push('\n'); + } else { + output.push_str("\n /** "); + output.push_str(&javadoc(&column.comment)); + output.push_str(" */\n"); + } + if column.primary_key == Some(true) && !table_id_written { + output.push_str(" @TableId(\""); + table_id_written = true; + } else { + output.push_str(" @TableField(\""); + } + output.push_str(&java_string(&column.name)); + output.push_str("\")\n private "); + output.push_str(java_type(&column.column_type)); + output.push(' '); + output.push_str(&lower_camel(&column.name)); + output.push_str(";\n"); + } + output.push_str("}\n"); + output +} + +fn render_mapper(class_name: &str, table_name: &str) -> String { + let mapper_name = format!("{}Mapper", upper_camel(table_name)); + format!( + "package com.my.mapper;\n\n\ + import com.baomidou.mybatisplus.core.mapper.BaseMapper;\n\ + import com.my.entity.{class_name};\n\ + import org.apache.ibatis.annotations.Mapper;\n\n\ + @Mapper\n\ + public interface {mapper_name} extends BaseMapper<{class_name}> {{\n}}\n" + ) +} + +fn render_mapper_xml(table_name: &str) -> String { + let mapper_name = format!("{}Mapper", upper_camel(table_name)); + format!( + "\n\ + \n\ + \n\ + \n" + ) +} + +fn java_type(mysql_type: &str) -> &'static str { + match mysql_type + .split_ascii_whitespace() + .next() + .unwrap_or_default() + .to_ascii_uppercase() + .as_str() + { + "BIGINT" => "Long", + "TINYINT" | "SMALLINT" | "MEDIUMINT" | "INT" | "INTEGER" | "YEAR" => "Integer", + "DECIMAL" | "NUMERIC" => "BigDecimal", + "FLOAT" => "Float", + "DOUBLE" | "REAL" => "Double", + "BIT" | "BOOL" | "BOOLEAN" => "Boolean", + "DATE" => "LocalDate", + "TIME" => "LocalTime", + "DATETIME" | "TIMESTAMP" => "LocalDateTime", + "BINARY" | "VARBINARY" | "TINYBLOB" | "BLOB" | "MEDIUMBLOB" | "LONGBLOB" => "byte[]", + _ => "String", + } +} + +fn upper_camel(value: &str) -> String { + let mut output = String::new(); + let mut uppercase = true; + for character in value.chars() { + if character.is_alphanumeric() { + if uppercase { + output.extend(character.to_uppercase()); + uppercase = false; + } else { + output.push(character); + } + } else { + uppercase = true; + } + } + valid_java_identifier(output, "Generated") +} + +fn lower_camel(value: &str) -> String { + let upper = upper_camel(value); + let mut characters = upper.chars(); + let Some(first) = characters.next() else { + return "generated".to_owned(); + }; + let mut output = first.to_lowercase().collect::(); + output.extend(characters); + valid_java_identifier(output, "generated") +} + +fn valid_java_identifier(mut value: String, fallback: &str) -> String { + if value.is_empty() { + value.push_str(fallback); + } + if value.starts_with(|character: char| character.is_ascii_digit()) { + value.insert(0, '_'); + } + if JAVA_KEYWORDS.contains(&value.as_str()) { + value.push('_'); + } + value +} + +fn atomic_write(path: &Path, contents: &[u8]) -> Result<(), AppError> { + let file_name = path + .file_name() + .and_then(|value| value.to_str()) + .ok_or_else(AppError::internal)?; + let part = path.with_file_name(format!(".{file_name}.{}.part", Uuid::new_v4())); + let result = (|| { + let mut file = OpenOptions::new() + .create_new(true) + .write(true) + .open(&part) + .map_err(file_error)?; + file.write_all(contents).map_err(file_error)?; + file.sync_all().map_err(file_error)?; + drop(file); + fs::rename(&part, path).map_err(file_error)?; + sync_parent(path)?; + Ok(()) + })(); + if result.is_err() { + let _ = fs::remove_file(part); + } + result +} + +fn sync_parent(path: &Path) -> Result<(), AppError> { + let parent = path.parent().ok_or_else(AppError::internal)?; + File::open(parent) + .and_then(|directory| directory.sync_all()) + .map_err(file_error) +} + +fn java_string(value: &str) -> String { + value + .replace('\\', "\\\\") + .replace('"', "\\\"") + .replace('\r', "\\r") + .replace('\n', "\\n") +} + +fn javadoc(value: &str) -> String { + value + .replace("*/", "* /") + .replace(['\r', '\n'], " ") + .trim() + .to_owned() +} + +fn file_error(error: std::io::Error) -> AppError { + tracing::warn!(%error, "MyBatis Plus class export filesystem operation failed"); + drop(error); + AppError::unavailable( + "class_export_failed", + "The MyBatis Plus class files could not be written", + ) +} + +fn zip_error(error: &zip::result::ZipError) -> AppError { + tracing::warn!(%error, "MyBatis Plus class archive generation failed"); + AppError::unavailable( + "class_archive_failed", + "The MyBatis Plus class archive could not be generated", + ) +} + +const JAVA_KEYWORDS: &[&str] = &[ + "abstract", + "assert", + "boolean", + "break", + "byte", + "case", + "catch", + "char", + "class", + "const", + "continue", + "default", + "do", + "double", + "else", + "enum", + "extends", + "final", + "finally", + "float", + "for", + "goto", + "if", + "implements", + "import", + "instanceof", + "int", + "interface", + "long", + "native", + "new", + "package", + "private", + "protected", + "public", + "return", + "short", + "static", + "strictfp", + "super", + "switch", + "synchronized", + "this", + "throw", + "throws", + "transient", + "try", + "void", + "volatile", + "while", +]; + +#[cfg(test)] +mod tests { + use std::io::{Cursor, Read as _}; + + use super::{ + RenderedClassSet, java_type, lower_camel, safe_archive_component, upper_camel, + write_class_archive, + }; + + #[test] + fn java_names_and_types_are_stable() { + assert_eq!(upper_camel("audit_log"), "AuditLog"); + assert_eq!(lower_camel("user-id"), "userId"); + assert_eq!(lower_camel("class"), "class_"); + assert_eq!(java_type("BIGINT UNSIGNED"), "Long"); + assert_eq!(java_type("TIMESTAMP"), "LocalDateTime"); + } + + #[test] + fn archive_entries_are_relative_and_reuse_rendered_bytes() { + let rendered = RenderedClassSet { + directory_name: "audit log".to_owned(), + files: vec![( + "AuditLogDO.java".to_owned(), + "class AuditLogDO {}\n".to_owned(), + )], + }; + let mut output = Cursor::new(Vec::new()); + write_class_archive(&mut output, &rendered).expect("archive writes"); + output.set_position(0); + let mut archive = zip::ZipArchive::new(output).expect("archive opens"); + let mut entry = archive + .by_name("audit_log/AuditLogDO.java") + .expect("safe relative entry exists"); + let mut contents = String::new(); + entry.read_to_string(&mut contents).expect("entry reads"); + assert_eq!(contents, "class AuditLogDO {}\n"); + assert_eq!(safe_archive_component("../unsafe"), ".._unsafe"); + } +} diff --git a/crates/chat2db-core/src/transfer/format.rs b/crates/chat2db-core/src/transfer/format.rs new file mode 100644 index 0000000..e7abf94 --- /dev/null +++ b/crates/chat2db-core/src/transfer/format.rs @@ -0,0 +1,1069 @@ +use std::{ + fs::File, + io::{Read, Seek, SeekFrom, Write}, + path::Path, +}; + +use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD}; +use chat2db_contract::{ApiError, TabularImportEncoding, TransferFileFormat}; +use xls::core::{Cell, Workbook}; +use zip::ZipArchive; + +use crate::{AppError, AppErrorKind}; + +pub(crate) const MAX_IMPORT_FILE_BYTES: u64 = 128 * 1024 * 1024; +const MAX_TABULAR_CELLS: usize = 2_000_000; +const MAX_TABULAR_COLUMNS: usize = 16_384; +const MAX_CELL_BYTES: usize = 16 * 1024 * 1024; +const CELL_ENCODING_PREFIX: &str = "__CHAT2DB_TRANSFER_V1__:"; +const NULL_ENCODING: &str = "__CHAT2DB_TRANSFER_V1__:NULL"; +const TEXT_ENCODING_PREFIX: &str = "__CHAT2DB_TRANSFER_V1__:TEXT:"; +const BYTES_ENCODING_PREFIX: &str = "__CHAT2DB_TRANSFER_V1__:BASE64:"; +const MAX_XLSX_ZIP_ENTRIES: usize = 1_024; +const MAX_XLSX_ENTRY_BYTES: u64 = 64 * 1024 * 1024; +const MAX_XLSX_TOTAL_BYTES: u64 = 256 * 1024 * 1024; +const MIN_XLSX_RATIO_CHECK_BYTES: u64 = 1024 * 1024; +const MAX_XLSX_COMPRESSION_RATIO: u64 = 200; +const ZIP_EOCD_SIGNATURE: [u8; 4] = *b"PK\x05\x06"; +const ZIP64_EOCD_SIGNATURE: [u8; 4] = *b"PK\x06\x06"; +const ZIP64_LOCATOR_SIGNATURE: [u8; 4] = *b"PK\x06\x07"; +const ZIP_EOCD_BYTES: usize = 22; +const ZIP_MAX_COMMENT_BYTES: usize = u16::MAX as usize; +const ZIP64_LOCATOR_BYTES: u64 = 20; +const ZIP64_EOCD_MIN_BYTES: usize = 56; + +/// Version-one tabular values are stored as readable text except for values that +/// need an explicit envelope: NULL, reserved-prefix text, and raw bytes. +/// `CELL_ENCODING_PREFIX` is a reserved namespace on import; exports escape any +/// user text in that namespace through the TEXT envelope before writing it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum TabularValue { + Null, + Text(String), + Bytes(Vec), +} + +#[derive(Debug)] +pub(crate) struct ImportedTable { + pub(crate) columns: Option>, + pub(crate) rows: Vec>, +} + +pub(crate) trait TabularSink: Send { + fn write_header(&mut self, columns: &[String]) -> Result<(), AppError>; + fn write_row(&mut self, values: &[TabularValue]) -> Result<(), AppError>; + fn finish(&mut self) -> Result<(), AppError>; +} + +pub(crate) fn tabular_sink<'a>( + format: TransferFileFormat, + file: &'a mut File, + contains_header: bool, +) -> Result, AppError> { + match format { + TransferFileFormat::Csv => Ok(Box::new(CsvSink { + writer: csv::WriterBuilder::new().from_writer(file), + contains_header, + columns: None, + })), + TransferFileFormat::Xls | TransferFileFormat::Xlsx => Ok(Box::new(SpreadsheetSink { + file, + format, + contains_header, + workbook: Workbook::new(), + next_row: 0, + columns: 0, + cells: 0, + finished: false, + })), + TransferFileFormat::Sql => Err(AppError::invalid( + "invalid_transfer_format", + "SQL exports use the SQL dump writer", + )), + } +} + +pub(crate) fn read_tabular_file( + path: &Path, + format: TransferFileFormat, + contains_header: bool, + tabular_encoding: TabularImportEncoding, +) -> Result { + validate_import_file(path)?; + match format { + TransferFileFormat::Csv => read_csv(path, contains_header, tabular_encoding), + TransferFileFormat::Xls | TransferFileFormat::Xlsx => { + read_spreadsheet(path, format, contains_header, tabular_encoding) + } + TransferFileFormat::Sql => Err(AppError::invalid( + "invalid_transfer_format", + "SQL input is not a tabular file", + )), + } +} + +pub(crate) fn validate_import_file(path: &Path) -> Result { + let metadata = std::fs::symlink_metadata(path).map_err(|_| { + AppError::not_found( + "import_file_not_found", + "The selected import file does not exist", + ) + })?; + if metadata.file_type().is_symlink() || !metadata.is_file() { + return Err(AppError::invalid( + "invalid_import_file", + "The selected import path must be a regular file", + )); + } + if metadata.len() > MAX_IMPORT_FILE_BYTES { + return Err(resource_error( + "import_file_too_large", + format!("Import files are limited to {MAX_IMPORT_FILE_BYTES} bytes"), + )); + } + Ok(metadata.len()) +} + +fn read_csv( + path: &Path, + contains_header: bool, + tabular_encoding: TabularImportEncoding, +) -> Result { + let mut reader = csv::ReaderBuilder::new() + .has_headers(contains_header) + .flexible(false) + .from_path(path) + .map_err(format_error)?; + let columns = contains_header + .then(|| { + reader + .headers() + .map(|headers| headers.iter().map(str::to_owned).collect::>()) + .map_err(format_error) + }) + .transpose()?; + validate_columns(columns.as_deref())?; + let mut rows = Vec::new(); + let mut cells = columns.as_ref().map_or(0, Vec::len); + for record in reader.records() { + let record = record.map_err(format_error)?; + cells = cells + .checked_add(record.len()) + .ok_or_else(|| resource_error("tabular_file_too_large", "Too many cells"))?; + enforce_cell_budget(cells)?; + let mut row = Vec::with_capacity(record.len()); + for value in &record { + enforce_cell_size(value)?; + row.push(decode_tabular_value(value, tabular_encoding)?); + } + rows.push(row); + } + validate_row_widths(columns.as_deref(), &rows)?; + Ok(ImportedTable { columns, rows }) +} + +fn read_spreadsheet( + path: &Path, + format: TransferFileFormat, + contains_header: bool, + tabular_encoding: TabularImportEncoding, +) -> Result { + if format == TransferFileFormat::Xlsx { + validate_xlsx_archive(path)?; + } + let file = File::open(path).map_err(|_| { + AppError::not_found( + "import_file_not_found", + "The selected import file could not be opened", + ) + })?; + let workbook = match format { + TransferFileFormat::Xls => xls::core::xls::read(file), + TransferFileFormat::Xlsx => xls::core::xlsx::read(file), + TransferFileFormat::Csv | TransferFileFormat::Sql => unreachable!(), + } + .map_err(format_error)?; + let sheet = workbook.sheets.first().ok_or_else(|| { + AppError::invalid( + "invalid_spreadsheet", + "The spreadsheet does not contain a worksheet", + ) + })?; + let (row_count, column_count) = sheet.dimensions(); + let row_count = usize::try_from(row_count).map_err(|_| AppError::internal())?; + let column_count = usize::try_from(column_count).map_err(|_| AppError::internal())?; + if column_count == 0 || column_count > MAX_TABULAR_COLUMNS { + return Err(resource_error( + "tabular_file_too_wide", + format!("Tabular files are limited to {MAX_TABULAR_COLUMNS} columns"), + )); + } + enforce_cell_budget(row_count.saturating_mul(column_count))?; + + let start_row = usize::from(contains_header); + let columns = contains_header.then(|| { + (0..column_count) + .map(|column| workbook.display_cell(0, 0, u32::try_from(column).unwrap_or(u32::MAX))) + .collect::>() + }); + validate_columns(columns.as_deref())?; + let mut rows = Vec::with_capacity(row_count.saturating_sub(start_row)); + for row in start_row..row_count { + let row = u32::try_from(row).map_err(|_| AppError::internal())?; + let mut values = Vec::with_capacity(column_count); + for column in 0..column_count { + let column = u32::try_from(column).map_err(|_| AppError::internal())?; + let value = if sheet.get(row, column).is_none() { + TabularValue::Null + } else { + let value = workbook.display_cell(0, row, column); + enforce_cell_size(&value)?; + decode_tabular_value(&value, tabular_encoding)? + }; + values.push(value); + } + rows.push(values); + } + validate_row_widths(columns.as_deref(), &rows)?; + Ok(ImportedTable { columns, rows }) +} + +struct CsvSink { + writer: csv::Writer, + contains_header: bool, + columns: Option, +} + +impl TabularSink for CsvSink { + fn write_header(&mut self, columns: &[String]) -> Result<(), AppError> { + validate_export_columns(columns)?; + self.columns = Some(columns.len()); + if self.contains_header { + self.writer.write_record(columns).map_err(format_error)?; + } + Ok(()) + } + + fn write_row(&mut self, values: &[TabularValue]) -> Result<(), AppError> { + validate_export_row(self.columns, values)?; + self.writer + .write_record(values.iter().map(encode_tabular_value)) + .map_err(format_error) + } + + fn finish(&mut self) -> Result<(), AppError> { + self.writer.flush().map_err(format_error) + } +} + +struct SpreadsheetSink<'a> { + file: &'a mut File, + format: TransferFileFormat, + contains_header: bool, + workbook: Workbook, + next_row: u32, + columns: usize, + cells: usize, + finished: bool, +} + +impl TabularSink for SpreadsheetSink<'_> { + fn write_header(&mut self, columns: &[String]) -> Result<(), AppError> { + validate_export_columns(columns)?; + self.columns = columns.len(); + if self.contains_header { + self.write_values( + &columns + .iter() + .cloned() + .map(TabularValue::Text) + .collect::>(), + )?; + } + Ok(()) + } + + fn write_row(&mut self, values: &[TabularValue]) -> Result<(), AppError> { + validate_export_row(Some(self.columns), values)?; + self.write_values(values) + } + + fn finish(&mut self) -> Result<(), AppError> { + if self.finished { + return Ok(()); + } + self.file.rewind().map_err(format_error)?; + self.file.set_len(0).map_err(format_error)?; + match self.format { + TransferFileFormat::Xls => xls::core::xls::write(&self.workbook, &mut *self.file), + TransferFileFormat::Xlsx => xls::core::xlsx::write(&self.workbook, &mut *self.file), + TransferFileFormat::Csv | TransferFileFormat::Sql => unreachable!(), + } + .map_err(format_error)?; + self.finished = true; + Ok(()) + } +} + +impl SpreadsheetSink<'_> { + fn write_values(&mut self, values: &[TabularValue]) -> Result<(), AppError> { + self.cells = self + .cells + .checked_add(values.len()) + .ok_or_else(|| resource_error("tabular_export_too_large", "Too many cells"))?; + enforce_cell_budget(self.cells)?; + let sheet = self.workbook.sheet_mut(0).ok_or_else(AppError::internal)?; + for (column, value) in values.iter().enumerate() { + let value = encode_tabular_value(value); + enforce_cell_size(&value)?; + sheet.set( + self.next_row, + u32::try_from(column).map_err(|_| AppError::internal())?, + Cell::Text(value), + ); + } + self.next_row = self + .next_row + .checked_add(1) + .ok_or_else(|| resource_error("tabular_export_too_large", "Too many rows"))?; + Ok(()) + } +} + +fn encode_tabular_value(value: &TabularValue) -> String { + match value { + TabularValue::Null => NULL_ENCODING.to_owned(), + TabularValue::Bytes(value) => { + format!("{BYTES_ENCODING_PREFIX}{}", URL_SAFE_NO_PAD.encode(value)) + } + TabularValue::Text(value) if value.starts_with(CELL_ENCODING_PREFIX) => { + format!("{TEXT_ENCODING_PREFIX}{}", URL_SAFE_NO_PAD.encode(value)) + } + TabularValue::Text(value) => value.clone(), + } +} + +fn decode_tabular_value( + value: &str, + tabular_encoding: TabularImportEncoding, +) -> Result { + if tabular_encoding == TabularImportEncoding::Plain { + return Ok(TabularValue::Text(value.to_owned())); + } + if value == NULL_ENCODING { + return Ok(TabularValue::Null); + } + if let Some(value) = value.strip_prefix(BYTES_ENCODING_PREFIX) { + return URL_SAFE_NO_PAD + .decode(value) + .map(TabularValue::Bytes) + .map_err(|_| invalid_cell_encoding()); + } + if let Some(value) = value.strip_prefix(TEXT_ENCODING_PREFIX) { + let value = URL_SAFE_NO_PAD + .decode(value) + .map_err(|_| invalid_cell_encoding())?; + return String::from_utf8(value) + .map(TabularValue::Text) + .map_err(|_| invalid_cell_encoding()); + } + if value.starts_with(CELL_ENCODING_PREFIX) { + return Err(invalid_cell_encoding()); + } + Ok(TabularValue::Text(value.to_owned())) +} + +fn invalid_cell_encoding() -> AppError { + AppError::invalid( + "invalid_tabular_cell_encoding", + "The Chat2DB transfer file contains an invalid encoded cell", + ) +} + +fn validate_xlsx_archive(path: &Path) -> Result<(), AppError> { + let raw_entry_count = preflight_xlsx_entry_count(path)?; + let file = File::open(path).map_err(format_error)?; + let mut archive = ZipArchive::new(file).map_err(format_error)?; + if archive.len() != raw_entry_count { + return Err(AppError::invalid( + "xlsx_archive_duplicate_entries", + "XLSX archives cannot contain duplicate ZIP entry names", + )); + } + if raw_entry_count > MAX_XLSX_ZIP_ENTRIES { + return Err(resource_error( + "xlsx_archive_too_many_entries", + format!("XLSX archives are limited to {MAX_XLSX_ZIP_ENTRIES} ZIP entries"), + )); + } + + let mut declared_total = 0_u64; + let mut actual_total = 0_u64; + for index in 0..archive.len() { + let mut entry = archive.by_index(index).map_err(format_error)?; + let declared_size = entry.size(); + let compressed_size = entry.compressed_size(); + if declared_size > MAX_XLSX_ENTRY_BYTES { + return Err(resource_error( + "xlsx_archive_entry_too_large", + format!("One XLSX ZIP entry exceeds {MAX_XLSX_ENTRY_BYTES} bytes"), + )); + } + declared_total = declared_total + .checked_add(declared_size) + .ok_or_else(|| resource_error("xlsx_archive_too_large", "XLSX archive is too large"))?; + if declared_total > MAX_XLSX_TOTAL_BYTES { + return Err(resource_error( + "xlsx_archive_too_large", + format!("XLSX archives may expand to at most {MAX_XLSX_TOTAL_BYTES} bytes"), + )); + } + if declared_size >= MIN_XLSX_RATIO_CHECK_BYTES + && (compressed_size == 0 + || declared_size > compressed_size.saturating_mul(MAX_XLSX_COMPRESSION_RATIO)) + { + return Err(resource_error( + "xlsx_archive_compression_ratio_too_high", + "One XLSX ZIP entry has an unsafe compression ratio", + )); + } + if entry.is_dir() { + continue; + } + + let remaining_total = MAX_XLSX_TOTAL_BYTES.saturating_sub(actual_total); + let read_limit = MAX_XLSX_ENTRY_BYTES.min(remaining_total).saturating_add(1); + let actual_size = std::io::copy(&mut (&mut entry).take(read_limit), &mut std::io::sink()) + .map_err(format_error)?; + if actual_size > MAX_XLSX_ENTRY_BYTES { + return Err(resource_error( + "xlsx_archive_entry_too_large", + format!("One XLSX ZIP entry exceeds {MAX_XLSX_ENTRY_BYTES} bytes"), + )); + } + if actual_size >= MIN_XLSX_RATIO_CHECK_BYTES + && (compressed_size == 0 + || actual_size > compressed_size.saturating_mul(MAX_XLSX_COMPRESSION_RATIO)) + { + return Err(resource_error( + "xlsx_archive_compression_ratio_too_high", + "One XLSX ZIP entry has an unsafe compression ratio", + )); + } + actual_total = actual_total + .checked_add(actual_size) + .ok_or_else(|| resource_error("xlsx_archive_too_large", "XLSX archive is too large"))?; + if actual_total > MAX_XLSX_TOTAL_BYTES { + return Err(resource_error( + "xlsx_archive_too_large", + format!("XLSX archives may expand to at most {MAX_XLSX_TOTAL_BYTES} bytes"), + )); + } + } + Ok(()) +} + +fn preflight_xlsx_entry_count(path: &Path) -> Result { + let mut file = File::open(path).map_err(format_error)?; + let file_len = file.metadata().map_err(format_error)?.len(); + let tail_budget = ZIP_EOCD_BYTES + .checked_add(ZIP_MAX_COMMENT_BYTES) + .and_then(|value| value.checked_add(usize::try_from(ZIP64_LOCATOR_BYTES).ok()?)) + .ok_or_else(invalid_xlsx_archive)?; + let tail_len = usize::try_from(file_len.min(u64::try_from(tail_budget).unwrap_or(u64::MAX))) + .map_err(|_| invalid_xlsx_archive())?; + if tail_len < ZIP_EOCD_BYTES { + return Err(invalid_xlsx_archive()); + } + let tail_start = file_len + .checked_sub(u64::try_from(tail_len).map_err(|_| invalid_xlsx_archive())?) + .ok_or_else(invalid_xlsx_archive)?; + file.seek(SeekFrom::Start(tail_start)) + .map_err(format_error)?; + let mut tail = vec![0_u8; tail_len]; + file.read_exact(&mut tail).map_err(format_error)?; + + let mut eocd_relative = None; + for offset in (0..=tail_len - ZIP_EOCD_BYTES).rev() { + if tail.get(offset..offset + 4) != Some(ZIP_EOCD_SIGNATURE.as_slice()) { + continue; + } + let Some(comment_len) = read_u16_at(&tail, offset + 20) else { + continue; + }; + let Some(candidate_end) = offset + .checked_add(ZIP_EOCD_BYTES) + .and_then(|value| value.checked_add(usize::from(comment_len))) + else { + continue; + }; + if candidate_end == tail_len { + eocd_relative = Some(offset); + break; + } + } + let eocd_relative = eocd_relative.ok_or_else(invalid_xlsx_archive)?; + let eocd_position = tail_start + .checked_add(u64::try_from(eocd_relative).map_err(|_| invalid_xlsx_archive())?) + .ok_or_else(invalid_xlsx_archive)?; + let disk_number = read_u16_at(&tail, eocd_relative + 4).ok_or_else(invalid_xlsx_archive)?; + let central_disk = read_u16_at(&tail, eocd_relative + 6).ok_or_else(invalid_xlsx_archive)?; + let disk_entries = read_u16_at(&tail, eocd_relative + 8).ok_or_else(invalid_xlsx_archive)?; + let total_entries = read_u16_at(&tail, eocd_relative + 10).ok_or_else(invalid_xlsx_archive)?; + let central_size = read_u32_at(&tail, eocd_relative + 12).ok_or_else(invalid_xlsx_archive)?; + let central_offset = read_u32_at(&tail, eocd_relative + 16).ok_or_else(invalid_xlsx_archive)?; + if disk_number != 0 || central_disk != 0 { + return Err(invalid_xlsx_archive()); + } + + let uses_zip64 = disk_entries == u16::MAX + || total_entries == u16::MAX + || central_size == u32::MAX + || central_offset == u32::MAX; + let (entry_count, central_size, central_offset, central_limit) = if uses_zip64 { + read_zip64_directory_metadata(&mut file, eocd_position, file_len)? + } else { + if disk_entries != total_entries { + return Err(invalid_xlsx_archive()); + } + ( + u64::from(total_entries), + u64::from(central_size), + u64::from(central_offset), + eocd_position, + ) + }; + if entry_count > u64::try_from(MAX_XLSX_ZIP_ENTRIES).unwrap_or(u64::MAX) { + return Err(resource_error( + "xlsx_archive_too_many_entries", + format!("XLSX archives are limited to {MAX_XLSX_ZIP_ENTRIES} ZIP entries"), + )); + } + validate_central_directory_bounds(central_offset, central_size, central_limit, file_len)?; + usize::try_from(entry_count).map_err(|_| invalid_xlsx_archive()) +} + +fn read_zip64_directory_metadata( + file: &mut File, + eocd_position: u64, + file_len: u64, +) -> Result<(u64, u64, u64, u64), AppError> { + let locator_position = eocd_position + .checked_sub(ZIP64_LOCATOR_BYTES) + .ok_or_else(invalid_xlsx_archive)?; + let locator = read_exact_at::<20>(file, locator_position)?; + if locator[..4] != ZIP64_LOCATOR_SIGNATURE + || read_u32_at(&locator, 4) != Some(0) + || read_u32_at(&locator, 16) != Some(1) + { + return Err(invalid_xlsx_archive()); + } + let zip64_position = read_u64_at(&locator, 8).ok_or_else(invalid_xlsx_archive)?; + let fixed = read_exact_at::(file, zip64_position)?; + if fixed[..4] != ZIP64_EOCD_SIGNATURE { + return Err(invalid_xlsx_archive()); + } + let record_payload_size = read_u64_at(&fixed, 4).ok_or_else(invalid_xlsx_archive)?; + if record_payload_size < 44 { + return Err(invalid_xlsx_archive()); + } + let record_end = zip64_position + .checked_add(12) + .and_then(|value| value.checked_add(record_payload_size)) + .ok_or_else(invalid_xlsx_archive)?; + if record_end > locator_position || record_end > file_len { + return Err(invalid_xlsx_archive()); + } + let disk_number = read_u32_at(&fixed, 16).ok_or_else(invalid_xlsx_archive)?; + let central_disk = read_u32_at(&fixed, 20).ok_or_else(invalid_xlsx_archive)?; + let disk_entries = read_u64_at(&fixed, 24).ok_or_else(invalid_xlsx_archive)?; + let total_entries = read_u64_at(&fixed, 32).ok_or_else(invalid_xlsx_archive)?; + if disk_number != 0 || central_disk != 0 || disk_entries != total_entries { + return Err(invalid_xlsx_archive()); + } + let central_size = read_u64_at(&fixed, 40).ok_or_else(invalid_xlsx_archive)?; + let central_offset = read_u64_at(&fixed, 48).ok_or_else(invalid_xlsx_archive)?; + Ok((total_entries, central_size, central_offset, zip64_position)) +} + +fn validate_central_directory_bounds( + offset: u64, + size: u64, + metadata_position: u64, + file_len: u64, +) -> Result<(), AppError> { + let end = offset.checked_add(size).ok_or_else(invalid_xlsx_archive)?; + if offset > file_len || size > file_len || end > metadata_position || end > file_len { + return Err(invalid_xlsx_archive()); + } + Ok(()) +} + +fn read_exact_at(file: &mut File, offset: u64) -> Result<[u8; N], AppError> { + file.seek(SeekFrom::Start(offset)).map_err(format_error)?; + let mut bytes = [0_u8; N]; + file.read_exact(&mut bytes).map_err(format_error)?; + Ok(bytes) +} + +fn read_u16_at(bytes: &[u8], offset: usize) -> Option { + let value = bytes.get(offset..offset.checked_add(2)?)?.try_into().ok()?; + Some(u16::from_le_bytes(value)) +} + +fn read_u32_at(bytes: &[u8], offset: usize) -> Option { + let value = bytes.get(offset..offset.checked_add(4)?)?.try_into().ok()?; + Some(u32::from_le_bytes(value)) +} + +fn read_u64_at(bytes: &[u8], offset: usize) -> Option { + let value = bytes.get(offset..offset.checked_add(8)?)?.try_into().ok()?; + Some(u64::from_le_bytes(value)) +} + +fn invalid_xlsx_archive() -> AppError { + AppError::invalid( + "invalid_xlsx_archive", + "The XLSX ZIP directory is invalid or unsupported", + ) +} + +fn validate_columns(columns: Option<&[String]>) -> Result<(), AppError> { + let Some(columns) = columns else { + return Ok(()); + }; + validate_export_columns(columns)?; + if columns.iter().any(|column| column.trim().is_empty()) { + return Err(AppError::invalid( + "invalid_tabular_header", + "Column headers cannot be empty", + )); + } + let mut normalized = std::collections::HashSet::with_capacity(columns.len()); + if columns + .iter() + .any(|column| !normalized.insert(column.to_ascii_lowercase())) + { + return Err(AppError::invalid( + "invalid_tabular_header", + "Column headers must be unique", + )); + } + Ok(()) +} + +fn validate_export_columns(columns: &[String]) -> Result<(), AppError> { + if columns.is_empty() || columns.len() > MAX_TABULAR_COLUMNS { + return Err(resource_error( + "tabular_file_too_wide", + format!("Tabular files require 1 to {MAX_TABULAR_COLUMNS} columns"), + )); + } + for column in columns { + enforce_cell_size(column)?; + } + Ok(()) +} + +fn validate_export_row(expected: Option, values: &[TabularValue]) -> Result<(), AppError> { + if expected != Some(values.len()) { + return Err(AppError::invalid( + "invalid_tabular_row", + "Every tabular row must match the column count", + )); + } + for value in values { + let encoded = encode_tabular_value(value); + enforce_cell_size(&encoded)?; + } + Ok(()) +} + +fn validate_row_widths( + columns: Option<&[String]>, + rows: &[Vec], +) -> Result<(), AppError> { + let expected = columns + .map(<[String]>::len) + .or_else(|| rows.first().map(Vec::len)); + if expected.is_none() { + return Err(AppError::invalid( + "empty_tabular_file", + "The tabular import file does not contain rows", + )); + } + if rows.iter().any(|row| Some(row.len()) != expected) { + return Err(AppError::invalid( + "invalid_tabular_row", + "Every tabular row must match the column count", + )); + } + Ok(()) +} + +fn enforce_cell_budget(cells: usize) -> Result<(), AppError> { + if cells > MAX_TABULAR_CELLS { + return Err(resource_error( + "tabular_file_too_large", + format!("Tabular files are limited to {MAX_TABULAR_CELLS} cells"), + )); + } + Ok(()) +} + +fn enforce_cell_size(value: &str) -> Result<(), AppError> { + if value.len() > MAX_CELL_BYTES { + return Err(resource_error( + "tabular_cell_too_large", + format!("One tabular cell exceeds {MAX_CELL_BYTES} bytes"), + )); + } + Ok(()) +} + +fn format_error(error: impl std::fmt::Display) -> AppError { + AppError::invalid( + "invalid_transfer_file", + format!("The transfer file could not be processed: {error}"), + ) +} + +fn resource_error(code: impl Into, message: impl Into) -> AppError { + AppError::new( + AppErrorKind::ResourceExhausted, + ApiError::new(code, message), + ) +} + +#[cfg(test)] +mod tests { + use std::{fs, fs::File, io::Write as _}; + + use chat2db_contract::{TabularImportEncoding, TransferFileFormat}; + use tempfile::NamedTempFile; + use xls::core::{Cell, Workbook}; + use zip::{CompressionMethod, ZipWriter, write::SimpleFileOptions}; + + use super::{ + BYTES_ENCODING_PREFIX, CELL_ENCODING_PREFIX, MAX_XLSX_COMPRESSION_RATIO, + MAX_XLSX_ENTRY_BYTES, MAX_XLSX_TOTAL_BYTES, MAX_XLSX_ZIP_ENTRIES, + MIN_XLSX_RATIO_CHECK_BYTES, NULL_ENCODING, TEXT_ENCODING_PREFIX, TabularValue, + preflight_xlsx_entry_count, read_tabular_file, tabular_sink, validate_xlsx_archive, + }; + + #[test] + fn tabular_round_trip_preserves_null_empty_utf8_binary_and_reserved_text() { + for format in [ + TransferFileFormat::Csv, + TransferFileFormat::Xls, + TransferFileFormat::Xlsx, + ] { + let temporary = NamedTempFile::new().expect("temp file"); + let mut file = File::options() + .read(true) + .write(true) + .open(temporary.path()) + .expect("temp file opens"); + { + let mut sink = tabular_sink(format, &mut file, true).expect("sink creates"); + sink.write_header(&[ + "plain_value".to_owned(), + "null_value".to_owned(), + "empty_value".to_owned(), + "utf8_value".to_owned(), + "binary_value".to_owned(), + "reserved_value".to_owned(), + ]) + .expect("header writes"); + sink.write_row(&[ + TabularValue::Text("1".to_owned()), + TabularValue::Null, + TabularValue::Text(String::new()), + TabularValue::Text("中文,quoted".to_owned()), + TabularValue::Bytes(vec![0x00, 0xff]), + TabularValue::Text(format!("{CELL_ENCODING_PREFIX}NULL")), + ]) + .expect("row writes"); + sink.finish().expect("sink finishes"); + } + if format == TransferFileFormat::Csv { + let mut reader = csv::Reader::from_path(temporary.path()).expect("CSV reads"); + let record = reader + .records() + .next() + .expect("CSV row exists") + .expect("CSV row decodes"); + assert_eq!(&record[0], "1", "ordinary numeric text remains readable"); + assert_eq!(&record[1], NULL_ENCODING); + assert_eq!(&record[2], "", "empty text remains an empty CSV field"); + assert_eq!(&record[3], "中文,quoted"); + assert!(record[4].starts_with(BYTES_ENCODING_PREFIX)); + assert!(record[5].starts_with(TEXT_ENCODING_PREFIX)); + } + let imported = read_tabular_file( + temporary.path(), + format, + true, + TabularImportEncoding::Chat2dbV1, + ) + .expect("tabular file reads"); + assert_eq!( + imported.rows, + vec![vec![ + TabularValue::Text("1".to_owned()), + TabularValue::Null, + TabularValue::Text(String::new()), + TabularValue::Text("中文,quoted".to_owned()), + TabularValue::Bytes(vec![0x00, 0xff]), + TabularValue::Text(format!("{CELL_ENCODING_PREFIX}NULL")), + ]], + "{format:?} must preserve typed values" + ); + } + } + + #[test] + fn plain_tabular_import_preserves_the_v1_namespace_as_external_text() { + let values = [ + NULL_ENCODING.to_owned(), + format!("{BYTES_ENCODING_PREFIX}YWJj"), + format!("{CELL_ENCODING_PREFIX}UNKNOWN"), + ]; + for format in [ + TransferFileFormat::Csv, + TransferFileFormat::Xls, + TransferFileFormat::Xlsx, + ] { + let temporary = NamedTempFile::new().expect("temp file"); + match format { + TransferFileFormat::Csv => { + let mut writer = csv::Writer::from_path(temporary.path()).expect("CSV opens"); + writer + .write_record(["null_marker", "bytes_marker", "unknown_marker"]) + .expect("CSV header writes"); + writer.write_record(&values).expect("CSV row writes"); + writer.flush().expect("CSV flushes"); + } + TransferFileFormat::Xls | TransferFileFormat::Xlsx => { + let mut workbook = Workbook::new(); + let sheet = workbook.sheet_mut(0).expect("default sheet exists"); + for (column, value) in ["null_marker", "bytes_marker", "unknown_marker"] + .into_iter() + .enumerate() + { + sheet.set( + 0, + u32::try_from(column).unwrap(), + Cell::Text(value.to_owned()), + ); + } + for (column, value) in values.iter().enumerate() { + sheet.set(1, u32::try_from(column).unwrap(), Cell::Text(value.clone())); + } + let mut file = temporary.reopen().expect("spreadsheet opens"); + match format { + TransferFileFormat::Xls => xls::core::xls::write(&workbook, &mut file), + TransferFileFormat::Xlsx => xls::core::xlsx::write(&workbook, &mut file), + TransferFileFormat::Csv | TransferFileFormat::Sql => unreachable!(), + } + .expect("spreadsheet writes"); + } + TransferFileFormat::Sql => unreachable!(), + } + let imported = + read_tabular_file(temporary.path(), format, true, TabularImportEncoding::Plain) + .expect("plain tabular file reads"); + assert_eq!( + imported.rows, + vec![ + values + .iter() + .cloned() + .map(TabularValue::Text) + .collect::>() + ], + "{format:?} must not interpret an unmarked external cell" + ); + } + } + + #[test] + fn xlsx_rejects_a_declared_entry_larger_than_the_budget() { + let temporary = zip_fixture(1, CompressionMethod::Stored, &[]); + patch_central_uncompressed_sizes(temporary.path(), MAX_XLSX_ENTRY_BYTES + 1); + let error = validate_xlsx_archive(temporary.path()).expect_err("oversized entry rejects"); + assert_eq!(error.api_error().code, "xlsx_archive_entry_too_large"); + } + + #[test] + fn xlsx_rejects_a_declared_cumulative_size_larger_than_the_budget() { + let declared_size = MIN_XLSX_RATIO_CHECK_BYTES - 1; + let entries = usize::try_from(MAX_XLSX_TOTAL_BYTES / declared_size + 1) + .expect("entry count fits usize"); + assert!(entries < MAX_XLSX_ZIP_ENTRIES); + let temporary = zip_fixture(entries, CompressionMethod::Stored, &[]); + patch_central_uncompressed_sizes(temporary.path(), declared_size); + let error = validate_xlsx_archive(temporary.path()).expect_err("oversized archive rejects"); + assert_eq!(error.api_error().code, "xlsx_archive_too_large"); + } + + #[test] + fn xlsx_rejects_too_many_zip_entries() { + let temporary = zip_fixture(MAX_XLSX_ZIP_ENTRIES + 1, CompressionMethod::Stored, &[]); + let error = validate_xlsx_archive(temporary.path()).expect_err("entry flood rejects"); + assert_eq!(error.api_error().code, "xlsx_archive_too_many_entries"); + } + + #[test] + fn xlsx_rejects_a_high_compression_ratio_before_workbook_parsing() { + let repeated = vec![0_u8; usize::try_from(MIN_XLSX_RATIO_CHECK_BYTES).unwrap()]; + let temporary = zip_fixture(1, CompressionMethod::Deflated, &repeated); + let compressed = fs::metadata(temporary.path()).expect("ZIP metadata").len(); + assert!( + MIN_XLSX_RATIO_CHECK_BYTES > compressed * MAX_XLSX_COMPRESSION_RATIO, + "fixture must exceed the configured compression ratio" + ); + let error = validate_xlsx_archive(temporary.path()).expect_err("ZIP bomb rejects"); + assert_eq!( + error.api_error().code, + "xlsx_archive_compression_ratio_too_high" + ); + } + + #[test] + fn xlsx_rechecks_compression_ratio_with_the_actual_uncompressed_size() { + let repeated = vec![0_u8; usize::try_from(MIN_XLSX_RATIO_CHECK_BYTES).unwrap()]; + let temporary = zip_fixture(1, CompressionMethod::Deflated, &repeated); + patch_central_uncompressed_sizes(temporary.path(), MIN_XLSX_RATIO_CHECK_BYTES - 1); + let error = validate_xlsx_archive(temporary.path()) + .expect_err("actual high compression ratio rejects"); + assert_eq!( + error.api_error().code, + "xlsx_archive_compression_ratio_too_high" + ); + } + + #[test] + fn xlsx_rejects_duplicate_central_directory_names() { + let temporary = zip_fixture(2, CompressionMethod::Stored, &[]); + patch_second_central_name_to_match_first(temporary.path()); + let error = validate_xlsx_archive(temporary.path()).expect_err("duplicates reject"); + assert_eq!(error.api_error().code, "xlsx_archive_duplicate_entries"); + } + + #[test] + fn xlsx_preflights_zip64_entry_counts_before_archive_construction() { + let accepted = zip64_fixture(1); + assert_eq!( + preflight_xlsx_entry_count(accepted.path()).expect("ZIP64 count reads"), + 1 + ); + + let rejected = zip64_fixture(u64::try_from(MAX_XLSX_ZIP_ENTRIES).unwrap() + 1); + let error = preflight_xlsx_entry_count(rejected.path()).expect_err("ZIP64 flood rejects"); + assert_eq!(error.api_error().code, "xlsx_archive_too_many_entries"); + } + + fn zip_fixture( + entries: usize, + compression: CompressionMethod, + contents: &[u8], + ) -> NamedTempFile { + let temporary = NamedTempFile::new().expect("temp file"); + let file = temporary.reopen().expect("ZIP fixture opens"); + let mut writer = ZipWriter::new(file); + let options = SimpleFileOptions::default().compression_method(compression); + for index in 0..entries { + writer + .start_file(format!("xl/worksheets/sheet{index}.xml"), options) + .expect("ZIP entry starts"); + writer.write_all(contents).expect("ZIP entry writes"); + } + writer.finish().expect("ZIP fixture finishes"); + temporary + } + + fn patch_central_uncompressed_sizes(path: &std::path::Path, size: u64) { + let size = u32::try_from(size).expect("test declaration fits classic ZIP"); + let mut bytes = fs::read(path).expect("ZIP fixture reads"); + let mut cursor = 0; + let mut patched = 0; + while let Some(relative) = bytes[cursor..] + .windows(4) + .position(|window| window == b"PK\x01\x02") + { + let header = cursor + relative; + bytes[header + 24..header + 28].copy_from_slice(&size.to_le_bytes()); + patched += 1; + cursor = header + 46; + } + assert_ne!(patched, 0, "at least one central-directory entry patches"); + fs::write(path, bytes).expect("patched ZIP fixture writes"); + } + + fn patch_second_central_name_to_match_first(path: &std::path::Path) { + let mut bytes = fs::read(path).expect("ZIP fixture reads"); + let headers = bytes + .windows(4) + .enumerate() + .filter_map(|(offset, signature)| (signature == b"PK\x01\x02").then_some(offset)) + .collect::>(); + assert_eq!(headers.len(), 2, "fixture has two central entries"); + let first_length = usize::from(u16::from_le_bytes( + bytes[headers[0] + 28..headers[0] + 30] + .try_into() + .expect("first name length reads"), + )); + let second_length = usize::from(u16::from_le_bytes( + bytes[headers[1] + 28..headers[1] + 30] + .try_into() + .expect("second name length reads"), + )); + assert_eq!(first_length, second_length); + let first_name = bytes[headers[0] + 46..headers[0] + 46 + first_length].to_vec(); + bytes[headers[1] + 46..headers[1] + 46 + second_length].copy_from_slice(&first_name); + fs::write(path, bytes).expect("duplicate-name ZIP fixture writes"); + } + + fn zip64_fixture(entry_count: u64) -> NamedTempFile { + let temporary = zip_fixture(1, CompressionMethod::Stored, &[]); + let bytes = fs::read(temporary.path()).expect("ZIP fixture reads"); + let eocd = bytes + .windows(4) + .rposition(|signature| signature == b"PK\x05\x06") + .expect("EOCD exists"); + let central_size = u32::from_le_bytes( + bytes[eocd + 12..eocd + 16] + .try_into() + .expect("central size reads"), + ); + let central_offset = u32::from_le_bytes( + bytes[eocd + 16..eocd + 20] + .try_into() + .expect("central offset reads"), + ); + let mut classic_eocd = bytes[eocd..].to_vec(); + classic_eocd[8..12].fill(0xff); + classic_eocd[12..20].fill(0xff); + + let zip64_position = u64::try_from(eocd).expect("fixture offset fits u64"); + let mut output = bytes[..eocd].to_vec(); + output.extend_from_slice(b"PK\x06\x06"); + output.extend_from_slice(&44_u64.to_le_bytes()); + output.extend_from_slice(&45_u16.to_le_bytes()); + output.extend_from_slice(&45_u16.to_le_bytes()); + output.extend_from_slice(&0_u32.to_le_bytes()); + output.extend_from_slice(&0_u32.to_le_bytes()); + output.extend_from_slice(&entry_count.to_le_bytes()); + output.extend_from_slice(&entry_count.to_le_bytes()); + output.extend_from_slice(&u64::from(central_size).to_le_bytes()); + output.extend_from_slice(&u64::from(central_offset).to_le_bytes()); + output.extend_from_slice(b"PK\x06\x07"); + output.extend_from_slice(&0_u32.to_le_bytes()); + output.extend_from_slice(&zip64_position.to_le_bytes()); + output.extend_from_slice(&1_u32.to_le_bytes()); + output.extend_from_slice(&classic_eocd); + fs::write(temporary.path(), output).expect("ZIP64 fixture writes"); + temporary + } +} diff --git a/crates/chat2db-core/src/transfer/mod.rs b/crates/chat2db-core/src/transfer/mod.rs new file mode 100644 index 0000000..4f51e05 --- /dev/null +++ b/crates/chat2db-core/src/transfer/mod.rs @@ -0,0 +1,771 @@ +mod class_generation; +mod format; +mod mysql; + +use std::{ + collections::HashMap, + fmt::Write as _, + fs::File, + path::{Path, PathBuf}, + time::Duration, +}; + +use chat2db_contract::{ + DmlExportRequest, GenerateMysqlClassRequest, GeneratedMysqlClassSet, ImportFileRequest, + OtherFileExportRequest, SqlFileExportRequest, TransferArtifact, TransferTask, + TransferTaskAccepted, TransferTaskKind, TransferTaskPage, TransferTaskStatus, +}; +use chat2db_storage::{ + CreateTransferTask, ResolvedTransferArtifact, Storage, StorageError, StoredTransferTaskKind, + StoredTransferTaskStatus, TransferArtifactRecord, TransferArtifactWriter, TransferTaskRecord, +}; +use tokio::{ + sync::{Mutex, oneshot}, + task::JoinHandle, +}; +use tokio_util::sync::CancellationToken; + +use crate::{AppError, Application, native_mysql, storage_call}; + +const MAX_TASK_PAGE_SIZE: u32 = 100; + +pub(crate) struct TransferTaskHub { + tasks: Mutex>, +} + +struct ActiveTransferTask { + cancellation: CancellationToken, + handle: JoinHandle<()>, +} + +pub struct TransferArtifactDownload { + pub artifact: TransferArtifact, + pub path: PathBuf, + pub file: File, +} + +impl std::fmt::Debug for TransferArtifactDownload { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("TransferArtifactDownload") + .field("artifact", &self.artifact) + .field("path", &self.path) + .field("file", &self.file) + .finish() + } +} + +impl TransferTaskHub { + pub(crate) fn new() -> Self { + Self { + tasks: Mutex::new(HashMap::new()), + } + } + + async fn insert( + &self, + task_id: i64, + cancellation: CancellationToken, + handle: JoinHandle<()>, + ) -> Option { + self.tasks.lock().await.insert( + task_id, + ActiveTransferTask { + cancellation, + handle, + }, + ) + } + + async fn remove(&self, task_id: i64) { + self.tasks.lock().await.remove(&task_id); + } + + async fn cancel(&self, task_id: i64) -> bool { + let tasks = self.tasks.lock().await; + let Some(task) = tasks.get(&task_id) else { + return false; + }; + task.cancellation.cancel(); + true + } + + async fn cancel_all(&self) -> Vec { + let tasks = self.tasks.lock().await; + for task in tasks.values() { + task.cancellation.cancel(); + } + tasks.keys().copied().collect() + } + + async fn take_all(&self) -> HashMap { + std::mem::take(&mut *self.tasks.lock().await) + } +} + +pub(super) struct TransferContext { + storage: Storage, + task_id: i64, + cancellation: CancellationToken, +} + +impl TransferContext { + fn new(storage: Storage, task_id: i64, cancellation: CancellationToken) -> Self { + Self { + storage, + task_id, + cancellation, + } + } + + pub(super) fn cancellation(&self) -> &CancellationToken { + &self.cancellation + } + + pub(super) fn check_cancelled(&self) -> Result<(), TransferRunError> { + if self.cancellation.is_cancelled() { + Err(TransferRunError::Cancelled) + } else { + Ok(()) + } + } + + pub(super) fn begin_artifact( + &self, + file_name: &str, + media_type: &str, + format: &str, + extension: &str, + ) -> Result { + self.check_cancelled()?; + self.storage + .begin_transfer_artifact( + Some(self.task_id), + file_name, + media_type, + format, + extension, + None, + ) + .map_err(TransferRunError::from) + } + + pub(super) async fn progress( + &self, + current: u64, + total: Option, + description: &str, + info: Option<&str>, + ) -> Result<(), TransferRunError> { + self.check_cancelled()?; + let storage = self.storage.clone(); + let task_id = self.task_id; + let description = description.to_owned(); + let info = info.map(str::to_owned); + storage_call(move || { + storage.update_transfer_progress(task_id, current, total, &description, info.as_deref()) + }) + .await + .map_err(TransferRunError::from) + } +} + +pub(super) enum TransferRunError { + Cancelled, + Failed(AppError), +} + +impl TransferRunError { + pub(super) fn into_app_error(self) -> AppError { + match self { + Self::Cancelled => { + AppError::unavailable("transfer_cancelled", "The transfer operation was cancelled") + } + Self::Failed(error) => error, + } + } +} + +impl From for TransferRunError { + fn from(error: AppError) -> Self { + Self::Failed(error) + } +} + +impl From for TransferRunError { + fn from(error: StorageError) -> Self { + Self::Failed(error.into()) + } +} + +pub(super) enum TaskCompletion { + WithoutArtifact(String), + Artifact(TransferArtifactRecord), +} + +enum TransferJob { + Import(ImportFileRequest), + SqlExport(SqlFileExportRequest), + OtherExport(OtherFileExportRequest), +} + +impl Application { + /// Starts a durable native-MySQL CSV, XLS, XLSX, or SQL import task. + /// + /// # Errors + /// + /// Returns validation, datasource, storage, or runtime-shutdown failures. + pub async fn import_mysql_file( + &self, + request: ImportFileRequest, + ) -> Result { + validate_import_request(&request)?; + native_mysql::resolve_native_connection(self, &request.datasource_id).await?; + let file_name = Path::new(&request.file_path) + .file_name() + .and_then(|value| value.to_str()) + .unwrap_or("file"); + self.start_transfer_job( + CreateTransferTask { + datasource_id: request.datasource_id.clone(), + database_name: request.database_name.clone(), + schema_name: request.schema_name.clone(), + table_name: request.table_name.clone(), + kind: StoredTransferTaskKind::ImportFile, + task_name: format!("Import {file_name}"), + }, + TransferJob::Import(request), + ) + .await + } + + /// Starts a durable native-MySQL SQL dump export task. + /// + /// # Errors + /// + /// Returns validation, datasource, storage, or runtime-shutdown failures. + pub async fn export_mysql_sql_file( + &self, + request: SqlFileExportRequest, + ) -> Result { + validate_transfer_scope( + &request.datasource_id, + &request.database_name, + request.export_path.as_deref(), + )?; + native_mysql::resolve_native_connection(self, &request.datasource_id).await?; + self.start_transfer_job( + CreateTransferTask { + datasource_id: request.datasource_id.clone(), + database_name: request.database_name.clone(), + schema_name: request.schema_name.clone(), + table_name: single_table(&request.table_names), + kind: StoredTransferTaskKind::ExportSql, + task_name: format!("Export SQL {}", request.database_name), + }, + TransferJob::SqlExport(request), + ) + .await + } + + /// Starts a durable native-MySQL table file export task. + /// + /// # Errors + /// + /// Returns validation, datasource, storage, or runtime-shutdown failures. + pub async fn export_mysql_other_file( + &self, + request: OtherFileExportRequest, + ) -> Result { + validate_transfer_scope( + &request.datasource_id, + &request.database_name, + request.export_path.as_deref(), + )?; + if request.table_names.is_empty() { + return Err(AppError::invalid( + "missing_export_tables", + "tableNames must contain at least one table", + )); + } + native_mysql::resolve_native_connection(self, &request.datasource_id).await?; + self.start_transfer_job( + CreateTransferTask { + datasource_id: request.datasource_id.clone(), + database_name: request.database_name.clone(), + schema_name: request.schema_name.clone(), + table_name: single_table(&request.table_names), + kind: StoredTransferTaskKind::ExportFile, + task_name: format!( + "Export {} {} table(s)", + request.format.extension().to_ascii_uppercase(), + request.table_names.len() + ), + }, + TransferJob::OtherExport(request), + ) + .await + } + + /// Lists retained transfer tasks newest first. + /// + /// # Errors + /// + /// Returns invalid paging or durable-storage failures. + pub async fn list_transfer_tasks( + &self, + page_no: u32, + page_size: u32, + ) -> Result { + self.list_transfer_tasks_by_statuses(page_no, page_size, &[]) + .await + } + + /// Lists retained transfer tasks after applying an optional status set. + /// + /// An empty status set selects every task. Filtering happens before + /// pagination so legacy Community task tabs keep accurate totals. + /// + /// # Errors + /// + /// Returns invalid paging or durable-storage failures. + pub async fn list_transfer_tasks_by_statuses( + &self, + page_no: u32, + page_size: u32, + statuses: &[TransferTaskStatus], + ) -> Result { + if page_no == 0 || page_size == 0 || page_size > MAX_TASK_PAGE_SIZE { + return Err(AppError::invalid( + "invalid_transfer_task_page", + "pageNo must be positive and pageSize must be between 1 and 100", + )); + } + let storage = self.require_storage()?; + let tasks: Vec = storage_call(move || storage.list_transfer_tasks()) + .await? + .into_iter() + .map(transfer_task) + .filter(|task| statuses.is_empty() || statuses.contains(&task.status)) + .collect(); + let total = u64::try_from(tasks.len()).map_err(|_| AppError::internal())?; + let start = usize::try_from((page_no - 1).saturating_mul(page_size)) + .map_err(|_| AppError::internal())?; + let items = tasks + .into_iter() + .skip(start) + .take(usize::try_from(page_size).map_err(|_| AppError::internal())?) + .collect(); + Ok(TransferTaskPage { + items, + total, + page_no, + page_size, + }) + } + + /// Reads one retained transfer task. + /// + /// # Errors + /// + /// Returns not-found or durable-storage failures. + pub async fn transfer_task(&self, task_id: i64) -> Result { + let storage = self.require_storage()?; + storage_call(move || storage.get_transfer_task(task_id)) + .await? + .map(transfer_task) + .ok_or_else(|| { + AppError::not_found( + "transfer_task_not_found", + format!("Transfer task {task_id} does not exist"), + ) + }) + } + + /// Requests cooperative cancellation of one queued or running transfer. + /// + /// # Errors + /// + /// Returns not-found or durable-storage failures. + pub async fn stop_transfer_task(&self, task_id: i64) -> Result<(), AppError> { + let storage = self.require_storage()?; + let changed = storage_call(move || storage.request_transfer_cancel(task_id)).await?; + if changed { + self.inner.transfer_tasks.cancel(task_id).await; + } + Ok(()) + } + + /// Resolves a managed artifact and its owner-only local path for a delivery adapter. + /// + /// # Errors + /// + /// Returns not-found, expiry, corruption, or durable-storage failures. + pub async fn transfer_artifact_download( + &self, + artifact_id: &str, + ) -> Result { + if artifact_id.trim().is_empty() { + return Err(AppError::invalid( + "invalid_transfer_artifact", + "artifactId cannot be empty", + )); + } + let storage = self.require_storage()?; + let artifact_id = artifact_id.to_owned(); + let resolved = + storage_call(move || storage.resolve_transfer_artifact(&artifact_id)).await?; + Ok(artifact_download(resolved)) + } + + /// Resolves the managed artifact produced by one completed transfer task. + /// + /// # Errors + /// + /// Returns not-found, incomplete-task, expiry, corruption, or durable-storage failures. + pub async fn transfer_task_artifact_download( + &self, + task_id: i64, + ) -> Result { + let task = self.transfer_task(task_id).await?; + let artifact_id = task.artifact_id.ok_or_else(|| { + AppError::not_found( + "transfer_artifact_not_found", + format!("Transfer task {task_id} has no downloadable artifact"), + ) + })?; + self.transfer_artifact_download(&artifact_id).await + } + + /// Streams one DML result into a temporary managed CSV, XLSX, or INSERT artifact. + /// + /// # Errors + /// + /// Returns SQL analysis, datasource, query, format, or storage failures. + pub async fn export_mysql_dml( + &self, + request: DmlExportRequest, + ) -> Result { + native_mysql::resolve_native_connection(self, &request.datasource_id).await?; + mysql::export_dml(self, request) + .await + .map(transfer_artifact) + } + + /// Generates `MyBatis` Plus entity, Mapper, and Mapper XML files from native `MySQL` metadata. + /// + /// # Errors + /// + /// Returns validation, metadata, datasource, or filesystem failures. + pub async fn generate_mysql_classes( + &self, + request: GenerateMysqlClassRequest, + ) -> Result { + native_mysql::resolve_native_connection(self, &request.datasource_id).await?; + class_generation::generate(self, request).await + } + + /// Generates the same `MyBatis` Plus files as Desktop in a temporary managed ZIP. + /// + /// # Errors + /// + /// Returns validation, metadata, datasource, archive, or storage failures. + pub async fn generate_mysql_class_archive( + &self, + request: GenerateMysqlClassRequest, + ) -> Result { + native_mysql::resolve_native_connection(self, &request.datasource_id).await?; + class_generation::generate_archive(self, request) + .await + .map(transfer_artifact) + } + + async fn start_transfer_job( + &self, + task: CreateTransferTask, + job: TransferJob, + ) -> Result { + let accepting_work = self.inner.accepting_work.lock().await; + if !*accepting_work { + return Err(AppError::unavailable( + "runtime_shutting_down", + "The Chat2DB runtime is shutting down", + )); + } + let storage = self.require_storage()?; + let task_record = storage_call({ + let storage = storage.clone(); + move || storage.create_transfer_task(&task) + }) + .await?; + let task_id = task_record.id; + let cancellation = CancellationToken::new(); + let run_cancellation = cancellation.clone(); + let application = self.clone(); + let (registered, wait_for_registration) = oneshot::channel(); + let handle = tokio::spawn(async move { + if wait_for_registration.await.is_err() { + return; + } + application + .run_transfer_task(task_id, job, run_cancellation) + .await; + application.inner.transfer_tasks.remove(task_id).await; + }); + let replaced = self + .inner + .transfer_tasks + .insert(task_id, cancellation, handle) + .await; + debug_assert!(replaced.is_none(), "transfer task ids must be unique"); + if registered.send(()).is_err() { + if let Some(task) = self + .inner + .transfer_tasks + .tasks + .lock() + .await + .remove(&task_id) + { + task.handle.abort(); + } + let storage = storage.clone(); + let _ = storage_call(move || { + storage.fail_transfer_task(task_id, "Transfer task registration failed") + }) + .await; + return Err(AppError::internal()); + } + drop(accepting_work); + Ok(TransferTaskAccepted { task_id }) + } + + async fn run_transfer_task( + &self, + task_id: i64, + job: TransferJob, + cancellation: CancellationToken, + ) { + let Some(storage) = self.storage().cloned() else { + return; + }; + if cancellation.is_cancelled() { + let _ = storage_call(move || storage.request_transfer_cancel(task_id)).await; + return; + } + let start_storage = storage.clone(); + if let Err(error) = storage_call(move || start_storage.start_transfer_task(task_id)).await { + if !cancellation.is_cancelled() { + tracing::warn!(task_id, %error, "transfer task could not enter running state"); + } + return; + } + let context = TransferContext::new(storage.clone(), task_id, cancellation.clone()); + let result = match job { + TransferJob::Import(request) => mysql::import_file(self, request, &context).await, + TransferJob::SqlExport(request) => mysql::export_sql(self, request, &context).await, + TransferJob::OtherExport(request) => mysql::export_other(self, request, &context).await, + }; + match result { + Ok(TaskCompletion::WithoutArtifact(message)) => { + let complete_storage = storage.clone(); + if let Err(error) = + storage_call(move || complete_storage.complete_transfer_task(task_id, &message)) + .await + { + tracing::warn!(task_id, %error, "transfer task completion could not be persisted"); + } + } + Ok(TaskCompletion::Artifact(artifact)) => { + debug_assert_eq!(artifact.task_id, Some(task_id)); + } + Err(TransferRunError::Cancelled) => { + let cancel_storage = storage.clone(); + let _ = storage_call(move || { + cancel_storage.cancel_transfer_task(task_id, "Transfer cancelled by request") + }) + .await; + } + Err(TransferRunError::Failed(error)) => { + let message = error.api_error().message; + tracing::warn!(task_id, code = %error.api_error().code, "transfer task failed"); + let fail_storage = storage.clone(); + let _ = + storage_call(move || fail_storage.fail_transfer_task(task_id, &message)).await; + } + } + } + + pub(crate) async fn begin_transfer_shutdown(&self) { + let task_ids = self.inner.transfer_tasks.cancel_all().await; + let Some(storage) = self.storage().cloned() else { + return; + }; + for task_id in task_ids { + let storage = storage.clone(); + let _ = storage_call(move || storage.request_transfer_cancel(task_id)).await; + } + } + + pub(crate) async fn join_transfer_tasks(&self, timeout: Duration) { + let tasks = self.inner.transfer_tasks.take_all().await; + let deadline = tokio::time::Instant::now() + timeout; + let storage = self.storage().cloned(); + for (task_id, mut task) in tasks { + let terminal_message = match tokio::time::timeout_at(deadline, &mut task.handle).await { + Ok(Ok(())) => None, + Ok(Err(_)) => Some("Transfer worker stopped unexpectedly"), + Err(_) => { + task.handle.abort(); + Some("Transfer stopped during runtime shutdown") + } + }; + if let (Some(storage), Some(message)) = (storage.clone(), terminal_message) { + let _ = storage_call(move || storage.cancel_transfer_task(task_id, message)).await; + } + } + } +} + +fn validate_import_request(request: &ImportFileRequest) -> Result<(), AppError> { + validate_transfer_scope(&request.datasource_id, &request.database_name, None)?; + if request.file_path.trim().is_empty() || request.file_path.contains('\0') { + return Err(AppError::invalid( + "invalid_import_file", + "filePath cannot be empty", + )); + } + if request.format != chat2db_contract::TransferFileFormat::Sql + && request.table_name.as_deref().is_none_or(str::is_empty) + { + return Err(AppError::invalid( + "missing_import_table", + "tableName is required for tabular imports", + )); + } + Ok(()) +} + +fn validate_transfer_scope( + datasource_id: &str, + database_name: &str, + export_path: Option<&str>, +) -> Result<(), AppError> { + if datasource_id.trim().is_empty() { + return Err(AppError::invalid( + "invalid_transfer_request", + "datasourceId cannot be empty", + )); + } + native_mysql::quote_identifier(database_name, "databaseName")?; + if export_path.is_some_and(|path| path.trim().is_empty() || path.contains('\0')) { + return Err(AppError::invalid( + "invalid_export_path", + "exportPath must be a local directory", + )); + } + Ok(()) +} + +fn single_table(table_names: &[String]) -> Option { + (table_names.len() == 1).then(|| table_names[0].clone()) +} + +fn transfer_task(record: TransferTaskRecord) -> TransferTask { + TransferTask { + id: record.id, + datasource_id: record.datasource_id, + database_name: record.database_name, + schema_name: record.schema_name, + table_name: record.table_name, + kind: match record.kind { + StoredTransferTaskKind::ImportFile => TransferTaskKind::ImportFile, + StoredTransferTaskKind::ExportSql => TransferTaskKind::ExportSql, + StoredTransferTaskKind::ExportFile => TransferTaskKind::ExportFile, + }, + status: match record.status { + StoredTransferTaskStatus::Queued => TransferTaskStatus::Queued, + StoredTransferTaskStatus::Running => TransferTaskStatus::Running, + StoredTransferTaskStatus::Succeeded => TransferTaskStatus::Succeeded, + StoredTransferTaskStatus::Failed => TransferTaskStatus::Failed, + StoredTransferTaskStatus::Cancelled => TransferTaskStatus::Cancelled, + StoredTransferTaskStatus::Interrupted => TransferTaskStatus::Interrupted, + }, + task_name: record.task_name, + progress_current: record.progress_current.to_string(), + progress_total: record.progress_total.map(|value| value.to_string()), + progress_description: record.progress_description, + info_log: record.info_log, + error_log: record.error_log, + artifact_id: record.artifact_id, + cancel_requested: record.cancel_requested, + created_at_ms: record.created_at_ms.to_string(), + updated_at_ms: record.updated_at_ms.to_string(), + finished_at_ms: record.finished_at_ms.map(|value| value.to_string()), + } +} + +fn artifact_download(resolved: ResolvedTransferArtifact) -> TransferArtifactDownload { + TransferArtifactDownload { + artifact: transfer_artifact(resolved.record), + path: resolved.path, + file: resolved.file, + } +} + +fn transfer_artifact(record: TransferArtifactRecord) -> TransferArtifact { + TransferArtifact { + id: record.id, + task_id: record.task_id, + file_name: record.file_name, + media_type: record.media_type, + format: record.format, + byte_count: record.byte_count.to_string(), + sha256: sha256_hex(&record.sha256), + created_at_ms: record.created_at_ms.to_string(), + } +} + +fn sha256_hex(digest: &[u8; 32]) -> String { + digest + .iter() + .fold(String::with_capacity(64), |mut hex, byte| { + let _ = write!(hex, "{byte:02x}"); + hex + }) +} + +#[cfg(test)] +mod tests { + use chat2db_storage::{StoredTransferTaskKind, StoredTransferTaskStatus, TransferTaskRecord}; + + use super::transfer_task; + + #[test] + fn durable_interrupted_status_is_preserved_for_transport_projection() { + let projected = transfer_task(TransferTaskRecord { + id: 7, + datasource_id: "mysql".to_owned(), + database_name: "app".to_owned(), + schema_name: String::new(), + table_name: None, + kind: StoredTransferTaskKind::ExportSql, + status: StoredTransferTaskStatus::Interrupted, + task_name: "export".to_owned(), + progress_current: 1, + progress_total: Some(2), + progress_description: "Interrupted".to_owned(), + info_log: String::new(), + error_log: "stopped".to_owned(), + cancel_requested: false, + created_at_ms: 1, + updated_at_ms: 2, + finished_at_ms: Some(2), + artifact_id: None, + }); + assert_eq!( + projected.status, + chat2db_contract::TransferTaskStatus::Interrupted + ); + } +} diff --git a/crates/chat2db-core/src/transfer/mysql.rs b/crates/chat2db-core/src/transfer/mysql.rs new file mode 100644 index 0000000..0d58aee --- /dev/null +++ b/crates/chat2db-core/src/transfer/mysql.rs @@ -0,0 +1,1478 @@ +use std::{ + collections::HashSet, + fmt::Write as _, + fs::{self, File, OpenOptions}, + io::{Read as _, Write as _}, + path::{Path, PathBuf}, + time::{SystemTime, UNIX_EPOCH}, +}; + +use chat2db_contract::{ + DmlExportFormat, DmlExportRequest, DmlExportSize, ImportFileRequest, OtherFileExportRequest, + SqlFileExportRequest, TabularImportEncoding, TransferFileFormat, TransferSqlScope, +}; +use chat2db_storage::{TransferArtifactRecord, TransferArtifactWriter}; +use mysql_async::{ + Column, Conn, Error as MysqlError, Params, Row, TxOpts, Value, + consts::{ColumnFlags, ColumnType}, + prelude::Queryable, +}; +use sqlparser::{ + ast::{ObjectName, Query, SetExpr, Statement, TableFactor}, + dialect::MySqlDialect, + parser::Parser, +}; +use tempfile::TempDir; +use tokio_util::sync::CancellationToken; +use uuid::Uuid; +use zip::{CompressionMethod, ZipWriter, write::SimpleFileOptions}; + +use super::{TaskCompletion, TransferContext, TransferRunError, format}; +use crate::{AppError, AppErrorKind, Application, native_mysql}; + +const IMPORT_BATCH_ROWS: usize = 256; +const PROGRESS_ROW_INTERVAL: u64 = 250; +const DML_ARTIFACT_TTL_MS: i64 = 24 * 60 * 60 * 1_000; + +pub(super) async fn import_file( + application: &Application, + request: ImportFileRequest, + context: &TransferContext, +) -> Result { + context.check_cancelled()?; + let path = PathBuf::from(&request.file_path); + let input = match request.format { + TransferFileFormat::Sql => ImportInput::Sql(read_sql_file(path).await?), + format => ImportInput::Table( + read_table_file( + path, + format, + request.contains_header, + request.tabular_encoding, + ) + .await?, + ), + }; + context + .progress(0, input.total(), "Import file validated", None) + .await?; + + let mut conn = open_connection(application, &request.datasource_id, true).await?; + let result = match input { + ImportInput::Sql(statements) => { + import_sql(&mut conn, &request.database_name, statements, context).await + } + ImportInput::Table(table) => { + let table_name = request.table_name.as_deref().ok_or_else(|| { + TransferRunError::from(AppError::invalid( + "missing_import_table", + "tableName is required for tabular imports", + )) + })?; + import_table( + &mut conn, + &request.database_name, + table_name, + table, + context, + ) + .await + } + }; + finish_connection(conn, result).await?; + Ok(TaskCompletion::WithoutArtifact( + "Import completed successfully".to_owned(), + )) +} + +pub(super) async fn export_sql( + application: &Application, + request: SqlFileExportRequest, + context: &TransferContext, +) -> Result { + validate_database_name(&request.database_name).map_err(TransferRunError::into_app_error)?; + let mut conn = open_connection(application, &request.datasource_id, false).await?; + let table_names = + resolve_tables(&mut conn, &request.database_name, &request.table_names).await?; + let full_database = request.table_names.is_empty(); + let file_name = timestamped_file_name(&request.database_name, "sql"); + let mut writer = + context.begin_artifact(&file_name, "application/sql; charset=utf-8", "SQL", "sql")?; + let write_result = write_sql_export( + &mut conn, + writer.file_mut(), + &request.database_name, + &table_names, + request.scope, + full_database, + context, + ) + .await; + finish_connection(conn, write_result).await?; + publish_task_artifact(writer, request.export_path.as_deref(), &file_name, context).await +} + +pub(super) async fn export_other( + application: &Application, + request: OtherFileExportRequest, + context: &TransferContext, +) -> Result { + validate_database_name(&request.database_name).map_err(TransferRunError::into_app_error)?; + if request.table_names.is_empty() { + return Err(AppError::invalid( + "missing_export_tables", + "tableNames must contain at least one table", + ) + .into()); + } + let mut conn = open_connection(application, &request.datasource_id, false).await?; + let table_names = + resolve_tables(&mut conn, &request.database_name, &request.table_names).await?; + let (file_name, media_type, artifact_format, extension) = if table_names.len() == 1 { + let extension = request.format.extension(); + ( + timestamped_file_name(&table_names[0], extension), + media_type(request.format), + request.format.extension().to_ascii_uppercase(), + extension, + ) + } else { + ( + timestamped_file_name(&request.database_name, "zip"), + "application/zip", + "ZIP".to_owned(), + "zip", + ) + }; + let mut writer = context.begin_artifact(&file_name, media_type, &artifact_format, extension)?; + let write_result = if request.format == TransferFileFormat::Sql && table_names.len() == 1 { + write_sql_data_export( + &mut conn, + writer.file_mut(), + &request.database_name, + &table_names, + context, + ) + .await + } else if request.format == TransferFileFormat::Sql { + write_sql_zip( + &mut conn, + writer.file_mut(), + &request.database_name, + &table_names, + context, + ) + .await + } else if table_names.len() == 1 { + write_table_tabular( + &mut conn, + writer.file_mut(), + &request.database_name, + &table_names[0], + request.format, + request.contains_header, + 0, + context.cancellation(), + Some(context), + ) + .await + .map(|_| ()) + } else { + write_tabular_zip( + &mut conn, + writer.file_mut(), + &request.database_name, + &table_names, + request.format, + request.contains_header, + context, + ) + .await + }; + finish_connection(conn, write_result).await?; + publish_task_artifact(writer, request.export_path.as_deref(), &file_name, context).await +} + +pub(super) async fn export_dml( + application: &Application, + request: DmlExportRequest, +) -> Result { + validate_database_name(&request.database_name).map_err(TransferRunError::into_app_error)?; + let sql = match request.export_size { + DmlExportSize::CurrentPage if !request.sql.trim().is_empty() => request.sql.trim(), + DmlExportSize::CurrentPage | DmlExportSize::All => request.original_sql.trim(), + }; + let table_name = select_table_name(sql)?; + let (format, extension, media_type) = match request.format { + DmlExportFormat::Csv => (TransferFileFormat::Csv, "csv", "text/csv; charset=utf-8"), + DmlExportFormat::Xlsx => ( + TransferFileFormat::Xlsx, + "xlsx", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ), + DmlExportFormat::Insert => ( + TransferFileFormat::Sql, + "sql", + "application/sql; charset=utf-8", + ), + }; + let stem = table_name + .as_deref() + .unwrap_or(request.database_name.as_str()); + let file_name = timestamped_file_name(stem, extension); + let storage = application.require_storage()?; + let expires_at = now_millis()?.saturating_add(DML_ARTIFACT_TTL_MS); + let mut writer = storage + .begin_transfer_artifact( + None, + &file_name, + media_type, + &extension.to_ascii_uppercase(), + extension, + Some(expires_at), + ) + .map_err(AppError::from)?; + let cancellation = CancellationToken::new(); + let mut conn = open_connection(application, &request.datasource_id, false) + .await + .map_err(TransferRunError::into_app_error)?; + let selected_result_set = request.result_set_id.unwrap_or(0); + let write_result = match request.format { + DmlExportFormat::Csv | DmlExportFormat::Xlsx => write_query_tabular( + &mut conn, + writer.file_mut(), + sql, + format, + true, + selected_result_set, + &cancellation, + None, + ) + .await + .map(|_| ()), + DmlExportFormat::Insert => { + let table_name = table_name.ok_or_else(|| { + AppError::invalid( + "sql_analysis_error", + "INSERT export requires a SELECT from a table", + ) + })?; + write_query_inserts( + &mut conn, + writer.file_mut(), + sql, + &request.database_name, + &table_name, + selected_result_set, + &cancellation, + None, + ) + .await + .map(|_| ()) + } + }; + finish_connection(conn, write_result) + .await + .map_err(TransferRunError::into_app_error)?; + writer.finish().map_err(AppError::from) +} + +enum ImportInput { + Sql(Vec), + Table(format::ImportedTable), +} + +impl ImportInput { + fn total(&self) -> Option { + match self { + Self::Sql(statements) => u64::try_from(statements.len()).ok(), + Self::Table(table) => u64::try_from(table.rows.len()).ok(), + } + } +} + +async fn read_sql_file(path: PathBuf) -> Result, TransferRunError> { + tokio::task::spawn_blocking(move || { + format::validate_import_file(&path)?; + let mut file = File::open(&path).map_err(import_file_error)?; + let mut script = String::new(); + file.read_to_string(&mut script).map_err(|error| { + tracing::warn!(%error, "SQL import file could not be decoded"); + AppError::invalid( + "invalid_sql_import_file", + "The SQL import file must contain UTF-8 text", + ) + })?; + native_mysql::split_mysql_script(&script) + }) + .await + .map_err(|_| TransferRunError::from(AppError::internal()))? + .map_err(TransferRunError::from) +} + +async fn read_table_file( + path: PathBuf, + format: TransferFileFormat, + contains_header: bool, + tabular_encoding: TabularImportEncoding, +) -> Result { + tokio::task::spawn_blocking(move || { + format::read_tabular_file(&path, format, contains_header, tabular_encoding) + }) + .await + .map_err(|_| TransferRunError::from(AppError::internal()))? + .map_err(TransferRunError::from) +} + +async fn import_sql( + conn: &mut Conn, + database_name: &str, + statements: Vec, + context: &TransferContext, +) -> Result<(), TransferRunError> { + let database_name = native_mysql::quote_identifier(database_name, "databaseName")?; + conn.query_drop(format!("USE {database_name}")) + .await + .map_err(mysql_error)?; + let total = u64::try_from(statements.len()).map_err(|_| AppError::internal())?; + for (index, statement) in statements.into_iter().enumerate() { + context.check_cancelled()?; + let query = conn.query_drop(statement); + tokio::pin!(query); + tokio::select! { + () = context.cancellation().cancelled() => return Err(TransferRunError::Cancelled), + result = &mut query => result.map_err(mysql_error)?, + } + let current = u64::try_from(index + 1).map_err(|_| AppError::internal())?; + context + .progress( + current, + Some(total), + "Executing SQL import", + (current == total).then_some("SQL statements imported"), + ) + .await?; + } + Ok(()) +} + +async fn import_table( + conn: &mut Conn, + database_name: &str, + table_name: &str, + table: format::ImportedTable, + context: &TransferContext, +) -> Result<(), TransferRunError> { + validate_database_name(database_name)?; + let available_columns = table_columns(conn, database_name, table_name).await?; + if available_columns.is_empty() { + return Err(AppError::not_found( + "mysql_table_not_found", + "The selected MySQL table does not exist", + ) + .into()); + } + let columns = match table.columns { + Some(headers) => canonical_import_columns(headers, &available_columns)?, + None => available_columns, + }; + if table.rows.iter().any(|row| row.len() != columns.len()) { + return Err(AppError::invalid( + "invalid_tabular_row", + "Every imported row must match the target column count", + ) + .into()); + } + let qualified = qualified_table(database_name, table_name)?; + let placeholders = std::iter::repeat_n("?", columns.len()) + .collect::>() + .join(", "); + let columns = columns + .iter() + .map(|column| native_mysql::quote_identifier(column, "columnName")) + .collect::, _>>()? + .join(", "); + let sql = format!("INSERT INTO {qualified} ({columns}) VALUES ({placeholders})"); + let total = u64::try_from(table.rows.len()).map_err(|_| AppError::internal())?; + let mut transaction = conn + .start_transaction(TxOpts::default()) + .await + .map_err(mysql_error)?; + let statement = match transaction.prep(sql).await { + Ok(statement) => statement, + Err(error) => { + let _ = transaction.rollback().await; + return Err(mysql_error(error)); + } + }; + let mut current = 0_u64; + for rows in table.rows.chunks(IMPORT_BATCH_ROWS) { + if context.cancellation().is_cancelled() { + let _ = transaction.rollback().await; + return Err(TransferRunError::Cancelled); + } + let parameters = rows.iter().map(|row| { + Params::Positional( + row.iter() + .map(|value| match value { + format::TabularValue::Null => Value::NULL, + format::TabularValue::Text(value) => { + Value::Bytes(value.as_bytes().to_vec()) + } + format::TabularValue::Bytes(value) => Value::Bytes(value.clone()), + }) + .collect(), + ) + }); + if let Err(error) = transaction.exec_batch(&statement, parameters).await { + let _ = transaction.rollback().await; + return Err(mysql_error(error)); + } + current = current + .checked_add(u64::try_from(rows.len()).map_err(|_| AppError::internal())?) + .ok_or_else(AppError::internal)?; + if let Err(error) = context + .progress(current, Some(total), "Importing table rows", None) + .await + { + let _ = transaction.rollback().await; + return Err(error); + } + } + if context.cancellation().is_cancelled() { + let _ = transaction.rollback().await; + return Err(TransferRunError::Cancelled); + } + transaction.commit().await.map_err(mysql_error)?; + Ok(()) +} + +async fn write_sql_export( + conn: &mut Conn, + output: &mut File, + database_name: &str, + table_names: &[String], + scope: TransferSqlScope, + full_database: bool, + context: &TransferContext, +) -> Result<(), TransferRunError> { + writeln!(output, "-- Chat2DB MySQL export").map_err(export_file_error)?; + writeln!(output, "SET FOREIGN_KEY_CHECKS=0;").map_err(export_file_error)?; + writeln!( + output, + "USE {};", + native_mysql::quote_identifier(database_name, "databaseName")? + ) + .map_err(export_file_error)?; + + if matches!(scope, TransferSqlScope::All | TransferSqlScope::Schema) { + write_table_definitions(conn, output, database_name, table_names, context).await?; + if full_database { + write_database_objects(conn, output, database_name, context).await?; + } + } + if matches!(scope, TransferSqlScope::All | TransferSqlScope::Table) { + write_table_data(conn, output, database_name, table_names, context).await?; + } + writeln!(output, "SET FOREIGN_KEY_CHECKS=1;").map_err(export_file_error)?; + output.flush().map_err(export_file_error) +} + +async fn write_sql_data_export( + conn: &mut Conn, + output: &mut File, + database_name: &str, + table_names: &[String], + context: &TransferContext, +) -> Result<(), TransferRunError> { + writeln!(output, "-- Chat2DB MySQL table-data export").map_err(export_file_error)?; + writeln!(output, "SET FOREIGN_KEY_CHECKS=0;").map_err(export_file_error)?; + write_table_data(conn, output, database_name, table_names, context).await?; + writeln!(output, "SET FOREIGN_KEY_CHECKS=1;").map_err(export_file_error)?; + output.flush().map_err(export_file_error) +} + +async fn write_table_definitions( + conn: &mut Conn, + output: &mut File, + database_name: &str, + table_names: &[String], + context: &TransferContext, +) -> Result<(), TransferRunError> { + for (index, table_name) in table_names.iter().enumerate() { + context.check_cancelled()?; + let qualified = qualified_table(database_name, table_name)?; + let create = show_create(conn, &format!("SHOW CREATE TABLE {qualified}"), 1).await?; + writeln!( + output, + "\nDROP TABLE IF EXISTS {};", + native_mysql::quote_identifier(table_name, "tableName")? + ) + .and_then(|()| writeln!(output, "{create};")) + .map_err(export_file_error)?; + context + .progress( + u64::try_from(index + 1).map_err(|_| AppError::internal())?, + u64::try_from(table_names.len()).ok(), + "Exporting table definitions", + None, + ) + .await?; + } + Ok(()) +} + +async fn write_database_objects( + conn: &mut Conn, + output: &mut File, + database_name: &str, + context: &TransferContext, +) -> Result<(), TransferRunError> { + let views: Vec = conn + .exec( + "SELECT TABLE_NAME FROM information_schema.VIEWS WHERE TABLE_SCHEMA = ? ORDER BY TABLE_NAME", + (database_name,), + ) + .await + .map_err(mysql_error)?; + for view in views { + context.check_cancelled()?; + let create = show_create( + conn, + &format!( + "SHOW CREATE VIEW {}", + qualified_table(database_name, &view)? + ), + 1, + ) + .await?; + writeln!( + output, + "\nDROP VIEW IF EXISTS {};\n{create};", + native_mysql::quote_identifier(&view, "viewName")? + ) + .map_err(export_file_error)?; + } + + let routines: Vec<(String, String)> = conn + .exec( + "SELECT ROUTINE_NAME, ROUTINE_TYPE FROM information_schema.ROUTINES \ + WHERE ROUTINE_SCHEMA = ? ORDER BY ROUTINE_TYPE, ROUTINE_NAME", + (database_name,), + ) + .await + .map_err(mysql_error)?; + for (name, kind) in routines { + context.check_cancelled()?; + let kind = if kind.eq_ignore_ascii_case("FUNCTION") { + "FUNCTION" + } else { + "PROCEDURE" + }; + let create = show_create( + conn, + &format!( + "SHOW CREATE {kind} {}", + qualified_table(database_name, &name)? + ), + 2, + ) + .await?; + write_delimited_object(output, kind, &name, &create)?; + } + + let triggers: Vec = conn + .exec( + "SELECT TRIGGER_NAME FROM information_schema.TRIGGERS \ + WHERE TRIGGER_SCHEMA = ? ORDER BY TRIGGER_NAME", + (database_name,), + ) + .await + .map_err(mysql_error)?; + for trigger in triggers { + context.check_cancelled()?; + let create = show_create( + conn, + &format!( + "SHOW CREATE TRIGGER {}", + qualified_table(database_name, &trigger)? + ), + 2, + ) + .await?; + write_delimited_object(output, "TRIGGER", &trigger, &create)?; + } + Ok(()) +} + +fn write_delimited_object( + output: &mut File, + kind: &str, + name: &str, + create: &str, +) -> Result<(), TransferRunError> { + writeln!(output, "\nDELIMITER $$") + .and_then(|()| { + writeln!( + output, + "DROP {kind} IF EXISTS {}$$", + native_mysql::quote_identifier(name, "objectName") + .map_err(|_| std::io::Error::other("invalid MySQL object name"))? + ) + }) + .and_then(|()| writeln!(output, "{create}$$")) + .and_then(|()| writeln!(output, "DELIMITER ;")) + .map_err(export_file_error) +} + +async fn write_table_data( + conn: &mut Conn, + output: &mut File, + database_name: &str, + table_names: &[String], + context: &TransferContext, +) -> Result<(), TransferRunError> { + let cancellation = context.cancellation(); + let mut exported_rows = 0_u64; + for table_name in table_names { + context.check_cancelled()?; + writeln!(output, "\n-- Data for table {table_name}").map_err(export_file_error)?; + let sql = format!( + "SELECT * FROM {}", + qualified_table(database_name, table_name)? + ); + let rows = write_query_inserts( + conn, + output, + &sql, + database_name, + table_name, + 0, + cancellation, + Some(context), + ) + .await?; + exported_rows = exported_rows.saturating_add(rows); + context + .progress( + exported_rows, + None, + "Exporting table data", + Some(&format!("Exported table {table_name}")), + ) + .await?; + } + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +async fn write_query_inserts( + conn: &mut Conn, + output: &mut File, + sql: &str, + database_name: &str, + table_name: &str, + selected_result_set: u32, + cancellation: &CancellationToken, + context: Option<&TransferContext>, +) -> Result { + let no_backslash_escapes = session_no_backslash_escapes(conn).await?; + let mut result = conn.query_iter(sql).await.map_err(mysql_error)?; + let mut result_set = 0_u32; + let mut found = false; + let mut row_count = 0_u64; + let qualified = qualified_table(database_name, table_name)?; + loop { + if result.is_empty() { + break; + } + let columns = result + .columns_ref() + .iter() + .map(|column| column.name_str().into_owned()) + .collect::>(); + let selected = !columns.is_empty() && result_set == selected_result_set; + if selected { + found = true; + } + loop { + let row = tokio::select! { + () = cancellation.cancelled() => return Err(TransferRunError::Cancelled), + row = result.next() => row.map_err(mysql_error)?, + }; + let Some(row) = row else { + break; + }; + if !selected { + continue; + } + let values = row.unwrap(); + if values.len() != columns.len() { + return Err(AppError::internal().into()); + } + let column_sql = columns + .iter() + .map(|column| native_mysql::quote_identifier(column, "columnName")) + .collect::, _>>()? + .join(", "); + let value_sql = values + .iter() + .map(|value| value.as_sql(no_backslash_escapes)) + .collect::>() + .join(", "); + writeln!( + output, + "INSERT INTO {qualified} ({column_sql}) VALUES ({value_sql});" + ) + .map_err(export_file_error)?; + row_count = row_count.saturating_add(1); + if row_count.is_multiple_of(PROGRESS_ROW_INTERVAL) + && let Some(context) = context + { + context + .progress(row_count, None, "Exporting table rows", None) + .await?; + } + } + if !columns.is_empty() { + result_set = result_set.saturating_add(1); + if selected { + break; + } + } + } + drop(result); + if !found { + return Err(AppError::invalid( + "result_set_not_found", + "The selected result set does not exist", + ) + .into()); + } + Ok(row_count) +} + +#[allow(clippy::too_many_arguments)] +async fn write_table_tabular( + conn: &mut Conn, + output: &mut File, + database_name: &str, + table_name: &str, + format: TransferFileFormat, + contains_header: bool, + selected_result_set: u32, + cancellation: &CancellationToken, + context: Option<&TransferContext>, +) -> Result { + let sql = format!( + "SELECT * FROM {}", + qualified_table(database_name, table_name)? + ); + write_query_tabular( + conn, + output, + &sql, + format, + contains_header, + selected_result_set, + cancellation, + context, + ) + .await +} + +#[allow(clippy::too_many_arguments)] +async fn write_query_tabular( + conn: &mut Conn, + output: &mut File, + sql: &str, + format: TransferFileFormat, + contains_header: bool, + selected_result_set: u32, + cancellation: &CancellationToken, + context: Option<&TransferContext>, +) -> Result { + let mut result = conn.query_iter(sql).await.map_err(mysql_error)?; + let mut sink = format::tabular_sink(format, output, contains_header)?; + let mut result_set = 0_u32; + let mut found = false; + let mut row_count = 0_u64; + loop { + if result.is_empty() { + break; + } + let column_metadata = result.columns_ref().to_vec(); + let columns = column_metadata + .iter() + .map(|column| column.name_str().into_owned()) + .collect::>(); + let binary_columns = column_metadata + .iter() + .map(is_binary_tabular_column) + .collect::>(); + let selected = !columns.is_empty() && result_set == selected_result_set; + if selected { + sink.write_header(&columns)?; + found = true; + } + loop { + let row = tokio::select! { + () = cancellation.cancelled() => return Err(TransferRunError::Cancelled), + row = result.next() => row.map_err(mysql_error)?, + }; + let Some(row) = row else { + break; + }; + if !selected { + continue; + } + let values = row + .unwrap() + .into_iter() + .zip(binary_columns.iter().copied()) + .map(|(value, binary)| tabular_value(value, binary)) + .collect::>(); + sink.write_row(&values)?; + row_count = row_count.saturating_add(1); + if row_count.is_multiple_of(PROGRESS_ROW_INTERVAL) + && let Some(context) = context + { + context + .progress(row_count, None, "Exporting table rows", None) + .await?; + } + } + if !columns.is_empty() { + result_set = result_set.saturating_add(1); + if selected { + break; + } + } + } + drop(result); + if !found { + return Err(AppError::invalid( + "result_set_not_found", + "The selected result set does not exist", + ) + .into()); + } + sink.finish()?; + Ok(row_count) +} + +async fn write_tabular_zip( + conn: &mut Conn, + output: &mut File, + database_name: &str, + table_names: &[String], + format: TransferFileFormat, + contains_header: bool, + context: &TransferContext, +) -> Result<(), TransferRunError> { + let temporary = TempDir::new().map_err(export_file_error)?; + let cancellation = context.cancellation(); + let mut paths = Vec::with_capacity(table_names.len()); + for (index, table_name) in table_names.iter().enumerate() { + context.check_cancelled()?; + let name = format!("{}.{}", safe_file_stem(table_name), format.extension()); + let path = temporary.path().join(&name); + let mut file = OpenOptions::new() + .create_new(true) + .read(true) + .write(true) + .open(&path) + .map_err(export_file_error)?; + write_table_tabular( + conn, + &mut file, + database_name, + table_name, + format, + contains_header, + 0, + cancellation, + Some(context), + ) + .await?; + file.sync_all().map_err(export_file_error)?; + paths.push((name, path)); + context + .progress( + u64::try_from(index + 1).map_err(|_| AppError::internal())?, + u64::try_from(table_names.len()).ok(), + "Preparing table archive", + None, + ) + .await?; + } + context.check_cancelled()?; + write_zip_entries(output, paths) +} + +async fn write_sql_zip( + conn: &mut Conn, + output: &mut File, + database_name: &str, + table_names: &[String], + context: &TransferContext, +) -> Result<(), TransferRunError> { + let temporary = TempDir::new().map_err(export_file_error)?; + let mut paths = Vec::with_capacity(table_names.len()); + for (index, table_name) in table_names.iter().enumerate() { + context.check_cancelled()?; + let name = format!("{}.sql", safe_file_stem(table_name)); + let path = temporary.path().join(&name); + let mut file = OpenOptions::new() + .create_new(true) + .read(true) + .write(true) + .open(&path) + .map_err(export_file_error)?; + write_sql_data_export( + conn, + &mut file, + database_name, + std::slice::from_ref(table_name), + context, + ) + .await?; + file.sync_all().map_err(export_file_error)?; + paths.push((name, path)); + context + .progress( + u64::try_from(index + 1).map_err(|_| AppError::internal())?, + u64::try_from(table_names.len()).ok(), + "Preparing SQL table archive", + None, + ) + .await?; + } + context.check_cancelled()?; + write_zip_entries(output, paths) +} + +fn write_zip_entries( + output: &mut File, + paths: Vec<(String, PathBuf)>, +) -> Result<(), TransferRunError> { + let mut zip = ZipWriter::new(output); + let options = SimpleFileOptions::default().compression_method(CompressionMethod::Deflated); + for (name, path) in paths { + zip.start_file(name, options).map_err(zip_error)?; + let mut source = File::open(path).map_err(export_file_error)?; + std::io::copy(&mut source, &mut zip).map_err(export_file_error)?; + } + zip.finish().map_err(zip_error)?; + Ok(()) +} + +async fn session_no_backslash_escapes(conn: &mut Conn) -> Result { + let sql_mode = conn + .query_first::("SELECT @@SESSION.sql_mode") + .await + .map_err(mysql_error)? + .unwrap_or_default(); + Ok(sql_mode_has_no_backslash_escapes(&sql_mode)) +} + +fn sql_mode_has_no_backslash_escapes(sql_mode: &str) -> bool { + sql_mode + .split(',') + .any(|mode| mode.trim().eq_ignore_ascii_case("NO_BACKSLASH_ESCAPES")) +} + +async fn resolve_tables( + conn: &mut Conn, + database_name: &str, + requested: &[String], +) -> Result, TransferRunError> { + let available: Vec = conn + .exec( + "SELECT TABLE_NAME FROM information_schema.TABLES \ + WHERE TABLE_SCHEMA = ? AND TABLE_TYPE = 'BASE TABLE' ORDER BY TABLE_NAME", + (database_name,), + ) + .await + .map_err(mysql_error)?; + if requested.is_empty() { + return Ok(available); + } + let available_set = available.iter().collect::>(); + let mut seen = HashSet::with_capacity(requested.len()); + let mut selected = Vec::with_capacity(requested.len()); + for table in requested { + if !available_set.contains(table) { + return Err(AppError::not_found( + "mysql_table_not_found", + format!("MySQL table {database_name}.{table} does not exist"), + ) + .into()); + } + if seen.insert(table) { + selected.push(table.clone()); + } + } + Ok(selected) +} + +async fn table_columns( + conn: &mut Conn, + database_name: &str, + table_name: &str, +) -> Result, TransferRunError> { + conn.exec( + "SELECT COLUMN_NAME FROM information_schema.COLUMNS \ + WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? ORDER BY ORDINAL_POSITION", + (database_name, table_name), + ) + .await + .map_err(mysql_error) +} + +fn canonical_import_columns( + requested: Vec, + available: &[String], +) -> Result, TransferRunError> { + requested + .into_iter() + .map(|column| { + available + .iter() + .find(|available| available.eq_ignore_ascii_case(&column)) + .cloned() + .ok_or_else(|| { + AppError::invalid( + "unknown_import_column", + format!("Import column {column} does not exist in the target table"), + ) + .into() + }) + }) + .collect() +} + +async fn show_create( + conn: &mut Conn, + sql: &str, + value_index: usize, +) -> Result { + let row = conn + .query_first::(sql) + .await + .map_err(mysql_error)? + .ok_or_else(|| { + TransferRunError::from(AppError::not_found( + "mysql_object_not_found", + "The MySQL object no longer exists", + )) + })?; + row.get_opt::(value_index) + .ok_or_else(|| TransferRunError::from(AppError::internal()))? + .map_err(|_| TransferRunError::from(AppError::internal())) +} + +async fn open_connection( + application: &Application, + datasource_id: &str, + writable: bool, +) -> Result { + let resolved = native_mysql::resolve_native_connection(application, datasource_id).await?; + if writable && resolved.connection.read_only { + return Err(AppError::new( + AppErrorKind::Conflict, + chat2db_contract::ApiError::new( + "datasource_read_only", + "The datasource is configured as read-only", + ), + ) + .into()); + } + native_mysql::open_resolved_connection(&resolved) + .await + .map_err(TransferRunError::from) +} + +async fn finish_connection( + conn: native_mysql::ManagedMysqlConnection, + result: Result, +) -> Result { + match result { + Ok(value) => native_mysql::finish_connection(conn, Ok(value)) + .await + .map_err(TransferRunError::from), + Err(TransferRunError::Cancelled) => { + drop(conn); + Err(TransferRunError::Cancelled) + } + Err(TransferRunError::Failed(error)) => native_mysql::finish_connection(conn, Err(error)) + .await + .map_err(TransferRunError::from), + } +} + +async fn publish_task_artifact( + mut writer: TransferArtifactWriter, + export_path: Option<&str>, + file_name: &str, + context: &TransferContext, +) -> Result { + writer.file_mut().flush().map_err(export_file_error)?; + writer.file_mut().sync_all().map_err(export_file_error)?; + let pending = match export_path { + Some(path) => Some( + stage_user_copy( + writer.path().to_owned(), + path.to_owned(), + file_name.to_owned(), + ) + .await?, + ), + None => None, + }; + context.check_cancelled()?; + let artifact = writer.finish().map_err(AppError::from)?; + if let Some(pending) = pending + && let Err(error) = pending.publish().await + { + tracing::warn!(%error, artifact_id = %artifact.id, "managed transfer succeeded but exportPath publication failed"); + } + Ok(TaskCompletion::Artifact(artifact)) +} + +struct PendingUserCopy { + part_path: PathBuf, + final_path: PathBuf, + published: bool, +} + +impl PendingUserCopy { + async fn publish(mut self) -> Result<(), AppError> { + let part_path = self.part_path.clone(); + let final_path = self.final_path.clone(); + tokio::task::spawn_blocking(move || { + fs::rename(&part_path, &final_path).map_err(export_file_app_error)?; + sync_parent(&final_path) + }) + .await + .map_err(|_| AppError::internal())??; + self.published = true; + Ok(()) + } +} + +impl Drop for PendingUserCopy { + fn drop(&mut self) { + if !self.published { + let _ = fs::remove_file(&self.part_path); + } + } +} + +async fn stage_user_copy( + source: PathBuf, + export_path: String, + file_name: String, +) -> Result { + tokio::task::spawn_blocking(move || { + let directory = PathBuf::from(export_path); + fs::create_dir_all(&directory).map_err(export_file_app_error)?; + let directory = fs::canonicalize(directory).map_err(export_file_app_error)?; + let final_path = directory.join(&file_name); + let part_path = directory.join(format!(".{file_name}.{}.part", Uuid::new_v4())); + let result: Result = (|| { + let mut input = File::open(source).map_err(export_file_app_error)?; + let mut output = OpenOptions::new() + .create_new(true) + .write(true) + .open(&part_path) + .map_err(export_file_app_error)?; + std::io::copy(&mut input, &mut output).map_err(export_file_app_error)?; + output.sync_all().map_err(export_file_app_error)?; + Ok(PendingUserCopy { + part_path: part_path.clone(), + final_path, + published: false, + }) + })(); + if result.is_err() { + let _ = fs::remove_file(part_path); + } + result + }) + .await + .map_err(|_| TransferRunError::from(AppError::internal()))? + .map_err(TransferRunError::from) +} + +fn select_table_name(sql: &str) -> Result, AppError> { + if sql.trim().is_empty() { + return Err(AppError::invalid( + "invalid_dml_export_sql", + "A SELECT statement is required for export", + )); + } + let statements = Parser::parse_sql(&MySqlDialect {}, sql).map_err(|_| { + AppError::invalid( + "sql_analysis_error", + "The export SQL could not be parsed as one MySQL SELECT statement", + ) + })?; + let [Statement::Query(query)] = statements.as_slice() else { + return Err(AppError::invalid( + "sql_analysis_error", + "The export SQL must be exactly one SELECT statement", + )); + }; + Ok(first_query_table(query)) +} + +fn first_query_table(query: &Query) -> Option { + query + .with + .as_ref() + .and_then(|with| { + with.cte_tables + .iter() + .find_map(|cte| first_query_table(&cte.query)) + }) + .or_else(|| first_set_table(&query.body)) +} + +fn first_set_table(expression: &SetExpr) -> Option { + match expression { + SetExpr::Select(select) => select + .from + .iter() + .find_map(|table| table_factor_name(&table.relation)), + SetExpr::Query(query) => first_query_table(query), + SetExpr::SetOperation { left, right, .. } => { + first_set_table(left).or_else(|| first_set_table(right)) + } + SetExpr::Table(table) => table.table_name.as_ref().map(ToString::to_string), + SetExpr::Values(_) + | SetExpr::Insert(_) + | SetExpr::Update(_) + | SetExpr::Delete(_) + | SetExpr::Merge(_) => None, + } +} + +fn table_factor_name(factor: &TableFactor) -> Option { + match factor { + TableFactor::Table { name, .. } => object_name_last(name), + TableFactor::Derived { subquery, .. } => first_query_table(subquery), + _ => None, + } +} + +fn object_name_last(name: &ObjectName) -> Option { + name.0 + .last() + .and_then(|part| part.as_ident()) + .map(|identifier| identifier.value.clone()) +} + +fn tabular_value(value: Value, binary: bool) -> format::TabularValue { + match value { + Value::NULL => format::TabularValue::Null, + Value::Bytes(bytes) if binary => format::TabularValue::Bytes(bytes), + Value::Bytes(bytes) => match String::from_utf8(bytes) { + Ok(value) => format::TabularValue::Text(value), + Err(error) => format::TabularValue::Bytes(error.into_bytes()), + }, + Value::Int(value) => format::TabularValue::Text(value.to_string()), + Value::UInt(value) => format::TabularValue::Text(value.to_string()), + Value::Float(value) => format::TabularValue::Text(value.to_string()), + Value::Double(value) => format::TabularValue::Text(value.to_string()), + Value::Date(year, month, day, hour, minute, second, micros) => { + let mut value = format!("{year:04}-{month:02}-{day:02}"); + if hour != 0 || minute != 0 || second != 0 || micros != 0 { + let _ = write!(value, " {hour:02}:{minute:02}:{second:02}"); + if micros != 0 { + let _ = write!(value, ".{micros:06}"); + } + } + format::TabularValue::Text(value) + } + Value::Time(negative, days, hours, minutes, seconds, micros) => { + let total_hours = days.saturating_mul(24).saturating_add(u32::from(hours)); + let sign = if negative { "-" } else { "" }; + let mut value = format!("{sign}{total_hours:02}:{minutes:02}:{seconds:02}"); + if micros != 0 { + let _ = write!(value, ".{micros:06}"); + } + format::TabularValue::Text(value) + } + } +} + +fn is_binary_tabular_column(column: &Column) -> bool { + use ColumnType as Type; + + matches!( + column.column_type(), + Type::MYSQL_TYPE_BIT | Type::MYSQL_TYPE_GEOMETRY | Type::MYSQL_TYPE_VECTOR + ) || (matches!( + column.column_type(), + Type::MYSQL_TYPE_VARCHAR + | Type::MYSQL_TYPE_VAR_STRING + | Type::MYSQL_TYPE_STRING + | Type::MYSQL_TYPE_TINY_BLOB + | Type::MYSQL_TYPE_MEDIUM_BLOB + | Type::MYSQL_TYPE_LONG_BLOB + | Type::MYSQL_TYPE_BLOB + ) && (column.character_set() == 63 || column.flags().contains(ColumnFlags::BINARY_FLAG))) +} + +fn qualified_table(database_name: &str, table_name: &str) -> Result { + Ok(format!( + "{}.{}", + native_mysql::quote_identifier(database_name, "databaseName")?, + native_mysql::quote_identifier(table_name, "tableName")? + )) +} + +fn validate_database_name(database_name: &str) -> Result<(), TransferRunError> { + native_mysql::quote_identifier(database_name, "databaseName")?; + Ok(()) +} + +fn media_type(format: TransferFileFormat) -> &'static str { + match format { + TransferFileFormat::Csv => "text/csv; charset=utf-8", + TransferFileFormat::Xls => "application/vnd.ms-excel", + TransferFileFormat::Xlsx => { + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + } + TransferFileFormat::Sql => "application/sql; charset=utf-8", + } +} + +fn timestamped_file_name(stem: &str, extension: &str) -> String { + format!( + "{}_{}.{}", + safe_file_stem(stem), + now_millis().unwrap_or(0), + extension + ) +} + +fn safe_file_stem(value: &str) -> String { + let output = value + .chars() + .map(|character| { + if character.is_alphanumeric() || matches!(character, '-' | '_') { + character + } else { + '_' + } + }) + .take(128) + .collect::(); + if output.is_empty() { + "mysql_export".to_owned() + } else { + output + } +} + +fn now_millis() -> Result { + let millis = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|_| AppError::internal())? + .as_millis(); + i64::try_from(millis).map_err(|_| AppError::internal()) +} + +fn sync_parent(path: &Path) -> Result<(), AppError> { + File::open(path.parent().ok_or_else(AppError::internal)?) + .and_then(|directory| directory.sync_all()) + .map_err(export_file_app_error) +} + +fn import_file_error(error: std::io::Error) -> AppError { + tracing::warn!(%error, "MySQL import file could not be opened"); + drop(error); + AppError::not_found( + "import_file_not_found", + "The selected import file could not be opened", + ) +} + +fn export_file_error(error: std::io::Error) -> TransferRunError { + export_file_app_error(error).into() +} + +fn export_file_app_error(error: std::io::Error) -> AppError { + tracing::warn!(%error, "MySQL export file operation failed"); + drop(error); + AppError::unavailable( + "transfer_file_write_failed", + "The export file could not be written", + ) +} + +fn zip_error(error: zip::result::ZipError) -> TransferRunError { + tracing::warn!(%error, "MySQL export archive operation failed"); + drop(error); + AppError::unavailable( + "transfer_archive_failed", + "The export archive could not be written", + ) + .into() +} + +fn mysql_error(error: MysqlError) -> TransferRunError { + let (kind, code, message) = match &error { + MysqlError::Server(server) => ( + AppErrorKind::InvalidRequest, + "mysql_transfer_query_failed", + format!( + "MySQL rejected the transfer operation (server error {})", + server.code + ), + ), + _ => ( + AppErrorKind::Unavailable, + "mysql_transfer_unavailable", + "The MySQL transfer operation could not be completed".to_owned(), + ), + }; + tracing::warn!(%error, "native MySQL transfer operation failed"); + drop(error); + AppError::new(kind, chat2db_contract::ApiError::new(code, message)).into() +} + +#[cfg(test)] +mod tests { + use super::{safe_file_stem, select_table_name, sql_mode_has_no_backslash_escapes}; + + #[test] + fn insert_export_uses_parser_for_the_select_table() { + assert_eq!( + select_table_name("SELECT u.id FROM `app`.`user_account` AS u") + .expect("select parses") + .as_deref(), + Some("user_account") + ); + assert!(select_table_name("UPDATE user_account SET name = 'x'").is_err()); + assert!(select_table_name("SELECT 1; SELECT 2").is_err()); + } + + #[test] + fn artifact_file_stems_do_not_create_paths() { + assert_eq!(safe_file_stem("../../audit/log"), "______audit_log"); + } + + #[test] + fn sql_mode_detection_is_case_insensitive_and_token_based() { + assert!(sql_mode_has_no_backslash_escapes( + "STRICT_TRANS_TABLES,NO_BACKSLASH_ESCAPES" + )); + assert!(sql_mode_has_no_backslash_escapes(" no_backslash_escapes ")); + assert!(!sql_mode_has_no_backslash_escapes( + "STRICT_TRANS_TABLES,NO_BACKSLASH_ESCAPES_EXTRA" + )); + } +} diff --git a/crates/chat2db-core/src/workspace.rs b/crates/chat2db-core/src/workspace.rs new file mode 100644 index 0000000..554eea0 --- /dev/null +++ b/crates/chat2db-core/src/workspace.rs @@ -0,0 +1,281 @@ +use std::collections::HashMap; + +use chat2db_contract::{ + AssignDatasourceNamespaceRequest, CreateWorkspaceNamespaceRequest, MoveWorkspaceNodeRequest, + UpdateWorkspaceNamespaceRequest, WorkspaceDatasourceGroup, WorkspaceDatasourceList, + WorkspaceNamespace, WorkspaceNodeKind, WorkspaceNodeRef, WorkspaceTree, WorkspaceTreeNode, +}; +use chat2db_storage::{ + WorkspaceNodeKind as StoredWorkspaceNodeKind, WorkspaceNodeLocator, WorkspaceNodeRecord, +}; + +use crate::{AppError, Application, storage_call}; + +impl Application { + /// Returns the persisted local datasource workspace tree. + /// + /// # Errors + /// + /// Returns availability, storage, or persisted-tree integrity failures. + pub async fn workspace_tree(&self) -> Result { + let storage = self.require_storage()?; + let records = storage_call(move || storage.list_workspace_nodes()).await?; + build_tree(records) + } + + /// Creates a root or child namespace at the end of its sibling list. + /// + /// # Errors + /// + /// Returns validation, parent-not-found, availability, or storage failures. + pub async fn create_workspace_namespace( + &self, + request: CreateWorkspaceNamespaceRequest, + ) -> Result { + let parent_id = parse_optional_namespace_id(request.parent_id.as_deref())?; + let storage = self.require_storage()?; + let record = + storage_call(move || storage.create_workspace_namespace(&request.name, parent_id)) + .await?; + Ok(WorkspaceNamespace { + id: record.id.to_string(), + name: record.name, + parent_id: record.parent_id.map(|id| id.to_string()), + }) + } + + /// Renames one persisted namespace. + /// + /// # Errors + /// + /// Returns validation, namespace-not-found, availability, or storage failures. + pub async fn update_workspace_namespace( + &self, + request: UpdateWorkspaceNamespaceRequest, + ) -> Result { + let id = parse_namespace_id(&request.id)?; + let storage = self.require_storage()?; + let record = + storage_call(move || storage.update_workspace_namespace(id, &request.name)).await?; + Ok(WorkspaceNamespace { + id: record.id.to_string(), + name: record.name, + parent_id: record.parent_id.map(|id| id.to_string()), + }) + } + + /// Deletes a namespace and promotes its direct children into its parent. + /// + /// # Errors + /// + /// Returns validation, namespace-not-found, availability, or storage failures. + pub async fn delete_workspace_namespace(&self, id: &str) -> Result<(), AppError> { + let id = parse_namespace_id(id)?; + let storage = self.require_storage()?; + storage_call(move || storage.delete_workspace_namespace(id)).await + } + + /// Applies Community-compatible namespace/datasource tree movement. + /// + /// # Errors + /// + /// Returns invalid-position, missing-node, cycle, availability, or storage failures. + pub async fn move_workspace_node( + &self, + request: MoveWorkspaceNodeRequest, + ) -> Result<(), AppError> { + let drag = node_locator(request.drag_node)?; + let target = node_locator(request.drop_to_node)?; + let storage = self.require_storage()?; + storage_call(move || storage.move_workspace_node(&drag, &target, request.drop_position)) + .await + } + + /// Assigns a datasource to the end of one namespace or the root. + /// + /// # Errors + /// + /// Returns datasource/namespace-not-found, validation, availability, or storage failures. + pub async fn assign_datasource_namespace( + &self, + request: AssignDatasourceNamespaceRequest, + ) -> Result<(), AppError> { + let namespace_id = parse_optional_namespace_id(request.namespace_id.as_deref())?; + let storage = self.require_storage()?; + storage_call(move || { + storage.assign_datasource_namespace(&request.datasource_id, namespace_id) + }) + .await + } + + /// Returns direct datasource membership for root and every namespace. + /// + /// # Errors + /// + /// Returns availability, storage, or persisted-tree integrity failures. + pub async fn workspace_datasource_list(&self) -> Result { + let storage = self.require_storage()?; + let records = storage_call(move || storage.list_workspace_nodes()).await?; + let mut datasource_ids = HashMap::, Vec>::new(); + let mut namespaces = Vec::::new(); + for record in records { + match record.kind { + StoredWorkspaceNodeKind::Namespace => { + namespaces.push(parse_namespace_id(&record.id)?); + } + StoredWorkspaceNodeKind::DataSource => datasource_ids + .entry(record.parent_namespace_id) + .or_default() + .push(record.id), + } + } + let mut groups = Vec::with_capacity(namespaces.len() + 1); + groups.push(WorkspaceDatasourceGroup { + namespace_id: None, + datasource_ids: datasource_ids.remove(&None).unwrap_or_default(), + }); + groups.extend(namespaces.into_iter().map(|namespace_id| { + WorkspaceDatasourceGroup { + namespace_id: Some(namespace_id.to_string()), + datasource_ids: datasource_ids + .remove(&Some(namespace_id)) + .unwrap_or_default(), + } + })); + if !datasource_ids.is_empty() { + return Err(AppError::internal()); + } + Ok(WorkspaceDatasourceList { groups }) + } +} + +fn build_tree(records: Vec) -> Result { + let mut children = HashMap::, Vec>::new(); + for record in records { + children + .entry(record.parent_namespace_id) + .or_default() + .push(record); + } + for siblings in children.values_mut() { + siblings.sort_by(|left, right| { + left.position + .cmp(&right.position) + .then_with(|| left.id.cmp(&right.id)) + }); + } + let items = build_children(None, &mut children, 0)?; + if children.values().any(|records| !records.is_empty()) { + return Err(AppError::internal()); + } + Ok(WorkspaceTree { items }) +} + +fn build_children( + parent: Option, + children: &mut HashMap, Vec>, + depth: usize, +) -> Result, AppError> { + if depth > 1_024 { + return Err(AppError::internal()); + } + let records = children.remove(&parent).unwrap_or_default(); + records + .into_iter() + .map(|record| match record.kind { + StoredWorkspaceNodeKind::Namespace => { + let namespace_id = parse_namespace_id(&record.id)?; + Ok(WorkspaceTreeNode { + id: record.id.clone(), + node_type: WorkspaceNodeKind::Namespace, + name: record.name, + datasource_id: None, + namespace_id: Some(record.id), + children: build_children(Some(namespace_id), children, depth + 1)?, + }) + } + StoredWorkspaceNodeKind::DataSource => Ok(WorkspaceTreeNode { + id: record.id.clone(), + node_type: WorkspaceNodeKind::DataSource, + name: record.name, + datasource_id: Some(record.id), + namespace_id: None, + children: Vec::new(), + }), + }) + .collect() +} + +fn node_locator(reference: WorkspaceNodeRef) -> Result { + let kind = match reference.node_type { + WorkspaceNodeKind::Namespace => { + parse_namespace_id(&reference.id)?; + StoredWorkspaceNodeKind::Namespace + } + WorkspaceNodeKind::DataSource => { + if reference.id.trim().is_empty() { + return Err(AppError::invalid( + "invalid_workspace_operation", + "datasource id cannot be empty", + )); + } + StoredWorkspaceNodeKind::DataSource + } + }; + Ok(WorkspaceNodeLocator { + id: reference.id, + kind, + }) +} + +fn parse_optional_namespace_id(value: Option<&str>) -> Result, AppError> { + value.map(parse_namespace_id).transpose() +} + +fn parse_namespace_id(value: &str) -> Result { + value + .parse::() + .ok() + .filter(|id| *id > 0) + .ok_or_else(|| { + AppError::invalid( + "invalid_workspace_operation", + "namespace id must be a positive integer", + ) + }) +} + +#[cfg(test)] +mod tests { + use chat2db_storage::{WorkspaceNodeKind, WorkspaceNodeRecord}; + + use super::build_tree; + + #[test] + fn tree_builder_combines_namespaces_and_datasources_without_secret_fields() { + let tree = build_tree(vec![ + WorkspaceNodeRecord { + id: "1".to_owned(), + kind: WorkspaceNodeKind::Namespace, + name: "Development".to_owned(), + parent_namespace_id: None, + position: 0, + }, + WorkspaceNodeRecord { + id: "mysql-local".to_owned(), + kind: WorkspaceNodeKind::DataSource, + name: "Local MySQL".to_owned(), + parent_namespace_id: Some(1), + position: 0, + }, + ]) + .expect("tree builds"); + assert_eq!(tree.items.len(), 1); + assert_eq!(tree.items[0].children.len(), 1); + let json = serde_json::to_string(&tree).expect("tree serializes"); + assert!(json.contains("mysql-local")); + for forbidden in ["password", "jdbcUrl", "secretRef"] { + assert!(!json.contains(forbidden)); + } + } +} diff --git a/crates/chat2db-core/tests/java_community_mysql_product.rs b/crates/chat2db-core/tests/java_community_mysql_product.rs index 1fca3be..4c257d2 100644 --- a/crates/chat2db-core/tests/java_community_mysql_product.rs +++ b/crates/chat2db-core/tests/java_community_mysql_product.rs @@ -24,7 +24,7 @@ use futures_util::FutureExt as _; use tempfile::TempDir; use uuid::Uuid; -const COMMUNITY_COMMIT: &str = "37a34be858f2566b6b7fcf6c3f64183c1f560853"; +const COMMUNITY_COMMIT: &str = "3cb8af54cad5bd5caa20bb25f10d9b0e4f01931c"; const MYSQL_DATABASE_TYPE: &str = "MYSQL"; const MYSQL_DRIVER_CLASS: &str = "com.mysql.cj.jdbc.Driver"; const MYSQL_DRIVER_VERSION: &str = "8.0.30"; @@ -213,11 +213,25 @@ impl MysqlProductHarness { let drivers = application.list_drivers(); assert_eq!( - drivers.items.len(), + drivers + .items + .iter() + .filter(|driver| driver.pack_id == "native:mysql_async") + .count(), 1, - "test pack root must contain only MySQL" + "inventory must contain exactly one native MySQL driver" + ); + let mut managed_mysql = drivers + .items + .iter() + .filter(|driver| driver.pack_id == "mysql"); + let installed = managed_mysql + .next() + .expect("managed Connector/J pack must be discovered beside native MySQL"); + assert!( + managed_mysql.next().is_none(), + "inventory must contain exactly one managed Connector/J pack" ); - let installed = &drivers.items[0]; assert_eq!(installed.pack_id, "mysql"); assert_eq!(installed.version, MYSQL_DRIVER_VERSION); assert_eq!(installed.driver_class, MYSQL_DRIVER_CLASS); @@ -275,6 +289,7 @@ async fn provision_mysql_database( jdbc_url: server_url.clone(), properties: config.product_properties(), read_only: false, + ssh: None, }), }) .await @@ -395,6 +410,7 @@ async fn verify_database_vertical( jdbc_url: database_url.clone(), properties: config.product_properties(), read_only: false, + ssh: None, }, }, }, diff --git a/crates/chat2db-core/tests/java_community_product.rs b/crates/chat2db-core/tests/java_community_product.rs index 2cd929d..c7d7da3 100644 --- a/crates/chat2db-core/tests/java_community_product.rs +++ b/crates/chat2db-core/tests/java_community_product.rs @@ -29,7 +29,7 @@ use chat2db_java_bridge::{ use chat2db_storage::{EncryptedFileVault, Storage}; use tempfile::TempDir; -const COMMUNITY_COMMIT: &str = "37a34be858f2566b6b7fcf6c3f64183c1f560853"; +const COMMUNITY_COMMIT: &str = "3cb8af54cad5bd5caa20bb25f10d9b0e4f01931c"; const H2_DRIVER_CLASS: &str = "org.h2.Driver"; const EVENT_TIMEOUT: Duration = Duration::from_secs(30); @@ -67,6 +67,7 @@ async fn product_services_invoke_the_fixed_community_h2_compatibility_slice() { jdbc_url: jdbc_url.to_owned(), properties: Vec::new(), read_only: false, + ssh: None, }), }) .await @@ -184,7 +185,7 @@ async fn verify_namespace_builder( }) .await .expect("Core must invoke the H2 namespace DROP SCHEMA builder"); - assert_eq!(drop.sql, "DROP SCHEMA PRODUCT_NAMESPACE_ONLY"); + assert_eq!(drop.sql, "DROP SCHEMA \"PRODUCT_NAMESPACE_ONLY\""); assert_eq!( query_values( &session, diff --git a/crates/chat2db-core/tests/java_h2_product.rs b/crates/chat2db-core/tests/java_h2_product.rs index a35826a..d025c85 100644 --- a/crates/chat2db-core/tests/java_h2_product.rs +++ b/crates/chat2db-core/tests/java_h2_product.rs @@ -2,16 +2,17 @@ use std::{ fmt::Write as _, fs, path::{Path, PathBuf}, + process::Command, sync::Arc, time::Duration, }; use base64::{Engine as _, engine::general_purpose::STANDARD}; use chat2db_contract::{ - CancelDisposition, ComponentState, CreateDatasourceRequest, DatasourceConnection, JdbcValue, - ListCommunityColumnsRequest, ListCommunityDatabasesRequest, ListCommunityIndexesRequest, - ListCommunityTablesRequest, OperationEvent, OperationStatus, QueryLimits, ResultPageRequest, - StartQueryRequest, + CancelDisposition, ComponentState, CreateDatasourceRequest, DatasourceConnection, JdbcDriver, + JdbcValue, ListCommunityColumnsRequest, ListCommunityDatabasesRequest, + ListCommunityIndexesRequest, ListCommunityTablesRequest, OperationEvent, OperationStatus, + QueryLimits, ResultPageRequest, StartQueryRequest, }; use chat2db_core::{Application, RuntimeConfig, RuntimeHost}; use chat2db_java_bridge::{ @@ -24,6 +25,24 @@ use tempfile::TempDir; const H2_DRIVER_CLASS: &str = "org.h2.Driver"; const EVENT_TIMEOUT: Duration = Duration::from_secs(10); +fn assert_native_mysql_driver(drivers: &[JdbcDriver]) { + assert!( + drivers + .iter() + .any(|driver| driver.pack_id == "native:mysql_async"), + "the native MySQL driver must be discoverable without starting Java" + ); +} + +fn managed_driver<'a>(drivers: &'a [JdbcDriver], pack_id: &str) -> &'a JdbcDriver { + drivers + .iter() + .find(|driver| driver.pack_id == pack_id) + .unwrap_or_else(|| { + panic!("managed {pack_id} driver must be discovered beside native MySQL") + }) +} + struct H2ProductHarness { _directory: TempDir, host: RuntimeHost, @@ -178,7 +197,9 @@ async fn runtime_host_open_keeps_java_dormant() { .await .expect("opening storage must not spawn the missing Java executable"); assert_engine_available_on_demand(&host.application()); - assert!(host.application().list_drivers().items.is_empty()); + let inventory = host.application().list_drivers(); + assert_eq!(inventory.items.len(), 1); + assert_native_mysql_driver(&inventory.items); host.shutdown() .await .expect("a dormant runtime must shut down cleanly"); @@ -212,8 +233,9 @@ async fn managed_h2_starts_on_demand_and_reloads_after_idle_shutdown() { let application = host.application(); assert_engine_available_on_demand(&application); let inventory = application.list_drivers(); - assert_eq!(inventory.items.len(), 1); - let installed = &inventory.items[0]; + assert_eq!(inventory.items.len(), 2); + assert_native_mysql_driver(&inventory.items); + let installed = managed_driver(&inventory.items, "h2"); assert_eq!(installed.pack_id, "h2"); assert_eq!(installed.version, "test"); assert_eq!(installed.driver_class, H2_DRIVER_CLASS); @@ -235,6 +257,7 @@ async fn managed_h2_starts_on_demand_and_reloads_after_idle_shutdown() { .to_owned(), properties: Vec::new(), read_only: true, + ssh: None, }), }) .await @@ -289,6 +312,97 @@ async fn managed_h2_starts_on_demand_and_reloads_after_idle_shutdown() { assert_directory_empty(&driver_runtime_directory); } +#[tokio::test] +async fn imports_legacy_community_mysql_from_a_read_only_h2_snapshot() { + let engine_jar = required_jar("CHAT2DB_JAVA_ENGINE_JAR"); + let h2_jar = required_jar("CHAT2DB_H2_DRIVER_JAR"); + let directory = TempDir::new().expect("temporary migration directory"); + let legacy_base = directory.path().join("legacy/chat2db"); + fs::create_dir_all(legacy_base.parent().expect("legacy parent")) + .expect("legacy directory creates"); + let legacy_url = format!("jdbc:h2:file:{};MODE=MYSQL", legacy_base.to_string_lossy()); + let sql = "CREATE TABLE DATA_SOURCE (\ + ID BIGINT PRIMARY KEY, ALIAS VARCHAR, TYPE VARCHAR, URL VARCHAR, \ + USER_NAME VARCHAR, \"PASSWORD\" VARCHAR, SSH VARCHAR, SSL VARCHAR, \ + DRIVER_CONFIG VARCHAR, EXTEND_INFO VARCHAR, HOST VARCHAR, PORT VARCHAR, \ + JDBC VARCHAR, SERVICE_NAME VARCHAR); \ + INSERT INTO DATA_SOURCE VALUES \ + (1, 'Legacy MySQL', 'MYSQL', 'jdbc:mysql://127.0.0.1:3306/demo', \ + 'developer', 'must-not-migrate', NULL, NULL, NULL, NULL, \ + '127.0.0.1', '3306', NULL, 'demo'), \ + (2, 'Legacy PostgreSQL', 'POSTGRESQL', 'jdbc:postgresql://127.0.0.1/demo', \ + 'developer', 'ignored', NULL, NULL, NULL, NULL, \ + '127.0.0.1', '5432', NULL, 'demo');"; + let output = Command::new("java") + .args(["-cp"]) + .arg(&h2_jar) + .arg("org.h2.tools.Shell") + .args(["-url", &legacy_url, "-user", "sa", "-sql", sql]) + .output() + .expect("H2 fixture command starts"); + assert!( + output.status.success(), + "H2 fixture failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + let legacy_file = legacy_base.with_extension("mv.db"); + assert!(legacy_file.is_file(), "legacy H2 fixture must exist"); + + let driver_pack_root = directory.path().join("driver-packs"); + write_driver_pack( + &driver_pack_root, + "02-h2-migration", + "h2-legacy-migration", + H2_DRIVER_CLASS, + &h2_jar, + ); + let config = managed_runtime_config( + &engine_jar, + &directory.path().join("data"), + &driver_pack_root, + &STANDARD.encode([0x5a; 32]), + ); + let mut host = RuntimeHost::open(config) + .await + .expect("managed migration runtime opens"); + let application = host.application(); + let imported = application + .import_legacy_community_datasources_from_file(&legacy_file) + .await + .expect("legacy MySQL datasource imports"); + assert!(imported.database_found); + assert_eq!(imported.imported, 1); + assert_eq!(imported.skipped_unsupported, 1); + assert_eq!(imported.password_fields_omitted, 1); + + let datasources = application + .list_datasources() + .await + .expect("imported datasource lists"); + assert_eq!(datasources.items.len(), 1); + assert_eq!(datasources.items[0].name, "Legacy MySQL"); + let storage = application.storage().expect("storage configured"); + let (_, secret) = storage + .get_datasource_with_secret(&datasources.items[0].id) + .expect("imported connection reads"); + let connection: DatasourceConnection = + serde_json::from_slice(secret.expect("imported connection exists").expose_secret()) + .expect("imported connection decodes"); + assert!( + connection + .properties + .iter() + .any(|property| property.key == "user" && property.value == "developer") + ); + assert!( + connection + .properties + .iter() + .all(|property| !property.key.eq_ignore_ascii_case("password")) + ); + host.shutdown().await.expect("migration runtime shuts down"); +} + #[tokio::test] async fn partial_managed_driver_preload_cleans_generation_and_releases_storage() { let engine_jar = required_jar("CHAT2DB_JAVA_ENGINE_JAR"); @@ -315,7 +429,7 @@ async fn partial_managed_driver_preload_cleans_generation_and_releases_storage() )) .await .expect("driver discovery must not start Java"); - assert_eq!(host.application().list_drivers().items.len(), 2); + assert_eq!(host.application().list_drivers().items.len(), 3); let error = host .acquire_engine() .await @@ -337,7 +451,7 @@ async fn partial_managed_driver_preload_cleans_generation_and_releases_storage() )) .await .expect("storage and driver discovery must reopen immediately"); - assert_eq!(host.application().list_drivers().items.len(), 1); + assert_eq!(host.application().list_drivers().items.len(), 2); let lease = host .acquire_engine() .await @@ -363,6 +477,7 @@ async fn jdbc_stream_is_retained_and_read_through_product_services() { .to_owned(), properties: Vec::new(), read_only: true, + ssh: None, }), }) .await @@ -412,6 +527,7 @@ async fn active_jdbc_query_is_explicitly_cancelled_through_product_services() { jdbc_url: "jdbc:h2:mem:stage5_cancel;DB_CLOSE_DELAY=-1".to_owned(), properties: Vec::new(), read_only: true, + ssh: None, }), }) .await @@ -465,6 +581,7 @@ async fn local_result_failure_settles_query_before_releasing_session_and_driver( jdbc_url: "jdbc:h2:mem:stage5_cleanup;DB_CLOSE_DELAY=-1".to_owned(), properties: Vec::new(), read_only: true, + ssh: None, }), }) .await diff --git a/crates/chat2db-core/tests/native_mysql_account_docker.rs b/crates/chat2db-core/tests/native_mysql_account_docker.rs new file mode 100644 index 0000000..62666bf --- /dev/null +++ b/crates/chat2db-core/tests/native_mysql_account_docker.rs @@ -0,0 +1,570 @@ +use std::panic::AssertUnwindSafe; + +use base64::{Engine as _, engine::general_purpose::STANDARD}; +use chat2db_contract::{ + CommunityAccountAction, CommunityAccountCommandRequest, CommunityAccountGrantsRequest, + CommunityAccountPrivilegeScope, ComponentState, CreateDatasourceRequest, DatasourceConnection, + DatasourceConnectionProperty, +}; +use chat2db_core::{Application, RuntimeConfig, RuntimeHost}; +use chat2db_java_bridge::{EngineCommand, EngineConfig}; +use futures_util::FutureExt as _; +use mysql_async::{Conn, Error as MysqlError, Opts, OptsBuilder, prelude::Queryable}; +use tempfile::TempDir; +use uuid::Uuid; + +const REQUIRED_MYSQL_ENV: [&str; 4] = [ + "MYSQL_TEST_HOST", + "MYSQL_TEST_PORT", + "MYSQL_TEST_USER", + "MYSQL_TEST_PASSWORD", +]; + +struct MysqlTestConfig { + host: String, + port: u16, + user: String, + password: String, +} + +impl MysqlTestConfig { + fn from_environment() -> Option { + let required = mysql_test_required(); + let configured = REQUIRED_MYSQL_ENV + .iter() + .filter(|name| std::env::var_os(name).is_some()) + .count(); + if configured == 0 { + assert!( + !required, + "MYSQL_TEST_REQUIRED is enabled but the MySQL endpoint is absent" + ); + eprintln!("skipping native MySQL account test; MYSQL_TEST_* variables are absent"); + return None; + } + assert_eq!( + configured, + REQUIRED_MYSQL_ENV.len(), + "native MySQL integration is partially configured" + ); + let host = required_env("MYSQL_TEST_HOST"); + assert!( + !host.trim().is_empty() + && !host.chars().any(char::is_control) + && !host.contains(['/', '?', '#']), + "MYSQL_TEST_HOST is invalid" + ); + let port = required_env("MYSQL_TEST_PORT") + .parse::() + .expect("MYSQL_TEST_PORT must be a TCP port"); + assert_ne!(port, 0, "MYSQL_TEST_PORT cannot be zero"); + let user = required_env("MYSQL_TEST_USER"); + assert!(!user.is_empty(), "MYSQL_TEST_USER cannot be empty"); + Some(Self { + host, + port, + user, + password: required_env("MYSQL_TEST_PASSWORD"), + }) + } + + fn native_options(&self) -> Opts { + OptsBuilder::default() + .ip_or_hostname(self.host.clone()) + .tcp_port(self.port) + .user(Some(self.user.clone())) + .pass(Some(self.password.clone())) + .prefer_socket(Some(false)) + .into() + } + + fn connection(&self, database_name: &str) -> DatasourceConnection { + let host = if self.host.contains(':') + && !(self.host.starts_with('[') && self.host.ends_with(']')) + { + format!("[{}]", self.host) + } else { + self.host.clone() + }; + DatasourceConnection { + jdbc_url: format!( + "jdbc:mysql://{host}:{}/{database_name}?useSSL=false&serverTimezone=UTC", + self.port + ), + properties: vec![ + DatasourceConnectionProperty { + key: "user".to_owned(), + value: self.user.clone(), + sensitive: false, + }, + DatasourceConnectionProperty { + key: "password".to_owned(), + value: self.password.clone(), + sensitive: true, + }, + ], + read_only: false, + ssh: None, + } + } +} + +#[tokio::test] +async fn native_mysql_account_lifecycle_keeps_java_dormant() { + let Some(config) = MysqlTestConfig::from_environment() else { + return; + }; + let suffix = Uuid::new_v4().simple().to_string(); + let database_name = format!("chat2db_account_{}", &suffix[..12]); + let account_user = format!("c2d'\\{}", &suffix[..12]); + let account_host = "%"; + let metadata_user = format!("c2dh{}", &suffix[..12]); + let metadata_host = format!("h'\\{}", &suffix[..12]); + provision( + &config, + &database_name, + &account_user, + account_host, + &metadata_user, + &metadata_host, + ) + .await; + + let verification = AssertUnwindSafe(verify_account_lifecycle( + &config, + &database_name, + &account_user, + account_host, + &metadata_user, + &metadata_host, + )) + .catch_unwind() + .await; + let cleanup = cleanup( + &config, + &database_name, + &account_user, + account_host, + &metadata_user, + &metadata_host, + ) + .await; + if let Err(payload) = verification { + if let Err(error) = cleanup { + eprintln!("native MySQL account cleanup also failed: {error}"); + } + std::panic::resume_unwind(payload); + } + cleanup.expect("native MySQL account fixture must be removed"); +} + +#[allow(clippy::too_many_lines)] +async fn verify_account_lifecycle( + config: &MysqlTestConfig, + database_name: &str, + user: &str, + host_name: &str, + metadata_user: &str, + metadata_host: &str, +) { + let directory = TempDir::new().expect("temporary native MySQL account runtime"); + let missing_java = directory.path().join("missing-java"); + let runtime = RuntimeConfig::new(EngineConfig::new(EngineCommand::new(missing_java))) + .with_data_dir(directory.path().join("data")) + .with_vault_master_key_base64(STANDARD.encode([0x71; 32])); + let mut host = RuntimeHost::open(runtime) + .await + .expect("native MySQL account runtime must open without Java"); + let application = host.application(); + assert_java_dormant(&application); + + let datasource = application + .create_datasource(CreateDatasourceRequest { + name: "Native MySQL account admin".to_owned(), + driver_id: "mysql".to_owned(), + connection: Some(config.connection(database_name)), + }) + .await + .expect("native MySQL account datasource must persist"); + + let capability = application + .mysql_account_capability(&datasource.id) + .await + .expect("account capability must load"); + assert_eq!(capability.db_type, "MYSQL"); + assert_eq!(capability.editable_privileges.len(), 14); + assert!(capability.account_list_readable); + assert!(capability.account_lock_supported); + assert_eq!( + capability.connection_user.as_deref(), + Some(config.user.as_str()) + ); + assert_java_dormant(&application); + + let password = format!("Pa'ss\\word-{user}"); + let mut create = command( + &datasource.id, + user, + host_name, + CommunityAccountAction::CreateUser, + ); + create.password = Some(password.clone()); + execute_success(&application, &mut create).await; + assert_account_metadata(config, user, host_name).await; + assert_account_login(config, user, host_name, &password).await; + assert_java_dormant(&application); + + let duplicate = execute(&application, &mut create).await; + assert!(!duplicate.success); + assert_eq!( + duplicate.failure_code.as_deref(), + Some("mysql.account.executeFailed") + ); + assert!(duplicate.error_code.is_some()); + assert!( + !duplicate + .message + .as_deref() + .unwrap_or_default() + .contains(&password) + ); + + let accounts = application + .list_mysql_accounts(&datasource.id) + .await + .expect("account list must load"); + let created = accounts + .items + .iter() + .find(|account| account.user == user && account.host == host_name) + .expect("created account must be listed"); + assert_eq!(created.display_name, format!("{user}@{host_name}")); + assert_eq!(created.locked, Some(false)); + + let mut alter = command( + &datasource.id, + user, + host_name, + CommunityAccountAction::AlterPassword, + ); + let changed_password = format!("Changed'\\{password}"); + alter.password = Some(changed_password.clone()); + execute_success(&application, &mut alter).await; + assert_account_login_rejected(config, user, &password).await; + assert_account_login(config, user, host_name, &changed_password).await; + + let mut special_host = command( + &datasource.id, + metadata_user, + metadata_host, + CommunityAccountAction::CreateUser, + ); + special_host.password = Some(format!("Meta'\\{password}")); + execute_success(&application, &mut special_host).await; + assert_account_metadata(config, metadata_user, metadata_host).await; + let mut drop_special_host = command( + &datasource.id, + metadata_user, + metadata_host, + CommunityAccountAction::DropUser, + ); + execute_success(&application, &mut drop_special_host).await; + + let mut lock = command( + &datasource.id, + user, + host_name, + CommunityAccountAction::LockAccount, + ); + execute_success(&application, &mut lock).await; + assert_eq!( + listed_lock_state(&application, &datasource.id, user, host_name).await, + Some(true) + ); + + let mut unlock = command( + &datasource.id, + user, + host_name, + CommunityAccountAction::UnlockAccount, + ); + execute_success(&application, &mut unlock).await; + assert_eq!( + listed_lock_state(&application, &datasource.id, user, host_name).await, + Some(false) + ); + + let mut grant = command( + &datasource.id, + user, + host_name, + CommunityAccountAction::GrantPrivilege, + ); + grant.scope = Some(CommunityAccountPrivilegeScope::Table); + grant.database_name = Some(database_name.to_owned()); + grant.table_name = Some("account_items".to_owned()); + grant.privileges = vec!["SELECT".to_owned(), "UPDATE".to_owned()]; + grant.grant_option = true; + execute_success(&application, &mut grant).await; + + let grants = application + .mysql_account_grants(&CommunityAccountGrantsRequest { + datasource_id: datasource.id.clone(), + user: user.to_owned(), + host: host_name.to_owned(), + }) + .await + .expect("SHOW GRANTS must succeed"); + assert!(grants.items.iter().any(|grant| { + grant.contains("SELECT") && grant.contains("UPDATE") && grant.contains(database_name) + })); + + let mut revoke = grant.clone(); + revoke.action_type = CommunityAccountAction::RevokePrivilege; + revoke.grant_option = false; + execute_success(&application, &mut revoke).await; + + let mut drop_account = command( + &datasource.id, + user, + host_name, + CommunityAccountAction::DropUser, + ); + execute_success(&application, &mut drop_account).await; + assert!( + application + .list_mysql_accounts(&datasource.id) + .await + .expect("account list after drop") + .items + .iter() + .all(|account| account.user != user || account.host != host_name) + ); + assert_java_dormant(&application); + + host.shutdown() + .await + .expect("native-only account runtime must shut down cleanly"); +} + +async fn execute_success(application: &Application, request: &mut CommunityAccountCommandRequest) { + let result = execute(application, request).await; + assert!(result.success, "account operation failed: {result:?}"); +} + +async fn execute( + application: &Application, + request: &mut CommunityAccountCommandRequest, +) -> chat2db_contract::CommunityAccountExecution { + let preview = application + .preview_mysql_account(request) + .expect("account preview must succeed"); + if let Some(password) = request.password.as_deref() { + assert!(!preview.sql.contains(password)); + } + request.preview_token = Some(preview.preview_token); + application + .execute_mysql_account(request) + .await + .expect("authorized account execution must return a structured result") +} + +async fn listed_lock_state( + application: &Application, + datasource_id: &str, + user: &str, + host: &str, +) -> Option { + application + .list_mysql_accounts(datasource_id) + .await + .expect("account list must load") + .items + .into_iter() + .find(|account| account.user == user && account.host == host) + .and_then(|account| account.locked) +} + +fn command( + datasource_id: &str, + user: &str, + host: &str, + action_type: CommunityAccountAction, +) -> CommunityAccountCommandRequest { + CommunityAccountCommandRequest { + datasource_id: datasource_id.to_owned(), + user: user.to_owned(), + host: host.to_owned(), + action_type, + scope: None, + database_name: None, + table_name: None, + privileges: Vec::new(), + grant_option: false, + password: None, + preview_token: None, + } +} + +async fn provision( + config: &MysqlTestConfig, + database_name: &str, + user: &str, + host: &str, + metadata_user: &str, + metadata_host: &str, +) { + cleanup( + config, + database_name, + user, + host, + metadata_user, + metadata_host, + ) + .await + .expect("stale native MySQL account fixture must be removable"); + let mut conn = Conn::new(config.native_options()) + .await + .expect("native MySQL account fixture must connect"); + conn.query_drop(format!("CREATE DATABASE `{database_name}`")) + .await + .expect("account fixture database must be created"); + conn.query_drop(format!( + "CREATE TABLE `{database_name}`.`account_items` (id BIGINT PRIMARY KEY, note VARCHAR(64))" + )) + .await + .expect("account fixture table must be created"); + conn.disconnect() + .await + .expect("account fixture connection must close"); +} + +async fn cleanup( + config: &MysqlTestConfig, + database_name: &str, + user: &str, + host: &str, + metadata_user: &str, + metadata_host: &str, +) -> Result<(), MysqlError> { + let mut conn = Conn::new(config.native_options()).await?; + enforce_no_backslash_escapes(&mut conn).await?; + let account = format!("{}@{}", mysql_literal(user), mysql_literal(host)); + let metadata_account = format!( + "{}@{}", + mysql_literal(metadata_user), + mysql_literal(metadata_host) + ); + let account_result = conn + .query_drop(format!("DROP USER IF EXISTS {account}, {metadata_account}")) + .await; + let database_result = conn + .query_drop(format!("DROP DATABASE IF EXISTS `{database_name}`")) + .await; + let disconnect = conn.disconnect().await; + account_result?; + database_result?; + disconnect +} + +fn mysql_literal(value: &str) -> String { + format!("'{}'", value.replace('\'', "''")) +} + +async fn enforce_no_backslash_escapes(conn: &mut Conn) -> Result<(), MysqlError> { + let current = conn + .query_first::("SELECT @@SESSION.sql_mode") + .await? + .unwrap_or_default(); + if current + .split(',') + .any(|mode| mode.trim().eq_ignore_ascii_case("NO_BACKSLASH_ESCAPES")) + { + return Ok(()); + } + let mode = if current.trim().is_empty() { + "NO_BACKSLASH_ESCAPES".to_owned() + } else { + format!("{},NO_BACKSLASH_ESCAPES", current.trim()) + }; + conn.exec_drop("SET SESSION sql_mode = ?", (mode,)).await +} + +async fn assert_account_metadata(config: &MysqlTestConfig, user: &str, host: &str) { + let mut conn = Conn::new(config.native_options()) + .await + .expect("metadata verifier must connect"); + let row = conn + .exec_first::<(String, String), _, _>( + "SELECT User, Host FROM mysql.user WHERE User = ? AND Host = ?", + (user, host), + ) + .await + .expect("account metadata query must succeed") + .expect("account metadata must preserve the exact user and host"); + assert_eq!(row, (user.to_owned(), host.to_owned())); + conn.disconnect() + .await + .expect("metadata verifier must disconnect"); +} + +async fn assert_account_login(config: &MysqlTestConfig, user: &str, host: &str, password: &str) { + let options: Opts = OptsBuilder::default() + .ip_or_hostname(config.host.clone()) + .tcp_port(config.port) + .user(Some(user.to_owned())) + .pass(Some(password.to_owned())) + .prefer_socket(Some(false)) + .into(); + let mut conn = Conn::new(options) + .await + .expect("the exact generated MySQL credentials must authenticate"); + let current = conn + .query_first::("SELECT CURRENT_USER()") + .await + .expect("authenticated account identity query must succeed") + .expect("authenticated account identity must exist"); + assert_eq!(current, format!("{user}@{host}")); + conn.disconnect() + .await + .expect("authenticated account must disconnect"); +} + +async fn assert_account_login_rejected(config: &MysqlTestConfig, user: &str, password: &str) { + let options: Opts = OptsBuilder::default() + .ip_or_hostname(config.host.clone()) + .tcp_port(config.port) + .user(Some(user.to_owned())) + .pass(Some(password.to_owned())) + .prefer_socket(Some(false)) + .into(); + assert!( + Conn::new(options).await.is_err(), + "the superseded password must no longer authenticate" + ); +} + +fn assert_java_dormant(application: &Application) { + let engine = application + .health() + .components + .into_iter() + .find(|component| component.id == "database-engine") + .expect("database engine health must be present"); + assert_eq!(engine.state, ComponentState::Ready); + assert_eq!(engine.detail, "Available on demand; Java is not running"); +} + +fn mysql_test_required() -> bool { + std::env::var("MYSQL_TEST_REQUIRED").is_ok_and(|value| { + matches!( + value.trim().to_ascii_lowercase().as_str(), + "1" | "true" | "yes" | "on" + ) + }) +} + +fn required_env(name: &str) -> String { + std::env::var(name).unwrap_or_else(|_| panic!("{name} must be configured")) +} diff --git a/crates/chat2db-core/tests/native_mysql_console_docker.rs b/crates/chat2db-core/tests/native_mysql_console_docker.rs index 8c5c0b7..c86197a 100644 --- a/crates/chat2db-core/tests/native_mysql_console_docker.rs +++ b/crates/chat2db-core/tests/native_mysql_console_docker.rs @@ -84,6 +84,7 @@ impl MysqlTestConfig { }, ], read_only: false, + ssh: None, } } } diff --git a/crates/chat2db-core/tests/native_mysql_dashboard_docker.rs b/crates/chat2db-core/tests/native_mysql_dashboard_docker.rs new file mode 100644 index 0000000..a008acc --- /dev/null +++ b/crates/chat2db-core/tests/native_mysql_dashboard_docker.rs @@ -0,0 +1,519 @@ +use std::panic::AssertUnwindSafe; + +use base64::{Engine as _, engine::general_purpose::STANDARD}; +use chat2db_contract::{ + ComponentState, CreateCommunityChartRequest, CreateDatasourceRequest, DatasourceConnection, + DatasourceConnectionProperty, UpdateCommunityChartRequest, +}; +use chat2db_core::{Application, RuntimeConfig, RuntimeHost}; +use chat2db_java_bridge::{EngineCommand, EngineConfig}; +use chat2db_storage::OperationLogListQuery; +use futures_util::FutureExt as _; +use mysql_async::{Conn, Error as MysqlError, Opts, OptsBuilder, prelude::Queryable}; +use serde_json::{Value, json}; +use tempfile::TempDir; +use uuid::Uuid; + +const REQUIRED_MYSQL_ENV: [&str; 4] = [ + "MYSQL_TEST_HOST", + "MYSQL_TEST_PORT", + "MYSQL_TEST_USER", + "MYSQL_TEST_PASSWORD", +]; +const CHART_CONSOLE_ID: &str = "native-dashboard-chart-it"; + +struct MysqlTestConfig { + host: String, + port: u16, + user: String, + password: String, +} + +impl MysqlTestConfig { + fn from_environment() -> Option { + let required = mysql_test_required(); + let configured = REQUIRED_MYSQL_ENV + .iter() + .filter(|name| std::env::var_os(name).is_some()) + .count(); + if configured == 0 { + assert!( + !required, + "MYSQL_TEST_REQUIRED is enabled but the MySQL endpoint is absent" + ); + eprintln!("skipping native MySQL Dashboard test; MYSQL_TEST_* variables are absent"); + return None; + } + assert_eq!( + configured, + REQUIRED_MYSQL_ENV.len(), + "native MySQL integration is partially configured" + ); + + let host = required_env("MYSQL_TEST_HOST"); + assert!( + !host.trim().is_empty() + && !host.chars().any(char::is_control) + && !host.contains(['/', '?', '#']), + "MYSQL_TEST_HOST is invalid" + ); + let port = required_env("MYSQL_TEST_PORT") + .parse::() + .expect("MYSQL_TEST_PORT must be a TCP port"); + assert_ne!(port, 0, "MYSQL_TEST_PORT cannot be zero"); + let user = required_env("MYSQL_TEST_USER"); + assert!(!user.is_empty(), "MYSQL_TEST_USER cannot be empty"); + Some(Self { + host, + port, + user, + password: required_env("MYSQL_TEST_PASSWORD"), + }) + } + + fn native_options(&self) -> Opts { + OptsBuilder::default() + .ip_or_hostname(self.host.clone()) + .tcp_port(self.port) + .user(Some(self.user.clone())) + .pass(Some(self.password.clone())) + .prefer_socket(Some(false)) + .into() + } + + fn connection(&self, database_name: &str) -> DatasourceConnection { + let host = if self.host.contains(':') + && !(self.host.starts_with('[') && self.host.ends_with(']')) + { + format!("[{}]", self.host) + } else { + self.host.clone() + }; + DatasourceConnection { + jdbc_url: format!( + "jdbc:mysql://{host}:{}/{database_name}?useSSL=false&serverTimezone=UTC", + self.port + ), + properties: vec![ + DatasourceConnectionProperty { + key: "user".to_owned(), + value: self.user.clone(), + sensitive: false, + }, + DatasourceConnectionProperty { + key: "password".to_owned(), + value: self.password.clone(), + sensitive: true, + }, + ], + read_only: false, + ssh: None, + } + } +} + +#[tokio::test] +async fn native_mysql_dashboard_refresh_is_bounded_read_only_and_keeps_java_dormant() { + let Some(config) = MysqlTestConfig::from_environment() else { + return; + }; + let suffix = Uuid::new_v4().simple().to_string(); + let default_database = format!("chat2db_dash_default_{}", &suffix[..12]); + let selected_database = format!("chat2db_dash_selected_{}", &suffix[..12]); + + let verification = AssertUnwindSafe(async { + provision_databases(&config, &default_database, &selected_database).await; + verify_dashboard_refresh(&config, &default_database, &selected_database).await; + }) + .catch_unwind() + .await; + let cleanup = cleanup_databases(&config, &default_database, &selected_database).await; + if let Err(payload) = verification { + if let Err(error) = cleanup { + eprintln!("native MySQL Dashboard cleanup also failed: {error}"); + } + std::panic::resume_unwind(payload); + } + cleanup.expect("native MySQL Dashboard fixtures must be removed"); +} + +#[allow(clippy::too_many_lines)] +async fn verify_dashboard_refresh( + config: &MysqlTestConfig, + default_database: &str, + selected_database: &str, +) { + let directory = TempDir::new().expect("temporary native MySQL Dashboard runtime"); + let missing_java = directory.path().join("missing-java"); + let runtime = RuntimeConfig::new(EngineConfig::new(EngineCommand::new(missing_java))) + .with_data_dir(directory.path().join("data")) + .with_vault_master_key_base64(STANDARD.encode([0x64; 32])); + let mut host = RuntimeHost::open(runtime) + .await + .expect("native MySQL Dashboard runtime must open without Java"); + let application = host.application(); + assert_java_dormant(&application); + + let datasource = application + .create_datasource(CreateDatasourceRequest { + name: "Native MySQL Dashboard".to_owned(), + driver_id: "mysql".to_owned(), + connection: Some(config.connection(default_database)), + }) + .await + .expect("native MySQL Dashboard datasource must persist"); + + let persisted_metadata = json!({ + "dataList": [["persisted-only"]], + "headerList": [{"name": "persisted"}] + }); + let chart_id = application + .create_community_chart(CreateCommunityChartRequest { + name: Some("Native MySQL selected-database chart".to_owned()), + description: Some("real native MySQL Dashboard integration".to_owned()), + schema: None, + data_source_id: None, + data_source_name: Some("Native MySQL Dashboard".to_owned()), + schema_name: Some(default_database.to_owned()), + r#type: Some("TABLE".to_owned()), + database_name: Some(default_database.to_owned()), + ddl: None, + deleted: Some("N".to_owned()), + user_id: None, + chart_schema: Some(json!({"title": "Native MySQL selected-database chart"})), + meta_data: Some(persisted_metadata.clone()), + database_info: Some(database_info( + &datasource.id, + selected_database, + "UPDATE `chart_rows` SET `label` = 'refresh-false-ran' WHERE `id` = 1", + )), + refresh_type: Some("MANUAL".to_owned()), + refresh_cycle: None, + }) + .await + .expect("Community chart must persist"); + + let without_refresh = application + .get_community_chart_detail(chart_id, false) + .await + .expect("refresh=false chart detail must load") + .expect("created chart must exist"); + assert_eq!(without_refresh.meta_data, Some(persisted_metadata.clone())); + assert_target_unchanged(config, selected_database).await; + assert_java_dormant(&application); + + let select_sql = "SELECT `id`, `label`, `optional_note`, `payload`, `enabled` FROM `chart_rows` ORDER BY `id`"; + update_chart_sql( + &application, + chart_id, + &datasource.id, + selected_database, + select_sql, + ) + .await; + let refreshed = application + .get_community_chart_detail(chart_id, true) + .await + .expect("selected-database chart refresh must succeed") + .expect("created chart must exist"); + let metadata = refreshed.meta_data.expect("refreshed metadata"); + let data_list = metadata["dataList"] + .as_array() + .expect("refreshed dataList must be an array"); + assert_eq!(data_list.len(), 200, "chart rows must be capped at 200"); + assert_eq!( + data_list[0], + json!(["1", "target-001", null, "{\"row\": 1}", "true"]) + ); + assert_eq!( + data_list[199], + json!(["200", "target-200", null, "{\"row\": 200}", "true"]) + ); + assert_eq!(metadata["headerList"][0]["name"], "id"); + assert_eq!(metadata["headerList"][0]["dataType"], "NUMERIC"); + assert_eq!(metadata["headerList"][0]["primaryKey"], true); + assert_eq!(metadata["headerList"][0]["autoIncrement"], 1); + assert_eq!(metadata["headerList"][0]["nullable"], 0); + assert_eq!(metadata["headerList"][0]["editorType"], "TEXT"); + assert_eq!(metadata["headerList"][1]["name"], "label"); + assert_eq!(metadata["headerList"][1]["dataType"], "STRING"); + assert_eq!(metadata["headerList"][1]["comment"], "Chart label"); + assert_eq!(metadata["headerList"][1]["defaultValue"], "unset"); + assert_eq!(metadata["headerList"][1]["nullable"], 0); + assert_eq!(metadata["headerList"][2]["nullable"], 1); + assert_eq!(metadata["headerList"][3]["name"], "payload"); + assert_eq!(metadata["headerList"][3]["dataType"], "STRING"); + assert_eq!(metadata["headerList"][3]["editorType"], "TEXT"); + assert_eq!(metadata["headerList"][4]["name"], "enabled"); + assert_eq!(metadata["headerList"][4]["dataType"], "BIT"); + assert_eq!(metadata["headerList"][4]["editorType"], "TEXT"); + + let persisted_after_refresh = application + .get_community_chart(chart_id) + .await + .expect("persisted chart must load") + .expect("created chart must remain present"); + assert_eq!( + persisted_after_refresh.meta_data, + Some(persisted_metadata.clone()), + "refreshed metadata must remain response-only" + ); + assert_java_dormant(&application); + + let cte_sql = "WITH `selected` AS (SELECT `id`, `label` FROM `chart_rows` WHERE `id` = 201) SELECT `id`, `label` FROM `selected`"; + update_chart_sql( + &application, + chart_id, + &datasource.id, + selected_database, + cte_sql, + ) + .await; + let cte = application + .get_community_chart_detail(chart_id, true) + .await + .expect("SELECT CTE chart refresh must succeed") + .expect("created chart must exist"); + assert_eq!( + cte.meta_data.expect("CTE metadata")["dataList"], + json!([["201", "target-201"]]) + ); + + let rejected_sql = [ + "UPDATE `chart_rows` SET `label` = 'mutated' WHERE `id` = 1".to_owned(), + "SELECT `id` FROM `chart_rows` LIMIT 1; UPDATE `chart_rows` SET `label` = 'mutated' WHERE `id` = 1".to_owned(), + "SELECT `id` FROM `chart_rows` WHERE `id` = 1 FOR UPDATE".to_owned(), + "SELECT `id` FROM `chart_rows` WHERE `id` = 1 FOR SHARE".to_owned(), + format!( + "SELECT `id` /*! INTO OUTFILE '/tmp/{selected_database}-comment' */ FROM `chart_rows` LIMIT 1" + ), + "SELECT `id` FROM `chart_rows` /*M! FOR SHARE */ LIMIT 1".to_owned(), + format!( + "SELECT `id` INTO OUTFILE '/tmp/{selected_database}' FROM `chart_rows` LIMIT 1" + ), + ]; + for sql in &rejected_sql { + update_chart_sql( + &application, + chart_id, + &datasource.id, + selected_database, + sql, + ) + .await; + let error = application + .get_community_chart_detail(chart_id, true) + .await + .expect_err("unsafe chart SQL must fail closed"); + assert_eq!(error.api_error().code, "chart_query_must_be_read_only"); + } + assert_target_unchanged(config, selected_database).await; + assert_java_dormant(&application); + + let history = application + .storage() + .expect("Dashboard runtime storage") + .list_operation_logs(&OperationLogListQuery { + data_source_id: Some(datasource.id.clone()), + database_name: Some(selected_database.to_owned()), + schema_name: Some(selected_database.to_owned()), + operation_type: Some("SQL_EXECUTE".to_owned()), + search_key: None, + page_no: 1, + page_size: 50, + }) + .expect("chart operation history must load"); + assert_eq!(history.total, 9, "refresh=false must not create history"); + assert_eq!( + history + .records + .iter() + .filter(|record| record.status == "success") + .count(), + 2 + ); + assert_eq!( + history + .records + .iter() + .filter(|record| record.status == "fail") + .count(), + 7 + ); + for record in &history.records { + let extend_info: Value = serde_json::from_str( + record + .extend_info + .as_deref() + .expect("chart history extendInfo"), + ) + .expect("chart history extendInfo must be JSON"); + assert_eq!(extend_info["source"], "CHART"); + assert_eq!(extend_info["chartId"], chart_id); + assert_eq!(extend_info["consoleId"], CHART_CONSOLE_ID); + assert_eq!( + record.data_source_id.as_deref(), + Some(datasource.id.as_str()) + ); + assert_eq!(record.database_name.as_deref(), Some(selected_database)); + assert_eq!(record.schema_name.as_deref(), Some(selected_database)); + } + assert_java_dormant(&application); + + host.shutdown() + .await + .expect("native-only Dashboard runtime must shut down cleanly"); +} + +async fn update_chart_sql( + application: &Application, + chart_id: i64, + datasource_id: &str, + database_name: &str, + sql: &str, +) { + application + .update_community_chart( + chart_id, + UpdateCommunityChartRequest { + database_info: Some(database_info(datasource_id, database_name, sql)), + ..UpdateCommunityChartRequest::default() + }, + ) + .await + .expect("chart databaseInfo update must persist"); +} + +fn database_info(datasource_id: &str, database_name: &str, sql: &str) -> Value { + json!({ + "dataSourceId": datasource_id, + "databaseName": database_name, + "schemaName": database_name, + "consoleId": CHART_CONSOLE_ID, + "sql": sql, + }) +} + +async fn assert_target_unchanged(config: &MysqlTestConfig, database_name: &str) { + let mut conn = Conn::new(config.native_options()) + .await + .expect("target-state probe must connect"); + let count = conn + .query_first::(format!( + "SELECT COUNT(*) FROM `{database_name}`.`chart_rows`" + )) + .await + .expect("target row count must load") + .expect("target row count must exist"); + let label = conn + .query_first::(format!( + "SELECT `label` FROM `{database_name}`.`chart_rows` WHERE `id` = 1" + )) + .await + .expect("target label must load") + .expect("target row 1 must exist"); + conn.disconnect() + .await + .expect("target-state probe must disconnect"); + assert_eq!(count, 201); + assert_eq!(label, "target-001"); +} + +async fn provision_databases( + config: &MysqlTestConfig, + default_database: &str, + selected_database: &str, +) { + cleanup_databases(config, default_database, selected_database) + .await + .expect("stale native MySQL Dashboard fixtures must be removable"); + let mut conn = Conn::new(config.native_options()) + .await + .expect("native MySQL Dashboard fixture must connect"); + conn.query_drop(format!( + "CREATE DATABASE `{default_database}` CHARACTER SET utf8mb4" + )) + .await + .expect("default fixture database must be created"); + conn.query_drop(format!( + "CREATE DATABASE `{selected_database}` CHARACTER SET utf8mb4" + )) + .await + .expect("selected fixture database must be created"); + conn.query_drop(format!( + "CREATE TABLE `{default_database}`.`chart_rows` (\ + `id` BIGINT NOT NULL PRIMARY KEY, `label` VARCHAR(64) NOT NULL, \ + `optional_note` VARCHAR(64) NULL) ENGINE=InnoDB" + )) + .await + .expect("default fixture table must be created"); + conn.query_drop(format!( + "INSERT INTO `{default_database}`.`chart_rows` (`id`, `label`) VALUES (999, 'decoy')" + )) + .await + .expect("default fixture decoy row must be inserted"); + conn.query_drop(format!( + "CREATE TABLE `{selected_database}`.`chart_rows` (\ + `id` BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY, \ + `label` VARCHAR(64) NOT NULL DEFAULT 'unset' COMMENT 'Chart label', \ + `optional_note` VARCHAR(64) NULL, `payload` JSON NULL, \ + `enabled` BIT(1) NOT NULL DEFAULT b'1') ENGINE=InnoDB" + )) + .await + .expect("selected fixture table must be created"); + let values = (1_u16..=201) + .map(|id| format!("({id}, 'target-{id:03}', NULL, JSON_OBJECT('row', {id}), b'1')")) + .collect::>() + .join(", "); + conn.query_drop(format!( + "INSERT INTO `{selected_database}`.`chart_rows` \ + (`id`, `label`, `optional_note`, `payload`, `enabled`) VALUES {values}" + )) + .await + .expect("selected fixture rows must be inserted"); + conn.disconnect() + .await + .expect("native MySQL Dashboard fixture must disconnect"); +} + +async fn cleanup_databases( + config: &MysqlTestConfig, + default_database: &str, + selected_database: &str, +) -> Result<(), MysqlError> { + let mut conn = Conn::new(config.native_options()).await?; + let selected = conn + .query_drop(format!("DROP DATABASE IF EXISTS `{selected_database}`")) + .await; + let default = conn + .query_drop(format!("DROP DATABASE IF EXISTS `{default_database}`")) + .await; + let disconnect = conn.disconnect().await; + selected?; + default?; + disconnect +} + +fn assert_java_dormant(application: &Application) { + let engine = application + .health() + .components + .into_iter() + .find(|component| component.id == "database-engine") + .expect("database engine health must be present"); + assert_eq!(engine.state, ComponentState::Ready); + assert_eq!(engine.detail, "Available on demand; Java is not running"); +} + +fn mysql_test_required() -> bool { + std::env::var("MYSQL_TEST_REQUIRED").is_ok_and(|value| { + matches!( + value.trim().to_ascii_lowercase().as_str(), + "1" | "true" | "yes" | "on" + ) + }) +} + +fn required_env(name: &str) -> String { + std::env::var(name).unwrap_or_else(|_| panic!("{name} must be configured")) +} diff --git a/crates/chat2db-core/tests/native_mysql_product.rs b/crates/chat2db-core/tests/native_mysql_product.rs index 96e0457..111f1f2 100644 --- a/crates/chat2db-core/tests/native_mysql_product.rs +++ b/crates/chat2db-core/tests/native_mysql_product.rs @@ -2,15 +2,17 @@ use std::{panic::AssertUnwindSafe, time::Duration}; use base64::{Engine as _, engine::general_purpose::STANDARD}; use chat2db_contract::{ - CancelDisposition, ComponentState, CreateDatasourceRequest, DatasourceConnection, - DatasourceConnectionProperty, GetCommunityFunctionRequest, GetCommunityProcedureRequest, - GetCommunityTriggerRequest, JdbcValue, ListCommunityColumnsRequest, - ListCommunityDatabasesRequest, ListCommunityFunctionsRequest, ListCommunityIndexesRequest, - ListCommunityProceduresRequest, ListCommunitySchemasRequest, ListCommunityTableKeysRequest, - ListCommunityTablesRequest, ListCommunityTriggersRequest, ListCommunityViewsRequest, - OperationEvent, OperationStatus, PreviewCommunityRoutineInvocationRequest, QueryLimits, - QueryParameter, ResultMetadata, ResultPageRequest, StartCommunityTablePreviewRequest, - StartQueryRequest, + ApiError, CancelDisposition, CommunityErPositionRequest, CommunityErQueryRequest, + CommunityPinnedTableRequest, CommunityRoutineMigrationRequest, ComponentState, + CreateDatasourceRequest, DatabaseWriteState, DatasourceConnection, + DatasourceConnectionProperty, ExecuteDatabaseWriteRequest, GetCommunityFunctionRequest, + GetCommunityProcedureRequest, GetCommunityTriggerRequest, JdbcValue, + ListCommunityColumnsRequest, ListCommunityDatabasesRequest, ListCommunityFunctionsRequest, + ListCommunityIndexesRequest, ListCommunityProceduresRequest, ListCommunitySchemasRequest, + ListCommunityTableKeysRequest, ListCommunityTablesRequest, ListCommunityTriggersRequest, + ListCommunityViewsRequest, OperationEvent, OperationStatus, + PreviewCommunityRoutineInvocationRequest, QueryLimits, QueryParameter, ResultMetadata, + ResultPageRequest, StartCommunityTablePreviewRequest, StartQueryRequest, }; use chat2db_core::{ Application, MysqlConsoleCancellation, MysqlConsoleRequest, RuntimeConfig, RuntimeHost, @@ -115,6 +117,7 @@ impl MysqlTestConfig { }, ], read_only: false, + ssh: None, } } } @@ -151,7 +154,14 @@ async fn verify_native_product(config: &MysqlTestConfig, database_name: &str) { .expect("native MySQL runtime must open without Java"); let application = host.application(); assert_java_dormant(&application); - assert!(application.list_drivers().items.is_empty()); + let drivers = application.list_drivers(); + let mysql_driver = drivers + .items + .iter() + .find(|driver| driver.driver_id == "mysql") + .expect("native MySQL driver must be present in the driver inventory"); + assert_eq!(mysql_driver.driver_class, "rust:mysql_async"); + assert_eq!(mysql_driver.artifact_count, 0); let unknown_driver = application .test_datasource_connection("notmysql", config.connection(None)) @@ -177,9 +187,13 @@ async fn verify_native_product(config: &MysqlTestConfig, database_name: &str) { verify_native_metadata(&application, &datasource.id, database_name).await; verify_native_object_metadata(&application, &datasource.id, database_name).await; + verify_native_workspace_metadata(&application, &datasource.id, database_name).await; verify_native_routine_invocation(&application, &datasource.id, database_name).await; + verify_native_routine_migration(&application, &datasource.id, database_name).await; verify_native_preview(&application, &datasource.id, database_name).await; verify_native_console(&application, &datasource.id).await; + verify_native_bind_parameters(&application, &datasource.id, config, database_name).await; + verify_native_confirmed_writes(&application, &datasource.id, config, database_name).await; verify_rejected_native_selects(&application, &datasource.id).await; verify_native_truncation(&application, &datasource.id).await; verify_native_cancellation(&application, &datasource.id, config, database_name).await; @@ -189,6 +203,298 @@ async fn verify_native_product(config: &MysqlTestConfig, database_name: &str) { .expect("native-only runtime must shut down cleanly"); } +async fn verify_confirmed_write_execution( + application: &Application, + datasource_id: &str, + table_name: &str, +) { + let create_sql = format!( + "CREATE TABLE `{table_name}` (`id` BIGINT PRIMARY KEY, `label` VARCHAR(64) NOT NULL)" + ); + let unconfirmed = application + .execute_confirmed_database_write(ExecuteDatabaseWriteRequest { + datasource_id: datasource_id.to_owned(), + sql: create_sql.clone(), + confirmed: false, + }) + .await; + assert_eq!(unconfirmed.state, DatabaseWriteState::NotStarted); + assert_eq!( + unconfirmed.error.as_ref().map(|error| error.code.as_str()), + Some("database_write_confirmation_required") + ); + + let created = application + .execute_confirmed_database_write(ExecuteDatabaseWriteRequest { + datasource_id: datasource_id.to_owned(), + sql: create_sql, + confirmed: true, + }) + .await; + assert_eq!(created.state, DatabaseWriteState::Succeeded); + assert_eq!(created.affected_rows.as_deref(), Some("0")); + + let inserted = application + .execute_confirmed_database_write(ExecuteDatabaseWriteRequest { + datasource_id: datasource_id.to_owned(), + sql: format!("INSERT INTO `{table_name}` VALUES (1, 'created by automation')"), + confirmed: true, + }) + .await; + assert_eq!(inserted.state, DatabaseWriteState::Succeeded); + assert_eq!(inserted.affected_rows.as_deref(), Some("1")); +} + +async fn verify_read_only_write_and_drop( + application: &Application, + datasource_id: &str, + config: &MysqlTestConfig, + database_name: &str, + table_name: &str, +) { + let mut read_only_connection = config.connection(Some(database_name)); + read_only_connection.read_only = true; + let read_only = application + .create_datasource(CreateDatasourceRequest { + name: "Native MySQL read-only automation".to_owned(), + driver_id: "mysql".to_owned(), + connection: Some(read_only_connection), + }) + .await + .expect("read-only native MySQL datasource must persist"); + let rejected = application + .execute_confirmed_database_write(ExecuteDatabaseWriteRequest { + datasource_id: read_only.id, + sql: format!("UPDATE `{table_name}` SET `label` = 'blocked' WHERE `id` = 1"), + confirmed: true, + }) + .await; + assert_eq!(rejected.state, DatabaseWriteState::NotStarted); + assert_eq!( + rejected.error.as_ref().map(|error| error.code.as_str()), + Some("datasource_read_only") + ); + + let dropped = application + .execute_confirmed_database_write(ExecuteDatabaseWriteRequest { + datasource_id: datasource_id.to_owned(), + sql: format!("DROP TABLE `{table_name}`"), + confirmed: true, + }) + .await; + assert_eq!(dropped.state, DatabaseWriteState::Succeeded); +} + +async fn verify_native_confirmed_writes( + application: &Application, + datasource_id: &str, + config: &MysqlTestConfig, + database_name: &str, +) { + let table_name = "automation_write_probe"; + verify_confirmed_write_execution(application, datasource_id, table_name).await; + + let mut probe = Conn::new(config.native_options()) + .await + .expect("write safety probe must connect"); + probe + .query_drop(format!("USE `{database_name}`")) + .await + .expect("write safety probe must select the fixture database"); + probe + .exec_drop( + format!("UPDATE `{table_name}` SET `label` = 'unsafe'; DELETE FROM `{table_name}`"), + (), + ) + .await + .expect_err("the prepared protocol must reject a multi-statement payload as one unit"); + let unchanged = probe + .query_first::<(u64, String), _>(format!( + "SELECT COUNT(*), MIN(`label`) FROM `{table_name}`" + )) + .await + .expect("write safety probe must inspect the table") + .expect("aggregate query always returns one row"); + assert_eq!( + unchanged, + (1, "created by automation".to_owned()), + "prepared multi-statement rejection must execute neither statement" + ); + + let known_failure = application + .execute_confirmed_database_write(ExecuteDatabaseWriteRequest { + datasource_id: datasource_id.to_owned(), + sql: format!("INSERT INTO `{table_name}` VALUES (1, 'duplicate')"), + confirmed: true, + }) + .await; + assert_eq!(known_failure.state, DatabaseWriteState::Unknown); + assert_eq!( + known_failure + .error + .as_ref() + .map(|error| error.code.as_str()), + Some("database_write_outcome_unknown") + ); + + let delimiter_script = application + .execute_confirmed_database_write(ExecuteDatabaseWriteRequest { + datasource_id: datasource_id.to_owned(), + sql: format!( + "DELIMITER $$\nUPDATE `{table_name}` SET `label` = 'unsafe'; DELETE FROM `{table_name}`$$\nDELIMITER ;" + ), + confirmed: true, + }) + .await; + assert_eq!(delimiter_script.state, DatabaseWriteState::NotStarted); + assert_eq!( + delimiter_script + .error + .as_ref() + .map(|error| error.code.as_str()), + Some("invalid_database_write") + ); + + let retained_rows = probe + .query_first::(format!("SELECT COUNT(*) FROM `{table_name}`")) + .await + .expect("write safety probe must query retained rows") + .expect("COUNT always returns one row"); + probe + .disconnect() + .await + .expect("write safety probe must disconnect"); + assert_eq!( + retained_rows, 1, + "rejected scripts must dispatch no statement" + ); + + let script = application + .execute_confirmed_database_write(ExecuteDatabaseWriteRequest { + datasource_id: datasource_id.to_owned(), + sql: format!("UPDATE `{table_name}` SET `label` = 'one'; DELETE FROM `{table_name}`"), + confirmed: true, + }) + .await; + assert_eq!(script.state, DatabaseWriteState::NotStarted); + assert_eq!( + script.error.as_ref().map(|error| error.code.as_str()), + Some("invalid_database_write") + ); + + verify_read_only_write_and_drop( + application, + datasource_id, + config, + database_name, + table_name, + ) + .await; + assert_java_dormant(application); +} + +async fn verify_native_workspace_metadata( + application: &Application, + datasource_id: &str, + database_name: &str, +) { + let pin = CommunityPinnedTableRequest { + data_source_id: datasource_id.to_owned(), + database_name: database_name.to_owned(), + schema_name: String::new(), + table_name: "items".to_owned(), + }; + application + .pin_community_mysql_table(pin.clone()) + .await + .expect("table pin must persist"); + application + .pin_community_mysql_table(pin.clone()) + .await + .expect("duplicate table pin must be idempotent"); + assert_eq!( + application + .list_community_mysql_pinned_tables(pin.clone()) + .await + .expect("table pins must list") + .items, + vec!["items"] + ); + + let er_request = CommunityErQueryRequest { + data_source_id: datasource_id.to_owned(), + database_name: database_name.to_owned(), + schema_name: String::new(), + }; + let model = application + .community_mysql_er_model(er_request.clone()) + .await + .expect("native MySQL ER metadata must load"); + assert!(model.position.is_none()); + let categories = model + .tables + .iter() + .find(|table| table.name == "categories") + .expect("category table must be present in ER metadata"); + assert!( + categories + .column_list + .iter() + .any(|column| column.name == "id" && column.primary_key) + ); + let items = model + .tables + .iter() + .find(|table| table.name == "items") + .expect("items table must be present in ER metadata"); + assert!( + items + .column_list + .iter() + .any(|column| column.name == "amount" && column.column_type == "DECIMAL") + ); + assert!(items.foreign_key_list.iter().any(|key| { + key.pk_table_name == "categories" + && key.pk_column_name == "id" + && key.fk_table_name == "items" + && key.fk_column_name == "category_id" + })); + + for position in [r#"{"version":1}"#, r#"{"version":2}"#] { + application + .save_community_mysql_er_position(CommunityErPositionRequest { + data_source_id: datasource_id.to_owned(), + database_name: database_name.to_owned(), + schema_name: String::new(), + position: position.to_owned(), + }) + .await + .expect("ER layout must upsert"); + } + assert_eq!( + application + .community_mysql_er_model(er_request) + .await + .expect("native MySQL ER metadata must reload") + .position + .as_deref(), + Some(r#"{"version":2}"#) + ); + application + .unpin_community_mysql_table(pin.clone()) + .await + .expect("table pin must delete"); + assert!( + application + .list_community_mysql_pinned_tables(pin) + .await + .expect("table pins must relist") + .items + .is_empty() + ); + assert_java_dormant(application); +} + async fn verify_native_metadata( application: &Application, datasource_id: &str, @@ -369,6 +675,90 @@ async fn verify_native_routine_invocation( assert_java_dormant(application); } +async fn verify_native_routine_migration( + application: &Application, + datasource_id: &str, + database_name: &str, +) { + let request = CommunityRoutineMigrationRequest { + datasource_id: datasource_id.to_owned(), + database_type: MYSQL_DATABASE_TYPE.to_owned(), + database_name: database_name.to_owned(), + schema_name: String::new(), + routine_type: "FUNCTION".to_owned(), + routine_name: "double_amount".to_owned(), + ddl: format!( + "CREATE FUNCTION `{database_name}`.`double_amount`(input_value DECIMAL(12,2)) \ + RETURNS DECIMAL(12,2) DETERMINISTIC RETURN input_value * 3" + ), + }; + let preview = application + .preview_community_routine_migration(&request) + .expect("native MySQL routine migration must preview"); + assert!(preview.sql.starts_with(&format!( + "DROP FUNCTION IF EXISTS `{database_name}`.`double_amount`;" + ))); + assert!(preview.sql.ends_with("RETURN input_value * 3;")); + + let migrated = application + .execute_community_routine_migration(request.clone()) + .await + .expect("native MySQL routine migration must return a product result"); + assert!(migrated.success, "migration failed: {}", migrated.message); + assert_eq!(migrated.failure_stage, None); + assert!(!migrated.restore_attempted); + let results = execute_console_preview( + application, + datasource_id, + database_name, + "SELECT double_amount(4)".to_owned(), + ) + .await; + assert!(matches!( + results + .iter() + .find(|result| !result.rows.is_empty()) + .expect("migrated function must return a row") + .rows[0] + .values + .as_slice(), + [JdbcValue::Decimal { value }] if value == "12.00" + )); + + let failed = application + .execute_community_routine_migration(CommunityRoutineMigrationRequest { + ddl: format!( + "CREATE FUNCTION `{database_name}`.`double_amount`(input_value DECIMAL(12,2)) \ + RETURNS DECIMAL(12,2) RETURN" + ), + ..request + }) + .await + .expect("failed migration must return its compensation result"); + assert!(!failed.success); + assert_eq!(failed.failure_stage.as_deref(), Some("APPLY")); + assert!(failed.restore_attempted); + assert!(failed.restore_succeeded); + let restored = execute_console_preview( + application, + datasource_id, + database_name, + "SELECT double_amount(4)".to_owned(), + ) + .await; + assert!(matches!( + restored + .iter() + .find(|result| !result.rows.is_empty()) + .expect("restored function must return a row") + .rows[0] + .values + .as_slice(), + [JdbcValue::Decimal { value }] if value == "12.00" + )); + assert_java_dormant(application); +} + fn routine_preview_request( datasource_id: &str, database_name: &str, @@ -722,13 +1112,214 @@ async fn verify_native_console(application: &Application, datasource_id: &str) { assert_java_dormant(application); } +#[allow(clippy::too_many_lines)] +async fn verify_native_bind_parameters( + application: &Application, + datasource_id: &str, + config: &MysqlTestConfig, + database_name: &str, +) { + let mut parameters = vec![ + QueryParameter { + position: 1, + value: JdbcValue::Null, + }, + QueryParameter { + position: 2, + value: JdbcValue::SignedInteger { + value: "-42".to_owned(), + }, + }, + QueryParameter { + position: 3, + value: JdbcValue::UnsignedInteger { + value: "18446744073709551615".to_owned(), + }, + }, + QueryParameter { + position: 4, + value: JdbcValue::Decimal { + value: "12345678901234.123456".to_owned(), + }, + }, + QueryParameter { + position: 5, + value: JdbcValue::Boolean { value: true }, + }, + QueryParameter { + position: 6, + value: JdbcValue::Text { + value: "你好,Chat2DB".to_owned(), + }, + }, + QueryParameter { + position: 7, + value: JdbcValue::Binary { + value: "AAH/".to_owned(), + }, + }, + QueryParameter { + position: 8, + value: JdbcValue::Date { + value: "2026-08-03".to_owned(), + }, + }, + QueryParameter { + position: 9, + value: JdbcValue::Time { + value: "12:34:56.123456".to_owned(), + }, + }, + QueryParameter { + position: 10, + value: JdbcValue::Timestamp { + value: "2026-08-03T12:34:56.654321".to_owned(), + }, + }, + QueryParameter { + position: 11, + value: JdbcValue::TimestampWithTimeZone { + value: "2026-08-03T12:34:56+08:00".to_owned(), + }, + }, + ]; + parameters.reverse(); + let query = application + .start_query(StartQueryRequest { + datasource_id: datasource_id.to_owned(), + sql: "SELECT ?, CAST(? AS SIGNED), CAST(? AS UNSIGNED), \ + CAST(? AS DECIMAL(20, 6)), IF(?, 1, 0), \ + CAST(? AS CHAR CHARACTER SET utf8mb4), HEX(?), \ + CAST(? AS DATE), CAST(? AS TIME(6)), \ + CAST(? AS DATETIME(6)), CAST(? AS DATETIME(6))" + .to_owned(), + parameters, + limits: query_limits("10"), + }) + .await + .expect("typed native MySQL bind parameters must be accepted"); + let result = wait_for_result(application, &query.operation_id).await; + let page = result_page(application, &result).await; + assert_eq!(page.rows.len(), 1); + assert_eq!( + page.rows[0].values, + vec![ + JdbcValue::Null, + JdbcValue::SignedInteger { + value: "-42".to_owned(), + }, + JdbcValue::UnsignedInteger { + value: "18446744073709551615".to_owned(), + }, + JdbcValue::Decimal { + value: "12345678901234.123456".to_owned(), + }, + JdbcValue::SignedInteger { + value: "1".to_owned(), + }, + JdbcValue::Text { + value: "你好,Chat2DB".to_owned(), + }, + JdbcValue::Text { + value: "0001FF".to_owned(), + }, + JdbcValue::Date { + value: "2026-08-03".to_owned(), + }, + JdbcValue::Time { + value: "12:34:56.123456".to_owned(), + }, + JdbcValue::Timestamp { + value: "2026-08-03T12:34:56.654321".to_owned(), + }, + JdbcValue::Timestamp { + value: "2026-08-03T04:34:56".to_owned(), + }, + ] + ); + + let mut read_only_connection = config.connection(Some(database_name)); + read_only_connection.read_only = true; + let read_only = application + .create_datasource(CreateDatasourceRequest { + name: "Read-only native MySQL binds".to_owned(), + driver_id: "mysql".to_owned(), + connection: Some(read_only_connection), + }) + .await + .expect("read-only native MySQL datasource must persist"); + let read_only_query = application + .start_query(StartQueryRequest { + datasource_id: read_only.id, + sql: "SELECT label FROM items WHERE id = ?".to_owned(), + parameters: vec![QueryParameter { + position: 1, + value: JdbcValue::SignedInteger { + value: "1".to_owned(), + }, + }], + limits: query_limits("10"), + }) + .await + .expect("read-only native MySQL bind query must be accepted"); + let read_only_result = wait_for_result(application, &read_only_query.operation_id).await; + assert_eq!( + result_page(application, &read_only_result).await.rows[0].values, + vec![JdbcValue::Text { + value: "mysql-ready".to_owned(), + }] + ); + + for (sql, parameters, expected_required, expected_supplied) in [ + ( + "SELECT ?, ?", + vec![QueryParameter { + position: 1, + value: JdbcValue::Null, + }], + "2", + "1", + ), + ( + "SELECT ?", + vec![ + QueryParameter { + position: 1, + value: JdbcValue::Null, + }, + QueryParameter { + position: 2, + value: JdbcValue::Null, + }, + ], + "1", + "2", + ), + ] { + let accepted = application + .start_query(StartQueryRequest { + datasource_id: datasource_id.to_owned(), + sql: sql.to_owned(), + parameters, + limits: query_limits("10"), + }) + .await + .expect("parameter-count validation occurs after MySQL prepares the statement"); + let error = wait_for_failure(application, &accepted.operation_id).await; + assert_eq!(error.code, "invalid_query_parameter_count"); + assert!(error.message.contains(expected_required)); + assert!(error.message.contains(expected_supplied)); + } + assert_java_dormant(application); +} + async fn verify_rejected_native_selects(application: &Application, datasource_id: &str) { - let parameterized = application + let invalid_position = application .start_query(StartQueryRequest { datasource_id: datasource_id.to_owned(), sql: "SELECT ?".to_owned(), parameters: vec![QueryParameter { - position: 1, + position: 2, value: JdbcValue::SignedInteger { value: "1".to_owned(), }, @@ -736,8 +1327,8 @@ async fn verify_rejected_native_selects(application: &Application, datasource_id limits: query_limits("10"), }) .await - .expect_err("parameterized native SELECT must fail without starting Java"); - assert_eq!(parameterized.api_error().code, "invalid_query_request"); + .expect_err("non-contiguous native parameter positions must fail before execution"); + assert_eq!(invalid_position.api_error().code, "invalid_query_parameter"); let cte = application .start_query(StartQueryRequest { @@ -879,6 +1470,34 @@ async fn wait_for_result(application: &Application, operation_id: &str) -> Resul .expect("native MySQL query must finish before timeout") } +async fn wait_for_failure(application: &Application, operation_id: &str) -> ApiError { + let mut subscription = application + .subscribe_operation(operation_id, None) + .await + .expect("failed query operation must be subscribable"); + tokio::time::timeout(EVENT_TIMEOUT, async { + while let Some(envelope) = subscription + .next_event() + .await + .expect("operation event must decode") + { + match envelope.event { + OperationEvent::Failed { error } => return error, + OperationEvent::Completed { result } => { + panic!("native MySQL query unexpectedly completed: {result:?}") + } + OperationEvent::Cancelled { reason } => { + panic!("native MySQL query was cancelled: {reason:?}") + } + OperationEvent::Started | OperationEvent::Progress { .. } => {} + } + } + panic!("native MySQL operation ended without a failure event") + }) + .await + .expect("native MySQL query must fail before timeout") +} + async fn result_page( application: &Application, result: &ResultMetadata, diff --git a/crates/chat2db-core/tests/native_mysql_schema_diff_docker.rs b/crates/chat2db-core/tests/native_mysql_schema_diff_docker.rs new file mode 100644 index 0000000..6498838 --- /dev/null +++ b/crates/chat2db-core/tests/native_mysql_schema_diff_docker.rs @@ -0,0 +1,649 @@ +use std::panic::AssertUnwindSafe; + +use base64::{Engine as _, engine::general_purpose::STANDARD}; +use chat2db_contract::{ + CommunitySchemaDiffEndpoint, CommunitySchemaDiffRequest, ComponentState, + CreateDatasourceRequest, DatasourceConnection, DatasourceConnectionProperty, +}; +use chat2db_core::{Application, RuntimeConfig, RuntimeHost}; +use chat2db_java_bridge::{EngineCommand, EngineConfig}; +use futures_util::FutureExt as _; +use mysql_async::{Conn, Error as MysqlError, Opts, OptsBuilder, prelude::Queryable}; +use tempfile::TempDir; +use uuid::Uuid; + +const REQUIRED_MYSQL_ENV: [&str; 4] = [ + "MYSQL_TEST_HOST", + "MYSQL_TEST_PORT", + "MYSQL_TEST_USER", + "MYSQL_TEST_PASSWORD", +]; + +struct MysqlTestConfig { + host: String, + port: u16, + user: String, + password: String, +} + +impl MysqlTestConfig { + fn from_environment() -> Option { + let required = mysql_test_required(); + let configured = REQUIRED_MYSQL_ENV + .iter() + .filter(|name| std::env::var_os(name).is_some()) + .count(); + if configured == 0 { + assert!( + !required, + "MYSQL_TEST_REQUIRED is enabled but the MySQL endpoint is absent" + ); + eprintln!("skipping native MySQL schema diff test; MYSQL_TEST_* variables are absent"); + return None; + } + assert_eq!( + configured, + REQUIRED_MYSQL_ENV.len(), + "native MySQL integration is partially configured" + ); + Some(Self { + host: required_env("MYSQL_TEST_HOST"), + port: required_env("MYSQL_TEST_PORT") + .parse::() + .expect("MYSQL_TEST_PORT must be a TCP port"), + user: required_env("MYSQL_TEST_USER"), + password: required_env("MYSQL_TEST_PASSWORD"), + }) + } + + fn native_options(&self) -> Opts { + OptsBuilder::default() + .ip_or_hostname(self.host.clone()) + .tcp_port(self.port) + .user(Some(self.user.clone())) + .pass(Some(self.password.clone())) + .prefer_socket(Some(false)) + .into() + } + + fn connection(&self, database_name: &str) -> DatasourceConnection { + let host = if self.host.contains(':') + && !(self.host.starts_with('[') && self.host.ends_with(']')) + { + format!("[{}]", self.host) + } else { + self.host.clone() + }; + DatasourceConnection { + jdbc_url: format!( + "jdbc:mysql://{host}:{}/{database_name}?useSSL=false&serverTimezone=UTC", + self.port + ), + properties: vec![ + DatasourceConnectionProperty { + key: "user".to_owned(), + value: self.user.clone(), + sensitive: false, + }, + DatasourceConnectionProperty { + key: "password".to_owned(), + value: self.password.clone(), + sensitive: true, + }, + ], + read_only: false, + ssh: None, + } + } +} + +#[tokio::test] +async fn native_mysql_schema_diff_is_preview_only_executable_and_keeps_java_dormant() { + let Some(config) = MysqlTestConfig::from_environment() else { + return; + }; + let suffix = Uuid::new_v4().simple().to_string(); + let source_database = format!("chat2db_diff_src_{}", &suffix[..12]); + let target_database = format!("chat2db_diff_dst_{}", &suffix[..12]); + provision(&config, &source_database, &target_database).await; + + let verification = AssertUnwindSafe(verify_schema_diff( + &config, + &source_database, + &target_database, + )) + .catch_unwind() + .await; + let cleanup = cleanup(&config, &source_database, &target_database).await; + if let Err(payload) = verification { + if let Err(error) = cleanup { + eprintln!("native MySQL schema diff cleanup also failed: {error}"); + } + std::panic::resume_unwind(payload); + } + cleanup.expect("native MySQL schema diff fixtures must be removed"); +} + +#[allow(clippy::too_many_lines)] +async fn verify_schema_diff( + config: &MysqlTestConfig, + source_database: &str, + target_database: &str, +) { + let directory = TempDir::new().expect("temporary native MySQL schema diff runtime"); + let missing_java = directory.path().join("missing-java"); + let runtime = RuntimeConfig::new(EngineConfig::new(EngineCommand::new(missing_java))) + .with_data_dir(directory.path().join("data")) + .with_vault_master_key_base64(STANDARD.encode([0x53; 32])); + let mut host = RuntimeHost::open(runtime) + .await + .expect("native MySQL schema diff runtime must open without Java"); + let application = host.application(); + assert_java_dormant(&application); + + let source = application + .create_datasource(CreateDatasourceRequest { + name: "MySQL schema diff source".to_owned(), + driver_id: "mysql".to_owned(), + connection: Some(config.connection(source_database)), + }) + .await + .expect("source datasource must persist"); + let target = application + .create_datasource(CreateDatasourceRequest { + name: "MySQL schema diff target".to_owned(), + driver_id: "mysql".to_owned(), + connection: Some(config.connection(target_database)), + }) + .await + .expect("target datasource must persist"); + let request = CommunitySchemaDiffRequest { + source: CommunitySchemaDiffEndpoint { + datasource_id: source.id, + database_name: source_database.to_owned(), + schema_name: String::new(), + }, + target: CommunitySchemaDiffEndpoint { + datasource_id: target.id, + database_name: target_database.to_owned(), + schema_name: String::new(), + }, + }; + + let preview = application + .preview_mysql_schema_diff(&request) + .await + .expect("schema diff preview must succeed"); + let sql = preview.as_str(); + assert!(sql.contains(format!("CREATE TABLE `{target_database}`.`added_only`").as_str())); + assert!(sql.contains(format!("DROP TABLE `{target_database}`.`removed_only`;").as_str())); + assert!(sql.contains( + format!("ALTER TABLE `{target_database}`.`changed` DROP INDEX `idx_old`;").as_str() + )); + assert!(sql.contains( + format!("ALTER TABLE `{target_database}`.`changed` DROP COLUMN `old_col`;").as_str() + )); + assert!(sql.contains("MODIFY COLUMN `id` bigint NOT NULL FIRST;")); + assert!( + sql.contains( + "MODIFY COLUMN `title` varchar(100) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '' AFTER `id`;" + ), + "generated schema diff:\n{sql}" + ); + assert!(sql.contains("ADD COLUMN `new_col` int DEFAULT NULL AFTER `title`;")); + assert!( + sql.contains( + format!("ALTER TABLE `{target_database}`.`changed` ADD KEY `idx_title` (`title`);") + .as_str() + ) + ); + assert!( + sql.contains( + format!( + "ALTER TABLE `{target_database}`.`relations` DROP FOREIGN KEY `fk_parent_old`;" + ) + .as_str() + ) + ); + assert!(sql.contains( + format!( + "ALTER TABLE `{target_database}`.`relations` ADD CONSTRAINT `fk_parent` FOREIGN KEY (`parent_id`) REFERENCES `{target_database}`.`parents` (`id`) ON DELETE CASCADE;" + ) + .as_str() + )); + assert!(sql.contains( + format!( + "ALTER TABLE `{target_database}`.`options_only` ENGINE=InnoDB, DEFAULT CHARACTER SET=utf8mb4, COLLATE=utf8mb4_unicode_ci, COMMENT='source option';" + ) + .as_str() + )); + assert!(!sql.contains("AUTO_INCREMENT=42")); + assert!(sql.contains(format!("DROP VIEW `{target_database}`.`removed_view`;").as_str())); + assert!(sql.contains(format!("CREATE VIEW `{target_database}`.`added_view` AS").as_str())); + assert!(sql.contains(format!("`{target_database}`.`added_only`").as_str())); + assert!(sql.contains( + format!("CREATE OR REPLACE VIEW `{target_database}`.`changed_view` AS").as_str() + )); + let parent_view = sql + .find(format!("CREATE VIEW `{target_database}`.`z_parent_view`").as_str()) + .expect("parent view must be created"); + let child_view = sql + .find(format!("CREATE VIEW `{target_database}`.`a_child_view`").as_str()) + .expect("dependent view must be created"); + assert!(parent_view < child_view); + assert!(sql.contains( + format!( + "ALTER TABLE `{target_database}`.`pk_swap` DROP PRIMARY KEY, ADD PRIMARY KEY (`code`);" + ) + .as_str() + )); + assert!(!sql.contains(format!("`{source_database}`.").as_str())); + assert!(!sql.contains("chat2db_database_change_")); + assert_java_dormant(&application); + + let relation_before = show_create(config, target_database, "relations").await; + let options_before = show_create(config, target_database, "options_only").await; + let changed_view_before = view_definition(config, target_database, "changed_view").await; + let before = target_shape(config, target_database).await; + assert_eq!( + before, + TargetShape { + added_table: false, + removed_table: true, + old_column: true, + new_column: false, + old_index: true, + new_index: false, + }, + "preview must not mutate the target database" + ); + assert_eq!( + relation_before, + show_create(config, target_database, "relations").await + ); + assert_eq!( + options_before, + show_create(config, target_database, "options_only").await + ); + assert_eq!( + changed_view_before, + view_definition(config, target_database, "changed_view").await + ); + + // Generated DDL must target the requested database even when the connection currently points + // at a different catalog. + execute_preview(config, source_database, sql).await; + let after = target_shape(config, target_database).await; + assert_eq!( + after, + TargetShape { + added_table: true, + removed_table: false, + old_column: false, + new_column: true, + old_index: false, + new_index: true, + } + ); + let no_diff = application + .preview_mysql_schema_diff(&request) + .await + .expect("applied target must compare cleanly"); + if no_diff.as_str() != "-- No differences. " { + let source_added = show_create(config, source_database, "added_only").await; + let target_added = show_create(config, target_database, "added_only").await; + let source_changed = show_create(config, source_database, "changed").await; + let target_changed = show_create(config, target_database, "changed").await; + panic!( + "schema diff did not converge:\n{}\nsource added:\n{}\ntarget added:\n{}\nsource changed:\n{}\ntarget changed:\n{}", + no_diff.as_str(), + source_added, + target_added, + source_changed, + target_changed + ); + } + assert_java_dormant(&application); + + host.shutdown() + .await + .expect("native-only schema diff runtime must shut down cleanly"); +} + +#[allow(clippy::struct_excessive_bools)] +#[derive(Debug, PartialEq, Eq)] +struct TargetShape { + added_table: bool, + removed_table: bool, + old_column: bool, + new_column: bool, + old_index: bool, + new_index: bool, +} + +async fn target_shape(config: &MysqlTestConfig, database_name: &str) -> TargetShape { + let mut conn = Conn::new(config.native_options()) + .await + .expect("target shape connection"); + let table_exists = async |conn: &mut Conn, table_name: &str| { + conn.exec_first::( + "SELECT 1 FROM information_schema.TABLES \ + WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? AND TABLE_TYPE = 'BASE TABLE' LIMIT 1", + (database_name, table_name), + ) + .await + .expect("table shape query") + .is_some() + }; + let added_table = table_exists(&mut conn, "added_only").await; + let removed_table = table_exists(&mut conn, "removed_only").await; + let old_column = metadata_exists( + &mut conn, + "SELECT 1 FROM information_schema.COLUMNS \ + WHERE TABLE_SCHEMA = ? AND TABLE_NAME = 'changed' AND COLUMN_NAME = ? LIMIT 1", + database_name, + "old_col", + ) + .await; + let new_column = metadata_exists( + &mut conn, + "SELECT 1 FROM information_schema.COLUMNS \ + WHERE TABLE_SCHEMA = ? AND TABLE_NAME = 'changed' AND COLUMN_NAME = ? LIMIT 1", + database_name, + "new_col", + ) + .await; + let old_index = metadata_exists( + &mut conn, + "SELECT 1 FROM information_schema.STATISTICS \ + WHERE TABLE_SCHEMA = ? AND TABLE_NAME = 'changed' AND INDEX_NAME = ? LIMIT 1", + database_name, + "idx_old", + ) + .await; + let new_index = metadata_exists( + &mut conn, + "SELECT 1 FROM information_schema.STATISTICS \ + WHERE TABLE_SCHEMA = ? AND TABLE_NAME = 'changed' AND INDEX_NAME = ? LIMIT 1", + database_name, + "idx_title", + ) + .await; + conn.disconnect() + .await + .expect("target shape connection must close"); + TargetShape { + added_table, + removed_table, + old_column, + new_column, + old_index, + new_index, + } +} + +async fn metadata_exists( + conn: &mut Conn, + query: &str, + database_name: &str, + object_name: &str, +) -> bool { + conn.exec_first::(query, (database_name, object_name)) + .await + .expect("metadata shape query") + .is_some() +} + +async fn show_create(config: &MysqlTestConfig, database_name: &str, table_name: &str) -> String { + let mut conn = Conn::new(config.native_options()) + .await + .expect("show create connection"); + let row = conn + .query_first::<(String, String), _>(format!( + "SHOW CREATE TABLE `{database_name}`.`{table_name}`" + )) + .await + .expect("show create query") + .expect("show create row"); + conn.disconnect() + .await + .expect("show create connection must close"); + row.1 +} + +async fn view_definition(config: &MysqlTestConfig, database_name: &str, view_name: &str) -> String { + let mut conn = Conn::new(config.native_options()) + .await + .expect("view definition connection"); + let definition = conn + .exec_first::( + "SELECT VIEW_DEFINITION FROM information_schema.VIEWS \ + WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?", + (database_name, view_name), + ) + .await + .expect("view definition query") + .expect("view definition row"); + conn.disconnect() + .await + .expect("view definition connection must close"); + definition +} + +async fn execute_preview(config: &MysqlTestConfig, database_name: &str, sql: &str) { + let mut conn = Conn::new(config.native_options()) + .await + .expect("schema diff execution connection"); + conn.query_drop(format!("USE `{database_name}`")) + .await + .expect("target database must be selected"); + for statement in sql + .split(";\n\n") + .map(str::trim) + .filter(|sql| !sql.is_empty()) + { + conn.query_drop(statement).await.unwrap_or_else(|error| { + panic!("generated schema diff statement failed: {error}: {statement}") + }); + } + conn.disconnect() + .await + .expect("schema diff execution connection must close"); +} + +#[allow(clippy::too_many_lines)] +async fn provision(config: &MysqlTestConfig, source_database: &str, target_database: &str) { + cleanup(config, source_database, target_database) + .await + .expect("stale schema diff fixtures must be removable"); + let mut conn = Conn::new(config.native_options()) + .await + .expect("schema diff fixture connection"); + // MySQL images ship different server collations, so pin the fixture metadata explicitly. + for database_name in [source_database, target_database] { + conn.query_drop(format!( + "CREATE DATABASE `{database_name}` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci" + )) + .await + .expect("schema diff fixture database must be created"); + } + conn.query_drop(format!( + "CREATE TABLE `{source_database}`.`added_only` (\ + id BIGINT NOT NULL, label VARCHAR(64) NOT NULL, \ + PRIMARY KEY (id), UNIQUE KEY uq_label (label)) ENGINE=InnoDB" + )) + .await + .expect("source-only table must be created"); + conn.query_drop(format!( + "CREATE TABLE `{source_database}`.`changed` (\ + id BIGINT NOT NULL, title VARCHAR(100) NOT NULL DEFAULT '', new_col INT DEFAULT NULL, \ + PRIMARY KEY (id), KEY idx_title (title)) ENGINE=InnoDB" + )) + .await + .expect("source changed table must be created"); + conn.query_drop(format!( + "CREATE TABLE `{target_database}`.`changed` (\ + id INT NOT NULL, old_col VARCHAR(10) DEFAULT NULL, title VARCHAR(20) DEFAULT NULL, \ + PRIMARY KEY (id), KEY idx_old (old_col)) ENGINE=InnoDB" + )) + .await + .expect("target changed table must be created"); + conn.query_drop(format!( + "CREATE TABLE `{target_database}`.`removed_only` (id BIGINT NOT NULL PRIMARY KEY) ENGINE=InnoDB" + )) + .await + .expect("target-only table must be created"); + for database_name in [source_database, target_database] { + conn.query_drop(format!( + "CREATE TABLE `{database_name}`.`same_table` (id BIGINT NOT NULL PRIMARY KEY) ENGINE=InnoDB" + )) + .await + .expect("matching table must be created"); + } + conn.query_drop(format!( + "CREATE TABLE `{source_database}`.`parents` (id BIGINT NOT NULL PRIMARY KEY) ENGINE=InnoDB" + )) + .await + .expect("source foreign-key parent table must be created"); + conn.query_drop(format!( + "CREATE TABLE `{target_database}`.`parents` (id INT NOT NULL PRIMARY KEY) ENGINE=InnoDB" + )) + .await + .expect("target foreign-key parent table must be created"); + let source_relation_sql = format!( + "CREATE TABLE `{source_database}`.`relations` (\ + id BIGINT NOT NULL PRIMARY KEY, parent_id BIGINT NOT NULL, \ + KEY idx_parent (parent_id), \ + CONSTRAINT `fk_parent` FOREIGN KEY (`parent_id`) REFERENCES `{source_database}`.`parents` (`id`) \ + ON DELETE CASCADE) \ + ENGINE=InnoDB" + ); + conn.query_drop(&source_relation_sql) + .await + .unwrap_or_else(|error| { + panic!("source foreign-key table must be created: {error}: {source_relation_sql}") + }); + let target_relation_sql = format!( + "CREATE TABLE `{target_database}`.`relations` (\ + id BIGINT NOT NULL PRIMARY KEY, parent_id INT NOT NULL, \ + KEY idx_parent (parent_id), \ + CONSTRAINT `fk_parent_old` FOREIGN KEY (`parent_id`) REFERENCES `{target_database}`.`parents` (`id`)) \ + ENGINE=InnoDB" + ); + conn.query_drop(&target_relation_sql) + .await + .unwrap_or_else(|error| { + panic!("target foreign-key table must be created: {error}: {target_relation_sql}") + }); + conn.query_drop(format!( + "CREATE TABLE `{source_database}`.`options_only` (\ + id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY) \ + ENGINE=InnoDB AUTO_INCREMENT=42 DEFAULT CHARSET=utf8mb4 \ + COLLATE=utf8mb4_unicode_ci COMMENT='source option'" + )) + .await + .expect("source table options fixture must be created"); + conn.query_drop(format!( + "CREATE TABLE `{target_database}`.`options_only` (\ + id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY) \ + ENGINE=MyISAM AUTO_INCREMENT=7 DEFAULT CHARSET=latin1 \ + COLLATE=latin1_swedish_ci COMMENT='target option'" + )) + .await + .expect("target table options fixture must be created"); + conn.query_drop(format!( + "CREATE TABLE `{source_database}`.`pk_swap` (\ + id BIGINT NOT NULL AUTO_INCREMENT, code BIGINT NOT NULL, \ + PRIMARY KEY (code), KEY idx_id (id)) ENGINE=InnoDB" + )) + .await + .expect("source primary-key replacement fixture must be created"); + conn.query_drop(format!( + "CREATE TABLE `{target_database}`.`pk_swap` (\ + id BIGINT NOT NULL AUTO_INCREMENT, code BIGINT NOT NULL, \ + PRIMARY KEY (id)) ENGINE=InnoDB" + )) + .await + .expect("target primary-key replacement fixture must be created"); + conn.query_drop(format!( + "CREATE VIEW `{source_database}`.`added_view` AS \ + SELECT id, label FROM `{source_database}`.`added_only`" + )) + .await + .expect("source-only view must be created"); + conn.query_drop(format!( + "CREATE VIEW `{source_database}`.`changed_view` AS \ + SELECT id, title FROM `{source_database}`.`changed`" + )) + .await + .expect("source changed view must be created"); + conn.query_drop(format!( + "CREATE VIEW `{target_database}`.`changed_view` AS \ + SELECT id, old_col FROM `{target_database}`.`changed`" + )) + .await + .expect("target changed view must be created"); + conn.query_drop(format!( + "CREATE VIEW `{target_database}`.`removed_view` AS \ + SELECT id FROM `{target_database}`.`removed_only`" + )) + .await + .expect("target-only view must be created"); + conn.query_drop(format!( + "CREATE VIEW `{source_database}`.`z_parent_view` AS \ + SELECT id FROM `{source_database}`.`same_table`" + )) + .await + .expect("source parent view must be created"); + conn.query_drop(format!( + "CREATE VIEW `{source_database}`.`a_child_view` AS \ + SELECT id FROM `{source_database}`.`z_parent_view`" + )) + .await + .expect("source dependent view must be created"); + conn.disconnect() + .await + .expect("schema diff fixture connection must close"); +} + +async fn cleanup( + config: &MysqlTestConfig, + source_database: &str, + target_database: &str, +) -> Result<(), MysqlError> { + let mut conn = Conn::new(config.native_options()).await?; + let source_result = conn + .query_drop(format!("DROP DATABASE IF EXISTS `{source_database}`")) + .await; + let target_result = conn + .query_drop(format!("DROP DATABASE IF EXISTS `{target_database}`")) + .await; + let disconnect = conn.disconnect().await; + source_result?; + target_result?; + disconnect +} + +fn assert_java_dormant(application: &Application) { + let engine = application + .health() + .components + .into_iter() + .find(|component| component.id == "database-engine") + .expect("database engine health must be present"); + assert_eq!(engine.state, ComponentState::Ready); + assert_eq!(engine.detail, "Available on demand; Java is not running"); +} + +fn mysql_test_required() -> bool { + std::env::var("MYSQL_TEST_REQUIRED").is_ok_and(|value| { + matches!( + value.trim().to_ascii_lowercase().as_str(), + "1" | "true" | "yes" | "on" + ) + }) +} + +fn required_env(name: &str) -> String { + std::env::var(name).unwrap_or_else(|_| panic!("{name} must be configured")) +} diff --git a/crates/chat2db-core/tests/native_mysql_ssh_tunnel_docker.rs b/crates/chat2db-core/tests/native_mysql_ssh_tunnel_docker.rs new file mode 100644 index 0000000..24ff15a --- /dev/null +++ b/crates/chat2db-core/tests/native_mysql_ssh_tunnel_docker.rs @@ -0,0 +1,291 @@ +use std::{net::Ipv4Addr, time::Duration}; + +use base64::{Engine as _, engine::general_purpose::STANDARD}; +use chat2db_contract::{ + ComponentState, CreateDatasourceRequest, DatasourceConnection, DatasourceConnectionProperty, + SshAuthentication, SshHostKeyVerification, SshTunnelConfig, +}; +use chat2db_core::{ + Application, MysqlConsoleCancellation, MysqlConsoleRequest, MysqlConsoleResult, RuntimeConfig, + RuntimeHost, +}; +use chat2db_java_bridge::{EngineCommand, EngineConfig}; +use tempfile::TempDir; +use tokio::net::{TcpListener, TcpStream}; + +const QUERY_TIMEOUT: Duration = Duration::from_secs(30); +const PORT_STATE_TIMEOUT: Duration = Duration::from_secs(10); + +struct MysqlSshTestConfig { + mysql_host: String, + mysql_port: u16, + mysql_user: String, + mysql_password: String, + ssh: SshTunnelConfig, +} + +impl MysqlSshTestConfig { + fn from_environment() -> Self { + let mysql_host = required_host("CHAT2DB_TEST_MYSQL_HOST"); + let mysql_port = required_port("CHAT2DB_TEST_MYSQL_PORT"); + let mysql_user = required_env("CHAT2DB_TEST_MYSQL_USER"); + assert!( + !mysql_user.is_empty(), + "CHAT2DB_TEST_MYSQL_USER cannot be empty" + ); + + let ssh_host = required_host("CHAT2DB_TEST_SSH_HOST"); + let ssh_port = required_port("CHAT2DB_TEST_SSH_PORT"); + let ssh_user = required_env("CHAT2DB_TEST_SSH_USER"); + assert!( + !ssh_user.is_empty(), + "CHAT2DB_TEST_SSH_USER cannot be empty" + ); + let local_port = required_port("CHAT2DB_TEST_SSH_LOCAL_PORT"); + let authentication = ssh_authentication(); + + Self { + mysql_host, + mysql_port, + mysql_user, + mysql_password: required_env("CHAT2DB_TEST_MYSQL_PASSWORD"), + ssh: SshTunnelConfig { + host_name: ssh_host, + port: ssh_port, + user_name: ssh_user, + authentication, + host_key_verification: SshHostKeyVerification::KnownHosts, + local_port: Some(local_port), + }, + } + } + + fn connection(&self) -> DatasourceConnection { + let host = url_host(&self.mysql_host); + DatasourceConnection { + jdbc_url: format!( + "jdbc:mysql://{host}:{}/?useSSL=false&serverTimezone=UTC", + self.mysql_port + ), + properties: vec![ + DatasourceConnectionProperty { + key: "user".to_owned(), + value: self.mysql_user.clone(), + sensitive: false, + }, + DatasourceConnectionProperty { + key: "password".to_owned(), + value: self.mysql_password.clone(), + sensitive: true, + }, + ], + read_only: false, + ssh: Some(self.ssh.clone()), + } + } + + fn local_port(&self) -> u16 { + self.ssh + .local_port + .expect("test requires a fixed local port") + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "requires CHAT2DB_TEST_MYSQL_* and CHAT2DB_TEST_SSH_* endpoints plus a known_hosts entry"] +async fn native_mysql_concurrent_queries_share_one_fixed_ssh_tunnel() { + let config = MysqlSshTestConfig::from_environment(); + assert_port_available(config.local_port()).await; + + let directory = TempDir::new().expect("temporary native MySQL SSH runtime"); + let missing_java = directory.path().join("missing-java"); + let runtime = RuntimeConfig::new(EngineConfig::new(EngineCommand::new(missing_java))) + .with_data_dir(directory.path().join("data")) + .with_vault_master_key_base64(STANDARD.encode([0x73; 32])); + let mut host = RuntimeHost::open(runtime) + .await + .expect("native MySQL SSH runtime opens without Java"); + let application = host.application(); + let datasource = application + .create_datasource(CreateDatasourceRequest { + name: "Native MySQL shared SSH tunnel".to_owned(), + driver_id: "mysql".to_owned(), + connection: Some(config.connection()), + }) + .await + .expect("SSH-backed native MySQL datasource persists"); + assert_java_dormant(&application); + + let first = spawn_sleep_query(application.clone(), datasource.id.clone()); + wait_for_port_bound(config.local_port()).await; + let second = spawn_sleep_query(application.clone(), datasource.id); + + let first_results = await_query(first, "first").await; + let second_results = await_query(second, "second").await; + assert_query_succeeded(&first_results); + assert_query_succeeded(&second_results); + wait_for_port_released(config.local_port()).await; + assert_java_dormant(&application); + + host.shutdown() + .await + .expect("native-only SSH runtime shuts down cleanly"); +} + +fn spawn_sleep_query( + application: Application, + datasource_id: String, +) -> tokio::task::JoinHandle, chat2db_core::AppError>> { + tokio::spawn(async move { + application + .execute_mysql_console( + MysqlConsoleRequest { + datasource_id, + database_name: String::new(), + sql: "SELECT SLEEP(2), CONNECTION_ID()".to_owned(), + page_no: 1, + page_size: 10, + result_set_id: None, + single: false, + page_size_all: false, + explain: false, + error_continue: false, + }, + MysqlConsoleCancellation::new(), + ) + .await + }) +} + +async fn await_query( + task: tokio::task::JoinHandle, chat2db_core::AppError>>, + label: &str, +) -> Vec { + tokio::time::timeout(QUERY_TIMEOUT, task) + .await + .unwrap_or_else(|_| panic!("{label} tunneled query timed out")) + .unwrap_or_else(|error| panic!("{label} tunneled query task panicked: {error}")) + .unwrap_or_else(|error| panic!("{label} tunneled query failed: {error}")) +} + +fn assert_query_succeeded(results: &[MysqlConsoleResult]) { + assert_eq!(results.len(), 1); + assert!(results[0].success); + assert_eq!(results[0].rows.len(), 1); +} + +async fn assert_port_available(port: u16) { + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, port)) + .await + .unwrap_or_else(|error| { + panic!("CHAT2DB_TEST_SSH_LOCAL_PORT {port} is unavailable: {error}") + }); + drop(listener); +} + +async fn wait_for_port_bound(port: u16) { + tokio::time::timeout(PORT_STATE_TIMEOUT, async move { + loop { + match TcpStream::connect((Ipv4Addr::LOCALHOST, port)).await { + Ok(stream) => { + drop(stream); + break; + } + Err(error) if error.kind() == std::io::ErrorKind::ConnectionRefused => { + tokio::time::sleep(Duration::from_millis(10)).await; + } + Err(error) => panic!("could not inspect SSH tunnel port {port}: {error}"), + } + } + }) + .await + .expect("SSH tunnel did not bind its configured local port"); +} + +async fn wait_for_port_released(port: u16) { + tokio::time::timeout(PORT_STATE_TIMEOUT, async move { + loop { + match TcpListener::bind((Ipv4Addr::LOCALHOST, port)).await { + Ok(listener) => { + drop(listener); + break; + } + Err(error) if error.kind() == std::io::ErrorKind::AddrInUse => { + tokio::task::yield_now().await; + } + Err(error) => panic!("could not inspect released SSH tunnel port {port}: {error}"), + } + } + }) + .await + .expect("SSH tunnel did not release its configured local port"); +} + +fn ssh_authentication() -> SshAuthentication { + let password = optional_env("CHAT2DB_TEST_SSH_PASSWORD"); + let private_key = optional_env("CHAT2DB_TEST_SSH_PRIVATE_KEY"); + match (password, private_key) { + (Some(password), None) if !password.is_empty() => SshAuthentication::Password { password }, + (None, Some(key_file)) if !key_file.trim().is_empty() => SshAuthentication::PrivateKey { + key_file, + passphrase: optional_env("CHAT2DB_TEST_SSH_PRIVATE_KEY_PASSPHRASE"), + }, + (Some(_), Some(_)) => panic!( + "configure only one of CHAT2DB_TEST_SSH_PASSWORD or CHAT2DB_TEST_SSH_PRIVATE_KEY" + ), + _ => panic!( + "configure a non-empty CHAT2DB_TEST_SSH_PASSWORD or CHAT2DB_TEST_SSH_PRIVATE_KEY" + ), + } +} + +fn url_host(host: &str) -> String { + if host.contains(':') && !(host.starts_with('[') && host.ends_with(']')) { + format!("[{host}]") + } else { + host.to_owned() + } +} + +fn required_host(name: &str) -> String { + let value = required_env(name); + assert!( + !value.trim().is_empty() + && !value.chars().any(char::is_control) + && !value.contains(['/', '?', '#']), + "{name} is invalid" + ); + value +} + +fn required_port(name: &str) -> u16 { + let port = required_env(name) + .parse::() + .unwrap_or_else(|_| panic!("{name} must be a TCP port")); + assert_ne!(port, 0, "{name} cannot be zero"); + port +} + +fn required_env(name: &str) -> String { + std::env::var(name).unwrap_or_else(|_| panic!("{name} must be configured")) +} + +fn optional_env(name: &str) -> Option { + match std::env::var(name) { + Ok(value) if value.trim().is_empty() => None, + Ok(value) => Some(value), + Err(std::env::VarError::NotPresent) => None, + Err(std::env::VarError::NotUnicode(_)) => panic!("{name} must be valid UTF-8"), + } +} + +fn assert_java_dormant(application: &Application) { + let engine = application + .health() + .components + .into_iter() + .find(|component| component.id == "database-engine") + .expect("database engine health is present"); + assert_eq!(engine.state, ComponentState::Ready); + assert_eq!(engine.detail, "Available on demand; Java is not running"); +} diff --git a/crates/chat2db-core/tests/native_mysql_transfer_docker.rs b/crates/chat2db-core/tests/native_mysql_transfer_docker.rs new file mode 100644 index 0000000..a0c08c3 --- /dev/null +++ b/crates/chat2db-core/tests/native_mysql_transfer_docker.rs @@ -0,0 +1,1077 @@ +use std::{ + fs::{self, File, OpenOptions}, + io::Read as _, + panic::AssertUnwindSafe, + path::Path, + time::Duration, +}; + +use base64::{Engine as _, engine::general_purpose::STANDARD}; +use chat2db_contract::{ + ComponentState, CreateDatasourceRequest, DatasourceConnection, DatasourceConnectionProperty, + DmlExportFormat, DmlExportRequest, DmlExportSize, GenerateMysqlClassRequest, ImportFileRequest, + OtherFileExportRequest, SqlFileExportRequest, TabularImportEncoding, TransferFileFormat, + TransferSqlScope, TransferTask, TransferTaskStatus, +}; +use chat2db_core::{Application, RuntimeConfig, RuntimeHost, TransferArtifactDownload}; +use chat2db_java_bridge::{EngineCommand, EngineConfig}; +use futures_util::FutureExt as _; +use mysql_async::{Conn, Opts, OptsBuilder, prelude::Queryable}; +use tempfile::TempDir; +use uuid::Uuid; +use xls::core::{Cell, Workbook}; +use zip::ZipArchive; + +const REQUIRED_MYSQL_ENV: [&str; 4] = [ + "MYSQL_TEST_HOST", + "MYSQL_TEST_PORT", + "MYSQL_TEST_USER", + "MYSQL_TEST_PASSWORD", +]; +const TASK_TIMEOUT: Duration = Duration::from_secs(30); +type TabularRoundTripRow = ( + u64, + Option, + String, + String, + String, + String, + Vec, + Vec, + Vec, +); + +struct MysqlTestConfig { + host: String, + port: u16, + user: String, + password: String, +} + +impl MysqlTestConfig { + fn from_environment() -> Option { + let required = mysql_test_required(); + let configured = REQUIRED_MYSQL_ENV + .iter() + .filter(|name| std::env::var_os(name).is_some()) + .count(); + if configured == 0 { + assert!( + !required, + "MYSQL_TEST_REQUIRED is enabled but the MySQL endpoint is absent" + ); + eprintln!("skipping native MySQL transfer test; MYSQL_TEST_* variables are absent"); + return None; + } + assert_eq!( + configured, + REQUIRED_MYSQL_ENV.len(), + "native MySQL integration is partially configured" + ); + let host = required_env("MYSQL_TEST_HOST"); + assert!( + !host.trim().is_empty() + && !host.chars().any(char::is_control) + && !host.contains(['/', '?', '#']), + "MYSQL_TEST_HOST is invalid" + ); + let port = required_env("MYSQL_TEST_PORT") + .parse::() + .expect("MYSQL_TEST_PORT must be a TCP port"); + assert_ne!(port, 0, "MYSQL_TEST_PORT cannot be zero"); + let user = required_env("MYSQL_TEST_USER"); + assert!(!user.is_empty(), "MYSQL_TEST_USER cannot be empty"); + Some(Self { + host, + port, + user, + password: required_env("MYSQL_TEST_PASSWORD"), + }) + } + + fn native_options(&self, database_name: Option<&str>) -> Opts { + let mut builder = OptsBuilder::default() + .ip_or_hostname(self.host.clone()) + .tcp_port(self.port) + .user(Some(self.user.clone())) + .pass(Some(self.password.clone())) + .prefer_socket(Some(false)); + if let Some(database_name) = database_name { + builder = builder.db_name(Some(database_name.to_owned())); + } + builder.into() + } + + fn connection(&self, database_name: &str) -> DatasourceConnection { + let host = if self.host.contains(':') + && !(self.host.starts_with('[') && self.host.ends_with(']')) + { + format!("[{}]", self.host) + } else { + self.host.clone() + }; + DatasourceConnection { + jdbc_url: format!( + "jdbc:mysql://{host}:{}/{database_name}?useSSL=false&serverTimezone=UTC", + self.port + ), + properties: vec![ + DatasourceConnectionProperty { + key: "user".to_owned(), + value: self.user.clone(), + sensitive: false, + }, + DatasourceConnectionProperty { + key: "password".to_owned(), + value: self.password.clone(), + sensitive: true, + }, + ], + read_only: false, + ssh: None, + } + } +} + +#[tokio::test] +async fn native_mysql_transfer_product_keeps_java_dormant() { + let Some(config) = MysqlTestConfig::from_environment() else { + return; + }; + let suffix = Uuid::new_v4().simple().to_string(); + let database_name = format!("chat2db_transfer_{}", &suffix[..12]); + provision_database(&config, &database_name).await; + + let verification = AssertUnwindSafe(verify_transfer_product(&config, &database_name)) + .catch_unwind() + .await; + let cleanup = cleanup_database(&config, &database_name).await; + if let Err(payload) = verification { + if let Err(error) = cleanup { + eprintln!("native MySQL transfer cleanup also failed: {error}"); + } + std::panic::resume_unwind(payload); + } + cleanup.expect("native MySQL transfer fixture must be removed"); +} + +#[allow(clippy::too_many_lines)] +async fn verify_transfer_product(config: &MysqlTestConfig, database_name: &str) { + let directory = TempDir::new().expect("temporary native MySQL transfer runtime"); + let data_dir = directory.path().join("data"); + let missing_java = directory.path().join("missing-java"); + let mut host = RuntimeHost::open(runtime_config(&data_dir, &missing_java)) + .await + .expect("native MySQL transfer runtime must open without Java"); + let application = host.application(); + assert_java_dormant(&application); + + let datasource = application + .create_datasource(CreateDatasourceRequest { + name: "Native MySQL transfer".to_owned(), + driver_id: "mysql".to_owned(), + connection: Some(config.connection(database_name)), + }) + .await + .expect("native MySQL transfer datasource must persist"); + + verify_imports( + &application, + config, + &datasource.id, + database_name, + directory.path(), + ) + .await; + assert_java_dormant(&application); + + let durable_export_task = verify_task_exports( + &application, + &datasource.id, + database_name, + directory.path(), + ) + .await; + verify_tabular_round_trip(&application, config, &datasource.id, database_name).await; + assert_java_dormant(&application); + + verify_dml_exports_and_replay(&application, config, &datasource.id, database_name).await; + verify_class_generation( + &application, + &datasource.id, + database_name, + directory.path(), + ) + .await; + verify_cancellation( + &application, + &datasource.id, + database_name, + directory.path(), + ) + .await; + assert_java_dormant(&application); + + let tasks = application + .list_transfer_tasks(1, 100) + .await + .expect("transfer task history must list"); + assert!(tasks.total >= 14, "all product tasks must be retained"); + assert!(tasks.total <= 20, "task retention must remain bounded"); + + drop(application); + host.shutdown() + .await + .expect("native-only transfer runtime must shut down cleanly"); + drop(host); + + let mut reopened = RuntimeHost::open(runtime_config(&data_dir, &missing_java)) + .await + .expect("native MySQL transfer runtime must reopen"); + let application = reopened.application(); + assert_java_dormant(&application); + let task = application + .transfer_task(durable_export_task) + .await + .expect("completed task must survive restart"); + assert_eq!(task.status, TransferTaskStatus::Succeeded); + let download = application + .transfer_task_artifact_download(durable_export_task) + .await + .expect("completed artifact must survive restart"); + assert!(download.path.is_file()); + drop(application); + reopened + .shutdown() + .await + .expect("reopened native runtime must shut down cleanly"); +} + +async fn verify_imports( + application: &Application, + config: &MysqlTestConfig, + datasource_id: &str, + database_name: &str, + directory: &Path, +) { + let csv_path = directory.join("import.csv"); + fs::write( + &csv_path, + "id,value_text\n1,csv-one\n2,csv-two\n3,__CHAT2DB_TRANSFER_V1__:NULL\n", + ) + .expect("CSV fixture must write"); + import_and_succeed( + application, + ImportFileRequest { + datasource_id: datasource_id.to_owned(), + database_name: database_name.to_owned(), + schema_name: String::new(), + table_name: Some("import_csv".to_owned()), + file_path: csv_path.to_string_lossy().into_owned(), + format: TransferFileFormat::Csv, + contains_header: true, + tabular_encoding: TabularImportEncoding::Plain, + }, + ) + .await; + + for (format, table_name, file_name, value) in [ + ( + TransferFileFormat::Xls, + "import_xls", + "import.xls", + "xls-one", + ), + ( + TransferFileFormat::Xlsx, + "import_xlsx", + "import.xlsx", + "xlsx-one", + ), + ] { + let path = directory.join(file_name); + write_spreadsheet(&path, format, value); + import_and_succeed( + application, + ImportFileRequest { + datasource_id: datasource_id.to_owned(), + database_name: database_name.to_owned(), + schema_name: String::new(), + table_name: Some(table_name.to_owned()), + file_path: path.to_string_lossy().into_owned(), + format, + contains_header: true, + tabular_encoding: TabularImportEncoding::Plain, + }, + ) + .await; + } + + let sql_path = directory.join("import.sql"); + fs::write( + &sql_path, + "CREATE TABLE sql_loaded (id BIGINT PRIMARY KEY, value_text VARCHAR(64));\n\ + INSERT INTO sql_loaded VALUES (1, 'selected-database');\n", + ) + .expect("SQL fixture must write"); + import_and_succeed( + application, + ImportFileRequest { + datasource_id: datasource_id.to_owned(), + database_name: database_name.to_owned(), + schema_name: String::new(), + table_name: None, + file_path: sql_path.to_string_lossy().into_owned(), + format: TransferFileFormat::Sql, + contains_header: false, + tabular_encoding: TabularImportEncoding::Plain, + }, + ) + .await; + + let mut conn = Conn::new(config.native_options(Some(database_name))) + .await + .expect("import verification connection must open"); + for (table_name, expected) in [ + ( + "import_csv", + vec!["csv-one", "csv-two", "__CHAT2DB_TRANSFER_V1__:NULL"], + ), + ("import_xls", vec!["xls-one"]), + ("import_xlsx", vec!["xlsx-one"]), + ("sql_loaded", vec!["selected-database"]), + ] { + let values: Vec = conn + .query(format!("SELECT value_text FROM `{table_name}` ORDER BY id")) + .await + .expect("imported rows must query"); + assert_eq!(values, expected); + } + conn.disconnect() + .await + .expect("import verification connection must close"); +} + +async fn verify_sql_task_exports( + application: &Application, + datasource_id: &str, + database_name: &str, + user_export_path: &Path, +) -> i64 { + let mut durable_task_id = 0_i64; + for scope in [ + TransferSqlScope::All, + TransferSqlScope::Schema, + TransferSqlScope::Table, + ] { + let task = application + .export_mysql_sql_file(SqlFileExportRequest { + datasource_id: datasource_id.to_owned(), + database_name: database_name.to_owned(), + schema_name: String::new(), + table_names: vec!["source_a".to_owned()], + scope, + export_path: (scope == TransferSqlScope::All) + .then(|| user_export_path.to_string_lossy().into_owned()), + }) + .await + .expect("SQL export task must start"); + let completed = wait_for_terminal_task(application, task.task_id).await; + assert_task_succeeded(&completed); + let download = application + .transfer_task_artifact_download(task.task_id) + .await + .expect("SQL task artifact must download by task id"); + let sql = fs::read_to_string(&download.path).expect("SQL artifact must be UTF-8"); + match scope { + TransferSqlScope::All => { + assert!(sql.contains("CREATE TABLE")); + assert!(sql.contains("INSERT INTO")); + assert!( + user_export_path + .join(&download.artifact.file_name) + .is_file() + ); + durable_task_id = task.task_id; + } + TransferSqlScope::Schema => { + assert!(sql.contains("CREATE TABLE")); + assert!(!sql.contains("INSERT INTO")); + } + TransferSqlScope::Table => { + assert!(!sql.contains("CREATE TABLE")); + assert!(sql.contains("INSERT INTO")); + } + } + } + durable_task_id +} + +async fn verify_task_exports( + application: &Application, + datasource_id: &str, + database_name: &str, + directory: &Path, +) -> i64 { + let user_export_path = directory.join("user-exports"); + let durable_task_id = + verify_sql_task_exports(application, datasource_id, database_name, &user_export_path).await; + + for format in [ + TransferFileFormat::Csv, + TransferFileFormat::Xls, + TransferFileFormat::Xlsx, + ] { + let download = export_other_and_download( + application, + OtherFileExportRequest { + datasource_id: datasource_id.to_owned(), + database_name: database_name.to_owned(), + schema_name: String::new(), + table_names: vec!["source_a".to_owned()], + format, + contains_header: true, + export_path: None, + }, + ) + .await; + assert_single_tabular_export(&download, format); + } + + let csv_zip = export_other_and_download( + application, + OtherFileExportRequest { + datasource_id: datasource_id.to_owned(), + database_name: database_name.to_owned(), + schema_name: String::new(), + table_names: vec!["source_a".to_owned(), "source_b".to_owned()], + format: TransferFileFormat::Csv, + contains_header: true, + export_path: None, + }, + ) + .await; + assert_zip_entries(&csv_zip.path, &["source_a.csv", "source_b.csv"], "id"); + + let sql_zip = export_other_and_download( + application, + OtherFileExportRequest { + datasource_id: datasource_id.to_owned(), + database_name: database_name.to_owned(), + schema_name: String::new(), + table_names: vec!["source_a".to_owned(), "source_b".to_owned()], + format: TransferFileFormat::Sql, + contains_header: true, + export_path: None, + }, + ) + .await; + assert_eq!(sql_zip.artifact.format, "ZIP"); + assert_zip_entries( + &sql_zip.path, + &["source_a.sql", "source_b.sql"], + "INSERT INTO", + ); + durable_task_id +} + +async fn verify_tabular_round_trip( + application: &Application, + config: &MysqlTestConfig, + datasource_id: &str, + database_name: &str, +) { + for (format, target_table) in [ + (TransferFileFormat::Csv, "roundtrip_csv"), + (TransferFileFormat::Xls, "roundtrip_xls"), + (TransferFileFormat::Xlsx, "roundtrip_xlsx"), + ] { + let download = export_other_and_download( + application, + OtherFileExportRequest { + datasource_id: datasource_id.to_owned(), + database_name: database_name.to_owned(), + schema_name: String::new(), + table_names: vec!["source_roundtrip".to_owned()], + format, + contains_header: true, + export_path: None, + }, + ) + .await; + assert_round_trip_export_is_readable(&download.path, format); + import_and_succeed( + application, + ImportFileRequest { + datasource_id: datasource_id.to_owned(), + database_name: database_name.to_owned(), + schema_name: String::new(), + table_name: Some(target_table.to_owned()), + file_path: download.path.to_string_lossy().into_owned(), + format, + contains_header: true, + tabular_encoding: TabularImportEncoding::Chat2dbV1, + }, + ) + .await; + } + + let mut conn = Conn::new(config.native_options(Some(database_name))) + .await + .expect("tabular round-trip verification connection must open"); + for table_name in ["roundtrip_csv", "roundtrip_xls", "roundtrip_xlsx"] { + let row: Option = conn + .exec_first( + format!( + "SELECT id, nullable_text, empty_text, utf8_text, decimal_value, \ + CAST(timestamp_value AS CHAR), bit_value, payload, blob_value \ + FROM `{table_name}`" + ), + (), + ) + .await + .expect("round-tripped row must query"); + let row = row.expect("round-tripped row must exist"); + assert_eq!(row.0, 1); + assert_eq!(row.1, None, "{table_name} must preserve NULL"); + assert_eq!(row.2, "", "{table_name} must preserve empty text"); + assert_eq!( + row.3, "utf8-\u{4e2d}\u{6587}", + "{table_name} must preserve UTF-8 text" + ); + assert_eq!( + row.4, "1234567890.123400", + "{table_name} must preserve readable DECIMAL" + ); + assert_eq!( + row.5, "2024-02-03 04:05:06.123456", + "{table_name} must preserve readable TIMESTAMP" + ); + assert_eq!( + row.6, + vec![0x01, 0x01], + "{table_name} must preserve BIT bytes" + ); + assert_eq!( + row.7, + vec![0x00, 0xff], + "{table_name} must preserve VARBINARY bytes" + ); + assert_eq!( + row.8, + vec![0x00, 0xff, b'B'], + "{table_name} must preserve BLOB bytes" + ); + } + conn.disconnect() + .await + .expect("tabular round-trip verification connection must close"); +} + +fn assert_round_trip_export_is_readable(path: &Path, format: TransferFileFormat) { + match format { + TransferFileFormat::Csv => { + let mut reader = + csv::Reader::from_path(path).expect("round-trip CSV artifact must decode"); + let record = reader + .records() + .next() + .expect("round-trip CSV row must exist") + .expect("round-trip CSV row must decode"); + assert_eq!(&record[0], "1", "ordinary numeric values stay readable"); + assert_eq!(&record[1], "__CHAT2DB_TRANSFER_V1__:NULL"); + assert_eq!(&record[2], "", "empty text stays an empty CSV field"); + assert_eq!(&record[3], "utf8-\u{4e2d}\u{6587}"); + assert_eq!(&record[4], "1234567890.123400"); + assert_eq!(&record[5], "2024-02-03 04:05:06.123456"); + for index in [6, 7, 8] { + assert!(record[index].starts_with("__CHAT2DB_TRANSFER_V1__:BASE64:")); + } + } + TransferFileFormat::Xls | TransferFileFormat::Xlsx => { + let file = File::open(path).expect("round-trip spreadsheet must open"); + let workbook = match format { + TransferFileFormat::Xls => xls::core::xls::read(file), + TransferFileFormat::Xlsx => xls::core::xlsx::read(file), + TransferFileFormat::Csv | TransferFileFormat::Sql => unreachable!(), + } + .expect("round-trip spreadsheet must decode"); + assert_eq!(workbook.display_cell(0, 1, 0), "1"); + assert_eq!(workbook.display_cell(0, 1, 2), ""); + assert_eq!(workbook.display_cell(0, 1, 3), "utf8-\u{4e2d}\u{6587}"); + assert_eq!(workbook.display_cell(0, 1, 4), "1234567890.123400"); + assert_eq!(workbook.display_cell(0, 1, 5), "2024-02-03 04:05:06.123456"); + for column in [6, 7, 8] { + assert!( + workbook + .display_cell(0, 1, column) + .starts_with("__CHAT2DB_TRANSFER_V1__:BASE64:") + ); + } + } + TransferFileFormat::Sql => panic!("SQL is not a tabular round-trip format"), + } +} + +async fn verify_current_page_dml_csv( + application: &Application, + datasource_id: &str, + database_name: &str, + original_sql: &str, +) { + let csv = application + .export_mysql_dml(DmlExportRequest { + datasource_id: datasource_id.to_owned(), + database_name: database_name.to_owned(), + schema_name: String::new(), + sql: format!("{original_sql} LIMIT 1"), + original_sql: original_sql.to_owned(), + result_set_id: Some(0), + export_size: DmlExportSize::CurrentPage, + format: DmlExportFormat::Csv, + }) + .await + .expect("current-page DML CSV must export"); + let csv = application + .transfer_artifact_download(&csv.id) + .await + .expect("DML CSV artifact must resolve"); + let mut csv_reader = csv::Reader::from_path(csv.path).expect("DML CSV must decode"); + assert_eq!( + csv_reader.records().count(), + 1, + "current-page export uses sql" + ); +} + +async fn verify_dml_exports_and_replay( + application: &Application, + config: &MysqlTestConfig, + datasource_id: &str, + database_name: &str, +) { + let original_sql = + format!("SELECT id, note, payload FROM `{database_name}`.`source_a` ORDER BY id"); + verify_current_page_dml_csv(application, datasource_id, database_name, &original_sql).await; + + let xlsx = application + .export_mysql_dml(DmlExportRequest { + datasource_id: datasource_id.to_owned(), + database_name: database_name.to_owned(), + schema_name: String::new(), + sql: String::new(), + original_sql: original_sql.clone(), + result_set_id: Some(0), + export_size: DmlExportSize::All, + format: DmlExportFormat::Xlsx, + }) + .await + .expect("all-row DML XLSX must export"); + let xlsx = application + .transfer_artifact_download(&xlsx.id) + .await + .expect("DML XLSX artifact must resolve"); + let workbook = xls::core::xlsx::read(File::open(xlsx.path).expect("XLSX must open")) + .expect("DML XLSX must decode"); + assert_eq!(workbook.sheets[0].dimensions().0, 3); + + let inserts = application + .export_mysql_dml(DmlExportRequest { + datasource_id: datasource_id.to_owned(), + database_name: database_name.to_owned(), + schema_name: String::new(), + sql: String::new(), + original_sql, + result_set_id: Some(0), + export_size: DmlExportSize::All, + format: DmlExportFormat::Insert, + }) + .await + .expect("DML INSERT must export"); + let inserts = application + .transfer_artifact_download(&inserts.id) + .await + .expect("DML INSERT artifact must resolve"); + let insert_sql = fs::read_to_string(&inserts.path).expect("INSERT export must be UTF-8"); + assert_eq!(insert_sql.matches("INSERT INTO").count(), 2); + + let mut conn = Conn::new(config.native_options(Some(database_name))) + .await + .expect("DML replay verification connection must open"); + conn.query_drop("TRUNCATE TABLE source_a") + .await + .expect("DML replay table must clear"); + conn.disconnect() + .await + .expect("DML replay preparation connection must close"); + import_and_succeed( + application, + ImportFileRequest { + datasource_id: datasource_id.to_owned(), + database_name: database_name.to_owned(), + schema_name: String::new(), + table_name: None, + file_path: inserts.path.to_string_lossy().into_owned(), + format: TransferFileFormat::Sql, + contains_header: false, + tabular_encoding: TabularImportEncoding::Plain, + }, + ) + .await; + + let mut conn = Conn::new(config.native_options(Some(database_name))) + .await + .expect("DML replay result connection must open"); + let rows: Vec<(String, Vec)> = conn + .query("SELECT note, payload FROM source_a ORDER BY id") + .await + .expect("replayed INSERT rows must query"); + assert_eq!(rows[0].0, "quote ' slash \\ newline\nnext"); + assert_eq!(rows[0].1, vec![0x00, 0x01, 0xff]); + assert_eq!(rows[1].0, "plain"); + conn.disconnect() + .await + .expect("DML replay result connection must close"); +} + +async fn verify_class_generation( + application: &Application, + datasource_id: &str, + database_name: &str, + directory: &Path, +) { + let output = directory.join("generated"); + let generated = application + .generate_mysql_classes(GenerateMysqlClassRequest { + datasource_id: datasource_id.to_owned(), + database_name: database_name.to_owned(), + schema_name: String::new(), + table_name: "source_a".to_owned(), + export_path: output.to_string_lossy().into_owned(), + }) + .await + .expect("MyBatis Plus classes must generate"); + assert_eq!(generated.files.len(), 3); + let contents = generated + .files + .iter() + .map(|path| fs::read_to_string(path).expect("generated file must read")) + .collect::>() + .join("\n"); + assert!(contents.contains("@TableName(\"source_a\")")); + assert!(contents.contains("SourceAMapper")); + assert!(contents.contains(" TransferArtifactDownload { + let accepted = application + .export_mysql_other_file(request) + .await + .expect("other-file export task must start"); + let task = wait_for_terminal_task(application, accepted.task_id).await; + assert_task_succeeded(&task); + application + .transfer_task_artifact_download(accepted.task_id) + .await + .expect("other-file artifact must download by task id") +} + +async fn wait_for_running_task(application: &Application, task_id: i64) { + tokio::time::timeout(TASK_TIMEOUT, async { + loop { + let task = application + .transfer_task(task_id) + .await + .expect("transfer task must remain readable"); + match task.status { + TransferTaskStatus::Running => return, + TransferTaskStatus::Queued => tokio::time::sleep(Duration::from_millis(20)).await, + status => panic!("task became {status:?} before cancellation"), + } + } + }) + .await + .expect("transfer task must enter running state"); +} + +async fn wait_for_terminal_task(application: &Application, task_id: i64) -> TransferTask { + tokio::time::timeout(TASK_TIMEOUT, async { + loop { + let task = application + .transfer_task(task_id) + .await + .expect("transfer task must remain readable"); + if matches!( + task.status, + TransferTaskStatus::Succeeded + | TransferTaskStatus::Failed + | TransferTaskStatus::Cancelled + | TransferTaskStatus::Interrupted + ) { + return task; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("transfer task must finish before timeout") +} + +fn assert_task_succeeded(task: &TransferTask) { + assert_eq!( + task.status, + TransferTaskStatus::Succeeded, + "transfer task failed: {}", + task.error_log + ); +} + +fn assert_single_tabular_export(download: &TransferArtifactDownload, format: TransferFileFormat) { + match format { + TransferFileFormat::Csv => { + let value = fs::read_to_string(&download.path).expect("CSV export must decode"); + assert!(value.starts_with("id,note,payload")); + } + TransferFileFormat::Xls => { + let bytes = fs::read(&download.path).expect("XLS export must read"); + assert_eq!(&bytes[..4], &[0xd0, 0xcf, 0x11, 0xe0]); + let workbook = xls::core::xls::read(File::open(&download.path).expect("XLS opens")) + .expect("XLS export must decode"); + assert_eq!(workbook.display_cell(0, 0, 0), "id"); + } + TransferFileFormat::Xlsx => { + let bytes = fs::read(&download.path).expect("XLSX export must read"); + assert_eq!(&bytes[..4], b"PK\x03\x04"); + let workbook = xls::core::xlsx::read(File::open(&download.path).expect("XLSX opens")) + .expect("XLSX export must decode"); + assert_eq!(workbook.display_cell(0, 0, 0), "id"); + } + TransferFileFormat::Sql => panic!("SQL is not a tabular export in this assertion"), + } +} + +fn assert_zip_entries(path: &Path, expected_names: &[&str], expected_text: &str) { + let bytes = fs::read(path).expect("ZIP export must read"); + assert_eq!(&bytes[..4], b"PK\x03\x04"); + let mut archive = ZipArchive::new(File::open(path).expect("ZIP export must open")) + .expect("ZIP export must decode"); + assert_eq!(archive.len(), expected_names.len()); + for expected_name in expected_names { + let mut entry = archive + .by_name(expected_name) + .unwrap_or_else(|_| panic!("ZIP entry {expected_name} must exist")); + let mut contents = String::new(); + entry + .read_to_string(&mut contents) + .expect("ZIP text entry must decode"); + assert!(contents.contains(expected_text)); + } +} + +fn write_spreadsheet(path: &Path, format: TransferFileFormat, value: &str) { + let mut workbook = Workbook::new(); + let sheet = workbook.sheet_mut(0).expect("default worksheet must exist"); + for (row, values) in [["id", "value_text"], ["1", value]].into_iter().enumerate() { + for (column, value) in values.into_iter().enumerate() { + sheet.set( + u32::try_from(row).expect("fixture row fits"), + u32::try_from(column).expect("fixture column fits"), + Cell::Text(value.to_owned()), + ); + } + } + let mut file = OpenOptions::new() + .create_new(true) + .read(true) + .write(true) + .open(path) + .expect("spreadsheet fixture must create"); + match format { + TransferFileFormat::Xls => xls::core::xls::write(&workbook, &mut file), + TransferFileFormat::Xlsx => xls::core::xlsx::write(&workbook, &mut file), + TransferFileFormat::Csv | TransferFileFormat::Sql => unreachable!(), + } + .expect("spreadsheet fixture must encode"); +} + +fn runtime_config(data_dir: &Path, missing_java: &Path) -> RuntimeConfig { + RuntimeConfig::new(EngineConfig::new(EngineCommand::new( + missing_java.to_owned(), + ))) + .with_data_dir(data_dir) + .with_vault_master_key_base64(STANDARD.encode([0x74; 32])) +} + +async fn provision_database(config: &MysqlTestConfig, database_name: &str) { + let mut conn = Conn::new(config.native_options(None)) + .await + .expect("native MySQL transfer fixture connection must open"); + conn.query_drop(format!( + "CREATE DATABASE `{database_name}` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci" + )) + .await + .expect("native MySQL transfer fixture database must create"); + for table_name in ["import_csv", "import_xls", "import_xlsx"] { + conn.query_drop(format!( + "CREATE TABLE `{database_name}`.`{table_name}` (\ + id BIGINT PRIMARY KEY, value_text VARCHAR(64) NOT NULL) ENGINE=InnoDB" + )) + .await + .expect("native MySQL import target must create"); + } + conn.query_drop(format!( + "CREATE TABLE `{database_name}`.`source_a` (\ + id BIGINT PRIMARY KEY, note VARCHAR(255) NOT NULL, payload VARBINARY(255) NOT NULL\ + ) ENGINE=InnoDB" + )) + .await + .expect("native MySQL source A must create"); + conn.exec_drop( + format!("INSERT INTO `{database_name}`.`source_a` (id, note, payload) VALUES (?, ?, ?)"), + ( + 1_u64, + "quote ' slash \\ newline\nnext", + vec![0x00, 0x01, 0xff], + ), + ) + .await + .expect("native MySQL special source row must insert"); + conn.exec_drop( + format!("INSERT INTO `{database_name}`.`source_a` (id, note, payload) VALUES (?, ?, ?)"), + (2_u64, "plain", vec![0x02, 0x03]), + ) + .await + .expect("native MySQL plain source row must insert"); + conn.query_drop(format!( + "CREATE TABLE `{database_name}`.`source_b` (\ + id BIGINT PRIMARY KEY, label VARCHAR(64) NOT NULL\ + ) ENGINE=InnoDB" + )) + .await + .expect("native MySQL source B must create"); + conn.query_drop(format!( + "INSERT INTO `{database_name}`.`source_b` VALUES (1, 'second-table')" + )) + .await + .expect("native MySQL source B row must insert"); + for table_name in [ + "source_roundtrip", + "roundtrip_csv", + "roundtrip_xls", + "roundtrip_xlsx", + ] { + conn.query_drop(format!( + "CREATE TABLE `{database_name}`.`{table_name}` (\ + id BIGINT PRIMARY KEY, nullable_text VARCHAR(64) NULL, \ + empty_text VARCHAR(64) NOT NULL, utf8_text VARCHAR(64) NOT NULL, \ + decimal_value DECIMAL(20, 6) NOT NULL, timestamp_value TIMESTAMP(6) NOT NULL, \ + bit_value BIT(9) NOT NULL, payload VARBINARY(64) NOT NULL, \ + blob_value BLOB NOT NULL\ + ) ENGINE=InnoDB" + )) + .await + .expect("native MySQL tabular round-trip table must create"); + } + conn.exec_drop( + format!( + "INSERT INTO `{database_name}`.`source_roundtrip` \ + (id, nullable_text, empty_text, utf8_text, decimal_value, timestamp_value, \ + bit_value, payload, blob_value) \ + VALUES (?, ?, ?, ?, ?, ?, b'100000001', ?, ?)" + ), + ( + 1_u64, + Option::::None, + String::new(), + "utf8-\u{4e2d}\u{6587}", + "1234567890.123400", + "2024-02-03 04:05:06.123456", + vec![0x00, 0xff], + vec![0x00, 0xff, b'B'], + ), + ) + .await + .expect("native MySQL tabular round-trip source row must insert"); + conn.disconnect() + .await + .expect("native MySQL transfer fixture connection must close"); +} + +async fn cleanup_database(config: &MysqlTestConfig, database_name: &str) -> Result<(), String> { + let mut conn = Conn::new(config.native_options(None)) + .await + .map_err(|error| error.to_string())?; + conn.query_drop(format!("DROP DATABASE IF EXISTS `{database_name}`")) + .await + .map_err(|error| error.to_string())?; + conn.disconnect().await.map_err(|error| error.to_string()) +} + +fn assert_java_dormant(application: &Application) { + let engine = application + .health() + .components + .into_iter() + .find(|component| component.id == "database-engine") + .expect("database engine health must be present"); + assert_eq!(engine.state, ComponentState::Ready); + assert_eq!(engine.detail, "Available on demand; Java is not running"); +} + +fn required_env(name: &str) -> String { + std::env::var(name).unwrap_or_else(|_| panic!("{name} must be configured")) +} + +fn mysql_test_required() -> bool { + match std::env::var("MYSQL_TEST_REQUIRED") { + Err(std::env::VarError::NotPresent) => false, + Ok(value) if value == "1" || value.eq_ignore_ascii_case("true") => true, + Ok(value) if value == "0" || value.eq_ignore_ascii_case("false") => false, + Ok(_) | Err(std::env::VarError::NotUnicode(_)) => { + panic!("MYSQL_TEST_REQUIRED must be 1, 0, true, or false") + } + } +} diff --git a/crates/chat2db-java-bridge/tests/java_community_h2.rs b/crates/chat2db-java-bridge/tests/java_community_h2.rs index b521edd..9b2843b 100644 --- a/crates/chat2db-java-bridge/tests/java_community_h2.rs +++ b/crates/chat2db-java-bridge/tests/java_community_h2.rs @@ -23,7 +23,7 @@ use chat2db_java_bridge::{ }; use tempfile::TempDir; -const COMMUNITY_COMMIT: &str = "37a34be858f2566b6b7fcf6c3f64183c1f560853"; +const COMMUNITY_COMMIT: &str = "3cb8af54cad5bd5caa20bb25f10d9b0e4f01931c"; const H2_DRIVER_CLASS: &str = "org.h2.Driver"; const COMMUNITY_CLASSPATH_LOCK: &str = include_str!("../../../third_party/community-h2-classpath.lock"); @@ -174,7 +174,7 @@ async fn verify_namespace_builder(community: &CommunityClient, session: &Session }) .await .expect("real H2 plugin must build namespace DROP SCHEMA SQL"); - assert_eq!(drop, "DROP SCHEMA NAMESPACE_ONLY"); + assert_eq!(drop, "DROP SCHEMA \"NAMESPACE_ONLY\""); assert_eq!( query_values( session, diff --git a/crates/chat2db-java-bridge/tests/supervisor.rs b/crates/chat2db-java-bridge/tests/supervisor.rs index c445cf4..bee51ee 100644 --- a/crates/chat2db-java-bridge/tests/supervisor.rs +++ b/crates/chat2db-java-bridge/tests/supervisor.rs @@ -9,7 +9,7 @@ use chat2db_java_bridge::{ Session, SessionConfig, SessionState, TransactionOptions, UpdateRequest, }; -const COMMUNITY_COMMIT: &str = "37a34be858f2566b6b7fcf6c3f64183c1f560853"; +const COMMUNITY_COMMIT: &str = "3cb8af54cad5bd5caa20bb25f10d9b0e4f01931c"; fn fixture_command(arguments: &[&str]) -> EngineCommand { arguments.iter().fold( diff --git a/crates/chat2db-local/Cargo.toml b/crates/chat2db-local/Cargo.toml index 5ceab2d..b9c68c3 100644 --- a/crates/chat2db-local/Cargo.toml +++ b/crates/chat2db-local/Cargo.toml @@ -26,6 +26,7 @@ tracing.workspace = true uuid.workspace = true [dev-dependencies] +chat2db-java-bridge = { path = "../chat2db-java-bridge" } tempfile = "3" [target.'cfg(unix)'.dependencies] diff --git a/crates/chat2db-local/src/client.rs b/crates/chat2db-local/src/client.rs index 9a042eb..a4b3877 100644 --- a/crates/chat2db-local/src/client.rs +++ b/crates/chat2db-local/src/client.rs @@ -5,8 +5,9 @@ use std::fs; use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD}; use chat2db_contract::{ - CancelOperationResponse, DatasourceList, HealthResponse, OperationSnapshot, QueryAccepted, - ResultPage, ResultPageRequest, StartQueryRequest, + ApiError, CancelOperationResponse, DatabaseWriteResult, DatabaseWriteState, DatasourceList, + ExecuteDatabaseWriteRequest, HealthResponse, OperationSnapshot, QueryAccepted, ResultPage, + ResultPageRequest, StartQueryRequest, }; use uuid::Uuid; @@ -85,6 +86,38 @@ impl LocalClient { } } + /// Executes one explicitly confirmed database write in the attached runtime. + /// + /// Transport failures after request delivery are returned as an `unknown` + /// write outcome so callers never retry a potentially committed statement. + pub async fn execute_database_write( + &self, + request: ExecuteDatabaseWriteRequest, + ) -> DatabaseWriteResult { + let probe = AttachmentRequest { + protocol_version: PROTOCOL_VERSION, + request_id: "0".repeat(64), + token: "0".repeat(43), + command: AttachmentCommand::ExecuteDatabaseWrite { request }, + }; + if transport::encode_message(&probe, MAX_REQUEST_BYTES).is_err() { + return write_failure( + DatabaseWriteState::NotStarted, + ApiError::new( + "invalid_database_write", + "The database write request exceeds the local transport limit", + ), + ); + } + match self.call(probe.command).await { + Ok(payload) => match *payload { + AttachmentPayload::DatabaseWrite(value) => *value, + _ => unknown_write_result(), + }, + Err(error) => local_write_failure(error), + } + } + /// Reads the current state of one attached database operation. /// /// # Errors @@ -254,3 +287,146 @@ impl LocalClient { fn unexpected_payload() -> LocalError { LocalError::Protocol("runtime returned an unexpected payload type".to_owned()) } + +fn local_write_failure(error: LocalError) -> DatabaseWriteResult { + match error { + LocalError::Remote(error) if remote_rejected_before_dispatch(&error.0.code) => { + write_failure(DatabaseWriteState::NotStarted, error.0) + } + LocalError::Unavailable(_) => write_failure( + DatabaseWriteState::NotStarted, + retryable_error( + "local_runtime_unavailable", + "The Chat2DB local runtime is unavailable", + ), + ), + LocalError::Timeout("connect") => write_failure( + DatabaseWriteState::NotStarted, + retryable_error( + "local_runtime_timeout", + "The Chat2DB local runtime could not be reached in time", + ), + ), + LocalError::Io { operation, .. } if write_was_not_dispatched(operation) => write_failure( + DatabaseWriteState::NotStarted, + retryable_error( + "local_runtime_io_error", + "The Chat2DB local runtime could not be reached", + ), + ), + LocalError::Remote(_) + | LocalError::Timeout(_) + | LocalError::Io { .. } + | LocalError::Protocol(_) + | LocalError::Json(_) + | LocalError::Task(_) => unknown_write_result(), + } +} + +fn remote_rejected_before_dispatch(code: &str) -> bool { + matches!( + code, + "local_protocol_version_mismatch" + | "invalid_local_request" + | "local_attachment_unauthorized" + ) +} + +fn write_was_not_dispatched(operation: &str) -> bool { + operation.contains("metadata") + || operation.contains("data directory") + || operation.contains("connect") + || operation.contains("socket") + || operation.contains("named pipe") +} + +fn unknown_write_result() -> DatabaseWriteResult { + write_failure( + DatabaseWriteState::Unknown, + ApiError::new( + "database_write_outcome_unknown", + "The database write outcome is unknown; do not retry it blindly", + ), + ) +} + +fn write_failure(state: DatabaseWriteState, error: ApiError) -> DatabaseWriteResult { + DatabaseWriteResult { + state, + affected_rows: None, + error: Some(error), + } +} + +fn retryable_error(code: &'static str, message: &'static str) -> ApiError { + let mut error = ApiError::new(code, message); + error.retryable = true; + error +} + +#[cfg(test)] +mod tests { + use chat2db_contract::{ApiError, DatabaseWriteState, ExecuteDatabaseWriteRequest}; + + use super::{LocalClient, LocalError, MAX_REQUEST_BYTES, RemoteError, local_write_failure}; + + #[test] + fn write_transport_timeout_distinguishes_before_and_after_dispatch() { + let before_dispatch = local_write_failure(LocalError::Timeout("connect")); + assert_eq!(before_dispatch.state, DatabaseWriteState::NotStarted); + assert_eq!( + before_dispatch + .error + .as_ref() + .map(|error| error.code.as_str()), + Some("local_runtime_timeout") + ); + + let after_dispatch = local_write_failure(LocalError::Timeout("response read")); + assert_eq!(after_dispatch.state, DatabaseWriteState::Unknown); + assert_eq!( + after_dispatch + .error + .as_ref() + .map(|error| error.code.as_str()), + Some("database_write_outcome_unknown") + ); + assert!(!after_dispatch.error.unwrap().retryable); + } + + #[test] + fn only_known_local_rejections_are_classified_as_not_started() { + let unauthorized = + local_write_failure(LocalError::Remote(Box::new(RemoteError(ApiError::new( + "local_attachment_unauthorized", + "Local attachment authentication failed", + ))))); + assert_eq!(unauthorized.state, DatabaseWriteState::NotStarted); + + let oversized_response = + local_write_failure(LocalError::Remote(Box::new(RemoteError(ApiError::new( + "local_response_too_large", + "The local response exceeds the maximum transport frame", + ))))); + assert_eq!(oversized_response.state, DatabaseWriteState::Unknown); + } + + #[tokio::test] + async fn oversized_write_is_rejected_before_local_delivery() { + let result = LocalClient::new("missing-runtime") + .execute_database_write(ExecuteDatabaseWriteRequest { + datasource_id: "datasource-1".to_owned(), + sql: format!( + "UPDATE items SET label = '{}';", + "x".repeat(MAX_REQUEST_BYTES) + ), + confirmed: true, + }) + .await; + assert_eq!(result.state, DatabaseWriteState::NotStarted); + assert_eq!( + result.error.as_ref().map(|error| error.code.as_str()), + Some("invalid_database_write") + ); + } +} diff --git a/crates/chat2db-local/src/lib.rs b/crates/chat2db-local/src/lib.rs index fc1273a..d4a9dc7 100644 --- a/crates/chat2db-local/src/lib.rs +++ b/crates/chat2db-local/src/lib.rs @@ -7,8 +7,9 @@ mod transport; use std::{fmt, io}; use chat2db_contract::{ - ApiError, CancelOperationResponse, DatasourceList, HealthResponse, OperationSnapshot, - QueryAccepted, ResultPage, ResultPageRequest, StartQueryRequest, + ApiError, CancelOperationResponse, DatabaseWriteResult, DatasourceList, + ExecuteDatabaseWriteRequest, HealthResponse, OperationSnapshot, QueryAccepted, ResultPage, + ResultPageRequest, StartQueryRequest, }; use serde::{Deserialize, Serialize}; use thiserror::Error; @@ -59,6 +60,9 @@ enum AttachmentCommand { StartReadQuery { request: StartQueryRequest, }, + ExecuteDatabaseWrite { + request: ExecuteDatabaseWriteRequest, + }, OperationSnapshot { operation_id: String, }, @@ -92,6 +96,7 @@ enum AttachmentPayload { Health(Box), Datasources(Box), QueryAccepted(Box), + DatabaseWrite(Box), OperationSnapshot(Box), CancelOperation(Box), ResultPage(Box), diff --git a/crates/chat2db-local/src/server.rs b/crates/chat2db-local/src/server.rs index 9fc35bd..ff7d274 100644 --- a/crates/chat2db-local/src/server.rs +++ b/crates/chat2db-local/src/server.rs @@ -243,6 +243,11 @@ async fn dispatch(application: Application, command: AttachmentCommand) -> Attac .start_read_query(request) .await .map(|value| AttachmentPayload::QueryAccepted(Box::new(value))), + AttachmentCommand::ExecuteDatabaseWrite { request } => { + Ok(AttachmentPayload::DatabaseWrite(Box::new( + application.execute_confirmed_database_write(request).await, + ))) + } AttachmentCommand::OperationSnapshot { operation_id } => application .operation_snapshot(&operation_id) .await diff --git a/crates/chat2db-local/src/tests.rs b/crates/chat2db-local/src/tests.rs index 6d19879..cec7122 100644 --- a/crates/chat2db-local/src/tests.rs +++ b/crates/chat2db-local/src/tests.rs @@ -2,8 +2,8 @@ use std::{fs, sync::Arc, time::Duration}; use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD}; use chat2db_contract::{ - ApiError, CreateDatasourceRequest, JdbcValue, QueryLimits, ResultMetadata, ResultPage, - ResultPageRequest, ResultRow, StartQueryRequest, + ApiError, CreateDatasourceRequest, DatabaseWriteState, ExecuteDatabaseWriteRequest, JdbcValue, + QueryLimits, ResultMetadata, ResultPage, ResultPageRequest, ResultRow, StartQueryRequest, }; use chat2db_core::Application; use chat2db_storage::{SecretRef, SecretValue, SecretVault, SecretVaultError, Storage}; @@ -139,6 +139,26 @@ async fn serves_real_application_state_and_cleans_discovery_files() { assert!(!directory.path().join(SOCKET_FILE).exists()); } +#[tokio::test] +async fn local_runtime_enforces_database_write_confirmation() { + let (directory, application) = setup(); + let mut server = LocalServer::start(application).expect("server starts"); + let result = LocalClient::new(directory.path()) + .execute_database_write(ExecuteDatabaseWriteRequest { + datasource_id: "missing-datasource".to_owned(), + sql: "UPDATE items SET label = 'changed'".to_owned(), + confirmed: false, + }) + .await; + + assert_eq!(result.state, DatabaseWriteState::NotStarted); + assert_eq!( + result.error.as_ref().map(|error| error.code.as_str()), + Some("database_write_confirmation_required") + ); + server.shutdown().await.expect("server shuts down"); +} + #[tokio::test] async fn rejects_a_second_listener_for_the_same_runtime() { let (_directory, application) = setup(); diff --git a/crates/chat2db-local/tests/native_mysql_write_docker.rs b/crates/chat2db-local/tests/native_mysql_write_docker.rs new file mode 100644 index 0000000..ca5f5de --- /dev/null +++ b/crates/chat2db-local/tests/native_mysql_write_docker.rs @@ -0,0 +1,407 @@ +use std::time::Duration; + +use base64::{Engine as _, engine::general_purpose::STANDARD}; +use chat2db_contract::{ + ComponentState, CreateDatasourceRequest, DatabaseWriteState, DatasourceConnection, + DatasourceConnectionProperty, ExecuteDatabaseWriteRequest, JdbcValue, OperationStatus, + QueryLimits, ResultPageRequest, StartQueryRequest, +}; +use chat2db_core::{Application, RuntimeConfig, RuntimeHost}; +use chat2db_java_bridge::{EngineCommand, EngineConfig}; +use chat2db_local::{LocalClient, LocalServer}; +use tempfile::TempDir; +use uuid::Uuid; + +const REQUIRED_MYSQL_ENV: [&str; 4] = [ + "MYSQL_TEST_HOST", + "MYSQL_TEST_PORT", + "MYSQL_TEST_USER", + "MYSQL_TEST_PASSWORD", +]; +const QUERY_TIMEOUT: Duration = Duration::from_secs(15); + +#[derive(Clone)] +struct MysqlTestConfig { + host: String, + port: u16, + user: String, + password: String, +} + +struct AutomationDatasources { + writable: String, + read_only: String, +} + +impl MysqlTestConfig { + fn from_environment() -> Option { + let required = mysql_test_required(); + let configured = REQUIRED_MYSQL_ENV + .iter() + .filter(|name| std::env::var_os(name).is_some()) + .count(); + if configured == 0 { + assert!( + !required, + "MYSQL_TEST_REQUIRED is enabled but the MySQL endpoint is absent" + ); + eprintln!("skipping local automation MySQL test; MYSQL_TEST_* variables are absent"); + return None; + } + assert_eq!( + configured, + REQUIRED_MYSQL_ENV.len(), + "local automation MySQL integration is partially configured" + ); + let host = required_env("MYSQL_TEST_HOST"); + assert!( + !host.trim().is_empty() + && !host.chars().any(char::is_control) + && !host.contains(['/', '?', '#']), + "MYSQL_TEST_HOST is invalid" + ); + let port = required_env("MYSQL_TEST_PORT") + .parse::() + .expect("MYSQL_TEST_PORT must be a TCP port"); + assert_ne!(port, 0, "MYSQL_TEST_PORT cannot be zero"); + let user = required_env("MYSQL_TEST_USER"); + assert!(!user.is_empty(), "MYSQL_TEST_USER cannot be empty"); + Some(Self { + host, + port, + user, + password: required_env("MYSQL_TEST_PASSWORD"), + }) + } + + fn connection(&self, database_name: &str, read_only: bool) -> DatasourceConnection { + let host = if self.host.contains(':') + && !(self.host.starts_with('[') && self.host.ends_with(']')) + { + format!("[{}]", self.host) + } else { + self.host.clone() + }; + DatasourceConnection { + jdbc_url: format!( + "jdbc:mysql://{host}:{}/{database_name}?useSSL=false&serverTimezone=UTC", + self.port + ), + properties: vec![ + DatasourceConnectionProperty { + key: "user".to_owned(), + value: self.user.clone(), + sensitive: false, + }, + DatasourceConnectionProperty { + key: "password".to_owned(), + value: self.password.clone(), + sensitive: true, + }, + ], + read_only, + ssh: None, + } + } +} + +#[tokio::test] +async fn local_automation_writes_real_mysql_and_keeps_java_dormant() { + let Some(config) = MysqlTestConfig::from_environment() else { + return; + }; + let database_name = format!("chat2db_local_it_{}", Uuid::new_v4().simple()); + let directory = TempDir::new().expect("temporary local automation runtime"); + let runtime = RuntimeConfig::new(EngineConfig::new(EngineCommand::new( + directory.path().join("missing-java"), + ))) + .with_data_dir(directory.path().join("data")) + .with_vault_master_key_base64(STANDARD.encode([0x4c; 32])); + let mut host = RuntimeHost::open(runtime) + .await + .expect("local automation runtime must open without Java"); + let application = host.application(); + assert_java_dormant(&application); + let admin_datasource = application + .create_datasource(CreateDatasourceRequest { + name: "Local automation MySQL administrator".to_owned(), + driver_id: "mysql".to_owned(), + connection: Some(config.connection("mysql", false)), + }) + .await + .expect("native MySQL administrator datasource must persist"); + let data_dir = application + .storage() + .expect("runtime storage must be configured") + .data_dir() + .to_path_buf(); + let mut server = LocalServer::start(application.clone()).expect("local server must start"); + let client = LocalClient::new(data_dir); + assert_java_dormant(&application); + + let verification = tokio::spawn(verify_local_automation( + config, + database_name.clone(), + application.clone(), + client.clone(), + admin_datasource.id.clone(), + )) + .await; + let cleanup = client + .execute_database_write(ExecuteDatabaseWriteRequest { + datasource_id: admin_datasource.id, + sql: format!("DROP DATABASE IF EXISTS `{database_name}`"), + confirmed: true, + }) + .await; + if cleanup.state != DatabaseWriteState::Succeeded { + eprintln!("local automation MySQL cleanup failed: {cleanup:?}"); + } + assert_java_dormant(&application); + server.shutdown().await.expect("local server must stop"); + assert_java_dormant(&application); + host.shutdown() + .await + .expect("native-only runtime must shut down cleanly"); + + match verification { + Ok(()) => assert_eq!(cleanup.state, DatabaseWriteState::Succeeded), + Err(error) if error.is_panic() => std::panic::resume_unwind(error.into_panic()), + Err(error) => panic!("local automation verification task failed: {error}"), + } +} + +async fn verify_local_automation( + config: MysqlTestConfig, + database_name: String, + application: Application, + client: LocalClient, + admin_datasource_id: String, +) { + let version = read_text(&client, &admin_datasource_id, "SELECT VERSION()").await; + assert!( + version.starts_with("8.4."), + "local automation acceptance requires MySQL 8.4, found {version}" + ); + let datasources = provision_fixture( + &config, + &database_name, + &application, + &client, + admin_datasource_id, + ) + .await; + let update_sql = "UPDATE `automation_write_probe` SET `label` = 'written-through-local-server' WHERE `id` = 1"; + + let unconfirmed = client + .execute_database_write(ExecuteDatabaseWriteRequest { + datasource_id: datasources.writable.clone(), + sql: update_sql.to_owned(), + confirmed: false, + }) + .await; + assert_eq!(unconfirmed.state, DatabaseWriteState::NotStarted); + assert_eq!( + unconfirmed.error.as_ref().map(|error| error.code.as_str()), + Some("database_write_confirmation_required") + ); + assert_eq!( + read_text( + &client, + &datasources.writable, + "SELECT `label` FROM `automation_write_probe` WHERE `id` = 1", + ) + .await, + "initial" + ); + assert_java_dormant(&application); + + let read_only = client + .execute_database_write(ExecuteDatabaseWriteRequest { + datasource_id: datasources.read_only, + sql: update_sql.to_owned(), + confirmed: true, + }) + .await; + assert_eq!(read_only.state, DatabaseWriteState::NotStarted); + assert_eq!( + read_only.error.as_ref().map(|error| error.code.as_str()), + Some("datasource_read_only") + ); + assert_eq!( + read_text( + &client, + &datasources.writable, + "SELECT `label` FROM `automation_write_probe` WHERE `id` = 1", + ) + .await, + "initial" + ); + assert_java_dormant(&application); + + let written = client + .execute_database_write(ExecuteDatabaseWriteRequest { + datasource_id: datasources.writable.clone(), + sql: update_sql.to_owned(), + confirmed: true, + }) + .await; + assert_eq!(written.state, DatabaseWriteState::Succeeded); + assert_eq!(written.affected_rows.as_deref(), Some("1")); + assert!(written.error.is_none()); + assert_eq!( + read_text( + &client, + &datasources.writable, + "SELECT `label` FROM `automation_write_probe` WHERE `id` = 1", + ) + .await, + "written-through-local-server" + ); + assert_java_dormant(&application); +} + +async fn provision_fixture( + config: &MysqlTestConfig, + database_name: &str, + application: &Application, + client: &LocalClient, + admin_datasource_id: String, +) -> AutomationDatasources { + let created_database = client + .execute_database_write(ExecuteDatabaseWriteRequest { + datasource_id: admin_datasource_id, + sql: format!( + "CREATE DATABASE `{database_name}` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci" + ), + confirmed: true, + }) + .await; + assert_write_succeeded(&created_database, "fixture database creation"); + assert_java_dormant(application); + + let writable = application + .create_datasource(CreateDatasourceRequest { + name: "Local automation MySQL".to_owned(), + driver_id: "mysql".to_owned(), + connection: Some(config.connection(database_name, false)), + }) + .await + .expect("writable native MySQL datasource must persist"); + let read_only = application + .create_datasource(CreateDatasourceRequest { + name: "Local automation read-only MySQL".to_owned(), + driver_id: "mysql".to_owned(), + connection: Some(config.connection(database_name, true)), + }) + .await + .expect("read-only native MySQL datasource must persist"); + let created_table = client + .execute_database_write(ExecuteDatabaseWriteRequest { + datasource_id: writable.id.clone(), + sql: "CREATE TABLE `automation_write_probe` (`id` BIGINT NOT NULL, `label` VARCHAR(128) NOT NULL, PRIMARY KEY (`id`)) ENGINE=InnoDB".to_owned(), + confirmed: true, + }) + .await; + assert_write_succeeded(&created_table, "fixture table creation"); + let inserted_row = client + .execute_database_write(ExecuteDatabaseWriteRequest { + datasource_id: writable.id.clone(), + sql: "INSERT INTO `automation_write_probe` VALUES (1, 'initial')".to_owned(), + confirmed: true, + }) + .await; + assert_write_succeeded(&inserted_row, "fixture row insertion"); + AutomationDatasources { + writable: writable.id, + read_only: read_only.id, + } +} + +async fn read_text(client: &LocalClient, datasource_id: &str, sql: &str) -> String { + let accepted = client + .start_read_query(StartQueryRequest { + datasource_id: datasource_id.to_owned(), + sql: sql.to_owned(), + parameters: Vec::new(), + limits: QueryLimits { + max_rows: "10".to_owned(), + max_result_bytes: "1048576".to_owned(), + batch_rows: 10, + batch_bytes: 65_536, + result_ttl_seconds: 60, + }, + }) + .await + .expect("local read query must be accepted"); + let snapshot = tokio::time::timeout(QUERY_TIMEOUT, async { + loop { + let snapshot = client + .operation_snapshot(&accepted.operation_id) + .await + .expect("local read query snapshot must remain available"); + match snapshot.status { + OperationStatus::Running => tokio::time::sleep(Duration::from_millis(20)).await, + OperationStatus::Completed => break snapshot, + OperationStatus::Failed | OperationStatus::Cancelled => { + panic!("local read query ended unexpectedly: {snapshot:?}") + } + } + } + }) + .await + .expect("local read query must complete before timeout"); + let result = snapshot + .result + .expect("completed local read query must retain a result"); + let page = client + .result_page( + result.id, + ResultPageRequest { + offset: "0".to_owned(), + max_rows: "10".to_owned(), + max_bytes: "262144".to_owned(), + }, + ) + .await + .expect("local read result must be pageable"); + assert_eq!(page.rows.len(), 1); + assert_eq!(page.rows[0].values.len(), 1); + let JdbcValue::Text { value } = &page.rows[0].values[0] else { + panic!("local read result must contain text: {:?}", page.rows[0]); + }; + value.clone() +} + +fn assert_write_succeeded(result: &chat2db_contract::DatabaseWriteResult, operation: &str) { + assert_eq!( + result.state, + DatabaseWriteState::Succeeded, + "{operation} must succeed: {result:?}" + ); + assert!(result.error.is_none(), "{operation} returned an error"); +} + +fn assert_java_dormant(application: &Application) { + let engine = application + .health() + .components + .into_iter() + .find(|component| component.id == "database-engine") + .expect("database engine health must be present"); + assert_eq!(engine.state, ComponentState::Ready); + assert_eq!(engine.detail, "Available on demand; Java is not running"); +} + +fn mysql_test_required() -> bool { + std::env::var("MYSQL_TEST_REQUIRED").is_ok_and(|value| { + matches!( + value.trim().to_ascii_lowercase().as_str(), + "1" | "true" | "yes" | "on" + ) + }) +} + +fn required_env(name: &str) -> String { + std::env::var(name).unwrap_or_else(|_| panic!("{name} must be configured")) +} diff --git a/crates/chat2db-storage/migrations/005_workspace_namespace.sql b/crates/chat2db-storage/migrations/005_workspace_namespace.sql new file mode 100644 index 0000000..d7132a8 --- /dev/null +++ b/crates/chat2db-storage/migrations/005_workspace_namespace.sql @@ -0,0 +1,65 @@ +BEGIN IMMEDIATE; + +CREATE TABLE workspace_namespaces ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL CHECK (length(name) BETWEEN 1 AND 512), + created_at_ms INTEGER NOT NULL, + updated_at_ms INTEGER NOT NULL CHECK (updated_at_ms >= created_at_ms) +) STRICT; + +CREATE TABLE workspace_nodes ( + node_key TEXT PRIMARY KEY NOT NULL, + node_type TEXT NOT NULL CHECK (node_type IN ('NAMESPACE', 'DATA_SOURCE')), + namespace_id INTEGER UNIQUE REFERENCES workspace_namespaces(id) ON DELETE CASCADE, + datasource_id TEXT UNIQUE REFERENCES datasources(id) ON DELETE CASCADE, + parent_namespace_id INTEGER REFERENCES workspace_namespaces(id) ON DELETE SET NULL, + position INTEGER NOT NULL CHECK (position >= 0), + created_at_ms INTEGER NOT NULL, + CHECK ( + (node_type = 'NAMESPACE' AND namespace_id IS NOT NULL AND datasource_id IS NULL) + OR + (node_type = 'DATA_SOURCE' AND namespace_id IS NULL AND datasource_id IS NOT NULL) + ) +) STRICT; + +CREATE INDEX workspace_nodes_parent_position_idx + ON workspace_nodes (parent_namespace_id, position, node_key); + +INSERT INTO workspace_nodes ( + node_key, node_type, namespace_id, datasource_id, + parent_namespace_id, position, created_at_ms +) +SELECT + 'datasource:' || id, + 'DATA_SOURCE', + NULL, + id, + NULL, + ROW_NUMBER() OVER (ORDER BY created_at_ms, id) - 1, + created_at_ms +FROM datasources; + +CREATE TRIGGER workspace_datasource_insert +AFTER INSERT ON datasources +BEGIN + INSERT INTO workspace_nodes ( + node_key, node_type, namespace_id, datasource_id, + parent_namespace_id, position, created_at_ms + ) VALUES ( + 'datasource:' || NEW.id, + 'DATA_SOURCE', + NULL, + NEW.id, + NULL, + COALESCE( + (SELECT MAX(position) + 1 + FROM workspace_nodes + WHERE parent_namespace_id IS NULL), + 0 + ), + NEW.created_at_ms + ); +END; + +PRAGMA user_version = 5; +COMMIT; diff --git a/crates/chat2db-storage/migrations/006_transfer.sql b/crates/chat2db-storage/migrations/006_transfer.sql new file mode 100644 index 0000000..b6650bd --- /dev/null +++ b/crates/chat2db-storage/migrations/006_transfer.sql @@ -0,0 +1,51 @@ +BEGIN IMMEDIATE; + +CREATE TABLE transfer_tasks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + datasource_id TEXT NOT NULL, + database_name TEXT NOT NULL, + schema_name TEXT NOT NULL, + table_name TEXT, + kind TEXT NOT NULL CHECK (kind IN ('import_file', 'export_sql', 'export_file')), + status TEXT NOT NULL CHECK ( + status IN ('queued', 'running', 'succeeded', 'failed', 'cancelled', 'interrupted') + ), + task_name TEXT NOT NULL, + progress_current INTEGER NOT NULL DEFAULT 0 CHECK (progress_current >= 0), + progress_total INTEGER CHECK (progress_total IS NULL OR progress_total >= 0), + progress_description TEXT NOT NULL DEFAULT '', + info_log TEXT NOT NULL DEFAULT '', + error_log TEXT NOT NULL DEFAULT '', + cancel_requested INTEGER NOT NULL DEFAULT 0 CHECK (cancel_requested IN (0, 1)), + created_at_ms INTEGER NOT NULL, + updated_at_ms INTEGER NOT NULL CHECK (updated_at_ms >= created_at_ms), + finished_at_ms INTEGER CHECK (finished_at_ms IS NULL OR finished_at_ms >= created_at_ms) +) STRICT; + +CREATE INDEX transfer_tasks_created_idx + ON transfer_tasks (created_at_ms DESC, id DESC); + +CREATE INDEX transfer_tasks_status_idx + ON transfer_tasks (status, updated_at_ms DESC, id DESC); + +CREATE TABLE transfer_artifacts ( + id TEXT PRIMARY KEY NOT NULL, + task_id INTEGER REFERENCES transfer_tasks(id) ON DELETE CASCADE, + storage_name TEXT NOT NULL UNIQUE, + file_name TEXT NOT NULL, + media_type TEXT NOT NULL, + format TEXT NOT NULL, + byte_count INTEGER NOT NULL CHECK (byte_count >= 0), + sha256 BLOB NOT NULL CHECK (length(sha256) = 32), + created_at_ms INTEGER NOT NULL, + expires_at_ms INTEGER CHECK (expires_at_ms IS NULL OR expires_at_ms >= created_at_ms) +) STRICT; + +CREATE UNIQUE INDEX transfer_artifacts_task_idx + ON transfer_artifacts (task_id) WHERE task_id IS NOT NULL; + +CREATE INDEX transfer_artifacts_expiry_idx + ON transfer_artifacts (expires_at_ms) WHERE expires_at_ms IS NOT NULL; + +PRAGMA user_version = 6; +COMMIT; diff --git a/crates/chat2db-storage/migrations/007_mysql_workspace.sql b/crates/chat2db-storage/migrations/007_mysql_workspace.sql new file mode 100644 index 0000000..18e715e --- /dev/null +++ b/crates/chat2db-storage/migrations/007_mysql_workspace.sql @@ -0,0 +1,22 @@ +BEGIN IMMEDIATE; + +CREATE TABLE mysql_pinned_tables ( + datasource_id TEXT NOT NULL REFERENCES datasources(id) ON DELETE CASCADE, + database_name TEXT NOT NULL, + schema_name TEXT NOT NULL, + table_name TEXT NOT NULL, + created_at_ms INTEGER NOT NULL, + PRIMARY KEY (datasource_id, database_name, schema_name, table_name) +) STRICT; + +CREATE TABLE mysql_er_positions ( + datasource_id TEXT NOT NULL REFERENCES datasources(id) ON DELETE CASCADE, + database_name TEXT NOT NULL, + schema_name TEXT NOT NULL, + position TEXT NOT NULL, + updated_at_ms INTEGER NOT NULL, + PRIMARY KEY (datasource_id, database_name, schema_name) +) STRICT; + +PRAGMA user_version = 7; +COMMIT; diff --git a/crates/chat2db-storage/migrations/008_community_dashboard.sql b/crates/chat2db-storage/migrations/008_community_dashboard.sql new file mode 100644 index 0000000..5291233 --- /dev/null +++ b/crates/chat2db-storage/migrations/008_community_dashboard.sql @@ -0,0 +1,83 @@ +BEGIN IMMEDIATE; + +CREATE TABLE community_charts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + gmt_create_ms INTEGER NOT NULL CHECK (gmt_create_ms >= 0), + gmt_modified_ms INTEGER NOT NULL CHECK (gmt_modified_ms >= gmt_create_ms), + name TEXT CHECK (name IS NULL OR length(CAST(name AS BLOB)) <= 1024), + description TEXT CHECK (description IS NULL OR length(CAST(description AS BLOB)) <= 1048576), + schema_text TEXT CHECK (schema_text IS NULL OR length(CAST(schema_text AS BLOB)) <= 16777216), + data_source_id INTEGER, + data_source_name TEXT CHECK (data_source_name IS NULL OR length(CAST(data_source_name AS BLOB)) <= 1024), + schema_name TEXT CHECK (schema_name IS NULL OR length(CAST(schema_name AS BLOB)) <= 1024), + chart_type TEXT CHECK (chart_type IS NULL OR length(CAST(chart_type AS BLOB)) <= 256), + database_name TEXT CHECK (database_name IS NULL OR length(CAST(database_name AS BLOB)) <= 1024), + ddl TEXT CHECK (ddl IS NULL OR length(CAST(ddl AS BLOB)) <= 16777216), + deleted TEXT CHECK (deleted IS NULL OR length(CAST(deleted AS BLOB)) <= 32), + user_id INTEGER, + chart_schema_json TEXT CHECK ( + chart_schema_json IS NULL OR ( + json_valid(chart_schema_json) + AND length(CAST(chart_schema_json AS BLOB)) <= 16777216 + ) + ), + meta_data_json TEXT CHECK ( + meta_data_json IS NULL OR ( + json_valid(meta_data_json) + AND length(CAST(meta_data_json AS BLOB)) <= 16777216 + ) + ), + database_info_json TEXT CHECK ( + database_info_json IS NULL OR ( + json_valid(database_info_json) + AND length(CAST(database_info_json AS BLOB)) <= 16777216 + ) + ), + refresh_type TEXT CHECK (refresh_type IS NULL OR length(CAST(refresh_type AS BLOB)) <= 256), + refresh_cycle_json TEXT CHECK ( + refresh_cycle_json IS NULL OR ( + json_valid(refresh_cycle_json) + AND length(CAST(refresh_cycle_json AS BLOB)) <= 16777216 + ) + ) +) STRICT; + +CREATE TABLE community_dashboards ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + gmt_create_ms INTEGER NOT NULL CHECK (gmt_create_ms >= 0), + gmt_modified_ms INTEGER NOT NULL CHECK (gmt_modified_ms >= gmt_create_ms), + name TEXT CHECK (name IS NULL OR length(CAST(name AS BLOB)) <= 1024), + description TEXT CHECK (description IS NULL OR length(CAST(description AS BLOB)) <= 1048576), + data_source_collection_id INTEGER, + chart_ids_json TEXT NOT NULL DEFAULT '[]' CHECK ( + json_valid(chart_ids_json) + AND json_type(chart_ids_json) = 'array' + AND length(CAST(chart_ids_json AS BLOB)) <= 2097152 + ), + schema_text TEXT CHECK (schema_text IS NULL OR length(CAST(schema_text AS BLOB)) <= 16777216), + refresh_type TEXT CHECK (refresh_type IS NULL OR length(CAST(refresh_type AS BLOB)) <= 256), + refresh_cycle_json TEXT CHECK ( + refresh_cycle_json IS NULL OR ( + json_valid(refresh_cycle_json) + AND length(CAST(refresh_cycle_json AS BLOB)) <= 16777216 + ) + ), + user_id INTEGER +) STRICT; + +CREATE INDEX community_dashboards_modified_idx + ON community_dashboards (gmt_modified_ms DESC, id DESC); + +CREATE TRIGGER community_dashboards_delete_charts +AFTER DELETE ON community_dashboards +BEGIN + DELETE FROM community_charts + WHERE id IN ( + SELECT value + FROM json_each(OLD.chart_ids_json) + WHERE type = 'integer' + ); +END; + +PRAGMA user_version = 8; +COMMIT; diff --git a/crates/chat2db-storage/src/community_dashboard.rs b/crates/chat2db-storage/src/community_dashboard.rs new file mode 100644 index 0000000..c2d5827 --- /dev/null +++ b/crates/chat2db-storage/src/community_dashboard.rs @@ -0,0 +1,1229 @@ +use chat2db_contract::{ + CommunityChart, CommunityDashboard, CommunityDashboardListQuery, CommunityDashboardPage, + CreateCommunityChartRequest, CreateCommunityDashboardRequest, UpdateCommunityChartRequest, + UpdateCommunityDashboardRequest, +}; +use rusqlite::{Connection, OptionalExtension, TransactionBehavior, params}; +use serde_json::Value; + +use crate::{Storage, StorageError, now_millis}; + +const MAX_NAME_BYTES: usize = 1_024; +const MAX_DESCRIPTION_BYTES: usize = 1024 * 1024; +const MAX_SCOPE_BYTES: usize = 1_024; +const MAX_SHORT_TEXT_BYTES: usize = 256; +const MAX_DELETED_BYTES: usize = 32; +const MAX_LARGE_TEXT_BYTES: usize = 16 * 1024 * 1024; +const MAX_JSON_BYTES: usize = 16 * 1024 * 1024; +const MAX_CHART_IDS: usize = 100_000; +const MAX_CHART_IDS_JSON_BYTES: usize = 2 * 1024 * 1024; +const MAX_SEARCH_KEY_BYTES: usize = 1_024; +const MAX_PAGE_SIZE: u32 = 1_000; + +const DASHBOARD_COLUMNS: &str = "id, gmt_create_ms, gmt_modified_ms, name, description, + data_source_collection_id, chart_ids_json, schema_text, refresh_type, + refresh_cycle_json, user_id"; + +const CHART_COLUMNS: &str = "id, gmt_create_ms, gmt_modified_ms, name, description, schema_text, + data_source_id, data_source_name, schema_name, chart_type, database_name, + ddl, deleted, user_id, chart_schema_json, meta_data_json, + database_info_json, refresh_type, refresh_cycle_json"; + +impl Storage { + /// Lists Community dashboards ordered by most recent modification. + /// + /// # Errors + /// + /// Returns validation, numeric-range, persisted-data, or `SQLite` failures. + pub fn list_community_dashboards( + &self, + query: &CommunityDashboardListQuery, + ) -> Result { + let page_no = query.page_no.max(1); + let page_size = query.page_size.max(1); + if page_size > MAX_PAGE_SIZE { + return Err(StorageError::InvalidCommunityDashboard( + "page size must be between 1 and 1000", + )); + } + if query + .search_key + .as_ref() + .is_some_and(|value| value.len() > MAX_SEARCH_KEY_BYTES) + { + return Err(StorageError::InvalidCommunityDashboard( + "search key must be at most 1024 UTF-8 bytes", + )); + } + + let offset = u64::from(page_no - 1) + .checked_mul(u64::from(page_size)) + .ok_or(StorageError::NumericRange( + "Community dashboard page offset", + ))?; + let offset = i64::try_from(offset) + .map_err(|_| StorageError::NumericRange("Community dashboard page offset"))?; + let search_pattern = query + .search_key + .as_deref() + .filter(|value| !value.trim().is_empty()) + .map(like_pattern); + let connection = self.connection()?; + let total: i64 = connection.query_row( + "SELECT COUNT(*) FROM community_dashboards + WHERE ?1 IS NULL + OR name COLLATE NOCASE LIKE ?1 ESCAPE '\\' + OR description COLLATE NOCASE LIKE ?1 ESCAPE '\\'", + [search_pattern.as_deref()], + |row| row.get(0), + )?; + let sql = format!( + "SELECT {DASHBOARD_COLUMNS} FROM community_dashboards + WHERE ?1 IS NULL + OR name COLLATE NOCASE LIKE ?1 ESCAPE '\\' + OR description COLLATE NOCASE LIKE ?1 ESCAPE '\\' + ORDER BY gmt_modified_ms DESC, id DESC + LIMIT ?2 OFFSET ?3" + ); + let mut statement = connection.prepare(&sql)?; + let rows = statement.query_map( + params![search_pattern.as_deref(), i64::from(page_size), offset], + raw_dashboard, + )?; + let mut data = Vec::new(); + for row in rows { + data.push(decode_dashboard(row?)?); + } + let total = u64::try_from(total) + .map_err(|_| StorageError::NumericRange("Community dashboard total"))?; + let consumed = u64::from(page_no).checked_mul(u64::from(page_size)).ok_or( + StorageError::NumericRange("Community dashboard page boundary"), + )?; + Ok(CommunityDashboardPage { + data, + total, + page_no, + page_size, + // The historical controller re-wraps this page through + // WebPageResult, whose exact-boundary behavior uses `<=`. + has_next_page: consumed <= total, + }) + } + + /// Loads one Community dashboard by id, or `None` when it is absent. + /// + /// # Errors + /// + /// Returns persisted-data or `SQLite` failures. + pub fn get_community_dashboard( + &self, + id: i64, + ) -> Result, StorageError> { + if id <= 0 { + return Ok(None); + } + load_dashboard(&self.connection()?, id) + } + + /// Creates one Community dashboard and returns its generated id. + /// + /// # Errors + /// + /// Returns validation, clock, numeric-range, or `SQLite` failures. + pub fn create_community_dashboard( + &self, + input: CreateCommunityDashboardRequest, + ) -> Result { + let CreateCommunityDashboardRequest { + name, + description, + data_source_collection_id, + chart_ids, + schema, + refresh_type, + refresh_cycle, + user_id, + } = input; + validate_dashboard_fields( + name.as_deref(), + description.as_deref(), + &chart_ids, + schema.as_deref(), + refresh_type.as_deref(), + refresh_cycle.as_ref(), + )?; + let chart_ids_json = encode_chart_ids(&chart_ids)?; + let refresh_cycle_json = encode_dashboard_json(refresh_cycle.as_ref(), "refresh cycle")?; + let timestamp = now_millis()?; + let mut connection = self.connection()?; + let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + transaction.execute( + "INSERT INTO community_dashboards ( + gmt_create_ms, gmt_modified_ms, name, description, + data_source_collection_id, chart_ids_json, schema_text, + refresh_type, refresh_cycle_json, user_id + ) VALUES (?1, ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)", + params![ + timestamp, + name, + description, + data_source_collection_id, + chart_ids_json, + schema, + refresh_type, + refresh_cycle_json, + user_id, + ], + )?; + let id = transaction.last_insert_rowid(); + if id <= 0 { + return Err(StorageError::Integrity( + "Community dashboard generated a non-positive id".to_owned(), + )); + } + transaction.commit()?; + Ok(id) + } + + /// Applies a non-null partial update to one Community dashboard. + /// + /// # Errors + /// + /// Returns [`StorageError::CommunityDashboardNotFound`] when absent, or + /// validation, clock, numeric-range, and `SQLite` failures. + pub fn update_community_dashboard( + &self, + id: i64, + input: UpdateCommunityDashboardRequest, + ) -> Result<(), StorageError> { + let mut connection = self.connection()?; + let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + let current = load_dashboard(&transaction, id)? + .ok_or(StorageError::CommunityDashboardNotFound(id))?; + let next = CommunityDashboard { + id, + gmt_create: current.gmt_create, + gmt_modified: next_modified(current.gmt_modified)?, + name: input.name.or(current.name), + description: input.description.or(current.description), + data_source_collection_id: input + .data_source_collection_id + .or(current.data_source_collection_id), + chart_ids: input.chart_ids.unwrap_or(current.chart_ids), + schema: input.schema.or(current.schema), + refresh_type: input.refresh_type.or(current.refresh_type), + refresh_cycle: input.refresh_cycle.or(current.refresh_cycle), + user_id: input.user_id.or(current.user_id), + }; + validate_dashboard(&next)?; + let chart_ids_json = encode_chart_ids(&next.chart_ids)?; + let refresh_cycle_json = + encode_dashboard_json(next.refresh_cycle.as_ref(), "refresh cycle")?; + let changed = transaction.execute( + "UPDATE community_dashboards + SET gmt_modified_ms = ?1, name = ?2, description = ?3, + data_source_collection_id = ?4, chart_ids_json = ?5, + schema_text = ?6, refresh_type = ?7, refresh_cycle_json = ?8, + user_id = ?9 + WHERE id = ?10", + params![ + next.gmt_modified, + next.name, + next.description, + next.data_source_collection_id, + chart_ids_json, + next.schema, + next.refresh_type, + refresh_cycle_json, + next.user_id, + id, + ], + )?; + if changed != 1 { + return Err(StorageError::CommunityDashboardNotFound(id)); + } + transaction.commit()?; + Ok(()) + } + + /// Deletes one Community dashboard and every chart id it references. + /// + /// # Errors + /// + /// Returns `SQLite` failures. The dashboard and chart deletes are atomic. + pub fn delete_community_dashboard(&self, id: i64) -> Result { + if id <= 0 { + return Ok(false); + } + let mut connection = self.connection()?; + let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + let deleted = + transaction.execute("DELETE FROM community_dashboards WHERE id = ?1", [id])?; + transaction.commit()?; + Ok(deleted == 1) + } + + /// Loads one Community chart by id, or `None` when it is absent. + /// + /// # Errors + /// + /// Returns persisted-data or `SQLite` failures. + pub fn get_community_chart(&self, id: i64) -> Result, StorageError> { + if id <= 0 { + return Ok(None); + } + load_chart(&self.connection()?, id) + } + + /// Creates one Community chart and returns its generated id. + /// + /// Blank names fall back to `chartSchema.title`, then + /// `chartSchema.summary`, matching Community. + /// + /// # Errors + /// + /// Returns validation, clock, numeric-range, or `SQLite` failures. + pub fn create_community_chart( + &self, + mut input: CreateCommunityChartRequest, + ) -> Result { + input.name = chart_name_for_create(input.name.take(), input.chart_schema.as_ref()); + validate_chart_fields( + input.name.as_deref(), + input.description.as_deref(), + input.schema.as_deref(), + input.data_source_name.as_deref(), + input.schema_name.as_deref(), + input.r#type.as_deref(), + input.database_name.as_deref(), + input.ddl.as_deref(), + input.deleted.as_deref(), + input.chart_schema.as_ref(), + input.meta_data.as_ref(), + input.database_info.as_ref(), + input.refresh_type.as_deref(), + input.refresh_cycle.as_ref(), + )?; + let chart_schema_json = encode_chart_json(input.chart_schema.as_ref(), "chart schema")?; + let meta_data_json = encode_chart_json(input.meta_data.as_ref(), "metadata")?; + let database_info_json = encode_chart_json(input.database_info.as_ref(), "database info")?; + let refresh_cycle_json = encode_chart_json(input.refresh_cycle.as_ref(), "refresh cycle")?; + let timestamp = now_millis()?; + let mut connection = self.connection()?; + let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + transaction.execute( + "INSERT INTO community_charts ( + gmt_create_ms, gmt_modified_ms, name, description, schema_text, + data_source_id, data_source_name, schema_name, chart_type, + database_name, ddl, deleted, user_id, chart_schema_json, + meta_data_json, database_info_json, refresh_type, + refresh_cycle_json + ) VALUES ( + ?1, ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, + ?13, ?14, ?15, ?16, ?17 + )", + params![ + timestamp, + input.name, + input.description, + input.schema, + input.data_source_id, + input.data_source_name, + input.schema_name, + input.r#type, + input.database_name, + input.ddl, + input.deleted, + input.user_id, + chart_schema_json, + meta_data_json, + database_info_json, + input.refresh_type, + refresh_cycle_json, + ], + )?; + let id = transaction.last_insert_rowid(); + if id <= 0 { + return Err(StorageError::Integrity( + "Community chart generated a non-positive id".to_owned(), + )); + } + transaction.commit()?; + Ok(id) + } + + /// Applies a non-null partial update to one Community chart. + /// + /// # Errors + /// + /// Returns [`StorageError::CommunityChartNotFound`] when absent, or + /// validation, clock, numeric-range, and `SQLite` failures. + pub fn update_community_chart( + &self, + id: i64, + input: UpdateCommunityChartRequest, + ) -> Result<(), StorageError> { + let mut connection = self.connection()?; + let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + let current = + load_chart(&transaction, id)?.ok_or(StorageError::CommunityChartNotFound(id))?; + let name = chart_name_for_update( + input.name, + input.chart_schema.as_ref(), + current.name.clone(), + ); + let next = CommunityChart { + id, + gmt_create: current.gmt_create, + gmt_modified: next_modified(current.gmt_modified)?, + name, + description: input.description.or(current.description), + schema: input.schema.or(current.schema), + data_source_id: input.data_source_id.or(current.data_source_id), + data_source_name: input.data_source_name.or(current.data_source_name), + schema_name: input.schema_name.or(current.schema_name), + r#type: input.r#type.or(current.r#type), + database_name: input.database_name.or(current.database_name), + ddl: input.ddl.or(current.ddl), + deleted: input.deleted.or(current.deleted), + user_id: input.user_id.or(current.user_id), + chart_schema: input.chart_schema.or(current.chart_schema), + meta_data: input.meta_data.or(current.meta_data), + database_info: input.database_info.or(current.database_info), + refresh_type: input.refresh_type.or(current.refresh_type), + refresh_cycle: input.refresh_cycle.or(current.refresh_cycle), + }; + validate_chart(&next)?; + let chart_schema_json = encode_chart_json(next.chart_schema.as_ref(), "chart schema")?; + let meta_data_json = encode_chart_json(next.meta_data.as_ref(), "metadata")?; + let database_info_json = encode_chart_json(next.database_info.as_ref(), "database info")?; + let refresh_cycle_json = encode_chart_json(next.refresh_cycle.as_ref(), "refresh cycle")?; + let changed = transaction.execute( + "UPDATE community_charts + SET gmt_modified_ms = ?1, name = ?2, description = ?3, + schema_text = ?4, data_source_id = ?5, data_source_name = ?6, + schema_name = ?7, chart_type = ?8, database_name = ?9, + ddl = ?10, deleted = ?11, user_id = ?12, + chart_schema_json = ?13, meta_data_json = ?14, + database_info_json = ?15, refresh_type = ?16, + refresh_cycle_json = ?17 + WHERE id = ?18", + params![ + next.gmt_modified, + next.name, + next.description, + next.schema, + next.data_source_id, + next.data_source_name, + next.schema_name, + next.r#type, + next.database_name, + next.ddl, + next.deleted, + next.user_id, + chart_schema_json, + meta_data_json, + database_info_json, + next.refresh_type, + refresh_cycle_json, + id, + ], + )?; + if changed != 1 { + return Err(StorageError::CommunityChartNotFound(id)); + } + transaction.commit()?; + Ok(()) + } + + /// Deletes one Community chart, returning whether it existed. + /// + /// # Errors + /// + /// Returns `SQLite` failures. + pub fn delete_community_chart(&self, id: i64) -> Result { + if id <= 0 { + return Ok(false); + } + let mut connection = self.connection()?; + let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + let deleted = transaction.execute("DELETE FROM community_charts WHERE id = ?1", [id])?; + transaction.commit()?; + Ok(deleted == 1) + } +} + +fn next_modified(current: i64) -> Result { + let next = current.checked_add(1).ok_or(StorageError::NumericRange( + "Community modification timestamp", + ))?; + Ok(now_millis()?.max(next)) +} + +fn chart_name_for_create(name: Option, chart_schema: Option<&Value>) -> Option { + match name { + Some(name) if !name.trim().is_empty() => Some(name), + _ => chart_fallback_name(chart_schema), + } +} + +fn chart_name_for_update( + name: Option, + update_chart_schema: Option<&Value>, + current_name: Option, +) -> Option { + match name { + Some(name) if !name.trim().is_empty() => Some(name), + _ => chart_fallback_name(update_chart_schema).or(current_name), + } +} + +fn chart_fallback_name(chart_schema: Option<&Value>) -> Option { + let schema = chart_schema?.as_object()?; + for field in ["title", "summary"] { + if let Some(value) = schema + .get(field) + .and_then(Value::as_str) + .filter(|value| !value.trim().is_empty()) + { + return Some(value.to_owned()); + } + } + None +} + +fn like_pattern(value: &str) -> String { + let mut pattern = String::with_capacity(value.len() + 2); + pattern.push('%'); + for character in value.chars() { + if matches!(character, '%' | '_' | '\\') { + pattern.push('\\'); + } + pattern.push(character); + } + pattern.push('%'); + pattern +} + +fn encode_chart_ids(chart_ids: &[i64]) -> Result { + if chart_ids.len() > MAX_CHART_IDS { + return Err(StorageError::InvalidCommunityDashboard( + "chartIds must contain at most 100000 ids", + )); + } + let encoded = serde_json::to_string(chart_ids).map_err(|_| { + StorageError::Integrity("failed to encode Community dashboard chartIds".to_owned()) + })?; + if encoded.len() > MAX_CHART_IDS_JSON_BYTES { + return Err(StorageError::InvalidCommunityDashboard( + "encoded chartIds exceeds the 2 MiB limit", + )); + } + Ok(encoded) +} + +fn encode_dashboard_json( + value: Option<&Value>, + field: &'static str, +) -> Result, StorageError> { + encode_json( + value, + MAX_JSON_BYTES, + StorageError::InvalidCommunityDashboard, + field, + ) +} + +fn encode_chart_json( + value: Option<&Value>, + field: &'static str, +) -> Result, StorageError> { + encode_json( + value, + MAX_JSON_BYTES, + StorageError::InvalidCommunityChart, + field, + ) +} + +fn encode_json( + value: Option<&Value>, + limit: usize, + invalid: F, + field: &'static str, +) -> Result, StorageError> +where + F: FnOnce(&'static str) -> StorageError, +{ + let Some(value) = value else { + return Ok(None); + }; + let encoded = serde_json::to_string(value) + .map_err(|_| StorageError::Integrity(format!("failed to encode Community {field}")))?; + if encoded.len() > limit { + return Err(invalid(match field { + "chart schema" => "chartSchema exceeds the 16 MiB encoded JSON limit", + "metadata" => "metaData exceeds the 16 MiB encoded JSON limit", + "database info" => "databaseInfo exceeds the 16 MiB encoded JSON limit", + _ => "refreshCycle exceeds the 16 MiB encoded JSON limit", + })); + } + Ok(Some(encoded)) +} + +fn decode_json(value: Option, field: &'static str) -> Result, StorageError> { + value + .map(|encoded| { + serde_json::from_str(&encoded).map_err(|_| { + StorageError::Integrity(format!("persisted Community {field} JSON is invalid")) + }) + }) + .transpose() +} + +fn validate_dashboard(dashboard: &CommunityDashboard) -> Result<(), StorageError> { + if dashboard.id <= 0 { + return Err(StorageError::InvalidCommunityDashboard( + "persisted id must be a positive signed 64-bit integer", + )); + } + if dashboard.gmt_create < 0 || dashboard.gmt_modified < dashboard.gmt_create { + return Err(StorageError::InvalidCommunityDashboard( + "persisted timestamps are invalid", + )); + } + validate_dashboard_fields( + dashboard.name.as_deref(), + dashboard.description.as_deref(), + &dashboard.chart_ids, + dashboard.schema.as_deref(), + dashboard.refresh_type.as_deref(), + dashboard.refresh_cycle.as_ref(), + ) +} + +fn validate_dashboard_fields( + name: Option<&str>, + description: Option<&str>, + chart_ids: &[i64], + schema: Option<&str>, + refresh_type: Option<&str>, + refresh_cycle: Option<&Value>, +) -> Result<(), StorageError> { + validate_dashboard_text( + name, + MAX_NAME_BYTES, + "name must be at most 1024 UTF-8 bytes", + )?; + validate_dashboard_text( + description, + MAX_DESCRIPTION_BYTES, + "description must be at most 1 MiB", + )?; + validate_dashboard_text( + schema, + MAX_LARGE_TEXT_BYTES, + "schema must be at most 16 MiB", + )?; + validate_dashboard_text( + refresh_type, + MAX_SHORT_TEXT_BYTES, + "refreshType must be at most 256 UTF-8 bytes", + )?; + encode_chart_ids(chart_ids)?; + encode_dashboard_json(refresh_cycle, "refresh cycle")?; + Ok(()) +} + +fn validate_dashboard_text( + value: Option<&str>, + limit: usize, + message: &'static str, +) -> Result<(), StorageError> { + if value.is_some_and(|value| value.len() > limit || value.contains('\0')) { + return Err(StorageError::InvalidCommunityDashboard(message)); + } + Ok(()) +} + +fn validate_chart(chart: &CommunityChart) -> Result<(), StorageError> { + if chart.id <= 0 { + return Err(StorageError::InvalidCommunityChart( + "persisted id must be a positive signed 64-bit integer", + )); + } + if chart.gmt_create < 0 || chart.gmt_modified < chart.gmt_create { + return Err(StorageError::InvalidCommunityChart( + "persisted timestamps are invalid", + )); + } + validate_chart_fields( + chart.name.as_deref(), + chart.description.as_deref(), + chart.schema.as_deref(), + chart.data_source_name.as_deref(), + chart.schema_name.as_deref(), + chart.r#type.as_deref(), + chart.database_name.as_deref(), + chart.ddl.as_deref(), + chart.deleted.as_deref(), + chart.chart_schema.as_ref(), + chart.meta_data.as_ref(), + chart.database_info.as_ref(), + chart.refresh_type.as_deref(), + chart.refresh_cycle.as_ref(), + ) +} + +#[allow(clippy::too_many_arguments)] +fn validate_chart_fields( + name: Option<&str>, + description: Option<&str>, + schema: Option<&str>, + data_source_name: Option<&str>, + schema_name: Option<&str>, + chart_type: Option<&str>, + database_name: Option<&str>, + ddl: Option<&str>, + deleted: Option<&str>, + chart_schema: Option<&Value>, + meta_data: Option<&Value>, + database_info: Option<&Value>, + refresh_type: Option<&str>, + refresh_cycle: Option<&Value>, +) -> Result<(), StorageError> { + validate_chart_text( + name, + MAX_NAME_BYTES, + "name must be at most 1024 UTF-8 bytes", + )?; + validate_chart_text( + description, + MAX_DESCRIPTION_BYTES, + "description must be at most 1 MiB", + )?; + validate_chart_text( + schema, + MAX_LARGE_TEXT_BYTES, + "schema must be at most 16 MiB", + )?; + for value in [data_source_name, schema_name, database_name] { + validate_chart_text( + value, + MAX_SCOPE_BYTES, + "datasource, schema, and database names must be at most 1024 UTF-8 bytes", + )?; + } + validate_chart_text( + chart_type, + MAX_SHORT_TEXT_BYTES, + "type must be at most 256 UTF-8 bytes", + )?; + validate_chart_text(ddl, MAX_LARGE_TEXT_BYTES, "ddl must be at most 16 MiB")?; + validate_chart_text( + deleted, + MAX_DELETED_BYTES, + "deleted must be at most 32 UTF-8 bytes", + )?; + validate_chart_text( + refresh_type, + MAX_SHORT_TEXT_BYTES, + "refreshType must be at most 256 UTF-8 bytes", + )?; + encode_chart_json(chart_schema, "chart schema")?; + encode_chart_json(meta_data, "metadata")?; + encode_chart_json(database_info, "database info")?; + encode_chart_json(refresh_cycle, "refresh cycle")?; + Ok(()) +} + +fn validate_chart_text( + value: Option<&str>, + limit: usize, + message: &'static str, +) -> Result<(), StorageError> { + if value.is_some_and(|value| value.len() > limit || value.contains('\0')) { + return Err(StorageError::InvalidCommunityChart(message)); + } + Ok(()) +} + +struct RawDashboard { + id: i64, + gmt_create: i64, + gmt_modified: i64, + name: Option, + description: Option, + data_source_collection_id: Option, + chart_ids_json: String, + schema: Option, + refresh_type: Option, + refresh_cycle_json: Option, + user_id: Option, +} + +fn raw_dashboard(row: &rusqlite::Row<'_>) -> rusqlite::Result { + Ok(RawDashboard { + id: row.get(0)?, + gmt_create: row.get(1)?, + gmt_modified: row.get(2)?, + name: row.get(3)?, + description: row.get(4)?, + data_source_collection_id: row.get(5)?, + chart_ids_json: row.get(6)?, + schema: row.get(7)?, + refresh_type: row.get(8)?, + refresh_cycle_json: row.get(9)?, + user_id: row.get(10)?, + }) +} + +fn decode_dashboard(raw: RawDashboard) -> Result { + let chart_ids = serde_json::from_str(&raw.chart_ids_json).map_err(|_| { + StorageError::Integrity("persisted Community dashboard chartIds JSON is invalid".to_owned()) + })?; + let dashboard = CommunityDashboard { + id: raw.id, + gmt_create: raw.gmt_create, + gmt_modified: raw.gmt_modified, + name: raw.name, + description: raw.description, + data_source_collection_id: raw.data_source_collection_id, + chart_ids, + schema: raw.schema, + refresh_type: raw.refresh_type, + refresh_cycle: decode_json(raw.refresh_cycle_json, "dashboard refreshCycle")?, + user_id: raw.user_id, + }; + validate_dashboard(&dashboard)?; + Ok(dashboard) +} + +fn load_dashboard( + connection: &Connection, + id: i64, +) -> Result, StorageError> { + let sql = format!("SELECT {DASHBOARD_COLUMNS} FROM community_dashboards WHERE id = ?1"); + connection + .query_row(&sql, [id], raw_dashboard) + .optional()? + .map(decode_dashboard) + .transpose() +} + +struct RawChart { + id: i64, + gmt_create: i64, + gmt_modified: i64, + name: Option, + description: Option, + schema: Option, + data_source_id: Option, + data_source_name: Option, + schema_name: Option, + chart_type: Option, + database_name: Option, + ddl: Option, + deleted: Option, + user_id: Option, + chart_schema_json: Option, + meta_data_json: Option, + database_info_json: Option, + refresh_type: Option, + refresh_cycle_json: Option, +} + +fn raw_chart(row: &rusqlite::Row<'_>) -> rusqlite::Result { + Ok(RawChart { + id: row.get(0)?, + gmt_create: row.get(1)?, + gmt_modified: row.get(2)?, + name: row.get(3)?, + description: row.get(4)?, + schema: row.get(5)?, + data_source_id: row.get(6)?, + data_source_name: row.get(7)?, + schema_name: row.get(8)?, + chart_type: row.get(9)?, + database_name: row.get(10)?, + ddl: row.get(11)?, + deleted: row.get(12)?, + user_id: row.get(13)?, + chart_schema_json: row.get(14)?, + meta_data_json: row.get(15)?, + database_info_json: row.get(16)?, + refresh_type: row.get(17)?, + refresh_cycle_json: row.get(18)?, + }) +} + +fn decode_chart(raw: RawChart) -> Result { + let chart = CommunityChart { + id: raw.id, + gmt_create: raw.gmt_create, + gmt_modified: raw.gmt_modified, + name: raw.name, + description: raw.description, + schema: raw.schema, + data_source_id: raw.data_source_id, + data_source_name: raw.data_source_name, + schema_name: raw.schema_name, + r#type: raw.chart_type, + database_name: raw.database_name, + ddl: raw.ddl, + deleted: raw.deleted, + user_id: raw.user_id, + chart_schema: decode_json(raw.chart_schema_json, "chartSchema")?, + meta_data: decode_json(raw.meta_data_json, "metaData")?, + database_info: decode_json(raw.database_info_json, "databaseInfo")?, + refresh_type: raw.refresh_type, + refresh_cycle: decode_json(raw.refresh_cycle_json, "chart refreshCycle")?, + }; + validate_chart(&chart)?; + Ok(chart) +} + +fn load_chart(connection: &Connection, id: i64) -> Result, StorageError> { + let sql = format!("SELECT {CHART_COLUMNS} FROM community_charts WHERE id = ?1"); + connection + .query_row(&sql, [id], raw_chart) + .optional()? + .map(decode_chart) + .transpose() +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use serde_json::{Value, json}; + use tempfile::TempDir; + + use super::MAX_JSON_BYTES; + use crate::{ + CommunityDashboardListQuery, CreateCommunityChartRequest, CreateCommunityDashboardRequest, + SecretRef, SecretValue, SecretVault, SecretVaultError, Storage, StorageError, + UpdateCommunityChartRequest, UpdateCommunityDashboardRequest, + }; + + #[derive(Debug)] + struct EmptyVault; + + impl SecretVault for EmptyVault { + fn probe(&self) -> Result<(), SecretVaultError> { + Ok(()) + } + + fn create( + &self, + _reference: &SecretRef, + _value: &SecretValue, + ) -> Result<(), SecretVaultError> { + Ok(()) + } + + fn get(&self, _reference: &SecretRef) -> Result, SecretVaultError> { + Ok(None) + } + + fn delete(&self, _reference: &SecretRef) -> Result<(), SecretVaultError> { + Ok(()) + } + } + + fn open(directory: &TempDir) -> Storage { + Storage::open(directory.path(), Arc::new(EmptyVault)).expect("storage opens") + } + + fn chart(name: Option<&str>, chart_schema: Value) -> CreateCommunityChartRequest { + CreateCommunityChartRequest { + name: name.map(str::to_owned), + description: Some("Revenue by region".to_owned()), + schema: Some(r#"{"legacy":true}"#.to_owned()), + data_source_id: Some(42), + data_source_name: Some("Local MySQL".to_owned()), + schema_name: Some("analytics".to_owned()), + r#type: Some("BAR".to_owned()), + database_name: Some("warehouse".to_owned()), + ddl: Some("select region, sum(amount) from sales group by region".to_owned()), + deleted: Some("N".to_owned()), + user_id: Some(7), + chart_schema: Some(chart_schema), + meta_data: Some(json!({"rows": [{"region": "east", "amount": 10.50}]})), + database_info: Some(json!({ + "dataSourceId": 42, + "databaseName": "warehouse", + "schemaName": "analytics", + "sql": "select 1" + })), + refresh_type: Some("AUTO".to_owned()), + refresh_cycle: Some(json!({"unit": "seconds", "value": 30})), + } + } + + #[test] + fn dashboard_and_chart_round_trip_across_restart_with_partial_updates() { + let directory = TempDir::new().expect("temp dir"); + let storage = open(&directory); + let chart_id = storage + .create_community_chart(chart(Some(" "), json!({"title": "Revenue"}))) + .expect("chart creates"); + let dashboard_id = storage + .create_community_dashboard(CreateCommunityDashboardRequest { + name: Some("Executive".to_owned()), + description: Some("Initial description".to_owned()), + data_source_collection_id: Some(12), + chart_ids: vec![chart_id], + schema: Some(r#"[{"i":"chart"}]"#.to_owned()), + refresh_type: Some("MANUAL".to_owned()), + refresh_cycle: Some(json!({"cron": "0 * * * *"})), + user_id: Some(7), + }) + .expect("dashboard creates"); + let before = storage + .get_community_dashboard(dashboard_id) + .expect("dashboard reads") + .expect("dashboard exists"); + storage + .update_community_dashboard( + dashboard_id, + UpdateCommunityDashboardRequest { + description: Some("Updated description".to_owned()), + ..UpdateCommunityDashboardRequest::default() + }, + ) + .expect("dashboard updates"); + storage + .update_community_chart( + chart_id, + UpdateCommunityChartRequest { + name: Some(String::new()), + chart_schema: Some(json!({"summary": "Revenue summary"})), + ..UpdateCommunityChartRequest::default() + }, + ) + .expect("chart updates"); + drop(storage); + + let reopened = open(&directory); + let dashboard = reopened + .get_community_dashboard(dashboard_id) + .expect("dashboard rereads") + .expect("dashboard survives restart"); + assert_eq!(dashboard.name.as_deref(), Some("Executive")); + assert_eq!( + dashboard.description.as_deref(), + Some("Updated description") + ); + assert_eq!(dashboard.chart_ids, vec![chart_id]); + assert_eq!(dashboard.refresh_cycle, Some(json!({"cron": "0 * * * *"}))); + assert!(dashboard.gmt_modified > before.gmt_modified); + + let chart = reopened + .get_community_chart(chart_id) + .expect("chart rereads") + .expect("chart survives restart"); + assert_eq!(chart.name.as_deref(), Some("Revenue summary")); + assert_eq!(chart.description.as_deref(), Some("Revenue by region")); + assert_eq!( + chart.chart_schema, + Some(json!({"summary": "Revenue summary"})) + ); + assert_eq!( + chart.meta_data, + Some(json!({"rows": [{"region": "east", "amount": 10.50}]})) + ); + assert_eq!( + chart + .database_info + .as_ref() + .and_then(|value| value.get("sql")), + Some(&json!("select 1")) + ); + } + + #[test] + fn dashboard_list_is_modified_desc_paged_and_case_insensitive() { + let directory = TempDir::new().expect("temp dir"); + let storage = open(&directory); + let first = storage + .create_community_dashboard(CreateCommunityDashboardRequest { + name: Some("Alpha".to_owned()), + description: Some("Quarterly SALES".to_owned()), + ..CreateCommunityDashboardRequest::default() + }) + .expect("first creates"); + let second = storage + .create_community_dashboard(CreateCommunityDashboardRequest { + name: Some("Beta".to_owned()), + description: Some("Operations".to_owned()), + ..CreateCommunityDashboardRequest::default() + }) + .expect("second creates"); + let third = storage + .create_community_dashboard(CreateCommunityDashboardRequest { + name: Some("Gamma".to_owned()), + description: Some("Sales forecast".to_owned()), + ..CreateCommunityDashboardRequest::default() + }) + .expect("third creates"); + storage + .update_community_dashboard( + first, + UpdateCommunityDashboardRequest { + description: Some("Quarterly SALES updated".to_owned()), + ..UpdateCommunityDashboardRequest::default() + }, + ) + .expect("first becomes newest"); + + let page = storage + .list_community_dashboards(&CommunityDashboardListQuery { + page_no: 1, + page_size: 2, + search_key: None, + }) + .expect("page lists"); + assert_eq!(page.total, 3); + assert_eq!(page.data.len(), 2); + assert!(page.has_next_page); + assert_eq!(page.data[0].id, first); + assert_eq!(page.data[1].id, third); + + let exact_page = storage + .list_community_dashboards(&CommunityDashboardListQuery { + page_no: 1, + page_size: 3, + search_key: None, + }) + .expect("exact boundary page lists"); + assert_eq!(exact_page.total, 3); + assert_eq!(exact_page.data.len(), 3); + assert!( + exact_page.has_next_page, + "Community WebPageResult reports another page at an exact boundary" + ); + + let search = storage + .list_community_dashboards(&CommunityDashboardListQuery { + search_key: Some("sAlEs".to_owned()), + ..CommunityDashboardListQuery::default() + }) + .expect("search lists"); + assert_eq!(search.total, 2); + assert_eq!(search.data[0].id, first); + assert_eq!(search.data[1].id, third); + + let literal_wildcard = storage + .list_community_dashboards(&CommunityDashboardListQuery { + search_key: Some("%".to_owned()), + ..CommunityDashboardListQuery::default() + }) + .expect("literal wildcard searches"); + assert_eq!(literal_wildcard.total, 0); + assert_ne!(second, third); + } + + #[test] + fn dashboard_delete_cascades_only_referenced_charts() { + let directory = TempDir::new().expect("temp dir"); + let storage = open(&directory); + let first = storage + .create_community_chart(chart(None, json!({"title": "First"}))) + .expect("first chart creates"); + let second = storage + .create_community_chart(chart(None, json!({"summary": "Second"}))) + .expect("second chart creates"); + let unrelated = storage + .create_community_chart(chart(Some("Unrelated"), json!({}))) + .expect("unrelated chart creates"); + let dashboard = storage + .create_community_dashboard(CreateCommunityDashboardRequest { + chart_ids: vec![first, second, first], + ..CreateCommunityDashboardRequest::default() + }) + .expect("dashboard creates"); + + assert!( + storage + .delete_community_dashboard(dashboard) + .expect("dashboard deletes") + ); + assert!( + storage + .get_community_dashboard(dashboard) + .expect("dashboard absence reads") + .is_none() + ); + assert!( + storage + .get_community_chart(first) + .expect("first absence reads") + .is_none() + ); + assert!( + storage + .get_community_chart(second) + .expect("second absence reads") + .is_none() + ); + assert!( + storage + .get_community_chart(unrelated) + .expect("unrelated reads") + .is_some() + ); + assert!( + !storage + .delete_community_dashboard(dashboard) + .expect("missing delete is idempotent") + ); + } + + #[test] + fn defaults_missing_records_and_json_limits_are_enforced() { + let directory = TempDir::new().expect("temp dir"); + let storage = open(&directory); + let dashboard = storage + .create_community_dashboard(CreateCommunityDashboardRequest::default()) + .expect("default dashboard creates"); + assert!( + storage + .get_community_dashboard(dashboard) + .expect("dashboard reads") + .expect("dashboard exists") + .chart_ids + .is_empty() + ); + assert!( + storage + .get_community_dashboard(9_999_999) + .expect("missing dashboard reads") + .is_none() + ); + assert!( + storage + .get_community_chart(9_999_999) + .expect("missing chart reads") + .is_none() + ); + assert!(matches!( + storage + .update_community_dashboard(9_999_999, UpdateCommunityDashboardRequest::default()), + Err(StorageError::CommunityDashboardNotFound(9_999_999)) + )); + assert!(matches!( + storage.update_community_chart(9_999_999, UpdateCommunityChartRequest::default()), + Err(StorageError::CommunityChartNotFound(9_999_999)) + )); + + let oversized = "x".repeat(MAX_JSON_BYTES + 1); + let error = storage + .create_community_chart(CreateCommunityChartRequest { + chart_schema: Some(json!({"value": oversized})), + ..CreateCommunityChartRequest::default() + }) + .expect_err("oversized JSON is rejected"); + assert!(matches!(error, StorageError::InvalidCommunityChart(_))); + } +} diff --git a/crates/chat2db-storage/src/error.rs b/crates/chat2db-storage/src/error.rs index e7ba7eb..ba9470c 100644 --- a/crates/chat2db-storage/src/error.rs +++ b/crates/chat2db-storage/src/error.rs @@ -62,6 +62,36 @@ pub enum StorageError { /// A datasource field violates the durable contract. #[error("invalid datasource: {0}")] InvalidDatasource(&'static str), + /// The requested workspace namespace does not exist. + #[error("workspace namespace not found: {0}")] + WorkspaceNamespaceNotFound(String), + /// The requested namespace or datasource tree node does not exist. + #[error("workspace node not found: {0}")] + WorkspaceNodeNotFound(String), + /// A workspace namespace or tree operation violates the durable contract. + #[error("invalid workspace operation: {0}")] + InvalidWorkspace(&'static str), + /// The requested Community dashboard does not exist. + #[error("Community dashboard not found: {0}")] + CommunityDashboardNotFound(i64), + /// A Community dashboard field or page request violates the durable contract. + #[error("invalid Community dashboard: {0}")] + InvalidCommunityDashboard(&'static str), + /// The requested Community chart does not exist. + #[error("Community chart not found: {0}")] + CommunityChartNotFound(i64), + /// A Community chart field violates the durable contract. + #[error("invalid Community chart: {0}")] + InvalidCommunityChart(&'static str), + /// The requested transfer task does not exist. + #[error("transfer task not found: {0}")] + TransferTaskNotFound(i64), + /// The requested managed transfer artifact does not exist. + #[error("transfer artifact not found: {0}")] + TransferArtifactNotFound(String), + /// A transfer task or artifact violates the durable contract. + #[error("invalid transfer operation: {0}")] + InvalidTransfer(&'static str), /// The requested saved Console does not exist. #[error("saved Console not found: {0}")] SavedConsoleNotFound(i64), diff --git a/crates/chat2db-storage/src/lib.rs b/crates/chat2db-storage/src/lib.rs index 7f5f59f..3c30b48 100644 --- a/crates/chat2db-storage/src/lib.rs +++ b/crates/chat2db-storage/src/lib.rs @@ -1,14 +1,18 @@ //! Durable `Chat2DB` product state and retained query-result storage. mod agent; +mod community_dashboard; mod datasource; mod error; +mod mysql_workspace; mod operation_log; mod provider; mod result_store; mod saved_console; mod secret; +mod transfer; mod vault; +mod workspace; use std::{ collections::HashSet, @@ -33,6 +37,11 @@ pub use agent::{ SqlPermissionMode, StartAgentRun, StartedAgentRun, ToolPermissionDecision, ToolPermissionRecord, ToolPermissionStatus, UnknownAgentWrite, UpdateAgentSession, }; +pub use chat2db_contract::{ + CommunityChart, CommunityDashboard, CommunityDashboardListQuery, CommunityDashboardPage, + CreateCommunityChartRequest, CreateCommunityDashboardRequest, UpdateCommunityChartRequest, + UpdateCommunityDashboardRequest, +}; pub use datasource::{ CreateDatasource, DatasourceRecord, SecretChange, SecretCleanupReport, UpdateDatasource, }; @@ -52,14 +61,22 @@ pub use saved_console::{ UpdateSavedConsole, }; pub use secret::{SecretRef, SecretValue, SecretVault, SecretVaultError}; +pub use transfer::{ + CreateTransferTask, ResolvedTransferArtifact, StoredTransferTaskKind, StoredTransferTaskStatus, + TransferArtifactRecord, TransferArtifactWriter, TransferRecoveryReport, TransferTaskRecord, +}; pub use vault::EncryptedFileVault; #[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))] pub use vault::OsSecretVault; +pub use workspace::{ + WorkspaceNamespaceRecord, WorkspaceNodeKind, WorkspaceNodeLocator, WorkspaceNodeRecord, +}; const DATABASE_FILE: &str = "chat2db.sqlite3"; const LOCK_FILE: &str = ".chat2db.lock"; const RESULTS_DIRECTORY: &str = "results"; -const CURRENT_SCHEMA_VERSION: i64 = 4; +const ARTIFACTS_DIRECTORY: &str = "artifacts"; +const CURRENT_SCHEMA_VERSION: i64 = 8; #[cfg(test)] #[derive(Clone, Copy)] @@ -135,6 +152,8 @@ pub struct StartupReport { pub secrets: SecretCleanupReport, /// Agent runs, permissions, and result handles recovered at startup. pub agents: AgentRecoveryReport, + /// Interrupted transfer tasks and managed artifact cleanup. + pub transfers: TransferRecoveryReport, } impl Default for StorageOptions { @@ -155,6 +174,7 @@ pub(crate) struct StorageInner { data_dir: PathBuf, database_path: PathBuf, results_dir: PathBuf, + artifacts_dir: PathBuf, secret_gate: Mutex<()>, result_gate: Mutex>, max_retained_bytes: u64, @@ -254,11 +274,17 @@ impl Storage { fs::create_dir_all(&results_dir).map_err(|error| StorageError::io(&results_dir, error))?; secure_directory(&results_dir)?; + let artifacts_dir = data_dir.join(ARTIFACTS_DIRECTORY); + fs::create_dir_all(&artifacts_dir) + .map_err(|error| StorageError::io(&artifacts_dir, error))?; + secure_directory(&artifacts_dir)?; + let database_path = data_dir.join(DATABASE_FILE); let inner = Arc::new(StorageInner { data_dir, database_path, results_dir, + artifacts_dir, secret_gate: Mutex::new(()), result_gate: Mutex::new(HashSet::new()), max_retained_bytes: options.max_retained_bytes, @@ -287,6 +313,7 @@ impl Storage { let timestamp = now_millis()?; let recovery = storage.recover_at(timestamp)?; let agents = storage.recover_agents_at(timestamp)?; + let transfers = storage.recover_transfers_at(timestamp)?; let secrets = storage.reconcile_secrets()?; storage .inner @@ -295,6 +322,7 @@ impl Storage { results: recovery, secrets, agents, + transfers, }) .map_err(|_| { StorageError::Integrity("startup recovery initialized twice".to_owned()) @@ -394,6 +422,31 @@ fn migrate(connection: &Connection) -> Result<(), StorageError> { connection, include_str!("../migrations/004_operation_log.sql"), )?; + version = 4; + } + if version == 4 { + apply_migration( + connection, + include_str!("../migrations/005_workspace_namespace.sql"), + )?; + version = 5; + } + if version == 5 { + apply_migration(connection, include_str!("../migrations/006_transfer.sql"))?; + version = 6; + } + if version == 6 { + apply_migration( + connection, + include_str!("../migrations/007_mysql_workspace.sql"), + )?; + version = 7; + } + if version == 7 { + apply_migration( + connection, + include_str!("../migrations/008_community_dashboard.sql"), + )?; } Ok(()) } @@ -485,8 +538,8 @@ mod tests { use tempfile::TempDir; use super::{ - DATABASE_FILE, RecoveryReport, SecretRef, SecretValue, SecretVault, SecretVaultError, - Storage, + CURRENT_SCHEMA_VERSION, DATABASE_FILE, RecoveryReport, SecretRef, SecretValue, SecretVault, + SecretVaultError, Storage, }; use crate::StorageError; @@ -519,6 +572,28 @@ mod tests { Arc::new(TestVault) } + fn assert_current_schema_tables(connection: &Connection) { + for table in [ + "workspace_namespaces", + "workspace_nodes", + "transfer_tasks", + "transfer_artifacts", + "mysql_pinned_tables", + "mysql_er_positions", + "community_dashboards", + "community_charts", + ] { + let table_count: i64 = connection + .query_row( + "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = ?1", + [table], + |row| row.get(0), + ) + .expect("current schema table count reads"); + assert_eq!(table_count, 1, "current schema table {table} must exist"); + } + } + #[test] fn migration_and_required_pragmas_are_idempotent() { let directory = TempDir::new().expect("temp dir"); @@ -536,7 +611,8 @@ mod tests { let synchronous: i64 = connection .pragma_query_value(None, "synchronous", |row| row.get(0)) .expect("synchronous reads"); - assert_eq!(version, 4); + assert_eq!(version, CURRENT_SCHEMA_VERSION); + assert_current_schema_tables(&connection); assert_eq!(foreign_keys, 1); assert_eq!(journal_mode.to_ascii_lowercase(), "wal"); assert_eq!(synchronous, 2); @@ -545,6 +621,12 @@ mod tests { let reopened = Storage::open(directory.path(), vault()).expect("storage reopens"); assert_eq!(reopened.startup_report().results, RecoveryReport::default()); + let connection = reopened.connection().expect("reopened connection opens"); + let version: i64 = connection + .pragma_query_value(None, "user_version", |row| row.get(0)) + .expect("reopened schema version reads"); + assert_eq!(version, CURRENT_SCHEMA_VERSION); + assert_current_schema_tables(&connection); } #[test] @@ -553,24 +635,25 @@ mod tests { let storage = Storage::open(directory.path(), vault()).expect("storage opens"); drop(storage); let database = directory.path().join(DATABASE_FILE); + let unsupported_version = CURRENT_SCHEMA_VERSION + 1; Connection::open(&database) .expect("database opens") - .execute_batch("PRAGMA user_version = 5") + .pragma_update(None, "user_version", unsupported_version) .expect("test version updates"); let error = Storage::open(directory.path(), vault()).expect_err("newer schema must fail"); - assert!(matches!( - error, - StorageError::UnsupportedSchema { - found: 5, - supported: 4 + match error { + StorageError::UnsupportedSchema { found, supported } => { + assert_eq!(found, unsupported_version); + assert_eq!(supported, CURRENT_SCHEMA_VERSION); } - )); + other => panic!("unexpected newer schema error: {other}"), + } let version: i64 = Connection::open(database) .expect("database opens") .pragma_query_value(None, "user_version", |row| row.get(0)) .expect("schema version reads"); - assert_eq!(version, 5); + assert_eq!(version, unsupported_version); } #[test] @@ -604,8 +687,9 @@ mod tests { |row| row.get(0), ) .expect("provider table count reads"); - assert_eq!(version, 4); + assert_eq!(version, CURRENT_SCHEMA_VERSION); assert_eq!(provider_table, 1); + assert_current_schema_tables(&connection); assert!( storage .get_datasource("existing") @@ -648,8 +732,9 @@ mod tests { |row| row.get(0), ) .expect("saved Console table count reads"); - assert_eq!(version, 4); + assert_eq!(version, CURRENT_SCHEMA_VERSION); assert_eq!(saved_console_table, 1); + assert_current_schema_tables(&connection); assert!( storage .get_datasource("existing-v2") @@ -746,8 +831,9 @@ mod tests { |row| row.get(0), ) .expect("operation log table count reads"); - assert_eq!(version, 4); + assert_eq!(version, CURRENT_SCHEMA_VERSION); assert_eq!(operation_log_table, 1); + assert_current_schema_tables(&connection); assert!( storage .get_saved_console(42) diff --git a/crates/chat2db-storage/src/mysql_workspace.rs b/crates/chat2db-storage/src/mysql_workspace.rs new file mode 100644 index 0000000..4ca1de1 --- /dev/null +++ b/crates/chat2db-storage/src/mysql_workspace.rs @@ -0,0 +1,293 @@ +use rusqlite::{OptionalExtension, params}; + +use crate::{Storage, StorageError, now_millis}; + +const MAX_DATASOURCE_ID_BYTES: usize = 512; +const MAX_IDENTIFIER_BYTES: usize = 256; +const MAX_ER_POSITION_BYTES: usize = 16 * 1024 * 1024; + +impl Storage { + /// Persists one pinned `MySQL` table idempotently. + /// + /// # Errors + /// + /// Returns validation, datasource foreign-key, clock, or `SQLite` failures. + pub fn pin_mysql_table( + &self, + datasource_id: &str, + database_name: &str, + schema_name: &str, + table_name: &str, + ) -> Result<(), StorageError> { + validate_scope(datasource_id, database_name, schema_name)?; + validate_table_name(table_name)?; + let connection = self.connection()?; + connection.execute( + "INSERT INTO mysql_pinned_tables ( + datasource_id, database_name, schema_name, table_name, created_at_ms + ) VALUES (?1, ?2, ?3, ?4, ?5) + ON CONFLICT (datasource_id, database_name, schema_name, table_name) DO NOTHING", + params![ + datasource_id, + database_name, + schema_name, + table_name, + now_millis()? + ], + )?; + Ok(()) + } + + /// Removes one pinned `MySQL` table idempotently. + /// + /// # Errors + /// + /// Returns validation or `SQLite` failures. + pub fn unpin_mysql_table( + &self, + datasource_id: &str, + database_name: &str, + schema_name: &str, + table_name: &str, + ) -> Result<(), StorageError> { + validate_scope(datasource_id, database_name, schema_name)?; + validate_table_name(table_name)?; + let connection = self.connection()?; + connection.execute( + "DELETE FROM mysql_pinned_tables + WHERE datasource_id = ?1 AND database_name = ?2 + AND schema_name = ?3 AND table_name = ?4", + params![datasource_id, database_name, schema_name, table_name], + )?; + Ok(()) + } + + /// Lists pinned table names for one `MySQL` database/schema scope. + /// + /// # Errors + /// + /// Returns validation or `SQLite` failures. + pub fn list_mysql_pinned_tables( + &self, + datasource_id: &str, + database_name: &str, + schema_name: &str, + ) -> Result, StorageError> { + validate_scope(datasource_id, database_name, schema_name)?; + let connection = self.connection()?; + let mut statement = connection.prepare( + "SELECT table_name FROM mysql_pinned_tables + WHERE datasource_id = ?1 AND database_name = ?2 AND schema_name = ?3 + ORDER BY created_at_ms, table_name", + )?; + statement + .query_map(params![datasource_id, database_name, schema_name], |row| { + row.get(0) + })? + .collect::, _>>() + .map_err(Into::into) + } + + /// Reads the saved ER layout for one `MySQL` database/schema scope. + /// + /// # Errors + /// + /// Returns validation or `SQLite` failures. + pub fn mysql_er_position( + &self, + datasource_id: &str, + database_name: &str, + schema_name: &str, + ) -> Result, StorageError> { + validate_scope(datasource_id, database_name, schema_name)?; + let connection = self.connection()?; + connection + .query_row( + "SELECT position FROM mysql_er_positions + WHERE datasource_id = ?1 AND database_name = ?2 AND schema_name = ?3", + params![datasource_id, database_name, schema_name], + |row| row.get(0), + ) + .optional() + .map_err(Into::into) + } + + /// Inserts or replaces the saved ER layout for one `MySQL` database/schema scope. + /// + /// # Errors + /// + /// Returns validation, datasource foreign-key, clock, or `SQLite` failures. + pub fn save_mysql_er_position( + &self, + datasource_id: &str, + database_name: &str, + schema_name: &str, + position: &str, + ) -> Result<(), StorageError> { + validate_scope(datasource_id, database_name, schema_name)?; + if position.len() > MAX_ER_POSITION_BYTES { + return Err(StorageError::InvalidWorkspace( + "ER position exceeds the 16 MiB UTF-8 limit", + )); + } + let connection = self.connection()?; + connection.execute( + "INSERT INTO mysql_er_positions ( + datasource_id, database_name, schema_name, position, updated_at_ms + ) VALUES (?1, ?2, ?3, ?4, ?5) + ON CONFLICT (datasource_id, database_name, schema_name) DO UPDATE SET + position = excluded.position, + updated_at_ms = excluded.updated_at_ms", + params![ + datasource_id, + database_name, + schema_name, + position, + now_millis()? + ], + )?; + Ok(()) + } +} + +fn validate_scope( + datasource_id: &str, + database_name: &str, + schema_name: &str, +) -> Result<(), StorageError> { + if datasource_id.trim().is_empty() || datasource_id.len() > MAX_DATASOURCE_ID_BYTES { + return Err(StorageError::InvalidWorkspace( + "datasource id must be non-empty and at most 512 UTF-8 bytes", + )); + } + validate_identifier(database_name, "database name")?; + validate_identifier(schema_name, "schema name") +} + +fn validate_table_name(table_name: &str) -> Result<(), StorageError> { + if table_name.is_empty() || table_name.len() > MAX_IDENTIFIER_BYTES || table_name.contains('\0') + { + return Err(StorageError::InvalidWorkspace( + "table name must be non-empty, NUL-free, and at most 256 UTF-8 bytes", + )); + } + Ok(()) +} + +fn validate_identifier(value: &str, field: &'static str) -> Result<(), StorageError> { + if value.len() > MAX_IDENTIFIER_BYTES || value.contains('\0') { + return Err(StorageError::InvalidWorkspace(match field { + "database name" => "database name must be NUL-free and at most 256 UTF-8 bytes", + _ => "schema name must be NUL-free and at most 256 UTF-8 bytes", + })); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use tempfile::TempDir; + + use crate::{CreateDatasource, SecretRef, SecretValue, SecretVault, SecretVaultError, Storage}; + + #[derive(Debug)] + struct EmptyVault; + + impl SecretVault for EmptyVault { + fn probe(&self) -> Result<(), SecretVaultError> { + Ok(()) + } + + fn create( + &self, + _reference: &SecretRef, + _value: &SecretValue, + ) -> Result<(), SecretVaultError> { + Ok(()) + } + + fn get(&self, _reference: &SecretRef) -> Result, SecretVaultError> { + Ok(None) + } + + fn delete(&self, _reference: &SecretRef) -> Result<(), SecretVaultError> { + Ok(()) + } + } + + fn open(directory: &TempDir) -> Storage { + Storage::open(directory.path(), Arc::new(EmptyVault)).expect("storage opens") + } + + fn datasource(storage: &Storage) -> String { + storage + .create_datasource( + CreateDatasource { + name: "Local MySQL".to_owned(), + driver_id: "mysql".to_owned(), + }, + None, + ) + .expect("datasource creates") + .id + } + + #[test] + fn pins_are_idempotent_scoped_and_restart_safe() { + let directory = TempDir::new().expect("temp dir"); + let storage = open(&directory); + let datasource_id = datasource(&storage); + storage + .pin_mysql_table(&datasource_id, "shop", "", "orders") + .expect("pin creates"); + storage + .pin_mysql_table(&datasource_id, "shop", "", "orders") + .expect("duplicate pin is idempotent"); + storage + .pin_mysql_table(&datasource_id, "shop", "", "users") + .expect("second pin creates"); + drop(storage); + + let reopened = open(&directory); + assert_eq!( + reopened + .list_mysql_pinned_tables(&datasource_id, "shop", "") + .expect("pins list"), + vec!["orders", "users"] + ); + reopened + .unpin_mysql_table(&datasource_id, "shop", "", "orders") + .expect("pin deletes"); + assert_eq!( + reopened + .list_mysql_pinned_tables(&datasource_id, "shop", "") + .expect("pins relist"), + vec!["users"] + ); + } + + #[test] + fn er_position_upsert_replaces_the_existing_layout_and_survives_restart() { + let directory = TempDir::new().expect("temp dir"); + let storage = open(&directory); + let datasource_id = datasource(&storage); + storage + .save_mysql_er_position(&datasource_id, "shop", "", r#"{"version":1}"#) + .expect("first layout saves"); + storage + .save_mysql_er_position(&datasource_id, "shop", "", r#"{"version":2}"#) + .expect("second layout replaces first"); + drop(storage); + + let reopened = open(&directory); + assert_eq!( + reopened + .mysql_er_position(&datasource_id, "shop", "") + .expect("layout reads") + .as_deref(), + Some(r#"{"version":2}"#) + ); + } +} diff --git a/crates/chat2db-storage/src/transfer.rs b/crates/chat2db-storage/src/transfer.rs new file mode 100644 index 0000000..bdaa271 --- /dev/null +++ b/crates/chat2db-storage/src/transfer.rs @@ -0,0 +1,1545 @@ +use std::{ + collections::HashSet, + fs::{self, File, OpenOptions}, + io::{Read, Seek, SeekFrom, Write}, + path::{Path, PathBuf}, +}; + +use rusqlite::{OptionalExtension, TransactionBehavior, params}; +use sha2::{Digest, Sha256}; +use uuid::Uuid; + +use crate::{Storage, StorageError, now_millis, secure_file, sync_directory}; + +const MAX_RETAINED_TASKS: usize = 20; +const MAX_TASK_NAME_BYTES: usize = 1_024; +const MAX_SCOPE_BYTES: usize = 1_024; +const MAX_LOG_BYTES: usize = 256 * 1024; +const MAX_LOG_BYTES_I64: i64 = 256 * 1024; +const MAX_FILE_NAME_BYTES: usize = 1_024; +const MAX_MEDIA_TYPE_BYTES: usize = 255; + +/// Durable transfer category. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StoredTransferTaskKind { + ImportFile, + ExportSql, + ExportFile, +} + +impl StoredTransferTaskKind { + const fn as_str(self) -> &'static str { + match self { + Self::ImportFile => "import_file", + Self::ExportSql => "export_sql", + Self::ExportFile => "export_file", + } + } + + fn parse(value: &str) -> Result { + match value { + "import_file" => Ok(Self::ImportFile), + "export_sql" => Ok(Self::ExportSql), + "export_file" => Ok(Self::ExportFile), + _ => Err(StorageError::Integrity( + "transfer task has an invalid kind".to_owned(), + )), + } + } +} + +/// Durable transfer lifecycle state. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StoredTransferTaskStatus { + Queued, + Running, + Succeeded, + Failed, + Cancelled, + Interrupted, +} + +impl StoredTransferTaskStatus { + const fn as_str(self) -> &'static str { + match self { + Self::Queued => "queued", + Self::Running => "running", + Self::Succeeded => "succeeded", + Self::Failed => "failed", + Self::Cancelled => "cancelled", + Self::Interrupted => "interrupted", + } + } + + fn parse(value: &str) -> Result { + match value { + "queued" => Ok(Self::Queued), + "running" => Ok(Self::Running), + "succeeded" => Ok(Self::Succeeded), + "failed" => Ok(Self::Failed), + "cancelled" => Ok(Self::Cancelled), + "interrupted" => Ok(Self::Interrupted), + _ => Err(StorageError::Integrity( + "transfer task has an invalid status".to_owned(), + )), + } + } + + const fn is_terminal(self) -> bool { + matches!( + self, + Self::Succeeded | Self::Failed | Self::Cancelled | Self::Interrupted + ) + } +} + +/// Input for one durable task. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CreateTransferTask { + pub datasource_id: String, + pub database_name: String, + pub schema_name: String, + pub table_name: Option, + pub kind: StoredTransferTaskKind, + pub task_name: String, +} + +/// Durable transfer task record. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TransferTaskRecord { + pub id: i64, + pub datasource_id: String, + pub database_name: String, + pub schema_name: String, + pub table_name: Option, + pub kind: StoredTransferTaskKind, + pub status: StoredTransferTaskStatus, + pub task_name: String, + pub progress_current: u64, + pub progress_total: Option, + pub progress_description: String, + pub info_log: String, + pub error_log: String, + pub cancel_requested: bool, + pub created_at_ms: i64, + pub updated_at_ms: i64, + pub finished_at_ms: Option, + pub artifact_id: Option, +} + +/// Durable artifact metadata without a filesystem path. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TransferArtifactRecord { + pub id: String, + pub task_id: Option, + pub file_name: String, + pub media_type: String, + pub format: String, + pub byte_count: u64, + pub sha256: [u8; 32], + pub created_at_ms: i64, + pub expires_at_ms: Option, +} + +/// Owner-only path resolved from a durable artifact record. +#[derive(Debug)] +pub struct ResolvedTransferArtifact { + pub record: TransferArtifactRecord, + pub path: PathBuf, + pub file: File, +} + +/// Startup cleanup and task recovery report. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct TransferRecoveryReport { + pub interrupted_tasks: usize, + pub expired_artifacts: usize, + pub partial_files_removed: usize, + pub orphan_files_removed: usize, +} + +/// Poison-on-failure writer for one managed transfer artifact. +pub struct TransferArtifactWriter { + storage: Storage, + file: Option, + id: String, + task_id: Option, + part_path: PathBuf, + final_path: PathBuf, + storage_name: String, + file_name: String, + media_type: String, + format: String, + expires_at_ms: Option, + cleanup_on_drop: bool, +} + +impl std::fmt::Debug for TransferArtifactWriter { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("TransferArtifactWriter") + .field("id", &self.id) + .field("task_id", &self.task_id) + .finish_non_exhaustive() + } +} + +impl Write for TransferArtifactWriter { + fn write(&mut self, buffer: &[u8]) -> std::io::Result { + self.file_mut().write(buffer) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.file_mut().flush() + } +} + +impl Drop for TransferArtifactWriter { + fn drop(&mut self) { + self.file.take(); + if self.cleanup_on_drop { + let _ = fs::remove_file(&self.part_path); + let _ = sync_directory(&self.storage.inner.artifacts_dir); + } + } +} + +impl TransferArtifactWriter { + /// Returns the private partial path for format writers requiring seekable files. + #[must_use] + pub fn path(&self) -> &Path { + &self.part_path + } + + /// Returns the seekable partial artifact file. + /// + /// # Panics + /// + /// Panics only when called after the writer has already been consumed by [`Self::finish`]. + pub fn file_mut(&mut self) -> &mut File { + self.file + .as_mut() + .expect("artifact file remains open until finish") + } + + /// Atomically publishes the file and its metadata. For task artifacts this + /// also transitions the task to `succeeded` in the same `SQLite` transaction. + /// + /// # Errors + /// + /// Returns filesystem, integrity, task-state, clock, or `SQLite` failures. + pub fn finish(mut self) -> Result { + let mut file = self.file.take().ok_or(StorageError::InvalidTransfer( + "artifact writer is already closed", + ))?; + file.flush() + .map_err(|error| StorageError::io(&self.part_path, error))?; + file.sync_all() + .map_err(|error| StorageError::io(&self.part_path, error))?; + let byte_count = file + .metadata() + .map_err(|error| StorageError::io(&self.part_path, error))? + .len(); + file.seek(SeekFrom::Start(0)) + .map_err(|error| StorageError::io(&self.part_path, error))?; + let mut hasher = Sha256::new(); + let mut buffer = vec![0_u8; 64 * 1024].into_boxed_slice(); + loop { + let read = file + .read(&mut buffer) + .map_err(|error| StorageError::io(&self.part_path, error))?; + if read == 0 { + break; + } + hasher.update(&buffer[..read]); + } + let sha256: [u8; 32] = hasher.finalize().into(); + drop(file); + + fs::rename(&self.part_path, &self.final_path) + .map_err(|error| StorageError::io(&self.final_path, error))?; + sync_directory(&self.storage.inner.artifacts_dir)?; + + let created_at_ms = now_millis()?; + let result = (|| { + let mut connection = self.storage.connection()?; + let transaction = + connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + transaction.execute( + "INSERT INTO transfer_artifacts ( + id, task_id, storage_name, file_name, media_type, format, + byte_count, sha256, created_at_ms, expires_at_ms + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)", + params![ + self.id, + self.task_id, + self.storage_name, + self.file_name, + self.media_type, + self.format, + i64::try_from(byte_count) + .map_err(|_| StorageError::NumericRange("artifact byte count"))?, + sha256.as_slice(), + created_at_ms, + self.expires_at_ms, + ], + )?; + if let Some(task_id) = self.task_id { + let updated = transaction.execute( + "UPDATE transfer_tasks + SET status = 'succeeded', progress_description = 'Completed', + cancel_requested = 0, updated_at_ms = ?2, finished_at_ms = ?2 + WHERE id = ?1 AND status IN ('queued', 'running') AND cancel_requested = 0", + params![task_id, created_at_ms], + )?; + if updated != 1 { + return Err(StorageError::InvalidTransfer( + "task cannot publish an artifact in its current state", + )); + } + } + transaction.commit()?; + Ok(TransferArtifactRecord { + id: self.id.clone(), + task_id: self.task_id, + file_name: self.file_name.clone(), + media_type: self.media_type.clone(), + format: self.format.clone(), + byte_count, + sha256, + created_at_ms, + expires_at_ms: self.expires_at_ms, + }) + })(); + + if result.is_err() { + let _ = fs::remove_file(&self.final_path); + let _ = sync_directory(&self.storage.inner.artifacts_dir); + } else { + self.cleanup_on_drop = false; + if self.task_id.is_some() { + let _ = self.storage.prune_transfer_tasks(); + } + } + result + } +} + +impl Storage { + /// Creates one queued task and performs recoverable best-effort retention cleanup. + /// + /// # Errors + /// + /// Returns validation, clock, or pre-commit `SQLite` failures. + pub fn create_transfer_task( + &self, + input: &CreateTransferTask, + ) -> Result { + validate_create_task(input)?; + let timestamp = now_millis()?; + let mut connection = self.connection()?; + let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + transaction.execute( + "INSERT INTO transfer_tasks ( + datasource_id, database_name, schema_name, table_name, kind, status, + task_name, created_at_ms, updated_at_ms + ) VALUES (?1, ?2, ?3, ?4, ?5, 'queued', ?6, ?7, ?7)", + params![ + input.datasource_id, + input.database_name, + input.schema_name, + input.table_name, + input.kind.as_str(), + input.task_name, + timestamp, + ], + )?; + let id = transaction.last_insert_rowid(); + transaction.commit()?; + let record = TransferTaskRecord { + id, + datasource_id: input.datasource_id.clone(), + database_name: input.database_name.clone(), + schema_name: input.schema_name.clone(), + table_name: input.table_name.clone(), + kind: input.kind, + status: StoredTransferTaskStatus::Queued, + task_name: input.task_name.clone(), + progress_current: 0, + progress_total: None, + progress_description: String::new(), + info_log: String::new(), + error_log: String::new(), + cancel_requested: false, + created_at_ms: timestamp, + updated_at_ms: timestamp, + finished_at_ms: None, + artifact_id: None, + }; + + // The queued task is durable after commit. Retention cleanup is recoverable + // maintenance and must not make Core believe that no worker should start. + let _ = self.prune_transfer_tasks(); + Ok(record) + } + + /// Lists at most the 20 retained tasks, newest first. + /// + /// # Errors + /// + /// Returns `SQLite` or durable-record decoding failures. + pub fn list_transfer_tasks(&self) -> Result, StorageError> { + let connection = self.connection()?; + let mut statement = connection.prepare( + "SELECT t.id, t.datasource_id, t.database_name, t.schema_name, t.table_name, + t.kind, t.status, t.task_name, t.progress_current, t.progress_total, + t.progress_description, t.info_log, t.error_log, t.cancel_requested, + t.created_at_ms, t.updated_at_ms, t.finished_at_ms, a.id + FROM transfer_tasks t + LEFT JOIN transfer_artifacts a ON a.task_id = t.id + ORDER BY t.created_at_ms DESC, t.id DESC + LIMIT 20", + )?; + let rows = statement.query_map([], raw_task)?; + rows.map(|row| row.map_err(StorageError::from)).collect() + } + + /// Gets one task and its optional artifact id. + /// + /// # Errors + /// + /// Returns `SQLite` or durable-record decoding failures. + pub fn get_transfer_task(&self, id: i64) -> Result, StorageError> { + let connection = self.connection()?; + connection + .query_row( + "SELECT t.id, t.datasource_id, t.database_name, t.schema_name, t.table_name, + t.kind, t.status, t.task_name, t.progress_current, t.progress_total, + t.progress_description, t.info_log, t.error_log, t.cancel_requested, + t.created_at_ms, t.updated_at_ms, t.finished_at_ms, a.id + FROM transfer_tasks t + LEFT JOIN transfer_artifacts a ON a.task_id = t.id + WHERE t.id = ?1", + [id], + raw_task, + ) + .optional() + .map_err(StorageError::from) + } + + /// Transitions a queued task to running. + /// + /// # Errors + /// + /// Returns not-found, invalid-state, clock, or `SQLite` failures. + pub fn start_transfer_task(&self, id: i64) -> Result<(), StorageError> { + let timestamp = now_millis()?; + let connection = self.connection()?; + let updated = connection.execute( + "UPDATE transfer_tasks + SET status = 'running', progress_description = 'Running', updated_at_ms = ?2 + WHERE id = ?1 AND status = 'queued' AND cancel_requested = 0", + params![id, timestamp], + )?; + if updated == 1 { + return Ok(()); + } + ensure_task_exists(&connection, id)?; + Err(StorageError::InvalidTransfer( + "task cannot start in its current state", + )) + } + + /// Persists bounded, monotonic task progress and appends a bounded log line. + /// + /// # Errors + /// + /// Returns validation, numeric-range, not-found, invalid-state, clock, or `SQLite` failures. + pub fn update_transfer_progress( + &self, + id: i64, + current: u64, + total: Option, + description: &str, + info: Option<&str>, + ) -> Result<(), StorageError> { + validate_text(description, MAX_TASK_NAME_BYTES, "progress description")?; + if let Some(info) = info { + validate_text(info, MAX_LOG_BYTES, "task info log entry")?; + } + let timestamp = now_millis()?; + let current = + i64::try_from(current).map_err(|_| StorageError::NumericRange("transfer progress"))?; + let total = total + .map(|value| { + i64::try_from(value) + .map_err(|_| StorageError::NumericRange("transfer progress total")) + }) + .transpose()?; + let connection = self.connection()?; + let updated = connection.execute( + "UPDATE transfer_tasks + SET progress_current = MAX(progress_current, ?2), + progress_total = COALESCE(?3, progress_total), + progress_description = ?4, + info_log = CASE WHEN ?5 IS NULL THEN info_log + ELSE substr(info_log || CASE WHEN info_log = '' THEN '' ELSE char(10) END || ?5, ?6) + END, + updated_at_ms = ?7 + WHERE id = ?1 AND status = 'running'", + params![ + id, + current, + total, + description, + info, + -MAX_LOG_BYTES_I64, + timestamp, + ], + )?; + if updated == 1 { + return Ok(()); + } + ensure_task_exists(&connection, id)?; + Err(StorageError::InvalidTransfer( + "task progress can only update while running", + )) + } + + /// Requests cooperative cancellation. Queued tasks become cancelled immediately. + /// + /// # Errors + /// + /// Returns not-found, clock, or `SQLite` failures. + pub fn request_transfer_cancel(&self, id: i64) -> Result { + let timestamp = now_millis()?; + let connection = self.connection()?; + let changed = connection.execute( + "UPDATE transfer_tasks + SET cancel_requested = 1, + status = CASE WHEN status = 'queued' THEN 'cancelled' ELSE status END, + progress_description = CASE WHEN status = 'queued' THEN 'Cancelled' ELSE progress_description END, + finished_at_ms = CASE WHEN status = 'queued' THEN ?2 ELSE finished_at_ms END, + updated_at_ms = ?2 + WHERE id = ?1 AND status IN ('queued', 'running') AND cancel_requested = 0", + params![id, timestamp], + )?; + if changed == 1 { + return Ok(true); + } + ensure_task_exists(&connection, id)?; + Ok(false) + } + + /// Reports whether cancellation was durably requested. + /// + /// # Errors + /// + /// Returns not-found or `SQLite` failures. + pub fn transfer_cancel_requested(&self, id: i64) -> Result { + let connection = self.connection()?; + connection + .query_row( + "SELECT cancel_requested FROM transfer_tasks WHERE id = ?1", + [id], + |row| row.get::<_, bool>(0), + ) + .optional()? + .ok_or(StorageError::TransferTaskNotFound(id)) + } + + /// Marks a running task cancelled after cooperative cleanup completes. + /// + /// # Errors + /// + /// Returns validation, not-found, invalid-state, clock, or pre-commit `SQLite` failures. + pub fn cancel_transfer_task(&self, id: i64, message: &str) -> Result<(), StorageError> { + self.finish_transfer_without_artifact( + id, + StoredTransferTaskStatus::Cancelled, + "Cancelled", + message, + ) + } + + /// Marks a queued or running task failed. + /// + /// # Errors + /// + /// Returns validation, not-found, invalid-state, clock, or pre-commit `SQLite` failures. + pub fn fail_transfer_task(&self, id: i64, message: &str) -> Result<(), StorageError> { + self.finish_transfer_without_artifact( + id, + StoredTransferTaskStatus::Failed, + "Failed", + message, + ) + } + + /// Marks a running task complete when the operation does not produce an artifact. + /// + /// # Errors + /// + /// Returns validation, not-found, invalid-state, clock, or pre-commit `SQLite` failures. + pub fn complete_transfer_task(&self, id: i64, message: &str) -> Result<(), StorageError> { + validate_text(message, MAX_LOG_BYTES, "task completion message")?; + let timestamp = now_millis()?; + let connection = self.connection()?; + let updated = connection.execute( + "UPDATE transfer_tasks + SET status = 'succeeded', progress_description = 'Completed', + info_log = CASE WHEN ?2 = '' THEN info_log + ELSE substr(info_log || CASE WHEN info_log = '' THEN '' ELSE char(10) END || ?2, ?3) + END, + cancel_requested = 0, updated_at_ms = ?4, finished_at_ms = ?4 + WHERE id = ?1 AND status = 'running' AND cancel_requested = 0", + params![ + id, + message, + -MAX_LOG_BYTES_I64, + timestamp, + ], + )?; + if updated != 1 { + ensure_task_exists(&connection, id)?; + return Err(StorageError::InvalidTransfer( + "task cannot complete in its current state", + )); + } + // The terminal state is already durable. Retention is recoverable + // maintenance and must not make callers retry a committed transition. + let _ = self.prune_transfer_tasks(); + Ok(()) + } + + fn finish_transfer_without_artifact( + &self, + id: i64, + status: StoredTransferTaskStatus, + description: &str, + message: &str, + ) -> Result<(), StorageError> { + if !status.is_terminal() || status == StoredTransferTaskStatus::Succeeded { + return Err(StorageError::InvalidTransfer("invalid terminal task state")); + } + validate_text(message, MAX_LOG_BYTES, "task terminal message")?; + let timestamp = now_millis()?; + let connection = self.connection()?; + let updated = connection.execute( + "UPDATE transfer_tasks + SET status = ?2, progress_description = ?3, error_log = ?4, + updated_at_ms = ?5, finished_at_ms = ?5 + WHERE id = ?1 AND status IN ('queued', 'running')", + params![id, status.as_str(), description, message, timestamp], + )?; + if updated != 1 { + ensure_task_exists(&connection, id)?; + return Err(StorageError::InvalidTransfer( + "task is already in a terminal state", + )); + } + // The terminal state is already durable. Retention is recoverable + // maintenance and must not make callers retry a committed transition. + let _ = self.prune_transfer_tasks(); + Ok(()) + } + + /// Begins a private managed artifact. Dropping the writer removes its `.part` file. + /// + /// # Errors + /// + /// Returns validation, not-found, invalid-state, filesystem, or `SQLite` failures. + pub fn begin_transfer_artifact( + &self, + task_id: Option, + file_name: &str, + media_type: &str, + format: &str, + extension: &str, + expires_at_ms: Option, + ) -> Result { + validate_artifact_fields(file_name, media_type, format, extension)?; + if let Some(task_id) = task_id { + let task = self + .get_transfer_task(task_id)? + .ok_or(StorageError::TransferTaskNotFound(task_id))?; + if !matches!( + task.status, + StoredTransferTaskStatus::Queued | StoredTransferTaskStatus::Running + ) || task.cancel_requested + { + return Err(StorageError::InvalidTransfer( + "task cannot create an artifact in its current state", + )); + } + } + let id = Uuid::new_v4().to_string(); + let storage_name = format!("{id}.{extension}"); + let part_path = self.inner.artifacts_dir.join(format!("{id}.part")); + let final_path = self.inner.artifacts_dir.join(&storage_name); + let file = OpenOptions::new() + .create_new(true) + .read(true) + .write(true) + .open(&part_path) + .map_err(|error| StorageError::io(&part_path, error))?; + secure_file(&part_path)?; + Ok(TransferArtifactWriter { + storage: self.clone(), + file: Some(file), + id, + task_id, + part_path, + final_path, + storage_name, + file_name: file_name.to_owned(), + media_type: media_type.to_owned(), + format: format.to_owned(), + expires_at_ms, + cleanup_on_drop: true, + }) + } + + /// Resolves an artifact to an owner-only regular file under the managed directory. + /// + /// # Errors + /// + /// Returns not-found, expiry, integrity, filesystem, clock, or `SQLite` failures. + pub fn resolve_transfer_artifact( + &self, + id: &str, + ) -> Result { + let connection = self.connection()?; + let stored = connection + .query_row( + "SELECT id, task_id, storage_name, file_name, media_type, format, + byte_count, sha256, created_at_ms, expires_at_ms + FROM transfer_artifacts WHERE id = ?1", + [id], + raw_artifact_with_storage_name, + ) + .optional()? + .ok_or_else(|| StorageError::TransferArtifactNotFound(id.to_owned()))?; + if stored + .record + .expires_at_ms + .is_some_and(|expiry| expiry <= now_millis().unwrap_or(i64::MAX)) + { + return Err(StorageError::TransferArtifactNotFound(id.to_owned())); + } + validate_storage_name(&stored.storage_name)?; + let path = self.inner.artifacts_dir.join(stored.storage_name); + let metadata = + fs::symlink_metadata(&path).map_err(|error| StorageError::io(&path, error))?; + if !metadata.file_type().is_file() || metadata.file_type().is_symlink() { + return Err(StorageError::Integrity( + "transfer artifact is not a regular managed file".to_owned(), + )); + } + let canonical_directory = fs::canonicalize(&self.inner.artifacts_dir) + .map_err(|error| StorageError::io(&self.inner.artifacts_dir, error))?; + let canonical_path = + fs::canonicalize(&path).map_err(|error| StorageError::io(&path, error))?; + if !canonical_path.starts_with(canonical_directory) { + return Err(StorageError::Integrity( + "transfer artifact escaped the managed directory".to_owned(), + )); + } + let file = open_verified_transfer_artifact(&canonical_path, &stored.record)?; + Ok(ResolvedTransferArtifact { + record: stored.record, + path: canonical_path, + file, + }) + } + + /// Deletes an unmanaged temporary artifact and its durable metadata. + /// + /// Task-owned output artifacts are deliberately excluded so a delivery + /// adapter cannot remove a retained download by mistake. + /// + /// # Errors + /// + /// Returns integrity, filesystem, or `SQLite` failures. + pub fn delete_temporary_transfer_artifact(&self, id: &str) -> Result { + let mut connection = self.connection()?; + let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + let storage_name = transaction + .query_row( + "SELECT storage_name FROM transfer_artifacts + WHERE id = ?1 AND task_id IS NULL", + [id], + |row| row.get::<_, String>(0), + ) + .optional()?; + let Some(storage_name) = storage_name else { + transaction.commit()?; + return Ok(false); + }; + validate_storage_name(&storage_name)?; + let path = self.inner.artifacts_dir.join(storage_name); + match fs::remove_file(&path) { + Ok(()) => sync_directory(&self.inner.artifacts_dir)?, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(StorageError::io(path, error)), + } + transaction.execute( + "DELETE FROM transfer_artifacts WHERE id = ?1 AND task_id IS NULL", + [id], + )?; + transaction.commit()?; + Ok(true) + } + + pub(crate) fn recover_transfers_at( + &self, + timestamp_ms: i64, + ) -> Result { + let mut connection = self.connection()?; + let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + let interrupted_tasks = transaction.execute( + "UPDATE transfer_tasks + SET status = 'interrupted', progress_description = 'Interrupted', + error_log = CASE WHEN error_log = '' THEN 'Application stopped before the task completed' + ELSE error_log END, + updated_at_ms = ?1, finished_at_ms = ?1 + WHERE status IN ('queued', 'running')", + [timestamp_ms], + )?; + let expired_names = { + let mut statement = transaction.prepare( + "SELECT storage_name FROM transfer_artifacts + WHERE task_id IS NULL AND expires_at_ms IS NOT NULL AND expires_at_ms <= ?1", + )?; + let rows = statement.query_map([timestamp_ms], |row| row.get::<_, String>(0))?; + rows.collect::, _>>()? + }; + let expired_artifacts = transaction.execute( + "DELETE FROM transfer_artifacts + WHERE task_id IS NULL AND expires_at_ms IS NOT NULL AND expires_at_ms <= ?1", + [timestamp_ms], + )?; + let known_names = { + let mut statement = + transaction.prepare("SELECT storage_name FROM transfer_artifacts")?; + let rows = statement.query_map([], |row| row.get::<_, String>(0))?; + rows.collect::, _>>()? + }; + transaction.commit()?; + + for name in expired_names { + if validate_storage_name(&name).is_ok() { + let _ = fs::remove_file(self.inner.artifacts_dir.join(name)); + } + } + let mut partial_files_removed = 0; + let mut orphan_files_removed = 0; + for entry in fs::read_dir(&self.inner.artifacts_dir) + .map_err(|error| StorageError::io(&self.inner.artifacts_dir, error))? + { + let entry = + entry.map_err(|error| StorageError::io(&self.inner.artifacts_dir, error))?; + let file_type = entry + .file_type() + .map_err(|error| StorageError::io(entry.path(), error))?; + if !file_type.is_file() || file_type.is_symlink() { + continue; + } + let Some(name) = entry.file_name().to_str().map(str::to_owned) else { + continue; + }; + let remove = if Path::new(&name) + .extension() + .is_some_and(|extension| extension.eq_ignore_ascii_case("part")) + { + partial_files_removed += 1; + true + } else if !known_names.contains(&name) { + orphan_files_removed += 1; + true + } else { + false + }; + if remove { + fs::remove_file(entry.path()) + .map_err(|error| StorageError::io(entry.path(), error))?; + } + } + sync_directory(&self.inner.artifacts_dir)?; + self.prune_transfer_tasks()?; + Ok(TransferRecoveryReport { + interrupted_tasks, + expired_artifacts, + partial_files_removed, + orphan_files_removed, + }) + } + + fn prune_transfer_tasks(&self) -> Result<(), StorageError> { + let mut connection = self.connection()?; + let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + let candidates = { + let mut statement = transaction.prepare( + "SELECT id, status FROM transfer_tasks + ORDER BY created_at_ms DESC, id DESC LIMIT -1 OFFSET ?1", + )?; + let rows = statement.query_map( + [i64::try_from(MAX_RETAINED_TASKS).expect("task bound fits i64")], + |row| Ok((row.get::<_, i64>(0)?, row.get::<_, String>(1)?)), + )?; + rows.collect::, _>>()? + .into_iter() + .filter_map(|(id, status)| { + StoredTransferTaskStatus::parse(&status) + .ok() + .filter(|status| status.is_terminal()) + .map(|_| id) + }) + .collect::>() + }; + if candidates.is_empty() { + transaction.commit()?; + return Ok(()); + } + let mut storage_names = Vec::new(); + for id in &candidates { + if let Some(name) = transaction + .query_row( + "SELECT storage_name FROM transfer_artifacts WHERE task_id = ?1", + [id], + |row| row.get::<_, String>(0), + ) + .optional()? + { + storage_names.push(name); + } + transaction.execute("DELETE FROM transfer_tasks WHERE id = ?1", [id])?; + } + transaction.commit()?; + for name in storage_names { + validate_storage_name(&name)?; + let path = self.inner.artifacts_dir.join(name); + match fs::remove_file(&path) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(StorageError::io(path, error)), + } + } + sync_directory(&self.inner.artifacts_dir) + } +} + +struct StoredArtifactRow { + record: TransferArtifactRecord, + storage_name: String, +} + +fn open_verified_transfer_artifact( + path: &Path, + record: &TransferArtifactRecord, +) -> Result { + let mut file = File::open(path).map_err(|error| StorageError::io(path, error))?; + let metadata = file + .metadata() + .map_err(|error| StorageError::io(path, error))?; + if metadata.len() != record.byte_count { + return Err(StorageError::Integrity( + "transfer artifact size does not match durable metadata".to_owned(), + )); + } + + let mut byte_count = 0_u64; + let mut hasher = Sha256::new(); + let mut buffer = vec![0_u8; 64 * 1024].into_boxed_slice(); + loop { + let read = file + .read(&mut buffer) + .map_err(|error| StorageError::io(path, error))?; + if read == 0 { + break; + } + byte_count = byte_count + .checked_add(u64::try_from(read).expect("read buffer length fits u64")) + .ok_or(StorageError::NumericRange("artifact byte count"))?; + hasher.update(&buffer[..read]); + } + if byte_count != record.byte_count { + return Err(StorageError::Integrity( + "transfer artifact size changed while it was verified".to_owned(), + )); + } + let sha256: [u8; 32] = hasher.finalize().into(); + if sha256 != record.sha256 { + return Err(StorageError::Integrity( + "transfer artifact SHA-256 does not match durable metadata".to_owned(), + )); + } + file.seek(SeekFrom::Start(0)) + .map_err(|error| StorageError::io(path, error))?; + Ok(file) +} + +fn raw_task(row: &rusqlite::Row<'_>) -> rusqlite::Result { + let kind: String = row.get(5)?; + let status: String = row.get(6)?; + let progress_current: i64 = row.get(8)?; + let progress_total: Option = row.get(9)?; + Ok(TransferTaskRecord { + id: row.get(0)?, + datasource_id: row.get(1)?, + database_name: row.get(2)?, + schema_name: row.get(3)?, + table_name: row.get(4)?, + kind: StoredTransferTaskKind::parse(&kind).map_err(storage_decode_error)?, + status: StoredTransferTaskStatus::parse(&status).map_err(storage_decode_error)?, + task_name: row.get(7)?, + progress_current: u64::try_from(progress_current) + .map_err(|_| rusqlite::Error::IntegralValueOutOfRange(8, progress_current))?, + progress_total: progress_total + .map(|value| { + u64::try_from(value).map_err(|_| rusqlite::Error::IntegralValueOutOfRange(9, value)) + }) + .transpose()?, + progress_description: row.get(10)?, + info_log: row.get(11)?, + error_log: row.get(12)?, + cancel_requested: row.get(13)?, + created_at_ms: row.get(14)?, + updated_at_ms: row.get(15)?, + finished_at_ms: row.get(16)?, + artifact_id: row.get(17)?, + }) +} + +fn raw_artifact_with_storage_name(row: &rusqlite::Row<'_>) -> rusqlite::Result { + let digest: Vec = row.get(7)?; + let sha256: [u8; 32] = digest.try_into().map_err(|_| { + rusqlite::Error::FromSqlConversionFailure( + 32, + rusqlite::types::Type::Blob, + "artifact digest is not SHA-256".into(), + ) + })?; + let byte_count: i64 = row.get(6)?; + Ok(StoredArtifactRow { + record: TransferArtifactRecord { + id: row.get(0)?, + task_id: row.get(1)?, + file_name: row.get(3)?, + media_type: row.get(4)?, + format: row.get(5)?, + byte_count: u64::try_from(byte_count) + .map_err(|_| rusqlite::Error::IntegralValueOutOfRange(6, byte_count))?, + sha256, + created_at_ms: row.get(8)?, + expires_at_ms: row.get(9)?, + }, + storage_name: row.get(2)?, + }) +} + +fn validate_create_task(input: &CreateTransferTask) -> Result<(), StorageError> { + validate_nonempty_text(&input.datasource_id, MAX_SCOPE_BYTES, "datasource id")?; + validate_nonempty_text(&input.database_name, MAX_SCOPE_BYTES, "database name")?; + validate_text(&input.schema_name, MAX_SCOPE_BYTES, "schema name")?; + if let Some(table_name) = &input.table_name { + validate_nonempty_text(table_name, MAX_SCOPE_BYTES, "table name")?; + } + validate_nonempty_text(&input.task_name, MAX_TASK_NAME_BYTES, "task name") +} + +fn validate_artifact_fields( + file_name: &str, + media_type: &str, + format: &str, + extension: &str, +) -> Result<(), StorageError> { + validate_nonempty_text(file_name, MAX_FILE_NAME_BYTES, "artifact file name")?; + if Path::new(file_name) + .file_name() + .and_then(|name| name.to_str()) + != Some(file_name) + { + return Err(StorageError::InvalidTransfer( + "artifact file name must not contain a path", + )); + } + validate_nonempty_text(media_type, MAX_MEDIA_TYPE_BYTES, "artifact media type")?; + validate_nonempty_text(format, 32, "artifact format")?; + if extension.is_empty() + || extension.len() > 16 + || !extension.bytes().all(|byte| byte.is_ascii_alphanumeric()) + { + return Err(StorageError::InvalidTransfer( + "artifact extension is invalid", + )); + } + Ok(()) +} + +fn validate_storage_name(value: &str) -> Result<(), StorageError> { + if value.is_empty() + || value.len() > 128 + || value.contains(['/', '\\']) + || value == "." + || value == ".." + || !value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.')) + { + return Err(StorageError::Integrity( + "transfer artifact storage name is invalid".to_owned(), + )); + } + Ok(()) +} + +fn validate_nonempty_text( + value: &str, + maximum: usize, + field: &'static str, +) -> Result<(), StorageError> { + if value.trim().is_empty() { + return Err(StorageError::InvalidTransfer(field)); + } + validate_text(value, maximum, field) +} + +fn validate_text(value: &str, maximum: usize, field: &'static str) -> Result<(), StorageError> { + if value.len() > maximum { + return Err(StorageError::InvalidTransfer(field)); + } + Ok(()) +} + +fn ensure_task_exists(connection: &rusqlite::Connection, id: i64) -> Result<(), StorageError> { + let exists = connection.query_row( + "SELECT EXISTS(SELECT 1 FROM transfer_tasks WHERE id = ?1)", + [id], + |row| row.get::<_, bool>(0), + )?; + if exists { + Ok(()) + } else { + Err(StorageError::TransferTaskNotFound(id)) + } +} + +fn storage_decode_error(error: StorageError) -> rusqlite::Error { + rusqlite::Error::FromSqlConversionFailure(0, rusqlite::types::Type::Text, Box::new(error)) +} + +#[cfg(test)] +mod tests { + use std::{ + fs, + io::{Read as _, Write as _}, + sync::Arc, + }; + + use tempfile::TempDir; + + use crate::{SecretRef, SecretValue, SecretVault, SecretVaultError, now_millis}; + + use super::{ + CreateTransferTask, Storage, StorageError, StoredTransferTaskKind, StoredTransferTaskStatus, + }; + + #[derive(Debug)] + struct EmptyVault; + + impl SecretVault for EmptyVault { + fn probe(&self) -> Result<(), SecretVaultError> { + Ok(()) + } + + fn create( + &self, + _reference: &SecretRef, + _value: &SecretValue, + ) -> Result<(), SecretVaultError> { + Ok(()) + } + + fn get(&self, _reference: &SecretRef) -> Result, SecretVaultError> { + Ok(None) + } + + fn delete(&self, _reference: &SecretRef) -> Result<(), SecretVaultError> { + Ok(()) + } + } + + fn open(directory: &TempDir) -> Storage { + Storage::open(directory.path(), Arc::new(EmptyVault)).expect("storage opens") + } + + fn task(storage: &Storage, suffix: usize) -> i64 { + storage + .create_transfer_task(&CreateTransferTask { + datasource_id: "mysql-local".to_owned(), + database_name: "inventory".to_owned(), + schema_name: String::new(), + table_name: Some(format!("items_{suffix}")), + kind: StoredTransferTaskKind::ExportFile, + task_name: format!("Export items {suffix}"), + }) + .expect("task creates") + .id + } + + #[test] + fn artifact_publish_is_atomic_and_completes_the_task() { + let directory = TempDir::new().expect("temp directory"); + let storage = open(&directory); + let id = task(&storage, 1); + storage.start_transfer_task(id).expect("task starts"); + let mut writer = storage + .begin_transfer_artifact(Some(id), "items.csv", "text/csv", "CSV", "csv", None) + .expect("artifact begins"); + writer + .write_all(b"id,name\n1,alpha\n") + .expect("artifact writes"); + let artifact = writer.finish().expect("artifact finishes"); + + let task = storage + .get_transfer_task(id) + .expect("task reads") + .expect("task exists"); + assert_eq!(task.status, StoredTransferTaskStatus::Succeeded); + assert_eq!(task.artifact_id.as_deref(), Some(artifact.id.as_str())); + let resolved = storage + .resolve_transfer_artifact(&artifact.id) + .expect("artifact resolves"); + assert_eq!( + fs::read(resolved.path).expect("artifact reads"), + b"id,name\n1,alpha\n" + ); + } + + #[test] + fn artifact_display_names_never_create_paths() { + let directory = TempDir::new().expect("temp directory"); + let storage = open(&directory); + + assert!(matches!( + storage + .begin_transfer_artifact(None, "../outside.csv", "text/csv", "CSV", "csv", None,), + Err(StorageError::InvalidTransfer( + "artifact file name must not contain a path" + )) + )); + assert_eq!( + fs::read_dir(directory.path().join("artifacts")) + .expect("artifact directory reads") + .count(), + 0 + ); + } + + #[test] + fn artifact_resolution_rejects_tampered_and_truncated_files() { + let directory = TempDir::new().expect("temp directory"); + let storage = open(&directory); + + let mut tampered = storage + .begin_transfer_artifact(None, "tampered.csv", "text/csv", "CSV", "csv", None) + .expect("artifact begins"); + tampered + .write_all(b"id,name\n1,alpha\n") + .expect("artifact writes"); + let tampered = tampered.finish().expect("artifact finishes"); + let tampered_path = storage + .resolve_transfer_artifact(&tampered.id) + .expect("untampered artifact resolves") + .path; + fs::write(&tampered_path, b"id,name\n1,omega\n").expect("artifact is tampered"); + assert!(matches!( + storage.resolve_transfer_artifact(&tampered.id), + Err(StorageError::Integrity(message)) if message.contains("SHA-256") + )); + + let mut truncated = storage + .begin_transfer_artifact(None, "truncated.csv", "text/csv", "CSV", "csv", None) + .expect("artifact begins"); + truncated + .write_all(b"id,name\n1,alpha\n") + .expect("artifact writes"); + let truncated = truncated.finish().expect("artifact finishes"); + let truncated_path = storage + .resolve_transfer_artifact(&truncated.id) + .expect("complete artifact resolves") + .path; + fs::write(&truncated_path, b"id\n").expect("artifact is truncated"); + assert!(matches!( + storage.resolve_transfer_artifact(&truncated.id), + Err(StorageError::Integrity(message)) if message.contains("size") + )); + } + + #[test] + fn temporary_artifact_deletion_removes_metadata_and_file() { + let directory = TempDir::new().expect("temp directory"); + let storage = open(&directory); + let mut writer = storage + .begin_transfer_artifact(None, "upload.csv", "text/csv", "CSV", "csv", None) + .expect("artifact begins"); + writer.write_all(b"id\n1\n").expect("artifact writes"); + let artifact = writer.finish().expect("artifact finishes"); + let path = storage + .resolve_transfer_artifact(&artifact.id) + .expect("artifact resolves") + .path; + + assert!( + storage + .delete_temporary_transfer_artifact(&artifact.id) + .expect("temporary artifact deletes") + ); + assert!(!path.exists()); + assert!( + !storage + .delete_temporary_transfer_artifact(&artifact.id) + .expect("repeated deletion is idempotent") + ); + assert!(matches!( + storage.resolve_transfer_artifact(&artifact.id), + Err(StorageError::TransferArtifactNotFound(_)) + )); + + let task_id = task(&storage, 90); + storage.start_transfer_task(task_id).expect("task starts"); + let mut writer = storage + .begin_transfer_artifact( + Some(task_id), + "retained.csv", + "text/csv", + "CSV", + "csv", + None, + ) + .expect("task artifact begins"); + writer.write_all(b"id\n2\n").expect("artifact writes"); + let retained = writer.finish().expect("task artifact finishes"); + let retained_path = storage + .resolve_transfer_artifact(&retained.id) + .expect("task artifact resolves") + .path; + assert!( + !storage + .delete_temporary_transfer_artifact(&retained.id) + .expect("task artifact is not eligible for temporary cleanup") + ); + assert!(retained_path.is_file()); + } + + #[cfg(unix)] + #[test] + fn resolved_artifact_keeps_the_verified_file_when_the_path_is_replaced() { + let directory = TempDir::new().expect("temp directory"); + let storage = open(&directory); + let mut writer = storage + .begin_transfer_artifact(None, "download.csv", "text/csv", "CSV", "csv", None) + .expect("artifact begins"); + writer + .write_all(b"verified-content") + .expect("artifact writes"); + let artifact = writer.finish().expect("artifact finishes"); + let mut resolved = storage + .resolve_transfer_artifact(&artifact.id) + .expect("artifact resolves"); + let replaced = resolved.path.with_extension("replaced"); + fs::rename(&resolved.path, &replaced).expect("verified inode is renamed"); + fs::write(&resolved.path, b"different-content").expect("path is replaced"); + + let mut content = Vec::new(); + resolved + .file + .read_to_end(&mut content) + .expect("verified descriptor reads"); + assert_eq!(content, b"verified-content"); + } + + fn terminal_transition_with_broken_prune( + transition: impl FnOnce(&Storage, i64) -> Result<(), StorageError>, + ) { + let directory = TempDir::new().expect("temp directory"); + let storage = open(&directory); + let oldest = task(&storage, 0); + for suffix in 1..21 { + task(&storage, suffix); + } + let artifacts = directory.path().join("artifacts"); + let displaced = directory.path().join("artifacts-displaced"); + fs::rename(&artifacts, &displaced).expect("artifact directory is displaced"); + + let result = transition(&storage, oldest); + + fs::rename(&displaced, &artifacts).expect("artifact directory is restored"); + result.expect("post-commit pruning cannot reverse a terminal transition"); + } + + #[test] + fn terminal_transitions_ignore_post_commit_prune_failures() { + terminal_transition_with_broken_prune(|storage, id| { + storage.start_transfer_task(id)?; + storage.complete_transfer_task(id, "done") + }); + terminal_transition_with_broken_prune(|storage, id| { + storage.fail_transfer_task(id, "expected failure") + }); + terminal_transition_with_broken_prune(|storage, id| { + storage.cancel_transfer_task(id, "cancelled") + }); + } + + #[test] + fn committed_task_survives_post_commit_prune_cleanup_failure() { + let directory = TempDir::new().expect("temp directory"); + let storage = open(&directory); + let oldest = task(&storage, 100); + storage.start_transfer_task(oldest).expect("task starts"); + let mut writer = storage + .begin_transfer_artifact(Some(oldest), "old.csv", "text/csv", "CSV", "csv", None) + .expect("artifact begins"); + writer.write_all(b"id\n1\n").expect("artifact writes"); + let artifact = writer.finish().expect("artifact finishes"); + let artifact_path = storage + .resolve_transfer_artifact(&artifact.id) + .expect("artifact resolves") + .path; + fs::remove_file(&artifact_path).expect("artifact file removes"); + fs::create_dir(&artifact_path).expect("artifact path becomes an unremovable directory"); + + for suffix in 101..120 { + task(&storage, suffix); + } + let accepted = storage + .create_transfer_task(&CreateTransferTask { + datasource_id: "mysql-local".to_owned(), + database_name: "inventory".to_owned(), + schema_name: String::new(), + table_name: Some("items_120".to_owned()), + kind: StoredTransferTaskKind::ExportFile, + task_name: "Export items 120".to_owned(), + }) + .expect("committed task remains accepted when cleanup fails"); + + assert_eq!(accepted.status, StoredTransferTaskStatus::Queued); + assert_eq!( + storage + .get_transfer_task(accepted.id) + .expect("task reads") + .expect("task exists"), + accepted + ); + assert!( + artifact_path.is_dir(), + "cleanup failure fixture must remain isolated from task acceptance" + ); + } + + #[test] + fn dropping_writer_removes_partial_file_and_restart_interrupts_running_tasks() { + let directory = TempDir::new().expect("temp directory"); + let storage = open(&directory); + let id = task(&storage, 2); + storage.start_transfer_task(id).expect("task starts"); + let writer = storage + .begin_transfer_artifact(Some(id), "items.sql", "application/sql", "SQL", "sql", None) + .expect("artifact begins"); + let partial = writer.path().to_path_buf(); + drop(writer); + assert!(!partial.exists()); + drop(storage); + + let reopened = open(&directory); + assert_eq!(reopened.startup_report().transfers.interrupted_tasks, 1); + assert_eq!( + reopened + .get_transfer_task(id) + .expect("task reads") + .expect("task exists") + .status, + StoredTransferTaskStatus::Interrupted + ); + } + + #[test] + fn failed_artifact_publish_removes_the_renamed_file() { + let directory = TempDir::new().expect("temp directory"); + let storage = open(&directory); + let id = task(&storage, 3); + storage.start_transfer_task(id).expect("task starts"); + let mut writer = storage + .begin_transfer_artifact(Some(id), "items.csv", "text/csv", "CSV", "csv", None) + .expect("artifact begins"); + writer.write_all(b"id\n1\n").expect("artifact writes"); + storage + .request_transfer_cancel(id) + .expect("running task cancellation records"); + + assert!(writer.finish().is_err(), "cancelled task cannot publish"); + assert_eq!( + fs::read_dir(directory.path().join("artifacts")) + .expect("artifact directory reads") + .count(), + 0, + "failed publication must remove partial and final files" + ); + } + + #[test] + fn recovery_removes_expired_partial_and_orphan_artifacts() { + let directory = TempDir::new().expect("temp directory"); + let storage = open(&directory); + let timestamp = now_millis().expect("clock reads"); + let mut writer = storage + .begin_transfer_artifact( + None, + "temporary.csv", + "text/csv", + "CSV", + "csv", + Some(timestamp + 10_000), + ) + .expect("temporary artifact begins"); + writer.write_all(b"id\n1\n").expect("artifact writes"); + let artifact = writer.finish().expect("temporary artifact finishes"); + let resolved = storage + .resolve_transfer_artifact(&artifact.id) + .expect("temporary artifact resolves"); + let artifact_path = resolved.path; + let artifacts = directory.path().join("artifacts"); + let partial = artifacts.join("stranded.part"); + let orphan = artifacts.join("orphan.bin"); + fs::write(&partial, b"partial").expect("partial fixture writes"); + fs::write(&orphan, b"orphan").expect("orphan fixture writes"); + + let report = storage + .recover_transfers_at(timestamp + 20_000) + .expect("transfer recovery succeeds"); + assert_eq!(report.expired_artifacts, 1); + assert_eq!(report.partial_files_removed, 1); + assert_eq!(report.orphan_files_removed, 1); + assert!(!artifact_path.exists()); + assert!(!partial.exists()); + assert!(!orphan.exists()); + } + + #[test] + fn only_twenty_terminal_tasks_are_retained_with_their_artifacts() { + let directory = TempDir::new().expect("temp directory"); + let storage = open(&directory); + for suffix in 0..21 { + let id = task(&storage, suffix); + storage.start_transfer_task(id).expect("task starts"); + storage + .fail_transfer_task(id, "expected test failure") + .expect("task fails"); + } + let tasks = storage.list_transfer_tasks().expect("tasks list"); + assert_eq!(tasks.len(), 20); + assert!(tasks.iter().all(|task| task.id != 1)); + } +} diff --git a/crates/chat2db-storage/src/workspace.rs b/crates/chat2db-storage/src/workspace.rs new file mode 100644 index 0000000..8ec3976 --- /dev/null +++ b/crates/chat2db-storage/src/workspace.rs @@ -0,0 +1,746 @@ +use rusqlite::{Connection, OptionalExtension, Transaction, TransactionBehavior, params}; + +use crate::{Storage, StorageError, now_millis}; + +const MAX_NAMESPACE_NAME_BYTES: usize = 512; + +/// Durable workspace node discriminator. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum WorkspaceNodeKind { + /// A user-created namespace. + Namespace, + /// A datasource owned by the workspace. + DataSource, +} + +/// Disambiguated storage-level workspace node reference. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct WorkspaceNodeLocator { + /// Namespace decimal id or opaque datasource id. + pub id: String, + /// Node category. + pub kind: WorkspaceNodeKind, +} + +/// Flat durable workspace node used to build transport-specific trees. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct WorkspaceNodeRecord { + /// Namespace decimal id or opaque datasource id. + pub id: String, + /// Node category. + pub kind: WorkspaceNodeKind, + /// Current display name. + pub name: String, + /// Direct parent namespace, or root when absent. + pub parent_namespace_id: Option, + /// Stable zero-based order among direct siblings. + pub position: u32, +} + +/// Persisted namespace metadata and placement. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct WorkspaceNamespaceRecord { + /// Positive `SQLite` namespace id. + pub id: i64, + /// Current display name. + pub name: String, + /// Direct parent namespace, or root when absent. + pub parent_id: Option, +} + +impl Storage { + /// Lists every namespace and datasource node in deterministic tree order. + /// + /// # Errors + /// + /// Returns `SQLite` or persisted-data validation failures. + pub fn list_workspace_nodes(&self) -> Result, StorageError> { + let connection = self.connection()?; + let mut statement = connection.prepare( + "SELECT n.node_type, n.namespace_id, n.datasource_id, + COALESCE(ns.name, ds.name), n.parent_namespace_id, n.position + FROM workspace_nodes n + LEFT JOIN workspace_namespaces ns ON ns.id = n.namespace_id + LEFT JOIN datasources ds ON ds.id = n.datasource_id + ORDER BY n.parent_namespace_id IS NOT NULL, + n.parent_namespace_id, n.position, n.node_key", + )?; + let rows = statement.query_map([], decode_workspace_node)?; + rows.collect::, _>>().map_err(Into::into) + } + + /// Creates a namespace at the end of the selected parent's children. + /// + /// # Errors + /// + /// Returns validation, parent-not-found, or `SQLite` failures. + pub fn create_workspace_namespace( + &self, + name: &str, + parent_id: Option, + ) -> Result { + let name = validate_namespace_name(name)?; + let mut connection = self.connection()?; + let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + validate_parent_namespace(&transaction, parent_id)?; + let timestamp = now_millis()?; + transaction.execute( + "INSERT INTO workspace_namespaces (name, created_at_ms, updated_at_ms) + VALUES (?1, ?2, ?2)", + params![name, timestamp], + )?; + let id = transaction.last_insert_rowid(); + let position = next_position(&transaction, parent_id)?; + transaction.execute( + "INSERT INTO workspace_nodes ( + node_key, node_type, namespace_id, datasource_id, + parent_namespace_id, position, created_at_ms + ) VALUES (?1, 'NAMESPACE', ?2, NULL, ?3, ?4, ?5)", + params![namespace_key(id), id, parent_id, position, timestamp], + )?; + transaction.commit()?; + Ok(WorkspaceNamespaceRecord { + id, + name, + parent_id, + }) + } + + /// Renames one namespace without changing placement. + /// + /// # Errors + /// + /// Returns validation, namespace-not-found, or `SQLite` failures. + pub fn update_workspace_namespace( + &self, + id: i64, + name: &str, + ) -> Result { + validate_namespace_id(id)?; + let name = validate_namespace_name(name)?; + let connection = self.connection()?; + let changed = connection.execute( + "UPDATE workspace_namespaces SET name = ?1, updated_at_ms = ?2 WHERE id = ?3", + params![name, now_millis()?, id], + )?; + if changed != 1 { + return Err(StorageError::WorkspaceNamespaceNotFound(id.to_string())); + } + let parent_id = connection.query_row( + "SELECT parent_namespace_id FROM workspace_nodes WHERE namespace_id = ?1", + [id], + |row| row.get(0), + )?; + Ok(WorkspaceNamespaceRecord { + id, + name, + parent_id, + }) + } + + /// Deletes one namespace and promotes its ordered children into its parent. + /// + /// # Errors + /// + /// Returns namespace-not-found, integrity, or `SQLite` failures. + pub fn delete_workspace_namespace(&self, id: i64) -> Result<(), StorageError> { + validate_namespace_id(id)?; + let mut connection = self.connection()?; + let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + let deleted = load_node( + &transaction, + &WorkspaceNodeLocator { + id: id.to_string(), + kind: WorkspaceNodeKind::Namespace, + }, + )?; + let parent = deleted.parent_namespace_id; + let siblings = list_node_keys(&transaction, parent, Some(&namespace_key(id)))?; + let children = list_node_keys(&transaction, Some(id), None)?; + let insertion = siblings + .iter() + .position(|sibling| sibling.position > deleted.position) + .unwrap_or(siblings.len()); + let mut ordered = Vec::with_capacity(siblings.len() + children.len()); + ordered.extend(siblings[..insertion].iter().map(|node| node.key.clone())); + ordered.extend(children.iter().map(|node| node.key.clone())); + ordered.extend(siblings[insertion..].iter().map(|node| node.key.clone())); + + transaction.execute( + "UPDATE workspace_nodes SET parent_namespace_id = ?1 + WHERE parent_namespace_id = ?2", + params![parent, id], + )?; + let changed = + transaction.execute("DELETE FROM workspace_namespaces WHERE id = ?1", [id])?; + if changed != 1 { + return Err(StorageError::WorkspaceNamespaceNotFound(id.to_string())); + } + apply_order(&transaction, parent, &ordered)?; + transaction.commit()?; + Ok(()) + } + + /// Applies Community's `before`, `after`, `first child`, and `last child` tree movement. + /// + /// # Errors + /// + /// Returns invalid-position, missing-node, cycle, or `SQLite` failures. + pub fn move_workspace_node( + &self, + drag: &WorkspaceNodeLocator, + target: &WorkspaceNodeLocator, + drop_position: i8, + ) -> Result<(), StorageError> { + if !matches!(drop_position, -1..=2) { + return Err(StorageError::InvalidWorkspace( + "drop position must be -1, 0, 1, or 2", + )); + } + if drag == target { + return Err(StorageError::InvalidWorkspace( + "a workspace node cannot be dropped onto itself", + )); + } + let mut connection = self.connection()?; + let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + let drag_record = load_node(&transaction, drag)?; + let target_record = load_node(&transaction, target)?; + let drag_key = node_key(drag)?; + let target_key = node_key(target)?; + + let (new_parent, mut ordered, insertion) = if matches!(drop_position, 0 | 2) { + if target_record.kind != WorkspaceNodeKind::Namespace { + return Err(StorageError::InvalidWorkspace( + "only namespaces can receive child nodes", + )); + } + let namespace_id = parse_namespace_id(&target_record.id)?; + let mut children = list_node_keys(&transaction, Some(namespace_id), None)?; + children.retain(|node| node.key != drag_key); + let insertion = if drop_position == 0 { + 0 + } else { + children.len() + }; + (Some(namespace_id), children, insertion) + } else { + let parent = target_record.parent_namespace_id; + let mut siblings = list_node_keys(&transaction, parent, None)?; + siblings.retain(|node| node.key != drag_key); + let target_index = siblings + .iter() + .position(|node| node.key == target_key) + .ok_or_else(|| { + StorageError::Integrity( + "workspace drop target disappeared during a transaction".to_owned(), + ) + })?; + let insertion = if drop_position < 0 { + target_index + } else { + target_index + 1 + }; + (parent, siblings, insertion) + }; + + if drag_record.kind == WorkspaceNodeKind::Namespace { + let drag_namespace_id = parse_namespace_id(&drag_record.id)?; + reject_namespace_cycle(&transaction, drag_namespace_id, new_parent)?; + } + + ordered.insert( + insertion, + OrderedNode { + key: drag_key.clone(), + position: 0, + }, + ); + transaction.execute( + "UPDATE workspace_nodes SET parent_namespace_id = ?1 WHERE node_key = ?2", + params![new_parent, drag_key], + )?; + apply_order( + &transaction, + new_parent, + &ordered + .iter() + .map(|node| node.key.clone()) + .collect::>(), + )?; + if drag_record.parent_namespace_id != new_parent { + normalize_parent(&transaction, drag_record.parent_namespace_id)?; + } + transaction.commit()?; + Ok(()) + } + + /// Moves one datasource to the end of a namespace or the root. + /// + /// # Errors + /// + /// Returns datasource/namespace-not-found or `SQLite` failures. + pub fn assign_datasource_namespace( + &self, + datasource_id: &str, + namespace_id: Option, + ) -> Result<(), StorageError> { + if datasource_id.trim().is_empty() { + return Err(StorageError::InvalidWorkspace( + "datasource id cannot be empty", + )); + } + let mut connection = self.connection()?; + let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + validate_parent_namespace(&transaction, namespace_id)?; + let locator = WorkspaceNodeLocator { + id: datasource_id.to_owned(), + kind: WorkspaceNodeKind::DataSource, + }; + let current = load_node(&transaction, &locator)?; + let key = node_key(&locator)?; + transaction.execute( + "UPDATE workspace_nodes SET parent_namespace_id = ?1 WHERE node_key = ?2", + params![namespace_id, key], + )?; + let mut destination = list_node_keys(&transaction, namespace_id, None)?; + destination.retain(|node| node.key != key); + destination.push(OrderedNode { key, position: 0 }); + apply_order( + &transaction, + namespace_id, + &destination + .iter() + .map(|node| node.key.clone()) + .collect::>(), + )?; + if current.parent_namespace_id != namespace_id { + normalize_parent(&transaction, current.parent_namespace_id)?; + } + transaction.commit()?; + Ok(()) + } +} + +#[derive(Debug)] +struct OrderedNode { + key: String, + position: u32, +} + +fn decode_workspace_node(row: &rusqlite::Row<'_>) -> rusqlite::Result { + let node_type: String = row.get(0)?; + let namespace_id: Option = row.get(1)?; + let datasource_id: Option = row.get(2)?; + let kind = match node_type.as_str() { + "NAMESPACE" => WorkspaceNodeKind::Namespace, + "DATA_SOURCE" => WorkspaceNodeKind::DataSource, + _ => { + return Err(rusqlite::Error::FromSqlConversionFailure( + 0, + rusqlite::types::Type::Text, + "invalid workspace node type".into(), + )); + } + }; + let id = match kind { + WorkspaceNodeKind::Namespace => namespace_id.map(|id| id.to_string()), + WorkspaceNodeKind::DataSource => datasource_id, + } + .ok_or_else(|| { + rusqlite::Error::FromSqlConversionFailure( + 0, + rusqlite::types::Type::Null, + "workspace node identity is missing".into(), + ) + })?; + let position: i64 = row.get(5)?; + Ok(WorkspaceNodeRecord { + id, + kind, + name: row.get(3)?, + parent_namespace_id: row.get(4)?, + position: u32::try_from(position) + .map_err(|_| rusqlite::Error::IntegralValueOutOfRange(5, position))?, + }) +} + +fn load_node( + connection: &Connection, + locator: &WorkspaceNodeLocator, +) -> Result { + let key = node_key(locator)?; + connection + .query_row( + "SELECT n.node_type, n.namespace_id, n.datasource_id, + COALESCE(ns.name, ds.name), n.parent_namespace_id, n.position + FROM workspace_nodes n + LEFT JOIN workspace_namespaces ns ON ns.id = n.namespace_id + LEFT JOIN datasources ds ON ds.id = n.datasource_id + WHERE n.node_key = ?1", + [key], + decode_workspace_node, + ) + .optional()? + .ok_or_else(|| StorageError::WorkspaceNodeNotFound(locator.id.clone())) +} + +fn list_node_keys( + connection: &Connection, + parent: Option, + excluded_key: Option<&str>, +) -> Result, StorageError> { + let mut statement = connection.prepare( + "SELECT node_key, position FROM workspace_nodes + WHERE parent_namespace_id IS ?1 AND (?2 IS NULL OR node_key <> ?2) + ORDER BY position, node_key", + )?; + let rows = statement.query_map(params![parent, excluded_key], |row| { + let position: i64 = row.get(1)?; + Ok(OrderedNode { + key: row.get(0)?, + position: u32::try_from(position) + .map_err(|_| rusqlite::Error::IntegralValueOutOfRange(1, position))?, + }) + })?; + rows.collect::, _>>().map_err(Into::into) +} + +fn apply_order( + transaction: &Transaction<'_>, + parent: Option, + ordered_keys: &[String], +) -> Result<(), StorageError> { + for (position, key) in ordered_keys.iter().enumerate() { + transaction.execute( + "UPDATE workspace_nodes + SET parent_namespace_id = ?1, position = ?2 + WHERE node_key = ?3", + params![ + parent, + i64::try_from(position) + .map_err(|_| StorageError::NumericRange("workspace node position"))?, + key, + ], + )?; + } + Ok(()) +} + +fn normalize_parent( + transaction: &Transaction<'_>, + parent: Option, +) -> Result<(), StorageError> { + let keys = list_node_keys(transaction, parent, None)? + .into_iter() + .map(|node| node.key) + .collect::>(); + apply_order(transaction, parent, &keys) +} + +fn next_position(connection: &Connection, parent: Option) -> Result { + connection + .query_row( + "SELECT COALESCE(MAX(position) + 1, 0) + FROM workspace_nodes WHERE parent_namespace_id IS ?1", + [parent], + |row| row.get(0), + ) + .map_err(Into::into) +} + +fn validate_parent_namespace( + connection: &Connection, + parent: Option, +) -> Result<(), StorageError> { + let Some(parent) = parent else { + return Ok(()); + }; + validate_namespace_id(parent)?; + let exists: bool = connection.query_row( + "SELECT EXISTS(SELECT 1 FROM workspace_namespaces WHERE id = ?1)", + [parent], + |row| row.get(0), + )?; + if exists { + Ok(()) + } else { + Err(StorageError::WorkspaceNamespaceNotFound(parent.to_string())) + } +} + +fn reject_namespace_cycle( + connection: &Connection, + dragged_namespace: i64, + mut candidate_parent: Option, +) -> Result<(), StorageError> { + let mut visited = 0_usize; + while let Some(parent) = candidate_parent { + if parent == dragged_namespace { + return Err(StorageError::InvalidWorkspace( + "a namespace cannot be moved into its own descendant", + )); + } + candidate_parent = connection + .query_row( + "SELECT parent_namespace_id FROM workspace_nodes WHERE namespace_id = ?1", + [parent], + |row| row.get(0), + ) + .optional()? + .flatten(); + visited += 1; + if visited > 1_024 { + return Err(StorageError::Integrity( + "workspace namespace ancestry is cyclic".to_owned(), + )); + } + } + Ok(()) +} + +fn node_key(locator: &WorkspaceNodeLocator) -> Result { + match locator.kind { + WorkspaceNodeKind::Namespace => { + let id = parse_namespace_id(&locator.id)?; + Ok(namespace_key(id)) + } + WorkspaceNodeKind::DataSource => { + if locator.id.trim().is_empty() || locator.id.len() > 512 { + return Err(StorageError::InvalidWorkspace( + "datasource id must be non-empty and at most 512 UTF-8 bytes", + )); + } + Ok(format!("datasource:{}", locator.id)) + } + } +} + +fn namespace_key(id: i64) -> String { + format!("namespace:{id}") +} + +fn parse_namespace_id(id: &str) -> Result { + let id = id + .parse::() + .map_err(|_| StorageError::InvalidWorkspace("namespace id must be a positive integer"))?; + validate_namespace_id(id)?; + Ok(id) +} + +fn validate_namespace_id(id: i64) -> Result<(), StorageError> { + if id <= 0 { + return Err(StorageError::InvalidWorkspace( + "namespace id must be a positive integer", + )); + } + Ok(()) +} + +fn validate_namespace_name(name: &str) -> Result { + let name = name.trim(); + if name.is_empty() || name.len() > MAX_NAMESPACE_NAME_BYTES { + return Err(StorageError::InvalidWorkspace( + "namespace name must be non-empty and at most 512 UTF-8 bytes", + )); + } + Ok(name.to_owned()) +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use tempfile::TempDir; + + use super::{WorkspaceNodeKind, WorkspaceNodeLocator}; + use crate::{ + CreateDatasource, SecretRef, SecretValue, SecretVault, SecretVaultError, Storage, + StorageError, + }; + + #[derive(Debug)] + struct EmptyVault; + + impl SecretVault for EmptyVault { + fn probe(&self) -> Result<(), SecretVaultError> { + Ok(()) + } + + fn create( + &self, + _reference: &SecretRef, + _value: &SecretValue, + ) -> Result<(), SecretVaultError> { + Ok(()) + } + + fn get(&self, _reference: &SecretRef) -> Result, SecretVaultError> { + Ok(None) + } + + fn delete(&self, _reference: &SecretRef) -> Result<(), SecretVaultError> { + Ok(()) + } + } + + fn open(directory: &TempDir) -> Storage { + Storage::open(directory.path(), Arc::new(EmptyVault)).expect("storage opens") + } + + fn namespace(id: i64) -> WorkspaceNodeLocator { + WorkspaceNodeLocator { + id: id.to_string(), + kind: WorkspaceNodeKind::Namespace, + } + } + + fn datasource(id: &str) -> WorkspaceNodeLocator { + WorkspaceNodeLocator { + id: id.to_owned(), + kind: WorkspaceNodeKind::DataSource, + } + } + + #[test] + fn namespace_tree_survives_restart_and_delete_promotes_children() { + let directory = TempDir::new().expect("temp dir"); + let storage = open(&directory); + let datasource = storage + .create_datasource( + CreateDatasource { + name: "Local MySQL".to_owned(), + driver_id: "mysql".to_owned(), + }, + None, + ) + .expect("datasource creates"); + let root = storage + .create_workspace_namespace("Root", None) + .expect("root namespace creates"); + let child = storage + .create_workspace_namespace("Child", Some(root.id)) + .expect("child namespace creates"); + storage + .assign_datasource_namespace(&datasource.id, Some(child.id)) + .expect("datasource moves"); + drop(storage); + + let reopened = open(&directory); + let before = reopened + .list_workspace_nodes() + .expect("workspace nodes list"); + assert!(before.iter().any(|node| { + node.kind == WorkspaceNodeKind::DataSource + && node.id == datasource.id + && node.parent_namespace_id == Some(child.id) + })); + reopened + .delete_workspace_namespace(root.id) + .expect("root namespace deletes"); + let after = reopened + .list_workspace_nodes() + .expect("workspace nodes relist"); + assert!(after.iter().any(|node| { + node.kind == WorkspaceNodeKind::Namespace + && node.id == child.id.to_string() + && node.parent_namespace_id.is_none() + })); + assert!(after.iter().any(|node| { + node.kind == WorkspaceNodeKind::DataSource + && node.id == datasource.id + && node.parent_namespace_id == Some(child.id) + })); + } + + #[test] + fn workspace_reorder_supports_siblings_and_first_or_last_child() { + let directory = TempDir::new().expect("temp dir"); + let storage = open(&directory); + let first = storage + .create_workspace_namespace("First", None) + .expect("first creates"); + let second = storage + .create_workspace_namespace("Second", None) + .expect("second creates"); + storage + .move_workspace_node(&namespace(second.id), &namespace(first.id), -1) + .expect("second moves before first"); + let roots = storage + .list_workspace_nodes() + .expect("nodes list") + .into_iter() + .filter(|node| node.parent_namespace_id.is_none()) + .collect::>(); + assert_eq!(roots[0].id, second.id.to_string()); + assert_eq!(roots[1].id, first.id.to_string()); + + storage + .move_workspace_node(&namespace(first.id), &namespace(second.id), 0) + .expect("first moves inside second"); + let moved = storage + .list_workspace_nodes() + .expect("nodes relist") + .into_iter() + .find(|node| node.id == first.id.to_string()) + .expect("first remains"); + assert_eq!(moved.parent_namespace_id, Some(second.id)); + assert_eq!(moved.position, 0); + + let first_datasource = storage + .create_datasource( + CreateDatasource { + name: "First datasource".to_owned(), + driver_id: "mysql".to_owned(), + }, + None, + ) + .expect("first datasource creates"); + let last_datasource = storage + .create_datasource( + CreateDatasource { + name: "Last datasource".to_owned(), + driver_id: "mysql".to_owned(), + }, + None, + ) + .expect("last datasource creates"); + storage + .assign_datasource_namespace(&first_datasource.id, Some(second.id)) + .expect("first datasource moves inside second"); + storage + .move_workspace_node(&datasource(&last_datasource.id), &namespace(second.id), 2) + .expect("last datasource appends inside second"); + let children = storage + .list_workspace_nodes() + .expect("nodes relist after append") + .into_iter() + .filter(|node| node.parent_namespace_id == Some(second.id)) + .collect::>(); + assert_eq!(children[0].id, first.id.to_string()); + assert_eq!(children[1].id, first_datasource.id); + assert_eq!(children[2].id, last_datasource.id); + } + + #[test] + fn namespace_cycles_and_unknown_datasources_are_rejected() { + let directory = TempDir::new().expect("temp dir"); + let storage = open(&directory); + let root = storage + .create_workspace_namespace("Root", None) + .expect("root creates"); + let child = storage + .create_workspace_namespace("Child", Some(root.id)) + .expect("child creates"); + let cycle = storage + .move_workspace_node(&namespace(root.id), &namespace(child.id), 0) + .expect_err("cycle rejected"); + assert!(matches!(cycle, StorageError::InvalidWorkspace(_))); + + let missing = storage + .assign_datasource_namespace("missing", None) + .expect_err("unknown datasource rejected"); + assert!(matches!(missing, StorageError::WorkspaceNodeNotFound(_))); + assert_eq!(datasource("missing").kind, WorkspaceNodeKind::DataSource); + } +} diff --git a/docs/architecture.md b/docs/architecture.md index 91aec5b..48e90e6 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -17,8 +17,9 @@ without embedding H2 in the compatibility-engine JAR. The Web and Tauri hosts open the production vault, SQLite storage, and verified driver catalog before exposing a shared `Application`; they do not start Java during host bootstrap. Native MySQL connection, object metadata, editable -result-grid operations, database/table/view DDL, preview, and Console -reads/writes/scripts do not acquire a Java lease. The Core +result-grid operations, database/table/view DDL, preview, Console +reads/writes/scripts, and Dashboard/Chart refresh do not acquire a Java lease. +The Core `EngineManager` starts one Java generation on the first JDBC-only database, parser, formatter, completion, builder, or advanced metadata request. It shares that single-flight startup across concurrent callers and issues generation-scoped @@ -41,8 +42,10 @@ building, parsing, syntax validation, formatting, and datasource-aware SQL completion plus datasource-free typed DML, namespace SQL, and bounded table-preview SQL generation when the exact locked classpath is configured. The current product frontend is not the former repository-owned replacement -workbench. The build exports the unmodified Community frontend tree pinned by -`scripts/community-frontend.lock.json`. Web maps its historical `/api` +workbench. The build exports a locked Community frontend commit that retains +the original pages, components, interactions, and styles while applying a +reviewable host-transport patch for CSP-safe callbacks and Web/Desktop file +flows. `scripts/community-frontend.lock.json` binds both commit and tree. Web maps its historical `/api` contract through Axum; desktop maps the existing `window.javaQuery` contract through one Tauri `legacy_request` command. Both paths call the same Rust legacy dispatcher. The implemented product slice covers native MySQL connection @@ -74,8 +77,11 @@ drag-only column reordering, type-preserving ENUM/SET and `UNSIGNED` ALTERs, composite primary-key ordering, and fail-closed generated/invisible-column reorders that retained their live definitions. View metadata and full database/table/view cleanup also passed while Java remained dormant. The -complete repository `make verify` gate and an explicit real-MySQL rerun passed -after the final implementation. +Dashboard/Chart integration separately passed selected-database refresh, +200-row bounding, SELECT CTEs, response-only metadata with Community column +attributes, unsafe-SQL rejection, `CHART` history, cleanup, and dormant Java +against MySQL 8.4. The complete repository `make verify` gate and an explicit +real-MySQL rerun pass with this Dashboard/Chart increment included. ## Ownership @@ -88,8 +94,8 @@ after the final implementation. | Durable state | Rust | SQLite, retained-result files, and a mandatory injected secret-vault contract | | AI agent | Rust | Provider adapters, tool loop, limits, compaction, and cancellation | | MCP and CLI | Rust | Adapters around the same product services and policy | -| Native MySQL product slice | Rust / `mysql_async` | Connection, object metadata, Community-compatible routes/envelopes, editable result-grid DML, database/table/view DDL, preview, unparameterized Console reads/writes/scripts, transactions, paging, limits, cancellation, large values, and durable history | -| Compatibility databases and remaining MySQL operations | Java 17 | Existing SPI/plugins, JDBC bind parameters, SQL builders, parsing, formatting, completion, unmapped MySQL features, and non-MySQL metadata | +| Native MySQL product slice | Rust / `mysql_async` | Connection and SSH, datasource lifecycle/portability, object metadata, typed SELECT binds, editable DML/DDL, Console, Dashboard/Chart refresh, routines/migration, transfer and class generation, accounts, schema diff, workspace state, Agent/CLI/MCP writes, cancellation, large values, and historical HTTP/IPC envelopes | +| Compatibility databases and exact Community helpers | Java 17 | Existing SPI/plugins for non-MySQL databases plus Community parsing, formatting, completion, SQL builders, and plugin-specific behavior | | SQL parsing, formatting, and completion | Java 17 | Existing Java ANTLR grammars, parser behavior, formatter behavior, and completion | | Rust-to-Java IPC | Shared Protobuf contract | Length-prefixed frames over private stdin/stdout | @@ -104,9 +110,9 @@ React in system WebView React in browser -> Rust application services <- owner-only local attachment <- CLI <- owner-only local attachment <- rmcp stdio server <- MCP client - -> SQLite and result store + -> SQLite dashboard/chart/workspace state and result store -> AI agent runtime - -> native MySQL connection / metadata / editable DDL / Console + -> native MySQL connection / metadata / editable DDL / Console / chart refresh -> Java process supervisor -> Protobuf stdin/stdout -> Java database compatibility engine @@ -145,12 +151,17 @@ cross-language acceptance gates pass. ## Database boundary Java/JDBC remains the compatibility implementation for other databases and for -unmigrated MySQL operations. The native route uses upstream -`mysql_async 0.37.0` for MySQL connection testing, object metadata, editable -result-grid execution, database/table/view DDL, preview, and unparameterized -Console execution. Core selects this backend before requesting an -`EngineLease`; unrecognized drivers cannot enter it. Console bind parameters -remain unsupported rather than silently starting Java. +fixed Community parser, formatter, completion, builder, and plugin behavior. +The native route uses upstream `mysql_async 0.37.0` for the complete MySQL +product data plane: connection and SSH, metadata, editable DML and DDL, +Console, typed SELECT bind parameters, Dashboard/Chart refresh, routines, +migration, transfer, accounts, schema diff, workspace state, and approved +automation writes. Rust also renders +MyBatis Plus entity, Mapper, and Mapper XML files from native MySQL metadata, +writing local files for Desktop or a bounded ZIP artifact for Web. Core selects +the native backend before requesting an `EngineLease`; unrecognized drivers +cannot enter it. The pinned Community write/script contract has no bind field, +so only ordered single-statement SELECT binds are exposed. The native MySQL baseline implements: @@ -178,7 +189,12 @@ The native MySQL baseline implements: - active-query cancellation through a second bounded connection issuing `KILL CONNECTION`, followed by deterministic cleanup; and - SQLite-backed saved Consoles and per-statement execution history exposed by - the original Web and desktop contracts. + the original Web and desktop contracts; and +- SQLite-backed Dashboard/Chart CRUD through all ten historical Web/Tauri + routes. Chart detail refresh executes only one SELECT or SELECT CTE through a + forced native MySQL read-only transaction, caps page 1 at 200 rows, returns + Community-shaped response-only metadata, records `CHART` history, and rejects + writes, multiple statements, locking reads, and server-file output. The JDBC baseline implements: @@ -191,7 +207,7 @@ The JDBC baseline implements: Stage 7B additionally implements: - a Git submodule fixed at Community commit - `37a34be858f2566b6b7fcf6c3f64183c1f560853`; + `3cb8af54cad5bd5caa20bb25f10d9b0e4f01931c`; - a reproducible H2 compatibility classpath, established with 148 JARs and extended in Stage 7J to 149 JARs for the retained Community domain-core completion implementation, whose filenames, byte lengths, and SHA-256 @@ -214,6 +230,12 @@ inside Java. The Community classpath and each JDBC driver classloader are separate; JDBC driver JARs are not added to the Community classpath. Only bounded, process-neutral DTOs cross Protobuf. +For H2 completion, a compatibility proxy resolves the pinned plugin's +`ParserUtil` call from the active external JDBC driver classloader. Its +thread-local binding is removed after each completion, preserving classloader +isolation and driver unload while matching the pinned Community identifier +quoting behavior. + Stage 7C composes that boundary into the product runtime. The Web and desktop bootstrap paths accept `CHAT2DB_COMMUNITY_CLASSPATH_DIR`, but the source commit and 149 filenames, byte lengths, and SHA-256 digests come only from the lock @@ -407,14 +429,10 @@ SQL into the editor and observes the accepted operation in the existing result surface. A table/scope change aborts the pending request, and a late accepted operation is cancelled instead of replacing newer state. The Core path is runtime-tested against MySQL 8.4 through both the historical Connector/J gate -and the native `mysql_async` gate. Product writes -and Agent, CLI, and MCP MySQL conformance remain outside Stage 7M; PostgreSQL and -long-tail plugin conformance do not block this MySQL milestone. - -Remaining builder operations, complete MySQL type conformance, native bind -parameters and CTE-first SELECT, non-relational -operations, script execution, import/export, and per-dialect conformance are not -implemented yet. +and the native `mysql_async` gate. The complete Issue `#14` MySQL milestone +additionally covers product writes, CTE-first and typed SELECT binds, scripts, +import/export, Agent, CLI, and MCP. PostgreSQL, non-relational, and long-tail +plugin conformance remain separate work and do not block this MySQL milestone. Spring Boot, Spring Web, Spring AI, MCP, JCEF, product storage, and updater logic do not belong in the final Java engine. @@ -556,7 +574,7 @@ product UI from those intermediate slices with the exact original Community frontend while retaining the Rust capabilities behind explicit historical API adapters. Signing, installation, hot reload, downloading, compatibility selection, updates, -rollback, and the remaining compatibility operations are not implemented. +rollback, and non-MySQL compatibility operations are not implemented. ## Local attachment and MCP boundary @@ -569,23 +587,28 @@ authenticate each request with a random 32-byte token, and enforce bounded length-prefixed JSON frames and I/O deadlines. The local protocol exposes health, secret-free datasource listing, -forced-read-only query start, operation snapshot, idempotent cancellation, and -row/byte-bounded result paging. The CLI maps these operations to structured JSON -commands. It does not start another product runtime or contact Java directly. +forced-read-only query start, operation snapshot, idempotent cancellation, +row/byte-bounded result paging, and one explicitly confirmed MySQL write. The +CLI maps these operations to structured JSON commands and requires +`--confirm-write` for the write command. It does not start another product +runtime or contact Java directly. -`chat2db-mcp` uses `rmcp` 2.2 over standard stdio and maps five tools onto the +`chat2db-mcp` uses `rmcp` 2.2 over standard stdio and maps six tools onto the same `LocalClient`: `list_datasources`, `query_database`, -`inspect_query_operation`, `cancel_database_query`, and -`inspect_query_result`. Query start returns only an operation id. Query +`inspect_query_operation`, `cancel_database_query`, `inspect_query_result`, and +`execute_database_write`. Query start returns only an operation id. Query retention is capped at 10,000 rows, 16 MiB, and 900 seconds; each result page is capped at 1,000 rows and 512 KiB. Product `ApiError` values retain their stable codes, while local paths and transport details are redacted. Stdout is protocol-only, and dependency logging cannot be raised above `WARN` through the MCP log setting. -This MCP slice has no write tool, Agent-run tool, or JDBC bind-parameter input. -Those capabilities are not implied by the built-in Agent's broader SQL tool -set. +The write tool accepts only datasource identity and SQL from the model. It asks +the trusted MCP client for form elicitation that displays the datasource, exact +SQL SHA-256, and bounded SQL preview; approval is bound to those exact values +and consumed once. A model cannot approve itself with `confirm` or an approval +token, and clients without form elicitation fail closed. MCP does not expose an +Agent-run tool or JDBC bind-parameter input. ## Security baseline diff --git a/docs/mysql-community-parity.md b/docs/mysql-community-parity.md index 9d407cf..5a12e75 100644 --- a/docs/mysql-community-parity.md +++ b/docs/mysql-community-parity.md @@ -2,143 +2,162 @@ ## Status -- Community baseline: `OtterMind/Chat2DB` `main@3cb8af54cad5bd5caa20bb25f10d9b0e4f01931c`. -- Rust baseline: `OtterMind/Chat2DB-Rust` - `main@0d39236b724efdf4fd2c74a0d57f96579745e7d9`; PR `#11` merged the - three-stage Issue `#10` table-DDL retrieval work and closed that issue. -- Current Issue `#12` branch: MySQL `FUNCTION` and `PROCEDURE` - `preview_invocation` is implemented with native `mysql_async` parameter - metadata and Community-compatible invocation SQL. The original POST route is - registered for both Axum HTTP and desktop `legacy_request`, using the same - handler and envelope. `preview_migration` and `execute_migration` remain not - implemented. Focused unit tests cover function return-row filtering, - parameter modes, type-based defaults, identifier quoting, and zero-parameter - or unknown-routine preview rendering. The real-MySQL product test provisions - a function plus `IN`/`OUT`/`INOUT` and zero-parameter procedures, executes the - generated SQL through the native Console path, and asserts Java dormancy. -- Product target: the original Community React frontend running against the Rust Web or Tauri host. -- Runtime-tested now: datasource CRUD/test; database, schema, table, column, - index, foreign-key, primary-key, view, function, procedure, trigger, and - routine-parameter metadata; all first-stage historical metadata routes; - bounded table preview; saved Console CRUD; and native unparameterized MySQL - Console execution. Console coverage includes DDL/DML, semicolon scripts, - `DELIMITER` procedure scripts, explicit transactions, error-continue policy, - preserved-single dispatch, `EXPLAIN`, bounded all-row paging, datasource - read-only enforcement, multiple result sets, exact affected-row counts, - cancellation, a shared 64 MiB retained-result budget, durable history with - cancelled-state projection, and bounded large-cell retrieval/download. Base64 - and hex large-text chunks use UTF-8 byte offsets as the original frontend - expects. Native metadata and Console execution keep Java dormant. Paged table - name/comment search, complete-list filtering, page-size validation, HTTP - binding-error envelopes, and nullable column defaults match the locked - Community baseline. The original Web and Tauri contracts now also expose - editable table previews, create/update/delete SQL generation and execution, - copy-as-SQL helpers, bounded counts, table metadata/query, database/schema - create and confirmed delete, table create/alter/drop/truncate/copy, and view - query/metadata/create-or-replace/drop. A real Web-to-native-MySQL 8.4 vertical - exercises these mutations while proving Java remains dormant. The retained - editor now accepts its explicit-null column and index payloads, recognizes - `IN_VALUES`, infers `FIRST`/`AFTER` changes from drag-only array order, and - preserves `UNSIGNED`, empty and quoted ENUM/SET values, and composite - primary-key order across native metadata and subsequent ALTER statements. - Type modifiers are parsed outside ENUM/SET value lists. Dragging a generated, - invisible, `ZEROFILL`, or otherwise unmodeled column is rejected after a live - metadata check instead of emitting a lossy `MODIFY COLUMN`. MySQL `view_meta` - returns the original six form configurations and creation template without - requiring an existing view. Native `SHOW CREATE TABLE` now backs both - `/api/rdb/ddl/export` and `/api/rdb/table/export`; the four Community - create/update example aliases preserve MySQL's successful `data: null` - contract. HTTP and desktop dispatch return identical envelopes, and the real - MySQL 8.4 vertical proves Java remains dormant. -- Complete parity: not implemented. - -This file is the acceptance contract for MySQL work. Community frontend routes -and user-visible behavior define parity. A modern Core, Axum, Tauri, Java, or -native MySQL capability does not count as complete until the original frontend -route reaches it and a real MySQL product test covers the behavior. +- Community baseline: `OtterMind/Chat2DB` + `main@3cb8af54cad5bd5caa20bb25f10d9b0e4f01931c`. +- Rust milestone: Issue `#14`, branch + `feat/mysql-community-complete-parity`. +- Implementation: complete in the milestone working tree for every MySQL + workbench capability reached by the pinned Community frontend. +- Runtime-tested: yes, with local MySQL 8.4 for native metadata, Console, + editable data, DDL, routines, transfer, account administration, schema diff, + Dashboard/Chart refresh, views, workspace state, Web HTTP, and + Desktop-compatible dispatch. These native paths keep Java dormant. +- SSH status: implemented and runtime-tested through a temporary Docker SSH + endpoint without enabling macOS Remote Login. Two concurrent MySQL queries + shared one datasource/revision-scoped fixed-port tunnel; the final lease + released the listener, the fixture was removed, and Java remained dormant. +- Release status: the local repository, Community frontend, Java/H2, real + MySQL, and real SSH gates pass in the milestone working tree. After the + Dashboard/Chart increment, the complete `rtk make verify` gate and its Rust, + process, Java, IPC/JDBC/H2, frontend, and Desktop checks also pass. The normal + public CI workflow must pass on the staged commits before Issue `#14` is + closed. + +This file is the acceptance contract for MySQL work. A Core capability counts +only when the original Community UI can reach it through both Axum HTTP and the +Tauri legacy bridge. Host-specific transport patches are allowed only when the +locked frontend commit, tree, and production build reproduce them. ## Ownership -- `mysql_async` owns MySQL connections, metadata, query/update execution, - transactions, cancellation, large values, and data transfer. -- The fixed Community Java compatibility process retains the exact Community - ANTLR parser, formatter, completion engine, and plugin SQL builders where - reproducing their behavior in Rust would create unnecessary divergence. -- Rust remains the only product host. Java has no HTTP port and starts only for - compatibility operations that require it. -- The original Community frontend and its styles remain unchanged. Compatibility - is implemented behind its existing HTTP and `window.javaQuery` contracts. +- `mysql_async 0.37.0` owns native MySQL connections, metadata, queries, + updates, transactions, cancellation, account administration, schema diff, + chart refresh, and transfer. +- Rust owns the Axum/Tauri product host, SQLite workspace/task/dashboard/chart + state, encrypted datasource secrets, SSH tunnels, MyBatis Plus class + generation, AI Agent, CLI, and MCP. +- The fixed Community Java compatibility process owns only parser, formatter, + completion, SQL-builder, and plugin behavior where exact Community semantics + are required. It starts on demand and has no HTTP port. +- The original Community pages, components, interaction model, and styles are + retained. A locked host-adapter patch changes only transport-facing source + for CSP-safe callbacks and Web/Desktop file upload and download behavior. ## Capability Matrix -| Area | Community frontend contract | Rust baseline | Required parity | -| --- | --- | --- | --- | -| Runtime bootstrap | `/api/system`, `/api/common/environment`, `/api/jdbc/driver/list` | Implemented | Preserve exact envelopes and immutable driver inventory. | -| Datasource CRUD and test | `/api/connection/datasource/list`, `/datasource`, `/datasource/create`, `/datasource/pre_connect`, `/datasource/update`, `DELETE /datasource` | Implemented | Keep secret-safe persistence and native MySQL connection testing. | -| Datasource lifecycle | `/api/connection/datasource/connect`, `/datasource/close`, `/connection/close`, `/connection/console/connect`, `/datasource/clone` | Not implemented | Match explicit connect/close/clone behavior and frontend refresh semantics. | -| SSH and JDBC driver management | `/api/connection/ssh/pre_connect`, `/api/jdbc/driver/download`, `/upload`, `/save`, `/delete` | Not implemented | Match Community SSH testing and local driver lifecycle without exposing secrets. | -| Datasource import/export and namespaces | converter upload routes, `/api/connection/datasource/import_community`, `/datasource/export`, `/api/namespaces/*` | Not implemented | Support Community, Chat2DB, Navicat, DBeaver, DataGrip, export, grouping, and ordering. | -| Database and schema metadata | `/api/rdb/database/list`, `/database_schema_list`, `/api/rdb/schema/list` | Database/schema list implemented | Match filtering, system flags, charset/collation, comments, and pagination envelopes. | -| Database and schema mutation | database create/modify/delete and `/api/rdb/delete/{database,schema}/{prepare,execute}` | Historical create-SQL routes and two-phase confirmed database/schema deletion are implemented; database create/delete is real-MySQL tested | Add unsupported database alteration fields and close remaining exact projection differences. | -| Table inventory and detail | `/api/rdb/table/list`, `/table_list`, `/table_meta`, `/column_list`, `/index_list`, `/key_list`, `/query` | List, compact list, table metadata/query, column, index, and key routes are implemented with native MySQL metadata; nullable defaults, type-suffix-aware `UNSIGNED`/`ZEROFILL`, empty and quoted ENUM/SET values, composite primary-key order, and legacy envelopes match the retained editor and are real-MySQL tested | Close remaining field-level differences as original editor scenarios expose them. | -| Table data operations | `/api/rdb/dml/execute_table`, `/execute_update`, `/get_update_sql`, `/copy_update_sql`, `/copy_in_values_sql`, `/count` | Editable previews, PK-first optimistic insert/update/delete SQL, bounded native execution, copy-as-INSERT/UPDATE/WHERE, frontend `IN_VALUES`, and protected count queries are implemented and real-MySQL tested | Close remaining clipboard and uncommon result-type differences. | -| Table DDL | `/api/rdb/ddl/*`, `/api/rdb/table/modify/sql`, `/delete`, `/truncate`, `/copy`, create/update examples, DDL export | Create/alter/drop/truncate/copy previews and execution are implemented for columns, indexes, engine, charset, collation, comments, auto-increment, and MySQL editor types; explicit-null editor rows and drag-only `FIRST`/`AFTER` reordering are real-MySQL tested, while live metadata rejects generated, invisible, `ZEROFILL`, and other unmodeled columns before a lossy reorder. Native `SHOW CREATE TABLE` backs both export aliases with the Community trailing semicolon; all four MySQL example aliases preserve Community's null response. Foreign keys are implemented as read-only metadata; pinned Community `MysqlSqlBuilder` and `MysqlIndexTypeEnum` do not generate or modify `foreignKeyList`, and the Community MySQL editor exposes no foreign-key mutation contract. | Add remaining table options and close field-level edge cases; foreign-key mutation is not a current Community parity requirement, while foreign-key metadata remains available to read-only metadata and future ER flows. | -| Views | `/api/rdb/view/list`, `/column_list`, `/detail`, `/query`, `/view_meta`, `/modify/sql`, `/delete`, `/drop` | Native list/detail plus historical query, the six-option Community `view_meta` creation template, create-or-replace preview/execution, and drop are implemented and real-MySQL tested | Add any remaining delete alias and uncommon definer/security projection differences. | -| Functions, procedures, and triggers | `/api/rdb/{function,procedure,trigger}/{list,detail}`, `/api/rdb/routine/{preview_invocation,preview_migration,execute_migration}` | Native list/detail and routine-parameter projections are implemented; every original list/detail route is mapped. The current Issue `#12` slice implements MySQL `FUNCTION` and `PROCEDURE` invocation previews from `information_schema.PARAMETERS`, preserving parameter order, `IN`/`OUT`/`INOUT` handling, type-based input defaults, quoted routine names, and trailing separators. Function previews use `SELECT`; procedure previews emit the required `SET`, `CALL`, and output `SELECT` statements. | `preview_migration` and `execute_migration` are not implemented; add migration preview and replacement execution with compensating restore semantics. | -| Console SELECT | `/api/rdb/dml/execute`, desktop `sql-execute`/`sql-cancel` | Native unparameterized MySQL reads, CTEs, normal/all-row paging, preserved-single dispatch, `EXPLAIN`, limits, multiple result sets, affected-row counts, datasource read-only enforcement, and cancellation implemented | Add JDBC-style bind parameters and close remaining warning/error/result-shape differences. | -| Console scripts and writes | `/api/rdb/dml/execute`, `/execute_ddl` | Native unparameterized DDL/DML, semicolon and `DELIMITER` scripts, explicit transactions, error-continue policy, cancellation, and per-statement results implemented | Add bind parameters and complete exact Community conformance for unsupported edge-case scripts. | -| Large cell values | `/api/rdb/cell/value`, `/download`, `/download_path` | Bounded UTF-8/Base64 previews, owner-scoped expiring tokens, byte-oriented Base64/hex chunk reads, character-oriented text reads, and full-value downloads implemented | Add long-running export/task integration and close remaining content-type/display-mode differences. | -| Saved consoles and SQL history | `/api/operation/saved/*`, `/api/operation/log/{create,list}` and detail | Restart-safe saved Console CRUD plus durable history create/list/detail, filtering, paging, per-statement recording, and cancelled-state projection implemented | Complete remaining Community audit/history fields and non-Console producers. | -| SQL parser, formatter, validation, completion | `/api/sql/format`, `/valid_select`, `/api/sql_parser/get_keywords`, `/context/{parser,quick_parser,tip,hover}` | Modern parser/validation/formatter/completion contracts implemented through Java; legacy routes absent | Map every original endpoint to the fixed Community implementation with matching UTF-16 offsets and envelopes. | -| Import, export, and tasks | `/api/import/{sql_file,other_file}`, `/api/export/{sql_file,other_file}`, `/api/task/*`, `/api/rdb/dml/export`, table class generation | Not implemented | Add bounded streaming import/export, progress, stop, download, cleanup, and failure recovery. | -| Account administration | `/api/rdb/account/{capability,list,grants,preview,execute}` | Not implemented | Match MySQL users, hosts, authentication, privileges, role/grant previews, execution, and current Community escaping rules. | -| Structure comparison | `/api/diff/sql` | Not implemented | Match Community structure projection and MySQL synchronization SQL without changing shared query parsing behavior. | -| Pins and ER metadata | `/api/pin/table/*`, `/api/er/*` | Not implemented | Persist pinned tables and expose the metadata needed by the existing ER view. | -| AI, CLI, and MCP | Original `/api/ai` UI plus Community CLI/MCP database actions | Rust Agent, owner-only CLI attachment, and read-only MCP exist behind modern contracts | Map the original AI workspace and make MySQL read/write tools pass the same product conformance gates. | - -## Delivery Order - -1. Complete: native read-only object metadata and every matching original - metadata route, with Axum/dispatch contracts and a real MySQL 8.4 product - vertical that proves Java remains dormant. -2. Implemented slice: native unparameterized Console statement execution, - multi-result handling, writes, transactions, history, cancellation, and - large-cell retrieval. Bind parameters and remaining exact Community edge-case - conformance are still required for complete parity. -3. Implemented slice: table data editing plus database/schema/table/view DDL - preview and execution, native table DDL retrieval/export, and the Community - create/update example route aliases. - Foreign-key mutation is not a current Community MySQL editor requirement; - foreign keys remain read-only metadata for metadata and future ER flows. - Remaining exact Community edge cases are still required for complete parity. -4. Current slice: native MySQL `FUNCTION` and `PROCEDURE` invocation preview. - `preview_migration` and `execute_migration` remain not implemented. Also add - import/export/tasks, datasource lifecycle/SSH/import, account - administration, structure comparison, pins, and ER metadata. -5. Original AI mapping and MySQL conformance for Agent, CLI, and MCP. - -Each stage requires focused unit tests, a real MySQL product vertical with Java -dormancy assertions for native operations, original Web and Tauri contract tests, -the complete repository verification gate, and all GitHub Actions jobs. +| Area | Community contract | Milestone status | +| --- | --- | --- | +| Runtime bootstrap | `/api/system`, `/api/common/environment`, `/api/jdbc/driver/list` | Implemented for Web and Desktop with the native MySQL driver inventory. | +| Datasource CRUD and lifecycle | datasource list/get/create/update/delete, clone, connect, close, console connect, grouping | Implemented with revision CAS, safe edit projections, secret-preserving empty-password updates, clone/close semantics, and namespace persistence. Create responses do not echo submitted connection data; edit/list responses return only sanitized URL, username, non-sensitive properties, and `readOnly`. | +| SSH and driver lifecycle | `/api/connection/ssh/pre_connect`, datasource `ssh`, JDBC driver download/upload/save/delete | Implemented. Password/private-key SSH settings persist inside the encrypted datasource descriptor; passwords and passphrases never leave the vault. Native MySQL callers share managed tunnel leases rather than bypassing SSH in metadata or Console paths. | +| Datasource portability | converter upload routes, Community import/export, Navicat, DBeaver, DataGrip | Implemented for Chat2DB JSON, Navicat NCX v11/v12, DBeaver DBP/AES credentials, and DataGrip text, with bounded ZIP/XML/file parsing and secret-safe export. | +| Database and schema | list, create SQL, create, confirmed delete, metadata projections | Implemented natively with pagination, system flags, charset/collation/comment fields, and two-phase destructive confirmation. | +| Tables and metadata | table/list/query/meta, columns, indexes, primary/imported/exported keys | Implemented natively. Composite keys, nullable defaults, generated/invisible safety checks, `UNSIGNED`/`ZEROFILL`, ENUM/SET values, and editor ordering are covered. | +| Editable data | preview, count, insert/update/delete SQL and execution, copy SQL/IN values | Implemented with bounded reads, PK-first optimistic writes, explicit nulls, Community result envelopes, and large-cell retention. | +| Table DDL | create/alter/drop/truncate/copy, example aliases, `SHOW CREATE`, export | Implemented for the pinned MySQL editor surface. Foreign-key mutation is not exposed by that editor; foreign-key metadata remains available to ER and schema diff. | +| Views | list/columns/detail/query/meta/create-or-replace/drop | Implemented with the six-field Community editor projection and real MySQL execution. | +| Functions, procedures, triggers | list/detail/parameters, invocation preview, migration preview/execute | Implemented. Migration replacement uses compensating restore behavior; generated invocations preserve modes, order, defaults, and quoted identifiers. | +| Console | `/api/rdb/dml/execute`, DDL/update aliases, Desktop SQL stream/cancel | Implemented for SELECT/CTE, DDL/DML, scripts, custom `DELIMITER`, transactions, `EXPLAIN`, paging, multiple result sets, cancellation, error-continue, read-only enforcement, durable history, and bounded retained results. The pinned Community request has no write-bind field; typed SELECT binds are an additional Rust capability. | +| Large values | cell preview/read/download/path | Implemented with owner-scoped expiring tokens, UTF-8 byte/character boundaries, Base64/hex modes, and bounded fallback previews. | +| Import, export, and tasks | SQL/CSV/XLS/XLSX import/export, task list/get/stop/download, DML export, class generation | Implemented with durable tasks, cancellation/recovery, streamed Web attachments, Desktop paths, and generated SQL ZIPs. Rust renders MyBatis Plus entity, Mapper, and Mapper XML files from native MySQL metadata, writing local files for Desktop or a bounded ZIP artifact for Web. | +| Account administration | capability/list/grants/preview/execute | Implemented for seven actions, three scopes, and fourteen privileges, with preview-token authorization, escaping, password redaction, and read-only rejection. | +| Structure comparison | `/api/diff/sql` | Implemented as read-only source-to-target preview. It covers tables, columns/order, primary and secondary indexes, foreign keys, engine/charset/collation/comment, and views. DDL is target-qualified, foreign keys are retargeted, dependent views are topologically ordered, and the output is real-MySQL tested to converge. Runtime AUTO_INCREMENT counters are intentionally not treated as schema. | +| Workspace state | saved consoles/history, namespaces, pins, ER metadata/positions | Implemented in SQLite with restart-safe ownership and migration coverage. | +| Dashboards and charts | Dashboard/Chart CRUD plus chart detail refresh | Implemented through all ten historical Web/Tauri routes. Dashboard and chart documents persist in SQLite; refresh reads the chart's `databaseInfo` and executes native MySQL under the boundary below. | +| SQL compatibility | parser, formatter, validation, keywords, context parser/tip/hover/completion | Every historical route is mapped to the fixed Community Java implementation with shared Web/Desktop envelopes and lazy Java startup. | +| AI workspace | `/api/v3/ai/chat/stream`, history, model list/options/config/test, attachments | Implemented as a compatibility facade over the Rust Agent. Web SSE and Desktop `ai_sse_message` are mapped; model secrets are retained safely; legacy runs are forced read-only and unexpected write approval requests are denied. | +| Agent, CLI, and MCP writes | Rust Agent tools, CLI write, MCP write tool | Implemented through native `mysql_async` with explicit confirmation, single-statement validation, datasource read-only enforcement, prepared-protocol dispatch, and `not_started` versus `unknown` retry semantics. Non-MySQL writes never fall back to Java. | + +## Dashboard and Chart Boundary + +The original Community frontend reaches Dashboard/Chart through the same Rust +legacy dispatcher on Web and desktop: + +| Method | Historical route | Operation | +| --- | --- | --- | +| `GET` | `/api/dashboard/list` | Search and page dashboards. | +| `GET` | `/api/dashboard` | Load one dashboard. | +| `DELETE` | `/api/dashboard` | Delete one dashboard. | +| `POST` | `/api/dashboard/create` | Create one dashboard. | +| `POST` | `/api/dashboard/update` | Partially update one dashboard. | +| `GET` | `/api/v1/chart` | Load one persisted chart without SQL execution. | +| `GET` | `/api/chart/detail` | Load a chart and optionally refresh its query result. | +| `POST` | `/api/v1/chart/create` | Create one chart. | +| `POST` | `/api/v1/chart/update` | Partially update one chart. | +| `DELETE` | `/api/chart` | Delete one chart. | + +SQLite migration 8 stores dashboards and charts, including chart schema, +persisted metadata, database context, refresh settings, and dashboard chart-id +relations. A `refresh=false` detail request performs no database query. For a +refresh, Rust reads `dataSourceId`, `sql`, `databaseName`, `schemaName`, and +`consoleId` from `databaseInfo`, selects the requested database, and accepts +exactly one parsed MySQL `SELECT` or SELECT CTE. Writes, multiple statements, +locking reads, and `INTO OUTFILE`/`DUMPFILE` are rejected before dispatch. + +Accepted SQL runs through native `mysql_async` inside `START TRANSACTION READ +ONLY` and is rolled back before disconnect. The response is page 1 capped at +200 rows and 8 MiB. `dataList` contains string-or-null cells and `headerList` +uses the Community column shape. Simple single-table results enrich headers +from native MySQL column metadata, including primary key, auto-increment, +integer nullability, default, comment, size, scale, and editor type. Refreshed +`metaData` exists only on the detached response and never replaces the chart's +persisted `metaData`. Successful and rejected refreshes write durable +`SQL_EXECUTE` history with `extendInfo.source = "CHART"`, chart id, and console +id. The complete path is native and does not acquire a Java lease. + +## Schema Diff Boundary + +The pinned Community Liquibase comparison surface does not expose independent +CHECK-constraint editing, partition editing, or MySQL runtime counters. The +Rust diff therefore does not claim those as editable parity. It fails closed on +case-only object conflicts under case-insensitive MySQL naming and on cyclic +view dependencies instead of emitting ambiguous SQL. + +## Verification Gates + +Before merge, the milestone requires: + +1. Rust format, workspace tests, all-target/all-feature checks, and strict + Clippy with locked dependencies. +2. Locked Community frontend source verification, typecheck, tests, and + production build without UI/component/style changes; the host transport + patch must be committed and reproducible. +3. Web and Desktop contract tests, including AI SSE and legacy dispatch. +4. Real MySQL 8.4 product tests for Core Console, metadata, transfer, accounts, + schema diff, Dashboard/Chart refresh, and the Web editable/DDL vertical, plus + a real SSH-forwarded concurrent-query test with Java dormancy and fixture + cleanup assertions. +5. Knowledge-base lint and staged commits followed by the normal public CI + workflow. The macOS package workflow remains manual-only and is not invoked + by this milestone. ## Source Anchors -- `third_party/chat2db-community/chat2db-community-client/src/service/connection.ts` -- `third_party/chat2db-community/chat2db-community-client/src/service/sql.ts` -- `third_party/chat2db-community/chat2db-community-client/src/service/executeSql.ts` -- `third_party/chat2db-community/chat2db-community-client/src/service/importExport.ts` -- `third_party/chat2db-community/chat2db-community-client/src/service/accountAdmin.ts` -- `third_party/chat2db-community/chat2db-community-client/src/service/schemaSync.ts` +- `third_party/chat2db-community/chat2db-community-client/src/service/` - `third_party/chat2db-community/chat2db-community-server/chat2db-community-web/src/main/java/ai/chat2db/community/web/api/controller/` -- `third_party/chat2db-community/chat2db-community-server/chat2db-community-plugins/chat2db-community-mysql/src/main/java/ai/chat2db/plugin/mysql/` +- `apps/chat2db-web/src/legacy.rs` +- `apps/chat2db-web/src/legacy_ai.rs` +- `apps/chat2db-desktop/src/lib.rs` +- `apps/chat2db-desktop/src/legacy_files.rs` +- `crates/chat2db-contract/src/` +- `crates/chat2db-contract/src/community_dashboard.rs` - `crates/chat2db-core/src/native_mysql.rs` -- `crates/chat2db-contract/src/community.rs` -- `crates/chat2db-core/src/mysql_ddl.rs` -- `crates/chat2db-core/src/large_value.rs` -- `crates/chat2db-core/tests/native_mysql_console_docker.rs` -- `crates/chat2db-storage/src/operation_log.rs` -- `crates/chat2db-storage/migrations/004_operation_log.sql` -- `crates/chat2db-core/tests/native_mysql_product.rs` +- `crates/chat2db-core/src/mysql_dashboard.rs` +- `crates/chat2db-core/src/mysql_account.rs` +- `crates/chat2db-core/src/mysql_schema_diff.rs` +- `crates/chat2db-core/src/mysql_workspace.rs` +- `crates/chat2db-core/src/ssh.rs` +- `crates/chat2db-core/src/transfer/` +- `crates/chat2db-storage/migrations/005_workspace_namespace.sql` +- `crates/chat2db-storage/migrations/006_transfer.sql` +- `crates/chat2db-storage/migrations/007_mysql_workspace.sql` +- `crates/chat2db-storage/migrations/008_community_dashboard.sql` +- `crates/chat2db-storage/src/community_dashboard.rs` - `apps/chat2db-web/tests/native_mysql_editable_ddl_docker.rs` -- `crates/chat2db-core/src/community.rs` -- `apps/chat2db-web/src/legacy.rs` +- `crates/chat2db-core/tests/native_mysql_product.rs` +- `crates/chat2db-core/tests/native_mysql_transfer_docker.rs` +- `crates/chat2db-core/tests/native_mysql_account_docker.rs` +- `crates/chat2db-core/tests/native_mysql_schema_diff_docker.rs` +- `crates/chat2db-core/tests/native_mysql_dashboard_docker.rs` +- `crates/chat2db-core/tests/native_mysql_ssh_tunnel_docker.rs` +- `java/compat-runtime/src/main/java/ai/chat2db/rust/compat/CommunityH2IdentifierCompatibility.java` diff --git a/docs/protocol.md b/docs/protocol.md index 2353fdb..0bc8fd2 100644 --- a/docs/protocol.md +++ b/docs/protocol.md @@ -33,7 +33,7 @@ not committed. The retained Community SPI and its implementations come from the Community 5.3.0 submodule fixed at commit -`37a34be858f2566b6b7fcf6c3f64183c1f560853`. The Protobuf messages are +`3cb8af54cad5bd5caa20bb25f10d9b0e4f01931c`. The Protobuf messages are compatibility-layer-owned DTOs, not serialized Community Java types; Community plugin, JDBC, parser, and exception objects remain inside Java. The catalog's `source_commit` is provenance that Rust checks against the configured commit. @@ -408,6 +408,14 @@ can consume the response and close the session. These product rules sit above the compatibility wire contract. Typed-DML, namespace, and DQL generation remain datasource-free. +The fixed H2 plugin's identifier processor calls H2 `ParserUtil` while building +completion candidates. The compatibility bridge wraps only the H2 plugin's +metadata and identifier interfaces, binds the active connection's external +driver classloader for that completion call, and invokes `ParserUtil` +reflectively from that loader. The binding is thread-local and restored after +the call, so the Community loader does not gain the H2 driver and the bridge +does not retain a driver loader after session cleanup. + For table preview, Core applies a default of 200 and a maximum of 1,000 rows, parses the generated SQL, requires `is_select`, at most one projected SELECT statement, a SELECT prefix, and no semicolon, and only then submits it through diff --git a/docs/stages.md b/docs/stages.md index 02c2c03..f726f01 100644 --- a/docs/stages.md +++ b/docs/stages.md @@ -31,9 +31,9 @@ new generation on later use. Host health reports a dormant configured engine as ready and available on demand rather than disabled or degraded. Frontend checkpoints `928e62c` and `cf9ab8a` supersede the repository-owned -Stage 5/7G replacement workbench. Current builds export the pinned Community -Umi frontend without local page or style patches; the pinned source includes a -CSP-safe utility fix that preserves callback references without `new Function`. +Stage 5/7G replacement workbench. Current builds export a locked Community Umi +frontend that retains the original pages, components, and styles. A reviewable +host-adapter patch provides CSP-safe callbacks and Web/Desktop file transport. Web uses historical `/api` compatibility routes; desktop preserves `window.javaQuery` through one Tauri command; both converge on the same Rust dispatcher. Earlier stage descriptions @@ -74,13 +74,17 @@ channels; the frontend has matching HTTP/Tauri observers with bounded recovery. Web and desktop start the same owner-only local attachment around their shared `Application`. The JSON CLI exposes health, datasource listing, -forced-read-only query start/status/cancel, and bounded retained-result pages. -The `rmcp` stdio server exposes the matching five datasource/query lifecycle -tools, returns only an operation id from query start, and requires result -polling and paging. MCP retention is capped at 10,000 rows, 16 MiB, and 900 -seconds; pages are capped at 1,000 rows and 512 KiB. The current MCP contract is -read-only and accepts no JDBC bind parameters; it does not claim the built-in -Agent's write tool. +forced-read-only query start/status/cancel, bounded retained-result pages, and +one MySQL write command gated by `--confirm-write`. The `rmcp` stdio server +exposes the matching five datasource/query lifecycle tools plus +`execute_database_write`. It requires protocol-level Form elicitation from the +trusted client, bound to the exact datasource and SQL; model arguments expose +neither `confirm` nor an approval token, approval is single-use, and clients +without Form elicitation fail closed. Query start still +returns only an operation id and requires polling and paging. MCP retention is +capped at 10,000 rows, 16 MiB, and 900 seconds; pages are capped at 1,000 rows +and 512 KiB. MCP accepts no JDBC bind-parameter input and exposes no Agent-run +tool. Stage 7A implements strict local JDBC driver-pack discovery, bounded artifact hashing, immutable inventory through Core, Axum, Tauri, and generated frontend @@ -89,7 +93,7 @@ Host-owned staging remains valid across idle restarts. Downloading, signing, installation, update, rollback, and hot reload remain incomplete. Stage 7B fixes the Community source at commit -`37a34be858f2566b6b7fcf6c3f64183c1f560853`, builds its H2 compatibility +`3cb8af54cad5bd5caa20bb25f10d9b0e4f01931c`, builds its H2 compatibility classpath reproducibly, and initially locks 148 JAR filenames, lengths, and SHA-256 digests. Before lock verification, the fixed build strips dependency-manifest `Class-Path` entries deterministically, rounds the commit timestamp down to ZIP's @@ -333,9 +337,10 @@ metadata page-size validation, HTTP 200 error envelopes, and the distinction between a null and empty-string column default. The original asynchronous SELECT path still emits typed retained-result wire messages from a read-only transaction. The native Console path owns broader -unparameterized statements and scripts on one session. Bind parameters remain -unsupported. Cancellation terminates the active MySQL connection through a -separate bounded control connection. The +unparameterized statements and scripts on one session. The pinned Community +write request has no bind field; the native API additionally supports ordered +single-statement SELECT binds. Cancellation terminates the active MySQL +connection through a separate bounded control connection. The explicit `native-mysql-integration` target and MySQL CI job use a deliberately missing Java executable and verify connection, first-stage object metadata, two-row preview, typed three-row Console output, one-row truncation, active @@ -352,8 +357,8 @@ The editable-grid and DDL follow-up adds structured MySQL insert/update/delete generation and native execution, copy-as-SQL and bounded count helpers, table editor metadata, database/schema create and confirmed delete, table create/alter/drop/truncate/copy, and view query/create-or-replace/drop. Axum and -desktop `legacy_request` use the same historical dispatcher, so the unchanged -Community frontend reaches one Rust implementation on both transports. The SQL +desktop `legacy_request` use the same historical dispatcher, so the retained +Community UI reaches one Rust implementation on both transports. The SQL builders validate identifier segments, closed type/options, values, and view bodies; updates prefer primary keys and otherwise match the complete old row with `LIMIT 1`. @@ -366,11 +371,13 @@ fixture cleanup, and a dormant Java assertion after every product operation. Core and Web focused tests, strict workspace Clippy, formatting, whitespace, and the complete repository `make verify` gate passed. -Stage 7 remains incomplete. Complete MySQL type conformance, native bind -parameters, remaining exact Community Console edge cases, data import/export, -non-relational behavior, remaining builder operations and plugin inventory, -driver distribution, and per-dialect conformance are not -implemented. +The Issue `#14` Community MySQL milestone is complete: native type handling, +ordered SELECT binds, exact Console edge cases, import/export and durable tasks, +Rust MyBatis Plus class generation, SSH, routines, accounts, schema diff, +workspace state, and Agent/CLI/MCP safety boundaries are implemented. Stage 7 +as a multi-database program remains in progress for non-MySQL and non-relational +behavior, remaining plugin inventory, driver distribution, and per-dialect +conformance. Before Stage 8 may produce any Object-form distribution containing Community 5.3.0 code, the release must record written commercial authorization compatible diff --git a/java/compat-runtime/src/main/java/ai/chat2db/rust/compat/CommunityH2IdentifierCompatibility.java b/java/compat-runtime/src/main/java/ai/chat2db/rust/compat/CommunityH2IdentifierCompatibility.java new file mode 100644 index 0000000..2a2eeb6 --- /dev/null +++ b/java/compat-runtime/src/main/java/ai/chat2db/rust/compat/CommunityH2IdentifierCompatibility.java @@ -0,0 +1,205 @@ +package ai.chat2db.rust.compat; + +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.Proxy; +import java.sql.Connection; +import java.util.Map; + +/** Keeps H2 identifier checks on the separately supplied JDBC driver classloader. */ +final class CommunityH2IdentifierCompatibility { + + private static final String H2_DATABASE_TYPE = "H2"; + private static final String PLUGIN_CLASS = "ai.chat2db.spi.IPlugin"; + private static final String METADATA_CLASS = "ai.chat2db.spi.IDbMetaData"; + private static final String IDENTIFIER_PROCESSOR_CLASS = + "ai.chat2db.spi.ISQLIdentifierProcessor"; + private static final String H2_PARSER_UTIL_CLASS = "org.h2.util.ParserUtil"; + + private final ClassLoader communityLoader; + private final Class metadataType; + private final Class identifierProcessorType; + private final ThreadLocal driverLoader = new ThreadLocal<>(); + + private CommunityH2IdentifierCompatibility( + ClassLoader communityLoader, + Class metadataType, + Class identifierProcessorType) { + this.communityLoader = communityLoader; + this.metadataType = metadataType; + this.identifierProcessorType = identifierProcessorType; + } + + static CommunityH2IdentifierCompatibility install( + ClassLoader communityLoader, Class contextType) + throws ReflectiveOperationException { + Class pluginType = Class.forName(PLUGIN_CLASS, true, communityLoader); + Class metadataType = Class.forName(METADATA_CLASS, true, communityLoader); + Class identifierProcessorType = + Class.forName(IDENTIFIER_PROCESSOR_CLASS, true, communityLoader); + CommunityH2IdentifierCompatibility compatibility = + new CommunityH2IdentifierCompatibility( + communityLoader, metadataType, identifierProcessorType); + + Object value = contextType.getField("PLUGIN_MAP").get(null); + if (!(value instanceof Map rawPlugins)) { + throw new IllegalStateException("Community plugin registry is not a map"); + } + @SuppressWarnings("unchecked") + Map plugins = (Map) rawPlugins; + Object h2Plugin = plugins.get(H2_DATABASE_TYPE); + if (h2Plugin == null) { + throw new IllegalStateException("Community H2 plugin is unavailable"); + } + if (Proxy.isProxyClass(h2Plugin.getClass())) { + InvocationHandler handler = Proxy.getInvocationHandler(h2Plugin); + if (handler instanceof PluginHandler installed + && installed.compatibility.communityLoader == communityLoader) { + return installed.compatibility; + } + } + Object wrappedPlugin = Proxy.newProxyInstance( + communityLoader, + new Class[] {pluginType}, + compatibility.pluginHandler(h2Plugin)); + plugins.put(H2_DATABASE_TYPE, wrappedPlugin); + return compatibility; + } + + DriverBinding bind(String databaseType, Connection connection) + throws ReflectiveOperationException { + if (!H2_DATABASE_TYPE.equalsIgnoreCase(databaseType)) { + return DriverBinding.NOOP; + } + ClassLoader connectionLoader = connection.getClass().getClassLoader(); + if (connectionLoader == null) { + throw new IllegalStateException("Community H2 connection has no driver classloader"); + } + Class parserUtil = Class.forName(H2_PARSER_UTIL_CLASS, false, connectionLoader); + ClassLoader parserLoader = parserUtil.getClassLoader(); + if (parserLoader == null || parserLoader == communityLoader) { + throw new IllegalStateException( + "Community H2 ParserUtil must come from the external driver classloader"); + } + + ClassLoader previous = driverLoader.get(); + driverLoader.set(parserLoader); + return () -> { + if (previous == null) { + driverLoader.remove(); + } else { + driverLoader.set(previous); + } + }; + } + + private InvocationHandler pluginHandler(Object delegate) { + return new PluginHandler(this, delegate); + } + + private static final class PluginHandler implements InvocationHandler { + private final CommunityH2IdentifierCompatibility compatibility; + private final Object delegate; + + private PluginHandler(CommunityH2IdentifierCompatibility compatibility, Object delegate) { + this.compatibility = compatibility; + this.delegate = delegate; + } + + @Override + public Object invoke(Object proxy, Method method, Object[] arguments) throws Throwable { + if (isNoArgMethod(method, "getDbMetaData")) { + return compatibility.wrapMetadata( + CommunityH2IdentifierCompatibility.invoke(delegate, method, arguments)); + } + if (isNoArgMethod(method, "getSQLIdentifierProcessor")) { + return compatibility.wrapIdentifierProcessor( + CommunityH2IdentifierCompatibility.invoke(delegate, method, arguments)); + } + return CommunityH2IdentifierCompatibility.invoke(delegate, method, arguments); + } + } + + private Object wrapMetadata(Object delegate) { + if (delegate == null) { + return null; + } + return Proxy.newProxyInstance( + communityLoader, + new Class[] {metadataType}, + (proxy, method, arguments) -> isNoArgMethod(method, "getSQLIdentifierProcessor") + ? wrapIdentifierProcessor(invoke(delegate, method, arguments)) + : invoke(delegate, method, arguments)); + } + + private Object wrapIdentifierProcessor(Object delegate) { + if (delegate == null) { + return null; + } + return Proxy.newProxyInstance( + communityLoader, + new Class[] {identifierProcessorType}, + (proxy, method, arguments) -> isConditionalQuote(method) + ? quoteIdentifier((String) arguments[0]) + : invoke(delegate, method, arguments)); + } + + private String quoteIdentifier(String identifier) throws ReflectiveOperationException { + if (isBlank(identifier)) { + return identifier; + } + ClassLoader currentDriverLoader = driverLoader.get(); + if (currentDriverLoader == null) { + throw new IllegalStateException( + "Community H2 identifier quoting requires an active driver binding"); + } + Class parserUtil = Class.forName(H2_PARSER_UTIL_CLASS, true, currentDriverLoader); + Method isSimpleIdentifier = + parserUtil.getMethod("isSimpleIdentifier", String.class, boolean.class, boolean.class); + boolean simple = (Boolean) isSimpleIdentifier.invoke(null, identifier, true, false); + return simple ? identifier : '"' + identifier.replace("\"", "\"\"") + '"'; + } + + private static boolean isConditionalQuote(Method method) { + if (!method.getName().equals("quoteIdentifier")) { + return false; + } + Class[] parameters = method.getParameterTypes(); + return parameters.length == 1 && parameters[0] == String.class + || parameters.length == 3 && parameters[0] == String.class; + } + + private static boolean isNoArgMethod(Method method, String name) { + return method.getName().equals(name) && method.getParameterCount() == 0; + } + + private static boolean isBlank(String value) { + if (value == null || value.isEmpty()) { + return true; + } + for (int index = 0; index < value.length(); index++) { + if (!Character.isWhitespace(value.charAt(index))) { + return false; + } + } + return true; + } + + private static Object invoke(Object delegate, Method method, Object[] arguments) + throws Throwable { + try { + return method.invoke(delegate, arguments); + } catch (InvocationTargetException failure) { + throw failure.getCause(); + } + } + + @FunctionalInterface + interface DriverBinding extends AutoCloseable { + DriverBinding NOOP = () -> {}; + + @Override + void close(); + } +} diff --git a/java/compat-runtime/src/main/java/ai/chat2db/rust/compat/CommunitySqlCompletionBridge.java b/java/compat-runtime/src/main/java/ai/chat2db/rust/compat/CommunitySqlCompletionBridge.java index 3fb47e4..c8c04ae 100644 --- a/java/compat-runtime/src/main/java/ai/chat2db/rust/compat/CommunitySqlCompletionBridge.java +++ b/java/compat-runtime/src/main/java/ai/chat2db/rust/compat/CommunitySqlCompletionBridge.java @@ -75,6 +75,7 @@ final class CommunitySqlCompletionBridge { private final Class requestType; private final Class activeSnippetSlotType; private final Object service; + private final CommunityH2IdentifierCompatibility h2IdentifierCompatibility; private CommunitySqlCompletionBridge( ClassLoader loader, @@ -82,13 +83,15 @@ private CommunitySqlCompletionBridge( Class contextType, Class requestType, Class activeSnippetSlotType, - Object service) { + Object service, + CommunityH2IdentifierCompatibility h2IdentifierCompatibility) { this.loader = loader; this.connectInfoType = connectInfoType; this.contextType = contextType; this.requestType = requestType; this.activeSnippetSlotType = activeSnippetSlotType; this.service = service; + this.h2IdentifierCompatibility = h2IdentifierCompatibility; } static CommunitySqlCompletionBridge open(ClassLoader loader) throws ReflectiveOperationException { @@ -114,13 +117,15 @@ static CommunitySqlCompletionBridge open(ClassLoader loader) throws ReflectiveOp new Class[] {converterType, genericEngine.getClass()}, converter, genericEngine); + Class contextType = Class.forName(CONTEXT_CLASS, true, loader); return new CommunitySqlCompletionBridge( loader, Class.forName(CONNECT_INFO_CLASS, true, loader), - Class.forName(CONTEXT_CLASS, true, loader), + contextType, Class.forName(COMPLETION_REQUEST_CLASS, true, loader), Class.forName(ACTIVE_SNIPPET_SLOT_CLASS, true, loader), - service); + service, + CommunityH2IdentifierCompatibility.install(loader, contextType)); } finally { thread.setContextClassLoader(previous); } @@ -190,8 +195,11 @@ CommunitySqlCompletion complete( ClassLoader previous = thread.getContextClassLoader(); Object connectInfo = null; RuntimeFailure operationFailure = null; + CommunityH2IdentifierCompatibility.DriverBinding driverBinding = + CommunityH2IdentifierCompatibility.DriverBinding.NOOP; thread.setContextClassLoader(loader); try { + driverBinding = h2IdentifierCompatibility.bind(canonicalDatabaseType, connection); connectInfo = connectInfo(canonicalDatabaseType, connection, request); contextType.getMethod("putContext", connectInfoType).invoke(null, connectInfo); Object response = service.getClass().getMethod("complete", requestType) @@ -221,6 +229,7 @@ CommunitySqlCompletion complete( operationFailure = translated; throw translated; } finally { + driverBinding.close(); RuntimeFailure connectionFailure = connectionOwnershipFailure(connection); Throwable cleanupFailure = clearCompletionState(request.getDatasourceScope()); thread.setContextClassLoader(previous); diff --git a/java/compat-runtime/src/test/java/ai/chat2db/rust/compat/CommunityPluginRegistryTest.java b/java/compat-runtime/src/test/java/ai/chat2db/rust/compat/CommunityPluginRegistryTest.java index 29287b3..81efc6e 100644 --- a/java/compat-runtime/src/test/java/ai/chat2db/rust/compat/CommunityPluginRegistryTest.java +++ b/java/compat-runtime/src/test/java/ai/chat2db/rust/compat/CommunityPluginRegistryTest.java @@ -4,6 +4,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assumptions.assumeTrue; @@ -46,6 +47,11 @@ import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import java.util.jar.Attributes; import java.util.jar.JarEntry; import java.util.jar.JarOutputStream; @@ -466,6 +472,7 @@ void realCommunityH2SqlCompletionClearsRequestStateAndKeepsTheConnectionOpen() } long failureScope = tableScope + 2; + assertNull(activeH2DriverLoader(registry)); try (Connection closedConnection = h2Connection(driverLoader)) { String closedDatabaseName = closedConnection.getCatalog(); closedConnection.close(); @@ -482,12 +489,130 @@ void realCommunityH2SqlCompletionClearsRequestStateAndKeepsTheConnectionOpen() } assertEquals(previous, Thread.currentThread().getContextClassLoader()); assertCompletionStateCleared(communityLoader, failureScope); + assertNull(activeH2DriverLoader(registry)); + + long recoveryScope = failureScope + 1; + var recovered = registry.completeSql( + connection, + completionRequest( + databaseName, + "select * from ", + "select * from ".length(), + recoveryScope)); + assertEquals("SUCCESS", recovered.getStatus()); + assertTrue(recovered.getCandidatesList().stream().anyMatch(candidate -> + candidate.getLabel().equalsIgnoreCase("completion_users"))); + assertCompletionStateCleared(communityLoader, recoveryScope); + assertNull(activeH2DriverLoader(registry)); removeCommunityCache(communityLoader, adjacentCacheKey); removeCommunityCache(communityLoader, unrelatedCacheKey); } } + @Test + void repeatedSqlCompletionOpenKeepsEarlierAndLatestH2BindingsUsable() throws Exception { + Path communityClasspath = communityClasspathDirectory(); + assumeTrue( + Files.isDirectory(communityClasspath), + "the fixed Community H2 classpath is built by the extended integration lane"); + + try (URLClassLoader driverLoader = new URLClassLoader( + new URL[] {h2DriverJar().toUri().toURL()}, + ClassLoader.getPlatformClassLoader()); + URLClassLoader communityLoader = communityLoader(communityClasspath); + Connection connection = h2Connection(driverLoader)) { + try (Statement statement = connection.createStatement()) { + statement.executeUpdate("CREATE SCHEMA IF NOT EXISTS APP"); + statement.executeUpdate( + "CREATE TABLE IF NOT EXISTS APP.repeated_open_items " + + "(id BIGINT PRIMARY KEY, label VARCHAR(64))"); + } + + CommunitySqlCompletionBridge first = CommunitySqlCompletionBridge.open(communityLoader); + CommunitySqlCompletionBridge latest = CommunitySqlCompletionBridge.open(communityLoader); + assertSame(h2IdentifierCompatibility(first), h2IdentifierCompatibility(latest)); + String databaseName = connection.getCatalog(); + String sql = "select * from "; + + var latestResult = latest.complete( + "H2", connection, completionRequest(databaseName, sql, sql.length(), 7_101L)); + assertEquals("SUCCESS", latestResult.getStatus()); + assertNull(activeH2DriverLoader(latest)); + + var firstResult = first.complete( + "H2", connection, completionRequest(databaseName, sql, sql.length(), 7_102L)); + assertEquals("SUCCESS", firstResult.getStatus()); + assertTrue(firstResult.getCandidatesList().stream().anyMatch(candidate -> + candidate.getLabel().equalsIgnoreCase("repeated_open_items"))); + assertNull(activeH2DriverLoader(first)); + } + } + + @Test + void concurrentH2IdentifierBindingsStayIsolatedByDriverClassloader() throws Exception { + Path communityClasspath = communityClasspathDirectory(); + assumeTrue( + Files.isDirectory(communityClasspath), + "the fixed Community H2 classpath is built by the extended integration lane"); + + URL h2Driver = h2DriverJar().toUri().toURL(); + try (URLClassLoader communityLoader = communityLoader(communityClasspath); + URLClassLoader firstDriverLoader = new URLClassLoader( + new URL[] {h2Driver}, + ClassLoader.getPlatformClassLoader()); + URLClassLoader secondDriverLoader = new URLClassLoader( + new URL[] {h2Driver}, + ClassLoader.getPlatformClassLoader()); + Connection firstConnection = h2Connection(firstDriverLoader); + Connection secondConnection = h2Connection(secondDriverLoader)) { + CommunitySqlCompletionBridge bridge = CommunitySqlCompletionBridge.open(communityLoader); + Class contextType = + Class.forName("ai.chat2db.spi.sql.Chat2DBContext", true, communityLoader); + CommunityH2IdentifierCompatibility compatibility = + h2IdentifierCompatibility(bridge); + Object identifierProcessor = h2IdentifierProcessor(contextType, communityLoader); + Method quoteIdentifier = Class.forName( + "ai.chat2db.spi.ISQLIdentifierProcessor", true, communityLoader) + .getMethod("quoteIdentifier", String.class); + + CountDownLatch ready = new CountDownLatch(2); + CountDownLatch start = new CountDownLatch(1); + ExecutorService executor = Executors.newFixedThreadPool(2); + try { + Future first = executor.submit(() -> quoteConcurrently( + compatibility, + identifierProcessor, + quoteIdentifier, + firstConnection, + firstDriverLoader, + "IDENTIFIER", + "IDENTIFIER", + ready, + start)); + Future second = executor.submit(() -> quoteConcurrently( + compatibility, + identifierProcessor, + quoteIdentifier, + secondConnection, + secondDriverLoader, + "IDENTIFIER", + "IDENTIFIER", + ready, + start)); + + assertTrue(ready.await(5, TimeUnit.SECONDS)); + start.countDown(); + assertTrue(first.get(10, TimeUnit.SECONDS)); + assertTrue(second.get(10, TimeUnit.SECONDS)); + } finally { + start.countDown(); + executor.shutdownNow(); + executor.awaitTermination(5, TimeUnit.SECONDS); + } + } + } + private static URLClassLoader h2DriverWithTrigger(Path temporaryDirectory) throws Exception { Path h2Jar = h2DriverJar(); @@ -918,16 +1043,7 @@ private static boolean hasPlugin(CommunityPluginRegistry registry, String databa } private static CommunityPluginRegistry openRegistry(Path directory) throws Exception { - var paths = CommunityPluginRegistry.validateClasspath(directory.toRealPath().toString()); - URLClassLoader loader = new URLClassLoader( - paths.stream().map(Path::toUri).map(uri -> { - try { - return uri.toURL(); - } catch (java.net.MalformedURLException failure) { - throw new IllegalStateException(failure); - } - }).toArray(URL[]::new), - ClassLoader.getPlatformClassLoader()); + URLClassLoader loader = communityLoader(directory); try { Method discover = CommunityPluginRegistry.class.getDeclaredMethod( "discover", URLClassLoader.class); @@ -942,7 +1058,7 @@ private static CommunityPluginRegistry openRegistry(Path directory) throws Excep CommunitySqlCompletionBridge.class); constructor.setAccessible(true); return constructor.newInstance( - "37a34be858f2566b6b7fcf6c3f64183c1f560853", + "3cb8af54cad5bd5caa20bb25f10d9b0e4f01931c", loader, plugins, CommunitySqlCompletionBridge.open(loader)); @@ -952,6 +1068,19 @@ private static CommunityPluginRegistry openRegistry(Path directory) throws Excep } } + private static URLClassLoader communityLoader(Path directory) throws Exception { + var paths = CommunityPluginRegistry.validateClasspath(directory.toRealPath().toString()); + return new URLClassLoader( + paths.stream().map(Path::toUri).map(uri -> { + try { + return uri.toURL(); + } catch (java.net.MalformedURLException failure) { + throw new IllegalStateException(failure); + } + }).toArray(URL[]::new), + ClassLoader.getPlatformClassLoader()); + } + private static CompleteCommunitySqlRequest completionRequest( String databaseName, String sql, int cursorUtf16, long datasourceScope) { return CompleteCommunitySqlRequest.newBuilder() @@ -1051,6 +1180,70 @@ private static ClassLoader registryLoader(CommunityPluginRegistry registry) thro return (ClassLoader) field.get(registry); } + private static Object activeH2DriverLoader(CommunityPluginRegistry registry) throws Exception { + var field = CommunityPluginRegistry.class.getDeclaredField("sqlCompletion"); + field.setAccessible(true); + return activeH2DriverLoader((CommunitySqlCompletionBridge) field.get(registry)); + } + + private static Object activeH2DriverLoader(CommunitySqlCompletionBridge bridge) throws Exception { + return activeH2DriverLoader(h2IdentifierCompatibility(bridge)); + } + + private static CommunityH2IdentifierCompatibility h2IdentifierCompatibility( + CommunitySqlCompletionBridge bridge) throws Exception { + var field = CommunitySqlCompletionBridge.class.getDeclaredField("h2IdentifierCompatibility"); + field.setAccessible(true); + return (CommunityH2IdentifierCompatibility) field.get(bridge); + } + + private static Object activeH2DriverLoader( + CommunityH2IdentifierCompatibility compatibility) throws Exception { + var field = CommunityH2IdentifierCompatibility.class.getDeclaredField("driverLoader"); + field.setAccessible(true); + return ((ThreadLocal) field.get(compatibility)).get(); + } + + private static Object h2IdentifierProcessor(Class contextType, ClassLoader communityLoader) + throws Exception { + @SuppressWarnings("unchecked") + Map plugins = (Map) contextType.getField("PLUGIN_MAP").get(null); + Object plugin = plugins.get("H2"); + assertNotNull(plugin); + Class pluginType = Class.forName("ai.chat2db.spi.IPlugin", true, communityLoader); + return pluginType.getMethod("getSQLIdentifierProcessor").invoke(plugin); + } + + private static boolean quoteConcurrently( + CommunityH2IdentifierCompatibility compatibility, + Object identifierProcessor, + Method quoteIdentifier, + Connection connection, + ClassLoader expectedDriverLoader, + String identifier, + String expected, + CountDownLatch ready, + CountDownLatch start) + throws Exception { + try (CommunityH2IdentifierCompatibility.DriverBinding ignored = + compatibility.bind("H2", connection)) { + if (activeH2DriverLoader(compatibility) != expectedDriverLoader) { + return false; + } + ready.countDown(); + if (!start.await(5, TimeUnit.SECONDS)) { + return false; + } + for (int attempt = 0; attempt < 1_000; attempt++) { + if (activeH2DriverLoader(compatibility) != expectedDriverLoader + || !expected.equals(quoteIdentifier.invoke(identifierProcessor, identifier))) { + return false; + } + } + } + return activeH2DriverLoader(compatibility) == null; + } + private static void assertCompletionStateCleared(ClassLoader loader, long datasourceScope) throws Exception { Class contextType = Class.forName("ai.chat2db.spi.sql.Chat2DBContext", true, loader); diff --git a/packaging/macos/THIRD_PARTY_NOTICES.md b/packaging/macos/THIRD_PARTY_NOTICES.md index 7b3d835..615c0cc 100644 --- a/packaging/macos/THIRD_PARTY_NOTICES.md +++ b/packaging/macos/THIRD_PARTY_NOTICES.md @@ -7,6 +7,10 @@ revision, artifact names, byte lengths, and SHA-256 digests are recorded in: - `scripts/community-frontend.lock.json` - `third_party/community-h2-classpath.lock` - `target/mysql-driver-packs/01-mysql/driver-pack.json` +- `target/mysql-driver-packs/02-h2-migration/driver-pack.json` + +The H2 2.1.214 driver is bundled only for read-only migration of the previous +Chat2DB local store. H2 is available under MPL 2.0 or EPL 1.0. The package includes copies of both the Chat2DB Rust license and the pinned Chat2DB Community license under `Contents/Resources/chat2db/licenses`. diff --git a/scripts/build-community-h2-classpath.sh b/scripts/build-community-h2-classpath.sh index 3a7d04b..e26f1df 100755 --- a/scripts/build-community-h2-classpath.sh +++ b/scripts/build-community-h2-classpath.sh @@ -10,7 +10,7 @@ maven_repository="${repository_root}/target/community-m2" classpath_lock="${repository_root}/third_party/community-h2-classpath.lock" classpath_lock_tool="${repository_root}/scripts/community-classpath-lock.sh" classpath_sanitizer="${repository_root}/scripts/CommunityClasspathSanitizer.java" -expected_commit="37a34be858f2566b6b7fcf6c3f64183c1f560853" +expected_commit="3cb8af54cad5bd5caa20bb25f10d9b0e4f01931c" community_version="5.3.0" if [[ ! -e "${community_root}/.git" ]]; then @@ -19,8 +19,15 @@ if [[ ! -e "${community_root}/.git" ]]; then fi actual_commit="$(git -C "${community_root}" rev-parse HEAD)" -if [[ "${actual_commit}" != "${expected_commit}" ]]; then - echo "Community source must be pinned to ${expected_commit}; found ${actual_commit}" >&2 +if ! git -C "${community_root}" merge-base --is-ancestor "${expected_commit}" "${actual_commit}"; then + echo "Community source ${actual_commit} must descend from compatibility baseline ${expected_commit}" >&2 + exit 1 +fi + +expected_server_tree="$(git -C "${community_root}" rev-parse "${expected_commit}:chat2db-community-server")" +actual_server_tree="$(git -C "${community_root}" rev-parse "${actual_commit}:chat2db-community-server")" +if [[ "${actual_server_tree}" != "${expected_server_tree}" ]]; then + echo "Community server tree must match compatibility baseline ${expected_commit}" >&2 exit 1 fi diff --git a/scripts/community-frontend.lock.json b/scripts/community-frontend.lock.json index e53017f..9583e89 100644 --- a/scripts/community-frontend.lock.json +++ b/scripts/community-frontend.lock.json @@ -2,7 +2,7 @@ "repository": "https://github.com/OtterMind/Chat2DB.git", "submodulePath": "third_party/chat2db-community", "sourcePath": "chat2db-community-client", - "commit": "37a34be858f2566b6b7fcf6c3f64183c1f560853", - "tree": "07aacbd4db8f917c99c62a3b5d2aea0529c6945a", + "commit": "1c650f0e8a61d80b6b570e2cdcfc9c1b01f2a4e4", + "tree": "b08643418e5dd71a1ba4c955f16278a96e604939", "packageManager": "yarn@1.22.22" } diff --git a/scripts/community-frontend.mjs b/scripts/community-frontend.mjs index 3d917d4..ec98a05 100644 --- a/scripts/community-frontend.mjs +++ b/scripts/community-frontend.mjs @@ -193,9 +193,10 @@ function test() { setupUmi(worktree); runYarn(['test:chat-answer-update'], worktree); runYarn(['test:tree-title-highlight'], worktree); - runYarn(['test:canvas-lifecycle'], worktree); - runYarn(['test:result-resource-activity'], worktree); - runYarn(['test:workspace-resource-activity'], worktree); + runYarn(['test:deep-clone'], worktree); + runYarn(['test:ai-model-select'], worktree); + runYarn(['test:export-connections'], worktree); + runYarn(['test:host-file-transfer'], worktree); } function build() { diff --git a/scripts/prepare-h2-driver-pack.sh b/scripts/prepare-h2-driver-pack.sh new file mode 100755 index 0000000..5cab48c --- /dev/null +++ b/scripts/prepare-h2-driver-pack.sh @@ -0,0 +1,133 @@ +#!/usr/bin/env bash +set -euo pipefail + +readonly h2_version="2.1.214" +readonly h2_filename="h2-${h2_version}.jar" +readonly h2_sha256="d623cdc0f61d218cf549a8d09f1c391ff91096116b22e2475475fce4fbe72bd0" +readonly h2_bytes="2543012" +readonly h2_url="https://repo.maven.apache.org/maven2/com/h2database/h2/${h2_version}/${h2_filename}" +readonly pack_directory_name="02-h2-migration" + +repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +output_root="${1:-${repository_root}/target/mysql-driver-packs}" +source_jar="${H2_MIGRATION_DRIVER_JAR:-}" +staging_directory="" +backup_directory="" +pack_directory="" + +sha256_file() { + local path="$1" + local output + local digest + + if command -v sha256sum >/dev/null 2>&1; then + output="$(sha256sum -- "${path}")" + elif command -v shasum >/dev/null 2>&1; then + output="$(shasum -a 256 -- "${path}")" + elif command -v openssl >/dev/null 2>&1; then + output="$(openssl dgst -sha256 -r "${path}")" + else + echo "sha256sum, shasum, or openssl is required" >&2 + return 1 + fi + + digest="${output%% *}" + digest="$(printf '%s' "${digest}" | tr '[:upper:]' '[:lower:]')" + if [[ ! "${digest}" =~ ^[0-9a-f]{64}$ ]]; then + echo "invalid SHA-256 output for ${path}" >&2 + return 1 + fi + printf '%s' "${digest}" +} + +cleanup() { + if [[ -n "${staging_directory}" && -d "${staging_directory}" ]]; then + rm -rf -- "${staging_directory}" + fi + if [[ -n "${backup_directory}" && -d "${backup_directory}" ]]; then + if [[ -n "${pack_directory}" && ! -e "${pack_directory}" ]]; then + mv -- "${backup_directory}" "${pack_directory}" + else + rm -rf -- "${backup_directory}" + fi + fi +} +trap cleanup EXIT + +if [[ -e "${output_root}" && ( ! -d "${output_root}" || -L "${output_root}" ) ]]; then + echo "driver-pack root must be a non-symbolic directory: ${output_root}" >&2 + exit 1 +fi +mkdir -p "${output_root}" +output_root="$(cd "${output_root}" && pwd -P)" + +staging_directory="$(mktemp -d "${output_root}/.${pack_directory_name}.staging.XXXXXX")" +staged_jar="${staging_directory}/${h2_filename}" + +if [[ -n "${source_jar}" ]]; then + if [[ ! -f "${source_jar}" || -L "${source_jar}" ]]; then + echo "H2_MIGRATION_DRIVER_JAR must be a non-symbolic regular file" >&2 + exit 1 + fi + cp -- "${source_jar}" "${staged_jar}" +else + if ! command -v curl >/dev/null 2>&1; then + echo "curl is required when H2_MIGRATION_DRIVER_JAR is not set" >&2 + exit 1 + fi + curl --fail --location --silent --show-error \ + --retry 3 --retry-all-errors \ + --output "${staged_jar}" \ + "${h2_url}" +fi + +actual_bytes="$(LC_ALL=C wc -c < "${staged_jar}" | tr -d '[:space:]')" +if [[ "${actual_bytes}" != "${h2_bytes}" ]]; then + echo "H2 driver byte length mismatch: expected ${h2_bytes}, found ${actual_bytes}" >&2 + exit 1 +fi + +actual_sha256="$(sha256_file "${staged_jar}")" +if [[ "${actual_sha256}" != "${h2_sha256}" ]]; then + echo "H2 driver SHA-256 mismatch: expected ${h2_sha256}, found ${actual_sha256}" >&2 + exit 1 +fi + +printf '%s\n' \ + '{' \ + ' "schemaVersion": 1,' \ + ' "id": "h2-legacy-migration",' \ + ' "name": "H2 legacy migration",' \ + " \"version\": \"${h2_version}\"," \ + ' "driverClass": "org.h2.Driver",' \ + ' "artifacts": [' \ + ' {' \ + " \"path\": \"${h2_filename}\"," \ + " \"sha256\": \"${actual_sha256}\"" \ + ' }' \ + ' ]' \ + '}' > "${staging_directory}/driver-pack.json" +chmod 0644 "${staged_jar}" "${staging_directory}/driver-pack.json" + +pack_directory="${output_root}/${pack_directory_name}" +if [[ -e "${pack_directory}" ]]; then + if [[ ! -d "${pack_directory}" || -L "${pack_directory}" ]]; then + echo "refusing to replace unsafe pack path: ${pack_directory}" >&2 + exit 1 + fi + backup_directory="${output_root}/.${pack_directory_name}.previous.$$" + if [[ -e "${backup_directory}" ]]; then + echo "refusing to replace existing backup path: ${backup_directory}" >&2 + exit 1 + fi + mv -- "${pack_directory}" "${backup_directory}" +fi + +mv -- "${staging_directory}" "${pack_directory}" +staging_directory="" +if [[ -n "${backup_directory}" ]]; then + rm -rf -- "${backup_directory}" + backup_directory="" +fi + +echo "Prepared H2 ${h2_version} legacy-migration driver pack at ${output_root}" diff --git a/scripts/verify-community-h2-reproducibility.sh b/scripts/verify-community-h2-reproducibility.sh index ac0df94..49217cc 100755 --- a/scripts/verify-community-h2-reproducibility.sh +++ b/scripts/verify-community-h2-reproducibility.sh @@ -6,7 +6,7 @@ build_tool="${repository_root}/scripts/build-community-h2-classpath.sh" lock_tool="${repository_root}/scripts/community-classpath-lock.sh" sanitizer="${repository_root}/scripts/CommunityClasspathSanitizer.java" output_directory="${repository_root}/target/community-h2-classpath" -expected_commit="37a34be858f2566b6b7fcf6c3f64183c1f560853" +expected_commit="3cb8af54cad5bd5caa20bb25f10d9b0e4f01931c" first_lock="$(mktemp "${TMPDIR:-/tmp}/chat2db-community-first-lock.XXXXXX")" second_lock="$(mktemp "${TMPDIR:-/tmp}/chat2db-community-second-lock.XXXXXX")" diff --git a/scripts/verify-macos-package.sh b/scripts/verify-macos-package.sh index 88b09f0..b589371 100755 --- a/scripts/verify-macos-package.sh +++ b/scripts/verify-macos-package.sh @@ -49,7 +49,7 @@ fi "${repository_root}/scripts/community-classpath-lock.sh" verify \ "${community_classpath}" \ "${repository_root}/third_party/community-h2-classpath.lock" \ - "37a34be858f2566b6b7fcf6c3f64183c1f560853" + "3cb8af54cad5bd5caa20bb25f10d9b0e4f01931c" driver_manifest="${driver_root}/01-mysql/driver-pack.json" driver_jar="${driver_root}/01-mysql/mysql-connector-java-8.0.30.jar" @@ -62,6 +62,17 @@ if [[ -z "${expected_driver_sha}" || "${actual_driver_sha}" != "${expected_drive exit 1 fi +h2_manifest="${driver_root}/02-h2-migration/driver-pack.json" +h2_jar="${driver_root}/02-h2-migration/h2-2.1.214.jar" +require_file "${h2_manifest}" +require_file "${h2_jar}" +expected_h2_sha="$(awk -F '"' '/"sha256"/ { print $4; exit }' "${h2_manifest}")" +actual_h2_sha="$(shasum -a 256 -- "${h2_jar}" | awk '{ print $1 }')" +if [[ -z "${expected_h2_sha}" || "${actual_h2_sha}" != "${expected_h2_sha}" ]]; then + echo "packaged H2 migration driver digest does not match its manifest" >&2 + exit 1 +fi + "${java_bin}" -version while IFS= read -r module; do [[ -z "${module}" || "${module}" == \#* ]] && continue diff --git a/third_party/chat2db-community b/third_party/chat2db-community index 37a34be..1c650f0 160000 --- a/third_party/chat2db-community +++ b/third_party/chat2db-community @@ -1 +1 @@ -Subproject commit 37a34be858f2566b6b7fcf6c3f64183c1f560853 +Subproject commit 1c650f0e8a61d80b6b570e2cdcfc9c1b01f2a4e4 diff --git a/third_party/community-h2-classpath.lock b/third_party/community-h2-classpath.lock index 16dcef1..71324e6 100644 --- a/third_party/community-h2-classpath.lock +++ b/third_party/community-h2-classpath.lock @@ -1,5 +1,5 @@ format_version 1 -source_commit 37a34be858f2566b6b7fcf6c3f64183c1f560853 +source_commit 3cb8af54cad5bd5caa20bb25f10d9b0e4f01931c artifact_count 149 artifact FastInfoset-2.1.1.jar 75d6635e09101ef1545d9a59bb4d9524ec6bf15246540af5435750f271612765 317728 artifact HikariCP-5.0.1.jar 26d492397e6775b4296737a8919bf04047afe5827fdd2c08b4557595436b3a2b 161902 @@ -17,12 +17,12 @@ artifact bson-4.10.1.jar 739a338b2aa0b74d8df9273aac71e37a4e2f3d609658da3d48c9ab5 artifact cache-api-1.1.1.jar 9f34e007edfa82a7b2a2e1b969477dcf5099ce7f4f926fb54ce7e27c4a0cd54b 51281 artifact calcite-core-1.38.0.jar b55427c5e98fdc59e4722062d1414b18db3bc08998a00f42959751200ff09624 8315387 artifact calcite-linq4j-1.38.0.jar 1c4bb46a6ed58f62923ec20c64401f015ef59813647ddb2ca2fc60d7959a21df 518500 -artifact chat2db-community-domain-api-5.3.0.jar 5a8e98442095276ac70473a6f6f9a37fe79de29d158a7108951b619b551e2870 2100744 -artifact chat2db-community-domain-core-5.3.0.jar 85f6ebaee61ae8159cb667eb2a67e956faad4d89d1bad8c19365c3fa338fa8f5 1003427 -artifact chat2db-community-h2-5.3.0.jar b8043a40f149b3f32d48da8537856f56dbf6d51d15f01f40c1dc9368da9f9873 30480 -artifact chat2db-community-mysql-5.3.0.jar e9f4fbbdcccfbbb12a2a2bbbf90903456477a325ad454fdae1ec6d3da72f3ab8 5696156 -artifact chat2db-community-spi-5.3.0.jar 30a102ac21bb32a3de8682df5bc146823b5af3297f1bd77e0391f25bc3a25bcd 779815 -artifact chat2db-community-tools-5.3.0.jar 5f58b14c1e03bcb560b95f131283c6d2e692aeac04b2d9a68c80d871eed08a58 412683 +artifact chat2db-community-domain-api-5.3.0.jar 7db8f199d8eaea2acab9679baf06a5dbc4e89b010ee510ab1d733ffdeee292c3 2103037 +artifact chat2db-community-domain-core-5.3.0.jar a42c697b66c44a1a9cd7c470f7cb309bb77592e8e97bccaa35a7f2365016ac98 1007128 +artifact chat2db-community-h2-5.3.0.jar 673575ecd0b7bbb9e855c1816762ff667f8ff164b4fa6ee740a229961ee58538 50698 +artifact chat2db-community-mysql-5.3.0.jar 8c3410cc174687e0945909121b90979db357b67cb645435bb2bbf48ec17693b5 5716850 +artifact chat2db-community-spi-5.3.0.jar 6ee6a9cdae374cb1a82afbc08422b6cf9483d063b6f9033db39c9d14c4624ea0 782296 +artifact chat2db-community-tools-5.3.0.jar b3be839172759c7a2c730ebab168ed7ff31680ade457fdce0808f68817c50392 412670 artifact checker-qual-3.43.0.jar 3fbc2e98f05854c3df16df9abaa955b91b15b3ecac33623208ed6424640ef0f6 231525 artifact classmate-1.7.3.jar 75fbda45456f123fb6e2028a6189442d8d0730b357adce4c0a6d7e789f70669b 68249 artifact commons-beanutils-1.9.4.jar 7d938c81789028045c08c065e94be75fc280527620d5bd62b519d5838532368a 246918 @@ -77,7 +77,7 @@ artifact jakarta.validation-api-3.0.2.jar 291c25e6910cc6a7ebd96d4c6baebf6d7c3767 artifact janino-3.1.12.jar 5699878c31e71c9a94cf472f95a2dad52ca7b0449c099276b8bd7e992a43595b 956369 artifact javax.activation-api-1.2.0.jar 43fdef0b5b6ceb31b0424b208b930c74ab58fac2ceeb7b3f6fd3aeb8b5ca4393 56674 artifact jaxb-api-2.3.1.jar 88b955a0df57880a26a74708bc34f74dcaf8ebf4e78843a28b50eae945732b06 128076 -artifact jaxb-runtime-2.3.1.jar b82d10a3d67f8a8ab31d17b605de7f88fc0935e251d8261915ca87522fa9b8da 2461167 +artifact jaxb-runtime-2.3.1.jar 4f91b09f841152bac69de099f7f7c1547993bd10b289f0ac1f01e455839d5b20 2461167 artifact jboss-logging-3.6.3.Final.jar 7c12ee575508f81e22b1db9334b969d0ec54ef0fd1dcba24eeab1a44235fc366 62672 artifact jsch-0.2.9.jar 6d0ccb63c6d6003e2c55e46d41dec770276c7b305c0031236c67e1def544bdd6 525631 artifact json-path-2.9.0.jar 11a9ee6f88bb31f1450108d1cf6441377dec84aca075eb6bb2343be157575bea 276633