From 0a51913b05d6e4f395555cddceba32d2585dfc1a Mon Sep 17 00:00:00 2001 From: Lionel Lalande Date: Mon, 22 Jun 2026 13:54:50 +0200 Subject: [PATCH 01/17] fix: dont anonymize anonymized files --- src/Cli/AnonymizerService.cs | 3 +++ src/Cli/FilesWatcher.cs | 3 --- steps.md | 18 ++++++++++++++++++ 3 files changed, 21 insertions(+), 3 deletions(-) create mode 100644 steps.md diff --git a/src/Cli/AnonymizerService.cs b/src/Cli/AnonymizerService.cs index 4e2c7c9..8e604dd 100644 --- a/src/Cli/AnonymizerService.cs +++ b/src/Cli/AnonymizerService.cs @@ -29,6 +29,9 @@ public async Task AnonymizeFilesAsync(FileSystemInfo fileOrDir, CancellationToke public async Task AnonymizeFileAsync(FileInfo file, CancellationToken cancellationToken = default) { + // skip if already anonymized + if (file.Name.EndsWith(".anon.pdf")) return; + await _semaphore.WaitAsync(cancellationToken); try { diff --git a/src/Cli/FilesWatcher.cs b/src/Cli/FilesWatcher.cs index 5b15ebd..f10b314 100644 --- a/src/Cli/FilesWatcher.cs +++ b/src/Cli/FilesWatcher.cs @@ -79,9 +79,6 @@ private void RecreateWatcher(CancellationToken cancellationToken) { await Task.Delay(200, cancellationToken); - // skip if anonymized file - if (e.FullPath.EndsWith(".anon.pdf")) return; - LogFileFound(e.FullPath); FileInfo file = new(e.FullPath); _ = Task.Run(() => _anonymizer.AnonymizeFileAsync(file, cancellationToken), cancellationToken); diff --git a/steps.md b/steps.md new file mode 100644 index 0000000..77aa1cc --- /dev/null +++ b/steps.md @@ -0,0 +1,18 @@ +# Steps + +## Compilation + +- compilation Anonymizer.Cli.csproj -> anonymizer(.exe) +- téléchargement de UV +- création fichier compressé (7z) avec scripts/*.py et tools/uv(.exe) +- création fichier setup(.exe) en concaténant anonymizer(.exe) et le fichier compressé + magic + +## Installation + +- télécharge setup(.exe) +- exécute `setup` (commande `install` ou sans commande -> appel commande `install` en arrière plan ?) qui : + - copie le fichier compressé dans un répertoire temporaire + - l'extrait dans le répertoire d'installation + - crée anonymizer(.exe) sans le fichier compressé dans le répertoire d'installation + - ajoute le fichier de metadata (version, date, etc.) + - lance le téléchargement des modèles (`anonymizer(.exe) download`) dans le répertoire d'installation From fe9e69b10b90f36adf7a5be461f6235f8bd4ccd3 Mon Sep 17 00:00:00 2001 From: Lionel Lalande Date: Mon, 22 Jun 2026 13:58:35 +0200 Subject: [PATCH 02/17] feat: create aot executable - No runtime required - Smaller file --- src/Cli/Anonymizer.Cli.csproj | 15 ++++++++++++--- src/Cli/Lifetime/Metadata.cs | 20 ++++++++++++++++++-- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/src/Cli/Anonymizer.Cli.csproj b/src/Cli/Anonymizer.Cli.csproj index 885e2e7..322e00c 100644 --- a/src/Cli/Anonymizer.Cli.csproj +++ b/src/Cli/Anonymizer.Cli.csproj @@ -7,9 +7,18 @@ enable enable Anonymizer.Cli - true - true - true + + + + true + true + true + Speed + + + + true + full diff --git a/src/Cli/Lifetime/Metadata.cs b/src/Cli/Lifetime/Metadata.cs index d64f320..b681876 100644 --- a/src/Cli/Lifetime/Metadata.cs +++ b/src/Cli/Lifetime/Metadata.cs @@ -1,5 +1,6 @@ using System.Runtime.Versioning; using System.Text.Json; +using System.Text.Json.Serialization; namespace Anonymizer.Cli.Lifetime; @@ -13,13 +14,28 @@ internal sealed record class Metadata( public static async Task Load() { using var stream = File.OpenRead(Application.Metadata.Path); - var metadata = await JsonSerializer.DeserializeAsync(stream); + var metadata = await JsonSerializer.DeserializeAsync(stream, MetadataJsonContext.Default.Metadata); return metadata ?? new(null, null, string.Empty); } public async Task Save() { using var stream = File.Open(Application.Metadata.Path, FileMode.Create, FileAccess.Write); - await JsonSerializer.SerializeAsync(stream, this, Defaults.JsonOptions); + await JsonSerializer.SerializeAsync(stream, this, MetadataJsonContext.Default.Metadata); } } + +[JsonSerializable(typeof(Metadata))] +[JsonSourceGenerationOptions(Converters = new[] { typeof(VersionJsonConverter) })] +internal partial class MetadataJsonContext : JsonSerializerContext +{ +} + +internal sealed class VersionJsonConverter : JsonConverter +{ + public override Version? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => + Version.TryParse(reader.GetString(), out var v) ? v : null; + + public override void Write(Utf8JsonWriter writer, Version value, JsonSerializerOptions options) => + writer.WriteStringValue(value.ToString()); +} From 01aeab96d6f22be7459e9fcc6fb0770d186a929f Mon Sep 17 00:00:00 2001 From: Lionel Lalande Date: Mon, 22 Jun 2026 14:00:03 +0200 Subject: [PATCH 03/17] ci: create changelog from latest tag --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index dbe8cc7..9b536fd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -47,7 +47,7 @@ jobs: uses: orhun/git-cliff-action@v4 with: config: cliff.toml - args: --tag ${{ github.event.inputs.tag || github.ref_name }} -o artifacts/CHANGELOG.md + args: --latest --tag ${{ github.event.inputs.tag || github.ref_name }} -o artifacts/CHANGELOG.md env: GIT_CLIFF_OUTPUT: content From a388c047cb8db0134e99ef0d13fc75f434f326e2 Mon Sep 17 00:00:00 2001 From: Lionel Lalande Date: Fri, 3 Jul 2026 18:28:06 +0200 Subject: [PATCH 04/17] feat: use lib in Rust to anonymize pii --- src/Core/.gitignore | 1 + src/Core/Cargo.lock | 2190 ++++++++++++++++++++++++++++ src/Core/Cargo.toml | 17 + src/Core/build.ps1 | 4 + src/Core/src/_main.rs | 66 + src/Core/src/face_detector.rs | 97 ++ src/Core/src/font_map.rs | 277 ++++ src/Core/src/layout_analysis.rs | 203 +++ src/Core/src/lib.rs | 223 +++ src/Core/src/models.rs | 62 + src/Core/src/pdf_generator.rs | 189 +++ src/Core/src/pdf_parser.rs | 345 +++++ src/Core/src/pii_ner_detector.rs | 531 +++++++ src/Core/src/pii_regex_detector.rs | 91 ++ 14 files changed, 4296 insertions(+) create mode 100644 src/Core/.gitignore create mode 100644 src/Core/Cargo.lock create mode 100644 src/Core/Cargo.toml create mode 100644 src/Core/build.ps1 create mode 100644 src/Core/src/_main.rs create mode 100644 src/Core/src/face_detector.rs create mode 100644 src/Core/src/font_map.rs create mode 100644 src/Core/src/layout_analysis.rs create mode 100644 src/Core/src/lib.rs create mode 100644 src/Core/src/models.rs create mode 100644 src/Core/src/pdf_generator.rs create mode 100644 src/Core/src/pdf_parser.rs create mode 100644 src/Core/src/pii_ner_detector.rs create mode 100644 src/Core/src/pii_regex_detector.rs diff --git a/src/Core/.gitignore b/src/Core/.gitignore new file mode 100644 index 0000000..2f7896d --- /dev/null +++ b/src/Core/.gitignore @@ -0,0 +1 @@ +target/ diff --git a/src/Core/Cargo.lock b/src/Core/Cargo.lock new file mode 100644 index 0000000..5d87ac3 --- /dev/null +++ b/src/Core/Cargo.lock @@ -0,0 +1,2190 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "Anonymizer_Core" +version = "0.1.0" +dependencies = [ + "image", + "lopdf", + "ort", + "regex", + "serde", + "serde_json", + "tokenizers", + "ttf-parser", +] + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "aligned" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee4508988c62edf04abd8d92897fca0c2995d907ce1dfeaf369dac3716a40685" +dependencies = [ + "as-slice", +] + +[[package]] +name = "aligned-vec" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc890384c8602f339876ded803c97ad529f3842aba97f6392b3dba0dd171769b" +dependencies = [ + "equator", +] + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" + +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" + +[[package]] +name = "arg_enum_proc_macro" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ae92a5119aa49cdbcf6b9f893fe4e1d98b04ccbf82ee0584ad948a44a734dea" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + +[[package]] +name = "as-slice" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "516b6b4f0e40d50dcda9365d53964ec74560ad4284da2e7fc97122cd83174516" +dependencies = [ + "stable_deref_trait", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "av-scenechange" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f321d77c20e19b92c39e7471cf986812cbb46659d2af674adc4331ef3f18394" +dependencies = [ + "aligned", + "anyhow", + "arg_enum_proc_macro", + "arrayvec", + "log", + "num-rational", + "num-traits", + "pastey", + "rayon", + "thiserror 2.0.18", + "v_frame", + "y4m", +] + +[[package]] +name = "av1-grain" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cfddb07216410377231960af4fcab838eaa12e013417781b78bd95ee22077f8" +dependencies = [ + "anyhow", + "arrayvec", + "log", + "nom 8.0.0", + "num-rational", + "v_frame", +] + +[[package]] +name = "avif-serialize" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7178fe5f7d460b13895ebb9dcb28a3a6216d2df2574a0806cb51b555d297f38" +dependencies = [ + "arrayvec", +] + +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bit_field" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e4b40c7323adcfc0a41c4b88143ed58346ff65a288fc144329c5c45e05d70c6" + +[[package]] +name = "bitflags" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" + +[[package]] +name = "bitstream-io" +version = "4.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eff00be299a18769011411c9def0d827e8f2d7bf0c3dbf53633147a8867fd1f" +dependencies = [ + "no_std_io2", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "built" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c0e531d93d39c34eef561e929e8a7f86d77a5af08aac4f6d6e39976c51858e9" + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytemuck" +version = "1.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + +[[package]] +name = "bytes" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" + +[[package]] +name = "cc" +version = "1.2.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "num-traits", + "windows-link", +] + +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + +[[package]] +name = "console" +version = "0.15.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8" +dependencies = [ + "encode_unicode", + "libc", + "once_cell", + "unicode-width", + "windows-sys 0.59.0", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" + +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "either" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" + +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equator" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4711b213838dfee0117e3be6ac926007d7f433d7bbe33595975d4190cb07e6fc" +dependencies = [ + "equator-macro", +] + +[[package]] +name = "equator-macro" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44f23cf4b44bfce11a86ace86f8a73ffdec849c9fd00a386a53d278bd9e81fb3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "esaxx-rs" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" +dependencies = [ + "cc", +] + +[[package]] +name = "exr" +version = "1.74.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4300e043a56aa2cb633c01af81ca8f699a321879a7854d3896a0ba89056363be" +dependencies = [ + "bit_field", + "half", + "lebe", + "miniz_oxide", + "rayon-core", + "smallvec", + "zune-inflate", +] + +[[package]] +name = "fax" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caf1079563223d5d59d83c85886a56e586cfd5c1a26292e971a0fa266531ac5a" + +[[package]] +name = "fdeflate" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" +dependencies = [ + "simd-adler32", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + +[[package]] +name = "gif" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee8cfcc411d9adbbaba82fb72661cc1bcca13e8bba98b364e62b2dba8f960159" +dependencies = [ + "color_quant", + "weezl", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "hmac-sha256" +version = "1.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec9d92d097f4749b64e8cc33d924d9f40a2d4eb91402b458014b781f5733d60f" + +[[package]] +name = "http" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "image" +version = "0.25.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104" +dependencies = [ + "bytemuck", + "byteorder-lite", + "color_quant", + "exr", + "gif", + "image-webp", + "moxcms", + "num-traits", + "png", + "qoi", + "ravif", + "rayon", + "rgb", + "tiff", + "zune-core", + "zune-jpeg", +] + +[[package]] +name = "image-webp" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525e9ff3e1a4be2fbea1fdf0e98686a6d98b4d8f937e1bf7402245af1909e8c3" +dependencies = [ + "byteorder-lite", + "quick-error", +] + +[[package]] +name = "imgref" +version = "1.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89194689a993ab15268672e99e7b0e19da2da3268ac682e8f02d29d4d1434cd7" + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "indicatif" +version = "0.17.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235" +dependencies = [ + "console", + "number_prefix", + "portable-atomic", + "unicode-width", + "web-time", +] + +[[package]] +name = "interpolate_name" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c34819042dc3d3971c46c2190835914dfbe0c3c13f61449b2997f4e9722dfa60" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "lebe" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a79a3332a6609480d7d0c9eab957bca6b455b91bb84e66d19f5ff66294b85b8" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libfuzzer-sys" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9fd2f41a1cba099f79a0b6b6c35656cf7c03351a7bae8ff0f28f25270f929d2" +dependencies = [ + "arbitrary", + "cc", +] + +[[package]] +name = "linked-hash-map" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "loop9" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fae87c125b03c1d2c0150c90365d7d6bcc53fb73a9acaef207d2d065860f062" +dependencies = [ + "imgref", +] + +[[package]] +name = "lopdf" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5c14afa083a906d49e1bda105ddbf8175016e2658954e6d0c3e612f886df3db" +dependencies = [ + "chrono", + "encoding_rs", + "flate2", + "indexmap", + "itoa", + "linked-hash-map", + "log", + "md-5", + "nom 7.1.3", + "rayon", + "time", + "weezl", +] + +[[package]] +name = "lzma-rust2" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e20f57f9918e5bd7bc58c22cdd70a6afc7375d4dd9683af5f2b34bd3d2bba619" + +[[package]] +name = "macro_rules_attribute" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65049d7923698040cd0b1ddcced9b0eb14dd22c5f86ae59c3740eab64a676520" +dependencies = [ + "macro_rules_attribute-proc_macro", + "paste", +] + +[[package]] +name = "macro_rules_attribute-proc_macro" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670fdfda89751bc4a84ac13eaa63e205cf0fd22b4c9a5fbfa085b63c1f1d3a30" + +[[package]] +name = "matrixmultiply" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06de3016e9fae57a36fd14dba131fccf49f74b40b7fbdb472f96e361ec71a08" +dependencies = [ + "autocfg", + "rawpointer", +] + +[[package]] +name = "maybe-rayon" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea1f30cedd69f0a2954655f7188c6a834246d2bcf1e315e2ac40c4b24dc9519" +dependencies = [ + "cfg-if", + "rayon", +] + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + +[[package]] +name = "memchr" +version = "2.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "monostate" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3341a273f6c9d5bef1908f17b7267bbab0e95c9bf69a0d4dcf8e9e1b2c76ef67" +dependencies = [ + "monostate-impl", + "serde", + "serde_core", +] + +[[package]] +name = "monostate-impl" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4db6d5580af57bf992f59068d4ea26fd518574ff48d7639b255a36f9de6e7e9" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "moxcms" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b" +dependencies = [ + "num-traits", + "pxfm", +] + +[[package]] +name = "ndarray" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520080814a7a6b4a6e9070823bb24b4531daac8c4627e08ba5de8c5ef2f2752d" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + +[[package]] +name = "new_debug_unreachable" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" + +[[package]] +name = "no_std_io2" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418abd1b6d34fbf6cae440dc874771b0525a604428704c76e48b29a5e67b8003" +dependencies = [ + "memchr", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + +[[package]] +name = "noop_proc_macro" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0676bb32a98c1a483ce53e500a81ad9c3d5b3f7c920c28c24e9cb0980d0b5bc8" + +[[package]] +name = "num-bigint" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c863e9ab5e7bf9c99ba75e1050f1e4d624ae87ed3532d6238ffbdc7b585dbbe6" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "onig" +version = "6.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc3cbf698f9438986c11a880c90a6d04b9de27575afd28bbf45b154b6c709e2" +dependencies = [ + "bitflags", + "libc", + "once_cell", + "onig_sys", +] + +[[package]] +name = "onig_sys" +version = "69.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e68317604e77e53b85896388e1a803c1d21b74c899ec9e5e1112db90735edd7" +dependencies = [ + "cc", + "pkg-config", +] + +[[package]] +name = "ort" +version = "2.0.0-rc.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7de3af33d24a745ffb8fab904b13478438d1cd52868e6f17735ef6e1f8bf133" +dependencies = [ + "ndarray", + "ort-sys", + "smallvec", + "tracing", + "ureq", +] + +[[package]] +name = "ort-sys" +version = "2.0.0-rc.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7b497d21a8b6fbb4b5a544f8fadb77e801a09ae0add9e411d31c6f89e3c1e90" +dependencies = [ + "hmac-sha256", + "lzma-rust2", + "ureq", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pastey" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35fb2e5f958ec131621fdd531e9fc186ed768cbe395337403ae56c17a74c68ec" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "png" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61" +dependencies = [ + "bitflags", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide", +] + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "profiling" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d595e54a326bc53c1c197b32d295e14b169e3cfeaa8dc82b529f947fba6bcf5" +dependencies = [ + "profiling-procmacros", +] + +[[package]] +name = "profiling-procmacros" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4488a4a36b9a4ba6b9334a32a39971f77c1436ec82c38707bce707699cc3bbcb" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "pxfm" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0c5ccf5294c6ccd63a74f1565028353830a9c2f5eb0c682c355c471726a6e3f" + +[[package]] +name = "qoi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6d64c71eb498fe9eae14ce4ec935c555749aef511cca85b5568910d6e48001" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "quick-error" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" + +[[package]] +name = "quote" +version = "1.0.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rav1e" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43b6dd56e85d9483277cde964fd1bdb0428de4fec5ebba7540995639a21cb32b" +dependencies = [ + "aligned-vec", + "arbitrary", + "arg_enum_proc_macro", + "arrayvec", + "av-scenechange", + "av1-grain", + "bitstream-io", + "built", + "cfg-if", + "interpolate_name", + "itertools 0.14.0", + "libc", + "libfuzzer-sys", + "log", + "maybe-rayon", + "new_debug_unreachable", + "noop_proc_macro", + "num-derive", + "num-traits", + "paste", + "profiling", + "rand 0.9.4", + "rand_chacha 0.9.0", + "simd_helpers", + "thiserror 2.0.18", + "v_frame", + "wasm-bindgen", +] + +[[package]] +name = "ravif" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e52310197d971b0f5be7fe6b57530dcd27beb35c1b013f29d66c1ad73fbbcc45" +dependencies = [ + "avif-serialize", + "imgref", + "loop9", + "quick-error", + "rav1e", + "rayon", + "rgb", +] + +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-cond" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "059f538b55efd2309c9794130bc149c6a553db90e9d99c2030785c82f0bd7df9" +dependencies = [ + "either", + "itertools 0.11.0", + "rayon", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regex" +version = "1.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rgb" +version = "0.8.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b34b781b31e5d73e9fbc8689c70551fd1ade9a19e3e28cfec8580a79290cc4" + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls" +version = "0.23.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" +dependencies = [ + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + +[[package]] +name = "simd_helpers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95890f873bec569a0362c235787f3aca6e1e887302ba4840839bcc6459c42da6" +dependencies = [ + "quote", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "socks" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c3dbbd9ae980613c6dd8e28a9407b50509d3803b57624d5dfe8315218cd58b" +dependencies = [ + "byteorder", + "libc", + "winapi", +] + +[[package]] +name = "spm_precompiled" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326" +dependencies = [ + "base64 0.13.1", + "nom 7.1.3", + "serde", + "unicode-segmentation", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tiff" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63feaf3343d35b6ca4d50483f94843803b0f51634937cc2ec519fc32232bc52" +dependencies = [ + "fax", + "flate2", + "half", + "quick-error", + "weezl", + "zune-jpeg", +] + +[[package]] +name = "time" +version = "0.3.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18dfaaeddcb932337b5e7866ee7d0ce9b76d2fd092997146f187ec09b4558a50" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c431b87111666e491a90baa837f914fb45cd5dc3c268591b0220ff5057f2085f" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tokenizers" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e500fad1dd3af3d626327e6a3fe5050e664a6eaa4708b8ca92f1794aaf73e6fd" +dependencies = [ + "aho-corasick", + "derive_builder", + "esaxx-rs", + "getrandom 0.2.17", + "indicatif", + "itertools 0.12.1", + "lazy_static", + "log", + "macro_rules_attribute", + "monostate", + "onig", + "paste", + "rand 0.8.6", + "rayon", + "rayon-cond", + "regex", + "regex-syntax", + "serde", + "serde_json", + "spm_precompiled", + "thiserror 1.0.69", + "unicode-normalization-alignments", + "unicode-segmentation", + "unicode_categories", +] + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "ttf-parser" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17f77d76d837a7830fe1d4f12b7b4ba4192c1888001c7164257e4bc6d21d96b4" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-normalization-alignments" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de" +dependencies = [ + "smallvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "ureq" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dea7109cdcd5864d4eeb1b58a1648dc9bf520360d7af16ec26d0a9354bafcfc0" +dependencies = [ + "base64 0.22.1", + "log", + "percent-encoding", + "rustls", + "rustls-pki-types", + "socks", + "ureq-proto", + "utf8-zero", + "webpki-roots", +] + +[[package]] +name = "ureq-proto" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e994ba84b0bd1b1b0cf92878b7ef898a5c1760108fe7b6010327e274917a808c" +dependencies = [ + "base64 0.22.1", + "http", + "httparse", + "log", +] + +[[package]] +name = "utf8-zero" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" + +[[package]] +name = "v_frame" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "666b7727c8875d6ab5db9533418d7c764233ac9c0cff1d469aec8fa127597be2" +dependencies = [ + "aligned-vec", + "num-traits", + "wasm-bindgen", +] + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "weezl" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "y4m" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5a4b21e1a62b67a2970e6831bc091d7b87e119e7f9791aef9702e3bef04448" + +[[package]] +name = "zerocopy" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zune-core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb8a0807f7c01457d0379ba880ba6322660448ddebc890ce29bb64da71fb40f9" + +[[package]] +name = "zune-inflate" +version = "0.2.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ab332fe2f6680068f3582b16a24f90ad7096d5d39b974d1c0aff0125116f02" +dependencies = [ + "simd-adler32", +] + +[[package]] +name = "zune-jpeg" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296" +dependencies = [ + "zune-core", +] diff --git a/src/Core/Cargo.toml b/src/Core/Cargo.toml new file mode 100644 index 0000000..f78a3fe --- /dev/null +++ b/src/Core/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "Anonymizer_Core" +version = "0.1.0" +edition = "2024" + +[dependencies] +image = "0.25" +lopdf = "0.33.0" +ort = { version = "2.0.0-rc.4", default-features = false, features = [ "download-binaries", "tls-rustls", "std" ] } +regex = "1.10" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +tokenizers = "0.19.1" +ttf-parser = "0.20" + +[lib] +crate-type = ["cdylib"] diff --git a/src/Core/build.ps1 b/src/Core/build.ps1 new file mode 100644 index 0000000..ac0b9b8 --- /dev/null +++ b/src/Core/build.ps1 @@ -0,0 +1,4 @@ +$env:CFLAGS = "/MD" +$env:CXXFLAGS = "/MD" +cargo build --release +mv target/release/Anonymizer_Core.dll target/release/Anonymizer.Core.dll diff --git a/src/Core/src/_main.rs b/src/Core/src/_main.rs new file mode 100644 index 0000000..d5d6477 --- /dev/null +++ b/src/Core/src/_main.rs @@ -0,0 +1,66 @@ +// src/main.rs +// Test file to create a demo program. +mod models; + +use std::env; +use std::ffi::CString; + +fn main() { + // DEBUG + // let mut detector = PiiDetector::new("../Anonymizer/models"); + // match detector.find_pii("Salut Lionel ! GitHub : http://github.com/LionelLalande ") { + // Ok(spans) => println!("PII found: {:?}", spans), + // Err(e) => eprintln!("find_pii ERROR: {e}"), + // } + + // 1. Handling the 3 arguments required by the new pipeline + let args: Vec = env::args().collect(); + if args.len() < 4 { + eprintln!( + "Usage: {} ", + args[0] + ); + std::process::exit(1); + } + + let input_path = &args[1]; + let models_dir = &args[2]; + let output_path = &args[3]; + + println!("--- [Test CLI] Launching the pipeline via the FFI interface ---"); + + // 2. Simulation of C# behavior (Conversion to C-compatible strings for P/Invoke) + let c_input_path = CString::new(input_path.as_str()).expect("Failed to convert input_path"); + let c_models_dir = CString::new(models_dir.as_str()).expect("Failed to convert models_dir"); + let c_output_path = CString::new(output_path.as_str()).expect("Failed to convert output_path"); + + // 3. Call the function from your library (lib.rs) + let status = redact::redact_pdf( + c_input_path.as_ptr(), + c_models_dir.as_ptr(), + c_output_path.as_ptr(), + ); + + // 4. Handling the integer return code + match status { + 0 => println!("\n[Success CLI] The pipeline returned code 0."), + -1 => { + eprintln!("\n[Error CLI] Code -1: One of the passed pointers was null."); + std::process::exit(1); + } + -2 => { + eprintln!("\n[Error CLI] Code -2: UTF-8 conversion error in the arguments."); + std::process::exit(1); + } + -3 => { + eprintln!( + "\n[Error CLI] Code -3: An internal error occurred while processing the PDF." + ); + std::process::exit(1); + } + _ => { + eprintln!("\n[Error CLI] Unknown return code: {}", status); + std::process::exit(1); + } + } +} diff --git a/src/Core/src/face_detector.rs b/src/Core/src/face_detector.rs new file mode 100644 index 0000000..5c19ed0 --- /dev/null +++ b/src/Core/src/face_detector.rs @@ -0,0 +1,97 @@ +// src/face_detector.rs +use ort::session::Session; +use ort::value::Value; +use std::path::Path; + +pub struct FaceDetector { + session: Session, + input_name: String, + conf_threshold: f32, +} + +impl FaceDetector { + pub fn new>( + model_path: P, + conf_threshold: f32, + ) -> Result> { + let session = Session::builder()?.commit_from_file(model_path)?; + + let input_name = session.inputs()[0].name().to_string(); + + Ok(Self { + session, + input_name, + conf_threshold, + }) + } + + pub fn detect_faces(&mut self, image_bytes: &[u8]) -> bool { + if image_bytes.is_empty() { + return false; + } + + let input_tensor = match self.preprocess(image_bytes) { + Ok(tensor) => tensor, + Err(e) => { + eprintln!("[FaceDetector] Error during preprocessing: {e}"); + return false; + } + }; + + let outputs = match self.session.run(ort::inputs![ + &self.input_name => input_tensor + ]) { + Ok(out) => out, + Err(e) => { + eprintln!("[FaceDetector] Error during ONNX inference: {e}"); + return false; + } + }; + + let (shape, data) = match outputs[0].try_extract_tensor::() { + Ok(res) => res, + Err(_) => return false, + }; + + if shape.len() < 3 { + return false; + } + + let num_boxes = shape[1] as usize; + for i in 0..num_boxes { + let score_idx = (i * 2) + 1; + if score_idx < data.len() { + let score = data[score_idx]; + if score > self.conf_threshold { + return true; + } + } + } + + false + } + + fn preprocess(&self, image_bytes: &[u8]) -> Result> { + let img = image::load_from_memory(image_bytes)?; + + let resized = img.resize_exact(320, 240, image::imageops::FilterType::Triangle); + let rgb_image = resized.to_rgb8(); + let mut float_data = vec![0.0f32; 1 * 3 * 240 * 320]; + for y in 0..240 { + for x in 0..320 { + let pixel = rgb_image.get_pixel(x, y); + + let r_idx = 0 * (240 * 320) + (y as usize) * 320 + (x as usize); + let g_idx = 1 * (240 * 320) + (y as usize) * 320 + (x as usize); + let b_idx = 2 * (240 * 320) + (y as usize) * 320 + (x as usize); + + float_data[r_idx] = (pixel[0] as f32 - 127.0) / 128.0; + float_data[g_idx] = (pixel[1] as f32 - 127.0) / 128.0; + float_data[b_idx] = (pixel[2] as f32 - 127.0) / 128.0; + } + } + + let tensor = Value::from_array(([1, 3, 240, 320], float_data))?; + Ok(tensor.into()) + } +} diff --git a/src/Core/src/font_map.rs b/src/Core/src/font_map.rs new file mode 100644 index 0000000..063c070 --- /dev/null +++ b/src/Core/src/font_map.rs @@ -0,0 +1,277 @@ +// src/font_map.rs +// Decode character codes from a PDF stream (Tj/TJ) into real Unicode characters, +// relying on the cmap table of the actually embedded TrueType font. +// Necessary because many PDF generators (browsers, LibreOffice, etc.) +// subset fonts with an Identity-H encoding: the stream codes are arbitrary glyph +// indices (GID), NOT ASCII/Unicode codes. Interpreting them as raw UTF-8 produces +// random text and silently drops any glyph whose code represents an invalid UTF-8 +// byte (common: accents, ligatures, space...). +use lopdf::{Dictionary, Document, Object}; +use std::collections::HashMap; + +pub struct FontDecoder { + pub bytes_per_code: usize, // 1 for simple font, 2 for Type0/Identity-H + pub code_to_char: HashMap, + pub code_to_width: HashMap, // font_size ratio (horizontal advance, varies per glyph: legitimate) + pub default_width: f64, + // Height/offset: constants on ALL characters in the font (global ascender/descender), + // not the actual bounding box of each glyph. A comma, an "x", and an "É" should + // produce boxes of the same height aligned on the same line — like + // pdftotext/pdfplumber — rather than the actual visual bounding box of each + // glyphe qui varie par nature (jambages, hampes, ponctuation basse...). + pub height_ratio: f64, + pub y_offset_ratio: f64, +} + +pub struct GlyphInfo { + pub c: char, + pub width: f64, // font_size ratio +} + +impl FontDecoder { + /// Returns (glyph, offset_in_bytes) for each resolved code. + /// Important: a code absent from code_to_char is silently skipped, so + /// the offset CANNOT be deduced afterwards by simple position in the + /// result — we return it explicitly here, calculated during the only pass + /// that truly knows the position of each code in `bytes`. + pub fn decode(&self, bytes: &[u8]) -> Vec<(GlyphInfo, usize)> { + let mut out = Vec::new(); + let mut i = 0; + while i + self.bytes_per_code <= bytes.len() { + let code = if self.bytes_per_code == 2 { + ((bytes[i] as u32) << 8) | bytes[i + 1] as u32 + } else { + bytes[i] as u32 + }; + if let Some(&c) = self.code_to_char.get(&code) { + let w = self + .code_to_width + .get(&code) + .copied() + .unwrap_or(self.default_width); + out.push((GlyphInfo { c, width: w }, i)); + } + // If the code is absent from the mapping, we silently skip THIS glyph + // (better than a wrong character), rather than failing the entire run. + i += self.bytes_per_code; + } + out + } +} + +/// Builds a decoder for each font referenced by the page. +pub fn build_font_decoders(doc: &Document, page_id: (u32, u16)) -> HashMap, FontDecoder> { + let mut decoders = HashMap::new(); + let fonts = doc.get_page_fonts(page_id); + + for (name, font_dict) in fonts { + if let Some(decoder) = build_one_decoder(doc, font_dict) { + decoders.insert(name, decoder); + } + } + decoders +} + +fn build_one_decoder(doc: &Document, font_dict: &Dictionary) -> Option { + let subtype = font_dict.get(b"Subtype").ok()?.as_name_str().ok()?; + + if subtype == "Type0" { + // Composite font (CID), almost always Identity-H => 2 bytes per glyph. + let descendants = font_dict.get(b"DescendantFonts").ok()?.as_array().ok()?; + let desc_font_dict = resolve_dict(doc, descendants.first()?)?; + let font_descriptor = resolve_dict(doc, desc_font_dict.get(b"FontDescriptor").ok()?)?; + + // The embedded font gives us the actual widths (hmtx table), whether or not we have + // a ToUnicode CMap for the characters. We parse it in all cases. + let font_file = font_descriptor.get(b"FontFile2").ok()?; + let stream = doc + .get_object(font_file.as_reference().ok()?) + .and_then(Object::as_stream) + .ok()?; + let data = stream.decompressed_content().ok()?; + let face = ttf_parser::Face::parse(&data, 0).ok()?; + let units_per_em = face.units_per_em() as f64; + + let width_for_gid = |gid: u16| -> Option { + face.glyph_hor_advance(ttf_parser::GlyphId(gid)) + .map(|adv| adv as f64 / units_per_em) + }; + + // 1) Priority to the ToUnicode CMap if it exists for the characters (standard mechanism, + // the most reliable), but widths are always taken from the font. + if let Ok(tounicode_ref) = font_dict.get(b"ToUnicode") { + if let Ok(stream) = doc + .get_object(tounicode_ref.as_reference().ok()?) + .and_then(Object::as_stream) + { + if let Ok(content) = stream.decompressed_content() { + let map = parse_tounicode_cmap(&content); + if !map.is_empty() { + let mut code_to_width = HashMap::new(); + for &code in map.keys() { + // CIDToGIDMap Identity implicit: CID == GID for these subsets. + if let Some(w) = width_for_gid(code as u16) { + code_to_width.insert(code, w); + } + } + let (height_ratio, y_offset_ratio) = + ink_metrics(&face, map.keys().copied(), units_per_em); + return Some(FontDecoder { + bytes_per_code: 2, + code_to_char: map, + code_to_width, + default_width: 0.5, + height_ratio, + y_offset_ratio, + }); + } + } + } + } + + // 2) Otherwise, reconstruction from the cmap table of the embedded TrueType font + // (CIDToGIDMap Identity implicit: CID == GID for these subsets). + let mut gid_to_char = HashMap::new(); + let mut code_to_width = HashMap::new(); + // We test all reasonable Unicode code points and see which + // glyph the font associates with them, then we invert (gid -> char). + for cp in 0x20u32..=0x2FFFu32 { + if let Some(ch) = char::from_u32(cp) { + if let Some(gid) = face.glyph_index(ch) { + gid_to_char.entry(gid.0 as u32).or_insert(ch); + if let Some(w) = width_for_gid(gid.0) { + code_to_width.entry(gid.0 as u32).or_insert(w); + } + } + } + } + let (height_ratio, y_offset_ratio) = + ink_metrics(&face, gid_to_char.keys().copied(), units_per_em); + return Some(FontDecoder { + bytes_per_code: 2, + code_to_char: gid_to_char, + code_to_width, + default_width: 0.5, + height_ratio, + y_offset_ratio, + }); + } else { + // Simple font (1 byte per character). Default WinAnsiEncoding: + // essentially identical to Latin-1 in the useful range. + let mut code_to_char = HashMap::new(); + for b in 0x20u32..=0xFFu32 { + if let Some(c) = char::from_u32(b) { + code_to_char.insert(b, c); + } + } + + // Actual widths via the /Widths array + /FirstChar from the font dictionary + // (expressed in 1/1000 of an em in the PDF, as for standard fonts). + let mut code_to_width = HashMap::new(); + if let (Ok(first_char), Ok(widths)) = ( + font_dict.get(b"FirstChar").and_then(Object::as_i64), + font_dict.get(b"Widths").and_then(Object::as_array), + ) { + for (i, w) in widths.iter().enumerate() { + if let Ok(w) = w.as_float() { + code_to_width.insert(first_char as u32 + i as u32, w as f64 / 1000.0); + } + } + } + // No embedded font used here for the height: reasonable default values + // (simple font = mostly decorative/icon glyphs in this document, not critical). + return Some(FontDecoder { + bytes_per_code: 1, + code_to_char, + code_to_width, + default_width: 0.5, + height_ratio: 0.9, + y_offset_ratio: -0.2, + }); + } +} + +/// Uniform height/offset for the entire font, calibrated on the ink actually +/// used by this subset (max of the top, min of the bottom among the present glyphs) — +/// not on the declarative ascender/descender metrics of the font (hhea table), +/// often inflated for line spacing and therefore much too high. +fn ink_metrics( + face: &ttf_parser::Face, + gids: impl Iterator, + units_per_em: f64, +) -> (f64, f64) { + let mut min_y = i16::MAX; + let mut max_y = i16::MIN; + let mut found = false; + for gid in gids { + if let Some(r) = face.glyph_bounding_box(ttf_parser::GlyphId(gid as u16)) { + min_y = min_y.min(r.y_min); + max_y = max_y.max(r.y_max); + found = true; + } + } + if !found { + return (0.7, 0.0); + } + ( + (max_y - min_y) as f64 / units_per_em, + min_y as f64 / units_per_em, + ) +} + +fn resolve_dict<'a>(doc: &'a Document, obj: &'a Object) -> Option<&'a Dictionary> { + match obj { + Object::Reference(id) => doc.get_dictionary(*id).ok(), + Object::Dictionary(d) => Some(d), + _ => None, + } +} + +/// Minimal parser for a CMap ToUnicode (bfchar / bfrange sections in hexadecimal). +fn parse_tounicode_cmap(content: &[u8]) -> HashMap { + let text = String::from_utf8_lossy(content); + let mut map = HashMap::new(); + + for block in text.split("beginbfchar").skip(1) { + let block = block.split("endbfchar").next().unwrap_or(""); + for line in block.lines() { + let hexes: Vec<&str> = line + .split(|c| c == '<' || c == '>') + .filter(|s| !s.trim().is_empty()) + .collect(); + if hexes.len() >= 2 { + if let (Ok(src), Ok(dst)) = ( + u32::from_str_radix(hexes[0], 16), + u32::from_str_radix(&hexes[1][..hexes[1].len().min(4)], 16), + ) { + if let Some(c) = char::from_u32(dst) { + map.insert(src, c); + } + } + } + } + } + for block in text.split("beginbfrange").skip(1) { + let block = block.split("endbfrange").next().unwrap_or(""); + for line in block.lines() { + let hexes: Vec<&str> = line + .split(|c| c == '<' || c == '>') + .filter(|s| !s.trim().is_empty()) + .collect(); + if hexes.len() >= 3 { + if let (Ok(lo), Ok(hi), Ok(dst)) = ( + u32::from_str_radix(hexes[0], 16), + u32::from_str_radix(hexes[1], 16), + u32::from_str_radix(&hexes[2][..hexes[2].len().min(4)], 16), + ) { + for (offset, code) in (lo..=hi).enumerate() { + if let Some(c) = char::from_u32(dst + offset as u32) { + map.insert(code, c); + } + } + } + } + } + } + map +} diff --git a/src/Core/src/layout_analysis.rs b/src/Core/src/layout_analysis.rs new file mode 100644 index 0000000..cc0e9c2 --- /dev/null +++ b/src/Core/src/layout_analysis.rs @@ -0,0 +1,203 @@ +// src/layout_analysis.rs +use crate::models::{Block, Line, Paragraph, Word}; + +pub fn build_segmented_layout(pages_words: Vec>) -> Vec { + let mut all_blocks = Vec::new(); + let mut global_font_sum = 0.0; + let mut global_word_count = 0.0; + + for words in pages_words { + if words.is_empty() { + continue; + } + for w in &words { + global_font_sum += w.font_size; + global_word_count += 1.0; + } + + let lines = words_to_lines(words); + let blocks = lines_to_blocks(lines); + all_blocks.extend(blocks); + } + + let avg_font_size = if global_word_count > 0.0 { + global_font_sum / global_word_count + } else { + 10.0 + }; + let mut paragraphs = Vec::new(); + if all_blocks.is_empty() { + return paragraphs; + } + + let mut current_para_text = String::new(); + let mut current_para_blocks = Vec::new(); + + for block in all_blocks { + let mut block_text = String::new(); + let mut block_font_sum = 0.0; + let mut block_word_count = 0.0; + + for line in &block.lines { + for word in &line.words { + block_text.push_str(&word.text); + block_text.push(' '); + block_font_sum += word.font_size; + block_word_count += 1.0; + } + } + let block_avg_font = if block_word_count > 0.0 { + block_font_sum / block_word_count + } else { + 10.0 + }; + let is_heading = block_avg_font > (avg_font_size * 1.25); + + if is_heading { + if !current_para_blocks.is_empty() { + paragraphs.push(Paragraph { + text: current_para_text.trim().to_string(), + blocks: current_para_blocks.clone(), + is_heading: false, + }); + current_para_text.clear(); + current_para_blocks.clear(); + } + paragraphs.push(Paragraph { + text: block_text.trim().to_string(), + blocks: vec![block], + is_heading: true, + }); + } else { + let ends_with_punctuation = block_text.trim().ends_with('.') + || block_text.trim().ends_with(':') + || block_text.trim().ends_with('!'); + current_para_text.push_str(&block_text); + current_para_blocks.push(block); + + if ends_with_punctuation { + paragraphs.push(Paragraph { + text: current_para_text.trim().to_string(), + blocks: current_para_blocks.clone(), + is_heading: false, + }); + current_para_text.clear(); + current_para_blocks.clear(); + } + } + } + + if !current_para_blocks.is_empty() { + paragraphs.push(Paragraph { + text: current_para_text.trim().to_string(), + blocks: current_para_blocks, + is_heading: false, + }); + } + + paragraphs +} + +fn words_to_lines(mut words: Vec) -> Vec { + words.sort_by(|a, b| { + b.baseline_y + .partial_cmp(&a.baseline_y) + .unwrap() + .then(a.bbox.x.partial_cmp(&b.bbox.x).unwrap()) + }); + let mut lines: Vec = Vec::new(); + + let is_special_str = |s: &str| { + if s.len() != 1 { + return false; + } + let c = s.chars().next().unwrap(); + c.is_ascii_punctuation() + }; + + for word in words { + let mut added = false; + for line in lines.iter_mut() { + // Lenient alignment to integrate isolated punctuation into the current line + let lenient = is_special_str(&word.text); + let max_v = if lenient { word.font_size * 0.8 } else { 6.0 }; + + // Comparison on the baseline (stable), not on the visual bbox which + // varies according to the stems/descenders of the word (a "g" and a "T" do not have the + // same bbox.y even though they are on the same line). + let inline = (word.baseline_y - line.baseline_y).abs() < max_v; + let no_column_gap = + (word.bbox.x - (line.bbox.x + line.bbox.width)).abs() < (word.font_size * 2.5); + + if inline && no_column_gap { + // Physical fusion of BBoxes in the line (purely visual, the reference baseline + // of the line itself does not change) + let min_x = line.bbox.x.min(word.bbox.x); + let max_x = (line.bbox.x + line.bbox.width).max(word.bbox.x + word.bbox.width); + let min_y = line.bbox.y.min(word.bbox.y); + let max_y = (line.bbox.y + line.bbox.height).max(word.bbox.y + word.bbox.height); + + line.bbox.x = min_x; + line.bbox.width = max_x - min_x; + line.bbox.y = min_y; + line.bbox.height = max_y - min_y; + + line.words.push(word.clone()); + added = true; + break; + } + } + if !added { + lines.push(Line { + bbox: word.bbox.clone(), + page_id: word.page_id, + baseline_y: word.baseline_y, + words: vec![word], + }); + } + } + lines +} + +fn lines_to_blocks(mut lines: Vec) -> Vec { + lines.sort_by(|a, b| b.baseline_y.partial_cmp(&a.baseline_y).unwrap()); + let mut blocks: Vec = Vec::new(); + + for line in lines { + let mut added = false; + for block in blocks.iter_mut() { + // Distance to the baseline of the LAST added line (actual line spacing), + // rather than to block.bbox.y which drifts with cumulative stems/descenders. + let close_y = (line.baseline_y - block.baseline_y).abs() < 18.0; + // Slight horizontal margin to capture punctuation shifted at the end of the block + let x_overlap = line.bbox.x < (block.bbox.x + block.bbox.width + 10.0) + && (line.bbox.x + line.bbox.width) > (block.bbox.x - 10.0); + + if close_y && x_overlap { + let min_x = block.bbox.x.min(line.bbox.x); + let min_y = block.bbox.y.min(line.bbox.y); + let max_x = (block.bbox.x + block.bbox.width).max(line.bbox.x + line.bbox.width); + let max_y = (block.bbox.y + block.bbox.height).max(line.bbox.y + line.bbox.height); + + block.bbox.x = min_x; + block.bbox.y = min_y; + block.bbox.width = max_x - min_x; + block.bbox.height = max_y - min_y; + block.baseline_y = line.baseline_y; + + block.lines.push(line.clone()); + added = true; + break; + } + } + if !added { + blocks.push(Block { + bbox: line.bbox.clone(), + page_id: line.page_id, + baseline_y: line.baseline_y, + lines: vec![line], + }); + } + } + blocks +} diff --git a/src/Core/src/lib.rs b/src/Core/src/lib.rs new file mode 100644 index 0000000..b824518 --- /dev/null +++ b/src/Core/src/lib.rs @@ -0,0 +1,223 @@ +// src/lib.rs +mod face_detector; +mod font_map; +mod layout_analysis; +mod models; +mod pdf_generator; +mod pdf_parser; +mod pii_ner_detector; +mod pii_regex_detector; + +use face_detector::FaceDetector; +use lopdf::{Dictionary, Document, Object}; +use pii_ner_detector::PiiNerDetector; +use pii_regex_detector::PiiRegexDetector; + +use std::ffi::CStr; +use std::os::raw::c_char; +use std::path::Path; + +/// Redact PII information from a PDF file using a hybrid approach (NER + Regex + Face Detection). +/// Returns an integer status code indicating the result of the operation: +/// 0 = Success +/// -1 = One or more input pointers are null +/// -2 = UTF-8 conversion error on input strings +/// -3 = Internal error during pipeline execution +#[unsafe(no_mangle)] +pub extern "C" fn redact_pdf( + c_input_path: *const c_char, + c_models_dir: *const c_char, + c_output_path: *const c_char, +) -> i32 { + let result = std::panic::catch_unwind(|| { + // Safety check for null pointers + if c_input_path.is_null() || c_models_dir.is_null() || c_output_path.is_null() { + return -1; + } + + // Safe conversion of C strings (*const c_char) to Rust &str + let input_path = unsafe { + match CStr::from_ptr(c_input_path).to_str() { + Ok(s) => s, + Err(_) => return -2, + } + }; + + let models_dir = unsafe { + match CStr::from_ptr(c_models_dir).to_str() { + Ok(s) => s, + Err(_) => return -2, + } + }; + + let output_path = unsafe { + match CStr::from_ptr(c_output_path).to_str() { + Ok(s) => s, + Err(_) => return -2, + } + }; + + // Execute the pipeline and capture errors + match run_redaction_pipeline(input_path, models_dir, output_path) { + Ok(_) => 0, + Err(e) => { + eprintln!("[Error] Pipeline execution failed: {}", e); + -3 + } + } + }); + match result { + Ok(code) => code, + Err(_) => { + eprintln!("[Error] Internal panic."); + -4 + } + } +} + +// Internal logic of the global redaction pipeline +fn run_redaction_pipeline( + input_path: &str, + models_dir: &str, + output_path: &str, +) -> Result<(), Box> { + let file_prefix = Path::new(input_path) + .file_stem() + .and_then(|s| s.to_str()) + .unwrap_or("document"); + + let output_path_path = Path::new(output_path); + + println!("Loading the source file: {}...", input_path); + let mut doc = Document::load(input_path)?; + + // Low-level extraction (Letters -> Physical Words) + let mut all_pages_words = Vec::new(); + for page_id in doc.page_iter() { + let words = pdf_parser::extract_words(&doc, page_id)?; + all_pages_words.push(words); + } + + println!("Semantic analysis..."); + let paragraphs = layout_analysis::build_segmented_layout(all_pages_words); + + println!("Initializing detection engines (Multi-agents)..."); + + // NER Detector (ONNX Language Model) + let mut ner_detector = PiiNerDetector::new(models_dir); + + // Deterministic Detector (Regex) + let regex_detector = PiiRegexDetector::new(); + + // Face Detector (UltraFace ONNX) + let face_model_path = Path::new(models_dir) + .join("face") + .join("model.onnx"); + let mut face_detector = FaceDetector::new(&face_model_path, 0.7)?; + + println!("Analyzing text... (could take a while for large documents)"); + let mut redaction_targets = ner_detector.scan_pii(¶graphs); + let ner_count = redaction_targets.len(); + let mut regex_count = 0; + for paragraph in ¶graphs { + let regex_spans = regex_detector.analyze_text(¶graph.text); + if regex_spans.is_empty() { + continue; + } + + let mut search_offset = 0; + for word in paragraph + .blocks + .iter() + .flat_map(|b| &b.lines) + .flat_map(|l| &l.words) + { + let Some(local_idx) = paragraph.text[search_offset..].find(&word.text) else { + continue; + }; + let word_start = search_offset + local_idx; + let word_end = word_start + word.text.len(); + + let is_regex_pii = regex_spans + .iter() + .any(|span| word_start.max(span.start) < word_end.min(span.end)); + + if is_regex_pii { + // CORRECTION 1 : Remplacement de std::ptr::eq par une comparaison structurelle de valeur + let already_exists = redaction_targets.iter().any(|w| { + w.page_id == word.page_id && w.bbox == word.bbox && w.text == word.text + }); + + if !already_exists { + redaction_targets.push(word.clone()); + regex_count += 1; + } + } + search_offset = word_end; + } + } + println!( + " -> NER Engine : {} occurrence(s) identified.", + ner_count + ); + println!( + " -> Regex Engine : {} new occurrence(s) added.", + regex_count + ); + + println!("Analyzing graphic resources (Face Detection)..."); + let mut face_detected_total = 0; + + let object_ids: Vec = doc.objects.keys().copied().collect(); + for id in object_ids { + if let Ok(Object::Stream(stream)) = doc.get_object_mut(id) { + if is_image_stream(&stream.dict) { + let _ = stream.decompress(); + let image_bytes = &stream.content; + + if face_detector.detect_faces(image_bytes) { + face_detected_total += 1; + + stream.content = vec![]; + stream + .dict + .set("Filter", Object::Name(b"FlateDecode".to_vec())); + } + } + } + } + + if face_detected_total > 0 { + println!( + " -> Face Engine : {} image(s) containing faces purged.", + face_detected_total + ); + } else { + println!(" -> Face Engine : No faces detected in images."); + } + + let output_str = output_path_path + .to_str() + .ok_or("Invalid final output path")?; + println!("Applying text redaction -> {}...", output_str); + + // Save by applying black masks on textual `Word` structures + pdf_generator::apply_text_redaction(&mut doc, &redaction_targets)?; + + doc.save(output_str)?; + + println!("\n[Success] Anonymization pipeline executed successfully."); + Ok(()) +} + +/// Utility function to identify if a lopdf object dictionary corresponds to an image +fn is_image_stream(dict: &Dictionary) -> bool { + if let Ok(Object::Name(type_name)) = dict.get(b"Type") { + if type_name == b"XObject" { + if let Ok(Object::Name(subtype_name)) = dict.get(b"Subtype") { + return subtype_name == b"Image"; + } + } + } + false +} diff --git a/src/Core/src/models.rs b/src/Core/src/models.rs new file mode 100644 index 0000000..35be4df --- /dev/null +++ b/src/Core/src/models.rs @@ -0,0 +1,62 @@ +// src/models.rs +use lopdf::ObjectId; + +#[derive(Debug, Clone, PartialEq)] +pub struct BBox { + pub x: f64, + pub y: f64, + pub width: f64, + pub height: f64, +} + +// Precisely locate the source byte of a letter in the content stream, to +// be able to actually erase it when writing (not just paint over it). +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct SourceRef { + pub op_index: usize, // index in content.operations + pub array_item: Option, // Some(i) if the operand is a TJ array, None if Tj/'/" + pub byte_start: usize, // offset in the bytes of the relevant PDF string + pub byte_len: usize, // number of bytes occupied by the code of this glyph (1 or 2) +} + +#[derive(Debug, Clone, PartialEq)] +pub struct Letter { + pub value: char, + pub bbox: BBox, + pub font_size: f64, + pub baseline_y: f64, // position of the baseline (stable, independent of stems/descenders) — to be used for any grouping/alignment + pub src: SourceRef, +} + +#[derive(Debug, Clone)] +pub struct Word { + pub text: String, + pub bbox: BBox, + pub font_size: f64, + pub page_id: ObjectId, + pub letters: Vec, // Memorization of letters for light green tracing + pub baseline_y: f64, +} + +#[derive(Debug, Clone)] +pub struct Line { + pub words: Vec, + pub bbox: BBox, + pub page_id: ObjectId, + pub baseline_y: f64, +} + +#[derive(Debug, Clone)] +pub struct Block { + pub lines: Vec, + pub bbox: BBox, + pub page_id: ObjectId, + pub baseline_y: f64, // baseline of the last added line, used to judge line spacing +} + +#[derive(Debug, Clone)] +pub struct Paragraph { + pub text: String, + pub blocks: Vec, + pub is_heading: bool, +} diff --git a/src/Core/src/pdf_generator.rs b/src/Core/src/pdf_generator.rs new file mode 100644 index 0000000..51f0612 --- /dev/null +++ b/src/Core/src/pdf_generator.rs @@ -0,0 +1,189 @@ +// src/pdf_generator.rs +use crate::models::{Paragraph, Word}; +use lopdf::{ + Document, Object, ObjectId, + content::{Content, Operation}, +}; + +pub fn apply_text_redaction( + doc: &mut Document, + redactions: &[Word], +) -> Result<(), Box> { + let page_ids: Vec = doc.page_iter().collect(); + + for page_id in page_ids { + let page_redactions: Vec<&Word> = + redactions.iter().filter(|w| w.page_id == page_id).collect(); + if page_redactions.is_empty() { + continue; + } + + let content_data = doc.get_page_content(page_id)?; + let mut content = Content::decode(&content_data)?; + + // 1. Actual erasure of the source text + for word in &page_redactions { + for letter in &word.letters { + let src = &letter.src; + let Some(op) = content.operations.get_mut(src.op_index) else { + continue; + }; + let string_obj = match src.array_item { + Some(idx) => match op.operands.first_mut() { + Some(Object::Array(arr)) => arr.get_mut(idx), + _ => None, + }, + None => op.operands.last_mut(), + }; + if let Some(Object::String(bytes, _)) = string_obj { + for b in bytes.iter_mut().skip(src.byte_start).take(src.byte_len) { + *b = 0; + } + } + } + } + + // 2. Visual masking (white box with black border) + content.operations.insert(0, Operation::new("q", vec![])); + content.operations.push(Operation::new("Q", vec![])); + + for word in &page_redactions { + let bbox = &word.bbox; + content.operations.push(Operation::new("q", vec![])); + content + .operations + .push(Operation::new("w", vec![Object::Real(1.0)])); + content.operations.push(Operation::new( + "rg", + vec![Object::Real(1.0), Object::Real(1.0), Object::Real(1.0)], + )); + content.operations.push(Operation::new( + "RG", + vec![Object::Real(0.0), Object::Real(0.0), Object::Real(0.0)], + )); + content.operations.push(Operation::new( + "re", + vec![ + Object::Real((bbox.x - 1.0) as f32), + Object::Real((bbox.y - 1.0) as f32), + Object::Real((bbox.width + 2.0) as f32), + Object::Real((bbox.height + 2.0) as f32), + ], + )); + content.operations.push(Operation::new("B", vec![])); + content.operations.push(Operation::new("Q", vec![])); + } + + doc.change_page_content(page_id, content.encode()?)?; + } + + Ok(()) +} + +// keep for DEBUG +pub fn write_layout_debug( + doc_path: &str, + paragraphs: &[Paragraph], + out_path: &str, +) -> Result<(), Box> { + let mut doc = Document::load(doc_path)?; + let page_ids: Vec = doc.page_iter().collect(); + + for page_id in page_ids { + let content_data = doc.get_page_content(page_id)?; + let mut content = Content::decode(&content_data)?; + + content.operations.insert(0, Operation::new("q", vec![])); + content.operations.push(Operation::new("Q", vec![])); + + let mut has_drawings = false; + + for para in paragraphs { + for block in ¶.blocks { + if block.page_id != page_id { + continue; + } + has_drawings = true; + + // 1. RED BLOCKS (Paragraphs) - Expanded by 3.5 pixels outward + let color = if para.is_heading { + vec![Object::Real(0.7), Object::Real(0.0), Object::Real(1.0)] // Violet + } else { + vec![Object::Real(1.0), Object::Real(0.0), Object::Real(0.0)] // Red + }; + + content.operations.push(Operation::new("q", vec![])); + content + .operations + .push(Operation::new("w", vec![Object::Real(1.0)])); + content.operations.push(Operation::new("RG", color)); + content.operations.push(Operation::new( + "re", + vec![ + Object::Real((block.bbox.x - 3.5) as f32), + Object::Real((block.bbox.y - 3.5) as f32), + Object::Real((block.bbox.width + 7.0) as f32), + Object::Real((block.bbox.height + 7.0) as f32), + ], + )); + content.operations.push(Operation::new("S", vec![])); + content.operations.push(Operation::new("Q", vec![])); + + for line in &block.lines { + for word in &line.words { + // 2. BLUE BLOCKS (Words) - Expanded by 1.5 pixels outward + content.operations.push(Operation::new("q", vec![])); + content + .operations + .push(Operation::new("w", vec![Object::Real(0.5)])); + content.operations.push(Operation::new( + "RG", + vec![Object::Real(0.0), Object::Real(0.0), Object::Real(1.0)], + )); + content.operations.push(Operation::new( + "re", + vec![ + Object::Real((word.bbox.x - 1.5) as f32), + Object::Real((word.bbox.y - 1.5) as f32), + Object::Real((word.bbox.width + 3.0) as f32), + Object::Real((word.bbox.height + 3.0) as f32), + ], + )); + content.operations.push(Operation::new("S", vec![])); + content.operations.push(Operation::new("Q", vec![])); + + // 3. LIGHT GREEN BLOCKS (Letters) - Exact pixel-perfect coordinates + for letter in &word.letters { + content.operations.push(Operation::new("q", vec![])); + content + .operations + .push(Operation::new("w", vec![Object::Real(0.3)])); + content.operations.push(Operation::new( + "RG", + vec![Object::Real(0.3), Object::Real(0.8), Object::Real(0.3)], + )); // Light green + content.operations.push(Operation::new( + "re", + vec![ + Object::Real(letter.bbox.x as f32), + Object::Real(letter.bbox.y as f32), + Object::Real(letter.bbox.width as f32), + Object::Real(letter.bbox.height as f32), + ], + )); + content.operations.push(Operation::new("S", vec![])); + content.operations.push(Operation::new("Q", vec![])); + } + } + } + } + } + + if has_drawings { + doc.change_page_content(page_id, content.encode()?)?; + } + } + + doc.save(out_path)?; + Ok(()) +} diff --git a/src/Core/src/pdf_parser.rs b/src/Core/src/pdf_parser.rs new file mode 100644 index 0000000..9d15ceb --- /dev/null +++ b/src/Core/src/pdf_parser.rs @@ -0,0 +1,345 @@ +// src/pdf_parser.rs +use crate::font_map::{self, FontDecoder}; +use crate::models::{BBox, Letter, Word}; +use lopdf::{Document, Object, ObjectId, content::Content}; +use std::collections::HashMap; + +fn multiply_matrices(m2: &[f64; 6], m1: &[f64; 6]) -> [f64; 6] { + [ + m2[0] * m1[0] + m2[1] * m1[2], + m2[0] * m1[1] + m2[1] * m1[3], + m2[2] * m1[0] + m2[3] * m1[2], + m2[2] * m1[1] + m2[3] * m1[3], + m2[4] * m1[0] + m2[5] * m1[2] + m1[4], + m2[4] * m1[1] + m2[5] * m1[3] + m1[5], + ] +} + +fn obj_to_f64(obj: &Object) -> f64 { + match obj { + Object::Integer(i) => *i as f64, + Object::Real(f) => *f as f64, + _ => 0.0, + } +} + +pub fn extract_words( + doc: &Document, + page_id: ObjectId, +) -> Result, Box> { + let content_data = doc.get_page_content(page_id)?; + let content = Content::decode(&content_data)?; + let decoders: HashMap, FontDecoder> = font_map::build_font_decoders(doc, page_id); + + let mut letters = Vec::new(); + let mut ctm = [1.0, 0.0, 0.0, 1.0, 0.0, 0.0]; + let mut ctm_stack = Vec::new(); + let mut tm = [1.0, 0.0, 0.0, 1.0, 0.0, 0.0]; + let mut lm = [1.0, 0.0, 0.0, 1.0, 0.0, 0.0]; + let mut font_size = 10.0; + let mut current_font: Option> = None; + + for (op_index, op) in content.operations.iter().enumerate() { + match op.operator.as_str() { + "q" => ctm_stack.push(ctm), + "Q" => { + if let Some(old) = ctm_stack.pop() { + ctm = old; + } + } + "cm" => { + let m = [ + obj_to_f64(&op.operands[0]), + obj_to_f64(&op.operands[1]), + obj_to_f64(&op.operands[2]), + obj_to_f64(&op.operands[3]), + obj_to_f64(&op.operands[4]), + obj_to_f64(&op.operands[5]), + ]; + ctm = multiply_matrices(&m, &ctm); + } + "Tf" => { + if let Some(Object::Name(name)) = op.operands.first() { + current_font = Some(name.clone()); + } + if let Some(size) = op.operands.get(1).and_then(|o| o.as_float().ok()) { + font_size = size as f64; + } + } + "BT" => { + tm = [1.0, 0.0, 0.0, 1.0, 0.0, 0.0]; + lm = tm; + } + "Td" | "TD" => { + let x = obj_to_f64(&op.operands[0]); + let y = obj_to_f64(&op.operands[1]); + lm = multiply_matrices(&[1.0, 0.0, 0.0, 1.0, x, y], &lm); + tm = lm; + } + "Tm" => { + tm = [ + obj_to_f64(&op.operands[0]), + obj_to_f64(&op.operands[1]), + obj_to_f64(&op.operands[2]), + obj_to_f64(&op.operands[3]), + obj_to_f64(&op.operands[4]), + obj_to_f64(&op.operands[5]), + ]; + lm = tm; + } + "T*" => { + lm = multiply_matrices(&[1.0, 0.0, 0.0, 1.0, 0.0, -font_size], &lm); + tm = lm; + } + "Tj" | "'" | "\"" => { + if let Some(Object::String(bytes, _)) = op.operands.last() { + let decoder = current_font.as_ref().and_then(|f| decoders.get(f)); + parse_letters( + bytes, + decoder, + &mut tm, + &ctm, + font_size, + &mut letters, + op_index, + None, + ); + } + } + "TJ" => { + if let Some(Object::Array(arr)) = op.operands.first() { + let decoder = current_font.as_ref().and_then(|f| decoders.get(f)); + for (item_index, item) in arr.iter().enumerate() { + match item { + Object::String(bytes, _) => parse_letters( + bytes, + decoder, + &mut tm, + &ctm, + font_size, + &mut letters, + op_index, + Some(item_index), + ), + Object::Integer(i) => { + tm = multiply_matrices( + &[1.0, 0.0, 0.0, 1.0, -(*i as f64 / 1000.0) * font_size, 0.0], + &tm, + ); + } + Object::Real(f) => { + tm = multiply_matrices( + &[1.0, 0.0, 0.0, 1.0, -(*f as f64 / 1000.0) * font_size, 0.0], + &tm, + ); + } + _ => {} + } + } + } + } + _ => {} + } + } + + Ok(reconstruct_words(letters, page_id)) +} + +fn parse_letters( + bytes: &[u8], + decoder: Option<&FontDecoder>, + tm: &mut [f64; 6], + ctm: &[f64; 6], + font_size: f64, + letters: &mut Vec, + op_index: usize, + array_item: Option, +) { + // Decode via the actual font table (CID/GID -> Unicode + actual width). + // (glyph, byte_offset, byte_length) to be able to precisely erase + // each glyph later without affecting neighboring glyphs. + let (glyphs, height_ratio, y_offset_ratio): ( + Vec<(font_map::GlyphInfo, usize, usize)>, + f64, + f64, + ) = match decoder { + Some(d) => ( + d.decode(bytes) + .into_iter() + .map(|(g, off)| (g, off, d.bytes_per_code)) + .collect(), + d.height_ratio, + d.y_offset_ratio, + ), + None => { + let mut offset = 0usize; + let list = String::from_utf8(bytes.to_vec()) + .unwrap_or_default() + .chars() + .map(|c| { + let len = c.len_utf8(); + let start = offset; + offset += len; + (font_map::GlyphInfo { c, width: 0.45 }, start, len) + }) + .collect(); + (list, 0.9, -0.2) + } + }; + + for (g, byte_start, byte_len) in glyphs { + let src = crate::models::SourceRef { + op_index, + array_item, + byte_start, + byte_len, + }; + + let m_abs = multiply_matrices(tm, ctm); + if m_abs[0].abs() < 0.1 && m_abs[3].abs() < 0.1 { + continue; + } + + if (g.c.is_control() || g.c == '\u{200B}' || g.c == '\u{FEFF}') && g.c != '\u{0020}' { + continue; + } + + let width = font_size * g.width; + let height = font_size * height_ratio; + // Constant offset per font: the bottom of the box extends below the baseline (descenders), + // identical for all letters of this font/size. + let baseline_y = m_abs[5]; + let y = baseline_y + font_size * y_offset_ratio; + + letters.push(Letter { + value: g.c, + bbox: BBox { + x: m_abs[4], + y, + width, + height, + }, + font_size, + baseline_y, + src, + }); + *tm = multiply_matrices(&[1.0, 0.0, 0.0, 1.0, width, 0.0], tm); + } +} + +fn reconstruct_words(mut letters: Vec, page_id: ObjectId) -> Vec { + if letters.is_empty() { + return vec![]; + } + letters.sort_by(|a, b| { + b.baseline_y + .partial_cmp(&a.baseline_y) + .unwrap() + .then(a.bbox.x.partial_cmp(&b.bbox.x).unwrap()) + }); + + let mut words = Vec::new(); + let mut current_word = String::new(); + let mut current_word_letters = Vec::new(); + let mut word_bbox: Option = None; + let mut last_letter: Option = None; + + // Only true isolated punctuation, never letters (accented or not) + let is_special = |c: char| c.is_ascii_punctuation(); + let mut saw_space = false; + + for letter in letters { + if letter.value.is_whitespace() || letter.value == '\u{00a0}' { + saw_space = true; + continue; + } + + // A true space in the text flow is a reliable word separator: + // it takes precedence over any geometric heuristic (kerning, attached punctuation...). + let is_new_word = if saw_space { + true + } else { + match &last_letter { + Some(last) => { + // Comparison on the baseline (stable), not on the visual bbox + // which varies with the stems/descenders of each glyph. + let vertical_dist = (letter.baseline_y - last.baseline_y).abs(); + let horizontal_dist = letter.bbox.x - (last.bbox.x + last.bbox.width); + + // Leniency only for attaching isolated punctuation to the current word, + // never for attaching a normal word to the previous one. + let lenient = is_special(letter.value); + let max_v = if lenient { letter.font_size * 0.8 } else { 6.0 }; + // Expanded threshold: no need to be strict for detecting spaces, + // which prevents cutting a word in the middle due to kerning TJ. + let max_h = if lenient { + letter.font_size * 0.4 + } else { + letter.font_size * 0.5 + }; + + let is_overlapping_h = + horizontal_dist < max_h && horizontal_dist > -last.bbox.width; + + vertical_dist > max_v || !is_overlapping_h + } + None => true, + } + }; + saw_space = false; + + if is_new_word && !current_word.is_empty() { + if let Some(bbox) = word_bbox.take() { + let trimmed = current_word.trim(); + if !trimmed.is_empty() { + words.push(Word { + text: trimmed.to_string(), + bbox, + font_size: last_letter.as_ref().unwrap().font_size, + page_id, + letters: current_word_letters.clone(), // Injection + baseline_y: current_word_letters[0].baseline_y, + }); + } + } + current_word.clear(); + current_word_letters.clear(); + } + + if current_word.is_empty() { + word_bbox = Some(letter.bbox.clone()); + } else if let Some(ref mut bbox) = word_bbox { + let min_x = bbox.x.min(letter.bbox.x); + let max_x = (bbox.x + bbox.width).max(letter.bbox.x + letter.bbox.width); + let min_y = bbox.y.min(letter.bbox.y); + let max_y = (bbox.y + bbox.height).max(letter.bbox.y + letter.bbox.height); + + bbox.x = min_x; + bbox.width = max_x - min_x; + bbox.y = min_y; + bbox.height = max_y - min_y; + } + + current_word.push(letter.value); + current_word_letters.push(letter.clone()); + last_letter = Some(letter); + } + + if !current_word.is_empty() { + if let Some(bbox) = word_bbox { + let trimmed = current_word.trim(); + if !trimmed.is_empty() { + words.push(Word { + text: trimmed.to_string(), + bbox, + font_size: last_letter.unwrap().font_size, + page_id, + baseline_y: current_word_letters[0].baseline_y, + letters: current_word_letters, + }); + } + } + } + + words +} diff --git a/src/Core/src/pii_ner_detector.rs b/src/Core/src/pii_ner_detector.rs new file mode 100644 index 0000000..a734264 --- /dev/null +++ b/src/Core/src/pii_ner_detector.rs @@ -0,0 +1,531 @@ +// src/pii_ner_detector.rs +use ort::session::Session; +use ort::value::Value; +use std::fs; +use std::ops::Range; +use std::path::Path; +use tokenizers::Tokenizer; + +use crate::models::{Paragraph, Word}; + +#[derive(Debug, Clone)] +struct RawEntity { + label: String, + start: usize, + end: usize, + score: f32, +} + +pub struct PiiNerDetector { + session: Option, + tokenizer: Option, + labels: Vec, + max_seq_len: usize, + expects_token_type_ids: bool, +} + +impl PiiNerDetector { + pub fn new(models_directory: &str) -> Self { + let model_dir = Path::new(models_directory).join("pii"); + let onnx_file = model_dir.join("model.onnx"); + let tokenizer_file = model_dir.join("tokenizer.json"); + let config_file = model_dir.join("config.json"); + + if !tokenizer_file.exists() { + panic!("Tokenizer introuvable : {:?}", tokenizer_file); + } + if !onnx_file.exists() { + panic!("Modèle ONNX introuvable : {:?}", onnx_file); + } + if !config_file.exists() { + panic!("Fichier de configuration introuvable : {:?}", config_file); + } + + let session = Session::builder() + .unwrap() + .commit_from_file(onnx_file) + .unwrap(); + let tokenizer = Tokenizer::from_file(tokenizer_file).unwrap(); + + // Some models (RoBERTa/CamemBERT-like) do not expose + // the `token_type_ids` input, unlike a "classic" BERT. + // We only call it if the model actually declares it, + // otherwise ONNX Runtime returns "Invalid input name: token_type_ids". + let expects_token_type_ids = session + .inputs() + .iter() + .any(|input| input.name() == "token_type_ids"); + + if cfg!(debug_assertions) { + let input_names: Vec<&str> = session.inputs().iter().map(|i| i.name()).collect(); + eprintln!("[PiiNerDetector] ONNX model inputs: {:?}", input_names); + } + + let config_content = + fs::read_to_string(config_file).expect("Impossible de lire config.json"); + let config: serde_json::Value = + serde_json::from_str(&config_content).expect("JSON invalide"); + + let mut labels = Vec::new(); + if let Some(id2label) = config.get("id2label").and_then(|v| v.as_object()) { + let mut sorted_keys: Vec = id2label + .keys() + .filter_map(|k| k.parse::().ok()) + .collect(); + sorted_keys.sort_unstable(); + + for key in sorted_keys { + if let Some(label_str) = id2label.get(&key.to_string()).and_then(|v| v.as_str()) { + labels.push(label_str.to_string()); + } + } + } + + if labels.is_empty() { + eprintln!( + "[PiiNerDetector] Warning: no labels loaded from config.json \ + (key 'id2label' missing, empty, or malformed)." + ); + } else if cfg!(debug_assertions) { + // Useful for diagnosing an unexpected label schema + // (e.g., flat labels "PER" instead of "B-PER"/"I-PER"). + eprintln!("[PiiNerDetector] Labels loaded: {:?}", labels); + } + + Self { + session: Some(session), + tokenizer: Some(tokenizer), + labels, + max_seq_len: 512, + expects_token_type_ids, + } + } + + pub fn scan_pii(&mut self, paragraphs: &[Paragraph]) -> Vec { + let mut words_to_redact = Vec::new(); + + for paragraph in paragraphs { + let pii_spans = self.find_pii(¶graph.text).unwrap_or_else(|e| { + eprintln!("[PiiDetector] Erreur find_pii : {e}"); + Vec::new() + }); + + let mut search_offset = 0; + + for word in paragraph + .blocks + .iter() + .flat_map(|b| &b.lines) + .flat_map(|l| &l.words) + { + // Si le mot n'est pas trouvé à partir du curseur courant + // (OCR imparfait, doublon, etc.), on passe au suivant sans + // faire avancer le curseur plutôt que de tout bloquer. + let Some(local_idx) = paragraph.text[search_offset..].find(&word.text) else { + continue; + }; + let word_start = search_offset + local_idx; + let word_end = word_start + word.text.len(); + + let is_pii = pii_spans + .iter() + .any(|span| word_start.max(span.start) < word_end.min(span.end)); + + if is_pii { + words_to_redact.push(word.clone()); + } + + search_offset = word_end; + } + } + + words_to_redact + } + + pub fn find_pii( + &mut self, + text: &str, + ) -> Result>, Box> { + if text.trim().is_empty() { + return Ok(Vec::new()); + } + + let tokenizer = self.tokenizer.as_ref().unwrap(); + let session = self.session.as_mut().unwrap(); + + let encoding = tokenizer + .encode(text, true) + .map_err(|e| Box::::from(e.to_string()))?; + + let all_input_ids = encoding.get_ids(); + let all_attention_mask = encoding.get_attention_mask(); + let all_type_ids = encoding.get_type_ids(); + let all_tokens = encoding.get_tokens(); + let all_special_mask = encoding.get_special_tokens_mask(); + + // Some versions of `tokenizers` return offsets in terms of + // Unicode characters rather than bytes. Since Rust strings + // are indexed by bytes, we secure the conversion to avoid + // a panic or incorrect slicing on accented text + // ("Léa", "François"...). + let byte_offsets = to_byte_offsets(text, encoding.get_offsets()); + + let total_tokens = all_input_ids.len(); + let max_len = self.max_seq_len; + let num_labels = self.labels.len(); + let mut chunk_entities = Vec::new(); + + for i in (0..total_tokens).step_by(max_len) { + let current_len = std::cmp::min(max_len, total_tokens - i); + + let mut input_ids: Vec = all_input_ids[i..i + current_len] + .iter() + .map(|&x| x as i64) + .collect(); + let mut attention_mask: Vec = all_attention_mask[i..i + current_len] + .iter() + .map(|&x| x as i64) + .collect(); + let mut type_ids: Vec = all_type_ids[i..i + current_len] + .iter() + .map(|&x| x as i64) + .collect(); + + let offsets = &byte_offsets[i..i + current_len]; + let tokens = &all_tokens[i..i + current_len]; + let special_mask = &all_special_mask[i..i + current_len]; + + if current_len < max_len { + input_ids.resize(max_len, 0); + attention_mask.resize(max_len, 0); + type_ids.resize(max_len, 0); + } + + let input_tensor = Value::from_array(([1, max_len], input_ids))?; + let attention_tensor = Value::from_array(([1, max_len], attention_mask))?; + + let outputs = if self.expects_token_type_ids { + let type_ids_tensor = Value::from_array(([1, max_len], type_ids))?; + session.run(ort::inputs![ + "input_ids" => input_tensor, + "attention_mask" => attention_tensor, + "token_type_ids" => type_ids_tensor + ])? + } else { + session.run(ort::inputs![ + "input_ids" => input_tensor, + "attention_mask" => attention_tensor + ])? + }; + + // NB: if the ONNX model exposes multiple named outputs, + // ensure that the output at index 0 is indeed the token-by-token + // classification logits (and not a pooler_output or + // something else) — a confusion here would also produce a silent "no + // entity" result. + let (_shape, data) = outputs[0].try_extract_tensor::()?; + + let (preds, confs) = decode_predictions(data, max_len, num_labels); + + let is_special = |idx: usize| -> bool { + if idx < special_mask.len() && special_mask[idx] == 1 { + return true; + } + if idx < offsets.len() { + return offsets[idx].0 == 0 + && offsets[idx].1 == 0 + && tokens[idx].starts_with('['); + } + false + }; + + let mut current: Option = None; + let mut current_score_sum = 0.0f32; + let mut current_count = 0usize; + + for t in 0..current_len { + if std::env::var("PII_DEBUG").is_ok() { + let raw_label = self + .labels + .get(preds[t]) + .map(|s| s.as_str()) + .unwrap_or(""); + eprintln!( + "[PII_DEBUG] t={t:>3} token={:<15?} offset={:?} special={} pred_idx={} label={:<25} conf={:.3}", + tokens[t], + offsets[t], + is_special(t), + preds[t], + raw_label, + confs[t] + ); + } + + if is_special(t) { + continue; + } + + // A token whose offset directly touches that of the + // previous one is a subword of the same word (e.g., "Lion"/"##el"): + // we extend the current entity without revalidating the label, + // only the first subword of the word counts. + if current.is_some() + && t > 0 + && !is_special(t - 1) + && offsets[t - 1].1 == offsets[t].0 + { + if let Some(ref mut ent) = current { + ent.end = ent.end.max(offsets[t].1); + current_score_sum += confs[t]; + current_count += 1; + continue; + } + } + + let p = preds[t]; + let label = if p < self.labels.len() { + self.labels[p].as_str() + } else { + "O" + }; + + if label == "O" { + finalize_entity( + &mut current, + current_score_sum, + current_count, + text, + &mut chunk_entities, + ); + continue; + } + + let (bio, tag) = split_bio(label); + + if tag == "ORG" { + finalize_entity( + &mut current, + current_score_sum, + current_count, + text, + &mut chunk_entities, + ); + continue; + } + + let off = offsets[t]; + if off.0 == 0 && off.1 == 0 && tokens[t].starts_with('[') { + continue; + } + + let matches_current = current.as_ref().map_or(false, |e| e.label == tag); + + if bio == "B" || !matches_current { + let mut start_idx = t; + while start_idx > 0 && !is_special(start_idx - 1) { + if offsets[start_idx - 1].1 == offsets[start_idx].0 { + start_idx -= 1; + } else { + break; + } + } + + finalize_entity( + &mut current, + current_score_sum, + current_count, + text, + &mut chunk_entities, + ); + + current = Some(RawEntity { + label: tag.to_string(), + start: offsets[start_idx].0, + end: off.1, + score: confs[t], + }); + current_score_sum = confs[t]; + current_count = 1; + } else if let Some(ref mut ent) = current { + ent.end = ent.end.max(off.1); + current_score_sum += confs[t]; + current_count += 1; + } + } + finalize_entity( + &mut current, + current_score_sum, + current_count, + text, + &mut chunk_entities, + ); + } + + let merged_entities = merge_adjacent_entities(chunk_entities, text); + Ok(merged_entities + .into_iter() + .map(|e| e.start..e.end) + .collect()) + } +} + +/// Splits a label of the form "B-PER" / "I-PER" into (bio, tag). +/// Also accepts the '_' separator ("B_PER"). If the model does not follow an +/// explicit BIO scheme (flat label, e.g., "PER" alone), each +/// occurrence is treated as the beginning of an entity ("B"): the merging of +/// adjacent words of the same type is still handled by `merge_adjacent_entities`. +fn split_bio(label: &str) -> (&str, &str) { + for sep in ['-', '_'] { + if let Some((bio, tag)) = label.split_once(sep) { + if bio == "B" || bio == "I" { + return (bio, tag); + } + } + } + ("B", label) +} + +/// Calculates for each token the predicted class (argmax) and its confidence +/// (softmax) from the raw logits tensor returned by ONNX Runtime. +fn decode_predictions(data: &[f32], max_len: usize, num_labels: usize) -> (Vec, Vec) { + let mut preds = vec![0usize; max_len]; + let mut confs = vec![0.0f32; max_len]; + + if num_labels == 0 { + return (preds, confs); + } + + for t in 0..max_len { + let start_idx = t * num_labels; + let end_idx = start_idx + num_labels; + if end_idx > data.len() { + break; + } + let token_logits = &data[start_idx..end_idx]; + + let (argmax, &max_val) = token_logits + .iter() + .enumerate() + .max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)) + .unwrap(); + + let mut sum_exp = 0.0f32; + let mut exps = vec![0.0f32; num_labels]; + for (l, &logit) in token_logits.iter().enumerate() { + let e = (logit - max_val).exp(); + exps[l] = e; + sum_exp += e; + } + + preds[t] = argmax; + confs[t] = exps[argmax] / sum_exp; + } + + (preds, confs) +} + +/// Converts the offsets returned by the tokenizer into valid byte offsets +/// for `text`. If an offset already corresponds to a valid byte boundary, +/// it is kept as is; otherwise, it is interpreted as a character count and +/// converted using a character-to-byte mapping table. +fn to_byte_offsets(text: &str, raw_offsets: &[(usize, usize)]) -> Vec<(usize, usize)> { + let mut char_to_byte: Vec = text.char_indices().map(|(b, _)| b).collect(); + char_to_byte.push(text.len()); + + let convert = |pos: usize| -> usize { + if pos <= text.len() && text.is_char_boundary(pos) { + pos + } else { + char_to_byte.get(pos).copied().unwrap_or(text.len()) + } + }; + + raw_offsets + .iter() + .map(|&(s, e)| { + let s = convert(s); + let e = convert(e).max(s); + (s, e) + }) + .collect() +} + +fn finalize_entity( + current: &mut Option, + score_sum: f32, + count: usize, + text: &str, + out: &mut Vec, +) { + if let Some(mut ent) = current.take() { + normalize_entity_span(text, &mut ent.start, &mut ent.end); + if ent.end > ent.start { + ent.score = score_sum / (count as f32).max(1.0); + out.push(ent); + } + } +} + +fn normalize_entity_span(text: &str, start: &mut usize, end: &mut usize) { + if *start >= text.len() || *end <= *start { + return; + } + if *end > text.len() { + *end = text.len(); + } + + while *start < *end { + match text[*start..].chars().next() { + Some(c) if c.is_whitespace() => *start += c.len_utf8(), + _ => break, + } + } + while *end > *start { + match text[..*end].chars().next_back() { + Some(c) if c.is_whitespace() => *end -= c.len_utf8(), + _ => break, + } + } + while *end > *start { + match text[..*end].chars().next_back() { + Some(c) if c.is_ascii_punctuation() => *end -= c.len_utf8(), + _ => break, + } + } + while *start < *end { + match text[*start..].chars().next() { + Some(c) if c.is_ascii_punctuation() => *start += c.len_utf8(), + _ => break, + } + } +} + +fn merge_adjacent_entities(entities: Vec, text: &str) -> Vec { + if entities.len() <= 1 { + return entities; + } + + let mut merged = Vec::new(); + let mut current = entities[0].clone(); + + for next in entities.into_iter().skip(1) { + let only_spaces = current.label == next.label + && next.start >= current.end + && next.start <= text.len() + && text[current.end..next.start] + .chars() + .all(char::is_whitespace); + + if only_spaces { + current.end = next.end; + normalize_entity_span(text, &mut current.start, &mut current.end); + current.score = (current.score + next.score) / 2.0; + continue; + } + + merged.push(current); + current = next; + } + merged.push(current); + merged +} diff --git a/src/Core/src/pii_regex_detector.rs b/src/Core/src/pii_regex_detector.rs new file mode 100644 index 0000000..d76aa63 --- /dev/null +++ b/src/Core/src/pii_regex_detector.rs @@ -0,0 +1,91 @@ +// src/pii_regex_detector.rs +use regex::Regex; + +#[derive(Debug, Clone)] +pub struct RegexEntity { + // pub label: String, + pub start: usize, + pub end: usize, + // pub text: String, + // pub score: f32, +} + +pub struct PiiRegexDetector { + email_regex: Regex, + phone_regex: Regex, + date_regex: Regex, + url_regex: Regex, + person_full_name_regex: Regex, +} + +impl PiiRegexDetector { + pub fn new() -> Self { + Self { + email_regex: Regex::new(r"(?i)\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b").unwrap(), + phone_regex: Regex::new(r"(?:(?:\+|00)33|0)\s*[1-9](?:[\s.-]*\d+)+\b").unwrap(), + date_regex: Regex::new(r"\b\d{2}[/-]\d{2}[/-]\d{4}\b").unwrap(), + url_regex: Regex::new(r"(?i)https?://(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)").unwrap(), + person_full_name_regex: Regex::new(r"(?m)^(?:#+\s*)?[A-ZÀ-ÖØ-Ý][a-zà-öø-ÿ]+(?:[-'][A-ZÀ-ÖØ-Ý][a-zà-öø-ÿ]+)?[ \t\u00A0\u202F]+(?:[A-ZÀ-ÖØ-Ý]{3,}(?:[-'’][A-ZÀ-ÖØ-Ý]{2,})*|(?:[A-ZÀ-ÖØ-Ý][ \t\u00A0\u202F]+){2,}[A-ZÀ-ÖØ-Ý])\b").unwrap(), + } + } + + pub fn analyze_text(&self, text: &str) -> Vec { + let mut entities = Vec::new(); + + if text.trim().is_empty() { + return entities; + } + + // 1. Detect full names (NOM_PERSONNE) with a regex that matches capitalized first and last names, optionally preceded by hashtags or whitespace. + for mat in self.person_full_name_regex.find_iter(text) { + let mut start = mat.start(); + let end = mat.end(); + + while start < end { + if let Some(c) = text[start..].chars().next() { + if c == '#' || c.is_whitespace() { + start += c.len_utf8(); + } else { + break; + } + } else { + break; + } + } + + if end <= start { + continue; + } + + entities.push(RegexEntity { + // label: "NOM_PERSONNE".to_string(), + start, + end, + // text: text[start..end].to_string(), + // score: 1.0, + }); + } + + let standard_regexes = [ + (&self.email_regex, "EMAIL"), + (&self.phone_regex, "PHONE"), + (&self.date_regex, "DATE"), + (&self.url_regex, "URL"), + ]; + + for (regex, _label) in standard_regexes { + for mat in regex.find_iter(text) { + entities.push(RegexEntity { + // label: label.to_string(), + start: mat.start(), + end: mat.end(), + // text: mat.as_str().to_string(), + // score: 1.0, + }); + } + } + + entities.sort_by_key(|e| e.start); + entities + } +} From 75c9bcca2e39fd5650c52522c60fdcce86d7a954 Mon Sep 17 00:00:00 2001 From: Lionel Lalande Date: Sun, 5 Jul 2026 14:55:41 +0200 Subject: [PATCH 05/17] refactor(cli): replace python runtime with native core interop --- src/Cli/Anonymizer.Cli.csproj | 27 +++-- src/Cli/AnonymizerService.cs | 76 ++++++++++--- src/Cli/Application.cs | 41 +------ src/Cli/Core/NativeMethods.cs | 9 ++ src/Cli/ExternalScriptRunner.cs | 49 -------- src/Cli/Program.cs | 2 - src/Cli/Python/UVRunner.cs | 21 ---- src/Cli/scripts/anonymize.py | 190 ------------------------------- src/Cli/scripts/face_detector.py | 48 -------- 9 files changed, 87 insertions(+), 376 deletions(-) create mode 100644 src/Cli/Core/NativeMethods.cs delete mode 100644 src/Cli/ExternalScriptRunner.cs delete mode 100644 src/Cli/Python/UVRunner.cs delete mode 100644 src/Cli/scripts/anonymize.py delete mode 100644 src/Cli/scripts/face_detector.py diff --git a/src/Cli/Anonymizer.Cli.csproj b/src/Cli/Anonymizer.Cli.csproj index 322e00c..3997ddc 100644 --- a/src/Cli/Anonymizer.Cli.csproj +++ b/src/Cli/Anonymizer.Cli.csproj @@ -6,37 +6,39 @@ net10.0 enable enable + true Anonymizer.Cli - + true true true Speed - + true full - - - PreserveNewest - PreserveNewest - true - - + + PreserveNewest PreserveNewest - true - + + + + PreserveNewest PreserveNewest - true + + + PreserveNewest PreserveNewest @@ -48,6 +50,7 @@ + diff --git a/src/Cli/AnonymizerService.cs b/src/Cli/AnonymizerService.cs index 8e604dd..f177312 100644 --- a/src/Cli/AnonymizerService.cs +++ b/src/Cli/AnonymizerService.cs @@ -1,10 +1,10 @@ -using Anonymizer.Cli.Python; +using Anonymizer.Cli.Core; using Microsoft.Extensions.Logging; namespace Anonymizer.Cli; -internal sealed partial class AnonymizerService(UVRunner runner, ILogger logger) +internal sealed partial class AnonymizerService(ILogger logger) { public async Task AnonymizeFilesAsync(IEnumerable filesOrDirs, CancellationToken cancellationToken = default) { @@ -31,11 +31,12 @@ public async Task AnonymizeFileAsync(FileInfo file, CancellationToken cancellati { // skip if already anonymized if (file.Name.EndsWith(".anon.pdf")) return; + if (File.Exists(Path.ChangeExtension(file.FullName, ".anon.pdf"))) return; await _semaphore.WaitAsync(cancellationToken); try { - await LaunchProcessAsync(file, cancellationToken); + await RunPipeline(file, cancellationToken); } finally { @@ -43,30 +44,75 @@ public async Task AnonymizeFileAsync(FileInfo file, CancellationToken cancellati } } - private async Task LaunchProcessAsync(FileInfo file, CancellationToken cancellationToken) + private async Task RunPipeline(FileInfo file, CancellationToken cancellationToken) { - var resultCode = await runner.ExecuteScriptAsync( - Application.Scripts.Anonymize.File, - arguments: [Application.Models.Path, file.FullName], - Application.Scripts.Dir, - cancellationToken); - if (resultCode is 0) - LogProcessTerminated(file.Name); - else - LogProcessErrorOutput(file.Name); + OutputPaths output = OutputPaths.From(file); + await Task.Run(() => + { + Console.WriteLine("[DEBUG] Avant RunPipeline()"); + var result = NativeMethods.RedactPdf(file.FullName, ResolveModelsDirectory().FullName, output.AnonymizedPdfPath); + var error = result switch + { + 0 => null, + -1 => "Some paths are empty.", + -2 => "Cannot read some paths.", + -3 => "[Error] Pipeline execution failed", + -4 => "[Error] Internal panic.", + _ => throw new NotSupportedException("Invalid error code!"), + }; + if (error is not null) + LogProcessError(file.FullName, error); + else + LogProcessTerminated(file.FullName); + Console.WriteLine("[DEBUG] Après RunPipeline()"); + }, cancellationToken).ConfigureAwait(false); } + private static DirectoryInfo ResolveModelsDirectory() + { + if (HasPiiModel(Application.Models.Dir)) + return Application.Models.Dir; + + DirectoryInfo? current = new(Environment.CurrentDirectory); + while (current is not null) + { + DirectoryInfo candidate = new(Path.Combine(current.FullName, "models")); + if (HasPiiModel(candidate)) + return candidate; + + current = current.Parent; + } + + return Application.Models.Dir; + } + + private static bool HasPiiModel(DirectoryInfo modelsDir) => + File.Exists(Path.Combine(modelsDir.FullName, "pii", "config.json")) && + File.Exists(Path.Combine(modelsDir.FullName, "pii", "model.onnx")) && + File.Exists(Path.Combine(modelsDir.FullName, "pii", "tokenizer.json")); + #pragma warning disable CA1822 [LoggerMessage(LogLevel.Information, "File {FileName} processed")] private partial void LogProcessTerminated(string fileName); - [LoggerMessage(LogLevel.Error, "Error when processing {FileName}")] - private partial void LogProcessErrorOutput(string fileName); + [LoggerMessage(LogLevel.Error, "Error when processing {FileName}: {Error}")] + private partial void LogProcessError(string fileName, string error); #pragma warning restore CA1822 // Maximum 4 files processed at same time! private readonly SemaphoreSlim _semaphore = new(4); + // private readonly RegexPiiDetector _regexDetector = new(); private readonly ILogger _logger = logger; + + private readonly record struct OutputPaths(string AnonymizedPdfPath) + { + public static OutputPaths From(FileInfo file) + { + string basePath = Path.Combine(file.DirectoryName!, Path.GetFileNameWithoutExtension(file.Name)); + return new( + AnonymizedPdfPath: basePath + ".anon.pdf"); + } + } } diff --git a/src/Cli/Application.cs b/src/Cli/Application.cs index c2ba369..217e898 100644 --- a/src/Cli/Application.cs +++ b/src/Cli/Application.cs @@ -52,9 +52,9 @@ internal static class Models internal static class PII { - public static readonly string File = Join(Models.Path, "pii"); + public static readonly string Path = Join(Models.Path, "pii"); - public static DirectoryInfo Dir = new(File); + public static DirectoryInfo Dir = new(Path); } internal static class Face @@ -65,43 +65,6 @@ internal static class Face } } - internal static class Scripts - { - public static readonly string Path = Join(Base.Path, "scripts")!; - - public static readonly DirectoryInfo Dir = new(Path); - - internal static class Anonymize - { - public static readonly string Path = Join(Scripts.Path, "anonymize.py"); - - public static readonly FileInfo File = new(Path); - } - - internal static class DownloadModel - { - public static readonly string Path = Join(Scripts.Path, "download-model.py"); - - public static readonly FileInfo File = new(Path); - } - } - - internal static class Tools - { - private static readonly string Extension = OperatingSystem.IsWindows() ? ".exe" : string.Empty; - - public static readonly string Path = Join(Base.Path, "tools")!; - - public static readonly DirectoryInfo Dir = new(Path); - - internal static class UV - { - public static readonly string Path = Join(Tools.Path, ChangeExtension("uv", Extension)); - - public static FileInfo File = new(Path); - } - } - internal static class AppSettings { public static readonly string Path = Join(Base.Path, "appsettings.json")!; diff --git a/src/Cli/Core/NativeMethods.cs b/src/Cli/Core/NativeMethods.cs new file mode 100644 index 0000000..7530d08 --- /dev/null +++ b/src/Cli/Core/NativeMethods.cs @@ -0,0 +1,9 @@ +using System.Runtime.InteropServices; + +namespace Anonymizer.Cli.Core; + +internal static unsafe partial class NativeMethods +{ + [LibraryImport("Anonymizer.Core", EntryPoint = "redact_pdf", StringMarshalling = StringMarshalling.Utf8)] + public static partial int RedactPdf(string inputPath, string modelsDir, string outputPath); +} diff --git a/src/Cli/ExternalScriptRunner.cs b/src/Cli/ExternalScriptRunner.cs deleted file mode 100644 index 6e6a52d..0000000 --- a/src/Cli/ExternalScriptRunner.cs +++ /dev/null @@ -1,49 +0,0 @@ -using System.Diagnostics; -using System.Text; - -namespace Anonymizer.Cli; - -internal abstract class ExternalScriptRunner -{ - public abstract Task ExecuteScriptAsync(FileInfo scriptFile, string[] arguments, DirectoryInfo workingDirectory, CancellationToken cancellationToken); - - protected static async Task RunProcessAsync( - ProcessStartInfo startInfo, - CancellationToken cancellationToken) - { - using var process = new Process { StartInfo = startInfo }; - process.OutputDataReceived += (_, e) => - { - if (e.Data is null) return; - Console.WriteLine(e.Data); - }; - - process.ErrorDataReceived += (_, e) => - { - if (e.Data is null) return; - var previous = Console.ForegroundColor; - Console.ForegroundColor = ConsoleColor.Red; - Console.WriteLine(e.Data); - Console.ForegroundColor = previous; - }; - - process.Start(); - process.BeginOutputReadLine(); - process.BeginErrorReadLine(); - await process.WaitForExitAsync(cancellationToken); - - return process.ExitCode; - } - - protected static ProcessStartInfo CreateBaseStartInfo(FileInfo exeFile, DirectoryInfo? workingDir = null) => new() - { - FileName = exeFile.FullName, - UseShellExecute = false, - RedirectStandardOutput = true, - RedirectStandardError = true, - CreateNoWindow = true, - StandardOutputEncoding = Encoding.UTF8, - StandardErrorEncoding = Encoding.UTF8, - WorkingDirectory = workingDir?.FullName ?? Application.Base.Path, - }; -} diff --git a/src/Cli/Program.cs b/src/Cli/Program.cs index 804097d..e5ce37c 100644 --- a/src/Cli/Program.cs +++ b/src/Cli/Program.cs @@ -2,7 +2,6 @@ using Anonymizer.Cli.Commands; using Anonymizer.Cli.Downloaders; using Anonymizer.Cli.Lifetime; -using Anonymizer.Cli.Python; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; @@ -16,7 +15,6 @@ { services.AddSingleton(); services.AddHttpClient(); - services.AddSingleton((sp) => ActivatorUtilities.CreateInstance(sp, Application.Tools.UV.File)); services.AddSingleton(); if (Path.GetFileNameWithoutExtension(Environment.ProcessPath) is "setup") diff --git a/src/Cli/Python/UVRunner.cs b/src/Cli/Python/UVRunner.cs deleted file mode 100644 index 89e0ff5..0000000 --- a/src/Cli/Python/UVRunner.cs +++ /dev/null @@ -1,21 +0,0 @@ -namespace Anonymizer.Cli.Python; - -internal sealed class UVRunner(FileInfo uvFile) : ExternalScriptRunner -{ - public override async Task ExecuteScriptAsync(FileInfo scriptFile, string[] arguments, DirectoryInfo workingDirectory, CancellationToken cancellationToken) - { - if (!scriptFile.Exists) - throw new FileNotFoundException($"No Python script at {scriptFile.FullName}"); - - var startInfo = CreateBaseStartInfo(uvFile, workingDirectory); - startInfo.ArgumentList.Add("run"); - startInfo.ArgumentList.Add(scriptFile.FullName); - if (arguments is not null) - { - foreach (var arg in arguments) - startInfo.ArgumentList.Add(arg); - } - - return await RunProcessAsync(startInfo, cancellationToken); - } -} diff --git a/src/Cli/scripts/anonymize.py b/src/Cli/scripts/anonymize.py deleted file mode 100644 index b6fbf1f..0000000 --- a/src/Cli/scripts/anonymize.py +++ /dev/null @@ -1,190 +0,0 @@ -# /// script -# requires-python = ">=3.12" -# dependencies = [ -# "numpy", -# "onnxruntime", -# "opencv-python", -# "PyMuPDF", -# ] -# /// -import sys -import re -import fitz -import os -from face_detector import FaceDetector - -REGEX_PATTERNS = [ - (r'\b[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}\b', "EMAIL"), - (r'(?:\+33|0033|0)\s*[1-9](?:[\s.\-]?\d{2}){4}', "PHONE"), -] - -def get_full_name(text): - lines = [l.strip() for l in text.split("\n") if l.strip()] - if lines: - first = lines[0] - if re.search(r'[:;?!_*/\\]', first): - return None - if ( - 1 < len(first.split()) <= 4 - and not any(c.isdigit() for c in first) - and not first.isupper() - and len(first) > 3 - ): - return first.strip() - return None - - -def redact_standard_text(page, text, full_name, page_summary): - entities = [] - if full_name: - entities.append({"text": full_name, "label": "NAME"}) - - for pattern, label in REGEX_PATTERNS: - for m in re.finditer(pattern, text, re.IGNORECASE): - entities.append({"text": m.group(), "label": label}) - - entities_sorted = sorted(entities, key=lambda x: len(x["text"]), reverse=True) - seen = set() - - for ent in entities_sorted: - if ent["text"] in seen: - continue - rects = page.search_for(ent["text"]) - for rect in rects: - page.add_redact_annot( - rect, text="[REDACTED]", fill=(0, 0, 0), - text_color=(0, 0, 0), fontname="helv", fontsize=8 - ) - page_summary.setdefault(ent["label"], set()).add(ent["text"]) - seen.add(ent["text"]) - - -def redact_urls_robust(page, full_name, page_summary): - blocks = page.get_text("dict")["blocks"] - name_parts = [] - if full_name: - name_parts = [full_name.lower().replace(" ", "")] + [ - p.lower() for p in full_name.split() if len(p) > 2 - ] - - for b in blocks: - if "lines" not in b: - continue - for line in b["lines"]: - line_text = "".join(span["text"] for span in line["spans"]).strip() - line_lower = line_text.lower() - is_url = bool(re.search(r'https?://|www\.|linkedin\.com/|github\.com/', line_lower)) - - if is_url: - rect = fitz.Rect(line["bbox"]) - label = "URL" - if "linkedin.com" in line_lower or "github.com" in line_lower: - label = "NETWORK" - if name_parts and any(part in line_lower for part in name_parts): - label = "URL_NAME" - - page.add_redact_annot( - rect, text="[REDACTED]", fill=(0, 0, 0), - text_color=(0, 0, 0), fontname="helv", fontsize=8 - ) - page_summary.setdefault(label, set()).add(line_text) - - -def redact_images_with_faces(doc, page, detector, page_summary): - for img_info in page.get_images(full=True): - xref = img_info[0] - - try: - base_image = doc.extract_image(xref) - image_bytes = base_image["image"] - except Exception as e: - print(f"Cannot extract image xref {xref}: {e}") - continue - - if detector.detect_faces(image_bytes): - for rect in page.get_image_rects(xref): - page.add_redact_annot(rect, text="", fill=(0.5, 0.5, 0.5)) - page_summary.setdefault("FACE_PHOTO", set()).add(f"Image XREF {xref}") - - -def redact_page(doc, page, global_summary, detector, full_name): - for link in page.get_links(): - page.delete_link(link) - - text = page.get_text("text") - page_summary = {} - - # Process images before text to avoid annotation overlap - if detector: - redact_images_with_faces(doc, page, detector, page_summary) - - redact_standard_text(page, text, full_name, page_summary) - redact_urls_robust(page, full_name, page_summary) - - page.apply_redactions(images=fitz.PDF_REDACT_IMAGE_REMOVE) - - for k, v in page_summary.items(): - global_summary.setdefault(k, set()).update(v) - - -def find_onnx_model(model_dir): - """Find a .onnx file in the specified directory.""" - if os.path.isfile(model_dir) and model_dir.endswith(".onnx"): - return model_dir - if os.path.isdir(model_dir): - for f in os.listdir(model_dir): - if f.endswith(".onnx"): - return os.path.join(model_dir, f) - return None - - -def anonymize(model_dir, pdf_file): - model_path = find_onnx_model(os.path.join(model_dir, "face")) - - detector = None - if model_path and os.path.exists(model_path): - detector = FaceDetector(model_path) - else: - print(f"WARNING: No .onnx model found in '{model_dir}'. Face detection disabled.") - - doc = fitz.open(pdf_file) - global_summary = {} - - full_name = None - if len(doc) > 0: - first_page_text = doc[0].get_text("text") - full_name = get_full_name(first_page_text) - - for page in doc: - redact_page(doc, page, global_summary, detector, full_name) - - out = pdf_file.replace(".pdf", ".anon.pdf") - - doc.set_metadata({}) - doc.save(out, garbage=4, deflate=True, clean=True) - doc.close() - - report = {k: sorted(list(v)) for k, v in global_summary.items()} - - total = sum(len(v) for v in report.values()) - print(f"Output : {out}") - print(f"Name : {full_name or 'not detected'}") - print(f"Redacted: {total} item(s) across {len(report)} categor{'y' if len(report) == 1 else 'ies'}") - if report: - for k, v in report.items(): - print(f" {k:<12} {len(v):>2} " + ", ".join(v)) - - -if __name__ == "__main__": - if len(sys.argv) < 3: - print("Usage: python anonymizer.py ") - sys.exit(1) - - model_dir = sys.argv[1] - pdf_file = sys.argv[2] - - if not os.path.exists(pdf_file): - print(f"Error: PDF file '{pdf_file}' not found.") - sys.exit(1) - - anonymize(model_dir, pdf_file) diff --git a/src/Cli/scripts/face_detector.py b/src/Cli/scripts/face_detector.py deleted file mode 100644 index 1b5f470..0000000 --- a/src/Cli/scripts/face_detector.py +++ /dev/null @@ -1,48 +0,0 @@ -import cv2 -import numpy as np -import onnxruntime as ort -import sys - - -class FaceDetector: - def __init__(self, model_path, conf_threshold=0.7): - self.conf_threshold = conf_threshold - try: - opts = ort.SessionOptions() - opts.log_severity_level = 3 # 0=VERBOSE, 1=INFO, 2=WARNING, 3=ERROR - self.ort_session = ort.InferenceSession(model_path, sess_options=opts) - self.input_name = self.ort_session.get_inputs()[0].name - except Exception as e: - print(f"Error loading ONNX model: {e}") - sys.exit(1) - - def _preprocess(self, image): - # UltraFace RFB-320 expects 320x240 input normalized to [-1, 1] - image = cv2.resize(image, (320, 240)) - image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) - image_norm = (image_rgb - 127.0) / 128.0 - image_chw = np.transpose(image_norm, (2, 0, 1)) - return np.expand_dims(image_chw, axis=0).astype(np.float32) - - def detect_faces(self, image_bytes): - """Return True if at least one face is detected above the confidence threshold.""" - if not image_bytes: - return False - - try: - nparr = np.frombuffer(image_bytes, np.uint8) - image = cv2.imdecode(nparr, cv2.IMREAD_COLOR) - - if image is None: - return False - - input_tensor = self._preprocess(image) - outputs = self.ort_session.run(None, {self.input_name: input_tensor}) - - # Column 1 holds the face confidence score - confidences = outputs[0] - return any(score > self.conf_threshold for score in confidences[0][:, 1]) - - except Exception as e: - print(f"Error during face detection: {e}") - return False From 4b2e238fcb693fd24eed4ab18083eb0da6d66cc0 Mon Sep 17 00:00:00 2001 From: Lionel Lalande Date: Sun, 5 Jul 2026 14:56:30 +0200 Subject: [PATCH 06/17] feat(cli): unify ONNX model download flow --- .gitignore | 2 + src/Cli/Commands/DownloadCommand.cs | 66 +++++++++++++++++-- ...eModelDownloader.cs => ModelDownloader.cs} | 23 ++++--- src/Cli/Downloaders/PIIModelDownloader.cs | 54 --------------- src/Cli/Models.cs | 12 +++- src/Cli/Program.cs | 3 +- 6 files changed, 84 insertions(+), 76 deletions(-) rename src/Cli/Downloaders/{FaceModelDownloader.cs => ModelDownloader.cs} (66%) delete mode 100644 src/Cli/Downloaders/PIIModelDownloader.cs diff --git a/.gitignore b/.gitignore index c40acad..bf35663 100644 --- a/.gitignore +++ b/.gitignore @@ -482,3 +482,5 @@ $RECYCLE.BIN/ *.swp *.lscache + +models/ diff --git a/src/Cli/Commands/DownloadCommand.cs b/src/Cli/Commands/DownloadCommand.cs index 60159a6..fbe91aa 100644 --- a/src/Cli/Commands/DownloadCommand.cs +++ b/src/Cli/Commands/DownloadCommand.cs @@ -1,4 +1,5 @@ using System.CommandLine; +using System.IO.Compression; using Anonymizer.Cli.Downloaders; @@ -10,7 +11,7 @@ namespace Anonymizer.Cli.Commands; internal sealed class DownloadCommand : Command { private const string HelpDesc = """ - Download PII and face-recognition models. + Download PII NER and face-recognition models. """; public DownloadCommand(IHostBuilder builder) : base("download", HelpDesc) => @@ -18,10 +19,63 @@ public DownloadCommand(IHostBuilder builder) : base("download", HelpDesc) => { var app = builder.Build(); - var piiDownloader = app.Services.GetRequiredService(); - await piiDownloader.DownloadAsync(Models.PII.Name, Application.Models.PII.Dir, cancellationToken); - - var faceDownloader = app.Services.GetRequiredService(); - await faceDownloader.DownloadAsync(Models.Face.RemoteUri, Application.Models.Face.Dir, cancellationToken); + var modelDownloader = app.Services.GetRequiredService(); + var piiModelTask = DownloadAndUncompress(modelDownloader, Models.PII.Model.RemoteUri, Application.Models.PII.Dir, cancellationToken); + var faceModelTask = DownloadAndUncompress(modelDownloader, Models.Face.Model.RemoteUri, Application.Models.Face.Dir, cancellationToken); + await Task.WhenAll(piiModelTask, faceModelTask); }); + + private static async Task DownloadAndUncompress(ModelDownloader modelDownloader, Uri remoteUri, DirectoryInfo outputDir, CancellationToken cancellationToken) + { + DirectoryInfo? modelsRoot = outputDir.Parent; + if (modelsRoot is null) + throw new InvalidOperationException("Unable to resolve models root directory."); + + outputDir.Refresh(); + if (outputDir.Exists) + outputDir.Delete(true); + + string tempDirPath = Path.Combine(Path.GetTempPath(), "anonymizer-models"); + Directory.CreateDirectory(tempDirPath); + + string archiveName = Path.GetFileName(remoteUri.LocalPath); + FileInfo archiveFile = new(Path.Combine(tempDirPath, archiveName)); + if (archiveFile.Exists) + archiveFile.Delete(); + + try + { + await modelDownloader.DownloadAsync(remoteUri, archiveFile, cancellationToken); + ZipFile.ExtractToDirectory(archiveFile.FullName, modelsRoot.FullName, overwriteFiles: true); + } + finally + { + if (archiveFile.Exists) + archiveFile.Delete(); + } + + EnsureExpectedFiles(remoteUri, outputDir); + } + + private static void EnsureExpectedFiles(Uri remoteUri, DirectoryInfo outputDir) + { + string archiveName = Path.GetFileName(remoteUri.LocalPath); + if (archiveName.StartsWith("pii_", StringComparison.OrdinalIgnoreCase)) + { + string[] required = ["config.json", "model.onnx", "tokenizer.json"]; + foreach (string fileName in required) + { + if (!File.Exists(Path.Combine(outputDir.FullName, fileName))) + throw new FileNotFoundException($"Missing extracted PII model file: {fileName}"); + } + return; + } + + if (archiveName.StartsWith("face_", StringComparison.OrdinalIgnoreCase) && + !File.Exists(Path.Combine(outputDir.FullName, "model.onnx"))) + { + throw new FileNotFoundException("Missing extracted face model file: model.onnx"); + } + } } + diff --git a/src/Cli/Downloaders/FaceModelDownloader.cs b/src/Cli/Downloaders/ModelDownloader.cs similarity index 66% rename from src/Cli/Downloaders/FaceModelDownloader.cs rename to src/Cli/Downloaders/ModelDownloader.cs index 051aa76..b931864 100644 --- a/src/Cli/Downloaders/FaceModelDownloader.cs +++ b/src/Cli/Downloaders/ModelDownloader.cs @@ -4,16 +4,15 @@ namespace Anonymizer.Cli.Downloaders; -internal sealed partial class FaceModelDownloader(HttpClient http, ILogger logger) +internal sealed partial class ModelDownloader(HttpClient http, ILogger logger) { - public async Task DownloadAsync(Uri remoteUri, DirectoryInfo outputDir, CancellationToken cancellationToken) + public async Task DownloadAsync(Uri remoteUri, FileInfo outputFile, CancellationToken cancellationToken) { - outputDir.Create(); + outputFile.Directory?.Create(); - FileInfo localFile = new(Path.Join(outputDir.FullName, Path.GetFileName(remoteUri.AbsolutePath))); - if (localFile.Exists) + if (outputFile.Exists) { - LogAlreadyExists(localFile.FullName); + LogAlreadyExists(outputFile.FullName); return; } @@ -36,7 +35,7 @@ public async Task DownloadAsync(Uri remoteUri, DirectoryInfo outputDir, Cancella ProgressBar progress = new(); await using var input = await response.Content.ReadAsStreamAsync(cancellationToken); - await using var output = localFile.Open(FileMode.Create, FileAccess.Write); + await using var output = outputFile.Open(FileMode.Create, FileAccess.Write); var buffer = new byte[81920]; long totalRead = 0; @@ -49,7 +48,7 @@ public async Task DownloadAsync(Uri remoteUri, DirectoryInfo outputDir, Cancella } progress.Finish(); - LogSuccess(outputDir.FullName); + LogSuccess(outputFile.FullName); } catch (Exception ex) { @@ -60,19 +59,19 @@ public async Task DownloadAsync(Uri remoteUri, DirectoryInfo outputDir, Cancella #pragma warning disable CA1822 - [LoggerMessage(LogLevel.Information, "Face Detection ONNX model already exists at {Path}.")] + [LoggerMessage(LogLevel.Information, "ONNX model already exists at {Path}.")] private partial void LogAlreadyExists(string path); - [LoggerMessage(LogLevel.Information, "Downloading Face Detection ONNX model…")] + [LoggerMessage(LogLevel.Information, "Downloading ONNX model…")] private partial void LogStartDownload(); - [LoggerMessage(LogLevel.Warning, "Unable to download Face Detection ONNX model.")] + [LoggerMessage(LogLevel.Warning, "Unable to download ONNX model.")] private partial void LogUnableToDownloadModel(); [LoggerMessage(LogLevel.Information, "Model successfully downloaded to {Path}.")] private partial void LogSuccess(string path); - [LoggerMessage(LogLevel.Error, "Failed to download Face Detection model.")] + [LoggerMessage(LogLevel.Error, "Failed to download ONNX model.")] private partial void LogFailure(Exception exception); #pragma warning restore CA1822 diff --git a/src/Cli/Downloaders/PIIModelDownloader.cs b/src/Cli/Downloaders/PIIModelDownloader.cs deleted file mode 100644 index 3fb74ec..0000000 --- a/src/Cli/Downloaders/PIIModelDownloader.cs +++ /dev/null @@ -1,54 +0,0 @@ -using Anonymizer.Cli.Python; - -using Microsoft.Extensions.Logging; - -namespace Anonymizer.Cli.Downloaders; - -internal sealed partial class PIIModelDownloader( - UVRunner runner, - ILogger logger) -{ - public async Task DownloadAsync(string model, DirectoryInfo outputDir, CancellationToken cancellationToken) - { - if (outputDir.Exists && outputDir.GetFiles().Length > 0) - { - LogAlreadyExists(outputDir.FullName); - return; - } - - LogStartDownload(); - - var result = await runner.ExecuteScriptAsync( - Application.Scripts.DownloadModel.File, - arguments: ["--model", model, "--out", outputDir.FullName], - Application.Scripts.Dir, - cancellationToken - ); - - if (result is not 0) - { - LogFailure(); - return; - } - - LogSuccess(outputDir.FullName); - } - -#pragma warning disable CA1822 - - [LoggerMessage(LogLevel.Information, "PII model already exists at {Path}")] - private partial void LogAlreadyExists(string path); - - [LoggerMessage(LogLevel.Information, "Starting PII model download script…")] - private partial void LogStartDownload(); - - [LoggerMessage(LogLevel.Information, "Successfully downloaded PII model to: {Path}")] - private partial void LogSuccess(string path); - - [LoggerMessage(LogLevel.Error, "Error while downloading PII model")] - private partial void LogFailure(); - -#pragma warning restore CA1822 - - private readonly ILogger _logger = logger; -} diff --git a/src/Cli/Models.cs b/src/Cli/Models.cs index ed0f71f..fb23edd 100644 --- a/src/Cli/Models.cs +++ b/src/Cli/Models.cs @@ -2,13 +2,21 @@ namespace Anonymizer.Cli; internal static class Models { + public static Uri BaseUri = new("https://github.com/LsquaredTechnologies/Anonymizer/releases/latest/download/"); + internal static class PII { - public const string Name = "yalen-ai/distilbert_pii_ner_yalen"; + internal static class Model + { + public static readonly Uri RemoteUri = new(BaseUri, "pii_model.zip"); + } } internal static class Face { - public static readonly Uri RemoteUri = new("https://github.com/Linzaer/Ultra-Light-Fast-Generic-Face-Detector-1MB/raw/master/models/onnx/version-RFB-320.onnx"); + internal static class Model + { + public static readonly Uri RemoteUri = new(BaseUri, "face_model.zip"); + } } } diff --git a/src/Cli/Program.cs b/src/Cli/Program.cs index e5ce37c..ecd0cc7 100644 --- a/src/Cli/Program.cs +++ b/src/Cli/Program.cs @@ -13,8 +13,7 @@ var builder = Host.CreateDefaultBuilder(); builder.ConfigureServices((context, services) => { - services.AddSingleton(); - services.AddHttpClient(); + services.AddHttpClient(); services.AddSingleton(); if (Path.GetFileNameWithoutExtension(Environment.ProcessPath) is "setup") From 40289bbd3fcd3a8e294d5e798f73b05071c980c7 Mon Sep 17 00:00:00 2001 From: Lionel Lalande Date: Sun, 5 Jul 2026 14:57:36 +0200 Subject: [PATCH 07/17] chore(cli): cleanup commands and config serialization --- src/Cli/Commands/ConfigSetCommand.cs | 13 +++++++++---- src/Cli/Commands/StartCommand.cs | 1 - src/Cli/Defaults.cs | 11 ----------- src/Cli/Lifetime/SingleInstance.cs | 3 --- 4 files changed, 9 insertions(+), 19 deletions(-) delete mode 100644 src/Cli/Defaults.cs diff --git a/src/Cli/Commands/ConfigSetCommand.cs b/src/Cli/Commands/ConfigSetCommand.cs index cb5d0f6..ee0f3b1 100644 --- a/src/Cli/Commands/ConfigSetCommand.cs +++ b/src/Cli/Commands/ConfigSetCommand.cs @@ -1,6 +1,6 @@ using System.CommandLine; using System.Text.Json; -using System.Text.Json.Nodes; +using System.Text.Json.Serialization; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; @@ -40,12 +40,17 @@ public ConfigSetCommand(IHostBuilder builder) : base("set", HelpDesc) if (!Application.Config.ValidConfigKey.Contains(key)) return; - JsonObject o = new() + Dictionary o = new() { - { key, value } + [key] = value, }; using var stream = File.Open(Application.AppSettings.Path, FileMode.Create, FileAccess.Write); - await JsonSerializer.SerializeAsync(stream, o, Defaults.JsonOptions); + await JsonSerializer.SerializeAsync(stream, o, ConfigJsonContext.Default.DictionaryStringString); }); } } + +[JsonSerializable(typeof(Dictionary))] +internal partial class ConfigJsonContext : JsonSerializerContext +{ +} diff --git a/src/Cli/Commands/StartCommand.cs b/src/Cli/Commands/StartCommand.cs index dffba93..2e8e1af 100644 --- a/src/Cli/Commands/StartCommand.cs +++ b/src/Cli/Commands/StartCommand.cs @@ -4,7 +4,6 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; -using Microsoft.VisualBasic; namespace Anonymizer.Cli.Commands; diff --git a/src/Cli/Defaults.cs b/src/Cli/Defaults.cs deleted file mode 100644 index 996374f..0000000 --- a/src/Cli/Defaults.cs +++ /dev/null @@ -1,11 +0,0 @@ -using System.Text.Json; - -namespace Anonymizer.Cli; - -internal static class Defaults -{ - public static readonly JsonSerializerOptions JsonOptions = new() - { - WriteIndented = false, - }; -} diff --git a/src/Cli/Lifetime/SingleInstance.cs b/src/Cli/Lifetime/SingleInstance.cs index 7b066c5..b634b59 100644 --- a/src/Cli/Lifetime/SingleInstance.cs +++ b/src/Cli/Lifetime/SingleInstance.cs @@ -1,7 +1,4 @@ using System.Runtime.Versioning; -using System.Security.AccessControl; -using System.Security.Cryptography; -using System.Security.Principal; namespace Anonymizer.Cli.Lifetime; From 15f7d6573d77d87f79dacacc91c11529a590a054 Mon Sep 17 00:00:00 2001 From: Lionel Lalande Date: Sun, 5 Jul 2026 14:59:07 +0200 Subject: [PATCH 08/17] chore(vscode): add vscode settings --- .vscode/settings.json | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..b8d8b06 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,8 @@ +{ + "cSpell.ignoreWords": [ + "onnx" + ], + "files.associations": { + "launchsettings.json": "jsonc" + }, +} From 9450468f6fae553cf4567d5a53672b4c72b84dc8 Mon Sep 17 00:00:00 2001 From: Lionel Lalande Date: Sun, 5 Jul 2026 15:01:47 +0200 Subject: [PATCH 09/17] fix(cli): remove UpdateCommand from "normal" commands UpdateCommand is only available in "setup" program for now. --- src/Cli/Commands/RootCommand.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Cli/Commands/RootCommand.cs b/src/Cli/Commands/RootCommand.cs index 9da0ae3..83c3180 100644 --- a/src/Cli/Commands/RootCommand.cs +++ b/src/Cli/Commands/RootCommand.cs @@ -26,9 +26,8 @@ public RootCommand(IHostBuilder builder) : base(HelpDesc) { Add(new InstallCommand(builder)); Add(new UninstallCommand(builder)); + Add(new UpdateCommand()); } - - Add(new UpdateCommand()); } TreatUnmatchedTokensAsErrors = true; From bb1ff1a7dbd242c80710d900d248118db63324c5 Mon Sep 17 00:00:00 2001 From: Lionel Lalande Date: Sun, 5 Jul 2026 15:02:01 +0200 Subject: [PATCH 10/17] fix(cli): update download URL construction for setup files --- src/Cli/Commands/UpdateCommand.cs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/Cli/Commands/UpdateCommand.cs b/src/Cli/Commands/UpdateCommand.cs index 379e7d9..ee4332e 100644 --- a/src/Cli/Commands/UpdateCommand.cs +++ b/src/Cli/Commands/UpdateCommand.cs @@ -53,22 +53,19 @@ private static async Task Update() ProcessManager.KillRunningInstances(); - string setupName = OperatingSystem.IsWindows() ? "setup.exe" : "setup"; - string releaseUrl = - $"https://github.com/LsquaredTechnologies/Anonymizer/releases/latest/download/{setupName}"; - string tempPath = Path.Combine(Path.GetTempPath(), "anonymizer-update"); DirectoryInfo tempDir = new(tempPath); if (tempDir.Exists) tempDir.Delete(recursive: true); tempDir.Create(); + string setupName = OperatingSystem.IsWindows() ? "setup.exe" : "setup"; + Uri releaseSetupUri = new(Models.BaseUri, setupName); FileInfo downloadedSetup = new(Path.Combine(tempPath, setupName)); - - Console.WriteLine($"Downloading: {releaseUrl}"); + Console.WriteLine($"Downloading: {releaseSetupUri}"); using (var http = new HttpClient()) - using (var stream = await http.GetStreamAsync(releaseUrl)) + using (var stream = await http.GetStreamAsync(releaseSetupUri)) using (var file = downloadedSetup.Open(FileMode.Create, FileAccess.Write)) await stream.CopyToAsync(file); From e2373dedebb267e27dc2ca44c6235024d371c054 Mon Sep 17 00:00:00 2001 From: Lionel Lalande Date: Sun, 5 Jul 2026 15:06:18 +0200 Subject: [PATCH 11/17] fix(cli): avoid console when running at startup --- src/Cli/Anonymizer.Cli.csproj | 2 +- src/Cli/Internals/ConsoleA.cs | 20 ++++++++++++++++++++ src/Cli/Program.cs | 5 ++++- 3 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 src/Cli/Internals/ConsoleA.cs diff --git a/src/Cli/Anonymizer.Cli.csproj b/src/Cli/Anonymizer.Cli.csproj index 3997ddc..1a92a61 100644 --- a/src/Cli/Anonymizer.Cli.csproj +++ b/src/Cli/Anonymizer.Cli.csproj @@ -1,7 +1,7 @@  - Exe + WinExe anonymizer net10.0 enable diff --git a/src/Cli/Internals/ConsoleA.cs b/src/Cli/Internals/ConsoleA.cs new file mode 100644 index 0000000..d8d19ce --- /dev/null +++ b/src/Cli/Internals/ConsoleA.cs @@ -0,0 +1,20 @@ +using System.Runtime.InteropServices; + +namespace Anonymizer.Cli.Internals; + +internal static partial class ConsoleA +{ + public static void AttachToConsole() + { + if (OperatingSystem.IsWindows()) + { + AttachConsole(0xFFFFFFFF); + Console.SetOut(new StreamWriter(Console.OpenStandardOutput()) { AutoFlush = true }); + Console.SetError(new StreamWriter(Console.OpenStandardError()) { AutoFlush = true }); + } + } + + [LibraryImport("kernel32.dll")] + [return: MarshalAs(UnmanagedType.I1)] + private static partial bool AttachConsole(uint dwProcessId); +} diff --git a/src/Cli/Program.cs b/src/Cli/Program.cs index ecd0cc7..4285ee4 100644 --- a/src/Cli/Program.cs +++ b/src/Cli/Program.cs @@ -8,7 +8,10 @@ using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; -Console.OutputEncoding = System.Text.Encoding.UTF8; +using static Anonymizer.Cli.Internals.ConsoleA; + +AttachToConsole(); +Console.OutputEncoding = Console.InputEncoding = System.Text.Encoding.UTF8; var builder = Host.CreateDefaultBuilder(); builder.ConfigureServices((context, services) => From 271b2d5c933f946adecfcd684c2ec8b71dff30fa Mon Sep 17 00:00:00 2001 From: Lionel Lalande Date: Sun, 5 Jul 2026 15:09:52 +0200 Subject: [PATCH 12/17] chore(cli): add commandLineArgs samples for various CLI commands --- src/Cli/Properties/launchSettings.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Cli/Properties/launchSettings.json b/src/Cli/Properties/launchSettings.json index 1d97e52..2c8ea44 100644 --- a/src/Cli/Properties/launchSettings.json +++ b/src/Cli/Properties/launchSettings.json @@ -2,6 +2,10 @@ "profiles": { "cli": { "commandName": "Project", + //"commandLineArgs": "download", + //"commandLineArgs": "config get filesdir", + //"commandLineArgs": "config set filesdir %USERPROFILE%/Documents", + //"commandLineArgs": "run \"../../samples/document.pdf\"", "commandLineArgs": "start", "dotnetRunMessages": false, "workingDirectory": "." From 409163c09db8e4eb368363bf583d364c9860f052 Mon Sep 17 00:00:00 2001 From: Lionel Lalande Date: Sun, 5 Jul 2026 15:55:22 +0200 Subject: [PATCH 13/17] fix(cli): improve single instance handling --- src/Cli/Commands/StartCommand.cs | 8 +++++- src/Cli/Lifetime/SingleInstance.cs | 44 ++++++++++++++++++++---------- 2 files changed, 36 insertions(+), 16 deletions(-) diff --git a/src/Cli/Commands/StartCommand.cs b/src/Cli/Commands/StartCommand.cs index 2e8e1af..cb05677 100644 --- a/src/Cli/Commands/StartCommand.cs +++ b/src/Cli/Commands/StartCommand.cs @@ -18,7 +18,13 @@ public StartCommand(IHostBuilder builder) : base("start", HelpDesc) Hidden = true; SetAction(async (_) => { - using var alreadyRunning = SingleInstance.TryAcquire("App"); + using var alreadyRunning = SingleInstance.TryAcquire("AppStart"); + if (!alreadyRunning.IsAcquired) + { + Console.Error.WriteLine("Anonymizer is already running."); + return; + } + builder.ConfigureServices((context, services) => { services.AddHostedService(); diff --git a/src/Cli/Lifetime/SingleInstance.cs b/src/Cli/Lifetime/SingleInstance.cs index b634b59..4fd66f9 100644 --- a/src/Cli/Lifetime/SingleInstance.cs +++ b/src/Cli/Lifetime/SingleInstance.cs @@ -4,10 +4,7 @@ namespace Anonymizer.Cli.Lifetime; internal static class SingleInstance { - /// - /// Returns true if this process is the only running instance. - /// - public static IDisposable TryAcquire(string name) + public static InstanceHandle TryAcquire(string name) { if (OperatingSystem.IsWindows()) return AcquireWindowsMutex(name); if (OperatingSystem.IsLinux()) return AcquireLinuxLockFile(name); @@ -15,30 +12,36 @@ public static IDisposable TryAcquire(string name) } [SupportedOSPlatform("Windows")] - private static IDisposable AcquireWindowsMutex(string name) + private static InstanceHandle AcquireWindowsMutex(string name) { try { Mutex mutex = new( - false, + true, $"Global\\Anonymizer.{name}.Instance", new() { CurrentUserOnly = true }, - out var createdNew); + out bool createdNew); - return new Disposable(() => + if (!createdNew) + { + mutex.Dispose(); + return InstanceHandle.NotAcquired(); + } + + return InstanceHandle.Acquired(new Disposable(() => { mutex?.ReleaseMutex(); mutex?.Dispose(); - }); + })); } catch { - return new NoopDisposable(); + return InstanceHandle.NotAcquired(); } } [SupportedOSPlatform("Linux")] - private static IDisposable AcquireLinuxLockFile(string name) + private static InstanceHandle AcquireLinuxLockFile(string name) { try { @@ -50,24 +53,35 @@ private static IDisposable AcquireLinuxLockFile(string name) FileAccess.ReadWrite, FileShare.None); - return new Disposable(() => + return InstanceHandle.Acquired(new Disposable(() => { stream.Dispose(); if (lockFile.Exists) lockFile.Delete(); - }); + })); } catch (IOException) { // File is locked by another instance - return new NoopDisposable(); + return InstanceHandle.NotAcquired(); } catch { - return new NoopDisposable(); + return InstanceHandle.NotAcquired(); } } + internal sealed class InstanceHandle(bool isAcquired, IDisposable releaser) : IDisposable + { + public bool IsAcquired { get; } = isAcquired; + + public static InstanceHandle Acquired(IDisposable releaser) => new(true, releaser); + + public static InstanceHandle NotAcquired() => new(false, new NoopDisposable()); + + public void Dispose() => releaser.Dispose(); + } + private sealed class NoopDisposable : IDisposable { public void Dispose() { } From 55848e61f67d63137ce0a245adf21b9f30491da2 Mon Sep 17 00:00:00 2001 From: Lionel Lalande Date: Sun, 5 Jul 2026 15:55:40 +0200 Subject: [PATCH 14/17] fix(cli): improve file processing error handling in FilesWatcher --- src/Cli/FilesWatcher.cs | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/Cli/FilesWatcher.cs b/src/Cli/FilesWatcher.cs index f10b314..6dfcae7 100644 --- a/src/Cli/FilesWatcher.cs +++ b/src/Cli/FilesWatcher.cs @@ -58,6 +58,7 @@ public override Task StopAsync(CancellationToken cancellationToken) protected override Task ExecuteAsync(CancellationToken stoppingToken) { + // Only register for process termination! stoppingToken.Register(() => _tcs.TrySetResult()); return Task.CompletedTask; } @@ -73,21 +74,20 @@ private void RecreateWatcher(CancellationToken cancellationToken) EnableRaisingEvents = true, }; _watcher.Created += async (_, e) => - _ = Task.Run(async () => + { + try + { + await Task.Delay(200, cancellationToken); + + LogFileFound(e.FullPath); + FileInfo file = new(e.FullPath); + await _anonymizer.AnonymizeFileAsync(file, cancellationToken); + } + catch (Exception ex) { - try - { - await Task.Delay(200, cancellationToken); - - LogFileFound(e.FullPath); - FileInfo file = new(e.FullPath); - _ = Task.Run(() => _anonymizer.AnonymizeFileAsync(file, cancellationToken), cancellationToken); - } - catch (Exception ex) - { - LogErrorWhileProcessing(ex, e.FullPath); - } - }); + LogErrorWhileProcessing(ex, e.FullPath); + } + }; } #pragma warning disable CA1822 From a35dc719357074f9c332509b067db1b60bbd3a34 Mon Sep 17 00:00:00 2001 From: Lionel Lalande Date: Sun, 5 Jul 2026 14:52:24 +0200 Subject: [PATCH 15/17] ci: build Rust core and publish ONNX artifacts --- .github/workflows/build-csharp.yml | 112 +++++++++++++++++ .github/workflows/build-rust.yml | 56 +++++++++ .github/workflows/ci.yml | 119 +----------------- .github/workflows/download-models.yml | 44 +++++++ .github/workflows/release.yml | 65 +++++++++- .github/workflows/tag.yml | 2 +- scripts/downlaod_uv.ps1 | 54 -------- .../Cli/scripts => scripts}/download-model.py | 0 scripts/download_uv.sh | 50 -------- .../scripts => scripts}/model_downloader.py | 15 ++- 10 files changed, 287 insertions(+), 230 deletions(-) create mode 100644 .github/workflows/build-csharp.yml create mode 100644 .github/workflows/build-rust.yml create mode 100644 .github/workflows/download-models.yml delete mode 100644 scripts/downlaod_uv.ps1 rename {src/Cli/scripts => scripts}/download-model.py (100%) delete mode 100644 scripts/download_uv.sh rename {src/Cli/scripts => scripts}/model_downloader.py (73%) diff --git a/.github/workflows/build-csharp.yml b/.github/workflows/build-csharp.yml new file mode 100644 index 0000000..9828a94 --- /dev/null +++ b/.github/workflows/build-csharp.yml @@ -0,0 +1,112 @@ +name: Build CSharp + +on: + workflow_call: + inputs: + tag: + required: false + type: string + use_rust_artifacts: + required: false + type: boolean + default: true + +jobs: + build-csharp: + name: Build C# (${{ matrix.target }}) + runs-on: ${{ matrix.os }} + strategy: + matrix: + include: + - target: linux + os: ubuntu-latest + rid: linux-x64 + ext: "" + prefix: linux + rust_artifact: rust-core-linux + - target: windows + os: windows-latest + rid: win-x64 + ext: .exe + prefix: win + rust_artifact: rust-core-windows + + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Download Rust artifact (${{ matrix.target }}) + if: inputs.use_rust_artifacts + uses: actions/download-artifact@v8 + with: + name: ${{ matrix.rust_artifact }} + path: src/Core/target/release + + - name: Compute version + id: version + shell: bash + run: | + TAG="${{ inputs.tag || '' }}" + if [ -z "$TAG" ]; then + echo "version=0.0.0" >> $GITHUB_OUTPUT + else + CLEAN="${TAG#v}" + echo "version=$CLEAN" >> $GITHUB_OUTPUT + fi + + - name: Build .Net (${{ matrix.target }}) + shell: bash + run: | + VERSION="${{ steps.version.outputs.version }}" + dotnet publish \ + src/Cli \ + -c Release \ + -r ${{ matrix.rid }} \ + -p:Version=$VERSION \ + -o artifacts/${{ matrix.prefix }} + + - name: Create setup-linux.zip + if: github.event_name != 'pull_request' && matrix.target == 'linux' + shell: bash + run: | + cd artifacts/${{ matrix.prefix }} + zip -r payload.zip appsettings.json Anonymizer.Core.so libonnxruntime.so libonnxruntime_providers_shared.so + + - name: Create setup-win.zip + if: github.event_name != 'pull_request' && matrix.target == 'windows' + shell: pwsh + run: | + Set-Location "artifacts/${{ matrix.prefix }}" + $items = @( + "appsettings.json" + "Anonymizer.Core.dll", + "onnxruntime.dll", + "onnxruntime_providers_shared.dll") + Compress-Archive -Path $items -DestinationPath payload.zip -CompressionLevel Optimal -Force + + - name: Create setup (${{ matrix.target }}) + if: github.event_name != 'pull_request' + shell: bash + run: | + cd artifacts/${{ matrix.prefix }} + MAGIC="SETUP-PAYLOAD" + PAYLOAD_SIZE=$(stat -c%s payload.zip) + cp anonymizer${{ matrix.ext }} setup${{ matrix.ext }} + cat payload.zip >> setup${{ matrix.ext }} + printf "%s" "$MAGIC" | iconv -f ASCII -t ASCII >> setup${{ matrix.ext }} + python3 - <> $GITHUB_OUTPUT - - build: - name: Build - runs-on: ubuntu-latest - needs: [download-uv] - - strategy: - matrix: - target: [linux, windows] - include: - - target: linux - rid: linux-x64 - ext: "" # no extension - prefix: linux - - target: windows - rid: win-x64 - ext: .exe - prefix: win - - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - - name: Compute version - id: version - run: | - TAG="${{ inputs.tag }}" - if [ -z "$TAG" ]; then - echo "version=0.0.0" >> $GITHUB_OUTPUT - else - CLEAN="${TAG#v}" - echo "version=$CLEAN" >> $GITHUB_OUTPUT - fi - - - name: Download uv tools - uses: actions/download-artifact@v8 - with: - name: uv-tools - path: src/Cli/tools - - - name: Build (${{ matrix.target }}) - run: | - VERSION="${{ steps.version.outputs.version }}" - dotnet publish \ - src/Cli \ - -c Release \ - -r ${{ matrix.rid }} \ - -p:Version=$VERSION \ - -o artifacts/${{ matrix.prefix }} - - - name: Create setup.zip (${{ matrix.target }}) - if: github.event_name != 'pull_request' - run: | - cd artifacts/${{ matrix.prefix }} - zip -r payload.zip scripts tools appsettings.json - - - name: Create setup (${{ matrix.target }}) - if: github.event_name != 'pull_request' - run: | - cd artifacts/${{ matrix.prefix }} - MAGIC="SETUP-PAYLOAD" - PAYLOAD_SIZE=$(stat -c%s payload.zip) - cp anonymizer${{ matrix.ext }} setup${{ matrix.ext }} - cat payload.zip >> setup${{ matrix.ext }} - printf "%s" "$MAGIC" | iconv -f ASCII -t ASCII >> setup${{ matrix.ext }} - python3 - < artifact.name === 'onnx-models' && !artifact.expired, + ); + if (hasModels) { + selectedRunId = run.id; + break; + } + } + if (!selectedRunId) { + core.setFailed('No successful run with non-expired onnx-models artifact was found for download-models.yml.'); + return; + } + core.info(`Using onnx-models artifact from run ${selectedRunId}`); + core.setOutput('run_id', selectedRunId.toString()); + + - name: Download model archives artifact + uses: actions/download-artifact@v8 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + repository: ${{ github.repository }} + run-id: ${{ steps.latest-models-run.outputs.run_id }} + name: onnx-models + path: artifacts + - name: Copy install scripts run: | cp ./scripts/install.* ./artifacts/ - name: Changelog id: changelog - uses: orhun/git-cliff-action@v4 + uses: orhun/git-cliff-action@v4.8.0 with: config: cliff.toml args: --latest --tag ${{ github.event.inputs.tag || github.ref_name }} -o artifacts/CHANGELOG.md @@ -59,6 +115,7 @@ jobs: body_path: artifacts/CHANGELOG.md files: | artifacts/CHANGELOG.md + artifacts/*_model.zip artifacts/install.* artifacts/setup* env: diff --git a/.github/workflows/tag.yml b/.github/workflows/tag.yml index 3adcc9f..8e0bc26 100644 --- a/.github/workflows/tag.yml +++ b/.github/workflows/tag.yml @@ -43,7 +43,7 @@ jobs: git push origin "v${{ steps.semver.outputs.majorMinorPatch }}" - name: Trigger release workflow - uses: benc-uk/workflow-dispatch@v1 + uses: benc-uk/workflow-dispatch@v1.3.2 with: workflow: release.yml ref: main diff --git a/scripts/downlaod_uv.ps1 b/scripts/downlaod_uv.ps1 deleted file mode 100644 index a1de97a..0000000 --- a/scripts/downlaod_uv.ps1 +++ /dev/null @@ -1,54 +0,0 @@ -param( - [string]$TargetDir = "./src/Cli/tools" -) - -$UV_URL_LINUX = "https://github.com/astral-sh/uv/releases/latest/download/uv-x86_64-unknown-linux-gnu.tar.gz" -$UV_URL_WIN = "https://github.com/astral-sh/uv/releases/latest/download/uv-x86_64-pc-windows-msvc.zip" - -$TarPath = "uv_linux.tar.gz" -$ZipPath = "uv_windows.zip" -$ExtractLinux = "uv_linux_temp" -$ExtractWin = "uv_windows_temp" - -Write-Host "Creating target directory: $TargetDir..." -New-Item -ItemType Directory -Force -Path $TargetDir | Out-Null - -Write-Host "Downloading uv (Linux)..." -Invoke-WebRequest -Uri $UV_URL_LINUX -OutFile $TarPath - -Write-Host "Downloading uv.exe (Windows)..." -Invoke-WebRequest -Uri $UV_URL_WIN -OutFile $ZipPath - -Write-Host "Extracting Linux archive..." -New-Item -ItemType Directory -Force -Path $ExtractLinux | Out-Null -tar -xzf $TarPath -C $ExtractLinux - -Write-Host "Extracting Windows archive (using .NET)..." -New-Item -ItemType Directory -Force -Path $ExtractWin | Out-Null -Add-Type -AssemblyName System.IO.Compression.FileSystem - -try { - [System.IO.Compression.ZipFile]::ExtractToDirectory($ZipPath, $ExtractWin) -} catch { - Write-Host "ZIP already extracted, continuing..." -} - -Write-Host "Searching for uv.exe..." -$UvExe = Get-ChildItem -Path $ExtractWin -Recurse -Filter "uv.exe" | Select-Object -First 1 - -if (-not $UvExe) { - Write-Host "`nERROR: uv.exe not found in extracted ZIP!" -ForegroundColor Red - exit 1 -} - -Write-Host "Found uv.exe at: $($UvExe.FullName)" -Copy-Item $UvExe.FullName "$TargetDir/uv.exe" -Force - -Write-Host "Copying uv (Linux)..." -Copy-Item "$ExtractLinux/uv-x86_64-unknown-linux-gnu/uv" "$TargetDir/uv" -Force - -Write-Host "Cleaning up..." -Remove-Item $TarPath, $ZipPath -Force -Remove-Item $ExtractLinux, $ExtractWin -Recurse -Force - -Write-Host "`nDone! uv and uv.exe are ready in '$TargetDir'." diff --git a/src/Cli/scripts/download-model.py b/scripts/download-model.py similarity index 100% rename from src/Cli/scripts/download-model.py rename to scripts/download-model.py diff --git a/scripts/download_uv.sh b/scripts/download_uv.sh deleted file mode 100644 index c7f2ca9..0000000 --- a/scripts/download_uv.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash -set -e - -TARGET_DIR="${1:-"./src/Cli/tools"}" - -UV_URL_LINUX="https://github.com/astral-sh/uv/releases/latest/download/uv-x86_64-unknown-linux-gnu.tar.gz" -UV_URL_WIN="https://github.com/astral-sh/uv/releases/latest/download/uv-x86_64-pc-windows-msvc.zip" - -TAR_PATH="uv_linux.tar.gz" -ZIP_PATH="uv_windows.zip" -EXTRACT_LINUX="uv_linux_temp" -EXTRACT_WIN="uv_windows_temp" - -echo "Creating target directory: $TARGET_DIR..." -mkdir -p "$TARGET_DIR" - -echo "Downloading uv (Linux)..." -curl -L "$UV_URL_LINUX" -o "$TAR_PATH" - -echo "Downloading uv.exe (Windows)..." -curl -L "$UV_URL_WIN" -o "$ZIP_PATH" - -echo "Extracting Linux archive..." -mkdir -p "$EXTRACT_LINUX" -tar -xzf "$TAR_PATH" -C "$EXTRACT_LINUX" - -echo "Extracting Windows archive..." -mkdir -p "$EXTRACT_WIN" -unzip -q "$ZIP_PATH" -d "$EXTRACT_WIN" - -echo "Copying uv (Linux) to $TARGET_DIR/uv..." -cp "$EXTRACT_LINUX/uv-x86_64-unknown-linux-gnu/uv" "$TARGET_DIR/uv" -chmod +x "$TARGET_DIR/uv" - -echo "Searching for uv.exe in extracted Windows files..." -UV_EXE_PATH=$(find "$EXTRACT_WIN" -type f -name "uv.exe" | head -n 1) - -if [ -z "$UV_EXE_PATH" ]; then - echo "ERROR: uv.exe not found in extracted ZIP!" - exit 1 -fi - -echo "Found uv.exe at: $UV_EXE_PATH" -cp "$UV_EXE_PATH" "$TARGET_DIR/uv.exe" - -echo "Cleaning up..." -rm -f "$TAR_PATH" "$ZIP_PATH" -rm -rf "$EXTRACT_LINUX" "$EXTRACT_WIN" - -echo "Done! uv and uv.exe are ready in '$TARGET_DIR'." diff --git a/src/Cli/scripts/model_downloader.py b/scripts/model_downloader.py similarity index 73% rename from src/Cli/scripts/model_downloader.py rename to scripts/model_downloader.py index f59deb0..bb6b515 100644 --- a/src/Cli/scripts/model_downloader.py +++ b/scripts/model_downloader.py @@ -1,5 +1,5 @@ from pathlib import Path -from transformers import AutoTokenizer, AutoModelForTokenClassification +from transformers import AutoTokenizer from optimum.onnxruntime import ORTModelForTokenClassification @@ -14,11 +14,12 @@ def download_tokenizer(self): tokenizer.save_pretrained(self.output_dir) def try_download_onnx(self): + """Tente de télécharger un modèle déjà exporté en ONNX sur le Hub.""" try: model = ORTModelForTokenClassification.from_pretrained( self.model_id, export=False, - local_files_only=False + local_files_only=False, ) model.save_pretrained(self.output_dir) return True @@ -26,16 +27,14 @@ def try_download_onnx(self): return False def export_pytorch_to_onnx(self): - tokenizer = AutoTokenizer.from_pretrained(self.model_id) + """Exporte depuis PyTorch si aucun ONNX n'est disponible sur le Hub.""" + # from_transformers est déprécié depuis Optimum 1.14 — export=True suffit ORTModelForTokenClassification.from_pretrained( self.model_id, - from_transformers=True, export=True, - tokenizer=tokenizer, - save_dir=self.output_dir - ) + ).save_pretrained(self.output_dir) - def run(self): + def run(self) -> str: self.download_tokenizer() if self.try_download_onnx(): From 0bdd2a0e2b9e1cf81f2fa7608cfcbfaa6e4c0b6e Mon Sep 17 00:00:00 2001 From: Lionel Lalande Date: Sun, 5 Jul 2026 22:34:24 +0200 Subject: [PATCH 16/17] fix(cli): mutex initialization was incorrect --- src/Cli/Lifetime/SingleInstance.cs | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src/Cli/Lifetime/SingleInstance.cs b/src/Cli/Lifetime/SingleInstance.cs index 4fd66f9..2e5a964 100644 --- a/src/Cli/Lifetime/SingleInstance.cs +++ b/src/Cli/Lifetime/SingleInstance.cs @@ -16,12 +16,7 @@ private static InstanceHandle AcquireWindowsMutex(string name) { try { - Mutex mutex = new( - true, - $"Global\\Anonymizer.{name}.Instance", - new() { CurrentUserOnly = true }, - out bool createdNew); - + Mutex mutex = new(true, $"Anonymizer.{name}.Instance", out bool createdNew); if (!createdNew) { mutex.Dispose(); @@ -30,11 +25,11 @@ private static InstanceHandle AcquireWindowsMutex(string name) return InstanceHandle.Acquired(new Disposable(() => { - mutex?.ReleaseMutex(); - mutex?.Dispose(); + mutex.ReleaseMutex(); + mutex.Dispose(); })); } - catch + catch (Exception) { return InstanceHandle.NotAcquired(); } From 7ae9b48a84ed2a03762e9ae265b7cccc371bc96b Mon Sep 17 00:00:00 2001 From: Lionel Lalande Date: Sun, 5 Jul 2026 22:34:59 +0200 Subject: [PATCH 17/17] fix(cli): remove unnecessary output directory deletion --- src/Cli/Commands/DownloadCommand.cs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/Cli/Commands/DownloadCommand.cs b/src/Cli/Commands/DownloadCommand.cs index fbe91aa..18012bf 100644 --- a/src/Cli/Commands/DownloadCommand.cs +++ b/src/Cli/Commands/DownloadCommand.cs @@ -31,10 +31,6 @@ private static async Task DownloadAndUncompress(ModelDownloader modelDownloader, if (modelsRoot is null) throw new InvalidOperationException("Unable to resolve models root directory."); - outputDir.Refresh(); - if (outputDir.Exists) - outputDir.Delete(true); - string tempDirPath = Path.Combine(Path.GetTempPath(), "anonymizer-models"); Directory.CreateDirectory(tempDirPath);