From 4820f9995c05b3d421e9b03f6cc7ac4177463e15 Mon Sep 17 00:00:00 2001 From: sysangels | Michal Koeckeis-Fresel Date: Thu, 29 Jan 2026 08:57:39 +0100 Subject: [PATCH] plugin malware-scan v.0.8.0 --- malware-scan/GDPR.md | 916 +++++++++ malware-scan/README.md | 389 ++++ malware-scan/SECURITY.md | 75 + malware-scan/TECHNICAL.md | 119 ++ malware-scan/install-docker-clamav.md | 1494 +++++++++++++++ malware-scan/install-docker-clamav.sh | 866 +++++++++ malware-scan/install.md | 573 ++++++ malware-scan/install.sh | 275 +++ malware-scan/jobs/README.md | 331 ++++ .../jobs/malware-scan-cache-cleanup.py | 334 ++++ malware-scan/jobs/malware-scan-update.lua | 96 + malware-scan/jobs/malware-scan-update.py | 1569 ++++++++++++++++ malware-scan/malware-scan.lua | 437 +++++ malware-scan/malware_scan_api.lua | 739 ++++++++ malware-scan/malware_scan_cache.lua | 1634 +++++++++++++++++ malware-scan/malware_scan_clamav.lua | 435 +++++ malware-scan/malware_scan_config.lua | 884 +++++++++ malware-scan/malware_scan_custom_hashes.lua | 688 +++++++ malware-scan/malware_scan_database.lua | 515 ++++++ malware-scan/malware_scan_file.lua | 1286 +++++++++++++ malware-scan/malware_scan_filetypes.lua | 347 ++++ malware-scan/malware_scan_init.lua | 423 +++++ malware-scan/malware_scan_logger.lua | 180 ++ malware-scan/malware_scan_malwarebazaar.lua | 186 ++ malware-scan/malware_scan_metrics.lua | 271 +++ .../malware_scan_multipart_full_scan.lua | 462 +++++ malware-scan/malware_scan_orchestrator.lua | 1539 ++++++++++++++++ malware-scan/malware_scan_scheduler.lua | 925 ++++++++++ malware-scan/malware_scan_sentinelone.lua | 270 +++ malware-scan/malware_scan_ui_metrics.lua | 188 ++ malware-scan/malware_scan_utils.lua | 538 ++++++ malware-scan/malware_scan_virustotal.lua | 531 ++++++ malware-scan/malware_scan_webhook.lua | 745 ++++++++ .../pictures/discord-message-on-hit.png | Bin 0 -> 112846 bytes malware-scan/plugin.json | 556 ++++++ malware-scan/ui/actions.py | 506 +++++ malware-scan/ui/blueprints/__init__.py | 5 + malware-scan/ui/blueprints/malware_scan.py | 418 +++++ malware-scan/ui/template.html | 689 +++++++ malware-scan/variables.env.example | 157 ++ 40 files changed, 22591 insertions(+) create mode 100644 malware-scan/GDPR.md create mode 100644 malware-scan/README.md create mode 100644 malware-scan/SECURITY.md create mode 100644 malware-scan/TECHNICAL.md create mode 100644 malware-scan/install-docker-clamav.md create mode 100644 malware-scan/install-docker-clamav.sh create mode 100644 malware-scan/install.md create mode 100755 malware-scan/install.sh create mode 100644 malware-scan/jobs/README.md create mode 100644 malware-scan/jobs/malware-scan-cache-cleanup.py create mode 100644 malware-scan/jobs/malware-scan-update.lua create mode 100755 malware-scan/jobs/malware-scan-update.py create mode 100644 malware-scan/malware-scan.lua create mode 100644 malware-scan/malware_scan_api.lua create mode 100644 malware-scan/malware_scan_cache.lua create mode 100644 malware-scan/malware_scan_clamav.lua create mode 100644 malware-scan/malware_scan_config.lua create mode 100644 malware-scan/malware_scan_custom_hashes.lua create mode 100644 malware-scan/malware_scan_database.lua create mode 100644 malware-scan/malware_scan_file.lua create mode 100644 malware-scan/malware_scan_filetypes.lua create mode 100644 malware-scan/malware_scan_init.lua create mode 100644 malware-scan/malware_scan_logger.lua create mode 100644 malware-scan/malware_scan_malwarebazaar.lua create mode 100644 malware-scan/malware_scan_metrics.lua create mode 100644 malware-scan/malware_scan_multipart_full_scan.lua create mode 100644 malware-scan/malware_scan_orchestrator.lua create mode 100644 malware-scan/malware_scan_scheduler.lua create mode 100644 malware-scan/malware_scan_sentinelone.lua create mode 100644 malware-scan/malware_scan_ui_metrics.lua create mode 100644 malware-scan/malware_scan_utils.lua create mode 100644 malware-scan/malware_scan_virustotal.lua create mode 100644 malware-scan/malware_scan_webhook.lua create mode 100644 malware-scan/pictures/discord-message-on-hit.png create mode 100644 malware-scan/plugin.json create mode 100644 malware-scan/ui/actions.py create mode 100644 malware-scan/ui/blueprints/__init__.py create mode 100644 malware-scan/ui/blueprints/malware_scan.py create mode 100644 malware-scan/ui/template.html create mode 100644 malware-scan/variables.env.example diff --git a/malware-scan/GDPR.md b/malware-scan/GDPR.md new file mode 100644 index 0000000..897aa5f --- /dev/null +++ b/malware-scan/GDPR.md @@ -0,0 +1,916 @@ +# GDPR Compliance Guide for Malware-Scan Plugin + +**Version**: 0.8.0 +**Last Updated**: 2026-01-27 +**Scope**: European Union General Data Protection Regulation (GDPR) compliance + +## Table of Contents + +1. [Executive Summary](#executive-summary) +2. [Data Processing Overview](#data-processing-overview) +3. [Legal Basis](#legal-basis) +4. [Data Collected](#data-collected) +5. [GDPR Compliance Measures](#gdpr-compliance-measures) +6. [Administrator Responsibilities](#administrator-responsibilities) +7. [Data Subject Rights](#data-subject-rights) +8. [Recommended Privacy Policy](#recommended-privacy-policy) +9. [Compliance Checklist](#compliance-checklist) + +--- + +## Executive Summary + +The malware-scan plugin processes personal data (IP addresses) as part of its security functionality. This document outlines the GDPR compliance measures implemented in the plugin and the responsibilities of administrators who deploy it. + +### Key Points + +- **Legal Basis**: Legitimate Interest (Article 6(1)(f)) - Security monitoring and infrastructure protection +- **Data Controller**: The administrator deploying BunkerWeb, not the plugin itself +- **Personal Data**: Client IP addresses, upload timestamps, file hashes +- **Retention**: 90 days (cache), up to 90 days (logs) +- **Data Minimization**: IP anonymization in webhooks (default: enabled) +- **Rights**: Access and erasure requests handled by administrators + +### Compliance Status + +โœ… **Implemented**: +- IP anonymization in webhooks (Article 5(1)(c) - Data Minimization) +- Configurable retention periods (Article 5(1)(e) - Storage Limitation) +- Legitimate interest legal basis (Article 6(1)(f)) +- Security measures for data protection (Article 32) +- **File uploads to public databases DISABLED by default** (VirusTotal, MalwareBazaar) - GDPR-safe configuration + +โš ๏ธ **Administrator Action Required**: +- Privacy policy documentation (Article 13-14 - Transparency) +- Data subject rights implementation (Articles 15-17) +- Log retention policy enforcement (Article 5(1)(e)) + +๐Ÿšจ **CRITICAL - File Uploads to Public Databases**: +- **Default**: DISABLED for both services + - `MALWARE_SCAN_API_VIRUSTOTAL_UPLOAD_UNKNOWN=no` + - `MALWARE_SCAN_API_MALWAREBAZAAR_UPLOAD_UNKNOWN=no` +- **If enabling**: Uploaded malware files become **publicly accessible** in VirusTotal and/or MalwareBazaar databases +- **GDPR Risk**: Files may contain personal data (documents, emails, images) +- **Required Action**: Consult legal counsel, document legal basis, add explicit privacy notice +- **Recommendation**: Keep DISABLED unless you have strong legal justification + +--- + +## Data Processing Overview + +### What is Personal Data? + +Under GDPR Article 4(1), **personal data** means: +> "any information relating to an identified or identifiable natural person" + +**IP addresses are personal data** per CJEU Case C-582/14 (Breyer v Germany). + +### Purpose of Data Processing + +The plugin processes personal data for: +1. **Malware detection** - Scanning uploaded files for threats +2. **Security monitoring** - Tracking repeat malware upload attempts +3. **Incident response** - Alerting administrators of attacks +4. **Infrastructure protection** - Blocking malicious uploads + +--- + +## Legal Basis + +### Article 6(1)(f) - Legitimate Interest + +The plugin relies on **legitimate interest** as the legal basis for processing IP addresses: + +**Purpose**: Security monitoring - tracking attackers and distributed malware to fulfill security expectations + +**Necessity**: Full IP addresses (not anonymized) are needed for: +- Forensic evidence of malicious intent +- Proving attacks in legal proceedings (Article 17(3)(e) exception) +- Protecting servers, data, and users from malware + +**Balancing Test**: +- **Controller's Interest**: Protecting infrastructure from malware attacks (high priority) +- **Data Subject's Interest**: Privacy expectations when uploading malware (low expectation of privacy for malicious activity) +- **Result**: Security interest outweighs attacker's privacy expectations + +### Why Not Consent? + +Consent (Article 6(1)(a)) is **not appropriate** for security monitoring because: +- Attackers would not provide consent +- Security measures must be effective regardless of consent +- Consent can be withdrawn, undermining security +- Legitimate interest is the proper legal basis for security purposes + +--- + +## Data Collected + +### 1. Attacker Tracking Data + +**Enabled when**: `MALWARE_SCAN_TRACK_ATTACKERS=yes` (default) + +| Data Element | Example | Purpose | Retention | +|--------------|---------|---------|-----------| +| IP Address | `192.168.1.123` | Identify repeat offenders | 90 days (configurable) | +| First Upload | `2026-01-19 10:30:00 UTC` | Track attack timeline | 90 days (configurable) | +| Last Upload | `2026-01-19 14:22:15 UTC` | Track attack timeline | 90 days (configurable) | +| Upload Count | `5` | Identify repeat offenders | 90 days (configurable) | +| Checksums | `["abc123...", "def456..."]` | Link malware samples | 90 days (configurable) | + +**Storage Location**: Redis cache +**Cache Key Format**: `plugin_malware_scan_attacker_[ipv4|ipv6]_[IP_ADDRESS]` +**TTL**: Configurable via `MALWARE_SCAN_TRACK_ATTACKER_TTL` (default: 7776000 seconds / 90 days) +**TTL Behavior**: TTL resets on each new malware upload from the same IP. The 90-day window is always from the LAST upload, not the first. +**Rationale**: Extended 90-day retention allows sufficient time to gather evidence for law enforcement and legal proceedings + +### 2. Scan Results Cache + +**Purpose**: Performance optimization - avoid re-scanning known files + +| Data Element | Example | Purpose | Retention | +|--------------|---------|---------|-----------| +| File Hash (SHA256) | `abc123def456...` | Identify file | 5 min (clean) / 90 days (malicious) | +| Detection Results | `Win.Trojan.Zusy-9935890-0` | Cache scanner responses | 5 min (clean) / 90 days (malicious) | +| Scanner Names | `ClamAV, VirusTotal` | Cache scanner responses | 5 min (clean) / 90 days (malicious) | + +**Storage Location**: Redis cache +**Cache Key Format**: `plugin_malware_scan_result_[SHA256]` +**TTL**: 300 seconds (clean files), 7776000 seconds (malicious files) +**Rationale**: Long retention for malicious hashes reduces API calls, improves performance for repeat attacks, and maintains detection history + +**Note**: File hashes alone are **not personal data**, but combined with IP address tracking may constitute personal data through linkability. + +### 3. Application Logs + +**Purpose**: Security monitoring, debugging, incident investigation + +| Data Element | Example | Storage | Retention | +|--------------|---------|---------|-----------| +| IP Addresses | `192.168.1.123` | BunkerWeb logs | Administrator controlled (recommend โ‰ค90 days) | +| Timestamps | `2026-01-19 10:30:00` | BunkerWeb logs | Administrator controlled (recommend โ‰ค90 days) | +| Detection Results | `ClamAV: Win.Trojan.Zusy` | BunkerWeb logs | Administrator controlled (recommend โ‰ค90 days) | +| File Checksums | `abc123def456...` | BunkerWeb logs | Administrator controlled (recommend โ‰ค90 days) | +| Server Names | `www.example.com` | BunkerWeb logs | Administrator controlled (recommend โ‰ค90 days) | +| Request URIs | `/upload.php` | BunkerWeb logs | Administrator controlled (recommend โ‰ค90 days) | + +**Storage Location**: `/var/log/bunkerweb/` (default) or custom log path +**Retention**: Not automatically managed - administrator responsibility +**Recommendation**: Implement log rotation with maximum 90-day retention + +### 4. Webhook Notifications (Optional) + +**Enabled when**: `MALWARE_SCAN_TEAMS_WEBHOOK_URL` or `MALWARE_SCAN_DISCORD_WEBHOOK_URL` configured + +**Data Minimization**: IP addresses in webhooks are **anonymized by default** (GDPR Article 5(1)(c)) + +| Data Element | Original | Anonymized (default) | Purpose | +|--------------|----------|----------------------|---------| +| IPv4 Address | `192.168.1.123` | `192.168.1.x` | Alert security team | +| IPv6 Address | `2001:db8::1234:5678` | `2001:db8::1234:x` | Alert security team | +| Server Name | `www.example.com` | `www.example.com` | Identify affected service | +| Detection Results | `ClamAV: Win.Trojan` | `ClamAV: Win.Trojan` | Understand threat | +| File Checksum | `abc123def456...` | `abc123def456...` | Identify malware sample | + +**Configuration**: `MALWARE_SCAN_ANONYMIZE_WEBHOOK_IPS=yes` (default: yes) +**Third Parties**: Microsoft Teams, Discord (US companies - international data transfer) +**Retention by Third Parties**: Potentially indefinite (no control) + +### 5. File Uploads to Third Parties (Optional but HIGH IMPACT) + +**โš ๏ธ CRITICAL GDPR CONCERN**: When enabled, malware files (which may contain personal data) are uploaded to third-party threat intelligence services. + +**Enabled when**: +- `MALWARE_SCAN_API_VIRUSTOTAL_UPLOAD_UNKNOWN=yes` +- `MALWARE_SCAN_API_MALWAREBAZAAR_UPLOAD_UNKNOWN=yes` + +**What Gets Uploaded**: +- **Complete file contents** - The entire malware file detected by ClamAV is uploaded +- **Files may contain personal data** - Documents, emails, images, or other files with embedded personal information +- **Trigger condition (VirusTotal)**: ClamAV detects malware AND VirusTotal doesn't have the file in their database +- **Trigger condition (MalwareBazaar)**: ClamAV detects malware AND MalwareBazaar doesn't have the file in their database + +**Third-Party Data Sharing**: + +| Recipient | Data Shared | Purpose | Retention | Public Access | +|-----------|-------------|---------|-----------|---------------| +| **VirusTotal** (Google LLC, USA) | Complete file contents, SHA256 hash, file metadata | Malware analysis and threat intelligence | Indefinite | **YES** - Files become part of VirusTotal's public database accessible to security researchers | +| **MalwareBazaar** (abuse.ch, Switzerland) | Complete file contents, SHA256 hash, file metadata, tags | Malware sample sharing and threat intelligence | Indefinite | **YES** - Files become part of MalwareBazaar's public database accessible to security researchers worldwide | + +**GDPR Implications**: +- **Article 5(1)(c) - Data Minimization**: Uploading entire files (vs. hashes) is extensive data sharing +- **Article 6 - Legal Basis**: Legitimate interest may not cover sharing files containing personal data with public databases +- **Article 44-50 - International Transfer**: Transfer to USA (Google) without adequate protections +- **Article 9 - Special Categories**: If files contain health data, biometric data, etc., this may violate GDPR +- **Transparency**: Users may not expect uploaded malware files to become publicly accessible + +**Recommendation**: +- **File uploads are DISABLED by default**: + - `MALWARE_SCAN_API_VIRUSTOTAL_UPLOAD_UNKNOWN=no` (default) + - `MALWARE_SCAN_API_MALWAREBAZAAR_UPLOAD_UNKNOWN=no` (default) +- **Only enable** if you understand the legal implications and have documented justification +- **Add explicit notice** in privacy policy about file sharing with VirusTotal and/or MalwareBazaar +- **Consider consent requirement** if files might contain personal data + +**โœ… Automatic GDPR Protection (Built-In File Type Filtering)**: + +Even if file upload is enabled, the plugin **automatically protects user data** with file type filtering: + +- **Only uploads**: Executables (PE, ELF, Mach-O), scripts (Python, PHP, shell, JavaScript), bytecode (Java, Python .pyc), WebAssembly binaries +- **NEVER uploads**: Documents (PDF, Word, Excel), images (JPEG, PNG, GIF, RAW), databases (SQLite, Access), archives (ZIP, RAR, 7z, TAR), disk images (ISO, DMG), emails, audio, video, or any file that may contain personal data + +**How It Works**: +1. Plugin detects file type using Unix `file` command (magic byte analysis) at scan start +2. Checks file type against user data categories (129 MIME types) before ANY upload +3. If file may contain personal data โ†’ BLOCKED from upload with log message +4. If file is executable malware โ†’ Allowed to upload (shares malware with community) +5. **Protection applies to ALL scanners**: ClamAV direct uploads AND SentinelOne/MalwareBazaar sharing via `detection_share()` + +**Example Log Messages**: +- Blocked (ClamAV): `"ClamAV detected malware but file may contain user data (type: application/pdf) - GDPR protection: NOT uploading to VirusTotal"` +- Blocked (Unknown): `"[SHARE] MalwareBazaar detected malware but file type unknown - GDPR protection: NOT uploading to VirusTotal (cannot verify if file contains personal data)"` +- Allowed (Executable): `"ClamAV detected malware but VT doesn't have file (type: application/x-dosexec) - uploading to VirusTotal"` +- Allowed (Share): `"[SHARE] File unknown to VirusTotal and safe to share (type: text/x-php) - uploading for threat intelligence (detected by SentinelOne)"` + +**File Type Categories Protected**: +- Documents: PDF, DOC/DOCX, XLS/XLSX, PPT/PPTX, ODT, ODS, ODP +- Images: JPEG, PNG, GIF, BMP, TIFF, WebP, HEIC, AVIF, PSD, Camera RAW (CR2, RAF, ORF) +- Databases: SQLite, Microsoft Access, GeoPackage, dBase +- Medical: DICOM medical imaging (HIPAA/GDPR Article 9 special category data) +- Financial: GnuCash financial data +- Archives: ZIP, RAR, 7z, TAR, GZIP, BZIP2, XZ (may contain personal documents) +- Disk Images: ISO, DMG, VHD, OVA (may contain personal file systems) +- Emails: .eml, MBOX mailbox files +- Audio: MP3, WAV, FLAC, OGG, AAC (voice recordings) +- Video: MP4, AVI, MKV, MOV, WebM (may contain faces, locations) + +**Result**: This automated protection significantly reduces GDPR risk even if upload is enabled. Only true executable malware is shared with threat intelligence services. + +**Hash-Only Sharing (Lower Risk)**: + +Unlike file uploads, hash-only operations do not transfer personal data: + +**VirusTotal Voting**: +- **Enabled when**: `MALWARE_SCAN_API_VIRUSTOTAL_VOTE_MALICIOUS=yes` or `MALWARE_SCAN_API_VIRUSTOTAL_VOTE_HARMLESS=yes` +- **Data Shared**: Only SHA256 hash + verdict (malicious/harmless) - **No file contents** +- **Risk Level**: Low - Hash voting does not transfer personal data + +**Hash Lookups** (VirusTotal, SentinelOne, MalwareBazaar): +- **Data Shared**: Only SHA256 hash - **No file contents** +- **Risk Level**: Low - Hashes alone are not personal data + +--- + +## GDPR Compliance Measures + +### 1. Data Minimization (Article 5(1)(c)) + +**Issue**: Sharing full IP addresses with third parties (Microsoft Teams/Discord) excessive for security alerting. + +**Solution**: Implemented IP anonymization in webhooks (v0.7.2) +- IPv4: `192.168.1.123` โ†’ `192.168.1.x` +- IPv6: `2001:db8:1234:5678:90ab:cdef:1234:5678` โ†’ `2001:db8:1234:5678:90ab:cdef:1234:x` +- Enabled by default: `MALWARE_SCAN_ANONYMIZE_WEBHOOK_IPS=yes` +- Network-level information sufficient for security teams + +**Code**: [malware_scan_webhook.lua:133-161](malware_scan_webhook.lua#L133-L161) + +### 2. Storage Limitation (Article 5(1)(e)) + +**Requirement**: Personal data must be kept only as long as necessary for the purpose. + +**Implementation**: +- Attacker tracking: 90 days default (configurable: `MALWARE_SCAN_TRACK_ATTACKER_TTL`) +- Scan results: 5 minutes (clean) / 90 days (malicious) +- Logs: Administrator controlled (recommend maximum 90 days) + +**Rationale**: +- 90 days provides adequate time for law enforcement and legal proceedings +- Allows gathering comprehensive evidence of attack patterns +- Malicious file hashes rarely change, making long-term caching beneficial +- Balances security needs with privacy rights + +### 3. Security of Processing (Article 32) + +**Measures Implemented** (v0.7.14): +- โœ… **Input validation** - Prevents command injection, path traversal (malware_scan_utils.lua) +- โœ… **Data sanitization** - Prevents log injection attacks (malware_scan_utils.lua) +- โœ… **Credential masking** - Automatic API key/token masking in logs (malware_scan_logger.lua) +- โœ… **Filename sanitization** - Path traversal protection with validation (malware_scan_file.lua) +- โœ… **Download integrity** - SHA256 checksum verification for all downloads (jobs/malware-scan-update.py) +- โœ… **Access control** - Restrictive file system permissions +- โœ… **Encryption in transit** - HTTPS webhooks and API calls +- โœ… **Pseudonymization** - IP anonymization in webhooks (default: enabled) +- โœ… **DoS protection** - File size limits and rate limiting +- โœ… **Secure temporary files** - mktemp with Redis tracking for cleanup +- โœ… **Email anonymization** - Automatic email redaction in logs (GDPR Article 5(1)(c)) + +**Enhanced Security Features** (v0.7.14): +- **Automatic Credential Masking**: All log messages automatically mask: + - API keys (64 hex chars) โ†’ `[APIKEY:abcd****9876]` + - Long tokens (32+ chars) โ†’ `[TOKEN:abc1****6789]` + - Bearer tokens โ†’ `Bearer [MASKED]` + - Webhook URLs โ†’ `https://hooks.***/[MASKED]` + - Email addresses โ†’ `***@***.***` + - Password/key parameters in URLs โ†’ `[MASKED]` + +- **Download Integrity Verification**: + - SHA256 checksum calculated for all downloaded files + - **Published checksum verification** - Compares against official `.sha256` files from MalwareBazaar + - Corrupted downloads detected and removed automatically + - **Tampering detection** - Rejects files that don't match published checksums + - Cached file tampering detection with checksum comparison + - State persistence includes checksums for audit trail + - **Double verification layer**: Calculated checksum + published checksum comparison + +**Code References**: +- Input validation: [malware_scan_utils.lua:246-376](malware_scan_utils.lua#L246-L376) +- Credential masking: [malware_scan_logger.lua:52-100](malware_scan_logger.lua#L52-L100) +- Download verification: [jobs/malware-scan-update.py:313-385](jobs/malware-scan-update.py#L313-L385) +- Filename validation: [malware_scan_file.lua:156-163](malware_scan_file.lua#L156-L163) + +### 4. Data Retention Policy (Article 5(1)(e)) + +**Automatic Data Expiry Implementation**: + +The plugin implements automatic data expiry using TTL (Time To Live) mechanisms to ensure personal data is not retained longer than necessary: + +#### Cache-Based Data (Redis TTL) +All cache entries automatically expire without manual intervention: + +| Data Type | TTL | Auto-Deletion | Configuration | +|-----------|-----|---------------|---------------| +| Attacker tracking | 90 days | โœ… Automatic | `MALWARE_SCAN_TRACK_ATTACKER_TTL` | +| Malicious file results | 90 days | โœ… Automatic | `MALWARE_SCAN_CACHE_VIRUS_TTL` | +| Clean file results | 5 minutes | โœ… Automatic | `MALWARE_SCAN_CACHE_CLEAN_TTL` | +| Shared database cache | 300 seconds | โœ… Automatic | `MALWARE_SCAN_CACHE_SHARED_TTL` | + +**Important TTL Behavior**: +- Attacker tracking TTL **resets on each new upload** from the same IP +- 90-day retention window is always measured from the **LAST upload**, not the first +- This extended retention for persistent attackers is justified by legitimate interest in gathering comprehensive evidence for law enforcement + +#### Log-Based Data (Administrator Managed) +Application logs require administrator configuration: + +| Data Type | Recommended Retention | Implementation | Responsibility | +|-----------|----------------------|----------------|----------------| +| Application logs | โ‰ค90 days | logrotate or equivalent | Administrator | +| Audit logs (GDPR requests) | Longer (accountability) | Manual configuration | Administrator | +| Backup archives | Backup rotation period | Backup system | Administrator | + +**Logrotate Example Configuration**: +```bash +/var/log/bunkerweb/*.log { + daily + rotate 90 + compress + delaycompress + missingok + notifempty + create 0640 nginx adm +} +``` + +#### Automatic Cleanup on Restart +The plugin supports cache cleanup on restart for enhanced privacy: + +| Configuration Variable | Effect | +|------------------------|--------| +| `MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_ALL=yes` | Clear all cache on restart | +| `MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_CLAMAV=yes` | Clear ClamAV cache only | +| `MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_VIRUSTOTAL=yes` | Clear VirusTotal cache only | +| `MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_SENTINELONE=yes` | Clear SentinelOne cache only | +| `MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_MALWAREBAZAAR=yes` | Clear MalwareBazaar cache only | +| `MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_ATTACKER_IPV4=yes` | Clear attacker tracking (IPv4) | +| `MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_ATTACKER_IPV6=yes` | Clear attacker tracking (IPv6) | + +**GDPR Erasure Compliance**: +- Manual deletion **not required** for cache data (TTL handles expiry automatically) +- Administrators should log erasure requests for audit trail per Article 5(2) +- Maximum retention period: 90 days from last upload (automatic expiry) + +### 5. Transparency (Articles 13-14) + +**Requirement**: Data subjects must be informed about data processing. + +**Administrator Action Required**: Add privacy policy notice (see [Recommended Privacy Policy](#recommended-privacy-policy)) + +**Minimum Information to Provide**: +- Identity and contact details of data controller +- Purpose of processing (malware detection and security) +- Legal basis (legitimate interest) +- Legitimate interests pursued (infrastructure protection) +- Retention period (90 days cache, 90 days logs) +- Data subject rights (access, erasure, object) +- Right to lodge complaint with supervisory authority + +--- + +## Administrator Responsibilities + +### You Are the Data Controller + +As a BunkerWeb administrator deploying the malware-scan plugin, **you** are the **data controller** under GDPR Article 4(7): + +> "the natural or legal person [...] which, alone or jointly with others, determines the purposes and means of the processing of personal data" + +The plugin is a tool/processor - you determine how it's used and are responsible for GDPR compliance. + +### Your Obligations + +#### 1. Privacy Policy (Articles 13-14) + +**Required**: Yes +**Deadline**: Before collecting personal data +**Action**: Add malware scanning section to your website's privacy policy (see template below) + +#### 2. Log Retention Policy (Article 5(1)(e)) + +**Required**: Yes +**Recommendation**: Maximum 90 days for malware-related logs +**Action**: Configure log rotation + +```bash +# Example: logrotate configuration +/var/log/bunkerweb/*.log { + daily + rotate 90 + compress + delaycompress + missingok + notifempty + create 0640 nginx adm +} +``` + +#### 3. Data Subject Rights (Articles 15-17, 20) + +**Required**: Yes +**Action**: Implement process for handling access and erasure requests (see [Data Subject Rights](#data-subject-rights)) + +#### 4. Records of Processing Activities (Article 30) + +**Required**: If your organization has 250+ employees OR processes special categories of personal data OR processing poses high risk +**Recommendation**: Document processing activities even if not required +**Action**: Maintain internal documentation of: +- What data is collected (IP addresses, timestamps, hashes) +- Why it's collected (security monitoring) +- Legal basis (legitimate interest) +- Retention period (90 days cache, 90 days logs) +- Security measures (encryption, access control) + +#### 5. Data Protection Impact Assessment (Article 35) + +**Required**: Only if processing is "likely to result in a high risk" (not typical for security monitoring) +**Action**: Not usually required for malware scanning + +#### 6. International Data Transfer (Articles 44-50) + +**Applies If**: Using webhooks to US-based services (Microsoft Teams, Discord) +**Issue**: US does not have adequate data protection per GDPR +**Mitigation**: +- Plugin anonymizes IPs by default (reduces risk) +- Consider EU-based webhook alternatives +- Document reliance on legitimate interest +- Add data transfer notice in privacy policy + +--- + +## Data Subject Rights + +### Overview + +Data subjects (including attackers) have rights under GDPR even when legitimate interest applies: + +| Right | Article | Your Obligation | +|-------|---------|-----------------| +| **Access** | 15 | Provide copy of data you hold | +| **Rectification** | 16 | Correct inaccurate data | +| **Erasure** | 17 | Delete data (with exceptions) | +| **Restriction** | 18 | Limit processing | +| **Portability** | 20 | Provide machine-readable format | +| **Object** | 21 | Stop processing unless compelling grounds | + +### Important Exception: Article 17(3)(e) + +You may **refuse erasure requests** if the data is needed: +> "for the establishment, exercise or defence of legal claims" + +**This includes**: Evidence of malware attacks for potential legal proceedings. + +--- + +## Handling Data Subject Access Requests (Article 15) + +### Request Format + +Data subjects must prove their identity. Common approaches: +- Request from the same IP address tracked +- Email with verification +- Formal written request with ID proof + +### Steps to Process Access Request + +#### 1. Verify Identity +Ensure requester is the data subject or authorized representative. + +#### 2. Query Redis Cache + +Check attacker tracking data: + +```bash +# For IPv4 +redis-cli GET "plugin_malware_scan_attacker_ipv4_192.168.1.123" + +# For IPv6 +redis-cli GET "plugin_malware_scan_attacker_ipv6_2001:db8:1234:5678:90ab:cdef:1234:5678" +``` + +**Output Example**: +```json +{ + "upload_count": 3, + "first_upload": 1737277800, + "last_upload": 1737291720, + "checksums": ["abc123def456...", "789xyz012..."] +} +``` + +#### 3. Search Application Logs + +Search for all log entries containing the IP address: + +```bash +# Search logs +grep "192.168.1.123" /var/log/bunkerweb/*.log + +# Search with date filter +find /var/log/bunkerweb/ -name "*.log" -mtime -90 -exec grep "192.168.1.123" {} + +``` + +#### 4. Prepare Response + +Compile information into structured format: + +```json +{ + "data_subject_ip": "192.168.1.123", + "data_controller": "Your Organization Name", + "request_date": "2026-01-19", + "data_collected": { + "attacker_tracking": { + "upload_count": 3, + "first_upload": "2026-01-19 10:30:00 UTC", + "last_upload": "2026-01-19 14:22:15 UTC", + "malware_hashes": ["abc123...", "789xyz..."] + }, + "log_entries": [ + { + "timestamp": "2026-01-19 10:30:00", + "server": "www.example.com", + "detection": "ClamAV: Win.Trojan.Zusy-9935890-0", + "hash": "abc123def456..." + } + ] + }, + "processing_details": { + "purpose": "Malware detection and infrastructure protection", + "legal_basis": "Legitimate interest (GDPR Article 6(1)(f))", + "retention_period": "90 days (cache), up to 90 days (logs)", + "recipients": "Microsoft Teams / Discord (if webhooks enabled, anonymized)", + "rights": "Access, erasure (with exceptions), restriction, object" + } +} +``` + +#### 5. Deliver Response + +- **Deadline**: Within 1 month (Article 12(3)) +- **Format**: Free of charge, electronic format preferred +- **Language**: Same language as request + +--- + +## Handling Erasure Requests (Article 17) + +### Evaluation Process + +#### Step 1: Assess Legal Grounds for Refusal + +You may refuse erasure if data is needed for **legal claims** (Article 17(3)(e)): + +**Refuse if**: +- Evidence of ongoing attack +- Potential legal proceedings anticipated +- Required for law enforcement cooperation +- Needed to defend against lawsuits + +**Grant if**: +- Single, isolated detection months ago +- No ongoing security concern +- No legal proceedings anticipated + +#### Step 2: If Granting Erasure + +**Automatic Data Expiry**: + +The plugin uses TTL (Time To Live) for automatic data expiry: +- **Attacker tracking cache**: Expires automatically after 90 days (default, configurable via `MALWARE_SCAN_TRACK_ATTACKER_TTL`) + - **Important**: TTL resets on each new malware upload from the same IP. For continuous attackers, the 90-day window is from the LAST upload. +- **Scan result cache**: Expires automatically after 5 minutes (clean files) or 90 days (malicious files) + +**No manual deletion required** - the cache system handles expiry automatically. + +**Action Required**: Log the erasure request for audit trail + +```bash +# Log the erasure request (IMPORTANT: for audit trail) +echo "[$(date)] GDPR_ERASURE: Request from 192.168.1.123 - Status: GRANTED - Data expires via TTL (90 days)" >> /var/log/gdpr_requests.log +``` + +**Important Notes**: +- Data will be automatically deleted when TTL expires (maximum 90 days) +- Do NOT delete from application logs if needed for audit/legal purposes +- Keep erasure request log for accountability (Article 5(2)) +- Inform data subject that cache data expires within 90 days automatically + +#### Step 3: If Refusing Erasure + +Document the refusal: + +```bash +# Log the erasure request and reason for refusal +echo "[$(date)] GDPR_ERASURE: Request from 192.168.1.123 - Status: REFUSED - Reason: Evidence needed for legal claims (Art 17(3)(e))" >> /var/log/gdpr_requests.log +``` + +#### Step 4: Respond to Data Subject + +**If granted**: +``` +Your erasure request has been granted as of [DATE]. + +Data Retention Details: +- Cached data (Redis): Will expire automatically within 90 days via TTL (Time To Live) +- Application logs: May be retained for audit purposes per Article 17(3)(e) +- Backups: May contain data until backup rotation completes +- Erasure request: Logged for accountability per Article 5(2) + +Your cached data will be automatically removed from our systems when the retention period +expires (maximum 90 days from last upload attempt). + +Important: If you continue uploading malware, the 90-day window resets with each new upload, +extending the retention period. This extended period allows us to maintain evidence for +potential law enforcement cooperation and legal proceedings. +``` + +**If refused**: +``` +Your erasure request has been evaluated and refused pursuant to GDPR Article 17(3)(e). + +Reason: The data is necessary for the establishment, exercise, or defence of legal claims. +Specifically, the malware upload attempts from your IP address constitute evidence that may be +required in potential legal proceedings. + +Your rights: +- You may object to processing under Article 21 +- You may lodge a complaint with the supervisory authority: [Your DPA contact info] +- You may re-submit this request when legal proceedings are no longer anticipated +``` + +--- + +## Recommended Privacy Policy + +Add this section to your website's privacy policy: + +```markdown +## Malware Scanning and File Upload Security + +### Overview + +We use automated malware scanning to protect our infrastructure, services, and users from malicious uploads. This processing involves collecting and analyzing certain personal data as described below. + +### What Data We Collect + +When you upload files to our services, we collect and process: + +- **IP address**: Your network address +- **Timestamp**: Date and time of upload attempt +- **File hash**: Cryptographic fingerprint (SHA256) of uploaded files +- **Detection results**: Malware scanner findings (if malicious content detected) +- **Upload history**: Number of malware upload attempts (if multiple detections) + +### Why We Collect This Data + +**Legal Basis**: Legitimate interest (GDPR Article 6(1)(f)) + +We have a legitimate interest in protecting our infrastructure and users from malware attacks. This interest includes: +- Detecting and blocking malware uploads +- Identifying repeat attackers +- Investigating security incidents +- Maintaining evidence for potential legal proceedings +- Alerting our security team to threats + +**Balancing Test**: Our security interest in protecting infrastructure outweighs the privacy expectations of individuals attempting to upload malware. + +### How Long We Keep Your Data + +| Data Type | Retention Period | Reason | +|-----------|------------------|--------| +| Attacker tracking (cache) | 90 days | Evidence for law enforcement and legal proceedings | +| Scan results (cache) | 5 minutes to 90 days | Performance optimization and detection history | +| Application logs | Up to 90 days | Security monitoring and incident investigation | +| Audit logs | [Your retention period] | Legal compliance and accountability | + +**Important**: The 90-day retention period for attacker tracking resets with each new malware upload from the same IP address. For continuous attackers, the retention window is always from the last upload attempt. + +Data is automatically deleted when retention periods expire. + +### Who We Share Your Data With + +**Security Notifications** (Optional): + +If malware is detected, we may send anonymized alerts to: +- **Microsoft Teams** (Microsoft Corporation, USA) - Security team notifications +- **Discord** (Discord Inc., USA) - Security team notifications + +**Data Minimization**: IP addresses shared with third parties are anonymized (e.g., `192.168.1.123` becomes `192.168.1.x`) to minimize data exposure while maintaining network-level visibility for security purposes. + +**โš ๏ธ Malware File Uploads** (Optional but HIGH IMPACT): + +**IMPORTANT**: If file upload sharing is enabled, detected malware files are uploaded to: + +- **VirusTotal** (Google LLC, USA) - When `MALWARE_SCAN_API_VIRUSTOTAL_UPLOAD_UNKNOWN=yes` +- **MalwareBazaar** (abuse.ch, Switzerland) - When `MALWARE_SCAN_API_MALWAREBAZAAR_UPLOAD_UNKNOWN=yes` + +**What This Means**: +- **Complete file contents** are transferred to third-party threat intelligence services +- **Files become part of public databases** accessible to security researchers worldwide +- **Files may contain personal data** (documents, emails, images, etc.) +- **Retention is indefinite** - Files remain in these databases permanently +- **You cannot request deletion** from VirusTotal or MalwareBazaar after upload + +**Legal Basis**: This extensive data sharing may **not** be covered by legitimate interest alone. If you enable this feature: +- **Consult legal counsel** about appropriate legal basis +- **Consider obtaining explicit consent** from file uploaders +- **Add prominent notice** before file upload forms +- **Document justification** for this processing + +**Configuration**: File uploads are **DISABLED by default** for GDPR compliance. Only enable if you have proper legal basis and user notification in place. + +**International Transfer**: Microsoft, Discord, and Google (VirusTotal) are US-based companies. abuse.ch (MalwareBazaar) is based in Switzerland but makes files publicly accessible worldwide. The EU does not consider the USA to have adequate data protection. We rely on legitimate interest for security purposes and apply data minimization measures where possible. + +### Your Rights + +Under GDPR, you have the following rights: + +#### Right to Access (Article 15) +You may request a copy of the data we hold about you by contacting: [Your contact email] + +#### Right to Erasure (Article 17) +You may request deletion of your data by contacting: [Your contact email] + +**Important**: We may refuse erasure requests if the data is necessary for establishing, exercising, or defending legal claims (e.g., evidence of malware attacks). This is permitted under Article 17(3)(e). + +#### Right to Object (Article 21) +You may object to processing based on legitimate interest. We will cease processing unless we demonstrate compelling legitimate grounds that override your interests. + +#### Right to Lodge a Complaint +If you believe we have violated GDPR, you may lodge a complaint with your national data protection authority: +- **EU Data Protection Authorities**: https://edpb.europa.eu/about-edpb/board/members_en +- **Your Local DPA**: [Insert local DPA contact information] + +### Data Controller + +The data controller responsible for this processing is: + +**[Your Organization Name]** +[Your Address] +[Your Email] +[Your Phone] + +### Data Protection Officer (if applicable) + +If you have questions about data protection, you may contact our Data Protection Officer: + +**[DPO Name]** (if required) +[DPO Email] + +### Changes to This Policy + +This privacy notice was last updated on [DATE]. We may update it periodically to reflect changes in our processing activities or legal requirements. + +--- + +*Last updated: 2026-01-19* +``` + +--- + +## Compliance Checklist + +Use this checklist to verify GDPR compliance before deploying in the EU: + +### Configuration + +- [ ] **IP Anonymization**: Verify `MALWARE_SCAN_ANONYMIZE_WEBHOOK_IPS=yes` (default) +- [ ] **Retention Period**: Set `MALWARE_SCAN_TRACK_ATTACKER_TTL` appropriately (default: 7776000 = 90 days) +- [ ] **Attacker Tracking**: Confirm `MALWARE_SCAN_TRACK_ATTACKERS` setting matches your needs +- [ ] **Webhooks**: If using Teams/Discord, understand international transfer implications +- [ ] **โš ๏ธ VirusTotal File Uploads**: Verify `MALWARE_SCAN_API_VIRUSTOTAL_UPLOAD_UNKNOWN=no` (default: disabled) + - [ ] If enabled: Documented legal basis (legitimate interest may be insufficient) + - [ ] If enabled: Added prominent notice about public file sharing to privacy policy + - [ ] If enabled: Considered consent requirement for file uploads + - [ ] If enabled: Consulted legal counsel about GDPR Article 9 implications (special categories of data) +- [ ] **โš ๏ธ MalwareBazaar File Uploads**: Verify `MALWARE_SCAN_API_MALWAREBAZAAR_UPLOAD_UNKNOWN=no` (default: disabled) + - [ ] If enabled: Documented legal basis (legitimate interest may be insufficient) + - [ ] If enabled: Added prominent notice about public file sharing to privacy policy + - [ ] If enabled: Considered consent requirement for file uploads + - [ ] If enabled: Consulted legal counsel about GDPR Article 9 implications (special categories of data) + +### Documentation + +- [ ] **Privacy Policy**: Added malware scanning section (see template above) +- [ ] **Legal Basis**: Documented legitimate interest justification +- [ ] **Retention Policy**: Defined and documented data retention periods +- [ ] **Records of Processing**: Created internal documentation per Article 30 (if required) +- [ ] **Contact Information**: Provided data controller contact details in privacy policy + +### Operational + +- [ ] **Log Rotation**: Configured log rotation with maximum retention (recommend 90 days) +- [ ] **Access Request Process**: Established procedure for handling data subject access requests +- [ ] **Erasure Request Process**: Established procedure for evaluating and processing erasure requests +- [ ] **Audit Logging**: Implemented logging for GDPR-related requests (access, erasure) +- [ ] **Staff Training**: Trained relevant staff on GDPR compliance and data subject rights + +### Technical + +- [ ] **Security Measures**: Verified Article 32 security measures in place (see section above) +- [ ] **Access Control**: Restricted file system permissions for logs and cache +- [ ] **Backup Policy**: Understood that backups may retain data beyond retention period + +### Supervisory Authority + +- [ ] **Identified Your DPA**: Know which data protection authority has jurisdiction +- [ ] **DPA Contact Info**: Added to privacy policy for complaint lodging + +--- + +## Legal Disclaimer + +This document provides guidance on GDPR compliance for the malware-scan plugin. It does not constitute legal advice. Each organization's GDPR obligations depend on specific circumstances, including: +- Jurisdiction and applicable regulations +- Organization size and structure +- Nature and scope of processing +- Supervisory authority interpretations + +**Recommendation**: Consult with a qualified data protection lawyer or Data Protection Officer before deploying in production, especially for: +- High-risk processing activities +- Large-scale processing +- International data transfers +- Sensitive or special categories of personal data + +--- + +## Additional Resources + +### GDPR Official Sources + +- **GDPR Full Text**: https://eur-lex.europa.eu/eli/reg/2016/679/oj +- **European Data Protection Board**: https://edpb.europa.eu/ +- **Article 29 Working Party Guidelines**: https://ec.europa.eu/justice/article-29/documentation/opinion-recommendation/index_en.htm + +### Key GDPR Concepts + +- **Article 5**: Principles (lawfulness, fairness, transparency, purpose limitation, data minimization, storage limitation, accuracy, integrity, confidentiality, accountability) +- **Article 6**: Lawfulness of processing (consent, contract, legal obligation, vital interests, public task, legitimate interests) +- **Article 32**: Security of processing +- **Articles 15-22**: Data subject rights + +### Case Law + +- **CJEU Case C-582/14 (Breyer v Germany)**: IP addresses are personal data +- **CJEU Case C-311/18 (Schrems II)**: Invalidated Privacy Shield for EU-US data transfers + +--- + +## Document History + +| Version | Date | Changes | +|---------|------|---------| +| 1.0 | 2026-01-19 | Initial GDPR compliance documentation | +| 1.1 | 2026-01-25 | Added v0.7.13 security enhancements: automatic credential masking in logs, SHA256 download verification, enhanced data retention policy documentation, email anonymization | +| 1.2 | 2026-01-27 | Updated for v0.8.0: Enhanced architecture documentation for 21 modular components, improved security features, optional lfs dependency handling + +--- + +## Contact + +For questions about GDPR compliance in the malware-scan plugin, please: +- Open an issue: https://github.com/bunkerity/bunkerweb-plugins/issues + + +--- + +**Note to Administrators**: This document is maintained as part of the malware-scan plugin. As the data controller, you are responsible for adapting this guidance to your specific deployment and legal requirements. diff --git a/malware-scan/README.md b/malware-scan/README.md new file mode 100644 index 0000000..d427f1b --- /dev/null +++ b/malware-scan/README.md @@ -0,0 +1,389 @@ +# BunkerWeb Malware Scan Plugin + +**Version**: 0.8.0 | **Status**: Production Ready | **Release**: 2026-01-27 + +Multi-layer malware scanning plugin for BunkerWeb with advanced caching, webhook notifications, and modular architecture. + +## โœจ Key Features + +- **๐Ÿ›ก๏ธ Multi-layer scanning**: ClamAV (local) + VirusTotal + SentinelOne (cloud) +- **โšก Smart caching**: SHA256-based result caching with differential TTL (clean: 5min, malicious: 24h, max 7d, configurable) +- **๐Ÿ“ข Webhook notifications**: Real-time alerts to Discord and Microsoft Teams +- **๐Ÿ”ง Modular architecture**: 21 specialized modules for maintainability +- **๐ŸŒ Cluster-ready**: Works correctly on manager and worker nodes + +## โšก Performance Metrics (v0.8.0) + +| Metric | Result | Details | +|--------|--------|---------| +| **First scan (cache miss)** | ~4 seconds | Full ClamAV INSTREAM scan | +| **Repeat scan (cache hit)** | ~1 second | 75% faster - hash calculation + cache lookup | +| **Code maintainability** | 21 modules | Specialized components for scanning, caching, APIs | +| **Webhook delivery** | Real-time | Discord & Microsoft Teams notifications | +| **Module initialization** | 100% | All 21 modules load successfully | + +## ๐ŸŽฏ Real-World Examples + +### Example 1: First Upload (Cache Miss) - Full Scan + +Uploading EICAR test file for the first time: + +```log +[MALWARE-SCAN] file upload detected, scanning for malware +[MALWARE-SCAN] file SHA256: 2546dcffc5ad854d4ddc64fbf056871cd5a00f2471cb7a5bfd4ac23b6e9eedad +[CACHE] ClamAV local cache MISS for checksum: 2546dcffc5ad854d... + +# Full ClamAV INSTREAM scan performed +[MALWARE-SCAN] scanning file via INSTREAM +[MALWARE-SCAN] INSTREAM: Read chunk 1, size: 703 +[MALWARE-SCAN] INSTREAM: All chunks sent - 1 chunks, 703 bytes total +clamd[922]: instream(127.0.0.1@34668): Eicar-Test-Signature FOUND + +# Detection and caching +[MALWARE-SCAN] MALWARE DETECTED BY CLAMAV +[MALWARE-SCAN] ClamAV signature: Eicar-Test-Signature +[CACHE] Added ClamAV result to cache: 2546dcffc5ad854d... = Eicar-Test-Signature (TTL: 7776000s) + +# Webhook notification +[WEBHOOK] Processing Discord webhook +[WEBHOOK] Cleaned URL: https://discord.co*** + +# Request blocked +POST /wp-admin/async-upload.php HTTP/3.0" 403 (4 seconds total) +``` + +**Timeline**: 4 seconds (file buffering + ClamAV scan + webhook) + +--- + +### Example 2: Second Upload (Cache Hit) - Instant Block + +Uploading the same EICAR file again within 90 days: + +```log +[MALWARE-SCAN] file upload detected, scanning for malware +[MALWARE-SCAN] file SHA256: 2546dcffc5ad854d4ddc64fbf056871cd5a00f2471cb7a5bfd4ac23b6e9eedad +[CACHE] ClamAV local cache HIT for checksum: 2546dcffc5ad854d... + +# Cache hit - no ClamAV scan needed! +[CACHE_HIT] ClamAV cache shows malicious - skipping ClamAV scan +[MALWARE DETECTED BY CLAMAV (cached)] +[MALWARE-SCAN] ClamAV signature: Eicar-Test-Signature (SHA256: 2546dcffc5ad854d...) + +# Webhook notification (still fires) +[WEBHOOK] Processing Discord webhook +[WEBHOOK] Cleaned URL: https://discord.co*** + +# Request blocked instantly +POST /wp-admin/async-upload.php HTTP/3.0" 403 (1 second total) +``` + +**Timeline**: 1 second (hash calculation + cache lookup + webhook) - **75% faster!** + +--- + +### Example 3: Multi-Layer Mode with Cloud Scanners + +Configuration: `MALWARE_SCAN_SKIP_OTHERS_ON_CLAMAV_DETECT=no` + +```log +# ClamAV detects malware +[MALWARE-SCAN] MALWARE DETECTED BY CLAMAV +[MALWARE-SCAN] ClamAV signature: Eicar-Test-Signature + +# Continue to cloud scanners for additional intelligence +[MALWARE-SCAN] ClamAV detected malware, continuing to VirusTotal for additional analysis +[MALWARE-SCAN] calculated SHA256: 2546dcffc5ad854d4ddc64fbf056871cd5a00f2471cb7a5bfd4ac23b6e9eedad + +# VirusTotal confirmation +[MALWARE-SCAN] checking file against VirusTotal API +[MALWARE-SCAN] file checksum found in VT cache: 0 suspicious and 61 malicious + +# Dual confirmation +[MALWARE-SCAN] MALWARE DETECTED BY VIRUSTOTAL - BLOCKING REQUEST +[MALWARE-SCAN] VirusTotal detection: 61 malicious engines confirmed + +# Webhook with multiple detections +[WEBHOOK] Processing Discord webhook +Scanner: ClamAV โ†’ Eicar-Test-Signature +Scanner: VirusTotal โ†’ 61/61 engines detected malware + +POST /wp-admin/async-upload.php HTTP/3.0" 403 +``` + +**Benefits**: +- โœ… Dual confirmation from multiple sources +- โœ… Rich threat intelligence +- โœ… Lower false positive rate +- โœ… Better for security research + +--- + +### Comparison: Fast vs Full Mode + +| Feature | Fast Mode (default) | Full Multi-Layer Mode | +|---------|--------------------|-----------------------| +| **Config** | `SKIP_OTHERS_ON_CLAMAV_DETECT=yes` | `SKIP_OTHERS_ON_CLAMAV_DETECT=no` | +| **ClamAV scan** | โœ… Always | โœ… Always | +| **Cache check** | โœ… Before ClamAV | โœ… Before ClamAV | +| **Cloud scanners** | Only if ClamAV clean | Always (additional intelligence) | +| **Response time** | ~1s cached, ~4s uncached | ~1s cached, ~5s uncached | +| **Detection sources** | Single (ClamAV) | Multiple (ClamAV + VT + TF + S1) | +| **API quota usage** | Lower | Higher | +| **Best for** | Production speed | Security research | + +## ๐Ÿ“ฅ Installation + +### Option 1: Docker ClamAV (Recommended for New Deployments) + +For production environments, use the automated Docker installation script for ClamAV: + +```bash +# Supports: Debian 11+, Ubuntu 20.04+, CentOS 7+, Rocky 8+, AlmaLinux 8+, Fedora 33+ +sudo ./install-docker-clamav.sh + +# Container runs on port 3310 with 4GB memory limit +# Signature updates: Every 2 hours (12 checks per day) +# Logging: JSON-file (docker logs clamav) +``` + +**Features:** +- โœ… Automatic OS detection (Debian/CentOS) +- โœ… Auto-start on reboot (systemd service) +- โœ… 24/7 availability (concurrent database reload) +- โœ… Comprehensive troubleshooting guides included + +See [install-docker-clamav.md](install-docker-clamav.md) for detailed instructions. + +### Option 2: Plugin Installation + +```bash +# Clone the repository +git clone https://github.com/bunkerity/bunkerweb-plugins.git +cd bunkerweb-plugins/malware-scan + +# Deploy files +sudo cp *.lua plugin.json /etc/bunkerweb/plugins/malware-scan/ +sudo chmod 644 /etc/bunkerweb/plugins/malware-scan/* + +# Restart BunkerWeb +sudo systemctl restart bunkerweb +``` + +### Verification + +Check that all modules loaded successfully: + +```bash +journalctl -u bunkerweb -n 50 | grep -i "malware-scan" +``` + +Expected output: +``` +[MALWARE-SCAN] cache module loaded successfully +[MALWARE-SCAN] file_ops module loaded successfully +[MALWARE-SCAN] clamav module loaded successfully +[MALWARE-SCAN] virustotal module loaded successfully +[MALWARE-SCAN] sentinelone module loaded successfully +[MALWARE-SCAN] webhook module loaded successfully +``` + +### ClamAV Debugging + +The install script automatically enables ClamAV debugging for detailed scanning logs. To manually enable or check: + +```bash +# Check current ClamAV configuration +grep -E "^(LogVerbose|Debug|LogClean)" /etc/clamav/clamd.conf + +# Enable debugging (if not already enabled) +sudo sed -i 's/^LogVerbose false/LogVerbose true/' /etc/clamav/clamd.conf +sudo sed -i 's/^Debug false/Debug true/' /etc/clamav/clamd.conf +sudo sed -i 's/^LogClean false/LogClean true/' /etc/clamav/clamd.conf +sudo systemctl restart clamav-daemon + +# View ClamAV logs +sudo tail -f /var/log/clamav/clamav.log + +# View BunkerWeb malware-scan logs +sudo journalctl -u bunkerweb -f | grep MALWARE +``` + +**What you'll see with debugging enabled:** +- `LogVerbose`: Detailed information about each scan (file paths, scan methods) +- `Debug`: Daemon-level debugging (connections, protocol details) +- `LogClean`: Logs for clean files (not just infected ones) + +## โš™๏ธ Configuration + +Edit `/etc/bunkerweb/variables.env`: + +### Core Settings + +```bash +USE_MALWARE-SCAN=yes # Enable malware scanning +MALWARE_SCAN_DEBUG=no # Debug logging (disable in production) +``` + +### ClamAV (Local Scanning) + +```bash +MALWARE_SCAN_CLAMAV_HOST=127.0.0.1 # ClamAV daemon host +MALWARE_SCAN_CLAMAV_PORT=3310 # ClamAV daemon port +MALWARE_SCAN_CLAMAV_TIMEOUT=10000 # Connection timeout (ms) +MALWARE_SCAN_CLAMAV_MAX_SIZE=26214400 # Max file size (25MB) +``` + +### Multi-Layer Scanning + +```bash +MALWARE_SCAN_SKIP_OTHERS_ON_CLAMAV_DETECT=yes # Fast mode (skip cloud on ClamAV hit) +MALWARE_SCAN_RESUME_SCAN_ON_FIRST_HIT=no # WARNING: DOS risk! Continue scanning after first detection +MALWARE_SCAN_SCAN_FULL_MULTIPART_BODY=no # Skip full multipart body scan (recommended - see note below) +MALWARE_SCAN_HASH_MAX_SIZE=67108864 # Max file size for hash scanning (64MB) +``` + +**Note on SCAN_FULL_MULTIPART_BODY**: + +By default (`no`), the plugin skips scanning the entire multipart request body and only scans extracted individual files. This is **recommended** because: +- HTTP multipart boundaries contain random strings that change on every upload +- Full body SHA256 is different each time, making caching completely ineffective +- Individual file extraction provides consistent SHA256s for reliable caching +- Wastes ClamAV resources scanning the same files repeatedly + +Only enable (`yes`) if you suspect malware might be split across multipart boundaries (extremely rare). + +**Note on RESUME_SCAN_ON_FIRST_HIT**: + +The plugin performs **dual scanning** for comprehensive malware detection: +1. **Full-body scan** (optional): Scans entire multipart upload - disabled by default (see SCAN_FULL_MULTIPART_BODY above) +2. **Per-file scan** (when resume enabled): Parses multipart data and scans each uploaded file individually + +When `RESUME_SCAN_ON_FIRST_HIT=yes`: +- All scanners complete even after detection (comprehensive threat intelligence) +- Continues to next file after detecting malware in one file +- Aggregates detections from whole-body + all individual files before blocking + +When `RESUME_SCAN_ON_FIRST_HIT=no` (default): +- Blocks immediately on first detection (fast response, minimal resources) + +**DOS Risk**: Attackers can force expensive operations (multiple file scans, API calls) even after detection. Only enable for security research or when you need complete visibility into multi-file attacks. + +### VirusTotal (70+ AV Engines) + +```bash +MALWARE_SCAN_API_USE_VIRUSTOTAL=yes # Enable VirusTotal +MALWARE_SCAN_API_VIRUSTOTAL_KEY=your_api_key # API key (required) +MALWARE_SCAN_API_VIRUSTOTAL_SUSPICIOUS=5 # Suspicious threshold +MALWARE_SCAN_API_VIRUSTOTAL_MALICIOUS=3 # Malicious threshold +MALWARE_SCAN_API_VIRUSTOTAL_UPLOAD_UNKNOWN=yes # Upload unknown malware +MALWARE_SCAN_API_VIRUSTOTAL_AUTO_VOTE=yes # Auto-vote on uploads +MALWARE_SCAN_API_VIRUSTOTAL_MAX_UPLOAD_SIZE=33554432 # Max upload size (32MB) +``` + +### SentinelOne (Enterprise) + +```bash +MALWARE_SCAN_API_USE_SENTINELONE=yes # Enable SentinelOne +MALWARE_SCAN_API_SENTINELONE_URL=https://... # Console URL +MALWARE_SCAN_API_SENTINELONE_TOKEN=your_token # API token +``` + +### Webhooks (Discord & Teams) + +```bash +MALWARE_SCAN_DISCORD_WEBHOOK_URL=https://discord.com/api/webhooks/... +MALWARE_SCAN_TEAMS_WEBHOOK_URL=https://outlook.office.com/webhook/... +``` + +### False Positive Handling + +```bash +MALWARE_SCAN_ALLOWLIST_HASHES=sha256_hash1,sha256_hash2 # Comma-separated hashes +MALWARE_SCAN_ALLOWLIST_FILE=/path/to/allowlist.txt # File with hashes (one per line) +``` + +### Cleanup + +```bash +MALWARE_SCAN_CLEANUP_FILES=yes # Delete temp files after scan +MALWARE_SCAN_CLEANUP_ORPHANED=yes # Clean up old orphaned files +MALWARE_SCAN_CLEANUP_AGE=3600 # Max age for orphaned files (seconds) +``` + +## ๐Ÿงช Testing + +### Test with EICAR File + +```bash +# Download EICAR standard test file +curl -o eicar.txt https://secure.eicar.org/eicar.com.txt + +# Upload through your web application +# Example: WordPress media uploader +``` + +Expected result: +``` +HTTP 403 Forbidden +malware detected by ClamAV: Eicar-Test-Signature +``` + +### Test Caching + +1. Upload EICAR file - should take ~4 seconds (full scan) +2. Upload same file again - should take ~1 second (cache hit) +3. Check logs for `[CACHE_HIT]` messages + +## ๐Ÿ“‹ API Rate Limits + +### VirusTotal Public API +- **Rate limits**: 4 requests/minute, 500 requests/day +- **Free tier**: https://www.virustotal.com/gui/join-us +- **Premium**: Unlimited requests + +### SentinelOne API +- **Rate limits**: 10 req/sec (50 burst) +- **License**: Requires SentinelOne subscription + +## ๐Ÿ“š Documentation + +- **[install-docker-clamav.md](install-docker-clamav.md)** - **NEW!** Automated Docker ClamAV setup (Debian & CentOS) +- **[install-docker-clamav.sh](install-docker-clamav.sh)** - Deployment script with OS auto-detection +- **[install.md](install.md)** - Detailed installation guide +- **[TECHNICAL.md](TECHNICAL.md)** - Architecture, security features, development details +- **[GDPR.md](GDPR.md)** - GDPR compliance guide with privacy policy template +- **[SECURITY.md](SECURITY.md)** - Vulnerability reporting and security policy +- **[all-features-enabled.md](all-features-enabled.md)** - Complete configuration with all scanners +- **[variables.env.example](variables.env.example)** - Complete configuration reference +- **[ALLOWLIST.md](ALLOWLIST.md)** - False positive handling with SHA256 allowlists +- **[GRANULAR_CACHE_CLEANUP.md](GRANULAR_CACHE_CLEANUP.md)** - Per-scanner cache cleanup configuration + +## ๐Ÿ”’ Security + +### Security Best Practices + +- Keep ClamAV signatures updated: `sudo freshclam` +- Use default settings: `BLOCK_ON_FILENAME_INJECTION_ATTACK=yes` +- Avoid `RESUME_SCAN_ON_FIRST_HIT=yes` in production (DOS risk) +- Store API keys in environment variables, never commit to git +- Review logs regularly for attack patterns + +### Reporting Vulnerabilities + +See [SECURITY.md](SECURITY.md) for vulnerability disclosure policy and security contacts. + +## ๐Ÿ’ฌ Support + +- **Documentation**: [docs.bunkerweb.io](https://docs.bunkerweb.io/) +- **Issues**: Check logs first: `journalctl -u bunkerweb` +- **ClamAV**: [docs.clamav.net](https://docs.clamav.net/) +- **Security**: [SECURITY.md](SECURITY.md) + +## ๐Ÿ“œ License + +This plugin is part of the BunkerWeb project. + +--- + +**User-Agent**: API requests include version: `bunkerweb - https://github.com/bunkerity/bunkerweb - malware-scan module v0.8.0` diff --git a/malware-scan/SECURITY.md b/malware-scan/SECURITY.md new file mode 100644 index 0000000..b51b92d --- /dev/null +++ b/malware-scan/SECURITY.md @@ -0,0 +1,75 @@ +# Security Policy + +## Vulnerability Reporting + +If you discover a security vulnerability in the BunkerWeb malware-scan plugin, please report it responsibly: + +1. **Do NOT** open a public GitHub issue +2. Email security details to both: + security [@] bunkerity.com (BunkerWeb) + mkf [@] sysangels.ai (the main dev of this module) +3. Include: + - Description of the vulnerability + - Steps to reproduce + - Potential impact + - Suggested fix (if available) + +We will respond within 48 hours and work with you to address the issue. + +## Security Best Practices + +### Configuration Hardening + +1. **API Keys**: Store in environment variables, never commit to git +2. **ClamAV**: Keep virus signatures updated (`freshclam`) +3. **File Validation**: Keep `BLOCK_ON_FILENAME_INJECTION_ATTACK=yes` (default) +4. **Resume Mode**: Only enable `RESUME_SCAN_ON_FIRST_HIT` for research (DOS risk) +5. **Webhooks**: Mask URLs in logs (leave `UNMASK_KEYS` empty) + +### Network Security + +- ClamAV should only listen on `127.0.0.1` (localhost) +- Redis should require authentication if exposed +- API calls use HTTPS for VirusTotal, SentinelOne +- Webhook URLs should be kept confidential + +### File System Security + +- Temporary files use `os.tmpname()` with restrictive permissions +- Files cleaned up after scanning (`CLEANUP_FILES=yes`) +- Never trust attacker-provided filenames (validated by `utils.validate_filename()`) + +### Log Injection Protection + +All user-provided strings are sanitized before logging: +- Filenames: `utils.sanitize_for_logging()` escapes control characters +- Prevents newline injection, ANSI escape codes, null bytes +- Limits log message length to prevent flooding + +### Input Validation + +Multiple layers of validation: +1. **Filename validation**: Path traversal, null bytes, excessive length +2. **File size limits**: ClamAV max 25MB, hash scanning max 64MB +3. **Content-Type validation**: Boundary extraction from multipart uploads +4. **SHA256 verification**: Hash-based deduplication and cache lookups + +## Security Features + +### Defense in Depth + +- **Multi-layer scanning**: Local (ClamAV) + Cloud (VT, S1) +- **Dual scanning mode**: Whole-body + per-file scanning +- **Cache validation**: Differential TTL (clean: 5min, malicious: 24h) +- **Filename attack detection**: 8 separate validation checks + +### Monitoring & Alerting + +- Webhook notifications (Discord, Teams) +- Attacker IP tracking +- Per-file detection reports +- Cache hit/miss metrics + +--- + +**Last Updated**: 2026-01-27 diff --git a/malware-scan/TECHNICAL.md b/malware-scan/TECHNICAL.md new file mode 100644 index 0000000..ce7604f --- /dev/null +++ b/malware-scan/TECHNICAL.md @@ -0,0 +1,119 @@ +# Technical Documentation - Malware Scan Plugin + +**Version**: 0.8.0 | **Last Updated**: 2026-01-27 + +This document provides technical details for developers working on the malware-scan plugin. + +## Table of Contents + +- [Architecture](#architecture) +- [How It Works](#how-it-works) +- [Security Features](#security-features) +- [Caching](#caching-strategy) +- [Error Handling](#error-handling) +- [Adding Features](#development) + +## Architecture + +### Plugin Structure + +The plugin consists of 21 specialized modules that work together: + +- **Core scanning** - ClamAV (local) + VirusTotal/SentinelOne (cloud) +- **Caching layer** - Fast result storage for repeated scans +- **File operations** - Safe file handling and hashing +- **Notifications** - Discord and Teams webhook alerts +- **Logging** - Secure logging with sensitive data masking +- **Configuration and monitoring** - Settings management and metrics + +For complete module details, see the source code comments. + +### Design Principles + +The plugin is designed to be: + +- **Modular** - Each component handles one specific task +- **Reliable** - If one scanner fails, others continue working +- **Cluster-aware** - Works correctly with BunkerWeb manager and worker nodes +- **Maintainable** - Clear separation between scanning, caching, APIs, and logging + +## How It Works + +### File Scanning Process + +The plugin scans individual files extracted from uploads rather than the entire request. This is faster because: + +- Each file has a consistent hash that can be cached +- Full request scans have random boundaries that change every upload (breaking caching) +- Individual file scanning is 75% faster on repeat uploads due to effective caching + +The plugin only scans the full request body if you explicitly enable it, which is only needed in rare cases where malware might be split across file boundaries. + +### Request Processing + +When a file upload is detected: + +1. Check if it's a file upload (multipart content type) +2. Extract uploaded files from request +3. Calculate file hash (SHA256) +4. Check if file is in allowlist +5. Check local ClamAV cache - if malware found, block immediately +6. Check cloud scanner caches - if malware found, block immediately +7. Run ClamAV scan if not cached +8. Run cloud scanners (VirusTotal, SentinelOne) if enabled +9. Block file if any scanner detects malware +10. Cache the result for faster future scans +11. Send webhook alert to Discord/Teams if blocked + +## Security Features + +The plugin protects against malware through: + +- **Multiple scanning layers** - ClamAV (local) + VirusTotal + SentinelOne (cloud) +- **File type validation** - Checks file contents, not just extension +- **Filename safety** - Validates against path traversal and injection attacks +- **File size limits** - Prevents resource exhaustion (25MB for ClamAV, 64MB for hashing) +- **Caching** - Known malware blocked instantly without rescanning +- **Safe temporary files** - Uses secure temp storage, cleaned up after scanning + +## Caching Strategy + +Files are cached after scanning to speed up detection: + +- **Clean files** are cached for 5 minutes (shorter cache in case new threats emerge) +- **Malicious files** are cached for 90 days (longer cache since threats rarely change) +- **Cache lookup** takes ~1 second vs ~4 seconds for full scan +- **Result**: 75% faster on repeat uploads + +In typical production: +- Legitimate files: ~60% cache hit (same attachments uploaded repeatedly) +- Malware: ~95% cache hit (attackers reuse same payloads) + +## Error Handling + +If a scanner fails or becomes unavailable, the plugin: + +- Logs the error for troubleshooting +- Continues with other available scanners +- Allows the file through if all scanners fail (better to allow than block legitimate uploads) +- Uses cached results when APIs hit rate limits + +This "graceful degradation" means the plugin keeps working even if one component fails. + +## Adding Features + +To add a new scanner: + +1. Create a new module file for the scanner +2. Implement cache check and storage functions +3. Register it in the plugin configuration +4. Add it to the scanning flow +5. Update webhook notifications to include results + +Testing should verify that: + +- First file upload takes ~4 seconds (full scan) +- Repeated uploads take ~1 second (cache hit) +- Files are properly cached and retrieved +- Errors in one scanner don't crash the plugin + diff --git a/malware-scan/install-docker-clamav.md b/malware-scan/install-docker-clamav.md new file mode 100644 index 0000000..fa64135 --- /dev/null +++ b/malware-scan/install-docker-clamav.md @@ -0,0 +1,1494 @@ +# ClamAV Docker Installation Guide - Debian & CentOS + +This guide covers installing and configuring ClamAV using official Docker images from Docker Hub on Debian and CentOS-based systems. + +**Supported Operating Systems:** +- Debian 11+ +- Ubuntu 20.04+ +- CentOS 7+ +- Rocky Linux 8+ +- AlmaLinux 8+ +- Fedora 33+ + +The installation script automatically detects your OS and uses the appropriate package manager. + +## Quick Installation (Automated) + +For a fully automated installation with optimized configuration: + +```bash +# Download and make executable +chmod +x install-docker-clamav.sh + +# Run as root (default installation directory: /DATA/clamav) +sudo ./install-docker-clamav.sh + +# Or specify custom installation directory +sudo ./install-docker-clamav.sh --install_dir=/opt/clamav +``` + +### Installation Options + +- `--install_dir=PATH` - Custom installation directory (default: `/DATA/clamav`) +- `--help` - Show help message + +### Examples + +```bash +# Default installation to /DATA/clamav +sudo ./install-docker-clamav.sh + +# Custom installation directory +sudo ./install-docker-clamav.sh --install_dir=/opt/clamav +sudo ./install-docker-clamav.sh --install_dir=/home/user/clamav +``` + +The script will: +1. Check if ClamAV is already running (prevents duplicate installations) +2. Validate system requirements (RAM, disk space) +3. Install Docker if needed +4. Create installation directory structure (default: `/DATA/clamav`) +5. Generate optimized configuration files (DetectPUA disabled, verbose logging) +6. Start ClamAV container with memory limits +7. **Configure syslog logging** (logs sent to host syslog/journald) +8. Wait for signature database download +9. Test installation with EICAR +10. Create helper scripts (start.sh, stop.sh, status.sh, logs.sh, update.sh) + +**Logging**: By default, the script configures ClamAV to send logs to the host's syslog daemon using Docker's syslog logging driver. View logs with `sudo journalctl -t clamav -f` (not `docker logs`). + +**Note**: If ClamAV is already running on port 3310, the script will detect it and exit without making changes. + +## Table of Contents + +- [Prerequisites](#prerequisites) +- [Docker Installation](#docker-installation) +- [ClamAV Docker Setup](#clamav-docker-setup) +- [Container Configuration](#container-configuration) +- [Memory Optimization](#memory-optimization) +- [Testing ClamAV](#testing-clamav) +- [Integration with BunkerWeb](#integration-with-bunkerweb) +- [Troubleshooting](#troubleshooting) + +## Prerequisites + +### System Requirements + +- **RAM**: **Minimum 4GB** for 24/7 ClamAV operation with concurrent database reload +- **Disk Space**: Minimum 2GB for signature databases +- **OS**: + - Debian 11+ + - Ubuntu 20.04+ + - CentOS 7+ + - Rocky Linux 8+ + - AlmaLinux 8+ + - Fedora 33+ +- **Architecture**: amd64, arm64, or arm/v7 + +### Why Use Docker for ClamAV? + +- **Isolation**: ClamAV runs in isolated container environment +- **Easy Updates**: Pull new images for updates +- **Persistent Storage**: Database signatures survive container restarts +- **Simplified Deployment**: No package dependency conflicts + +### Automated Installation Script + +This guide includes an automated installation script (`install-docker-clamav.sh`) that: +- Checks if ClamAV is already running on port 3310 before installation +- Validates system requirements (RAM, disk space) +- Installs Docker if not present +- Creates optimized configuration files +- Sets up persistent data directory (default: `/DATA/clamav`, customizable with `--install_dir`) +- Tests installation with EICAR +- Creates helper scripts for management + +**Safety Feature**: The script will detect if ClamAV is already running and exit gracefully to prevent conflicts. + +## Docker Installation + +### Automatic Docker Installation (Recommended) + +The installation script handles Docker installation automatically for all supported operating systems. It will: +- Detect your OS (Debian/Ubuntu or CentOS/RHEL/Rocky/AlmaLinux) +- Use the appropriate package manager (`apt-get` or `yum`/`dnf`) +- Install Docker from the official repository +- Enable Docker service for auto-start on boot + +**No manual Docker installation needed!** Just run the script and it takes care of everything. + +### Manual Docker Installation + +#### 1a. Install Docker Engine on Debian/Ubuntu + +```bash +# Update package index +sudo apt update + +# Install prerequisites +sudo apt install -y \ + ca-certificates \ + curl \ + gnupg \ + lsb-release + +# Add Docker's official GPG key +sudo install -m 0755 -d /etc/apt/keyrings +curl -fsSL https://download.docker.com/linux/debian/gpg | \ + sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg +sudo chmod a+r /etc/apt/keyrings/docker.gpg + +# Set up Docker repository +echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] \ + https://download.docker.com/linux/debian \ + $(lsb_release -cs) stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + +# Install Docker Engine +sudo apt update +sudo apt install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + +# Verify installation +sudo docker --version +``` + +#### 1b. Install Docker Engine on CentOS/RHEL/Rocky/AlmaLinux + +```bash +# Update package index +sudo yum update -y +# Or on newer systems (9+): +sudo dnf update -y + +# Install prerequisites +sudo yum install -y yum-utils device-mapper-persistent-data lvm2 +# Or on newer systems (9+): +sudo dnf install -y dnf-plugins-core + +# Add Docker repository +sudo yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo +# Or on newer systems: +sudo dnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo + +# Install Docker Engine +sudo yum install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin +# Or on newer systems: +sudo dnf install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + +# Verify installation +sudo docker --version +``` + +### 2. Configure Docker (Optional) + +```bash +# Add your user to docker group (to run docker without sudo) +sudo usermod -aG docker $USER + +# Log out and back in for group changes to take effect +# Or run: newgrp docker + +# Enable Docker to start on boot +sudo systemctl enable docker +sudo systemctl start docker +``` + +### Important: Using the Automated Installation Script + +The `install-docker-clamav.sh` script handles all Docker installation automatically. You don't need to run these manual steps if you use the script. The script will: + +1. **Detect your OS** from `/etc/os-release` or `/etc/redhat-release` +2. **Normalize OS names** (debian, ubuntu, centos, rhel, rocky, almalinux, fedora) +3. **Use the correct package manager** (apt-get for Debian/Ubuntu, yum/dnf for CentOS/RHEL) +4. **Install Docker** from the official repository for your OS +5. **Enable Docker service** for auto-start on boot + +Example of OS detection in action: +``` +[INFO] Detected OS: debian (Type: debian) Version: 13 +# or +[INFO] Detected OS: centos (Type: rhel) Version: 7 +# or +[INFO] Detected OS: rocky (Type: rhel) Version: 8 +``` + +## ClamAV Docker Setup + +### 1. Pull Official ClamAV Image + +```bash +# Pull stable ClamAV image from Docker Hub +docker pull clamav/clamav:stable + +# Verify image downloaded +docker images | grep clamav +``` + +**Available Image Tags:** +- `clamav/clamav:stable` - Full ClamAV with clamd and freshclam +- `clamav/clamav:stable_base` - Minimal image (no freshclam) +- `clamav/clamav:latest` - Latest development version (not recommended for production) + +### 2. Create Persistent Volume for Signature Databases + +ClamAV signature databases need to persist across container restarts: + +```bash +# Create dedicated volume for ClamAV databases +docker volume create clamav-db + +# Verify volume created +docker volume ls | grep clamav-db + +# Inspect volume location +docker volume inspect clamav-db +``` + +**Volume Details:** +- **Mount Point**: `/var/lib/clamav` inside container +- **Purpose**: Stores virus signature databases (~1-2GB) +- **Persistence**: Survives container deletion and recreation + +### 3. Run ClamAV Container + +#### Basic Configuration (Recommended) + +```bash +docker run -d \ + --name clamav \ + --restart unless-stopped \ + -p 3310:3310 \ + -v clamav-db:/var/lib/clamav \ + -e CLAMAV_NO_FRESHCLAMD=false \ + -e CLAMD_STARTUP_TIMEOUT=1800 \ + clamav/clamav:stable +``` + +**Configuration Breakdown:** +- `-d`: Run in detached mode (background) +- `--name clamav`: Container name for easy reference +- `--restart unless-stopped`: Auto-restart on failure/reboot +- `-p 3310:3310`: Expose clamd TCP socket on port 3310 +- `-v clamav-db:/var/lib/clamav`: Mount persistent database volume +- `-e CLAMAV_NO_FRESHCLAMD=false`: Enable automatic signature updates +- `-e CLAMD_STARTUP_TIMEOUT=1800`: Allow 30 minutes for initial database download + +#### Production Configuration with 24/7 Availability + +For production use with BunkerWeb integration: + +```bash +docker run -d \ + --name clamav \ + --restart unless-stopped \ + --memory="4g" \ + --memory-swap="4g" \ + -p 3310:3310 \ + -v clamav-db:/var/lib/clamav \ + -e CLAMAV_NO_FRESHCLAMD=false \ + -e CLAMD_STARTUP_TIMEOUT=1800 \ + -e FRESHCLAM_CHECKS=12 \ + clamav/clamav:stable +``` + +**Memory Settings:** +- `--memory="4g"`: Limit container to 4GB RAM (required for concurrent reload) +- `--memory-swap="4g"`: Disable swap to prevent OOM issues +- `-e FRESHCLAM_CHECKS=12`: Check for updates every 2 hours + +**Why 4GB:** Concurrent database reload (enabled by default) requires ~4GB peak memory but ensures the ClamAV socket remains available 24/7 without interruption during signature updates. + +### 4. Monitor Container Startup + +```bash +# Watch container logs during startup +docker logs -f clamav + +# Initial database download takes 10-30 minutes +# Look for: "clamd[1]: Self checking every 600 seconds" +# This indicates ClamAV is ready +``` + +**Expected Startup Messages:** +``` ++++ Starting initial checks +++ ++++ Database check passed +++ ++++ Starting FreshClam +++ ++++ Starting ClamAV +++ +clamd[1]: Self checking every 600 seconds +``` + +## Container Configuration + +### Environment Variables + +Configure ClamAV behavior using environment variables: + +```bash +# Create config file for easier management +cat > clamav.env < clamd.conf < freshclam.conf <> clamd.conf < clamd.conf < freshclam.conf < eicar.txt <<'EOF' +X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H* +EOF + +# Scan using clamdscan (from host) +# Install clamav-daemon package for clamdscan tool +sudo apt install -y clamav-daemon + +# Configure clamdscan to use Docker container +sudo tee /etc/clamav/clamd.conf </dev/null || true + +# Disable all ClamAV systemd units +sudo systemctl disable clamav.socket clamav-daemon.socket clamav-daemon clamav-freshclam 2>/dev/null || true + +# Uninstall ClamAV packages +sudo apt remove -y clamav clamav-daemon clamav-freshclam clamav-base + +# Remove ClamAV configuration directories +sudo rm -rf /etc/clamav /var/lib/clamav + +# Verify port is free +sudo ss -tlnp | grep 3310 + +# Now install Docker ClamAV +sudo ./install-docker-clamav.sh +``` + +### Port 3310 in Use by Init but No ClamAV Units Found + +**Problem**: `netstat` shows init process listening on port 3310, but ClamAV systemd units don't exist: +```bash +tcp 0 0 127.0.0.1:3310 0.0.0.0:* LISTEN 1/init +# and... +Failed to stop clamavd.service: Unit clamavd.service not loaded. +``` + +**Cause**: This typically indicates a leftover socket file or a previous ClamAV installation's artifacts. The init process (PID 1/systemd) has inherited the socket binding but there's no active service managing it. + +**Solution**: + +```bash +# 1. Use lsof to see exactly what has the port open +sudo lsof -i :3310 + +# 2. Look for socket files left behind +sudo find /var/run -name "*clamav*" -o -name "*clamd*" 2>/dev/null +sudo find /tmp -name "*clamav*" -o -name "*clamd*" 2>/dev/null +sudo find /var/lib -path "*clamav*" -type s 2>/dev/null + +# 3. Remove any leftover socket files +sudo rm -f /var/run/clamav/clamd.ctl +sudo rm -f /var/run/clamav/clamd.sock +sudo rm -f /tmp/clamd.sock + +# 4. Remove leftover ClamAV package (if installed) +sudo apt remove -y clamav clamav-daemon clamav-freshclam clamav-base 2>/dev/null || true + +# 5. Clean up ClamAV directories +sudo rm -rf /etc/clamav +sudo rm -rf /var/lib/clamav + +# 6. Force release the port by restarting systemd +# (last resort if socket persists) +sudo systemctl daemon-reload + +# 7. Verify port is now free +sleep 1 +sudo ss -tlnp | grep 3310 # Should return nothing + +# 8. Now run the installation script +sudo ./install-docker-clamav.sh +``` + +**If clamd process is actually running (found via lsof):** + +If `lsof -i :3310` shows clamd process is holding the socket: +```bash +# Kill the clamd daemon directly using the PID from lsof output +# From lsof output: clamd 804923 clamav 4u IPv4 2919 0t0 TCP localhost:3310 (LISTEN) +sudo kill 804923 # Replace 804923 with the actual PID from your lsof output + +# Or kill all clamd processes: +sudo kill $(pgrep clamd) 2>/dev/null || true + +# If it doesn't stop gracefully, force kill it: +sudo kill -9 $(pgrep clamd) 2>/dev/null || true + +# Wait for cleanup +sleep 2 + +# Verify port is now free +sudo ss -tlnp | grep 3310 # Should now only show systemd or be empty + +# Clean up remaining artifacts +sudo rm -rf /var/run/clamav +sudo rm -rf /tmp/clamd.* + +# Reload systemd to clear socket binding +sudo systemctl daemon-reload + +# Verify port is completely free +sleep 1 +sudo ss -tlnp | grep 3310 # Should return nothing now + +# Run the installation script +sudo ./install-docker-clamav.sh +``` + +**Complete ClamAV Uninstall (Recommended for Docker-only setup)** + +To completely remove system ClamAV and free up port 3310: + +```bash +# 1. Stop and kill any running clamd processes +sudo kill -9 $(pgrep clamd) 2>/dev/null || true +sudo kill -9 $(pgrep freshclam) 2>/dev/null || true + +# 2. Stop systemd services and sockets +sudo systemctl stop clamav-daemon clamav-freshclam 2>/dev/null || true +sudo systemctl stop clamav.socket clamav-daemon.socket 2>/dev/null || true + +# 3. Disable from auto-start +sudo systemctl disable clamav-daemon clamav-freshclam 2>/dev/null || true +sudo systemctl disable clamav.socket clamav-daemon.socket 2>/dev/null || true + +# 4. Uninstall ClamAV packages +sudo apt remove -y clamav clamav-daemon clamav-freshclam clamav-base 2>/dev/null || true + +# 5. Remove ClamAV configuration and data directories +sudo rm -rf /etc/clamav +sudo rm -rf /var/lib/clamav +sudo rm -rf /var/run/clamav +sudo rm -rf /tmp/clamd* + +# 6. Clean up package manager +sudo apt autoclean +sudo apt autoremove + +# 7. Reload systemd to clear all socket bindings +sudo systemctl daemon-reload + +# 8. Verify port 3310 is completely free +sleep 2 +sudo ss -tlnp | grep 3310 # Should return nothing + +# 9. Now run the Docker installation script +sudo ./install-docker-clamav.sh +``` + +**If port still shows as in use after all cleanup:** + +```bash +# The socket may be stuck in the kernel. Reboot required. +sudo reboot + +# After reboot, verify: +sudo ss -tlnp | grep 3310 # Should be empty + +# Then run installation: +sudo ./install-docker-clamav.sh +``` + +### Syslog Logging Driver Not Available + +**Problem**: Docker fails with "protocol wrong type for socket" when starting container: +``` +docker: Error response from daemon: failed to create task for container: failed to initialize logging driver: dial unix /dev/log: connect: protocol wrong type for socket +``` + +**Cause**: The syslog logging driver is not available on this system (common on minimal Debian/Alpine installations). + +**Solution**: The installation script has already created the container but failed on logging setup. Restart without syslog logging: + +```bash +# Stop the failed container +docker stop clamav 2>/dev/null || true +docker rm clamav 2>/dev/null || true + +# Start container with default json-file logging instead +docker run -d \ + --name clamav \ + --restart unless-stopped \ + --memory="4g" \ + --memory-swap="4g" \ + -p 3310:3310 \ + -v clamav-db:/var/lib/clamav \ + -e CLAMAV_NO_FRESHCLAMD=false \ + -e CLAMD_STARTUP_TIMEOUT=1800 \ + -e FRESHCLAM_CHECKS=12 \ + clamav/clamav:stable + +# Check if it started +sleep 5 +docker ps | grep clamav + +# View logs +docker logs clamav +``` + +**Note**: With the default json-file logging driver, use `docker logs clamav` to view logs instead of `journalctl`. If you prefer syslog logging, see the **Send Logs to Host Syslog** section below. + +### Container Won't Start + +**Problem**: Container exits immediately after starting + +```bash +# Check container logs +docker logs clamav + +# Common causes: +# 1. Port 3310 already in use +sudo netstat -tulpn | grep 3310 + +# 2. Insufficient memory +free -h + +# 3. Permission issues with volume +docker volume inspect clamav-db +``` + +**Solution**: +```bash +# Remove existing container and volume +docker stop clamav +docker rm clamav +docker volume rm clamav-db + +# Recreate with correct configuration +docker volume create clamav-db +docker run -d --name clamav -p 3310:3310 \ + -v clamav-db:/var/lib/clamav \ + clamav/clamav:stable +``` + +### Database Download Fails + +**Problem**: FreshClam can't download signature databases + +```bash +# Check logs +docker logs clamav | grep -i freshclam + +# Common error: "Can't download daily.cvd" +``` + +**Solution**: +```bash +# 1. Check internet connectivity +docker exec clamav ping -c 3 database.clamav.net + +# 2. Try alternative mirror +docker run -d --name clamav \ + -e FRESHCLAM_MIRRORS=https://database.clamav.net \ + clamav/clamav:stable + +# 3. Manual database download +docker exec clamav freshclam --verbose +``` + +### High Memory Usage + +**Problem**: Container using more than 4GB RAM + +```bash +# Check memory usage +docker stats clamav + +# Check concurrent reload setting +docker exec clamav grep ConcurrentDatabaseReload /etc/clamav/clamd.conf +``` + +**Expected Behavior**: +- With `ConcurrentDatabaseReload yes` (default): Peak ~4GB during database updates, normal ~2GB +- This is **intentional** for 24/7 availability + +**If Memory is Critical Issue**: + +Only if you can tolerate ~30-second scanning interruption during updates: + +```bash +# Stop container +docker stop clamav + +# Create memory-constrained config (NOT RECOMMENDED for production) +cat > clamd.conf < clamd.conf </-json.log +``` + +To find your ClamAV container's log file: +```bash +# Get container ID +docker inspect clamav --format='{{.Id}}' + +# View log file location +docker inspect clamav --format='{{.LogPath}}' + +# View log file directly +sudo cat $(docker inspect clamav --format='{{.LogPath}}') +``` + +### View Container Logs + +**Note**: If you used the automated installation script (`install-docker-clamav.sh`), json-file logging is used by default (most compatible). Use `docker logs -f clamav` to view logs. If you customized the installation with syslog driver, use `sudo journalctl -t clamav -f` instead. + +```bash +# Follow logs in real-time +docker logs -f clamav + +# Show last 100 lines +docker logs --tail 100 clamav + +# Show logs with timestamps +docker logs -t clamav + +# Filter for errors only +docker logs clamav 2>&1 | grep -i error +``` + +### Send Logs to Host Syslog + +To send ClamAV container logs to the host's syslog daemon, use the `syslog` logging driver: + +#### Option 1: Using docker run + +```bash +docker run -d \ + --name clamav \ + --restart unless-stopped \ + --memory="3g" \ + --memory-swap="3g" \ + -p 3310:3310 \ + -v clamav-db:/var/lib/clamav \ + --log-driver=syslog \ + --log-opt syslog-address=unix:///dev/log \ + --log-opt tag="clamav" \ + clamav/clamav:stable +``` + +**Log Options:** +- `--log-driver=syslog` - Use syslog logging driver +- `--log-opt syslog-address=unix:///dev/log` - Send to host's syslog socket (default) +- `--log-opt tag="clamav"` - Tag logs with "clamav" identifier +- `--log-opt syslog-facility=daemon` - Optional: Set syslog facility (default: daemon) + +#### Option 2: Using docker-compose.yml + +```yaml +version: '3.8' + +services: + clamav: + image: clamav/clamav:stable + container_name: clamav + restart: unless-stopped + ports: + - "3310:3310" + volumes: + - clamav-db:/var/lib/clamav + - /DATA/clamav/config/clamd.conf:/etc/clamav/clamd.conf:ro + - /DATA/clamav/config/freshclam.conf:/etc/clamav/freshclam.conf:ro + logging: + driver: syslog + options: + syslog-address: "unix:///dev/log" + tag: "clamav" + syslog-facility: "daemon" + environment: + - CLAMAV_NO_FRESHCLAMD=false + - CLAMD_STARTUP_TIMEOUT=1800 + deploy: + resources: + limits: + memory: 4g + reservations: + memory: 2g + +volumes: + clamav-db: + driver: local +``` + +#### Option 3: Send to Remote Syslog Server + +```bash +docker run -d \ + --name clamav \ + --log-driver=syslog \ + --log-opt syslog-address=tcp://192.168.1.100:514 \ + --log-opt tag="clamav" \ + clamav/clamav:stable +``` + +**Remote syslog addresses:** +- `tcp://hostname:port` - TCP connection +- `udp://hostname:port` - UDP connection (default port: 514) +- `tcp+tls://hostname:port` - TLS-encrypted TCP + +### View Syslog Logs + +After configuring syslog logging, view ClamAV logs through your system's syslog: + +```bash +# For systemd systems (journald) +sudo journalctl -t clamav -f + +# For traditional syslog +sudo tail -f /var/log/syslog | grep clamav + +# Or check /var/log/messages on some systems +sudo tail -f /var/log/messages | grep clamav +``` + +### Send Logs to Systemd Journal + +Alternative to syslog, use the `journald` driver to integrate with systemd: + +```bash +docker run -d \ + --name clamav \ + --log-driver=journald \ + --log-opt tag="clamav" \ + clamav/clamav:stable + +# View logs via journalctl +sudo journalctl CONTAINER_NAME=clamav -f + +# Or by container ID +sudo journalctl CONTAINER_ID=$(docker inspect clamav --format='{{.Id}}') -f +``` + +**Note**: When using `syslog` or `journald` drivers, `docker logs` command will not work. Use `journalctl` or syslog viewers instead. + +### Update ClamAV + +```bash +# Pull latest stable image +docker pull clamav/clamav:stable + +# Stop and remove old container +docker stop clamav +docker rm clamav + +# Run new container with same volume (preserves databases) +docker run -d \ + --name clamav \ + --restart unless-stopped \ + -p 3310:3310 \ + -v clamav-db:/var/lib/clamav \ + clamav/clamav:stable +``` + +### Backup Signature Databases + +```bash +# Create backup directory +mkdir -p ~/clamav-backups + +# Backup database volume +docker run --rm \ + -v clamav-db:/data \ + -v ~/clamav-backups:/backup \ + alpine tar czf /backup/clamav-db-$(date +%Y%m%d).tar.gz /data + +# Verify backup +ls -lh ~/clamav-backups/ +``` + +### Restore Signature Databases + +```bash +# Stop ClamAV container +docker stop clamav + +# Restore from backup +docker run --rm \ + -v clamav-db:/data \ + -v ~/clamav-backups:/backup \ + alpine tar xzf /backup/clamav-db-20260127.tar.gz -C / + +# Start ClamAV container +docker start clamav +``` + +### Remove ClamAV Completely + +```bash +# Stop and remove container +docker stop clamav +docker rm clamav + +# Remove database volume (WARNING: Deletes all signature data) +docker volume rm clamav-db + +# Remove image +docker rmi clamav/clamav:stable + +# Clean up configuration files +rm -f clamd.conf freshclam.conf clamav.env +``` + +## Additional Resources + +- **Official ClamAV Documentation**: https://docs.clamav.net/ +- **ClamAV Docker Hub**: https://hub.docker.com/r/clamav/clamav +- **ClamAV GitHub Repository**: https://github.com/Cisco-Talos/clamav +- **BunkerWeb Documentation**: https://docs.bunkerweb.io/ +- **BunkerWeb Malware-Scan Plugin**: https://github.com/bunkerity/bunkerweb-plugins + +## License + +ClamAV is licensed under the GNU General Public License v2.0. See the [ClamAV license](https://github.com/Cisco-Talos/clamav/blob/main/COPYING) for details. + +## Support + +For ClamAV issues: +- ClamAV Users Mailing List: https://www.clamav.net/contact.html#ml +- ClamAV GitHub Issues: https://github.com/Cisco-Talos/clamav/issues + +For BunkerWeb integration issues: +- BunkerWeb GitHub Issues: https://github.com/bunkerity/bunkerweb/issues +- BunkerWeb Discord: https://discord.gg/bunkerweb diff --git a/malware-scan/install-docker-clamav.sh b/malware-scan/install-docker-clamav.sh new file mode 100644 index 0000000..9c0d544 --- /dev/null +++ b/malware-scan/install-docker-clamav.sh @@ -0,0 +1,866 @@ +#!/bin/bash + +############################################################################# +# ClamAV Docker Installation Script (Debian & CentOS) +# +# Purpose: Automated installation and configuration of ClamAV using Docker +# Supported: Debian 11+, CentOS 7+, Rocky Linux 8+, AlmaLinux 8+ +# Target Directory: /DATA/clamav (default, customizable with --install_dir) +# Container Name: clamav +# TCP Port: 3310 +# +# Usage: +# sudo ./install-docker-clamav.sh +# sudo ./install-docker-clamav.sh --install_dir=/custom/path +# +# Options: +# --install_dir=PATH Custom installation directory (default: /DATA/clamav) +# +# Features: +# - Auto-detects OS (Debian/CentOS/Rocky/AlmaLinux) +# - Installs Docker if not present +# - Creates persistent data directory +# - Configures memory-optimized ClamAV +# - Sets up systemd service for auto-start +# - Tests installation with EICAR +# +############################################################################# + +set -e # Exit on error +set -u # Exit on undefined variable + +# Default Configuration +INSTALL_DIR="/DATA/clamav" +CONTAINER_NAME="clamav" +CLAMAV_IMAGE="clamav/clamav:stable" +CLAMAV_PORT="3310" +MEMORY_LIMIT="4g" # 4GB required for concurrent database reload (24/7 availability) +LOG_DRIVER="json-file" # Options: json-file, syslog, journald (json-file is most compatible) +SYSLOG_TAG="clamav" + +# ============================================================================ +# OS DETECTION +# ============================================================================ + +# Detect OS type +detect_os() { + if [[ -f /etc/os-release ]]; then + . /etc/os-release + OS_ID="$ID" + OS_VERSION="${VERSION_ID:-}" + elif [[ -f /etc/redhat-release ]]; then + OS_ID="rhel" + OS_VERSION=$(cat /etc/redhat-release | grep -oP '\d+' | head -1) + elif [[ -f /etc/debian_version ]]; then + OS_ID="debian" + OS_VERSION=$(cat /etc/debian_version | grep -oP '\d+' | head -1) + else + OS_ID="unknown" + OS_VERSION="unknown" + fi + + # Normalize OS names + case "$OS_ID" in + debian|ubuntu) + OS_TYPE="debian" + ;; + centos|rhel|rocky|almalinux|fedora) + OS_TYPE="rhel" + ;; + *) + OS_TYPE="unknown" + ;; + esac +} + +# Check if OS is supported +check_os_support() { + detect_os + + if [[ "$OS_TYPE" == "unknown" ]]; then + log_error "Unsupported OS: $OS_ID" + log_error "Supported: Debian 11+, CentOS 7+, Rocky Linux 8+, AlmaLinux 8+" + exit 1 + fi + + log_info "Detected OS: $OS_ID (Type: $OS_TYPE) Version: $OS_VERSION" +} + +# Package manager wrapper for OS-agnostic operations +pkg_update() { + if [[ "$OS_TYPE" == "debian" ]]; then + apt-get update -qq + else + yum update -y -q 2>/dev/null || dnf update -y -q + fi +} + +pkg_install() { + local packages="$@" + if [[ "$OS_TYPE" == "debian" ]]; then + apt-get install -y -qq $packages + else + yum install -y -q $packages 2>/dev/null || dnf install -y -q $packages + fi +} + +# Parse command-line arguments +for arg in "$@"; do + case $arg in + --install_dir=*) + INSTALL_DIR="${arg#*=}" + ;; + --help|-h) + echo "ClamAV Docker Installation Script" + echo "" + echo "Usage: sudo $0 [OPTIONS]" + echo "" + echo "Options:" + echo " --install_dir=PATH Custom installation directory (default: /DATA/clamav)" + echo " --help, -h Show this help message" + echo "" + echo "Examples:" + echo " sudo $0" + echo " sudo $0 --install_dir=/opt/clamav" + echo " sudo $0 --install_dir=/home/user/clamav" + exit 0 + ;; + *) + echo "Unknown option: $arg" + echo "Use --help for usage information" + exit 1 + ;; + esac +done + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Logging functions +log_info() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +log_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +log_warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# Check if running as root +check_root() { + if [[ $EUID -ne 0 ]]; then + log_error "This script must be run as root (use sudo)" + exit 1 + fi +} + +# Check system requirements +check_requirements() { + log_info "Checking system requirements..." + + # Check if OS is supported + check_os_support + + # Check RAM + total_ram=$(free -g | awk '/^Mem:/{print $2}') + if [[ $total_ram -lt 4 ]]; then + log_warning "System has ${total_ram}GB RAM. Minimum 4GB required for 24/7 ClamAV availability." + log_warning "(Concurrent database reload requires ~4GB peak memory)" + read -p "Continue anyway? (y/N): " -n 1 -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + exit 1 + fi + else + log_success "System has ${total_ram}GB RAM (sufficient)" + fi + + # Check disk space + available_space=$(df -BG / | awk 'NR==2 {print $4}' | sed 's/G//') + if [[ $available_space -lt 5 ]]; then + log_error "Insufficient disk space. Need at least 5GB, have ${available_space}GB" + exit 1 + else + log_success "Disk space: ${available_space}GB available" + fi +} + +# Install Docker if not present +install_docker() { + if command -v docker &> /dev/null; then + log_success "Docker already installed ($(docker --version))" + else + log_info "Installing Docker..." + pkg_update + fi + + # Install netcat if not present (needed for port checking) + if ! command -v nc &> /dev/null && ! command -v netcat &> /dev/null; then + log_info "Installing netcat for port checking..." + if [[ "$OS_TYPE" == "debian" ]]; then + pkg_install netcat-openbsd + else + pkg_install nmap-ncat + fi + fi + + # Continue Docker installation if not already present + if ! command -v docker &> /dev/null; then + if [[ "$OS_TYPE" == "debian" ]]; then + # Debian/Ubuntu Docker installation + log_info "Installing Docker on Debian-based system..." + + # Install prerequisites + pkg_install ca-certificates curl gnupg lsb-release + + # Add Docker's GPG key + install -m 0755 -d /etc/apt/keyrings + curl -fsSL https://download.docker.com/linux/debian/gpg | \ + gpg --dearmor -o /etc/apt/keyrings/docker.gpg + chmod a+r /etc/apt/keyrings/docker.gpg + + # Set up Docker repository + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] \ + https://download.docker.com/linux/debian \ + $(lsb_release -cs) stable" | \ + tee /etc/apt/sources.list.d/docker.list > /dev/null + + # Install Docker Engine + pkg_update + pkg_install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + + else + # CentOS/RHEL/Rocky/AlmaLinux Docker installation + log_info "Installing Docker on RHEL-based system..." + + # Install prerequisites + pkg_install yum-utils device-mapper-persistent-data lvm2 + + # Add Docker repository + if command -v yum &> /dev/null; then + yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo -q 2>/dev/null || true + pkg_install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + else + dnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo 2>/dev/null || true + pkg_install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + fi + fi + + # Enable Docker service + systemctl daemon-reload + systemctl enable docker + systemctl start docker + + log_success "Docker installed successfully ($(docker --version))" + fi +} + +# Create installation directory +create_directories() { + log_info "Creating installation directory: ${INSTALL_DIR}" + + # Create main directory + mkdir -p "${INSTALL_DIR}" + mkdir -p "${INSTALL_DIR}/data" + mkdir -p "${INSTALL_DIR}/config" + mkdir -p "${INSTALL_DIR}/logs" + + log_success "Directories created" +} + +# Create optimized ClamAV configuration files +create_configs() { + log_info "Creating ClamAV configuration files..." + + # Create clamd.conf (memory optimized, aligned with install.sh defaults) + cat > "${INSTALL_DIR}/config/clamd.conf" <<'EOF' +# ClamAV Daemon Configuration +# Memory-optimized for 3-4GB RAM systems +# Aligned with malware-scan plugin defaults + +# Network Configuration +TCPSocket 3310 +TCPAddr 127.0.0.1 + +# Performance Settings +MaxThreads 12 +MaxConnectionQueueLength 30 +ReadTimeout 300 +CommandReadTimeout 30 +SendBufTimeout 500 + +# File Size Limits (aligned with plugin: MALWARE_SCAN_CLAMAV_MAX_SIZE=26214400 = 25MB) +MaxScanSize 150M +MaxFileSize 30M +MaxRecursion 16 +MaxFiles 10000 +MaxEmbeddedPE 10M +MaxHTMLNormalize 10M +MaxHTMLNoTags 2M +MaxScriptNormalize 5M +MaxZipTypeRcg 1M +StreamMaxLength 25M + +# Scan Options +ScanPE yes +ScanELF yes +ScanMail yes +ScanArchive yes +ScanHTML yes +ScanOLE2 yes +ScanPDF yes +ScanSWF yes +ScanXMLDOCS yes +ScanHWP3 yes + +# IMPORTANT: Disable PUA detection (focus on real malware only) +# This prevents false positives from legitimate software +# and ensures only actual malware is detected and shared +DetectPUA no + +# Self-check interval (seconds) +SelfCheck 600 + +# Idle timeout (seconds) +IdleTimeout 60 + +# Foreground mode (for Docker) +Foreground yes + +# IMPORTANT: Enable concurrent reload for 24/7 availability +# Allows scanning to continue during database updates (no interruption) +# Trade-off: Higher peak RAM usage (~4GB) during database reload +# Ensures ClamAV socket is available 24/7 without downtime +ConcurrentDatabaseReload yes + +# Logging (verbose mode for troubleshooting) +LogTime yes +LogVerbose yes +Debug yes +LogClean yes +EOF + + # Create freshclam.conf (memory optimized) + cat > "${INSTALL_DIR}/config/freshclam.conf" <<'EOF' +# FreshClam Configuration +# Configured for 24/7 availability + +# Database Mirror +DatabaseMirror database.clamav.net + +# Update frequency (checks per day: 1-50) +# 12 checks per day = every 2 hours (recommended for production) +Checks 12 + +# Database directory +DatabaseDirectory /var/lib/clamav + +# Logging +LogSyslog no +LogVerbose yes +LogTime yes + +# Connection settings +ConnectTimeout 30 +ReceiveTimeout 60 + +# Database testing enabled for reliability +# Ensures database integrity before loading +TestDatabases yes + +# Bytecode +Bytecode yes + +# Foreground mode (for Docker) +Foreground yes +EOF + + # Create docker-compose.yml for easier management + cat > "${INSTALL_DIR}/docker-compose.yml" < /dev/null || command -v nc &> /dev/null; then + # Use netcat to check port + if nc -z localhost ${CLAMAV_PORT} 2>/dev/null; then + # Port is in use - test if it's ClamAV + local response=$(echo "PING" | nc -w 2 localhost ${CLAMAV_PORT} 2>/dev/null || echo "") + + if echo "$response" | grep -q "PONG"; then + log_warning "ClamAV is already running on port ${CLAMAV_PORT}" + log_info "Testing existing ClamAV daemon..." + + # Test with EICAR to verify it's working + if echo 'X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*' | nc -w 5 localhost ${CLAMAV_PORT} 2>/dev/null | grep -q "FOUND"; then + log_success "Existing ClamAV daemon is working correctly" + log_info "No installation needed - ClamAV is already operational" + log_info "" + log_info "If you want to reinstall, first stop the existing daemon:" + log_info " - For Docker: docker stop clamav && docker rm clamav" + log_info " - For systemd: systemctl stop clamav-daemon && systemctl disable clamav-daemon" + log_info " (Note: Disable prevents auto-start on reboot)" + log_info "" + exit 0 + else + log_warning "Port ${CLAMAV_PORT} responds to PING but may not be working correctly" + read -p "Continue with installation anyway? (y/N): " -n 1 -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + exit 1 + fi + fi + else + log_error "Port ${CLAMAV_PORT} is in use but not responding as ClamAV" + log_error "Please free up port ${CLAMAV_PORT} before continuing" + log_info "Check what's using the port with: sudo netstat -tlnp | grep ${CLAMAV_PORT}" + exit 1 + fi + else + log_success "Port ${CLAMAV_PORT} is available" + fi + else + # Netcat not available - install it now for better port checking + log_info "Netcat not found, installing for better port checking..." + apt-get update -qq + apt-get install -y -qq netcat-openbsd + log_success "Netcat installed" + + # Now check port again with netcat + if nc -z localhost ${CLAMAV_PORT} 2>/dev/null; then + # Port is in use - test if it's ClamAV + local response=$(echo "PING" | nc -w 2 localhost ${CLAMAV_PORT} 2>/dev/null || echo "") + + if echo "$response" | grep -q "PONG"; then + log_warning "ClamAV is already running on port ${CLAMAV_PORT}" + log_info "Testing existing ClamAV daemon..." + + # Test with EICAR to verify it's working + if echo 'X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*' | nc -w 5 localhost ${CLAMAV_PORT} 2>/dev/null | grep -q "FOUND"; then + log_success "Existing ClamAV daemon is working correctly" + log_info "No installation needed - ClamAV is already operational" + log_info "" + log_info "If you want to reinstall, first stop the existing daemon:" + log_info " - For Docker: docker stop clamav && docker rm clamav" + log_info " - For systemd: systemctl stop clamav-daemon && systemctl disable clamav-daemon" + log_info " (Note: Disable prevents auto-start on reboot)" + log_info "" + exit 0 + else + log_warning "Port ${CLAMAV_PORT} responds to PING but may not be working correctly" + read -p "Continue with installation anyway? (y/N): " -n 1 -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + exit 1 + fi + fi + else + log_error "Port ${CLAMAV_PORT} is in use but not responding as ClamAV" + log_error "Please free up port ${CLAMAV_PORT} before continuing" + log_info "Check what's using the port with: sudo netstat -tlnp | grep ${CLAMAV_PORT}" + exit 1 + fi + else + log_success "Port ${CLAMAV_PORT} is available" + fi + fi +} + +# Stop and remove existing container if present +cleanup_existing() { + if docker ps -a --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then + log_warning "Existing ClamAV container found, removing..." + docker stop ${CONTAINER_NAME} 2>/dev/null || true + docker rm ${CONTAINER_NAME} 2>/dev/null || true + log_success "Existing container removed" + fi +} + +# Start ClamAV container +start_container() { + log_info "Starting ClamAV container..." + + cd "${INSTALL_DIR}" + + # Build logging options based on LOG_DRIVER + local log_opts="" + if [[ "${LOG_DRIVER}" == "syslog" ]]; then + log_opts="--log-driver=syslog --log-opt syslog-address=unix:///dev/log --log-opt tag=${SYSLOG_TAG}" + log_info "Using syslog logging driver (logs will go to host syslog, not 'docker logs')" + elif [[ "${LOG_DRIVER}" == "journald" ]]; then + log_opts="--log-driver=journald --log-opt tag=${SYSLOG_TAG}" + log_info "Using journald logging driver (view with 'journalctl -t ${SYSLOG_TAG}')" + else + log_opts="--log-driver=json-file" + log_info "Using json-file logging driver (default, view with 'docker logs')" + fi + + docker run -d \ + --name ${CONTAINER_NAME} \ + --restart unless-stopped \ + --memory="${MEMORY_LIMIT}" \ + --memory-swap="${MEMORY_LIMIT}" \ + -p ${CLAMAV_PORT}:3310 \ + -v "${INSTALL_DIR}/data:/var/lib/clamav" \ + -v "${INSTALL_DIR}/config/clamd.conf:/etc/clamav/clamd.conf:ro" \ + -v "${INSTALL_DIR}/config/freshclam.conf:/etc/clamav/freshclam.conf:ro" \ + ${log_opts} \ + -e CLAMAV_NO_FRESHCLAMD=false \ + -e CLAMD_STARTUP_TIMEOUT=1800 \ + -e FRESHCLAM_CHECKS=12 \ + ${CLAMAV_IMAGE} + + log_success "Container started successfully" +} + +# Wait for ClamAV to be ready +wait_for_clamav() { + log_info "Waiting for ClamAV to download signature databases..." + log_info "This may take 10-30 minutes on first run..." + + local max_wait=1800 # 30 minutes + local elapsed=0 + local check_interval=10 + + while [[ $elapsed -lt $max_wait ]]; do + # Check if clamd is ready by testing PING command + if echo "PING" | nc -w 2 localhost ${CLAMAV_PORT} 2>/dev/null | grep -q "PONG"; then + log_success "ClamAV is ready!" + return 0 + fi + + # Check if container crashed + if ! docker ps --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then + log_error "Container stopped unexpectedly" + log_info "Check logs with: ${LOG_DRIVER == syslog && 'sudo journalctl -t ${SYSLOG_TAG} -n 50' || 'docker logs --tail 50 ${CONTAINER_NAME}'}" + return 1 + fi + + # Show progress every minute + if [[ $((elapsed % 60)) -eq 0 ]] && [[ $elapsed -gt 0 ]]; then + log_info "Still waiting... (${elapsed}s elapsed)" + fi + + sleep $check_interval + elapsed=$((elapsed + check_interval)) + done + + log_error "Timeout waiting for ClamAV to start" + return 1 +} + +# Test ClamAV installation +test_installation() { + log_info "Testing ClamAV installation..." + + # Test 1: TCP connection + log_info "Test 1: TCP connection to port ${CLAMAV_PORT}" + if echo "PING" | nc -w 5 localhost ${CLAMAV_PORT} | grep -q "PONG"; then + log_success "TCP connection test passed" + else + log_error "TCP connection test failed" + return 1 + fi + + # Test 2: EICAR test file + log_info "Test 2: EICAR malware detection" + + # Create EICAR test file + local eicar_file="${INSTALL_DIR}/eicar.txt" + echo 'X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*' > "${eicar_file}" + + # Scan using clamdscan inside container + if docker exec ${CONTAINER_NAME} clamdscan /tmp/eicar.txt 2>&1 | grep -q "Eicar.*FOUND"; then + log_success "EICAR detection test passed" + else + # Try alternative method: pipe to container + if cat "${eicar_file}" | docker exec -i ${CONTAINER_NAME} clamdscan - 2>&1 | grep -q "Eicar.*FOUND"; then + log_success "EICAR detection test passed" + else + log_warning "EICAR detection test inconclusive (may still be working)" + fi + fi + + # Clean up test file + rm -f "${eicar_file}" + + # Test 3: Database info + log_info "Test 3: Checking signature database" + local db_count=$(docker exec ${CONTAINER_NAME} sigtool --info=/var/lib/clamav/main.cvd 2>/dev/null | grep "Signatures:" | awk '{print $2}' || echo "0") + if [[ $db_count -gt 0 ]]; then + log_success "Signature database loaded (${db_count} signatures)" + else + log_warning "Could not verify signature count" + fi + + log_success "Installation tests completed" +} + +# Create systemd service for auto-start +create_systemd_service() { + log_info "Creating systemd service for auto-start..." + + cat > /etc/systemd/system/clamav-docker.service < "${INSTALL_DIR}/start.sh" <<'EOF' +#!/bin/bash +docker start clamav && echo "ClamAV started" +EOF + chmod +x "${INSTALL_DIR}/start.sh" + + # Create stop script + cat > "${INSTALL_DIR}/stop.sh" <<'EOF' +#!/bin/bash +docker stop clamav && echo "ClamAV stopped" +EOF + chmod +x "${INSTALL_DIR}/stop.sh" + + # Create restart script + cat > "${INSTALL_DIR}/restart.sh" <<'EOF' +#!/bin/bash +docker restart clamav && echo "ClamAV restarted" +EOF + chmod +x "${INSTALL_DIR}/restart.sh" + + # Create logs script (adapts based on logging driver) + if [[ "${LOG_DRIVER}" == "syslog" ]] || [[ "${LOG_DRIVER}" == "journald" ]]; then + cat > "${INSTALL_DIR}/logs.sh" < "${INSTALL_DIR}/logs.sh" <<'EOF' +#!/bin/bash +docker logs -f clamav +EOF + fi + chmod +x "${INSTALL_DIR}/logs.sh" + + # Create status script + cat > "${INSTALL_DIR}/status.sh" <<'EOF' +#!/bin/bash +echo "=== Container Status ===" +docker ps -a | grep clamav + +echo -e "\n=== Memory Usage ===" +docker stats --no-stream clamav + +echo -e "\n=== TCP Connection Test ===" +if echo "PING" | nc -w 5 localhost 3310 | grep -q "PONG"; then + echo "โœ… ClamAV is responding on port 3310" +else + echo "โŒ ClamAV is not responding on port 3310" +fi + +echo -e "\n=== Recent Logs ===" +docker logs --tail 20 clamav +EOF + chmod +x "${INSTALL_DIR}/status.sh" + + # Create update script + cat > "${INSTALL_DIR}/update.sh" <> /etc/clamd.d/scan.conf +grep -q "^TCPAddr" /etc/clamd.d/scan.conf || echo "TCPAddr 127.0.0.1" >> /etc/clamd.d/scan.conf +``` + +Configure the systemd socket at `/usr/lib/systemd/system/clamd@scan.socket`: + +```bash +# Check if ListenStream is already configured +grep -q "^ListenStream=127.0.0.1:3310" /usr/lib/systemd/system/clamd@scan.socket || \ + sed -i '/\[Socket\]/a ListenStream=127.0.0.1:3310' /usr/lib/systemd/system/clamd@scan.socket +``` + +After modifying the configuration, reload systemd and restart the service: + +```bash +systemctl daemon-reload +systemctl restart clamd@scan +``` + +### Disable PUA Detection (Recommended) + +By default, ClamAV may detect PUA (Potentially Unwanted Applications). For malware scanning purposes, it's recommended to disable PUA detection to focus only on real malware threats. + +**Important**: Sharing examples and threat intelligence should only be done for real malware, not for PUA. PUA can include legitimate software that users may intentionally want to use, and flagging these could lead to false positives and unnecessary blocking. By disabling PUA detection, you ensure that only actual malicious threats are detected and reported. + +Check if PUA detection is enabled: + +**Debian/Ubuntu:** +```bash +grep "^DetectPUA" /etc/clamav/clamd.conf +``` + +**Red Hat/CentOS:** +```bash +grep "^DetectPUA" /etc/clamd.d/scan.conf +``` + +If `DetectPUA` is set to `yes`, disable it: + +**Debian/Ubuntu:** +```bash +# Disable PUA detection if enabled +sed -i 's/^DetectPUA yes/DetectPUA no/' /etc/clamav/clamd.conf + +# Or ensure it's explicitly disabled if the line doesn't exist +grep -q "^DetectPUA" /etc/clamav/clamd.conf || echo "DetectPUA no" >> /etc/clamav/clamd.conf +``` + +**Red Hat/CentOS:** +```bash +# Disable PUA detection if enabled +sed -i 's/^DetectPUA yes/DetectPUA no/' /etc/clamd.d/scan.conf + +# Or ensure it's explicitly disabled if the line doesn't exist +grep -q "^DetectPUA" /etc/clamd.d/scan.conf || echo "DetectPUA no" >> /etc/clamd.d/scan.conf +``` + +Verify your configuration: + +```bash +clamconf | grep "DetectPUA" +``` + +Expected output: +``` +DetectPUA disabled +``` + +After changing this setting, restart the daemon: + +**Debian/Ubuntu:** +```bash +systemctl restart clamav-daemon +``` + +**Red Hat/CentOS:** +```bash +systemctl restart clamd@scan +``` + +### Enable ClamAV Debugging (Recommended) + +For detailed scanning logs and troubleshooting, enable ClamAV debugging output: + +**Debian/Ubuntu:** +```bash +# Enable verbose logging +sed -i 's/^LogVerbose false/LogVerbose true/' /etc/clamav/clamd.conf +sed -i 's/^Debug false/Debug true/' /etc/clamav/clamd.conf +sed -i 's/^LogClean false/LogClean true/' /etc/clamav/clamd.conf + +# If lines don't exist, add them +grep -q "^LogVerbose" /etc/clamav/clamd.conf || echo "LogVerbose true" >> /etc/clamav/clamd.conf +grep -q "^Debug" /etc/clamav/clamd.conf || echo "Debug true" >> /etc/clamav/clamd.conf +grep -q "^LogClean" /etc/clamav/clamd.conf || echo "LogClean true" >> /etc/clamav/clamd.conf + +# Restart daemon +systemctl restart clamav-daemon + +# View logs +tail -f /var/log/clamav/clamav.log +``` + +**Red Hat/CentOS:** +```bash +# Enable verbose logging +sed -i 's/^LogVerbose false/LogVerbose true/' /etc/clamd.d/scan.conf +sed -i 's/^Debug false/Debug true/' /etc/clamd.d/scan.conf +sed -i 's/^LogClean false/LogClean true/' /etc/clamd.d/scan.conf + +# If lines don't exist, add them +grep -q "^LogVerbose" /etc/clamd.d/scan.conf || echo "LogVerbose true" >> /etc/clamd.d/scan.conf +grep -q "^Debug" /etc/clamd.d/scan.conf || echo "Debug true" >> /etc/clamd.d/scan.conf +grep -q "^LogClean" /etc/clamd.d/scan.conf || echo "LogClean true" >> /etc/clamd.d/scan.conf + +# Restart daemon +systemctl restart clamd@scan + +# View logs +tail -f /var/log/clamd.scan +``` + +**What debugging enables:** +- **LogVerbose**: Shows detailed scan information (file paths, scan duration, methods used) +- **Debug**: Shows daemon-level details (connections, INSTREAM protocol, network activity) +- **LogClean**: Logs clean files (not just infected ones) - helps confirm scanning is working + +## BunkerWeb Plugin Configuration + +After ClamAV is installed and running, configure the BunkerWeb plugin by adding these variables to `/etc/bunkerweb/variables.env`: + +### Basic Configuration (Copy-Paste Ready) + +```bash +# Core Settings +USE_MALWARE-SCAN=yes +MALWARE_SCAN_DEBUG=yes # Enable until plugin is stable, then set to 'no' +MALWARE_SCAN_EXCLUDED_EXTENSIONS=jpg,jpeg,png,gif,bmp,ico,webp,mp3,mp4,avi,mov,wmv,flv,webm,ogg,wav,flac + +# ClamAV Settings +MALWARE_SCAN_CLAMAV_HOST=127.0.0.1 +MALWARE_SCAN_CLAMAV_PORT=3310 +MALWARE_SCAN_CLAMAV_TIMEOUT=10000 +MALWARE_SCAN_CLAMAV_MAX_SIZE=26214400 # 25MB +MALWARE_SCAN_HASH_MAX_SIZE=67108864 # 64MB + +# Multi-Layer Scanning +MALWARE_SCAN_SKIP_OTHERS_ON_CLAMAV_DETECT=yes # Fast mode: skip cloud checks if ClamAV detects +MALWARE_SCAN_SCAN_FULL_MULTIPART_BODY=no # Skip full body scan (recommended - random boundaries prevent caching) + +# File Handling +MALWARE_SCAN_BLOCK_MEMORY_UPLOADS=no +MALWARE_SCAN_CLEANUP_FILES=yes +MALWARE_SCAN_CLEANUP_ORPHANED=no +MALWARE_SCAN_CLEANUP_AGE=3600 + +# Caching +MALWARE_SCAN_CACHE_VIRUS_TTL=7776000 # 90 days + +# VirusTotal Integration (Optional) +MALWARE_SCAN_API_USE_VIRUSTOTAL=no +MALWARE_SCAN_API_VIRUSTOTAL_KEY= # Required if enabled +MALWARE_SCAN_API_VIRUSTOTAL_SUSPICIOUS=5 +MALWARE_SCAN_API_VIRUSTOTAL_MALICIOUS=3 +MALWARE_SCAN_API_VIRUSTOTAL_UPLOAD_UNKNOWN=no +MALWARE_SCAN_API_VIRUSTOTAL_AUTO_VOTE=no +MALWARE_SCAN_API_VIRUSTOTAL_MAX_UPLOAD_SIZE=33554432 # 32MB +MALWARE_SCAN_SHARE_FILES=no # Share detected malware with threat intelligence + +# SentinelOne Integration (Optional) +MALWARE_SCAN_API_USE_SENTINELONE=no +MALWARE_SCAN_API_SENTINELONE_URL= # Required if enabled +MALWARE_SCAN_API_SENTINELONE_TOKEN= # Required if enabled +``` + +### Configuration Notes + +**Important Settings:** +- `MALWARE_SCAN_DEBUG=yes` - Keep enabled during initial testing. Generates verbose logs for troubleshooting. Set to `no` once stable. +- `MALWARE_SCAN_CLAMAV_MAX_SIZE=26214400` - Must match or be less than ClamAV's `StreamMaxLength` setting +- `MALWARE_SCAN_HASH_MAX_SIZE=67108864` - Prevents performance issues with large files (ISOs, VMs) + +**Cloud Scanner Setup:** +- **VirusTotal**: Get free API key from https://www.virustotal.com/gui/join-us (4 req/min, 500 req/day) +- **SentinelOne**: Requires SentinelOne license and management console access + +**Performance Modes:** +- Fast Mode: `MALWARE_SCAN_SKIP_OTHERS_ON_CLAMAV_DETECT=yes` (default) - Skip cloud checks if ClamAV detects +- Full Analysis: `MALWARE_SCAN_SKIP_OTHERS_ON_CLAMAV_DETECT=no` - Always run all scanners for complete threat intelligence + +**Threat Intelligence Sharing:** +- `MALWARE_SCAN_SHARE_FILES=yes` - Share malware detected by any scanner (SentinelOne) with VirusTotal if the file is unknown to VT +- Helps improve global threat intelligence by contributing new malware samples +- Requires VirusTotal integration to be enabled +- Only real malware is shared, not PUA (Potentially Unwanted Applications) +- Files are automatically voted as malicious if `MALWARE_SCAN_API_VIRUSTOTAL_AUTO_VOTE=yes` + +After updating variables, restart BunkerWeb: +```bash +systemctl restart bunkerweb +``` + +## Testing the Installation + +### Basic Connectivity Test + +Verify that ClamAV is listening on the correct port by sending a PING command: + +```bash +echo "PING" | nc 127.0.0.1 3310 +``` + +Expected response: +``` +PONG +``` + +If you receive `PONG`, ClamAV is correctly installed and listening on the network socket. + +### Malware Detection Test (EICAR) + +To verify that ClamAV can detect malware, use the EICAR test file - a harmless test string recognized by all antivirus software. + +**Download EICAR test files:** + +Official source: https://www.eicar.org/download-anti-malware-testfile/ + +```bash +# Download EICAR test file (plain text) +wget https://secure.eicar.org/eicar.com -O eicar.com + +# Or download the ZIP version +wget https://secure.eicar.org/eicar_com.zip -O eicar_com.zip +``` + +**Test with ClamAV:** + +```bash +# Scan the EICAR test file +clamdscan eicar.com +``` + +Expected output: +``` +/path/to/eicar.com: Eicar-Test-Signature FOUND + +----------- SCAN SUMMARY ----------- +Infected files: 1 +Time: 0.001 sec (0 m 0 s) +``` + +**Important Notes:** +- EICAR is NOT malware - it's a safe test string designed for AV testing +- Modern browsers may block the download (this is normal security behavior) +- If your browser blocks it, use `wget` or `curl` from the command line +- The EICAR signature: `X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*` + +**Alternative - Create EICAR manually:** + +```bash +# Create EICAR test file manually (if download is blocked) +echo 'X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*' > eicar.com + +# Test the file +clamdscan eicar.com +``` + +If ClamAV detects the EICAR file correctly, your malware scanning setup is working properly. + +### Testing Socket Connection with INSTREAM + +Test uploading a file directly to ClamAV's TCP socket using the INSTREAM protocol (this is how the BunkerWeb plugin communicates with ClamAV): + +**1. Create EICAR test file:** + +```bash +echo 'X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*' > /tmp/eicar.com +``` + +**2. Send file via INSTREAM protocol:** + +```bash +# Using a simple bash script to test INSTREAM +( + echo -n "zINSTREAM" + echo -ne "\x00\x00\x00\x44" # 68 bytes in big-endian + cat /tmp/eicar.com + echo -ne "\x00\x00\x00\x00" # End marker +) | nc 127.0.0.1 3310 +``` + +**Expected response:** +``` +stream: Eicar-Test-Signature FOUND +``` + +**Alternative using a Python script:** + +```bash +# Create a test script +cat > test_instream.py << 'EOF' +#!/usr/bin/env python3 +import socket +import struct + +def scan_file(filepath): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect(("127.0.0.1", 3310)) + + # Send INSTREAM command + sock.sendall(b"zINSTREAM\0") + + # Read and send file in chunks + with open(filepath, 'rb') as f: + while True: + chunk = f.read(2048) + if not chunk: + break + # Send chunk size (4 bytes, big-endian) followed by chunk + sock.sendall(struct.pack(b'>L', len(chunk))) + sock.sendall(chunk) + + # Send zero-length chunk to signal end + sock.sendall(struct.pack(b'>L', 0)) + + # Receive response + response = sock.recv(4096).decode('utf-8') + sock.close() + + print(f"ClamAV response: {response.strip()}") + return response + +if __name__ == "__main__": + scan_file("/tmp/eicar.com") +EOF + +# Make it executable and run +chmod +x test_instream.py +python3 test_instream.py +``` + +**Expected output:** +``` +ClamAV response: stream: Eicar-Test-Signature FOUND +``` + +**3. Test with a clean file:** + +```bash +echo "This is a clean file" > /tmp/clean.txt +python3 test_instream.py # (modify filepath to /tmp/clean.txt) +``` + +**Expected output:** +``` +ClamAV response: stream: OK +``` + +This confirms that: +- ClamAV's TCP socket is working correctly +- The INSTREAM protocol is functioning +- Malware detection works over the network socket +- The plugin's communication method with ClamAV is validated + +### Testing BunkerWeb Plugin Integration + +After configuring the BunkerWeb plugin, test the complete malware scanning workflow: + +**1. Prepare EICAR test file:** + +```bash +# Create EICAR test file +echo 'X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*' > /tmp/eicar.com + +# Or download it +wget https://secure.eicar.org/eicar.com -O /tmp/eicar.com +``` + +**2. Test file upload via HTTP:** + +```bash +# Upload EICAR to your BunkerWeb-protected application +curl -X POST -F "file=@/tmp/eicar.com" http://your-app.com/upload +``` + +**Expected behavior:** +- Request should be blocked with HTTP 403 +- BunkerWeb logs should show malware detection: + ``` + [MALWARE-SCAN] MALWARE DETECTED BY CLAMAV + [MALWARE-SCAN] ClamAV signature: Eicar-Test-Signature + [MALWARE-SCAN] BLOCKING REQUEST + ``` + +**3. Check BunkerWeb logs:** + +```bash +# View recent logs +journalctl -u bunkerweb -n 100 | grep MALWARE-SCAN + +# Or if using Docker +docker logs bunkerweb | grep MALWARE-SCAN +``` + +**4. Test clean file upload:** + +```bash +# Create a clean test file +echo "This is a clean test file" > /tmp/clean.txt + +# Upload clean file +curl -X POST -F "file=@/tmp/clean.txt" http://your-app.com/upload +``` + +**Expected behavior:** +- Request should be allowed (HTTP 200 or normal response) +- Logs should show clean scan result + +**5. Cleanup:** + +```bash +# Remove test files +rm /tmp/eicar.com /tmp/clean.txt +``` + +## Updating Virus Definitions + +To ensure ClamAV has the latest virus definitions, run: + +```bash +freshclam +``` + +You can also enable automatic updates by ensuring the `clamav-freshclam` service is running: + +```bash +systemctl enable clamav-freshclam +systemctl start clamav-freshclam +``` + +## Troubleshooting + +If the service is not responding: + +1. Check the service status: + + **Debian/Ubuntu:** + ```bash + systemctl status clamav-daemon + ``` + + **Red Hat/CentOS:** + ```bash + systemctl status clamd@scan + ``` + +2. Check if the port is listening: + ```bash + netstat -tlnp | grep 3310 + # or + ss -tlnp | grep 3310 + ``` + +3. Review logs for errors: + + **Debian/Ubuntu:** + ```bash + journalctl -u clamav-daemon -n 50 + ``` + + **Red Hat/CentOS:** + ```bash + journalctl -u clamd@scan -n 50 + ``` diff --git a/malware-scan/install.sh b/malware-scan/install.sh new file mode 100755 index 0000000..4bcc36d --- /dev/null +++ b/malware-scan/install.sh @@ -0,0 +1,275 @@ +#!/bin/bash +set -e + +# ClamAV Installation Script for Malware Scanning +# Supports: Debian 13, Ubuntu, Red Hat, CentOS, Rocky Linux, AlmaLinux + +echo "==========================================" +echo "ClamAV Network Scanner Installation" +echo "==========================================" +echo "" + +# Check if running as root +if [ "$EUID" -ne 0 ]; then + echo "Error: This script must be run as root" + exit 1 +fi + +# Detect OS +detect_os() { + if [ -f /etc/os-release ]; then + . /etc/os-release + OS=$ID + VER=$VERSION_ID + elif [ -f /etc/redhat-release ]; then + OS="rhel" + else + echo "Error: Unable to detect operating system" + exit 1 + fi +} + +# Install ClamAV based on OS +install_clamav() { + detect_os + echo "Detected OS: $OS $VER" + echo "" + + case "$OS" in + debian|ubuntu) + echo "Installing ClamAV on Debian/Ubuntu..." + apt-get update + apt-get install -y clamav-daemon clamav netcat-openbsd + SOCKET_FILE="/lib/systemd/system/clamav-daemon.socket" + CONFIG_FILE="/etc/clamav/clamd.conf" + SERVICE_NAME="clamav-daemon" + ;; + rhel|centos|rocky|almalinux|fedora) + echo "Installing ClamAV on Red Hat/CentOS..." + # Enable EPEL repository if needed + if ! rpm -q epel-release &>/dev/null; then + if [ "$OS" = "rhel" ] || [ "$OS" = "centos" ]; then + yum install -y epel-release + fi + fi + yum install -y clamav clamd clamav-update nc + SOCKET_FILE="/usr/lib/systemd/system/clamd@scan.socket" + CONFIG_FILE="/etc/clamd.d/scan.conf" + SERVICE_NAME="clamd@scan" + ;; + *) + echo "Error: Unsupported operating system: $OS" + exit 1 + ;; + esac + + echo "โœ“ ClamAV packages installed" + echo "" +} + +# Configure systemd socket for network listening +configure_socket() { + echo "Configuring systemd socket to listen on 127.0.0.1:3310..." + + if [ ! -f "$SOCKET_FILE" ]; then + echo "Error: Socket file not found: $SOCKET_FILE" + exit 1 + fi + + # Check if ListenStream is already configured + if grep -q "^ListenStream=127.0.0.1:3310" "$SOCKET_FILE"; then + echo "โœ“ Socket already configured for 127.0.0.1:3310" + else + # Add ListenStream after [Socket] section + sed -i '/\[Socket\]/a ListenStream=127.0.0.1:3310' "$SOCKET_FILE" + echo "โœ“ Added ListenStream=127.0.0.1:3310 to socket configuration" + fi + echo "" +} + +# Configure clamd.conf +configure_clamd() { + echo "Configuring ClamAV daemon..." + + if [ ! -f "$CONFIG_FILE" ]; then + echo "Error: Configuration file not found: $CONFIG_FILE" + exit 1 + fi + + # For Red Hat/CentOS, comment out Example line if present + if [ "$OS" = "rhel" ] || [ "$OS" = "centos" ] || [ "$OS" = "rocky" ] || [ "$OS" = "almalinux" ]; then + if grep -q "^Example" "$CONFIG_FILE"; then + sed -i 's/^Example/#Example/' "$CONFIG_FILE" + echo "โœ“ Disabled Example directive" + fi + fi + + # Ensure TCP socket settings are present + if ! grep -q "^TCPSocket" "$CONFIG_FILE"; then + echo "TCPSocket 3310" >> "$CONFIG_FILE" + echo "โœ“ Added TCPSocket 3310" + fi + + if ! grep -q "^TCPAddr" "$CONFIG_FILE"; then + echo "TCPAddr 127.0.0.1" >> "$CONFIG_FILE" + echo "โœ“ Added TCPAddr 127.0.0.1" + fi + + # Disable PUA detection + if grep -q "^DetectPUA yes" "$CONFIG_FILE"; then + sed -i 's/^DetectPUA yes/DetectPUA no/' "$CONFIG_FILE" + echo "โœ“ Disabled PUA detection (was enabled)" + elif ! grep -q "^DetectPUA" "$CONFIG_FILE"; then + echo "DetectPUA no" >> "$CONFIG_FILE" + echo "โœ“ Disabled PUA detection (added configuration)" + else + echo "โœ“ PUA detection already disabled" + fi + + # Enable debugging output for malware scanning + echo "Enabling ClamAV debugging output..." + + # Enable LogVerbose for detailed scanning information + if grep -q "^LogVerbose false" "$CONFIG_FILE"; then + sed -i 's/^LogVerbose false/LogVerbose true/' "$CONFIG_FILE" + echo "โœ“ Enabled LogVerbose (was disabled)" + elif grep -q "^LogVerbose true" "$CONFIG_FILE"; then + echo "โœ“ LogVerbose already enabled" + elif ! grep -q "^LogVerbose" "$CONFIG_FILE"; then + echo "LogVerbose true" >> "$CONFIG_FILE" + echo "โœ“ Enabled LogVerbose (added configuration)" + fi + + # Enable Debug for detailed daemon debugging + if grep -q "^Debug false" "$CONFIG_FILE"; then + sed -i 's/^Debug false/Debug true/' "$CONFIG_FILE" + echo "โœ“ Enabled Debug mode (was disabled)" + elif grep -q "^Debug true" "$CONFIG_FILE"; then + echo "โœ“ Debug mode already enabled" + elif ! grep -q "^Debug" "$CONFIG_FILE"; then + echo "Debug true" >> "$CONFIG_FILE" + echo "โœ“ Enabled Debug mode (added configuration)" + fi + + # Enable LogClean to log clean files (helpful for troubleshooting) + if grep -q "^LogClean false" "$CONFIG_FILE"; then + sed -i 's/^LogClean false/LogClean true/' "$CONFIG_FILE" + echo "โœ“ Enabled LogClean (was disabled)" + elif grep -q "^LogClean true" "$CONFIG_FILE"; then + echo "โœ“ LogClean already enabled" + elif ! grep -q "^LogClean" "$CONFIG_FILE"; then + echo "LogClean true" >> "$CONFIG_FILE" + echo "โœ“ Enabled LogClean (added configuration)" + fi + + echo "" +} + +# Update virus definitions +update_definitions() { + echo "Updating virus definitions..." + if command -v freshclam &> /dev/null; then + # Stop freshclam service if running to avoid conflicts + systemctl stop clamav-freshclam 2>/dev/null || systemctl stop clamav-freshclam.service 2>/dev/null || true + + freshclam || { + echo "Warning: freshclam update failed. Will retry after service start." + } + echo "โœ“ Virus definitions updated" + else + echo "Warning: freshclam not found, skipping virus definition update" + fi + echo "" +} + +# Reload and restart services +restart_services() { + echo "Reloading systemd and restarting ClamAV daemon..." + systemctl daemon-reload + systemctl enable "$SERVICE_NAME" 2>/dev/null || true + systemctl restart "$SERVICE_NAME" + + # Give the service a moment to start + sleep 2 + + if systemctl is-active --quiet "$SERVICE_NAME"; then + echo "โœ“ ClamAV daemon is running" + else + echo "Warning: ClamAV daemon may not be running properly" + systemctl status "$SERVICE_NAME" --no-pager + fi + + # Enable and start freshclam for automatic updates + if [ "$OS" = "debian" ] || [ "$OS" = "ubuntu" ]; then + systemctl enable clamav-freshclam 2>/dev/null || true + systemctl start clamav-freshclam 2>/dev/null || true + echo "โœ“ Enabled automatic virus definition updates" + fi + + echo "" +} + +# Test the installation +test_installation() { + echo "Testing ClamAV network scanner..." + + # Check if port is listening + if ss -tlnp 2>/dev/null | grep -q ":3310" || netstat -tlnp 2>/dev/null | grep -q ":3310"; then + echo "โœ“ Port 3310 is listening" + else + echo "Warning: Port 3310 does not appear to be listening" + fi + + # Send PING command + echo "Sending PING command to 127.0.0.1:3310..." + RESPONSE=$(echo "PING" | nc 127.0.0.1 3310 2>/dev/null || echo "FAILED") + + if echo "$RESPONSE" | grep -q "PONG"; then + echo "โœ“ Received PONG response" + echo "" + echo "==========================================" + echo "Installation completed successfully!" + echo "==========================================" + echo "" + echo "ClamAV is now listening on 127.0.0.1:3310" + + # Show configuration summary + echo "" + echo "Configuration Summary:" + echo " - DetectPUA: disabled (real malware only)" + echo " - TCP Socket: 127.0.0.1:3310" + echo " - LogVerbose: enabled (detailed scanning logs)" + echo " - Debug: enabled (daemon debugging)" + echo " - LogClean: enabled (logs clean files)" + if command -v clamconf &> /dev/null; then + VIRUS_COUNT=$(clamconf 2>/dev/null | grep "Total number of signatures:" | awk '{print $NF}') + if [ -n "$VIRUS_COUNT" ]; then + echo " - Virus Signatures: $VIRUS_COUNT" + fi + fi + echo "" + echo "Debug logs available at: /var/log/clamav/clamav.log" + + return 0 + else + echo "โœ— Failed to receive PONG response" + echo "Response: $RESPONSE" + echo "" + echo "Installation completed but service may not be responding correctly." + echo "Please check the logs: journalctl -u $SERVICE_NAME -n 50" + return 1 + fi +} + +# Main installation flow +main() { + install_clamav + configure_socket + configure_clamd + update_definitions + restart_services + test_installation +} + +# Run main installation +main diff --git a/malware-scan/jobs/README.md b/malware-scan/jobs/README.md new file mode 100644 index 0000000..a5c380e --- /dev/null +++ b/malware-scan/jobs/README.md @@ -0,0 +1,331 @@ +# Malware Scan Job Scripts + +This directory contains job scripts for the malware-scan plugin's hash database update functionality. + +--- + +## Overview + +BunkerWeb uses a job system for scheduled tasks. The malware-scan plugin provides hash database updates through: + +1. **Primary (Lua)**: `../malware_scan_scheduler.lua` - Core implementation +2. **Job Scripts** (this directory): + - `malware-scan-update.lua` - Lua job reference + - `malware-scan-update.py` - Python job (future use) + +--- + +## Architecture + +### Current (Lua-Based) + +``` +BunkerWeb Manager + โ†“ +malware-scan.lua + โ†“ +scheduler() hook + โ†“ +malware_scan_scheduler.lua + โ”œโ”€ init_on_startup() + โ””โ”€ update_hashes() + โ†“ +Redis (hash buckets) +``` + +### Future (Python Job) + +``` +BunkerWeb Job Scheduler + โ†“ +jobs/malware-scan-update.py + โ”œโ”€ Download with retry + โ”œโ”€ Extract ZIP + โ””โ”€ Import to Redis + โ†“ +Redis (hash buckets) +``` + +--- + +## Job Scripts + +### malware-scan-update.lua + +**Status**: Reference implementation + +A Lua job script demonstrating how the hash update could be structured as a standalone job. Currently not used - the Lua scheduler module in the main plugin is the primary mechanism. + +**Usage**: For future restructuring or alternative job frameworks. + +### malware-scan-update.py + +**Status**: Future-ready implementation + +A Python job script following the BunkerWeb pattern (similar to blacklist-download.py). Provides: + +- **Retry Logic**: 3 attempts with exponential backoff (3s, 6s, 9s) +- **Error Handling**: Graceful degradation on network failures +- **State Persistence**: ETag + file size for change detection +- **Redis Bucketing**: Distributes 1M+ hashes across 16 buckets +- **Logging**: Detailed progress and error reporting +- **Cleanup**: Automatic removal of temporary files + +**When to Use**: +- Large-scale deployments (100+ instances) +- If you prefer Python over Lua +- When you need centralized job scheduling +- For environments where the Lua scheduler is inadequate + +**Requirements**: +- Python 3.6+ +- `requests` module +- `redis` module +- BunkerWeb utilities + +**Configuration**: Uses same environment variables as Lua scheduler +```env +USE_MALWARE-SCAN=yes +MALWARE_SCAN_CUSTOM_HASHES_ENABLED=yes +MALWARE_SCAN_CUSTOM_HASHES_AUTO_UPDATE=yes +MALWARE_SCAN_CUSTOM_HASHES_UPDATE_URL=... +MALWARE_SCAN_CUSTOM_HASHES_CSV_PATH=... +``` + +**Exit Codes**: +- `0`: Success, no changes +- `1`: Success, hashes added +- `2`: Error occurred + +--- + +## Migration Guide + +### Current Setup (Recommended) + +Uses Lua scheduler in main plugin: + +```bash +# In malware-scan.lua +function malware_scan:scheduler() + local init_ok = deps.modules.ms_scheduler.init_on_startup(self) + local update_ok = deps.modules.ms_scheduler.update_hashes(self) + return self:ret(true, "success") +end +``` + +**Pros**: +- Single Lua ecosystem +- No Python dependency +- No separate job configuration +- Integrated with plugin lifecycle + +**Cons**: +- Less isolation from main plugin +- Scheduler runs in worker processes +- No centralized job management + +### Future: Python Job (Optional) + +For large deployments, you could switch to Python: + +```lua +-- In malware-scan.lua scheduler() +-- Call Python job instead of Lua scheduler +os.execute("/path/to/jobs/malware-scan-update.py") +``` + +**Pros**: +- Better separation of concerns +- Can be run independently +- More debugging flexibility +- Matches BunkerWeb blacklist pattern + +**Cons**: +- Adds Python dependency +- Separate job configuration +- Less integrated + +--- + +## Testing the Python Job + +### Dry Run + +```bash +# Test without actually importing +cd /opt/bunkerweb/plugins/malware-scan +python3 jobs/malware-scan-update.py +``` + +### Debug Mode + +```bash +# Enable debug logging +DEBUG=yes python3 jobs/malware-scan-update.py +``` + +### With Custom Config + +```bash +# Override environment variables +USE_MALWARE-SCAN=yes \ +MALWARE_SCAN_CUSTOM_HASHES_ENABLED=yes \ +MALWARE_SCAN_CUSTOM_HASHES_AUTO_UPDATE=yes \ +REDIS_HOST=localhost \ +REDIS_PORT=6379 \ +python3 jobs/malware-scan-update.py +``` + +### Verify Results + +```bash +# Check hashes imported +redis-cli hlen malware_scan:hashes:0 +redis-cli hlen malware_scan:hashes:A +redis-cli hlen malware_scan:hashes:F + +# Check state file +cat /var/cache/bunkerweb/malware-scan/hashes.state +``` + +--- + +## Performance Comparison + +### Lua Scheduler (Current) + +``` +| Metric | Value | +|--------|-------| +| Startup | ~1ms (compiled in) | +| Download | 10-30s (depends on network) | +| Extract + Import | 10-20s | +| Total | 20-50s | +| Memory | ~200MB (temporary) | +| CPU | Single thread | +``` + +### Python Job (Future) + +``` +| Metric | Value | +|--------|-------| +| Startup | ~100ms (interpreter) | +| Download | 10-30s (same network) | +| Extract + Import | 10-20s (same algorithm) | +| Total | 20-50s (same) | +| Memory | ~200MB (temporary) | +| CPU | Single thread | +``` + +**Conclusion**: Same performance, different approach. + +--- + +## BunkerWeb Pattern + +This structure follows the BunkerWeb convention: + +``` +plugin-name/ + plugin-name.lua โ† Main plugin logic + helper_modules.lua โ† Shared functions + jobs/ โ† Job scripts directory + plugin-name-job.py โ† Python job (if used) + plugin-name-job.lua โ† Lua job (reference) +``` + +**Examples in BunkerWeb**: +``` +core_plugins/ + blacklist/ + blacklist.lua + jobs/ + blacklist-download.py + + bans/ + bans.lua + jobs/ + bans-save.py +``` + +--- + +## Future Enhancements + +### 1. Centralized Job Scheduler + +Instead of scheduler hook in plugin, use BunkerWeb Jobs system: + +```python +# jobs/malware-scan-update.py +class MalwareScanUpdateJob(Job): + def execute(self): + # Update logic + return status +``` + +### 2. Multiple Source Support + +Download from multiple mirrors and merge: + +```python +SOURCES = [ + "https://bazaar.mirror.sysangels.ai/...", + "https://bazaar.abuse.ch/...", + "file:///local/hashes.csv", +] + +for source in SOURCES: + download_and_import(source) +``` + +### 3. Partial Import Recovery + +Resume import if interrupted: + +```python +if state.get("import_progress"): + # Skip already imported lines + start_line = int(state["import_progress"]) +else: + start_line = 0 +``` + +### 4. Rollback Support + +Keep previous version for rollback: + +```python +# Before import +backup_hashes() + +# On import failure +restore_hashes() +``` + +--- + +## References + +- **BunkerWeb Blacklist**: `/core_plugins/blacklist/jobs/blacklist-download.py` +- **BunkerWeb Jobs**: `/src/scheduler/jobs.py` +- **Malware-Scan Scheduler**: `../malware_scan_scheduler.lua` + +--- + +## Support + +For issues with the job scripts: + +1. Check logs: `/var/log/bunkerweb/access.log` +2. Verify Redis: `redis-cli ping` +3. Verify network: `curl -I https://bazaar.mirror.sysangels.ai/...` +4. Check hashes: `redis-cli hlen malware_scan:hashes:0` + +--- + +**Version**: 0.7.44 +**Date**: 2026-01-27 +**Status**: Production Ready (Lua), Future-Ready (Python) diff --git a/malware-scan/jobs/malware-scan-cache-cleanup.py b/malware-scan/jobs/malware-scan-cache-cleanup.py new file mode 100644 index 0000000..fe93597 --- /dev/null +++ b/malware-scan/jobs/malware-scan-cache-cleanup.py @@ -0,0 +1,334 @@ +#!/usr/bin/env python3 + +""" +BunkerWeb Malware-Scan Cache Cleanup Job + +Purpose: + Explicit cache cleanup job that runs once at manager startup. + Removes stale cache entries from Redis cache. + Runs BEFORE hash download/update jobs to ensure clean state. + +Usage: + Called automatically by BunkerWeb scheduler as "malware-scan-cache-cleanup" job. + Configured to run once at startup with "every": "once". + +Job Configuration: + Name: malware-scan-cache-cleanup + Trigger: once (at startup) + Reload: false + Async: true + +Behavior: + When USE_MALWARE_SCANNER != "yes" (disabled): + - Force cleanup of ALL caches to free Redis memory + - Deletes hash databases (32 keys), metadata/tracking (42 keys), scanner caches (7 patterns) + - Total: 74 fixed keys + variable scanner cache keys + - Ignores individual cleanup flags + When USE_MALWARE_SCANNER == "yes" (enabled): + - Check cleanup flags to determine what to clean + - Only delete caches for enabled cleanup flags + + Cleanup Process: + 1. Check scanner status and cleanup flags + 2. Connect to Redis and scan for matching patterns + 3. Delete keys matching enabled cache patterns + 4. Log cleanup statistics + +Environment: + USE_MALWARE_SCANNER: Controls cleanup behavior + - "yes" = Respect cleanup flags (selective cleanup) + - != "yes" = Force full cleanup (free Redis memory) + MALWARE_SCAN_USE_SHARED_DATABASE: Must be "yes" for Redis cleanup + MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_ALL: Optional, default "no" (only when scanner enabled) + MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_*: Individual cleanup flags (only when scanner enabled) + +Returns: + 0 if no cleanup needed or cleanup completed successfully + 1 if cleanup was performed (with changes) + 2 if error occurred +""" + +from sys import exit as sys_exit, path as sys_path +from os import getenv, sep +from os.path import join +from logging import getLogger, StreamHandler, Formatter, DEBUG, INFO +import traceback +import time + +# Add BunkerWeb dependencies to path +for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("db",))]: + if deps_path not in sys_path: + sys_path.append(deps_path) + +# Setup logging +LOGGER = getLogger("MALWARE-SCAN-CLEANUP") +handler = StreamHandler() +formatter = Formatter("[MALWARE-SCAN-CLEANUP] [%(levelname)s] - %(message)s") +handler.setFormatter(formatter) +LOGGER.addHandler(handler) +LOGGER.setLevel(DEBUG if getenv("LOG_LEVEL") == "debug" else INFO) + +try: + # Add startup delay to prevent overlapping with update job during scheduler restart + LOGGER.info("Waiting 11 seconds before starting cache cleanup to prevent job overlap...") + time.sleep(11) + LOGGER.info("Starting cache cleanup job") + + # Check if malware scanning is enabled + scanner_enabled = getenv("USE_MALWARE_SCANNER", "no") == "yes" + + if not scanner_enabled: + # Scanner disabled - force cleanup of ALL caches to free Redis memory + LOGGER.info("Malware scan disabled, forcing cleanup to free Redis memory") + cleanup_all = True + cleanup_clamav = True + cleanup_virustotal = True + cleanup_sentinelone = True + cleanup_malwarebazaar = True + cleanup_attacker_ipv4 = True + cleanup_attacker_ipv6 = True + cleanup_enabled = True + else: + # Scanner enabled - check cleanup flags + cleanup_all = getenv("MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_ALL", "no") == "yes" + cleanup_clamav = getenv("MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_CLAMAV", "no") == "yes" + cleanup_virustotal = getenv("MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_VIRUSTOTAL", "no") == "yes" + cleanup_sentinelone = getenv("MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_SENTINELONE", "no") == "yes" + cleanup_malwarebazaar = getenv("MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_MALWAREBAZAAR", "no") == "yes" + cleanup_attacker_ipv4 = getenv("MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_ATTACKER_IPV4", "no") == "yes" + cleanup_attacker_ipv6 = getenv("MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_ATTACKER_IPV6", "no") == "yes" + + cleanup_enabled = ( + cleanup_all + or cleanup_clamav + or cleanup_virustotal + or cleanup_sentinelone + or cleanup_malwarebazaar + or cleanup_attacker_ipv4 + or cleanup_attacker_ipv6 + ) + + if not cleanup_enabled: + LOGGER.info("Malware scan enabled but no cleanup flags set, skipping cleanup") + sys_exit(0) + + # Try to import Redis + try: + import redis + except ImportError: + LOGGER.error("Redis module not available") + sys_exit(2) + + # Get Redis connection configuration + try: + sentinel_hosts_str = getenv("REDIS_SENTINEL_HOSTS", "").strip() + if sentinel_hosts_str: + # Redis Sentinel configuration + sentinel_master = getenv("REDIS_SENTINEL_MASTER", "bw-master").strip() + sentinel_username = getenv("REDIS_SENTINEL_USERNAME", "").strip() or None + sentinel_password = getenv("REDIS_SENTINEL_PASSWORD", "").strip() or None + redis_db_str = getenv("REDIS_DATABASE", "").strip() or "0" + redis_username = getenv("REDIS_USERNAME", "").strip() or None + redis_password = getenv("REDIS_PASSWORD", "").strip() or None + redis_ssl = getenv("REDIS_SSL", "no").strip() == "yes" + + try: + redis_db = int(redis_db_str) + except ValueError: + redis_db = 0 + + # Parse sentinel hosts: convert port strings to integers + # Format: "host1:port1 host2:port2 ..." (space-separated, not comma-separated) + sentinel_hosts = [] + for h in sentinel_hosts_str.split(): + h = h.strip() + if ":" in h: + host, port = h.rsplit(":", 1) + sentinel_hosts.append((host.strip(), int(port.strip()))) + else: + sentinel_hosts.append((h, 26379)) # Default sentinel port + + sentinel = redis.Sentinel( + sentinel_hosts, + username=sentinel_username, + password=sentinel_password, + ssl=redis_ssl, + socket_connect_timeout=10, + socket_timeout=10, + ) + redis_conn = sentinel.master_for( + sentinel_master, + socket_connect_timeout=10, + socket_timeout=10, + db=redis_db, + username=redis_username, + password=redis_password, + ssl=redis_ssl, + decode_responses=True, + ) + else: + # Direct Redis configuration + redis_host = getenv("REDIS_HOST", "localhost").strip() + redis_port = int(getenv("REDIS_PORT", "6379").strip()) + redis_db_str = getenv("REDIS_DATABASE", "").strip() or "0" + redis_username = getenv("REDIS_USERNAME", "").strip() or None + redis_password = getenv("REDIS_PASSWORD", "").strip() or None + redis_ssl = getenv("REDIS_SSL", "no").strip() == "yes" + + try: + redis_db = int(redis_db_str) + except ValueError: + redis_db = 0 + + redis_conn = redis.Redis( + host=redis_host, + port=redis_port, + db=redis_db, + username=redis_username, + password=redis_password, + ssl=redis_ssl, + decode_responses=True, + socket_connect_timeout=10, + socket_timeout=10, + ) + + # Test connection + redis_conn.ping() + + # Log connection success + if sentinel_hosts_str: + LOGGER.info(f"Connected to Redis Sentinel master '{sentinel_master}' via sentinels {sentinel_hosts} db={redis_db}") + else: + LOGGER.info(f"Connected to Redis at {redis_host}:{redis_port} db={redis_db}") + + except Exception as e: + LOGGER.error(f"Failed to connect to Redis: {e}") + sys_exit(2) + + # Build pattern list based on enabled flags + patterns_to_delete = [] + + if cleanup_all or cleanup_clamav: + patterns_to_delete.append("plugin_malware_scan_clamav_*") + if cleanup_all or cleanup_virustotal: + patterns_to_delete.append("plugin_malware_scan_virustotal_*") + if cleanup_all or cleanup_sentinelone: + patterns_to_delete.append("plugin_malware_scan_sentinelone_*") + if cleanup_all or cleanup_malwarebazaar: + patterns_to_delete.append("plugin_malware_scan_malwarebazaar_*") + if cleanup_all or cleanup_attacker_ipv4: + patterns_to_delete.append("plugin_malware_scan_attacker_ipv4_*") + if cleanup_all or cleanup_attacker_ipv6: + patterns_to_delete.append("plugin_malware_scan_attacker_ipv6_*") + + # Always clean composite results cache if any scanner is being cleaned + if patterns_to_delete: + patterns_to_delete.append("plugin_malware_scan_results_*") + + # When scanner is disabled, also clean MalwareBazaar hash databases (16 bucketed keys) + if not scanner_enabled: + # Add full hash table buckets: malware_scan:hashes:0 through F + for i in range(16): + patterns_to_delete.append(f"malware_scan:hashes:{i:X}") + # Add recent cache buckets: malware_scan:hashes:recent:0 through F + for i in range(16): + patterns_to_delete.append(f"malware_scan:hashes:recent:{i:X}") + + # Add metadata and tracking keys (exact keys, not patterns) + # State keys + patterns_to_delete.append("malware-scan:hashes:state:full") + patterns_to_delete.append("malware-scan:hashes:state:recent") + # Import tracking + patterns_to_delete.append("malware-scan:hashes:import_state") + patterns_to_delete.append("malware-scan:hashes:import_lock") + patterns_to_delete.append("malware-scan:hashes:import_start_time:full") + patterns_to_delete.append("malware-scan:hashes:import_start_time:recent") + patterns_to_delete.append("malware-scan:hashes:import_end_time:full") + patterns_to_delete.append("malware-scan:hashes:import_end_time:recent") + # Progress tracking + patterns_to_delete.append("malware-scan:hashes:import_total:full") + patterns_to_delete.append("malware-scan:hashes:import_total:recent") + # Per-bucket progress counters (32 total: 16 for full, 16 for recent) + for i in range(16): + patterns_to_delete.append(f"malware-scan:hashes:import_processed:full:bucket_{i:X}") + patterns_to_delete.append(f"malware-scan:hashes:import_processed:recent:bucket_{i:X}") + + if not patterns_to_delete: + LOGGER.info("No patterns to delete") + sys_exit(0) + + # Separate exact keys from patterns for efficient deletion + exact_keys = [] + pattern_keys = [] + + for item in patterns_to_delete: + # Patterns have wildcards (*), exact keys don't + if "*" in item: + pattern_keys.append(item) + else: + exact_keys.append(item) + + LOGGER.info(f"Starting Redis cache cleanup - {len(pattern_keys)} patterns + {len(exact_keys)} exact keys") + + # Delete keys matching patterns + total_deleted = 0 + pattern_details = {} + + # Handle pattern-based deletion (SCAN + UNLINK) + for pattern in pattern_keys: + pattern_deleted = 0 + cursor = 0 + safe_prefixes = ["plugin_malware_scan_", "malware_scan:hashes:", "malware-scan:hashes:"] + + # Validate pattern prefix for safety + if not any(pattern.startswith(prefix) for prefix in safe_prefixes): + LOGGER.error(f"Safety error: Pattern '{pattern}' does not start with expected prefixes {safe_prefixes}") + sys_exit(2) + + try: + while True: + cursor, keys = redis_conn.scan(cursor, match=pattern, count=100) + if keys: + pattern_deleted += redis_conn.unlink(*keys) + if cursor == 0: + break + + pattern_details[pattern] = pattern_deleted + total_deleted += pattern_deleted + if pattern_deleted > 0: + LOGGER.info(f"Deleted {pattern_deleted} keys matching pattern '{pattern}'") + + except Exception as e: + LOGGER.error(f"Error deleting pattern '{pattern}': {e}") + sys_exit(2) + + # Handle exact key deletion (direct UNLINK) + if exact_keys: + # Validate all exact keys for safety + safe_prefixes = ["plugin_malware_scan_", "malware_scan:hashes:", "malware-scan:hashes:"] + for key in exact_keys: + if not any(key.startswith(prefix) for prefix in safe_prefixes): + LOGGER.error(f"Safety error: Key '{key}' does not start with expected prefixes {safe_prefixes}") + sys_exit(2) + + try: + # UNLINK multiple exact keys at once (non-blocking batch delete) + exact_deleted = redis_conn.unlink(*exact_keys) + total_deleted += exact_deleted + if exact_deleted > 0: + LOGGER.info(f"Deleted {exact_deleted} exact keys (metadata and tracking)") + pattern_details["exact_keys"] = exact_deleted + except Exception as e: + LOGGER.error(f"Error deleting exact keys: {e}") + sys_exit(2) + + LOGGER.info(f"โœ… Cache cleanup completed - Total keys deleted: {total_deleted}") + for pattern, count in pattern_details.items(): + LOGGER.debug(f" {pattern}: {count} keys") + + sys_exit(1 if total_deleted > 0 else 0) + +except Exception as e: + LOGGER.error(f"Cache cleanup job failed with exception: {e}") + LOGGER.error(traceback.format_exc()) + sys_exit(2) diff --git a/malware-scan/jobs/malware-scan-update.lua b/malware-scan/jobs/malware-scan-update.lua new file mode 100644 index 0000000..198cc88 --- /dev/null +++ b/malware-scan/jobs/malware-scan-update.lua @@ -0,0 +1,96 @@ +#!/usr/bin/env lua +-- BunkerWeb Malware Scan - Update Job +-- Scheduled task for periodic MalwareBazaar hash database updates +-- +-- This job is called by the scheduler hook at configured intervals +-- and handles downloading and importing malware hashes into Redis +-- +-- Features: +-- - Scheduled downloads with intelligent change detection (ETag + filesize) +-- - Configurable update intervals (seconds precision) +-- - State persistence for bandwidth optimization (99.9% savings) +-- - Atomic database updates with bucketed key distribution (16 buckets) +-- - Infinite persistence (no TTL - freshness maintained by scheduler) +-- - Support for three strategies: full, recent, hybrid +-- - Automatic retry on network failures +-- - Graceful error handling +-- +-- Version: 0.8.0 +-- Date: 2026-01-27 + +local _M = {} + +-- ============================================================================ +-- LOGGING HELPERS (works without ngx in standalone job context) +-- ============================================================================ + +local function log_info(msg) + print("[MALWARE-SCAN] " .. msg) +end + +local function log_error(msg) + io.stderr:write("[MALWARE-SCAN] ERROR: " .. msg .. "\n") +end + +local function log_warn(msg) + print("[MALWARE-SCAN] WARNING: " .. msg) +end + +-- ============================================================================ +-- SCHEDULER JOB DEFINITION +-- ============================================================================ + +-- Job metadata for BunkerWeb scheduler +_M.metadata = { + id = "malware-scan-update", + name = "Malware Scan Hash Update", + description = "Download and import malware hash database from MalwareBazaar", + every = "10m", -- Run every 10 minutes + reload = true, -- Reload nginx after update +} + +-- Job execution function called by BunkerWeb scheduler +--- @param logger table BunkerWeb logger instance +--- @param env table Environment/context object +--- @return number status Status code (0=success, 1=changed, 2=error) +function _M.execute(logger, env) + if not logger then + -- Fallback if no logger provided + logger = { + log = function(_, level, msg) + if level == "ERROR" then + log_error(msg) + elseif level == "WARN" then + log_warn(msg) + else + log_info(msg) + end + end + } + end + + logger:log("INFO", "[SCHEDULER] Starting MalwareBazaar hash update job") + + -- This would normally call the scheduler module + -- For now, return success status + return 0 +end + +-- ============================================================================ +-- METADATA FOR JOB FRAMEWORK +-- ============================================================================ + +-- BunkerWeb scheduler expects these to be available +function _M.get_metadata() + return _M.metadata +end + +function _M.get_schedule() + return _M.metadata.every +end + +-- ============================================================================ +-- MODULE RETURN +-- ============================================================================ + +return _M diff --git a/malware-scan/jobs/malware-scan-update.py b/malware-scan/jobs/malware-scan-update.py new file mode 100755 index 0000000..8537ce3 --- /dev/null +++ b/malware-scan/jobs/malware-scan-update.py @@ -0,0 +1,1569 @@ +#!/usr/bin/env python3 +""" +BunkerWeb Malware Scan - Hash Update Job + +Scheduled task for periodic MalwareBazaar hash database updates. +This script is called by the BunkerWeb scheduler at configured intervals +and handles downloading, extracting, and importing malware hashes into Redis. + +Features: +- Time-based update window (22-hour threshold) with success verification (retries immediately if last import failed) +- Intelligent change detection using ETag + file size +- Automatic retry with exponential backoff (3 retries) +- State persistence for bandwidth optimization (99.9% savings) +- Dual hash tables: malware_scan:hashes (full) and malware_scan:hashes:recent (recent) +- Bucketed key distribution (16 buckets per table for parallel import) +- Parallel import with ThreadPoolExecutor (3-6x faster) +- 10-second progress indicators with rate tracking +- Full table: infinite persistence (no TTL) +- Recent table: automatic 48-hour TTL (auto-expires old entries) +- Automatic processing of both full and recent files (scheduler controls frequency) +- Graceful error handling (continues on failures) +- SSL verification disabled for testing (temporary workaround) +- Automatic Redis connection management via Job class + +Version: 0.7.44 +Date: 2026-01-27 +""" + +from contextlib import suppress +from datetime import datetime, timezone +from os import getenv, sep +from os.path import join +from sys import exit as sys_exit, path as sys_path +from traceback import format_exc +from typing import Tuple, Optional +import os +import time +import zipfile +import tempfile +import shutil +import threading +import logging +import hashlib +from concurrent.futures import ThreadPoolExecutor, as_completed + +# ============================================================================ +# EARLY LOGGER SETUP - Set up logging before any imports that might fail +# ============================================================================ + +# Check debug mode early +_is_debug = getenv("MALWARE_SCAN_DEBUG", "no") == "yes" + +# Create basic Python logger that writes to stderr (goes to scheduler.log via Docker) +_early_logger = logging.getLogger("MALWARE-SCAN") +_early_logger.setLevel(logging.DEBUG if _is_debug else logging.INFO) +_handler = logging.StreamHandler() # Writes to stderr by default +_handler.setFormatter(logging.Formatter( + '[%(asctime)s] [%(name)s] [%(process)d] - %(levelname)s - %(message)s', + datefmt='%Y-%m-%d %H:%M:%S' +)) +_early_logger.addHandler(_handler) + +# Add BunkerWeb dependencies to path +for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("db",))]: + if deps_path not in sys_path: + sys_path.append(deps_path) + +try: + from requests import get, head + from requests.exceptions import ConnectionError, Timeout +except ImportError: + _early_logger.error("requests module not available") + sys_exit(2) + +try: + import redis +except ImportError: + _early_logger.error("redis module not available") + sys_exit(2) + +try: + from logger import getLogger # type: ignore + from jobs import Job # type: ignore + # Use BunkerWeb logger + LOGGER = getLogger("MALWARE-SCAN") + _early_logger.info("Using BunkerWeb logger") +except ImportError: + # Fallback: use the early logger we already set up + _early_logger.warning("BunkerWeb logger not available, using fallback logger") + LOGGER = _early_logger + + # Fallback Job class that doesn't require BunkerWeb + class Job: + def __init__(self, logger, filepath): + self.logger = logger + + def cache_file(self, key, data): + """Fallback cache_file - returns (False, error_message) as caching not available""" + return False, "Caching not available in fallback mode" + + def get_cache(self, key): + """Fallback get_cache - returns None as caching not available""" + return None + + +VERSION = "0.8.0" +status = 0 + +# Debug mode detection +DEBUG_MODE = getenv("MALWARE_SCAN_DEBUG", "no") == "yes" + +# Log script information +SCRIPT_PATH = os.path.abspath(__file__) +SCRIPT_NAME = os.path.basename(__file__) +LOGGER.info(f"[v{VERSION}] Script: {SCRIPT_NAME}") +LOGGER.info(f"[v{VERSION}] Path: {SCRIPT_PATH}") +if DEBUG_MODE: + LOGGER.info(f"[v{VERSION}] DEBUG MODE ENABLED") + LOGGER.debug(f"[v{VERSION}] Working directory: {os.getcwd()}") + LOGGER.debug(f"[v{VERSION}] Python executable: {os.sys.executable}") + LOGGER.debug(f"[v{VERSION}] Python version: {os.sys.version}") + +# Initialize Job instance for Redis access +try: + JOB = Job(LOGGER, __file__) +except Exception as e: + LOGGER.error(f"[v{VERSION}] Failed to initialize Job: {e}") + sys_exit(2) + +# ============================================================================ +# CHECKSUM VALIDATION (Security Enhancement) +# ============================================================================ + +def calculate_sha256(file_path: str, buffer_size: int = 65536) -> Optional[str]: + """ + Calculate SHA256 checksum of a file. + + Uses buffered reading to handle large files efficiently. + Prevents tampering detection and ensures download integrity. + + Args: + file_path: Path to the file to checksum + buffer_size: Buffer size for reading (default: 64KB) + + Returns: + Hexadecimal SHA256 checksum string, or None on error + """ + try: + sha256_hash = hashlib.sha256() + with open(file_path, "rb") as f: + while True: + data = f.read(buffer_size) + if not data: + break + sha256_hash.update(data) + return sha256_hash.hexdigest() + except Exception as e: + LOGGER.error(f"[v{VERSION}] Failed to calculate SHA256 for {file_path}: {e}") + return None + +def verify_file_integrity(file_path: str, expected_size: Optional[int] = None) -> Tuple[bool, Optional[str], Optional[int]]: + """ + Verify file integrity through size check and checksum calculation. + + Provides defense against: + - Truncated downloads + - Corrupted files + - Tampering during transfer + + Args: + file_path: Path to the file to verify + expected_size: Expected file size in bytes (optional) + + Returns: + Tuple of (is_valid, sha256_checksum, actual_size) + """ + try: + if not os.path.exists(file_path): + LOGGER.error(f"[v{VERSION}] File does not exist: {file_path}") + return False, None, None + + actual_size = os.path.getsize(file_path) + + # Size validation if expected size provided + if expected_size is not None and actual_size != expected_size: + LOGGER.error(f"[v{VERSION}] File size mismatch: expected {expected_size} bytes, got {actual_size} bytes") + return False, None, actual_size + + # Calculate checksum + checksum = calculate_sha256(file_path) + if not checksum: + return False, None, actual_size + + return True, checksum, actual_size + + except Exception as e: + LOGGER.error(f"[v{VERSION}] File integrity check failed: {e}") + return False, None, None + +try: + # Log environment variables in debug mode + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] Environment variables:") + for key in sorted(os.environ.keys()): + if key.startswith("MALWARE_SCAN_") or key.startswith("REDIS_"): + value = os.environ[key] + # Mask sensitive values (passwords, API keys, webhooks, secrets) + sensitive_keywords = ["PASSWORD", "SECRET", "API_KEY", "TOKEN", "WEBHOOK"] + if any(keyword in key for keyword in sensitive_keywords): + if value: + # Show partial value for debugging (first 10 chars + "...") + if len(value) > 10: + value = value[:10] + "...***MASKED***" + else: + value = "***MASKED***" + else: + value = "(empty)" + LOGGER.debug(f"[v{VERSION}] {key}={value}") + + # Add startup delay to prevent overlapping with cleanup job during scheduler restart + LOGGER.info(f"[v{VERSION}] Waiting 20 seconds before starting update job to prevent job overlap...") + time.sleep(20) + LOGGER.info(f"[v{VERSION}] Starting update job") + + # Check if enabled + if getenv("USE_MALWARE_SCANNER", "no") != "yes": + LOGGER.info(f"[v{VERSION}] Malware scan disabled, skipping update") + sys_exit(0) + + if getenv("MALWARE_SCAN_MALWAREBAZAAR_HASH_LOOKUP", "yes") != "yes": + LOGGER.info(f"[v{VERSION}] MalwareBazaar hash lookup disabled, skipping update") + sys_exit(0) + + if getenv("MALWARE_SCAN_MALWAREBAZAAR_AUTO_UPDATE", "yes") != "yes": + LOGGER.info(f"[v{VERSION}] Auto-update disabled, skipping update") + sys_exit(0) + + # Get Redis connection using BunkerWeb-configured environment variables + try: + # Check if using Redis Sentinel + sentinel_hosts_str = getenv("REDIS_SENTINEL_HOSTS", "").strip() + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] REDIS_SENTINEL_HOSTS={repr(sentinel_hosts_str)}") + + if sentinel_hosts_str: + # Redis Sentinel configuration + sentinel_master = getenv("REDIS_SENTINEL_MASTER", "bw-master").strip() + sentinel_username = getenv("REDIS_SENTINEL_USERNAME", "").strip() or None + sentinel_password = getenv("REDIS_SENTINEL_PASSWORD", "").strip() or None + redis_db_str = getenv("REDIS_DATABASE", "").strip() or "0" + redis_username = getenv("REDIS_USERNAME", "").strip() or None + redis_password = getenv("REDIS_PASSWORD", "").strip() or None + redis_ssl = getenv("REDIS_SSL", "no").strip() == "yes" + + try: + redis_db = int(redis_db_str) + except ValueError as e: + LOGGER.error(f"[v{VERSION}] Invalid Redis database: {e}") + sys_exit(2) + + # Parse sentinel hosts (format: "host1:port1 host2:port2 ...") + parsed_hosts = [] + for host_spec in sentinel_hosts_str.split(): + if ":" in host_spec: + h, p = host_spec.rsplit(":", 1) + try: + parsed_hosts.append((h, int(p))) + except ValueError: + LOGGER.warning(f"[v{VERSION}] Invalid sentinel host spec: {host_spec}") + else: + parsed_hosts.append((host_spec, 26379)) + + if not parsed_hosts: + LOGGER.error(f"[v{VERSION}] Failed to parse Redis Sentinel hosts") + sys_exit(2) + + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] Using Redis Sentinel: master={sentinel_master}, hosts={parsed_hosts}") + LOGGER.debug(f"[v{VERSION}] Sentinel config: username={sentinel_username is not None}, password={sentinel_password is not None}, ssl={redis_ssl}") + LOGGER.debug(f"[v{VERSION}] Redis config: db={redis_db}, username={redis_username is not None}, password={redis_password is not None}, ssl={redis_ssl}") + + try: + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] Creating Sentinel instance...") + sentinel = redis.sentinel.Sentinel( + parsed_hosts, + username=sentinel_username, + password=sentinel_password, + sentinel_kwargs=dict(ssl=redis_ssl), + socket_connect_timeout=5, + socket_timeout=5 + ) + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] Sentinel instance created, getting connection...") + redis_conn = sentinel.master_for( + sentinel_master, + db=redis_db, + username=redis_username, + password=redis_password, + ssl=redis_ssl + ) + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] Got connection from sentinel.master_for") + except Exception as e: + LOGGER.error(f"[v{VERSION}] Sentinel connection error: {type(e).__name__}: {e}") + raise + else: + # Standard Redis configuration + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] REDIS_HOST={repr(getenv('REDIS_HOST', ''))}") + LOGGER.debug(f"[v{VERSION}] REDIS_PORT={repr(getenv('REDIS_PORT', ''))}") + LOGGER.debug(f"[v{VERSION}] REDIS_DATABASE={repr(getenv('REDIS_DATABASE', ''))}") + + redis_host = getenv("REDIS_HOST", "").strip() or "127.0.0.1" + redis_port_str = getenv("REDIS_PORT", "").strip() or "6379" + redis_db_str = getenv("REDIS_DATABASE", "").strip() or "0" + redis_ssl = getenv("REDIS_SSL", "no").strip() == "yes" + redis_username = getenv("REDIS_USERNAME", "").strip() or None + redis_password = getenv("REDIS_PASSWORD", "").strip() or None + + try: + redis_port = int(redis_port_str) + redis_db = int(redis_db_str) + except ValueError as e: + LOGGER.error(f"[v{VERSION}] Invalid Redis configuration: {e}") + sys_exit(2) + + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] Using standard Redis: host={redis_host}, port={redis_port}, db={redis_db}") + LOGGER.debug(f"[v{VERSION}] Redis config: username={redis_username is not None}, password={redis_password is not None}, ssl={redis_ssl}") + + try: + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] Creating Redis connection...") + redis_conn = redis.Redis( + host=redis_host, + port=redis_port, + db=redis_db, + username=redis_username, + password=redis_password, + ssl=redis_ssl, + decode_responses=False, + socket_connect_timeout=5, + socket_timeout=5 + ) + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] Redis connection created") + except Exception as e: + LOGGER.error(f"[v{VERSION}] Redis creation error: {type(e).__name__}: {e}") + raise + + try: + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] Pinging Redis...") + redis_conn.ping() + LOGGER.info(f"[v{VERSION}] Redis connected successfully") + except Exception as ping_err: + LOGGER.error(f"[v{VERSION}] Redis ping failed: {type(ping_err).__name__}: {ping_err}") + raise + except Exception as e: + LOGGER.error(f"[v{VERSION}] Redis connection failed: {type(e).__name__}: {e}") + if DEBUG_MODE: + import traceback + LOGGER.debug(f"[v{VERSION}] Traceback: {traceback.format_exc()}") + sys_exit(2) + + # ============================================================================ + # PERSISTENT TEMP DIRECTORY MANAGEMENT + # ============================================================================ + + def get_or_create_temp_dir(redis_conn): + """ + Get or create persistent temp directory for malware-scan downloads. + + Stores path in Redis to survive container restarts. If /var/cache/bunkerweb + gets cleaned up, this function ensures we use a persistent /tmp directory. + + Returns: + str: Path to the persistent temp directory + """ + temp_dir_key = "malware_scan:scheduler:temp_dir" + directory_missing = False + old_dir_path = None + + try: + # Try to get existing temp dir from Redis + existing_dir = redis_conn.get(temp_dir_key) + if existing_dir: + existing_dir = existing_dir.decode() + # Verify directory still exists + if os.path.isdir(existing_dir): + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] Using existing temp directory from Redis: {existing_dir}") + # Refresh TTL to 24 hours + redis_conn.expire(temp_dir_key, 86400) + return existing_dir + else: + # Directory from Redis no longer exists (likely after reboot) + directory_missing = True + old_dir_path = existing_dir + LOGGER.warning(f"[v{VERSION}] Temp directory from Redis no longer exists (likely after reboot): {existing_dir}") + # Delete stale Redis key + redis_conn.delete(temp_dir_key) + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] Deleted stale temp directory key from Redis") + except Exception as e: + LOGGER.warning(f"[v{VERSION}] Failed to get temp directory from Redis: {e}") + + # Create new temp directory in /tmp + try: + temp_dir = tempfile.mkdtemp(prefix='malware-scan-', dir='/tmp') + if directory_missing: + LOGGER.info(f"[v{VERSION}] โ™ป๏ธ Created new temp directory after reboot: {temp_dir} (old: {old_dir_path})") + else: + LOGGER.info(f"[v{VERSION}] Created new persistent temp directory: {temp_dir}") + + # Store in Redis with 24-hour TTL + redis_conn.set(temp_dir_key, temp_dir, ex=86400) + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] Stored temp directory path in Redis with 24h TTL") + + return temp_dir + except Exception as e: + LOGGER.error(f"[v{VERSION}] Failed to create temp directory: {e}") + # Fallback to /tmp/malware-scan if all else fails + fallback_dir = "/tmp/malware-scan" + os.makedirs(fallback_dir, exist_ok=True) + return fallback_dir + + # Get or create the persistent temp directory + # NOTE: This directory stores downloaded ZIP files and metadata (survives across scheduler runs) + # Separate extraction temp directories are created during import (cleaned up after each import) + persistent_temp_dir = get_or_create_temp_dir(redis_conn) + LOGGER.info(f"[v{VERSION}] Using persistent temp directory for downloads: {persistent_temp_dir}") + + # ============================================================================ + # FILE PATHS CONFIGURATION (using persistent temp directory) + # ============================================================================ + + # Always use persistent temp directory (ignoring env variable to avoid /var/cache/bunkerweb cleanup issues) + csv_path = os.path.join(persistent_temp_dir, "hashes.csv") + + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] Configuration:") + LOGGER.debug(f"[v{VERSION}] CSV path: {csv_path}") + + # Define both full and recent database files with configurable URLs + full_url = getenv( + "MALWARE_SCAN_MALWAREBAZAAR_UPDATE_URL_FULL", + "https://bazaar.sysangels.ai/bunkerweb/malwarebazaar_full_split.zip" + ).strip() + recent_url = getenv( + "MALWARE_SCAN_MALWAREBAZAAR_UPDATE_URL_RECENT", + "https://bazaar.sysangels.ai/bunkerweb/malwarebazaar_recent_split.zip" + ).strip() + + files_to_update = [ + { + "name": "recent", + "url": recent_url, + "zip_file": csv_path.replace(".csv", "_recent.zip"), + "state_key": "malware-scan:hashes:state:recent", + }, + { + "name": "full", + "url": full_url, + "zip_file": csv_path.replace(".csv", "_full.zip"), + "state_key": "malware-scan:hashes:state:full", + }, + ] + + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] MalwareBazaar URLs:") + LOGGER.debug(f"[v{VERSION}] Full: {full_url}") + LOGGER.debug(f"[v{VERSION}] Recent: {recent_url}") + + # ============================================================================ + # CHECK CLEANUP VARIABLES (before init detection) + # ============================================================================ + + # Check if any cleanup flag is enabled + cleanup_all = getenv("MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_ALL", "no") == "yes" + cleanup_clamav = getenv("MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_CLAMAV", "no") == "yes" + cleanup_virustotal = getenv("MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_VIRUSTOTAL", "no") == "yes" + cleanup_sentinelone = getenv("MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_SENTINELONE", "no") == "yes" + cleanup_malwarebazaar = getenv("MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_MALWAREBAZAAR", "no") == "yes" + cleanup_attacker_ipv4 = getenv("MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_ATTACKER_IPV4", "no") == "yes" + cleanup_attacker_ipv6 = getenv("MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_ATTACKER_IPV6", "no") == "yes" + + cleanup_requested = ( + cleanup_all or cleanup_clamav or cleanup_virustotal + or cleanup_sentinelone or cleanup_malwarebazaar or cleanup_attacker_ipv4 or cleanup_attacker_ipv6 + ) + + # ============================================================================ + # DETECT IF THIS IS FIRST RUN (based on Redis state keys) + # ============================================================================ + + # Check if state keys exist in Redis - if they don't, this is first run/init mode + state_key_full = "malware-scan:hashes:state:full" + state_key_recent = "malware-scan:hashes:state:recent" + + state_full_exists = redis_conn.exists(state_key_full) + state_recent_exists = redis_conn.exists(state_key_recent) + + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] Initialization detection:") + LOGGER.debug(f"[v{VERSION}] State key (full): {state_key_full}") + LOGGER.debug(f"[v{VERSION}] State key (recent): {state_key_recent}") + LOGGER.debug(f"[v{VERSION}] Full exists: {state_full_exists}") + LOGGER.debug(f"[v{VERSION}] Recent exists: {state_recent_exists}") + # Try to get values if they exist + if state_full_exists: + try: + full_value = redis_conn.get(state_key_full) + LOGGER.debug(f"[v{VERSION}] Full value: {full_value}") + except Exception as e: + LOGGER.debug(f"[v{VERSION}] Failed to get full value: {e}") + if state_recent_exists: + try: + recent_value = redis_conn.get(state_key_recent) + LOGGER.debug(f"[v{VERSION}] Recent value: {recent_value}") + except Exception as e: + LOGGER.debug(f"[v{VERSION}] Failed to get recent value: {e}") + + # First run = neither state key exists in Redis + is_init_run = not (state_full_exists and state_recent_exists) + + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] Is init run: {is_init_run}") + LOGGER.debug(f"[v{VERSION}] Cleanup requested: {cleanup_requested}") + + # Determine run mode + if cleanup_requested: + LOGGER.info(f"[v{VERSION}] [CLEANUP] Cache cleanup requested via environment variables") + elif is_init_run: + LOGGER.info(f"[v{VERSION}] [INIT] First run detected - initialization mode enabled") + else: + LOGGER.info(f"[v{VERSION}] [UPDATE] Periodic update run - no cleanup requested") + + # ============================================================================ + # CACHE CLEANUP (runs if cleanup requested OR first init run) + # ============================================================================ + + if cleanup_requested or is_init_run: + if cleanup_requested: + LOGGER.info(f"[v{VERSION}] [CLEANUP] Starting cache cleanup...") + else: + LOGGER.info(f"[v{VERSION}] [INIT] Starting cache cleanup before hash import...") + + # Build pattern list based on enabled flags + patterns_to_delete = [] + if cleanup_all or cleanup_clamav: + patterns_to_delete.append("plugin_malware_scan_clamav_*") + if cleanup_all or cleanup_virustotal: + patterns_to_delete.append("plugin_malware_scan_virustotal_*") + if cleanup_all or cleanup_sentinelone: + patterns_to_delete.append("plugin_malware_scan_sentinelone_*") + # Clean when cleanup_all or cleanup_malwarebazaar is enabled + if cleanup_all or cleanup_malwarebazaar: + patterns_to_delete.append("plugin_malware_scan_malwarebazaar_*") + if cleanup_all or cleanup_attacker_ipv4: + patterns_to_delete.append("plugin_malware_scan_attacker_ipv4_*") + if cleanup_all or cleanup_attacker_ipv6: + patterns_to_delete.append("plugin_malware_scan_attacker_ipv6_*") + + # Always clean composite results cache + if patterns_to_delete: + patterns_to_delete.append("plugin_malware_scan_results_*") + + # Clean Redis cache + if patterns_to_delete: + LOGGER.info(f"[v{VERSION}] [CACHE_CLEANUP] Cleaning Redis patterns: {', '.join(patterns_to_delete)}") + total_deleted = 0 + for pattern in patterns_to_delete: + pattern_deleted = 0 + cursor = 0 + safe_prefix = "plugin_malware_scan_" + + # Validate pattern for safety + if not pattern.startswith(safe_prefix): + LOGGER.error(f"[v{VERSION}] [CACHE_CLEANUP] Safety error: Pattern '{pattern}' invalid") + continue + + try: + while True: + cursor, keys = redis_conn.scan(cursor, match=pattern, count=100) + if keys: + pattern_deleted += redis_conn.unlink(*keys) + if cursor == 0: + break + + if pattern_deleted > 0: + total_deleted += pattern_deleted + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] [CACHE_CLEANUP] Deleted {pattern_deleted} keys for pattern: {pattern}") + except Exception as e: + LOGGER.warning(f"[v{VERSION}] [CACHE_CLEANUP] Error cleaning pattern '{pattern}': {e}") + + # Log cleanup results + if total_deleted > 0: + LOGGER.info(f"[v{VERSION}] [CACHE_CLEANUP] Redis cleanup complete - {total_deleted} keys deleted") + else: + LOGGER.info(f"[v{VERSION}] [CACHE_CLEANUP] Redis cleanup complete - 0 keys existing") + + # Cleanup MalwareBazaar hash databases and Redis buckets + # Clean when cleanup_all or cleanup_malwarebazaar is enabled + # NOTE: ZIP files are preserved to allow re-import without re-download + if cleanup_all or cleanup_malwarebazaar: + LOGGER.info(f"[v{VERSION}] [CACHE_CLEANUP] Cleaning MalwareBazaar Redis hash buckets...") + mb_buckets_deleted = 0 + + # Skip CSV, ZIP, and metadata file deletion to allow re-import from existing files + LOGGER.info(f"[v{VERSION}] [CACHE_CLEANUP] Preserving ZIP files and metadata for re-import (no re-download needed)") + + # Delete all hash buckets unconditionally (will be re-imported from preserved ZIPs) + bucket_prefixes = ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "A", "B", "C", "D", "E", "F"] + + # Delete full hash buckets + for prefix in bucket_prefixes: + bucket = f"malware_scan:hashes:{prefix}" + try: + if redis_conn.delete(bucket): + mb_buckets_deleted += 1 + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] [CACHE_CLEANUP] Deleted hash bucket: {bucket}") + except Exception as e: + LOGGER.warning(f"[v{VERSION}] [CACHE_CLEANUP] Failed to delete bucket {bucket}: {e}") + + # Delete recent hash buckets + for prefix in bucket_prefixes: + bucket = f"malware_scan:hashes:recent:{prefix}" + try: + if redis_conn.delete(bucket): + mb_buckets_deleted += 1 + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] [CACHE_CLEANUP] Deleted hash bucket: {bucket}") + except Exception as e: + LOGGER.warning(f"[v{VERSION}] [CACHE_CLEANUP] Failed to delete bucket {bucket}: {e}") + + LOGGER.info(f"[v{VERSION}] [CACHE_CLEANUP] Deleted {mb_buckets_deleted} hash buckets from Redis") + + # Clear BunkerWeb cache state files (will trigger re-import from preserved ZIPs) + try: + JOB.cache_file("malware-scan:hashes:state:full", b"") + JOB.cache_file("malware-scan:hashes:state:recent", b"") + LOGGER.info(f"[v{VERSION}] [CACHE_CLEANUP] Cleared state files (will trigger re-import from local ZIPs)") + except Exception as e: + LOGGER.warning(f"[v{VERSION}] [CACHE_CLEANUP] Failed to clear cache state files: {e}") + + LOGGER.info(f"[v{VERSION}] [CACHE_CLEANUP] โœ… Cache cleanup phase completed") + + # Lock key for preventing concurrent imports (defined here for closure access) + lock_key = "malware-scan:hashes:import_lock" + + # ============================================================================ + # METADATA FILE FUNCTIONS (for HTTP header caching) + # ============================================================================ + + def save_zip_metadata(zip_path, last_modified=None, etag=None, content_length=None, sha256=None): + """ + Save HTTP metadata for a ZIP file to a .meta file. + This allows checking if remote file has changed without re-downloading. + + Args: + zip_path: Path to the ZIP file + last_modified: Last-Modified header from HTTP response + etag: ETag header from HTTP response + content_length: Content-Length header from HTTP response + sha256: SHA256 checksum of the file + """ + meta_path = zip_path + ".meta" + try: + metadata = { + "last_modified": last_modified, + "etag": etag, + "content_length": content_length, + "sha256": sha256, + "saved_at": datetime.now(timezone.utc).isoformat() + } + with open(meta_path, 'w') as f: + import json + json.dump(metadata, f, indent=2) + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] Saved metadata to {meta_path}") + return True + except Exception as e: + LOGGER.warning(f"[v{VERSION}] Failed to save metadata file: {e}") + return False + + def load_zip_metadata(zip_path): + """ + Load HTTP metadata for a ZIP file from its .meta file. + + Args: + zip_path: Path to the ZIP file + + Returns: + dict: Metadata dictionary or None if file doesn't exist or is invalid + """ + meta_path = zip_path + ".meta" + try: + if os.path.exists(meta_path): + with open(meta_path, 'r') as f: + import json + metadata = json.load(f) + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] Loaded metadata from {meta_path}") + return metadata + return None + except Exception as e: + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] Failed to load metadata file: {e}") + return None + + # Define helper function to process one file + def process_file(file_config, previous_import_state=None): + """Download and import a single malware hash file (full or recent) + + Args: + file_config: Configuration dict for the file to process + previous_import_state: The import state from before this run started + """ + file_name = file_config["name"] + file_url = file_config["url"] + zip_path = file_config["zip_file"] + state_key = file_config["state_key"] + file_status = 0 + + try: + # Load state from BunkerWeb cache + state = {} + cached_state = JOB.get_cache(state_key) + if cached_state: + try: + for line in cached_state.decode().split("\n"): + if line and "=" in line: + key, value = line.strip().split("=", 1) + state[key] = value.strip('"') + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] [{file_name}] Loaded state from cache: {len(state)} keys") + LOGGER.debug(f"[v{VERSION}] [{file_name}] State contents:") + for k, v in state.items(): + LOGGER.debug(f"[v{VERSION}] [{file_name}] {k}={v}") + except Exception as e: + LOGGER.warning(f"[v{VERSION}] [{file_name}] Failed to parse cached state: {e}") + else: + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] [{file_name}] No cached state found") + + # Check time-based update window + # - Full file: 22 hours (79200 seconds) + # - Recent file: 5 minutes (300 seconds) + # Only skip remote check if: + # 1. Last import completed successfully (IMPORT_SUCCESS="yes" in state) + # 2. Redis confirms last import was successful (import_state == "success") + # 3. Less than the configured time window has passed since last download + last_download_time = state.get("DOWNLOAD_TIME", "") + import_success = state.get("IMPORT_SUCCESS", "") + + # Use the previous import state (from before this run started) + redis_import_state = previous_import_state + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] [{file_name}] Previous Redis import state: {redis_import_state}") + + # Set update window based on file type + is_recent = file_name == "recent" + update_window_seconds = 300 if is_recent else 79200 # 5 minutes vs 22 hours + update_window_display = "5m" if is_recent else "22h" + + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] [{file_name}] Time window check:") + LOGGER.debug(f"[v{VERSION}] [{file_name}] last_download_time: {last_download_time}") + LOGGER.debug(f"[v{VERSION}] [{file_name}] import_success: {import_success}") + LOGGER.debug(f"[v{VERSION}] [{file_name}] redis_import_state: {redis_import_state}") + LOGGER.debug(f"[v{VERSION}] [{file_name}] update_window: {update_window_seconds}s ({update_window_display})") + + if last_download_time and import_success == "yes" and redis_import_state == "success": + try: + last_download = datetime.fromisoformat(last_download_time) + now = datetime.now(timezone.utc) + elapsed_seconds = (now - last_download).total_seconds() + elapsed_hours = elapsed_seconds / 3600 + elapsed_minutes = elapsed_seconds / 60 + + # Skip remote check if last download was recent AND successful AND cached file exists + if elapsed_seconds < update_window_seconds: + # Check if cached ZIP file still exists (may be deleted after temp dir recreation) + if not os.path.exists(zip_path): + LOGGER.warning(f"[v{VERSION}] [{file_name}] Cached file missing despite recent import - forcing re-download") + LOGGER.warning(f"[v{VERSION}] [{file_name}] Expected location: {zip_path}") + LOGGER.warning(f"[v{VERSION}] [{file_name}] (Likely temp directory changed after reboot)") + else: + if is_recent: + LOGGER.info(f"[v{VERSION}] [{file_name}] โœ… Last successful import was {elapsed_minutes:.1f}m ago (< {update_window_display} window), skipping remote check") + else: + LOGGER.info(f"[v{VERSION}] [{file_name}] โœ… Last successful import was {elapsed_hours:.1f}h ago (< {update_window_display} window), skipping remote check") + return 0 + else: + if is_recent: + LOGGER.debug(f"[v{VERSION}] [{file_name}] Last successful import was {elapsed_minutes:.1f}m ago (>= {update_window_display} window), checking for updates") + else: + LOGGER.debug(f"[v{VERSION}] [{file_name}] Last successful import was {elapsed_hours:.1f}h ago (>= {update_window_display} window), checking for updates") + except Exception as e: + LOGGER.debug(f"[v{VERSION}] [{file_name}] Failed to parse DOWNLOAD_TIME '{last_download_time}': {e}, proceeding with remote check") + elif redis_import_state != "success": + LOGGER.info(f"[v{VERSION}] [{file_name}] Last import did not complete successfully (Redis state: {redis_import_state}), forcing remote check to retry") + elif import_success != "yes": + LOGGER.info(f"[v{VERSION}] [{file_name}] Last import did not complete successfully (state marker missing), forcing remote check to retry") + else: + LOGGER.debug(f"[v{VERSION}] [{file_name}] No previous successful import found, proceeding with remote check") + + # Record import start time for this file type + import time as time_module + import_start_time_key = f"malware-scan:hashes:import_start_time:{file_name}" + try: + redis_conn.set(import_start_time_key, str(int(time_module.time()))) + except Exception as e: + LOGGER.warning(f"[v{VERSION}] [{file_name}] Failed to set import start time: {e}") + + # Get remote metadata + LOGGER.info(f"[v{VERSION}] [{file_name}] Checking remote file metadata...") + max_retries = 3 + remote_etag = None + remote_size = None + remote_modified = None + headers = {"User-Agent": f"bunkerweb - https://github.com/bunkerity/bunkerweb - malware-scan module v{VERSION}"} + + for attempt in range(1, max_retries + 1): + try: + response = head(file_url, timeout=30, headers=headers) + if response.status_code == 200: + remote_etag = response.headers.get("etag", "").strip('"') + remote_size = response.headers.get("content-length", "") + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] [{file_name}] Remote: ETag={remote_etag}, Size={remote_size}") + break + else: + LOGGER.warning(f"[v{VERSION}] [{file_name}] HTTP {response.status_code}") + except (ConnectionError, Timeout) as e: + LOGGER.warning(f"[v{VERSION}] [{file_name}] Network error: {e}") + if attempt < max_retries: + delay = 3 * attempt + LOGGER.info(f"[v{VERSION}] [{file_name}] Retrying in {delay}s (attempt {attempt}/{max_retries})...") + time.sleep(delay) + except Exception as e: + LOGGER.error(f"[v{VERSION}] [{file_name}] Error checking metadata: {e}") + + if not remote_size or not remote_etag: + LOGGER.error(f"[v{VERSION}] [{file_name}] Failed to get remote metadata after retries") + return 2 + + # Check if changed using ETag and Size + saved_etag = state.get("SAVED_ETAG", "") + saved_size = state.get("SAVED_SIZE", "") + + etag_match = saved_etag and saved_etag == remote_etag + size_match = saved_size and saved_size == remote_size + + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] [{file_name}] ETag comparison: saved='{saved_etag}' vs remote='{remote_etag}' (match: {etag_match})") + LOGGER.debug(f"[v{VERSION}] [{file_name}] Size comparison: saved='{saved_size}' vs remote='{remote_size}' (match: {size_match})") + + # File is unchanged if both match AND last import was successful + # If last import failed, force re-download even if file is unchanged + if etag_match and size_match and import_success == "yes" and redis_import_state == "success": + # Verify cached file integrity with checksum (tamper detection) + if os.path.exists(zip_path): + saved_checksum = state.get("SAVED_SHA256", "") + if saved_checksum: + LOGGER.info(f"[v{VERSION}] [{file_name}] Verifying cached file integrity...") + is_valid, current_checksum, _ = verify_file_integrity(zip_path) + if is_valid and current_checksum == saved_checksum: + LOGGER.info(f"[v{VERSION}] [{file_name}] โœ… Cached file integrity verified (SHA256 match)") + LOGGER.info(f"[v{VERSION}] [{file_name}] โœ… File unchanged (ETag & Size match), skipping download") + return 0 + elif is_valid and current_checksum != saved_checksum: + LOGGER.warning(f"[v{VERSION}] [{file_name}] โš ๏ธ Cached file checksum mismatch - file may have been tampered") + LOGGER.warning(f"[v{VERSION}] [{file_name}] Expected: {saved_checksum[:16]}...{saved_checksum[-16:]}") + LOGGER.warning(f"[v{VERSION}] [{file_name}] Got: {current_checksum[:16]}...{current_checksum[-16:]}") + LOGGER.info(f"[v{VERSION}] [{file_name}] Forcing re-download for security") + # Continue to download section + else: + LOGGER.warning(f"[v{VERSION}] [{file_name}] Failed to verify cached file - forcing re-download") + # Continue to download section + else: + LOGGER.info(f"[v{VERSION}] [{file_name}] No saved checksum available - skipping integrity check") + LOGGER.info(f"[v{VERSION}] [{file_name}] โœ… File unchanged (ETag & Size match), skipping download") + return 0 + else: + LOGGER.warning(f"[v{VERSION}] [{file_name}] Cached file not found at {zip_path} - forcing re-download") + # Continue to download section + elif etag_match and size_match: + LOGGER.info(f"[v{VERSION}] [{file_name}] File unchanged but previous import failed (import_success={import_success}, redis_state={redis_import_state}), forcing re-download to retry") + # Continue to download section + + if DEBUG_MODE: + if not etag_match: + LOGGER.debug(f"[v{VERSION}] [{file_name}] ETag mismatch: '{saved_etag}' != '{remote_etag}'") + if not size_match: + LOGGER.debug(f"[v{VERSION}] [{file_name}] Size mismatch: '{saved_size}' != '{remote_size}'") + + # Check if we have local metadata file (survives Redis cleanup) + local_metadata = load_zip_metadata(zip_path) if os.path.exists(zip_path) else None + if local_metadata and DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] [{file_name}] Found local metadata: ETag={local_metadata.get('etag')}, Last-Modified={local_metadata.get('last_modified')}, SHA256={local_metadata.get('sha256', '')[:16]}...") + + # Optimization: Check remote SHA256 checksum before downloading ZIP + # This avoids downloading large ZIP files if they haven't changed + LOGGER.info(f"[v{VERSION}] [{file_name}] Checking remote SHA256 checksum...") + sha256_url = file_url + ".sha256" + published_checksum = None + skip_download = False + download_ok = False + download_checksum = None + remote_modified = None + + try: + sha256_response = get(sha256_url, timeout=30, headers=headers) + if sha256_response.status_code == 200: + # Parse checksum from .sha256 file + # Format is typically: "checksum filename" or just "checksum" + published_checksum_line = sha256_response.text.strip() + published_checksum = published_checksum_line.split()[0].strip().lower() + + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] [{file_name}] Published checksum: {published_checksum}") + + # Validate checksum format (should be 64 hex chars) + if len(published_checksum) == 64 and all(c in '0123456789abcdef' for c in published_checksum): + LOGGER.info(f"[v{VERSION}] [{file_name}] Retrieved remote checksum: {published_checksum[:16]}...{published_checksum[-16:]}") + + # If cached file exists, compare checksums + if os.path.exists(zip_path): + LOGGER.info(f"[v{VERSION}] [{file_name}] Calculating checksum of cached file...") + is_valid, cached_checksum, _ = verify_file_integrity(zip_path) + + if is_valid and cached_checksum: + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] [{file_name}] Cached checksum: {cached_checksum}") + + # Compare checksums + if cached_checksum.lower() == published_checksum: + LOGGER.info(f"[v{VERSION}] [{file_name}] โœ… Cached file matches remote checksum - skipping download") + LOGGER.info(f"[v{VERSION}] [{file_name}] Using cached file: {zip_path}") + + # Update state with current metadata (but don't re-import) + state_data = f'SAVED_ETAG="{remote_etag}"\nSAVED_SIZE="{remote_size}"\nSAVED_SHA256="{cached_checksum}"\nDOWNLOAD_TIME="{datetime.now(timezone.utc).isoformat()}"\nIMPORT_SUCCESS="yes"\n' + cached, err = JOB.cache_file(state_key, state_data.encode()) + if not cached: + LOGGER.warning(f"[v{VERSION}] [{file_name}] Failed to update cache state: {err}") + + # Also save/update local metadata file + save_zip_metadata( + zip_path, + last_modified=None, # Don't have Last-Modified from SHA256 check + etag=remote_etag, + content_length=remote_size, + sha256=cached_checksum + ) + + # Proceed with import using cached file + download_ok = True + download_checksum = cached_checksum + remote_modified = datetime.now(timezone.utc).strftime("%a, %d %b %Y %H:%M:%S GMT") + # Skip to import section by setting flag and jumping past download + skip_download = True + else: + LOGGER.info(f"[v{VERSION}] [{file_name}] Cached file checksum differs - downloading new version") + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] [{file_name}] Cached: {cached_checksum}") + LOGGER.debug(f"[v{VERSION}] [{file_name}] Published: {published_checksum}") + skip_download = False + else: + LOGGER.warning(f"[v{VERSION}] [{file_name}] Failed to calculate cached file checksum - will download") + skip_download = False + else: + LOGGER.info(f"[v{VERSION}] [{file_name}] No cached file found - will download") + skip_download = False + else: + LOGGER.warning(f"[v{VERSION}] [{file_name}] Invalid checksum format: {published_checksum}") + skip_download = False + else: + LOGGER.warning(f"[v{VERSION}] [{file_name}] SHA256 checksum file not available (HTTP {sha256_response.status_code})") + skip_download = False + except Exception as e: + LOGGER.warning(f"[v{VERSION}] [{file_name}] Failed to check remote checksum: {e}") + skip_download = False + + # Download file (only if checksum comparison indicated it's needed) + if not skip_download: + LOGGER.info(f"[v{VERSION}] [{file_name}] Downloading from {file_url}...") + + # Ensure cache directory exists with explicit error handling + cache_dir = os.path.dirname(zip_path) + try: + os.makedirs(cache_dir, exist_ok=True) + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] [{file_name}] Cache directory ready: {cache_dir}") + except Exception as dir_error: + LOGGER.error(f"[v{VERSION}] [{file_name}] Failed to create cache directory '{cache_dir}': {dir_error}") + return 2 + + # Verify directory exists + if not os.path.isdir(cache_dir): + LOGGER.error(f"[v{VERSION}] [{file_name}] Cache directory does not exist after creation: {cache_dir}") + return 2 + + for attempt in range(1, max_retries + 1): + try: + response = get(file_url, timeout=60, stream=True, headers=headers) + if response.status_code == 200: + with open(zip_path, "wb") as f: + for chunk in response.iter_content(chunk_size=8192): + if chunk: + f.write(chunk) + file_size = os.path.getsize(zip_path) + file_mtime = os.path.getmtime(zip_path) + file_mtime_str = datetime.fromtimestamp(file_mtime, tz=timezone.utc).strftime("%a, %d %b %Y %H:%M:%S GMT") + LOGGER.info(f"[v{VERSION}] [{file_name}] Download complete: {file_size} bytes") + + # Verify file integrity with checksum (security enhancement) + LOGGER.info(f"[v{VERSION}] [{file_name}] Calculating SHA256 checksum for integrity verification...") + is_valid, checksum, actual_size = verify_file_integrity(zip_path, file_size) + if not is_valid or not checksum: + LOGGER.error(f"[v{VERSION}] [{file_name}] File integrity check failed - file may be corrupted or tampered") + # Remove corrupted file + try: + os.remove(zip_path) + LOGGER.info(f"[v{VERSION}] [{file_name}] Removed corrupted download") + except Exception as e: + LOGGER.warning(f"[v{VERSION}] [{file_name}] Failed to remove corrupted file: {e}") + # Continue to retry + continue + + LOGGER.info(f"[v{VERSION}] [{file_name}] โœ… Integrity verified - SHA256: {checksum[:16]}...{checksum[-16:]}") + + # Save HTTP metadata for future comparisons + last_modified_header = response.headers.get('Last-Modified') + etag_header = response.headers.get('ETag') + content_length_header = response.headers.get('Content-Length') + save_zip_metadata( + zip_path, + last_modified=last_modified_header, + etag=etag_header, + content_length=content_length_header, + sha256=checksum + ) + + # Verify against published SHA256 checksum file (additional security layer) + # Reuse published_checksum if we already fetched it during pre-download check + if not published_checksum: + LOGGER.info(f"[v{VERSION}] [{file_name}] Downloading published SHA256 checksum for verification...") + sha256_url = file_url + ".sha256" + try: + sha256_response = get(sha256_url, timeout=30, headers=headers) + if sha256_response.status_code == 200: + # Parse checksum from .sha256 file + # Format is typically: "checksum filename" or just "checksum" + published_checksum_line = sha256_response.text.strip() + published_checksum = published_checksum_line.split()[0].strip().lower() + + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] [{file_name}] Published checksum file content: {published_checksum_line[:100]}") + LOGGER.debug(f"[v{VERSION}] [{file_name}] Extracted published checksum: {published_checksum}") + else: + LOGGER.warning(f"[v{VERSION}] [{file_name}] SHA256 checksum file not available (HTTP {sha256_response.status_code})") + except Exception as e: + LOGGER.warning(f"[v{VERSION}] [{file_name}] Failed to download/verify published checksum: {e}") + else: + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] [{file_name}] Using published checksum from pre-download check") + + # Compare checksums if published checksum is available + if published_checksum: + # Validate checksum format (should be 64 hex chars) + if len(published_checksum) == 64 and all(c in '0123456789abcdef' for c in published_checksum): + # Compare checksums (case-insensitive) + if checksum.lower() == published_checksum: + LOGGER.info(f"[v{VERSION}] [{file_name}] โœ… Published checksum verified - matches calculated checksum") + else: + LOGGER.error(f"[v{VERSION}] [{file_name}] โŒ CHECKSUM MISMATCH - File may be tampered or corrupted!") + LOGGER.error(f"[v{VERSION}] [{file_name}] Published: {published_checksum}") + LOGGER.error(f"[v{VERSION}] [{file_name}] Calculated: {checksum.lower()}") + # Remove potentially tampered file + try: + os.remove(zip_path) + LOGGER.info(f"[v{VERSION}] [{file_name}] Removed file with checksum mismatch") + except Exception as e: + LOGGER.warning(f"[v{VERSION}] [{file_name}] Failed to remove file: {e}") + # Continue to retry + continue + else: + LOGGER.warning(f"[v{VERSION}] [{file_name}] Invalid checksum format in .sha256 file: {published_checksum}") + LOGGER.warning(f"[v{VERSION}] [{file_name}] Proceeding with calculated checksum only") + else: + LOGGER.warning(f"[v{VERSION}] [{file_name}] Proceeding with calculated checksum only") + + remote_modified = file_mtime_str + download_checksum = checksum # Store for state persistence + download_ok = True + break + else: + LOGGER.warning(f"[v{VERSION}] [{file_name}] HTTP {response.status_code}") + except (ConnectionError, Timeout) as e: + LOGGER.warning(f"[v{VERSION}] [{file_name}] Network error: {e}") + if attempt < max_retries: + delay = 3 * attempt + LOGGER.info(f"[v{VERSION}] [{file_name}] Retrying in {delay}s (attempt {attempt}/{max_retries})...") + time.sleep(delay) + except Exception as e: + LOGGER.error(f"[v{VERSION}] [{file_name}] Error downloading: {e}") + + if not download_ok: + LOGGER.error(f"[v{VERSION}] [{file_name}] Download failed") + return 2 + + # At this point, either download succeeded or we're using cached file (skip_download=True) + # Extract and import + # Create extraction subdirectory within persistent temp dir (cleaned up after import) + temp_dir = os.path.join(persistent_temp_dir, f"extract_{file_name}") + os.makedirs(temp_dir, exist_ok=True) + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] [{file_name}] Created extraction directory: {temp_dir}") + LOGGER.debug(f"[v{VERSION}] [{file_name}] Extracting from ZIP: {zip_path}") + LOGGER.debug(f"[v{VERSION}] [{file_name}] ZIP size: {os.path.getsize(zip_path)} bytes") + try: + with zipfile.ZipFile(zip_path, "r") as zf: + zf.extractall(temp_dir) + + # Find all 16 split hash files + split_files = {} + bucket_prefixes = ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "A", "B", "C", "D", "E", "F"] + is_recent = file_name == "recent" + + for root, dirs, files in os.walk(temp_dir): + for fname in files: + if ("malwarebazaar_full_split_" in fname or "malwarebazaar_recent_split_" in fname) and fname.endswith(".txt"): + try: + bucket_id = fname.split("_")[-1].replace(".txt", "").upper() + if bucket_id in bucket_prefixes: + split_files[bucket_id] = os.path.join(root, fname) + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] [{file_name}] Found split file for bucket {bucket_id}: {fname}") + except (IndexError, ValueError): + pass + + if not split_files or len(split_files) < 16: + LOGGER.error(f"[v{VERSION}] [{file_name}] Expected 16 split files, found {len(split_files)}") + return 2 + + # Count total lines across all 16 split files before starting import + total_lines_all_files = 0 + LOGGER.info(f"[v{VERSION}] [{file_name}] Counting total lines in all split files...") + for bucket_id, file_path in split_files.items(): + try: + with open(file_path, "r") as f: + next(f, None) # Skip header + lines = sum(1 for line in f if line.strip() and not line.strip().startswith("#")) + total_lines_all_files += lines + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] [{file_name}] Bucket {bucket_id}: {lines:,} lines") + except Exception as e: + LOGGER.warning(f"[v{VERSION}] [{file_name}] Failed to count lines for bucket {bucket_id}: {e}") + + LOGGER.info(f"[v{VERSION}] [{file_name}] Total lines to import: {total_lines_all_files:,}") + + # Initialize progress tracking in Redis for UI + # Set total and reset all per-bucket progress counters to 0 + try: + redis_conn.set(f"malware-scan:hashes:import_total:{file_name}", str(total_lines_all_files)) + # Initialize all per-bucket progress counters to 0 + for bucket in bucket_prefixes: + redis_conn.set(f"malware-scan:hashes:import_processed:{file_name}:bucket_{bucket}", "0") + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] [{file_name}] Initialized progress tracking: total={total_lines_all_files:,}, per-bucket=0") + except Exception as e: + LOGGER.warning(f"[v{VERSION}] [{file_name}] Failed to initialize progress tracking in Redis: {e}") + + # Import split files in parallel + bucket_prefix = "recent" if is_recent else "" + bucket_prefix_display = " (recent)" if is_recent else " (full)" + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] [{file_name}] Importing to {bucket_prefix if bucket_prefix else 'full'} hash table{bucket_prefix_display}") + + added_total = 0 + duplicates_total = 0 + processed_total_all_buckets = 0 + import_lock = threading.Lock() + import_results = {} + + def import_bucket_file(bucket_id, file_path, bucket_prefix=""): + """Import a single bucket's split file to appropriate hash table""" + try: + added = 0 + duplicates = 0 + bucket_key_suffix = f"recent:{bucket_id}" if bucket_prefix else bucket_id + bucket = f"malware_scan:hashes:{bucket_key_suffix}" + + # Get existing count before import + existing_before = redis_conn.hlen(bucket) or 0 + + # Count total lines in file (excluding header) + total_lines = 0 + try: + with open(file_path, "r") as f: + next(f, None) # Skip header + total_lines = sum(1 for line in f if line.strip() and not line.strip().startswith("#")) + except Exception as e: + LOGGER.warning(f"[v{VERSION}] [{file_name}] Failed to count lines for bucket {bucket_id}: {e}") + total_lines = 0 # Fallback to unknown count + + start_time = time.time() + last_progress_time = start_time + + with open(file_path, "r") as f: + next(f, None) # Skip header + for line in f: + line = line.strip() + if not line or line.startswith("#"): + continue + + fields = line.split(",") + if len(fields) < 3: + continue + + sha256 = fields[0].strip().upper() + signature = fields[1].strip() + clamav = fields[2].strip() + + if not sha256 or len(sha256) != 64: + continue + + # Store as "signature|clamav" (pipe-delimited) + value = f"{signature}|{clamav}" + + if redis_conn.hexists(bucket, sha256): + duplicates += 1 + else: + redis_conn.hset(bucket, sha256, value) + added += 1 + + # Progress update every 100k hashes or every 10 seconds + current_time = time.time() + # Log progress every 10 seconds (regardless of debug mode) + if (current_time - last_progress_time) >= 10: + elapsed = current_time - start_time + rate = (added + duplicates) / elapsed if elapsed > 0 else 0 + total_after = existing_before + added + processed = added + duplicates + + # Format progress message with "nnn of xxx" if total is known + if total_lines > 0: + progress_str = f"{processed:,} of {total_lines:,}" + percent = (processed / total_lines * 100) if total_lines > 0 else 0 + LOGGER.info(f"[v{VERSION}] [{file_name}] Bucket {bucket_id} progress: {progress_str} ({percent:.1f}%), added={added:,}, duplicates={duplicates:,}, rate={rate:.0f}h/s") + else: + LOGGER.info(f"[v{VERSION}] [{file_name}] Bucket {bucket_id} progress: processed={processed:,}, added={added:,}, duplicates={duplicates:,}, rate={rate:.0f}h/s") + + # Update global progress counter in Redis for UI tracking + try: + # Use SET to track per-bucket progress (will be summed by API) + redis_conn.set(f"malware-scan:hashes:import_processed:{file_name}:bucket_{bucket_id}", str(processed)) + except Exception as e: + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] [{file_name}] Failed to update progress counter: {e}") + + # Renew import lock TTL to prevent expiration during long imports + try: + redis_conn.expire(lock_key, 60) + except Exception: + pass # Lock renewal failure is non-critical + last_progress_time = current_time + + # Set TTL for recent buckets (48 hours) + if bucket_prefix == "recent": + ttl_seconds = 48 * 60 * 60 + redis_conn.expire(bucket, ttl_seconds) + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] [{file_name}] Bucket {bucket_id} TTL set to 48 hours") + + # Get final count after import + total_after = redis_conn.hlen(bucket) or 0 + + with import_lock: + import_results[bucket_id] = (added, duplicates) + + # Update final progress for this bucket + try: + final_processed = added + duplicates + redis_conn.set(f"malware-scan:hashes:import_processed:{file_name}:bucket_{bucket_id}", str(final_processed)) + except Exception as e: + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] [{file_name}] Failed to update final progress counter: {e}") + + # Log completion with total lines info + if DEBUG_MODE: + processed_total = added + duplicates + if total_lines > 0: + LOGGER.debug(f"[v{VERSION}] [{file_name}] Bucket {bucket_id} import complete: processed {processed_total} of {total_lines} ({processed_total/total_lines*100:.1f}%), before={existing_before}, added={added}, duplicates={duplicates}, after={total_after}") + else: + LOGGER.debug(f"[v{VERSION}] [{file_name}] Bucket {bucket_id} import complete: before={existing_before}, added={added}, duplicates={duplicates}, after={total_after}") + return True + except Exception as e: + LOGGER.error(f"[v{VERSION}] [{file_name}] Import error for bucket {bucket_id}: {e}") + return False + + # Import all 16 files in parallel + LOGGER.info(f"[v{VERSION}] [{file_name}] Starting parallel import of 16 buckets{bucket_prefix_display}...") + with ThreadPoolExecutor(max_workers=16) as executor: + futures = { + executor.submit(import_bucket_file, bucket_id, file_path, bucket_prefix): bucket_id + for bucket_id, file_path in split_files.items() + } + + failed_buckets = [] + for future in as_completed(futures): + bucket_id = futures[future] + try: + success = future.result() + if not success: + failed_buckets.append(bucket_id) + except Exception as e: + LOGGER.error(f"[v{VERSION}] [{file_name}] Thread error for bucket {bucket_id}: {e}") + failed_buckets.append(bucket_id) + + if failed_buckets: + LOGGER.warning(f"[v{VERSION}] [{file_name}] Failed to import buckets: {', '.join(failed_buckets)}") + return 2 + + # Sum results from all buckets + for added, duplicates in import_results.values(): + added_total += added + duplicates_total += duplicates + + LOGGER.info(f"[v{VERSION}] [{file_name}] Import complete: new_entries={added_total:,}, already_existed={duplicates_total:,}, processed={added_total + duplicates_total:,}") + + # Count totals per bucket and show distribution + table_name = "recent" if is_recent else "full" + total = 0 + min_count = None + max_count = None + for prefix in bucket_prefixes: + bucket_key_suffix = f"recent:{prefix}" if is_recent else prefix + count = redis_conn.hlen(f"malware_scan:hashes:{bucket_key_suffix}") or 0 + total += count + if min_count is None or count < min_count: + min_count = count + if max_count is None or count > max_count: + max_count = count + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] [{file_name}] Bucket {prefix}: {count:,} hashes") + + avg_count = total // 16 if total > 0 else 0 + balance_ratio = (max_count / min_count) if min_count and min_count > 0 else 1.0 + LOGGER.info(f"[v{VERSION}] [{file_name}] Total hashes in {table_name} table: {total:,} across 16 buckets (avg={avg_count:,}, min={min_count:,}, max={max_count:,}, ratio={balance_ratio:.3f})") + + # Record import end time for this file type + import time as time_module + import_end_time_key = f"malware-scan:hashes:import_end_time:{file_name}" + try: + redis_conn.set(import_end_time_key, str(int(time_module.time()))) + except Exception as e: + LOGGER.warning(f"[v{VERSION}] [{file_name}] Failed to set import end time: {e}") + + # Update state in BunkerWeb cache with success marker + state_content = ( + f'SAVED_ETAG="{remote_etag}"\n' + f'SAVED_SIZE="{remote_size}"\n' + f'SAVED_MODIFIED="{remote_modified}"\n' + f'SAVED_SHA256="{download_checksum}"\n' + f'DOWNLOAD_TIME="{datetime.now(timezone.utc).isoformat()}"\n' + f'IMPORT_SUCCESS="yes"\n' + ) + cached, err = JOB.cache_file(state_key, state_content.encode()) + if not cached: + LOGGER.warning(f"[v{VERSION}] [{file_name}] Failed to cache state: {err}") + + # Keep downloaded ZIP file for potential future use + LOGGER.info(f"[v{VERSION}] [{file_name}] Downloaded ZIP file preserved at: {zip_path}") + + file_status = 1 if added_total > 0 else 0 + + finally: + shutil.rmtree(temp_dir, ignore_errors=True) + + return file_status + + except Exception as e: + LOGGER.error(f"[v{VERSION}] [{file_name}] Error: {e}") + return 2 + + # Process both full and recent files + LOGGER.info(f"[v{VERSION}] Starting malware hash database updates...") + + # Import state key for tracking interrupted imports + import_state_key = "malware-scan:hashes:import_state" + + # Check for interrupted import from previous run AND save previous state + force_reimport = False + previous_import_state = None + try: + import_state = redis_conn.get(import_state_key) + if import_state: + previous_import_state = import_state.decode() + if previous_import_state == "running": + # Previous import didn't complete - check if lock is still held + lock_exists = redis_conn.exists(lock_key) + if not lock_exists: + LOGGER.warning(f"[v{VERSION}] Detected interrupted import from previous run - forcing full reimport") + force_reimport = True + # Delete state keys to trigger full cleanup and reimport + try: + redis_conn.delete(state_key_full) + redis_conn.delete(state_key_recent) + LOGGER.info(f"[v{VERSION}] Deleted state keys to trigger full reimport") + except Exception as e: + LOGGER.warning(f"[v{VERSION}] Failed to delete state keys: {e}") + except Exception as e: + LOGGER.warning(f"[v{VERSION}] Failed to check import state: {e}") + + # Acquire lock to prevent concurrent imports + lock_acquired = False + lock_refresh_thread = None + lock_refresh_stop = threading.Event() + + def refresh_lock(): + """Background thread function that refreshes the import lock TTL every 2 seconds.""" + while not lock_refresh_stop.is_set(): + try: + # Sleep for 2 seconds before refreshing + if lock_refresh_stop.wait(timeout=2.0): + break # Stop event was set, exit loop + + # Refresh lock TTL to 10 seconds using main connection + redis_conn.expire(lock_key, 10) + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] Lock TTL refreshed to 10s") + except Exception as e: + LOGGER.warning(f"[v{VERSION}] Lock refresh error: {e}") + + try: + # Try to acquire lock with 10-second TTL + lock_acquired = redis_conn.set(lock_key, "1", nx=True, ex=10) + if not lock_acquired: + LOGGER.warning(f"[v{VERSION}] Another import is in progress, waiting 20 seconds for lock...") + # Wait 20 seconds and try one more time + time.sleep(20) + lock_acquired = redis_conn.set(lock_key, "1", nx=True, ex=10) + if not lock_acquired: + LOGGER.warning(f"[v{VERSION}] Could not acquire import lock after 20s wait, skipping this run") + sys_exit(0) + + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] Import lock acquired with 10s TTL") + + # Start background thread to refresh lock every 2 seconds + lock_refresh_thread = threading.Thread(target=refresh_lock, daemon=True, name="LockRefresh") + lock_refresh_thread.start() + LOGGER.info(f"[v{VERSION}] Lock refresh heartbeat started (10s TTL, 2s refresh interval)") + + # Set import state to "running" + try: + redis_conn.set(import_state_key, "running") + LOGGER.info(f"[v{VERSION}] Import state set to 'running'") + except Exception as e: + LOGGER.warning(f"[v{VERSION}] Failed to set import state: {e}") + + # If forced reimport, delete all hash buckets to start fresh + if force_reimport: + LOGGER.info(f"[v{VERSION}] [RECOVERY] Cleaning existing hash buckets for full reimport...") + mb_buckets_deleted = 0 + bucket_prefixes = ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "A", "B", "C", "D", "E", "F"] + + # Delete full hash buckets + for prefix in bucket_prefixes: + bucket = f"malware_scan:hashes:{prefix}" + try: + if redis_conn.delete(bucket): + mb_buckets_deleted += 1 + except Exception as e: + LOGGER.warning(f"[v{VERSION}] [RECOVERY] Failed to delete bucket {bucket}: {e}") + + # Delete recent hash buckets + for prefix in bucket_prefixes: + bucket = f"malware_scan:hashes:recent:{prefix}" + try: + if redis_conn.delete(bucket): + mb_buckets_deleted += 1 + except Exception as e: + LOGGER.warning(f"[v{VERSION}] [RECOVERY] Failed to delete bucket {bucket}: {e}") + + if mb_buckets_deleted > 0: + LOGGER.info(f"[v{VERSION}] [RECOVERY] Deleted {mb_buckets_deleted} hash buckets for clean reimport") + + for file_config in files_to_update: + file_status = process_file(file_config, previous_import_state) + if file_status == 2: + status = 2 # Error occurred + elif file_status == 1 and status != 2: + status = 1 # Changes made (if no error) + + LOGGER.info(f"[v{VERSION}] Malware hash database updates complete (status={status})") + + # Set import state to "success" if no errors occurred + if status != 2: + try: + redis_conn.set(import_state_key, "success") + LOGGER.info(f"[v{VERSION}] โœ… Import state set to 'success' (status={status})") + except Exception as e: + LOGGER.warning(f"[v{VERSION}] Failed to set import success state: {e}") + else: + LOGGER.warning(f"[v{VERSION}] Import completed with errors - state remains 'running' for recovery") + + finally: + # Stop lock refresh thread if it's running + if lock_refresh_thread and lock_refresh_thread.is_alive(): + lock_refresh_stop.set() + lock_refresh_thread.join(timeout=3.0) # Wait up to 3 seconds for thread to stop + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] Lock refresh heartbeat stopped") + + # Release lock if we acquired it + if lock_acquired: + try: + redis_conn.delete(lock_key) + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] Import lock released") + except Exception as e: + LOGGER.warning(f"[v{VERSION}] Failed to release lock (will expire in 10s): {e}") + + # Mark state keys in Redis to indicate initialization is complete + # This prevents cleanup from running on subsequent job executions + # Always set these keys (not just on first run) to ensure they persist across restarts + try: + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] [STATE] Setting Redis state keys to mark initialization complete") + LOGGER.debug(f"[v{VERSION}] [STATE] Key: malware-scan:hashes:state:full") + LOGGER.debug(f"[v{VERSION}] [STATE] Key: malware-scan:hashes:state:recent") + + redis_conn.set("malware-scan:hashes:state:full", "1") + redis_conn.set("malware-scan:hashes:state:recent", "1") + if DEBUG_MODE: + LOGGER.debug(f"[v{VERSION}] [STATE] Redis state keys marked for initialization completion") + + if DEBUG_MODE: + # Verify keys were set + full_check = redis_conn.exists("malware-scan:hashes:state:full") + recent_check = redis_conn.exists("malware-scan:hashes:state:recent") + LOGGER.debug(f"[v{VERSION}] [STATE] Verification:") + LOGGER.debug(f"[v{VERSION}] [STATE] Full key exists: {full_check}") + LOGGER.debug(f"[v{VERSION}] [STATE] Recent key exists: {recent_check}") + except Exception as e: + LOGGER.warning(f"[v{VERSION}] [STATE] Failed to set state keys in Redis: {e}") + +except SystemExit as e: + status = e.code +except BaseException as e: + status = 2 + LOGGER.debug(format_exc()) + LOGGER.error(f"[v{VERSION}] Exception while running malware-scan-update.py:\n{e}") + +# Log completion +if DEBUG_MODE: + status_name = "SUCCESS" if status == 0 else "CHANGED" if status == 1 else "ERROR" + LOGGER.debug(f"[v{VERSION}] Script completed: {SCRIPT_NAME}") + LOGGER.debug(f"[v{VERSION}] Exit status: {status} ({status_name})") + +sys_exit(status) diff --git a/malware-scan/malware-scan.lua b/malware-scan/malware-scan.lua new file mode 100644 index 0000000..1b216e0 --- /dev/null +++ b/malware-scan/malware-scan.lua @@ -0,0 +1,437 @@ +local class = require("middleclass") +local plugin = require("bunkerweb.plugin") +local utils = require("bunkerweb.utils") +local cjson = require("cjson") +local http = require("resty.http") +-- Note: sha256 and str (resty.string) are now only used in malware_scan_file.lua + +-- Load modules using initialization module +-- Add plugin directory to package path so require() can find the modules +package.path = package.path .. ";/etc/bunkerweb/plugins/malware-scan/?.lua" + +-- Load initialization module +local ms_init = require("malware_scan_init") + +-- Initialize all dependencies (loads modules and sets up cross-dependencies) +local deps = ms_init.initialize() + +-- Extract modules for easier access throughout the code +local ms_logger = deps.modules.logger +local cache = deps.modules.cache +local database = deps.modules.database +local file_ops = deps.modules.file_ops +local clamav = deps.modules.clamav +local virustotal = deps.modules.virustotal +local sentinelone = deps.modules.sentinelone +local malwarebazaar = deps.modules.malwarebazaar +local webhook = deps.modules.webhook +local multipart_full_scan = deps.modules.multipart_full_scan +local ms_utils = deps.modules.ms_utils +local ms_config = deps.modules.ms_config +local ms_orchestrator = deps.modules.ms_orchestrator +local ms_api = deps.modules.ms_api +local ms_scheduler = deps.modules.ms_scheduler +local ui_metrics = deps.modules.ui_metrics + +-- Extract load status for modules that need status checks +local multipart_full_scan_load_ok = deps.status.multipart_full_scan +local ms_utils_load_ok = deps.status.ms_utils +local ms_config_load_ok = deps.status.ms_config +local ms_orchestrator_load_ok = deps.status.ms_orchestrator + +-- Ngx constants +local ngx = ngx +local ERR = ngx.ERR +local WARN = ngx.WARN +local NOTICE = ngx.NOTICE +local INFO = ngx.INFO + +-- Plugin version +local VERSION = "0.8.0" + +local malware_scan = class("malware-scan", plugin) + +local socket = ngx.socket +local HTTP_INTERNAL_SERVER_ERROR = ngx.HTTP_INTERNAL_SERVER_ERROR +local HTTP_OK = ngx.HTTP_OK +local has_variable = utils.has_variable +local get_deny_status = utils.get_deny_status +local tonumber = tonumber +local tostring = tostring +local decode = cjson.decode +local encode = cjson.encode +local http_new = http.new +-- Note: to_hex (str.to_hex) now only used in malware_scan_file.lua + +-- User-Agent for external API calls +local USER_AGENT = "bunkerweb - https://github.com/bunkerity/bunkerweb - malware-scan module v" .. VERSION + +-- Version prefix for debug messages +local DEBUG_PREFIX = "[v" .. VERSION .. "] [DEBUG]" + +-- Initialize plugin instance with context. +function malware_scan:initialize(ctx) + plugin.initialize(self, "malware-scan", ctx) + -- Note: Redis initialization moved to access() phase after file upload check + -- to avoid wasteful initialization on every GET request +end + +-- Init phase: Runs once on manager process (not workers). +-- NOTE: Cleanup is handled by the scheduler job (malware-scan-cache-cleanup.py) +-- which runs at startup and correctly reads environment variables via getenv(). +-- The init() phase cannot reliably detect per-site or environment configuration, +-- as has_variable() only checks BunkerWeb's global database configuration. +function malware_scan:init() + self.logger:log(NOTICE, "[INIT] Entering init phase for malware-scan v" .. VERSION .. " (manager process)") + self.logger:log(NOTICE, "[INIT] Cache cleanup is handled by scheduler job 'malware-scan-cache-cleanup'") + self.logger:log(NOTICE, "[INIT] (Scheduler job correctly reads environment variables, init phase cannot)") + return self:ret(true, "init complete") +end + +-- Helper function to check if debug mode is enabled. +-- Returns true if MALWARE_SCAN_DEBUG=yes, false otherwise. +function malware_scan:is_debug() + return self.variables["MALWARE_SCAN_DEBUG"] == "yes" +end + +-- Log a message only if debug mode is enabled. +-- Uses INFO level for debug messages (semantically correct). +-- Includes version prefix for easier tracking across versions. +function malware_scan:log_debug(message) + if self:is_debug() then + self.logger:log(INFO, DEBUG_PREFIX .. " " .. message) + end +end + +-- Mask sensitive values (API keys, webhook URLs) for security. +-- Delegates to utils.data_mask_sensitive() with plugin context. +--- @param value The sensitive value to mask (API key, webhook URL, etc.) +--- @return string result Masked string or original value if unmasking is enabled +function malware_scan:mask_sensitive(value) + if ms_utils_load_ok and ms_utils then + return ms_utils.data_mask_sensitive(value, self.variables["MALWARE_SCAN_UNMASK_KEYS"]) + end + -- Fallback if utils not loaded: simple masking + return value and #value > 8 and value:sub(1, 4) .. "***" or "***" +end + +-- Sanitize external data (scanner results, filenames) before logging to prevent log injection (CWE-117). +-- Delegates to utils.log_sanitize() for safe logging. +--- @param data The external data to sanitize (scanner signature, API response, etc.) +--- @return string result Sanitized string safe for logging +function malware_scan:sanitize_external_data(data) + if ms_utils_load_ok and ms_utils then + return ms_utils.log_sanitize(data) + end + -- Fallback: simple tostring if utils not loaded + return tostring(data or "") +end + +-- Init worker phase: Runs in each worker process (workers only, not manager). +-- Used for worker-specific initialization like ClamAV connectivity checks. +-- Also handles cache cleanup with distributed locking to ensure only one worker cleans. +function malware_scan:init_worker() + -- Log entry into init_worker for diagnostics + self.logger:log(NOTICE, "[INIT_WORKER] Entering init_worker phase for malware-scan v" .. VERSION) + + local init_needed, err = has_variable("USE_MALWARE_SCANNER", "yes") + if init_needed == nil then + self.logger:log(ERR, "[INIT_WORKER] Failed to check USE_MALWARE_SCANNER: " .. err) + return self:ret(false, "can't check USE_MALWARE_SCANNER variable : " .. err) + end + + -- Check if initialization is needed + if not init_needed then + self.logger:log(NOTICE, "[INIT_WORKER] Malware scanner disabled - skipping worker initialization") + self.logger:log(NOTICE, "[INIT_WORKER] (Cleanup handled by manager process in init phase)") + return self:ret(true, "init_worker not needed (disabled)") + end + + if self.is_loading then + self.logger:log(NOTICE, "[INIT_WORKER] Skipping - plugin is in loading phase") + return self:ret(true, "init_worker not needed (loading)") + end + + self.logger:log(NOTICE, "[INIT_WORKER] Malware scanning enabled - proceeding with initialization") + + -- Load, validate, and dump configuration using config module + if ms_config_load_ok and ms_config then + local module_status = { + clamav = clamav, + cache = cache, + file_ops = file_ops, + virustotal = virustotal, + sentinelone = sentinelone, + webhook = webhook, + } + local cfg_ok, cfg_err = ms_config.load_validate_and_dump(self, module_status) + if not cfg_ok then + return self:ret(false, cfg_err) + end + else + -- Fallback if config module not loaded (log basic info only) + self.logger:log(WARN, "[INIT_WORKER] Config module not loaded - using fallback configuration logging") + self.logger:log(NOTICE, "[INIT_WORKER] USE_MALWARE_SCANNER = " .. (self.variables["USE_MALWARE_SCANNER"] or "(not set)")) + self.logger:log(NOTICE, "[INIT_WORKER] MALWARE_SCAN_CLAMAV_ENABLED = " .. (self.variables["MALWARE_SCAN_CLAMAV_ENABLED"] or "(not set)")) + end + + -- Test ClamAV connectivity if ClamAV scanning is enabled + if clamav and self.variables["MALWARE_SCAN_CLAMAV_ENABLED"] == "yes" then + local clamav_host = self.variables["MALWARE_SCAN_CLAMAV_HOST"] + local clamav_port = self.variables["MALWARE_SCAN_CLAMAV_PORT"] + + self.logger:log(NOTICE, "[INIT_WORKER] Testing ClamAV connectivity to " .. clamav_host .. ":" .. clamav_port .. " (timeout: 2s)") + + local test_ok, test_err = clamav.test_connectivity(self, 2000) + if not test_ok then + self.logger:log(ERR, "[INIT_WORKER] ClamAV connectivity test FAILED: " .. test_err) + return self:ret(false, "ClamAV connectivity test failed: " .. test_err) + end + + self.logger:log(NOTICE, "[INIT_WORKER] โœ“ ClamAV connectivity test PASSED - Worker ready for malware scanning") + else + self.logger:log(NOTICE, "[INIT_WORKER] ClamAV scanning disabled or module not loaded - skipping connectivity test") + end + + -- Cache cleanup with distributed locking (only first worker performs cleanup) + -- Check if any cleanup flag is enabled (granular or master switch) + local cleanup_enabled = self.variables["MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_ALL"] == "yes" + or self.variables["MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_CLAMAV"] == "yes" + or self.variables["MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_VIRUSTOTAL"] == "yes" + or self.variables["MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_SENTINELONE"] == "yes" + or self.variables["MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_MALWAREBAZAAR"] == "yes" + or self.variables["MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_ATTACKER_IPV4"] == "yes" + or self.variables["MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_ATTACKER_IPV6"] == "yes" + + if cleanup_enabled then + if cache then + -- Try to acquire distributed lock + local lock_acquired = cache.try_acquire_cleanup_lock(self) + if lock_acquired then + self.logger:log(NOTICE, "[CACHE_CLEANUP] Lock acquired - this worker will perform cleanup") + local cleanup_ok = cache.cleanup_all(self) + if not cleanup_ok then + self.logger:log(WARN, "[CACHE_CLEANUP] Cache cleanup encountered errors") + else + self.logger:log(NOTICE, "[CACHE_CLEANUP] Cache cleanup completed successfully") + end + -- Release lock + cache.release_cleanup_lock(self) + else + self.logger:log(NOTICE, "[CACHE_CLEANUP] Another worker is handling cleanup - skipping") + end + else + self.logger:log(WARN, "[CACHE_CLEANUP] Cache cleanup requested but cache module not loaded") + end + end + + -- Cleanup orphaned files (workers only - they handle uploads) + if self.variables["MALWARE_SCAN_CLEANUP_ORPHANED"] == "yes" and file_ops then + local cleanup_age = tonumber(self.variables["MALWARE_SCAN_CLEANUP_AGE"]) or 3600 + local cleanup_ok, cleanup_err = file_ops.orphan_cleanup(self, cleanup_age) + if not cleanup_ok then + self.logger:log( + WARN, + "failed to cleanup orphaned files: " .. cleanup_err + ) + end + end + + -- Initialize UI metrics to 0 so UI has data to display even before first scan + self:set_metric("counter_malware_detected", 0) + self:set_metric("counter_files_scanned", 0) + self:set_metric("counter_cache_hits", 0) + self.logger:log(NOTICE, "[INIT_WORKER] Initialized UI metrics counters") + + return self:ret(true, "success") +end + +-- Access phase handler - scan uploaded files for malware. +-- Checks for file uploads, scans with ClamAV and/or VirusTotal, and blocks infected files. +function malware_scan:access() + self:log_debug("=== MALWARE-SCAN ACCESS PHASE CALLED ===") + self:log_debug("Request: " .. tostring(self.ctx.bw.uri) .. " Method: " .. tostring(self.ctx.bw.request_method)) + self:log_debug("Content-Type: " .. tostring(self.ctx.bw.http_content_type)) + + -- PERFORMANCE: Check for file upload FIRST before any expensive operations + -- Exit immediately for non-upload requests to avoid unnecessary processing + if + not self.ctx.bw.http_content_type + or ( + not self.ctx.bw.http_content_type:match("boundary") + or not self.ctx.bw.http_content_type:match("multipart/form%-data") + ) + then + return self:ret(true, "no file upload detected") + end + + self:log_debug("File upload detected, initializing malware scan") + + -- Initialize Redis connection for MalwareBazaar hash lookups (deferred from initialize phase) + -- Only do this for actual file uploads to avoid wasteful initialization on GET requests + local redis_ok, redis_err = database.connect(self) + if not redis_ok then + self.logger:log(WARN, "Redis setup failed: " .. (redis_err or "unknown error")) + end + + -- Pre-scan validation and initialization using orchestrator + local scan_ctx, early_result + if ms_orchestrator_load_ok and ms_orchestrator then + scan_ctx, early_result = ms_orchestrator.pre_scan_validate(self, file_ops, ui_metrics, webhook, get_deny_status) + if not scan_ctx then + return early_result + end + else + -- Fallback if orchestrator not loaded (should not happen in production) + self.logger:log(ERR, "Orchestrator not loaded - using legacy code path") + return self:ret(false, "orchestrator module not available") + end + + -- Extract variables from scan context + local scanner_state = scan_ctx.scanner_state + local body_file = scan_ctx.body_file + local checksum = scan_ctx.checksum + local checksum_ok = scan_ctx.checksum_ok + local file_size = scan_ctx.file_size + local detections = scan_ctx.detections + + -- Extract scanner states for compatibility + local clamav_enabled = scanner_state.clamav + local virustotal_enabled = scanner_state.virustotal + local sentinelone_enabled = scanner_state.sentinelone + local malwarebazaar_enabled = scanner_state.malwarebazaar + + -- Comprehensive cache checking using orchestrator + local cache_result + if ms_orchestrator_load_ok and ms_orchestrator then + cache_result = ms_orchestrator.cache_check_comprehensive( + self, scan_ctx, cache, webhook, file_ops, ui_metrics, get_deny_status + ) + + -- If blocked by cache, return immediately + if cache_result.blocked then + return cache_result.block_result + end + + -- Update scanner state from cache results + scanner_state = cache_result.scanner_state + detections = cache_result.detections + + -- Extract updated scanner states + clamav_enabled = scanner_state.clamav + virustotal_enabled = scanner_state.virustotal + sentinelone_enabled = scanner_state.sentinelone + malwarebazaar_enabled = scanner_state.malwarebazaar + else + -- Fallback if orchestrator not loaded (should not happen in production) + self.logger:log(ERR, "Orchestrator cache check not available - using legacy code path") + end + + -- Configure scan limits using orchestrator + local scan_limits + if ms_orchestrator_load_ok and ms_orchestrator then + scan_limits = ms_orchestrator.configure_scan_limits(self, scan_ctx) + else + -- Fallback if orchestrator not loaded (should not happen in production) + self.logger:log(ERR, "Orchestrator configure_scan_limits not available - using legacy code path") + return self:ret(false, "orchestrator module not available") + end + + -- Extract configuration from scan limits + local clamav_actually_scanned = false + local clamav_skipped_reason = nil + + -- Update scanner state for body scan + scanner_state = scan_limits.scanner_state + clamav_enabled = scanner_state.clamav + virustotal_enabled = scanner_state.virustotal + sentinelone_enabled = scanner_state.sentinelone + malwarebazaar_enabled = scanner_state.malwarebazaar + + -- ============================================================================ + -- SCANNER INVOCATION: Orchestrated body scan across all enabled scanners + -- ============================================================================ + -- Call orchestrator to handle all scanner invocations (MalwareBazaar, ClamAV, VirusTotal, SentinelOne) + -- This centralizes ~400-500 lines of scanner logic into a single function call + + -- Update scanner state with current enabled flags (may have been disabled by size checks or multipart logic) + scanner_state.clamav = clamav_enabled + scanner_state.virustotal = virustotal_enabled + scanner_state.sentinelone = sentinelone_enabled + scanner_state.malwarebazaar = malwarebazaar_enabled + + local blocking_result, detected, scan_info + if ms_orchestrator_load_ok and ms_orchestrator then + blocking_result, detected, scan_info = ms_orchestrator.body_scan( + self, body_file, checksum, checksum_ok, file_size, + scanner_state, detections, + clamav, virustotal, sentinelone, malwarebazaar, + cache, webhook, file_ops, get_deny_status + ) + + -- Extract scan info + clamav_actually_scanned = scan_info.clamav_actually_scanned or clamav_actually_scanned + clamav_skipped_reason = scan_info.clamav_skipped_reason or clamav_skipped_reason + checksum = scan_info.checksum + checksum_ok = scan_info.checksum_ok + + -- If orchestrator returned a blocking result, return it immediately + if blocking_result then + return blocking_result + end + else + -- Fallback if orchestrator not loaded (graceful degradation) + self.logger:log(ERR, "[ORCHESTRATOR] body_scan() not available - scanner invocation disabled") + detected = "clean" + end + + -- ============================================================================ + -- POST-SCAN PROCESSING + -- ============================================================================ + -- From here onwards, all scanners have completed (either clean or resume mode) + + -- Build scan result object for post-scan processing + local scan_result = { + detected = detected, + checksum = checksum, + checksum_ok = checksum_ok, + clamav_actually_scanned = clamav_actually_scanned + } + + -- Call orchestrator for post-scan processing + if ms_orchestrator_load_ok and ms_orchestrator then + return ms_orchestrator.post_scan_complete( + self, scan_ctx, scan_limits, scan_result, + webhook, file_ops, multipart_full_scan, multipart_full_scan_load_ok, get_deny_status + ) + else + -- Fallback if orchestrator not loaded (should not happen in production) + self.logger:log(ERR, "Orchestrator post_scan_complete not available - using legacy fallback") + return self:ret(true, "scan complete (orchestrator unavailable)") + end +end + +-- API endpoint handler - delegates to API module +-- Handles test/diagnostic API endpoints (ping, test-eicar, virustotal-ping, etc.) +function malware_scan:api() + if ms_api then + return self:ret(ms_api.handle_request(self)) + end + return self:ret(false, "success") +end + +-- ============================================================================ +-- SCHEDULED TASKS +-- ============================================================================ + +-- Scheduler hook: Runs periodically to update MalwareBazaar custom hash database +-- Called by BunkerWeb scheduler at intervals defined by MALWARE_SCAN_CUSTOM_HASHES_UPDATE_INTERVAL +-- Uses smart download with ETag caching to minimize bandwidth usage +function malware_scan:scheduler() + if ms_scheduler then + ms_scheduler.run(self) + end + return self:ret(true, "success") +end + +return malware_scan diff --git a/malware-scan/malware_scan_api.lua b/malware-scan/malware_scan_api.lua new file mode 100644 index 0000000..e351fd7 --- /dev/null +++ b/malware-scan/malware_scan_api.lua @@ -0,0 +1,739 @@ +-- API Handler Module for Malware Scan Plugin +-- Handles test/diagnostic API endpoints +-- +-- Endpoints: +-- POST /malware-scan/ping - Test ClamAV connectivity +-- POST /malware-scan/test-eicar - Test ClamAV malware detection with EICAR +-- POST /malware-scan/virustotal-ping - Test VirusTotal API connectivity +-- POST /malware-scan/malwarebazaar-stats - Get MalwareBazaar hash database statistics +-- POST /malware-scan/clamav-stats - Get ClamAV version and definition statistics +-- POST /malware-scan/test-discord - Test Discord webhook connectivity +-- POST /malware-scan/test-teams - Test Microsoft Teams webhook connectivity +-- +-- Version: 0.8.0 +-- Date: 2026-01-27 + +local api = {} + +-- Module version +api.VERSION = "0.8.0" + +-- Get required modules +local cjson = require("cjson") +local logger = require("malware_scan_logger") +local utils = require("malware_scan_utils") +local database = require("malware_scan_database") +local clamav = require("malware_scan_clamav") +local virustotal = require("malware_scan_virustotal") +local http = require("resty.http") + +local encode = cjson.encode + +local ngx = ngx +local HTTP_OK = ngx.HTTP_OK +local HTTP_INTERNAL_SERVER_ERROR = ngx.HTTP_INTERNAL_SERVER_ERROR + +-- ============================================================================ +-- CLAMAV API ENDPOINTS +-- ============================================================================ + +-- Test ClamAV connectivity with PING command +-- Also includes MalwareBazaar and ClamAV statistics in response +--- @param plugin table Plugin instance +--- @return boolean success Success flag +--- @return string response JSON response +--- @return number status HTTP status code +function api.clamav_ping(plugin) + if plugin.variables["USE_MALWARE_SCANNER"] ~= "yes" then + local response = encode({ + status = "warning", + message = "Malware scan plugin not enabled" + }) + return true, response, HTTP_OK + end + + local ok, data = clamav.command(plugin, "PING") + if not ok then + local response = encode({ + status = "error", + message = "connectivity with ClamAV failed : " .. tostring(data) + }) + return true, response, HTTP_INTERNAL_SERVER_ERROR + end + if data ~= "PONG" then + local response = encode({ + status = "error", + message = "wrong data received from ClamAV : " .. tostring(data) + }) + return true, response, HTTP_INTERNAL_SERVER_ERROR + end + + -- Build response with stats + local response_data = { + status = "success", + message = "ClamAV is operational" + } + + -- Add MalwareBazaar stats if enabled + if plugin.variables["MALWARE_SCAN_MALWAREBAZAAR_HASH_LOOKUP"] == "yes" and database.is_available(plugin) then + local buckets = {"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "A", "B", "C", "D", "E", "F"} + local total_hashes = 0 + + for _, bucket in ipairs(buckets) do + local bucket_key = "malware_scan:hashes:" .. bucket + local count = database.hlen(plugin, bucket_key) + if count and type(count) == "number" then + total_hashes = total_hashes + count + end + end + + -- Get database state + local full_state = database.redis_get(plugin, "malware-scan:hashes:state:full") or "never" + local recent_state = database.redis_get(plugin, "malware-scan:hashes:state:recent") or "never" + local import_state = database.redis_get(plugin, "malware-scan:hashes:import_state") or "idle" + local full_start_time = database.redis_get(plugin, "malware-scan:hashes:import_start_time:full") + local full_end_time = database.redis_get(plugin, "malware-scan:hashes:import_end_time:full") + local recent_start_time = database.redis_get(plugin, "malware-scan:hashes:import_start_time:recent") + local recent_end_time = database.redis_get(plugin, "malware-scan:hashes:import_end_time:recent") + local full_download_ever = database.redis_get(plugin, "malware_scan:full_download_ever_done") + + response_data.malwarebazaar = { + total_hashes = total_hashes, + buckets = 16, + full_db_status = full_state, + recent_db_status = recent_state, + import_status = import_state, + full_import_start_time = (full_start_time and full_start_time ~= ngx.null) and full_start_time or nil, + full_import_end_time = (full_end_time and full_end_time ~= ngx.null) and full_end_time or nil, + recent_import_start_time = (recent_start_time and recent_start_time ~= ngx.null) and recent_start_time or nil, + recent_import_end_time = (recent_end_time and recent_end_time ~= ngx.null) and recent_end_time or nil, + initialized = full_download_ever ~= nil and full_download_ever ~= ngx.null, + import_progress = { + full = { + total = tonumber(database.redis_get(plugin, "malware-scan:hashes:import_total:full")) or 0, + processed = (function() + local sum = 0 + for _, bucket in ipairs(buckets) do + sum = sum + (tonumber(database.redis_get(plugin, "malware-scan:hashes:import_processed:full:bucket_" .. bucket)) or 0) + end + return sum + end)() + }, + recent = { + total = tonumber(database.redis_get(plugin, "malware-scan:hashes:import_total:recent")) or 0, + processed = (function() + local sum = 0 + for _, bucket in ipairs(buckets) do + sum = sum + (tonumber(database.redis_get(plugin, "malware-scan:hashes:import_processed:recent:bucket_" .. bucket)) or 0) + end + return sum + end)() + } + } + } + end + + -- Add ClamAV version stats + local version_ok, version_data = clamav.command(plugin, "VERSION") + if version_ok then + local clamav_version = (version_data and version_data:match("ClamAV ([^/]+)")) or "unknown" + local definition_version = (version_data and version_data:match("/(%d+)/")) or "unknown" + local definition_date = (version_data and version_data:match("/%d+/(.+)$")) or "unknown" + + response_data.clamav_stats = { + version = clamav_version, + definitions = definition_version, + updated = definition_date + } + end + + local response = encode(response_data) + return true, response, HTTP_OK +end + +-- Test ClamAV malware detection with EICAR test file +--- @param plugin table Plugin instance +--- @return boolean success Success flag +--- @return string response JSON response +--- @return number status HTTP status code +function api.clamav_test_eicar(plugin) + if plugin.variables["USE_MALWARE_SCANNER"] ~= "yes" then + local response = encode({ + status = "warning", + message = "Malware scan plugin not enabled" + }) + return true, response, HTTP_OK + end + + -- EICAR test virus string + local eicar = "X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*" + + logger.log_error("Testing ClamAV with EICAR test file") + local ok, detected = clamav.scan_content_instream(plugin, eicar) + + if not ok then + local response = encode({ + status = "error", + message = "ClamAV scan failed: " .. tostring(detected) + }) + return true, response, HTTP_INTERNAL_SERVER_ERROR + end + + if detected == "clean" then + local response = encode({ + status = "error", + message = "WARNING: ClamAV did not detect EICAR test file (expected: Eicar-Signature, got: clean)" + }) + return true, response, HTTP_INTERNAL_SERVER_ERROR + end + + -- Successfully detected EICAR + local response = encode({ + status = "success", + message = "ClamAV successfully detected EICAR test file: " .. tostring(detected) + }) + return true, response, HTTP_OK +end + +-- ============================================================================ +-- STATISTICS API ENDPOINTS +-- ============================================================================ + +-- Get MalwareBazaar hash database statistics +-- Returns count of hashes across all 16 Redis buckets +--- @param plugin table Plugin instance +--- @return boolean success Success flag +--- @return string response JSON response +--- @return number status HTTP status code +function api.malwarebazaar_stats(plugin) + if plugin.variables["USE_MALWARE_SCANNER"] ~= "yes" then + local response = encode({ + status = "warning", + message = "Malware scan plugin not enabled" + }) + return true, response, HTTP_OK + end + + if plugin.variables["MALWARE_SCAN_MALWAREBAZAAR_HASH_LOOKUP"] ~= "yes" then + local response = encode({ + status = "success", + total_hashes = 0, + message = "MalwareBazaar hash lookup not enabled" + }) + return true, response, HTTP_OK + end + + if not database.is_available(plugin) then + local response = encode({ + status = "error", + message = "Redis not available" + }) + return true, response, HTTP_INTERNAL_SERVER_ERROR + end + + -- Count hashes across all 16 buckets (0-9, A-F) + local buckets = {"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "A", "B", "C", "D", "E", "F"} + local total_hashes = 0 + local bucket_counts = {} + + for _, bucket in ipairs(buckets) do + local bucket_key = "malware_scan:hashes:" .. bucket + local count = database.hlen(plugin, bucket_key) + if count and type(count) == "number" then + total_hashes = total_hashes + count + bucket_counts[bucket] = count + else + bucket_counts[bucket] = 0 + end + end + + -- Get hash database state information + local full_state = database.redis_get(plugin, "malware-scan:hashes:state:full") or "never" + local recent_state = database.redis_get(plugin, "malware-scan:hashes:state:recent") or "never" + local import_state = database.redis_get(plugin, "malware-scan:hashes:import_state") or "unknown" + local full_download_ever = database.redis_get(plugin, "malware_scan:full_download_ever_done") + + -- Get import progress information + local full_import_total = tonumber(database.redis_get(plugin, "malware-scan:hashes:import_total:full")) or 0 + local recent_import_total = tonumber(database.redis_get(plugin, "malware-scan:hashes:import_total:recent")) or 0 + + -- Sum up processed counts from all 16 buckets + local full_import_processed = 0 + local recent_import_processed = 0 + for _, bucket in ipairs(buckets) do + local full_processed = tonumber(database.redis_get(plugin, "malware-scan:hashes:import_processed:full:bucket_" .. bucket)) or 0 + local recent_processed = tonumber(database.redis_get(plugin, "malware-scan:hashes:import_processed:recent:bucket_" .. bucket)) or 0 + full_import_processed = full_import_processed + full_processed + recent_import_processed = recent_import_processed + recent_processed + end + + local response = encode({ + status = "success", + total_hashes = total_hashes, + bucket_counts = bucket_counts, + buckets = 16, + database_state = { + full_download_status = full_state, + recent_download_status = recent_state, + import_status = import_state, + full_database_initialized = full_download_ever ~= nil and full_download_ever ~= ngx.null + }, + import_progress = { + full = { + total = full_import_total, + processed = full_import_processed + }, + recent = { + total = recent_import_total, + processed = recent_import_processed + } + } + }) + return true, response, HTTP_OK +end + +-- Get ClamAV virus definition statistics +-- Returns version and definition information +--- @param plugin table Plugin instance +--- @return boolean success Success flag +--- @return string response JSON response +--- @return number status HTTP status code +function api.clamav_stats(plugin) + if plugin.variables["USE_MALWARE_SCANNER"] ~= "yes" then + local response = encode({ + status = "warning", + message = "Malware scan plugin not enabled" + }) + return true, response, HTTP_OK + end + + -- Get ClamAV version info + local ok, version_data = clamav.command(plugin, "VERSION") + if not ok then + local response = encode({ + status = "error", + message = "Failed to get ClamAV version: " .. tostring(version_data) + }) + return true, response, HTTP_INTERNAL_SERVER_ERROR + end + + -- Parse version string (format: "ClamAV 1.0.0/26000/Mon Jan 1 12:00:00 2024") + local clamav_version = (version_data and version_data:match("ClamAV ([^/]+)")) or "unknown" + local definition_version = (version_data and version_data:match("/(%d+)/")) or "unknown" + local definition_date = (version_data and version_data:match("/%d+/(.+)$")) or "unknown" + + local response = encode({ + status = "success", + clamav_version = clamav_version, + definition_version = definition_version, + definition_date = definition_date, + raw_version = version_data + }) + return true, response, HTTP_OK +end + +-- ============================================================================ +-- WEBHOOK TEST ENDPOINTS +-- ============================================================================ + +-- Test Discord webhook connectivity +-- Sends a test notification to verify webhook configuration +--- @param plugin table Plugin instance +--- @return boolean success Success flag +--- @return string response JSON response +--- @return number status HTTP status code +function api.test_discord(plugin) + if plugin.variables["USE_MALWARE_SCANNER"] ~= "yes" then + local response = encode({ + status = "warning", + message = "Malware scan plugin not enabled" + }) + return true, response, HTTP_OK + end + + local discord_url = plugin.variables["MALWARE_SCAN_DISCORD_WEBHOOK_URL"] + if not discord_url or discord_url == "" or discord_url == "your_webhook_url" then + local response = encode({ + status = "error", + message = "Discord webhook not configured" + }) + return true, response, HTTP_INTERNAL_SERVER_ERROR + end + + -- Parse URL + local url_parts, err = utils.parse_webhook_url(discord_url) + if not url_parts then + local response = encode({ + status = "error", + message = "Invalid Discord webhook URL: " .. tostring(err) + }) + return true, response, HTTP_INTERNAL_SERVER_ERROR + end + + -- Build test message (Discord format) + local test_payload = encode({ + embeds = { + { + title = "โœ… Discord Webhook Test", + description = "This is a test message from BunkerWeb Malware Scanner", + color = 65280, -- Green color + fields = { + {name = "Status", value = "Webhook is configured correctly", inline = false}, + {name = "Instance", value = ngx.var.hostname or "unknown", inline = true}, + {name = "Test Time", value = os.date("!%Y-%m-%d %H:%M:%S UTC"), inline = true} + }, + timestamp = os.date("!%Y-%m-%dT%H:%M:%SZ") + } + } + }) + + -- Send test request + local httpc = http.new() + httpc:set_timeout(5000) + + local ok, conn_err = httpc:connect(url_parts.host, url_parts.port) + if not ok then + local response = encode({ + status = "error", + message = "Failed to connect to Discord: " .. tostring(conn_err) + }) + return true, response, HTTP_INTERNAL_SERVER_ERROR + end + + if url_parts.ssl then + local session, ssl_err = httpc:ssl_handshake(nil, url_parts.host, false) + if not session then + httpc:close() + local response = encode({ + status = "error", + message = "SSL handshake failed: " .. tostring(ssl_err) + }) + return true, response, HTTP_INTERNAL_SERVER_ERROR + end + end + + local res, req_err = httpc:request({ + method = "POST", + path = url_parts.path, + headers = { + ["Host"] = url_parts.host, + ["Content-Type"] = "application/json", + ["Content-Length"] = tostring(#test_payload) + }, + body = test_payload + }) + + httpc:close() + + if not res then + local response = encode({ + status = "error", + message = "Request failed: " .. tostring(req_err) + }) + return true, response, HTTP_INTERNAL_SERVER_ERROR + end + + if res.status >= 200 and res.status < 300 then + local response = encode({ + status = "success", + message = "Discord webhook test successful" + }) + return true, response, HTTP_OK + else + local response = encode({ + status = "error", + message = "Discord returned HTTP " .. res.status .. ": " .. (res.body or "no body") + }) + return true, response, HTTP_INTERNAL_SERVER_ERROR + end +end + +-- Test Microsoft Teams webhook connectivity +-- Sends a test notification to verify webhook configuration +--- @param plugin table Plugin instance +--- @return boolean success Success flag +--- @return string response JSON response +--- @return number status HTTP status code +function api.test_teams(plugin) + if plugin.variables["USE_MALWARE_SCANNER"] ~= "yes" then + local response = encode({ + status = "warning", + message = "Malware scan plugin not enabled" + }) + return true, response, HTTP_OK + end + + local teams_url = plugin.variables["MALWARE_SCAN_TEAMS_WEBHOOK_URL"] + if not teams_url or teams_url == "" or teams_url == "your_webhook_url" then + local response = encode({ + status = "error", + message = "Microsoft Teams webhook not configured" + }) + return true, response, HTTP_INTERNAL_SERVER_ERROR + end + + -- Parse URL + local url_parts, err = utils.parse_webhook_url(teams_url) + if not url_parts then + local response = encode({ + status = "error", + message = "Invalid Teams webhook URL: " .. tostring(err) + }) + return true, response, HTTP_INTERNAL_SERVER_ERROR + end + + -- Build test message (Teams MessageCard format) + local test_payload = encode({ + ["@type"] = "MessageCard", + ["@context"] = "https://schema.org/extensions", + themeColor = "00FF00", -- Green color + summary = "Webhook Test", + sections = { + { + activityTitle = "โœ… Microsoft Teams Webhook Test", + activitySubtitle = "This is a test message from BunkerWeb Malware Scanner", + facts = { + {name = "Status", value = "Webhook is configured correctly"}, + {name = "Instance", value = ngx.var.hostname or "unknown"}, + {name = "Test Time", value = os.date("!%Y-%m-%d %H:%M:%S UTC")} + }, + markdown = true + } + } + }) + + -- Send test request + local httpc = http.new() + httpc:set_timeout(5000) + + local ok, conn_err = httpc:connect(url_parts.host, url_parts.port) + if not ok then + local response = encode({ + status = "error", + message = "Failed to connect to Teams: " .. tostring(conn_err) + }) + return true, response, HTTP_INTERNAL_SERVER_ERROR + end + + if url_parts.ssl then + local session, ssl_err = httpc:ssl_handshake(nil, url_parts.host, false) + if not session then + httpc:close() + local response = encode({ + status = "error", + message = "SSL handshake failed: " .. tostring(ssl_err) + }) + return true, response, HTTP_INTERNAL_SERVER_ERROR + end + end + + local res, req_err = httpc:request({ + method = "POST", + path = url_parts.path, + headers = { + ["Host"] = url_parts.host, + ["Content-Type"] = "application/json", + ["Content-Length"] = tostring(#test_payload) + }, + body = test_payload + }) + + httpc:close() + + if not res then + local response = encode({ + status = "error", + message = "Request failed: " .. tostring(req_err) + }) + return true, response, HTTP_INTERNAL_SERVER_ERROR + end + + if res.status >= 200 and res.status < 300 then + local response = encode({ + status = "success", + message = "Microsoft Teams webhook test successful" + }) + return true, response, HTTP_OK + else + local response = encode({ + status = "error", + message = "Teams returned HTTP " .. res.status .. ": " .. (res.body or "no body") + }) + return true, response, HTTP_INTERNAL_SERVER_ERROR + end +end + +-- ============================================================================ +-- VIRUSTOTAL API ENDPOINTS +-- ============================================================================ + +-- Test VirusTotal API connectivity +-- Uses EICAR test file SHA256 to verify API access +--- @param plugin table Plugin instance +--- @return boolean success Success flag +--- @return string response JSON response +--- @return number status HTTP status code +function api.virustotal_ping(plugin) + if plugin.variables["USE_MALWARE_SCANNER"] ~= "yes" then + local response = encode({ + status = "warning", + message = "Malware scan plugin not enabled" + }) + return true, response, HTTP_OK + end + + if plugin.variables["MALWARE_SCAN_API_USE_VIRUSTOTAL"] ~= "yes" then + local response = encode({ + status = "error", + message = "VirusTotal integration not enabled" + }) + return true, response, HTTP_INTERNAL_SERVER_ERROR + end + + if not plugin.variables["MALWARE_SCAN_API_VIRUSTOTAL_KEY"] + or plugin.variables["MALWARE_SCAN_API_VIRUSTOTAL_KEY"] == "" then + local response = encode({ + status = "error", + message = "VirusTotal API key not configured" + }) + return true, response, HTTP_INTERNAL_SERVER_ERROR + end + + -- Test with EICAR test file SHA256 checksum + local ok, found, vt_response = virustotal.virustotal_request( + plugin, + "/files/275a021bbfb6489e54d471899f7db9d1663fc695ec2fe2a2c4538aabf651fd0f" + ) + if not ok then + local response = encode({ + status = "error", + message = "error while connecting to VirusTotal API: " .. tostring(found) + }) + return true, response, HTTP_INTERNAL_SERVER_ERROR + end + if not found then + local response = encode({ + status = "error", + message = "error while testing VirusTotal: EICAR test file not found on VirusTotal" + }) + return true, response, HTTP_INTERNAL_SERVER_ERROR + end + + local response = encode({ + status = "success", + message = "VirusTotal API is operational" + }) + return true, response, HTTP_OK +end + +-- ============================================================================ +-- MAIN API HANDLER +-- ============================================================================ + +-- Route API requests to appropriate handler based on endpoint +--- @param plugin table Plugin instance +--- @return boolean success Success flag +--- @return string message Status message +--- @return number|string|nil status HTTP status code or response content +function api.handle_request(plugin) + local uri = plugin.ctx.bw.uri + local method = plugin.ctx.bw.request_method + + -- ClamAV connectivity test + if uri == "/malware-scan/ping" and method == "POST" then + return api.clamav_ping(plugin) + end + + -- ClamAV EICAR test + if uri == "/malware-scan/test-eicar" and method == "POST" then + return api.clamav_test_eicar(plugin) + end + + -- VirusTotal connectivity test + if uri == "/malware-scan/virustotal-ping" and method == "POST" then + return api.virustotal_ping(plugin) + end + + -- MalwareBazaar statistics + if uri == "/malware-scan/malwarebazaar-stats" and method == "POST" then + return api.malwarebazaar_stats(plugin) + end + + -- ClamAV statistics + if uri == "/malware-scan/clamav-stats" and method == "POST" then + return api.clamav_stats(plugin) + end + + -- Discord webhook test + if uri == "/malware-scan/test-discord" and method == "POST" then + return api.test_discord(plugin) + end + + -- Teams webhook test + if uri == "/malware-scan/test-teams" and method == "POST" then + return api.test_teams(plugin) + end + + -- Download full hash database + if uri == "/malware-scan/download-full-db" and method == "GET" then + return api.download_hash_db(plugin, "full") + end + + -- Download recent hash database + if uri == "/malware-scan/download-recent-db" and method == "GET" then + return api.download_hash_db(plugin, "recent") + end + + -- Not an API endpoint we handle + return false, "success", nil +end + +-- Download hash database file (full or recent) +--- @param plugin table Plugin instance +--- @param db_type string Database type ("full" or "recent") +--- @return boolean success Success flag +--- @return string status Status message +--- @return string|nil response JSON response or nil +function api.download_hash_db(plugin, db_type) + local file_path + if db_type == "full" then + file_path = "/var/cache/bunkerweb/malware-scan/hashes_full.zip" + elseif db_type == "recent" then + file_path = "/var/cache/bunkerweb/malware-scan/hashes_recent.zip" + else + return true, "error", cjson.encode({status = "error", message = "Invalid database type"}) + end + + -- Check if file exists + local file = io.open(file_path, "rb") + if not file then + logger.log_error( string.format("[v%s] [DOWNLOAD] Hash database file not found: %s", api.VERSION, file_path)) + return true, "error", cjson.encode({status = "error", message = "Hash database file not found"}) + end + + -- Read file contents + local content = file:read("*all") + file:close() + + if not content or #content == 0 then + logger.log_error( string.format("[v%s] [DOWNLOAD] Hash database file is empty: %s", api.VERSION, file_path)) + return true, "error", cjson.encode({status = "error", message = "Hash database file is empty"}) + end + + -- Set response headers for file download + local filename = db_type == "full" and "malwarebazaar_full_split.zip" or "malwarebazaar_recent_split.zip" + ngx.header["Content-Type"] = "application/zip" + ngx.header["Content-Disposition"] = string.format('attachment; filename="%s"', filename) + ngx.header["Content-Length"] = #content + ngx.header["Cache-Control"] = "no-cache, no-store, must-revalidate" + ngx.header["Pragma"] = "no-cache" + ngx.header["Expires"] = "0" + + logger.log_notice( string.format("[v%s] [DOWNLOAD] Serving %s hash database: %s (%d bytes)", api.VERSION, db_type, file_path, #content)) + + -- Send file content + ngx.print(content) + return true, "success", nil +end + +return api diff --git a/malware-scan/malware_scan_cache.lua b/malware-scan/malware_scan_cache.lua new file mode 100644 index 0000000..e12ac1b --- /dev/null +++ b/malware-scan/malware_scan_cache.lua @@ -0,0 +1,1634 @@ +-- BunkerWeb Malware Scan - Cache Module +-- Handles all caching operations including local cache and Redis/Valkey shared cache +-- Includes circuit breaker, composite cache, attacker tracking, and differential TTL + +local cjson = require("cjson") +local utils = require("bunkerweb.utils") +local logger = require("malware_scan_logger") +local ms_utils = require("malware_scan_utils") +local database = require("malware_scan_database") +local lfs = require("lfs") + +-- Load metrics module (optional - gracefully handles if not available) +local metrics_ok, metrics = pcall(require, "malware_scan_metrics") +if not metrics_ok then + metrics = nil -- Metrics not available +end + +local ngx = ngx +local tonumber = tonumber +local tostring = tostring + +-- Module table +local cache = {} + +-- Module version +cache.VERSION = "0.8.0" + +-- ============================================================================ +-- TYPE ALIASES (for documentation and LuaLS type checking) +-- ============================================================================ + +---@alias Scanner string Scanner name (e.g., "virustotal", "sentinelone", "malwarebazaar") +---@alias Number number Numeric value +---@alias Timestamp number Unix timestamp in seconds +---@alias Error string Error message or type + +-- ============================================================================ +-- HELPER FUNCTIONS +-- ============================================================================ + +--- Remove directory recursively (like rm -rf) +--- @param path string Directory path to remove +--- @return boolean success True if removal succeeded, false otherwise +local function remove_directory_recursive(path) + if not path or path == "" then + return false + end + + -- Check if directory exists + local attr = lfs.attributes(path, "mode") + if not attr then + return true -- Already doesn't exist + end + + if attr ~= "directory" then + -- It's a file, just remove it + return os.remove(path) ~= nil + end + + -- Recursively remove directory contents + for entry in lfs.dir(path) do + if entry ~= "." and entry ~= ".." then + local entry_path = path .. "/" .. entry + local entry_attr = lfs.attributes(entry_path, "mode") + + if entry_attr == "directory" then + -- Recursively remove subdirectory + remove_directory_recursive(entry_path) + else + -- Remove file + os.remove(entry_path) + end + end + end + + -- Remove the now-empty directory + return lfs.rmdir(path) ~= nil +end + + +-- ============================================================================ +-- API ERROR CIRCUIT BREAKER FUNCTIONS +-- Protects against API failures (user_blacklisted, rate limits, etc.) with exponential backoff +-- Prevents repeated failed API calls that waste resources and may lead to permanent blocks +-- ============================================================================ + +-- Get API error circuit breaker state from local cache. +-- Returns failure_count, last_failure_time, and error_type. +--- @param plugin Plugin instance +--- @param scanner Scanner name (e.g., "malwarebazaar", "sentinelone") +--- @return number failure_count +--- @return number last_failure_time +--- @return string|nil error_type +function cache.api_error_get(plugin, scanner) + local cb_key = "plugin_malware_scan_api_error_" .. scanner + local ok, data = plugin.cachestore:get(cb_key) + if not ok or not data then + return 0, 0, nil -- No errors recorded + end + + -- Data format: "failure_count:last_failure_time:error_type" + local count, time, error_type = data:match("^(%d+):(%d+):(.+)$") + if not count or not time then + return 0, 0, nil + end + + return tonumber(count) or 0, tonumber(time) or 0, error_type +end + +-- Update API error circuit breaker state in local cache and Redis. +-- Stores failure count, timestamp, and error type for later retry logic. +--- @param plugin Plugin instance +--- @param scanner Scanner name (e.g., "malwarebazaar", "sentinelone") +--- @param failure_count Number of consecutive failures +--- @param last_failure_time Timestamp of last failure +--- @param error_type Error type (e.g., "user_blacklisted", "no_api_key", "rate_limit") +function cache.api_error_set(plugin, scanner, failure_count, last_failure_time, error_type) + -- Validate numeric inputs to prevent corruption (Medium severity issue #11) + failure_count = tonumber(failure_count) or 0 + last_failure_time = tonumber(last_failure_time) or 0 + + -- Bounds checking: prevent negative values and excessive counts + if failure_count < 0 or failure_count > 1000000 then + logger.log_warn("[CACHE] Invalid API error failure_count (" .. tostring(failure_count) .. "), resetting to 0") + failure_count = 0 + end + + if last_failure_time < 0 then + logger.log_warn("[CACHE] Invalid API error last_failure_time (" .. tostring(last_failure_time) .. "), resetting to 0") + last_failure_time = 0 + end + + local cb_key = "plugin_malware_scan_api_error_" .. scanner + local data = tostring(failure_count) .. ":" .. tostring(last_failure_time) .. ":" .. tostring(error_type) + + -- Calculate TTL based on exponential backoff (1 min to 24 hours) + local backoff_seconds = cache.api_error_backoff_calculate(failure_count) + -- Add 60 seconds buffer to TTL to ensure data persists through backoff period + local ttl = backoff_seconds + 60 + + -- Store in local cache + plugin.cachestore:set(cb_key, data, ttl) + + -- Also store in Redis for cluster-wide awareness + if database.is_enabled(plugin) then + local redis_key = "plugin_malware_scan_api_error_" .. scanner + database.set(plugin, redis_key, data, ttl) + end +end + +-- Calculate exponential backoff delay for API errors. +-- Starts at 1 minute (60s), doubles each time, up to maximum of 24 hours (86400s). +-- Formula: min(60 * 2^failure_count, 86400) +--- @param failure_count Number of consecutive failures +--- @return number backoff_seconds Backoff delay in seconds +function cache.api_error_backoff_calculate(failure_count) + -- Cap failure_count to prevent integer overflow (2^20 = ~1 million seconds = 12 days) + local capped_count = math.min(failure_count, 20) + -- Start at 60 seconds (1 minute), max 86400 seconds (24 hours) + return math.min(60 * (2 ^ capped_count), 86400) +end + +-- Check if API should be skipped based on error circuit breaker state. +-- Returns true if API should be skipped (still in backoff), false if retry is allowed. +-- Also checks Redis for cluster-wide error state. +--- @param plugin Plugin instance +--- @param scanner Scanner name (e.g., "malwarebazaar", "sentinelone") +--- @return boolean should_skip True if API should be skipped +--- @return string|nil error_type Error type or nil +--- @return number remaining_backoff Remaining backoff time in seconds +function cache.api_error_should_skip(plugin, scanner) + -- Check local cache first + local failure_count, last_failure_time, error_type = cache.api_error_get(plugin, scanner) + + -- Check Redis for cluster-wide errors (another worker may have recorded failure) + if database.is_enabled(plugin) and failure_count == 0 then + local redis_key = "plugin_malware_scan_api_error_" .. scanner + local ok, redis_data, err = database.get(plugin, redis_key) + if ok and redis_data then + -- Parse Redis data + local r_count, r_time, r_error = redis_data:match("^(%d+):(%d+):(.+)$") + if r_count and r_time then + failure_count = tonumber(r_count) or 0 + last_failure_time = tonumber(r_time) or 0 + error_type = r_error + -- Store in local cache for faster subsequent checks + cache.api_error_set(plugin, scanner, failure_count, last_failure_time, error_type) + end + end + end + + if failure_count == 0 then + return false, nil, 0 -- No errors, allow API call + end + + local now = ngx.time() + local backoff_seconds = cache.api_error_backoff_calculate(failure_count) + local time_since_failure = now - last_failure_time + + if time_since_failure >= backoff_seconds then + return false, error_type, 0 -- Backoff period expired, allow retry + end + + -- Still in backoff period + local remaining = backoff_seconds - time_since_failure + plugin:log_debug(string.format( + "[API_ERROR] Skipping %s API (circuit breaker): error=%s, failures=%d, backoff=%ds, remaining=%ds", + scanner, error_type or "unknown", failure_count, backoff_seconds, remaining + )) + return true, error_type, remaining +end + +-- Record a successful API operation (resets error circuit breaker). +--- @param plugin Plugin instance +--- @param scanner Scanner name +function cache.api_error_success(plugin, scanner) + local cb_key = "plugin_malware_scan_api_error_" .. scanner + plugin.cachestore:delete(cb_key) + + -- Also delete from Redis + if database.is_enabled(plugin) then + local redis_key = "plugin_malware_scan_api_error_" .. scanner + database.delete(plugin, redis_key) + end + + plugin:log_debug("[v" .. cache.VERSION .. "] [API_ERROR] " .. scanner .. " API operation successful, circuit breaker reset") +end + +-- Record a failed API operation (increments error circuit breaker). +--- @param plugin Plugin instance +--- @param scanner Scanner name +--- @param error_type Error type (e.g., "user_blacklisted", "no_api_key", "rate_limit") +function cache.api_error_failure(plugin, scanner, error_type) + local failure_count = cache.api_error_get(plugin, scanner) + local now = ngx.time() + failure_count = failure_count + 1 + + cache.api_error_set(plugin, scanner, failure_count, now, error_type) + + local backoff_seconds = cache.api_error_backoff_calculate(failure_count) + + -- Format backoff time in human-readable format + local backoff_str + if backoff_seconds < 60 then + backoff_str = tostring(backoff_seconds) .. "s" + elseif backoff_seconds < 3600 then + backoff_str = string.format("%.1fm", backoff_seconds / 60) + else + backoff_str = string.format("%.1fh", backoff_seconds / 3600) + end + + logger.log_error( string.format( + "[API_ERROR] %s API failure #%d (error=%s), backing off for %s", + scanner, failure_count, error_type, backoff_str + )) +end + +-- ============================================================================ +-- COMPOSITE REDIS CACHE FUNCTIONS +-- Single lookup/write for all scanner results (performance optimization) +-- ============================================================================ + +--- Get composite scan results from Redis (all scanners in one lookup). +--- Returns success boolean, results table (or nil), and error message. +--- Results table format: {vt="result", sentinelone="result"} +--- @param plugin Plugin instance +--- @param checksum string SHA256 checksum +--- @return boolean success True if operation succeeded +--- @return table|nil results Results table or nil if not found +--- @return string|nil error Error message if failed +function cache.redis_composite_get(plugin, checksum) + local cache_key = "plugin_malware_scan_results_" .. checksum + plugin:log_debug(string.format("[v" .. cache.VERSION .. "] [CACHE_SHARED] Attempting composite Redis lookup for %s...", checksum:sub(1, 16) .. "...")) + + local ok, data, err = database.get(plugin, cache_key) + if not ok then + return false, nil, err or "redis lookup failed" + end + + if not data then + -- Cache miss (not an error) + return true, nil, nil + end + + -- Parse JSON + local decode_ok, results = pcall(cjson.decode, data) + if not decode_ok then + logger.log_error( "[CACHE_SHARED] Failed to parse composite cache JSON: " .. tostring(results)) + return false, nil, "json parse error" + end + + -- Validate JSON decoded to expected data structure (Medium severity issue #11) + if type(results) ~= "table" then + logger.log_error( "[CACHE_SHARED] Invalid data type after JSON decode: " .. type(results)) + return false, nil, "invalid data structure" + end + + local count = (results.vt and 1 or 0) + (results.sentinelone and 1 or 0) + plugin:log_debug(string.format("[v" .. cache.VERSION .. "] [CACHE_SHARED] Composite cache HIT (%d scanner results)", count)) + return true, results, nil +end + +--- Store composite scan results in Redis (all scanners in one write). +--- Takes checksum and results table: {vt="result", sentinelone="result"} +--- Returns success boolean and error message. +--- @param plugin Plugin instance +--- @param checksum string SHA256 checksum +--- @param results table Results table with scanner names as keys +--- @return boolean success True if operation succeeded +--- @return string|nil error Error message if failed +function cache.redis_composite_set(plugin, checksum, results) + local cache_key = "plugin_malware_scan_results_" .. checksum + local shared_ttl = tonumber(plugin.variables["MALWARE_SCAN_CACHE_SHARED_TTL"]) or 21600 + + -- Encode as JSON + local encode_ok, json_data = pcall(cjson.encode, results) + if not encode_ok then + logger.log_error( "[CACHE_SHARED] Failed to encode composite cache JSON: " .. tostring(json_data)) + return false, "json encode error" + end + + plugin:log_debug(string.format("[v" .. cache.VERSION .. "] [CACHE_SHARED] Storing composite results in Redis (TTL=%ds)...", shared_ttl)) + + local ok, err = database.set(plugin, cache_key, json_data, shared_ttl) + if not ok then + logger.log_error( "[CACHE_SHARED] Failed to store composite results in Redis: " .. tostring(err)) + return false, err + end + + plugin:log_debug(string.format("[v" .. cache.VERSION .. "] [CACHE_SHARED] Composite results stored in Redis (TTL=%ds)", shared_ttl)) + return true, nil +end + +-- ============================================================================ +-- ATTACKER IP TRACKING +-- Tracks IPs that upload malware, with separate IPv4/IPv6 namespaces +-- ============================================================================ + +--- Detect if an IP address is IPv4 or IPv6. +--- Returns "ipv4" or "ipv6" +--- @param ip string IP address to check +--- @return string version "ipv4" or "ipv6" +function cache.ip_detect_version(ip) + -- IPv6 addresses contain colons + if ip:match(":") then + return "ipv6" + else + return "ipv4" + end +end + +--- Track attacker IP that uploaded malware. +--- Stores IP address, timestamp of last upload, and array of SHA256 checksums. +--- Data is stored in local cache and optionally in Redis when shared cache is enabled. +--- IPv4 and IPv6 addresses are stored in separate key namespaces for better organization. +--- @param plugin Plugin instance +--- @param client_ip string Client IP address +--- @param checksum string SHA256 checksum of malware +--- @return boolean success True if tracking succeeded +function cache.attacker_track(plugin, client_ip, checksum) + local track_enabled = plugin.variables["MALWARE_SCAN_TRACK_ATTACKERS"] == "yes" + if not track_enabled then + return true -- Feature disabled, not an error + end + + local attacker_ttl = tonumber(plugin.variables["MALWARE_SCAN_TRACK_ATTACKER_TTL"]) or 7776000 -- 90 days default + + -- Separate IPv4 and IPv6 into different key namespaces + local ip_version = cache.ip_detect_version(client_ip) + local cache_key = "plugin_malware_scan_attacker_" .. ip_version .. "_" .. client_ip + + -- Get existing attacker data from local cache + local existing_data = nil + local ok, cached = plugin.cachestore:get(cache_key) + if ok and cached then + local decode_ok, decoded = pcall(cjson.decode, cached) + if decode_ok then + existing_data = decoded + end + end + + -- If not in local cache, try Redis (if shared cache enabled) + if not existing_data and database.is_enabled(plugin) then + local redis_ok, redis_data = database.get(plugin, cache_key) + if redis_ok and redis_data then + local decode_ok, decoded = pcall(cjson.decode, redis_data) + if decode_ok then + existing_data = decoded + end + end + end + + -- Create or update attacker data + local current_time = os.time() + local attacker_data = existing_data or { + first_upload = current_time, + last_upload = current_time, + checksums = {}, + upload_count = 0 + } + + -- Update last upload timestamp (preserves first_upload from existing data) + attacker_data.last_upload = current_time + + -- Ensure first_upload exists (backwards compatibility for old cache entries) + if not attacker_data.first_upload then + attacker_data.first_upload = current_time + end + + -- Add checksum if not already present (avoid duplicates) + local checksum_exists = false + for _, existing_checksum in ipairs(attacker_data.checksums) do + if existing_checksum == checksum then + checksum_exists = true + break + end + end + + if not checksum_exists then + table.insert(attacker_data.checksums, checksum) + attacker_data.upload_count = attacker_data.upload_count + 1 + end + + -- Encode as JSON + local encode_ok, json_data = pcall(cjson.encode, attacker_data) + if not encode_ok then + logger.log_error( "[ATTACKER_TRACK] Failed to encode attacker data: " .. tostring(json_data)) + return false + end + + -- Store in local cache + local store_ok, store_err = plugin.cachestore:set(cache_key, json_data, attacker_ttl) + if not store_ok then + logger.log_error( "[ATTACKER_TRACK] Failed to store in local cache: " .. tostring(store_err)) + end + + -- Store in Redis if shared cache enabled + if database.is_enabled(plugin) then + local redis_ok, redis_err = database.set(plugin, cache_key, json_data, attacker_ttl) + if redis_ok then + logger.log_notice( string.format("[v" .. cache.VERSION .. "] [ATTACKER_TRACK] %s %s tracked: %d unique malware uploads (TTL=%ds, stored in Redis)", + ip_version:upper(), client_ip, attacker_data.upload_count, attacker_ttl)) + else + logger.log_error( "[ATTACKER_TRACK] Failed to store in Redis: " .. tostring(redis_err)) + end + else + logger.log_notice( string.format("[v" .. cache.VERSION .. "] [ATTACKER_TRACK] %s %s tracked: %d unique malware uploads (TTL=%ds, local only)", + ip_version:upper(), client_ip, attacker_data.upload_count, attacker_ttl)) + end + + return true +end + +--- Get attacker data for a specific IP. +--- Returns attacker data table or nil if not found. +--- Checks local cache first, then Redis if enabled. +--- IPv4 and IPv6 addresses are stored in separate key namespaces. +--- @param plugin Plugin instance +--- @param client_ip string Client IP address +--- @return table|nil attacker_data Attacker data or nil if not found +function cache.attacker_get(plugin, client_ip) + local track_enabled = plugin.variables["MALWARE_SCAN_TRACK_ATTACKERS"] == "yes" + if not track_enabled then + return nil -- Feature disabled + end + + -- Separate IPv4 and IPv6 into different key namespaces + local ip_version = cache.ip_detect_version(client_ip) + local cache_key = "plugin_malware_scan_attacker_" .. ip_version .. "_" .. client_ip + + -- Check local cache first + local ok, cached = plugin.cachestore:get(cache_key) + if ok and cached then + local decode_ok, decoded = pcall(cjson.decode, cached) + if decode_ok then + return decoded + end + end + + -- Try Redis if shared cache enabled + if database.is_enabled(plugin) then + local redis_ok, redis_data = database.get(plugin, cache_key) + if redis_ok and redis_data then + local decode_ok, decoded = pcall(cjson.decode, redis_data) + if decode_ok then + -- Cache locally for faster subsequent lookups + local ttl = tonumber(plugin.variables["MALWARE_SCAN_TRACK_ATTACKER_TTL"]) or 7776000 -- 90 days default + plugin.cachestore:set(cache_key, redis_data, ttl) + return decoded + end + end + end + + return nil -- Not found +end + +-- ============================================================================ +-- LOCAL CACHE FUNCTIONS +-- Fast local cache operations with differential TTL for clean/malicious files +-- ============================================================================ + +--- Check if file scan result is in local cache. +--- Returns success boolean and cached value or nil. +--- @param plugin Plugin instance +--- @param file_path string File path to check +--- @return boolean success True if operation succeeded +--- @return string|nil value Cached value or nil +function cache.cache_check(plugin, file_path) + local ok, data = plugin.cachestore:get("plugin_malware_scan_file_" .. file_path) + if not ok then + return false, nil + end + return true, data +end + +--- Add scan result to local cache. +--- Returns success boolean and error message if failed. +--- @param plugin Plugin instance +--- @param file_path string File path +--- @param value string Scan result value +--- @return boolean success True if operation succeeded +--- @return string|nil error Error message if failed +function cache.cache_add(plugin, file_path, value) + local ok, err = plugin.cachestore:set( + "plugin_malware_scan_file_" .. file_path, + value, + 3600 + ) + if not ok then + return false, err + end + return true +end + +--- Check if ClamAV scan result is in local cache (by SHA256 checksum). +--- Returns success boolean, cached result, and file size (for verification). +--- Cache format: "result|size|timestamp" or "error|error_message|timestamp" +--- For errors, returns: true, "error", 0, error_message +--- @param plugin Plugin instance +--- @param checksum string SHA256 checksum +--- @param file_size number|nil File size for verification +--- @return boolean success True if cache hit +--- @return string|nil result Cached result or nil +--- @return number|nil cached_size Cached file size or nil +--- @return string|nil error_msg Error message if result is "error" +function cache.clamav_check(plugin, checksum, file_size) + local cache_key = "plugin_malware_scan_clamav_" .. checksum + local ok, data = plugin.cachestore:get(cache_key) + if ok and data then + -- Parse cached data: "result|data|timestamp" (data = size for results, error_msg for errors) + local result, cached_data, timestamp = data:match("^([^|]+)|([^|]*)|([^|]+)$") + if result then + -- Handle cached errors (negative caching) + if result == "error" then + local cache_age = os.time() - tonumber(timestamp) + local error_ttl = tonumber(plugin.variables["MALWARE_SCAN_CACHE_ERROR_TTL"]) or 60 + if cache_age > error_ttl then + plugin:log_debug(string.format( + "[CACHE] ClamAV error cache entry expired (age: %ds, TTL: %ds) - treating as MISS", + cache_age, error_ttl)) + return false, nil, nil, nil + end + plugin:log_debug(string.format("[v" .. cache.VERSION .. "] [CACHE] ClamAV error cache HIT for checksum: %s (error: %s, age: %ds)", + checksum:sub(1, 16) .. "...", cached_data or "unknown", cache_age)) + -- Return error: true (cache hit), "error" (result type), 0 (no size), error_message + return true, "error", 0, cached_data or "unknown error" + end + + -- Handle normal scan results + local cached_size = tonumber(cached_data) or 0 + + -- Verify file size matches (prevents collisions and provides quick filter) + if file_size and cached_size ~= file_size then + plugin:log_debug(string.format( + "[CACHE] ClamAV cache size mismatch: expected %d, got %d - treating as MISS", + file_size, cached_size)) + return false, nil, nil, nil + end + -- Verify TTL hasn't expired (defensive check in addition to cachestore auto-expiration) + local cache_age = os.time() - tonumber(timestamp) + local ttl + if result == "clean" then + ttl = tonumber(plugin.variables["MALWARE_SCAN_CACHE_CLEAN_TTL"]) or 300 + else + ttl = tonumber(plugin.variables["MALWARE_SCAN_CACHE_VIRUS_TTL"]) or 7776000 -- Default 90 days for malicious + if ttl > 604800 then ttl = 604800 end -- Cap at 7 days max to prevent table growth + end + if cache_age > ttl then + plugin:log_debug(string.format( + "[CACHE] ClamAV cache entry expired (age: %ds, TTL: %ds) - treating as MISS", + cache_age, ttl)) + return false, nil, nil, nil + end + + -- Check if entry should be invalidated due to cleanup on restart + if cache.should_invalidate_on_cleanup and cache.should_invalidate_on_cleanup(plugin, tonumber(timestamp)) then + plugin:log_debug(string.format( + "[CACHE] ClamAV cache entry invalidated by cleanup marker - treating as MISS")) + -- Delete the invalidated entry + plugin.cachestore:delete(cache_key) + return false, nil, nil, nil + end + + plugin:log_debug(string.format("[v" .. cache.VERSION .. "] [CACHE] ClamAV local cache HIT for checksum: %s (size: %s bytes, age: %ds, TTL remaining: %ds)", + checksum:sub(1, 16) .. "...", cached_size, cache_age, ttl - cache_age)) + return true, result, cached_size, nil + end + end + plugin:log_debug("[v" .. cache.VERSION .. "] [CACHE] ClamAV local cache MISS for checksum: " .. checksum:sub(1, 16) .. "...") + -- Record cache miss metric + if metrics then + metrics.record_cache("clamav", false) + end + return false, nil, nil, nil +end + +-- Add ClamAV scan result to local cache (by SHA256 checksum). +-- Stores: result, file size, and timestamp for verification and debugging. +-- Cache format: "result|size|timestamp" or "error|error_message|timestamp" for errors +-- Returns success boolean and error message if failed. +--- @param is_error boolean If true, value is treated as error message (negative caching) +function cache.clamav_add(plugin, checksum, value, file_size, is_error) + local cache_key = "plugin_malware_scan_clamav_" .. checksum + -- Differential caching: clean results cached for shorter period, malicious for longer + -- Uses configurable TTL: MALWARE_SCAN_CACHE_CLEAN_TTL, MALWARE_SCAN_CACHE_VIRUS_TTL, MALWARE_SCAN_CACHE_ERROR_TTL + local cache_ttl + local cache_data + + if is_error then + -- Negative caching: cache errors with short TTL to prevent repeated failed attempts + cache_ttl = tonumber(plugin.variables["MALWARE_SCAN_CACHE_ERROR_TTL"]) or 60 -- Default 60 seconds for errors + cache_data = string.format("error|%s|%d", value or "unknown error", os.time()) + plugin:log_debug(string.format("[v" .. cache.VERSION .. "] [CACHE] Added ClamAV error to cache: %s = %s (TTL: %ds)", + checksum:sub(1, 16) .. "...", value or "unknown error", cache_ttl)) + elseif value == "clean" then + cache_ttl = tonumber(plugin.variables["MALWARE_SCAN_CACHE_CLEAN_TTL"]) or 300 -- Default 5 minutes for clean files + cache_data = string.format("%s|%d|%d", value, file_size or 0, os.time()) + plugin:log_debug(string.format("[v" .. cache.VERSION .. "] [CACHE] Added ClamAV result to cache: %s = %s (size: %d bytes, TTL: %ds)", + checksum:sub(1, 16) .. "...", value, file_size or 0, cache_ttl)) + else + cache_ttl = tonumber(plugin.variables["MALWARE_SCAN_CACHE_VIRUS_TTL"]) or 7776000 -- Default 90 days for malicious + if cache_ttl > 7776000 then cache_ttl = 7776000 end -- Cap at 90 days max to prevent unbounded growth + cache_data = string.format("%s|%d|%d", value, file_size or 0, os.time()) + plugin:log_debug(string.format("[v" .. cache.VERSION .. "] [CACHE] Added ClamAV result to cache: %s = %s (size: %d bytes, TTL: %ds)", + checksum:sub(1, 16) .. "...", value, file_size or 0, cache_ttl)) + end + + local ok, err = plugin.cachestore:set(cache_key, cache_data, cache_ttl) + if not ok then + return false, err + end + return true +end + +-- Try to acquire lock for ClamAV scan (atomic operation to prevent concurrent scans). +-- Uses cache's add() which only succeeds if key doesn't exist (atomic compare-and-set). +-- Returns: boolean lock_acquired +--- @param plugin Plugin instance +--- @param checksum string SHA256 checksum +--- @return boolean acquired True if lock acquired, false if another thread holds lock +function cache.clamav_try_lock(plugin, checksum) + local lock_key = "plugin_malware_scan_clamav_lock_" .. checksum + local lock_ttl = tonumber(plugin.variables["MALWARE_SCAN_CACHE_LOCK_TTL"]) or 60 + + -- add() is atomic - only succeeds if key doesn't exist + local ok, err = plugin.cachestore:add(lock_key, "scanning", lock_ttl) + + if ok then + plugin:log_debug(string.format("[v" .. cache.VERSION .. "] [CACHE_LOCK] Acquired lock for checksum: %s (TTL: %ds)", + checksum:sub(1, 16) .. "...", lock_ttl)) + -- Record lock acquisition metric + if metrics then + metrics.record_lock_acquired("clamav") + end + return true + else + plugin:log_debug(string.format("[v" .. cache.VERSION .. "] [CACHE_LOCK] Lock already held by another request: %s", + checksum:sub(1, 16) .. "...")) + return false + end +end + +-- Release lock for ClamAV scan. +-- Should be called after scan completes (success or failure) or before returning on error. +--- @param plugin Plugin instance +--- @param checksum string SHA256 checksum +function cache.clamav_release_lock(plugin, checksum) + local lock_key = "plugin_malware_scan_clamav_lock_" .. checksum + plugin.cachestore:delete(lock_key) + plugin:log_debug(string.format("[v" .. cache.VERSION .. "] [CACHE_LOCK] Released lock for checksum: %s", + checksum:sub(1, 16) .. "...")) +end + +-- Wait for another request to complete ClamAV scan and return the cached result. +-- Polls cache with exponential backoff until result appears or timeout occurs. +--- @param plugin Plugin instance +--- @param checksum string SHA256 checksum +--- @param file_size number File size for validation +--- @return boolean success +--- @return string|nil result +--- @return number|nil cached_size +--- @return string|nil error_msg +function cache.clamav_wait_for_result(plugin, checksum, file_size) + local wait_timeout_ms = tonumber(plugin.variables["MALWARE_SCAN_CACHE_WAIT_TIMEOUT"]) or 30000 + local start_time = ngx.now() * 1000 -- Convert to milliseconds + local wait_intervals = {10, 20, 50, 100, 200, 500} -- Exponential backoff in ms + local interval_index = 1 + local poll_count = 0 + + plugin:log_debug(string.format("[v" .. cache.VERSION .. "] [CACHE_LOCK] Waiting for scan result: %s (timeout: %dms)", + checksum:sub(1, 16) .. "...", wait_timeout_ms)) + + while true do + poll_count = poll_count + 1 + + -- Check if result is now available + local cache_ok, result, cached_size, error_msg = cache.clamav_check(plugin, checksum, file_size) + if cache_ok and result then + -- Got result (clean, malicious, or error) + plugin:log_debug(string.format("[v" .. cache.VERSION .. "] [CACHE_LOCK] Scan result available after %d polls: %s = %s", + poll_count, checksum:sub(1, 16) .. "...", result)) + return true, result, cached_size, error_msg + end + + -- Check timeout + local elapsed_ms = (ngx.now() * 1000) - start_time + if elapsed_ms >= wait_timeout_ms then + -- Timeout - fall back to scanning ourselves + logger.log_warn( string.format( + "[CACHE_LOCK] Timeout waiting for scan result after %dms (%d polls) - will scan independently", + elapsed_ms, poll_count)) + return false, nil, nil, nil + end + + -- Exponential backoff + local wait_ms = wait_intervals[interval_index] or 500 + ngx.sleep(wait_ms / 1000) -- Convert to seconds for ngx.sleep + interval_index = math.min(interval_index + 1, #wait_intervals) + end +end + +-------------------------------------------------------------------------------- +-- GENERIC LOCKING FUNCTIONS (reusable across all scanners) +-------------------------------------------------------------------------------- + +-- Generic wait function that works for all scanners. +-- Polls cache with exponential backoff until result appears or timeout occurs. +--- @param plugin Plugin instance +--- @param scanner_name string Scanner name ("clamav", "virustotal", "sentinelone") +--- @param checksum string SHA256 checksum +--- @param file_size number File size for validation +--- @return boolean success +--- @return any result Scanner-specific result +--- @return number|nil cached_size +--- @return string|nil error_msg For clamav only +local function cache_wait_for_result_generic(plugin, scanner_name, checksum, file_size) + -- Map scanner names to their check functions + local check_function_map = { + clamav = cache.clamav_check, + virustotal = cache.virustotal_check, + sentinelone = cache.sentinelone_check + } + + local check_function = check_function_map[scanner_name] + if not check_function then + logger.log_error( "[CACHE_LOCK] Unknown scanner: " .. scanner_name) + return false, nil, nil, nil + end + + local wait_timeout_ms = tonumber(plugin.variables["MALWARE_SCAN_CACHE_WAIT_TIMEOUT"]) or 30000 + local start_time = ngx.now() * 1000 -- Convert to milliseconds + local wait_intervals = {10, 20, 50, 100, 200, 500} -- Exponential backoff in ms + local interval_index = 1 + local poll_count = 0 + + plugin:log_debug(string.format("[v" .. cache.VERSION .. "] [CACHE_LOCK] [%s] Waiting for result: %s (timeout: %dms)", + scanner_name:upper(), checksum:sub(1, 16) .. "...", wait_timeout_ms)) + + while true do + poll_count = poll_count + 1 + + -- Check if result is now available (call scanner-specific check function) + local cache_ok, result, cached_size, error_msg = check_function(plugin, checksum, file_size) + if cache_ok and result then + -- Got result (clean, malicious, or error) + plugin:log_debug(string.format("[v" .. cache.VERSION .. "] [CACHE_LOCK] [%s] Result available after %d polls: %s = %s", + scanner_name:upper(), poll_count, checksum:sub(1, 16) .. "...", result)) + return true, result, cached_size, error_msg + end + + -- Check timeout + local elapsed_ms = (ngx.now() * 1000) - start_time + if elapsed_ms >= wait_timeout_ms then + -- Timeout - fall back to scanning/querying independently + logger.log_warn( string.format( + "[CACHE_LOCK] [%s] Timeout waiting for result after %dms (%d polls) - will query independently", + scanner_name:upper(), elapsed_ms, poll_count)) + return false, nil, nil, nil + end + + -- Exponential backoff + local wait_ms = wait_intervals[interval_index] or 500 + ngx.sleep(wait_ms / 1000) -- Convert to seconds for ngx.sleep + interval_index = math.min(interval_index + 1, #wait_intervals) + end +end + +-------------------------------------------------------------------------------- +-- VIRUSTOTAL LOCKING FUNCTIONS +-------------------------------------------------------------------------------- + +-- Try to acquire lock for VirusTotal API query (atomic operation to prevent concurrent queries). +-- Uses cache's add() which only succeeds if key doesn't exist (atomic compare-and-set). +--- @param plugin Plugin instance +--- @param checksum string SHA256 checksum +--- @return boolean acquired True if lock acquired, false if another thread holds lock +function cache.virustotal_try_lock(plugin, checksum) + local lock_key = "plugin_malware_scan_virustotal_lock_" .. checksum + local lock_ttl = tonumber(plugin.variables["MALWARE_SCAN_CACHE_LOCK_TTL"]) or 60 + + -- add() is atomic - only succeeds if key doesn't exist + local ok, err = plugin.cachestore:add(lock_key, "scanning", lock_ttl) + + if ok then + plugin:log_debug(string.format("[v" .. cache.VERSION .. "] [CACHE_LOCK] [VIRUSTOTAL] Acquired lock for checksum: %s (TTL: %ds)", + checksum:sub(1, 16) .. "...", lock_ttl)) + return true + else + plugin:log_debug(string.format("[v" .. cache.VERSION .. "] [CACHE_LOCK] [VIRUSTOTAL] Lock already held by another request: %s", + checksum:sub(1, 16) .. "...")) + return false + end +end + +-- Release lock for VirusTotal API query. +-- Should be called after query completes (success or failure) or before returning on error. +--- @param plugin Plugin instance +--- @param checksum string SHA256 checksum +function cache.virustotal_release_lock(plugin, checksum) + local lock_key = "plugin_malware_scan_virustotal_lock_" .. checksum + plugin.cachestore:delete(lock_key) + plugin:log_debug(string.format("[v" .. cache.VERSION .. "] [CACHE_LOCK] [VIRUSTOTAL] Released lock for checksum: %s", + checksum:sub(1, 16) .. "...")) +end + +-- Wait for another request to complete VirusTotal query and return the cached result. +-- Polls cache with exponential backoff until result appears or timeout occurs. +--- @param plugin Plugin instance +--- @param checksum string SHA256 checksum +--- @param file_size number File size for validation +--- @return boolean success +--- @return string|nil result +--- @return number|nil cached_size +--- @return string|nil error_msg +function cache.virustotal_wait_for_result(plugin, checksum, file_size) + return cache_wait_for_result_generic(plugin, "virustotal", checksum, file_size) +end + +-------------------------------------------------------------------------------- +-- SENTINELONE LOCKING FUNCTIONS +-------------------------------------------------------------------------------- + +-- Try to acquire lock for SentinelOne API query (atomic operation to prevent concurrent queries). +--- @param plugin Plugin instance +--- @param checksum string SHA256 checksum +--- @return boolean acquired True if lock acquired, false if another thread holds lock +function cache.sentinelone_try_lock(plugin, checksum) + local lock_key = "plugin_malware_scan_sentinelone_lock_" .. checksum + local lock_ttl = tonumber(plugin.variables["MALWARE_SCAN_CACHE_LOCK_TTL"]) or 60 + + local ok, err = plugin.cachestore:add(lock_key, "scanning", lock_ttl) + + if ok then + plugin:log_debug(string.format("[v" .. cache.VERSION .. "] [CACHE_LOCK] [SENTINELONE] Acquired lock for checksum: %s (TTL: %ds)", + checksum:sub(1, 16) .. "...", lock_ttl)) + return true + else + plugin:log_debug(string.format("[v" .. cache.VERSION .. "] [CACHE_LOCK] [SENTINELONE] Lock already held by another request: %s", + checksum:sub(1, 16) .. "...")) + return false + end +end + +-- Release lock for SentinelOne API query. +--- @param plugin Plugin instance +--- @param checksum string SHA256 checksum +function cache.sentinelone_release_lock(plugin, checksum) + local lock_key = "plugin_malware_scan_sentinelone_lock_" .. checksum + plugin.cachestore:delete(lock_key) + plugin:log_debug(string.format("[v" .. cache.VERSION .. "] [CACHE_LOCK] [SENTINELONE] Released lock for checksum: %s", + checksum:sub(1, 16) .. "...")) +end + +-- Wait for another request to complete SentinelOne query and return the cached result. +--- @param plugin Plugin instance +--- @param checksum string SHA256 checksum +--- @param file_size number File size for validation +--- @return boolean success +--- @return string|nil result +--- @return number|nil cached_size +--- @return string|nil error_msg +function cache.sentinelone_wait_for_result(plugin, checksum, file_size) + return cache_wait_for_result_generic(plugin, "sentinelone", checksum, file_size) +end + +-------------------------------------------------------------------------------- +-- CACHE CHECK FUNCTIONS +-------------------------------------------------------------------------------- + +--- Check if VirusTotal scan result is in local cache. +--- Returns success boolean, cached result, and file size (for verification). +--- Cache format: "result|size|timestamp" +--- Note: Redis lookup is handled by get_composite_redis_cache() for performance. +--- @param plugin Plugin instance +--- @param checksum string SHA256 checksum +--- @param file_size number|nil File size for verification +--- @return boolean success True if cache hit +--- @return string|nil result Cached result or nil +--- @return number|nil cached_size Cached file size or nil +function cache.virustotal_check(plugin, checksum, file_size) + local cache_key = "plugin_malware_scan_virustotal_" .. checksum + + plugin:log_debug(string.format("[v" .. cache.VERSION .. "] [CACHE] is_in_vt_cache: checksum=%s", checksum:sub(1, 16) .. "...")) + + -- Check local cache (fast, ~1ms) + local ok, data = plugin.cachestore:get(cache_key) + if ok and data then + -- Parse cached data: "result|size|timestamp" + local result, cached_size, timestamp = data:match("^([^|]+)|([^|]+)|([^|]+)$") + if result then + -- Verify file size matches + if file_size and tonumber(cached_size) ~= file_size then + plugin:log_debug(string.format( + "[CACHE] VirusTotal cache size mismatch: expected %d, got %d - treating as MISS", + file_size, tonumber(cached_size))) + return false, nil, nil + end + -- Verify TTL hasn't expired + local cache_age = os.time() - tonumber(timestamp) + local ttl + if result == "clean" then + ttl = tonumber(plugin.variables["MALWARE_SCAN_CACHE_CLEAN_TTL"]) or 300 + else + ttl = tonumber(plugin.variables["MALWARE_SCAN_CACHE_VIRUS_TTL"]) or 7776000 -- Default 90 days for malicious + if ttl > 604800 then ttl = 604800 end -- Cap at 7 days max to prevent table growth + end + if cache_age > ttl then + plugin:log_debug(string.format( + "[CACHE] VirusTotal cache entry expired (age: %ds, TTL: %ds) - treating as MISS", + cache_age, ttl)) + return false, nil, nil + end + + -- Check if entry should be invalidated due to cleanup on restart + if cache.should_invalidate_on_cleanup and cache.should_invalidate_on_cleanup(plugin, tonumber(timestamp)) then + plugin:log_debug("[v" .. cache.VERSION .. "] [CACHE] VirusTotal cache entry invalidated by cleanup marker - treating as MISS") + plugin.cachestore:delete(cache_key) + return false, nil, nil + end + + plugin:log_debug(string.format("[v" .. cache.VERSION .. "] [CACHE] VirusTotal local cache HIT (size: %s bytes, age: %ds, TTL remaining: %ds)", + cached_size, cache_age, ttl - cache_age)) + return true, result, tonumber(cached_size) + end + end + + plugin:log_debug("[v" .. cache.VERSION .. "] [CACHE] VirusTotal local cache MISS") + return false, nil, nil +end + +--- Add VirusTotal scan result to local cache. +--- Stores: result, file size, and timestamp for verification and debugging. +--- Cache format: "result|size|timestamp" +--- Returns success boolean and error message if failed. +--- Note: Redis storage is handled by set_composite_redis_cache() for performance. +--- @param plugin Plugin instance +--- @param checksum string SHA256 checksum +--- @param value string Scan result ("clean" or detection info) +--- @param file_size number|nil File size in bytes +--- @return boolean success True if operation succeeded +--- @return string|nil error Error message if failed +function cache.virustotal_add(plugin, checksum, value, file_size) + -- Differential caching: clean results cached for shorter period, malicious for longer + -- This prevents false negatives if a file becomes identified as malware shortly after upload + local cache_ttl + if value == "clean" then + cache_ttl = tonumber(plugin.variables["MALWARE_SCAN_CACHE_CLEAN_TTL"]) or 300 -- Default 5 minutes for clean files + logger.log_error( "[CACHE_CLEAN] VirusTotal result cached for " .. cache_ttl .. " seconds") + else + cache_ttl = tonumber(plugin.variables["MALWARE_SCAN_CACHE_VIRUS_TTL"]) or 7776000 -- Default 90 days for malicious + if cache_ttl > 7776000 then cache_ttl = 7776000 end -- Cap at 90 days max to prevent unbounded growth + logger.log_error( "[CACHE_MALWARE] VirusTotal result cached for " .. cache_ttl .. " seconds") + end + + local cache_key = "plugin_malware_scan_virustotal_" .. checksum + + plugin:log_debug(string.format("[v" .. cache.VERSION .. "] [CACHE] add_to_vt_cache: checksum=%s, value=%s, size=%d bytes, ttl=%ds", + checksum:sub(1, 16) .. "...", value, file_size or 0, cache_ttl)) + + -- Store in local cache with structured format: "result|size|timestamp" + local cache_data = string.format("%s|%d|%d", value, file_size or 0, os.time()) + local ok, err = plugin.cachestore:set(cache_key, cache_data, cache_ttl) + if not ok then + plugin:log_debug("[v" .. cache.VERSION .. "] [CACHE] Failed to store VirusTotal in local cache: " .. tostring(err)) + return false, err + end + plugin:log_debug(string.format("[v" .. cache.VERSION .. "] [CACHE] VirusTotal result stored in local cache (size: %d bytes)", file_size or 0)) + + return true +end + +--- Check if SentinelOne result is in local cache. +--- Cache format: "result|size|timestamp" +--- Returns success boolean, cached result, and file size (or nil values if not found). +--- @param plugin Plugin instance +--- @param checksum string SHA256 checksum +--- @param file_size number|nil File size for verification +--- @return boolean success True if cache hit +--- @return string|nil result Cached result or nil +--- @return number|nil cached_size Cached file size or nil +function cache.sentinelone_check(plugin, checksum, file_size) + local cache_key = "plugin_malware_scan_sentinelone_" .. checksum + + plugin:log_debug(string.format("[v" .. cache.VERSION .. "] [CACHE] is_in_sentinelone_cache: checksum=%s, file_size=%d", + checksum:sub(1, 16) .. "...", file_size or 0)) + + -- Check local cache (fast, ~1ms) + local ok, data = plugin.cachestore:get(cache_key) + if ok and data then + -- Parse cached data: "result|size|timestamp" + local result, cached_size, timestamp = data:match("^([^|]+)|([^|]+)|([^|]+)$") + if result then + -- Verify file size matches to prevent collisions (same size can have different SHA256) + if file_size and tonumber(cached_size) ~= file_size then + plugin:log_debug(string.format( + "[CACHE] SentinelOne cache size mismatch: expected %d, got %d - treating as MISS", + file_size, tonumber(cached_size))) + return false, nil, nil + end + + -- Verify TTL hasn't expired + local cache_age = os.time() - tonumber(timestamp) + local ttl + if result == "clean" then + ttl = tonumber(plugin.variables["MALWARE_SCAN_CACHE_CLEAN_TTL"]) or 300 + else + ttl = tonumber(plugin.variables["MALWARE_SCAN_CACHE_VIRUS_TTL"]) or 7776000 -- Default 90 days for malicious + if ttl > 604800 then ttl = 604800 end -- Cap at 7 days max to prevent table growth + end + if cache_age > ttl then + plugin:log_debug(string.format( + "[CACHE] SentinelOne cache entry expired (age: %ds, TTL: %ds) - treating as MISS", + cache_age, ttl)) + return false, nil, nil + end + + -- Check if entry should be invalidated due to cleanup on restart + if cache.should_invalidate_on_cleanup and cache.should_invalidate_on_cleanup(plugin, tonumber(timestamp)) then + plugin:log_debug("[v" .. cache.VERSION .. "] [CACHE] SentinelOne cache entry invalidated by cleanup marker - treating as MISS") + plugin.cachestore:delete(cache_key) + return false, nil, nil + end + + plugin:log_debug(string.format("[v" .. cache.VERSION .. "] [CACHE] SentinelOne local cache HIT (size: %s bytes, age: %ds, TTL remaining: %ds)", + cached_size, cache_age, ttl - cache_age)) + return true, result, tonumber(cached_size) + end + end + + plugin:log_debug("[v" .. cache.VERSION .. "] [CACHE] SentinelOne local cache MISS") + return false, nil, nil +end + +--- Add SentinelOne result to local cache. +--- Cache format: "result|size|timestamp" +--- Returns success boolean and error message if failed. +--- Note: Redis storage is handled by set_composite_redis_cache() for performance. +--- @param plugin Plugin instance +--- @param checksum string SHA256 checksum +--- @param value string Scan result ("clean" or detection info) +--- @param file_size number|nil File size in bytes +--- @return boolean success True if operation succeeded +--- @return string|nil error Error message if failed +function cache.sentinelone_add(plugin, checksum, value, file_size) + -- Differential caching: clean results cached for shorter period, malicious for longer + -- This prevents false negatives if a file becomes identified as malware shortly after upload + local cache_ttl + if value == "clean" then + cache_ttl = tonumber(plugin.variables["MALWARE_SCAN_CACHE_CLEAN_TTL"]) or 300 -- Default 5 minutes for clean files + logger.log_error( "[CACHE_CLEAN] SentinelOne result cached for " .. cache_ttl .. " seconds") + else + cache_ttl = tonumber(plugin.variables["MALWARE_SCAN_CACHE_VIRUS_TTL"]) or 7776000 -- Default 90 days for malicious + if cache_ttl > 7776000 then cache_ttl = 7776000 end -- Cap at 90 days max to prevent unbounded growth + logger.log_error( "[CACHE_MALWARE] SentinelOne result cached for " .. cache_ttl .. " seconds") + end + + local cache_key = "plugin_malware_scan_sentinelone_" .. checksum + + plugin:log_debug(string.format("[v" .. cache.VERSION .. "] [CACHE] add_to_sentinelone_cache: checksum=%s, value=%s, size=%d bytes, ttl=%ds", + checksum:sub(1, 16) .. "...", value, file_size or 0, cache_ttl)) + + -- Store in local cache with structured format: "result|size|timestamp" + local cache_data = string.format("%s|%d|%d", value, file_size or 0, os.time()) + local ok, err = plugin.cachestore:set(cache_key, cache_data, cache_ttl) + if not ok then + plugin:log_debug("[v" .. cache.VERSION .. "] [CACHE] Failed to store SentinelOne in local cache: " .. tostring(err)) + return false, err + end + plugin:log_debug(string.format("[v" .. cache.VERSION .. "] [CACHE] SentinelOne result stored in local cache (size: %d bytes)", file_size or 0)) + + return true +end + +-- ============================================================================ +-- MALWAREBAZAAR CACHE FUNCTIONS +-- ============================================================================ + +--- Check if MalwareBazaar result for checksum exists in local cache. +--- Returns: success boolean, cached value (or nil), cached file size (or nil) +--- @param plugin Plugin instance +--- @param checksum string SHA256 checksum +--- @param file_size number|nil File size for verification +--- @return boolean success True if cache hit +--- @return string|nil result Cached result or nil +--- @return number|nil cached_size Cached file size or nil +function cache.malwarebazaar_check(plugin, checksum, file_size) + local cache_key = "plugin_malware_scan_malwarebazaar_" .. checksum + + plugin:log_debug("[v" .. cache.VERSION .. "] [CACHE] Checking MalwareBazaar local cache for checksum: " .. checksum:sub(1, 16) .. "...") + + -- Check local cache first (fast path) + local ok, cached_data = plugin.cachestore:get(cache_key) + if ok and cached_data then + -- Parse cache data: "result|size|timestamp" + local result, cached_size, timestamp = cached_data:match("([^|]+)|([^|]+)|([^|]+)") + if result then + -- Check if entry is still valid based on TTL + local cache_age = os.time() - tonumber(timestamp) + local ttl + if result == "clean" then + ttl = tonumber(plugin.variables["MALWARE_SCAN_CACHE_CLEAN_TTL"]) or 300 -- Default 5 minutes for clean + else + ttl = tonumber(plugin.variables["MALWARE_SCAN_CACHE_VIRUS_TTL"]) or 7776000 -- Default 90 days for malicious + if ttl > 604800 then ttl = 604800 end -- Cap at 7 days max to prevent table growth + end + if cache_age > ttl then + plugin:log_debug(string.format( + "[CACHE] MalwareBazaar cache entry expired (age: %ds, TTL: %ds) - treating as MISS", + cache_age, ttl)) + return false, nil, nil + end + + -- Check if entry should be invalidated due to cleanup on restart + if cache.should_invalidate_on_cleanup and cache.should_invalidate_on_cleanup(plugin, tonumber(timestamp)) then + plugin:log_debug("[v" .. cache.VERSION .. "] [CACHE] MalwareBazaar cache entry invalidated by cleanup marker - treating as MISS") + plugin.cachestore:delete(cache_key) + return false, nil, nil + end + + plugin:log_debug(string.format("[v" .. cache.VERSION .. "] [CACHE] MalwareBazaar local cache HIT (size: %s bytes, age: %ds, TTL remaining: %ds)", + cached_size, cache_age, ttl - cache_age)) + return true, result, tonumber(cached_size) + end + end + + plugin:log_debug("[v" .. cache.VERSION .. "] [CACHE] MalwareBazaar local cache MISS") + return false, nil, nil +end + +--- Add MalwareBazaar result to local cache. +--- Cache format: "result|size|timestamp" +--- Returns success boolean and error message if failed. +--- Note: Redis storage is handled by set_composite_redis_cache() for performance. +--- @param plugin Plugin instance +--- @param checksum string SHA256 checksum +--- @param value string Scan result ("clean" or detection info) +--- @param file_size number|nil File size in bytes +--- @return boolean success True if operation succeeded +--- @return string|nil error Error message if failed +function cache.malwarebazaar_add(plugin, checksum, value, file_size) + -- Differential caching: clean results cached for shorter period, malicious for longer + -- This prevents false negatives if a file becomes identified as malware shortly after upload + local cache_ttl + if value == "clean" then + cache_ttl = tonumber(plugin.variables["MALWARE_SCAN_CACHE_CLEAN_TTL"]) or 300 -- Default 5 minutes for clean files + logger.log_error( "[CACHE_CLEAN] MalwareBazaar result cached for " .. cache_ttl .. " seconds") + else + cache_ttl = tonumber(plugin.variables["MALWARE_SCAN_CACHE_VIRUS_TTL"]) or 7776000 -- Default 90 days for malicious + if cache_ttl > 7776000 then cache_ttl = 7776000 end -- Cap at 90 days max to prevent unbounded growth + logger.log_error( "[CACHE_MALWARE] MalwareBazaar result cached for " .. cache_ttl .. " seconds") + end + + local cache_key = "plugin_malware_scan_malwarebazaar_" .. checksum + + plugin:log_debug(string.format("[v" .. cache.VERSION .. "] [CACHE] add_to_malwarebazaar_cache: checksum=%s, value=%s, size=%d bytes, ttl=%ds", + checksum:sub(1, 16) .. "...", value, file_size or 0, cache_ttl)) + + -- Store in local cache with structured format: "result|size|timestamp" + local cache_data = string.format("%s|%d|%d", value, file_size or 0, os.time()) + local ok, err = plugin.cachestore:set(cache_key, cache_data, cache_ttl) + if not ok then + plugin:log_debug("[v" .. cache.VERSION .. "] [CACHE] Failed to store MalwareBazaar in local cache: " .. tostring(err)) + return false, err + end + plugin:log_debug(string.format("[v" .. cache.VERSION .. "] [CACHE] MalwareBazaar result stored in local cache (size: %d bytes)", file_size or 0)) + + return true +end + +-- ============================================================================ +-- CACHE CLEANUP ON RESTART +-- ============================================================================ + +--- Check if a cache entry should be invalidated based on cleanup marker. +--- Returns true if entry is older than cleanup marker (should be invalidated). +--- @param plugin Plugin instance +--- @param entry_timestamp number Timestamp of cache entry +--- @return boolean should_invalidate True if entry should be invalidated +function cache.should_invalidate_on_cleanup(plugin, entry_timestamp) + local marker_key = "plugin_malware_scan_cleanup_marker" + local ok, marker_value = plugin.cachestore:get(marker_key) + + if not ok or not marker_value then + -- No cleanup marker exists, entry is valid + return false + end + + local cleanup_timestamp = tonumber(marker_value) + if not cleanup_timestamp then + -- Invalid marker format, treat entry as valid + return false + end + + -- Entry is older than cleanup marker, should be invalidated + return entry_timestamp < cleanup_timestamp +end + +--- Try to acquire distributed lock for cache cleanup. +--- Uses cachestore to ensure only one worker performs cleanup. +--- Returns true if lock was acquired, false if another worker has it. +--- +--- NOTE: Uses atomic add() operation to prevent TOCTOU race condition (CWE-362). +--- If cachestore doesn't support add(), falls back to get+set (with race condition risk). +--- @param plugin Plugin instance +--- @return boolean acquired True if lock was acquired +function cache.lock_acquire(plugin) + local lock_key = "plugin_malware_scan_cleanup_lock" + local lock_ttl = 60 -- Lock expires after 60 seconds (safety timeout) + + -- Try atomic add() first (only succeeds if key doesn't exist) + if plugin.cachestore.add then + local add_ok, add_err = plugin.cachestore:add(lock_key, os.time(), lock_ttl) + if add_ok then + logger.log_notice( "[CACHE_CLEANUP] Distributed lock acquired successfully (atomic)") + return true + else + -- Lock already exists or add() failed + logger.log_notice( "[CACHE_CLEANUP] Lock already held by another worker") + return false + end + end + + -- Fallback to get+set (has TOCTOU race condition, but better than nothing) + logger.log_warn( "[CACHE_CLEANUP] cachestore doesn't support add(), using non-atomic fallback") + + local ok, err = plugin.cachestore:get(lock_key) + if ok then + -- Lock already exists + logger.log_notice( "[CACHE_CLEANUP] Lock already held by another worker") + return false + end + + -- Lock doesn't exist, try to acquire it (race window here!) + local set_ok, set_err = plugin.cachestore:set(lock_key, os.time(), lock_ttl) + if not set_ok then + logger.log_warn( "[CACHE_CLEANUP] Failed to acquire lock: " .. tostring(set_err)) + return false + end + + logger.log_notice( "[CACHE_CLEANUP] Distributed lock acquired successfully (non-atomic)") + return true +end + +--- Release distributed lock for cache cleanup. +--- Removes the lock from cachestore to allow future cleanups. +--- @param plugin Plugin instance +--- @return boolean success True if lock was released +function cache.lock_release(plugin) + local lock_key = "plugin_malware_scan_cleanup_lock" + + -- Delete the lock key + local ok, err = plugin.cachestore:delete(lock_key) + if not ok then + logger.log_warn( "[CACHE_CLEANUP] Failed to release lock: " .. tostring(err)) + return false + end + + logger.log_notice( "[CACHE_CLEANUP] Distributed lock released") + return true +end + +--- Clear all cached scan results on plugin restart. +--- Clears both local cache (cachestore) and Redis cache (clusterstore). +--- Uses marker-based approach for local cache (lazy invalidation). +--- Uses SCAN + UNLINK for Redis cache (pattern-based deletion). +--- This is useful after ClamAV signature updates or to resolve cache corruption. +--- Returns success boolean and statistics about cleanup. +--- @param plugin Plugin instance +--- @return boolean success True if cleanup succeeded +--- @return table stats Statistics about cleanup operation +function cache.cleanup_all(plugin) + local stats = { + local_cleared = 0, + redis_cleared = 0, + errors = 0, + patterns_cleaned = 0, + pattern_details = {}, + hash_reinitialized = false, + start_time = os.time(), + end_time = nil, + duration_seconds = nil + } + + logger.log_notice( "[CACHE_CLEANUP] Starting cache cleanup on restart") + + -- Clear local cache by setting cleanup marker timestamp + -- BunkerWeb's cachestore doesn't have enumerate/flush methods, + -- so we use a marker-based approach: set current timestamp as cleanup marker + -- On subsequent cache lookups, entries older than this marker are treated as expired + local cleanup_marker_key = "plugin_malware_scan_cleanup_marker" + local cleanup_timestamp = os.time() + local marker_ttl = 86400 * 7 -- Keep marker for 7 days (max cache TTL) + + local marker_ok, marker_err = plugin.cachestore:set(cleanup_marker_key, tostring(cleanup_timestamp), marker_ttl) + if marker_ok then + logger.log_notice( string.format( + "[CACHE_CLEANUP] Local cache cleanup marker set (timestamp: %d). Old entries will be invalidated on next lookup.", + cleanup_timestamp + )) + stats.local_cleared = 1 + else + logger.log_warn( "[CACHE_CLEANUP] Failed to set local cache cleanup marker: " .. tostring(marker_err)) + stats.errors = stats.errors + 1 + end + + -- Clear Redis cache if shared database is enabled + local use_shared = plugin.variables["MALWARE_SCAN_USE_SHARED_DATABASE"] == "yes" + local use_redis = plugin.ctx.bw.use_redis == "yes" + + -- Support granular cleanup: only clean patterns where flag is enabled + local cleanup_all = plugin.variables["MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_ALL"] == "yes" + + if use_shared and use_redis then + -- Check if circuit breaker is open + if database.should_skip(plugin) then + logger.log_warn( "[CACHE_CLEANUP] Redis circuit breaker open, skipping Redis cleanup") + return true, stats + end + + logger.log_notice( "[CACHE_CLEANUP] Clearing Redis shared cache using SCAN + UNLINK") + + -- Connect to Redis + local connect_ok, connect_err = plugin.clusterstore:connect() + if not connect_ok then + logger.log_warn( "[CACHE_CLEANUP] Failed to connect to Redis: " .. tostring(connect_err)) + stats.errors = stats.errors + 1 + database.failure_record(plugin) + return true, stats + end + + -- Clear plugin-related keys from Redis using SCAN + UNLINK + -- UNLINK is non-blocking (async deletion), better than DEL + -- Build pattern list based on enabled flags + local redis_patterns = {} + + -- Check each scanner's cleanup flag (or cleanup_all master switch) + if cleanup_all or plugin.variables["MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_CLAMAV"] == "yes" then + table.insert(redis_patterns, "plugin_malware_scan_clamav_*") + end + if cleanup_all or plugin.variables["MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_VIRUSTOTAL"] == "yes" then + table.insert(redis_patterns, "plugin_malware_scan_virustotal_*") + end + if cleanup_all or plugin.variables["MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_SENTINELONE"] == "yes" then + table.insert(redis_patterns, "plugin_malware_scan_sentinelone_*") + end + if cleanup_all or plugin.variables["MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_MALWAREBAZAAR"] == "yes" then + table.insert(redis_patterns, "plugin_malware_scan_malwarebazaar_*") + end + if cleanup_all or plugin.variables["MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_ATTACKER_IPV4"] == "yes" then + table.insert(redis_patterns, "plugin_malware_scan_attacker_ipv4_*") + end + if cleanup_all or plugin.variables["MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_ATTACKER_IPV6"] == "yes" then + table.insert(redis_patterns, "plugin_malware_scan_attacker_ipv6_*") + end + + -- Always clean composite results cache if any scanner is being cleaned + if #redis_patterns > 0 then + table.insert(redis_patterns, "plugin_malware_scan_results_*") + end + + -- If no patterns to clean, skip Redis cleanup + if #redis_patterns == 0 then + logger.log_notice( "[CACHE_CLEANUP] No Redis cache cleanup flags enabled, skipping Redis cleanup") + plugin.clusterstore:close() + return true, stats + end + + -- Safety validation: verify all patterns have expected prefix + local SAFE_PREFIX = "plugin_malware_scan_" + for _, pattern in ipairs(redis_patterns) do + if not pattern:match("^" .. SAFE_PREFIX) then + logger.log_error( string.format( + "[CACHE_CLEANUP] SAFETY ERROR: Pattern '%s' does not start with expected prefix '%s' - ABORTING cleanup", + pattern, + SAFE_PREFIX + )) + plugin.clusterstore:close() + stats.errors = stats.errors + 1 + return false, stats + end + end + + logger.log_notice( string.format( + "[CACHE_CLEANUP] Cleaning %d Redis cache pattern(s): %s", + #redis_patterns, + table.concat(redis_patterns, ", ") + )) + + local total_keys_deleted = 0 + + for _, pattern in ipairs(redis_patterns) do + local cursor = "0" + local keys_for_pattern = 0 + + repeat + -- SCAN returns: cursor, array of keys + -- SCAN 0 MATCH pattern COUNT 100 + local scan_result, scan_err = plugin.clusterstore:call("scan", cursor, "MATCH", pattern, "COUNT", 100) + + if not scan_result then + logger.log_warn( "[CACHE_CLEANUP] Redis SCAN failed for pattern " .. pattern .. ": " .. tostring(scan_err)) + stats.errors = stats.errors + 1 + database.failure_record(plugin) + break + end + + -- scan_result is a table: {new_cursor, {key1, key2, ...}} + -- Validate scan_result structure to prevent nil dereference + if type(scan_result) ~= "table" or #scan_result < 2 then + logger.log_warn( "[CACHE_CLEANUP] Invalid Redis SCAN result format") + stats.errors = stats.errors + 1 + break + end + + cursor = scan_result[1] + local keys = scan_result[2] + + -- Delete keys using UNLINK (non-blocking) + if keys and #keys > 0 then + -- Safety validation: verify each key has expected prefix before deletion + local SAFE_PREFIX = "plugin_malware_scan_" + local validated_keys = {} + for _, key in ipairs(keys) do + if key:match("^" .. SAFE_PREFIX) then + table.insert(validated_keys, key) + else + logger.log_warn( string.format( + "[CACHE_CLEANUP] SAFETY WARNING: Skipping key '%s' - does not match expected prefix '%s'", + key, + SAFE_PREFIX + )) + stats.errors = stats.errors + 1 + end + end + + -- Only unlink validated keys + if #validated_keys > 0 then + local unlink_result, unlink_err = plugin.clusterstore:call("unlink", unpack(validated_keys)) + if unlink_result then + keys_for_pattern = keys_for_pattern + tonumber(unlink_result) + database.success_record(plugin) + else + logger.log_warn( "[CACHE_CLEANUP] Redis UNLINK failed: " .. tostring(unlink_err)) + stats.errors = stats.errors + 1 + database.failure_record(plugin) + end + end + end + + until cursor == "0" + + if keys_for_pattern > 0 then + logger.log_notice( string.format( + "[CACHE_CLEANUP] Deleted %d keys matching pattern: %s", + keys_for_pattern, + pattern + )) + total_keys_deleted = total_keys_deleted + keys_for_pattern + stats.patterns_cleaned = stats.patterns_cleaned + 1 + table.insert(stats.pattern_details, { + pattern = pattern, + keys_deleted = keys_for_pattern + }) + end + end + + -- Close Redis connection + plugin.clusterstore:close() + + stats.redis_cleared = total_keys_deleted + logger.log_notice( string.format( + "[CACHE_CLEANUP] Redis cleanup complete - deleted %d keys total", + total_keys_deleted + )) + else + logger.log_notice( "[CACHE_CLEANUP] Redis shared cache disabled, skipping Redis cleanup") + end + + -- After cache cleanup, check if MalwareBazaar is enabled and reinitialize hashes + if plugin.variables["MALWARE_SCAN_CUSTOM_HASHES_ENABLED"] == "yes" and + plugin.variables["MALWARE_SCAN_CUSTOM_HASHES_AUTO_UPDATE"] == "yes" then + + logger.log_notice( "[CACHE_CLEANUP] MalwareBazaar hashes enabled - attempting to reinitialize hash database") + + -- Try to call the hash update job script + local update_script = "/etc/bunkerweb/plugins/malware-scan/jobs/malware-scan-update.py" + -- Check if script exists using native Lua (non-blocking) + local script_attr = lfs.attributes(update_script, "mode") + if script_attr == "file" then + logger.log_notice( "[CACHE_CLEANUP] Found malware-scan-update.py, triggering hash redownload") + + -- Execute script in background asynchronously + -- Using nohup to detach from this process + local cmd = "nohup python3 " .. ms_utils.shell_escape_path(update_script) .. " > /dev/null 2>&1 &" + local result = os.execute(cmd) + + if result == 0 then + logger.log_notice( "[CACHE_CLEANUP] Hash update job triggered successfully") + stats.hash_reinitialized = true + else + logger.log_warn( "[CACHE_CLEANUP] Failed to trigger hash update job, will be handled by scheduler") + stats.errors = stats.errors + 1 + end + else + logger.log_warn( "[CACHE_CLEANUP] malware-scan-update.py not found at " .. update_script .. ", hashes will be updated by scheduler") + stats.errors = stats.errors + 1 + end + end + + -- Clean up downloaded MalwareBazaar files and caches + if cleanup_all or plugin.variables["MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_MALWAREBAZAAR"] == "yes" then + logger.log_notice( "[CACHE_CLEANUP] Cleaning up MalwareBazaar downloaded files and caches...") + + local mb_files_cleaned = 0 + + -- Remove CSV hash file if it exists + local csv_path = plugin.variables["MALWARE_SCAN_CUSTOM_HASHES_CSV_PATH"] + if csv_path and csv_path ~= "" then + if os.remove(csv_path) then + logger.log_notice( string.format("[v" .. cache.VERSION .. "] [CACHE_CLEANUP] Removed CSV hash file: %s", csv_path)) + mb_files_cleaned = mb_files_cleaned + 1 + end + end + + -- Remove temporary extraction directory if it exists (native Lua, non-blocking) + local temp_extract_dir = "/tmp/malware-scan-hashes" + local temp_attr = lfs.attributes(temp_extract_dir, "mode") + if temp_attr == "directory" then + if remove_directory_recursive(temp_extract_dir) then + logger.log_notice( string.format("[v" .. cache.VERSION .. "] [CACHE_CLEANUP] Removed temporary extraction directory: %s", temp_extract_dir)) + mb_files_cleaned = mb_files_cleaned + 1 + end + end + + -- Remove downloaded ZIP files from cache directory (native Lua, non-blocking) + local cache_dir = "/var/cache/bunkerweb/malware-scan" + local cache_attr = lfs.attributes(cache_dir, "mode") + if cache_attr == "directory" then + local zip_count = 0 + for filename in lfs.dir(cache_dir) do + if filename:match("%.zip$") then + local file_path = cache_dir .. "/" .. filename + local file_attr = lfs.attributes(file_path, "mode") + if file_attr == "file" then + if os.remove(file_path) then + zip_count = zip_count + 1 + end + end + end + end + if zip_count > 0 then + logger.log_notice( string.format("[v" .. cache.VERSION .. "] [CACHE_CLEANUP] Cleaned up %d ZIP file(s) from cache directory: %s", zip_count, cache_dir)) + mb_files_cleaned = mb_files_cleaned + 1 + end + end + + if mb_files_cleaned > 0 then + stats.local_cleared = stats.local_cleared + mb_files_cleaned + logger.log_notice( string.format("[v" .. cache.VERSION .. "] [CACHE_CLEANUP] MalwareBazaar file cleanup completed - %d items removed", mb_files_cleaned)) + else + logger.log_notice( "[CACHE_CLEANUP] MalwareBazaar file cleanup: no files to remove") + end + end + + -- Calculate total duration + stats.end_time = os.time() + stats.duration_seconds = stats.end_time - stats.start_time + + -- Log cleanup statistics summary + local pattern_summary = "" + if stats.patterns_cleaned > 0 then + local pattern_list = {} + for _, detail in ipairs(stats.pattern_details) do + table.insert(pattern_list, string.format("%s (%d)", detail.pattern, detail.keys_deleted)) + end + pattern_summary = " - Patterns: " .. table.concat(pattern_list, ", ") + end + + logger.log_notice( string.format( + "[CACHE_CLEANUP] โœ… CLEANUP STATISTICS - Local: %d, Redis patterns: %d (%d keys), Hash reinit: %s, Errors: %d, Duration: %ds%s", + stats.local_cleared, + stats.patterns_cleaned, + stats.redis_cleared, + stats.hash_reinitialized and "yes" or "no", + stats.errors, + stats.duration_seconds, + pattern_summary + )) + + return true, stats +end + +-- Return module +return cache diff --git a/malware-scan/malware_scan_clamav.lua b/malware-scan/malware_scan_clamav.lua new file mode 100644 index 0000000..33f53b8 --- /dev/null +++ b/malware-scan/malware_scan_clamav.lua @@ -0,0 +1,435 @@ +-- BunkerWeb Malware Scan - ClamAV Scanner Module +-- Handles all ClamAV-specific scanning operations via INSTREAM protocol + +local logger = require("malware_scan_logger") + +local ngx = ngx +local socket = ngx.socket +local tonumber = tonumber + +-- Module table +local clamav = {} + +-- Module version +clamav.VERSION = "0.8.0" + +-- Cache module reference (will be set when module is loaded) +local cache = nil + +-- Set cache module reference (called during initialization) +function clamav.set_cache_module(cache_module) + cache = cache_module +end + +-- ============================================================================ +-- SOCKET CONNECTION +-- ============================================================================ + +-- Create TCP socket connection to ClamAV. +-- Returns socket object or false and error message. +function clamav.socket(plugin) + local tcp_socket = socket.tcp() + tcp_socket:settimeout(tonumber(plugin.variables["MALWARE_SCAN_CLAMAV_TIMEOUT"])) + local ok, err = tcp_socket:connect( + plugin.variables["MALWARE_SCAN_CLAMAV_HOST"], + tonumber(plugin.variables["MALWARE_SCAN_CLAMAV_PORT"]) + ) + if not ok then + return false, err + end + return tcp_socket +end + +-- Send command to ClamAV daemon. +-- Used for PING and other simple commands. +-- Returns success boolean and response data or error. +function clamav.command(plugin, cmd) + local clamav_socket, err = clamav.socket(plugin) + if not clamav_socket then + return false, err + end + + local bytes + bytes, err = clamav_socket:send("n" .. cmd .. "\n") + if not bytes then + clamav_socket:close() + return false, err + end + + local data + data, err = clamav_socket:receive("*l") + if not data then + clamav_socket:close() + return false, err + end + + clamav_socket:close() + return true, data +end + +-- Test ClamAV connectivity with custom timeout. +-- Used during init_worker to verify ClamAV is accessible. +-- Returns success boolean and error message if failed. +-- +--- @param plugin Plugin instance +--- @param timeout_ms Timeout in milliseconds +--- @return boolean success +--- @return string|nil error_message +function clamav.test_connectivity(plugin, timeout_ms) + -- Save original timeout + local original_timeout = plugin.variables["MALWARE_SCAN_CLAMAV_TIMEOUT"] + + -- Set custom timeout for connectivity test + plugin.variables["MALWARE_SCAN_CLAMAV_TIMEOUT"] = tostring(timeout_ms) + + -- Execute PING command + local ok, data = clamav.command(plugin, "PING") + + -- Restore original timeout + plugin.variables["MALWARE_SCAN_CLAMAV_TIMEOUT"] = original_timeout + + -- Validate response + if not ok then + return false, data or "unknown error" + end + if data ~= "PONG" then + return false, "unexpected response: " .. tostring(data) + end + + return true, nil +end + +-- ============================================================================ +-- FILE SCANNING +-- ============================================================================ + +-- Scan a file using ClamAV (always uses INSTREAM method). +-- Takes file path, returns success boolean and detection result or error. +function clamav.scan_file(plugin, file_path) + -- Always use INSTREAM method (bypasses file permissions) + return clamav.scan_file_instream(plugin, file_path) +end + +-- Scan a file using ClamAV's INSTREAM command (stream content over socket). +-- This bypasses file permission issues by reading the file and sending it to ClamAV. +-- Takes file path, returns success boolean and detection result or error. +function clamav.scan_file_instream(plugin, file_path) + plugin:log_debug("=== INSTREAM: scan_file_instream() called for: " .. file_path .. " ===") + + -- Check cache if cache module is available + if cache then + local ok, cached = cache.cache_check(plugin, file_path) + if ok and cached then + plugin:log_debug("file found in cache: " .. cached) + return true, cached + end + end + + logger.log_error( "scanning file via INSTREAM: " .. file_path) + plugin:log_debug("INSTREAM: Opening file for reading") + + -- Open file for reading + local file, err = io.open(file_path, "rb") + if not file then + return false, "failed to open file: " .. (err or "unknown") + end + + plugin:log_debug("INSTREAM: File opened successfully, creating socket") + + local clamav_socket + clamav_socket, err = clamav.socket(plugin) + if not clamav_socket then + logger.log_error( "INSTREAM socket creation failed: " .. tostring(err)) + file:close() + return false, "socket failed: " .. err + end + + plugin:log_debug("INSTREAM: Socket created, sending INSTREAM command") + + -- Send INSTREAM command + local bytes + bytes, err = clamav_socket:send("zINSTREAM\0") + if not bytes then + logger.log_error( "INSTREAM command send failed: " .. tostring(err)) + clamav_socket:close() + file:close() + return false, "socket:send(INSTREAM) failed: " .. err + end + + plugin:log_debug("INSTREAM: Command sent, reading file chunks") + + -- Send file in chunks with length prefix + local chunk_size = 8192 + local total_bytes = 0 + local chunk_count = 0 + + while true do + local chunk = file:read(chunk_size) + if not chunk then + break + end + + local chunk_len = #chunk + total_bytes = total_bytes + chunk_len + chunk_count = chunk_count + 1 + + plugin:log_debug("INSTREAM: Read chunk " .. chunk_count .. ", size: " .. chunk_len) + + -- Send 4-byte length in network byte order (big-endian) + -- Using math operations instead of bit library for compatibility + local len_bytes = string.char( + math.floor(chunk_len / 16777216) % 256, -- byte 1 (most significant) + math.floor(chunk_len / 65536) % 256, -- byte 2 + math.floor(chunk_len / 256) % 256, -- byte 3 + chunk_len % 256 -- byte 4 (least significant) + ) + + bytes, err = clamav_socket:send(len_bytes .. chunk) + if not bytes then + logger.log_error( "INSTREAM failed to send chunk " .. chunk_count .. ": " .. tostring(err)) + clamav_socket:close() + file:close() + return false, "socket:send(chunk) failed: " .. err + end + + plugin:log_debug("INSTREAM: Chunk " .. chunk_count .. " sent successfully") + end + + file:close() + plugin:log_debug("INSTREAM: All chunks sent - " .. chunk_count .. " chunks, " .. total_bytes .. " bytes total") + + -- Send 0-length to indicate end of stream + plugin:log_debug("INSTREAM: Sending EOF marker") + bytes, err = clamav_socket:send("\0\0\0\0") + if not bytes then + logger.log_error( "INSTREAM EOF send failed: " .. tostring(err)) + clamav_socket:close() + return false, "socket:send(EOF) failed: " .. err + end + + plugin:log_debug("INSTREAM: EOF sent, waiting for response") + + -- Receive response + -- ClamAV may close connection quickly, so we need to handle partial reads + local data, err, partial + data, err, partial = clamav_socket:receive("*l") + + -- Log what we got for debugging + plugin:log_debug("INSTREAM receive result - data: " .. tostring(data) .. ", err: " .. tostring(err) .. ", partial: " .. tostring(partial)) + + -- If socket was closed, check if we got partial data + if not data and err == "closed" and partial and partial ~= "" then + -- We got the response but socket closed before newline + -- This is normal for ClamAV closing connection after detection + data = partial + plugin:log_debug("INSTREAM: Using partial response before socket close") + end + + -- If still no data, this is an error condition + if not data or data == "" then + clamav_socket:close() + logger.log_error( "socket:receive() failed: " .. (err or "unknown") .. " - no data received") + return false, "ClamAV connection error: " .. (err or "no response") + end + + clamav_socket:close() + + -- Debug: Show last few bytes of the response (only in debug mode) + if plugin:is_debug() then + local len = #data + local last_chars = "" + for i = math.max(1, len - 5), len do + local byte = string.byte(data, i) + last_chars = last_chars .. string.format(" %d(0x%02X)", byte, byte) + end + plugin:log_debug("INSTREAM: Last 6 bytes of response:" .. last_chars) + end + + -- Strip trailing whitespace/newline/null bytes from ClamAV response + -- ClamAV INSTREAM responses end with null byte + -- In Lua patterns, %z matches null bytes + data = data:gsub("[%z\r\n\t ]+$", "") + plugin:log_debug("INSTREAM: After trimming - data: [" .. data .. "], length: " .. #data) + + local detected = "clean" + + -- Parse response (format: "stream: OK" or "stream: Virus.Name FOUND") + plugin:log_debug("INSTREAM: Parsing response: [" .. data .. "]") + if data:match(" OK$") then + detected = "clean" + plugin:log_debug("INSTREAM result: CLEAN") + elseif data:match(" FOUND$") then + plugin:log_debug("INSTREAM: FOUND pattern matched, extracting virus name") + -- Extract virus name with non-greedy match and validate (Medium severity issue #14) + local virus_name = data:match(": ([^%c]+) FOUND$") -- [^%c]+ excludes control characters + if virus_name and virus_name ~= "" and #virus_name <= 256 then + -- Sanitize virus name to prevent log injection + virus_name = virus_name:gsub("[\r\n\t%z]", "_") + detected = virus_name + logger.log_notice( "INSTREAM: malware detected - " .. virus_name) + else + detected = "unknown_virus" + logger.log_warn( "INSTREAM: malware detected but virus name extraction failed or invalid") + end + elseif data:match("ERROR") then + -- Check if it's a size limit error + if data:match("INSTREAM size limit exceeded") or data:match("size limit") then + logger.log_error( + "ClamAV INSTREAM size limit exceeded: " .. data .. " - file too large for ClamAV, consider increasing MALWARE_SCAN_CLAMAV_MAX_SIZE" + ) + return false, "file size exceeds ClamAV StreamMaxLength limit" + end + logger.log_error( "ClamAV INSTREAM error: " .. data) + return false, "ClamAV error: " .. data + else + logger.log_warn( "unexpected ClamAV INSTREAM response: " .. data) + detected = "clean" + end + + -- Cache result if cache module is available + if cache then + local ok, err = cache.cache_add(plugin, file_path, detected) + if not ok then + logger.log_error( "can't cache result: " .. err) + end + end + + plugin:log_debug("INSTREAM: Scan completed, returning detected = " .. tostring(detected)) + return true, detected +end + +-- ============================================================================ +-- CONTENT SCANNING (for testing with EICAR) +-- ============================================================================ + +-- Scan content directly via ClamAV's INSTREAM command. +-- Takes content as string, returns success boolean and detection result or error. +-- Used for testing ClamAV with EICAR test string. +function clamav.scan_content_instream(plugin, content) + plugin:log_debug("=== INSTREAM: scan_content_instream() called with " .. #content .. " bytes ===") + + logger.log_error( "scanning content via INSTREAM: " .. #content .. " bytes") + + local clamav_socket, err = clamav.socket(plugin) + if not clamav_socket then + logger.log_error( "INSTREAM socket creation failed: " .. tostring(err)) + return false, "socket failed: " .. err + end + + plugin:log_debug("INSTREAM: Socket created, sending INSTREAM command") + + -- Send INSTREAM command + local bytes + bytes, err = clamav_socket:send("zINSTREAM\0") + if not bytes then + logger.log_error( "INSTREAM command send failed: " .. tostring(err)) + clamav_socket:close() + return false, "socket:send(INSTREAM) failed: " .. err + end + + plugin:log_debug("INSTREAM: Command sent, sending content") + + -- Send content in chunks with length prefix + local chunk_size = 8192 + local total_bytes = 0 + local chunk_count = 0 + local pos = 1 + + while pos <= #content do + local chunk = content:sub(pos, pos + chunk_size - 1) + local chunk_len = #chunk + total_bytes = total_bytes + chunk_len + chunk_count = chunk_count + 1 + + plugin:log_debug("INSTREAM: Sending chunk " .. chunk_count .. ", size: " .. chunk_len) + + -- Send 4-byte length in network byte order (big-endian) + local len_bytes = string.char( + math.floor(chunk_len / 16777216) % 256, + math.floor(chunk_len / 65536) % 256, + math.floor(chunk_len / 256) % 256, + chunk_len % 256 + ) + + bytes, err = clamav_socket:send(len_bytes .. chunk) + if not bytes then + logger.log_error( "INSTREAM failed to send chunk " .. chunk_count .. ": " .. tostring(err)) + clamav_socket:close() + return false, "socket:send(chunk) failed: " .. err + end + + pos = pos + chunk_size + end + + plugin:log_debug("INSTREAM: All chunks sent - " .. chunk_count .. " chunks, " .. total_bytes .. " bytes total") + + -- Send 0-length to indicate end of stream + plugin:log_debug("INSTREAM: Sending EOF marker") + bytes, err = clamav_socket:send("\0\0\0\0") + if not bytes then + logger.log_error( "INSTREAM EOF send failed: " .. tostring(err)) + clamav_socket:close() + return false, "socket:send(EOF) failed: " .. err + end + + plugin:log_debug("INSTREAM: EOF sent, waiting for response") + + -- Receive response + local data, err, partial + data, err, partial = clamav_socket:receive("*l") + + plugin:log_debug("INSTREAM receive result - data: " .. tostring(data) .. ", err: " .. tostring(err) .. ", partial: " .. tostring(partial)) + + -- If socket was closed, check if we got partial data + if not data and err == "closed" and partial and partial ~= "" then + data = partial + plugin:log_debug("INSTREAM: Using partial response before socket close") + end + + -- If still no data, this is an error condition + if not data or data == "" then + clamav_socket:close() + logger.log_error( "socket:receive() failed: " .. (err or "unknown") .. " - no data received") + return false, "ClamAV connection error: " .. (err or "no response") + end + + clamav_socket:close() + + -- Strip trailing whitespace/newline/null bytes from ClamAV response + data = data:gsub("[%z\r\n\t ]+$", "") + plugin:log_debug("INSTREAM: After trimming - data: [" .. data .. "], length: " .. #data) + + local detected = "clean" + + -- Parse response (format: "stream: OK" or "stream: Virus.Name FOUND") + plugin:log_debug("INSTREAM: Parsing response: [" .. data .. "]") + if data:match(" OK$") then + detected = "clean" + plugin:log_debug("INSTREAM result: CLEAN") + elseif data:match(" FOUND$") then + plugin:log_debug("INSTREAM: FOUND pattern matched, extracting virus name") + -- Extract virus name with non-greedy match and validate (Medium severity issue #14) + local virus_name = data:match(": ([^%c]+) FOUND$") -- [^%c]+ excludes control characters + if virus_name and virus_name ~= "" and #virus_name <= 256 then + -- Sanitize virus name to prevent log injection + virus_name = virus_name:gsub("[\r\n\t%z]", "_") + detected = virus_name + logger.log_notice( "INSTREAM: malware detected - " .. virus_name) + else + detected = "unknown_virus" + logger.log_warn( "INSTREAM: malware detected but virus name extraction failed or invalid") + end + elseif data:match("ERROR") then + logger.log_error( "ClamAV INSTREAM error: " .. data) + return false, "ClamAV error: " .. data + else + logger.log_warn( "unexpected ClamAV INSTREAM response: " .. data) + detected = "clean" + end + + plugin:log_debug("INSTREAM: Scan completed, returning detected = " .. tostring(detected)) + return true, detected +end + +-- Return module +return clamav diff --git a/malware-scan/malware_scan_config.lua b/malware-scan/malware_scan_config.lua new file mode 100644 index 0000000..3c86cd3 --- /dev/null +++ b/malware-scan/malware_scan_config.lua @@ -0,0 +1,884 @@ +-- BunkerWeb Malware Scan - Configuration Module +-- Centralized configuration parsing, validation, and logging +-- Provides clean interface for config handling across the plugin + +local logger = require("malware_scan_logger") +local lfs = require("lfs") + +-- Module table +local config = {} + +-- Module version +config.VERSION = "0.8.0" + +-- ============================================================================ +-- CONFIGURATION MASKING +-- ============================================================================ + +-- Mask sensitive configuration values for safe logging. +-- Identifies sensitive keys (API keys, tokens, webhook URLs) and masks their values. +-- This prevents accidental exposure of credentials in logs. +-- +-- Masking rules: +-- - Keys ending with _KEY, _TOKEN, or _WEBHOOK_URL: Show "[MASKED]" +-- - Empty or nil values: Show "(not set)" +-- - All other values: Show as-is +-- +-- Example: +-- config_mask_if_sensitive("MALWARE_SCAN_API_VIRUSTOTAL_KEY", "abc123xyz789") +-- Returns: "[MASKED]" +-- +-- config_mask_if_sensitive("MALWARE_SCAN_CLAMAV_HOST", "clamav") +-- Returns: "clamav" +-- +--- @param key string The configuration key name (used to detect sensitive fields) +--- @param value string The configuration value to potentially mask +--- @return string Masked string or original value, or "(not set)" if empty +local function config_mask_if_sensitive(key, value) + if not value or value == "" then + return "(not set)" + end + -- Mask API keys, tokens, and webhook URLs (pattern matching on key name) + if key:match("_KEY$") or key:match("_TOKEN$") or key:match("_WEBHOOK_URL$") then + return "[MASKED]" + end + return value +end + +-- ============================================================================ +-- CONFIGURATION LOADING +-- ============================================================================ + +-- Load and validate configuration from plugin variables. +-- Extracts all plugin configuration into a structured table for easy access. +-- Performs basic validation on critical settings. +-- +-- Configuration structure: +-- { +-- core = { enabled, debug, unmask_keys }, +-- clamav = { enabled, host, port, timeout, max_size }, +-- file_limits = { hash_max_size }, +-- behavior = { block_memory_uploads, cleanup_files, cleanup_orphaned, ... }, +-- cache = { use_shared_database, virus_ttl, clean_ttl, shared_ttl }, +-- cache_cleanup = { all, clamav, virustotal, sentinelone, attacker_ipv4, attacker_ipv6 }, +-- api = { virustotal = {...}, sentinelone = {...} }, +-- webhooks = { teams = {...}, discord = {...} }, +-- attackers = { track, ttl }, +-- multipart = { scan_full_body } +-- } +-- +--- @param plugin Plugin The plugin instance (provides access to self.variables) +--- @return table|nil cfg Configuration table with all settings +--- @return string|nil error Error message on failure +function config.load(plugin) + local vars = plugin.variables + + -- Validate that variables are accessible + if not vars then + return nil, "plugin.variables is nil" + end + + -- Build configuration table + local cfg = { + -- Core settings + core = { + enabled = vars["USE_MALWARE_SCANNER"] or "no", + debug = vars["MALWARE_SCAN_DEBUG"] or "no", + unmask_keys = vars["MALWARE_SCAN_UNMASK_KEYS"] or "", + }, + + -- ClamAV settings + clamav = { + enabled = vars["MALWARE_SCAN_CLAMAV_ENABLED"] or "yes", + host = vars["MALWARE_SCAN_CLAMAV_HOST"] or "clamav", + port = vars["MALWARE_SCAN_CLAMAV_PORT"] or "3310", + timeout = vars["MALWARE_SCAN_CLAMAV_TIMEOUT"] or "10000", + max_size = vars["MALWARE_SCAN_CLAMAV_MAX_SIZE"] or "26214400", + }, + + -- File size limits + file_limits = { + hash_max_size = vars["MALWARE_SCAN_HASH_MAX_SIZE"] or "67108864", + }, + + -- Behavior settings + behavior = { + block_memory_uploads = vars["MALWARE_SCAN_BLOCK_MEMORY_UPLOADS"] or "yes", + cleanup_files = vars["MALWARE_SCAN_CLEANUP_FILES"] or "yes", + cleanup_orphaned = vars["MALWARE_SCAN_CLEANUP_ORPHANED"] or "yes", + cleanup_age = vars["MALWARE_SCAN_CLEANUP_AGE"] or "3600", + skip_others_on_clamav_detect = vars["MALWARE_SCAN_SKIP_OTHERS_ON_CLAMAV_DETECT"] or "yes", + resume_scan_on_first_hit = vars["MALWARE_SCAN_RESUME_SCAN_ON_FIRST_HIT"] or "no", + block_on_filename_injection_attack = vars["MALWARE_SCAN_BLOCK_ON_FILENAME_INJECTION_ATTACK"] or "yes", + }, + + -- Cache settings + cache = { + use_shared_database = vars["MALWARE_SCAN_USE_SHARED_DATABASE"] or "no", + virus_ttl = vars["MALWARE_SCAN_CACHE_VIRUS_TTL"] or "86400", + clean_ttl = vars["MALWARE_SCAN_CACHE_CLEAN_TTL"] or "3600", + shared_ttl = vars["MALWARE_SCAN_CACHE_SHARED_TTL"] or "86400", + }, + + -- Cache cleanup on restart + cache_cleanup = { + all = vars["MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_ALL"] or "no", + clamav = vars["MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_CLAMAV"] or "no", + virustotal = vars["MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_VIRUSTOTAL"] or "no", + sentinelone = vars["MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_SENTINELONE"] or "no", + attacker_ipv4 = vars["MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_ATTACKER_IPV4"] or "no", + attacker_ipv6 = vars["MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_ATTACKER_IPV6"] or "no", + }, + + -- API integrations + api = { + virustotal = { + enabled = vars["MALWARE_SCAN_API_USE_VIRUSTOTAL"] or "no", + key = vars["MALWARE_SCAN_API_VIRUSTOTAL_KEY"] or "", + suspicious_threshold = vars["MALWARE_SCAN_API_VIRUSTOTAL_SUSPICIOUS"] or "5", + malicious_threshold = vars["MALWARE_SCAN_API_VIRUSTOTAL_MALICIOUS"] or "1", + }, + sentinelone = { + enabled = vars["MALWARE_SCAN_API_USE_SENTINELONE"] or "no", + url = vars["MALWARE_SCAN_API_SENTINELONE_URL"] or "", + token = vars["MALWARE_SCAN_API_SENTINELONE_TOKEN"] or "", + }, + }, + + -- Webhook notifications + webhooks = { + teams = { + url = vars["MALWARE_SCAN_TEAMS_WEBHOOK_URL"] or "", + rate_limit = vars["MALWARE_SCAN_TEAMS_WEBHOOK_RATE_LIMIT"] or "60", + }, + discord = { + url = vars["MALWARE_SCAN_DISCORD_WEBHOOK_URL"] or "", + rate_limit = vars["MALWARE_SCAN_DISCORD_WEBHOOK_RATE_LIMIT"] or "60", + }, + }, + + -- Attacker tracking + attackers = { + track = vars["MALWARE_SCAN_TRACK_ATTACKERS"] or "yes", + ttl = vars["MALWARE_SCAN_TRACK_ATTACKER_TTL"] or "3600", + }, + + -- Multipart scanning + multipart = { + scan_full_body = vars["MALWARE_SCAN_SCAN_FULL_MULTIPART_BODY"] or "no", + }, + } + + return cfg +end + +-- ============================================================================ +-- CONFIGURATION VALIDATION (BASIC) +-- ============================================================================ + +-- Validate critical configuration values. +-- Checks that required settings are present and within acceptable ranges. +-- Currently validates ClamAV connectivity settings. +-- +--- @param cfg Configuration table from config.load() +--- @return boolean success True on success +--- @return string|nil error Error message on validation failure +function config.validate(cfg) + if not cfg then + return false, "configuration is nil" + end + + -- Validate ClamAV settings if enabled + if cfg.clamav.enabled == "yes" then + if not cfg.clamav.host or cfg.clamav.host == "" then + return false, "MALWARE_SCAN_CLAMAV_HOST is required when ClamAV is enabled" + end + + local port = tonumber(cfg.clamav.port) + if not port or port < 1 or port > 65535 then + return false, "MALWARE_SCAN_CLAMAV_PORT must be between 1 and 65535" + end + + local timeout = tonumber(cfg.clamav.timeout) + if not timeout or timeout < 100 then + return false, "MALWARE_SCAN_CLAMAV_TIMEOUT must be at least 100ms" + end + end + + return true +end + +-- ============================================================================ +-- COMPREHENSIVE STARTUP VALIDATION +-- ============================================================================ + +-- Load utilities module for numerical validation (lazy load to avoid init issues) +local utils = nil +local function config_get_utils() + if not utils then + utils = require("malware_scan_utils") + end + return utils +end + +--- Create directory recursively (like mkdir -p) +--- @param path string Directory path to create +--- @return boolean success True if directory was created or already exists +--- @return string|nil error Error message on failure +local function config_mkdir_recursive(path) + if not path or path == "" then + return false, "path is empty" + end + + -- Check if directory already exists + local attr = lfs.attributes(path, "mode") + if attr == "directory" then + return true, nil + elseif attr then + return false, "path exists but is not a directory" + end + + -- Split path into components + local components = {} + for component in path:gmatch("[^/]+") do + table.insert(components, component) + end + + -- Build path incrementally + local current_path = path:sub(1, 1) == "/" and "/" or "" + for _, component in ipairs(components) do + current_path = current_path .. component .. "/" + + -- Check if this level exists + local level_attr = lfs.attributes(current_path, "mode") + if not level_attr then + -- Create this directory level + local ok, err = lfs.mkdir(current_path) + if not ok then + return false, "failed to create directory " .. current_path .. ": " .. (err or "unknown error") + end + elseif level_attr ~= "directory" then + return false, "path component exists but is not a directory: " .. current_path + end + end + + return true, nil +end + +--- Check if a directory exists and is writable +--- @param path string Directory path to check +--- @return boolean exists True if directory exists +--- @return boolean writable True if directory is writable +--- @return string|nil error Error message if any +local function config_check_directory(path) + if not path or path == "" then + return false, false, "path is empty" + end + + -- Validate path to prevent command injection + local u = config_get_utils() + local validated_path, err = u.validate_and_escape_path(path) + if not validated_path then + return false, false, err + end + + -- Check if directory exists using lfs.attributes (non-blocking) + local attr = lfs.attributes(path, "mode") + if not attr then + return false, false, "directory does not exist" + end + + if attr ~= "directory" then + return false, false, "path is not a directory" + end + + -- Check write permission by attempting to create a test file + -- Use os.tmpname() for safe temporary file creation + local tmpname = os.tmpname() + local test_file = path .. "/" .. tmpname:match("([^/]+)$") + local fh, open_err = io.open(test_file, "w") + if not fh then + return true, false, "directory is not writable: " .. (open_err or "unknown error") + end + fh:close() + + -- Clean up test file + local remove_ok, remove_err = os.remove(test_file) + if not remove_ok then + -- Log but don't fail - directory is writable even if cleanup failed + logger.log_warn(string.format("Failed to remove write test file %s: %s", test_file, remove_err or "unknown error")) + end + + return true, true, nil +end + +--- Check if a file exists and is readable +--- @param path string File path to check +--- @return boolean exists True if file exists +--- @return boolean readable True if file is readable +--- @return string|nil error Error message if any +local function config_check_file(path) + if not path or path == "" then + return false, false, "path is empty" + end + + -- Validate path to prevent command injection + local u = config_get_utils() + local validated_path, err = u.validate_and_escape_path(path) + if not validated_path then + return false, false, err + end + + -- Check if file exists using lfs.attributes (non-blocking) + local attr = lfs.attributes(path, "mode") + if not attr then + return false, false, "file does not exist" + end + + if attr ~= "file" then + return false, false, "path is not a file" + end + + -- Check read permission by attempting to open the file + local fh, open_err = io.open(path, "r") + if not fh then + return true, false, "file is not readable: " .. (open_err or "unknown error") + end + fh:close() + + return true, true, nil +end + +--- Check if a Unix socket exists +--- @param path string Socket path to check +--- @return boolean exists True if socket exists +--- @return string|nil error Error message if any +local function config_check_socket(path) + if not path or path == "" then + return false, "socket path is empty" + end + + -- Validate path to prevent command injection + local u = config_get_utils() + local validated_path, err = u.validate_and_escape_path(path) + if not validated_path then + return false, err + end + + -- Check if socket exists using lfs.attributes (non-blocking) + local attr = lfs.attributes(path, "mode") + if not attr or attr ~= "socket" then + return false, "socket does not exist or is not a socket file" + end + + return true, nil +end + +--- Validate path configurations +--- @param variables table Plugin configuration variables (plugin.variables) +--- @return boolean valid True if all paths are valid +--- @return table errors List of validation errors +local function config_validate_paths(variables) + local errors = {} + + -- Validate export path (if set) + local export_path = variables["MALWARE_SCAN_VIRUSFILE_LOCAL_EXPORT_PATH"] + if export_path and export_path ~= "" then + local exists, writable, err = config_check_directory(export_path) + if not exists then + -- Directory doesn't exist - try to create it recursively (like mkdir -p) + local u = config_get_utils() + local validated_path, escape_err = u.validate_and_escape_path(export_path) + if not validated_path then + table.insert(errors, string.format( + "Export path validation failed: %s (%s)", + export_path, escape_err or "unknown error" + )) + else + local mkdir_ok, mkdir_err = config_mkdir_recursive(export_path) + if not mkdir_ok then + table.insert(errors, string.format( + "Export path does not exist and could not be created: %s (error: %s)", + export_path, mkdir_err or "unknown error" + )) + end + end + elseif not writable then + table.insert(errors, string.format( + "Export path is not writable: %s (%s)", + export_path, err or "unknown error" + )) + end + end + + -- Validate temp directory + local temp_path = "/var/tmp/bunkerweb/client_temp" + local exists, writable, err = config_check_directory(temp_path) + if not exists then + table.insert(errors, string.format( + "Temp directory does not exist: %s (%s)", + temp_path, err or "unknown error" + )) + elseif not writable then + table.insert(errors, string.format( + "Temp directory is not writable: %s (%s)", + temp_path, err or "unknown error" + )) + end + + -- Validate allowlist file (if set) + local allowlist_file = variables["MALWARE_SCAN_ALLOWLIST_FILE"] + if allowlist_file and allowlist_file ~= "" then + local exists, readable, err = config_check_file(allowlist_file) + if not exists then + table.insert(errors, string.format( + "Allowlist file does not exist: %s (%s)", + allowlist_file, err or "unknown error" + )) + elseif not readable then + table.insert(errors, string.format( + "Allowlist file is not readable: %s (%s)", + allowlist_file, err or "unknown error" + )) + end + end + + return #errors == 0, errors +end + +--- Validate numerical configuration values +--- @param variables table Plugin configuration variables (plugin.variables) +--- @return boolean valid True if all numerical configs are valid +--- @return table errors List of validation errors +local function config_validate_numerical_configs(variables) + local errors = {} + local u = config_get_utils() + + -- Validate MALWARE_SCAN_MAX_FILE_SIZE (0 to 10GB) + local max_file_size = tonumber(variables["MALWARE_SCAN_MAX_FILE_SIZE"]) + if max_file_size then + local validated = u.validate_file_size(max_file_size, 10 * 1024 * 1024 * 1024) + if validated == 0 and max_file_size ~= 0 then + table.insert(errors, string.format( + "MALWARE_SCAN_MAX_FILE_SIZE is invalid: %s (must be 0 to 10GB)", + variables["MALWARE_SCAN_MAX_FILE_SIZE"] + )) + end + end + + -- Validate MALWARE_SCAN_HASH_MAX_SIZE (0 to 10GB) + local hash_max_size = tonumber(variables["MALWARE_SCAN_HASH_MAX_SIZE"]) + if hash_max_size then + local validated = u.validate_file_size(hash_max_size, 10 * 1024 * 1024 * 1024) + if validated == 0 and hash_max_size ~= 0 then + table.insert(errors, string.format( + "MALWARE_SCAN_HASH_MAX_SIZE is invalid: %s (must be 0 to 10GB)", + variables["MALWARE_SCAN_HASH_MAX_SIZE"] + )) + end + end + + -- Validate MALWARE_SCAN_MAX_FILES_PER_REQUEST (1 to 1000) + local max_files = tonumber(variables["MALWARE_SCAN_MAX_FILES_PER_REQUEST"]) + if max_files then + if max_files < 1 then + table.insert(errors, string.format( + "MALWARE_SCAN_MAX_FILES_PER_REQUEST is too low: %s (must be >= 1)", + variables["MALWARE_SCAN_MAX_FILES_PER_REQUEST"] + )) + elseif max_files > 1000 then + table.insert(errors, string.format( + "MALWARE_SCAN_MAX_FILES_PER_REQUEST is too high: %s (must be <= 1000)", + variables["MALWARE_SCAN_MAX_FILES_PER_REQUEST"] + )) + end + end + + -- Validate MALWARE_SCAN_TRACK_ATTACKER_TTL (3600 to 31536000 = 1 hour to 1 year) + local attacker_ttl = tonumber(variables["MALWARE_SCAN_TRACK_ATTACKER_TTL"]) + if attacker_ttl then + -- Check bounds directly (validate_ttl doesn't enforce our minimum of 3600) + if attacker_ttl < 3600 then + table.insert(errors, string.format( + "MALWARE_SCAN_TRACK_ATTACKER_TTL is too low: %s (must be >= 3600 seconds / 1 hour)", + variables["MALWARE_SCAN_TRACK_ATTACKER_TTL"] + )) + elseif attacker_ttl > 31536000 then + table.insert(errors, string.format( + "MALWARE_SCAN_TRACK_ATTACKER_TTL is too high: %s (must be <= 31536000 seconds / 1 year)", + variables["MALWARE_SCAN_TRACK_ATTACKER_TTL"] + )) + end + end + + -- Validate MALWARE_SCAN_CACHE_HASH_MAXTTL (60 to 86400 = 1 minute to 1 day) + local cache_ttl = tonumber(variables["MALWARE_SCAN_CACHE_HASH_MAXTTL"]) + if cache_ttl then + if cache_ttl < 60 then + table.insert(errors, string.format( + "MALWARE_SCAN_CACHE_HASH_MAXTTL is too low: %s (must be >= 60 seconds / 1 minute)", + variables["MALWARE_SCAN_CACHE_HASH_MAXTTL"] + )) + elseif cache_ttl > 86400 then + table.insert(errors, string.format( + "MALWARE_SCAN_CACHE_HASH_MAXTTL is too high: %s (must be <= 86400 seconds / 1 day)", + variables["MALWARE_SCAN_CACHE_HASH_MAXTTL"] + )) + end + end + + return #errors == 0, errors +end + +--- Validate VirusTotal API key format +--- @param api_key string VirusTotal API key +--- @return boolean valid True if API key is valid +--- @return string|nil error Error message if invalid +local function config_validate_virustotal_key(api_key) + if not api_key or api_key == "" then + return false, "API key is empty" + end + + -- VirusTotal API keys are 64 character hex strings + if #api_key ~= 64 then + return false, string.format("API key has invalid length: %d (expected 64)", #api_key) + end + + if not api_key:match("^[a-fA-F0-9]+$") then + return false, "API key contains invalid characters (expected hex string)" + end + + return true, nil +end + +--- Validate MalwareBazaar API key format +--- @param api_key string MalwareBazaar API key +--- @return boolean valid True if API key is valid +--- @return string|nil error Error message if invalid +local function config_validate_malwarebazaar_key(api_key) + if not api_key or api_key == "" then + return false, "API key is empty" + end + + -- MalwareBazaar API keys are variable length alphanumeric strings + if #api_key < 16 then + return false, string.format("API key is too short: %d characters (expected >= 16)", #api_key) + end + + if not api_key:match("^[a-zA-Z0-9]+$") then + return false, "API key contains invalid characters (expected alphanumeric)" + end + + return true, nil +end + +--- Validate scanner dependencies +--- @param variables table Plugin configuration variables (plugin.variables) +--- @return boolean valid True if all scanner dependencies are valid +--- @return table errors List of validation errors +--- @return table warnings List of validation warnings +local function config_validate_scanner_dependencies(variables) + local errors = {} + local warnings = {} + + -- Check if any scanners are enabled + local clamav_enabled = variables["MALWARE_SCAN_USE_CLAMAV"] == "yes" + local virustotal_enabled = variables["MALWARE_SCAN_API_USE_VIRUSTOTAL"] == "yes" + local sentinelone_enabled = variables["MALWARE_SCAN_API_USE_SENTINELONE"] == "yes" + local malwarebazaar_enabled = variables["MALWARE_SCAN_MALWAREBAZAAR_HASH_LOOKUP"] == "yes" + + if not clamav_enabled and not virustotal_enabled and not sentinelone_enabled and not malwarebazaar_enabled then + table.insert(warnings, "No scanners are enabled - plugin will not perform any malware scanning") + end + + -- Validate ClamAV socket if enabled + if clamav_enabled then + local socket_path = variables["MALWARE_SCAN_CLAMAV_SOCKET"] + if socket_path and socket_path ~= "" then + local valid, err = config_check_socket(socket_path) + if not valid then + table.insert(errors, string.format( + "ClamAV is enabled but socket is invalid: %s (%s)", + socket_path, err or "unknown error" + )) + end + end + end + + -- Validate VirusTotal API key if enabled + if virustotal_enabled then + local api_key = variables["MALWARE_SCAN_API_VIRUSTOTAL_KEY"] + local valid, err = config_validate_virustotal_key(api_key) + if not valid then + table.insert(errors, string.format( + "VirusTotal is enabled but API key is invalid: %s", + err or "unknown error" + )) + end + + -- Warn if upload is enabled + if variables["MALWARE_SCAN_API_VIRUSTOTAL_UPLOAD_UNKNOWN"] == "yes" then + table.insert(warnings, "VirusTotal file upload is enabled - files will be uploaded to public database (GDPR implications)") + end + end + + -- Validate MalwareBazaar API key if enabled + if malwarebazaar_enabled then + local api_key = variables["MALWARE_SCAN_API_MALWAREBAZAAR_KEY"] + local valid, err = config_validate_malwarebazaar_key(api_key) + if not valid then + table.insert(errors, string.format( + "MalwareBazaar is enabled but API key is invalid: %s", + err or "unknown error" + )) + end + + -- Warn if upload is enabled + if variables["MALWARE_SCAN_API_MALWAREBAZAAR_UPLOAD_UNKNOWN"] == "yes" then + table.insert(warnings, "MalwareBazaar file upload is enabled - files will be uploaded to public database (GDPR implications)") + end + end + + -- Warn if file sharing is enabled but VirusTotal is disabled + if variables["MALWARE_SCAN_SHARE_FILES"] == "yes" and not virustotal_enabled then + table.insert(warnings, "File sharing is enabled (MALWARE_SCAN_SHARE_FILES=yes) but VirusTotal is disabled - no files will be shared") + end + + return #errors == 0, errors, warnings +end + +--- Validate mutually exclusive or unusual option combinations +--- @param variables table Plugin configuration variables (plugin.variables) +--- @return boolean valid True if all option logic is valid +--- @return table errors List of validation errors +--- @return table warnings List of validation warnings +local function config_validate_option_logic(variables) + local errors = {} + local warnings = {} + + -- Check if both allow list and block list are configured (not actually mutually exclusive, just unusual) + local allowlist = variables["MALWARE_SCAN_ALLOWLIST"] + local allowlist_file = variables["MALWARE_SCAN_ALLOWLIST_FILE"] + if (allowlist and allowlist ~= "") or (allowlist_file and allowlist_file ~= "") then + table.insert(warnings, "Allow list is configured - false positives will bypass scanning") + end + + -- Check if attacker tracking is disabled but ban actions are enabled + if variables["MALWARE_SCAN_TRACK_ATTACKERS"] ~= "yes" then + if variables["MALWARE_SCAN_BAN_ATTACKERS"] == "yes" then + table.insert(warnings, "Ban attackers is enabled but attacker tracking is disabled - no attackers will be banned") + end + end + + return #errors == 0, errors, warnings +end + +--- Comprehensive configuration validation at startup +-- Validates all configuration settings including paths, numerical values, +-- scanner dependencies, and logical consistency. +-- Returns validation results with errors (critical) and warnings (non-critical). +-- +--- @param variables table Plugin configuration variables (plugin.variables) +--- @return boolean valid True if all validations pass +--- @return table errors List of critical validation errors +--- @return table warnings List of non-critical validation warnings +function config.validate_startup(variables) + local all_errors = {} + local all_warnings = {} + + -- Validate paths + local paths_valid, paths_errors = config_validate_paths(variables) + if not paths_valid then + for _, err in ipairs(paths_errors) do + table.insert(all_errors, "[PATH] " .. err) + end + end + + -- Validate numerical configs + local numeric_valid, numeric_errors = config_validate_numerical_configs(variables) + if not numeric_valid then + for _, err in ipairs(numeric_errors) do + table.insert(all_errors, "[CONFIG] " .. err) + end + end + + -- Validate scanner dependencies + local scanner_valid, scanner_errors, scanner_warnings = config_validate_scanner_dependencies(variables) + if not scanner_valid then + for _, err in ipairs(scanner_errors) do + table.insert(all_errors, "[SCANNER] " .. err) + end + end + if scanner_warnings then + for _, warn in ipairs(scanner_warnings) do + table.insert(all_warnings, "[SCANNER] " .. warn) + end + end + + -- Validate option logic + local logic_valid, logic_errors, logic_warnings = config_validate_option_logic(variables) + if not logic_valid then + for _, err in ipairs(logic_errors) do + table.insert(all_errors, "[LOGIC] " .. err) + end + end + if logic_warnings then + for _, warn in ipairs(logic_warnings) do + table.insert(all_warnings, "[LOGIC] " .. warn) + end + end + + return #all_errors == 0, all_errors, all_warnings +end + +-- ============================================================================ +-- CONFIGURATION LOGGING +-- ============================================================================ + +-- Dump full configuration to logs for diagnostics. +-- Logs all configuration settings in organized sections, masking sensitive values. +-- This is useful for troubleshooting and verifying configuration at worker startup. +-- +-- Log sections: +-- 1. Core Settings +-- 2. ClamAV Settings +-- 3. File Size Limits +-- 4. Behavior Settings +-- 5. Cache Settings +-- 6. Cache Cleanup on Restart +-- 7. API Integrations +-- 8. Webhook Notifications +-- 9. Attacker Tracking +-- 10. Multipart Scanning +-- 11. Module Availability +-- +--- @param plugin Plugin The plugin instance (provides logger) +--- @param cfg table Configuration table from config.load() +--- @param module_status table|nil Optional table with module availability status +function config.dump(plugin, cfg, module_status) + logger.log_notice( "[INIT_WORKER] ========== Configuration Dump ==========") + + -- Core settings + logger.log_notice( "[INIT_WORKER] ## Core Settings") + logger.log_notice( string.format("[v" .. config.VERSION .. "] [INIT_WORKER] USE_MALWARE_SCANNER = %s", config_mask_if_sensitive("ENABLED", cfg.core.enabled))) + logger.log_notice( string.format("[v" .. config.VERSION .. "] [INIT_WORKER] MALWARE_SCAN_DEBUG = %s", config_mask_if_sensitive("DEBUG", cfg.core.debug))) + + -- ClamAV settings + logger.log_notice( "[INIT_WORKER] ## ClamAV Settings") + logger.log_notice( string.format("[v" .. config.VERSION .. "] [INIT_WORKER] MALWARE_SCAN_CLAMAV_ENABLED = %s", config_mask_if_sensitive("ENABLED", cfg.clamav.enabled))) + logger.log_notice( string.format("[v" .. config.VERSION .. "] [INIT_WORKER] MALWARE_SCAN_CLAMAV_HOST = %s", config_mask_if_sensitive("HOST", cfg.clamav.host))) + logger.log_notice( string.format("[v" .. config.VERSION .. "] [INIT_WORKER] MALWARE_SCAN_CLAMAV_PORT = %s", config_mask_if_sensitive("PORT", cfg.clamav.port))) + logger.log_notice( string.format("[v" .. config.VERSION .. "] [INIT_WORKER] MALWARE_SCAN_CLAMAV_TIMEOUT = %s", config_mask_if_sensitive("TIMEOUT", cfg.clamav.timeout))) + logger.log_notice( string.format("[v" .. config.VERSION .. "] [INIT_WORKER] MALWARE_SCAN_CLAMAV_MAX_SIZE = %s", config_mask_if_sensitive("SIZE", cfg.clamav.max_size))) + + -- File size limits + logger.log_notice( "[INIT_WORKER] ## File Size Limits") + logger.log_notice( string.format("[v" .. config.VERSION .. "] [INIT_WORKER] MALWARE_SCAN_HASH_MAX_SIZE = %s", config_mask_if_sensitive("SIZE", cfg.file_limits.hash_max_size))) + + -- Behavior settings + logger.log_notice( "[INIT_WORKER] ## Behavior Settings") + logger.log_notice( string.format("[v" .. config.VERSION .. "] [INIT_WORKER] MALWARE_SCAN_BLOCK_MEMORY_UPLOADS = %s", config_mask_if_sensitive("BLOCK", cfg.behavior.block_memory_uploads))) + logger.log_notice( string.format("[v" .. config.VERSION .. "] [INIT_WORKER] MALWARE_SCAN_CLEANUP_FILES = %s", config_mask_if_sensitive("CLEANUP", cfg.behavior.cleanup_files))) + logger.log_notice( string.format("[v" .. config.VERSION .. "] [INIT_WORKER] MALWARE_SCAN_CLEANUP_ORPHANED = %s", config_mask_if_sensitive("CLEANUP", cfg.behavior.cleanup_orphaned))) + logger.log_notice( string.format("[v" .. config.VERSION .. "] [INIT_WORKER] MALWARE_SCAN_CLEANUP_AGE = %s", config_mask_if_sensitive("AGE", cfg.behavior.cleanup_age))) + logger.log_notice( string.format("[v" .. config.VERSION .. "] [INIT_WORKER] MALWARE_SCAN_SKIP_OTHERS_ON_CLAMAV_DETECT = %s", config_mask_if_sensitive("SKIP", cfg.behavior.skip_others_on_clamav_detect))) + logger.log_notice( string.format("[v" .. config.VERSION .. "] [INIT_WORKER] MALWARE_SCAN_RESUME_SCAN_ON_FIRST_HIT = %s", config_mask_if_sensitive("RESUME", cfg.behavior.resume_scan_on_first_hit))) + logger.log_notice( string.format("[v" .. config.VERSION .. "] [INIT_WORKER] MALWARE_SCAN_BLOCK_ON_FILENAME_INJECTION_ATTACK = %s", config_mask_if_sensitive("BLOCK", cfg.behavior.block_on_filename_injection_attack))) + + -- Cache settings + logger.log_notice( "[INIT_WORKER] ## Cache Settings") + logger.log_notice( string.format("[v" .. config.VERSION .. "] [INIT_WORKER] MALWARE_SCAN_USE_SHARED_DATABASE = %s", config_mask_if_sensitive("USE", cfg.cache.use_shared_database))) + logger.log_notice( string.format("[v" .. config.VERSION .. "] [INIT_WORKER] MALWARE_SCAN_CACHE_VIRUS_TTL = %s", config_mask_if_sensitive("TTL", cfg.cache.virus_ttl))) + logger.log_notice( string.format("[v" .. config.VERSION .. "] [INIT_WORKER] MALWARE_SCAN_CACHE_CLEAN_TTL = %s", config_mask_if_sensitive("TTL", cfg.cache.clean_ttl))) + logger.log_notice( string.format("[v" .. config.VERSION .. "] [INIT_WORKER] MALWARE_SCAN_CACHE_SHARED_TTL = %s", config_mask_if_sensitive("TTL", cfg.cache.shared_ttl))) + + -- Cache cleanup on restart + logger.log_notice( "[INIT_WORKER] ## Cache Cleanup on Restart") + logger.log_notice( string.format("[v" .. config.VERSION .. "] [INIT_WORKER] MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_ALL = %s", config_mask_if_sensitive("CLEANUP", cfg.cache_cleanup.all))) + logger.log_notice( string.format("[v" .. config.VERSION .. "] [INIT_WORKER] MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_CLAMAV = %s", config_mask_if_sensitive("CLEANUP", cfg.cache_cleanup.clamav))) + logger.log_notice( string.format("[v" .. config.VERSION .. "] [INIT_WORKER] MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_VIRUSTOTAL = %s", config_mask_if_sensitive("CLEANUP", cfg.cache_cleanup.virustotal))) + logger.log_notice( string.format("[v" .. config.VERSION .. "] [INIT_WORKER] MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_SENTINELONE = %s", config_mask_if_sensitive("CLEANUP", cfg.cache_cleanup.sentinelone))) + logger.log_notice( string.format("[v" .. config.VERSION .. "] [INIT_WORKER] MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_ATTACKER_IPV4 = %s", config_mask_if_sensitive("CLEANUP", cfg.cache_cleanup.attacker_ipv4))) + logger.log_notice( string.format("[v" .. config.VERSION .. "] [INIT_WORKER] MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_ATTACKER_IPV6 = %s", config_mask_if_sensitive("CLEANUP", cfg.cache_cleanup.attacker_ipv6))) + + -- API integrations + logger.log_notice( "[INIT_WORKER] ## API Integrations") + logger.log_notice( string.format("[v" .. config.VERSION .. "] [INIT_WORKER] MALWARE_SCAN_API_USE_VIRUSTOTAL = %s", config_mask_if_sensitive("USE", cfg.api.virustotal.enabled))) + logger.log_notice( string.format("[v" .. config.VERSION .. "] [INIT_WORKER] MALWARE_SCAN_API_VIRUSTOTAL_KEY = %s", config_mask_if_sensitive("MALWARE_SCAN_API_VIRUSTOTAL_KEY", cfg.api.virustotal.key))) + logger.log_notice( string.format("[v" .. config.VERSION .. "] [INIT_WORKER] MALWARE_SCAN_API_VIRUSTOTAL_SUSPICIOUS = %s", config_mask_if_sensitive("SUSPICIOUS", cfg.api.virustotal.suspicious_threshold))) + logger.log_notice( string.format("[v" .. config.VERSION .. "] [INIT_WORKER] MALWARE_SCAN_API_VIRUSTOTAL_MALICIOUS = %s", config_mask_if_sensitive("MALICIOUS", cfg.api.virustotal.malicious_threshold))) + logger.log_notice( string.format("[v" .. config.VERSION .. "] [INIT_WORKER] MALWARE_SCAN_API_USE_SENTINELONE = %s", config_mask_if_sensitive("USE", cfg.api.sentinelone.enabled))) + logger.log_notice( string.format("[v" .. config.VERSION .. "] [INIT_WORKER] MALWARE_SCAN_API_SENTINELONE_URL = %s", config_mask_if_sensitive("URL", cfg.api.sentinelone.url))) + logger.log_notice( string.format("[v" .. config.VERSION .. "] [INIT_WORKER] MALWARE_SCAN_API_SENTINELONE_TOKEN = %s", config_mask_if_sensitive("MALWARE_SCAN_API_SENTINELONE_TOKEN", cfg.api.sentinelone.token))) + + -- Webhook notifications + logger.log_notice( "[INIT_WORKER] ## Webhook Notifications") + logger.log_notice( string.format("[v" .. config.VERSION .. "] [INIT_WORKER] MALWARE_SCAN_TEAMS_WEBHOOK_URL = %s", config_mask_if_sensitive("MALWARE_SCAN_TEAMS_WEBHOOK_URL", cfg.webhooks.teams.url))) + logger.log_notice( string.format("[v" .. config.VERSION .. "] [INIT_WORKER] MALWARE_SCAN_TEAMS_WEBHOOK_RATE_LIMIT = %s", config_mask_if_sensitive("RATE", cfg.webhooks.teams.rate_limit))) + logger.log_notice( string.format("[v" .. config.VERSION .. "] [INIT_WORKER] MALWARE_SCAN_DISCORD_WEBHOOK_URL = %s", config_mask_if_sensitive("MALWARE_SCAN_DISCORD_WEBHOOK_URL", cfg.webhooks.discord.url))) + logger.log_notice( string.format("[v" .. config.VERSION .. "] [INIT_WORKER] MALWARE_SCAN_DISCORD_WEBHOOK_RATE_LIMIT = %s", config_mask_if_sensitive("RATE", cfg.webhooks.discord.rate_limit))) + + -- Attacker tracking + logger.log_notice( "[INIT_WORKER] ## Attacker Tracking") + logger.log_notice( string.format("[v" .. config.VERSION .. "] [INIT_WORKER] MALWARE_SCAN_TRACK_ATTACKERS = %s", config_mask_if_sensitive("TRACK", cfg.attackers.track))) + logger.log_notice( string.format("[v" .. config.VERSION .. "] [INIT_WORKER] MALWARE_SCAN_TRACK_ATTACKER_TTL = %s", config_mask_if_sensitive("TTL", cfg.attackers.ttl))) + + -- Multipart scanning + logger.log_notice( "[INIT_WORKER] ## Multipart Scanning") + logger.log_notice( string.format("[v" .. config.VERSION .. "] [INIT_WORKER] MALWARE_SCAN_SCAN_FULL_MULTIPART_BODY = %s", config_mask_if_sensitive("SCAN", cfg.multipart.scan_full_body))) + + -- Module availability (if provided) + if module_status then + logger.log_notice( "[INIT_WORKER] ## Module Availability") + logger.log_notice( string.format("[v" .. config.VERSION .. "] [INIT_WORKER] clamav_module = %s", module_status.clamav and "loaded" or "not loaded")) + logger.log_notice( string.format("[v" .. config.VERSION .. "] [INIT_WORKER] cache_module = %s", module_status.cache and "loaded" or "not loaded")) + logger.log_notice( string.format("[v" .. config.VERSION .. "] [INIT_WORKER] file_ops_module = %s", module_status.file_ops and "loaded" or "not loaded")) + logger.log_notice( string.format("[v" .. config.VERSION .. "] [INIT_WORKER] virustotal_module = %s", module_status.virustotal and "loaded" or "not loaded")) + logger.log_notice( string.format("[v" .. config.VERSION .. "] [INIT_WORKER] sentinelone_module = %s", module_status.sentinelone and "loaded" or "not loaded")) + logger.log_notice( string.format("[v" .. config.VERSION .. "] [INIT_WORKER] webhook_module = %s", module_status.webhook and "loaded" or "not loaded")) + end + + logger.log_notice( "[INIT_WORKER] ========== End Configuration Dump ==========") +end + +-- ============================================================================ +-- COMBINED WORKFLOW +-- ============================================================================ + +-- Combined workflow: Load, validate, and dump configuration. +-- Convenience function for init_worker to perform all config setup in one call. +-- Logs errors and returns unified success/error result. +-- +-- Example: +-- local cfg_ok, cfg_err = ms_config.load_validate_and_dump(plugin, module_status) +-- if not cfg_ok then +-- return self:ret(false, "configuration failed: " .. cfg_err) +-- end +-- +--- @param plugin Plugin The plugin instance (provides access to self.variables and self.logger) +--- @param module_status table|nil Optional table of module availability {clamav=true, cache=true, ...} +--- @return boolean success True on success +--- @return string|nil error_message Error message on failure +function config.load_validate_and_dump(plugin, module_status) + -- Load configuration + local cfg, cfg_err = config.load(plugin) + if not cfg then + logger.log_error( "[INIT_WORKER] Failed to load configuration: " .. (cfg_err or "unknown error")) + return false, "configuration load failed: " .. (cfg_err or "unknown error") + end + + -- Validate configuration + local valid, valid_err = config.validate(cfg) + if not valid then + logger.log_error( "[INIT_WORKER] Configuration validation failed: " .. (valid_err or "unknown error")) + return false, "configuration validation failed: " .. (valid_err or "unknown error") + end + + -- Dump configuration to logs + config.dump(plugin, cfg, module_status) + + return true, nil +end + +-- Return module +return config diff --git a/malware-scan/malware_scan_custom_hashes.lua b/malware-scan/malware_scan_custom_hashes.lua new file mode 100644 index 0000000..bb63af9 --- /dev/null +++ b/malware-scan/malware_scan_custom_hashes.lua @@ -0,0 +1,688 @@ +-- BunkerWeb Malware Scan - Custom Hash Database Module +-- Optimized for large threat feeds (1M+ hashes from MalwareBazaar, etc.) +-- +-- Features: +-- - Redis-backed storage for 1M+ hashes (~200MB) distributed across 16 buckets +-- - Bucketed hash keys (malware_scan:hashes:0-9, A-F) for 16x faster lookups +-- - Sub-millisecond hash lookups via first-character bucketing +-- - Batch import for large CSV files with deduplication +-- - Automatic daily updates from threat feeds with ETag caching +-- - Infinite persistence (freshness maintained by scheduler) +-- - Optional local cache for most-frequently-matched hashes + +local cjson = require("cjson") +local logger = require("malware_scan_logger") +local utils = require("malware_scan_utils") +local database = require("malware_scan_database") + +local ngx = ngx + +local custom_hashes = {} + +-- Module version +custom_hashes.VERSION = "0.8.0" + +-------------------------------------------------------------------------------- +-- REDIS-BASED STORAGE (Optimized for 1M+ hashes) +-------------------------------------------------------------------------------- + +-- Check if hash exists in custom database +--- @param plugin Plugin instance +--- @param checksum string SHA256 hash (64 hex chars) +--- @return boolean found True if hash was found in database +--- @return string|nil malware_name Malware name if found +function custom_hashes.check(plugin, checksum) + -- Check if feature is enabled and Redis is available + if not database.is_available(plugin) then + return false, nil + end + + -- Only check custom hashes if feature is enabled + if plugin.variables and plugin.variables["MALWARE_SCAN_CUSTOM_HASHES_ENABLED"] ~= "yes" then + return false, nil + end + + -- Use bucketed key based on first character of SHA256 (64x faster lookups) + -- Store format: HSET hash_db || + local hash_key = utils.get_hash_bucket(checksum) + local data = database.hget(plugin, hash_key, checksum) + + if not data or data == ngx.null then + -- Hash not found + return false, nil + end + + -- Parse stored data: "malware_name|signature_type|reporter" + local malware_name = data:match("^([^|]+)") + + if not malware_name then + return false, nil + end + + -- Found match + logger.log_error( string.format( + "[CUSTOM_HASH] โœ“ MATCH: %s = %s (bucket: %s)", + checksum:sub(1, 16) .. "...", + utils.log_sanitize(malware_name), + hash_key:sub(-1) + )) + + return true, malware_name +end + +-- Add single hash to database +--- @param plugin Plugin instance +--- @param checksum string SHA256 hash +--- @param malware_name string Malware detection name +--- @param metadata string Optional metadata (signature_type|reporter) +--- @return boolean success True if hash was added +--- @return string|nil error Error message if failed +function custom_hashes.add(plugin, checksum, malware_name, metadata) + if not database.is_available(plugin) then + return false, "Redis not available" + end + + -- Only add if custom hashes are enabled + if plugin.variables and plugin.variables["MALWARE_SCAN_CUSTOM_HASHES_ENABLED"] ~= "yes" then + return false, "Custom hashes feature is disabled" + end + + metadata = metadata or "" + -- Use bucketed key based on first character of SHA256 + local hash_key = utils.get_hash_bucket(checksum) + local data = malware_name .. "|" .. metadata + + database.hset(plugin, hash_key, checksum, data) + return true +end + +-- Bulk add hashes (optimized for batch import with bucketing) +--- @param plugin Plugin instance +--- @param hashes table Array of {sha256, malware_name, metadata} +--- @return number added Number of hashes added +--- @return number duplicates Number of duplicate hashes skipped +function custom_hashes.bulk_add(plugin, hashes) + if not database.is_available(plugin) then + return 0, 0 + end + + if #hashes == 0 then + return 0, 0 + end + + local duplicates = 0 + local to_add = {} + + -- First pass: identify duplicates vs new hashes + for _, hash_entry in ipairs(hashes) do + local sha256 = hash_entry[1] + + if custom_hashes.exists(plugin, sha256) then + duplicates = duplicates + 1 + else + table.insert(to_add, hash_entry) + end + end + + -- Second pass: add only new hashes (grouped by bucket for efficiency) + if #to_add == 0 then + return 0, duplicates + end + + local pipeline = database.pipeline(plugin, #to_add) + + for _, hash_entry in ipairs(to_add) do + local sha256 = hash_entry[1] + local malware_name = hash_entry[2] + local metadata = hash_entry[3] or "" + local data = malware_name .. "|" .. metadata + -- Use bucketed key based on first character of SHA256 + local hash_key = utils.get_hash_bucket(sha256) + + pipeline:hset(hash_key, sha256, data) + end + + -- Execute pipeline + local results = pipeline:commit() + local added = 0 + + for _, result in ipairs(results) do + if result then + added = added + 1 + end + end + + return added, duplicates +end + +-- Bulk add hashes with duplicate detection (for recent additions) +--- @param plugin Plugin instance +--- @param hashes table Array of {sha256, malware_name, metadata} +--- @return number added Number of hashes added +--- @return number duplicates Number of duplicate hashes skipped +function custom_hashes.bulk_add_with_duplicates(plugin, hashes) + if not database.is_available(plugin) then + return 0, 0 + end + + if #hashes == 0 then + return 0, 0 + end + + local duplicates = 0 + local to_add = {} + + -- First pass: identify duplicates vs new hashes + for _, hash_entry in ipairs(hashes) do + local sha256 = hash_entry[1] + + if custom_hashes.exists(plugin, sha256) then + duplicates = duplicates + 1 + else + table.insert(to_add, hash_entry) + end + end + + -- Second pass: add only new hashes (grouped by bucket) + if #to_add == 0 then + return 0, duplicates + end + + local pipeline = database.pipeline(plugin, #to_add) + + for _, hash_entry in ipairs(to_add) do + local sha256 = hash_entry[1] + local malware_name = hash_entry[2] + local metadata = hash_entry[3] or "" + local data = malware_name .. "|" .. metadata + -- Use bucketed key based on first character of SHA256 + local hash_key = utils.get_hash_bucket(sha256) + + pipeline:hset(hash_key, sha256, data) + end + + -- Execute pipeline + local results = pipeline:commit() + local added = 0 + + for _, result in ipairs(results) do + if result then + added = added + 1 + end + end + + return added, duplicates +end + +-- Check if hash already exists (duplicate detection) +--- @param plugin Plugin instance +--- @param checksum string SHA256 hash +--- @return boolean exists True if hash exists in database +function custom_hashes.exists(plugin, checksum) + if not database.is_available(plugin) then + return false + end + + -- Use bucketed key based on first character of SHA256 + local hash_key = utils.get_hash_bucket(checksum) + local exists = database.hexists(plugin, hash_key, checksum) + + return exists == 1 +end + +-- Count total hashes across all buckets +--- @param plugin Plugin instance +--- @return number total Total number of hashes across all buckets +function custom_hashes.count(plugin) + if not database.is_available(plugin) then + return 0 + end + + -- Sum hash counts across all 16 buckets + local total_count = 0 + local buckets = utils.get_all_hash_buckets() + + for _, bucket_key in ipairs(buckets) do + local count = database.hlen(plugin, bucket_key) + total_count = total_count + (tonumber(count) or 0) + end + + return total_count +end + +-- Clear all hashes from all buckets +--- @param plugin Plugin instance +--- @return boolean success True if all buckets were cleared +function custom_hashes.clear(plugin) + if not database.is_available(plugin) then + return false + end + + -- Delete all 16 bucket keys (using UNLINK for non-blocking deletion) + local buckets = utils.get_all_hash_buckets() + local pipeline = database.pipeline(plugin, #buckets) + + if not pipeline then + logger.log_error("[CUSTOM_HASHES] Failed to create Redis pipeline") + return false + end + + for _, bucket_key in ipairs(buckets) do + pipeline:unlink(bucket_key) + end + + pipeline:commit() + return true +end + +-------------------------------------------------------------------------------- +-- CSV IMPORT (Optimized for MalwareBazaar format) +-------------------------------------------------------------------------------- + +-- Parse MalwareBazaar CSV line +-- Format: "first_seen_utc","sha256_hash","md5_hash","sha1_hash","reporter","file_name","file_type_guess","mime_type","signature","clamav","vtpercent","imphash","ssdeep","tlsh" +--- @return string|nil sha256 SHA256 hash if line is valid +--- @return string|nil malware_name Malware name if line is valid +--- @return string|nil metadata Metadata string if line is valid +local function custom_hashes_parse_bazaar_line(line) + -- Extract quoted CSV fields + local fields = {} + for field in line:gmatch('"([^"]*)"') do + table.insert(fields, field) + end + + if #fields < 10 then + return nil -- Invalid line (need at least 10 fields) + end + + local first_seen = fields[1] + local sha256 = fields[2] + local reporter = fields[5] + local file_name = fields[6] + local signature = fields[9] + local clamav = fields[10] + + -- Validate SHA256 (64 hex chars) + if not sha256 or #sha256 ~= 64 or not sha256:match("^%x+$") then + return nil + end + + -- Determine malware name (prefer signature, then clamav, then Unknown) + -- reporter is not a valid malware description (it's the organization name) + local malware_name = signature + if malware_name == "n/a" or malware_name == "" then + malware_name = clamav + end + if malware_name == "n/a" or malware_name == "" then + malware_name = "Unknown" + end + + local metadata = signature .. "|" .. clamav + + return sha256, malware_name, metadata +end + +-- Import CSV file (MalwareBazaar format) +--- @param plugin Plugin instance +--- @param csv_path string Path to CSV file +--- @return boolean success True if import completed +--- @return number imported Number of hashes imported +--- @return number duplicates Number of duplicate hashes skipped +--- @return number errors Number of parsing errors +function custom_hashes.import_csv(plugin, csv_path) + if not database.is_available(plugin) then + return false, 0, 0, 0 + end + + local file, err = io.open(csv_path, "r") + if not file then + logger.log_error( "[CUSTOM_HASH] Failed to open CSV: " .. utils.log_sanitize(tostring(err))) + return false, 0, 0, 0 + end + + local imported = 0 + local duplicates = 0 + local errors = 0 + local line_num = 0 + local batch = {} + local batch_size = 1000 -- Import in batches of 1000 for efficiency + + logger.log_notice( "[CUSTOM_HASH] Starting CSV import: " .. csv_path) + + for line in file:lines() do + line_num = line_num + 1 + + -- Skip header and comment lines + if line:match("^#") then + goto continue_line + end + if line_num == 1 and line:match("^first_seen_utc") then + goto continue_line + end + + -- Parse line + local sha256, malware_name, metadata = custom_hashes_parse_bazaar_line(line) + + if not sha256 then + errors = errors + 1 + if errors <= 10 then -- Log first 10 errors only + logger.log_warn( string.format( + "[CUSTOM_HASH] Invalid line %d (skipping): %.50s", line_num, utils.log_sanitize(line))) + end + goto continue_line + end + + -- Add to batch + table.insert(batch, {sha256, malware_name, metadata}) + + -- Bulk add when batch is full + if #batch >= batch_size then + local added, dupes = custom_hashes.bulk_add(plugin, batch) + imported = imported + added + duplicates = duplicates + dupes + batch = {} + + -- Progress indicator (every 100K hashes) + if (imported + duplicates) % 100000 == 0 then + logger.log_notice( string.format( + "[CUSTOM_HASH] Progress: %d new, %d duplicates (total processed: %d)", + imported, duplicates, imported + duplicates)) + end + end + + ::continue_line:: + end + + -- Import remaining batch + if #batch > 0 then + local added, dupes = custom_hashes.bulk_add(plugin, batch) + imported = imported + added + duplicates = duplicates + dupes + end + + file:close() + + logger.log_notice( string.format( + "[CUSTOM_HASH] CSV import complete: %d new, %d duplicates, %d errors (total: %d hashes)", + imported, duplicates, errors, custom_hashes.count(plugin))) + + return true, imported, duplicates, errors +end + +-- Import CSV with duplicate detection (for recent additions) +--- @param plugin Plugin instance +--- @param csv_path string Path to CSV file +--- @return boolean success True if import completed +--- @return number imported Number of hashes imported +--- @return number duplicates Number of duplicate hashes skipped +--- @return number errors Number of parsing errors +function custom_hashes.import_csv_recent(plugin, csv_path) + if not database.is_available(plugin) then + return false, 0, 0, 0 + end + + local file, err = io.open(csv_path, "r") + if not file then + logger.log_error( "[CUSTOM_HASH] Failed to open CSV: " .. utils.log_sanitize(tostring(err))) + return false, 0, 0, 0 + end + + local imported = 0 + local duplicates = 0 + local errors = 0 + local line_num = 0 + local batch = {} + local batch_size = 1000 + + logger.log_notice( "[CUSTOM_HASH] Starting recent CSV import (with duplicate detection): " .. csv_path) + + for line in file:lines() do + line_num = line_num + 1 + + -- Skip header and comment lines + if line:match("^#") then + goto continue_line + end + if line_num == 1 and line:match("^first_seen_utc") then + goto continue_line + end + + -- Parse line + local sha256, malware_name, metadata = custom_hashes_parse_bazaar_line(line) + + if not sha256 then + errors = errors + 1 + if errors <= 10 then + logger.log_warn( string.format( + "[CUSTOM_HASH] Invalid line %d (skipping): %.50s", line_num, line)) + end + goto continue_line + end + + -- Add to batch + table.insert(batch, {sha256, malware_name, metadata}) + + -- Process batch when full + if #batch >= batch_size then + local added, dupes = custom_hashes.bulk_add_with_duplicates(plugin, batch) + imported = imported + added + duplicates = duplicates + dupes + batch = {} + + -- Progress indicator + if (imported + duplicates) % 100000 == 0 then + logger.log_notice( string.format( + "[CUSTOM_HASH] Progress: %d new, %d duplicates (total processed: %d)", + imported, duplicates, imported + duplicates)) + end + end + + ::continue_line:: + end + + -- Import remaining batch + if #batch > 0 then + local added, dupes = custom_hashes.bulk_add_with_duplicates(plugin, batch) + imported = imported + added + duplicates = duplicates + dupes + end + + file:close() + + logger.log_notice( string.format( + "[CUSTOM_HASH] Recent import complete: %d new, %d duplicates, %d errors (total: %d hashes)", + imported, duplicates, errors, custom_hashes.count(plugin))) + + return true, imported, duplicates, errors +end + +-- Export custom hashes to CSV from all buckets +--- @param plugin Plugin instance +--- @param csv_path string Path to output CSV file +--- @return boolean success True if export completed +--- @return number exported Number of hashes exported +function custom_hashes.export_csv(plugin, csv_path) + if not database.is_available(plugin) then + return false, 0 + end + + local file, file_err = io.open(csv_path, "w") + + if not file then + logger.log_error( "[CUSTOM_HASH] Failed to open output CSV: " .. tostring(file_err)) + return false, 0 + end + + -- Write CSV header (MalwareBazaar compatible) + file:write("sha256,malware_name,signature_type,reporter\n") + + -- Scan all hashes from all buckets + local exported = 0 + local buckets = utils.get_all_hash_buckets() + + for _, hash_key in ipairs(buckets) do + local cursor = "0" + + repeat + local result = database.hscan(plugin, hash_key, cursor) + if not result then + break + end + + cursor = tostring(result[1]) or "0" + local hashes = result[2] or {} + + -- Process hashes in pairs (key, value) + for i = 1, #hashes, 2 do + local sha256 = hashes[i] + local data = hashes[i + 1] + + if sha256 and data then + -- Parse data: "malware_name|signature_type|reporter" + local parts = {} + for part in data:gmatch("[^|]+") do + table.insert(parts, part) + end + + local malware_name = parts[1] or "unknown" + local signature = parts[2] or "" + local reporter = parts[3] or "" + + file:write(string.format("%s,%s,%s,%s\n", + sha256, malware_name, signature, reporter)) + + exported = exported + 1 + end + end + until cursor == "0" + end + + file:close() + + logger.log_notice( string.format( + "[CUSTOM_HASH] CSV export complete: %d hashes exported from %d buckets", + exported, #buckets)) + + return true, exported +end + +-------------------------------------------------------------------------------- +-- DAILY UPDATE MECHANISM +-------------------------------------------------------------------------------- + +-- Download CSV from URL +--- @return boolean success True if download completed +--- @return string|nil error Error message if download failed +local function custom_hashes_download_file(url, dest_path, timeout_ms) + timeout_ms = timeout_ms or 30000 -- 30 second timeout + + local httpc = require("resty.http").new() + httpc:set_timeout(timeout_ms) + + local res, err = httpc:request_uri(url, { + method = "GET", + ssl_verify = true + }) + + if not res then + return false, "HTTP request failed: " .. tostring(err) + end + + if res.status ~= 200 then + return false, "HTTP " .. res.status + end + + -- Write to temporary file + local temp_path = dest_path .. ".tmp" + local file, file_err = io.open(temp_path, "w") + + if not file then + return false, "Failed to write file: " .. tostring(file_err) + end + + file:write(res.body) + file:close() + + -- Atomic rename + local ok, rename_err = os.rename(temp_path, dest_path) + if not ok then + return false, "Failed to move file: " .. tostring(rename_err) + end + + return true +end + +-- Daily update task +--- @param plugin Plugin instance +--- @return boolean success True if daily update completed successfully +function custom_hashes.daily_update(plugin) + local enabled = plugin.variables["MALWARE_SCAN_CUSTOM_HASH_AUTO_UPDATE"] or "no" + if enabled ~= "yes" then + return true -- Auto-update disabled + end + + local update_url = plugin.variables["MALWARE_SCAN_CUSTOM_HASH_UPDATE_URL"] or "" + if update_url == "" then + logger.log_warn( "[CUSTOM_HASH] Auto-update enabled but no URL configured") + return false + end + + local csv_path = plugin.variables["MALWARE_SCAN_CUSTOM_HASH_CSV_PATH"] or "/var/cache/bunkerweb/malware-scan/hashes.csv" + + logger.log_notice( "[CUSTOM_HASH] Starting daily update from: " .. update_url) + + -- Download CSV + local ok, err = custom_hashes_download_file(update_url, csv_path) + if not ok then + logger.log_error( "[CUSTOM_HASH] Download failed: " .. tostring(err)) + return false + end + + logger.log_notice( "[CUSTOM_HASH] Downloaded, now importing...") + + -- Clear old hashes + custom_hashes.clear(plugin) + + -- Import new CSV + local import_ok, imported, import_errors = custom_hashes.import_csv(plugin, csv_path) + + if not import_ok then + logger.log_error( "[CUSTOM_HASH] Import failed") + return false + end + + logger.log_notice( string.format( + "[CUSTOM_HASH] Daily update complete: %d hashes imported, %d errors", + imported, import_errors)) + + return true +end + +-------------------------------------------------------------------------------- +-- METRICS INTEGRATION +-------------------------------------------------------------------------------- + +-- Record hash check metric (if metrics module available) +function custom_hashes.record_metric(scanner_name, found) + local metrics_ok, metrics = pcall(require, "malware_scan_metrics") + + if metrics_ok and metrics then + if found then + -- Hash database hit + if metrics.custom_hash_hits then + metrics.custom_hash_hits:inc(1) + end + end + + -- Always record check + if metrics.custom_hash_checks then + metrics.custom_hash_checks:inc(1) + end + end +end + +-------------------------------------------------------------------------------- +-- PUBLIC API +-------------------------------------------------------------------------------- + +return custom_hashes diff --git a/malware-scan/malware_scan_database.lua b/malware-scan/malware_scan_database.lua new file mode 100644 index 0000000..688327e --- /dev/null +++ b/malware-scan/malware_scan_database.lua @@ -0,0 +1,515 @@ +-- BunkerWeb Malware Scan - Database Module +-- Centralized Redis/database operations with circuit breaker and connection management +-- Provides high-level API for all Redis operations across the plugin + +local logger = require("malware_scan_logger") + +local ngx = ngx +local tonumber = tonumber +local tostring = tostring + +-- Module table +local database = {} + +-- Module version +database.VERSION = "0.8.0" + +-- ============================================================================ +-- TYPE ALIASES (for documentation and LuaLS type checking) +-- ============================================================================ + +---@alias Redis string Redis key name +---@alias Value string Value to store in Redis +---@alias Time-to-live number TTL in seconds +---@alias Hash string Redis hash key +---@alias Field string Redis hash field name +---@alias Scan string Scan cursor position +---@alias Pipeline table Redis pipeline object + +-- ============================================================================ +-- CONNECTION MANAGEMENT +-- ============================================================================ + +-- Initialize Redis connection for malware hash lookups. +-- Sets up plugin.redisdb from the clusterstore's Redis client. +-- This is required for MalwareBazaar hash lookups during request handling. +-- +--- @param plugin Plugin instance with clusterstore +--- @return boolean success True if connection successful +--- @return string|nil error Error message if failed +function database.connect(plugin) + -- Check if Redis is enabled + if not plugin.use_redis then + logger.log_error( "Redis not enabled (USE_REDIS != yes)") + return false, "Redis not enabled" + end + + -- Check if clusterstore is available + if not plugin.clusterstore then + logger.log_error( "Clusterstore not available") + return false, "Clusterstore not available" + end + + -- Connect to Redis via clusterstore + local ok, err = plugin.clusterstore:connect() + if not ok then + logger.log_error( "Failed to connect to Redis: " .. (err or "unknown error")) + return false, "Failed to connect to Redis: " .. (err or "unknown error") + end + + -- Assign redis_client to plugin.redisdb for malware hash lookups + plugin.redisdb = plugin.clusterstore.redis_client + + if not plugin.redisdb then + logger.log_error( "Redis client not available after connection") + return false, "Redis client not available" + end + + logger.log_error( "Redis initialized successfully for malware hash lookups") + return true, nil +end + +-- Check if Redis is enabled and available. +-- Returns true if both USE_REDIS and MALWARE_SCAN_USE_SHARED_DATABASE are enabled. +-- +--- @param plugin Plugin instance +--- @return boolean enabled True if both shared database and Redis are enabled +function database.is_enabled(plugin) + local use_shared = plugin.variables["MALWARE_SCAN_USE_SHARED_DATABASE"] == "yes" + local use_redis = plugin.variables["USE_REDIS"] == "yes" + return use_shared and use_redis +end + +-- Check if Redis client is available. +-- Returns true if plugin.redisdb exists. +-- +--- @param plugin Plugin instance +--- @return boolean available True if Redis client is available +function database.is_available(plugin) + return plugin.redisdb ~= nil +end + +-- ============================================================================ +-- CIRCUIT BREAKER FUNCTIONS +-- Protects against Redis failures with exponential backoff +-- ============================================================================ + +-- Get Redis circuit breaker state from local cache. +-- Returns failure_count and last_failure_time. +-- +--- @param plugin Plugin instance +--- @return number failure_count Number of consecutive failures +--- @return number last_failure_time Unix timestamp of last failure +function database.breaker_get(plugin) + local cb_key = "plugin_malware_scan_redis_circuit_breaker" + local ok, data = plugin.cachestore:get(cb_key) + if not ok or not data then + return 0, 0 -- No failures recorded + end + + -- Data format: "failure_count:last_failure_time" + local count, time = data:match("^(%d+):(%d+)$") + if not count or not time then + return 0, 0 + end + + return tonumber(count) or 0, tonumber(time) or 0 +end + +-- Update Redis circuit breaker state in local cache. +-- Sets failure_count and last_failure_time. +-- +--- @param plugin Plugin instance +--- @param failure_count number Number of consecutive failures +--- @param last_failure_time number Unix timestamp of last failure +function database.breaker_set(plugin, failure_count, last_failure_time) + -- Validate numeric inputs to prevent corruption (Medium severity issue #11) + failure_count = tonumber(failure_count) or 0 + last_failure_time = tonumber(last_failure_time) or 0 + + -- Bounds checking: prevent negative values and excessive counts + if failure_count < 0 or failure_count > 1000000 then + logger.log_warn("[DATABASE] Invalid failure_count (" .. tostring(failure_count) .. "), resetting to 0") + failure_count = 0 + end + + if last_failure_time < 0 then + logger.log_warn("[DATABASE] Invalid last_failure_time (" .. tostring(last_failure_time) .. "), resetting to 0") + last_failure_time = 0 + end + + local cb_key = "plugin_malware_scan_redis_circuit_breaker" + local data = tostring(failure_count) .. ":" .. tostring(last_failure_time) + -- Store for 60 seconds (longer than max backoff of 8s default) + plugin.cachestore:set(cb_key, data, 60) +end + +-- Check if Redis should be skipped based on circuit breaker state. +-- Returns true if Redis should be skipped, false otherwise. +-- Implements exponential backoff: 1ms, 2ms, 4ms, 8ms, 16ms, 32ms, 64ms, 128ms, 256ms, 512ms, 1s, 2s, 4s, up to max (default 8s). +-- +--- @param plugin Plugin instance +--- @return boolean should_skip True if Redis should be skipped +function database.should_skip(plugin) + local failure_count, last_failure_time = database.breaker_get(plugin) + + if failure_count == 0 then + return false -- No failures, try Redis + end + + local now = ngx.time() + local max_backoff = tonumber(plugin.variables["MALWARE_SCAN_REDIS_CIRCUIT_BREAKER_MAX_BACKOFF"]) or 8 + -- Cap failure_count to prevent integer overflow in exponential backoff (Medium severity issue #12) + local capped_failure_count = math.min(failure_count, 20) + -- Start at 1ms (0.001s) and double each time: 0.001 * 2^failure_count + local backoff_seconds = math.min(0.001 * (2 ^ capped_failure_count), max_backoff) + local time_since_failure = now - last_failure_time + + if time_since_failure >= backoff_seconds then + return false -- Backoff period expired, try Redis again + end + + -- Still in backoff period + plugin:log_debug(string.format( + "[DATABASE] Skipping Redis (circuit breaker): %d consecutive failures, backoff %.3fs, %.3fs remaining", + failure_count, backoff_seconds, backoff_seconds - time_since_failure + )) + return true +end + +-- Record a successful Redis operation (resets circuit breaker). +-- +--- @param plugin Plugin instance +function database.success_record(plugin) + database.breaker_set(plugin, 0, 0) + plugin:log_debug("[v" .. database.VERSION .. "] [DATABASE] Redis operation successful, circuit breaker reset") +end + +-- Record a failed Redis operation (increments circuit breaker). +-- +--- @param plugin Plugin instance +function database.failure_record(plugin) + local failure_count, last_failure_time = database.breaker_get(plugin) + local now = ngx.time() + failure_count = failure_count + 1 + + database.breaker_set(plugin, failure_count, now) + + local max_backoff = tonumber(plugin.variables["MALWARE_SCAN_REDIS_CIRCUIT_BREAKER_MAX_BACKOFF"]) or 8 + local backoff_seconds = math.min(0.001 * (2 ^ failure_count), max_backoff) + + -- Implement log throttling to prevent log flooding (Low severity issue #22) + -- Only log if: first failure, every 5th failure, or 60+ seconds since last log + local should_log = (failure_count == 1) or + (failure_count % 5 == 0) or + (last_failure_time > 0 and (now - last_failure_time) >= 60) + + if should_log then + -- Format backoff time appropriately (ms for small values, s for large) + local backoff_str + if backoff_seconds < 1 then + backoff_str = string.format("%.0fms", backoff_seconds * 1000) + else + backoff_str = string.format("%.1fs", backoff_seconds) + end + + logger.log_error( string.format( + "[DATABASE] Redis failure #%d recorded, backing off for %s", + failure_count, backoff_str + )) + end +end + +-- ============================================================================ +-- KEY-VALUE OPERATIONS (via plugin.datastore with circuit breaker) +-- ============================================================================ + +-- Get value from Redis with circuit breaker protection. +-- Uses plugin.datastore for key-value operations. +-- +--- @param plugin Plugin instance +--- @param key string Redis key +--- @return boolean success True if operation successful +--- @return any value Value from Redis (nil if not found or error) +--- @return string|nil error Error message if failed +function database.get(plugin, key) + if not database.is_enabled(plugin) then + return false, nil, "shared cache disabled" + end + + -- Check circuit breaker before attempting Redis + if database.should_skip(plugin) then + return false, nil, "circuit breaker open" + end + + local ok, data = plugin.datastore:get(key) + if not ok then + database.failure_record(plugin) + return false, nil, "redis get failed" + end + + database.success_record(plugin) + return true, data, nil +end + +-- Set value in Redis with circuit breaker protection. +-- Uses plugin.datastore for key-value operations. +-- +--- @param plugin Plugin instance +--- @param key string Redis key +--- @param value string Value to store +--- @param ttl number|nil Time-to-live in seconds (optional) +--- @return boolean success True if operation successful +--- @return string|nil error Error message if failed +function database.set(plugin, key, value, ttl) + if not database.is_enabled(plugin) then + return true -- Not an error, just disabled + end + + -- Skip if circuit breaker is open + if database.should_skip(plugin) then + return true -- Not a fatal error + end + + local ok, err = plugin.datastore:set(key, value, ttl) + if not ok then + database.failure_record(plugin) + return false, err + end + + database.success_record(plugin) + return true, nil +end + +-- Delete key from Redis with circuit breaker protection. +-- Uses plugin.datastore for key-value operations. +-- Note: This uses the datastore's delete method which may use DEL internally. +-- +--- @param plugin Plugin instance +--- @param key string Redis key +--- @return boolean success True if operation successful +--- @return string|nil error Error message if failed +function database.delete(plugin, key) + if not database.is_enabled(plugin) then + return true -- Not an error, just disabled + end + + -- Skip if circuit breaker is open + if database.should_skip(plugin) then + return true -- Not a fatal error + end + + local ok, err = plugin.datastore:delete(key) + if not ok then + database.failure_record(plugin) + return false, err + end + + database.success_record(plugin) + return true, nil +end + +-- ============================================================================ +-- HASH OPERATIONS (via plugin.redisdb - direct Redis client access) +-- ============================================================================ + +-- Get hash field value from Redis. +-- Direct access to Redis client for hash operations with auto-reconnect. +-- Always gets a fresh connection from the pool to avoid stale connection issues. +-- +--- @param plugin Plugin instance +--- @param key string Redis hash key +--- @param field string Hash field name +--- @return any value Value from hash field (nil if not found) +function database.hget(plugin, key, field) + -- Always get a fresh connection from the pool (handles keepalive automatically) + local conn_ok, conn_err = plugin.clusterstore:connect() + if not conn_ok then + logger.log_error("[DATABASE] Failed to get Redis connection for hget: " .. (conn_err or "unknown")) + return nil + end + + -- Update redis client reference (may be a different connection from the pool) + plugin.redisdb = plugin.clusterstore.redis_client + + -- Perform operation + local result = plugin.redisdb:hget(key, field) + + return result +end + +-- Set hash field value in Redis. +-- Direct access to Redis client for hash operations with fresh connection. +-- +--- @param plugin Plugin instance +--- @param key string Redis hash key +--- @param field string Hash field name +--- @param value string Field value +--- @return boolean success True if operation successful +function database.hset(plugin, key, field, value) + -- Always get a fresh connection from the pool + local conn_ok, conn_err = plugin.clusterstore:connect() + if not conn_ok then + logger.log_error("[DATABASE] Failed to get Redis connection for hset: " .. (conn_err or "unknown")) + return false + end + + plugin.redisdb = plugin.clusterstore.redis_client + return plugin.redisdb:hset(key, field, value) +end + +-- Check if hash field exists in Redis. +-- Direct access to Redis client for hash operations with fresh connection. +-- +--- @param plugin Plugin instance +--- @param key string Redis hash key +--- @param field string Hash field name +--- @return number exists 1 if exists, 0 if not +function database.hexists(plugin, key, field) + -- Always get a fresh connection from the pool + local conn_ok, conn_err = plugin.clusterstore:connect() + if not conn_ok then + logger.log_error("[DATABASE] Failed to get Redis connection for hexists: " .. (conn_err or "unknown")) + return 0 + end + + plugin.redisdb = plugin.clusterstore.redis_client + return plugin.redisdb:hexists(key, field) or 0 +end + +-- Get hash length (number of fields) from Redis. +-- Direct access to Redis client for hash operations with fresh connection. +-- +--- @param plugin Plugin instance +--- @param key string Redis hash key +--- @return number count Number of fields in hash +function database.hlen(plugin, key) + -- Always get a fresh connection from the pool + local conn_ok, conn_err = plugin.clusterstore:connect() + if not conn_ok then + logger.log_error("[DATABASE] Failed to get Redis connection for hlen: " .. (conn_err or "unknown")) + return 0 + end + + plugin.redisdb = plugin.clusterstore.redis_client + return plugin.redisdb:hlen(key) or 0 +end + +-- Scan hash fields from Redis. +-- Direct access to Redis client for hash operations with fresh connection. +-- +--- @param plugin Plugin instance +--- @param key string Redis hash key +--- @param cursor number Scan cursor (0 to start) +--- @return table|nil result Scan result {cursor, data} or nil on error +function database.hscan(plugin, key, cursor) + -- Always get a fresh connection from the pool + local conn_ok, conn_err = plugin.clusterstore:connect() + if not conn_ok then + logger.log_error("[DATABASE] Failed to get Redis connection for hscan: " .. (conn_err or "unknown")) + return nil + end + + plugin.redisdb = plugin.clusterstore.redis_client + return plugin.redisdb:hscan(key, cursor) +end + +-- Create Redis pipeline for batch operations. +-- Direct access to Redis client for pipeline operations with fresh connection. +-- +--- @param plugin Plugin instance +--- @param size Pipeline size (number of commands) +--- @return table|nil pipeline Pipeline object or nil on error +function database.pipeline(plugin, size) + -- Always get a fresh connection from the pool + local conn_ok, conn_err = plugin.clusterstore:connect() + if not conn_ok then + logger.log_error("[DATABASE] Failed to get Redis connection for pipeline: " .. (conn_err or "unknown")) + return nil + end + + plugin.redisdb = plugin.clusterstore.redis_client + return plugin.redisdb:pipeline(size) +end + +-- ============================================================================ +-- SIMPLE KEY OPERATIONS (via plugin.redisdb) +-- ============================================================================ + +-- Check if key exists in Redis. +-- Direct access to Redis client with fresh connection. +-- +--- @param plugin Plugin instance +--- @param key Redis key +--- @return number exists 1 if exists, 0 if not +function database.exists(plugin, key) + -- Always get a fresh connection from the pool + local conn_ok, conn_err = plugin.clusterstore:connect() + if not conn_ok then + logger.log_error("[DATABASE] Failed to get Redis connection for exists: " .. (conn_err or "unknown")) + return 0 + end + + plugin.redisdb = plugin.clusterstore.redis_client + return plugin.redisdb:exists(key) or 0 +end + +-- Set key with expiry in Redis. +-- Direct access to Redis client with fresh connection. +-- +--- @param plugin Plugin instance +--- @param key string Redis key +--- @param ttl number Time-to-live in seconds +--- @param value string Value to store +--- @return boolean success True if operation successful +function database.setex(plugin, key, ttl, value) + -- Always get a fresh connection from the pool + local conn_ok, conn_err = plugin.clusterstore:connect() + if not conn_ok then + logger.log_error("[DATABASE] Failed to get Redis connection for setex: " .. (conn_err or "unknown")) + return false + end + + plugin.redisdb = plugin.clusterstore.redis_client + return plugin.redisdb:setex(key, ttl, value) or true +end + +-- Delete key from Redis using UNLINK (non-blocking). +-- UNLINK performs asynchronous deletion in the background, avoiding blocking. +-- Direct access to Redis client with fresh connection. +-- +--- @param plugin Plugin instance +--- @param key Redis key (can be single key or array of keys) +--- @return number count Number of keys unlinked +function database.unlink(plugin, key) + -- Always get a fresh connection from the pool + local conn_ok, conn_err = plugin.clusterstore:connect() + if not conn_ok then + logger.log_error("[DATABASE] Failed to get Redis connection for unlink: " .. (conn_err or "unknown")) + return 0 + end + + plugin.redisdb = plugin.clusterstore.redis_client + return plugin.redisdb:unlink(key) or 0 +end + +-- Get key value from Redis (simple operation). +-- Direct access to Redis client with fresh connection. +-- +--- @param plugin Plugin instance +--- @param key Redis key +--- @return any value Value from Redis (nil if not found) +function database.redis_get(plugin, key) + -- Always get a fresh connection from the pool + local conn_ok, conn_err = plugin.clusterstore:connect() + if not conn_ok then + logger.log_error("[DATABASE] Failed to get Redis connection for get: " .. (conn_err or "unknown")) + return nil + end + + plugin.redisdb = plugin.clusterstore.redis_client + return plugin.redisdb:get(key) +end + +-- Return module +return database diff --git a/malware-scan/malware_scan_file.lua b/malware-scan/malware_scan_file.lua new file mode 100644 index 0000000..47d720c --- /dev/null +++ b/malware-scan/malware_scan_file.lua @@ -0,0 +1,1286 @@ +-- BunkerWeb Malware Scan - File Operations Module +-- Handles all file handling, content extraction, validation, and cleanup operations + +local sha256 = require("resty.sha256") +local str = require("resty.string") +local utils = require("malware_scan_utils") +local logger = require("malware_scan_logger") + +-- Load lfs with fallback if not available +local lfs = nil +local lfs_ok, lfs_err = pcall(function() + lfs = require("lfs") +end) +if not lfs_ok then + logger.log_warn("[FILE_OPS] Lua File System (lfs) not available: " .. tostring(lfs_err) .. " - directory operations will be limited") +end + +local ngx = ngx +local to_hex = str.to_hex + +-- Module table +local file_ops = {} + +-- Module version +file_ops.VERSION = "0.8.0" + +-- ============================================================================ +-- HELPER FUNCTIONS +-- ============================================================================ + +--- Create directory recursively (like mkdir -p) +--- @param path string Directory path to create +--- @return boolean success True if directory created successfully +--- @return string|nil error Error message if failed +local function mkdir_recursive(path) + if not path or path == "" then + return false, "path is empty" + end + + -- Check if lfs is available + if not lfs then + return false, "lfs module not available - cannot create directories" + end + + -- Check if directory already exists + local attr = lfs.attributes(path, "mode") + if attr == "directory" then + return true, nil + elseif attr then + return false, "path exists but is not a directory" + end + + -- Split path into components + local components = {} + for component in path:gmatch("[^/]+") do + table.insert(components, component) + end + + -- Build path incrementally + local current_path = path:sub(1, 1) == "/" and "/" or "" + for _, component in ipairs(components) do + current_path = current_path .. component .. "/" + + -- Check if this level exists + local level_attr = lfs.attributes(current_path, "mode") + if not level_attr then + -- Create this directory level + local ok, err = lfs.mkdir(current_path) + if not ok then + return false, "failed to create directory " .. current_path .. ": " .. (err or "unknown error") + end + elseif level_attr ~= "directory" then + return false, "path component exists but is not a directory: " .. current_path + end + end + + return true, nil +end + +--- Copy file from source to destination (like cp -p) +-- Preserves file modification time +--- @param src string Source file path +--- @param dst string Destination file path +--- @return boolean success True if file copied successfully +--- @return string|nil error Error message if failed +local function file_copy(src, dst) + if not src or src == "" then + return false, "source path is empty" + end + if not dst or dst == "" then + return false, "destination path is empty" + end + + -- Open source file + local src_file, src_err = io.open(src, "rb") + if not src_file then + return false, "failed to open source file: " .. (src_err or "unknown error") + end + + -- Read source content + local content = src_file:read("*a") + src_file:close() + + if not content then + return false, "failed to read source file" + end + + -- Open destination file + local dst_file, dst_err = io.open(dst, "wb") + if not dst_file then + return false, "failed to open destination file: " .. (dst_err or "unknown error") + end + + -- Write to destination + local write_ok, write_err = dst_file:write(content) + dst_file:close() + + if not write_ok then + return false, "failed to write to destination file: " .. (write_err or "unknown error") + end + + -- Preserve modification time using lfs.touch() (if available) + if lfs then + local src_mtime = lfs.attributes(src, "modification") + if src_mtime then + lfs.touch(dst, src_mtime) + end + end + + return true, nil +end + +-- ============================================================================ +-- FILE OPERATIONS +-- ============================================================================ + +-- Get file size in bytes. +-- Returns file size or nil if unable to determine. +-- +-- NOTE: This function has a known TOCTOU (Time-of-Check Time-of-Use) race condition +-- where the file could be modified between this size check and subsequent use. +-- This is acceptable because: +-- 1. Files are temporary uploads in a controlled directory (/tmp/bunkerweb/malware-scan/) +-- 2. File size is used primarily for logging and validation, not security decisions +-- 3. Actual file operations use independent file handles with their own error handling +-- (Low severity issue #21 - documented as acceptable) +--- @return number|nil size File size in bytes or nil if unable to determine +function file_ops.file_get_size(plugin, file_path) + local file, err = io.open(file_path, "rb") + if not file then + logger.log_error( "failed to open file for size check: " .. (err or "unknown")) + return nil + end + + local size = file:seek("end") + file:close() + + -- Validate file size (check for negative, NaN, or invalid values) + if not size then + return nil + end + + -- Validate size is a valid non-negative number + size = utils.validate_file_size(size, 10 * 1024 * 1024 * 1024) -- Max 10GB + + return size +end + +-- Detect file type using the `file` command. +-- Uses the Unix `file` command to identify file type by magic numbers and content analysis. +-- Returns MIME type string or nil if unable to determine. +-- +-- Example outputs: +-- - "text/plain; charset=us-ascii" +-- - "application/x-dosexec" +-- - "image/jpeg" +-- - "inode/x-empty; charset=binary" +-- +--- @param plugin Plugin instance +--- @param file_path Path to file to analyze +--- @return string|nil file_type MIME type of the file or nil if unable to determine +--- @return string|nil error Error message if failed +function file_ops.file_get_type(plugin, file_path) + if not file_path or file_path == "" then + return nil, "invalid file path" + end + + -- Validate file path to prevent command injection (CWE-78) + -- Only allow alphanumeric, slash, dash, underscore, and dot + if file_path:match("[^%w%/%-%_%.]") then + return nil, "file path contains unsafe characters" + end + + -- Build command with long-form flags for cross-system compatibility: + -- --brief: no filename in output + -- --mime-type: output MIME type instead of human-readable description + -- --no-dereference: don't follow symlinks (security) + -- --preserve-date: don't modify file access times + -- -P bytes=8192: limit to 8KB scan (security, prevent DoS on large files) + -- Note: File magic bytes are typically in first 16 bytes to a few KB. + -- 8KB is more than sufficient for all file type detection including complex PE/ELF headers. + local cmd = string.format("/usr/bin/file --brief --mime-type --no-dereference --preserve-date -P bytes=8192 %s 2>&1", utils.shell_escape_path(file_path)) + + local handle, err = io.popen(cmd) + if not handle then + logger.log_error( "failed to execute file command: " .. (err or "unknown")) + return nil, "failed to execute file command" + end + + -- Read output (should be single line) + local output = handle:read("*l") + handle:close() + + if not output or output == "" then + return nil, "no output from file command" + end + + -- Clean up output (remove trailing whitespace) + output = output:gsub("%s+$", "") + + plugin:log_debug("file type detected: " .. output) + return output, nil +end + +-- Extract original filename from multipart upload file. +-- Reads the first few KB of the file to find the Content-Disposition header. +-- Returns filename or nil if not found. +--- @return string|nil filename Extracted filename or nil if not found +function file_ops.filename_extract(plugin, file_path) + local file, err = io.open(file_path, "rb") + if not file then + plugin:log_debug("Could not open file to extract filename: " .. tostring(err)) + return nil + end + + -- Read first 4KB (should contain headers) + local header_data = file:read(4096) + file:close() + + if not header_data then + return nil + end + + -- Look for Content-Disposition header with filename + -- Format: Content-Disposition: form-data; name="file"; filename="example.jpg" + local filename = header_data:match('filename="([^"]+)"') + if not filename then + -- Try alternative format without quotes + filename = header_data:match("filename=([^%s;]+)") + end + + if not filename then + return nil + end + + -- Validate filename to prevent path traversal (CWE-22) and log injection (CWE-117) + local valid, err = utils.file_validate_name(filename) + if not valid then + plugin:log_debug("Invalid filename extracted: " .. utils.log_sanitize(filename) .. " - " .. err) + return nil + end + + -- Sanitize filename for safe logging (removes control characters) + return utils.log_sanitize(filename) +end + +-- ============================================================================ +-- MULTIPART PARSING +-- ============================================================================ + +-- Parse multipart/form-data body and extract all uploaded files. +-- Returns success boolean, array of file info tables, and optional attack indicator. +-- Each file info table contains: {filepath, filename, size, content_type} +-- +-- Filename injection attack handling (path traversal, null bytes, etc.): +-- Behavior matrix based on BLOCK_ON_FILENAME_INJECTION_ATTACK and RESUME_SCAN_ON_FIRST_HIT: +-- +-- BLOCK=yes, RESUME=yes: Skip file, continue scanning other files, block at end +-- BLOCK=yes, RESUME=no: Block immediately (default, fastest protection) +-- BLOCK=no, RESUME=yes: Log attack, scan file content anyway, continue to other files +-- BLOCK=no, RESUME=no: Log attack, scan file content anyway until first virus hit +-- +-- When BLOCK=no, files with suspicious filenames are still scanned for actual malware. +-- This is useful for forensics/research to understand what attackers are uploading. +-- +-- Returns: +-- - On success: true, files_array, nil +-- - On success with attacks: true, files_array, "filename_attack_detected" +-- - On immediate failure: false, error_message, nil +--- @return boolean success True if parsing succeeded +--- @return table|string files_or_error Array of file info tables on success, error message on failure +--- @return string|nil attack_indicator "filename_attack_detected" if attacks were detected +function file_ops.multipart_parse(plugin, body_file, content_type) + -- Extract boundary from Content-Type header + local boundary = content_type:match("boundary=([^;%s]+)") + if not boundary then + return false, "no boundary found in Content-Type" + end + + -- Read the entire multipart body + local file, err = io.open(body_file, "rb") + if not file then + return false, "failed to open body file: " .. (err or "unknown") + end + + -- Check file size to prevent DoS via large uploads (CWE-770) + local size = file:seek("end") + file:seek("set", 0) + + local MAX_MULTIPART_SIZE = 100 * 1024 * 1024 -- 100MB + if size > MAX_MULTIPART_SIZE then + file:close() + return false, "multipart body too large (max 100MB)" + end + + local body_content = file:read("*a") + file:close() + + if not body_content or body_content == "" then + return false, "body file is empty" + end + + plugin:log_debug("[v" .. file_ops.VERSION .. "] [MULTIPART] Parsing multipart body: " .. #body_content .. " bytes, boundary=" .. boundary) + + -- Split by boundary (format: --boundary) + -- Escape ALL non-alphanumeric characters to prevent pattern injection (Medium severity issue #13) + local boundary_pattern = "%-%-" .. boundary:gsub("([^%w])", "%%%1") + local files = {} + local created_temp_files = {} -- Track temp files for cleanup on error + local part_count = 0 + local filename_attack_detected = false + local block_on_filename_attack = plugin.variables["MALWARE_SCAN_BLOCK_ON_FILENAME_INJECTION_ATTACK"] == "yes" + local resume_on_hit = plugin.variables["MALWARE_SCAN_RESUME_SCAN_ON_FIRST_HIT"] == "yes" + + -- Resource limit: maximum files per request (prevents file descriptor exhaustion) + local max_files = tonumber(plugin.variables["MALWARE_SCAN_MAX_FILES_PER_REQUEST"]) or 100 + + -- Find all parts between boundaries + local pos = 1 + while true do + -- Find next boundary + local boundary_start, boundary_end = body_content:find(boundary_pattern, pos) + if not boundary_start then + break + end + + -- Move past the boundary and newline + pos = boundary_end + 1 + + -- Skip CR/LF after boundary + if body_content:sub(pos, pos) == "\r" then + pos = pos + 1 + end + if body_content:sub(pos, pos) == "\n" then + pos = pos + 1 + end + + -- Find next boundary to get this part's content + local next_boundary_start = body_content:find(boundary_pattern, pos) + if not next_boundary_start then + break + end + + -- Extract this part (between current position and next boundary) + local part_content = body_content:sub(pos, next_boundary_start - 1) + + -- Check if this part has a filename (is a file upload) + local filename = part_content:match('filename="([^"]+)"') + if not filename then + -- Try without quotes + filename = part_content:match("filename=([^%s;]+)") + end + + if filename then + -- Sanitize filename once for all logging (prevent log injection) + local safe_filename = utils.log_sanitize(filename) + + -- Validate filename for security (defense-in-depth) + local valid, validation_err = utils.file_validate_name(filename) + if not valid then + logger.log_error( "[MULTIPART] SECURITY ATTACK DETECTED - Invalid filename: " .. safe_filename) + logger.log_error( "[MULTIPART] Attack type: " .. validation_err) + + -- Behavior matrix for filename injection attacks: + -- BLOCK=yes, RESUME=yes: Log attack, extract file with sanitized name, scan content, block at end + -- BLOCK=yes, RESUME=no: Block immediately (fast protection) + -- BLOCK=no, RESUME=yes: Log attack, scan this file anyway, continue to other files + -- BLOCK=no, RESUME=no: Log attack, scan this file anyway until first virus hit + if block_on_filename_attack then + if resume_on_hit then + -- Resume mode: Extract and scan file content even with malicious filename + -- This allows detecting BOTH filename attack AND content malware + logger.log_error( "[MULTIPART] BLOCK=yes, RESUME=yes - extracting file with sanitized name to scan content, will block at end") + filename_attack_detected = true + -- Sanitize filename for safe disk storage (replace dangerous chars with _) + -- Keep original filename for reporting but use sanitized version for extraction + filename = filename:gsub("[%z/\\]", "_") + -- Fall through to normal file processing + else + -- Block immediately - cleanup temp files before returning + logger.log_error( "[MULTIPART] BLOCK=yes, RESUME=no - blocking immediately") + for _, temp_path in ipairs(created_temp_files) do + os.remove(temp_path) + end + return false, "malicious filename detected: " .. validation_err .. " (filename: " .. safe_filename .. ")", nil + end + else + -- BLOCK=no: Log the attack but continue processing this file + -- The file will still be scanned for actual malware content + logger.log_error( "[MULTIPART] BLOCK=no - logging attack but continuing to scan file content") + filename_attack_detected = true + -- Fall through to normal file processing (no skip, no return) + end + end + + part_count = part_count + 1 + plugin:log_debug("[v" .. file_ops.VERSION .. "] [MULTIPART] Found file part #" .. part_count .. ": " .. safe_filename) + + -- Extract content-type if present + local file_content_type = part_content:match("[Cc]ontent%-[Tt]ype:%s*([^%s\r\n]+)") + + -- Find end of headers (double newline) + local content_start = part_content:find("\r\n\r\n") + if content_start then + content_start = content_start + 4 + else + content_start = part_content:find("\n\n") + if content_start then + content_start = content_start + 2 + else + logger.log_error( "[MULTIPART] Could not find end of headers for file: " .. safe_filename) + goto continue + end + end + + -- Extract file content (trim trailing CR/LF before boundary) + local file_content = part_content:sub(content_start) + -- Remove trailing CRLF or LF + file_content = file_content:gsub("\r\n$", ""):gsub("\n$", "") + + -- Resource limit: check file count before creating temp file (CWE-770: DoS via file descriptor exhaustion) + if #files >= max_files then + logger.log_error( "[MULTIPART] RESOURCE LIMIT EXCEEDED - Maximum files per request reached: " .. max_files) + logger.log_error( "[MULTIPART] Blocked DoS attempt: " .. (#files + 1) .. " files in single request") + -- Cleanup already created temp files before returning + for _, temp_path in ipairs(created_temp_files) do + os.remove(temp_path) + end + return false, "too many files in single request (max: " .. max_files .. ")", nil + end + + -- Write to temporary file + local temp_path = os.tmpname() + local temp_file, temp_err = io.open(temp_path, "wb") + if not temp_file then + logger.log_error( "[MULTIPART] Failed to create temp file for " .. safe_filename .. ": " .. (temp_err or "unknown")) + goto continue + end + + -- Track temp file for cleanup on error + table.insert(created_temp_files, temp_path) + + temp_file:write(file_content) + temp_file:close() + + -- Detect file type using `file` command + local detected_type, type_err = file_ops.file_get_type(plugin, temp_path) + if not detected_type then + plugin:log_debug("[v" .. file_ops.VERSION .. "] [MULTIPART] Could not detect file type for " .. safe_filename .. ": " .. (type_err or "unknown")) + detected_type = "unknown" + end + + table.insert(files, { + filepath = temp_path, + filename = filename, + size = #file_content, + content_type = file_content_type or "application/octet-stream", + file_type = detected_type + }) + + logger.log_error( "[MULTIPART] Extracted file: " .. safe_filename .. " (" .. #file_content .. " bytes, type: " .. detected_type .. ") -> " .. temp_path) + end + + ::continue:: + pos = next_boundary_start + end + + if #files == 0 and not filename_attack_detected then + return false, "no file uploads found in multipart body", nil + end + + logger.log_error( "[MULTIPART] Parsed " .. #files .. " file(s) from multipart body") + + -- Return with attack indicator if filename attacks were detected + if filename_attack_detected then + logger.log_error( "[MULTIPART] Filename injection attack(s) detected - will block request after full scan") + return true, files, "filename_attack_detected" + end + + return true, files, nil +end + +-- ============================================================================ +-- FILE CONTENT EXTRACTION +-- ============================================================================ + +-- Extract actual file content from multipart body. +-- The temp file contains the entire multipart/form-data body including headers. +-- This function extracts just the file content from the part with filename=. +-- +-- SECURITY WARNING: This function searches for the filename pattern in the body +-- but does NOT validate the filename itself. Filename validation MUST be done +-- by the caller (see utils.file_validate_name() and multipart_parse()). +-- NEVER trust attacker-provided filenames for filesystem operations! +-- +-- Returns success boolean and extracted content or error message. +--- @return boolean success True if extraction succeeded +--- @return string content_or_error Extracted file content on success, error message on failure +function file_ops.content_extract(plugin, file_path) + local file, err = io.open(file_path, "rb") + if not file then + return false, "failed to open file: " .. (err or "unknown") + end + + -- Read the entire file (should be reasonable size for uploads) + local content = file:read("*a") + file:close() + + if not content or content == "" then + return false, "file is empty" + end + + -- Find the part that contains filename= (the actual file upload) + local filename_pos = content:find('filename="') + if not filename_pos then + -- Try without quotes + filename_pos = content:find("filename=") + if not filename_pos then + logger.log_error( "could not find filename in multipart body") + return false, "no file upload found in multipart body" + end + end + + -- Find the end of headers for this part (double newline after filename line) + local content_start = content:find("\r\n\r\n", filename_pos) + if not content_start then + -- Try just \n\n (some clients use this) + content_start = content:find("\n\n", filename_pos) + if not content_start then + logger.log_error( "could not find end of file part headers") + return false, "invalid multipart format" + end + content_start = content_start + 2 -- Skip \n\n + else + content_start = content_start + 4 -- Skip \r\n\r\n + end + + -- Find the ending boundary AFTER the file content (starts with --) + local boundary_pattern = "\r\n%-%-" + local content_end = content:find(boundary_pattern, content_start) + if not content_end then + -- Try just \n-- + boundary_pattern = "\n%-%-" + content_end = content:find(boundary_pattern, content_start) + if not content_end then + logger.log_error( "could not find ending multipart boundary") + return false, "invalid multipart format" + end + end + + -- Extract just the file content (between headers and ending boundary) + local file_content = content:sub(content_start, content_end - 1) + + plugin:log_debug("Extracted file content: " .. #file_content .. " bytes from " .. #content .. " bytes total") + + return true, file_content +end + +-- Calculate SHA256 checksum of a file. +-- Takes file path, returns success boolean and checksum or error message. +-- For multipart uploads, extracts the actual file content first. +-- Returns special value "empty_file" for 0-byte files to skip calculation. +--- @return boolean success True if hash calculation succeeded +--- @return string checksum_or_error SHA256 checksum on success (or "empty_file"), error message on failure +function file_ops.hash_calculate(plugin, file_path) + -- Extract actual file content from multipart body + local ok, file_content = file_ops.content_extract(plugin, file_path) + if not ok then + logger.log_error( "failed to extract file content: " .. file_content) + -- Fallback: hash the entire file as-is + local file, err = io.open(file_path, "rb") + if not file then + return false, "failed to open file: " .. (err or "unknown") + end + + -- Validate file size before reading to prevent DoS (Medium severity issue #17) + local size = file:seek("end") + file:seek("set", 0) + + local MAX_FALLBACK_SIZE = 100 * 1024 * 1024 -- 100MB + if size > MAX_FALLBACK_SIZE then + file:close() + return false, "file too large for fallback SHA256 calculation (max 100MB)" + end + + local sha = sha256:new() + if not sha then + file:close() + return false, "failed to create SHA256 instance" + end + + while true do + local chunk = file:read(8192) + if not chunk then + break + end + sha:update(chunk) + end + + file:close() + local checksum = to_hex(sha:final()) + logger.log_error( "calculated SHA256 of entire multipart body (fallback): " .. checksum) + return true, checksum + end + + -- Check if file is empty (0 bytes) - skip SHA256 calculation + if #file_content == 0 then + logger.log_error( "file is 0 bytes (empty), skipping SHA256 calculation") + return true, "empty_file" + end + + -- Calculate SHA256 of extracted file content + local sha = sha256:new() + if not sha then + return false, "failed to create SHA256 instance" + end + + sha:update(file_content) + local checksum = to_hex(sha:final()) + + logger.log_error( "calculated SHA256 of extracted file content (" .. #file_content .. " bytes): " .. checksum) + return true, checksum +end + +-- ============================================================================ +-- FILE CLEANUP OPERATIONS +-- ============================================================================ + +-- Delete temporary file after scanning. +-- Returns success boolean and error message if failed. +--- @return boolean success True if cleanup succeeded +--- @return string|nil error Error message if failed +function file_ops.file_cleanup(plugin, file_path) + if not file_path or file_path == "" then + return false, "invalid file path" + end + + -- Use EAFP pattern to avoid race condition (Medium severity issue #18) + -- Try to remove first, handle error if file doesn't exist + local ok, err = os.remove(file_path) + if not ok then + -- Check if error is "file not found" (already cleaned up, possibly by security software) + if err and (err:match("No such file") or err:match("cannot find") or err:match("not exist")) then + logger.log_notice( "temp file already removed (possibly by security software): " .. file_path) + return true + end + -- Other error (permissions, etc.) + return false, err or "unknown error" + end + + logger.log_notice( "cleaned up temp file: " .. file_path) + return true +end + +-- Clean up orphaned temporary files older than specified age. +-- Scans temp directory and deletes files older than max_age seconds. +-- Returns success boolean and error message if failed. +--- @return boolean success True if cleanup succeeded +--- @return string|nil error Error message if failed +function file_ops.orphan_cleanup(plugin, max_age) + -- Check if lfs is available (required for directory operations) + if not lfs then + logger.log_warn("lfs module not available - orphan cleanup disabled") + return true -- Not an error, just can't clean + end + + -- Validate max_age to prevent invalid values and command injection (CWE-78) + -- Use validation utility to ensure it's a positive integer <= 86400 (1 day) + local max_age_num = utils.validate_positive_integer(max_age, 3600, 86400, "max_age") + + -- Double-check after validation + if max_age_num <= 0 then + return false, "invalid max_age value (must be positive number <= 86400)" + end + + -- Use default nginx temp path for BunkerWeb + local temp_path = "/var/tmp/bunkerweb/client_temp" + + -- Check if directory exists + local dir_attr = lfs.attributes(temp_path, "mode") + if not dir_attr or dir_attr ~= "directory" then + logger.log_warn("Temp directory does not exist: " .. temp_path) + return true -- Not an error, just nothing to clean + end + + local cleaned_count = 0 + local error_count = 0 + local current_time = os.time() + + -- Iterate through directory using lfs.dir (native Lua, non-blocking) + for filename in lfs.dir(temp_path) do + -- Skip . and .. entries + if filename ~= "." and filename ~= ".." then + local file_path = temp_path .. "/" .. filename + + -- Get file attributes + local attr = lfs.attributes(file_path) + + -- Only process regular files (skip directories, symlinks, etc.) + if attr and attr.mode == "file" then + -- Check if file is older than max_age + local file_age = current_time - attr.modification + if file_age > max_age_num then + -- File is old enough, try to remove it + local ok, err = os.remove(file_path) + if ok then + cleaned_count = cleaned_count + 1 + else + -- Check if error is "file not found" (already deleted by security software) + if err and (err:match("No such file") or err:match("cannot find")) then + -- File already removed, count as cleaned + cleaned_count = cleaned_count + 1 + else + -- Actual error + error_count = error_count + 1 + logger.log_warn( + "failed to remove orphaned file " .. file_path .. ": " .. (err or "unknown") + ) + end + end + end + end + end + end + + if cleaned_count > 0 then + logger.log_notice( + "cleaned up " .. cleaned_count .. " orphaned temp files from " .. temp_path + ) + end + + return true +end + +-- ============================================================================ +-- MALWARE SAMPLE SHARING +-- Note: This function has dependencies on VirusTotal module +-- It will be called with virustotal module passed as parameter +-- ============================================================================ + +-- Share detected malware file with threat intelligence platforms. +-- Uploads unknown malware to VirusTotal for community threat intelligence. +-- Returns success boolean and error message if failed. +--- @return boolean success True if sharing succeeded or was skipped +--- @return string|nil error Error message if failed +function file_ops.detection_share(plugin, file_path, checksum, detector_name, virustotal, file_type) + -- Check if file sharing is enabled + if plugin.variables["MALWARE_SCAN_SHARE_FILES"] ~= "yes" then + plugin:log_debug("File sharing disabled, not uploading detected malware") + return true + end + + -- Check if VirusTotal is enabled for sharing + if plugin.variables["MALWARE_SCAN_API_USE_VIRUSTOTAL"] ~= "yes" then + logger.log_error( "File sharing enabled but VirusTotal not configured - cannot share detected file") + return true + end + + -- Verify virustotal module is provided + if not virustotal then + logger.log_error( "[SHARE] VirusTotal module not loaded - cannot share detected file") + return true + end + + logger.log_error( "[SHARE] File detected by " .. detector_name .. ", checking if already known to VirusTotal") + + -- Check if file is already known to VirusTotal + local vt_ok, vt_found, vt_response = virustotal.virustotal_request(plugin, "/files/" .. checksum) + if not vt_ok then + logger.log_error( "[SHARE] VirusTotal check failed: " .. vt_found .. " - cannot determine if file should be shared") + return true + end + + if vt_found then + logger.log_error( "[SHARE] File already known to VirusTotal (detected by " .. detector_name .. ") - skipping upload") + return true + end + + -- ======================================================================== + -- GDPR PROTECTION: Check file type before uploading + -- ======================================================================== + + -- Load filetypes module for GDPR filtering + local filetypes = require("malware_scan_filetypes") + + -- Use provided file_type or default to "unknown" + local detected_type = file_type or "unknown" + + -- Apply same GDPR logic as orchestrator: + -- Only upload if: file_type is known AND is executable AND NOT user data + local is_safe_to_upload = detected_type and detected_type ~= "unknown" and not filetypes.contains_user_data(detected_type) + + if not detected_type or detected_type == "unknown" then + -- Unknown file type - DO NOT UPLOAD (GDPR-safe default) + logger.log_error( string.format( + "[SHARE] %s detected malware but file type unknown - GDPR protection: NOT uploading to VirusTotal (cannot verify if file contains personal data)", + detector_name + )) + return true + elseif filetypes.contains_user_data(detected_type) then + -- File may contain personal data - DO NOT UPLOAD + logger.log_error( string.format( + "[SHARE] %s detected malware but file may contain user data (type: %s) - GDPR protection: NOT uploading to VirusTotal", + detector_name, + detected_type + )) + logger.log_error( "[SHARE] File type categories: documents, images, databases, archives may contain personal data and will not be uploaded") + return true + elseif not is_safe_to_upload then + -- Shouldn't reach here, but defensive check + logger.log_error( string.format( + "[SHARE] %s detected malware but file failed GDPR check (type: %s) - NOT uploading to VirusTotal", + detector_name, + detected_type or "nil" + )) + return true + end + + -- File is safe to upload: known executable type without user data + logger.log_error( string.format( + "[SHARE] File unknown to VirusTotal and safe to share (type: %s) - uploading for threat intelligence (detected by %s)", + detected_type, + detector_name + )) + local upload_ok, analysis_id, file_size = virustotal.file_upload(plugin, file_path) + + if not upload_ok then + logger.log_error( "[SHARE] Failed to upload file to VirusTotal: " .. analysis_id) + return false, analysis_id + end + + logger.log_error( "[SHARE] File uploaded successfully (" .. file_size .. " bytes), analysis ID: " .. tostring(analysis_id)) + + -- Auto-vote if enabled + if plugin.variables["MALWARE_SCAN_API_VIRUSTOTAL_AUTO_VOTE"] == "yes" then + logger.log_error( "[SHARE] Auto-voting file as malicious on VirusTotal") + local vote_ok, vote_err = virustotal.virustotal_vote(plugin, checksum, "malicious") + if vote_ok then + logger.log_error( "[SHARE] Malicious vote submitted successfully") + else + logger.log_error( "[SHARE] Failed to submit vote: " .. vote_err) + end + end + + return true +end + +-- Check available disk space for a given path. +-- Returns available space in bytes, or nil on error. +-- Uses `df` command to check filesystem space. +--- @param plugin Plugin instance +--- @param path Directory path to check (will check parent if doesn't exist) +--- @return number|nil available_bytes Available space in bytes or nil on error +--- @return string|nil error Error message if failed +function file_ops.disk_space_check(plugin, path) + if not path or path == "" then + return nil, "invalid path" + end + + -- Validate and escape path to prevent command injection (CWE-78) + local escaped_path, err = utils.validate_and_escape_path(path) + if not escaped_path then + return nil, err + end + + -- Get parent directory if path doesn't exist yet (native Lua, non-blocking) + local check_path = escaped_path + local parent_dir = path:match("(.+)/[^/]+$") + if parent_dir and lfs then + -- Check if path exists using lfs.attributes + local dir_attr = lfs.attributes(path, "mode") + if not dir_attr or dir_attr ~= "directory" then + -- Directory doesn't exist, use parent directory + local escaped_parent = utils.validate_and_escape_path(parent_dir) + if escaped_parent then + check_path = escaped_parent + plugin:log_debug("Directory doesn't exist yet, checking parent: " .. parent_dir) + end + end + end + + -- Run df command to check available space + -- -B1 = output in bytes + -- Format: Filesystem 1B-blocks Used Available Use% Mounted on + local cmd = string.format("df -B1 %s 2>&1 | tail -1 | awk '{print $4}'", check_path) + + local handle, err = io.popen(cmd) + if not handle then + logger.log_error( "Failed to execute df command: " .. (err or "unknown")) + return nil, "failed to execute df command" + end + + local output = handle:read("*l") + handle:close() + + if not output or output == "" then + return nil, "no output from df command" + end + + -- Parse available bytes + local available_bytes = tonumber(output) + if not available_bytes then + logger.log_error( "Failed to parse df output: " .. output) + return nil, "failed to parse df output" + end + + -- Validate disk space is a valid non-negative number (not negative, NaN, or infinity) + available_bytes = utils.validate_disk_space(available_bytes) + if available_bytes == 0 and output ~= "0" then + -- Validation failed (was negative or invalid) + logger.log_error( "Invalid disk space value from df: " .. output) + return nil, "invalid disk space value" + end + + plugin:log_debug(string.format("Available disk space: %d bytes (%.2f MB)", available_bytes, available_bytes / 1024 / 1024)) + return available_bytes, nil +end + +-- Export detected malware file to local directory for forensic analysis. +-- Saves file with SHA256 hash as filename for easy identification. +-- Also creates a .txt file with detection details (detector, result, timestamp, etc.) +-- Checks disk space before saving to prevent filling the filesystem. +--- @param plugin Plugin instance +--- @param file_path Path to malware file to export +--- @param checksum SHA256 hash of the file +--- @param detector_name Name of scanner that detected the malware (e.g., "ClamAV") +--- @param detection_result Detection result string (e.g., "Win.Trojan.Cerber") +--- @param file_type MIME type of the file (optional) +--- @return boolean success True if export succeeded or was skipped +--- @return string|nil error Error message if failed +function file_ops.virusfile_export(plugin, file_path, checksum, detector_name, detection_result, file_type) + -- Check if export is enabled + local export_path = plugin.variables["MALWARE_SCAN_VIRUSFILE_LOCAL_EXPORT_PATH"] + if not export_path or export_path == "" then + plugin:log_debug("[v" .. file_ops.VERSION .. "] [EXPORT] Local malware export disabled (MALWARE_SCAN_VIRUSFILE_LOCAL_EXPORT_PATH not set)") + return true + end + + -- Validate inputs + if not file_path or file_path == "" then + return false, "invalid file path" + end + + if not checksum or checksum == "" or checksum == "empty_file" then + logger.log_error( "[EXPORT] Cannot export file without valid checksum") + return false, "invalid checksum" + end + + -- Get file size + local file_size = file_ops.file_get_size(plugin, file_path) + if not file_size or file_size == 0 then + logger.log_error( "[EXPORT] Cannot export empty or non-existent file") + return false, "invalid file size" + end + + -- Validate file_size is a positive number + file_size = utils.validate_file_size(file_size, 10 * 1024 * 1024 * 1024) + + -- Check available disk space (require at least 2x file size + 100MB buffer) + -- Use safe arithmetic to prevent overflow + local buffer_size = 100 * 1024 * 1024 -- 100MB + local doubled_size = utils.safe_multiply(file_size, 2, 10 * 1024 * 1024 * 1024) + local required_space = utils.safe_add(doubled_size, buffer_size, 10 * 1024 * 1024 * 1024) + + local available_space, space_err = file_ops.disk_space_check(plugin, export_path) + + if not available_space then + logger.log_error( string.format( + "[EXPORT] Failed to check disk space for %s: %s - skipping export", + export_path, + space_err or "unknown error" + )) + return false, "disk space check failed: " .. (space_err or "unknown") + end + + if available_space < required_space then + logger.log_error( string.format( + "[EXPORT] Insufficient disk space: available %.2f MB, required %.2f MB (file size %.2f MB + 100MB buffer) - skipping export", + available_space / 1024 / 1024, + required_space / 1024 / 1024, + file_size / 1024 / 1024 + )) + return false, string.format("insufficient disk space: %.2f MB available, %.2f MB required", + available_space / 1024 / 1024, + required_space / 1024 / 1024) + end + + -- Validate and escape paths for shell commands (defense-in-depth) + local escaped_export_path, escape_err = utils.validate_and_escape_path(export_path) + if not escaped_export_path then + logger.log_error( string.format("[v" .. file_ops.VERSION .. "] [EXPORT] Export path validation failed: %s", escape_err or "unknown")) + return false, "export path validation failed: " .. (escape_err or "unknown") + end + + local escaped_file_path, file_escape_err = utils.validate_and_escape_path(file_path) + if not escaped_file_path then + logger.log_error( string.format("[v" .. file_ops.VERSION .. "] [EXPORT] File path validation failed: %s", file_escape_err or "unknown")) + return false, "file path validation failed: " .. (file_escape_err or "unknown") + end + + -- Create export directory if it doesn't exist (native Lua, non-blocking) + local mkdir_ok, mkdir_err = mkdir_recursive(export_path) + if not mkdir_ok then + logger.log_error( "[EXPORT] Failed to create export directory: " .. (mkdir_err or "unknown error")) + return false, "failed to create directory: " .. (mkdir_err or "unknown error") + end + + -- Build export file path: /export/path/SHA256 + local export_file = string.format("%s/%s", export_path, checksum) + local escaped_export_file, export_file_err = utils.validate_and_escape_path(export_file) + if not escaped_export_file then + logger.log_error( string.format("[v" .. file_ops.VERSION .. "] [EXPORT] Export file path validation failed: %s", export_file_err or "unknown")) + return false, "export file path validation failed: " .. (export_file_err or "unknown") + end + + -- Check if file already exists (native Lua, non-blocking) if lfs is available + if lfs then + local file_attr = lfs.attributes(export_file, "mode") + if file_attr == "file" then + logger.log_error( string.format( + "[EXPORT] File already exported: %s (detected by %s: %s) - skipping", + export_file, + detector_name or "unknown", + detection_result or "unknown" + )) + return true -- Not an error, just already exported + end + end + + -- Copy file to export directory (native Lua, non-blocking) + local cp_ok, cp_err = file_copy(file_path, export_file) + if not cp_ok then + logger.log_error( string.format("[v" .. file_ops.VERSION .. "] [EXPORT] Failed to copy file: %s", cp_err or "unknown error")) + return false, "failed to copy file: " .. (cp_err or "unknown error") + end + + -- Set restrictive permissions (600 = owner read/write only) + local chmod_cmd = string.format("chmod 600 %s 2>&1", escaped_export_file) + local chmod_handle = io.popen(chmod_cmd) + if chmod_handle then + local _ = chmod_handle:read("*a") -- Read and discard output + chmod_handle:close() + end + + -- Create detection details text file (SHA256.txt) + local info_file = export_file .. ".txt" + local info_handle, info_err = io.open(info_file, "w") + if info_handle then + -- Get current timestamp + local timestamp = os.date("%Y-%m-%d %H:%M:%S UTC", os.time()) + + -- Write detection details + info_handle:write("=============================================================================\n") + info_handle:write("MALWARE DETECTION REPORT\n") + info_handle:write("=============================================================================\n\n") + info_handle:write("Detection Date: " .. timestamp .. "\n") + info_handle:write("SHA256 Hash: " .. checksum .. "\n") + info_handle:write("File Size: " .. string.format("%.2f MB (%d bytes)", file_size / 1024 / 1024, file_size) .. "\n") + info_handle:write("File Type: " .. (file_type or "unknown") .. "\n") + info_handle:write("Detected By: " .. (detector_name or "unknown") .. "\n") + info_handle:write("Detection Result: " .. (detection_result or "unknown") .. "\n") + info_handle:write("\n") + info_handle:write("-----------------------------------------------------------------------------\n") + info_handle:write("Original Path: " .. file_path .. "\n") + info_handle:write("Export Path: " .. export_file .. "\n") + info_handle:write("-----------------------------------------------------------------------------\n\n") + info_handle:write("This file contains malicious content and should only be analyzed\n") + info_handle:write("in a secure, isolated environment. Do not execute or open without\n") + info_handle:write("proper security precautions.\n\n") + info_handle:write("For analysis assistance, you can upload the sample to:\n") + info_handle:write("- VirusTotal: https://www.virustotal.com/\n") + info_handle:write("- MalwareBazaar: https://bazaar.abuse.ch/\n") + info_handle:write("- Hybrid Analysis: https://www.hybrid-analysis.com/\n\n") + info_handle:write("=============================================================================\n") + info_handle:close() + + -- Set restrictive permissions on info file + local escaped_info_file, info_escape_err = utils.validate_and_escape_path(info_file) + if escaped_info_file then + local chmod_info_cmd = string.format("chmod 600 %s 2>&1", escaped_info_file) + local chmod_info_handle = io.popen(chmod_info_cmd) + if chmod_info_handle then + local _ = chmod_info_handle:read("*a") -- Read and discard output + chmod_info_handle:close() + end + else + logger.log_warn( string.format("[v" .. file_ops.VERSION .. "] [EXPORT] Could not set permissions on info file: %s", info_escape_err or "unknown")) + end + + plugin:log_debug(string.format("[v" .. file_ops.VERSION .. "] [EXPORT] Detection details written to: %s", info_file)) + else + logger.log_warn( string.format( + "[EXPORT] Failed to create detection details file %s: %s", + info_file, + info_err or "unknown" + )) + end + + -- Log successful export + logger.log_error( string.format( + "[EXPORT] Malware sample exported: %s (%.2f MB) - Detected by %s: %s - SHA256: %s", + export_file, + file_size / 1024 / 1024, + detector_name or "unknown", + detection_result or "unknown", + checksum + )) + + return true +end + +-- ============================================================================ +-- ALLOW LIST (FALSE POSITIVE HANDLING) +-- ============================================================================ + +-- Populate allow list cache from config and file. +-- Loads all hashes into memory cache for fast lookups. +-- Called once on first request (lazy loading). +--- @param plugin table Plugin instance +--- @return boolean success True if population succeeded +--- @return number hash_count Number of hashes loaded +function file_ops.allowlist_populate(plugin) + local hash_count = 0 + + logger.log_error( "[ALLOWLIST] Populating allow list cache from configuration") + + -- Load from comma-separated config variable + local allowlist_hashes = plugin.variables["MALWARE_SCAN_ALLOWLIST_HASHES"] or "" + if allowlist_hashes ~= "" then + for hash in allowlist_hashes:gmatch("[^,]+") do + -- Trim whitespace and normalize to lowercase + hash = hash:gsub("^%s+", ""):gsub("%s+$", ""):lower() + -- Validate hash format (64 hex characters) + if hash:match("^[a-f0-9]{64}$") then + local cache_key = "plugin_malware_scan_allowlist_" .. hash + plugin.cachestore:set(cache_key, "1", 0) -- 0 = never expire + hash_count = hash_count + 1 + else + logger.log_warn( string.format( + "[ALLOWLIST] Invalid hash format in MALWARE_SCAN_ALLOWLIST_HASHES: %s (must be 64 hex characters)", + hash + )) + end + end + if hash_count > 0 then + logger.log_error( string.format( + "[ALLOWLIST] Loaded %d hash(es) from config variable", + hash_count + )) + end + end + + -- Load from file + local allowlist_file = plugin.variables["MALWARE_SCAN_ALLOWLIST_FILE"] or "" + if allowlist_file ~= "" then + local file, err = io.open(allowlist_file, "r") + if not file then + logger.log_error( string.format( + "[ALLOWLIST] Failed to open allow list file %s: %s", + allowlist_file, err or "unknown" + )) + else + local file_hash_count = 0 + for line in file:lines() do + -- Trim whitespace + line = line:gsub("^%s+", ""):gsub("%s+$", "") + -- Skip empty lines and comments + if line ~= "" and not line:match("^#") then + -- Normalize to lowercase + line = line:lower() + -- Validate hash format (64 hex characters) + if line:match("^[a-f0-9]{64}$") then + local cache_key = "plugin_malware_scan_allowlist_" .. line + plugin.cachestore:set(cache_key, "1", 0) -- 0 = never expire + file_hash_count = file_hash_count + 1 + hash_count = hash_count + 1 + else + logger.log_warn( string.format( + "[ALLOWLIST] Invalid hash format in file %s: %s (must be 64 hex characters)", + allowlist_file, line + )) + end + end + end + file:close() + + if file_hash_count > 0 then + logger.log_error( string.format( + "[ALLOWLIST] Loaded %d hash(es) from file %s", + file_hash_count, allowlist_file + )) + end + end + end + + -- Mark cache as populated + plugin.cachestore:set("plugin_malware_scan_allowlist_populated", "1", 0) + + logger.log_error( string.format( + "[ALLOWLIST] Cache population complete - %d total hash(es) loaded", + hash_count + )) + + return true, hash_count +end + +-- Clear allow list cache (called on service reload). +--- @param plugin table Plugin instance +--- @return boolean success Always returns true +function file_ops.allowlist_clear(plugin) + logger.log_error( "[ALLOWLIST] Clearing allow list cache for reload") + + -- Clear the populated flag + plugin.cachestore:delete("plugin_malware_scan_allowlist_populated") + + -- Note: Individual hash entries will be overwritten on next population + -- No need to iterate and delete them all (performance optimization) + + return true +end + +-- Check if allow list cache is populated. +--- @param plugin table Plugin instance +--- @return boolean populated True if cache is populated +function file_ops.allowlist_is_populated(plugin) + local populated = plugin.cachestore:get("plugin_malware_scan_allowlist_populated") + return populated == "1" +end + +-- Check if a SHA256 hash is in the allow list. +-- Uses cached allow list for performance (populated on first request). +-- Falls back to direct file/config reading if cache is not available. +--- @param plugin table Plugin instance +--- @param checksum string SHA256 hash to check +--- @return boolean in_list True if hash is in allow list, false otherwise +function file_ops.allowlist_check_hash(plugin, checksum) + if not checksum or checksum == "" or checksum == "empty_file" then + return false + end + + -- Normalize checksum to lowercase for comparison + checksum = checksum:lower() + + -- Ensure allow list cache is populated (lazy loading on first request) + if not file_ops.allowlist_is_populated(plugin) then + plugin:log_debug("[v" .. file_ops.VERSION .. "] [ALLOWLIST] Cache not populated, loading allow list into cache") + file_ops.allowlist_populate(plugin) + end + + -- Fast path: Check cache for hash + local cache_key = "plugin_malware_scan_allowlist_" .. checksum + local cached = plugin.cachestore:get(cache_key) + if cached == "1" then + plugin:log_debug(string.format( + "[ALLOWLIST] File hash matches allow list (cached): %s", + checksum + )) + return true + end + + -- Not in allow list + return false +end + +-- Return module +return file_ops diff --git a/malware-scan/malware_scan_filetypes.lua b/malware-scan/malware_scan_filetypes.lua new file mode 100644 index 0000000..6eff1ea --- /dev/null +++ b/malware-scan/malware_scan_filetypes.lua @@ -0,0 +1,347 @@ +-- BunkerWeb Malware Scan - File Type Categories +-- Categorizes MIME types returned by `file --mime-type` command +-- Used for risk assessment, GDPR compliance, and filtering logic + +-- Module table +local filetypes = {} + +-- Module version +filetypes.VERSION = "0.8.0" + +-- ============================================================================ +-- EXECUTABLE FILES (High Risk - Should Always Be Scanned) +-- ============================================================================ + +-- Files that can execute code directly or contain executable content. +-- These are the primary vectors for malware delivery. +-- MIME types returned by `file --mime-type` command. + +filetypes.EXECUTABLE_TYPES = { + -- Windows Executables (Modern) + "application/x-dosexec", -- PE executables (.exe, .dll, .scr, .sys, .com) + "application/x-msdownload", -- Windows downloads + "application/x-ms-dos-executable", -- DOS executables + "application/vnd.microsoft.portable-executable", -- PE format + + -- Windows Executables (Legacy - Still Seen in the Wild) + "application/x-ms-ne-executable", -- 16-bit Windows (NE format) + "application/x-ms-w3-executable", -- Win32s executables + "application/x-ms-w4-executable", -- Windows 95 executables + "application/x-coff-executable", -- COFF format + + -- Linux/Unix Executables + "application/x-executable", -- ELF executables + "application/x-sharedlib", -- Shared libraries (.so) + "application/x-object", -- Object files + "application/x-pie-executable", -- Position Independent Executables + + -- macOS Executables + "application/x-mach-binary", -- Mach-O binaries + "application/x-mach-executable", -- Mach-O executables + + -- Scripts (Interpreted but Executable) + "text/x-shellscript", -- Shell scripts (#!/bin/bash, #!/bin/sh) + "text/x-python", -- Python scripts + "text/x-perl", -- Perl scripts + "text/x-ruby", -- Ruby scripts + "text/x-php", -- PHP scripts + "text/x-lua", -- Lua scripts + "text/x-tcl", -- TCL scripts + "text/x-awk", -- AWK scripts + "application/x-javascript", -- JavaScript files + "application/javascript", -- JavaScript + "application/x-node", -- Node.js scripts + + -- Bytecode and Compiled Scripts + "application/java-vm", -- Java class files + "application/x-bytecode.python", -- Python bytecode (.pyc) + "application/x-java-applet", -- Java applets + + -- Modern Web Threats + "application/wasm", -- WebAssembly binaries (can execute code) + "application/x-chrome-extension", -- Chrome extensions (.crx) + + -- Archives (Often Contain Executables) + "application/zip", -- ZIP archives + "application/x-rar", -- RAR archives + "application/x-7z-compressed", -- 7-Zip archives + "application/x-ace-compressed", -- ACE archives + "application/x-arj", -- ARJ archives + "application/x-arc", -- ARC archives + "application/x-zoo", -- ZOO archives + "application/x-stuffit", -- StuffIt archives (Mac) + "application/x-cpio", -- CPIO archives + "application/x-lzh-compressed", -- LZH archives + "application/gzip", -- Gzip compressed + "application/x-bzip", -- Bzip compressed + "application/x-bzip2", -- Bzip2 compressed + "application/x-xz", -- XZ compressed + "application/x-lzip", -- LZIP compressed + "application/x-lzma", -- LZMA compressed + "application/x-lzop", -- LZOP compressed + "application/x-tar", -- TAR archives + "application/x-compress", -- Unix compress + + -- Disk Images (Common Malware Delivery Method) + "application/x-iso9660-image", -- ISO disk images + "application/x-raw-disk-image", -- Raw disk images + "application/x-compressed-iso", -- Compressed ISO images + "application/x-qemu-disk", -- QEMU disk images + "application/x-virtualbox-vhd", -- VirtualBox VHD + "application/x-virtualbox-ova", -- VirtualBox OVA + + -- Windows Installers and Packages + "application/x-msi", -- MSI installers + "application/x-ms-wim", -- Windows Imaging Format + "application/x-installshield", -- InstallShield installers + "application/vnd.ms-cab-compressed", -- CAB archives + + -- Mobile Applications + "application/vnd.android.package-archive", -- Android APK + "application/x-apple-diskimage", -- macOS DMG + "application/x-ios-app", -- iOS applications + + -- Java Archives + "application/java-archive", -- JAR files + "application/x-java-archive", -- JAR files (alternate) + + -- Windows Shortcuts (Can Execute Commands) + "application/x-ms-shortcut", -- .lnk files + + -- Batch Files + "application/x-bat", -- Windows batch files + "text/x-msdos-batch", -- DOS batch files +} + +-- ============================================================================ +-- USER DATA FILES (May Contain Personal Information - GDPR Relevant) +-- ============================================================================ + +-- Files that commonly contain user-generated content or personal data. +-- Important for GDPR compliance when uploading to public databases. +-- MIME types returned by `file --mime-type` command. + +filetypes.USER_DATA_TYPES = { + -- Plain Text Documents + "text/plain", -- Plain text files + "text/csv", -- CSV data files + "text/tab-separated-values", -- TSV files + "text/rtf", -- Rich Text Format + "text/x-log", -- Log files (may contain user data) + + -- Microsoft Office Documents (Legacy) + "application/msword", -- Word documents (.doc) + "application/vnd.ms-excel", -- Excel spreadsheets (.xls) + "application/vnd.ms-powerpoint", -- PowerPoint presentations (.ppt) + "application/vnd.ms-access", -- Access databases + "application/vnd.ms-outlook", -- Outlook PST/OST files + + -- Microsoft Office Documents (Modern - Office 2007+) + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", -- .docx + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", -- .xlsx + "application/vnd.openxmlformats-officedocument.presentationml.presentation", -- .pptx + + -- OpenDocument Formats (LibreOffice/OpenOffice) + "application/vnd.oasis.opendocument.text", -- .odt (text documents) + "application/vnd.oasis.opendocument.spreadsheet", -- .ods (spreadsheets) + "application/vnd.oasis.opendocument.presentation", -- .odp (presentations) + + -- PDF Documents + "application/pdf", -- PDF files (often contain personal data) + + -- Images (May Contain Faces, Locations, Metadata) + "image/jpeg", -- JPEG images + "image/jpg", -- JPG images + "image/png", -- PNG images + "image/gif", -- GIF images + "image/bmp", -- Bitmap images + "image/tiff", -- TIFF images + "image/webp", -- WebP images + "image/heic", -- HEIC (iPhone photos) + "image/heif", -- HEIF format + "image/svg+xml", -- SVG vector graphics + "image/avif", -- AVIF (modern format) + "image/jxl", -- JPEG XL (modern format) + "image/jxr", -- JPEG XR (Microsoft) + "image/vnd.adobe.photoshop", -- PSD files (Adobe Photoshop) + + -- Camera RAW Formats (Photos = Personal Data) + "image/x-canon-cr2", -- Canon RAW photos + "image/x-canon-crw", -- Canon RAW photos (older) + "image/x-fuji-raf", -- Fuji RAW photos + "image/x-olympus-orf", -- Olympus RAW photos + + -- Audio Files (Voice Recordings, etc.) + "audio/mpeg", -- MP3 audio + "audio/mp3", -- MP3 (alternate) + "audio/mp4", -- M4A audio + "audio/wav", -- WAV audio + "audio/x-wav", -- WAV (alternate) + "audio/flac", -- FLAC audio + "audio/ogg", -- OGG audio + "audio/aac", -- AAC audio + + -- Video Files (May Contain Faces, Locations) + "video/mp4", -- MP4 video + "video/mpeg", -- MPEG video + "video/quicktime", -- QuickTime/MOV + "video/x-msvideo", -- AVI video + "video/x-matroska", -- MKV video + "video/webm", -- WebM video + "video/3gpp", -- 3GP mobile video + "video/3gpp2", -- 3GPP2 mobile video + "video/x-flv", -- Flash Video + "video/x-ms-asf", -- Windows Media (ASF) + + -- Email Files + "message/rfc822", -- Email messages (.eml) + "application/mbox", -- Unix mailbox format + + -- **CRITICAL** Database Files (Almost Always Contain Personal Data) + "application/vnd.sqlite3", -- SQLite databases + "application/x-sqlite3", -- SQLite databases (alternate) + "application/x-sqlite2", -- SQLite v2 databases + "application/vnd.geopackage+sqlite3", -- GeoPackage databases (geospatial data) + "application/x-dbf", -- dBase database files + "application/msaccess", -- Microsoft Access databases + "application/x-msaccess", -- Microsoft Access (alternate) + + -- **CRITICAL** Medical and Health Data (HIPAA/GDPR) + "application/dicom", -- Medical imaging (DICOM format) + + -- Financial Data + "application/x-gnucash", -- GnuCash financial data + + -- eBooks (May Contain Personal Notes/Highlights) + "application/epub+zip", -- EPUB ebooks + "application/x-mobipocket-ebook", -- MOBI ebooks + + -- Scientific Data Files + "application/x-hdf", -- HDF scientific data + "application/x-hdf5", -- HDF5 scientific data + + -- Archives (May Contain Personal Data - Documents, Photos, Databases, etc.) + -- NOTE: Archives are in BOTH executable and user data lists + -- - EXECUTABLE_TYPES: Can contain malware + -- - USER_DATA_TYPES: Can contain personal documents/photos/data + "application/zip", -- ZIP archives + "application/x-rar", -- RAR archives + "application/x-7z-compressed", -- 7-Zip archives + "application/x-ace-compressed", -- ACE archives + "application/x-arj", -- ARJ archives + "application/x-arc", -- ARC archives + "application/x-zoo", -- ZOO archives + "application/x-stuffit", -- StuffIt archives (Mac) + "application/x-cpio", -- CPIO archives + "application/x-lzh-compressed", -- LZH archives + "application/gzip", -- Gzip compressed + "application/x-bzip", -- Bzip compressed + "application/x-bzip2", -- Bzip2 compressed + "application/x-xz", -- XZ compressed + "application/x-lzip", -- LZIP compressed + "application/x-lzma", -- LZMA compressed + "application/x-lzop", -- LZOP compressed + "application/x-tar", -- TAR archives + "application/x-compress", -- Unix compress + "application/vnd.ms-cab-compressed", -- CAB archives + + -- Disk Images (May Contain Personal Data - Backups, File Systems) + -- NOTE: Disk images are in BOTH executable and user data lists + -- - EXECUTABLE_TYPES: Can contain bootable malware + -- - USER_DATA_TYPES: Can contain personal files/backups + "application/x-iso9660-image", -- ISO disk images + "application/x-raw-disk-image", -- Raw disk images + "application/x-compressed-iso", -- Compressed ISO images + "application/x-qemu-disk", -- QEMU disk images + "application/x-virtualbox-vhd", -- VirtualBox VHD + "application/x-virtualbox-ova", -- VirtualBox OVA + "application/x-apple-diskimage", -- macOS DMG (disk images) + + -- Markup and Data Formats + "text/html", -- HTML documents + "application/xhtml+xml", -- XHTML documents + "text/xml", -- XML files + "application/xml", -- XML (alternate) + "application/json", -- JSON data files + + -- Contact and Calendar Data + "text/vcard", -- vCard contact files + "text/calendar", -- iCalendar files +} + +-- ============================================================================ +-- HELPER FUNCTIONS +-- ============================================================================ + +-- Check if a MIME type is an executable file type. +--- @param mime_type MIME type string from file --mime-type +--- @return boolean is_executable True if executable type +function filetypes.is_executable(mime_type) + if not mime_type then + return false + end + + -- Normalize MIME type (lowercase, strip charset if present) + local normalized = mime_type:lower():match("^([^;]+)") + + for _, exec_type in ipairs(filetypes.EXECUTABLE_TYPES) do + if normalized == exec_type then + return true + end + end + + return false +end + +-- Check if a MIME type commonly contains user data. +-- Important for GDPR compliance when considering file uploads to public databases. +--- @param mime_type MIME type string from file --mime-type +--- @return boolean contains_user_data True if likely contains user data +function filetypes.contains_user_data(mime_type) + if not mime_type then + return false + end + + -- Normalize MIME type (lowercase, strip charset if present) + local normalized = mime_type:lower():match("^([^;]+)") + + for _, data_type in ipairs(filetypes.USER_DATA_TYPES) do + if normalized == data_type then + return true + end + end + + return false +end + +-- Get risk category for a MIME type. +-- Returns one of: "executable", "user_data", "archive", "unknown" +--- @param mime_type MIME type string from file --mime-type +--- @return string category Risk category (one of: "executable", "user_data", "archive", "unknown") +function filetypes.get_risk_category(mime_type) + if not mime_type then + return "unknown" + end + + -- Check executable first (highest risk) + if filetypes.is_executable(mime_type) then + -- Check if it's specifically an archive + local normalized = mime_type:lower():match("^([^;]+)") + if normalized:match("archive") or normalized:match("zip") or + normalized:match("rar") or normalized:match("7z") or + normalized:match("tar") or normalized:match("compress") then + return "archive" + end + return "executable" + end + + -- Check user data + if filetypes.contains_user_data(mime_type) then + return "user_data" + end + + return "unknown" +end + +-- Return module +return filetypes diff --git a/malware-scan/malware_scan_init.lua b/malware-scan/malware_scan_init.lua new file mode 100644 index 0000000..1784019 --- /dev/null +++ b/malware-scan/malware_scan_init.lua @@ -0,0 +1,423 @@ +-- BunkerWeb Malware Scan - Initialization Module +-- Centralized module loading and dependency initialization +-- Provides clean interface for plugin setup and dependency management + +local logger = require("malware_scan_logger") +local ngx = ngx + +-- Module table +local init = {} + +-- Module version +init.VERSION = "0.8.0" + +-- ============================================================================ +-- MODULE LOADING +-- ============================================================================ + +-- Load all plugin dependencies with error handling. +-- Uses pcall to safely load each module, allowing the plugin to continue +-- functioning even if some optional modules fail to load. +-- +-- Returns a table with: +-- - modules: Table of loaded modules (nil if failed to load) +-- - status: Table of load status (true/false for each module) +-- - errors: Table of error messages for failed loads +-- +-- Example: +-- local deps = init.deps_load() +-- if deps.modules.clamav then +-- -- Use ClamAV module +-- else +-- -- ClamAV not available: deps.errors.clamav +-- end +-- +--- @return table deps Table with modules, status, and errors +function init.deps_load() + local modules = {} + local status = {} + local errors = {} + + -- Load logger module first (may be used by other modules) + status.logger, errors.logger = pcall(function() + modules.logger = require("malware_scan_logger") + end) + if not status.logger then + modules.logger = nil + end + + -- Load utilities module (required by other modules) + status.utils, errors.utils = pcall(function() + modules.utils = require("malware_scan_utils") + end) + if not status.utils then + modules.utils = nil + end + + -- Load database module (Redis connection and operations) + status.database, errors.database = pcall(function() + modules.database = require("malware_scan_database") + end) + if not status.database then + modules.database = nil + end + + -- Load cache module + status.cache, errors.cache = pcall(function() + modules.cache = require("malware_scan_cache") + end) + if not status.cache then + modules.cache = nil + end + + -- Load file operations module + status.file_ops, errors.file_ops = pcall(function() + modules.file_ops = require("malware_scan_file") + end) + if not status.file_ops then + modules.file_ops = nil + end + + -- Load ClamAV scanner module + status.clamav, errors.clamav = pcall(function() + modules.clamav = require("malware_scan_clamav") + end) + if not status.clamav then + modules.clamav = nil + end + + -- Load VirusTotal API module + status.virustotal, errors.virustotal = pcall(function() + modules.virustotal = require("malware_scan_virustotal") + end) + if not status.virustotal then + modules.virustotal = nil + end + + -- Load SentinelOne API module + status.sentinelone, errors.sentinelone = pcall(function() + modules.sentinelone = require("malware_scan_sentinelone") + end) + if not status.sentinelone then + modules.sentinelone = nil + end + + -- Load MalwareBazaar API module + status.malwarebazaar, errors.malwarebazaar = pcall(function() + modules.malwarebazaar = require("malware_scan_malwarebazaar") + end) + if not status.malwarebazaar then + modules.malwarebazaar = nil + end + + -- Load webhook notification module + status.webhook, errors.webhook = pcall(function() + modules.webhook = require("malware_scan_webhook") + end) + if not status.webhook then + modules.webhook = nil + end + + -- Load multipart full scan module + status.multipart_full_scan, errors.multipart_full_scan = pcall(function() + modules.multipart_full_scan = require("malware_scan_multipart_full_scan") + end) + if not status.multipart_full_scan then + modules.multipart_full_scan = nil + end + + -- Load utilities module + status.ms_utils, errors.ms_utils = pcall(function() + modules.ms_utils = require("malware_scan_utils") + end) + if not status.ms_utils then + modules.ms_utils = nil + end + + -- Load file types module + status.filetypes, errors.filetypes = pcall(function() + modules.filetypes = require("malware_scan_filetypes") + end) + if not status.filetypes then + modules.filetypes = nil + end + + -- Load configuration module + status.ms_config, errors.ms_config = pcall(function() + modules.ms_config = require("malware_scan_config") + end) + if not status.ms_config then + modules.ms_config = nil + end + + -- Load orchestrator module + status.ms_orchestrator, errors.ms_orchestrator = pcall(function() + modules.ms_orchestrator = require("malware_scan_orchestrator") + end) + if not status.ms_orchestrator then + modules.ms_orchestrator = nil + end + + -- Load API handler module + status.ms_api, errors.ms_api = pcall(function() + modules.ms_api = require("malware_scan_api") + end) + if not status.ms_api then + modules.ms_api = nil + end + + -- Load scheduler module + status.ms_scheduler, errors.ms_scheduler = pcall(function() + modules.ms_scheduler = require("malware_scan_scheduler") + end) + if not status.ms_scheduler then + modules.ms_scheduler = nil + end + + -- Load UI metrics module + status.ui_metrics, errors.ui_metrics = pcall(function() + modules.ui_metrics = require("malware_scan_ui_metrics") + end) + if not status.ui_metrics then + modules.ui_metrics = nil + end + + return { + modules = modules, + status = status, + errors = errors + } +end + +-- ============================================================================ +-- DEPENDENCY INITIALIZATION +-- ============================================================================ + +-- Initialize cross-module dependencies. +-- Sets up modules that depend on other modules (e.g., scanners need cache). +-- This function should be called after deps_load(). +-- +-- Dependency relationships: +-- - Cache module โ†’ ClamAV, VirusTotal, SentinelOne, MalwareBazaar, Multipart +-- - File ops module โ†’ VirusTotal, Multipart +-- - Scanner modules (ClamAV, VirusTotal, SentinelOne, MalwareBazaar) โ†’ Multipart +-- - Webhook module โ†’ Multipart +-- +-- Example: +-- local deps = init.deps_load() +-- init.deps_initialize(deps.modules, deps.status) +-- +--- @param modules Table of loaded modules from deps_load() +--- @param status Table of load status from deps_load() +function init.deps_initialize(modules, status) + -- Set cache module for scanners (if cache is loaded) + if status.cache and modules.cache then + -- ClamAV needs cache for result caching + if status.clamav and modules.clamav then + modules.clamav.set_cache_module(modules.cache) + end + + -- VirusTotal needs cache for API response caching + if status.virustotal and modules.virustotal then + modules.virustotal.set_cache_module(modules.cache) + end + + -- SentinelOne needs cache for API response caching + if status.sentinelone and modules.sentinelone then + modules.sentinelone.set_cache_module(modules.cache) + end + + -- MalwareBazaar uses Redis directly (no API caching needed) + -- Cache setup not required - MalwareBazaar integrates with Redis hash lookup + + -- Multipart scanner needs cache for body hash caching + if status.multipart_full_scan and modules.multipart_full_scan then + modules.multipart_full_scan.set_cache_module(modules.cache) + end + end + + -- Set file operations module (if file_ops is loaded) + if status.file_ops and modules.file_ops then + -- VirusTotal needs file ops for file upload + if status.virustotal and modules.virustotal then + modules.virustotal.set_file_ops_module(modules.file_ops) + end + + -- Multipart scanner needs file ops for body extraction + if status.multipart_full_scan and modules.multipart_full_scan then + modules.multipart_full_scan.set_file_ops_module(modules.file_ops) + end + end + + -- Set scanner modules for multipart full scan (if multipart is loaded) + if status.multipart_full_scan and modules.multipart_full_scan then + -- ClamAV scanner for body scanning + if status.clamav and modules.clamav then + modules.multipart_full_scan.set_clamav_module(modules.clamav) + end + + -- VirusTotal scanner for body hash lookup + if status.virustotal and modules.virustotal then + modules.multipart_full_scan.set_virustotal_module(modules.virustotal) + end + + -- SentinelOne scanner for body hash lookup + if status.sentinelone and modules.sentinelone then + modules.multipart_full_scan.set_sentinelone_module(modules.sentinelone) + end + + -- MalwareBazaar scanner for body hash lookup + if status.malwarebazaar and modules.malwarebazaar then + modules.multipart_full_scan.set_malwarebazaar_module(modules.malwarebazaar) + end + + -- Webhook for malware detection notifications + if status.webhook and modules.webhook then + modules.multipart_full_scan.set_webhook_module(modules.webhook) + end + end +end + +-- ============================================================================ +-- PACKAGE PATH SETUP +-- ============================================================================ + +-- Setup package.path to include plugin directory. +-- This allows require() to find plugin modules in the BunkerWeb plugins directory. +-- Should be called before deps_load(). +-- +-- Default path: /etc/bunkerweb/plugins/malware-scan/ +-- Can be overridden by passing custom path. +-- +-- Example: +-- init.path_setup() -- Use default path +-- init.path_setup("/custom/path/?.lua") -- Use custom path +-- +--- @param custom_path Optional custom path to add (default: /etc/bunkerweb/plugins/malware-scan/?.lua) +function init.path_setup(custom_path) + local path = custom_path or "/etc/bunkerweb/plugins/malware-scan/?.lua" + + -- Only add if not already present (avoid duplicates) + if not package.path:match(path:gsub("([%.%-%?])", "%%%1")) then + package.path = package.path .. ";" .. path + end +end + +-- ============================================================================ +-- CONFIGURATION VALIDATION +-- ============================================================================ + +-- Validate plugin configuration at startup. +-- Performs comprehensive checks on paths, numerical values, scanner dependencies, +-- and logical consistency of configuration options. +-- +-- Validation errors are critical and should prevent plugin from starting. +-- Validation warnings are informational and logged but don't prevent startup. +-- +-- Example: +-- local deps = init.initialize() +-- local valid, errors, warnings = init.validate_configuration(plugin, deps) +-- if not valid then +-- -- Log errors and abort +-- end +-- +--- @param plugin Plugin instance (provides access to plugin.variables and plugin.logger) +--- @param deps Dependencies structure from deps_load() or initialize() +--- @return boolean valid True if validation passed +--- @return table errors List of validation errors +--- @return table warnings List of validation warnings +function init.validate_configuration(plugin, deps) + -- Check if config module is loaded + if not deps.status.ms_config or not deps.modules.ms_config then + return false, {"Configuration module failed to load: " .. tostring(deps.errors.ms_config)}, {} + end + + local config_module = deps.modules.ms_config + + -- Run comprehensive startup validation + local valid, errors, warnings = config_module.validate_startup(plugin.variables) + + -- Log validation results + if #errors > 0 then + logger.log_error( "[INIT] Configuration validation FAILED:") + for _, err in ipairs(errors) do + logger.log_error( "[INIT] " .. err) + end + end + + if #warnings > 0 then + logger.log_warn( "[INIT] Configuration validation warnings:") + for _, warn in ipairs(warnings) do + logger.log_warn( "[INIT] " .. warn) + end + end + + if valid and #warnings == 0 then + logger.log_notice( "[INIT] Configuration validation passed (no errors or warnings)") + elseif valid then + logger.log_notice( "[INIT] Configuration validation passed with " .. #warnings .. " warning(s)") + end + + return valid, errors, warnings +end + +-- ============================================================================ +-- COMPLETE INITIALIZATION +-- ============================================================================ + +-- Complete initialization workflow in one call. +-- Convenience function that runs all initialization steps: +-- 1. Setup package path +-- 2. Load all dependencies +-- 3. Initialize cross-module dependencies +-- +-- Returns the same structure as deps_load() for easy access to modules. +-- +-- Example: +-- local deps = init.initialize() +-- local clamav = deps.modules.clamav +-- local cache = deps.modules.cache +-- +--- @param custom_path Optional custom package path +--- @return table deps Table with modules, status, and errors +function init.initialize(custom_path) + -- Setup package path + init.path_setup(custom_path) + + -- Load all dependencies + local deps = init.deps_load() + + -- Initialize cross-module dependencies + init.deps_initialize(deps.modules, deps.status) + + return deps +end + +-- Complete initialization workflow with validation. +-- Like initialize() but also validates configuration at startup. +-- Returns validation results along with dependencies. +-- +-- Example: +-- local deps, valid, errors, warnings = init.initialize_and_validate(plugin) +-- if not valid then +-- -- Abort startup due to configuration errors +-- end +-- +--- @param plugin Plugin instance (provides access to plugin.variables and plugin.logger) +--- @param custom_path Optional custom package path +--- @return table deps Table with modules, status, and errors +--- @return boolean valid True if validation passed +--- @return table errors List of validation errors +--- @return table warnings List of validation warnings +function init.initialize_and_validate(plugin, custom_path) + -- Run normal initialization + local deps = init.initialize(custom_path) + + -- Validate configuration + local valid, errors, warnings = init.validate_configuration(plugin, deps) + + return deps, valid, errors, warnings +end + +-- Return module +return init diff --git a/malware-scan/malware_scan_logger.lua b/malware-scan/malware_scan_logger.lua new file mode 100644 index 0000000..5c01206 --- /dev/null +++ b/malware-scan/malware_scan_logger.lua @@ -0,0 +1,180 @@ +-- BunkerWeb Malware Scan - Centralized Logger Module +-- Logs to BunkerWeb default logs via errlog.raw_log() +-- +-- Features: +-- - Sanitizes sensitive data (API keys, tokens, webhooks, emails) +-- - Version-tagged log messages +-- - Standard nginx log levels (ERR, WARN, NOTICE, INFO) +-- - Uses BunkerWeb's logging pipeline with [MALWARE-SCAN] prefix +-- +-- Version: 0.8.0 +-- Date: 2026-01-27 + +local logger = {} + +-- Module version +logger.VERSION = "0.8.0" + +-- ============================================================================ +-- SHARED TYPE ALIASES (for Lua Language Server) +-- ============================================================================ +-- Plugin class is defined in malware_scan_scheduler.lua + +---@alias Optional table|nil Optional value (nil or typed) +---@alias Dependencies table Module dependencies and configuration +---@alias Path string File or directory path +---@alias Remote string Remote URL or resource identifier +---@alias Download string Downloaded file content or metadata +---@alias Local string Local file or cache reference +---@alias Temporary string Temporary storage reference +---@alias Base string Base URL or directory +---@alias Update table Update operation metadata +---@alias Timeout number Connection or operation timeout value +---@alias Configuration table Configuration settings and variables +---@alias Directory string Directory path +---@alias SHA256 string SHA256 hash string +---@alias Name string Name or identifier +---@alias Detection string Malware detection signature or result +---@alias MIME string MIME type string +---@alias Table table Lua table (generic) +---@alias Log table Logger instance +---@alias File table File operations or file path +---@alias UI table User interface module +---@alias Webhook table Webhook notification module +---@alias Function function Lua function +---@alias Cache table Cache storage module +---@alias The table Generic table (for descriptions) + +local errlog = require("ngx.errlog") +local ngx = ngx +local ERR = ngx.ERR +local WARN = ngx.WARN +local NOTICE = ngx.NOTICE +local INFO = ngx.INFO +local raw_log = errlog.raw_log + +-- ============================================================================ +-- LOG MESSAGE SANITIZATION +-- ============================================================================ + +-- Sanitize log messages to mask sensitive data (API keys, tokens, webhook URLs). +-- Automatically detects and masks common sensitive patterns to prevent credential leakage. +-- +-- Patterns masked: +-- - API keys (64 hex chars) -> [APIKEY:****] +-- - Tokens (32+ alphanumeric) -> [TOKEN:****] +-- - Bearer tokens -> Bearer [MASKED] +-- - Webhook URLs -> https://hooks.***[MASKED] +-- - Email addresses -> ***@***.*** (for GDPR compliance) +-- +--- @param message string log message to sanitize +--- @return string sanitized_message Sanitized message with sensitive data masked +local function logger_sanitize_log_message(message) + if not message or type(message) ~= "string" then + return tostring(message) + end + + local sanitized = message + + -- Mask VirusTotal-style API keys (64 hex characters) + -- Example: abcd1234...9876 -> [APIKEY:abcd****9876] + sanitized = sanitized:gsub("([a-fA-F0-9]{64})", function(key) + return "[APIKEY:" .. key:sub(1, 4) .. "****" .. key:sub(-4) .. "]" + end) + + -- Mask long alphanumeric tokens (32+ characters, likely auth tokens) + -- Example: abc123xyz456... -> [TOKEN:abc1****6789] + sanitized = sanitized:gsub("([a-zA-Z0-9_-]{32,})", function(token) + -- Skip if it looks like a hash checksum in valid context + if token:match("^[a-fA-F0-9]+$") and ( + message:match("checksum") or + message:match("hash") or + message:match("SHA256") or + message:match("MD5") + ) then + return token + end + return "[TOKEN:" .. token:sub(1, 4) .. "****" .. token:sub(-4) .. "]" + end) + + -- Mask Bearer tokens in Authorization headers + sanitized = sanitized:gsub("Bearer%s+([a-zA-Z0-9_-]+)", "Bearer [MASKED]") + + -- Mask webhook URLs (Teams, Discord, Slack, etc.) + -- Example: https://hooks.slack.com/services/T00/B00/XXXX -> https://hooks.***/[MASKED] + sanitized = sanitized:gsub("(https?://[^%s]*webhook[^%s]*)", function(url) + local host = url:match("^(https?://[^/]+)") + if host then + return host:gsub("([^/]+//[^/]+)", "%1") .. "/***[MASKED]" + end + return "https://***[MASKED]" + end) + sanitized = sanitized:gsub("(https?://hooks%.[^%s]+)", "https://hooks.***/[MASKED]") + + -- Mask email addresses for GDPR compliance + -- Example: user@example.com -> ***@***.*** + sanitized = sanitized:gsub("([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+%.[a-zA-Z]{2,})", "***@***.***") + + -- Mask common password/key patterns in URLs or config + sanitized = sanitized:gsub("([?&])password=([^&%s]+)", "%1password=[MASKED]") + sanitized = sanitized:gsub("([?&])token=([^&%s]+)", "%1token=[MASKED]") + sanitized = sanitized:gsub("([?&])key=([^&%s]+)", "%1key=[MASKED]") + + return sanitized +end + +-- ============================================================================ +-- CORE LOGGING FUNCTIONS +-- ============================================================================ + +-- Simple logging function - writes to BunkerWeb default logs via errlog.raw_log() +-- Uses same mechanism as BunkerWeb's core logger for consistent log routing +--- @param level Log level (ERR, WARN, NOTICE, INFO) +--- @param message string Log message +local function logger_log_simple(level, message) + -- Sanitize message to mask sensitive data (API keys, tokens, emails, webhooks) + local sanitized_message = logger_sanitize_log_message(message) + + -- Add MALWARE-SCAN prefix (like BunkerWeb's logger) + version tag + local prefixed_message = "[MALWARE-SCAN] [v" .. logger.VERSION .. "] " .. sanitized_message + + -- Write to BunkerWeb logs using errlog.raw_log (same as BunkerWeb's logger) + raw_log(level, prefixed_message) +end + +-- ============================================================================ +-- PUBLIC API - Convenience Wrappers +-- ============================================================================ + +-- Log at ERROR level +--- @param message string Log message +function logger.log_error(message) + logger_log_simple(ERR, message) +end + +-- Log at WARN level +--- @param message string Log message +function logger.log_warn(message) + logger_log_simple(WARN, message) +end + +-- Log at NOTICE level +--- @param message string Log message +function logger.log_notice(message) + logger_log_simple(NOTICE, message) +end + +-- Log at INFO level +--- @param message string Log message +function logger.log_info(message) + logger_log_simple(INFO, message) +end + +-- Generic log function with level parameter +--- @param level Log level (ERR, WARN, NOTICE, INFO) +--- @param message string Log message +function logger.log(level, message) + logger_log_simple(level, message) +end + +return logger diff --git a/malware-scan/malware_scan_malwarebazaar.lua b/malware-scan/malware_scan_malwarebazaar.lua new file mode 100644 index 0000000..1a1bbb3 --- /dev/null +++ b/malware-scan/malware_scan_malwarebazaar.lua @@ -0,0 +1,186 @@ +-- BunkerWeb Malware Scan - MalwareBazaar Hash Database Module +-- Redis-backed hash lookup for MalwareBazaar threat intelligence +-- +-- Features: +-- - 1M+ malware hashes from MalwareBazaar (abuse.ch) in Redis +-- - Bucketed storage across 16 Redis keys for 16x faster lookups +-- - Sub-millisecond hash lookups (<1ms) vs API calls (100-500ms) +-- - Automatic hourly updates from sysangels.ai mirror +-- - Infinite persistence (freshness maintained by scheduler) +-- +-- Data Source: https://bazaar.abuse.ch (MalwareBazaar Project) +-- Mirror: https://bazaar.sysangels.ai/bunkerweb/ + +local cjson = require("cjson") +local logger = require("malware_scan_logger") +local utils = require("malware_scan_utils") +local database = require("malware_scan_database") + +-- Module table +local malwarebazaar = {} + +-- Module version +malwarebazaar.VERSION = "0.8.0" + +local ngx = ngx + +-------------------------------------------------------------------------------- +-- MALWAREBAZAAR HASH LOOKUP (Redis-based) +-------------------------------------------------------------------------------- + +-- Check if SHA256 hash exists in MalwareBazaar database +--- @param plugin Plugin instance +--- @param checksum string SHA256 hash (64 hex chars) +--- @param file_size number Optional file size (not used for Redis lookup) +--- @return boolean success True if lookup succeeded +--- @return string result "clean" if not found, malware name if detected +function malwarebazaar.hash_check(plugin, checksum, file_size) + -- DEBUG: Log function entry + logger.log_error(string.format("[v%s] [HASH_CHECK_DEBUG] === hash_check() called ===", malwarebazaar.VERSION)) + logger.log_error(string.format("[v%s] [HASH_CHECK_DEBUG] Input checksum: %s", malwarebazaar.VERSION, tostring(checksum))) + logger.log_error(string.format("[v%s] [HASH_CHECK_DEBUG] Input file_size: %s", malwarebazaar.VERSION, tostring(file_size))) + + -- Check if feature is enabled + local feature_enabled = plugin.variables and plugin.variables["MALWARE_SCAN_MALWAREBAZAAR_HASH_LOOKUP"] == "yes" + logger.log_error(string.format("[v%s] [HASH_CHECK_DEBUG] Feature enabled check: %s (value: %s)", + malwarebazaar.VERSION, + tostring(feature_enabled), + tostring(plugin.variables and plugin.variables["MALWARE_SCAN_MALWAREBAZAAR_HASH_LOOKUP"] or "nil"))) + + if not feature_enabled then + logger.log_error(string.format("[v%s] [HASH_CHECK_DEBUG] Feature disabled, returning clean", malwarebazaar.VERSION)) + return true, "clean" + end + + -- Check if Redis is available + logger.log_error(string.format("[v%s] [HASH_CHECK_DEBUG] Redis available: %s", malwarebazaar.VERSION, tostring(database.is_available(plugin)))) + if not database.is_available(plugin) then + logger.log_error(string.format("[v%s] [HASH_CHECK_DEBUG] Redis not available, returning clean", malwarebazaar.VERSION)) + logger.log_error( "MalwareBazaar hash lookup enabled but Redis not available") + return true, "clean" + end + + -- Two-tier lookup strategy: + -- 1. Check recent cache (fast path for frequently seen malware) + -- 2. Check bucketed full hash table (slower but complete) + -- Store format: HSET malware_scan:hashes:X | + -- Uppercase checksum for case-insensitive matching (hashes stored uppercase in Redis) + local checksum_upper = checksum:upper() + logger.log_error(string.format("[v%s] [HASH_CHECK_DEBUG] Uppercase checksum: %s", malwarebazaar.VERSION, checksum_upper)) + + -- Track lookup source for logging + local lookup_source = "none" + + -- TIER 1: Check recent cache first (fast path) + local recent_key = "malware_scan:hashes:recent" + logger.log_error(string.format("[v%s] [HASH_CHECK_DEBUG] Checking recent cache: %s", malwarebazaar.VERSION, recent_key)) + + local data = database.hget(plugin, recent_key, checksum_upper) + logger.log_error(string.format("[v%s] [HASH_CHECK_DEBUG] Recent cache response: %s (type: %s)", + malwarebazaar.VERSION, + tostring(data), + type(data))) + + if data and data ~= ngx.null then + -- Found in recent cache - fast path! + lookup_source = "recent" + logger.log_error(string.format("[v%s] [HASH_CHECK_DEBUG] โœ“ FOUND IN RECENT CACHE (fast path)", malwarebazaar.VERSION)) + else + -- TIER 2: Not in recent cache, check bucketed full hash table + logger.log_error(string.format("[v%s] [HASH_CHECK_DEBUG] Not in recent cache, checking full hash table", malwarebazaar.VERSION)) + + local hash_key = utils.get_hash_bucket(checksum_upper) + logger.log_error(string.format("[v%s] [HASH_CHECK_DEBUG] Bucket key: %s", malwarebazaar.VERSION, hash_key)) + + data = database.hget(plugin, hash_key, checksum_upper) + logger.log_error(string.format("[v%s] [HASH_CHECK_DEBUG] Full table response: %s (type: %s)", + malwarebazaar.VERSION, + tostring(data), + type(data))) + + if data and data ~= ngx.null then + -- Found in full table - cache it in recent for future quick lookups + lookup_source = "bucket:" .. hash_key:sub(-1) + logger.log_error(string.format("[v%s] [HASH_CHECK_DEBUG] โœ“ FOUND IN FULL TABLE, caching to recent", malwarebazaar.VERSION)) + + -- Add to recent cache (fire and forget - don't check result) + pcall(function() + database.hset(plugin, recent_key, checksum_upper, data) + end) + end + end + + logger.log_error(string.format("[v%s] [HASH_CHECK_DEBUG] data == ngx.null: %s", + malwarebazaar.VERSION, + tostring(data == ngx.null))) + + if not data or data == ngx.null then + -- Hash not found in either recent or full table - file is clean or unknown + logger.log_error(string.format("[v%s] [HASH_CHECK_DEBUG] Hash not found in Redis (data is nil or ngx.null), returning clean", malwarebazaar.VERSION)) + return true, "clean" + end + + -- Parse stored data: "signature|clamav" (pipe-delimited, 2 fields) + logger.log_error(string.format("[v%s] [HASH_CHECK_DEBUG] Parsing data: %s", malwarebazaar.VERSION, data)) + + -- Split by pipe delimiter + local fields = {} + for field in string.gmatch(data, "([^|]+)") do + table.insert(fields, field) + end + + local signature = fields[1] or "" + local clamav = fields[2] or "" + + logger.log_error(string.format("[v%s] [HASH_CHECK_DEBUG] Parsed signature: %s", malwarebazaar.VERSION, signature)) + logger.log_error(string.format("[v%s] [HASH_CHECK_DEBUG] Parsed clamav: %s", malwarebazaar.VERSION, clamav)) + + -- Normalize empty/placeholder values + -- Treat "n/a" as empty since it's a placeholder for missing data + if signature == "n/a" or signature == "N/A" then + signature = "" + end + if clamav == "n/a" or clamav == "N/A" then + clamav = "" + end + + -- Build combined result intelligently: + -- IMPORTANT: Hash match in database = confirmed malware, even if signature names are missing + -- 1. If both fields have values, show both (signature is MalwareBazaar name, clamav is ClamAV signature) + -- 2. If only one has a value, use that one + -- 3. If both are empty, use generic detection name (hash is in database = malware) + -- 4. Prefer clamav field for consistency with ClamAV scanner output + local combined_result + if signature ~= "" and clamav ~= "" then + -- Both fields present - show both for maximum context + combined_result = clamav .. " (MalwareBazaar: " .. signature .. ")" + elseif clamav ~= "" then + -- Only ClamAV signature present - use it directly + combined_result = clamav + elseif signature ~= "" then + -- Only MalwareBazaar signature present - use it + combined_result = signature + else + -- Both empty but hash exists in database - still malware! + -- Use generic detection name to indicate confirmed malware with unknown signature + combined_result = "MalwareBazaar.Generic" + logger.log_error(string.format("[v%s] [HASH_CHECK_DEBUG] Hash match but both signatures empty - using generic name", malwarebazaar.VERSION)) + end + + -- Found match - return both signatures + logger.log_error( string.format( + "[v%s] [MALWAREBAZAAR] โœ“ HASH MATCH: %s... = %s (source: %s)", + malwarebazaar.VERSION, + checksum_upper:sub(1, 16), + combined_result, + lookup_source + )) + + logger.log_error(string.format("[v%s] [HASH_CHECK_DEBUG] Returning malware detection: %s", malwarebazaar.VERSION, combined_result)) + return true, combined_result +end + +-- Legacy alias for backward compatibility +malwarebazaar.check_malwarebazaar = malwarebazaar.hash_check + +return malwarebazaar diff --git a/malware-scan/malware_scan_metrics.lua b/malware-scan/malware_scan_metrics.lua new file mode 100644 index 0000000..fea7d53 --- /dev/null +++ b/malware-scan/malware_scan_metrics.lua @@ -0,0 +1,271 @@ +-- Malware Scan Plugin - Metrics Module +-- Provides Prometheus-compatible metrics for operational monitoring +-- +-- Metrics Categories: +-- - Scanner execution (total, results by outcome) +-- - Cache performance (hits, misses, hit rate) +-- - Atomic locking (acquired, wait, timeout) +-- - Latency histograms (P50, P95, P99) +-- - API cost tracking +-- +-- Usage: +-- local metrics = require("malware_scan_metrics") +-- metrics.init() +-- metrics.record_scan("clamav", "clean", 150) -- 150ms latency +-- metrics.record_cache("clamav", true) -- Cache hit +-- metrics.record_lock_acquired("clamav") +-- + +local metrics = {} + +-- Module version +metrics.VERSION = "0.8.0" + +-- Module state +metrics.enabled = false +metrics.prometheus = nil + +-------------------------------------------------------------------------------- +-- INITIALIZATION +-------------------------------------------------------------------------------- + +-- Initialize metrics module +-- Attempts to load Prometheus library and define metrics +-- Falls back gracefully if library not available +--- @return boolean success True if initialization succeeded +function metrics.init() + -- Try to load Prometheus library + local ok, prometheus = pcall(require, "prometheus") + + if not ok then + -- Prometheus library not available - metrics disabled + metrics.enabled = false + return false + end + + -- Initialize Prometheus with namespace + metrics.prometheus = prometheus.init("plugin_malware_scan", {sync_interval=1}) + metrics.enabled = true + + -- ========================================================================== + -- SCANNER EXECUTION METRICS + -- ========================================================================== + + -- Total scans/queries per scanner (labels: scanner) + metrics.scan_total = metrics.prometheus:counter( + "scanner_total", + "Total number of scans or API queries executed per scanner", + {"scanner"} -- scanner: clamav, virustotal, sentinelone + ) + + -- Scan results by scanner and outcome (labels: scanner, result) + metrics.scan_result = metrics.prometheus:counter( + "scanner_result_total", + "Scan results by scanner and outcome (clean, malicious, error)", + {"scanner", "result"} -- result: clean, malicious, error, api_circuit_breaker + ) + + -- ========================================================================== + -- CACHE PERFORMANCE METRICS + -- ========================================================================== + + -- Cache hits per scanner (labels: scanner) + metrics.cache_hits = metrics.prometheus:counter( + "cache_hits_total", + "Number of cache hits per scanner", + {"scanner"} + ) + + -- Cache misses per scanner (labels: scanner) + metrics.cache_misses = metrics.prometheus:counter( + "cache_misses_total", + "Number of cache misses per scanner", + {"scanner"} + ) + + -- ========================================================================== + -- ATOMIC LOCKING METRICS + -- ========================================================================== + + -- Locks acquired per scanner (labels: scanner) + metrics.lock_acquired = metrics.prometheus:counter( + "lock_acquired_total", + "Number of locks acquired (scanner performed scan/query)", + {"scanner"} + ) + + -- Lock waits per scanner (labels: scanner) + metrics.lock_wait = metrics.prometheus:counter( + "lock_wait_total", + "Number of times waited for concurrent scan/query result", + {"scanner"} + ) + + -- Lock wait timeouts per scanner (labels: scanner) + metrics.lock_timeout = metrics.prometheus:counter( + "lock_timeout_total", + "Number of lock wait timeouts (fell back to independent scan/query)", + {"scanner"} + ) + + -- ========================================================================== + -- LATENCY METRICS (HISTOGRAMS) + -- ========================================================================== + + -- Scanner latency distribution (labels: scanner) + -- Buckets: 10ms, 50ms, 100ms, 500ms, 1s, 2s, 5s, 10s + metrics.scan_latency = metrics.prometheus:histogram( + "scanner_latency_seconds", + "Scanner execution latency distribution in seconds", + {"scanner"}, + {0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10} + ) + + -- ========================================================================== + -- FILE PROCESSING METRICS + -- ========================================================================== + + -- Total files processed + metrics.files_total = metrics.prometheus:counter( + "files_processed_total", + "Total number of files processed" + ) + + -- Files blocked (malware detected) + metrics.files_blocked = metrics.prometheus:counter( + "files_blocked_total", + "Number of files blocked due to malware detection" + ) + + -- Files allowed (clean) + metrics.files_allowed = metrics.prometheus:counter( + "files_allowed_total", + "Number of clean files allowed" + ) + + -- ========================================================================== + -- API COST TRACKING + -- ========================================================================== + + -- API calls per scanner (for cost calculation) + -- Separate from scan_total because scan_total includes cache hits + metrics.api_calls_actual = metrics.prometheus:counter( + "api_calls_actual_total", + "Actual API calls made (excludes cache hits)", + {"scanner"} -- scanner: virustotal, sentinelone + ) + + -- API calls saved by caching/locking + metrics.api_calls_saved = metrics.prometheus:counter( + "api_calls_saved_total", + "API calls saved by caching and atomic locking", + {"scanner"} + ) + + return true +end + +-------------------------------------------------------------------------------- +-- METRIC RECORDING FUNCTIONS +-------------------------------------------------------------------------------- + +-- Record scanner execution +--- @param scanner string Scanner name (clamav, virustotal, sentinelone) +--- @param result string Result type (clean, malicious, error, api_circuit_breaker) +--- @param latency_ms number Optional latency in milliseconds +function metrics.record_scan(scanner, result, latency_ms) + if not metrics.enabled then return end + + -- Increment total scans counter + metrics.scan_total:inc(1, {scanner}) + + -- Increment result-specific counter + metrics.scan_result:inc(1, {scanner, result}) + + -- Record latency if provided + if latency_ms and latency_ms > 0 then + metrics.scan_latency:observe(latency_ms / 1000, {scanner}) + end +end + +-- Record cache hit or miss +--- @param scanner string Scanner name +--- @param hit boolean True for cache hit, false for cache miss +function metrics.record_cache(scanner, hit) + if not metrics.enabled then return end + + if hit then + metrics.cache_hits:inc(1, {scanner}) + -- Cache hit = API call saved (for API scanners) + if scanner ~= "clamav" then + metrics.api_calls_saved:inc(1, {scanner}) + end + else + metrics.cache_misses:inc(1, {scanner}) + end +end + +-- Record lock acquisition +--- @param scanner string Scanner name +function metrics.record_lock_acquired(scanner) + if not metrics.enabled then return end + + metrics.lock_acquired:inc(1, {scanner}) +end + +-- Record lock wait (thread waited for concurrent scan/query result) +--- @param scanner string Scanner name +--- @param timeout boolean True if wait timed out, false if got result +function metrics.record_lock_wait(scanner, timeout) + if not metrics.enabled then return end + + if timeout then + -- Timeout - fell back to independent scan/query + metrics.lock_timeout:inc(1, {scanner}) + else + -- Successful wait - got result from concurrent scan + metrics.lock_wait:inc(1, {scanner}) + -- Wait succeeded = API call/scan saved + metrics.api_calls_saved:inc(1, {scanner}) + end +end + +-- Record actual API call (not from cache) +-- Call this when actually making an HTTP request to an API +--- @param scanner string Scanner name (virustotal, sentinelone) +function metrics.record_api_call(scanner) + if not metrics.enabled then return end + + metrics.api_calls_actual:inc(1, {scanner}) +end + +-- Record file processed +--- @param blocked boolean True if file was blocked (malware), false if allowed (clean) +function metrics.record_file(blocked) + if not metrics.enabled then return end + + metrics.files_total:inc(1) + + if blocked then + metrics.files_blocked:inc(1) + else + metrics.files_allowed:inc(1) + end +end + +-------------------------------------------------------------------------------- +-- METRICS EXPORT +-------------------------------------------------------------------------------- + +-- Collect and return Prometheus-formatted metrics +-- Call this from /metrics endpoint handler +--- @return string metrics Prometheus text format metrics +function metrics.collect() + if not metrics.enabled or not metrics.prometheus then + return "# Metrics not enabled - install lua-resty-prometheus\n" + end + + return metrics.prometheus:collect() +end + +return metrics diff --git a/malware-scan/malware_scan_multipart_full_scan.lua b/malware-scan/malware_scan_multipart_full_scan.lua new file mode 100644 index 0000000..195329f --- /dev/null +++ b/malware-scan/malware_scan_multipart_full_scan.lua @@ -0,0 +1,462 @@ +-- BunkerWeb Malware Scan - Multipart Full Scan Module +-- Handles multipart parsing and per-file scanning when resume mode is enabled +-- Only activates when MALWARE_SCAN_RESUME_SCAN_ON_FIRST_HIT=yes +-- Provides comprehensive threat intelligence by scanning all files before blocking + +local logger = require("malware_scan_logger") +local ngx = ngx +local get_deny_status = require("bunkerweb.utils").get_deny_status +local utils = require("malware_scan_utils") +local orchestrator = require("malware_scan_orchestrator") + +-- Module table +local multipart_full_scan = {} + +-- Module version +multipart_full_scan.VERSION = "0.8.0" + +-- Module dependencies (set via setters) +local cache = nil +local file_ops = nil +local clamav = nil +local virustotal = nil +local sentinelone = nil +local malwarebazaar = nil +local webhook = nil + +-- Setter functions for module dependencies +function multipart_full_scan.set_cache_module(cache_module) + cache = cache_module +end + +function multipart_full_scan.set_file_ops_module(file_ops_module) + file_ops = file_ops_module +end + +function multipart_full_scan.set_clamav_module(clamav_module) + clamav = clamav_module +end + +function multipart_full_scan.set_virustotal_module(vt_module) + virustotal = vt_module +end + +function multipart_full_scan.set_sentinelone_module(s1_module) + sentinelone = s1_module +end + +function multipart_full_scan.set_malwarebazaar_module(mbz_module) + malwarebazaar = mbz_module +end + +function multipart_full_scan.set_webhook_module(webhook_module) + webhook = webhook_module +end + +-- No longer need internal parsing functions - we'll use file_ops.multipart_parse() + +-- Scan a single file with all enabled scanners +-- Scanner execution order optimized for performance: +-- 1. MalwareBazaar (fastest: ~1ms Redis lookup, 1M+ hashes, blocks immediately) +-- 2. ClamAV (local scan: 100-1000ms, no API costs) +-- 3. VirusTotal (API call: 100-500ms, 70+ engines, paid) +-- 4. SentinelOne (API call: 100-500ms, enterprise AV, paid) +--- @return table detections Table of detections (empty if clean), keyed by scanner name +local function multipart_full_scan_scan_individual_file(plugin, temp_file, file_checksum, file_size, filename, enabled_scanners) + local file_detections = {} + + -- Sanitize filename for safe logging (prevent log injection) + local safe_filename = utils.log_sanitize(filename) + + -- 1. Scan with MalwareBazaar if enabled (FASTEST - run first!) + logger.log_error( string.format("[v%s] [MULTIPART_SCAN_DEBUG] MalwareBazaar check: enabled=%s, checksum=%s, checksum_empty=%s", + multipart_full_scan.VERSION, tostring(enabled_scanners.malwarebazaar), tostring(file_checksum ~= nil), tostring(file_checksum == "empty_file"))) + + if enabled_scanners.malwarebazaar and file_checksum and file_checksum ~= "empty_file" then + logger.log_error( string.format("[v%s] [MULTIPART_SCAN_DEBUG] Entering MalwareBazaar scan for %s", multipart_full_scan.VERSION, safe_filename)) + + -- Check cache first + local mbz_cached = nil + if cache then + local cache_ok, cached_result, cached_size = cache.malwarebazaar_check(plugin, file_checksum, file_size) + if cache_ok and cached_result and orchestrator.result_is_malware(cached_result) then + logger.log_error( "[MULTIPART] MalwareBazaar cache hit for " .. safe_filename .. ": " .. cached_result) + file_detections["MalwareBazaar (cached)"] = cached_result + mbz_cached = cached_result + end + end + + -- If not in cache, query MalwareBazaar + if not mbz_cached and malwarebazaar then + logger.log_error( string.format("[v%s] [MULTIPART_SCAN_DEBUG] Calling malwarebazaar.check_malwarebazaar for %s", multipart_full_scan.VERSION, safe_filename)) + local mbz_ok, mbz_result = malwarebazaar.check_malwarebazaar(plugin, file_checksum, file_size) + logger.log_error( string.format("[v%s] [MULTIPART_SCAN_DEBUG] MalwareBazaar result: ok=%s, result=%s", multipart_full_scan.VERSION, tostring(mbz_ok), tostring(mbz_result))) + + if mbz_ok and mbz_result and orchestrator.result_is_malware(mbz_result) then + logger.log_error( "[MULTIPART] MalwareBazaar detected malware in " .. safe_filename .. ": " .. mbz_result) + file_detections["MalwareBazaar"] = mbz_result + + -- Cache the result + if cache then + cache.malwarebazaar_add(plugin, file_checksum, mbz_result, file_size) + end + end + elseif not mbz_cached and not malwarebazaar then + logger.log_error( string.format("[v%s] [MULTIPART_SCAN_DEBUG] MalwareBazaar module not loaded!", multipart_full_scan.VERSION)) + end + else + logger.log_error( string.format("[v%s] [MULTIPART_SCAN_DEBUG] Skipping MalwareBazaar: condition failed", multipart_full_scan.VERSION)) + end + + -- 2. Scan with ClamAV if enabled (local, no API costs) + if enabled_scanners.clamav then + -- Check cache first + local file_cached = nil + if file_checksum and file_checksum ~= "empty_file" and cache then + local cache_ok, cached_result, cached_size = cache.clamav_check(plugin, file_checksum, file_size) + if cache_ok and cached_result and orchestrator.result_is_malware(cached_result) then + logger.log_error( "[MULTIPART] ClamAV cache hit for " .. safe_filename .. ": " .. cached_result) + file_detections["ClamAV (cached)"] = cached_result + file_cached = cached_result + end + end + + -- If not in cache, scan with ClamAV + if not file_cached and clamav then + local scan_ok, scan_result = clamav.scan_file_instream(plugin, temp_file) + if scan_ok and scan_result and orchestrator.result_is_malware(scan_result) then + logger.log_error( "[MULTIPART] ClamAV detected malware in " .. safe_filename .. ": " .. scan_result) + file_detections["ClamAV"] = scan_result + + -- Cache the result + if file_checksum and file_checksum ~= "empty_file" and cache then + cache.clamav_add(plugin, file_checksum, scan_result, file_size) + end + end + end + end + + -- 3. Scan with VirusTotal if enabled (paid API) + if enabled_scanners.virustotal and file_checksum and file_checksum ~= "empty_file" then + -- Check cache first + local vt_cached = nil + if cache then + local cache_ok, cached_result = cache.is_in_virustotal_cache(plugin, file_checksum) + if cache_ok and cached_result and orchestrator.result_is_malware(cached_result) then + logger.log_error( "[MULTIPART] VirusTotal cache hit for " .. safe_filename .. ": " .. cached_result) + file_detections["VirusTotal (cached)"] = cached_result + vt_cached = cached_result + end + end + + -- If not in cache, query VirusTotal + if not vt_cached and virustotal then + local vt_ok, vt_result = virustotal.check_hash(plugin, file_checksum, file_size) + if vt_ok and vt_result and orchestrator.result_is_malware(vt_result) then + logger.log_error( "[MULTIPART] VirusTotal detected malware in " .. safe_filename .. ": " .. vt_result) + file_detections["VirusTotal"] = vt_result + + -- Cache the result + if cache then + cache.virustotal_add(plugin, file_checksum, vt_result) + end + end + end + end + + -- 4. Scan with SentinelOne if enabled (paid API) + if enabled_scanners.sentinelone and file_checksum and file_checksum ~= "empty_file" then + -- Check cache first + local s1_cached = nil + if cache then + local cache_ok, cached_result = cache.is_in_sentinelone_cache(plugin, file_checksum) + if cache_ok and cached_result and orchestrator.result_is_malware(cached_result) then + logger.log_error( "[MULTIPART] SentinelOne cache hit for " .. safe_filename .. ": " .. cached_result) + file_detections["SentinelOne (cached)"] = cached_result + s1_cached = cached_result + end + end + + -- If not in cache, query SentinelOne + if not s1_cached and sentinelone then + local s1_ok, s1_result = sentinelone.check_hash(plugin, file_checksum, file_size) + if s1_ok and s1_result and orchestrator.result_is_malware(s1_result) then + logger.log_error( "[MULTIPART] SentinelOne detected malware in " .. safe_filename .. ": " .. s1_result) + file_detections["SentinelOne"] = s1_result + + -- Cache the result + if cache then + cache.sentinelone_add(plugin, file_checksum, s1_result) + end + end + end + end + + return file_detections +end + +-- Main function: Parse multipart upload and scan each individual file +-- Called from main plugin when resume mode is enabled +--- @return boolean success True to continue, false to block +--- @return string|nil message Block message if success=false +--- @return number|nil status HTTP status code if blocking +--- @return table|nil data Block response data if blocking +function multipart_full_scan.scan_multipart_files(plugin, body_file, checksum, detections, enabled_scanners) + -- Check if multipart scanning is possible + if not plugin.ctx.bw.http_content_type or + not plugin.ctx.bw.http_content_type:match("multipart/form%-data") then + logger.log_error( "[MULTIPART] Not multipart/form-data, skipping per-file scan") + return true, nil, nil, nil + end + + -- Check if file_ops module is available + if not file_ops then + logger.log_error( "[MULTIPART] file_ops module not loaded, cannot parse multipart uploads") + return true, nil, nil, nil + end + + logger.log_error( "[MULTIPART] Starting per-file scanning in resume mode") + + -- Use file_ops to parse multipart upload (includes security validation) + local parse_ok, files_or_err, attack_indicator = file_ops.multipart_parse(plugin, body_file, plugin.ctx.bw.http_content_type) + if not parse_ok then + -- Check if this is a security attack (malicious filename detected) + if files_or_err and files_or_err:match("malicious filename detected") then + logger.log_error( "[MULTIPART] SECURITY ALERT: " .. files_or_err) + -- Return error to trigger immediate blocking + return false, + files_or_err, + get_deny_status(), + { + id = "malicious_filename_attack", + file = body_file, + error = files_or_err + } + end + + logger.log_error( "[MULTIPART] Failed to parse multipart upload: " .. tostring(files_or_err)) + return true, nil, nil, nil + end + + local files = files_or_err + logger.log_error( "[MULTIPART] Parsed " .. #files .. " file(s) from multipart upload") + + -- Track if filename attack was detected (will block after full scan) + local filename_attack_detected = (attack_indicator == "filename_attack_detected") + + -- Track per-file detections + local per_file_detections = {} + local files_scanned = 0 + + -- Scan each extracted file + for _, file_info in ipairs(files) do + local filename = file_info.filename + local temp_file = file_info.filepath + local file_size = file_info.size + local file_type = file_info.file_type or "unknown" + + -- Sanitize filename for logging (prevent log injection) + local safe_filename = utils.log_sanitize(filename) + logger.log_error( "[MULTIPART] Processing file: " .. safe_filename .. " (size: " .. file_size .. " bytes, type: " .. file_type .. ")") + files_scanned = files_scanned + 1 + + -- Calculate SHA256 for this file + local file_checksum, file_checksum_ok + if file_ops then + file_checksum_ok, file_checksum = file_ops.hash_calculate(plugin, temp_file) + else + file_checksum_ok, file_checksum = false, nil + end + + if file_checksum_ok and file_checksum and file_checksum ~= "empty_file" then + logger.log_error( "[MULTIPART] File SHA256: " .. file_checksum) + end + + -- DEBUG: Log checksum status and enabled scanners for MalwareBazaar + logger.log_error( string.format("[v%s] [MULTIPART_SCAN_DEBUG] file_checksum_ok=%s, file_checksum=%s, enabled_scanners.malwarebazaar=%s", + multipart_full_scan.VERSION, tostring(file_checksum_ok), tostring(file_checksum), tostring(enabled_scanners.malwarebazaar))) + + -- Scan this file with all enabled scanners + local file_detections = multipart_full_scan_scan_individual_file( + plugin, + temp_file, + file_checksum, + file_size, + filename, + enabled_scanners + ) + + -- Store per-file detections if any + if next(file_detections) ~= nil then + per_file_detections[filename] = { + checksum = file_checksum, + detections = file_detections, + size = file_size, + file_type = file_type + } + end + + -- Cleanup temp file + if file_ops then + file_ops.file_cleanup(plugin, temp_file) + else + os.remove(temp_file) + end + end + + logger.log_error( "[MULTIPART] Scanned " .. files_scanned .. " individual files") + + -- If filename attack or per-file detections exist, return blocking response + if filename_attack_detected or next(per_file_detections) ~= nil then + if filename_attack_detected then + logger.log_error( "[MULTIPART] Filename injection attack detected during parsing - blocking request") + end + if next(per_file_detections) ~= nil then + logger.log_error( "[MULTIPART] Malware detected in individual files - blocking request") + end + + -- Count total infected files for webhook truncation logic + local infected_file_count = 0 + for _ in pairs(per_file_detections) do + infected_file_count = infected_file_count + 1 + end + + -- Merge per-file detections into main detections table for webhook + -- Limit to first 5 files to prevent webhook message size issues + local MAX_FILES_IN_WEBHOOK = 5 + local files_added = 0 + local truncated = false + + for fname, fdata in pairs(per_file_detections) do + if files_added < MAX_FILES_IN_WEBHOOK then + for scanner, detection in pairs(fdata.detections) do + -- Create composite key with filename (sanitize for safety) + local safe_fname = utils.log_sanitize(fname) + local key = scanner .. " (file: " .. safe_fname .. ")" + detections[key] = detection + end + files_added = files_added + 1 + else + truncated = true + -- Don't break - continue to log all detections to server logs below + end + end + + -- Log ALL detections to server logs (not truncated) + local report_msg + if filename_attack_detected and infected_file_count > 0 then + report_msg = string.format( + "[MULTIPART] Full detection report: %d file(s) with content malware detected + filename injection attack(s)", + infected_file_count + ) + elseif filename_attack_detected then + report_msg = "[MULTIPART] Full detection report: filename injection attack(s) detected, no content malware found" + else + report_msg = string.format( + "[MULTIPART] Full detection report: %d file(s) with malware detected", + infected_file_count + ) + end + logger.log_error( report_msg) + for fname, fdata in pairs(per_file_detections) do + local safe_fname = utils.log_sanitize(fname) + for scanner, detection in pairs(fdata.detections) do + logger.log_error( string.format( + "[MULTIPART] - %s: %s = %s", + safe_fname, + scanner, + detection + )) + end + end + + -- Add truncation notice to webhook if we hit the limit + if truncated then + local remaining = infected_file_count - MAX_FILES_IN_WEBHOOK + detections["NOTICE"] = string.format( + "Showing first %d of %d infected files. %d more file(s) with malware detected. Check server logs for complete details.", + MAX_FILES_IN_WEBHOOK, + infected_file_count, + remaining + ) + logger.log_error( string.format( + "[MULTIPART] Webhook truncated: showing %d/%d infected files in notification (limit: %d per webhook)", + MAX_FILES_IN_WEBHOOK, + infected_file_count, + MAX_FILES_IN_WEBHOOK + )) + end + + -- Add filename attack to detections for webhook visibility + if filename_attack_detected then + detections["๐Ÿšจ Filename Injection Attack"] = "Malicious characters detected in filename (null byte or path traversal attempt)" + end + + -- Send ONE aggregated webhook notification at end (not per-file) + if checksum and next(detections) ~= nil then + logger.log_error( string.format( + "[MULTIPART] Sending single aggregated webhook for %d infected file(s)", + infected_file_count + )) + webhook.notify(webhook, plugin, checksum, detections) + + + + + + end + + -- Cleanup original body file + local cleanup_enabled = plugin.variables["MALWARE_SCAN_CLEANUP_FILES"] + if cleanup_enabled ~= "no" then + local cleanup_ok, cleanup_err + if file_ops then + cleanup_ok, cleanup_err = file_ops.file_cleanup(plugin, body_file) + else + cleanup_ok, cleanup_err = true, nil + end + if not cleanup_ok then + logger.log_error( "failed to cleanup temp file " .. body_file .. ": " .. cleanup_err) + end + end + + -- Build detailed response with per-file information + local infected_files = {} + for fname, fdata in pairs(per_file_detections) do + table.insert(infected_files, fname) + end + + -- Build appropriate error message based on detection type + local error_msg + local error_id + if filename_attack_detected and next(per_file_detections) ~= nil then + error_msg = "filename injection attack and malware detected in multipart files: " .. table.concat(infected_files, ", ") + error_id = "filename_attack_and_malware_detected" + elseif filename_attack_detected then + error_msg = "filename injection attack detected in multipart upload" + error_id = "filename_attack_detected" + else + error_msg = "malware detected in multipart files: " .. table.concat(infected_files, ", ") + error_id = "malware_detected_multipart" + end + + return false, + error_msg, + get_deny_status(), + { + id = error_id, + file = body_file, + checksum = checksum, + per_file_detections = per_file_detections, + infected_files = infected_files, + files_scanned = files_scanned, + filename_attack_detected = filename_attack_detected + } + end + + -- No detections in per-file scan, continue normally + return true, nil, nil, nil +end + +-- Return module +return multipart_full_scan diff --git a/malware-scan/malware_scan_orchestrator.lua b/malware-scan/malware_scan_orchestrator.lua new file mode 100644 index 0000000..3150e66 --- /dev/null +++ b/malware-scan/malware_scan_orchestrator.lua @@ -0,0 +1,1539 @@ +-- BunkerWeb Malware Scan - Scanner Orchestrator Module +-- Centralized scanner coordination and detection handling +-- Eliminates code duplication for cache checks, webhooks, and cleanup + +local logger = require("malware_scan_logger") +local utils = require("malware_scan_utils") + +-- OpenResty log level constants (with fallback values for module load time) +local ERR = ngx and ngx.ERR or 3 +local WARN = ngx and ngx.WARN or 4 +local NOTICE = ngx and ngx.NOTICE or 5 +local INFO = ngx and ngx.INFO or 6 +local DEBUG = ngx and ngx.DEBUG or 7 + +-- Try to load UI metrics module (optional) +local ui_metrics +local ui_metrics_ok = pcall(function() + ui_metrics = require("malware_scan_ui_metrics") +end) + +-- Module table +local orchestrator = {} + +-- Module version +orchestrator.VERSION = "0.8.0" + +-- ============================================================================ +-- SCANNER STATE MANAGEMENT +-- ============================================================================ + +--- @class ScannerState +--- @field clamav boolean Whether ClamAV scanner is enabled +--- @field virustotal boolean Whether VirusTotal API is enabled +--- @field sentinelone boolean Whether SentinelOne API is enabled +--- @field malwarebazaar boolean Whether MalwareBazaar hash lookup is enabled +--- @field resume_on_hit boolean Whether to resume scan on first detection + +--- @class ScanContext +--- @field scanner_state ScannerState Scanner state from scanner_get_state() +--- @field body_file string Path to uploaded file +--- @field checksum string|nil SHA256 checksum (may be nil initially) +--- @field checksum_ok boolean Whether checksum calculation succeeded +--- @field file_size number File size in bytes +--- @field detections table Table of detections found + +--- @class ScanLimits +--- @field scanner_state ScannerState Scanner state with enabled flags +--- @field scanner_state_for_files ScannerState Original scanner state for file scanning (saved before body scan disable) +--- @field clamav_skipped_reason string|nil Reason ClamAV was skipped (if applicable) +--- @field skip_full_body_scan boolean Whether to skip full body scan + +--- @class ScanResult +--- @field detected string|nil Detection result ("clean" or signature) +--- @field checksum string|nil Final SHA256 checksum after scanning +--- @field checksum_ok boolean Whether checksum is valid +--- @field clamav_actually_scanned boolean Whether ClamAV performed a scan + +-- Create scanner state tracker. +-- Tracks which scanners are enabled and provides unified interface. +-- +--- @param plugin Plugin The plugin instance +--- @return ScannerState state Scanner state table with enabled flags +function orchestrator.scanner_get_state(plugin) + return { + clamav = plugin.variables["USE_MALWARE_SCANNER"] == "yes", + virustotal = plugin.variables["MALWARE_SCAN_API_USE_VIRUSTOTAL"] == "yes", + sentinelone = plugin.variables["MALWARE_SCAN_API_USE_SENTINELONE"] == "yes", + malwarebazaar = plugin.variables["MALWARE_SCAN_MALWAREBAZAAR_HASH_LOOKUP"] == "yes", + resume_on_hit = plugin.variables["MALWARE_SCAN_RESUME_SCAN_ON_FIRST_HIT"] == "yes" + } +end + +-- Check if any scanner is enabled. +-- +--- @param state ScannerState Scanner state from scanner_get_state() +--- @return boolean any_enabled True if any scanner is enabled, false otherwise +function orchestrator.scanner_any_enabled(state) + return state.clamav or state.virustotal or state.sentinelone or state.malwarebazaar +end + +-- Check if a scanner result indicates malware detection. +-- Returns false for "clean", "api_error", "api_circuit_breaker" (fail-open). +-- Returns true for any other value (malware detection). +-- +--- @param result Scanner result string +--- @return boolean is_malware True if malware detected, false if clean or API error +function orchestrator.result_is_malware(result) + if not result then + return false + end + + -- Non-blocking statuses (fail-open for availability) + if result == "clean" or result == "api_error" or result == "api_circuit_breaker" then + return false + end + + -- Any other result is a malware detection + return true +end + +-- Get list of enabled scanner names. +-- +--- @param state ScannerState Scanner state from scanner_get_state() +--- @return table scanners Array of enabled scanner names +function orchestrator.scanner_get_enabled(state) + local scanners = {} + if state.malwarebazaar then table.insert(scanners, "MalwareBazaar") end + if state.clamav then table.insert(scanners, "ClamAV") end + if state.virustotal then table.insert(scanners, "VirusTotal") end + if state.sentinelone then table.insert(scanners, "SentinelOne") end + return scanners +end + +-- ============================================================================ +-- DETECTION HANDLING +-- ============================================================================ + +-- Handle malware detection (webhook, cleanup, blocking). +-- Centralized detection handling to eliminate code duplication. +-- +-- This function: +-- 1. Sends webhook notification if configured +-- 2. Cleans up uploaded file if enabled +-- 3. Returns blocking response with detection details +-- +--- @param plugin Plugin The plugin instance +--- @param checksum string|nil File SHA256 checksum +--- @param detections table Table of detections (scanner_name => detection_string) +--- @param body_file string Path to uploaded file +--- @param scanner_name string Name of scanner that detected malware +--- @param detection string Detection signature/result +--- @param webhook table|nil Webhook module (or nil if not loaded) +--- @param file_ops table|nil File operations module (or nil if not loaded) +--- @param get_deny_status function Function to get deny status code +--- @return table result plugin:ret() result for blocking the request +function orchestrator.detection_handle(plugin, checksum, detections, body_file, scanner_name, detection, webhook, file_ops, get_deny_status) + local ERR = ngx and ngx.ERR or 3 -- Fallback for tests + + -- Send webhook notification with all detections (centralized error handling) + if webhook then + webhook.notify(webhook, plugin, checksum, detections) + end + + -- Track malware detection metrics + if ui_metrics_ok and ui_metrics then + ui_metrics.track_detection_complete(plugin, scanner_name, detection, checksum, detections) + end + + -- Cleanup uploaded file if enabled + local cleanup_enabled = plugin.variables["MALWARE_SCAN_CLEANUP_FILES"] + if cleanup_enabled ~= "no" then + local cleanup_ok, cleanup_err + if file_ops then + cleanup_ok, cleanup_err = file_ops.file_cleanup(plugin, body_file) + else + cleanup_ok, cleanup_err = true, nil + end + if not cleanup_ok then + logger.log_error( "failed to cleanup temp file " .. body_file .. ": " .. cleanup_err) + end + end + + -- Return blocking response with detection details + return plugin:ret( + true, + "malware detected by " .. scanner_name .. " (SHA256: " .. checksum .. "): " .. detection, + get_deny_status(), + nil, + { + id = "malware_detected_" .. scanner_name:lower():gsub("[^a-z0-9]", "_"), + file = body_file, + checksum = checksum, + detection = detection, + scanner = scanner_name + } + ) +end + +-- ============================================================================ +-- CACHE CHECKING +-- ============================================================================ + +-- Check scanner cache for malware detection. +-- Unified cache checking logic for all scanners. +-- +-- Returns: +-- - cache_hit: true if cache was checked successfully +-- - result: "clean", "malicious", or nil (cache miss/error) +-- - detection: detection string if malicious, nil otherwise +-- +--- @param plugin Plugin The plugin instance +--- @param cache table|nil Cache module (or nil if not loaded) +--- @param scanner_name string Scanner name ("clamav", "virustotal", "sentinelone") +--- @param checksum string File SHA256 checksum +--- @param file_size number File size in bytes +--- @return boolean cache_hit True if cache was checked successfully +--- @return string|nil result "clean", "malicious", or nil (cache miss/error) +--- @return string|nil detection Detection string if malicious, nil otherwise +function orchestrator.cache_check(plugin, cache, scanner_name, checksum, file_size) + if not cache then + return true, nil, nil -- No cache module - treat as cache miss + end + + -- Map scanner name to cache function + local cache_functions = { + clamav = cache.clamav_check, + virustotal = cache.virustotal_check, + sentinelone = cache.sentinelone_check + } + + local cache_fn = cache_functions[scanner_name] + if not cache_fn then + return true, nil, nil -- Unknown scanner - treat as cache miss + end + + -- Check cache + local cache_ok, cached, cached_size = cache_fn(plugin, checksum, file_size) + + if not cache_ok then + return false, nil, nil -- Cache check failed + end + + if not cached then + return true, nil, nil -- Cache miss + end + + if cached == "clean" then + return true, "clean", nil -- Clean file + else + return true, "malicious", cached -- Malicious file with detection + end +end + +-- ============================================================================ +-- ORCHESTRATION +-- ============================================================================ + +-- Check all scanner caches and handle detections. +-- Centralized orchestration to eliminate code duplication. +-- +-- This function checks caches for all enabled scanners and handles: +-- - Cache hits (clean or malicious) +-- - Detection logging +-- - Webhook notifications +-- - Resume mode logic +-- - Immediate blocking or continued scanning +-- +-- Returns: +-- - blocked: true if request was blocked, false if should continue +-- - detections: table of all detections found +-- - clamav_can_skip: true if ClamAV can be skipped (malicious cache hit) +-- - scanners_to_disable: table of scanners that found cache hits +-- +--- @param plugin Plugin The plugin instance +--- @param state ScannerState Scanner state from scanner_get_state() +--- @param checksum string File SHA256 checksum +--- @param file_size number File size in bytes +--- @param body_file string Path to uploaded file +--- @param cache table|nil Cache module (or nil if not loaded) +--- @param webhook table|nil Webhook module (or nil if not loaded) +--- @param file_ops table|nil File operations module (or nil if not loaded) +--- @param get_deny_status function Function to get deny status code +--- @return boolean blocked True if request was blocked +--- @return table detections Table of all detections found +--- @return boolean clamav_can_skip True if ClamAV can be skipped +--- @return table scanners_to_disable Table of scanners that found cache hits +--- @return table|nil block_result Blocking result from detection_handle if blocked +function orchestrator.cache_check_all(plugin, state, checksum, file_size, body_file, cache, webhook, file_ops, get_deny_status) + local ERR = ngx and ngx.ERR or 3 + local detections = {} + local clamav_can_skip = false + local scanners_to_disable = {} + + -- Scanner cache check configuration + local scanners_to_check = { + {name = "clamav", display = "ClamAV", enabled = state.clamav}, + {name = "virustotal", display = "VirusTotal", enabled = state.virustotal}, + {name = "sentinelone", display = "SentinelOne", enabled = state.sentinelone} + } + + -- Check each scanner's cache + for _, scanner in ipairs(scanners_to_check) do + if scanner.enabled then + local cache_hit, result, detection = orchestrator.cache_check( + plugin, cache, scanner.name, checksum, file_size + ) + + if cache_hit and result then + if result == "clean" then + -- Cache shows clean - skip this scanner + logger.log_error( "[CACHE_HIT] " .. scanner.display .. " cache shows clean - skipping scan") + scanners_to_disable[scanner.name] = true + elseif result == "malicious" and detection then + -- Cache shows malicious - log detection + logger.log_error( "[CACHE_HIT] " .. scanner.display .. " cache shows malicious") + logger.log_error( "MALWARE DETECTED BY " .. scanner.display:upper() .. " (cached)") + logger.log_error( scanner.display .. " detection: " .. plugin:sanitize_external_data(detection) .. " (SHA256: " .. checksum .. ")") + + detections[scanner.display .. " (cached)"] = detection + clamav_can_skip = true -- Can skip ClamAV since we know it's malicious + + -- Check if we should block immediately or continue scanning + if not state.resume_on_hit then + -- Default behavior: block immediately + return true, detections, clamav_can_skip, scanners_to_disable, + orchestrator.detection_handle( + plugin, checksum, detections, body_file, + scanner.display .. " (cached)", detection, + webhook, file_ops, get_deny_status + ) + else + -- Resume mode: continue to other scanners + logger.log_error( "[RESUME_MODE] Malware detected by " .. scanner.display .. " cache, continuing to other scanners") + end + end + end + end + end + + -- No blocking detection found or resume mode active + return false, detections, clamav_can_skip, scanners_to_disable, nil +end + +-- Handle detection with custom message and data structure +-- Provides the same webhook + cleanup + return pattern as detection_handle(), +-- but allows custom message and return data for complex multi-scanner scenarios. +-- +-- Use this for: +-- - Multi-scanner detections (e.g., both ClamAV and VirusTotal detected) +-- - Special case messages (e.g., "ClamAV detected, VirusTotal clean") +-- - Custom data structures with scanner-specific field names +-- +--- @param plugin Plugin The plugin instance +--- @param checksum string|nil File SHA256 checksum (for webhook, may be nil if not yet calculated) +--- @param detections table Table of all detections (for webhook) +--- @param body_file string Path to uploaded file (for cleanup) +--- @param message string Custom message string for return +--- @param return_data table Custom data table for return (must include: id, file, and scanner-specific fields) +--- @param webhook table|nil Webhook module (or nil if not loaded) +--- @param file_ops table|nil File operations module (or nil if not loaded) +--- @param get_deny_status function Function to get deny status code +--- @return table result plugin:ret() result (5 values: blocked, message, status, nil, data) +function orchestrator.detection_handle_custom(plugin, checksum, detections, body_file, + message, return_data, + webhook, file_ops, get_deny_status) + local ERR = ngx and ngx.ERR or 3 + + -- Send webhook notification (centralized error handling) + if webhook then + webhook.notify(webhook, plugin, checksum, detections) + end + + -- Cleanup uploaded file if enabled + local cleanup_enabled = plugin.variables["MALWARE_SCAN_CLEANUP_FILES"] + if cleanup_enabled ~= "no" then + local cleanup_ok, cleanup_err + if file_ops then + cleanup_ok, cleanup_err = file_ops.file_cleanup(plugin, body_file) + else + cleanup_ok, cleanup_err = true, nil + end + if not cleanup_ok then + logger.log_error( "failed to cleanup temp file " .. body_file .. ": " .. cleanup_err) + end + end + + -- Return blocking response with custom message and data + return plugin:ret( + true, + message, + get_deny_status(), + nil, + return_data + ) +end + +-- ============================================================================ +-- SCANNER INVOCATION ORCHESTRATION +-- ============================================================================ + +-- Orchestrate body file scanning across all enabled scanners. +-- Handles MalwareBazaar, ClamAV, VirusTotal, and SentinelOne scanning with proper +-- cache management, detection sharing, and resume mode logic. +-- +-- This function centralizes the complex scanner invocation flow that was previously +-- scattered across ~500 lines in the main access() function. +-- +--- @param plugin Plugin The plugin instance +--- @param body_file string Path to uploaded file +--- @param checksum string|nil File SHA256 checksum (or nil if not calculated yet) +--- @param checksum_ok boolean Whether checksum calculation succeeded +--- @param file_size number File size in bytes +--- @param scanner_state ScannerState Scanner state table from scanner_get_state() +--- @param detections table Table of existing detections (updated in-place) +--- @param clamav table|nil ClamAV module (or nil) +--- @param virustotal table|nil VirusTotal module (or nil) +--- @param sentinelone table|nil SentinelOne module (or nil) +--- @param malwarebazaar table|nil MalwareBazaar module (or nil) +--- @param cache table|nil Cache module (or nil) +--- @param webhook table|nil Webhook module (or nil) +--- @param file_ops table|nil File operations module (or nil) +--- @param get_deny_status function Function to get HTTP deny status code +-- +--- @return table|nil blocking_result plugin:ret() result OR nil if clean/continue +--- @return string detected ClamAV detection result ("clean" or signature) +--- @return table scan_info Table with scan metadata +function orchestrator.body_scan(plugin, body_file, checksum, checksum_ok, file_size, + scanner_state, detections, + clamav, virustotal, sentinelone, malwarebazaar, + cache, webhook, file_ops, get_deny_status) + local ERR = ngx and ngx.ERR or 3 + + -- Extract scanner states for readability + local clamav_enabled = scanner_state.clamav + local virustotal_enabled = scanner_state.virustotal + local sentinelone_enabled = scanner_state.sentinelone + local malwarebazaar_enabled = scanner_state.malwarebazaar + local resume_on_hit = scanner_state.resume_on_hit + + -- Tracking variables + local clamav_actually_scanned = false + local clamav_skipped_reason = nil + local detected = "clean" + local vt_detected = "clean" + + -- ======================================================================== + -- FILE TYPE DETECTION (GDPR Protection) + -- ======================================================================== + + -- Detect file type for GDPR compliance + -- Only executable malware should be uploaded to public databases + -- Files containing user data (documents, images, databases) must NOT be uploaded + local file_type = "unknown" + local file_type_error = nil + if file_ops then + file_type, file_type_error = file_ops.file_get_type(plugin, body_file) + if file_type then + plugin:log_debug("File type detected: " .. file_type) + else + logger.log_error( "Could not detect file type: " .. (file_type_error or "unknown error")) + file_type = "unknown" + end + end + + -- ======================================================================== + -- CLAMAV SCANNER (with atomic locking to prevent duplicate scans) + -- ======================================================================== + + if clamav_enabled then + -- Try to acquire lock for this file (prevents concurrent duplicate scans) + local should_scan = true + local lock_acquired = false + + if checksum_ok and checksum and checksum ~= "empty_file" and cache then + lock_acquired = cache.clamav_try_lock(plugin, checksum) + + if not lock_acquired then + -- Another request is scanning this file - wait for result + plugin:log_debug("[v" .. orchestrator.VERSION .. "] [CACHE_LOCK] Another request is scanning, waiting for result...") + local wait_ok, wait_result, wait_size, wait_error = cache.clamav_wait_for_result(plugin, checksum, file_size) + + if wait_ok then + -- Got result from concurrent scan + plugin:log_debug(string.format("[v" .. orchestrator.VERSION .. "] [CACHE_LOCK] Got result from concurrent scan: %s", wait_result)) + detected = wait_result + should_scan = false -- Don't scan, use cached result + + -- Handle the waited result (same logic as if we scanned) + if wait_result == "error" then + -- Concurrent scan failed + logger.log_error( "Concurrent ClamAV scan failed: " .. (wait_error or "unknown")) + detected = "clean" -- Continue to other scanners + elseif orchestrator.result_is_malware(wait_result) then + -- Concurrent scan found malware + clamav_actually_scanned = false -- We didn't scan, but got result + -- Will be handled in malware detection logic below + else + -- Concurrent scan found clean + clamav_actually_scanned = false -- We didn't scan, but got result + end + else + -- Timeout waiting - fall back to scanning ourselves + logger.log_warn( "[CACHE_LOCK] Timeout waiting for result - scanning independently") + -- Try to acquire lock again (might be available now) + lock_acquired = cache.clamav_try_lock(plugin, checksum) + should_scan = true + end + end + end + + local ok = false + if should_scan then + clamav_actually_scanned = true + plugin:log_debug("Calling scan_file() with: " .. body_file) + + if clamav then + ok, detected = clamav.scan_file_instream(plugin, body_file) + else + logger.log_error( "ClamAV module not loaded") + ok, detected = false, "ClamAV module not loaded" + end + + plugin:log_debug("scan_file returned - ok: " .. tostring(ok) .. ", detected: " .. tostring(detected)) + end + + if not ok then + -- Check if the error is due to file size limit + if detected and detected:match("size limit") then + logger.log_error( "ClamAV scan failed due to size limit: " .. detected) + clamav_skipped_reason = "size limit exceeded" + clamav_actually_scanned = false + else + logger.log_error( "ClamAV scan failed: " .. detected) + -- Cache the error to prevent repeated failed scans (negative caching) + if checksum_ok and checksum and checksum ~= "empty_file" and cache then + cache.clamav_add(plugin, checksum, detected or "scan failed", file_size, true) -- is_error = true + plugin:log_debug("[v" .. orchestrator.VERSION .. "] [CACHE] Cached ClamAV error to prevent repeated failures") + end + end + -- Don't fail the request, continue to hash-based scanners if enabled + detected = "clean" + elseif detected and orchestrator.result_is_malware(detected) then + -- ClamAV detected malware + logger.log_error( "MALWARE DETECTED BY CLAMAV") + logger.log_error( "ClamAV signature: " .. plugin:sanitize_external_data(detected) .. " in file: " .. plugin:sanitize_external_data(body_file)) + + -- Cache the ClamAV detection result + if checksum_ok and checksum and checksum ~= "empty_file" and cache then + cache.clamav_add(plugin, checksum, detected, file_size) + end + + -- Export malware sample to local directory for forensic analysis (if enabled) + if file_ops and checksum_ok and checksum and checksum ~= "empty_file" then + local export_ok, export_err = file_ops.virusfile_export( + plugin, + body_file, + checksum, + "ClamAV", + detected, + file_type + ) + if not export_ok and export_err then + plugin:log_debug("[v" .. orchestrator.VERSION .. "] [EXPORT] Failed to export malware sample: " .. export_err) + end + end + + -- Check if we should skip cloud scanners and block immediately + local skip_others = plugin.variables["MALWARE_SCAN_SKIP_OTHERS_ON_CLAMAV_DETECT"] + local any_cloud_enabled = virustotal_enabled or sentinelone_enabled + if (skip_others == "yes" or not any_cloud_enabled) and not resume_on_hit then + -- Block immediately (default behavior) + logger.log_error( "BLOCKING REQUEST - skipping cloud scanner checks (faster response)") + if detected and orchestrator.result_is_malware(detected) then + detections["ClamAV"] = detected + end + + -- Use orchestrator detection handler + local deny_status = get_deny_status() + plugin:log_debug("Returning block status: " .. tostring(deny_status)) + + local blocking_result = orchestrator.detection_handle( + plugin, checksum, detections, body_file, + "ClamAV", detected, + webhook, file_ops, get_deny_status + ) + + -- Release lock before returning + if lock_acquired and checksum_ok and checksum and checksum ~= "empty_file" and cache then + cache.clamav_release_lock(plugin, checksum) + end + + return blocking_result, detected, { + clamav_actually_scanned = clamav_actually_scanned, + clamav_skipped_reason = clamav_skipped_reason, + vt_detected = vt_detected, + checksum = checksum, + checksum_ok = checksum_ok + } + else + -- Continue to cloud scanners for additional analysis + if resume_on_hit then + logger.log_error( "[RESUME_MODE] ClamAV detected malware, continuing to all remaining scanners") + else + logger.log_error( "ClamAV detected malware, continuing to cloud scanners for additional analysis") + end + -- Add ClamAV detection to detections table + if detected and orchestrator.result_is_malware(detected) then + detections["ClamAV"] = detected + end + -- Keep detected value, will be used in final decision + end + else + -- ClamAV reported clean - cache this result too + logger.log_error( "ClamAV scan: file is clean") + if checksum_ok and checksum and checksum ~= "empty_file" and cache then + cache.clamav_add(plugin, checksum, "clean", file_size) + end + end + + -- Release lock after scan completes (success, error, or clean) + if lock_acquired and checksum_ok and checksum and checksum ~= "empty_file" and cache then + cache.clamav_release_lock(plugin, checksum) + end + else + logger.log_error( "ClamAV scanning disabled, skipping to VirusTotal") + end + + -- ======================================================================== + -- HASH-BASED SCANNERS (MalwareBazaar, VirusTotal, SentinelOne) + -- ======================================================================== + + if malwarebazaar_enabled or virustotal_enabled or sentinelone_enabled then + -- Reuse checksum from early cache check if available, otherwise calculate now + if not checksum_ok or not checksum then + -- Check file size before calculating SHA256 + local file_size_for_hash = file_ops and file_ops.file_get_size(plugin, body_file) or nil + + -- Validate hash_max_size from config (ensure it's a positive number, default 64MB, max 10GB) + local hash_max_size = utils.validate_file_size( + tonumber(plugin.variables["MALWARE_SCAN_HASH_MAX_SIZE"]), + 10 * 1024 * 1024 * 1024 -- Max 10GB + ) + if hash_max_size == 0 then + hash_max_size = 67108864 -- Fallback to 64MB if validation failed + end + + -- Validate file_size_for_hash + if file_size_for_hash then + file_size_for_hash = utils.validate_file_size(file_size_for_hash, 10 * 1024 * 1024 * 1024) + end + + if file_size_for_hash and file_size_for_hash > 0 and file_size_for_hash > hash_max_size then + logger.log_error( "[SIZE_LIMIT] File size " .. file_size_for_hash .. " bytes exceeds hash scan limit " .. hash_max_size .. " bytes - skipping hash-based scanners") + checksum_ok = false + checksum = nil + else + logger.log_error( "Hash-based scanning enabled, calculating SHA256 checksum") + if file_ops then + checksum_ok, checksum = file_ops.hash_calculate(plugin, body_file) + else + checksum_ok, checksum = false, nil + end + end + else + logger.log_error( "Hash-based scanning enabled, reusing previously calculated SHA256") + end + + if checksum_ok then + logger.log_error( "file SHA256: " .. checksum) + + -- Check if file is empty (0 bytes) + if checksum == "empty_file" then + logger.log_error( "file is 0 bytes, skipping hash-based checks") + vt_detected = "clean" + else + -- ============================================================ + -- MALWAREBAZAAR HASH LOOKUP (check first - fastest: <1ms Redis lookup) + -- ============================================================ + + if malwarebazaar_enabled then + logger.log_error( "checking file against MalwareBazaar hash database (1M+ known malware hashes)") + local mbz_ok, mbz_result + if malwarebazaar then + mbz_ok, mbz_result = malwarebazaar.hash_check(plugin, checksum, file_size) + else + mbz_ok, mbz_result = true, "clean" + end + + if mbz_ok and orchestrator.result_is_malware(mbz_result) then + logger.log_error( "MALWARE DETECTED BY MALWAREBAZAAR") + logger.log_error( "MalwareBazaar detection: " .. plugin:sanitize_external_data(mbz_result) .. " (SHA256: " .. checksum .. ")") + detections["MalwareBazaar"] = mbz_result + + -- Share file with threat intelligence platforms if enabled + local share_ok, share_err = plugin:detection_share(body_file, checksum, "MalwareBazaar", virustotal, file_type) + if not share_ok then + logger.log_error( "[SHARE] Failed to share file: " .. share_err) + end + + -- Check if we should block immediately or continue scanning + if not resume_on_hit then + -- Default behavior: block immediately + logger.log_error( "BLOCKING REQUEST") + + -- Use orchestrator detection handler + local blocking_result = orchestrator.detection_handle( + plugin, checksum, detections, body_file, + "MalwareBazaar", mbz_result, + webhook, file_ops, get_deny_status + ) + + return blocking_result, detected, { + clamav_actually_scanned = clamav_actually_scanned, + clamav_skipped_reason = clamav_skipped_reason, + vt_detected = vt_detected, + checksum = checksum, + checksum_ok = checksum_ok + } + else + -- Resume mode: continue to remaining scanners + logger.log_error( "[RESUME_MODE] Malware detected by MalwareBazaar, continuing to remaining scanners") + end + else + logger.log_error( "MalwareBazaar hash database: hash not found (clean or unknown)") + end + end + + -- ============================================================ + -- VIRUSTOTAL SCANNER + -- ============================================================ + + if virustotal_enabled then + logger.log_error( "checking file against VirusTotal API") + local vt_ok, vt_result, vt_found + if virustotal then + vt_ok, vt_result, vt_found = virustotal.check_virustotal(plugin, body_file, checksum) + else + vt_ok, vt_result, vt_found = true, "clean", false + end + if vt_ok then + vt_detected = vt_result + + -- Check if we should upload unknown malware to VT + if not vt_found and detected and orchestrator.result_is_malware(detected) then + -- ClamAV detected malware, but VT doesn't have the file + if plugin.variables["MALWARE_SCAN_API_VIRUSTOTAL_UPLOAD_UNKNOWN"] == "yes" then + -- **GDPR PROTECTION**: Only upload executables, NEVER upload user data files + -- Check if file type is known AND safe to upload (not user data) + -- CRITICAL: Unknown types are NOT uploaded (GDPR-safe default) + local filetypes = require("malware_scan_filetypes") + local is_safe_to_upload = file_type and file_type ~= "unknown" and not filetypes.contains_user_data(file_type) + + if not file_type or file_type == "unknown" then + -- Unknown file type - DO NOT UPLOAD (GDPR-safe default) + logger.log_error( "ClamAV detected malware but file type unknown - GDPR protection: NOT uploading to VirusTotal (cannot verify if file contains personal data)") + elseif filetypes.contains_user_data(file_type) then + -- File may contain personal data - DO NOT UPLOAD + logger.log_error( string.format( + "ClamAV detected malware but file may contain user data (type: %s) - GDPR protection: NOT uploading to VirusTotal", + file_type + )) + logger.log_error( "File type categories: documents, images, databases, archives may contain personal data and will not be uploaded") + elseif is_safe_to_upload and virustotal then + -- File is known executable/script - SAFE TO UPLOAD + logger.log_error( string.format( + "ClamAV detected malware but VT doesn't have file (type: %s) - uploading to VirusTotal", + file_type + )) + local upload_ok, analysis_id, uploaded_size = virustotal.file_upload(plugin, body_file) + if upload_ok then + logger.log_error( "file uploaded successfully (" .. uploaded_size .. " bytes), analysis ID: " .. tostring(analysis_id)) + + -- Check if we should auto-vote + if plugin.variables["MALWARE_SCAN_API_VIRUSTOTAL_AUTO_VOTE"] == "yes" then + logger.log_error( "auto-voting file as malicious on VirusTotal") + local vote_ok, vote_err = virustotal.file_vote(plugin, checksum, "malicious") + if vote_ok then + logger.log_error( "malicious vote submitted successfully") + else + logger.log_error( "failed to submit vote: " .. vote_err) + end + end + else + logger.log_error( "failed to upload file to VirusTotal: " .. analysis_id) + end + end + else + logger.log_error( "ClamAV detected malware but VT doesn't have file (upload disabled)") + end + end + + if orchestrator.result_is_malware(detected) then + logger.log_error( "MALWARE DETECTED BY VIRUSTOTAL") + logger.log_error( "VirusTotal detection: " .. plugin:sanitize_external_data(vt_detected) .. " (SHA256: " .. checksum .. ")") + detections["VirusTotal"] = vt_detected + + -- Check if we should block immediately or continue scanning + if not resume_on_hit then + -- Default behavior: block immediately + logger.log_error( "BLOCKING REQUEST") + + -- Use orchestrator detection handler + local blocking_result = orchestrator.detection_handle( + plugin, checksum, detections, body_file, + "VirusTotal", vt_detected, + webhook, file_ops, get_deny_status + ) + + return blocking_result, detected, { + clamav_actually_scanned = clamav_actually_scanned, + clamav_skipped_reason = clamav_skipped_reason, + vt_detected = vt_detected, + checksum = checksum, + checksum_ok = checksum_ok + } + else + -- Resume mode: continue to remaining scanners + logger.log_error( "[RESUME_MODE] Malware detected by VirusTotal, continuing to remaining scanners") + end + else + logger.log_error( "VirusTotal scan: file is clean") + end + else + logger.log_error( "VirusTotal check failed: " .. vt_result) + end + end + + -- ============================================================ + -- SENTINELONE SCANNER + -- ============================================================ + + if sentinelone_enabled then + logger.log_error( "checking file against SentinelOne API") + local s1_ok, s1_result + if sentinelone then + s1_ok, s1_result = sentinelone.check_sentinelone(plugin, checksum, file_size) + else + s1_ok, s1_result = true, "clean" + end + if s1_ok then + local sentinelone_detected = s1_result + if orchestrator.result_is_malware(detected) then + logger.log_error( "MALWARE DETECTED BY SENTINELONE") + logger.log_error( "SentinelOne detection: " .. plugin:sanitize_external_data(sentinelone_detected) .. " (SHA256: " .. checksum .. ")") + detections["SentinelOne"] = sentinelone_detected + + -- Share file with threat intelligence platforms if enabled + local share_ok, share_err = plugin:detection_share(body_file, checksum, "SentinelOne", virustotal, file_type) + if not share_ok then + logger.log_error( "[SHARE] Failed to share file: " .. share_err) + end + + -- Check if we should block immediately or continue scanning + if not resume_on_hit then + -- Default behavior: block immediately + logger.log_error( "BLOCKING REQUEST") + + -- Use orchestrator detection handler + local blocking_result = orchestrator.detection_handle( + plugin, checksum, detections, body_file, + "SentinelOne", sentinelone_detected, + webhook, file_ops, get_deny_status + ) + + return blocking_result, detected, { + clamav_actually_scanned = clamav_actually_scanned, + clamav_skipped_reason = clamav_skipped_reason, + vt_detected = vt_detected, + checksum = checksum, + checksum_ok = checksum_ok + } + else + -- Resume mode: continue (all scanners complete) + logger.log_error( "[RESUME_MODE] Malware detected by SentinelOne") + end + else + logger.log_error( "SentinelOne scan: file is clean") + end + else + logger.log_error( "SentinelOne check failed: " .. s1_result) + end + end + end + else + logger.log_error( "failed to calculate SHA256: " .. tostring(checksum)) + end + end + + -- ======================================================================== + -- POST-SCAN PROCESSING + -- ======================================================================== + + -- Return scan info for caller to handle post-processing + -- Caller will handle: + -- - Webhook notification + -- - File cleanup + -- - Resume mode final block + -- - Multi-scanner comparison + -- - Success message building + return nil, detected, { + clamav_actually_scanned = clamav_actually_scanned, + clamav_skipped_reason = clamav_skipped_reason, + vt_detected = vt_detected, + checksum = checksum, + checksum_ok = checksum_ok + } +end + +-- Handle resume mode final blocking with all collected detections. +-- Called after all scanners complete in resume mode when at least one detection exists. +-- +--- @param plugin Plugin The plugin instance +--- @param checksum string|nil File SHA256 checksum (may be nil if not yet calculated) +--- @param detections table Table of all detections from all scanners +--- @param body_file string Path to uploaded file +--- @param webhook table|nil Webhook module (or nil) +--- @param file_ops table|nil File operations module (or nil) +--- @param get_deny_status function Function to get HTTP deny status code +--- @return boolean|string|number|table result Plugin return values (5 values via plugin:ret) +function orchestrator.detection_handle_resume_mode(plugin, checksum, detections, body_file, + webhook, file_ops, get_deny_status) + local ERR = ngx and ngx.ERR or 3 + + -- Build scanner list from all detections + local scanner_list = {} + for scanner, detection in pairs(detections) do + table.insert(scanner_list, scanner) + end + + logger.log_error( "[RESUME_MODE] All scanners complete - MALWARE DETECTED BY: " .. table.concat(scanner_list, ", ")) + logger.log_error( "[RESUME_MODE] Blocking request with comprehensive detection data") + + -- Send webhook notification (single webhook, not double) + if webhook then + webhook.notify(webhook, plugin, checksum, detections) + end + + -- Cleanup uploaded file if enabled + local cleanup_enabled = plugin.variables["MALWARE_SCAN_CLEANUP_FILES"] + if cleanup_enabled ~= "no" then + local cleanup_ok, cleanup_err + if file_ops then + cleanup_ok, cleanup_err = file_ops.file_cleanup(plugin, body_file) + else + cleanup_ok, cleanup_err = true, nil + end + if not cleanup_ok then + logger.log_error( "failed to cleanup temp file " .. body_file .. ": " .. cleanup_err) + end + end + + -- Return blocking response with resume mode data + return plugin:ret( + true, + "malware detected (resume mode - collected " .. #scanner_list .. " detections)", + get_deny_status(), + nil, + { + id = "malware_detected_resume_mode", + file = body_file, + checksum = checksum, + detections = detections, + scanner_count = #scanner_list + } + ) +end + +-- ============================================================================ +-- PRE-SCAN VALIDATION +-- ============================================================================ + +-- Perform pre-scan validation and initialization. +-- Handles scanner state setup, body file retrieval, and SHA256 calculation. +-- Returns scan context object with all necessary information for scanning. +-- +--- @param plugin Plugin instance +--- @param file_ops table|nil File operations module +--- @param ui_metrics_module table|nil UI metrics module (optional) +--- @param webhook table|nil Webhook module (optional) +--- @param get_deny_status function Function to get deny status code +--- @return table|nil scan_ctx Scan context object, or nil on error +--- @return table|nil result Plugin return value if early exit needed +function orchestrator.pre_scan_validate(plugin, file_ops, ui_metrics_module, webhook, get_deny_status) + -- Get scanner state + local scanner_state = orchestrator.scanner_get_state(plugin) + + -- Extract scanner states for compatibility + local clamav_enabled = scanner_state.clamav + local virustotal_enabled = scanner_state.virustotal + local sentinelone_enabled = scanner_state.sentinelone + local malwarebazaar_enabled = scanner_state.malwarebazaar + local resume_on_hit = scanner_state.resume_on_hit + + -- Table to collect all detections for single webhook notification + local detections = {} + + -- Log resume mode if enabled + if resume_on_hit then + logger.log_error( "[RESUME_MODE] Resume scan on first hit enabled - will collect all scanner results before blocking") + end + + -- Check if webhooks are configured + local teams_url = plugin.variables["MALWARE_SCAN_TEAMS_WEBHOOK_URL"] + local discord_url = plugin.variables["MALWARE_SCAN_DISCORD_WEBHOOK_URL"] + local webhook_enabled = webhook and + ((teams_url and teams_url ~= "" and teams_url ~= "your_webhook_url") or + (discord_url and discord_url ~= "" and discord_url ~= "your_webhook_url")) + + plugin:log_debug("[WEBHOOK_DEBUG] webhook_enabled=" .. tostring(webhook_enabled) .. + ", teams_url=" .. plugin:mask_sensitive(teams_url or "") .. + ", discord_url=" .. plugin:mask_sensitive(discord_url or "")) + + -- Check if any scanner is enabled + local any_scanner_enabled = clamav_enabled or virustotal_enabled or sentinelone_enabled or malwarebazaar_enabled + if not any_scanner_enabled then + logger.log_error( "MALWARE SCANNING NOT ENABLED - all scanners disabled") + return nil, plugin:ret(true, "malware scanning not enabled") + end + + -- Build scanner list message + local scanners = orchestrator.scanner_get_enabled(scanner_state) + + if #scanners > 1 then + logger.log_error( "Multi-layer scanning enabled (" .. table.concat(scanners, " + ") .. ")") + else + logger.log_error( scanners[1] .. "-only scanning enabled") + end + + logger.log_error( "file upload detected, scanning for malware") + + -- Track file scan metric + if ui_metrics_module then + ui_metrics_module.track_file_scanned(plugin) + end + + -- Read request body + ngx.req.read_body() + local body_file = ngx.req.get_body_file() + + plugin:log_debug("body_file = " .. tostring(body_file)) + + -- Validate body file availability + if not body_file then + logger.log_error( + "upload body in memory, not in file - increase client_body_buffer_size " .. + "or decrease it to force file buffering for virus scanning" + ) + + if plugin.variables["MALWARE_SCAN_BLOCK_MEMORY_UPLOADS"] == "yes" then + logger.log_error( "BLOCKING UPLOAD - cannot scan files in memory (need file buffering)") + return nil, plugin:ret( + true, + "upload in memory blocked - cannot scan (increase client_body_buffer_size)", + get_deny_status(), + nil, + { + id = "unscannable_memory", + reason = "body in memory, not file" + } + ) + else + return nil, plugin:ret(true, "upload in memory, skipping scan (WARNING)") + end + end + + -- Calculate SHA256 checksum (used for webhooks, logging, and hash-based scanners) + local checksum, checksum_ok + local file_size = file_ops and file_ops.file_get_size(plugin, body_file) or nil + local hash_max_size = tonumber(plugin.variables["MALWARE_SCAN_HASH_MAX_SIZE"]) or 67108864 -- default 64MB + + if file_size and file_size > hash_max_size then + logger.log_error( "[SIZE_LIMIT] File size " .. file_size .. " bytes exceeds hash scan limit " .. hash_max_size .. " bytes - skipping SHA256 calculation") + checksum_ok = false + checksum = nil + else + -- Always calculate checksum for webhooks and logging, even in ClamAV-only mode + if file_ops then + checksum_ok, checksum = file_ops.hash_calculate(plugin, body_file) + else + checksum_ok, checksum = false, nil + end + if checksum_ok and checksum ~= "empty_file" then + logger.log_error( "file SHA256: " .. checksum) + end + end + + -- Return scan context + return { + success = true, + body_file = body_file, + checksum = checksum, + checksum_ok = checksum_ok, + file_size = file_size, + scanner_state = scanner_state, + detections = detections, + webhook_enabled = webhook_enabled + }, nil +end + +-- ============================================================================ +-- COMPREHENSIVE CACHE CHECKING +-- ============================================================================ + +-- Perform comprehensive cache checking for all scanners. +-- Checks both ClamAV cache and hash-based scanner caches. +-- Returns updated scanner state and detection results. +-- +--- @param plugin Plugin instance +--- @param scan_ctx table Scan context from pre_scan_validate() +--- @param cache table Cache module +--- @param webhook table|nil Webhook module (optional) +--- @param file_ops table|nil File operations module +--- @param ui_metrics_module table|nil UI metrics module (optional) +--- @param get_deny_status function Function to get deny status code +--- @return table result Table with {blocked, detections, scanner_state, block_result} +function orchestrator.cache_check_comprehensive(plugin, scan_ctx, cache, webhook, file_ops, ui_metrics_module, get_deny_status) + local scanner_state = scan_ctx.scanner_state + local checksum = scan_ctx.checksum + local checksum_ok = scan_ctx.checksum_ok + local file_size = scan_ctx.file_size + local body_file = scan_ctx.body_file + local detections = scan_ctx.detections + + -- Extract scanner states + local clamav_enabled = scanner_state.clamav + local virustotal_enabled = scanner_state.virustotal + local sentinelone_enabled = scanner_state.sentinelone + local malwarebazaar_enabled = scanner_state.malwarebazaar + local resume_on_hit = scanner_state.resume_on_hit + + -- Get hash max size + local hash_max_size = tonumber(plugin.variables["MALWARE_SCAN_HASH_MAX_SIZE"]) or 67108864 -- default 64MB + + -- Check ClamAV cache first (works even in ClamAV-only mode) + if clamav_enabled and checksum_ok and checksum and checksum ~= "empty_file" then + local clamav_cache_ok, clamav_cached, _, error_msg + if cache then + clamav_cache_ok, clamav_cached, _, error_msg = cache.clamav_check(plugin, checksum, file_size) + else + clamav_cache_ok, clamav_cached, _, error_msg = true, nil, nil, nil + end + + if clamav_cache_ok and clamav_cached == "error" then + -- Cached error - skip ClamAV scan to avoid repeated failures + clamav_enabled = false -- Disable ClamAV scan since we have cached error + logger.log_error( "[CACHE_HIT] ClamAV previously failed for this file - skipping scan: " .. (error_msg or "unknown error")) + plugin:log_debug("[CACHE_HIT] File scan previously failed, continuing to hash-based scanners") + elseif clamav_cache_ok and clamav_cached and clamav_cached ~= "clean" then + -- Cached result is MALICIOUS + clamav_enabled = false -- Disable ClamAV scan since we have cached result + logger.log_error( "[CACHE_HIT] ClamAV cache shows malicious - skipping ClamAV scan") + logger.log_error( "MALWARE DETECTED BY CLAMAV (cached)") + logger.log_error( "ClamAV signature: " .. plugin:sanitize_external_data(clamav_cached) .. " (SHA256: " .. checksum .. ")") + detections["ClamAV (cached)"] = clamav_cached + + -- Track cache hit and malware detection + if ui_metrics_module then + ui_metrics_module.track_cache_hit(plugin, "clamav") + ui_metrics_module.track_detection_complete(plugin, "ClamAV (cached)", clamav_cached, checksum, detections) + end + + -- Check if we should block immediately or continue scanning + if not resume_on_hit then + -- Default behavior: block immediately + local block_result = orchestrator.detection_handle( + plugin, checksum, detections, body_file, + "ClamAV (cached)", clamav_cached, + webhook, file_ops, get_deny_status + ) + return { + blocked = true, + detections = detections, + scanner_state = { + clamav = clamav_enabled, + virustotal = virustotal_enabled, + sentinelone = sentinelone_enabled, + malwarebazaar = malwarebazaar_enabled, + resume_on_hit = resume_on_hit + }, + block_result = block_result + } + else + -- Resume mode: continue to other scanners for comprehensive intelligence + logger.log_error( "[RESUME_MODE] Malware detected by ClamAV cache, continuing to other scanners") + end + elseif clamav_cache_ok and clamav_cached and clamav_cached == "clean" then + -- Cached result is CLEAN - skip ClamAV scan, allow file + clamav_enabled = false -- Disable ClamAV scan for this request + logger.log_error( "[CACHE_HIT] ClamAV cache shows clean - skipping ClamAV scan") + plugin:log_debug("[CACHE_HIT] File is known clean from cache, allowing upload without scan") + + -- Track cache hit + if ui_metrics_module then + ui_metrics_module.track_cache_hit(plugin, "clamav") + end + end + end + + -- OPTIMIZATION: Check hash-based caches BEFORE ClamAV to avoid wasting resources + -- If we already know the file is malicious from cache, skip ClamAV scan entirely + local clamav_can_skip = false + if virustotal_enabled or sentinelone_enabled or malwarebazaar_enabled then + if file_size and file_size > hash_max_size then + logger.log_error( "[SIZE_LIMIT] Skipping hash-based scanners for oversized file") + elseif checksum_ok and checksum and checksum ~= "empty_file" then + logger.log_error( "[CACHE_CHECK] Checking hash-based caches before ClamAV scan") + + -- Check all hash-based scanner caches + local blocked, cache_detections, skip_clamav, scanners_disabled, block_result = + orchestrator.cache_check_all( + plugin, scanner_state, checksum, file_size, body_file, + cache, webhook, file_ops, get_deny_status + ) + + -- Merge cache detections into main detections table + for scanner, detection in pairs(cache_detections) do + detections[scanner] = detection + end + + -- Update clamav_can_skip flag + if skip_clamav then + clamav_can_skip = true + end + + -- Disable scanners that found cache hits + if scanners_disabled.virustotal then virustotal_enabled = false end + if scanners_disabled.sentinelone then sentinelone_enabled = false end + if scanners_disabled.malwarebazaar then malwarebazaar_enabled = false end + + -- If blocked, return immediately + if blocked and block_result then + return { + blocked = true, + detections = detections, + scanner_state = { + clamav = clamav_enabled, + virustotal = virustotal_enabled, + sentinelone = sentinelone_enabled, + malwarebazaar = malwarebazaar_enabled, + resume_on_hit = resume_on_hit + }, + block_result = block_result + } + end + end + end + + -- Return updated state (not blocked) + return { + blocked = false, + detections = detections, + scanner_state = { + clamav = clamav_enabled, + virustotal = virustotal_enabled, + sentinelone = sentinelone_enabled, + malwarebazaar = malwarebazaar_enabled, + resume_on_hit = resume_on_hit + }, + clamav_can_skip = clamav_can_skip, + block_result = nil + } +end + +-- Configure scan limits and determine which scanners to use. +-- Handles multipart body skip logic and file size limit checks. +-- Returns updated scanner configuration for body scan and file extraction. +-- +--- @param plugin Plugin instance +--- @param scan_ctx ScanContext Scan context from pre_scan_validate() +--- @return table config Table with scan configuration +function orchestrator.configure_scan_limits(plugin, scan_ctx) + local scanner_state = scan_ctx.scanner_state + local file_size = scan_ctx.file_size + + -- Extract current scanner states + local clamav_enabled = scanner_state.clamav + local virustotal_enabled = scanner_state.virustotal + local sentinelone_enabled = scanner_state.sentinelone + local malwarebazaar_enabled = scanner_state.malwarebazaar + + -- Check if we should skip full multipart body scan + -- For multipart uploads, the body SHA256 changes on every upload due to random HTTP boundaries, + -- making caching ineffective and wasting ClamAV resources. Individual file extraction always occurs. + local skip_full_body_scan = false + local is_multipart = plugin.ctx.bw.http_content_type and plugin.ctx.bw.http_content_type:match("multipart/form%-data") + local scan_full_body = plugin.variables["MALWARE_SCAN_SCAN_FULL_MULTIPART_BODY"] + + -- Save original scanner enabled states before potentially disabling for body scan + local clamav_enabled_for_files = clamav_enabled + local virustotal_enabled_for_files = virustotal_enabled + local sentinelone_enabled_for_files = sentinelone_enabled + local malwarebazaar_enabled_for_files = malwarebazaar_enabled + + if is_multipart and (scan_full_body == nil or scan_full_body == "" or scan_full_body == "no") then + skip_full_body_scan = true + plugin.logger:log(ERR, "[MULTIPART] Skipping full body scan (random boundaries prevent caching) - will extract and scan individual files") + -- Skip both ClamAV and external API scanners for the body + -- Individual files will be scanned in multipart_full_scan module + clamav_enabled = false + virustotal_enabled = false + sentinelone_enabled = false + malwarebazaar_enabled = false + end + + -- Check file size for ClamAV scan limit + local clamav_skipped_reason = nil + if clamav_enabled and file_size then + local max_size = tonumber(plugin.variables["MALWARE_SCAN_CLAMAV_MAX_SIZE"]) or 26214400 + if file_size > max_size then + plugin.logger:log( + ERR, + "file too large for ClamAV INSTREAM scan: " .. file_size .. " bytes (max: " .. max_size .. " bytes), skipping ClamAV scan" + ) + clamav_skipped_reason = "file too large (" .. file_size .. " bytes)" + clamav_enabled = false -- Skip ClamAV scan, continue with hash-based scanners + end + end + + -- Return updated configuration + return { + skip_full_body_scan = skip_full_body_scan, + scanner_state = { + clamav = clamav_enabled, + virustotal = virustotal_enabled, + sentinelone = sentinelone_enabled, + malwarebazaar = malwarebazaar_enabled, + resume_on_hit = scanner_state.resume_on_hit + }, + scanner_state_for_files = { + clamav = clamav_enabled_for_files, + virustotal = virustotal_enabled_for_files, + sentinelone = sentinelone_enabled_for_files, + malwarebazaar = malwarebazaar_enabled_for_files, + resume_on_hit = scanner_state.resume_on_hit + }, + clamav_skipped_reason = clamav_skipped_reason + } +end + +-- Handle post-scan processing including notifications, cleanup, and multipart scanning. +-- Processes scan results, handles resume mode, and performs final cleanup. +-- Returns plugin result (success/blocked). +-- +--- @param plugin Plugin instance +--- @param scan_ctx ScanContext Scan context with file information +--- @param scan_limits ScanLimits Scan limits from configure_scan_limits() +--- @param scan_result ScanResult Result from body_scan() {detected, checksum, checksum_ok, clamav_actually_scanned} +--- @param webhook table|nil Webhook module +--- @param file_ops table|nil File operations module +--- @param multipart_full_scan table|nil Multipart full scan module (optional) +--- @param multipart_full_scan_load_ok boolean Module load status +--- @param get_deny_status function Function to get deny status code +--- @return boolean|string|number|table result Plugin return value (multiple values via plugin:ret) +function orchestrator.post_scan_complete(plugin, scan_ctx, scan_limits, scan_result, webhook, file_ops, multipart_full_scan, multipart_full_scan_load_ok, get_deny_status) + local body_file = scan_ctx.body_file + local checksum = scan_result.checksum or scan_ctx.checksum + local detections = scan_ctx.detections + local scanner_state = scan_limits.scanner_state + local resume_on_hit = scanner_state.resume_on_hit + local skip_full_body_scan = scan_limits.skip_full_body_scan + local detected = scan_result.detected + local clamav_actually_scanned = scan_result.clamav_actually_scanned + local clamav_skipped_reason = scan_limits.clamav_skipped_reason + + -- Send webhook notification with all detections (centralized error handling) + if webhook then + webhook.notify(webhook, plugin, checksum, detections) + end + + -- Clean up temp file after all scans complete (unless resume mode needs it for multipart parsing) + local cleanup_enabled = plugin.variables["MALWARE_SCAN_CLEANUP_FILES"] + if cleanup_enabled ~= "no" and not resume_on_hit then + local cleanup_ok = false + local cleanup_err = nil + if file_ops then + cleanup_ok, cleanup_err = file_ops.file_cleanup(plugin, body_file) + end + if not cleanup_ok then + plugin.logger:log( + ERR, + "failed to cleanup temp file " .. body_file .. ": " .. cleanup_err + ) + end + elseif resume_on_hit then + plugin:log_debug("[v" .. orchestrator.VERSION .. "] [RESUME_MODE] Preserving body file for multipart per-file scanning") + end + + -- If resume mode is enabled and any detections exist, block now with all collected intelligence + if resume_on_hit and next(detections) ~= nil then + -- Use orchestrator resume mode handler (fixes duplicate webhook bug) + return orchestrator.detection_handle_resume_mode( + plugin, checksum, detections, body_file, + webhook, file_ops, get_deny_status + ) + end + + -- If we reach here and ClamAV detected, build comprehensive multi-scanner response + if detected and detected ~= "clean" then + -- Add ClamAV to detections table if not already there + if not detections["ClamAV"] then + detections["ClamAV"] = detected + end + + -- Build list of ALL scanners that detected malware + local scanner_list = {} + local scanner_details = {} + + -- Guaranteed: ClamAV detected (we're in this block) + table.insert(scanner_list, "ClamAV") + scanner_details["clamav_signature"] = detected + + -- Check other scanners from detections table + if detections["VirusTotal"] then + table.insert(scanner_list, "VirusTotal") + scanner_details["virustotal_detection"] = detections["VirusTotal"] + end + if detections["SentinelOne"] then + table.insert(scanner_list, "SentinelOne") + scanner_details["sentinelone_detection"] = detections["SentinelOne"] + end + + -- Build comprehensive log message + local scanner_count = #scanner_list + if scanner_count == 1 then + plugin.logger:log(ERR, "MALWARE DETECTED BY CLAMAV - BLOCKING REQUEST") + plugin.logger:log(ERR, "ClamAV: " .. plugin:sanitize_external_data(detected)) + else + plugin.logger:log(ERR, "MALWARE DETECTED BY " .. table.concat(scanner_list, ", "):upper() .. " - BLOCKING REQUEST") + -- Log individual scanner results + for scanner, detection in pairs(detections) do + plugin.logger:log(ERR, scanner .. ": " .. plugin:sanitize_external_data(detection)) + end + end + + -- Build message for return + local message + if scanner_count == 1 then + message = "malware detected by ClamAV (" .. detected .. ")" + elseif scanner_count == 2 then + message = "malware detected by " .. scanner_list[1] .. " and " .. scanner_list[2] + else + -- 3+ scanners: "MalwareBazaar, ClamAV, and VirusTotal" + local last_scanner = table.remove(scanner_list) + message = "malware detected by " .. table.concat(scanner_list, ", ") .. ", and " .. last_scanner + end + + -- Build return data structure with all scanner details + local return_data = { + id = scanner_count == 1 and "malware_detected_clamav" or "malware_detected_multi", + file = body_file, + checksum = checksum, + scanner = table.concat(scanner_list, "+"), + scanner_count = scanner_count + } + + -- Add scanner-specific fields + for field, value in pairs(scanner_details) do + return_data[field] = value + end + + -- Use orchestrator custom detection handler (handles any scanner combination) + return orchestrator.detection_handle_custom( + plugin, checksum, detections, body_file, + message, + return_data, + webhook, file_ops, get_deny_status + ) + end + + -- ============================================================================ + -- MULTIPART FULL SCAN: PER-FILE SCANNING + -- ============================================================================ + -- If we reach here, whole-body scan is complete (either clean or resume mode). + -- When resume mode is enabled OR full body scan was skipped, parse multipart upload and scan each file individually. + + if (resume_on_hit or skip_full_body_scan) and multipart_full_scan_load_ok and multipart_full_scan then + -- Build enabled scanners table for multipart full scan module + -- Always use saved states (before body scan disable) for individual file scanning + local enabled_scanners = { + clamav = scan_limits.scanner_state_for_files.clamav or clamav_actually_scanned, + virustotal = scan_limits.scanner_state_for_files.virustotal, + sentinelone = scan_limits.scanner_state_for_files.sentinelone, + malwarebazaar = scan_limits.scanner_state_for_files.malwarebazaar + } + + -- DEBUG: Log enabled scanners for multipart scanning + local VERSION = orchestrator.VERSION + plugin.logger:log(ERR, string.format("[v%s] [MULTIPART_DEBUG] enabled_scanners: clamav=%s, virustotal=%s, sentinelone=%s, malwarebazaar=%s", + VERSION, tostring(enabled_scanners.clamav), tostring(enabled_scanners.virustotal), + tostring(enabled_scanners.sentinelone), tostring(enabled_scanners.malwarebazaar))) + plugin.logger:log(ERR, string.format("[v%s] [MULTIPART_DEBUG] skip_full_body_scan=%s", + VERSION, tostring(skip_full_body_scan))) + + -- Call multipart full scan module (parses multipart and scans each file) + local success, message, status, data = multipart_full_scan.scan_multipart_files( + plugin, + body_file, + checksum, + detections, + enabled_scanners + ) + + -- If per-file scanning found malware, block immediately + if not success then + -- Cleanup body file before blocking + local cleanup_enabled = plugin.variables["MALWARE_SCAN_CLEANUP_FILES"] + if cleanup_enabled ~= "no" and file_ops then + local cleanup_ok, cleanup_err = file_ops.file_cleanup(plugin, body_file) + if not cleanup_ok then + plugin.logger:log(ERR, "failed to cleanup temp file after multipart scan: " .. cleanup_err) + end + end + return plugin:ret(true, message, status, nil, data) + end + elseif (resume_on_hit or skip_full_body_scan) and not multipart_full_scan_load_ok then + plugin.logger:log(ERR, "[MULTIPART_FULL_SCAN] Resume mode or multipart extraction enabled but multipart_full_scan module failed to load") + end + + -- Build success message based on what was scanned + local scan_message + local active_scanners = {} + if clamav_actually_scanned then table.insert(active_scanners, "ClamAV") end + if scanner_state.virustotal then table.insert(active_scanners, "VirusTotal") end + if scanner_state.sentinelone then table.insert(active_scanners, "SentinelOne") end + if scanner_state.malwarebazaar then table.insert(active_scanners, "MalwareBazaar") end + + if #active_scanners > 0 then + local scanners_str = table.concat(active_scanners, " + ") + scan_message = "file scanned by " .. scanners_str .. ", no malware detected" + if clamav_skipped_reason then + scan_message = scan_message .. " (ClamAV skipped: " .. clamav_skipped_reason .. ")" + end + plugin.logger:log(ERR, "SCAN COMPLETE - file is clean (" .. scanners_str .. ")") + if clamav_skipped_reason then + plugin.logger:log(ERR, "Note: ClamAV scan was skipped - " .. clamav_skipped_reason) + end + else + scan_message = "no scanning performed" + end + + -- Cleanup body file if resume mode was enabled (file preserved for multipart scanning) + if resume_on_hit then + local cleanup_enabled = plugin.variables["MALWARE_SCAN_CLEANUP_FILES"] + if cleanup_enabled ~= "no" and file_ops then + local cleanup_ok, cleanup_err = file_ops.file_cleanup(plugin, body_file) + if not cleanup_ok then + plugin.logger:log(ERR, "failed to cleanup temp file after successful scan: " .. cleanup_err) + else + local VERSION = orchestrator.VERSION + plugin:log_debug("[v" .. VERSION .. "] [RESUME_MODE] Cleaned up body file after multipart scanning") + end + end + end + + return plugin:ret(true, scan_message) +end + +-- Return module +return orchestrator diff --git a/malware-scan/malware_scan_scheduler.lua b/malware-scan/malware_scan_scheduler.lua new file mode 100644 index 0000000..3016086 --- /dev/null +++ b/malware-scan/malware_scan_scheduler.lua @@ -0,0 +1,925 @@ +-- BunkerWeb Malware Scan - Scheduler Module +-- Handles automatic MalwareBazaar custom hash database updates +-- Uses ETag-based caching for smart downloads and Redis for bulk imports +-- +-- Features: +-- - Scheduled downloads with intelligent change detection (ETag + filesize) +-- - Configurable update intervals (seconds precision) +-- - State persistence for bandwidth optimization (99.9% savings) +-- - Atomic database updates with bucketed key distribution (16 buckets) +-- - Infinite persistence (no TTL - freshness maintained by scheduler) +-- - Support for three strategies: full, recent, hybrid +-- +-- Version: 0.8.0 +-- Date: 2026-01-27 + +local scheduler = {} + +-- Module version +scheduler.VERSION = "0.8.0" + +local ngx = ngx +local tonumber = tonumber +local tostring = tostring +local lfs = require("lfs") + +-- Load centralized logger and utility modules +local logger = require("malware_scan_logger") +local utils = require("malware_scan_utils") +local database = require("malware_scan_database") + +-- ============================================================================ +-- TYPE DEFINITIONS +-- ============================================================================ + +--- @class Plugin +--- @field variables table Configuration variables from BunkerWeb +--- @field use_redis boolean Whether Redis is enabled +--- @field clusterstore table|nil Clusterstore instance for Redis access +--- @field redisdb table|nil Redis client instance +--- @field datastore table|nil Datastore instance for key-value operations +--- @field cachestore table Local cache store instance +--- @field ctx table Request context with nested tables +--- @field logger table Logger instance with logging methods +--- @field ret fun(self: Plugin, ...): table Plugin return method (variadic) +--- @field log_debug fun(self: Plugin, message: string) Debug logging method +--- @field sanitize_external_data fun(self: Plugin, data: string): string Data sanitization method +--- @field detection_share fun(self: Plugin, ...) Detection sharing method +--- @field mask_sensitive fun(...): string Sensitive data masking method +--- @field track_file_scanned fun(): boolean File tracking method +--- @field file_get_size fun(...): number|nil Get file size method +--- @field hash_calculate fun(...): string|nil Calculate hash method + +-- ============================================================================ +-- HELPER FUNCTIONS +-- ============================================================================ + +--- Create directory recursively (like mkdir -p) +--- @param path string Directory path to create +--- @return boolean success True if directory was created/exists +--- @return string|nil error Error message if failed +local function mkdir_recursive(path) + if not path or path == "" then + return false, "path is empty" + end + + -- Check if directory already exists + local attr = lfs.attributes(path, "mode") + if attr == "directory" then + return true, nil + elseif attr then + return false, "path exists but is not a directory" + end + + -- Split path into components + local components = {} + for component in path:gmatch("[^/]+") do + table.insert(components, component) + end + + -- Build path incrementally + local current_path = path:sub(1, 1) == "/" and "/" or "" + for _, component in ipairs(components) do + current_path = current_path .. component .. "/" + + -- Check if this level exists + local level_attr = lfs.attributes(current_path, "mode") + if not level_attr then + -- Create this directory level + local ok, err = lfs.mkdir(current_path) + if not ok then + return false, "failed to create directory " .. current_path .. ": " .. (err or "unknown error") + end + elseif level_attr ~= "directory" then + return false, "path component exists but is not a directory: " .. current_path + end + end + + return true, nil +end + +--- Create secure temporary directory using native Lua +--- @return string|nil temp_dir Temporary directory path, or nil on error +local function create_temp_dir() + -- Use os.tmpname() to get a unique temporary path + local tmpname = os.tmpname() + if not tmpname then + return nil + end + + -- Extract just the filename part to create a unique suffix + local suffix = tmpname:match("([^/]+)$") or tostring(os.time()) + + -- Build secure temp directory path + local temp_dir = "/tmp/malware_scheduler." .. suffix + + -- Create the directory + local ok, err = lfs.mkdir(temp_dir) + if not ok then + logger.log_error("[SCHEDULER] Failed to create temp directory: " .. (err or "unknown")) + return nil + end + + -- Verify directory was created + local attr = lfs.attributes(temp_dir, "mode") + if attr ~= "directory" then + return nil + end + + return temp_dir +end + +-- ============================================================================ +-- REDIS CONFIGURATION VALIDATION +-- ============================================================================ + +-- Validate Redis configuration and availability. +-- Checks that Redis is properly configured when the plugin is enabled. +-- +--- @param plugin Plugin instance +--- @return boolean valid True if Redis is available and configured +--- @return string|nil error_message Error message if validation fails +local function scheduler_validate_redis_config(plugin) + -- Check if BunkerWeb's global USE_REDIS is enabled + local use_redis = plugin.variables["USE_REDIS"] == "yes" + + if not use_redis then + return false, "Redis is not enabled in BunkerWeb configuration (USE_REDIS != yes). " .. + "Custom hash database requires Redis. " .. + "Please set USE_REDIS=yes and configure either REDIS_HOST or REDIS_SENTINEL_HOSTS." + end + + -- Check if Redis connection is available + if not database.is_available(plugin) then + return false, "Redis connection is not available. " .. + "USE_REDIS is enabled but Redis connection failed. " .. + "Please verify your Redis configuration: " .. + "either REDIS_HOST (for standalone Redis) or REDIS_SENTINEL_HOSTS (for Sentinel) must be properly configured." + end + + -- Redis is properly configured and connected + return true, nil +end + +-- ============================================================================ +-- STATE FILE MANAGEMENT +-- ============================================================================ + +-- Load state file from disk +--- @param state_file string Path to state file +--- @return table state State table with saved_etag, saved_size, saved_modified, etc. +local function scheduler_load_state(state_file) + local state = { + saved_etag = nil, + saved_size = nil, + saved_modified = nil, + last_update = nil, + full_update_delay_seconds = nil, + full_update_delay_date = nil, + last_full_update = nil + } + + local f, err = io.open(state_file, "r") + if not f then + return state -- No state file yet + end + + for line in f:lines() do + local key, value = line:match("^(%w+)=(.*)$") + if key and value then + value = value:gsub('^"', ''):gsub('"$', '') -- Remove quotes + state[key] = value + end + end + f:close() + + return state +end + +-- Save state file to disk +-- Can be called with individual parameters: scheduler_save_state(path, etag, size, modified) +-- Or with state table: scheduler_save_state(path, nil, nil, nil) and pass table as state_obj +--- @param state_file string Path to state file +--- @param etag string|table|nil Remote ETag value (or table with state if others are nil) +--- @param size number|nil Remote file size +--- @param modified string|nil Remote Last-Modified header +--- @param state_obj table|nil Optional - full state table to merge +local function scheduler_save_state(state_file, etag, size, modified, state_obj) + local state_dir = state_file:match("(.*/)") + if state_dir and state_dir ~= "" then + -- Create state directory using native Lua (non-blocking) + local mkdir_ok, mkdir_err = mkdir_recursive(state_dir) + if not mkdir_ok then + logger.log_error("[SCHEDULER] Failed to create state directory: " .. (mkdir_err or "unknown error")) + end + end + + -- If etag is a table, treat it as state object + if type(etag) == "table" then + state_obj = etag + etag = state_obj.saved_etag + size = state_obj.saved_size + modified = state_obj.saved_modified + end + + local f, err = io.open(state_file, "w") + if not f then + logger.log_error( "[SCHEDULER] Failed to save state: " .. (err or "unknown error")) + return false + end + + f:write('SAVED_ETAG="' .. (etag or "") .. '"\n') + f:write('SAVED_SIZE="' .. (size or "") .. '"\n') + f:write('SAVED_MODIFIED="' .. (modified or "") .. '"\n') + f:write('DOWNLOAD_TIME="' .. os.date("!%Y-%m-%dT%H:%M:%SZ") .. '"\n') + + -- Write full update delay info (for server load balancing) + if state_obj then + f:write('FULL_UPDATE_DELAY_SECONDS="' .. (state_obj.full_update_delay_seconds or "") .. '"\n') + f:write('FULL_UPDATE_DELAY_DATE="' .. (state_obj.full_update_delay_date or "") .. '"\n') + f:write('LAST_FULL_UPDATE="' .. (state_obj.last_full_update or "") .. '"\n') + end + + f:close() + return true +end + +-- ============================================================================ +-- REMOTE METADATA RETRIEVAL +-- ============================================================================ + +-- Get remote file metadata via HTTP HEAD request +--- @param url string Download URL +--- @return string|nil etag Remote ETag header, or nil if failed +--- @return string|nil size Remote Content-Length header, or nil if failed +--- @return string|nil modified Remote Last-Modified header, or nil if failed +local function scheduler_get_remote_metadata(url) + local httpc = require("resty.http").new() + if not httpc then + logger.log_error( "[SCHEDULER] Failed to create HTTP client") + return nil, nil, nil + end + + httpc:set_timeout(10000) + + local res, err = httpc:request_uri(url, { + method = "HEAD", + ssl_verify = true + }) + + if not res then + logger.log_error( "[SCHEDULER] HTTP HEAD failed: " .. (err or "unknown")) + return nil, nil, nil + end + + if res.status ~= 200 and res.status ~= 304 then + logger.log_error( "[SCHEDULER] HTTP " .. res.status .. " error") + return nil, nil, nil + end + + local headers = res.headers or {} + local etag = headers["etag"] or headers["ETag"] or "" + local size = headers["content-length"] or headers["Content-Length"] or "" + local modified = headers["last-modified"] or headers["Last-Modified"] or "" + + return etag, size, modified +end + +-- ============================================================================ +-- FILE DOWNLOAD +-- ============================================================================ + +-- Download file from URL +--- @param url string Download URL +--- @param output_path string Local file path to save to +--- @return boolean success True on success, false on failure +local function scheduler_download_file(url, output_path) + local output_dir = output_path:match("(.*/)") + if output_dir and output_dir ~= "" then + -- Create output directory using native Lua (non-blocking) + local mkdir_ok, mkdir_err = mkdir_recursive(output_dir) + if not mkdir_ok then + logger.log_error("[SCHEDULER] Failed to create output directory: " .. (mkdir_err or "unknown error")) + end + end + + local httpc = require("resty.http").new() + if not httpc then + logger.log_error( "[SCHEDULER] Failed to create HTTP client") + return false + end + + httpc:set_timeout(60000) -- 60 second timeout for large files + + local res, err = httpc:request_uri(url, { + method = "GET", + ssl_verify = true + }) + + if not res then + logger.log_error( "[SCHEDULER] Download failed: " .. (err or "unknown")) + return false + end + + if res.status ~= 200 then + logger.log_error( "[SCHEDULER] HTTP " .. res.status .. " error") + return false + end + + local f, ferr = io.open(output_path, "wb") + if not f then + logger.log_error( "[SCHEDULER] Failed to write file: " .. (ferr or "unknown")) + return false + end + + if res.body then + f:write(res.body) + end + f:close() + + return true +end + +-- ============================================================================ +-- CSV EXTRACTION AND IMPORT +-- ============================================================================ + +-- Create secure temporary directory using native Lua (non-blocking). +-- Returns a random directory path that is unpredictable. +--- @return string|nil temp_dir Temporary directory path, or nil on error +local function scheduler_create_temp_dir() + -- Use the native helper function defined at the top of the file + return create_temp_dir() +end + +-- Store temp directory in Redis for cleanup tracking. +-- Allows recovery of abandoned temp directories from interrupted runs. +--- @param plugin Plugin Plugin instance +--- @param temp_dir string Temporary directory path +local function scheduler_register_temp_dir(plugin, temp_dir) + if not plugin.redisdb then + return false + end + + -- Store with 24-hour TTL (cleanup will happen much sooner, but this prevents leaks) + local key = "malware_scan:scheduler:temp_dir" + database.setex(plugin, key, 86400, temp_dir) + return true +end + +-- Cleanup temp directory and unregister from Redis. +--- @param plugin Plugin Plugin instance +--- @param temp_dir string Temporary directory path +local function scheduler_cleanup_temp_dir(plugin, temp_dir) + if temp_dir and temp_dir ~= "" then + os.execute("rm -rf " .. utils.shell_escape_path(temp_dir)) + end + + if plugin.redisdb then + database.unlink(plugin, "malware_scan:scheduler:temp_dir") + end +end + +-- Cleanup any abandoned temp directories from previous interrupted runs. +-- Checks Redis for registered temp directories and removes them. +--- @param plugin Plugin instance +local function scheduler_cleanup_abandoned_temp_dirs(plugin) + if not plugin.redisdb then + return + end + + local key = "malware_scan:scheduler:temp_dir" + local old_temp_dir = database.redis_get(plugin, key) + + if old_temp_dir and old_temp_dir ~= ngx.null then + logger.log_notice( "[SCHEDULER] Cleaning up abandoned temp directory: " .. old_temp_dir) + os.execute("rm -rf " .. utils.shell_escape_path(old_temp_dir)) + database.unlink(plugin, key) + end +end + +-- Extract ZIP and import CSV to Redis +--- @param zip_file string Path to downloaded ZIP file +--- @param plugin Plugin Plugin instance (for Redis access) +--- @return number added Count of hashes added to database +--- @return number duplicates Count of duplicate hashes skipped +local function scheduler_import_from_zip(zip_file, plugin) + if not plugin.redisdb then + logger.log_error( "[SCHEDULER] Redis not available") + return 0, 0 + end + + -- Clean up any abandoned temp directories from previous runs + scheduler_cleanup_abandoned_temp_dirs(plugin) + + -- Extract ZIP in secure temp location using mktemp + local work_dir = scheduler_create_temp_dir() + if not work_dir then + logger.log_error( "[SCHEDULER] Failed to create temporary directory") + return 0, 0 + end + + -- Register temp directory for cleanup tracking + scheduler_register_temp_dir(plugin, work_dir) + + os.execute("cd " .. utils.shell_escape_path(work_dir) .. " && unzip -q " .. utils.shell_escape_path(zip_file)) + + -- Find CSV file using native Lua (non-blocking) + local csv_file = nil + for filename in lfs.dir(work_dir) do + if filename:match("%.csv$") then + local file_path = work_dir .. "/" .. filename + local attr = lfs.attributes(file_path, "mode") + if attr == "file" then + csv_file = file_path + break -- Use first CSV file found + end + end + end + + if not csv_file then + logger.log_error( "[SCHEDULER] No CSV file found in ZIP") + scheduler_cleanup_temp_dir(plugin, work_dir) + return 0, 0 + end + + local added = 0 + local duplicates = 0 + local hash_key = "malware_scan:hashes" + + -- Import CSV lines + local f, err = io.open(csv_file, "r") + if not f then + logger.log_error( "[SCHEDULER] Failed to read CSV: " .. (err or "unknown")) + scheduler_cleanup_temp_dir(plugin, work_dir) + return 0, 0 + end + + for line in f:lines() do + -- Skip comment lines + if not line:match("^#") and line ~= "" then + -- Simple CSV parsing (quoted fields) + local fields = {} + for field in line:gmatch('"([^"]*)"') do + table.insert(fields, field) + end + + if #fields >= 9 then + local sha256 = fields[2] + local file_type = fields[7] + local mime_type = fields[8] + local signature = fields[9] + + -- Validate SHA256 + if sha256 and #sha256 == 64 and sha256:match("^[a-fA-F0-9]+$") then + -- Skip if no signature + if signature and signature ~= "n/a" and signature ~= "" then + -- Check if exists + if not database.hexists(plugin, hash_key, sha256) then + local metadata = file_type .. "|" .. mime_type + local data = signature .. "|" .. metadata + database.hset(plugin, hash_key, sha256, data) + added = added + 1 + else + duplicates = duplicates + 1 + end + end + end + end + end + end + f:close() + + -- No TTL set on hashes - they persist indefinitely + -- Freshness maintained by: + -- - Daily full refresh (at 3 AM with hybrid strategy) + -- - Frequent incremental updates (every 10 minutes) + -- - ETag-based change detection (avoids unnecessary re-imports) + logger.log_notice( "[SCHEDULER] Import complete - hashes persist indefinitely (no TTL)") + + scheduler_cleanup_temp_dir(plugin, work_dir) + return added, duplicates +end + +-- ============================================================================ +-- UPDATE STRATEGY SUPPORT +-- ============================================================================ + +-- Determine which URL to use based on strategy +--- @param base_url string Base URL from configuration +--- @param strategy string Update strategy (full, recent, hybrid) +--- @return string url URL to download from +local function scheduler_get_download_url(base_url, strategy) + if not strategy or strategy == "full" then + -- Use full database + return base_url or "https://bazaar.sysangels.ai/bunkerweb/malwarebazaar_full_hash.txt.zip" + elseif strategy == "recent" or strategy == "hybrid" then + -- Use recent/incremental updates + return (base_url:gsub("full_hash", "recent_hash")) or + "https://bazaar.sysangels.ai/bunkerweb/malwarebazaar_recent_hash.txt.zip" + end + return base_url +end + +-- Generate random delay for full update (1-9 minutes) to prevent server overload +--- @return number delay Random delay in seconds (60-540) +local function scheduler_get_random_full_update_delay() + math.randomseed(os.time()) + -- Random 1-9 minutes = 60-540 seconds + return math.random(1, 9) * 60 +end + +-- Calculate scheduled time for full update (3 AM UTC + random delay) +--- @return number scheduled_time Seconds since midnight for scheduled full update +local function scheduler_get_full_update_scheduled_time(delay_seconds) + -- 3 AM = 3 * 3600 = 10800 seconds + -- Add random delay (1-9 minutes) + return 10800 + delay_seconds +end + +-- Get current time as seconds since midnight UTC +--- @return number seconds Seconds since midnight UTC +local function scheduler_get_current_time_of_day() + return tonumber(os.date("!%H")) * 3600 + tonumber(os.date("!%M")) * 60 + tonumber(os.date("!%S")) +end + +-- Check if we should do full update today (for hybrid strategy) +-- Incorporates random 1-9 minute delay to prevent server overload at exactly 3 AM +--- @param state_file string Path to state file +--- @return boolean should_update True if should do full update, false for recent +--- @return number delay_seconds Random delay in seconds (informational, for logging) +local function scheduler_should_do_full_today(state_file) + local state = scheduler_load_state(state_file) + local today = os.date("!%Y-%m-%d") + + -- Check if we've already done a full update today + if state.last_full_update then + local last_date = state.last_full_update:match("^(%d%d%d%d%-%d%d%-%d%d)") + if last_date == today then + -- Already did full update today + return false, 0 + end + end + + -- New day - check if we should do full update now + -- Generate delay if not already set for today + local delay_seconds = tonumber(state.full_update_delay_seconds) or 0 + + if delay_seconds == 0 or state.full_update_delay_date ~= today then + -- First check of the day - generate random delay + delay_seconds = scheduler_get_random_full_update_delay() + -- Save delay for rest of the day + state.full_update_delay_seconds = tostring(delay_seconds) + state.full_update_delay_date = today + scheduler_save_state(state_file, state.saved_etag, state.saved_size, state.saved_modified) + end + + -- Check if current time has passed 3 AM + delay + local scheduled_time = scheduler_get_full_update_scheduled_time(delay_seconds) + local current_time = scheduler_get_current_time_of_day() + + if current_time >= scheduled_time then + -- Time to do full update + logger.log_notice( string.format("[v" .. scheduler.VERSION .. "] [SCHEDULER] Full update scheduled at 3:00 AM + %d minutes, now executing", delay_seconds / 60)) + return true, delay_seconds + else + -- Still waiting for scheduled time + local time_until = scheduled_time - current_time + logger.log_notice( string.format("[v" .. scheduler.VERSION .. "] [SCHEDULER] Hybrid: waiting for full update (scheduled in %d seconds)", time_until)) + return false, delay_seconds + end +end + +-- ============================================================================ +-- STARTUP HASH INITIALIZATION +-- ============================================================================ + +-- Check if hash buckets exist in Redis +--- @param plugin Plugin instance +--- @return boolean exists True if any buckets have data, false if all empty +local function scheduler_hashes_exist(plugin) + if not plugin.redisdb then + return false + end + + -- Check if any of the 16 buckets have data + local bucket_prefixes = {"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "A", "B", "C", "D", "E", "F"} + + for _, prefix in ipairs(bucket_prefixes) do + local bucket_key = "malware_scan:hashes:" .. prefix + local count = database.hlen(plugin, bucket_key) or 0 + if count > 0 then + return true -- At least one bucket has data + end + end + + return false -- All buckets are empty +end + +-- Initialize hashes on startup if they don't exist +-- Called on first scheduler run after BunkerWeb restart +-- Forces an immediate download and import if hashes are missing +--- @param plugin Plugin instance +--- @return boolean success True if initialization succeeded or was skipped, false on error +function scheduler.init_on_startup(plugin) + -- Check if plugin is enabled + if plugin.variables["USE_MALWARE_SCANNER"] ~= "yes" then + return true -- Plugin disabled, skip + end + + -- Check if custom hashes feature is enabled + if plugin.variables["MALWARE_SCAN_CUSTOM_HASHES_ENABLED"] ~= "yes" then + return true -- Feature disabled, skip + end + + -- Check if auto-update is enabled + if plugin.variables["MALWARE_SCAN_CUSTOM_HASHES_AUTO_UPDATE"] ~= "yes" then + return true -- Auto-update disabled, skip + end + + -- Validate Redis configuration + local redis_valid, redis_error = scheduler_validate_redis_config(plugin) + if not redis_valid then + logger.log_error("[v" .. scheduler.VERSION .. "] [SCHEDULER] [INIT] Redis configuration error: " .. redis_error) + return false + end + + -- Check if hashes already exist + if scheduler_hashes_exist(plugin) then + logger.log_notice( "[SCHEDULER] Hash buckets already populated - skipping startup initialization") + return true + end + + -- Hashes don't exist - force immediate import + logger.log_notice( "[SCHEDULER] Hash buckets are empty - forcing import on startup") + + -- Clear state file to force fresh download + local csv_path = plugin.variables["MALWARE_SCAN_CUSTOM_HASHES_CSV_PATH"] or "/var/cache/bunkerweb/malware-scan/hashes.csv" + local state_file = csv_path:gsub("%.csv$", ".state") + + -- Delete state file to force fresh download (ETag will be different) - native Lua + os.remove(state_file) -- Ignore errors - file may not exist + logger.log_notice( "[SCHEDULER] State file cleared - fresh download will be triggered") + + -- Now trigger the normal update process + -- This will download and import hashes + return scheduler.update_hashes(plugin) +end + +-- ============================================================================ +-- REDIS-BASED FULL DOWNLOAD TRACKING +-- ============================================================================ +-- Tracks two Redis keys: +-- 1. malware_scan:last_full_download (22h TTL) - tracks recent full download +-- 2. malware_scan:full_download_ever_done (30h TTL) - tracks if ever downloaded + +-- Check if full download is needed based on Redis TTL +--- @param plugin Plugin instance +--- @return boolean needs_download True if full download is needed +--- @return boolean is_first_time True if never downloaded before, false if refreshing +local function scheduler_needs_full_download(plugin) + if not plugin.redisdb then + return true, true -- No Redis, default to allowing full download (treat as first time) + end + + local full_key = "malware_scan:last_full_download" + local full_key_ever = "malware_scan:full_download_ever_done" + + local exists = database.exists(plugin, full_key) + local ever_done = database.exists(plugin, full_key_ever) + + if exists == 0 then + -- Key doesn't exist or expired (older than 22 hours) + local is_first_time = (ever_done == 0) + return true, is_first_time + end + + -- Key exists and hasn't expired yet + return false, false +end + +-- Record full download timestamp in Redis with 22-hour TTL +-- Also sets a marker that full download has been done at least once +--- @param plugin Plugin instance +local function scheduler_record_full_download(plugin) + if not plugin.redisdb then + return + end + + local full_key = "malware_scan:last_full_download" + local full_key_ever = "malware_scan:full_download_ever_done" + local timestamp = os.date("!%Y-%m-%dT%H:%M:%SZ") --[[@as string]] + + -- Set with 22-hour TTL (79200 seconds) + database.setex(plugin, full_key, 79200, timestamp) + + -- Set marker with 30-hour TTL (108000 seconds) to track if ever done + database.setex(plugin, full_key_ever, 108000, timestamp) + + logger.log_notice( "[SCHEDULER] Full download recorded in Redis with 22h TTL: " .. timestamp) +end + +-- ============================================================================ +-- SCHEDULER MAIN FUNCTION +-- ============================================================================ + +-- Scheduled task for periodic MalwareBazaar hash updates +-- Called by BunkerWeb scheduler at configured intervals +-- +-- New behavior: +-- - Run recent files download every 5 minutes for 55 minutes (11 total downloads) +-- - Each download has 1-45 second random delay to distribute server load +-- - After 55 minutes complete, check for full download (based on 22h TTL) +-- - Full download behavior: +-- * First time (never downloaded): immediate download +-- * Refresh (expired after 22h): random delay of 1-14 minutes before download +-- +--- @param plugin Plugin instance +--- @return boolean success True on success, false on failure +function scheduler.update_hashes(plugin) + -- Check if plugin is enabled (auto-update requires main plugin to be enabled) + if plugin.variables["USE_MALWARE_SCANNER"] ~= "yes" then + return true -- Plugin disabled, skip auto-update + end + + -- Check if auto-update is explicitly enabled + if plugin.variables["MALWARE_SCAN_CUSTOM_HASHES_AUTO_UPDATE"] ~= "yes" then + return true -- Auto-update disabled + end + + -- Check if custom hashes feature is enabled + if plugin.variables["MALWARE_SCAN_CUSTOM_HASHES_ENABLED"] ~= "yes" then + return true -- Custom hashes disabled + end + + -- Validate Redis configuration (required for custom hashes) + local redis_valid, redis_error = scheduler_validate_redis_config(plugin) + if not redis_valid then + logger.log_error("[v" .. scheduler.VERSION .. "] [SCHEDULER] [UPDATE] Redis configuration error: " .. redis_error) + return false + end + + logger.log_notice("[v" .. scheduler.VERSION .. "] [SCHEDULER] Starting MalwareBazaar hash update") + + -- Get configuration + local base_url = plugin.variables["MALWARE_SCAN_CUSTOM_HASHES_UPDATE_URL"] + local csv_path = plugin.variables["MALWARE_SCAN_CUSTOM_HASHES_CSV_PATH"] or "/var/cache/bunkerweb/malware-scan/hashes.csv" + local state_file = csv_path:gsub("%.csv$", ".state") + local zip_file = csv_path:gsub("%.csv$", ".zip") + + -- Helper function to download and import a file + local function download_and_import(url, file_type) + logger.log_notice( "[SCHEDULER] Checking remote file metadata for " .. file_type .. " update") + local remote_etag, remote_size, remote_modified = scheduler_get_remote_metadata(url) + + if not remote_etag then + logger.log_error( "[SCHEDULER] Failed to get remote metadata for " .. file_type) + return false + end + + logger.log_notice( "[SCHEDULER] Remote (" .. file_type .. "): ETag=" .. remote_etag .. ", Size=" .. (remote_size or "?")) + + -- Load saved state + local saved_state = scheduler_load_state(state_file) + + -- Check if file has changed + if saved_state.saved_etag == remote_etag and saved_state.saved_size == remote_size then + logger.log_notice( "[SCHEDULER] " .. file_type .. " file unchanged - skipping download") + return true + end + + -- File changed - download new version + logger.log_notice( "[SCHEDULER] " .. file_type .. " file changed - downloading fresh copy") + if not scheduler_download_file(url, zip_file) then + logger.log_error( "[SCHEDULER] " .. file_type .. " download failed") + return false + end + + logger.log_notice( "[SCHEDULER] " .. file_type .. " download complete") + + -- Extract and import + logger.log_notice( "[SCHEDULER] Extracting and importing " .. file_type .. " CSV") + local added, duplicates = scheduler_import_from_zip(zip_file, plugin) + + logger.log_notice( "[SCHEDULER] " .. file_type .. " import complete: added=" .. added .. ", duplicates=" .. duplicates) + + -- Save state with ETag + local state_to_save = scheduler_load_state(state_file) + state_to_save.saved_etag = remote_etag + state_to_save.saved_size = remote_size + state_to_save.saved_modified = remote_modified + + scheduler_save_state(state_file, state_to_save) + + return true + end + + -- PHASE 1: Run recent files download every 5 minutes for 55 minutes (11 total downloads) + -- Downloads occur at: 0, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55 minutes + -- Each download has a random 1-45 second delay to distribute server load + local recent_url = scheduler_get_download_url(base_url, "recent") + local total_downloads = 11 + local sleep_interval = 300 -- 5 minutes = 300 seconds + + logger.log_notice( "[SCHEDULER] Phase 1: Starting 55-minute recent files update cycle (" .. total_downloads .. " downloads)") + + for i = 1, total_downloads do + -- Add random delay (1-45 seconds) to distribute server load + -- Prevents all instances from hitting server at exactly :00, :05, :10, etc. + math.randomseed(os.time() + ngx.worker.pid()) + local random_delay = math.random(1, 45) + logger.log_notice( string.format("[v" .. scheduler.VERSION .. "] [SCHEDULER] Random delay: %d seconds (load distribution)", random_delay)) + ngx.sleep(random_delay) + + logger.log_notice( string.format("[v" .. scheduler.VERSION .. "] [SCHEDULER] Recent download %d/%d (at +%d minutes +%ds)", i, total_downloads, (i-1) * 5, random_delay)) + + local success = download_and_import(recent_url, "recent") + if not success then + logger.log_error( string.format("[v" .. scheduler.VERSION .. "] [SCHEDULER] Recent download %d/%d failed - continuing", i, total_downloads)) + end + + -- Sleep between downloads (but not after the last one) + if i < total_downloads then + logger.log_notice( string.format("[v" .. scheduler.VERSION .. "] [SCHEDULER] Sleeping 5 minutes before next download (completed %d/%d)", i, total_downloads)) + ngx.sleep(sleep_interval) + end + end + + logger.log_notice( "[SCHEDULER] Phase 1 complete: 55-minute recent files update cycle finished") + + -- PHASE 2: Check if full download is needed (based on 22-hour TTL) + local needs_download, is_first_time = scheduler_needs_full_download(plugin) + + if needs_download then + if is_first_time then + -- First time download - do it immediately + logger.log_notice( "[SCHEDULER] Phase 2: Full download needed (never done before) - downloading immediately") + else + -- Refresh download - add random delay to distribute server load + math.randomseed(os.time() + ngx.worker.pid()) + local refresh_delay = math.random(1, 14) * 60 -- 1-14 minutes in seconds + logger.log_notice( string.format("[v" .. scheduler.VERSION .. "] [SCHEDULER] Phase 2: Full download refresh needed (expired) - random delay: %d minutes", refresh_delay / 60)) + ngx.sleep(refresh_delay) + end + + local full_url = scheduler_get_download_url(base_url, "full") + local success = download_and_import(full_url, "full") + + if success then + -- Record full download timestamp with 22h TTL + scheduler_record_full_download(plugin) + else + logger.log_error( "[SCHEDULER] Full download failed") + end + else + logger.log_notice( "[SCHEDULER] Phase 2: Skipping full download (still fresh, less than 22 hours old)") + end + + -- Get total count across all 16 buckets + local total_count = 0 + local bucket_prefixes = {"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "A", "B", "C", "D", "E", "F"} + + for _, prefix in ipairs(bucket_prefixes) do + local bucket_key = "malware_scan:hashes:" .. prefix + local count = database.hlen(plugin, bucket_key) or 0 + total_count = total_count + count + end + + logger.log_notice( "[SCHEDULER] Total hashes in database: " .. total_count .. " across 16 buckets") + + return true +end + +-- ============================================================================ +-- COMBINED SCHEDULER WORKFLOW +-- ============================================================================ + +--- Combined scheduler workflow: Run initialization and hash updates. +--- Convenience function for BunkerWeb scheduler hook to perform all scheduler tasks in one call. +--- Handles both startup initialization and periodic hash updates. +--- +--- Example: +--- scheduler.run(plugin) +--- +--- @param plugin Plugin instance (provides access to self.logger and self.variables) +--- @return boolean success Always returns true to avoid breaking other scheduled tasks +function scheduler.run(plugin) + -- Initialize hashes on startup if they don't exist + -- This runs on first scheduler invocation after BunkerWeb restart + local init_ok = scheduler.init_on_startup(plugin) + if not init_ok then + logger.log_warn( "[SCHEDULER] Startup initialization failed - continuing with regular update") + end + + -- Run hash update if auto-update is enabled + local update_ok = scheduler.update_hashes(plugin) + if not update_ok then + logger.log_error( "[SCHEDULER] Hash update failed") + end + + -- Always return true to avoid breaking other scheduled tasks + return true +end + +-- Return module +return scheduler diff --git a/malware-scan/malware_scan_sentinelone.lua b/malware-scan/malware_scan_sentinelone.lua new file mode 100644 index 0000000..66cb75c --- /dev/null +++ b/malware-scan/malware_scan_sentinelone.lua @@ -0,0 +1,270 @@ +-- BunkerWeb Malware Scan - SentinelOne Scanner Module +-- Handles all SentinelOne API integration for threat hash lookups + +local cjson = require("cjson") +local http = require("resty.http") + +local logger = require("malware_scan_logger") +local ngx = ngx +local decode = cjson.decode +local encode = cjson.encode +local http_new = http.new +local tostring = tostring +local string = string + +-- Module table +local sentinelone = {} + +-- Cache module reference (will be set when module is loaded) +local cache = nil + +-- Module version +sentinelone.VERSION = "0.8.0" + +-- Plugin version for User-Agent identification +local VERSION = "0.8.0" +local USER_AGENT = "bunkerweb - https://github.com/bunkerity/bunkerweb - malware-scan module v" .. VERSION + +-- Set cache module reference (called during initialization) +function sentinelone.set_cache_module(cache_module) + cache = cache_module +end + +-- ============================================================================ +-- SENTINELONE THREAT LOOKUP +-- ============================================================================ + +-- Check SHA256 hash against SentinelOne threat database (standard hash_check interface). +-- This is the standard interface for hash-only lookups used by multipart scanning. +-- Returns success boolean and result string ("clean" or threat info). +-- +--- @param plugin Plugin Plugin instance +--- @param checksum string SHA256 hash to check +--- @param file_size number|nil Optional file size for cache verification +--- @return boolean success True if check succeeded (even if clean) +--- @return string result "clean", "api_error", "api_circuit_breaker", or threat info +function sentinelone.hash_check(plugin, checksum, file_size) + return sentinelone.check_sentinelone(plugin, checksum, file_size) +end + +-- Check file hash against SentinelOne threat database. +-- Takes SHA256 checksum and optional file_size, returns success boolean and result. +-- Returns "clean" if not found, or threat info string if found. +--- @return boolean success True if check succeeded (even if clean) +--- @return string result "clean", "api_error", "api_circuit_breaker", or threat info +function sentinelone.check_sentinelone(plugin, checksum, file_size) + -- Check if SentinelOne is enabled + if plugin.variables["MALWARE_SCAN_API_USE_SENTINELONE"] ~= "yes" then + return true, "clean" + end + + -- Check API error circuit breaker (prevents repeated failed calls) + if cache then + local should_skip, error_type, remaining = cache.api_error_should_skip(plugin, "sentinelone") + if should_skip then + logger.log_error( string.format( + "SentinelOne API temporarily disabled due to previous error: %s (retry in %ds)", + error_type or "unknown", remaining + )) + return true, "api_circuit_breaker" + end + end + + -- Check cache and acquire lock if available (atomic locking to prevent duplicate API calls) + local should_query = true + local lock_acquired = false + local result = "clean" + + if cache then + -- Step 1: Check if result already cached + local ok, cached, cached_size = cache.sentinelone_check(plugin, checksum, file_size) + if ok and cached then + logger.log_error( "file checksum " .. checksum .. " found in SentinelOne cache: " .. cached) + return true, cached + end + + -- Step 2: Try to acquire lock atomically + lock_acquired = cache.sentinelone_try_lock(plugin, checksum) + + if not lock_acquired then + -- Another request is querying SentinelOne - wait for result + plugin:log_debug("[v" .. VERSION .. "] [CACHE_LOCK] Another request querying SentinelOne, waiting for result...") + local wait_ok, wait_result, wait_size = cache.sentinelone_wait_for_result(plugin, checksum, file_size) + + if wait_ok then + -- Got result from concurrent query + logger.log_error( string.format( + "[CACHE_LOCK] Got SentinelOne result from concurrent query: %s", wait_result)) + return true, wait_result + else + -- Timeout - fall back to querying independently + logger.log_warn( "[CACHE_LOCK] Timeout waiting for SentinelOne result - querying independently") + lock_acquired = cache.sentinelone_try_lock(plugin, checksum) + should_query = true + end + end + end + + if should_query then + -- Request from SentinelOne API + logger.log_error( "sending request to SentinelOne API for checksum: " .. checksum) + local found, threat_info + local ok, found, threat_info, error_type = sentinelone.api_request(plugin, checksum) + if not ok then + logger.log_error( "SentinelOne API request failed: " .. found) + + -- Record API error in circuit breaker if it's a persistent error + -- Circuit break on: 4xx client errors (auth, config), 5xx server errors + if cache and error_type then + local should_circuit_break = ( + error_type == "invalid_token" or error_type == "invalid_url" or + error_type:match("^http_4%d%d$") or -- Match any 4xx error (401, 403, 429, etc.) + error_type:match("^http_5%d%d$") -- Match any 5xx error (500, 502, 503, etc.) + ) + if should_circuit_break then + cache.api_error_failure(plugin, "sentinelone", error_type) + end + end + + -- Release lock before returning on error + if lock_acquired and cache then + cache.sentinelone_release_lock(plugin, checksum) + end + + return true, "api_error" + end + + -- API call succeeded, reset circuit breaker + if cache then + cache.api_error_success(plugin, "sentinelone") + end + + result = "clean" + if found then + logger.log_error( "file hash found in SentinelOne threat database") + result = threat_info + logger.log_error( "SentinelOne result: " .. result) + else + logger.log_error( "file checksum " .. checksum .. " not found in SentinelOne") + end + + -- Add to cache if cache module is available + if cache then + local ok, err = cache.sentinelone_add(plugin, checksum, result, file_size) + if not ok then + logger.log_error( "can't cache SentinelOne result: " .. err) + end + end + + -- Track new result for composite Redis write + if plugin.ctx.bw.new_scan_results then + plugin.ctx.bw.new_scan_results.sentinelone = result + end + end + + -- Release lock after query completes (all paths: success, error, clean) + if lock_acquired and cache then + cache.sentinelone_release_lock(plugin, checksum) + end + + return true, result +end + +-- ============================================================================ +-- SENTINELONE API REQUEST +-- ============================================================================ + +-- Make HTTP request to SentinelOne API (standard api_request interface). +-- Takes hash (SHA256), returns success boolean, found boolean, threat info, and error_type. +-- This is the standard interface for API requests following domain_action pattern. +--- @return boolean success True if API call succeeded +--- @return boolean|string found_or_error Boolean when success=true (found/not found), error message when success=false +--- @return string|nil threat_info Threat information when found=true, nil otherwise +--- @return string|nil error_type Error type when success=false, nil otherwise +function sentinelone.api_request(plugin, hash) + -- Get HTTP client + local httpc, err = http_new() + if not httpc then + return false, err, nil, "http_error" + end + + -- Get API token and management URL (required for SentinelOne) and check for default values + local api_token = plugin.variables["MALWARE_SCAN_API_SENTINELONE_TOKEN"] + if not api_token or api_token == "" or api_token == "your_api_key" or api_token == "your_token" then + return false, "SentinelOne API token is required but not configured or using default value", nil, "invalid_token" + end + + local mgmt_url = plugin.variables["MALWARE_SCAN_API_SENTINELONE_URL"] + if not mgmt_url or mgmt_url == "" or mgmt_url == "https://your-console.example.com" then + return false, "SentinelOne Management URL is required but not configured or using default value", nil, "invalid_url" + end + + -- Remove trailing slash from management URL if present + mgmt_url = mgmt_url:gsub("/$", "") + + -- Build API endpoint for threat lookup by hash + -- Using /web/api/v2.1/threats endpoint with contentHash filter + local api_url = mgmt_url .. "/web/api/v2.1/threats" + + -- Send request with contentHashes filter + local res + res, err = httpc:request_uri(api_url, { + method = "GET", + headers = { + ["Authorization"] = "ApiToken " .. api_token, + ["Content-Type"] = "application/json", + ["accept"] = "application/json", + ["User-Agent"] = USER_AGENT, + }, + query = { + contentHashes = hash, + limit = "1" + } + }) + + if not res then + return false, err, nil, "http_error" + end + + -- Check status + if res.status ~= 200 then + err = "received status " .. tostring(res.status) .. " from SentinelOne API" + local error_type = "http_" .. tostring(res.status) + local decode_ok, data = pcall(decode, res.body) + if decode_ok and data then + err = err .. " with data " .. encode(data) + end + return false, err, nil, error_type + end + + -- Parse JSON response + local decode_ok, data = pcall(decode, res.body) + if not decode_ok then + return false, "failed to decode JSON: " .. data, nil, "json_error" + end + + -- Check if threats were found + if not data.data or #data.data == 0 then + -- Hash not found in SentinelOne + return true, false, nil, nil + end + + -- Extract threat information from first result + local threat = data.data[1] + local threat_name = threat.threatName or threat.maliciousProcessArguments or "unknown threat" + local classification = threat.classification or "unknown" + local confidence_level = threat.confidenceLevel or "unknown" + + -- Build threat info string + local threat_info = string.format( + "%s (Classification: %s, Confidence: %s)", + threat_name, + classification, + confidence_level + ) + + return true, true, threat_info, nil +end + +-- Return module +return sentinelone diff --git a/malware-scan/malware_scan_ui_metrics.lua b/malware-scan/malware_scan_ui_metrics.lua new file mode 100644 index 0000000..55f26b5 --- /dev/null +++ b/malware-scan/malware_scan_ui_metrics.lua @@ -0,0 +1,188 @@ +-- BunkerWeb Malware Scan - UI Metrics Tracking Module +-- Tracks scan statistics, detections, and cache performance for UI dashboard +-- Uses BunkerWeb's built-in metrics system (self:set_metric) +-- +-- Metrics tracked: +-- - counters: +-- - malware_detected: Total malware files detected +-- - files_scanned: Total files scanned +-- - cache_hits: Total cache hits (all scanners) +-- - cache_hits_: Per-scanner cache hits +-- - signature_: Per-signature detection count +-- - scanner_: Per-scanner detection count +-- - tables: +-- - detections: Detailed detection history + +local cjson = require("cjson") + +-- Module table +local ui_metrics = {} + +-- Module version +ui_metrics.VERSION = "0.8.0" + +-- Ngx constants (with fallback for testing) +local ngx = ngx or {} +local ERR = ngx.ERR or 3 +local WARN = ngx.WARN or 4 +local INFO = ngx.INFO or 6 + +-- ============================================================================ +-- COUNTER METRICS +-- ============================================================================ + +-- Track file scan metric. +-- Increments counter for each file upload scanned. +-- +--- @param plugin Plugin The plugin instance +function ui_metrics.track_file_scanned(plugin) + plugin:set_metric("counters", "files_scanned", 1) +end + +-- Track cache hit metric. +-- Increments counter when a cached result is used. +-- +--- @param plugin Plugin The plugin instance +--- @param scanner_name Name of scanner that had cache hit (e.g., "clamav", "virustotal") +function ui_metrics.track_cache_hit(plugin, scanner_name) + plugin:set_metric("counters", "cache_hits", 1) + + -- Also track per-scanner cache hits + if scanner_name then + local scanner_key = scanner_name:gsub("[^%w_]", "_"):lower() + plugin:set_metric("counters", "cache_hits_" .. scanner_key, 1) + end +end + +-- Track malware detection metric. +-- Increments multiple counters: +-- - malware_detected (total) +-- - signature_ (per-signature) +-- - scanner_ (per-scanner) +-- +--- @param plugin Plugin The plugin instance +--- @param scanner_name string Name of scanner that detected malware (e.g., "ClamAV", "VirusTotal") +--- @param signature string The malware signature/name detected +function ui_metrics.track_malware_detected(plugin, scanner_name, signature) + -- Increment total malware counter + plugin:set_metric("counters", "malware_detected", 1) + + -- Increment per-scanner counter + if scanner_name then + -- Sanitize scanner name: "ClamAV (cached)" -> "clamav_cached" + local scanner_key = scanner_name:gsub("[^%w_]", "_"):lower() + plugin:set_metric("counters", "scanner_" .. scanner_key, 1) + end + + -- Increment per-signature counter + if signature and signature ~= "" then + -- Sanitize signature name for metric key (remove special chars, limit length) + local sig_key = signature:gsub("[^%w_%-]", "_"):lower() + if #sig_key > 100 then + sig_key = sig_key:sub(1, 100) + end + plugin:set_metric("counters", "signature_" .. sig_key, 1) + end +end + +-- ============================================================================ +-- DETECTION HISTORY TABLE +-- ============================================================================ + +-- Extract filename from multipart upload headers or request. +-- Tries to parse Content-Disposition header to get original filename. +-- +--- @param plugin Plugin The plugin instance +--- @return string filename Original filename or "uploaded_file" if not found +local function ui_metrics_extract_filename(plugin) + -- Try to get filename from request headers + if plugin.ctx.bw.http_content_disposition then + local filename = plugin.ctx.bw.http_content_disposition:match('filename="([^"]+)"') + if filename then + return filename + end + end + + -- Fallback to generic name + return "uploaded_file" +end + +-- Get client country from GeoIP if available. +-- +--- @param plugin Plugin The plugin instance +--- @return string country Country code or "unknown" if not available +local function ui_metrics_get_client_country(plugin) + -- Try to get country from BunkerWeb GeoIP integration + if plugin.ctx.bw.country_code then + return plugin.ctx.bw.country_code + end + + return "unknown" +end + +-- Build comprehensive detection data structure. +-- Collects all available information about the detection for metrics. +-- +--- @param plugin Plugin The plugin instance +--- @param scanner_name string Name of scanner (e.g., "ClamAV") +--- @param signature string Malware signature +--- @param checksum string File SHA256 hash +--- @param detections table Table of all detections from multiple scanners +--- @return table detection_data Detection data structure with metadata +local function ui_metrics_build_detection_data(plugin, scanner_name, signature, checksum, detections) + local detection_id = string.format("%d_%s", + os.time(), + checksum and checksum:sub(1, 8) or "unknown" + ) + + return { + id = detection_id, + timestamp = os.time(), + signature = signature or "unknown", + scanner = scanner_name or "unknown", + sha256 = checksum or "", + filename = ui_metrics_extract_filename(plugin), + ip = plugin.ctx.bw.remote_addr or "unknown", + country = ui_metrics_get_client_country(plugin), + url = plugin.ctx.bw.uri or "/", + method = plugin.ctx.bw.request_method or "POST", + -- Include count of all scanner detections + detection_count = detections and #detections or 1 + } +end + +-- Add detection to history table for UI display. +-- Stores structured detection data in BunkerWeb metrics system. +-- +--- @param plugin Plugin The plugin instance +--- @param scanner_name string Name of scanner +--- @param signature string Malware signature +--- @param checksum string File SHA256 hash +--- @param detections table Table of all detections from multiple scanners +function ui_metrics.add_detection(plugin, scanner_name, signature, checksum, detections) + local detection_data = ui_metrics_build_detection_data( + plugin, scanner_name, signature, checksum, detections + ) + + -- Store in BunkerWeb metrics tables + -- The UI will read this via get_metrics("malware-scan") + plugin:set_metric("tables", "detections", detection_data) +end + +-- Track complete malware detection with all metrics. +-- Convenience function that tracks counters and adds to detection table. +-- +--- @param plugin Plugin The plugin instance +--- @param scanner_name string Name of scanner +--- @param signature string Malware signature +--- @param checksum string File SHA256 hash +--- @param detections table Table of all detections +function ui_metrics.track_detection_complete(plugin, scanner_name, signature, checksum, detections) + -- Track counter metrics + ui_metrics.track_malware_detected(plugin, scanner_name, signature) + + -- Add to detection history table + ui_metrics.add_detection(plugin, scanner_name, signature, checksum, detections) +end + +return ui_metrics diff --git a/malware-scan/malware_scan_utils.lua b/malware-scan/malware_scan_utils.lua new file mode 100644 index 0000000..4120749 --- /dev/null +++ b/malware-scan/malware_scan_utils.lua @@ -0,0 +1,538 @@ +-- BunkerWeb Malware Scan - Utility Functions +-- Common utility functions for validation, sanitization, and type checking + +local utils = {} + +-- Module version +utils.VERSION = "0.8.0" + +-- ============================================================================ +-- TYPE ALIASES (BunkerWeb Variable Definition Types) +-- ============================================================================ +-- These type aliases represent BunkerWeb variable configuration types +-- from plugin.json. They're used in type annotations for validation functions. + +---@alias Input any +---@alias Default any +---@alias Minimum number +---@alias Maximum number +---@alias Variable string +---@alias First number +---@alias Second number +---@alias Numerator number +---@alias Denominator number + +-- ============================================================================ +-- NUMERICAL VALIDATION +-- ============================================================================ + +-- Validate and sanitize a numerical value. +-- Ensures value is a positive number within optional min/max bounds. +-- Returns sanitized number or default value if invalid. +-- +--- @overload fun(value: Input, default: Default): number +--- @overload fun(value: Input, default: Default, min: Minimum): number +--- @overload fun(value: Input, default: Default, min: Minimum, max: Maximum): number +--- @param value Input value to validate (any type) +--- @param default Default value if validation fails +--- @param min Minimum allowed value (optional) +--- @param max Maximum allowed value (optional) +--- @param name Variable name for error messages (optional) +--- @return number result Validated number, or default if invalid +function utils.validate_number(value, default, min, max, name) + -- Convert to number if string + local num = tonumber(value) + + -- Check if conversion succeeded + if not num then + return default + end + + -- Check for NaN or infinity + if num ~= num or num == math.huge or num == -math.huge then + return default + end + + -- Check minimum bound + if min and num < min then + return default + end + + -- Check maximum bound + if max and num > max then + return default + end + + return num +end + +-- Validate a positive integer (whole number > 0). +-- Returns sanitized integer or default value if invalid. +-- +--- @param value Input value to validate +--- @param default Default value if validation fails +--- @param max Maximum allowed value (optional) +--- @param name Variable name for error messages (optional) +--- @return number result Validated positive integer, or default +function utils.validate_positive_integer(value, default, max, name) + local num = utils.validate_number(value, default, 1, max, name) + + -- Ensure it's a whole number + if num ~= math.floor(num) then + return default + end + + return num +end + +-- Validate a non-negative integer (whole number >= 0). +-- Returns sanitized integer or default value if invalid. +-- +--- @param value Input value to validate +--- @param default Default value if validation fails +--- @param max Maximum allowed value (optional) +--- @param name Variable name for error messages (optional) +--- @return number result Validated non-negative integer, or default +function utils.validate_nonnegative_integer(value, default, max, name) + local num = utils.validate_number(value, default, 0, max, name) + + -- Ensure it's a whole number + if num ~= math.floor(num) then + return default + end + + return num +end + +-- Validate file size (positive integer with reasonable upper bound). +-- Returns sanitized file size or 0 if invalid. +-- +--- @param size Input size value +--- @param max_size Maximum allowed size (default: 10GB) +--- @return number result Validated file size in bytes (>= 0) +function utils.validate_file_size(size, max_size) + max_size = max_size or (10 * 1024 * 1024 * 1024) -- Default: 10GB + return utils.validate_nonnegative_integer(size, 0, max_size, "file_size") +end + +-- Validate disk space (positive integer). +-- Returns sanitized space value or 0 if invalid. +-- +--- @param space Input space value +--- @return number result Validated disk space in bytes (>= 0) +function utils.validate_disk_space(space) + -- Allow up to 1PB (petabyte) as upper bound + local max_space = 1024 * 1024 * 1024 * 1024 * 1024 + return utils.validate_nonnegative_integer(space, 0, max_space, "disk_space") +end + +-- Validate TTL (time-to-live) value in seconds. +-- Returns sanitized TTL or default if invalid. +-- +--- @param ttl Input TTL value +--- @param default Default TTL if validation fails (default: 86400 = 1 day) +--- @param max_ttl Maximum allowed TTL (default: 31536000 = 1 year) +--- @return number result Validated TTL in seconds +function utils.validate_ttl(ttl, default, max_ttl) + default = default or 86400 -- 1 day + max_ttl = max_ttl or 31536000 -- 1 year + return utils.validate_positive_integer(ttl, default, max_ttl, "ttl") +end + +-- Validate timeout value in milliseconds. +-- Returns sanitized timeout or default if invalid. +-- +--- @param timeout Input timeout value +--- @param default Default timeout if validation fails (default: 30000 = 30 seconds) +--- @param max_timeout Maximum allowed timeout (default: 300000 = 5 minutes) +--- @return number result Validated timeout in milliseconds +function utils.validate_timeout(timeout, default, max_timeout) + default = default or 30000 -- 30 seconds + max_timeout = max_timeout or 300000 -- 5 minutes + return utils.validate_positive_integer(timeout, default, max_timeout, "timeout") +end + +-- Validate retry count. +-- Returns sanitized retry count or default if invalid. +-- +--- @param retries Input retry count +--- @param default Default retry count if validation fails (default: 3) +--- @param max_retries Maximum allowed retries (default: 10) +--- @return number result Validated retry count +function utils.validate_retry_count(retries, default, max_retries) + default = default or 3 + max_retries = max_retries or 10 + return utils.validate_nonnegative_integer(retries, default, max_retries, "retries") +end + +-- Validate port number. +-- Returns sanitized port or default if invalid. +-- +--- @param port Input port number +--- @param default Default port if validation fails +--- @return number result Validated port number (1-65535) +function utils.validate_port(port, default) + return utils.validate_positive_integer(port, default, 65535, "port") +end + +-- ============================================================================ +-- SAFE ARITHMETIC OPERATIONS +-- ============================================================================ + +-- Safe addition with overflow protection. +-- Returns sum or max value if overflow would occur. +-- +--- @overload fun(a: First, b: Second): number +--- @param a First number +--- @param b Second number +--- @param max Maximum allowed result (optional) +--- @return number result Sum of a + b, capped at max +function utils.safe_add(a, b, max) + a = utils.validate_number(a, 0) + b = utils.validate_number(b, 0) + + local sum = a + b + + -- Check for overflow or infinity + if sum == math.huge or sum ~= sum then + return max or a -- Return max or original value + end + + -- Cap at maximum if specified + if max and sum > max then + return max + end + + return sum +end + +-- Safe multiplication with overflow protection. +-- Returns product or max value if overflow would occur. +-- +--- @overload fun(a: First, b: Second): number +--- @param a First number +--- @param b Second number +--- @param max Maximum allowed result (optional) +--- @return number result Product of a * b, capped at max +function utils.safe_multiply(a, b, max) + a = utils.validate_number(a, 0) + b = utils.validate_number(b, 0) + + local product = a * b + + -- Check for overflow or infinity + if product == math.huge or product ~= product then + return max or a -- Return max or original value + end + + -- Cap at maximum if specified + if max and product > max then + return max + end + + return product +end + +-- Safe division with zero-check. +-- Returns quotient or default if division by zero. +-- +--- @overload fun(a: Numerator, b: Denominator): number +--- @param a Numerator +--- @param b Denominator +--- @param default Default value if b is zero (default: 0) +--- @return number result Quotient of a / b, or default if b is zero +function utils.safe_divide(a, b, default) + a = utils.validate_number(a, 0) + b = utils.validate_number(b, 0) + default = default or 0 + + -- Prevent division by zero + if b == 0 then + return default + end + + local quotient = a / b + + -- Check for infinity or NaN + if quotient == math.huge or quotient == -math.huge or quotient ~= quotient then + return default + end + + return quotient +end + +-- ============================================================================ +-- PATH SANITIZATION (Command Injection Prevention) +-- ============================================================================ + +--- Validate path for shell command safety. +-- Checks if path contains only safe characters to prevent command injection. +-- Allows: alphanumeric, slash, dash, underscore, dot, space +-- Blocks: semicolons, pipes, backticks, $(), redirects, etc. +--- +--- @param path string Path to validate +--- @return boolean, string|nil +function utils.validate_safe_path(path) + if not path or path == "" then + return false, "path is empty" + end + + -- Check for dangerous characters that could enable command injection + -- Allow: alphanumeric, /, -, _, ., and space + -- Block: ; | & $ ` ( ) < > \ " ' * ? [ ] { } ! \n \r + if path:match("[^%w%/%-%_%. ]") then + return false, "path contains unsafe characters (potential command injection)" + end + + -- Check for path traversal attempts + if path:match("%.%.") then + return false, "path contains '..' (potential path traversal)" + end + + -- Check for suspicious patterns + if path:match("%$%(") or path:match("`") then + return false, "path contains command substitution characters" + end + + return true, nil +end + +--- Shell-escape a path for safe use in commands. +-- Wraps path in single quotes and escapes any single quotes within. +-- This is the most secure way to pass paths to shell commands. +--- +--- @param path string Path to escape +--- @return string escaped_path +function utils.shell_escape_path(path) + if not path then + return "''" + end + + -- Replace single quotes with '\'' (end quote, escaped quote, start quote) + local escaped = path:gsub("'", "'\\''") + + -- Wrap in single quotes + return "'" .. escaped .. "'" +end + +--- Validate and escape path for shell commands. +-- Combines validation and escaping for safe shell command usage. +-- Returns escaped path or nil with error message. +--- +--- @param path string Path to validate and escape +--- @return string|nil, string|nil +function utils.validate_and_escape_path(path) + local valid, err = utils.validate_safe_path(path) + if not valid then + return nil, err + end + + return utils.shell_escape_path(path), nil +end + +--- +--- Sanitize string for safe logging. +--- Prevents log injection attacks by escaping control characters +--- and truncating long strings. +--- +--- @param str string String to sanitize +--- @return string Sanitized string safe for logging +function utils.log_sanitize(str) + if not str or str == "" then + return "" + end + + -- Convert to string if not already + local sanitized = tostring(str) + + -- Escape control characters (CR, LF, TAB, NULL, etc.) + sanitized = sanitized:gsub("[\r\n\t%z\1-\31]", function(c) + return string.format("\\x%02x", string.byte(c)) + end) + + -- Truncate to prevent log flooding + if #sanitized > 200 then + sanitized = sanitized:sub(1, 197) .. "..." + end + + return sanitized +end + +--- +--- Validate filename for security. +--- Checks for path traversal, null bytes, and other injection attacks. +--- +--- @param filename string Filename to validate +--- @return boolean, string|nil +function utils.file_validate_name(filename) + if not filename or filename == "" then + return false, "empty filename" + end + + -- Check for path traversal attempts + if filename:match("%.%.") then + return false, "path traversal attempt (..)" + end + + -- Check for absolute paths + if filename:match("^/") or filename:match("^\\") then + return false, "absolute path not allowed" + end + + -- Check for null bytes + if filename:match("%z") then + return false, "null byte in filename" + end + + -- Check for other control characters + if filename:match("[\1-\31]") then + return false, "control characters in filename" + end + + -- Check length (max 255 for most filesystems) + if #filename > 255 then + return false, "filename too long" + end + + return true, nil +end + +-- ============================================================================ +-- URL PARSING AND VALIDATION +-- ============================================================================ + +--- Parse and validate webhook URL. +-- Validates URL format and security (SSRF prevention, hostname validation). +-- Returns parsed components or nil with error message. +--- +--- @param url string URL to parse +--- @return table|nil, string|nil +function utils.parse_webhook_url(url) + -- Validate URL starts with http:// or https:// to prevent SSRF (CWE-918) + if not url:match("^https?://") then + return nil, "URL must start with http:// or https://" + end + + local scheme, host, port, path = url:match("^(https?)://([^/:]+):?(%d*)(.*)$") + + if not scheme or not host then + return nil, "invalid URL format" + end + + -- Validate hostname doesn't contain suspicious characters that could enable URL injection + -- Reject: spaces, quotes, angle brackets, percent signs (used in URL encoding attacks) + if host:match("[%s<>\"'%%]") then + return nil, "invalid characters in hostname" + end + + -- Validate port is in valid range (1-65535) + if port ~= "" then + local port_num = tonumber(port) + if not port_num or port_num < 1 or port_num > 65535 then + return nil, "invalid port number (must be 1-65535)" + end + port = tostring(port_num) + end + + -- Default ports + if port == "" then + port = (scheme == "https") and "443" or "80" + end + + -- Default path + if path == "" then + path = "/" + end + + return { + scheme = scheme, + host = host, + port = tonumber(port), + path = path, + ssl = (scheme == "https") + }, nil +end + +-- ============================================================================ +-- REDIS HASH BUCKETING +-- ============================================================================ + +--- Get bucketed Redis key for SHA256 hash. +-- Distributes 1M+ hashes across 16 bucket keys (0-9, A-F) for faster lookups. +-- Uses first character of SHA256 hash for bucketing. +--- +--- @param sha256 string SHA256 hash (first character used for bucketing) +--- @return string Bucketed hash key like "malware_scan:hashes:A" +function utils.get_hash_bucket(sha256) + if not sha256 or #sha256 == 0 then + return "malware_scan:hashes:0" + end + + -- Get first character of SHA256 (uppercase for consistency) + local first_char = sha256:sub(1, 1):upper() + + -- Create bucket key with first character suffix + return "malware_scan:hashes:" .. first_char +end + +--- Get all 16 hash bucket keys. +-- Returns array of all bucket keys for full database operations. +--- +--- @return table Array of all 16 bucket keys +function utils.get_all_hash_buckets() + local buckets = {} + -- Add numeric buckets (0-9) + for i = 0, 9 do + table.insert(buckets, "malware_scan:hashes:" .. i) + end + -- Add hex buckets (A-F) + for i = 10, 15 do + table.insert(buckets, "malware_scan:hashes:" .. string.char(65 + i - 10)) -- 65 = 'A' + end + return buckets +end + +-- ============================================================================ +-- DATA MASKING FOR SECURITY +-- ============================================================================ + +--- Mask sensitive data (API keys, webhook URLs, tokens) for safe logging. +--- Prevents accidental exposure of credentials in logs, error messages, or debug output. +--- Only shows full values if unmask_setting matches the specific phrase "I want to expose everything, I know what I'm doing". +--- +--- Masking rules: +--- - Empty or nil values: Returns empty string "" +--- - Short values (<=8 chars): Returns "***" +--- - Long values (>8 chars): Shows first 4 characters + "***" (e.g., "abc1***") +--- - Unmask override: If unmask_setting == "I want to expose everything, I know what I'm doing", returns original value +--- +--- Security note: Never log masked values without verifying unmask_setting is not set in production. +--- +--- Example: +--- utils.data_mask_sensitive("secret_key_12345", nil) โ†’ "secr***" +--- utils.data_mask_sensitive("abc", nil) โ†’ "***" +--- utils.data_mask_sensitive("secret", "I want to expose everything, I know what I'm doing") โ†’ "secret" +--- +--- @param value string|nil The sensitive value to mask (API key, token, webhook URL, etc.) +--- @param unmask_setting string|nil The MALWARE_SCAN_UNMASK_KEYS setting (only specific phrase disables masking) +--- @return string Masked string or original value if unmasking is enabled +function utils.data_mask_sensitive(value, unmask_setting) + -- Check for unmask override (exact phrase match required) + if unmask_setting == "I want to expose everything, I know what I'm doing" then + return value or "" + end + + -- Mask empty or nil values + if not value or value == "" then + return "" + end + + -- Simple masking: show first 4 chars + "***" for long strings, "***" for short strings + if #value > 8 then + return value:sub(1, 4) .. "***" + end + return "***" +end + +-- Return module +return utils diff --git a/malware-scan/malware_scan_virustotal.lua b/malware-scan/malware_scan_virustotal.lua new file mode 100644 index 0000000..c6f20ee --- /dev/null +++ b/malware-scan/malware_scan_virustotal.lua @@ -0,0 +1,531 @@ +-- BunkerWeb Malware Scan - VirusTotal Scanner Module +-- Handles all VirusTotal API integration for file hash lookups, uploads, and voting + +local cjson = require("cjson") +local http = require("resty.http") + +local logger = require("malware_scan_logger") +local ngx = ngx +local decode = cjson.decode +local encode = cjson.encode +local http_new = http.new +local tonumber = tonumber +local tostring = tostring + +-- Module table +local virustotal = {} + +-- Cache module reference (will be set when module is loaded) +local cache = nil + +-- File operations module reference (will be set when module is loaded) +local file_ops = nil + +-- Module version +virustotal.VERSION = "0.8.0" + +-- Plugin version for User-Agent identification +local VERSION = "0.8.0" +local USER_AGENT = "bunkerweb - https://github.com/bunkerity/bunkerweb - malware-scan module v" .. VERSION + +-- Set cache module reference (called during initialization) +function virustotal.set_cache_module(cache_module) + cache = cache_module +end + +-- Set file operations module reference (called during initialization) +function virustotal.set_file_ops_module(file_ops_module) + file_ops = file_ops_module +end + +-- ============================================================================ +-- VIRUSTOTAL HASH LOOKUP (MULTIPART-COMPATIBLE) +-- ============================================================================ + +-- Check SHA256 hash against VirusTotal database (no file path required). +-- This is the standard interface for hash-only lookups used by multipart scanning. +-- Returns success boolean and result string ("clean" or detection info). +-- +--- @param plugin Plugin Plugin instance +--- @param checksum string SHA256 hash to check +--- @param file_size number|nil Optional file size for cache verification +--- @return boolean success True if check succeeded (even if clean) +--- @return string result "clean", "api_error", "api_circuit_breaker", or detection info +function virustotal.hash_check(plugin, checksum, file_size) + -- Check if VirusTotal is enabled + if plugin.variables["MALWARE_SCAN_API_USE_VIRUSTOTAL"] ~= "yes" then + return true, "clean" + end + + -- Check if API key is configured + local api_key = plugin.variables["MALWARE_SCAN_API_VIRUSTOTAL_KEY"] + if not api_key or api_key == "" or api_key == "your_api_key" or api_key == "your_token" then + logger.log_error( "VirusTotal enabled but API key not configured") + return true, "clean" + end + + -- Check API error circuit breaker (prevents repeated failed calls) + if cache then + local should_skip, error_type, remaining = cache.api_error_should_skip(plugin, "virustotal") + if should_skip then + logger.log_error( string.format( + "VirusTotal API temporarily disabled due to previous error: %s (retry in %ds)", + error_type or "unknown", remaining + )) + return true, "api_circuit_breaker" + end + end + + -- Check cache and acquire lock if available (atomic locking to prevent duplicate API calls) + local should_query = true + local lock_acquired = false + local result = "clean" + + if cache then + -- Step 1: Check if result already cached + local ok, cached, cached_size = cache.virustotal_check(plugin, checksum, file_size) + if ok and cached then + logger.log_error( "file checksum " .. checksum .. " found in VT cache: " .. cached) + return true, cached + end + + -- Step 2: Try to acquire lock atomically + lock_acquired = cache.virustotal_try_lock(plugin, checksum) + + if not lock_acquired then + -- Another request is querying VirusTotal - wait for result + plugin:log_debug("[v" .. VERSION .. "] [CACHE_LOCK] Another request querying VirusTotal, waiting for result...") + local wait_ok, wait_result, wait_size = cache.virustotal_wait_for_result(plugin, checksum, file_size) + + if wait_ok then + -- Got result from concurrent query + logger.log_error( string.format( + "[CACHE_LOCK] Got VirusTotal result from concurrent query: %s", wait_result)) + return true, wait_result + else + -- Timeout - fall back to querying independently + logger.log_warn( "[CACHE_LOCK] Timeout waiting for VirusTotal result - querying independently") + lock_acquired = cache.virustotal_try_lock(plugin, checksum) + should_query = true + end + end + end + + if should_query then + -- Request from VirusTotal API + logger.log_error( "sending request to VirusTotal API for checksum: " .. checksum) + local ok, found, response, error_type = virustotal.api_request(plugin, "/files/" .. checksum) + if not ok then + logger.log_error( "VirusTotal API request failed: " .. found) + + -- Record API error in circuit breaker if it's a persistent error + -- Circuit break on: VT error codes (quota, auth, transient), 4xx client errors, 5xx server errors + if cache and error_type then + local should_circuit_break = ( + error_type == "QuotaExceededError" or + error_type == "TooManyRequestsError" or + error_type == "WrongCredentialsError" or + error_type == "ForbiddenError" or + error_type == "TransientError" or + error_type:match("^http_4%d%d$") or -- Match any 4xx error (401, 403, 429, etc.) + error_type:match("^http_5%d%d$") -- Match any 5xx error (500, 502, 503, etc.) + ) + if should_circuit_break then + cache.api_error_failure(plugin, "virustotal", error_type) + end + end + + -- Release lock before returning on error + if lock_acquired and cache then + cache.virustotal_release_lock(plugin, checksum) + end + + return true, "api_error" + end + + -- API call succeeded, reset circuit breaker + if cache then + cache.api_error_success(plugin, "virustotal") + end + + result = "clean" + if found then + logger.log_error( "file found on VirusTotal, analyzing results") + result = virustotal.result_parse(plugin, response) + logger.log_error( "VirusTotal result: " .. result) + else + logger.log_error( "file checksum " .. checksum .. " not found on VirusTotal (new file)") + end + + -- Add to cache if available + if cache then + local ok, err = cache.virustotal_add(plugin, checksum, result, file_size) + if not ok then + logger.log_error( "can't cache VT result: " .. err) + end + end + + -- Track new result for composite Redis write + if plugin.ctx.bw.new_scan_results then + plugin.ctx.bw.new_scan_results.vt = result + end + end + + -- Release lock after query completes (all paths: success, error, clean) + if lock_acquired and cache then + cache.virustotal_release_lock(plugin, checksum) + end + + return true, result +end + +-- ============================================================================ +-- VIRUSTOTAL FILE LOOKUP +-- ============================================================================ + +-- Check file hash against VirusTotal database. +-- Takes file path and SHA256 checksum, returns success boolean, result, and found boolean. +-- Returns ("clean", false) if file not found on VT, or (verdict, true) if found. +function virustotal.check_virustotal(plugin, file_path, checksum) + -- Check if VirusTotal is enabled + if plugin.variables["MALWARE_SCAN_API_USE_VIRUSTOTAL"] ~= "yes" then + return true, "clean", false + end + + -- Check if API key is configured and not a default value + local api_key = plugin.variables["MALWARE_SCAN_API_VIRUSTOTAL_KEY"] + if not api_key or api_key == "" or api_key == "your_api_key" or api_key == "your_token" then + logger.log_error( "VirusTotal enabled but API key not configured or using default value") + return true, "clean", false + end + + -- Check API error circuit breaker (prevents repeated failed calls) + if cache then + local should_skip, error_type, remaining = cache.api_error_should_skip(plugin, "virustotal") + if should_skip then + logger.log_error( string.format( + "VirusTotal API temporarily disabled due to previous error: %s (retry in %ds)", + error_type or "unknown", remaining + )) + return true, "api_circuit_breaker", false + end + end + + -- Get file size for cache verification + local file_size = file_ops and file_ops.file_get_size(plugin, file_path) or nil + + -- Check cache if cache module is available + if cache then + local ok, cached, cached_size = cache.virustotal_check(plugin, checksum, file_size) + if ok and cached then + logger.log_error( "file checksum " .. checksum .. " found in VT cache: " .. cached) + -- Assume if it's in cache, it was found before + return true, cached, true + end + end + + -- Request from VirusTotal API + logger.log_error( "sending request to VirusTotal API for checksum: " .. checksum) + local found, response + local ok, found, response, error_type = virustotal.api_request(plugin, "/files/" .. checksum) + if not ok then + logger.log_error( "VirusTotal API request failed: " .. found) + + -- Record API error in circuit breaker if it's a persistent error + -- Circuit break on: VT error codes (quota, auth, transient), 4xx client errors, 5xx server errors + if cache and error_type then + local should_circuit_break = ( + error_type == "QuotaExceededError" or + error_type == "TooManyRequestsError" or + error_type == "WrongCredentialsError" or + error_type == "ForbiddenError" or + error_type == "TransientError" or + error_type:match("^http_4%d%d$") or -- Match any 4xx error (401, 403, 429, etc.) + error_type:match("^http_5%d%d$") -- Match any 5xx error (500, 502, 503, etc.) + ) + if should_circuit_break then + cache.api_error_failure(plugin, "virustotal", error_type) + end + end + + return true, "api_error", false + end + + -- API call succeeded, reset circuit breaker + if cache then + cache.api_error_success(plugin, "virustotal") + end + + local result = "clean" + if found then + logger.log_error( "file found on VirusTotal, analyzing results") + result = virustotal.result_parse(plugin, response) + logger.log_error( "VirusTotal result: " .. result) + else + logger.log_error( "file checksum " .. checksum .. " not found on VirusTotal (new file)") + end + + -- Add to cache if cache module is available + if cache then + local ok, err = cache.virustotal_add(plugin, checksum, result, file_size) + if not ok then + logger.log_error( "can't cache VT result: " .. err) + end + end + + -- Track new result for composite Redis write + if plugin.ctx.bw.new_scan_results then + plugin.ctx.bw.new_scan_results.vt = result + end + + return true, result, found +end + +-- ============================================================================ +-- VIRUSTOTAL API REQUEST +-- ============================================================================ + +-- Make HTTP request to VirusTotal API (standard api_request interface). +-- Takes API endpoint URL, returns success boolean, found boolean, and response data. +-- This is the standard interface for API requests following domain_action pattern. +function virustotal.api_request(plugin, url) + -- Get HTTP client + local httpc, err = http_new() + if not httpc then + return false, err, nil, "http_error" + end + + -- Send request + local res + res, err = httpc:request_uri("https://www.virustotal.com/api/v3" .. url, { + headers = { + ["x-apikey"] = plugin.variables["MALWARE_SCAN_API_VIRUSTOTAL_KEY"], + ["accept"] = "application/json", + ["User-Agent"] = USER_AGENT, + }, + }) + if not res then + return false, err, nil, "http_error" + end + + -- Check status + if res.status == 404 then + return true, false, nil, nil + end + if res.status ~= 200 then + err = "received status " .. tostring(res.status) .. " from VirusTotal API" + local error_type = "http_" .. tostring(res.status) + + -- Try to parse error response to extract VirusTotal error code + local ok, data = pcall(decode, res.body) + if ok and data and data.error and data.error.code then + error_type = tostring(data.error.code) + err = err .. " (error: " .. error_type .. ")" + if data.error.message then + err = err .. " - " .. data.error.message + end + else + err = err .. " with data " .. (ok and encode(data) or "unknown") + end + return false, err, nil, error_type + end + + -- Parse JSON response + local ok, data = pcall(decode, res.body) + if not ok then + return false, "failed to decode JSON: " .. data, nil, "json_error" + end + if not data.data or not data.data.attributes or not data.data.attributes.last_analysis_stats then + return false, "malformed JSON response from VirusTotal", nil, "json_error" + end + + return true, true, data.data.attributes.last_analysis_stats, nil +end + +-- ============================================================================ +-- VIRUSTOTAL RESULT PROCESSING +-- ============================================================================ + +-- Process VirusTotal analysis stats and determine if file is malicious (standard result_parse interface). +-- Takes response stats, returns result string ("clean" or description of detections). +-- This is the standard interface for result parsing following domain_action pattern. +function virustotal.result_parse(plugin, response) + local suspicious = response["suspicious"] or 0 + local malicious = response["malicious"] or 0 + + local suspicious_threshold = tonumber(plugin.variables["MALWARE_SCAN_API_VIRUSTOTAL_SUSPICIOUS"]) or 5 + local malicious_threshold = tonumber(plugin.variables["MALWARE_SCAN_API_VIRUSTOTAL_MALICIOUS"]) or 3 + + if suspicious > suspicious_threshold or malicious > malicious_threshold then + return tostring(suspicious) .. " suspicious and " .. tostring(malicious) .. " malicious" + end + + return "clean" +end + +-- ============================================================================ +-- VIRUSTOTAL FILE UPLOAD +-- ============================================================================ + +-- Upload file to VirusTotal for analysis (standard file_upload interface). +-- Takes file path, returns success boolean, analysis ID or error message, and file size. +-- Only uploads files smaller than the configured max size (default 32MB). +-- This is the standard interface for file uploads following domain_action pattern. +function virustotal.file_upload(plugin, file_path) + logger.log_error( "preparing to upload file to VirusTotal") + + -- Extract file content from multipart body + if not file_ops then + return false, "file_ops module not loaded - cannot upload file" + end + + local ok, file_content = file_ops.content_extract(plugin, file_path) + if not ok then + return false, "failed to extract file content: " .. file_content + end + + local file_size = #file_content + plugin:log_debug("extracted file size: " .. file_size .. " bytes") + + -- Check file size against max upload size + local max_size = tonumber(plugin.variables["MALWARE_SCAN_API_VIRUSTOTAL_MAX_UPLOAD_SIZE"]) or 33554432 + if file_size > max_size then + logger.log_error( "file too large for VirusTotal upload: " .. file_size .. " bytes (max: " .. max_size .. " bytes)") + return false, "file too large for upload (max " .. max_size .. " bytes)" + end + + logger.log_error( "uploading file to VirusTotal (" .. file_size .. " bytes)") + + -- Get HTTP client + local httpc, err = http_new() + if not httpc then + return false, err + end + + -- Create multipart/form-data boundary + local boundary = "----WebKitFormBoundary" .. ngx.time() + + -- Build multipart body + local body_parts = {} + table.insert(body_parts, "--" .. boundary .. "\r\n") + table.insert(body_parts, 'Content-Disposition: form-data; name="file"; filename="upload"\r\n') + table.insert(body_parts, "Content-Type: application/octet-stream\r\n\r\n") + table.insert(body_parts, file_content) + table.insert(body_parts, "\r\n--" .. boundary .. "--\r\n") + + local upload_body = table.concat(body_parts) + + -- Send upload request + local res + res, err = httpc:request_uri("https://www.virustotal.com/api/v3/files", { + method = "POST", + headers = { + ["x-apikey"] = plugin.variables["MALWARE_SCAN_API_VIRUSTOTAL_KEY"], + ["accept"] = "application/json", + ["Content-Type"] = "multipart/form-data; boundary=" .. boundary, + ["Content-Length"] = tostring(#upload_body), + ["User-Agent"] = USER_AGENT, + }, + body = upload_body, + }) + + if not res then + return false, err + end + + -- Check status + if res.status ~= 200 then + err = "received status " .. tostring(res.status) .. " from VirusTotal upload API" + local decode_ok, data = pcall(decode, res.body) + if decode_ok and data then + err = err .. " with data " .. encode(data) + end + return false, err + end + + -- Parse JSON response + local decode_ok, data = pcall(decode, res.body) + if not decode_ok then + return false, "failed to decode JSON: " .. data + end + + -- Extract analysis ID from response + local analysis_id = nil + if data.data and data.data.id then + analysis_id = data.data.id + logger.log_error( "file uploaded to VirusTotal successfully, analysis ID: " .. analysis_id) + else + logger.log_error( "file uploaded but no analysis ID returned") + end + + return true, analysis_id, file_size +end + +-- ============================================================================ +-- VIRUSTOTAL VOTING +-- ============================================================================ + +-- Vote on a file in VirusTotal (standard file_vote interface). +-- Takes SHA256 checksum and verdict ("harmless" or "malicious"). +-- Returns success boolean and error message if failed. +-- This is the standard interface for file voting following domain_action pattern. +function virustotal.file_vote(plugin, checksum, verdict) + logger.log_error( "voting on VirusTotal file " .. checksum .. " as " .. verdict) + + -- Validate verdict + if verdict ~= "harmless" and verdict ~= "malicious" then + return false, "invalid verdict: " .. verdict .. " (must be 'harmless' or 'malicious')" + end + + -- Get HTTP client + local httpc, err = http_new() + if not httpc then + return false, err + end + + -- Build vote request body + local vote_data = { + data = { + type = "vote", + attributes = { + verdict = verdict + } + } + } + + local body = encode(vote_data) + + -- Send vote request + local res + res, err = httpc:request_uri("https://www.virustotal.com/api/v3/files/" .. checksum .. "/votes", { + method = "POST", + headers = { + ["x-apikey"] = plugin.variables["MALWARE_SCAN_API_VIRUSTOTAL_KEY"], + ["accept"] = "application/json", + ["Content-Type"] = "application/json", + ["Content-Length"] = tostring(#body), + ["User-Agent"] = USER_AGENT, + }, + body = body, + }) + + if not res then + return false, err + end + + -- Check status (200 or 201 are success) + if res.status ~= 200 and res.status ~= 201 then + err = "received status " .. tostring(res.status) .. " from VirusTotal vote API" + local decode_ok, data = pcall(decode, res.body) + if decode_ok and data then + err = err .. " with data " .. encode(data) + end + return false, err + end + + logger.log_error( "vote submitted successfully to VirusTotal") + return true +end + +-- Return module +return virustotal diff --git a/malware-scan/malware_scan_webhook.lua b/malware-scan/malware_scan_webhook.lua new file mode 100644 index 0000000..9055882 --- /dev/null +++ b/malware-scan/malware_scan_webhook.lua @@ -0,0 +1,745 @@ +-- Webhook Notification Module for BunkerWeb Malware Scanner +-- Supports Microsoft Teams and Discord webhooks for malware detection alerts +-- Handles rate limiting, retry logic, and automatic webhook type detection + +local logger = require("malware_scan_logger") +local utils = require("malware_scan_utils") + +local webhook = {} + +-- Module version +webhook.VERSION = "0.8.0" + +local cjson = require("cjson.safe") +local http = require("resty.http") + +-- Try to load cache module for attacker tracking (optional) +local cache = pcall(require, "malware_scan_cache") and require("malware_scan_cache") or nil + +-- Mask sensitive webhook URL for logging +-- Only shows full URL if MALWARE_SCAN_UNMASK_KEYS is set to the specific phrase +--- @param url string The URL to mask +--- @param plugin table Plugin instance for checking settings +--- @return string result Masked or unmasked URL +local function webhook_mask_webhook_url(url, plugin) + if not url or url == "" then + return "" + end + + -- Check if unmasking is explicitly enabled + local unmask = plugin.variables["MALWARE_SCAN_UNMASK_KEYS"] + if unmask == "I_WANT_TO_LEAK_MY_SECRETS_TO_LOG" then + return url + end + + -- Mask webhook URL (show protocol and domain prefix only) + if url:match("^https?://") then + local protocol, rest = url:match("^(https?://)(.+)$") + if protocol then + local domain = rest:match("^([^/]+)") + if domain then + -- Show protocol and domain prefix only + local domain_prefix = domain:sub(1, math.min(#domain, 10)) + return protocol .. domain_prefix .. "***" + end + end + end + return "https://***" +end + +-- Clean webhook URL by removing quotes, brackets, and whitespace +--- @param webhook_url string The URL to clean +--- @param plugin table Plugin instance for logging +--- @return string result Cleaned URL +local function webhook_clean_webhook_url(webhook_url, plugin) + -- Remove common wrapping characters + webhook_url = webhook_url:gsub("^['\"%[]", ""):gsub("['\"%]]$", "") + -- Remove whitespace + webhook_url = webhook_url:gsub("^%s+", ""):gsub("%s+$", "") + + logger.log_error( "[WEBHOOK] Cleaned URL: " .. webhook_mask_webhook_url(webhook_url, plugin)) + return webhook_url +end + +-- Parse ClamAV signature name into components +-- Format: {platform}.{category}.{name}-{signature_id}-{revision} +-- Example: Win.Trojan.Zusy-9935890-0 +--- @param signature string The ClamAV signature name +--- @return table|nil parsed Parsed components table or nil if not parseable +local function webhook_parse_clamav_signature(signature) + if not signature or signature == "" then + return nil + end + + -- ClamAV signature format: Platform.Category.Name-SignatureID-Revision + local platform, category, name, sig_id, revision = signature:match("^([^%.]+)%.([^%.]+)%.([^%-]+)%-(%d+)%-(%d+)$") + + if platform and category and name and sig_id and revision then + return { + platform = platform, + category = category, + name = name, + signature_id = sig_id, + revision = revision, + full_name = signature + } + end + + -- Fallback: just return the full name if parsing fails + return { + full_name = signature + } +end + +-- Truncate virus/malware name to fit in webhook messages +-- Limits virus names to 20 characters to prevent webhook message overflow +--- @param virus_name string The virus/malware name to truncate +--- @return string result Truncated virus name with ellipsis if needed +local function webhook_truncate_virus_name(virus_name) + if not virus_name or virus_name == "" then + return "" + end + + local max_len = 20 + if #virus_name > max_len then + return virus_name:sub(1, max_len - 3) .. "..." + end + return virus_name +end + +-- Truncate string to maximum length with ellipsis if needed +--- @param str string The string to truncate +--- @param max_len number Maximum length (default 1024 for Discord/Teams limits) +--- @return string result Truncated string +local function webhook_truncate_string(str, max_len) + max_len = max_len or 1024 + if not str then + return "" + end + if #str <= max_len then + return str + end + return str:sub(1, max_len - 3) .. "..." +end + +-- Anonymize IP address for GDPR data minimization in webhooks +-- Masks the last octet/group to provide network-level information without exposing full IP +-- IPv4: 192.168.1.123 -> 192.168.1.x +-- IPv6: 2001:db8:1234:5678:90ab:cdef:1234:5678 -> 2001:db8:1234:5678:90ab:cdef:1234:x +--- @param ip string The IP address to anonymize +--- @return string result Anonymized IP address +local function webhook_anonymize_ip_for_webhook(ip) + if not ip or ip == "" or ip == "unknown" then + return ip + end + + -- Detect IPv6 (contains colons) + if ip:find(":") then + -- IPv6: Replace last group with x + -- Handle both full notation (2001:db8::1) and compressed notation + local last_colon = ip:match("^(.+):") + if last_colon then + return last_colon .. ":x" + end + return ip -- Fallback if parsing fails + else + -- IPv4: Replace last octet with x + local first_three = ip:match("^(%d+%.%d+%.%d+)%.") + if first_three then + return first_three .. ".x" + end + return ip -- Fallback if parsing fails + end +end + +-- Build JSON payload for webhook notification +-- Auto-detects webhook type (Discord, Power Automate, or Teams) from URL +--- @param webhook_url string The webhook URL to send to +--- @param checksum string SHA256 hash of the malicious file +--- @param detections table Scanner detection results {scanner_name = detection_info} +--- @param client_ip string Client IP address +--- @param server_name string Server/domain name +--- @param request_uri string Request URI path +--- @param hostname string Hostname of the BunkerWeb instance +--- @param worker_info string Worker process information (PID and ID) +--- @param attacker_count number Number of previous malware uploads from this IP (0 if none) +--- @param first_upload_time number Unix timestamp of first malware upload from this IP (nil if none) +--- @return string json JSON payload formatted for webhook +local function webhook_build_json_payload(webhook_url, checksum, detections, client_ip, server_name, request_uri, hostname, worker_info, attacker_count, first_upload_time) + -- Build detection summary and separate cached vs fresh results + local detection_list = {} + local scanners_detected = {} + local cached_scanners = {} + local fresh_scanners = {} + local signature_breakdown = {} -- Store parsed ClamAV signatures + + for scanner, detection in pairs(detections or {}) do + -- Check if this is a cached result + if scanner:match("%(cached%)") then + local base_name = scanner:gsub(" %(cached%)", "") + table.insert(cached_scanners, base_name) + else + table.insert(fresh_scanners, scanner) + end + table.insert(scanners_detected, scanner) + + -- Truncate virus name to 20 chars to prevent webhook message overflow + local detection_truncated = webhook_truncate_virus_name(detection) + + -- Parse ClamAV signatures for enhanced display + local detection_display = detection_truncated + if scanner:match("ClamAV") then + local parsed = webhook_parse_clamav_signature(detection) + if parsed and parsed.platform and parsed.category then + -- Store for later use + signature_breakdown[scanner] = parsed + -- Enhanced display with breakdown (using truncated name) + detection_display = string.format("%s (`%s` on `%s`)", + detection_truncated, parsed.category, parsed.platform) + end + end + + -- Additional truncate for safety (Discord/Teams 1024 char limit per field) + detection_display = webhook_truncate_string(detection_display, 900) + table.insert(detection_list, "**" .. scanner .. "**: " .. detection_display) + end + + local hit_rate = table.concat(scanners_detected, ", ") + if hit_rate == "" then + hit_rate = "unknown" + end + + -- Build cached info string + local cached_info = nil + if #cached_scanners > 0 then + cached_info = table.concat(cached_scanners, ", ") + end + + -- Build attacker history info strings + local attacker_info = nil + local tracking_duration = nil + if attacker_count and attacker_count > 0 and first_upload_time then + -- Format first upload time (UTC) + local first_upload_str = os.date("!%Y-%m-%d %H:%M:%S UTC", first_upload_time) + + -- Calculate tracking duration + local current_time = os.time() + local duration_seconds = current_time - first_upload_time + local duration_hours = math.floor(duration_seconds / 3600) + local duration_minutes = math.floor((duration_seconds % 3600) / 60) + + -- Format tracking duration + if duration_hours > 24 then + local days = math.floor(duration_hours / 24) + local hours = duration_hours % 24 + tracking_duration = string.format("%d day(s) %d hour(s)", days, hours) + elseif duration_hours > 0 then + tracking_duration = string.format("%d hour(s) %d min(s)", duration_hours, duration_minutes) + else + tracking_duration = string.format("%d minute(s)", duration_minutes) + end + + attacker_info = string.format("%d previous malware upload(s) from this IP\nFirst upload: %s\nTracking duration: %s", + attacker_count, first_upload_str, tracking_duration) + end + + -- Detect webhook type from URL + local is_discord = webhook_url:find("discord%.com/api/webhooks") or webhook_url:find("discordapp%.com/api/webhooks") + local is_power_automate = webhook_url:find("powerplatform%.com") or webhook_url:find("powerautomate") + + local card + + -- Format detection timestamp (UTC) + local detection_time = os.date("!%Y-%m-%d %H:%M:%S UTC") + + if is_discord then + -- Discord embed format + local fields = { + {name = "๐Ÿ•’ Detection Time", value = webhook_truncate_string(detection_time, 1024), inline = true}, + {name = "๐Ÿ–ฅ๏ธ BunkerWeb Instance", value = webhook_truncate_string(hostname or "unknown", 1024), inline = true}, + {name = "โš™๏ธ Worker", value = webhook_truncate_string(worker_info or "unknown", 1024), inline = true}, + {name = "Client IP", value = webhook_truncate_string(client_ip, 1024), inline = true}, + {name = "Destination Domain", value = webhook_truncate_string(server_name, 1024), inline = true}, + {name = "Request URI", value = webhook_truncate_string(request_uri, 1024), inline = false}, + {name = "SHA256 Checksum", value = webhook_truncate_string(checksum or "not available", 1024), inline = false}, + {name = "Detected By", value = webhook_truncate_string(hit_rate, 1024), inline = false} + } + + -- Add signature breakdown if ClamAV signatures were parsed + if next(signature_breakdown) then + local breakdown_lines = {} + for scanner, parsed in pairs(signature_breakdown) do + if parsed.platform and parsed.category then + -- Truncate name to 20 chars to keep breakdown compact + local name_truncated = webhook_truncate_virus_name(parsed.name) + table.insert(breakdown_lines, string.format( + "โ€ข **Platform**: %s | **Category**: %s | **Name**: %s\n ID: %s-r%s", + parsed.platform, parsed.category, name_truncated, + parsed.signature_id, parsed.revision + )) + end + end + if #breakdown_lines > 0 then + table.insert(fields, { + name = "๐Ÿ”ฌ Signature Details", + value = table.concat(breakdown_lines, "\n"), + inline = false + }) + end + end + + -- Add attacker history if available + if attacker_info then + table.insert(fields, { + name = "๐Ÿšจ Repeat Offender", + value = webhook_truncate_string(attacker_info, 1024), + inline = false + }) + end + + -- Add cached results indicator if any + if cached_info then + table.insert(fields, { + name = "โšก Cached Results", + value = webhook_truncate_string(cached_info .. " (from cache, not re-scanned)", 1024), + inline = false + }) + end + + -- Add detection details (truncate to prevent Discord field limit issues) + for scanner, detection in pairs(detections or {}) do + table.insert(fields, { + name = webhook_truncate_string(scanner, 256), -- Discord field name limit: 256 chars + value = webhook_truncate_string(detection, 1024), -- Discord field value limit: 1024 chars + inline = false + }) + end + + card = { + embeds = { + { + title = "๐Ÿ›ก๏ธ MALWARE DETECTED", + description = "BunkerWeb Malware Scanner Alert", + color = 16711680, -- Red color in decimal + fields = fields, + timestamp = os.date("!%Y-%m-%dT%H:%M:%SZ") + } + } + } + elseif is_power_automate then + -- Power Automate simple JSON format + card = { + title = "๐Ÿ›ก๏ธ MALWARE DETECTED", + detection_time = detection_time, + bunkerweb_instance = hostname or "unknown", + worker = worker_info or "unknown", + client_ip = client_ip, + server_name = server_name, + request_uri = request_uri, + checksum = checksum or "not available", + detected_by = hit_rate, + signature_breakdown = signature_breakdown, -- Include parsed signatures + previous_uploads = attacker_count or 0, + repeat_offender = (attacker_count and attacker_count > 0) or false, + first_upload = first_upload_time and os.date("!%Y-%m-%d %H:%M:%S UTC", first_upload_time) or nil, + tracking_duration = tracking_duration, + cached_results = cached_info, + detections = detections + } + else + -- Microsoft Teams MessageCard format (Incoming Webhook) + local facts = { + {name = "๐Ÿ•’ Detection Time", value = webhook_truncate_string(detection_time, 1024)}, + {name = "๐Ÿ–ฅ๏ธ BunkerWeb Instance", value = webhook_truncate_string(hostname or "unknown", 1024)}, + {name = "โš™๏ธ Worker", value = webhook_truncate_string(worker_info or "unknown", 1024)}, + {name = "Client IP", value = webhook_truncate_string(client_ip, 1024)}, + {name = "Destination Domain", value = webhook_truncate_string(server_name, 1024)}, + {name = "Request URI", value = webhook_truncate_string(request_uri, 1024)}, + {name = "SHA256 Checksum", value = webhook_truncate_string(checksum or "not available", 1024)}, + {name = "Detected By", value = webhook_truncate_string(hit_rate, 1024)} + } + + -- Add signature breakdown if ClamAV signatures were parsed + if next(signature_breakdown) then + local breakdown_lines = {} + for scanner, parsed in pairs(signature_breakdown) do + if parsed.platform and parsed.category then + -- Truncate name to 20 chars to keep breakdown compact + local name_truncated = webhook_truncate_virus_name(parsed.name) + table.insert(breakdown_lines, string.format( + "โ€ข Platform: %s | Category: %s | Name: %s | ID: %s-r%s", + parsed.platform, parsed.category, name_truncated, + parsed.signature_id, parsed.revision + )) + end + end + if #breakdown_lines > 0 then + table.insert(facts, { + name = "๐Ÿ”ฌ Signature Details", + value = table.concat(breakdown_lines, "\n") + }) + end + end + + -- Add attacker history if available + if attacker_info then + table.insert(facts, {name = "๐Ÿšจ Repeat Offender", value = webhook_truncate_string(attacker_info, 1024)}) + end + + -- Add cached results indicator if any + if cached_info then + table.insert(facts, {name = "โšก Cached Results", value = webhook_truncate_string(cached_info .. " (from cache, not re-scanned)", 1024)}) + end + + card = { + ["@type"] = "MessageCard", + ["@context"] = "https://schema.org/extensions", + themeColor = "FF0000", -- Red color for alerts + summary = "Malware Detected", + sections = { + { + activityTitle = "๐Ÿ›ก๏ธ MALWARE DETECTED", + activitySubtitle = "BunkerWeb Malware Scanner Alert", + facts = facts, + markdown = true + } + } + } + + -- Add detection details section if available + if #detection_list > 0 then + table.insert(card.sections, { + title = "Detection Details", + text = webhook_truncate_string(table.concat(detection_list, "\n\n"), 5000) -- Teams text limit + }) + end + end + + return cjson.encode(card) +end + +-- Attempt a single webhook HTTP request (helper for retry logic) +--- @param url_parts table Parsed URL components +--- @param json_body string JSON payload to send +--- @param plugin table Plugin instance for logging +--- @param webhook_type string Type of webhook ("Teams" or "Discord") for logging +--- @return table|nil response Response object on success, nil on connection/SSL failure +--- @return string|nil error Error message on failure +local function webhook_attempt_webhook_request(url_parts, json_body, plugin, webhook_type) + local httpc = http.new() + httpc:set_timeout(5000) -- 5 second timeout + + -- Connect to webhook server + local ok, err = httpc:connect(url_parts.host, url_parts.port) + if not ok then + return nil, "connection failed: " .. (err or "unknown") + end + + -- Perform SSL handshake if needed + if url_parts.ssl then + local session, ssl_err = httpc:ssl_handshake(nil, url_parts.host, false) + if not session then + httpc:close() + return nil, "SSL handshake failed: " .. (ssl_err or "unknown") + end + end + + -- Send POST request + local res, req_err = httpc:request({ + method = "POST", + path = url_parts.path, + headers = { + ["Host"] = url_parts.host, + ["Content-Type"] = "application/json", + ["Content-Length"] = tostring(#json_body) + }, + body = json_body + }) + + httpc:close() + return res, req_err +end + +-- Send webhook HTTP request with retry logic +--- @param url_parts table Parsed URL components +--- @param json_body string JSON payload to send +--- @param plugin table Plugin instance for logging +--- @param webhook_type string Type of webhook ("Teams" or "Discord") for logging +--- @return boolean success True if webhook sent successfully, false otherwise +local function webhook_send_webhook_request(url_parts, json_body, plugin, webhook_type) + -- Retry configuration + local max_retries = 3 + local base_delay = 1 -- 1 second base delay + local max_delay = 20 -- 20 seconds max delay + + -- Transient error status codes that should be retried + local transient_errors = { + [412] = true, -- Precondition Failed + [429] = true, -- Too Many Requests (rate limit) + [502] = true, -- Bad Gateway + [504] = true, -- Gateway Timeout + } + + -- Retry loop with exponential backoff + for attempt = 1, max_retries do + -- Attempt the webhook request + local res, err = webhook_attempt_webhook_request(url_parts, json_body, plugin, webhook_type) + + -- Success case + if res and res.status >= 200 and res.status < 300 then + if attempt > 1 then + logger.log_notice( "[WEBHOOK] " .. webhook_type .. " notification sent successfully after " .. attempt .. " attempts") + else + logger.log_notice( "[WEBHOOK] " .. webhook_type .. " notification sent successfully") + end + return true + end + + -- Determine if error is transient and should be retried + local should_retry = false + local error_msg = "" + + if not res then + -- Connection error, SSL error, or timeout + should_retry = (attempt < max_retries) + error_msg = err or "unknown error" + elseif transient_errors[res.status] then + -- Transient HTTP error + should_retry = (attempt < max_retries) + error_msg = "HTTP " .. res.status .. ": " .. (res.body or "no body") + else + -- Non-retryable error (4xx other than 412/429, or 5xx other than 502/504) + should_retry = false + error_msg = "HTTP " .. res.status .. ": " .. (res.body or "no body") + end + + -- Log the error + if should_retry then + logger.log_notice( "[WEBHOOK] " .. webhook_type .. " attempt " .. attempt .. "/" .. max_retries .. " failed (" .. error_msg .. "), retrying...") + else + logger.log_error( "[WEBHOOK] " .. webhook_type .. " failed: " .. error_msg) + return false + end + + -- Calculate exponential backoff delay with jitter (only if retrying) + if should_retry and attempt < max_retries then + local delay = math.min(math.pow(2, attempt - 1) * base_delay, max_delay) + -- Add random jitter (ยฑ20%) + local jitter = delay * 0.2 * (math.random() * 2 - 1) + delay = delay + jitter + + logger.log_notice( "[WEBHOOK] " .. webhook_type .. " backing off for " .. string.format("%.2f", delay) .. " seconds") + ngx.sleep(delay) + end + end + + -- All retries exhausted + logger.log_error( "[WEBHOOK] " .. webhook_type .. " failed after " .. max_retries .. " attempts") + return false +end + +-- Safe wrapper: Send webhook notification with built-in error handling. +-- Handles nil module checks, empty data validation, and error logging. +-- This wrapper centralizes all error handling to eliminate code duplication. +-- +-- Use this function from other modules instead of calling notification_send directly. +-- +--- @param webhook_module table The webhook module (can be nil) +--- @param plugin table Plugin instance +--- @param checksum string SHA256 hash of the malicious file (can be nil) +--- @param detections table Scanner detection results (can be empty) +--- @return boolean success True if notification sent successfully, false otherwise +function webhook.notify(webhook_module, plugin, checksum, detections) + -- Silent return if webhook module not loaded + if not webhook_module then + return false + end + + -- Silent return if no checksum or detections + if not checksum or not detections or next(detections) == nil then + return false + end + + -- Call notification_send with error handling + local ok, err = pcall(webhook.notification_send, plugin, checksum, detections) + if not ok then + logger.log_error( "[WEBHOOK] Failed to send notification: " .. tostring(err)) + return false + end + + return true +end + +-- Main entry point: Send notification to configured webhooks +-- Processes both Teams and Discord webhooks if configured +--- @param plugin table Plugin instance with variables and logger +--- @param checksum string SHA256 hash of the malicious file +--- @param detections table Scanner detection results {scanner_name = detection_info} +--- @return boolean success True if all configured webhooks succeeded, false otherwise +function webhook.notification_send(plugin, checksum, detections) + local success = true + + -- Get request information + local client_ip = ngx.var.remote_addr or "unknown" + local server_name = ngx.var.server_name or ngx.var.host or "unknown" + local request_uri = plugin.ctx.bw.uri or "unknown" + + -- Get worker/instance information + local hostname = ngx.var.hostname or "unknown" + local worker_pid = ngx.worker.pid() + local worker_id = ngx.worker.id() + local worker_info = string.format("PID:%d ID:%d", worker_pid or 0, worker_id or 0) + + -- Get attacker history if tracking is enabled (using ORIGINAL IP before anonymization) + local attacker_count = 0 + local first_upload_time = nil + if cache and cache.attacker_get then + local attacker_data = cache.attacker_get(plugin, client_ip) + if attacker_data and attacker_data.upload_count then + attacker_count = attacker_data.upload_count + first_upload_time = attacker_data.first_upload + logger.log_notice( string.format("[v" .. webhook.VERSION .. "] [WEBHOOK] Attacker history: IP %s has %d previous malware upload(s)", client_ip, attacker_count)) + end + end + + -- Apply IP anonymization for GDPR data minimization (Article 5(1)(c)) + -- Anonymize by default to minimize data shared with third parties (Microsoft/Discord) + local anonymize_ips = plugin.variables["MALWARE_SCAN_ANONYMIZE_WEBHOOK_IPS"] + if anonymize_ips == nil or anonymize_ips == "" or anonymize_ips == "yes" then + local original_ip = client_ip + client_ip = webhook_anonymize_ip_for_webhook(client_ip) + if client_ip ~= original_ip then + logger.log_notice( string.format("[v" .. webhook.VERSION .. "] [WEBHOOK] IP anonymized for third-party sharing: %s -> %s", original_ip, client_ip)) + end + end + + -- Process Teams webhook + local teams_url = plugin.variables["MALWARE_SCAN_TEAMS_WEBHOOK_URL"] + if teams_url and teams_url ~= "" and teams_url ~= "your_webhook_url" then + logger.log_error( "[WEBHOOK] Processing Teams webhook") + + -- Check rate limit with hard cap at Microsoft Teams' limit (200/min) + local TEAMS_HARD_LIMIT = 200 + local user_rate_limit = tonumber(plugin.variables["MALWARE_SCAN_TEAMS_WEBHOOK_RATE_LIMIT"]) or 3 + local rate_limit = user_rate_limit + + if user_rate_limit == 0 then + rate_limit = TEAMS_HARD_LIMIT + elseif user_rate_limit > TEAMS_HARD_LIMIT then + rate_limit = TEAMS_HARD_LIMIT + logger.log_warn( "[WEBHOOK] Teams rate limit " .. user_rate_limit .. " exceeds maximum (" .. TEAMS_HARD_LIMIT .. "/min) - enforcing hard cap") + end + + -- Apply rate limiting + local cache_key = "plugin_malware_scan_teams_webhook_count" + local ok, count = plugin.cachestore:get(cache_key) + + if ok and count then + count = tonumber(count) or 0 + if count >= rate_limit then + logger.log_notice( "[WEBHOOK] Teams rate limit exceeded (" .. count .. "/" .. rate_limit .. " per minute) - skipping notification") + else + -- Increment counter + plugin.cachestore:set(cache_key, count + 1, 60) + + -- Clean URL and parse + teams_url = webhook_clean_webhook_url(teams_url, plugin) + local url_parts, err = utils.parse_webhook_url(teams_url) + if not url_parts then + logger.log_error( "[WEBHOOK] Teams URL parse failed: " .. err) + success = false + else + -- Build and send + local json_body = webhook_build_json_payload(teams_url, checksum, detections, client_ip, server_name, request_uri, hostname, worker_info, attacker_count, first_upload_time) + if not webhook_send_webhook_request(url_parts, json_body, plugin, "Teams") then + success = false + end + end + end + else + -- Initialize counter with 60 second TTL + plugin.cachestore:set(cache_key, 1, 60) + + -- Clean URL and parse + teams_url = webhook_clean_webhook_url(teams_url, plugin) + local url_parts, err = utils.parse_webhook_url(teams_url) + if not url_parts then + logger.log_error( "[WEBHOOK] Teams URL parse failed: " .. err) + success = false + else + -- Build and send + local json_body = webhook_build_json_payload(teams_url, checksum, detections, client_ip, server_name, request_uri, hostname, worker_info, attacker_count, first_upload_time) + if not webhook_send_webhook_request(url_parts, json_body, plugin, "Teams") then + success = false + end + end + end + end + + -- Process Discord webhook + local discord_url = plugin.variables["MALWARE_SCAN_DISCORD_WEBHOOK_URL"] + if discord_url and discord_url ~= "" and discord_url ~= "your_webhook_url" then + logger.log_error( "[WEBHOOK] Processing Discord webhook") + + -- Check rate limit with hard cap at Discord's limit (30 per 60 seconds) + local DISCORD_HARD_LIMIT = 30 + local user_rate_limit = tonumber(plugin.variables["MALWARE_SCAN_DISCORD_WEBHOOK_RATE_LIMIT"]) or 3 + local rate_limit = user_rate_limit + + if user_rate_limit == 0 then + rate_limit = DISCORD_HARD_LIMIT + elseif user_rate_limit > DISCORD_HARD_LIMIT then + rate_limit = DISCORD_HARD_LIMIT + logger.log_warn( "[WEBHOOK] Discord rate limit " .. user_rate_limit .. " exceeds maximum (" .. DISCORD_HARD_LIMIT .. "/60s) - enforcing hard cap") + end + + -- Apply rate limiting (60 second window for Discord) + local cache_key = "plugin_malware_scan_discord_webhook_count" + local ok, count = plugin.cachestore:get(cache_key) + + if ok and count then + count = tonumber(count) or 0 + if count >= rate_limit then + logger.log_notice( "[WEBHOOK] Discord rate limit exceeded (" .. count .. "/" .. rate_limit .. " per 60s) - skipping notification") + else + -- Increment counter + plugin.cachestore:set(cache_key, count + 1, 60) + + -- Clean URL and parse + discord_url = webhook_clean_webhook_url(discord_url, plugin) + local url_parts, err = utils.parse_webhook_url(discord_url) + if not url_parts then + logger.log_error( "[WEBHOOK] Discord URL parse failed: " .. err) + success = false + else + -- Build and send + local json_body = webhook_build_json_payload(discord_url, checksum, detections, client_ip, server_name, request_uri, hostname, worker_info, attacker_count, first_upload_time) + if not webhook_send_webhook_request(url_parts, json_body, plugin, "Discord") then + success = false + end + end + end + else + -- Initialize counter with 60 second TTL + plugin.cachestore:set(cache_key, 1, 60) + + -- Clean URL and parse + discord_url = webhook_clean_webhook_url(discord_url, plugin) + local url_parts, err = utils.parse_webhook_url(discord_url) + if not url_parts then + logger.log_error( "[WEBHOOK] Discord URL parse failed: " .. err) + success = false + else + -- Build and send + local json_body = webhook_build_json_payload(discord_url, checksum, detections, client_ip, server_name, request_uri, hostname, worker_info, attacker_count, first_upload_time) + if not webhook_send_webhook_request(url_parts, json_body, plugin, "Discord") then + success = false + end + end + end + end + + return success +end + +return webhook diff --git a/malware-scan/pictures/discord-message-on-hit.png b/malware-scan/pictures/discord-message-on-hit.png new file mode 100644 index 0000000000000000000000000000000000000000..fc4c4e1d3b6ce695a29830046d30638ad883c07c GIT binary patch literal 112846 zcmeFZWmp``7B-3n50KzaaCdh}a0>)?cXtnjV8NXbEVw%acL#3aP*tnD0??G21fURpTWTN`*NvOz#l$M{57%5*8B^|qDY;H%`1%Vm53#M!u8-E@ZgX5%@!+|mzNvHC4Ac{lusD>I02bn6&lh8HCubcj6E$Sq#l&b2( z1!!$(#p^`#JIjsp`XeZg;F@jgl760eboFi&e)}or?}1Q+(?Yk^{%vn{9J!X2GT!5P z@ASuzQYCi}+P5=9C#Z`BDaE-(9=Ku8D@4@3qteeJ3;YsNn}A}^DM4k`H}iv^nH5I6 zwqY61|A$0MiK_pkvi=vC@C8Lso!YSF^2Ihk^AUCTEO|69Oe4b!!N|ZZBIQ$P`)zMX z=zK%;oJ15Ypx0rHq{I}FpF%*|La!bI-xE#LrA+1IAZUSOc!(#E&mo|JBS_#M0um46 z$zR715YmwNzmAn4ssFhL3IZb70s`irdo+R1$G>Rc4V3=p6FN2s0uK209C!z2LjAlO zdMp$A=P}eQa1BC4Sxia__*6D>G%>MtGPiS1gQCa;P9WGzYB)haU{O52A*Gbaj{p^# zvUsEJtS%?ZYh-7`Xkct-Xu{}jV-MU70m1Li3mn>*I2(|-+gRH=@wy9;{&fd0aQt|h ziIn88Tb!)~NY&*ONyO|NO-MKynHiZ$1rbR|NcbI%O?j1GivLp__$EMV?(A&O%f#g7 z=Ems8#%Skg#>B$I!^6bP%EZdb0Nlag`=iIb6|g}t+doh`}Z za}5mbT$}|+Ngpfv_4l`&Chiu$YqE9vr&&M)nI5k&u`n_-{jXxq7N-9X#U8KxUF@%R z{aqdZPvuSofUkkB_WlVxdl z2cyef!MPS=iH-+zJZ&HYk>@0k|M~K7eJ)+7#VrCa3cj zv&U*gjN{IaLYNo2U+hIpLFr@>_+0sd7zN1vT zBjLCAMw0*?XS5V}_|M4=AAt8_7uE-sXjQMw^e;o%>_(wCJqR3S3Vf?2^z*XU?=b5a8 zb@hpaWdGH0$mj4_khEhRq=TMPZ($_=^VOLMk+5CSQoD!q=MyX76;n{KG$v&JL%>IQ z(du9dWP|rhP5S=palqGrS8oTLE526g{%7|CLitxvM88NDkjW5iF&>DQ8qEihY>&RS zL#4Euko%*GXiu1cyk)^S3UO5vKB~(+J|>+N`F!0dcZxrr3p9|`zu-B=K=+dyJUbmi zvT@dK&o^49#*|U)l>aewfA^@cCK=jMip7wmmvYV+9!_1-e|HMd3h!s|1*|%+hsEpr z*LcYGcUNSo{^(W`{~Ae>%mL@IaL9Tbg%7gRrhka$&rF4;Xwqm2OFL$8-(2DKUxN*N z4HFhl*baa!jkpZ{A7Y5c2@8)eo^_^de{UYP#{6@PNDyG&q@Wmgsbl|f4FVyM78v&| zPqQh;KLv}X5H{g8wtI>%{zuoN6+MpbPMSf}pQ9`Ks4vki9M=CsUkV;Iu*fZFpY)He zg4ck0lP7jDQDN|B4`fLL3RAFvTle9Q4j?f`090^Vt1IY#Oosp4xc?7F?kif+QzX-- z0kx8XVkWh$&++Bz9gP?(w$pVc)z}@ZlOGv8Y-ZAqn7czMY=gz+t=2_Vk&mk?I;CvY z%7JjB^Y&WpWRP?a^6?$Zb=z%~MxH{3!RJt-*Psf6a>?HCq^Fp5HnWUzR0?uu+870@ z`>GAQWqhSoLbpfRL!WqN!@!8_md;J=7NcJW7QP%~r?9Mw(WTF|%Wz&rrGyOVsw)kFH+f4@qC&Atnt2gE=|@`6|Y_Qre0K5R_YGEa9C=;kL7b? z%(!l4jdfg{zZ}xp+#WMq+sxqVzd7GES!}&Ku+BJ+uWJ!`=Q&zs;*@a?j&s3hqK{Q6Zawvo;^#}_b&2OYw5GZ~Sx+F~afBcJ_KAm#O8!%H ztyNpn>+FS?)jkF89qKm)nPN*;)&T!4Yk^7b*bSQa= z<#9QGyyID0QZhV1-)hwNO?6I9f9{JL%s|{TY7!319~>jb#1fGNFXU1=>%Kjlf`oD% zRAfbl<@B3@VaudHhm$=EZ?&ofY3uMC|Yo$w) zzqu_|2g-L=+wV=WCTvOo9Jc{c{*Q9tI{R zMXl9jJX6~(sm26s;b@_;ZXoCtYsWGB&yB>t;ssjbK#d0W{-Lb1^kLrXoH#87XJ?j- z{La81NpDI$<0ebGO$Ya=I*{w!kMZ}URHu@vY%wJb^n1dX2swn03*PjPArK&r7pti; z51Kz*vCg*~G(tsss+DT)hopW@*yEe|o+n3VLuxk@hZX6vJCW$Jt4iU)Dadd(oWUud z%&aQ(p>@EJawc^Rl%cTT=T}r;QX%=_Y?GAUmXuHAXXi&7L*d4H!zvs}>-XKSdNOce zT$iTCZR~?@cFQj{sh%4^9mvtXp=MII+Lz0W*Y&dTz1}S+Tc{$mU{*P&C|C``|GXY0 z-rb9r7x z<@J=RcK@)MO|^7nVY9ZZkJ#@G=F#~c%hJ`hhkL8{^Zv=nI2|0f&+4VX zgl=;O>fiKtuJ-E*Sq-}-YsEd&bdmHLooPG{rt@<>BP{bm1hJTOwKYVY$L^7J{qE#A zs(r2-xSn`jY%xwdY1h$rc%FNlZlqfG9Rn3PcJIgJIM!!ap~CGpROr1k_^Bn3&n1fi zTLNzD+x}GkQ1keWg>@|ll`hqB(?-CP@>310FdPEQyR$KI30&@0X-Vk>+U+FPwB+ji zJ|eG|DK!@|TS_zoeD+J3N>gE)E^p3PhKe0l#fie$<$h(49jehK2?=iIN6@(2WPlR&PEetrrbx5pCk z=;dds=t~8NiO=vkZ;f+3kjQKIaUf25%H0m(2%shPe9j|=(gh}GO?>in8lC&i?gSNX z{Cba;(|7axA_)Ke$y~Q)?zxm z6ImnRb-}wr2~7V;@J#t4TOPs<{F5fhn9j%=#SgL{*Ctt~Y2wr8n%O@X|iCJ_G zxu0&x5w1hwI+UB$T@Gq*=B4r{n63fdlgjW37FW+%rRaOd$S+>vi^jTzt%q|BW9%Kg zGiDp5-DpVdKOP2|+FWZ8<@>A5KpT=slaf(H^8`9wCxK=eJ_|7$46J6~_)Sy=mXv&8b^5s8rXUcGN=%4SDOw`Xc^cn-lvkasFfc?x|5JC zdY_hN^Cmx^os1UK2pZj!d9+2o(kkVIkVU%G+BU@cv|Vqc&JVu^%Rd~%V1*&vfl=-G zlgzWrIqtHi?&E9v_i<3)vEN(c2U|trbBR3H>`RMvec#y-bE_Q)z`pc((L$ zo32k+50@x+`KX_9w%!iAB%ubRzTl)_=08R%8Alnh*E3EuUVU~W>t*P>&QJehd#)kQ z=qbHybY(f?+TEp)z4h#yMrR8F=xwWg2hm9>dQ=xfo$C3(I^7mtR-@kFn;%Vw2U`VD zR+h^+!|(%m{@6}U@0gG!ztk^^PScIMA%`_zZ|vqZ+`u1%@`QOEH01k~fjJWlP`C6y zG^ky4M1*0|4HT&?UN<`z)XcM^eLDzx>echIg=qN<39&+F4q7+E`=n5SZ>_ViUw{Yn z>MNwtyWj2x2`;c~P(f>wDENQ2#CFGz3=PF>E)lvhC#FM656qHX&0P=#BL+(kCY7Uc z6I3I`p9QX^do^-b)!WWfGGhnR>o&g(LiUhieuv~=g}SM5q)veB@6q66x0PyFW=S*l z6z_+V_Vc{ju{^oDQD!65!_m!eX>nS1F5hlCp*htXDUyBc7MfE|0`{A+k7l$h^hIc1 zfBw>ii#cYafqhiRa-~NGUcLSY|is!m7D#KnKF`i-9+jrh2NNIM2v*dMO zqA>zOI(`#t!~@SKvm~2Y-&zSV#!oYIaW~q51k9WvKS%T}Osq)zbx?9RqPz3}d_4WO z_9;>>+DQi-F%K;xV8p57&DL7&C-(VDT#*W_{Ehda+cSw;+>)yxv{h@Hqf54H>x*F^)l?*$|K zkqUP==V$uCS}(eWu9rFxK(V~M7*H`}5|P}RufN!v^C>v7bEd@Xmb)<$3Uh!j4;4&p zmOV7d1+kYssHW+DuJismXql{O7@8u~S|!P2uah;jLRUN9!Ijd9TI@|7%X1WOys=zf zf(J|_VpRlaIR72rWWIVGfFLDDuwi&kj;SvnTmQB=I$2TMqz)b8&<{G z5j!%iI_vQcz)nad_hHx4ZkRkQ(|gFyCLqlkbxW=jI{Th2GTSwqy$UH^k-C~aFqRL3 zDcJa43JXqYqR^@v^b%=0C1oKB1-&9)FuO?k+2id}B=6Yph$?NgIAt6O{GWWj{Nk## zcC(wt=H>Tr7v*v@nmd?l1_GD3?@llfZp0iy`k2>ium>adoW z`6|zIJR9DKX#=t-!J7*Ht0j{$r?~Xd)tTBwzM}S-P3?xVXW8VbvdI}@rr$o5UL@O^ z#pZfnJMt0`>c|ogXC%S`S8L_Fl>ua-Qf1#sek2j-imGt!C=trwrZ5@nBKgmu4r=YXkTdE{aK`c0*NURog@Bm?k*5|3B!)w7#DYx7qSawiz4J|msOX*zTbaZeO(45f@UY6C6_ zJq2zkbEt037J3@$nCsaj_xnzI{)>p9cT8mFWCRV<6dX6L23_L8V{GnJhx1_HK4iX` z-Eu1P8o%4C(1DVxw=HyP^m_|QbM$OSp+Sw#%z+xTH@E$1xjRwK zJSpZK94ZgScfj1kf&lT4fwC%Q;3#AX3r8*iyR?6*a2^B#+BEQ@iv+RTaF z;c%fP@+u5d57@s0Qf8ur(v3dfgfLj{Db;M9Ulh3Nt5zC5kH(-x;&ujce> zUld)iw!D*l{iV$(ud5@y$h(iHo1kE0nK{ZCW*wW!14g<mB7MyCv3M3 zqcDHKvAFU1$a~+_`Mp3TDgpOaWls{1Sr6i->`KA`xbyvoT!oAU%c!G4l+Aa>qp zzVimHwllLAn2hCJxojL69}qzZ6XJ8-@6{HS zx87(1cEAS-B8>gE`^IKsN6U-(Y?it}7Ng$5qva&Cg4MJ*0g`O-79+ZKTwqg;)MIPU zSI9^VdW!F*x+D<>PB$nfySy>?;^+#FI(?-UJ6y#<&g_1&b>R=&dIUk{};k5 z`d)a7cFf82TfB%Eopi!2_S*LY`ohlHlJSd6L#{ddciCHjk4NZ$4*1AnOL57!*PDV8 z&%g`iQxgLW=v1B`%Nhqs1nm}}`aUWQeN`!TEmo4A@S-bAfUkGcD}SP5c(KP*QnEkm zcUmOdL|B@FZaVyNUnRxWt5truHMP!zk5TtFR4Gr_KKaJ=z+3I}0Q)X?EkpJCKSZLjMKle8PhnFLvpOl@=GBD+x+F0mkITpvy&i+u?mh^<}15J$3{nS1; zAzOy2oa;N-a{>=V?v~*IKEyjAR+|~*sPlfG8~&|Q^RQ9CYu$AajIYm?5xSI=fV%P8 z7{ZO>uMgf<+yUoj_->dpVKmjO;Nn%^0a2Mt^@sf}#bk*idZ20LdetYePfT;bm&DWn zCX^+GSWE^iCXH!($iEho=kS50tPw6_y?c1oV;#?pRw7c4a-0fUxS5?ZWZQE z(30}DT=P6lg|QawyD_A(JF@@<7&Rw2$y?LaP4%Cp>iB>VuxDOThTr`wkCRrpjc;j(%P`;LJDy3O&D8GZ zs1#oWaGc(My*}L}=wZ}3u`OMBEFv^FKc2tgwA{+M`fC*;N0Wy6EFDjcUcq(5!GrF5 zT#(Mz!JS`@`uwbK`uT+s~`)2S0h@KpgD%M(J z(x{bvs$vgSCH$bAHy4~R-c1}JLI~D>?SB5T<;A~nclcS3u`jjBX}#eN!A*xK12fx7 z?^n$dUwmpxi+cHrErtFHhob$jR05pT3%bH*Me{voSyV7aaGuR521HCH1}1 zTduBOnyN2OB3Km*UWgZRY8i=y+P=>IvTlwXIi%~EZk5A9fB)}}eQdQqJSEy|wy_KF zUV5&$F68<#xMJqRO^!^cIMHyUh`;DhFY{-s$swSg@v+&o-Cbx{dhwcm-ZmbR4>b7M z+TUFYSkX3KD4B1>032SJVy5@K$e;I60WlqUqVPWhBGJO1VMtA_N?uA;{DKz#b#Nv1 zcAzFBQ$76m67j1ik%4Ch!Tbvm{w4U*V|Bg(hAe*y@|+p)(eR)1|95;8sPkRMc31MleYr#Q^DFbdddAI{t6N{{J4a!1hYOX{mBt zcf9LQVKalTp=MA0UaY1tY`!ssDpu_eX^oi?n$tBDPh&Kyu!8$SPJbwQ0e)bhDa-LA&#)bxPZ9!##;~?#NFh@b`$FJr4;FAw>EKCDDP?Qe1SV5ng zAFa)O=YYjU54h?v;kaxA@id41W*H3U)N%2Fa70@aK%71nu+wBVzg?2Azmd?G)3O(B zf@0Vn&5Z*LYUwv`;>$dj+OgLs_3EkIQ}FFd=$V?Cg_A-&L;=8>{_y9 ztf#9~V4vcB65hTl*)3=K{IKbe^Ls@VE%lmYjASxddg+qQ}ybDDUE8LR*gC1q>fvfJ~P>yTwvo5h*_@qjs#Td86H2OG7xeMy9 zz#mjck1oyw)2lFy7_LPEnumM*#zh2MQ5^dZ5fLs_Z1O*sD>Q$YPNfvi%VQw6R%J6= zPFWPCQEfU>tv0Fc6ruW@&vlPP=;8K*_i+!R1dsR%0s)_^?8#bxcEV0PtA8xQbkij$ zUKO+ER0hCt3z|I7EhE411z_){+7_)yGU_y9Bv`=3EWN^_Po@L5@p0^{%SFHSQ&3X> z!F*Hhn?j{WCWX=$dxH*v02P=L0ZQ3qoaWvN(>8bjqMW3KLyArp@G3~zd9A8h0L$yr zDAVZHrVvTMLwY$k#_%S!-}Tp$m_zDcp_=D*Fh_$ba8{xA2{27!{h>1^J??G#%peDg(?H01d=dyJ+3zp^Q<((P98nT@z zRMt`jA`_Ba&yq`1eN$>ydhDXdi`&><6TN$x2{<8%q<43hyX!F$M5kY#)4veB(r@H^ zb^zEKdy8#)iS*Y$vI)?Yf+6npj)b0c>@mXfe&n)Cq>SIb*q;ULorWL6(=k8%dy0SQ zP9VR3h>LKY`H10kv&V;45d@mB7k+T>=7g&!Fqm4B_fIM7=NzkjMS5i%v&HMZ=ssgA z#U1b07t1oQMZWsbn`&j$QpzIHwwYcgI6GaaERp86(v8sIyd8^zO{Y3!=3xpGDI9?} z?qcHL;BYc!q~Kn9^*u{$G)s)C7Vf4U)4$p+Kc-w##F(%C0BQUwulQyo)voknvPfkNF%<#>kFN>P zb0pcVc|+5(tSjr8n&~t`*5lnWy3ARqS2sW$cfz2gqmOL`%RPcoIWJsZx6ZmjUjs5n zzhn0-#xFpI#A>3%R|i4{^CO2!HX&52U2@@75|OaH_(Evydf zGr(UvnR7bcSfO;<`u<`d+un42`I@3-;)~a%7XaEI#xl%~JDzk)UAvCcdfc6J#qM?I z0gC|O5eb}`EWP)>y&BvV%Gz@|T#(bOw=I>eDjykG)bqQec#JP26AO0k*2?lNS2h9w ze1L{3b+BUO!>v=soR1r}%dK*?2;Ky2+vNiLn4FI_5IYqF6kc@+8PSS_&-u$(*R*m9 z>oLmbNMfObkwhREB)QnCfEcM5EuSG+S`JK3_v`KaK)KG5LqQ*+*qw2^arp>)AMS^b zf<8g*Hzb9zLVCVFh#1W7%)xQICAB>c%bl+xcPbVZsHc_l<oGkAU~Lqvo_#hW*Muq8LCC27r+A{q*e0KQx^2@d`y)SD$J_HB8r=@dh9ia_P+HM5k0S8SBinb{Q^}LYfX!pQc2XCc`@HLcZen|thu8DW&g0<@ zmk5MgHeX=?twWSZT5SeW8B`;Rp8?uw)*7PS?EV^sh@V<4sM+!<7aT@orP70q}YrTwzT-S{Gt)J);8VU*ufV}42V-M__km^Tk_vvzT z;Vk-bEo!Um^M9ABzn-xwAD7^k!{>1fL|Fw@8uhIwsOi@Ji%anm#Xm$PkHsr2>Gjt{ zhU1Nei`j)Lljco}=C__|d4qIg>%14rB}YKiu1pW__$2iF?Z%VmPflo_5Ft_%Xw_Pb zE$~uR3h&R<{?J@|c9V;bh#3E>X()|PD-46iT0!ZU^u`IiUoTazi)zWH<^?(S&G+uc z(qtrEt@-MyDh7&M#kykX^PW<6^!7nY82P!SJp>!!7F$sN^=Thn;Z*DpBY>8xK_it< z$DIvv9)P+-SI-A|G1%>(DBPAiuLs23Z>_6n-uW5{@SDa5BjZ}e-tn3arpqPqd0$ni zDn`=wz+EDTRcoc#&elaRBE5ng;EE@AKksdnu3s|oo2tmtq^a68@M3aL_uELxk2f4w zYIHpZRlWw0X^vgi+(CEhG3NTi(cJ3uIRn_E`bF0%ree4N+s19m4w8_VwYq1Bg!kCY zgCWy>;t>bTDwd}Yv?{hWD~jD%jGxR(Q(wW(07!vUZnRM0gP!NbUM^f=Nt`!K>$PbD z|E)c78hEc7RM7F>y!PT^pPa^6F$bG5x%{y5j`au~;_&op(PFwPLM#gfhRS+zyhKA8 zOmTikz?|8}>QT0IJZCsBh+`!D^qq?9;c?I+_^^!%2OQeyT-pXru<%TajKU3cBru_z@io1U>lOGA+8ONu=;NB{IhS5bC(O?;`mK_9BAz zy(;jaRS-UMtj^Iu&k7hiAQ+Kw1tov%r_p6 zBhP!!GjW4E;Hq<}$BI;%Pt4M;9w2maUMLa>>D9^J--C(gCmzhKqP7(C3_5MFslS;b zRVfH+kgg9TC^OBcorhCnps)?bt9fW~dWk+9b)l+ajtop=W5~kC^Oj_Fx5LVcc%AQL zmAe6A84rKt&?S5;68e_(>FEx@F^~qm?fgU_jd&C8yT9nyZp1ln`8El{bn*rGVt=wv zA(1|YNT6}oA7;&82nH1M+8m2syzx3L`nK7#Nnt)AU@KQmKeisqXcJ!K;#WJ0*Ul34Fu{GTDXPZf4(qk&l=Pb6*$y*9RBUobylvV-P~2R>PrKW01XZ5 zm@k{=PQ|J=rJFoxpX{TK=oz%Xs0-9DZb4G#EPFC&)hW*U$YGXTx+2{tkyP*k>rFZl z`F268gk=#0&K7Db)!cwHKr_MB@K)>eAi1AB5(+!S^UDQRsb@Do{;Ug?G6A2tB zd}n9vh(qrWRASL@M@q~;I&9U-mSbQaB0SW`GD*H=r~A-&z-}2K-$lHK8jedt-Q_kn(yip3`Qz>2=U1oa;a4A+ zP<~ubgw{bM*VxX>--8c@(!y?7=(gN(MLNFMmkkg+Dl!AnpO>r9h&bAuuhuzOtRFn1rdmQp|*Vif2t@%Ir_XVOg_+ zeTV~)4SIcP#+c9Fvqrdz`0G@yRRxUJSQyQ7p606#f21skx@v15qVB%uhbf8x2?>1{ z%j&8XkMveXjea}ba=EaX%+UAIY0iu~Hn}4(wC#?k?|K1dPoL))2#2#fRWNDQh2OWc z!mwqcQ#S98H}RUj9De&X)h_Z!-o*yq7Z57{cNvD_Eeb`-4By5KT}J;~Vf)T%|A+M} zNSqPSle*(~ubv#x3{CpbB;sr}oPLX(V8gJByidF*YA1OP%RKY|J%bN_z)TTL@l>5k zg2?;G*9Y&Q#j6mW;Q0#p3AF1`>wyhg$0WzQyE`}&JmRmPc&c;MD-CbJpq3g5>?Pu# zF28>W+0wQOO(&XchM)>41SWen`GxDDUPO}i5Ut2_r#Bqt-;sa`SZy>E*?@^mT^jB8 zqfsk+Ydi4ap#ymUSZth!;%hg_!Zlv%g|VOP%YW6kwe8-+28KrW&JeW+!b;WW&f z5@53|Lpn8qk4LYc<{3RE(7!+*FmCa5iC#+*-nRa}#M#m^TBg%<1_3AkQs7IzUCVhM z-cfQ^f~@ijSnqH#fr<NCJR*4Xx6`q6aRl}Kts8+49%fBND03KxlKoDfY8QsCA76)HK${S9`1TIO6cq%zs1LOFf)gjlx@+X zy20 z>ttw>+mI{nn3TdS@oEWnWDvPz}*`IpgjoMBr>1sViG|3u;A7GwRuJt+ULA zH0}p*5(@ctrW?P{nlj{k8osF-nipqwW#@YHH{cWC)Wa7jT+>#IGO9RJY~fv^Mm9lp4S`1EM5%r-cXc>+~z z8sZ%mgQmLG!|+@93+*7s4Vl<6aen*-m;ruS;}>nTOgF6uTzRaY}X!&>u_O_>Cc zzGow{cOp%xTy|x0tul%9g$f-GGc`1pN9&A)<=PFg>+gvs;9A%ETNSPcsz&3w5Z=A8 z?PzIVdMbOr-2OdvDTVVL*M;|9wN1CZI6vB)K_p6_SwH3#{@Wf1V}M8~P}K223yZ+g zJ|WED@pe6^?Ku$l#eTYRnb8XI@XO(gevAFMyL)Yy%XiWmpXSY5#kI;GD>c3Y*9Aqg=1Pkk`Y;v-^y#xj(@Nd||O(?tbk49$hzt z{gV9b;^aV%0QFJyVU7%@~ zW8Oa2^lG{)4kIwwgu2%qw&DQCsxrdOC6Hw`0#|?N4JSaQRmuZm58J!#9G`@7OjNGE!(E_JrnIQO`26aF<%Rm7j%>;XM!w_1%9=`EliwiMJIe zQF7f$J8x8Pn~(OA(?3Kt&tW%$)O}n0~^L_c0MvaU_{)9<=hO2GJEdQb!W}Z zz`d7EStD?WYDtlu-R-Om?-#mQ2-=%vfK7jJulvvecUT{0 zxWvH8=_~go!5qn0ZFbZk2v3!VS)p&{ERdX|?cup4-03fbn>0!IR&@C5ep{^@t*P?9 zApxX^leY;IcFB>~q1>cOR|n=qt0;5o2W*d~c4I=vL58H^y!~r`#MRunuh~%-l@qC* zFf$Cj+jtlt0E;ZB&SFpXM5 zZz&{z+gND|>vuoha8GdEVM(?VBx-fvw^UA&3#E%r6$u2HoJ#!(^;|xeu#jltP_&mL9|sht%O| z%(fc#U|+**xP^o9jdgYH91t>Frao81{wi-O2KDg)VDi*|hm8{SH#Pe(ok- zfH0a6)Sf4o$^hyCg3@7RpqDGd7Wo5Rs6xiE1QI3X-Hj@CqveZni3t34@TU^`oQN#5 zm<=HRO)LBM&1%p}{hebH%SQk7kBgVCELO8E!=JI3eBaf=)!q^Kb)IfA{pew)HmKVT zFnOyJZWwW14S33+RwhjhAtxJPoH;*Q;W+Jp4|X@4j+WkI!+zZwvGX`zu-)l6j^wkQ z^o7cFm;1;*vXj&jj*u-+W(ol#os4VzHkALW;Yr>Sz$r}xn2e8!M{uce4f2_h{%h&+ zoCeWQ57}r+UJ|t57wyF33p6;AqLwwgn_YbF_uT$r`qopZQK^wwhVwT8sAEF zJ=RTj*^>kC%{1(5))uhfe0;WZv1@6La-`5pPGixTy_iy+>g^teVF)FXrSM<%7+Wti zPs5x)qZ}aiBWR1nv4p)%h7i(C@_Tk>0Y1nohnQa=4KkCA!Xesyh z?lNCv(sdG!%2Iq!epVI_gTg2=vZeb4omwwQHI>!A`-y)pM7#p^6W2H|WyS^@_Dw2R z0$%52-=Hjg1JJ@J9ws`9FMeG?Psg>2#tUAH>0hKr(`(gf^Wli$USYH&bHM8CC6WVJ zc_Lu_Z{3Z*MzQ*_ujT)gSWvVc?$YOY<%y+M)nMMW$zbC6WzP?@`XP8usv3^y=aL6f z7T6Pl_>aT^gd6@9z7FJD{ldWcY`z1+9Q)aRW){O;ZMIO9f_@6kX z@JqCiNuti0V)qmx0p5j|yjhGZPrUkLK?JXQ!*TNk%vULh*_5LME(%zO?<9s1v42Pt zV?1B|+V5Z(O&W|sIGQ(N&-`S4%zdDjw|k)lOuebBd`6EJDcOZ2)NG#0x$>%3uPtBs zMl_)McXK9jATr{7K(Pv7z0#Gg8G*^zoZw747R;0pU|nPv01|Ib-BV8yN`>^#{BPRK!l<9D z!PSn>?yGt)ce<}nlec?9FDtsx^}(b8p&dV3 zTAg@+ZroQz{#NQJQ-p~^H4X2rhuk4ygc+PR%T@p?VC&&68Npaw39;P6Mn~S2u}}j5 zFKB01EGtNZAH)0;Up*KHfs@K9O_&gRcs|3`pdH>S;}L#H%9dcV`;heg8ekpu zJAS@YJcR4EmJ8B>Iubyp@B;YCo)DD=U3_P5I+i{U%WhE60`6t1CF+bg++7fu>6pk> zuYYd~QSIx6?aVt(EPbcr6wT^l(|{^3=Bky$8MnX1<_|OUVKOC+n0^sYM7^n4=2qlV z4g|(7xckU6eZMw1EwotaHE;RY=~4TMGwOtKxv@JCj1W~P;v>F|0m2*oNKDA1KB9ON z_l63_)T7jXXJ218TrPn5F9hUE((-(r0`qOj3MRQ5SS>DA=NjY{Mp|V{k4ceDa08N* zs58Y2VZ5}VFTXPOTJcP@p%7es=4|Tq@!vGeh7p0>pC}DBN~>!@al@J9s50t1q)c8T zOMgn{t{dKbdqQWph=jwm?4I|%uRkuI)@j8dub|IirCVwSnp1Zr3oTy;`%rEUgFSB{ z#3*rQc45|a>=3}ml9dVj*6LU)Hz$0baofCm1$8gs06JL*5ZVlU288;m zZIZqt>qFed?hEU554~$k^M0Q&vUGxjP`xn+1c@AgNI4;3?}yEe*hqTC$X5NGYnI<{ z6}PYbd8?bf-;hA(cH#M5Fr-?wECdMgzRdvBfL=^n#~}V^+?)go{$&F7Ys%&N3UXof zxG`nOnhf33oXy6UUcS^pEL{P$c}K~&mx~f6(^X7odn(j*m^J~p4h@bb37->HCd&27 z_3pjf-UiDv>C=X4TyAX{diKJZe(fz8%}~XnQ+)&aW(Wbvwzbe44pvsE7+iYN3`Cv= zEJ@nDuh%Cn2O09Oy+kL2rMO|GnUyUw4CJ;xt*zzeMBMA5R=24uBlTIBfzo4uY$y0n z`?)V|n=7k0pNfN{3Lta#XX~vppiE2sL|w!n?WQXH7e339OEX`+#Wiml^!pHF<`TM*(z zmc0M7g98BRBQsNn%l@Qrzytw}ZnrmP__KrmbI9bCG--bR7Qn9tJ|6tv;rq`KMEidm zKK~HqJT~L6K$;-c!alEyW&h=3d7WSatx9-iU%>xRSIH3Jpg`8H7fe7{6iBW?l`2+c zAUFH=3CaY|aeI`4$KxdDF$JPGB9;2#?lL30>CqAd5=GT5fS750Cp^hvn%hDwBIW?F z4O;^=3!*ru&2KydIXzzi3lA_>L>sr^Pza)RTRgSq3jimMABeH5&S@>pHB_H~51ZEV zGdKtKrkyA_IVZGO@*WL)VfE)=?U1c9~M^%?^GsLaAnLJn+L2n#bV7Lk7TG ztz1zMnpg8Y`vMA!!e!I#jU=?|8M9mpCZ3S2vz~T7Rmc!z03v%`I|D73t_k;d8I>!c zPvk#wXX$<+nhbL>G)eg@!)RF;Vlm^2=v*RG8owt0!=c$znxa)e+Mr8U>P?I|9kZ5z?Hu3e=T~fP_&_%xNK7l`O7pI$ z(K@b1B$%vyoiv$mRH-uV@1z??_L;XTYY;aba#tv8I%uSuLDBv6xIBo)2m^c>=RNm) zAkTGDsZi-X-9k4&29W`!U-kz)arzOyDPE{q8zzy;tgeq?VQfe#sGRqMkowX*Ub_8I z_Te5hFp!WzXH`kuf*IquKEN{yBrPw6%fvn;F0NxkOCgS`U z#C!)Jw*;$>OWx6tAEeho1Lu&rU=|a3L6thX?F0v<H9K&`Myk_yNO}4%6v4d^S~?O(@WVDnFLT@PvugwP8Im)#T=kCSV#8Ufk7cvWx); zf7mi@a5MuR(F{zZc!cH5Ek2WM3?)u-G7!L*sAm}qaIp|)wu(ympy_C3hNn@yvFOp*`v4Lr1_X`V|u!F1Z1 zHcCXgZB1~qZpysC3n14& z&Z>O4{F@NPY&_Let4~H9e>66bMuUfelW8^;amxHAna|C(1#9k&pvTFNH%_n@r=T|O ziCL90Z7G^5p_IJ8nUBvD^dEy?;ETl?4G#Tw=_0w*!~xk$9XNdJ&8Zf!55UCX>rdq> zzc}?@EC$Hh3}oWr@r%B*%hYzHfa?e zZ9@_P+QmoZ^W)`UFFH6KLg%Zcj(3)F2h46?dk$*Z4f9WUnzpR}c^ zW2XW?QZWAw$Z;-S>^cPa6XM*k6$YWOwwQE@^+h5CzKY=`nzhp5Qe25nDI7gW*o?93 zgKEi!6P51AD>rWSfH0oh*ceAZ8bxu(g-KC4It;a#{E!0p&4BSs*iB*~U(tMl$4o)u z^RCq^HqhCnMOI?GSEMWX8Hy>W3x&)FEcjQ52;>OZzVRKEBHX=y20hFaTn3t$pVa}^R-wxKM_ykK`EtZd{StK!}5)qQ26;d&xMdZ zvXLDS&#{%{Fu@ZVb>(p+w@DeB{y*%UWmJ~k_Vp!{5NV{OySrOKQcAi70qO1r1tg`r zySuv^q&uV=B&GiwpHq+Le0o2>V=x>t9Nu;9dtYm>Ip=T94>LYin$Uqy&?k`iV+|BT zmtQut9(4i%H0v=--{<@cm%N$HWa`^!M$ab?!VAd54ffuSi*LtN-ZBMBLs@3?A=ez> zoHG?pqfCFZ!);na*?x%lHV$0;57?oaan)N_H@F7$2>R~36Al|73C|%ug#rcntI~M# zHI=>QT%spZ-r{$^Pi|(%JMi?S}ZH! zuo)M_bRS$@IDgo&zd3k;hba4KqXQL#h5(cuSGU+xzp@oWVJ7d1)&*ZNy#kIDB()CCBk$@%aj zQ6ilaa20!9r)?{jxEC-BSgg#n({NqBa+BP~8+>%RiJ#NkV@g+Kej437mw!&Vg!L`` zv0xUohWnfpS92!{Nxp<7(u8`l9uew9Id*6d`~vDFVKzAloQbV77)B8xB#p&$(`56f zuZe;%;Ti@2}BI05vmPNFQzQm6m)E{g@BPUpyH8f)v z+$S3YIrb>3E2lU*mt&Gp2RUNK3pO>xW)@GLdEDCZnh1pcRs|WM+oj`t(U?fE?3z&_ zBXWUjR$mk6#bd99o7Y`?89f})5wXcGs714dtZI$=){>bWg~3bba!pZTs?q{jmo z-BinTv+h$HQ3^NMcR1Vkj0vA^Ixv#35U`jWgMbpATRUab`~>Z{1079pry%{N1gCSy zYJd@}bMd_w-1$ivC)urk9e*$*4`M4X%`H#yD79c92VTwynup;rSct2s1d9|xho62tFZtbw0%{r4y;sP5{3rU+aZqSe7l2CnutxI z5VomOWS57#Jgq&ZugA{(pct|{l-#{MDixpVc}W+~wdxg6o74ts@cw*f&~skjEhwp8 zb-BE!8sX;=jB~x50=RjVH)|A*hh>Z1| z;jm>sn%eGvr0_UI(|fZo+f66!in^BHq2qNU$sa8tnHW-u{klEfD7RX3=A#MbeCtFM z>&C$#VlwS+LRGGkp+9EV_T-_>)64C9tMYkRF}yS$BWl z`-dHy9OsQp1L!}lc3MH2Fu)ckOf2(TekL}8_Q9wvM(I>8x$oBvRCPf3?1Ed_^D?Yf`QaAr z$WI>_!TJ+#%U>g*t?9tJ=FfDg?0z_g$ewvyS|J`mPA8>Mw4wE5#m_4ifPFrFnW6sp z908kof;pk5{uhYtw5b&Y>oT4 zrA-dyS9)de!?EQb8wTv>DCXcW6W1X57ZuqwL@GXd&dU6|9{68GmgVjucPERY`!sQwrw8PrXPYQb8k^<2J4A*l7}X z+*GVK+gD05Z%d_LA`${r2yPZDEgI*b)C=)p3#>#N90(s&vH+8D3sV#Q7$AwxWKbdM zHI<-HuF1k4csm5bp)Eb0KPLMaOZ==uu~ljWnOK0k0{Dp;WJ`zcSOc2z~Hjvv|A%Nc4s<5cO+}d3zz@g71_8#EsxZ$PWhOu8A-RDrS zbBJ#L6825pi_96AqI(u8^9LWCWO`_e|0qT({tV8{uex?H)_Q8`znpI)m&^Y7yIN`s6Cs;Hxvz~z*ZJA zH!VQ|u~*mhRWlAifO_K}9&jzmj_Np`{Z_)gGEv16)HEQQK1W10yAjk#@IKsDIPX2T z*o&8iHTVdqAJoU*vrRX5{2szZ@OQ9+OdRmci35$t?JLkDa}|a%-dAbya9gbRoc3^& z7ZtyEcS|z3^?tkHN!nzd$$tRM3jDV(hFZ5jk|&cf~cJDA9kV9`ol@E8vI z;R}LChGqo&2TOO&-{iAF=yO#z^YUQ6<8J8 z^*)cHrt~s)c$ODS)!Wn>wcl30P^h35+Nrh~GweOK3%$gMJlQhWx`XWpan@=e5#tDX z9JoFFj$O(=_T|hm_$Q$}GykE9Bi*p?ee~e_V$Y;U;#emL$^G;tL$V>2CMVK712C(# zT)Z!`G&aZnyrKFwP6P_1D{1&#w6us78*9h+7mYJxo;*Vs%aNftw42fjP!o5#JbbVI ztxFp!v6SMZ-fpkR6}sO3VAbDZwtNVsMLrD3Yk5@*Bb@Z1c*3c&Zx+-I&i9>CE=+@Y zNoHE+o0FX`4}YCckqcn;2rixKREF9;Hu^jTpvSJ>Q_@9wIn4)Q6L7@`$;|*Ui&7@V z?hJ`mi}|>t%z9%h+E6;tb9CFJ`0M32x9iGrgnVB!BpJ~SEn*Otw9IV&+08f*LB@Pc zB49T6v8viE50rvaa_m`QEQZ4>itEEk^J*-9F~k}%f3-#d&T$1T!{yTa+Q(}NfWmg~ z+y!ne>pLQGLy}5gM4;S8AIw%v3!hu`wt9`ITS2e1-RzaLd~YFpiz3X8Zjsq?K7|V? zpl)HAPd5iz9u5`8SU!qblyCHgwG=W!IxJW-M-~$&_?B9z7X3OKmzivR9faX$Q$JV5U;*>={hpA| zGqo#Nsd#kPo-~kmpNOKDls~%KV8MQbA$rk>J|3ODP?IG-j49>H3I)=l8}AdveQQk| zv}$D-#5j(}Nm8S?;ib^rZ_cFkpReJF|N{qAIgCXKv` zLlCwhKA$kHKKv z%Y5>5VM-mCS&{c6f`p!NPBAxV|EgaA3l>k6M6|iM_Hr_SHJTLg6UM`iLUgoE9ED0$ z%Dji$`y%!LmgS5rYT^KWh9QAo+PyA|Gr_(Lbu!Jmph$Kd7H)EVhoZ4 zlkAfM&WjS;l8Wvi_P4!gXhHBCCR1uFC=;+##E(by7IZ9&xZaMyAh(OQNSQ;ce)O$R z%d^{NlLN>$IdOTks3-bUOHvOknaylCd6$X5G_AxA0ly4ay_|Ho&4_sxJW0}?f z_P~6q7(KTE;PrNS9&(k*(8<>UZ!B9XiF-(+o16dk7)@(h_U0OoZO^(0-qb7e-uHa> z+%w+pQllWyYbOMchvyl_g1YGkWS8%$L?O$JjW^_9gPTmRgFwEo!5;6im+idq zYzoxJaxaY|q$agOn;X4c=n@5WbzACc5ot%7Z_I;}5k5^9>v^ktfjY>N4_4|zYSP6>fGLE#Q=ksCBG@8Qd)$^E|GuxDu1()RSJh41TF0Su?FB6sD+3$#BOAFNE zMsmaUe(j-%xNuo@oj~1pvRM)sS;AjYojdV|*^YhUVxdYMMM)>2=gmGm!yf*CdgEnN z^J;I}c)FMj{-x(rHt4LxaKe! zO_w=);@Kp{*zv2VSh2vDSyZ~D$?0f*i`7GBdydiKkvx_>xm2}>RK@RRR9zc+8GkLhzy0|Q3&>?(QFQtL$Me;~ezfeu26z19*Mv&eVjA_lsF+zH;d1Hg>6w9d zlIg361&n`4X@p~sI$9;q@clKZC*P`W2E3%yHS;xELUC+oCJk-bE{R-RQ!*yTP0h!~ z+Skn5zXZpqy5j$k43Qs+SEAi+1`KY<{^B_ocq5ht^2r6gWanQc9fCu{mm=22Hlm&@ z+l4=mn-hz+$ym=1v}wfr_=hMdsPd?BD%C_Ji3bUqDoAIsPRqjEzNm{#ilfyr@9f7|C}||h=Q@aYOKMAIKMy2xTWA|_Lda5u{0Zvm$PNFo_R)2JQJKGK|u5P6n}nqLlR6G5wAq>3g?4b1A) zhNunvXSbJ~XfYHZt-@8Q`kzLn7e=l>c-X2?4KYq5iGS@+pL0&i3|+(wAJWTs+#CMm zM_!t6KXNBFcWd%#GIg;Y$Xq zI-JWGVmv^PCjBO)u(bQic8VjCS(Q()a7>5Ic(15JRV%W>DrSORwP3P&e0&1umd>b@ z#FY8(dxIwfq>*%7h@8O!xa^2QTr~{TT4M|!7m)Beud#6F%)~B5%Mchc1xFIvsJ84w=tnuYq##=RX}mX0Y@`uu1!%sY%AoQh^}rRs_yqj6pIRSYO+pvPB%CI=&9w z#x}x4azVe*OzFQU`^d*3Rn(zvJfa)>k2NVToC+fU%+6=?^&$2L+`WAf8oH6hsl2}_ zYPY1{3?g@Ca(N}f@G+bMDpNR^XS*h%b&k{VNSs8*VlG%Il`9r@RuRSGaMLq4>8>(O z`-0^2W5vo|ms#K6Qe6S*XCuj>Tg=q7kh#GkYTrP(^J_oEeB2;|$;d_=gDX0UrUAuY z8@sm*A?WUVBaymYpS88p5?6@3Si z2!?SVH`Aa?FC(D8nlD`-yJcsrOWgLyf;@G7!NP1slPj;Dzx9OD?8npXKu@(b4|*GI z0H!U8Pt#g0Kp+3dErrk&79E&!mRu|X4rQv0Yx-?Q_z2kD^hCj$Cr%KIm(6DQ_$hTN zc`9U|2Sc|hG&ne1(<9)z6(Vu&kb2yNXsZ3Y`EcCe2$i3`dX(zj;dgUNbtZI zPxbM~Q2gZ+$uxrmL3%10CD5MADYK>NhzF0*^2UP?b`R4DEabc`;BuyO0lXuihqDG>camrqF6$*PR`@wVb5C zq8Q;lP4wf~0Afv_57JV}h-RbXS69&ZQsN_L)DiRmt?rus8>N>F_no9Jd_Z#-289n1 zz!5}&p5Zp&a#gCt!p1AVJ^D}|wA?7*u~wW9YM5kug~G9{oYglT&$7FP^J z(`8(rRBCv*q4b551wO(Mwt$|jJymqGr1b@(87RD2nhn+7MVa5(O;3LTR&_6U82#bI zIwdYNfad=8h~@^K35n990@woSimwZc$yARL zmj~3RJ(AF~Fp!S@2127d*T;z<@g@YUl&PE=XEoHCh#K{F4WiVF`IN9dU0r=&(J1tR zD;$7<6YAX^z5qo~?vkovF!pEKW4nnb!RmKkEaG=P>!aTX5e8sJ_sqq>YY|>z!#tnU zA9z_M&L2wgO;C;K+u8(Pf5YX^$}x@V2xzfZ7zj#Vv$@Kt01+@LgBLvjRQR8sBIbC2 z1P*{y%Z0>)_699DhQ>;pEY>H{sxT_km=45v5J5d_T%mO7!K-;hzNe4VYc*2VCaX7J zZAyaa8U;X@5$O)bEs4x7tM46`5!g$kZMm2$u%0)2VRsKwS&`w>U2=G@#OTy7;-q~} z*dA$h&3q>l@7*?v8@T5p3K{3-Dz@&TUz^5)tEhp)VQw5%0pon>E}Jy2Ndb`L!c=J# z6NEyF1p*i4)M6p-vyz81HFP69cw720G@<;rOuEhFfoK%oxBJ6VuC z!vrwkT~5rfM>!sxhL6Q+C=x-a|HVh%k4$|4l`B8IC7vy{v>aD$@$l~F_J*wIJ03ZU zweF#_R_{9;pg@KLf8YfvjjO^7bldHjK8yFK;0=MC0DobeTGR~WwoHWq-3DlQRAdDd zYPTr8@E3jr5>8z2T;e$e{T9B|$dFGAmWY+J6st`+;C_h^(o@H1Z>RwHC}AZWsNH2a zkm26|VG}QSD4H%ION=B#8c${evtsTWzykCEjHmwP;Uc#dfOA}bdKd?UdYX79y|+ag zYH-xDse%wiVkBlDUUUMusL%nl&=uY0Ue?B>fSd6~&UmptsxSKL6jSYk!xISyKSYP! zb?{Qvf#;W6b}7xYyR#E^E`Tna@f2RQ*0PKrz_lc)Tt>O~?*Z)Pwh90|vM`L+TTc21 z>ah%>fexE%V3iUM==q=QdmXQL+`S=SNC29#9NlP8N6-E;FDH5Q0MY{1AcQIP$2&`z zG%=u@NRQ|OMrcPs2PB@T6XoC|S!GRuGU;vDQiWmY=5Z&Q8|Zx%Db=A-Nar$}(TZ>b z`UC*4j#m-m;o{ObW?!E<2|q)09px^OiW?hid9YE!`}!2D5YYFpy#OsLA|IHRLc=gC z^XA(tg%p0NLYyS+5-m-(mD5~z{#N(CaG8y(Eg0%RufFRz5h4}1M{2J7WY6ozTl=>u zxX+Do4w1JMR_$i!W7j_27Zk#T4)!PthDi7&drT8eAA z@Cf;uBN&yV*#*!$XN4@0lR!Moi|lk{ad_|XbCqNJo&?dx7ryp)%RLA>+Hv0_;9kVzTRzDEyS%!=Atj6P#3$9dljoY%z+^8Gch}23 zg710aIIWpPmxW?yHh)ndZ<{jBQhff`+d>d+tZekFpcs~P*@@>vw=rM z_@0tdek?fCR>GfkEY%Y>*B=(un^;WpbVZ=WX}Eb2x5UR``N)<3@?d{(e}CWMa;e5} zIF7}A1Gv;gric^^o*}5M;IDB~_Iv}vTrcQZp7AxgUX^!2>Sw$SMn@3y)E>_s!aKaf zWjC#)T?F0_#g|Kc-VlK+(1#bu3pF5Vs5&_fdbog^sk|`1U0vu+h`Lf`vFv_uxZLbU z(Nl4JwKYD;(xN4wBVBYT=S8UY0p+o-{DtU6vBZow{ZM)IeKYEtCk@1TJNo4)*|f$X z@XvP2LU!+}aXeamk{aD^s$0F`nO-V3W!$d=&@KgN zzZ;hGnln+%CxCo}3Lw)JEsXoADSod3t&rV2U>;MwB8R}VfJIlWRef`N2Nfybv~~g` z;oKEqA?EWe_>3tVYb#GatN!fhAuMYmz8pk<`T@e@{E%hDF-7Uw1`0lu{nhnemD*1c zt-}R~AgL=*JLN>wnrDb5FjKA~zwcScq{T}84DNhGX<`W{U~TL{VDSoU^w+s?#5!LT zU307{t`tA)7GyOJD_`2`J z>yCDKFP?Ha6>t~KJv|1(<{Y=%TK9{#xh7=F-cnc`Ci?MGZyctV#{2VI_e#0QD_S8> zKFrq~O-u%&!&PdLaK_1<{g`_d`HIt05cAEfdaY&hM6u>L_td-m3~MkaT}Wy-_@>+S?0Wb!h&GUYO$oC8V#l8H)!X~y@by>6xp$B zy7`QxHd?dw2`YmYuS>Ob!fEWDAUR~whY+51Kwt^;e)_!LZAH~n2uTchQO?+F1y-K6 zoie#xAaHcI;Sp$XC9@$cv3`=(PIltE=-#qZM(kv+`-fU4+THKp9E&`o~EV1d;D*cmb5W05uN~Vpnp7mEPVpId5 zP(5hJyZ4Wt1MtPCFGv!RC@jm@D{Tbp>wU{~)7G`lz|5iEAf#u`+&Q@R7RLmeWCne+ zxA{WN2`}|I{p{PHzEK$PZ0W2d-mIINkGQR%^iZzp?x%PMPN=mI9o$IDrnNNS83=a9 zxA^dwa8ENpkZ%Jv@e8{pg13@T9H`j7#t+;Snd8_r1dnkUjiK+^mY%F~18Q_y-D~V_ zm}x&RSZMnW!Os*%llfop#d<-lWbQ@eT5ve!jI#A|b78(X`eOjnxnv+K~;Z1MFJ@WllslTgG_$%Ie&4Mzv z<=ykjw01i1shN@UP09n?!04mfauJpmC4f&Sn6M!1_pvwnf^~NDwK&$vM zQSg#_JJo?H255t$zOAnVon~Vj8r*J_*hFip)LFVu5CS=`p@0jZY{@&PKoM;EdB$H0 zxXUG}M&62lDTj6Vb-vf~l9o*$QzadfKCvVk#v~_ttUvz-df@)C{J|;}6GRiU)uwYfjXlk};{4`Wg5sG~^`YY#H#ci>(W5mDb!e`qROPjkWE^wWBO3&a z<}srBHW0%D6PmW&7KJ78+&dtN$3ct6Tb3}GhZFKW(JF^`l#rI&IYv`gxJt3N(~SS% z3laQ1DBLbw_yRaz`(MW>MH*2I+R`OLxBC@wwk+WGU}-kI2V$oLl(n$k&uCaS!`8Ux zDvju3(M*-ka-^;_roALtul9bWU?pwaD0`XO@7>Q#H+L-w<&kKZl0 zkrUOw2s+4wtuVB-QkS2lvzU?uv@V5e25Hx8cw~i3X7kM~+ zxN`dgwWg{EuT>~XVrUcHgt}3nCsgOB@xdD4B?<7b5#2XqiNVLYO?i{@w#c9n^O}t_k;VFaW*)>rs0g%vw+BA_!&{53g z&UT)tKHH<+)(3R*)nD6R#2iCDg-qwOSZ-cq&5F{i{r1NBQmKWmzX<2FV!CAc(_|S% zP#4Ziy}@AL8K^+B*^^*?e0T#e@4}8tBLD?up1zd3K_Z?tFY2VBx2(G&PcH>2L`S8# z+;)dKDT>7qQJ?DtY;QHNS(topoe*tF&2>L+yCEN#BaY`#c>@Sy#v2KHwsx4JPZLYp zJ?O8dAbv=-GKaY9CSL)YY?gM*z47<;cSZYeb{;kpittBtB5yZ`5>$P#iZQ?_fSgUf z&?)#*leio=gxErjIt_3vUsrP7lv+tVu-zF`+t_Z%d7^6>=0~_4Z&ktp^WiubkgyG6 z)E~dh{ut1@L14L|EI;^X)+)-Z3LwOuV$d8SGmPcHcAh>c@^uW6F6nKZQ1jb`HDZDP z(q=7&TzmOiN?0EhFogZ=ktmeS-PUHxpWgAA28&<`b*&`7C{wR1L&E&v&$V1uYe&z^r`6mL|(+l`D(grz0C(=kNev?V|+*Us{2z4 z`R2xyemG=GLs`h+R_;COKum_}SgZ|9n+(mTW*z0hb=$@L^E^irWyHE2nv4F}3zTxJ zig;}E>s!hwOHUg2lZtnWoZ!AD?H z#xA~!?LnNjCBdYJ9!>#JX#I$y5gtHrJQ!>FH0o+6z@nzFf?xEZ!`GQ$Bs~rdX#B7yLse15**7|{c6KA?Jh2Fav5TzU7l)6G>ieEEvy@MMUdbLx0iDa|}zvwf6Hu+O>Lf=75)U`4OekdQGNi2t}YOhTM2zlfi_9Iek7N5TmHDhY+2G zfzma^J&?_3iqd!hEa24FPJ;8X>gFk-CcB8(%nA)TIg&Bb3~0VwjVnG`y+LdkP>}u# zX3)_(gL%HIp@g3-^Lpf!3@qCRKX9CPG>EjkOTgo<#B@z1o!?k$aMxZlh%`K_?!XIO$1pkMN~|WWR@r?du5L3WF^NpDc9J-%zxhOyc2e=6ck&_j9ps;!UF183SdYY=${ zw`WxYpav_N-IZ%@^4{kdF+fdp=)7I1h1gRc0gW}PX(&7FVXxlfPiT-LSeFFm+iz1< zaGCae6Wn;GVRsFRv8Q20a1L6@2#WWYI$xb0e`syAv^bCb9m!$(DwwEIcf#c-(J^|% zCt>tLLPARhOTj8KQd2U`yRcInoos7WoH#_V{YD#p>Z4Kz$q6SSe;<@UxXR*qr|~*} zUZz--kScy8Pm)^9?Xadt)!{bZPmjz^B#H0RSi6gFP|2i7f1>4Le&e*Nav?2zcE8)z zRl%ziz@!B`F}-Eg`qCzKjklS|ir_AbzolRY7KUsoLoXly_s!Qe^a;8Uw5W~Z?qF0q ze9#kvFYp+V<2f=k`g=DzBnl*yIE9^+F4!>tPcdS7We+g)T->(+4+>c56e*B99v!fF-jJU5wM&>JPhQVl@wJLlfK zSbK+acX@g`emcYiLlEteL7Tw6@Yog>G$lmdi`+>N&k6LnZ7=(=bluHH&EV5{X+U}T zNyU!pFDav0N(Y3d)kb-&*Zo7D;J+`iTW)Te3rCQ=iKKepq}A=Pd@wq-1@OCIl$Ku8 zlpe%y6{a;poUpsz$q&P@-EY{*!{zHu$R1*a(BqVH!Q&8A5$<-!8!Vtv_1r zMRo`>cGxTVrzO*CT}+^x3qC&xFBJ~MxrOnYpEP|3qy$u5 z%+5YxT^^LzNELHCP*rT@j9KTm@sNL*elWn+AA3hT@%lEnl^WO zPybeln#BmGsK;yLkk&TCcqGy;LSs1Q4$Zt_a7kyA+~wTeYwMTaP4K-&--SkEFWBhg zkE*q1=`Zo2_>+C_wGEd1LgT@=H34q=J`1*Rzn=sBchHI(Lbu^^hmbD<&27b2Z%!v~ z*O#O0?tMy!2zt1e?w9k~BDcgKAD~99;CLy1zajAT`R9)?u9r%Yln?i-BsAWIGy%&B zcXW<~Uj-v;wK4-4oOw|VcGikiacbkgT0*T`qB-Fdo7Gt6%S7D0WtV$7sg3Py#UUS> z!cUgtjgiwDdBs*Jkk&9x3RP&jxQ5p}&K5QuRRm@qO4Z&trC6`sQ`FZFpZ=Ns6|6+{ zKku(ha(|UY+~Z;^;jw>3^iCl9SN<1Li9G52NDJ5S0vY(I9R8VjHA@hGHWq zqvxx=Z^VCDM%SSyzyBgg`8Zvej8W-dG4}uPEKrZTKFV=_{t!k2n}#1MR>6Pg+`m1EGUA8& zYajUApU4v66}*89n*1Lw70-Uwlz&@V}bq0Pb%|};eWpykoW{s0c2PYf1$?}h9{F~h(4ZH&G z$X1o4|N3-1!H08-{l`;3{^}+BI2&3*r|@5_@&9;WaC!d6VBv57kC$iFulycOn@=uX z7_jQgb#9z~u8MHE-*Q}m{06X#68IF2z@x~df^G)CvsZ7wm+C70eFf_1(E%OL;(*ifO$SD z!)N*9+Uhbuq`c2YBNZc@E$74B#}i3f?b410B2_<_shDXpooo&0B08aZU;Igq7i?7V zF5xdf-q;@<06KT4oPgW4Ef~iIGN$3YPS!^+Zk6S8Wg+r_7wc?)_T5~~(f%hW#r=a! znFoc^gTD@?|6T`z`y_%K7Iq^5!cJ9T@GjKNK_F2Xyl`F6{;DffDN}I2-huHd$Blqi z+Ru{26l?zsrzc-CtNiyB<3-`=Iii}a8MoBnAaj4c=S{-%#`i3o)kt0xzc31P!ox~h zw6=P;LL4B$FHzvWZKNoss0}OhPZmHj@0X~M!%h$f+!&0j2rfs2wo>M+~G*7O;SkSQ6aG$(>HttNV8+ibeG2(orhVEN4sP zH{R6*iY2fIpY2Y*X>dwt4=2-N6F2|kI*1VpL+chLTw1gweS7vIo|9020RL9Qqz7W* zs}*#yMjiFtwHqZU@2jqpc}0#-3yb=t{{2z@^(FHpUOA24Nj+pPi~-Y-B7gmaVkDJ} zA1rcSYILe3s}S_sva>8`a5(Je!ua8IbCJXhNZd8o7eUmD)RH;U2V_7{Lgg^m;Bq)j zG+n3#LS+IWXs{y->39$O{)#kjhPB zvs|G#xm?D4S61bAPz66$rpMD4&Q(Dw9j;=&P@A@x{1{Q3yhbblj^}$BkgcEvZFzJL z4-YPvi$c)b7Y14{rK~WK<-u?mNdxbVDfynO*oxZTq>Jd*-PKkK^QZ5~qE``>J|-Yf z6MTrOqI+Lc2O}ddFpK%;s)!IQRLTY*IwK%|@hO))Rii0G`eJ+gYczT_fLHkWND6Zb z=t9aYITysU8c#^kYy83$LG)^$%~!M$#UGwddn3)p(NGQ$z?0OH>5-Zly+YV@+Rcgd z9gW(``g5QTQpjD&8Te^9mPuva&x*eK>;)vc^?N4Jwlr?GYG;EsbN+m22&BP--9wQe<&Lq`=k{LGsb;ph$ft5F}vL>!DfBZ9H6)o6e(Yy~3x z^sMZwoOcPPR57I*py@60D53&UP%NI^j28GOzT2f?Dd#K5)md0f-B>Q(A9uKiYq(3a zID;wWq*I20$r;Ft4z`9M%_28JlkEHV+eE7}cztsaLG+4{<;!FGQ}5jahh3_rMEZVss{EMxAtG3F_kgaFU5X0kJN(jDV$olT8W zdDw9h(L@?6AAW*p7XuwET>a;XZq4SWza&y=o{IL@bRZ3_oplYakO3C*?*O%`f{r zS#ReR@QAM{x+W5zS^PSb`0*7C87p-cZ4AbkO*~nfEJ5)$Sc{L~{$2g_eKS(2SXmfT z8pZt200_OLg>;4PY?pbo*sxW5L@5kyfoAmz z3h9lXdwvX=&8yzu0>gjSEwB&}DazTJ*sa7@eH1?W$15_0>0o8#a*f~?Wu+Hux1?>N z2!S*;_@v1tCueICLrr84|G7wV1KI8Y+5y_gF9`q$%Zh2@Wl<~MeGb(u1PcikS z8wTDEB$)wcnRu$vQh`=6KeLrP=)8{PwfpT)3Q?A~qjn$X>`SYwP->0*rgoRBOppK} z#D~qV*?L2QJ9h!x#~6pKCI42Bflru-cm;ooTiCB_UxG%dFdPg)Rl$UKTITWKDK^>5 zYqMFX=-X*Cq7Sf8E!H=$cpYW8%x23c^B)dsZVFj7icm5K6jytY5X?8#3>@5iv#w?h zS-A2)_N`V9&pl^?)q$Q`U+eL(*<&g0IKwSow9nIO%2#5IVbD(V4yjTyU+!AiMAvIF zvOrBtsNS=(WT-)sZ?#5XfRGuhvwZIr{5c8pWOsw(VS;R!(UHpU+cW!61 z+YMe9m`Vi53)g|ztgYpmfG`pU6kN{T$G*ItKqgO$JGaV64*PkTl&TjbL#*ciszp|T z^ge?9lv>!&sko9N%0c24(d7J&;9*LAQh%njO-Wr0z_VMtPji5mB`g#mfQ)qLRKJ|OOPJ!`ZMzQKXk--S zb-6H&o@c+Fs5XNE5xLO8#H+eU%AeHn}Wfbrolrxh2PF$~X*c$A?2tiU?CAz zP%%L%L_jGCZue!TOe~(7#9`ZoMdv56r09NSDictdr&kT2KBN5=vpp}jKswdD_B%!^JrPN zm=WLWb9r!9{-|;V3rSXN`dK6Azx4-USDe=mWV>IV?Nm+9?t@g1cJty*(##f90<-Bz z%ES9Qc4axCqD1;0S!9zbXHoTxgYz(7X490C1GSJlV9Mn}*L@Gu({yK#SFvaH{JG=# zF55cbMdJ@x(Sf9Bl=TL+irwQ4ofh2NPE<-g8{khKQ-?rWwxg3joL+n?(O8g5K!j0v z4yHysUH$NTqlF~Y^5eC3qa#(>h-3``@`RvI(-QD?;GM2A&5V}oKQ9PqN)o+4aT#+LZ~jBz;dwZ++It#)AtLe&@kdF&Rq1iBLHNQ z1_QBd_K+JAYWhA)IszjlI4HD3>N`y4mz+&S`h5)a)B+~K@JDwh>;gc>TdB-9Re5oB zVo?7w+Bvpm|^fXs~rjd zUSvEz#{`SgtR>k`N8pR@xNOc{?4HLkde$a@;_c%$Kz}J|Nrej zE(~%9TqpJ8(Hj$-$$~*-@R@i5yBR`@UD*5`ndD0&B~XWkE})6%?L|>sM#a6#iX44S9Dzp#}4v(Sf)7c7*|2Eqy{ z)%)zoOz{XXzE;_r9Bn`n#FyVzfGKx+kzrzBoCT}-f@~6pNM;I)YWYX`LKQi;U~pUw zK0Eyh4JKHEFW5D|E-o4=IFi-3#IcBKdZak-Qn^W+{-fdkSM`PRhS0r}6^%uW3h0cB zqu?~r6NK1Pt_2Rj8mQz_l)*S`z~ru?n^cl{<< zSSHoshIO*i1~pr4)_}CsAL}7b;La&X-6ALcQuIVe6zt1nfNO^{lRcE6$Iq$tmk99J zVf^un5Q*l`)_?YB1pa}~b)pyfW)tqsTb&NyU~id^13;Qh3v~GS#^B~}h3Co|wC
    HPnDDYYiQ}pTS6t3?BKx54nL&H zaus#f&!D|5c5`924)8lCOD;zR2P80wjZUXx4fbZXOS_XMy!F19d(+C`uu{-(-@7}h z-B&TfPXjphG2m|0N0H*1@BK+Jnq#?;92n0rYF{qbTD_Sn0oHN~nJ^%%^9*B7;U!i8 z6dq{%2)W(&gc1yZNzwXXGB_|;l02Q>e`h>ZNCX(t(Puj;sIMF@_Eqf`9O^xmLh@wO z8g-nK_*^b_O|DjaF0%x^AmgDT$K7r&2DVaUrX{F11#3)aHTt9JNWhjJ12pBwVU8Oa zq0e@0>XI`+B@hmJT2~kIZn`Pc=EGmB!38DoKYQx24k5dhf@SBl6rVqz`Xe+JO(Sv@)`NsclBEj()Q`n0}{1T=Xh zk8RG1&VoZw8aS8s6seVH@lCMAi#rjRT00WP>XGcZpn*OM-bAr__01bvJ%!DPgX$kHqBo@|11FJ z9Qsb%-`h^+OfYO0!TOQ7yAFP}*ug!C%U(F?BpU)qY;Q0RI4$Nn)v#s z_sJ8Hy>S^o6~$6=8*qkM2OX<9MI6cCI{0TVz=EWEbgR><5;RmNs2bU{=kNn~; zBmlHNK$e=~9!^A)7y-DLqN`!0FAu0-O;$XU)EF3a2QcnTz;AQD9O5wfMAj4eC2FhT z=pqs%GvWFX9wgYS&pSE;)9vJOI>5o}SvaJ=nMkVGP>`2pez) zV+A}ynsY)B?`v!T3Cieu>s63T{ubQ`?+vBLT5vwTPrcje5V?35@t1nhge;5Y2i13R zdQ7Qk(MQ2GTiuB|ON|U?p3$>47FHfhPC3nbtxy}XFGWM~jhrfHLWOX^k+!M<=hxM6 zm6)an88+*-UXANV9pr<0oPCKDqNWvxRHY(r{JlDZxg&wO-bktl6O5)%AwGoly8dQ2 zm-9=EyDpu8oMi;p8W7CgWJlKhwR*{--R$WlG<{J<5Pp=Bvljc$a1hwOn4&xl>1eaH zg8ke(+D+5n%xg9xz51o4&SnK$7*GtUKM#wJG95Egv+O>AJ`kf_TsbDzDi1A!6Ws#B z2$B!?1bx7qurZN0UXPDHnD?Gy&sOCjl15ocz4<#sqehA9X_f17+e0LoO}!wtFQM~Q zZ&=5oG=yY?TxJ@2$JiH$r z;^5}i*CEY6^j|^pclbV~-sWb3_Ru*OSt*fm8&yiW zr5l4rkdQ7Bq(e$-A`Q~rCEX1ojevA_cXv0^ARUwDy;%D>d#(NK^YMH-;~ista5!Bs zncVkv|Np;y{Cy);Jzsldt@Zt>NR?9vKDCm3MU1%vl1q~~ITOo=R;#H3LKv-xSbA6n ze~?Q*sw|qkbCUBR)YA}!SEsO2Qk8)Bbl`2@39lp+UV?(*H|#TaDj?92$2&u*y8_;p zJfUdWKRH1swtB7>AHs=rXvYNZFQb*&T+X(CJ_~Gtff}}Y*G}EFina&C!J1tz3X(eN zppx-^Itpo*eB!i}vfAdW9TP4hL+@cJz~-%Zk|c!+0uJ2?RF>T;OCdfaPzCo)b|P=wJ-A)(@~&hamP*L(Q`ks z(xOm^IUjx+vs4dNFC6(^25e9?{l7oBF~p$Jm;eB?G)D6%6`C50DPaV-g9*VFSR2QH zsv>2#Go3#(qd=363w517h0mAI%#H&su%JyW zRJpIDGYwu396EEmsE?<7}Vt+dn48|*Dba156 z1zCt?DZLW0TGK4>?w6|(d$gHf1*n=i!9y};OkZLT+_y$&+oe->kO&kX$R)pU;14$A zh9-|(OXz_n`z8BF+0^Z&#+=);@!=^e$1a^x;FpTwM~Bs|A^&j1@Y|`y4uqMEUf)|R z)=4`RO39HijKlSx`HHPvc^wuy4ILNgVD9G^EAQ~z{h84ca9LWI2JXbd=a%Eo98(F3 zpVABnA*G->#uBvJxm|4_^Suk$^D~t3MJ`VI@()O$B?svd=RHNH6brxqLTk4f_T}Ie zoU$;D&Igmpy!7aj`yc#npL0S*$0vG0B!ofD5$wq87$h4z|Pq$gv_k^4HX%?Uee~ly3=)o1O$th5l^f_YJ z#5iNC7J$g5%;1&Jrf^=7-Z-l6-qIQZVQLu!FOpc@bMOhxaa4%x9U#@$QukaxuM{zA~qm}&^I(L zu#;k^fHdY2|JJbv%T-=4_6c6%qW#1>weqHsk2FjP%wO?UXDCZJoTElq1xwyh1A>F) zV>g}@i-jZbxhV7Vl6gm891YR_pZ#n@kS7Fca0k~K~hEOU@y?COnZ!v0F73` zFOcr1u%O3N74Wd~aex9jMb`pGe=Z4l(MIXKC;w&3{nsR3>Lr-gJNB%&3MBVl=cQdb z1^qyUZc5@a?{I|_H!5q@Yl!HuE9U`TF`xQ~>N7`-b;DO`x?=ghw3&qXk&%^~tF6o^ z;Fs>62IboiBL2i@yP*pA}2F?BkF&B~tI&n-5s2 zpv7Y?q2z$#H)}@{_CcZM*`gz-V?OL~XfCeoGO@*0H}qX-&?mIxY{85_fV!9>F;ZRI zhef_DvXzU>5B_zDB&7Cr6n!0Zy^gBiE;$nCjy5?qcIr4cQkgYfxe%fuKdeIZ{S(?2@(CrBh4Fu$NP#Ep8Z zGsYmO@`Cu_pRn4oG*}S64ptT&tnw`M0zp*-q@w18oIqJ4xTn=Cbzk9>^G~-PCQfsJ zuwEYxIl=mC=*#_UJBQ8%t>Njh$1J3#;uxNC?PA}Vqo zmcx{nWFO)IsB_dP6USg2cI5NR_+-1oA|?IkB{sHN+2nIEJ(hY#7G z|8lK4F?yE~48gL<%*X-2=!*5%UD?EwS{dQMc8Co45j$$`m^HTfb0$gf=k^62G{*eD zP5tDr*a{4~PQO}$1bNAult3Vn!Gk=WHQ_+0gs z5o~f(K4x`g`@^yV@KXCN7D(QUqW%eol~*ZZO5{hbgvCK%Eft$Ib|U*5vkFj#;m?=i z1fF3ZUr{9a;6|jlIu_U&>o!!W#03|q@i4e? z3j))G<>qZ-0UogjbJqB|vbw-7MA}R@_U(;<>1GxYzH0Pgzxo1m&7=r$QtTWnwD*ly zvPhAt0_A?AjD0-Q(>=A!Fuoa*3l>$~xSztGpui<(BuR&0*#FR%VBn9t#Zj}D7q@8$a_U%z0W4}IxHAy?#lhWP2C0GV#INOK;tWih`euw(rG3nwha zl14*&AC3<;5Y&yCw-%8+?s~Dl`h6PuJ7b)|kgBNEzZL2wHVp@eL3rgcpDy-*rCxe$E{x-=0$Z@2e_xok|e`TlAP zHakW&u?j9_Jgor{a&8VRQTqWcdSI0vwMtJGO8{QXv}~F z!R^W-Y0`!sT{M${^YfQ1eWbXr>cpr+oj zRQU%>i4KAlpixY|2ShM3Yyd87!fe%r#p&cdy&UOv06(zHOT>%pEaiVe{3ePyGK9Jd zX*8)RB=(vF8EUO@8HOi(X6+}9t!}=0LqBFm*Fy*q@9niy9qwD=C7lp11llU+!-Y7I|Wz9e;`T z8+#(VPK@Pp_syui1$ap)0+c~F!!y$E_Z{d%oIv^Cx)O}F9!f5tTz0o5(fm4LS=5H4 z`fn6#1qIHP^-1Dl=S_a`6r4*cP?Ej|(WsSL7g+2wpn!U*aOz!=p}xy!lYY8SNHnhi zKvU+<(xt!w4E<$OX`*W3WQ`=q^>;H7S-@>ZRbnj!IPX`glKA^XoU#S@W(c@{_H`g> zLOxv*t)H_QA%J9!0&;%jT@e;e=53??r2Lx3#@BqId%(9&QC7-{Z=oTg>IAlMS$YDw zA&62Z$-KvXErV?S&t6o?_aOq7D=wU$0#4kvM<{`ABbTY(dLt;QzG+@ar-y)5FM&+M z3p>CN0ht&f4((0|WdA@|%3?ZOQTXZ(|(+7>! zF<3RoDuvhC)2}c~P`zT(JVlPE!o)k|?jgNC-bR}?%k>V!qhYh0%Hm?_b4b$fts5@q z;lEPc-^)~=uMGY2WBN1rZVXsIs$%BA>BQCdheu~4aK#plm}68@C!Me48`N7qE1dL3 ziHIFtHDl17KicnnLR`-4q^FTl^vNhrX7Gr_B1isvgxy+jzQ4< z{>sn|G37=ok#*-3@?;ADjd`q4A-d@caGO2lbc|Y=Ygwj=Ga}w{J#2(l!jK$kO6AA5 z`DBD#Uly(17~!M3pMR1_cj8Zn6EW^MCZNbwD-XLjpARq1A^s(33yiu`YLL)Fytx(! z1RqPed&-qB_yuH;jsk7-i$Od$7)cV-IgaVzK)@GqjCwxA{}CmNfk(;7z9Wf&EGWIwng$DR6t_^Jl(#*-GmDet}k_@GzG@>tddg zx*NaaJI~h|eJld={EV+MhWhRQe!#4Je7$00zSPx8JQWQ@B@BY2PB;*FtqPjoaopC8 zaR5X%F^x2J7l*QRj}M%p?y#`@B#WUOQ1lo4g%bG9|8>DNplqg6BYvRR4cyu?0>XEo=6K9QsP$z3knvf9!`kR(gyFHFOr2a}lN_(Xs zAJZ)`*L%x~!!5D2I`w?ZN*$h~esjOF`UOY>9r zNru4ZaF{wkUU)Uf&e;$*7VsqQGh?_f#L7}=D4w}E#Xs0SoQy9ICj1T?D+faS@2~W) zKghpt*@`iS+W^5lh>yO0j}%;@0EA>fUfr>{31@AglQuB>=&G6rUW1D{D&pO-1Z`LE6g1Rcu$3Zl#YhWdOgCQ&icx|QUmWoxBkcl=N9J)Mlg8f z|1$cG?_EsN%FciDH46w~)X~XqDZSMVZ2o?KnB_>98f=FNZagzCe^0CbBfIfR5Nz3k ziRgse{zLuWIL6X{>4!h29asg{uZ0W+ay5F5B_gIX8!Ucs%OnZvCa;-+YRz0%yma7{1 z&2J6e$xdL^{<6ya<9G|CSV{Uk<6#72l(*UA z(`F>_44KFNYjMm;dYE&HJ;=4A0G>8Gdf{e}49)zg5X_b)&KLSdWPO^gxC0tqE9RXR ziLRNcolf+Yf^7M57v(ekB+K-?&b^7>Bj&Xdu`iAJVAlu~dE4kb0fGAlk8f&cuG0VO zV%=gT5%j-=<$Z`XrcCl6=$2>H4^7+UQsP8TvmfjsSGzKR_WDZ_SaZ<-J+ zFWp#CH{YEtLL!G*kZINWU{W_Da{(dX2drtvN{&1L3)42CQLMEHzwe}wrAhql3!m|r z<8jAGCDVU@-!eq-zyN^B=P0hzay;f5y9a<{ml==a)jA$Weabug4&+*SEk9>^Is$cn zTCT=-EI&l{2373S7vsi*bGg0}gVVMqgq-)%N1c#0jbb`aX>9wGqr z;rX{f7}X0n-WNb?TXOQ}pu^#CDOTYAondV4tCh`tX=Gf-R_R1Aqe>=Rf;!Llzy-1M zZW2hj?3%&U@uH+Ap5ZDFD8Yy2L9Lst-pvK6`ILj$%Y*+dt`~gq_}HoYT-k^I35efq zYT$~s?qU<1M!P1RhkiKeD8P}vMqZWa4%=^-`ZHFWSz$EWr3+%sHP3%oC;})?KpdEr z2Z{ihs>Od>l4`Hbndt7WPYR^t*3@87h~szTe^Z}62h1Ex@*aTE@xK2i`bx%jNUVF| z5FiUN01~rh^Y`)sw}cLhXB5zx8@JZ}hj!xe@r5}F=Q=bnrB6FX|GaHJJ1v>jKSx|x=0JmaQ z8r*D!R0$5Tz&5!fKA%A4x%NgHveF`CF5j`g4@kdpZweC-!cw)Fo%+s7L37v~^wGq2 zKZUtDzF>O?m`0Z{!;MF7+AK>t3G|@cKDrmVei#;Z@NEvb|hlyC`;y? zlfu95ez>U8X}=Cr1iA=yS}7|AeInnb7-JiXm|bmmE(3c~b(%eUE_(*crSiSGekAOU zc(u1VABQx!DyZy$DII9nTPJaO8Hjo)a5UUT=A!0H2idGQO-)*Hqu1>ceb zLS;;5iw??Ms_VLzySEo?k~hw9`jZEaa2GU z{`5j)7^I=_w?0F5^Fff_m?`x%d||QF;B47aQ3+`k_Gb;HaJ^V@0QZUw&A41xD5Rv7 zy^e3;@c848cYz8AD74|>pR?S@7Tr2H4Z`E#T)da(>ST&45y2-&fUB%4>aR2k z_pBW+Xc2(#E~b6D?nxYlK!Ba0OsJ8)HHcDS3G6d1bRu647G{A*_z!gpKEDx&9UwDz zEeRorkx=p4d&ggCGC7h8He&dC{z_wNpjLlHyIm~*q;TNPD~;#IE^p;XT7kJNwcsmF zptf6On{;FqT`t_g+qjo58?j?B#R$}h2ApnnHxxd!-FSF&oxxj@AOLX#9LE<_Vnji^ zpeoSTP0@3s>yOQR>3R}qjN1JY?9Qads*U^{`G*XI8AEnXDYB}X9sB}siqwhl9)NuV z-gwc0CxGzp`5K&5M{qbtwOr$tW`KeUEOdai}x(XAs#HQ4+5IJ zq&b?+fTUPGc)mNZH9H=|^bZokTY6bc7exoLE-mQ0pV8(^iI~ap z=a_QnFwXXjcmb(K2#B6!9$QSNld9;=Hf1O=rlx@UN4(KF6i!l9G`r#c+En>e&M}RK zFpAN`P0xa0*-sBKUUJsFqSGyh^gtCDR_eziclw zH&{y5T{s~W8S2$3k9DG?xo9>G)FMpUO&L^$&HA*Xj($Kn1W86douOB8AE#?}I4rub zee)a{w7%_4Td>vt=MzaV|RCuqDCAC_kBNqjV zR(~=?l9FB-Jd@CpGi3rsr>WASO0Jxvzwy9pno2 zMhjw*vV6%^`1+&YGyPz$a_^@Ka}H>v^g~NRj#&U7qj;I#$BDAfj0~JMF|xn_L_5U; zH{~J@4DYr?cdlC?H~~SavkyAREh$Hg!!d8|O9p4`(dGJKd!JxYIFX;arsyt3#lscus6K`hI1Q^S~PG_e3no=ZN^*lf~{l z=0|Q0?oBqBuUmN`H7T6t3nO$N^h^dAL7-c3?6a58 z`vYr{8CC^uONJG~g7H0OD}(0w5wPA^KJm(Vg1fttVO!6R-4^1HK^6ojGg=ka7xnUJ zE+UX?3fKPm-8i#|uXVHV?T*#pM1$+sGE zTbay&FC8W4)s?gD2JWLBiOL<^l-jiQsls`o=j9>)vFd#RVO1=3P@QdF7+Iyl>o~2G z9+EJpVav);RIf5b73=yIVdQF~n{K-U_6)&!pWMW??oi^jY;oPj4Ha$1eI=kkK&ZYu z?rurb$TdYEgt&29gBZZ@K1y7!dRAkVmch~@^@4IUeAOc#oPowraUdnrvsv2Tl8SoK z!fjS?NmD+IQ8kSwNffv1i?Zd+TVXsN=N#mjV~n&t>~eTM_H>k&Y~ZhEseeD)D7+A< zF-yOH{ANe*il)C z`;grA=+_?R8@jXUKx|*&_P*sqNj{jc)SX&imiVzQ1n?mx%x(bG*cpF|<@d){PW=zP z9Zk*6&Ak_Ocfe;C)?!TVIwFQC-mm_K71a@E*DBvD4W`91a!2C7y6*nrL8tQflvT^tNxV7ff9c%Bgo_Gick&7znfH)}8$jc_ z0vYyiWM`QG@FY*WkI(C_!B`2tXeD`G+>hNqGa%VjDB$@Hn8BqK0 z*S@47Rx`x3L9^CZiPUC96W*p_A9*8|-s?0(bMLzw+`6^W1ki;V27y8AxTdXxmwF6PPzYUeiAo}WE1M>E|7ywJu*y!Od;Z>m6Q(+e6~)dM;( zIy*Nj=YeHr)MA<&^omEgJaIwArrL<>e%4mQpA(hJC8MJ_ETB=#{ZLGrh?Y^4Uxau<^ ziC4Hh)N;9is4%kUGfyw@c(@Y^*H7>5NbAV@s5m*dL|WL+|KM5%3Ec)t#$BiVS7V^F zmNzqYrg{nah@<6E!2BRoueJuU1O@=FpanNkG4yKn-E>41Mah6x7}6LZwUKv1zcoGv zo-`nw`xJM{C*hR(0-z?j7~O)sUa&6btd=$Y1aC_F=PHTM4n{MZXUaEfx60@^FYe}I zm-t84ay$N)0n!VCbQz;hq%xl^?V>NbK=#Ks3Hi}=J6hBIeGzf7tjshF{J!v&@8=I zga;Yu_g653W3UR*jQl_t$0gEn8qov3ILRae>ZRP_nxkrQXv+qf1RGV(8b1-J1awn= zVM#Lw;ubhXor8OJK16=t`Ul7J7lM2S=PHwk;EhgN*8{=|_~^nBunMXWmwdN-?5&iT z$oQ;>xGJYbukY_%iqxFRPlNV@UKLcnmQ6WRolbEAW^=J8OsYJq20u8=wx@M2Ja7r% z{)7t*(N3RCgY0a+F!qG~t*#Rpuy8_aDzLrHl=41Fhmk)Fx(86-^)q+JC^gn;tFx?Tylr zqI?RA!S9uHmYV8f^3a3EiE?N?H3m;jHZTNkCwyK}?v2Bmd)cJnGqTvYcUAv|2R`~g zM-N@Z4xmZY9+~?>!=x{p_e^Mt$!T#1okVW~kf}2 zCqCZPWV_=N);w3-Wi(q$!##w6#_yg<9)99)utt7$h0h z%ETXCG^d-45o+ZIVl$J5JrP1cD)JiKyEdSA!W{Oo>U^jj1zMux^ApI~AHzTLm%s?Q zaXOr%e$y0nxgPEHD6qZ)p&dU>r^$r0@Mcy{k5=0$_V{*uBhOuZ*X!DC4#3qdxtKZh zXxROwb+;e!;~=mYXTbq`Sq0Skq=(jJVrg0Xyc-e!tc-?OuSq>cDx!+!%Y>!gk z9ouTMuww-M9{U+=3kA>}L-W1!ZD2zTuoC1MZn`Jj;Kkj5%yS&$<)&n3Em-YHkSAjZ zH1M(A?%%4z$%sRrCW1P^CHg}rm5kqP4ppt$U8LlkDnod4An{ya1UzFLzcK+hb*y;#=A`YExkqs)b!>PO z2$i0r+=-JNrp3C#u6zmI^YUgv6lTLga zX?)>|PJxM?ae??xzhJaL`|6Dp5ak>m&#W{>gX?T`I>b!%WsOaOJ;)=W1OvYc1`~)| zcyG9N;{nVK=R7>`(Mh?#=GB?)4=f;?5mH_;cJi(?qYWvY|(T?&JGlg49Y$?2EA)RG2bKj@tI%`)E71s^H-iA7aTX6i@- za4Giu;3_|hA6gQ6Ra<;JTPiXp z6NnX#&jB3jqOXp0F9Z@&M73dY3&}lhdzG*CUa&xPaE)(ufhsHH`UsW#st;12#wElIVelG(5+M)2TrbRg z24Xc%#Ve72M^AyDo3HCZi&7lJj{uFwJQxGforZoIW zD-fym=|u2e4iAvaw$n|&#KBsQm-`{()Gj)FBUJdAy%3s&g!8Hoqye0t!Z{rFU^l#N z0!+!&A>&0VuJKHX*V*>yU?Gnj9uOjDKjS{G2XWan)t~@HYHXR-<*7Tyk!Jg02XGCY zyKDe4;g@3U)ty|&*WEXi4ZfCU_N@;CXC{zR5&w(NsYYX2wH&B(dv8b^A4ln~fQ|}d zMFvqK>QS)UDQ0%4RNB>m9o@O&^&>HGMkruwFT@X*xdNv0a(u3_5wab|aLNZs+3Ra_z^x$BKQ~18W-ZmlarMQ@L#5 z6=eiAt8ur^Ctm}xi+Ayp0YLO|wrm6N2Gp(24W9Ax5;{&*l3p;(G$k$s99mMK zvpwGW>BFRHGw7OMuVkozzw|H$Dsfd#vpFj84I{yrK)a+rvC@h1T>#+)fQx=-zx&Nk z4BjaTO?kqQr(xmO*)at`Yu*v_fV*42)%0Mrk`Mfj>*lwHAAUVI{?n3Ywy=`^HHrTV zg$|%xPwt@0^z_aM?BWkqWVpWUq{=k_WnpMV8NDz{gHalM+kcoF069@S6_Q zVU^$-PnYE`U>rhUvblv0L4Cjtvs3h&v*W<`3{Z)Fo7loi4!4)3aa5#vu++7zP34RA z4wHsY!+~GRSoi6H?A3|8Lg8nnyJWD+7J-bkkAz_`gpD`mvfww!0|nZHN(ZEZp#&|@ zL&}0Xf(2*b1Y`FxW6I?BMl!s52P9^1%Bc*3qF4m~2A;unt_O98rfP zNf>C4VSMTpM&_ZrE2+7Q`sfGc;M|heUPrt3dhCAwez^B38~jc7RjXT;!y(j;yTAkB z|6kl#gvSxIW{~mS<;h`QS{*J~h}byBOtLe1ztfRB`0mK!3wJJ@P5{`loD~Ph)9LQP_kMMuK`F z!f(;~A|%+-F%=>Q-X?Ah+;EL}TH&Aod6_YIy+vn~kJteK}N%GKtQQLD|VY+U)&^^ot zCGjxlPO>Z>cyDA0kr%e4t~8`oBirGR?hTVQbm88rIqCD#X+NmjD+(u6_L=VNiOg&# zy99?lSBn3oRVmn7FFJcGo06QJADzy*c{GZ=!p6mk+AO`=t@x%AEJU4%)TIJGZSn!V8sHbnE)OxMxvv6`N z;9pMRw#6RoUKJH>Hzml9x%_$=gXgQ1XMtHF{xIPjxIS-(jPP_R4Qb@7%{5{7?8S;(z!UAI zrDQXsJcqk_-7qH^$uBr2L_3oFdZcCXqH^%jRT+ogk5NVrf<-%hTJHu@K1g>-_1aos zX!COr55l3CUto+Vs3M6vN|p#gk1@tc*#q`f$9}fus(z+=HiJKW`xM{Me|-og@^9xk za?R3Q$XAgSGh|oRog3_JkpAO3dtVgBaW%-fjJb<4t?~B7W^#XJMm+$J*aO@u3VBvM zVQQi?D}n1aboW(#)+;5Ll#Nj9raOtG%8uNn|5p_oep$VAt0tc?INzN z4LLd@b+a||Ak(NAG*QnIv?9K&nIqsWN|#M_x1y~ufkQ{_Vy9Xdzx1p5Nh{e51O|D` z=%FQ-A1HW?5A}krz4DZOo5;c1zBU$Hl9_j0OJWN*QgGFLE@IL@ z-qA*jOsjtOvm|lero6uePu2kWtU|f7Du*3~k=_=U%Z5|uxuT=$7rQ*>t$3pGTJ9)8 z1Sv1fpH5q^2QxQ;f6O>b>JV~dMX4&V5fg1@GqyZ$LB3gt#D*};wYBhOvBoMB_Gf;u z%C4LzLAPeyH<>5-)%pFTmqGMQr=jMKY&_*oApE@cXz5Wce47pG58pd*F{pD~xU_f* z0TbZMtZ{5fR$x}vx7ab@()^`z_|8M%WBBi{x?7puKs4DEo@_ARbaOzwsH9i>BE^DX ze!>QhKDSf3mG}#&xk{(HyWX?FLQkND%%oBd;>&@Ez+-5S-3_WF!=TdX|RDB z7I(H(@54!d@fTcU`ZZ#U2ExyogT#Jrs5rmpJ&zV%BU1n`$^PoiOcU3=C-$v{T}AJX zMxjcnv>fqvm{S70CzNbBUjWL9CmyOokf9 zY3FDBtp&%!C}QnFo#|crSp@XJO@3~6CBq;tl|{G9MB_a&@W@5JNBfTJUs$R+Z&qC> z;E=&$rkwjYurYt;tJuGtt72E;%+&8S;ELa(N;)mv(M9AoN~W^S9gP@%AHWAW#w&@! z3d!LGIvV)o4>5KF2|UqwiK`UAIGwp1j$+=h{&^XR!{ES|jdf?AS%zb|Vdc(ZLn7uL zwDcV%Nzcm0_?f|lv*`y}2xg!wHDRztM5zfyZ)ncVOJk*<8rsF00h7aNylCU)eznp^ z=FzT(Q+^%Cklw)8Dckrvs?xYNZDYVh)~jx*#@QdgPk(@4ZRMxQqDQIZR%W5_i7Q~Z zJ}c#LwcX-8857t!ENw3m3hK?h#0d8KLF-hXx|I9X2L7RhPtxZKt6tSNR7nTm%G7D%eD>7k9_^kIzzZQ5Z>om*UHos< zR^|pyPQhmN9Etb6?WgP)I#P&yQEZ-%KFDPjYytG7TLP58j&PAP;_*w1cE~M~d7M9q zb-=uCINkk^hFpSt*YnBgERt!r5g~Rr&v$)-nPCAjU@H1vzhA`@m6C{0X&&+;-er*G zSePr~(#^EaTp?Ry7r)!^Mq}OCTvgk;SCqd@z>6=p|NTNT8*O92T*wtw6b(5u{{`Dr z^W|^J!_}y~QGPvfA%CLKq+ljG+s1Y0F|01HV}9ZEozYClfLhtwnA0m9c5yjGV8=h0 zJ<`g`-15QTKf{V%TlU&X@hL4tJZ${U&??J&oTXyqxLNunVWa8A?hU(7F*+l$&ogax zJy$cTxCNDWpqqtj)Lh_Tg2**+>EoymBA(BihL`-p54Tq+4d*&tA&K|j*9r?pUB7ZA zIrr52-2^}Pk@?K9-mnZbCW={7_GIo*6qtNUSJ<-HZ0TW3kPdo|HCjqAo_I8=qWx^` z&YboO-L{& zp#98;nlJq&Ty<*>0Dip{#myw%y>1&84UBI+)c zyaTCREeMuPKsK$W*2jwTuX9-T49ycBC?O?)8t@H6LP=Fo7Er9nn?}eK z-dz0D>Ia>`YRH!q&MLA8$s1oo_xT5Jqz_P=WK0y zRQGG54xcRviB-qyQoDcE6VhK80=ztq{tFhp*!P=b9K3%@MBIoyxBM><_vsCWRvJYc z^7W$js~OeYYt2Dt3O6Z5NUg9VrI-L~!wjbRD~}rbjN8?0;bA7-O$D{*Tya4{urTSR zv0ssyQYj7ux}=WYP%uB!twLI9zxF40$CV1UQISA-s*6j^5n{*l!V|;e^^Xw>R0BK* ziRQk3t4*J`t`#lY?p+3$mjMu=P@?36SWLQ-P}AUSKJEZd)3R0sVg$TIH{p1md71ZS z!Fi!$M4`WfUpBknImb;roLIKooU8}7&Lp6;eTaVf8{YR#>f0YBa3@rkH-{eHEoe3* z$S$mu`;65KB=*j#+++s#n*&boR=GZkMG)I`RO}6$SP&nua(T8JFT5!v5Dm2&JYor8g3=O^;Fdjd^h^6amvl%3pPYPBHNHBqe zj^~byDmFT^3;5H-b%SadF^3!dt|{|!X`##?w8KTU1|?92v|~{|3JMiE9n=@Y;h-{u?m0Lat@f{zu-QjmqEj}r90+LKmY|Ov zW!nrbb=OWg6tymWr1{pMZWJ>9q+5hyR?PX&jFnJC()^u^mjgCdqO@FuMqQa%NluiDRi<;8Xk z_20ElAT=|HrhU}a(IbqEj|#QU;HC<>i~Xd#-r{84?o>y8XYk}a{&@9)%zj3P#sBz@Ycs(+EIom=vmmV_(HeqR zeJR4~)U*~JTI8@jvQ&Ihs9yTpXPar+s*TNjN8;7K=OypXOahmUp9)dN&34JkT_AP{ zOr(?oA2W@=t$}rTrf?44Av$g!SSdPPB|pYX-n0xUHoLnm3$?tp5N=Fss5Bwn$n2YU z39}G#bz4;@U-T<8mCq?R6I11KB%iGglqutPW2)4o^(c4ZXKdHl4cNeX=y&_M=m%8i z(Bt~(BE>kK1DeZ=F)cA;*%!4FgBo+78*qvFjG^2bD>A7Wz<}2~Ed`krb7X0uQidL^ ztBwAyq&q#O6`*Jp&7XaiBSq-ms2jwdGv0R1xw|_R13Mj`tAAN2l(uLKUvpxfcos;0 zNVY}cwVwRp?5%R^ax)~e=O)Pzz6q)u9D(=SfA!riP`CW9N_1E$s^2u3RT={sjqwG) zs#op$C^3g_S4DzE;Kq}U8&J=SMx;*~+)&dSPAp}K180gWR5}*B7H+YU6H|6pkR-*byGC|cl7B{dnbcNJ z+Tu#|*lez%mb|}*3zL|HrlaK4r4B?^c>0YEvSq507?gbI!JZ zH~FiR@HyM293AU5m?m?VBJ(#+sA#hHY`yPB#(TxBW+}g(G8*06&l-8u2A+)k2Y$pw zr{vDrwk_<304+r$N~!lVmtebC)AXL!TO9c5-bx8Zvv`}eSFXc+4*o%G&_ttj8mwPF-z{;aH(8 zXrf(~`qp8g9CJNeNkhE#A<`VbJ{kKTiKLpkRYTu6ou-sPQTEstN~SoBt+RFOnsVRp zCxl5Zk&~QGD9XZ(-?|YsU}&r~miZS;HoTRKz^OG;*V;nB#>LS~yoqR;T+=+jxZp_{B53_-Hm=!go{R4OARTJaqvjUIQ&Mg+5wq%_%=OyTb zGy#W+ZQkLh%+d(}I6kCQ58)8myc*@!ng8tp(ydjBrRF&yH~T1zJlP8!OJOji_T&70l64ZpqC&K`C+o7rZ?`{?3Fh#eQKH-{cj zZP|*l@!Y^M>F6%QeUxd{O%fu|bk*yZ_rgWXQ6wWaa7t1vB@w=Q!ThX`Hh19)$?_C010^ZO|(fyLo{4mlr*Lu2j zC)JH$XVnw27ey9lwz$qee$TG6m{r5UC4%=g0B>LGH`}pRTdv?1ralpyk!R{JPbm7n zeG2Iut}W)OTeSGaoat~$bezm}E-JCTWN!IN;aj(B)Q83FaX*0+kuf%NSLd^mGzQ|! z3qF1CmrD#2YI7`%pMPx3f8>33cdfa*r*^iYF=t4#>N=0v6wRycXt_FTuYtLdycJ)C zW5&R(of*|%&G=pRBU7!BHQOV$t@fPg}4$ zqY>;sAumwSUrN763b)7=9q(cuS2SyLGJX%hF3$T-9r6Kym0i;|1?(DnpG;Mn8a7K< z+!^PvBN02%1XLsFb{h^jQn)_Ze|NJX+Qm-Exy;LEfUaDSxjqOhLTDM=cz;bSVk%u> z2#R?7qv|4n^+e;MVHLJ%iT;-1f?BwKcSpzuDNeINVPX5+evT6aSW&r?k>E`;QWT}($3D;)Wy#LMV37L3~ zNZ9Fjc=koZvAz#mYSTqqGkJ#uzje6*2cj+0r8|}H(9m3EJFJ5oM9ZMFiG-JtY$ESV z7k?f06L~MV7YI{t8=wo4OLh(Tx%ds;I~418WU(Ol@Y2e-reUFyHg9wdekkd~0{p}S!i;@rGvowHtj*ZF&XeQV7xXJnpdp8LM{-q*gaYb!&} zAr{9?vJ930V)Ehmnt;5J*IB6-@5A=x&-isgVDF>v-^q|#LeSif>)*p&deMP3uja?! z`f|v*27UW!WZ%Jx3n+Nn*1kh8Cn7&~A?Hi|&Aa;fbU8gh9Irc>Yr<}h<8I4IbEy@H znYWm_dw_sjN+R27HAb#mMhwS_1264=7NZ0GQiQEFU8L$DwhW(|fFR}oUqRqmMAcEm zL#}~Aj+gix8>CLf{Eg?pW zl$9>)(j82GK|H?POs?$?GQ9^vLD_54wS6ovlY}#t-xE4M`#>KH)v|8tU{18S*>8_@ z$I#-IzdUpt*MVnLtvdyb9mp%tO^)_kOfNTL07$}_VYn5HgAhfg9%ix3*$;ZO9`Se8 z-IPA&{(W-Iej!YmUQOv@4fmnm`x9dO#ntez<=%+p5MI_#6%+#v21^JLzmQqR3(6;y zy5y_dI#S75V%E4PY3Ca#)HnfE56Er&_b~I0HX{Ti0|vpRbMe2%6*|TreD)|c&Xmd; z)RmNe^4SvoU|InNu9NiBX zO$zha(D46Zw&{AT!W$+oEp`u$oe6`=`3vO0cpB(ri0zMZHV9eaoZ>lHSq5NW1jC-% zY`y(>=28YJ_eBFjm-!SfEVCDn)s_5$MB2Umt7Mm^Y0Z6#i}-lGM+5*-K1?Ck>nHbj zUi7V5Pm+j0pXlc$6aICuQK#vy^+-Ljj9o;haNt z*07ee@1+@=%egTMwmZ|12T6-;ZCAlPvk*nx8C``i&Eok`meinxC_dRm-(GuOcSd^uD!v_&DU zRB?f4{n`B-^`mD;uBVuavN>5+vlnykiygOW=SEPi3H<{0&kri%@zVuj<%rGv+(dS9zFTk<&0B040Pn*q#JcoC8ZkS2^ZC04#~~3OQT49) zmTD#Z29sYyeoPORsWLWJg#aTUJznfgRx3s2AM_+nIX~}SOW|n5ROwowr5nOsNaryh z6I9JD5p3iIqqmKUT0t8fYJuG$#NMa}(0$4r>FEk=Qex+BgY}k_&S&GN-Ya4Z{!Q4H z6fg(qq-w@#TJWx7P1wtq2~MbK-Wn(Pa}{KrjwX5hyvRxyVcF~m@m_ypFbq_xy_QAt zE6A~ktBX44SO2y&vS`VP&J`e zRD05$5Ay49VA4IJH9U|!FC8zm52GX%>~(Bgv6u>|&iY-J$P~F}q3Ud{A%Lam_ig#Y zGISN>+5XlU(S0=!R{R?}z-dDKfy34;*Omb1u|Y=%ryD40TzC~+-{-XT=bxeWqBos@ zmNPcwC_8H$gGPscEO&oF-y5B%4cjLuWhmgmw4Nt|WS&_72oKdf_L}Q)sXU#%KlaPM z{49Ecb%VsiCP2DiNufz}3BN96VH~q36{^jRf7t5{!YhjzoT4KE_RZ$T8Q*+#LO@B{ z3!7R{wp~Lme@>J`g4{k)8c}oX0L8yFPtb9~ZM^GL-EV+u$@AD=PV`>fiWHTLUj}X4 zRo5m#O{FGnEX$VQ`8hn&Lnup_b_T=sPQ~}d4-Q?Fb!cfA{xYq}-&5*G%d|51PYJk~ZPc>4D<+-r4KQHhe|s^GbT3CqQDO7n;Xj{sHv27>R+_oqGB12t z7{SGBG%PNVRavt4zG#=gNpimteoao2u1Gt#MAOocW4AJUb@freN9!glkZLJmD{&EF5V`DcC(ceX2c8z>C(Z>jISqfYk^)oH?x?#R_OLfLrrnqTlu;g*%O5owUYe~Sb51DVO)Qh-bA1Y=Y z1-{S%=RC}V_gw>A&^xHT+Hue>0xZh_twS$YC#jEXc%c4hKmbvFDfO6iYa%O#ZnDA| zrZkviwO3y~*oGobx~^m_jXsan10CQ;+hHu)isuILX+>oi2HYGC4ax%usQFxbLd`P| z4Z_uPbAFfo_-up!dR_PC-IE{^9Gsc4Z=ZfmR6z1coEee}(6&YIhNoFoHCukB)U?nfmyE{?4G`x13&No%g9K8?2k$PPO7!Sya!|u%t zQ#y_+H_v5%O04X+HP5!1>g7tgh}*J{>aCCD-<1C9LbGI^|1tao^kVzhm5}EKl(M8o zAxLM?8gVg>lN1`fbANEB4lnfh6B#b@*S9-#v5Dv}jI%8?2soAVD=rT1a$BdUN@O4P z{M5F#6iT@OR+?yOPnNxhQ{@upK3yCart{*scXkdVhF{r;N!oO^Kc!(}L^^W|U94-1 z=qAvhJRl1qdxZRjL2Y!V5W1QmFSMM#xEhlCE-`2AZFQKBxs27N=bSo_7#)cj6Fx8F z{yivARzQ6duOQTt{qaqKM73c@t&|t2D=)p(y)%V9U*A#clu;>wJ*ex zE^O=Kym_Q*Yw6Qm8@!FrGOST_k|$p2Xbm1i9INjE^ECUt8g=iNLLWLE(#iM*L5F=M zP&*Tv&s<-P7b4IF2zMS=43v~88xduLd6ccc@6V}Y4m#QH_6r;M+;Do_p8ES#(6hPj zMTj>p>D0(1uJY{>^?FtczfR4kgV|J zD1Wg*R{z7y&yAJ^fu55@PFZ^L@pIv%uYnyHEz>1X`ww@xBFf$u#T%>qY-gMy(0l=VJ&TNb98KK=7*sz~PO2-A zBLb!MicV$G|Fh4Dt7C6|3h9qX&ip)XzX2>-!cN|DOnMII{r6d%kxO6luo8eF^d${F zN*pFbF!br~?_rueW>>@d;@R<);uwT)9|C}z09(ya?ZcyU4QaS>)5(3kaUFih zA^f|v@32*A??Kmc<$zRbB;S26`nIaRt~A?-4lGi7BKxh%lO<31mH=f6a$BXH8ogch z?2EmoX{&A5pA#H|EoZuW`PMSe9{Umo(}o|)4|9Jm_VyQSmIa}hnru?zjj6@Z)ojZ0h^ac%ftgH^-J-QxFN|i>vB)8cm9Swu* zv|r$aS1#tX+(U2Z;+EX;{5Z*|z=DevK5BuA9TD!f;ykwj6{9C==A}cpQ@a?6n$pbE z8h+iL>9EwJ-1(yC`TV?@E6+rA*hLpOIc)iIPTFc$c@%B9^BB*ZnjWSUdwf-(ifw(C z#&2q7ZziSfYDV2luQbMkHyyp$so(0nAEu!db4s&&D9tlyTI%RmiJ#LJrtknxBaCZ# zr0t(+{Ge$ut?uR;nU`wzWT>9-&n?=g-2R~5A|&R#b^+wj+X5@uU91}W-}F`*0!T!d zV*kvYEt62T5z3qP&U6Q%81}#-XQzd)JR58& zgYuWBG(}s}YBsff*X$N)#!lKv3gHcb&xg5_ZHj*OC=>cb>ysqRpC71oglA+eX+3(> z)gC|s4H$t23Ht^2B!9Xw&G1+~s!FdG@1mO~eb4y~x^Y@e-^)I-!(A+w9sYS*jSvQ! zr%{8QyAPSUeS%f{YK@w521Q`c8VA)P#`U~rwZiK08ZkKwRiG=H58u^CNU#n85(03z zJ<)pP$TfZOp73@FqN{no{GmT%Uh+Z8WX3mg>+#^G)AW0}&?dS{JVL|Mcfs|UG3fr! zr?mQhG(C-8F;KB6_sZ}%eQs- z=NAfdW6!&qRSAX0hy{+A7G4!obw9hfyruG($i#n=V%iO@mjNjCfeGW|8QA*El@Iaf z@RHVUC%3u^L7qgla1X|zzTDR{I8}}1UZ>lEZH%MEPakM;8YeCY$R^N3bUKAHE`nLK zp}%hqsrM8%)6kZ%AkQ=w+yZkewdr5=S+{M9n8=BK5iX5d)cZ;NmX7Yq{YXf(@_;;q zeQom-h6d6P;;I_rrmG-SWAIISWKmH&Ze6JTUFfuZZS*&fuU5JyH1r;qeJ=_Rf8|jc z>=_+OJy)P>gl`MH5$JlBvc@&eZ})#=j-u=3(qs%8NVii#-u-L$Ns6@SJO-=aV8`f5 zSAJOVCjDNv_VhA&7oq%CtU^+@nd_PGeAkx2bQ&b$j52!Y>VakX*%hD&`zBK7Ob?MZ$c=)|_eHPAI0L;5N9?B+`g7uV;}r#6Ky)Ng_Do8ddTUzZZAC zZ43tlt0dkr>kG0_&GA@%z!3csf{s&Oef&KI_>^OKV|UJbHkMh+o`%{Dw!bwTWckfA z$iRM2H|+R(9-%q81O>e~_Oid3l}BMjNTNUxwjJQ z!We@kqz?oI``*8b$s+O4hUlo>ZA^OblIpX?DK^1)hL02HfXJdE_axBBSRWwP>~=h> zNwyr_&U8J#8@hct-ssMp!5>s}!UF)3Q&plzcc5+LW;HR~Aj{4+`1p)T1@h|ADl}yY zH~Bl{EA)g($jfsq7vfIjI+^38jo&ob7Z{SV!Bj zIH)E$-(2}rS&5Mq@VWRrrx>qy$?mbd3 zt+5NVSE#&_7G0}nAMdO`Aj;K{R^zodi-A!==SS$-ORJtrRJ7?h`*R49_(1SPRfw?C zKBq=jz>^q9s-G68jb@5A`L*^ErbSwUEb`S{K+)ljzOx*dAW!(VbiP*HCH`kESgC-q zNW>Fu^OR$y=#>UH6&722ZxRB@_G3NTFYo~-3e}Tk0n-7&o8D^7j_x-9NVibNQky1M z5{{F!h52yV~vz54lb2z!FanHt5CdRpt<651821WTY2Ml^hW%F9}OEWh6Hsx_PX^m&9muFcf zZ(ZEp7ef|SnTK$)q!`bumLKka-mBB6eh6HfKq=4kgKtXCGDVQ7%>jbaa6(KW?YeZ2 z#ObuP$lgb)`4+lCFkll|TptN~+4=nJdt7d1)s}9x8N1J-n3+iF+y<=b^U5n?Ej#oTvld_BIli5oX%Cwk*z$$vnrsNG>x0|R6d zyA<@{P6`u}C_W#mU8hZ8+$i;??&d*u*4_vU5tsubk?7;Bp+`Hr6s&4|!$I2+iWc8_8+vvZmy>K?kjfVb{rwyeHNNIl##;xkRfh|_3&$t z&I38Kc)9iM1MhQl(x{UMm5OaT_s5* zq#L@D69~sMj&Z+{`*de8@!<6Y{=!p>xSnAnWTe4*qj zzaCyEsRis1n%;Mw+OoJz#_HD!UATch!v&c@xL4;UXFzu@AnRET=VE2iQoWaEl`n$9~uJcTh2>q`aCr)J#h((L^ z%qzjbMcDR5C8Gai2AW9AJ+RjV3Atg^;`_EXl0Qq0zFWR?DjOK@SiNr`xRz>F+q*Un zyjaSltD+=(0dryb%rJ>Z9S*Wgw3FTV-#W>t4*NF1Jw2Eo<}O*ZDAM!d^Jq}{fUQjJ ztkd)hb-wb_r=h+sLsbOk$3vO=#i@VH50`X4;|Py!yf(b7dRuxj%THnU4OlkH?Om9= zj07bKn5wO-MRFPue8~Zw%^Bx-<#ahPxnv1;*TmRu(Vxsh9Ry0EN4=*Cco36{ua-io z+#6P2p8U+tWr=~f5}7LK*QX_6>|=*3{*=i8`Z3EaKMrV zV9+1?+Qnvh{WK#V^}8y|PaHG=_AIe(DwYa^)pIUutoat!NSZQXlS~v;3SCw$*&Rmpx zV9r@;p~pV4=(;vCn|nZspX5Qey^(@!v{pd$2EsPmg{ zduflG$M+&q%$~WrSFUJJ5p*BQnYytJn$|l0G;w@A)In4THSK&L!Z|ZL}q`1` ze(X$|X5i}CZ5V~oZ4Ap|7ke{`aSLYiMn$rQ7q4r?0qahL?YoU-YqLI1`Lg$o#>w2! z$dRC$1hhDcj)xF#q)p{qzI+31leN1NmAlP{Z^zNN=48~FpFEd+V;;lMWLcSAhD7q1*KEoqG@3zDa3YbG&#kB zGxXYu@XmrR4Id>`T)t&}*(EQRtQTwFIK-l?CJ(ojIA{r|m+I|J1J|D7j5nT)X(8dS zVBr|zlfjV_p>|3L5o6yAZ^x*}(mw?DDG1<;hQjPnm3Sy|4_@KxhGs7iZ^>xbheJgv zQc1sMQhE|CSzm;L&rd`7h2PzuXw_TP61p@&&aEMbpAt!#r+!s+~X7N*&fACqF31hX>&ax=t z8)OKAj^jhxdRcl9J;3*h{_~;{W#^^De3^z|GLKMIP=aLXiAS5CfX?Xsb`P=aJ#KA! z_){k`?%_S`ILioc96}FCJ-p&LhT)O;6JYt=+Sy?l5aD_}}Vq#xNC(O*kN zMTGhy%yshj3xD6|G4A8rx(;~xi=nP)Ls-*!vIYfg#@_criZvx%tZ^PWyx(U_GnF`j z339Cxpi<98HfqUZj0MO4OGQlIaAoaAbB3xgOJ00%7wzb>9@Y8|H zhW!=K#Wt%(`4i|Fny$%#iy&~=n13y`NMmEBTC7Ypt2T3J5J=+GQy`NvaMXX`fStdL+GtsUY-y7yoH6BYq~H+n3l!!mdCJ(xWg-_ETY4^GVo?4i*+GBUmcV08DdC&C zTHUWG#8z2Y3OdL5k0jZYR8tpFsmO$g-;RrOl( z1+do%H|&X$MTQN$Ym}E?x0p7H32%22IZ0ScHUFkkkWP05t9PIN$jQ_i=P3K4j5F%} z@m<-1*JDbwtki?_@&gZMFz5EEF=`L)*7hjU0he3J&Jb<{F@uH*W~pe0=(m>CaVAdc zEMSa6s+{{abhx82Q`HF-UDd8<;ABW2%)3#%Y8LS^GhZ^kIL5fg+(+3w^5rP&6SRe?x>N6wkP+<{2CJjrI!I0;1y`m~r57Ab#UhaC}t#6=vyxLe2&K z$vm8EAS|bG^iFaijIh-RA1iZ4Pbyw>wQ?P)P#c;M`-GW9+i!10 zsWsE!yhxSisMcHlGyv)MzHM_HKmz-VXy61jf$|FX*r(656kIO786!5a_me^)@=alsWJzp9uzzZi!_U3{Hj-a zLP@p*{3rd-($@Os%(U4$Drg!a$6*~fj`>C;D&IZ&O32nk}d0lk?cA8Z!5_>S)3ZoW5;divfiLrqT-nr?~w z@?hKDfphV>uZ@-AAYwyYCRZBy6HvL;@l_a-s;$yI1ZXwWH!KEM5Pj7V+tHFnSLd8k z^AgT1LsV6-#2PU^QGb}FWv$W#i)1;;{1!YN*r%Rat|q|m38R~8e%gqQfkEj5ZlQHA z@&@nB%Z-OTn^Yu~QCMgTDsr|b^1=e|sI)*O=8XD;#9FF(;~<2)D}q|2b=K=f^+sp` zVa$lTbJzKZ<+s}Bne#B(m0x)tof9s_)AT>57Lr}0A}Jv1nK5DIfK_OZs3qwx_TyPj zGydTRbzTX%!wfyTM(&C1g1bRA+)RHUOp1mx6wR-g?QTP7DEPD$qxt%xC^O;MKeyFp zDY7eD1@Hhe-&{XD-W8%^LOIwVEIGpeQv}i0*PP^lqoI^WwKvxl7-;l(F zDcgxb?Y?eft@*HT??bB1W01m3WJRk7A$|`(QJ~XiAA6Eb<8?M$CoD}uy6KSj`H#?J z!&*ekp$!AJQ!I;!cKC)%nUaA@8Osr;oRre#xauuUEYIGuaCq6yiX{T)5jSQh$bN}PubFuHsCZTmdr@PtBIQmmWfY%V`n!qvx}(y2E& zd!7wykM8h~bMB{70u{&HNf`R`@R4t!8n%eFck?bhE}ah*C4GGqeB|cLpsAE@+ZA)O zU*Vis+^RE<##JIIuF@FjY`#$Scb>(N|8QM!L8#k?t~pR`S3E7X@KAI5^q_Hd7&kNG zuymyvc04RAip+b_DO<~8>t8~l;aZ68P`%HR+?Jw{J_(8)1Se`S;O7Y|-6A!~2VBJc z%$`Z+J$aM_8G1jq>IF0E$D$m1$>qDsMRuRw9TP@wk313Xj(OPB^^1A>Kj-@?x2RI^ z{2@5k5R0D9SaJCa{PJ$jG4eb(>992qH@dVnG(+O~UND6Vjj|YqX6fCLSL*hg=Q}Wp z`Rz0>>|9FL>5L%hVjQ>DvPGCy>h(C#SI4TMiogE%g@WR;wp!A_ghKn|-&dae>mCO) zBDL%Peal4(yfrubPUkCT(3ihoug!?Qa_UU^#limmckr>-a|B;G(;#)KWY7NnE?T82 z)Lgj)9)lJ3|9R^x1iZC2%lZc`_@D3kf*H$m_xVVkyapolf8JWA1aG-OWF1BS_xk_O zt^M;F8s(xQ4u z(rUHO46EwmY~E{jT+d@?&iRP-Y*?1Pel0_973gWk{mg1AjGm`-Bc6wQ7MsW(tmHpB*ikmQVDb^oNhVY z{7x2YZ1uSrDI#8?J;zhe|J{B`_!>pCjpLf8l;e#qTxZ)=*t2&rwiB@Gs)u)>@Q702 z+r4|xMU7RQ%bx4lw<+h)^PHgpSiaUL=;^MaKY+SBrn}apVJrG;S=A3&0x3_?PAY%Y z-)FwOM$-$io`zfdWi@8@6T;skCTKivcvH$~h`+J8nDgg|x^1_wUQ-mpB{& z2W1hRy*aXmbRqVjMaXKBJZ0%*^r7Oyk1ea9`k+slRf&`z2>xDM_}(m-%T31oBr#po zUTsZRrvg@>`Fq$+F6MEu5kB)P%J3sDpyN5}1mf<@CYkcs0#Z#Hgywp;VQ=O(#9KAm zesfz(gBT9a+jiMV=>oGZYiQaD!BZ>j@eG&MhmgG{m3Bwc&G9laC@sXkVYf1L+BqhC z7Kn$og|E+vAV8H{4~DLENX0PN_(L6#EP(dFeQmXa5x_}B5sh^FX0NFKq-<&;U&PAPD zGnftTflo3JI9lwIxiwW|58aB_3aAJ41QBgh15k%Mwg=2^k7<=H9022s4y&G6H^bpi z5`PCX{T=#pHRh3-i&o!2DCjx+)##Ts-6uAm{_^nzl*!tEzPnA(--M#2h|o&nQF}H} zy!>ndh3Pi^ShRCkgu*xdinJ^$T*5fRLtT2~``fQr@VP@_K>hWLx#zaEznHk2F4WKf z0N$?0vI`|Sk)Fa8ASKCbT0PSM zDWeT>a7Rbw6kCi)039Y!Z^1`0zYZpGjdW<4!$;_}2IEBm-)c6I7-lMsLp>M*R}iz1d5kec;MHcOF}rgpY$u27neJ4rC6N_v%jEe1QM)7S zzd67i)G=4%vpOQ+nX?q47L@Z&m8+13~3-z6#S#r;01g(UPJ;Hq*t_& zDK;!>Yv1}(&Vm~D+l1#>5un_{fq;|kdAT(8)@r{S3yT<(OXC~}X4`gRZNa^Io-MRa zax5%>^W>-apVbeg%@-d;bgv2v(&^t0yfM9H2e*sN?B@WFg`Sq0rgYz;ZZW+6N{RVa zn>H+*Hpg8rhx9|>8Pqu%`67$-pLA;MGs>1~Q7t66f3n)S_eK`0YD}>LuAxFdV8_7L ztU)iI?&YZ&^=Ynl&=WxVb)WnyIs+fS(>L}!=vaHaJAg?lQ=78IHO*GP{ARqu(2ca| zv73cQ;Gqep>v2$GVa4g>+UseO>p9505oEy|!*A=>1QjrIaK@Ea3*;NvwP8S?3!;Us z=M^X&f|IC`-;2<@pb^$p{bQ|<$0J>h%eCsLb^#0T{*l3ANu4gvIJKv9?bh^JrOhm9 zrSTW@bHMK+{*dQ|K#)O>IlB1x+2z<8_ua<&Dno^r@#dgUCtt~ zY`c#^K)UvUZHYc5U#r= zjp>AySdHY@G$41MdwTRGHq89aLK`v7`cn7#4{gxEwRe$H)|)4hL zA}0L&@`UfRsO9o@KF}gPVg37;icBr#fDFN=~#B=LaCysZgB_Nm2VFbGmlR z+i@39qd)xezpGrOD2&K0XY5k{TA*q!tvgtPJLblO%*pbH^_+&)U&3G?r~ zdOQh)%AUKc&UL%?0~>^4t-{>od$UiCRX6XG@Pv1##A%bWXzUyHsp|Im&X!EqbF7ss z_$7wZL5H)%rW0p8uU_)};6L(B;iQatp;5n)uLb8d)A>7sy+(b5;-72;eGVffH=M~5 z6S(Pe9{?~A&Z@R2V|ewa!3x&LNdAfgXDTKhD09QJ8g z-)cpw%`x`GZ8vE-NY&D=^I-vkuRKZC1&rL#rK$6{wCkrI3R=q!?O&53A+Nt$7@ti53XyZ-W*+z`^!Gjt&Z)PH2-2hCYX7A0pQD{0N$JreT+d zvUMMU*ht0!7XLq~>|Q~q%o%AUrJE`K*UuVqcAXOn$qwpaC&{1YQNK^wc2UkM2<(vW ziX9ZI6nH;*UCqmM2)DG8rCD4YVPkx61A{)Icw#(_p*Ni4rpH zI4W@ooHq{bRi*5`7Q7#NekfY61)~}2!$He0b@Tp8gF*Yq%+0m>x=)`cONZO8rjR6! z$pf3w57!`bMu$gJ@U1HIWXWmt!AZ=Be}1Etzi^gi9fO2=T8_tK`}VSJ?diI17&9l| z|lH?C=PMyqv3n~Cv7t(NMUir8{uD-xeHgo%*D0dis>j%LoR_J#H zeZsDH4ZhJXtmu7Fgr0)a12z2_Sk8TOZdSysC-PTI+a#Oh^QSl=hb`XfWXj(r&k$?= z@C+DoR{*kESU~oWNS+@x^cak_KZ!GLgsM_Y@b_6V@)c(8U7kq(0z$HJHV%v(YiS}~{(NjKKg-Gmou z9Ad?{`T#SAiL+E_(R+&V@5-iJ4b&XGd&7+>7>~;{1lBWs-CN3mXkeHCk#bPefdx-Y zf+PivF{`=bhfSuFaj~i#JG0E)YN;$?SEqBsSKC#gvs%_9IMSp-o$-)qOWxb!kP-$= zDSX|I`s;i-g)JLmuBmKVuD#n_G#1#JU0&cKhy3pT-d?bHsFV+nC?bM(soRNP1n8&+ z*IcTfSvRJfJugTlg&uf8e}rZo{eyPIa-qD-@14X^v6YRuGxRfoRcm{8w>$2O5zZrG zg|+u~rtcK~IiC@d`XyOBsYvP(glsMt%l%4BFVa*sLUlB{9_mkdxSGV*m61 zXupPhiQQ0jsAk{8T#y{(HaSp90vut6l?g=Sy)10Pzw47;Oa7Cz0d(Z zf(d1Dfcs+5GWe)TuF&+*26_gT`8_uDMn`=#{jp5sS;Wh#GN#eXuFSjF3)e$7Lr@ZVGGHR{+C05ko6UZr#rTgI7u z-ao$u_&yC^0BV(qp7YoI9mBx)y+%BHV*KO(_nYh8dbiiGCwyDXA9;ad-B@jWs@|T# z>vFl9sEXfHp4tJq)9!)5z}<%BhrV0BR4JHS>mADyL)y$wRODRMUSF}})tQO@^*j!f zS}qU{wt&cHgF}(N4dx6(#(y|YSF>4emDB;(`tSoYYG?f)AW1@A0i8Rq4Gf^#T9rm- z9}vZb;+9jXI|?FAww(nx;G%R26m+IEu6Lbiqti?;{U1|dcM#~_V*`f8i0i{&i68+Ro8#cQivP!e%0=b@Z zH1jhUUd7iq8c~0m+$wx4^`69`aMCJFlX^%-xBpuVPhjiokMkvLN=jD3zjnwkbrcul zBuVef^_Um0%r)8t&Kh=>)3uTFX-BRj`KD0I+R3RBZG`-QOPne?>j!SUy)~eR zk!8|aiMi`-UixsUGwk6C&!iF2c-(wycJ?t zl{KB(z5F~Xy}8*mq99OjSvNbT{>3yy=dlWb6QEo?3HpQPIZdMezy<~~;Tc8!%119x z?I80)9araY#ZoXO+zRx*AfT6$gggkOo^@#&fPm<=@NUH*iGa}@AVBv!sRr?`sjB*~ zAw?y%hpnln^Iotgd;Pwgm!#U`pmofU?tmxwf|cHq4Vd()+LGxyi`vM$HOTYMj)LHO zm4Uq2Dx3AgikSg+f*&h|kAI741K!eQEv3^$>&>kxo7MVnigpKJ59&)Hcs^y5=HHGh zF#6KWF5s!gBlwVQ#ti_a8nHdz>R?-~(QlwO^COmHv>Ti(AA3*OYy`54R#`PUI`1P( zjY*Drg;o;p;ZDrY>vgWFV8pX%E?G4mnC!iywtsEpZ-B9uHr@^r2s))E7GtPpbq>3k7sci2!G7S^vA%HdX5J5KrBdS|6=RjRpm9& zf9+0uZ+F_SkREQ{z~`kwOmcT}M-aw0CbK=|p zLY?i&CZ%cj75myZr7WMVk+r3TYi>K!j9E4lFCOMw9qR3Pv-yq>`DG2eAJk!EN5GNq}&C#RU(4@7IoDdeQ?_PDeMgkL&*=6Hy~JSw_rsH*=Z8au<|Y?c=@ zo=5P9fOfuim5I=DoWXQZ#~J9a*>IUEK-N<2t=@E;19a^Ng%xxF_W|@#h&wiN)=mKCkgOW2{3}E*<9Ii!=7aeR zJGasWiYpBkqUR&|3q$K&ODTnq0oZhn&yoy`sT>yQuS3)w=3Uli<*JyXsf^K$v3nU@ zL{2B%rBrQWD|_OY8~7a~9Dy$eqvQ~4^ji*!@4&GIhuX79J-0$qG0Cn>3wQ*7z1b}_ z?j@Z=3GoR!aTxtJBwWXkp}nrO{1pIu!_3Z%gSRGt)^L&4GM+{Lup@x_@0G#*$>JLf zBZWG%whe5hM1h;2nm3_aBxU@cJy7ZIM24n|C`;aw&Av(BIz)J0!ZP3%H_~@U)WtxuMZ^!fH8`Y{8!{wWp7a(3m z$GJ~CJZ<~lX8=Fe!H+MaE24?392d4*vIM+DAE0zu!oj2>LhbgYL^ml{L3X*QTMK0$ z`vJ!=-+;U0Z8iB%^*m1E)bn|#i{J2f$4UNmdiGG>KC$41tJM^1*Rw4!lL+Jflc@R| z_C~(G-Y|b$P3V51jaxo8bRykWvdSjR=`C8*8(YzIo#Wz$<>+RE3Z++4;g1$|DsAxw zB!ZxFUwL9n0Mo=lgNUG$f~Yn1D6{bNvv1f;G%X(2?O!;d_MiF`M@?cV1zt4{Q-N$gh`*=-i$_#z#D%wGZb z1pv~!`M<4)K)1jeA}><{n6Yf8P5R22${gtAkUqekpf5)~3)*-=&Sj$g0uS-DSr|A- zdK!4LWL9U7enb>-E%tE@-e;*rTS*aev&xjVURv!>fs<@{-$T052`L}D>~_1XrBSao zF@7U^Dmg={F^p%gQCiqLq~K=@mqfbBN(}<k*sh~ z68i5>+6eAc)6cz3y%J=Awg9&27xn0!L?f68su&f1Bg?bD65y|qsE2HKl4f&(`_?@B zQhX?dmFiTTHASgH;tLM#0i(OjhG6Q*8#Im>!%$gsdt(NfKrQPu{*y~${oqqz^i$Q( zgH1x4|CLjM-_-&*ZnvAa*YHvp6eGGBGBz*HsL5B25t*u-xMaX(PXDrIr+1aCgV^AK zJ*U}xb4yw(<_-uawoyhP%KgXCZ3gIS76E?#unHy>d88dmT4EThCR?uGO5{lZMnPD{ zu<7vo+q2&KH+=&F?Q_U6KYK2 z+RLNG&$Qz&yslC9?#mlqSC1%!Cb3Vkz%;N$u8ymIDQ~b7=H6e)=Js3AMm_%y{461W z!F|tsL&x*6)7vy`UUO9DPn9m!%1!`hV5JUEs#v6>amG|F`S53_vX@PGe~N#L_;@98 z%b9Jhr6=Ndca$7ve87tWXIi=#4-m>HZz@t{Z1POB1+50-P{`bgVn_76uKtv9)>3Ar zKf^u;Ij?oYh@v8u<8_KP;UK)psQX<8?~Q~`VI<$`x{W~BM@vPzTXQ$ov0T2(eKC)B z6VKe()}0N!RIwCeMbvDc@l%N?^IQHM!VYDgSjU-JVOo`-#(i{u5$;7?0`{nXya4fT zC8LI&ZB^%Y;5UzRNqxWX)=4sKjdInden5hJFART{FYblj3*~P`s)eacniWVJ64S?Q zJr-uq)x(e`seD$0!Ke~c>AylqsOe?iZPxH_8Lnn4r+7!82p_{aRgrRc5a>pM0&enrlhp%zN^U z=z1E1OQK8ACglBAMw@iBv;N%x9>2g>JMt#NAtohy|lfo$f*1AgL z4%(W|$`&4cDdtLIcFFce%Jbs6z3_F*FE6#Og=c^D`@G$seI1WCTaC=;h@|ZymV@pm zQ?S@SpN0-pIvZ9z;(~B^fgBOD(<5UyM~Uce%0``4(}~G=vqwR3$TdhrFIx2T1?hTG zAr{ZHdYM#|K#ufh&HGSLza=yAU=ond@&q<=C!|M(ZAovgh#7y$$;WReXW#u{GdLiQ zsciz^wY4{Ce7QSG|9}Fzk)J&xXNwm+)->xx!fIA{(|qx{wJC3o+Rb7+)~F-xvT9KH zXy{LpqJa9ONs7<4QgfhHx*xf-Ycqb(M@mb=|u1okW=`VzfP!*3Z7 za3v*0zC2s-=?J6JEnp*%CsO*cW5-?G1s1&kqY@f8iGmSr_kg}<-s;@&Ms#rF>C~1m zVoZxn{Kw$(#CbK3zgjIN#{uFW$hPN^ccXef1KQXpRDXugLaBoFFCN*M&BR6uU%cug zEwXDI#a`W(&p)m)vv*n;5TcRHf;)F+fZ_=v%xq5+G02bN&2`bEFvCF6x83b{gZHa|8PBK<-w)stgf?zK z$vosyE9kVnu0OXn*2?|<$RcJh$iTnlV`slpcJ=9wTX0F|B_JgAm)V!GHI_sAgtho? zE_J;FMYVV#yDTEL^R=c(;24a0h72a!yH6VgiJWX=OtiyZ)ZCbDuwg3w zlnv@PQVRDl1?Hj*KuT~vwqgBySI*)8ax+fmC?VB^zVY|5tpek7Gz%q`YLi9r8B`iU zv-G^>Y+=R{q^he3F4#oS{e-v;BdTU6&9lJ&p$f!vX}f+4}u~*~iqM z60PxueKw8##P%Js1Z$N|TeL{^Z^17jj7iX94>|{hkzvSdB=wBzRtW`Bq3iKF2Or55 zI395G9jAsUfmCr9R6BSx2^6X$PT(d(_M-t;T7|lBiX3M(s0qVlHy-$l&8cDxm z;cU%N#YB;w=VNNir460@C{?z5#oM`xt37zI+w_NIS3y+u2-=W4zN95Ey38EGE8TC&fT#l%)2HP&DW6xSK3S731=n}Mv zV~Y~jM5TP5uLN(F7{7UR^y&$l=%rA(!^MAtFZkYv3QzV}M`kj?l!AaWEQtkh2YBN| ztP>L-PLmubhc!{RMQ-FMu~Lq&ka+j=t6bZuLzaHNdPi&)8}EIu>{vxdvw3Jup*ltJ0rvw8GG|%#V)W^K`p@ z7t|3*9qsAThTWwy*6)Hkj(_y6cdFs_xUp*F5fqSwhO1M8+M-{3zz5I+_48q;PZKD^ zN!}8{7QYvjp31P@ryQyy>IUKW{9u~2@Npi`cWI)QEnta<9lQO0aS@wL`2S+>EyJp6 zw=Q5x8fg%u5hXSaA}NR<0+P}V0s_+AB@Gf%BHc*$raPp&ySqETn{&?dp2u_E@B8un zey@uk279ly?sdnUV~#n-D)2QW9D5`$uG1Yq3fzNHf!3KtLmyAB{^t~5cH2fgBfLI9 z)`o&oEaTdMeXotU#$0)yn7G1J03sq)77|9cEMrk)1vwV(!CVdPB0!llPWOPcNh(-l+4#2yE1A) zx9$mD>6v%FSoFK<>{I@&wWa`M@w=ZN zxJDMfx*NjIO9G`Ehh&O-e5iN-hisR$&*19QyG1Q$0OueekR_2%K#H-*bfzM9;295b z=82Z&5t8D`Nx5G1BmZsy?Cz1O9Tz=FaJ{65!k*U%@bkUz6gEBSS&KLXEMsu)sO$}9A@ueEmdpg{gZDp zfTz1KcelCL@6NY_C1yvGKn8NvuS%Jq)|t?VI?tpcXZq*)X(bJ~mQ@Wt)gH94?LZv( zw#eSdK>czvi|e=8V#r#?Y7g=PTz{+;Wa7imm-ej~VSlhO1nY5>8D?saPuDi9192Mp zYaEZG=R{mONEsd-TVgWCiK&Ul!z|~@rXP}R45XCeZQ{JhL;rhR3{JRmDi{?N?}DR9 zCfN4d3UoLlPLr``B-d)yh9CDNA9dJRhw@i6MpbNAfBy3*P@$mU+Apyn{D}AmmiP~} z-AY8zY;JYSE~n^^B;_9@RDDl?(*M7I4va>%27TTipuod3gMbj==8&03QGbHa5A~~L zbwEAUC@_mbW%y&}JzQLd02#paX!b(-|HZb!rd4{NrRGj&H2(*O0bDTh(L+_BSwn{C zKPqJ4uU+_{D)4`g_xC^${Mi${_5aVe$KzdN@p<&a%*0*|)LLrdrFUB7$3LKihmoe3 z2k%Sj_*)!RuX!bI#rtDgnm_WHe_XWl9e6pz(A-QhwV}aKSM#7h_w|4LjxribUIT%q zwP;MRgnccI(}>=@exx7HACveW^`8|&Sd~oqnf|wdWqCt%9!4?$Irs8t5I+5ovPwhY z@O8HTsHt4s^_!lQCRP7G=RJ_d>SRk;sDB}s{=R}wCX?pBCqEU1K93v{WI(xAB<>m1 zzECRn1OThs07Y&Q+wAIFV(l2Q>*ueey#YQ9x80J9s+DF$(I$hbFCHp-9}T(`6;Z8# zz}MH{M7OKM5;;he@E-#RCT5l6y?u>jxaN2JLXn`g0=~ z0)v#N$Mxv<3zHK6yBV+vhD_@QF*aL$DQ;U^Jc`qt0QGpit_mPf1qB69YW-pvC@?yY zX2alZ0JzjSfO)G_O4pzu-^iah(dTRe2N*Hi10%NEm{o!4k71!521^X7@`Y@2@rDM2 zX|4^h(4FBwm$U->O_h@}HzG7!A5{FAfEQ%@_w6Ay-JnG|Uu4Kg_zdCYgf_fV)zMy? zfneMNGMeCO#{dYbK^ldcgw}c;ne2VYNq8;4FoFl$4k*$ zRDa)X#A6#(b0uE{O=vW`$Qy9ETntdbmVhA29D6nR@HddD&$d6WK$p$7lcYk? z$2i4JU{gO2bf3amvuXiF2?f9Op^n69KG$^%{^y&AtX{|5CNM#PtSRS53V4Fj#(epg z$>(ctqjK?HkMXR*KDP&06vzI!N=Nvm?G zHvly~lAZf;My)%!a2v@qTTl_Hen2l3a^xmMn;wX@vw$DFlqL79dVT@7yh1=PLN+p^o5@yEUX&J7?rti;SaEPt@? zAf?tjBX@U!w|}7Qm;#LI`}X*LVC-#xBE7;?)9Jo@m6~CKt#11PAOXHKYxQro!bR9` zBShQQe!ePYqxqjekeGJ9Z00Z#IjK9m<)i+$Y{j3rKFhp*zU_mS6sRIy1}vk40$$bt z>MK62k#JD(rpRKhte*M(12cR)dyq&IJGlR=*b)-yM|Oi_z}CT|{l#15iy>|GA&zew zlx4_DUspHJ^Z{Tlz|0B&(m$W_IfU^7Jv{Y?KK!#{ucepfbCAlz_NN^?)VeR-0Tb4m z-&r$kAnJB0h}kBAr#zI~48Z1(^Uiy9E27%P0i#n`B<{xj;?OU4c%)E^`3aJHkdl}w z(qOa67g-B(u-Mwl&1U;tfzWEj^nGgHP!1M;yV0@zPVu{cfd4V-Rz}boVCQG{ps&6S zP3T}423w#7(EEeVhaM$;V++Z8TjykZ%Q|=;%L7rZAkg(q`XY6WU?`>NuSU-sXi~z$*cu zP;mIO<VQdpfEu-|YNUT5ZR2y4_Ck~Ouc_e!}NS)By(tc}1AkL;7!tQRH zBtTe|BcIh8zYpj_fCMH=gcPdJ^dZ!Oxw^?x>fju}YQ^eK22^&wMCOa2XAKO2S{xG3;}h zu-%y$Rf|}LkHjv6(2uKID_6o&9jGMO0aqkL;zse1xGp6 z4+=_A=S)k3Hb89Bzn~fr*j}`UTmb)X%h~Eviv->zT)lwy5&?&U?&99|f|m6m6T@nS)z+UZF2xs~wRXpv8rU`LmirYyP4vuE-q8u` z{8j`U|ED*S5MO}U9=8!Bt{j~77Hou?PC)5ZkDQi4&kbV#unWEFuz6Uh*j;8@Z zxm72ejNj?&Lxw?uO6EC$gCf53zyVsjp0Vz}nVF2X3*|p65~&27f{WMF9ynweTOvq} z!k8a0sSV!cKhTkQ+qY`IOrq7RM~{!>49^4;$=9jXn0B9WxmA~3xGdOqkfz-{F+T%B z)9t{~#Yv}xh}{4&5ytz5C-4ARfP1lSbdxa6BL`mUg)$y=57JcvO`laL)7Rpy;7~AP zsr5Q~ZOqtLCQ9fq1{Su{EB0e%Haep0mk z$$0y*VxcBuHMh(HYF-NKQ}3uHuoawC+>PJguBv-7nf|=le~6e$`(EkYMWC#T47Pe{ zfH*{lZ6*m+FNg|aR^ZN`6Jr%T$buxf8h_2YIO=8&nqiQSV3hrb7}DatqM9j@Sp%Sl z0IchskK#G#k8$8Sz?J%pm0s1(N~7fWSi>a&1(O~gZMA$xA0{F2@27tl(!=8z+*m!= zi)BxEncLmjPqLg|8U>m)nB%85fMFb(14AJ5H%$M)Y>JStUKBKxIW}1~_7X836hoA= zv$GgOvlxMJs9qjr6m*^BZ|{$PZtqOj*_+p%?|Gl^U6TqF)yVT+&(3xyavSy~he!+j zrYl+rd*Qeml|4kGuAJ9M8(KBsjp(XJtCu)6;0OwP6>|z|_DnO;Z`@u9HkK`PJ+=Bl zE0(7$gt{@DCG9;O+`r1&Q006+C+Y}YAs49aFDh#Zf|0Op!q@Bur+TKn>r~wAgDq=% z{Q6y$I&bawBN+=b7NY@{Dw!C;H?0u{!M z!KCPCZ>eQ5}tsHbhbi~aM&0z=OZ|>cKU!M!= ztIi2%QM(Cy`ie}E!<`%TXpD7HPQ|n61go!Ce$D6IkEFv5ZqxuwD*~`*e+;hy)~q_k zV&}WlT7p8OUM{?sr{ie!2S4nteKSsfnj==>wUwv=_xT73fy);WC%?aEF|Kz;(0mt6 zN}!R7pIACvZV3K7*6Ftt4V zE-!c*>UBqB_)Y^LP36OBdQh`FoMN9>)!|*lxTdwaEv8$;?I=Tq@E>O7%rSzE%tY41 z`+sQxgsd{x#**(ci>RLGNDyvU_sggP_%Sz?OIXF5lgDHXVnvs8*=3x+P04S&g*i&8 z{;k(5Sw>ZR*N)4caIhss#w6e=*EfAeBWNTh5u1$2iP>OkPAr#s#Xc#jl@?;_!tIe9eY)j-?*fe)wKz~}^QwL) z7SF1eNmFPxJJr?oMe56JwU6Alci%`1qzFU;@o=`s3$d-&`^t(MwSy*a$;ONri@ur& zC38vQ4h%K8n+NF@4xgTZ9vCMjjMh(`2jP}^_C5MbtpR!v6htmpzNcYfCu)`bjDMRX zFeqn^|J(*%!LlN7 zm_~wZWtdGR>bTD(qa_&2ZJu?y>)iV4pM&I)klZD#+20`nm~>aAk5F;HsEwdU<1&wb z{T|2fntd!mv+Q$y$%^ox%R1@y)NDJ-O&R=HrTP5JnX2L*1>m>c23)O2aTGhpg@zW~ z8f>=_Ey2_{1i`+ohW`i#ok|I`c8tmRgBjEoXxemXhh%hGgc$bUQ3308(;`9sJox-a z1^e0=`eS)U^b}9mqMVIlIvhXVE)nSe9 z#sIdlN+B50r0;4t6tMsP@!vrB{Nv0DsYN_|EX$|d`tHW(YGC{9Xq928uF6-|?D~w< zn20_BMg$d=luBC|<%$rydj@$;5kzD&I0NXY^muHgR2U*?CPIFS;bXJ0EIJVqMde_e zq*pq_XH)r7Z}w)LZ8^(*n$KmJ7&0H-wyLidDW;!ZuJx)V+mBZ0v^cf1KDh-FH0`X2 zblQ&Z(rxp_sB!r)$puooDA_I!7O6vcpkz=lus;nv;ShX|NQwBEBv^yyiKJzHr3LPI zVAPR)}Q*R zKx{VK6o5=Ngp}fb@5u)J$UvmMtyKAe50Ha^3yT)hkB_RLhpnF$$$nQfBu*7iG9h3! zeX8v84f(HpYrGwJq{m&>m3Z!VMi~AX42$X@ua!Rr)s62j#%R}+<)^;$Ij<4>qnW&r z{=wF38pfo)zn;WJ2q;WN93gIsc^^mZG`ahRYaLA_9JL-N&s@<1b&(Qij<6@Enm<@# zl#j_?eUeq|HPsV@GqDGO(5#r~Pf{M6XzNz*nZgjsDtP<-_K=@}@>yPMKbPkole}tW z!}8*hDdIAlfjpOr*EMS{abIr#!0L3OTghOK!u`kKOS=;|+yA3r-?=UpkNt*Z#j_oaTEDYfo}sUwXs^hGlU#o>lH z4ugnB&TJYPZtcA`H{@fs2asj_ZgXD8s91P6#x_4DuaV$)IjK_eDOzX{N4$F(V{^)R zB=zmK)!j_${3FyAd8XdN1~QE*uKn2>B|ewWg)=|^Qkfs zmHLG4Nm4y}7`SR3sS_|K&1-fD@&-XOkivMe`MevE>77RqTxf;&w0X@POUS>+0gMS1 zj;C0zq_bT5E$L?n6+hMv!yg+_7z|37hdo^QSTCwZ! zaoXbC2g<*#;lI|i7RpqI>Vn34* z_P+ml8F1ALNs1x5;qU1<|Jy}82eABY%;r)&+p7OqQvZB%7`Ts_mn8pj=MR4sWI>Vn z@%T9l!HfUpzUhZu5dZsT#VBGCvL%slp9KB)Er!8F841AuFE@ZC3GTof1*l{HZCd~J z-CZ0>|I5{Jke|bX=nA2I>3{pa5M(im5*&YG&R=5RpO*tGx`sAj=GO`qDis5&CfA>D z2|VQh8>yr=gzOAK;EJ|xqQbQF>M5|}#7Zavw%)()Y8^{8wqA&Z2|`p##7KC`w7@PS z0a0d=-kGj^mo66GoB_er17~p42x!R!su8b=EL&EC!{n^$7ic`!j=92p@eLO=fG3LIgH_dbyxuQT_IYrnV`ZC3|Hr`+O6|EO zo9%sZ#=~SvYBVXVGq`E52ij4I{FGuo-IV#8m6Cu_?stfz;UNd{W721+1dwcjz^ze}g$VMguYj@|_{vf@NUhQh(^TFqS1V1F zbJ-2>_XaC8b_Krm>^HpDQS%LWhLwo&Od|T@r@nYPKKpRjphkp)`#Eq(ip9DlnM)AZ9pNq{SMgIuf`X< zS$%XXk`oO<-}`C;cjYGFav=lJs%KL_k4m5h>)Z1{TpMr=+V5Qfh=yD^`>Eky>wh5w z7x|UG6WIRSN#^Jk=$#SN^OUip?m_pUuzuKd#5&hRT4EJUN4Lw9fHa%ryan(fF4Eu2+P^bo+Z0nYaO z0WVFb1CH0-@j>1hLMfvd@ho67d43)%oYywuaX$N!s+dnqS<)%Yhw zg^g9WRVHl)P;5o+@8E;=c0k;w@$uJ3Sqf$1wX~Ly1AuwRFrD006F?J8Ks`mR&g9GW z={EJ83%f56Mo#9=Y|-~fa9<_wEYNUn1+!$NzVKWbKl&aI$b=#0nLr|*f1t;rIS2g~ za{VxQyjfBSV{O6tZZCmq2PK$Z1*z)rnKIL9qq}VfOd_FkkVp<$fx!Y90Lp@4LEs~`)oIJ8gxg0a5)e{ zRlqzRI+Q%yJ`b@uZ_rozkf9TnW;#6vx0m`r(G5)e_W3ARF+W=3doMdg+YX~O__6{W z`J(D#<$FBbK1BDaI@Ch;WM@(h^ak`61d0Q?^2ej9yEocF>%{VcSlVVc7XfgVh&Pn; zF2mBbhoUxvY46N`ufPu$>F{3$8-|TvWI~zUpW5wDQv0G48KCcRC*hu#*zKyOibsi4 z+3ifwTk$Jmc}Os>UC-CMPRr>pG~O+N+C_}FsE{Q|waq3Zsj@?~ibf*ZkyH|O@0fH9 z(v)Jm9i8fMs2k|5H+w$cTCCU6+8XeErr`ct4j=Gm!~wBe#Q7q|X^qwHlr3<+mW*li zMx?0no)7f0LY3SaA^$b6$oCXPhmj3>)NMdV!<^~Y85$wev#vKM>ZLx{;69&GAk}y|*jKav zT&{un!hJ2q`rCQmCDEX5Q5$d2eNrR-u-d`if-C4*N_gdT5?htMdk2UdtH(FtPE5zE z-3(7ArwMRz|9MzbasM;={EX`cQ~Y|2Jv4u%{ywh5#a$mHWo zQx_b(S2Cujr#g=Ymr$LDQb<|r;~dYV`v?v)-aEw zkUPo(_h<`eZQdclVD61$n|8DcOBk2q5dw8w1Gn`4^k|FIx&u-QRCnQhA!73OhUs|0 zta_wF@72ZOvK0%Y?#gt0^_1ZJsN`s=~y(BFVF1;c161Vu6ZLN(llbj|J#gjl{2*U%S%!F&Tx0Wez8dEG9q=J48##_o#px zt3_CQYh-f?5cRs8CezWcv!d6t^AO<{ zd}PXv@d?c%!l`|U0#8C;Cw^3Ps--@#YUFRfZqRO0mn~2iX)#90K!F!NP=$p@j^rrC zvAa0XNn&oAKE9<=j;58Jkx2*g@ENr_+D3kyoB;BsPhLnBInFQ?eY9MC@Dq*(7tVx% zkA#Ib2Ee!6VKHjAOp9EPj0{Aq$MQCh>_--5z0dI)P{2G>add5TM(i;eFUXlg<1^`l z(tBxo@2%rwmWE9~^=?76ke#4Tr^cK42Zc`KDcZ}mx7X+2 z6<1zXL#LL`MHq%+n6+dH>>=_#^-JfE`rqu&@jpt`Z@pweORd{p#XPgw9xMoOVino+rbW@a=51(3p|d#HRHIWAx@5o5LVNX8NE+Df_vlD|UAINxSx^<0z-weIufwzM_!HEIen7}e(k@w*Y$tbXR7ejavlwFqKX ztEi>-=IaEJUX%pK8S8*-hPbV8l?d^SBoJWo*Erposw&Tw9DZ#fzdM?nCGPoVy4qGv zzCdOlHSbzRP6xZ4yIh;m4xPBnd|rq?Gs~uA>dj@}&+$bRnH)KM&rmnUpD**3y0$?g zl!0c08)Jyvg)t*W2?BU7^)J0Qy=3s!vp|ka+^3l}Rv}OMeK$;?TLCMKCtnd)_tq%G zh#)qiMi@EqV6-mn&=Yi)LK95;Gg4lX?HN6aM`N!UpDX}gG}D$vlb6rX)@xP69LCHn z4-7mGT2}x#m9FT)7_9Yd(&rnGZjreoLGbT+S&nVbUD7s!v~%3w?d-iv6UB%Oa^IhG z#mIQ#P5Ja?p28QF3y*CshKP&a$s(NuxolQ{L}sGf8u$Xxn-DYi3RfMG3&i;hv!^w5 zZ$PTcfru5J%s4#JajcOc&YAWhJ%o&3j{c~shye*D2o<_z!dlu+0c<00ZKJ2a9{v7S zv~jgYTj(7+<|pj|%xL}E&?jGvMzRAk$lR1(nyF_YHpYY4K=4hC-M8~=z?g`%eO)=? zlNCTlA?(ZB&2eY>0aCpJ#~eBR zm3PL>X^a}!nDJPexiM&LRI$oNg&6T z_MCv*Fr`IqyR!ko&DuhOmop!vZtWaL{czfdFblTe>dA>G)H&;5ZzNl`d+2#ri+DcJ zK0`cW5nRm^qb5xx!|J(wiRPuZH6L@f+rVO&>q3DON5hOA+P2|b|BjorkT7$Pq6FBo)VHzpu%Sf4 zWur|RYTztW?6L{O`#p<)NLL3y7OUQUIXG0(JVqVLivgb{*z*oT$|ndEP7qBZ`qUJy`Nm^-QGBkS0%5N?45yS2BwZVeTjbmn&KqO_~ay_AbDf!)sD?BFN;TL1Gww z>|5!)gR6c}j!w++R;uMU1NCl0j`|(;1-7@#6Mv7Ia>qXQt&Q{RF-9eG;@QkBM!b!O zsvHz!%DP$pGA1gEU**6ZKNI4J!zcfwsO8>#ZM|E`XHtGzRrdVM-6!j1?7M%#2%eG@ z$x)k^$Aq-{#N_G7a0t0-fbJ7=%?A=PD!~w!}5TYBp3|UgxdF|ic3Xi&i$tGA(zgnz^ ztIMG(rYINd`WE>tC>)SZhE3h+gm%}2kG`RuSf$vRvZG$-^nu?U zXo^L3$q2+eFSo@kC}tL4sLp@HR515e?W{KH*iax*mOpuA_htb-6wQ#!(*3MMLyb~D z;F>gko`%-wA|DK8V0?_0G$WDEIs^pFY-EDu_#@i@aeA3II+&h?-C9c$TVrA&^)4?M zI2Pp0qpau@!D$N0^b8uJQ+&oGT(4%nQf_$ts*t&#EfF7?kMrahq``HDe)Jp5ucwCg z`bzfn*dc#~_plORERFKWF@l)z29H*|>Gf4>nBSy;ModL$UGYU{D{<>ZP$PE~y@E`5 zEH>;@#Rl0l-ylQ`V!!Nr3+g^5#$Z)Sn{nhBa>|Pd^Z;$5=lNpf+*Th&$hrFa+dJ6Q zNp306Ga{b67IG;ZUkSBN3F)?Sf}v7r+1hz;6KM%etcS2Na4A3KdIc9mnQFQWPwLGa zd%pRFal{kavpZM6>ri>a$vinzQFBY~X2Cb^uztMGyizbB@u5TA`1U7)&tQU}(MaGd zkdK_fM=L!c{EZVxk@<6hdG5tq_IJsRpMFQsw-L%tAFncG7vMJp$s8=T1(ENVB=Ojm z{6eE~N(@c}7VaF~WntweN>2yty=S3G44^D6BI10Gw=lRD8B36w#B6_Dxw};*F$|w` zzBgy9^mX%J$TwCF%B$!3h%8<$_q~SR3UH{;BH=rIF@83m{5ovP6sdW>wY*@qICk61 z1+RcbTek;UBlG)MG(&*kjUIcyB_$vkC9a>qjmsv&Vr}rwg7$5V*BT&cIh3&k7;J+FeOLQb;p+kkvhcQp8P4%5PY{z9NZB>kvgQ@$}> zsE|5;vi0JUhy)qt|AZX5D}4IH%JAYK#ka&%N~`2k4z{H_Gxw*y4uK^i z5A436@?K#EU~iLVDTT0CirX>bO~f{2jTZBTe_$M_pl`e)# zj>jkHd3vM-II7Y0XLLdF)m}8oMR<{=6mS&;{2$=9^Mwzt$5XsbB289wRA}=iJTkTq zc;N>nigfOGCbf!+k;iPt8tkKkV{DR<*cbL@(*x>|BFSsSZw z6b|yCw~dcf`%T$jhr}iU!Xs>UDwEzD#WLfD2F!U1Q3| zENb#b`o-bWQ>aFmuDU(5Y6PZ38me^`l(hcpQC(5NF13Dl zR=JpT5^3xLJIM{mW&W_9fU~cLNMKP2WG6FJg|zzOE&09!n~j&zBe{^mI>%#!_8%6z z%9y0v5a9!KO#r0s>iy_ZgvzB^eIn}p&__Fm-GrxcH7mz$X64~?3+0?%T5iq7= zwoR3rgxnU*c%GJMH+glLwq-j=35&deZtawQ2ec29-BElscW~27( zWJAO%HrUlEg{gtCj=X0`i9s(ty<#4Pz;#|3db{poBfWOO(!&OcLpfh{hVL4<&l;g8Y`s5=D?~>%>tFv83 zr_-$zDji{z!)IBma+-ku7^awg5~!g3ICep=+@N=A2;a868u6r5B8FL^VLsfm$OeY; z?OS`z%E@8QuL@*{uK}`7u^!kbdyX6kLF7HS*{olh(H_f@H%xm9MqaL9Ugk*K)2|%$ z*iEnFTmDum25EX{_Q`x-qo=omARX;YtS%^do8e2Mh=KP}Q}5gp~gb58I(e;i1#XReGgV*xJ3l*S<~f2&&S z;0a||PZzkquJUJ`r{v%8XhN=`2#`(^kl)9dtE?UJ{&aoDNS~|)0_0Y}c}Xwfm}J(# zMCo3Le{$^H(M8k9QV!F<5XZ?d${wKYWx#Ly)8Rlkr-?`~XXbXwnKk1<7GB%eO#cRGpgR^f(FU39kik>DA* zbe?RDzGJl29~T8(rK=rF#4UBW?fF9vU0POEWg9gB+I99W+1)dZkt zz=`7vtd%{QstN&#d=RKf=6q6r>h)-Gss|O~rSN0jd5Py>5Z!k;vlk11!xm;^uiBkT zp_realDo>L)$6WQO>a(DCSB3`vW=9R5ZbrZW?nmm7SDloirw?rYuj(SKv7CZ1t{DV zbhCeQtG~OOzP4V;X>lS|me#i+6DVP|*LY{-ca(A+$Y$}HNj|f-Ezs?JkAG8Y`nNxA z@H&D7>SDbhT^_)Ghy@4ME#~ZYE;1(SoI@?O8pRlfWLmFJHn&q*uDZzAosL?I9~WHK z0uvWJr+hQ?y~d-QDE+b!I@k|!9Gt%dRR%OMpiEjK_b(?-VNC%oMj(P)-ICY-uItGF zj67|a!`NY;uGy6btoMvvhVwbu7XKSqbi9RI?+qi0t;XKhO^Z*}qs^o?UlI+Q9;ju{ zqwsxK4Id+^w%yjr37u~Qi-VZJnD4to!AV`qwQ`M}MwRbzXtYzvQ?97~yp}Yi25jX~ zn0~6n7BI<(8+@Y0j|3a^uH1OnyP*pO@9#4l^zx@GOkh9_17oa#SsPxWOfh|M|Lzv4 zQ|N~Z)9CrmBm?7^aLB!I<2!~S&}14Oy+_>6UO4&l>7YFOf&1t}lW?!fUpWyh4hyz_ z1igalxfWceSN+aj=&A~9r|avR;HzsZ{yTba*}Hh|QU)|Kff-v1BCRRua4MgYU&~L) znJQx?{4hz}QD7Wvcz&R7VVZvaVxa%~i_kKgXJLwE z2~6ZwiGA=f7A^=>#8BpOjB}eFU~6suJ0c^aZ9g$-# z`XV0&4n8vr*C!Lr?CY9~XbOZ1Ff3Xsz9*8(?oVZFb+CQZxO6#VsRyUagB)89!kV-Z z@~#)O6En4ykQ!7DB0Z==)?ZOUS+d@LS1nLT`hek}E5Lh4-eg|R)wF=l$8m7|b^Gi3 zcdpEf-Zl;rKwoLk>sm3IwcuGY9*CUakUx}Vm*`4?RdS-?|lagEdkl7}se z(#Tn&S{~PBif6XX)2Z;bj_izg2j$c6qhh)AMMXdwn=g7z*)Ir6_KkTe>`_!9RgAuzxZH2?j za4ANMWLxhYG)UX;Jw+q}OBDPe#@XI%?ou$HMk|00I` zjox$d17^1N7Co{z{BQsG&#?j@qaXHk4|VkUZxW0fpyp2X3r>>!1=#%0rv(?l_>x|n z$0~{a7x8Jq5HNB6?_&QRm;W2ae~-n(B>Vq8L9lxJcZ<1kka6Jvl>-|{Q@@{PfgpsK6NBq^ zy38ow0kq(JbO%LdI#8)+0;LmrfX-5+irNOxf;QB*ptAaW=Hv(IACx=KFf3$R{c6y- zn)-Q=3egY5N`XL5@@a=1Z<0%d{K?*IODB)@THfdR`m_fo)e~O3^vIbF?KyoS` z7+RuL2T6A>4p9>WUqu+wm=JFF-)4C)RvhBdHS5s5v4BKvxs^ z2iETO>ct%Gl25=C4(_t&l0W^9%PInswE(6A|9Y$fMR^}S?iaCKZusifbrG)Baz3}_mR9W`eqwLp+kKu zpc$xcNhb1rAk6~m*%666r+(3TWXdRwRG>&t$n8)pt9lfw%C2=Jd$KVo(&Bp>qB&FN ztb|T3(DiCfY692c?uL9YvzM?v0C<1lvwj}nYG<|o^T|`OAE-uo>Pm&0xPYIdcn)jL z!SFGg;-LY+SxftO8=orU`Tjg1Q2Z+8iD%P~Yvh}yl}+^rG?jM?uWHfOut00UOvX%oAI7uoV4fDx?ca$(q?D6!IY z8UV-)4Hv+w!f?fBoRl`eE^sN4e_DSa-{IG^#X6J1uP+XDUs*PlCQFitcJBbMMCVbG|Kv?+Dd0wTHUqW5OhkIDGH&;?bm z*;xeN>FjNQrg0zPVuwSyVv`t!#TF02N;9pT3fafvcU9I6fq=d(E^+c{)wh8DdXXGr zGE!m6APcpi#AZ;G9BuH!B>$x?3ksBMPd4O3$X|L^giFbF0M&ZVZy+S32E}q+ z5$dt-04_?ri{0 z*Cz7nN|{&fmZQ)dC?;jRYZa)995-GiHf7qc-G>_HizMF@vjHLF zb68MUC#8-S1Uu#1(O*%K<8X*1EFa+Z`vy{|5mKjWaA%l%LOVFAfAJ)C01jEVevxlg zuI~H$)fj6k*DD)(M#Vg3Dgaq5jpeIFv+@1KHvY$a@jQ^CSco)&(W-UG0Tw4rh|aX! zeO)%d94XjkCr_hh&MN--2V&XmFg-f*PY~sz0U7`J<3s^hdXSXp<|I9Pk7K)|J~s0h zRWOCChEmv)oniQMrF2qabVZ8Cr|xKxTSlclcG+TG&Wr(NRG$S3on{2x7GLu=paG|o zL&{~QE?e@+QnuDXw(3Mt*m6f%25+a?2TiqW3!a^qR$cW+yUQ8ae35Ht5?HW28&aZfR(;nO5Fl+J!QJFG% zXDqdUvt5OL`VB`YGNq1b{bMo8^X|tZcqEgtJd3Kw)5{_4Cv&1=~LJyyg!>>#VXe; z7^%HnCr~=^g!l6gCMp$2Gt!`@ItNMi4NjxcV5;L1=vn5F3iY1{V%p!C=1J3ojmnVKQ$+Hw;E`XaE&Y zkP(xNZ={B4x-E!MEJiHHe6Owzz{ewp;1qoeNR}D;brGlt zh6ZpM6FV4(V5$V=$iE;a8Eo7f1|DgFWNd=t%FB0Dq~ew>zP80C3=6*31J$<9+!RHV z3qj3B!S>P}LL--F2iYvN-Rp8I0Q-goZjutVpD%B`H?BMa zo@WdeA|a9mzitn2Hi!4eM(_U_0dA=GDVGkgyBYH0X@rt<8h>P0uiGB4q<;>W>l|G^ zpOCsF+X5ACm!iK2xbI99_xkCAt|0NrOAupNx!G~&?6}b}LYGHLR3w_T$pkGw#EmRQ zfBR*D#-vgbJlGY@sMZGL+k-O-lam#JbGB8U^SS1)cTdk^DxF9hEMz%D}!iehb*zP2 zXpxH%G~L9<`4p7d~)?#C``5EX$jg{bClLhwDszKv70IxWw+V>vW>e{i8r z?Y!z=_EqEg@00$+!$~jUu7d&eXzN9v87gliD>ofQQ@@iz(4nO+1>Pz{aqBzb)EyTZ z;q653H;3sgNCV9-29U{ey$&WL&zbbKeW$%2=2iM#krjAlXwM#rpjlg#<1uTF*gY=l zJ1^5H(hd&~Sx^I%(2oW4{Yh#XMf2{jZT1=t#8`qVD1HtXX@Sa$1kPU8S>JK0mVMrGviP8!MA(I;?rfQmf?cXWu)8zwK9Irv)(Ny-{;=W} zHSj_Z3XGD_VMZykv)dZc6hR-C?b-d+_=}jhiQS_KXZCTb=5&BH3kSO5L_h*nh?RdZ zuz_MA3a`lv%rsRZG^!?_2qk1@dMRFX#!2PQi#GNW^Y_(SDB0xe{wHNDXluVKKM(fA zuaty3z{_N&#G9-V`aC8_vAVU-S)qa5jh%k;*5$t*RdHD3?RH(vU0*Egv>aT)9GLB; z>5QtMG)jI)Tt;AOzh~5{FA3bkXd6&TCIKr%q8f?Iq7QYf*s*|vUe}S8*OM)u#>vK{ z)jVO(^tuU&u63<3FZU7arl3OMH8KEM0dX!6g3#?f- zmt@U~>1ed@$ui2wu#&&zL-R@l+AFHsx4=%}>95n!#S*g$tCwFU1SOLM7+PZh75;;S z2CWn@E%`}vI?f*}EU*T2dw#7?_f`oLZAPo}IARz)G&887V zy1P3hrMtVk8>CB+l5V6+K+?Ck_nxDk^AEf+-g7V*+uED`?e(oS=db31!2I&C^=a;N*mtioJ%j8TvP0CuTWcx7*-%OUsaEY z;&YsARHhB;v&uA^B-aPydNmZm?Ct6>j^Y}p6D&oHb|5Q0WOYn*Du+3^#-@v|st)!k zR;|jnxuPuOp6!Z8sGaT)f@9+WX`oZSOYPp`9m}i_Hz8 zvzEgLlSNHdAt`{#fP@{+S2XT`ut|*#5$I`~7=>(!((&(tU{juIqB90uT~&zQR?5rp z;?k`V6Efbo>0b3((=ScN!L;imFjAG=%a9K5%Z(Gg$q-$+^dwK`Az^XsVT1vR?h1~CQ2R zW6&cZq6dFtkoOOS;K-J52f1K0eDj!e@t;B_lVLT5oj`1sPhO#WKd;*rjTEmX^4;Av zC7ok@-I*Ow#h5~pfi{tB>rEWC;H|B=n!{S3r>s$kea0uh_bH~LWB%45PHpJ4;$XF5 z%GzVJKb(43)opdSc#AXmd?%UL;Sr$-s3a+P91hZfh?it{+JYe&wkC#AT|67;i&H3p zQ=Cys3?KEj&gQC`qmMAQpirx&P-Dw6NbSqlhGqeOLJ!b^ea`?rZ+twfUaVCJF|93R zst{Nx=!mChs|*|())mz!X|}jGzFi!xWGSjLQ0E-#0Ry*w^jQszP3g*jX;^Z}gQPnz zLhh-r_l8l)Zbe^eDV)A*$Cs1On%qT%{iq%1YSRsOa&x_Ff^&v8hZ5cb?aAkKpwu(Q zXNI&trg5L{10Qh)t!#S!XDQX!32Y+Et;WJFhpoKDsrr1B-}1OR>=L%5^@feMXb%== zbp-yb8xdikEfNsc_vaiM%LSaj*Gs8EsDz9JU1ta`@t6kGpr5IWri?CC>JwrMtLIT( zu`%?3>Z*pe4nQ5_VlSGBff>tEYY^`N=9Q6hHU>91ev+D!4YtRzadtT$kdfhTzT7>F z&R`e*FyU>_`jc0)vAPfi%fZ3hX?R4qOh`jksx*iyn$%sN^@o{(-fQVj!5YQHfP1`vsb{GcGwB)_Kbdho>yht&$9;TF(yk?$QlB{iX5Sd4f`Y%d#^%HL*Hi-%8sH4eTG zzTN{Oh6hLGEP`(=8T0;Jl%iDV-#3UbTn_cp@}#Zo&Qb*yc5?UTC!%PCPemdOl$}#f zhiqP@%yn3SdawKPw5{|mwFr}r?+4s?za!-=oehzM0x*LqHO<)dzbhTQ2ooW$teZ!10# zj9LEl6C2_4n4E*}JOPSvCa=p}$SJ=0sH|4H<)ykMhzAP_!g?AhHF&ZMBbN?NI0_Ph z>Jjx?UgqM4cpH@9X9(y30n*`xK;ZzXKqOOl87Z|+)G18BcE zr&coQ)h$%$RaMwx$Ae;7)T)~P;jSsz(Q@c-1B{)a4#59%;X}acx;Y2UG9B(FxIgl} z(3@%(a=>U`^?N|mIQ0Z@rc(B}ST-o}zB%s(qJy5lHYD*p-)stlF84Hov({_L$h3JBV~SS_fg7R z7loVBSYy^ZojMqOXw53#BtxfHX5ese1~!t2$9)f*jfc#B+7CZCZv1ZTsY7yafOS2g zb@WNdF4yLtmf^x`vV;N zPbJ)J3&<1gD@Gl}KYr5xc$z5)09AvrA6C)!`vUy$W9AfaiR0WK&iwgO|M_78#XSdX zMVs{Yn18#p|D|%;rD%louX2`P?EmeWCda8X!uLF(BW2uubP`RoP8v$Qa%{E;qY(5v;1@N^g6% zh-=~fc9lFmkVH@4JX@)XiDzBH0S08H^^p`QFieNAFbX7MKL)BCY*+*otR)W*3Rn9Z z>a)V%J=RiaAPfHiwc%Jpng1z5)VyC({wx`ErLyDuWEE6g0uxZ$230LNNB ziN~#-Lxuj}l7_XR~$ z`5MH=GR30LN|j3z>CAvv4cdBxybYy_nhC&s>1!8nX4=fx>LwzB4zBnjo z`~e*2!N5Y|yl3u@Lizz*KPQIAvln53#12AN{%#fk<#L@(BlD^5Q_m8M|RQ zC<$|{m}|`!o`7@=hFB&|s`T|5;CW;Kk|{p7s|ni2_zs|fVb-ZZ3=j*B{Q)mLn%W}P zkJ;V5N-cFePC|(60s{a8xkLC7cJ~{&jdKeJ<@(6!0tI#FXY4q#?UX)0uALqQJ20`$ zBWh9pT0p+IFwl9@dRlq_KF3-DXO0?{5h_5lJmMsUY*;<+h;oE#jlQS>`r?Jb%K^qs z_KWQ=qh-^rMql20wM7RClc`o_XS!Zm7gkM|EujPdp5@#z_}qfojmGnD?rEW9!JkDE zF?2a|Cte>{4!5`&78`_?hfIFmm0p1_$d;ugHwNH`hf~<}8|SVqLF^1D7x`n-1U7s9!!7Fk5OBxwj63NY17KrAhTqy6op8yoZ4Y3vD>}^$&2{*E-((r~tWpHZW$4 znvFx&tyHffC33P`BzG5E@))k;s7VotdELn)sa`QK)~jn2*LQH^&c1-Eu-zW-b7#N> zHf-$SBra^#s`ql!0hhp3>?|BY!r#Lg)WGcKH`xZNQwoC-qE!$xhY6S%6O$t>1IUqmNQ9 zshH>*sOBSIEA=0Pu%H)C+~A%U;zzr^a-#(9m<@T&XI_)I5oLgSq~0}98p|T(ccvOi zy%iHrKV&+(X^a9c`9Pp-2?WpyGHvU9(;U+7E+L~Gzg`+N{T)mNFKlXk$J1NPJvK&J%nT3D%y4mRw#waC7I7NSq}PvHg03-tk4|P*rLFf zC>qN^T4Zo|u+T&YFtPqxGfIq6TaZp9mMe`vDF9h$a?q~iAp`d8gOlb)vu|xEZin;y zqY&q^w_X0LRil!FTbIOVNhe1uBG|;Fqg5o$U$%}nQs`yL+MIiU;k(fL@;--k;33rE zF$jpXQ;03u1f7WMF_}~zi_YswJFz}oOamQZp{KPY&9RJjF~(Jo3zbmf`BNFR5~*+Q%c z0u~Z6UjYzjFa^cg=np+!XmtygPfR|wY&MxT9|L_RCH-HHVJqMmhR5PE5O$FrSczN& zZeU8@`mGn;945+i-S|@OZxbl0-_n3vUg1F$_o%uyXh`4IcPH&`-vUZjNU{BUJzghlP1P)6O1qO?8ClJnjq;5{o^@nGrFIa~|@rf%MkH*e3 zp071lyo;$y7I0Ie8wyR1nshZsQLIlI4c*GK-L7P^>9k8@(RQ6G=8m;LYpHTc&5$Tl zYk2YL4f>M&OHu%xh(eBCWjdrz6MK5!>V;fR-l?nGFtYq zabeYmM7S~Ca&NvtN)`Prc_U~Z3t{y$fl1qZ7mc<*V)!Eh1z_Q>mQT;iCT z$F3bt9o`MbI-jOFTVi&X+REs>$xOgf!UT~_m;kcHE+=`>dQ+Lz-q^MX=Yu6XfYJWQ zZ2C%MF%~0` zq>r_~J{e>hf6*l4Bta#U44FKe^4N0aGm*_F|HifXT2FwzxV_x-aHit5ScgODR+%1( z^>XS^jNBP$mKg!=W(Ws&vl?u_lLDilVNj%O_Rh1pyNAEa)AxPmwa-PlW)r7mgm_av zqRCH?*XirNRdH~0v{IUi+8DZQno-WI@q9@(`z>luI|?5>OMXD>T0+Ni1nbw#BVZurz!Y&2<^lDN|1P$1{iw-d{+DeOkxwiPn;aqEc% zQ6e>{`wd6JOVvg;S!lfWd(lh{^!JRyBqBe(cJ!Xsh>B6zkrWs{q?W6YrMo^C8K(>tV24VnL5&dHaGx=b7V? zZ1`4pq-uN+6t}}cv+Lb1fr;B%BTIa_Za15qg?Kp0Q$!x3HYpOqGxDleQR0PI^JAdN zHl6S+asyt|bvsuk%H}uh?v8i5rbx8dDD^SklB$>Q5 z%4uCMl_(z%x=ghzeRm*!*5veej<3e3&gq&3Jib>W~4G2r{eiK(;1|mC2kf>QSuD<)l2Q)hEAKj+-+Vq?|?U%7GZH%D3;i* zO>aMp@!>45_sQcD)_|ISe1p*L(#aj$1!%pTtxl*(l+qPI&N!Kj=eeQ@T3COyjQ#Qb z3tz4Ih5*ONZ&ez)Cmf7S!c^(d?1Sy?RPv*EoF*MPo;Ma}y)Gn1c0?@cZ4wD5uqhiC zW`-m1`m-sf^Yvf*yw4_*$Y$bVBPfYa+s-N$UqE-?+}~r{RN-^cPl-u?&ed{96Et-K zC5uY`5xb`nXLaU++1MkPJ`lz+jz@)LHkY3U3jG38*G%t%jp5z$MoMC@**&;{vXpj; zhhCBIV}bOwNTl4}VB4R|zsv`^(X1q!@}pLXnw=(0TE~M2Vh_AsGO%C9Ea!QU@Lzq& zp{D)laknioZ9Y8W_Z`beMeD<6@`iPYFutP;S)gjBFLH>hgB5%TjFvq*-fQ}GJSZe;=2D37IHFVNpVCe!jD~3IN>_R zYAa_BZGeC_XLq`)*1PhFK<@+_tZUP>Wp9tro1?TYBl zT;=sZ%O9n@{Tb%{-({GWC#4RD2Yma+TO$gjO)2Nc4B^@Eq#=uN=Hu}OA%UZ_Y6x6X zaK2z4F zK!n zsSrnWDC9~TMYAJfh$yaq4y?1eoaNMLs`;^6>T}&q!a?=@T^ArL= zBB}-2L9W^(VxD3LRB9#qS#fa|hjjMks|qH^{>U_Z8&yz@Z`9<-=SbpBoq_YQ4gEre zKTfTw&t*Rw4PIt|cAckB3Bm&mQFQN|uS1z04vVf|+*?~4 ziu2s0pLzxhHglspk$4+);D+P8*=Gy?Rqw#~3AI1{Vi`V#!!>kjd$Gc0I)g4kPQX3R zb)i+YSMP?O0tn=5Y<~o0(+`jmVU0T9PuCu0My{}mCU}z`w*0<$ z>s?aiqV!SGgYjiV z)f(d(%?Qhw+_(a7T)0%5s^y05@$QPfXm0-OYp|bNlnd0IU$N97s#r>se;d!KN)F%~ zPu6b&@iOKykEAR^2_{Vkm)*Wxd~#Cra*ut8Wl2otRWp}TB#>KFtX88TQzWrxEhCyX zy~4bKIEOB<*<6~LE7F$_5{QUOpi^Y) zx2|ER@KB%of+VT;~Wvpxfn`Nri z6pDYTM}zn@_IvEY5#kH%k*RUPnSo3R>5}l0qK>N%E*V*^YhM@Zt)fOA)v51Odz1NY zdZ9tiDmZJJgkd$%=g-NJ+Bb@H77Y2W9qlo=gaw=O(?Ej?SeE_dM%B!^mE|L+=`r=6$-)2KPvQ<|<4B5%FPf;cx(uj%lTJz-$yPp`v7e_gf(fe7T7X2g?eF86@z`Rl|5IMf3x2f(hLT2igZ&_d^VZ!^YvSTi3+2Mgn zs>k)d@#I#kn@bTiS#vLvWxfxLR;#Nji1(b$;f9+PX=iCZ9E@X@liL}~B@u0J;b^Nf zgfdalOvz$k;LeXak~!gw%4am;r4Ll(l&Jq6<9% zIm(qsZ&9ZuibHR{7j$864BmpM;}%GZbLfga(Mvk_t1KePU62E<8JoB>mQ|<0GssFq z7=)uin;{{>4J6XxYRZEL^W@xd=7{DKCR46OSV**@KLR=F$C3Kh8F+ZO$B$R`833yI z)c$ZWYgSaf9^qCvEc`&g?W&Qsb#~z5roqGoB2#R$RW^ahTQ5ZZ=_T(0#sL~bIxNBr zA}{c#h7$A%k&YokhBB&!zP79N^9)p65~MRk)2qlmeOWJ{A=2rKj4DPj)eN+5*VW%A zfOPlW#|oLAD)^pnf~Byq@LgH&FI{Y@Ek>4(rGw(YzcRjqZ7cVI5f z6CW{6;$VsEE;2vCAt`uFvnywklEsV-)5TTHJ74c?^AXg^!T z(EijwG^GQ?^NGhqdgiQHMtmxuGsFO}klh@Eca%|`8A~9}K6X}dtAqXMDd0wP=r0_* z(MO?23b2jJRa;&CoLfVB*2D4~9!Ozy~e zoV<)@lZK%&1;DW%BS~7!5MI zhw4@sI!s_B9~T$kv$SQp6SS?!lEcLs$y)wR}FyX@d1py||fSu265ge3?M zEcY`ozN4KsXME~tu>(E&K;>4+>-qsb)D<9PpxYn}DrWXJpa7ZqY@Igd1Io#={yOl_Fl? zq~gjhvXHt0aVHGUO(1Q&vYf25-@TSytrBH4CgsG)!K^=$8v9k$6Ko_F1gY9G7=J%~ z248j~`9V6lXk4U`;Eu45L2GrC0R;73HnneL#_C@k>O4sjNhATz9Zo4aRS z)UBc)R*l<{E3lU_GV}23c}c(gFU0FUowO1I$x)w&Z9&!meAnbHi}_W0{P#1_qVn>z zJ9UphcVha5rv8r^@ahu-q~LnvQnT8>{tj9YJ)LbIG0@cepU)^kW{rW=ms3G8&4|)C zn;{YqJW(JRZ8GYP8{NnM@8@ZItOLc?Myc=f>%0aG!hgQJWIW9mz1#zw-2VM{?tqh= zJ(T!L_KQdPUm-4(es4Sb<`6&qyAJxl&$;YyZ#xT%M*TOb3deKrar^%-_Sg0NfB7;7 z0=Ia+?sVz?&V3~RvEHE9B-V4VZ7PF&ZT@nm5S2gW9aQK>NTLt3>4D=Qq%U$qFO7K{ zj2^;)e}Hw~(98_4_bD(Vh*|=DDQ*>7>3yef1qEQ7b`odn4#~oZxQ!cHnOhBMf^eCi ztoDS`XNrEvJOQo@G9YIm=ds(NxVzjUmAYIe3LzC#cD<=sn+J*(bRe%kh42Gux-Jl~ z!35m`K_&)p!yG|U6aIOt=sZD}9k2z$tM2=x?N)*C>yD?p6&X+iL!c)J1YysRHg}Qo zQsB72p#Z0YcnTMYtexo{Nf(9{ji4xn#RWA37O>Ji2IddU$)agYyk(@)XeLd`^&x!Q zgP96F7c<=-N96wcu00_MT7y;p)eQXhJsw1lv$dGb*+&48xU+ol8DCCBuCRPj+vg*dKgnFK@)b76m7n_oR%RD4g1a0AtRo!X12Wnq zV1}tV92D#h!sgiQOXIc?*D0;$0(oNb8J45*tWrQ`b+}l_@~3PSyi*M%5d1#7vEwRz zpHL4(4vopDNt*4}P5`I20;UL%r|fEVwxcwsY&=y=4!{*A&?r*_#F7G_BD7Z*KYMD6 zWqj#vfx1cTWW7JeF$mh2wW8f3`R8O zY;B1_5go;QpnldkXU%En)2)Ye(SafGj`&utb;g(v_C9@Bn`lkl0z-UYwuAV^(&TH$ z)FwGFrFrkk*${LghBv7ik*|;yO3}B!p0kjTfTx$PQ!@ zpb9kVgy2~O0-`p{cjXLvAEDs=#qhD! zh~m6is)aIy4JGwm$jE)oAP-82@!%eW$UdwlixKm!DAvo!N75fyg~RNH_EIb_u3#V^ zxx7o=`d-`Owl9)r4@3qUz`O^xY_+6{D#af%s5;9t* z3r%s1E^n~0v116Bn@)j+vjF(1=FFQleyj%bopnu;PN?*TJ*6Ul#>~D)#rdlW{BNI^ z4g&+dzX_(hluUpuE>Q_Wt_=`qT?kCMNKAnPLqliIN_Ae{0as(<5_*&Dyw{W<3gw!@ zJan)s<;G_115K);*_`v^EMX$OZo=aQ$gsb#BNJp zJJ$!LTkPozI7%jN>J815@4NME2h$Z5U1A!5p;y`1O8bbW?D+P1&-b^%+-N2s)NK02 zMIupH$ee`bCOZ|2%iX2GxZf`2B|gf*Qjhskt65@YcU4C)l>Oa})38lewVRt0Uz8H? z1?*$p)c$DkwneUTztvJE}exM@o=eUsU{)T+@i=oto)P(_|(M|E=L9db?U9>^Rq z9*FJ-cRaUp{Cj<~*f>P=O!P6B+ccT+*M>>AhUU0&wat~b&8p105Xnl_SHrDqlUa0YWpANx)_PS~whjjo#%!v@U z`|%_C*)W%ugIS<@GHI>Dp}<$&NgVITQ`JTaO@$_y_WeTe+JSG*9dp&lLr>R;;Ni8n zjEVdm4|mG5v&DA4jg_s-JDNA=L#hokf`SQ+G;1n!1w7m%TZpL;P8}C9a;M9*>u&q| zU)2hHZ?xOq0n69EI1@q3f4Pyx5w`hMzAx!6 z(JN^nS!~yz5iaqJ(X3{lt#mQJ*A%5hhAyuH#*FT)S$_~r@YLg94jm9rDb)8-7)Zd-_7 zK4mC2FYF_B(F#PBGmBqhr9K_`rDW9YqH^}I8cRRepE$uD`lg|2Zv@NmEdx26v zABeKd(7cMyfvV|U&Rp^mimnn)MVEx{R7sKwAB7B_Ki*1Vv?l?(VR|B;Q>L3ZQXioA z2Dk~>X+J>6g&ALUO0uAg1}BaOs`OfTntlU3dck+5o-nt;5#$FCBIDp{MYw<*nzHGL zr2Qe>3@9s0h){c}^v@r96?9BYI0E`ny7-2| z!lBtll01N|pFz2#ZY-ZQ^8FrjSoZDt3Pe-9Fpr56J0r?xzk17-v$B%2p9zknQZ;E^ z6(654`HRLJ#E$C)K_>CR*2)UMEz0>|qesv#>*UuDE-zH+PuXD2m%)Zo_8o`BZGvzq z?1Q6txt06SC5!o$+~)28wuQKDz4O|0F0e@^fBmwsS@9b_{NF88h)hQ%_xLD(V?KBI z5NqC(JeTYwL|BKi2haYhx~}=LG6%@JF8f$aN4JCelv#gM&xFmM_xdyE*S0Yd0^+Pv zP5KYek5J7%99Y-tOVuW_teAZ`6T`8Ea+%?mqFEagPTkZ(iu{gtkN z=5=?6R_a%xcQu2-3e$tY87r|PofN-4G{i7GqfRrs4@HIFb%-QG>apkc3XCb^_KPEX za==ecuGQ4aOmF9Cx`#wx837gcSN7uH3n&Hy$&fXaN*DbLq+-5&1s0o-Fi+MWnZkL0 zwoP5Q5y)@`G0svVuuxv09rpo6JWfZzq``!R{j)rY!@0V$tD`PPweS47nn?x)VZ<%{ z!viFUb*7*6Meqo(_o_6$d~X1AYD$R@=Nqad-;92At9Pe)5>}?zE?BMZXhN zFWHZ33vM;+@4EH?;x3CBz5@9q^Cq69|BO9wpXAvlV}ei2?%rOB{OUvfR=P^yChv3W z(s$|bj(qn%%{?*eS(Qr~FOPG6Gm0tzqQX+{UZIi-NPFn+Jr%snUazZ7G5wnr`l~C& zhk<6yea1F5SZF(%>hYPwU%e0(@jQryGXe0L92qt?XsV{i$7|L3b96aHWVWX#)?!CW z>jEKQiq~C$^Vy_eKeueD@y!oW3|pDHcP`K`DH+X_p0b9{PAFA8rn%EkgRMys{5ijy zYChnpWc%pdm49H5o7vJ`gr=Xzil5+}@kXY+LX;Ud8`2sG%x+wi5%uLN;yloA_msf+ zQu?^)GcwmLaE%)|Tx_41`XIIy96dGZI|_}rCHPkbBi_#MSzVdsF0vN�hIanVcJY z2+64UN!O%ve`H&XvliprD{^QeCtc1VzR17Lf@Fr*FrP*DfgrUgMb5Q>;Bl z>YtdP&KWtJ_j2J@R()&9+Lj&_snl(`4@?%pcs;IxhOV+ZgA>vAyrZ3d!QX8(t3)gf zJirhfeLX}vLq12@Nb0!;9`oF_4G~EE^lQQkUChV?t1V6%gR+H^;@sMVZ|}^nhHyIF zz;GB&C3&6pL@Hj`LLl3bAj7p8l}vv0jKS5jew(2E1#AJkkZFB*OWHR zrrK5WFz%;M3kImDPwhf%P%?>&4mvtIoZ`7XRhK(JCBA%gUn6zx3fANV3w`B1MT?paXl|Yrz$Py$Me4 zn4)lv;kvHDe<%3dfDl=|T$9x{GX7gP;yM@M$~RfDYz&|jda#}NL+?2C?_xv+Q4LQsEi2!B7g!(1MyZu|87UtNp@*)p=MG~$i6C)7V* z-V^SIT8MR@nRVo^F+sm}dn88~tS`_%amk>5{n=k7h6nNu8YBMH;ZmB#2o(7BN=!zy J^o5T1{{cXB@~Qv; literal 0 HcmV?d00001 diff --git a/malware-scan/plugin.json b/malware-scan/plugin.json new file mode 100644 index 0000000..ce5ad7b --- /dev/null +++ b/malware-scan/plugin.json @@ -0,0 +1,556 @@ +{ + "id": "malware-scan", + "name": "Malware Scanner", + "description": "Scan uploaded files for malware", + "version": "0.8.0", + "stream": "no", + "ui": true, + "jobs": [ + { + "name": "malware-scan-cache-cleanup", + "file": "malware-scan-cache-cleanup.py", + "every": "once", + "reload": false, + "async": true + }, + { + "name": "malware-scan-update", + "file": "malware-scan-update.py", + "every": "hour", + "reload": false + } + ], + "settings": { + "USE_MALWARE_SCANNER": { + "context": "global", + "default": "no", + "help": "Enable malware scanning for file uploads", + "id": "use-malware-scanner", + "label": "Enable malware scanning", + "regex": "^(yes|no)$", + "type": "check" + }, + "MALWARE_SCAN_DEBUG": { + "context": "global", + "default": "no", + "help": "Enable detailed debug logging (shows every operation step, byte analysis, pattern matching). Only enable for troubleshooting as it generates verbose logs.", + "id": "malware-scan-debug", + "label": "Debug mode", + "regex": "^(yes|no)$", + "type": "check" + }, + "MALWARE_SCAN_CLAMAV_HOST": { + "context": "global", + "default": "127.0.0.1", + "help": "ClamAV server hostname or IP address", + "id": "malware-scan-clamav-host", + "label": "ClamAV host", + "regex": "^.*$", + "type": "text" + }, + "MALWARE_SCAN_CLAMAV_PORT": { + "context": "global", + "default": "3310", + "help": "ClamAV server TCP port", + "id": "malware-scan-clamav-port", + "label": "ClamAV port", + "regex": "^\\d+$", + "type": "text" + }, + "MALWARE_SCAN_CLAMAV_TIMEOUT": { + "context": "global", + "default": "10000", + "help": "ClamAV connection timeout in milliseconds", + "id": "malware-scan-clamav-timeout", + "label": "ClamAV timeout (ms)", + "regex": "^\\d+$", + "type": "text" + }, + "MALWARE_SCAN_CLAMAV_MAX_SIZE": { + "context": "global", + "default": "26214400", + "help": "Maximum file size (in bytes) for ClamAV scanning via INSTREAM. Files larger than this will skip ClamAV scan but continue with hash-based scanners. Default: 25MB (26214400 bytes). Should match or be less than ClamAV's StreamMaxLength setting (default 25M on Debian).", + "id": "malware-scan-clamav-max-size", + "label": "ClamAV max file size (bytes)", + "regex": "^\\d+$", + "type": "text" + }, + "MALWARE_SCAN_HASH_MAX_SIZE": { + "context": "global", + "default": "67108864", + "help": "Maximum file size (in bytes) for hash-based scanning (MalwareBazaar, VirusTotal, SentinelOne). Files larger than this will skip SHA256 calculation and hash-based scanners to avoid performance issues with large files like ISOs. Default: 64MB (67108864 bytes). ClamAV scan is unaffected by this limit.", + "id": "malware-scan-hash-max-size", + "label": "Hash scan max file size (bytes)", + "regex": "^\\d+$", + "type": "text" + }, + "MALWARE_SCAN_MAX_FILES_PER_REQUEST": { + "context": "global", + "default": "100", + "help": "Maximum number of files allowed in a single multipart upload request. Prevents resource exhaustion (file descriptor depletion) via DoS attacks with many small files. Default: 100 files. Requests exceeding this limit are blocked immediately.", + "id": "malware-scan-max-files-per-request", + "label": "Max files per request", + "regex": "^\\d+$", + "type": "text" + }, + "MALWARE_SCAN_ALLOWLIST_HASHES": { + "context": "global", + "default": "", + "help": "SHA256 hashes of known-good files to skip scanning (for false positive handling). Comma-separated list of full SHA256 hashes (64 hex characters each). Files matching these hashes will skip all scanning (ClamAV, MalwareBazaar, VirusTotal, SentinelOne) and be cached as clean. Example: abc123...,def456... Use this for legitimate files incorrectly flagged as malware. For large lists, use MALWARE_SCAN_ALLOWLIST_FILE instead. Leave empty to scan all files.", + "id": "malware-scan-allowlist-hashes", + "label": "Allow list SHA256 hashes (comma-separated)", + "regex": "^([a-fA-F0-9]{64}(,[a-fA-F0-9]{64})*)?$", + "type": "text" + }, + "MALWARE_SCAN_ALLOWLIST_FILE": { + "context": "global", + "default": "", + "help": "Path to file containing SHA256 hashes of known-good files to skip scanning (for false positive handling). File format: one SHA256 hash per line (64 hex characters), blank lines and lines starting with # are ignored. Files matching these hashes will skip all scanning (ClamAV, MalwareBazaar, VirusTotal, SentinelOne) and be cached as clean. Use this for large allow lists. Leave empty to disable file-based allow list.", + "id": "malware-scan-allowlist-file", + "label": "Allow list file path", + "regex": "^.*$", + "type": "text" + }, + "MALWARE_SCAN_BLOCK_MEMORY_UPLOADS": { + "context": "global", + "default": "no", + "help": "Block uploads that are buffered in memory instead of files (cannot scan memory uploads)", + "id": "malware-scan-block-memory", + "label": "Block memory uploads", + "regex": "^(yes|no)$", + "type": "check" + }, + "MALWARE_SCAN_CLEANUP_FILES": { + "context": "global", + "default": "yes", + "help": "Automatically delete temporary files after scanning", + "id": "malware-scan-cleanup", + "label": "Cleanup temp files", + "regex": "^(yes|no)$", + "type": "check" + }, + "MALWARE_SCAN_CLEANUP_ORPHANED": { + "context": "global", + "default": "no", + "help": "Clean up orphaned temporary files on worker start", + "id": "malware-scan-cleanup-orphaned", + "label": "Cleanup orphaned files", + "regex": "^(yes|no)$", + "type": "check" + }, + "MALWARE_SCAN_CLEANUP_AGE": { + "context": "global", + "default": "3600", + "help": "Maximum age (in seconds) for temporary files before cleanup", + "id": "malware-scan-cleanup-age", + "label": "Cleanup age (seconds)", + "regex": "^\\d+$", + "type": "text" + }, + "MALWARE_SCAN_USE_SHARED_DATABASE": { + "context": "global", + "default": "yes", + "help": "Enable Redis shared cache for scan results across cluster nodes. Automatically disabled if BunkerWeb's global USE_REDIS is disabled. Improves performance in multi-node deployments by sharing scan results between workers.", + "id": "malware-scan-use-shared-database", + "label": "Use Redis shared cache", + "regex": "^(yes|no)$", + "type": "check" + }, + "MALWARE_SCAN_CACHE_VIRUS_TTL": { + "context": "global", + "default": "7776000", + "help": "Cache duration (in seconds) for malicious scan results (all scanners). Default: 90 days (7776000 seconds). Extended retention improves performance by avoiding re-scanning of known malware, reduces API calls to external scanners (VirusTotal, SentinelOne), and maintains detection history. Malware file hashes (SHA256) rarely change, making long-term caching safe and beneficial.", + "id": "malware-scan-cache-virus-ttl", + "label": "Malicious scan cache TTL (seconds)", + "regex": "^\\d+$", + "type": "text" + }, + "MALWARE_SCAN_CACHE_CLEAN_TTL": { + "context": "global", + "default": "300", + "help": "Cache duration (in seconds) for clean scan results. Shorter than malicious TTL to quickly detect newly-identified threats. Default: 5 minutes (300 seconds). This implements differential caching for security.", + "id": "malware-scan-cache-clean-ttl", + "label": "Clean scan cache TTL (seconds)", + "regex": "^\\d+$", + "type": "text" + }, + "MALWARE_SCAN_CACHE_ERROR_TTL": { + "context": "global", + "default": "60", + "help": "Cache duration (in seconds) for scanner errors and failures. Prevents repeated failed scan attempts while allowing quick retry. Errors cached include scanner timeouts, connection failures, and service unavailability. Default: 60 seconds.", + "id": "malware-scan-cache-error-ttl", + "label": "Error cache TTL (seconds)", + "regex": "^\\d+$", + "type": "text" + }, + "MALWARE_SCAN_CACHE_LOCK_TTL": { + "context": "global", + "default": "60", + "help": "Lock duration (in seconds) for atomic cache operations across all scanners (ClamAV, MalwareBazaar, VirusTotal, SentinelOne). Prevents race conditions when multiple requests scan/query the same file concurrently. Lock automatically expires to prevent deadlocks if scanning process crashes. Default: 60 seconds.", + "id": "malware-scan-cache-lock-ttl", + "label": "Cache lock TTL (seconds)", + "regex": "^\\d+$", + "type": "text" + }, + "MALWARE_SCAN_CACHE_WAIT_TIMEOUT": { + "context": "global", + "default": "30000", + "help": "Maximum wait time (in milliseconds) for concurrent requests waiting for scan/query results across all scanners. If another request is scanning/querying the same file, wait up to this duration for the result before falling back to scanning/querying independently. Applies to ClamAV, MalwareBazaar, VirusTotal, and SentinelOne. Default: 30 seconds (30000 ms).", + "id": "malware-scan-cache-wait-timeout", + "label": "Cache wait timeout (ms)", + "regex": "^\\d+$", + "type": "text" + }, + "MALWARE_SCAN_CACHE_SHARED_TTL": { + "context": "global", + "default": "21600", + "help": "Cache duration (in seconds) for scan results stored in Redis (shared across cluster). Longer than local cache to benefit from cluster-wide knowledge. Default: 6 hours (21600 seconds).", + "id": "malware-scan-cache-shared-ttl", + "label": "Redis shared cache TTL (seconds)", + "regex": "^\\d+$", + "type": "text" + }, + "MALWARE_SCAN_REDIS_CIRCUIT_BREAKER_MAX_BACKOFF": { + "context": "global", + "default": "8", + "help": "Maximum backoff time (in seconds) for Redis circuit breaker. Circuit breaker uses exponential backoff starting at 1ms (2ms, 4ms, 8ms, 16ms, 32ms, 64ms, 128ms, 256ms, 512ms, 1s, 2s, 4s, up to this max). Protects against Redis outages by temporarily skipping Redis operations. Default: 8 seconds.", + "id": "malware-scan-redis-circuit-breaker-max-backoff", + "label": "Redis circuit breaker max backoff (seconds)", + "regex": "^\\d+$", + "type": "text" + }, + "MALWARE_SCAN_API_USE_VIRUSTOTAL": { + "context": "global", + "default": "no", + "help": "Enable VirusTotal API integration for cloud-based scanning. Can be used together with ClamAV (dual-layer) or independently (VirusTotal-only mode by disabling ClamAV)", + "id": "malware-scan-api-use-virustotal", + "label": "Enable VirusTotal", + "regex": "^(yes|no)$", + "type": "check" + }, + "MALWARE_SCAN_SKIP_OTHERS_ON_CLAMAV_DETECT": { + "context": "global", + "default": "yes", + "help": "Skip cloud scanner checks (MalwareBazaar, VirusTotal, SentinelOne) if ClamAV already detected malware (faster blocking). Disable to get additional cloud analysis even after ClamAV detection.", + "id": "malware-scan-skip-others-clamav", + "label": "Skip cloud scanners if ClamAV detects", + "regex": "^(yes|no)$", + "type": "check" + }, + "MALWARE_SCAN_RESUME_SCAN_ON_FIRST_HIT": { + "context": "global", + "default": "no", + "help": "WARNING: Possible denial of service risk. When malware is detected, continue scanning remaining files/scanners to collect comprehensive threat intelligence. Plugin performs dual scanning: (1) entire multipart body to catch malware split across boundaries, (2) individual files. Resume mode collects all detections before blocking. Default: no (block immediately on first detection).", + "id": "malware-scan-resume-on-first-hit", + "label": "Resume scan on first hit (DOS risk)", + "regex": "^(yes|no)$", + "type": "check" + }, + "MALWARE_SCAN_BLOCK_ON_FILENAME_INJECTION_ATTACK": { + "context": "global", + "default": "yes", + "help": "Block filename injection attacks (path traversal, null bytes). Behavior depends on RESUME_SCAN_ON_FIRST_HIT: (1) BLOCK=yes, RESUME=yes: Skip file, scan others, block at end. (2) BLOCK=yes, RESUME=no: Block immediately (default). (3) BLOCK=no, RESUME=yes: Log, scan content, continue. (4) BLOCK=no, RESUME=no: Log, scan until detection. When BLOCK=no, file content is still scanned for malware. Default: yes (immediate blocking).", + "id": "malware-scan-block-on-filename-injection", + "label": "Block on filename injection attack", + "regex": "^(yes|no)$", + "type": "check" + }, + "MALWARE_SCAN_SCAN_FULL_MULTIPART_BODY": { + "context": "global", + "default": "no", + "help": "Enable full multipart body scan (in addition to individual file extraction). WARNING: Full body SHA256 changes on every upload due to random HTTP boundary strings, making caching ineffective and wasting ClamAV resources. Individual file extraction always occurs regardless of this setting. Keep disabled (no) unless you suspect malware split across multipart boundaries. Default: no (skip full body scan, extract and scan individual files only).", + "id": "malware-scan-scan-full-multipart-body", + "label": "Scan full multipart body (not recommended)", + "regex": "^(yes|no)$", + "type": "check" + }, + "MALWARE_SCAN_API_VIRUSTOTAL_KEY": { + "context": "global", + "default": "", + "help": "VirusTotal API key for authentication", + "id": "malware-scan-api-virustotal-key", + "label": "VirusTotal API key", + "regex": "^.*$", + "type": "password" + }, + "MALWARE_SCAN_API_VIRUSTOTAL_SUSPICIOUS": { + "context": "global", + "default": "5", + "help": "Minimum number of suspicious reports before considering file as malicious", + "id": "malware-scan-api-virustotal-suspicious", + "label": "Suspicious threshold", + "regex": "^\\d+$", + "type": "text" + }, + "MALWARE_SCAN_API_VIRUSTOTAL_MALICIOUS": { + "context": "global", + "default": "3", + "help": "Minimum number of malicious reports before considering file as malicious", + "id": "malware-scan-api-virustotal-malicious", + "label": "Malicious threshold", + "regex": "^\\d+$", + "type": "text" + }, + "MALWARE_SCAN_API_VIRUSTOTAL_UPLOAD_UNKNOWN": { + "context": "global", + "default": "no", + "help": "\u26a0\ufe0f GDPR WARNING: Upload complete malware files to VirusTotal if ClamAV detects malware but file is unknown to VT. FILES BECOME PUBLICLY ACCESSIBLE in VirusTotal's database and may contain personal data (documents, emails, images). Default: DISABLED for GDPR compliance. Only enable if you have proper legal basis, user consent, and documented justification. See GDPR.md for full implications.", + "id": "malware-scan-api-virustotal-upload-unknown", + "label": "Upload unknown malware to VT", + "regex": "^(yes|no)$", + "type": "check" + }, + "MALWARE_SCAN_API_VIRUSTOTAL_AUTO_VOTE": { + "context": "global", + "default": "no", + "help": "Automatically vote on files uploaded to VirusTotal. If a file was uploaded (unknown to VT) and ClamAV detected it as malware, automatically add a 'malicious' vote.", + "id": "malware-scan-api-virustotal-auto-vote", + "label": "Auto-vote on uploaded files", + "regex": "^(yes|no)$", + "type": "check" + }, + "MALWARE_SCAN_API_VIRUSTOTAL_MAX_UPLOAD_SIZE": { + "context": "global", + "default": "33554432", + "help": "Maximum file size (in bytes) for uploading to VirusTotal. Default is 32MB (33554432 bytes), which is VirusTotal's limit for standard API.", + "id": "malware-scan-api-virustotal-max-upload-size", + "label": "Max VT upload size (bytes)", + "regex": "^\\d+$", + "type": "text" + }, + "MALWARE_SCAN_SHARE_FILES": { + "context": "global", + "default": "no", + "help": "Share detected malware files with threat intelligence platforms (currently VirusTotal) when any scanner (ClamAV, MalwareBazaar, SentinelOne) detects malware but the file is unknown to the platform. Helps improve global threat intelligence. Requires VirusTotal integration to be enabled. Only real malware is shared, not PUA.", + "id": "malware-scan-share-files", + "label": "Share detected malware files", + "regex": "^(yes|no)$", + "type": "check" + }, + "MALWARE_SCAN_VIRUSFILE_LOCAL_EXPORT_PATH": { + "context": "global", + "default": "", + "help": "Local directory path to export detected malware samples for forensic analysis. Files are saved with SHA256 hash as filename. Example: /tmp/malware-scan/samples. Leave empty to disable local export. Directory will be created if it doesn't exist. Disk space is checked before saving to prevent filling the filesystem.", + "id": "malware-scan-virusfile-local-export-path", + "label": "Local malware export path", + "regex": "^.*$", + "type": "text" + }, + "MALWARE_SCAN_API_USE_SENTINELONE": { + "context": "global", + "default": "no", + "help": "Enable SentinelOne API integration for threat intelligence lookup. Checks file hashes against SentinelOne's threat database. Can be used together with ClamAV and VirusTotal for multi-layer protection.", + "id": "malware-scan-api-use-sentinelone", + "label": "Enable SentinelOne", + "regex": "^(yes|no)$", + "type": "check" + }, + "MALWARE_SCAN_API_SENTINELONE_URL": { + "context": "global", + "default": "", + "help": "SentinelOne Management Console URL (e.g., https://your-console.sentinelone.net). Required for SentinelOne integration.", + "id": "malware-scan-api-sentinelone-url", + "label": "SentinelOne Console URL", + "regex": "^.*$", + "type": "text" + }, + "MALWARE_SCAN_API_SENTINELONE_TOKEN": { + "context": "global", + "default": "", + "help": "SentinelOne API token for authentication (REQUIRED). Generate from Console: POST /web/api/v2.0/users/generate-api-token", + "id": "malware-scan-api-sentinelone-token", + "label": "SentinelOne API token", + "regex": "^.*$", + "type": "password" + }, + "MALWARE_SCAN_TEAMS_WEBHOOK_URL": { + "context": "global", + "default": "", + "help": "Microsoft Teams webhook URL for malware detection notifications. Create an Incoming Webhook connector in Teams. Leave empty to disable Teams notifications.", + "id": "malware-scan-teams-webhook-url", + "label": "Teams webhook URL", + "regex": "^.*$", + "type": "password" + }, + "MALWARE_SCAN_TEAMS_WEBHOOK_RATE_LIMIT": { + "context": "global", + "default": "3", + "help": "Maximum Teams webhook notifications per minute. Microsoft Teams limit: 200/min. Set to 0 to use maximum (200). Default: 3/min.", + "id": "malware-scan-teams-webhook-rate-limit", + "label": "Teams rate limit (per min)", + "regex": "^\\d+$", + "type": "text" + }, + "MALWARE_SCAN_DISCORD_WEBHOOK_URL": { + "context": "global", + "default": "", + "help": "Discord webhook URL for malware detection notifications. Create a webhook in Discord channel settings. Leave empty to disable Discord notifications.", + "id": "malware-scan-discord-webhook-url", + "label": "Discord webhook URL", + "regex": "^.*$", + "type": "password" + }, + "MALWARE_SCAN_DISCORD_WEBHOOK_RATE_LIMIT": { + "context": "global", + "default": "3", + "help": "Maximum Discord webhook notifications per minute. Discord limit: 30/min. Set to 0 to use maximum (30). Default: 3/min.", + "id": "malware-scan-discord-webhook-rate-limit", + "label": "Discord rate limit (per min)", + "regex": "^\\d+$", + "type": "text" + }, + "MALWARE_SCAN_ANONYMIZE_WEBHOOK_IPS": { + "context": "global", + "default": "yes", + "help": "Anonymize IP addresses in webhook notifications for GDPR data minimization (Article 5(1)(c)). Masks last octet/group (192.168.1.123 -> 192.168.1.x) to provide network-level information without exposing full IPs to third parties (Microsoft Teams/Discord). Set to 'no' to send full IPs. Default: yes (recommended for GDPR compliance).", + "id": "malware-scan-anonymize-webhook-ips", + "label": "Anonymize IPs in webhooks", + "regex": "^(yes|no)$", + "type": "check" + }, + "MALWARE_SCAN_TRACK_ATTACKERS": { + "context": "global", + "default": "yes", + "help": "Track IP addresses that upload malware to identify repeat offenders. Shows upload history in webhook notifications and logs. Data stored in cache with configurable TTL.", + "id": "malware-scan-track-attackers", + "label": "Track attacker IPs", + "regex": "^(yes|no)$", + "type": "check" + }, + "MALWARE_SCAN_TRACK_ATTACKER_TTL": { + "context": "global", + "default": "7776000", + "help": "Duration (in seconds) to track attacker IP addresses. Default: 90 days (7776000 seconds). This extended retention period allows sufficient time to gather evidence for law enforcement and legal proceedings. After this period, the IP's malware upload history is cleared. TTL is reset on each new malware upload.", + "id": "malware-scan-track-attacker-ttl", + "label": "Attacker tracking TTL (seconds)", + "regex": "^\\d+$", + "type": "text" + }, + "MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_ALL": { + "context": "global", + "default": "no", + "help": "Clear ALL cached scan results on plugin startup/reload. Master switch that overrides individual scanner cleanup flags. Useful after signature updates. Clears both local and Redis caches.", + "id": "malware-scan-cache-cleanup-on-restart-all", + "label": "Clear all caches on restart", + "regex": "^(yes|no)$", + "type": "check" + }, + "MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_CLAMAV": { + "context": "global", + "default": "no", + "help": "Clear ClamAV cached scan results on plugin startup/reload. Useful after ClamAV signature database updates. Ignored if cleanup_all is enabled.", + "id": "malware-scan-cache-cleanup-on-restart-clamav", + "label": "Clear ClamAV cache on restart", + "regex": "^(yes|no)$", + "type": "check" + }, + "MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_VIRUSTOTAL": { + "context": "global", + "default": "no", + "help": "Clear VirusTotal cached scan results on plugin startup/reload. Use when you want fresh VirusTotal lookups. Ignored if cleanup_all is enabled.", + "id": "malware-scan-cache-cleanup-on-restart-virustotal", + "label": "Clear VirusTotal cache on restart", + "regex": "^(yes|no)$", + "type": "check" + }, + "MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_SENTINELONE": { + "context": "global", + "default": "no", + "help": "Clear SentinelOne cached scan results on plugin startup/reload. Use when you want fresh SentinelOne threat intelligence. Ignored if cleanup_all is enabled.", + "id": "malware-scan-cache-cleanup-on-restart-sentinelone", + "label": "Clear SentinelOne cache on restart", + "regex": "^(yes|no)$", + "type": "check" + }, + "MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_MALWAREBAZAAR": { + "context": "global", + "default": "no", + "help": "Clear MalwareBazaar cached scan results on plugin startup/reload. Use when you want fresh MalwareBazaar sample lookups. Ignored if cleanup_all is enabled.", + "id": "malware-scan-cache-cleanup-on-restart-malwarebazaar", + "label": "Clear MalwareBazaar cache on restart", + "regex": "^(yes|no)$", + "type": "check" + }, + "MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_ATTACKER_IPV4": { + "context": "global", + "default": "no", + "help": "Clear IPv4 attacker tracking data on plugin startup/reload. Removes malware upload history for all tracked IPv4 addresses. Ignored if cleanup_all is enabled.", + "id": "malware-scan-cache-cleanup-on-restart-attacker-ipv4", + "label": "Clear IPv4 attacker tracking on restart", + "regex": "^(yes|no)$", + "type": "check" + }, + "MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_ATTACKER_IPV6": { + "context": "global", + "default": "no", + "help": "Clear IPv6 attacker tracking data on plugin startup/reload. Removes malware upload history for all tracked IPv6 addresses. Ignored if cleanup_all is enabled.", + "id": "malware-scan-cache-cleanup-on-restart-attacker-ipv6", + "label": "Clear IPv6 attacker tracking on restart", + "regex": "^(yes|no)$", + "type": "check" + }, + "MALWARE_SCAN_UNMASK_KEYS": { + "context": "global", + "default": "", + "help": "SECURITY RISK: Set to 'I_WANT_TO_LEAK_MY_SECRETS_TO_LOG' to show full webhook URLs in logs (for debugging only). Leave empty to mask sensitive URLs.", + "id": "malware-scan-unmask-keys", + "label": "Unmask secrets in logs", + "regex": "^.*$", + "type": "text" + }, + "MALWARE_SCAN_MALWAREBAZAAR_HASH_LOOKUP": { + "context": "global", + "default": "yes", + "help": "Enable MalwareBazaar hash database for instant malware detection (<1ms). Stores 1M+ malware hashes from abuse.ch in Redis for sub-millisecond lookups. Can be used alone or combined with ClamAV/VirusTotal/SentinelOne for multi-layer protection.", + "id": "malware-scan-malwarebazaar-hash-lookup", + "label": "Enable MalwareBazaar hash lookup", + "regex": "^(yes|no)$", + "type": "check" + }, + "MALWARE_SCAN_MALWAREBAZAAR_AUTO_UPDATE": { + "context": "global", + "default": "yes", + "help": "Enable automatic scheduled updates of MalwareBazaar hash database. Uses BunkerWeb scheduler for hourly updates. Uses ETag caching to avoid downloading unchanged files (98% bandwidth savings). Updates from sysangels.ai mirror.", + "id": "malware-scan-malwarebazaar-auto-update", + "label": "Enable auto-updates", + "regex": "^(yes|no)$", + "type": "check" + }, + "MALWARE_SCAN_MALWAREBAZAAR_UPDATE_INTERVAL": { + "context": "global", + "default": "3600", + "help": "Interval (in seconds) between automatic hash database updates. Default: 3600 (1 hour). Use 86400 (daily) for FULL strategy. Set to lower values for more frequent updates (300=5min, 600=10min, 3600=1hour).", + "id": "malware-scan-malwarebazaar-update-interval", + "label": "Update interval (seconds)", + "regex": "^\\d+$", + "type": "text" + }, + "MALWARE_SCAN_MALWAREBAZAAR_UPDATE_URL_FULL": { + "context": "global", + "default": "https://bazaar.sysangels.ai/bunkerweb/malwarebazaar_full_split.zip", + "help": "URL for downloading MalwareBazaar full hash database (ZIP format with 16 split CSV files). Default: sysangels.ai optimized mirror with pre-split files for faster parallel import.", + "id": "malware-scan-malwarebazaar-update-url-full", + "label": "Full hash database download URL", + "regex": "^.*$", + "type": "text" + }, + "MALWARE_SCAN_MALWAREBAZAAR_UPDATE_URL_RECENT": { + "context": "global", + "default": "https://bazaar.sysangels.ai/bunkerweb/malwarebazaar_recent_split.zip", + "help": "URL for downloading MalwareBazaar recent hash database (ZIP format with 16 split CSV files, updated every 5 minutes). Contains hashes from past 24 hours.", + "id": "malware-scan-malwarebazaar-update-url-recent", + "label": "Recent hash database download URL", + "regex": "^.*$", + "type": "text" + }, + "MALWARE_SCAN_MALWAREBAZAAR_CSV_PATH": { + "context": "global", + "default": "/var/cache/bunkerweb/malware-scan/hashes.csv", + "help": "Path to store downloaded MalwareBazaar hash CSV file. Directory must be writable by BunkerWeb process.", + "id": "malware-scan-malwarebazaar-csv-path", + "label": "Hash database CSV path", + "regex": "^.*$", + "type": "text" + } + } +} \ No newline at end of file diff --git a/malware-scan/ui/actions.py b/malware-scan/ui/actions.py new file mode 100644 index 0000000..058f240 --- /dev/null +++ b/malware-scan/ui/actions.py @@ -0,0 +1,506 @@ +from logging import getLogger +from traceback import format_exc +import json + +# Import get_redis_client for consistent Redis access +try: + from app.routes.utils import get_redis_client + HAS_REDIS_CLIENT = True +except ImportError: + HAS_REDIS_CLIENT = False + + +def pre_render(**kwargs): + logger = getLogger("UI") + ret = { + "counter_malware_detected": { + "value": 0, + "title": "Malware", + "subtitle": "Detected", + "subtitle_color": "danger", + "svg_color": "danger", + }, + "counter_files_scanned": { + "value": 0, + "title": "Files", + "subtitle": "Scanned", + "subtitle_color": "info", + "svg_color": "info", + }, + "counter_cache_hits": { + "value": 0, + "title": "Cache", + "subtitle": "Hits", + "subtitle_color": "success", + "svg_color": "success", + }, + "worker_clamav_status": { + "title": "WORKER CLAMAV STATUS", + "workers": [], + }, + "malwarebazaar_stats": { + "total_hashes": 0, + "buckets": 16, + "status": "unknown", + "full_db_status": "unknown", + "recent_db_status": "unknown", + "import_status": "unknown", + "full_import_start_time": None, + "full_import_end_time": None, + "recent_import_start_time": None, + "recent_import_end_time": None, + "initialized": False, + "import_progress": { + "full": {"total": 0, "processed": 0}, + "recent": {"total": 0, "processed": 0}, + }, + }, + "clamav_stats": { + "clamav_version": "unknown", + "definition_version": "unknown", + "definition_date": "unknown", + "status": "unknown", + }, + } + + # Get ClamAV status from all worker instances + # The ping response now includes MalwareBazaar and ClamAV stats + stats_collected = False + try: + instances = kwargs["bw_instances_utils"].get_instances(status="up") + for instance in instances: + worker_status = { + "name": instance.name, + "hostname": instance.hostname, + "status": "error", + "message": "No response", + } + try: + resp, ping_data = instance.ping("malware-scan") + if resp and isinstance(ping_data, dict): + # ping_data format: {hostname: {"status": "success", "msg": "{\"status\":\"...\",\"message\":\"...\"}"}} + instance_data = ping_data.get(instance.hostname, {}) + + # The actual plugin response is JSON-encoded in the 'msg' field + msg_json = instance_data.get("msg", "{}") + try: + plugin_response = json.loads(msg_json) + worker_status["status"] = plugin_response.get("status", "error") + worker_status["message"] = plugin_response.get("message", "Unknown status") + + # Extract stats from first successful worker response + if not stats_collected and plugin_response.get("status") == "success": + # MalwareBazaar stats + if "malwarebazaar" in plugin_response: + mb_stats = plugin_response["malwarebazaar"] + ret["malwarebazaar_stats"]["total_hashes"] = mb_stats.get("total_hashes", 0) + ret["malwarebazaar_stats"]["buckets"] = mb_stats.get("buckets", 16) + ret["malwarebazaar_stats"]["full_db_status"] = mb_stats.get("full_db_status", "unknown") + ret["malwarebazaar_stats"]["recent_db_status"] = mb_stats.get("recent_db_status", "unknown") + ret["malwarebazaar_stats"]["import_status"] = mb_stats.get("import_status", "unknown") + ret["malwarebazaar_stats"]["full_import_start_time"] = mb_stats.get("full_import_start_time") + ret["malwarebazaar_stats"]["full_import_end_time"] = mb_stats.get("full_import_end_time") + ret["malwarebazaar_stats"]["recent_import_start_time"] = mb_stats.get("recent_import_start_time") + ret["malwarebazaar_stats"]["recent_import_end_time"] = mb_stats.get("recent_import_end_time") + ret["malwarebazaar_stats"]["initialized"] = mb_stats.get("initialized", False) + ret["malwarebazaar_stats"]["status"] = "success" + + # Extract import progress if available + if "import_progress" in mb_stats: + ret["malwarebazaar_stats"]["import_progress"] = mb_stats["import_progress"] + + # ClamAV stats + if "clamav_stats" in plugin_response: + cv_stats = plugin_response["clamav_stats"] + ret["clamav_stats"]["clamav_version"] = cv_stats.get("version", "unknown") + ret["clamav_stats"]["definition_version"] = cv_stats.get("definitions", "unknown") + ret["clamav_stats"]["definition_date"] = cv_stats.get("updated", "unknown") + ret["clamav_stats"]["status"] = "success" + + stats_collected = True + + except json.JSONDecodeError as e: + logger.error(f"Failed to parse JSON from worker {instance.name}: {e}") + worker_status["status"] = "error" + worker_status["message"] = "Invalid JSON response" + else: + worker_status["status"] = "error" + worker_status["message"] = "Invalid response from worker" + except BaseException as e: + logger.debug(f"Failed to get ping from {instance.name}: {format_exc()}") + worker_status["status"] = "error" + worker_status["message"] = f"Connection error: {str(e)}" + + ret["worker_clamav_status"]["workers"].append(worker_status) + + except BaseException as e: + logger.error(f"Failed to get worker instances: {e}") + logger.debug(format_exc()) + + # Always query Redis directly for import status (manager-side) + # The import status is set by scheduler jobs, not workers + # Workers return historical stats, but import status changes in real-time + if HAS_REDIS_CLIENT: + try: + # Use get_redis_client() for consistent Redis access (handles Sentinel automatically) + r = get_redis_client() + logger.debug(f"[Manager pre_render] get_redis_client() returned: {r is not None}") + + if r is not None: + # Test connection + try: + r.ping() + logger.debug("[Manager pre_render] Redis ping successful") + except Exception as ping_err: + logger.debug(f"[Manager pre_render] Redis ping failed: {ping_err}") + r = None + + if r is not None: + # Query import status and database state keys + import_status = r.get("malware-scan:hashes:import_state") + full_db_status = r.get("malware-scan:hashes:state:full") + recent_db_status = r.get("malware-scan:hashes:state:recent") + + # Decode bytes to strings if needed + if import_status: + import_status = import_status.decode() if isinstance(import_status, bytes) else import_status + if full_db_status: + full_db_status = full_db_status.decode() if isinstance(full_db_status, bytes) else full_db_status + if recent_db_status: + recent_db_status = recent_db_status.decode() if isinstance(recent_db_status, bytes) else recent_db_status + + logger.debug(f"[Manager pre_render] Retrieved from Redis: import_status={import_status}, full_db_status={full_db_status}, recent_db_status={recent_db_status}") + + # Set status to success if we got any data from Redis + if import_status or full_db_status or recent_db_status: + ret["malwarebazaar_stats"]["status"] = "success" + + if import_status: + ret["malwarebazaar_stats"]["import_status"] = import_status + logger.debug(f"[Manager pre_render] Set import_status to: {import_status}") + + if full_db_status: + ret["malwarebazaar_stats"]["full_db_status"] = full_db_status + + if recent_db_status: + ret["malwarebazaar_stats"]["recent_db_status"] = recent_db_status + + # Get full database timestamps + full_start = r.get("malware-scan:hashes:import_start_time:full") + full_end = r.get("malware-scan:hashes:import_end_time:full") + if full_start: + full_start = full_start.decode() if isinstance(full_start, bytes) else full_start + ret["malwarebazaar_stats"]["full_import_start_time"] = full_start + if full_end: + full_end = full_end.decode() if isinstance(full_end, bytes) else full_end + ret["malwarebazaar_stats"]["full_import_end_time"] = full_end + + # Get recent database timestamps + recent_start = r.get("malware-scan:hashes:import_start_time:recent") + recent_end = r.get("malware-scan:hashes:import_end_time:recent") + if recent_start: + recent_start = recent_start.decode() if isinstance(recent_start, bytes) else recent_start + ret["malwarebazaar_stats"]["recent_import_start_time"] = recent_start + if recent_end: + recent_end = recent_end.decode() if isinstance(recent_end, bytes) else recent_end + ret["malwarebazaar_stats"]["recent_import_end_time"] = recent_end + + # Get import progress data + buckets = ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "A", "B", "C", "D", "E", "F"] + + # Full database progress + full_import_total = r.get("malware-scan:hashes:import_total:full") + if full_import_total: + total_str = full_import_total.decode() if isinstance(full_import_total, bytes) else full_import_total + ret["malwarebazaar_stats"]["import_progress"]["full"]["total"] = int(total_str) + + # Sum up processed counts from all 16 buckets for full database + full_processed = 0 + for bucket in buckets: + bucket_processed = r.get(f"malware-scan:hashes:import_processed:full:bucket_{bucket}") + if bucket_processed: + count_str = bucket_processed.decode() if isinstance(bucket_processed, bytes) else bucket_processed + full_processed += int(count_str) + ret["malwarebazaar_stats"]["import_progress"]["full"]["processed"] = full_processed + + # Recent database progress + recent_import_total = r.get("malware-scan:hashes:import_total:recent") + if recent_import_total: + total_str = recent_import_total.decode() if isinstance(recent_import_total, bytes) else recent_import_total + ret["malwarebazaar_stats"]["import_progress"]["recent"]["total"] = int(total_str) + + # Sum up processed counts from all 16 buckets for recent database + recent_processed = 0 + for bucket in buckets: + bucket_processed = r.get(f"malware-scan:hashes:import_processed:recent:bucket_{bucket}") + if bucket_processed: + count_str = bucket_processed.decode() if isinstance(bucket_processed, bytes) else bucket_processed + recent_processed += int(count_str) + ret["malwarebazaar_stats"]["import_progress"]["recent"]["processed"] = recent_processed + + r.close() + else: + logger.debug("[Manager pre_render] Redis client is None, cannot query progress data") + except BaseException as e: + logger.debug(f"[Manager pre_render] Failed to query Redis for import progress: {e}") + logger.debug(format_exc()) + + # Get aggregated metrics from all instances + try: + data = kwargs["bw_instances_utils"].get_metrics("malware-scan") + + # Update counter values + ret["counter_malware_detected"]["value"] = data.get("counter_malware_detected", 0) + ret["counter_files_scanned"]["value"] = data.get("counter_files_scanned", 0) + ret["counter_cache_hits"]["value"] = data.get("counter_cache_hits", 0) + + except BaseException as e: + logger.debug(format_exc()) + logger.error(f"Failed to get malware-scan metrics: {e}") + ret["error"] = str(e) + + return ret + + +def test_discord(**kwargs): + """Test Discord webhook by sending test message from manager""" + logger = getLogger("UI") + try: + import requests + from datetime import datetime + import socket + + # Get Discord webhook URL from database settings + db = kwargs.get("Database") + if not db: + return {"status": "error", "message": "Database not available"} + + # Query for Discord webhook URL setting + discord_url = db.get_global_value("MALWARE_SCAN_DISCORD_WEBHOOK_URL") + + if not discord_url or discord_url == "" or discord_url == "your_webhook_url": + return {"status": "error", "message": "Discord webhook not configured"} + + # Build test message (Discord embed format) + hostname = socket.gethostname() + test_payload = { + "embeds": [{ + "title": "โœ… Discord Webhook Test", + "description": "This is a test message from BunkerWeb Malware Scanner", + "color": 65280, # Green color + "fields": [ + {"name": "Status", "value": "Webhook is configured correctly", "inline": False}, + {"name": "Instance", "value": hostname, "inline": True}, + {"name": "Test Time", "value": datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC"), "inline": True} + ], + "timestamp": datetime.utcnow().isoformat() + "Z" + }] + } + + # Send test request to Discord + response = requests.post(discord_url, json=test_payload, timeout=5) + + if response.status_code >= 200 and response.status_code < 300: + return {"status": "success", "message": "Discord webhook test successful"} + else: + return {"status": "error", "message": f"Discord returned HTTP {response.status_code}"} + + except ImportError: + return {"status": "error", "message": "requests library not available"} + except requests.RequestException as e: + logger.error(f"Discord webhook test failed: {e}") + return {"status": "error", "message": f"Request failed: {str(e)}"} + except BaseException as e: + logger.error(f"Discord webhook test failed: {e}") + logger.debug(format_exc()) + return {"status": "error", "message": str(e)} + + +def test_teams(**kwargs): + """Test Microsoft Teams webhook by sending test message from manager""" + logger = getLogger("UI") + try: + import requests + from datetime import datetime + import socket + + # Get Teams webhook URL from database settings + db = kwargs.get("Database") + if not db: + return {"status": "error", "message": "Database not available"} + + # Query for Teams webhook URL setting + teams_url = db.get_global_value("MALWARE_SCAN_TEAMS_WEBHOOK_URL") + + if not teams_url or teams_url == "" or teams_url == "your_webhook_url": + return {"status": "error", "message": "Teams webhook not configured"} + + # Build test message (Microsoft Teams MessageCard format) + hostname = socket.gethostname() + test_payload = { + "@type": "MessageCard", + "@context": "https://schema.org/extensions", + "summary": "Webhook Test", + "sections": [{ + "activityTitle": "โœ… Microsoft Teams Webhook Test", + "activitySubtitle": "BunkerWeb Malware Scanner", + "facts": [ + {"name": "Status", "value": "Webhook is configured correctly"}, + {"name": "Instance", "value": hostname}, + {"name": "Test Time", "value": datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC")} + ], + "markdown": True + }] + } + + # Send test request to Teams + response = requests.post(teams_url, json=test_payload, timeout=5) + + if response.status_code >= 200 and response.status_code < 300: + return {"status": "success", "message": "Teams webhook test successful"} + else: + return {"status": "error", "message": f"Teams returned HTTP {response.status_code}"} + + except ImportError: + return {"status": "error", "message": "requests library not available"} + except requests.RequestException as e: + logger.error(f"Teams webhook test failed: {e}") + return {"status": "error", "message": f"Request failed: {str(e)}"} + except BaseException as e: + logger.error(f"Teams webhook test failed: {e}") + logger.debug(format_exc()) + return {"status": "error", "message": str(e)} + + +def test_eicar(**kwargs): + """Test ClamAV with EICAR test file by calling worker API endpoint""" + logger = getLogger("UI") + try: + # Get first available worker instance + instances = kwargs["bw_instances_utils"].get_instances(status="up") + if not instances: + return {"status": "error", "message": "No worker instances available"} + + # Call the test-eicar endpoint on first worker + instance = instances[0] + resp, data = instance.api("/malware-scan/test-eicar", method="POST") + + if resp and isinstance(data, dict): + # Extract response from first worker + for hostname, response_data in data.items(): + msg_json = response_data.get("msg", "{}") + try: + result = json.loads(msg_json) + return result + except json.JSONDecodeError: + logger.error(f"Failed to parse response from {hostname}") + return {"status": "error", "message": "Invalid response from worker"} + + return {"status": "error", "message": "No response from worker"} + except BaseException as e: + logger.error(f"EICAR test failed: {e}") + logger.debug(format_exc()) + return {"status": "error", "message": str(e)} + + +def get_import_progress(**kwargs): + """Get current import progress for live updates""" + logger = getLogger("UI") + result = { + "import_status": "unknown", + "import_progress": { + "full": {"total": 0, "processed": 0}, + "recent": {"total": 0, "processed": 0}, + }, + } + + try: + # Try to access Redis through Database settings + db = kwargs.get("Database") + if db: + # Check if Redis is enabled globally + use_redis = db.get_global_value("USE_REDIS", default="no") + if use_redis == "yes": + # Try to import and use redis library + try: + import redis + + # Get Redis connection details from database + redis_host = db.get_global_value("REDIS_HOST", default="127.0.0.1") + redis_port = int(db.get_global_value("REDIS_PORT", default="6379")) + redis_db_num = int(db.get_global_value("REDIS_DB", default="0")) + sentinel_hosts_str = db.get_global_value("REDIS_SENTINEL_HOSTS", default="") + + # Check if using Redis Sentinel + if sentinel_hosts_str and sentinel_hosts_str.strip(): + # Parse sentinel hosts + sentinel_hosts = [] + for h in sentinel_hosts_str.split(): + h = h.strip() + if ":" in h: + host, port = h.rsplit(":", 1) + sentinel_hosts.append((host.strip(), int(port.strip()))) + else: + sentinel_hosts.append((h, 26379)) + + # Get service name + service_name = db.get_global_value("REDIS_SENTINEL_SERVICE_NAME", default="mymaster") + + # Connect via Sentinel + sentinel = redis.Sentinel(sentinel_hosts, socket_timeout=2, decode_responses=True) + r = sentinel.master_for(service_name, socket_timeout=2, db=redis_db_num, decode_responses=True) + logger.debug(f"[get_import_progress] Connected to Redis via Sentinel: {service_name}") + else: + # Connect to Redis directly + r = redis.Redis(host=redis_host, port=redis_port, db=redis_db_num, decode_responses=True, socket_connect_timeout=2) + logger.debug(f"[get_import_progress] Connected to Redis directly: {redis_host}:{redis_port}") + + # Query import status + import_status = r.get("malware-scan:hashes:import_state") + if import_status: + result["import_status"] = import_status + + # Get import progress data + buckets = ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "A", "B", "C", "D", "E", "F"] + + # Full database progress + full_import_total = r.get("malware-scan:hashes:import_total:full") + if full_import_total: + result["import_progress"]["full"]["total"] = int(full_import_total) + + # Sum up processed counts from all 16 buckets for full database + full_processed = 0 + for bucket in buckets: + bucket_processed = r.get(f"malware-scan:hashes:import_processed:full:bucket_{bucket}") + if bucket_processed: + full_processed += int(bucket_processed) + result["import_progress"]["full"]["processed"] = full_processed + + # Recent database progress + recent_import_total = r.get("malware-scan:hashes:import_total:recent") + if recent_import_total: + result["import_progress"]["recent"]["total"] = int(recent_import_total) + + # Sum up processed counts from all 16 buckets for recent database + recent_processed = 0 + for bucket in buckets: + bucket_processed = r.get(f"malware-scan:hashes:import_processed:recent:bucket_{bucket}") + if bucket_processed: + recent_processed += int(bucket_processed) + result["import_progress"]["recent"]["processed"] = recent_processed + + r.close() + except ImportError: + logger.debug("[get_import_progress] redis-py not available for direct Redis query") + except Exception as redis_err: + logger.warning(f"[get_import_progress] Redis connection failed: {redis_err}") + logger.debug(format_exc()) + except BaseException as e: + logger.debug(f"Failed to query Redis for import progress: {e}") + + return result + + +def malware_scan(**kwargs): + """Main entry point for malware-scan plugin UI page""" + return {"data": pre_render(**kwargs)} diff --git a/malware-scan/ui/blueprints/__init__.py b/malware-scan/ui/blueprints/__init__.py new file mode 100644 index 0000000..7755dff --- /dev/null +++ b/malware-scan/ui/blueprints/__init__.py @@ -0,0 +1,5 @@ +# BunkerWeb malware-scan plugin blueprints + +from .malware_scan import malware_scan + +__all__ = ["malware_scan"] diff --git a/malware-scan/ui/blueprints/malware_scan.py b/malware-scan/ui/blueprints/malware_scan.py new file mode 100644 index 0000000..cd0356d --- /dev/null +++ b/malware-scan/ui/blueprints/malware_scan.py @@ -0,0 +1,418 @@ +from logging import getLogger +from traceback import format_exc +from os.path import dirname, join, exists, basename +from flask import Blueprint, jsonify, request, send_file +from flask_login import login_required + +from app.dependencies import BW_CONFIG, DB # type: ignore +from app.routes.utils import cors_required, get_redis_client # type: ignore + +blueprint_path = dirname(__file__) + +malware_scan = Blueprint( + "malware_scan", + __name__, +) + + +@malware_scan.route("/api/plugins/malware-scan/test_discord", methods=["POST"]) +@login_required +@cors_required +def test_discord(): + """Test Discord webhook from manager UI""" + logger = getLogger("UI") + + # Get webhook URL from configuration + config = BW_CONFIG.get_config( + global_only=True, + methods=False, + filtered_settings=("MALWARE_SCAN_DISCORD_WEBHOOK_URL",) + ) + webhook_url = config.get("MALWARE_SCAN_DISCORD_WEBHOOK_URL", "") + + if not webhook_url or webhook_url.strip() == "": + return jsonify({"status": "error", "message": "Discord webhook URL not configured"}), 400 + + # Send test message to Discord + try: + import requests + + payload = { + "username": "BunkerWeb Malware Scanner", + "avatar_url": "https://docs.bunkerweb.io/img/logo-v2.0.png", + "embeds": [ + { + "title": "๐Ÿงช Test Message", + "description": "Discord webhook is operational! This is a test from BunkerWeb malware-scan plugin.", + "color": 3447003, # Blue color + "fields": [ + { + "name": "Status", + "value": "โœ… Connected", + "inline": True + }, + { + "name": "Source", + "value": "Manager UI", + "inline": True + } + ], + "footer": { + "text": "BunkerWeb Malware Scanner" + } + } + ] + } + + response = requests.post( + webhook_url, + json=payload, + timeout=10, + headers={"Content-Type": "application/json"} + ) + + if response.status_code in (200, 204): + logger.info("Discord webhook test successful") + return jsonify({"status": "success", "message": "Discord webhook is operational"}), 200 + else: + error_msg = f"Discord webhook returned status {response.status_code}" + try: + error_detail = response.json() + if "message" in error_detail: + error_msg = f"{error_msg}: {error_detail['message']}" + except: + pass + logger.error(error_msg) + return jsonify({"status": "error", "message": error_msg}), 500 + + except Exception as e: + logger.error(f"Discord webhook test failed: {e}") + logger.debug(format_exc()) + return jsonify({"status": "error", "message": str(e)}), 500 + + +@malware_scan.route("/api/plugins/malware-scan/test_teams", methods=["POST"]) +@login_required +@cors_required +def test_teams(): + """Test Microsoft Teams webhook from manager UI""" + logger = getLogger("UI") + + # Get webhook URL from configuration + config = BW_CONFIG.get_config( + global_only=True, + methods=False, + filtered_settings=("MALWARE_SCAN_TEAMS_WEBHOOK_URL",) + ) + webhook_url = config.get("MALWARE_SCAN_TEAMS_WEBHOOK_URL", "") + + if not webhook_url or webhook_url.strip() == "": + return jsonify({"status": "error", "message": "Teams webhook URL not configured"}), 400 + + # Send test message to Teams + try: + import requests + + payload = { + "@type": "MessageCard", + "@context": "https://schema.org/extensions", + "summary": "BunkerWeb Malware Scanner Test", + "themeColor": "0078D7", + "title": "๐Ÿงช Test Message", + "sections": [ + { + "activityTitle": "Webhook Test", + "activitySubtitle": "BunkerWeb malware-scan plugin", + "activityImage": "https://docs.bunkerweb.io/img/logo-v2.0.png", + "facts": [ + { + "name": "Status:", + "value": "โœ… Connected" + }, + { + "name": "Source:", + "value": "Manager UI" + } + ], + "text": "Microsoft Teams webhook is operational! This is a test from BunkerWeb malware-scan plugin." + } + ] + } + + response = requests.post( + webhook_url, + json=payload, + timeout=10, + headers={"Content-Type": "application/json"} + ) + + if response.status_code in (200, 204): + # Teams returns "1" in response body on success + if response.text.strip() == "1": + logger.info("Teams webhook test successful") + return jsonify({"status": "success", "message": "Microsoft Teams webhook is operational"}), 200 + else: + error_msg = f"Teams webhook returned unexpected response: {response.text}" + logger.error(error_msg) + return jsonify({"status": "error", "message": error_msg}), 500 + else: + error_msg = f"Teams webhook returned status {response.status_code}" + logger.error(error_msg) + return jsonify({"status": "error", "message": error_msg}), 500 + + except Exception as e: + logger.error(f"Teams webhook test failed: {e}") + logger.debug(format_exc()) + return jsonify({"status": "error", "message": str(e)}), 500 + + +@malware_scan.route("/api/plugins/malware-scan/test_eicar", methods=["POST"]) +@login_required +@cors_required +def test_eicar(): + """Test ClamAV with EICAR test file by calling worker API""" + logger = getLogger("UI") + + try: + from app.dependencies import BW_INSTANCES_UTILS # type: ignore + + instances = BW_INSTANCES_UTILS.get_instances(status="up") + + if not instances: + return jsonify({"status": "error", "message": "No active workers available"}), 500 + + # Call first available worker + instance = instances[0] + + # The worker endpoint is /malware-scan/test-eicar (POST) + # Use the instance's apiCaller to send the request + try: + resp, response_data = instance.apiCaller.send_to_apis("POST", "/malware-scan/test-eicar", response=True) + + if resp and instance.hostname in response_data: + worker_response = response_data[instance.hostname] + if worker_response.get("status") == "success": + message = worker_response.get("msg", "EICAR test successful") + return jsonify({"status": "success", "message": message}), 200 + else: + error_msg = worker_response.get("msg", "Unknown error from worker") + return jsonify({"status": "error", "message": error_msg}), 500 + else: + error_msg = "No response from worker" if not resp else f"Invalid response structure from {instance.hostname}" + return jsonify({"status": "error", "message": error_msg}), 500 + + except Exception as e: + logger.error(f"EICAR test failed when calling worker: {e}") + return jsonify({"status": "error", "message": f"Worker call failed: {str(e)}"}), 500 + + except Exception as e: + logger.error(f"EICAR test failed: {e}") + logger.debug(format_exc()) + return jsonify({"status": "error", "message": str(e)}), 500 + + +@malware_scan.route("/api/plugins/malware-scan/get_import_progress", methods=["GET"]) +@login_required +@cors_required +def get_import_progress(): + """Get current import progress for live updates""" + logger = getLogger("UI") + result = { + "import_status": "unknown", + "import_progress": { + "full": {"total": 0, "processed": 0}, + "recent": {"total": 0, "processed": 0}, + }, + } + + try: + # Get Redis client using BunkerWeb helper (handles Sentinel automatically) + r = get_redis_client() + logger.debug(f"get_redis_client() returned: {r is not None}") + + if r is not None: + # Test connection + try: + r.ping() + logger.debug("Redis ping successful") + except Exception as ping_err: + logger.debug(f"Redis ping failed: {ping_err}") + r = None + + if r is not None: + # Query import status + import_status = r.get("malware-scan:hashes:import_state") + logger.debug(f"import_state query returned: {import_status}") + if import_status: + # Decode bytes to string if needed + result["import_status"] = import_status.decode() if isinstance(import_status, bytes) else import_status + + # Get import progress data + buckets = ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "A", "B", "C", "D", "E", "F"] + + # Full database progress + full_import_total = r.get("malware-scan:hashes:import_total:full") + logger.debug(f"import_total:full query returned: {full_import_total}") + if full_import_total: + # Decode bytes to string if needed + total_str = full_import_total.decode() if isinstance(full_import_total, bytes) else full_import_total + result["import_progress"]["full"]["total"] = int(total_str) + + # Sum up processed counts from all 16 buckets for full database + full_processed = 0 + for bucket in buckets: + bucket_processed = r.get(f"malware-scan:hashes:import_processed:full:bucket_{bucket}") + if bucket_processed: + # Decode bytes to string if needed + count_str = bucket_processed.decode() if isinstance(bucket_processed, bytes) else bucket_processed + full_processed += int(count_str) + result["import_progress"]["full"]["processed"] = full_processed + + # Recent database progress + recent_import_total = r.get("malware-scan:hashes:import_total:recent") + if recent_import_total: + # Decode bytes to string if needed + total_str = recent_import_total.decode() if isinstance(recent_import_total, bytes) else recent_import_total + result["import_progress"]["recent"]["total"] = int(total_str) + + # Sum up processed counts from all 16 buckets for recent database + recent_processed = 0 + for bucket in buckets: + bucket_processed = r.get(f"malware-scan:hashes:import_processed:recent:bucket_{bucket}") + if bucket_processed: + # Decode bytes to string if needed + count_str = bucket_processed.decode() if isinstance(bucket_processed, bytes) else bucket_processed + recent_processed += int(count_str) + result["import_progress"]["recent"]["processed"] = recent_processed + + r.close() + else: + logger.debug("Redis client is None, cannot query progress data") + except Exception as e: + logger.debug(f"Failed to query Redis for import progress: {e}") + logger.debug(format_exc()) + + logger.debug(f"Returning progress result: {result}") + return jsonify(result), 200 + + +@malware_scan.route("/api/plugins/malware-scan/download-full-db", methods=["GET"]) +@login_required +@cors_required +def download_full_db(): + """Download the full MalwareBazaar hash database ZIP file""" + logger = getLogger("UI") + + try: + # Get Redis client using BunkerWeb helper (handles Sentinel automatically) + r = get_redis_client() + + if r is None: + logger.error("Redis client unavailable for download-full-db") + return jsonify({"status": "error", "message": "Redis unavailable"}), 500 + + # Test connection + try: + r.ping() + except Exception as ping_err: + logger.error(f"Redis ping failed: {ping_err}") + return jsonify({"status": "error", "message": "Redis connection failed"}), 500 + + # Get temp directory path from Redis + temp_dir_key = "malware_scan:scheduler:temp_dir" + temp_dir = r.get(temp_dir_key) + + if not temp_dir: + logger.error("Temp directory not found in Redis (downloads may not be available yet)") + return jsonify({"status": "error", "message": "Hash database not downloaded yet. Please wait for the scheduler to run."}), 404 + + # Decode bytes to string if needed + temp_dir = temp_dir.decode() if isinstance(temp_dir, bytes) else temp_dir + + # Build full path to ZIP file + zip_filename = "hashes_full.zip" + zip_path = join(temp_dir, zip_filename) + + logger.info(f"Attempting to download full DB from: {zip_path}") + + # Check if file exists + if not exists(zip_path): + logger.error(f"Full database ZIP file not found at: {zip_path}") + return jsonify({"status": "error", "message": f"Full database ZIP not found. File may not have been downloaded yet."}), 404 + + # Send file for download + return send_file( + zip_path, + mimetype='application/zip', + as_attachment=True, + download_name=zip_filename + ) + + except Exception as e: + logger.error(f"Failed to download full database: {e}") + logger.debug(format_exc()) + return jsonify({"status": "error", "message": str(e)}), 500 + finally: + if r: + r.close() + + +@malware_scan.route("/api/plugins/malware-scan/download-recent-db", methods=["GET"]) +@login_required +@cors_required +def download_recent_db(): + """Download the recent MalwareBazaar hash database ZIP file""" + logger = getLogger("UI") + + try: + # Get Redis client using BunkerWeb helper (handles Sentinel automatically) + r = get_redis_client() + + if r is None: + logger.error("Redis client unavailable for download-recent-db") + return jsonify({"status": "error", "message": "Redis unavailable"}), 500 + + # Test connection + try: + r.ping() + except Exception as ping_err: + logger.error(f"Redis ping failed: {ping_err}") + return jsonify({"status": "error", "message": "Redis connection failed"}), 500 + + # Get temp directory path from Redis + temp_dir_key = "malware_scan:scheduler:temp_dir" + temp_dir = r.get(temp_dir_key) + + if not temp_dir: + logger.error("Temp directory not found in Redis (downloads may not be available yet)") + return jsonify({"status": "error", "message": "Hash database not downloaded yet. Please wait for the scheduler to run."}), 404 + + # Decode bytes to string if needed + temp_dir = temp_dir.decode() if isinstance(temp_dir, bytes) else temp_dir + + # Build full path to ZIP file + zip_filename = "hashes_recent.zip" + zip_path = join(temp_dir, zip_filename) + + logger.info(f"Attempting to download recent DB from: {zip_path}") + + # Check if file exists + if not exists(zip_path): + logger.error(f"Recent database ZIP file not found at: {zip_path}") + return jsonify({"status": "error", "message": f"Recent database ZIP not found. File may not have been downloaded yet."}), 404 + + # Send file for download + return send_file( + zip_path, + mimetype='application/zip', + as_attachment=True, + download_name=zip_filename + ) + + except Exception as e: + logger.error(f"Failed to download recent database: {e}") + logger.debug(format_exc()) + return jsonify({"status": "error", "message": str(e)}), 500 + finally: + if r: + r.close() diff --git a/malware-scan/ui/template.html b/malware-scan/ui/template.html new file mode 100644 index 0000000..c159e93 --- /dev/null +++ b/malware-scan/ui/template.html @@ -0,0 +1,689 @@ + +
    + + {% if pre_render.get('data') %} + +
    +
    +
    +
    + + Webhook Testing +
    +
    +
    +
    +
    + +
    +
    + +
    +
    + +
    +
    +
    + + +
    +
    +
    +
    + + MalwareBazaar Hashes +
    +
    +
    + {% if pre_render['data']['malwarebazaar_stats']['status'] == 'success' %} +
    {{ "{:,}".format(pre_render['data']['malwarebazaar_stats']['total_hashes']) }}
    +
    Known Malware Hashes
    +
    + + {{ pre_render['data']['malwarebazaar_stats']['buckets'] }} Redis buckets + {% if pre_render['data']['malwarebazaar_stats'].get('import_status') == 'running' %} +
    + Import in progress + {% endif %} +
    + {% else %} +
    Hash lookup not enabled or Redis unavailable
    + {% endif %} +
    +
    +
    + + +
    +
    +
    +
    + + Full Database +
    +
    +
    + {% if pre_render['data']['malwarebazaar_stats'].get('import_status') == 'running' %} +
    โณ Import in progress
    +
    Full DB Status
    +
    + + Database import is running on manager +
    + Progress: + {% if pre_render['data']['malwarebazaar_stats']['import_progress']['full']['total'] > 0 %} + {{ "{:,}".format(pre_render['data']['malwarebazaar_stats']['import_progress']['full']['processed']) }} of {{ "{:,}".format(pre_render['data']['malwarebazaar_stats']['import_progress']['full']['total']) }} + ({{ "%.1f"|format(pre_render['data']['malwarebazaar_stats']['import_progress']['full']['processed'] / pre_render['data']['malwarebazaar_stats']['import_progress']['full']['total'] * 100) }}%) + {% else %} + 0 of 0 (0%) + {% endif %} + + {% if pre_render['data']['malwarebazaar_stats'].get('full_import_start_time') %} +
    + Duration: Calculating... + {% endif %} +
    + {% elif pre_render['data']['malwarebazaar_stats']['status'] == 'success' and pre_render['data']['malwarebazaar_stats'].get('full_db_status') %} + {% set full_status = pre_render['data']['malwarebazaar_stats']['full_db_status'] %} + {% if full_status == '1' %} +
    โœ“ Completed
    + {% elif full_status == 'never' %} +
    Never
    + {% else %} +
    {{ full_status }}
    + {% endif %} +
    Full DB Status
    +
    + + Complete database download + {% if pre_render['data']['malwarebazaar_stats'].get('full_import_start_time') and pre_render['data']['malwarebazaar_stats'].get('full_import_end_time') %} +
    + Last import: Calculating... +
    + Completed: Calculating... + {% endif %} +
    + +
    +
    + {% else %} +
    Status unavailable
    + {% endif %} +
    +
    +
    + + +
    +
    +
    +
    + + Recent Updates +
    +
    +
    + {% if pre_render['data']['malwarebazaar_stats'].get('import_status') == 'running' %} +
    โณ Import in progress
    +
    Recent DB Status
    +
    + + Database import is running on manager +
    + Progress: + {% if pre_render['data']['malwarebazaar_stats']['import_progress']['recent']['total'] > 0 %} + {{ "{:,}".format(pre_render['data']['malwarebazaar_stats']['import_progress']['recent']['processed']) }} of {{ "{:,}".format(pre_render['data']['malwarebazaar_stats']['import_progress']['recent']['total']) }} + ({{ "%.1f"|format(pre_render['data']['malwarebazaar_stats']['import_progress']['recent']['processed'] / pre_render['data']['malwarebazaar_stats']['import_progress']['recent']['total'] * 100) }}%) + {% else %} + 0 of 0 (0%) + {% endif %} + + {% if pre_render['data']['malwarebazaar_stats'].get('recent_import_start_time') %} +
    + Duration: Calculating... + {% endif %} +
    + {% elif pre_render['data']['malwarebazaar_stats']['status'] == 'success' and pre_render['data']['malwarebazaar_stats'].get('recent_db_status') %} + {% set recent_status = pre_render['data']['malwarebazaar_stats']['recent_db_status'] %} + {% if recent_status == '1' %} +
    โœ“ Completed
    + {% elif recent_status == 'never' %} +
    Never
    + {% else %} +
    {{ recent_status }}
    + {% endif %} +
    Recent DB Status
    +
    + + Incremental updates + {% if pre_render['data']['malwarebazaar_stats'].get('recent_import_start_time') and pre_render['data']['malwarebazaar_stats'].get('recent_import_end_time') %} +
    + Last import: Calculating... +
    + Completed: Calculating... + {% endif %} +
    + +
    +
    + {% else %} +
    Status unavailable
    + {% endif %} +
    +
    +
    + + +
    +
    +
    +
    + + ClamAV Definitions +
    +
    +
    + {% if pre_render['data']['clamav_stats']['status'] == 'success' %} +
    {{ pre_render['data']['clamav_stats']['definition_version'] }}
    +
    Definition Version
    +
    + ClamAV {{ pre_render['data']['clamav_stats']['clamav_version'] }}
    + Updated: {{ pre_render['data']['clamav_stats']['definition_date'] }} +
    + {% else %} +
    ClamAV information unavailable
    + {% endif %} +
    +
    +
    + {% endif %} + + + {% if pre_render.get('data') and pre_render['data'].get('worker_clamav_status') and pre_render['data']['worker_clamav_status'].get('workers') %} +
    +
    +
    +
    + + Worker ClamAV Status +
    +
    +
    +
    + + + + + + + + + + + + {% for worker in pre_render['data']['worker_clamav_status']['workers'] %} + + + + + + + + {% endfor %} + +
    Worker NameHostnameStatusMessageActions
    {{ worker.name }}{{ worker.hostname }} + {% if worker.status == 'success' %} + + Active + + {% elif worker.status == 'warning' %} + + Warning + + {% else %} + + Error + + {% endif %} + {{ worker.message }} + +
    +
    +
    +
    +
    + {% else %} +
    + +
    + {% endif %} +
    + + + + diff --git a/malware-scan/variables.env.example b/malware-scan/variables.env.example new file mode 100644 index 0000000..8386764 --- /dev/null +++ b/malware-scan/variables.env.example @@ -0,0 +1,157 @@ +# BunkerWeb Malware Scan Plugin - Configuration Variables +# Version: 0.8.0 +# Copy to /etc/bunkerweb/variables.env and customize + +# ============================================================================ +# CORE SETTINGS +# ============================================================================ + +USE_MALWARE_SCANNER=yes # Enable malware scanning +MALWARE_SCAN_DEBUG=no # Enable debug logging (disable in production) + +# ============================================================================ +# CLAMAV SETTINGS +# ============================================================================ + +MALWARE_SCAN_CLAMAV_HOST=127.0.0.1 # ClamAV daemon host +MALWARE_SCAN_CLAMAV_PORT=3310 # ClamAV daemon port +MALWARE_SCAN_CLAMAV_TIMEOUT=10000 # ClamAV connection timeout (ms) +MALWARE_SCAN_CLAMAV_MAX_SIZE=26214400 # Max file size for ClamAV (25MB) +MALWARE_SCAN_CLAMAV_SOCKET= # ClamAV socket path (alternative to host:port, e.g., /var/run/clamav/clamd.ctl) +MALWARE_SCAN_HASH_MAX_SIZE=67108864 # Max file size for hash scanners (64MB) + +# ============================================================================ +# ALLOW LIST (FALSE POSITIVE HANDLING) - v0.4.1+ +# ============================================================================ + +MALWARE_SCAN_ALLOWLIST_HASHES= # Comma-separated SHA256 hashes to skip scanning +MALWARE_SCAN_ALLOWLIST_FILE= # Path to file with SHA256 hashes (one per line) + +# ============================================================================ +# MULTI-LAYER SCANNING +# ============================================================================ + +MALWARE_SCAN_SKIP_OTHERS_ON_CLAMAV_DETECT=yes # Skip cloud scanners if ClamAV detects (faster response) +MALWARE_SCAN_RESUME_SCAN_ON_FIRST_HIT=no # WARNING: DOS risk! Continue scanning after detection +MALWARE_SCAN_BLOCK_ON_FILENAME_INJECTION_ATTACK=yes # Block on filename injection (path traversal, null bytes) +MALWARE_SCAN_SCAN_FULL_MULTIPART_BODY=no # Scan full multipart body (not recommended - random boundaries prevent caching) +MALWARE_SCAN_MAX_FILES_PER_REQUEST=100 # Maximum number of files per multipart request (DoS protection) + +# ============================================================================ +# VIRUSTOTAL (70+ AV ENGINES + ML) +# ============================================================================ + +MALWARE_SCAN_API_USE_VIRUSTOTAL=no # Enable VirusTotal scanning +MALWARE_SCAN_API_VIRUSTOTAL_KEY= # VirusTotal API key (REQUIRED) +MALWARE_SCAN_API_VIRUSTOTAL_SUSPICIOUS=5 # Suspicious if N+ engines flag +MALWARE_SCAN_API_VIRUSTOTAL_MALICIOUS=3 # Malicious if N+ engines flag +MALWARE_SCAN_API_VIRUSTOTAL_UPLOAD_UNKNOWN=no # Upload unknown malware to VirusTotal +MALWARE_SCAN_API_VIRUSTOTAL_AUTO_VOTE=yes # Auto-vote on uploaded files +MALWARE_SCAN_API_VIRUSTOTAL_MAX_UPLOAD_SIZE=33554432 # Max file size for VT upload (32MB) + +# ============================================================================ +# SENTINELONE (ENTERPRISE THREAT INTELLIGENCE) +# ============================================================================ + +MALWARE_SCAN_API_USE_SENTINELONE=no # Enable SentinelOne scanning +MALWARE_SCAN_API_SENTINELONE_URL= # SentinelOne console URL (REQUIRED) +MALWARE_SCAN_API_SENTINELONE_TOKEN= # SentinelOne API token (REQUIRED) + +# ============================================================================ +# THREAT INTELLIGENCE SHARING +# ============================================================================ + +MALWARE_SCAN_SHARE_FILES=no # Share detected files with VirusTotal (privacy/legal implications) + +# ============================================================================ +# WEBHOOK NOTIFICATIONS +# ============================================================================ + +MALWARE_SCAN_TEAMS_WEBHOOK_URL= # Microsoft Teams webhook URL +MALWARE_SCAN_TEAMS_WEBHOOK_RATE_LIMIT=3 # Teams notifications per minute (max: 200) +MALWARE_SCAN_DISCORD_WEBHOOK_URL= # Discord webhook URL +MALWARE_SCAN_DISCORD_WEBHOOK_RATE_LIMIT=3 # Discord notifications per minute (max: 30) +MALWARE_SCAN_ANONYMIZE_WEBHOOK_IPS=yes # Anonymize client IPs in webhook notifications (GDPR compliance) +# MALWARE_SCAN_UNMASK_KEYS=I_WANT_TO_LEAK_MY_SECRETS_TO_LOG # Show full URLs in logs (SECURITY RISK - debugging only) + +# ============================================================================ +# ATTACKER TRACKING +# ============================================================================ + +MALWARE_SCAN_TRACK_ATTACKERS=yes # Track IPs that upload malware +MALWARE_SCAN_TRACK_ATTACKER_TTL=7776000 # Track duration (90 days) + +# ============================================================================ +# CACHING +# ============================================================================ + +MALWARE_SCAN_USE_SHARED_DATABASE=yes # Enable Redis shared cache for clusters +MALWARE_SCAN_CACHE_VIRUS_TTL=7776000 # Local cache TTL for malicious results (90 days) +MALWARE_SCAN_CACHE_CLEAN_TTL=300 # Local cache TTL for clean results (5 minutes) +MALWARE_SCAN_CACHE_SHARED_TTL=21600 # Redis cache TTL (6 hours) +MALWARE_SCAN_REDIS_CIRCUIT_BREAKER_MAX_BACKOFF=8 # Redis circuit breaker max backoff (seconds) + +# Advanced cache configuration (most users don't need to change these) +MALWARE_SCAN_CACHE_ERROR_TTL=60 # TTL for API error caching (seconds) +MALWARE_SCAN_CACHE_LOCK_TTL=60 # Lock TTL for concurrent scan protection (seconds) +MALWARE_SCAN_CACHE_WAIT_TIMEOUT=30000 # Wait timeout for lock release (milliseconds) +MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_ALL=no # Clear ALL caches on restart (master switch) +MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_CLAMAV=no # Clear ClamAV cache (useful after signature updates) +MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_VIRUSTOTAL=no # Clear VirusTotal cache +MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_SENTINELONE=no # Clear SentinelOne cache +MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_ATTACKER_IPV4=no # Clear IPv4 attacker tracking cache on restart +MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_ATTACKER_IPV6=no # Clear IPv6 attacker tracking cache on restart +MALWARE_SCAN_CACHE_CLEANUP_ON_RESTART_MALWAREBAZAAR=no # Clear MalwareBazaar hash database on restart + +# ============================================================================ +# MALWAREBAZAAR HASH DATABASE +# ============================================================================ + +MALWARE_SCAN_MALWAREBAZAAR_HASH_LOOKUP=yes # Enable MalwareBazaar hash lookup (1M+ known malware hashes) +MALWARE_SCAN_MALWAREBAZAAR_AUTO_UPDATE=yes # Auto-update MalwareBazaar hash database from URL +MALWARE_SCAN_MALWAREBAZAAR_UPDATE_INTERVAL=86400 # Update interval in seconds (86400 = 24 hours) +MALWARE_SCAN_MALWAREBAZAAR_CSV_PATH=/var/lib/bunkerweb/malwarebazaar_hashes.csv # Local path for MalwareBazaar hash database + +# Advanced: Most users don't need to customize these URLs (used by scheduler jobs) +MALWARE_SCAN_MALWAREBAZAAR_UPDATE_URL_FULL=https://bazaar.sysangels.ai/bunkerweb/malwarebazaar_full_split.zip +MALWARE_SCAN_MALWAREBAZAAR_UPDATE_URL_RECENT=https://bazaar.sysangels.ai/bunkerweb/malwarebazaar_recent_split.zip + +# ============================================================================ +# CUSTOM HASHES (ADVANCED) +# ============================================================================ + +MALWARE_SCAN_CUSTOM_HASHES_ENABLED=no # Enable custom hash database +MALWARE_SCAN_CUSTOM_HASHES_AUTO_UPDATE=no # Auto-update custom hashes from URL +MALWARE_SCAN_CUSTOM_HASHES_UPDATE_URL= # URL to download custom hashes CSV +MALWARE_SCAN_CUSTOM_HASHES_CSV_PATH=/etc/bunkerweb/plugins/malware-scan/custom_hashes.csv # Local CSV path + +# ============================================================================ +# FILE CLEANUP +# ============================================================================ + +MALWARE_SCAN_CLEANUP_FILES=yes # Auto-cleanup detected malware files +MALWARE_SCAN_CLEANUP_ORPHANED=yes # Cleanup orphaned temp files (workers only) +MALWARE_SCAN_CLEANUP_AGE=3600 # Cleanup files older than N seconds (1 hour) + +# ============================================================================ +# MEMORY UPLOADS +# ============================================================================ + +MALWARE_SCAN_BLOCK_MEMORY_UPLOADS=yes # Block uploads buffered in memory (can't be scanned) + +# ============================================================================ +# FORENSICS & SAMPLE COLLECTION +# ============================================================================ + +MALWARE_SCAN_VIRUSFILE_LOCAL_EXPORT_PATH= # Path to export detected malware samples (for forensic analysis) + +# ============================================================================ +# NOTES +# ============================================================================ + +# - Replace empty values with actual API keys/URLs before use +# - For clusters: Deploy plugin files to ALL nodes (manager + workers) +# - SECURITY: Store this file as /etc/bunkerweb/variables.env with chmod 600 +# - Allow list (v0.4.1+): Use in-memory caching for 50-200x faster lookups +# - Cleanup runs on workers only (they handle file uploads) +# - See all-features-enabled.md for detailed documentation