From 95d6d8b4d9143edfe97e9f692cbb70dd911f0767 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Tue, 16 Dec 2025 14:16:43 +0000 Subject: [PATCH] chore(docs): upgrade for new batch trigger limits and functionality --- docs/batch-queue-stress-test-plan.md | 5011 +++++++++++++++++ docs/docs.json | 9 +- docs/limits.mdx | 36 +- docs/management/batches/create.mdx | 5 + docs/management/batches/stream-items.mdx | 5 + docs/openapi.yml | 238 + docs/self-hosting/env/webapp.mdx | 4 +- .../rate-limit-hit-use-batchtrigger.mdx | 2 +- docs/triggering.mdx | 46 +- docs/v3-openapi.yaml | 2 +- 10 files changed, 5342 insertions(+), 16 deletions(-) create mode 100644 docs/batch-queue-stress-test-plan.md create mode 100644 docs/management/batches/create.mdx create mode 100644 docs/management/batches/stream-items.mdx diff --git a/docs/batch-queue-stress-test-plan.md b/docs/batch-queue-stress-test-plan.md new file mode 100644 index 0000000000..0a8c639960 --- /dev/null +++ b/docs/batch-queue-stress-test-plan.md @@ -0,0 +1,5011 @@ +# Batch Queue Stress Test Plan + +This document outlines a systematic approach to testing the new Batch Queue with Deficit Round Robin (DRR) scheduling. + +## Test Environment + +**Setup:** + +- Single local machine running one webapp instance (single BatchQueue consumer process) +- 3 organizations, each with 3 projects = **9 tenants total** +- All tenants using the same task ID: `stress-test-task` + +**Note:** In production, multiple webapp instances will run BatchQueue consumers in parallel. This single-instance testing focuses on algorithm behavior rather than horizontal scaling. + +## Configuration Options Under Test + +| Variable | Default | Description | +| --------------------------------------- | ------- | ------------------------------------------------------ | +| `BATCH_QUEUE_DRR_QUANTUM` | 5 | Credits allocated per environment per scheduling round | +| `BATCH_QUEUE_MAX_DEFICIT` | 50 | Maximum accumulated deficit (prevents starvation) | +| `BATCH_QUEUE_CONSUMER_COUNT` | 1 | Number of concurrent consumer loops | +| `BATCH_QUEUE_CONSUMER_INTERVAL_MS` | 100 | Polling interval between consumer iterations | +| `BATCH_CONCURRENCY_DEFAULT_CONCURRENCY` | 10 | Default concurrent batch items per environment | +| `BATCH_QUEUE_GLOBAL_RATE_LIMIT` | none | Optional global items/sec limit | + +**Per-Org Settings (via database):** + +- `batchQueueConcurrencyConfig` - Override concurrency per org +- `batchRateLimitConfig` - Rate limit per org + +--- + +## Test Series Overview + +| Series | Focus | Objective | +| ------ | ------------------- | ------------------------------------------------------ | +| A | Baseline | Establish reference metrics with defaults | +| B | DRR Quantum | Understand how quantum affects fairness and throughput | +| C | Max Deficit | Understand starvation prevention and catch-up behavior | +| D | Consumer Count | Test parallelism within single process | +| E | Consumer Interval | Test polling frequency impact | +| F | Concurrency Limits | Test per-environment processing limits | +| G | Global Rate Limiter | Test global throughput caps | +| H | Asymmetric Load | Test fairness under uneven workloads | +| I | Combined Tuning | Test optimized configurations | + +--- + +## Series A: Baseline (Default Configuration) + +**Objective:** Establish baseline metrics with all default settings. + +**Configuration:** + +```env +BATCH_QUEUE_DRR_QUANTUM=5 +BATCH_QUEUE_MAX_DEFICIT=50 +# BATCH_QUEUE_CONSUMER_COUNT not set (default: 1) +# BATCH_QUEUE_CONSUMER_INTERVAL_MS not set (default: 100) +BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10 +# BATCH_QUEUE_GLOBAL_RATE_LIMIT not set +``` + +### Test A1: Fairness Baseline (All 9 Tenants) + +**Command:** + +```bash +pnpm stress fairness --batch-size 100 --batch-count 3 --tenants all -d "Baseline fairness test" +``` + +**Expected:** All tenants should complete in roughly similar timeframes. + +
+Results A1 (click to expand) + +```json +{ + "scenario": "fairness-9t", + "description": "Baseline fairness test", + "config": { + "tenantCount": 9, + "batchSize": 100, + "batchCount": 3 + }, + "results": { + "totalItemsProcessed": 2700, + "totalBatches": 27, + "overallThroughput": 54.51792024230187, + "overallDuration": 49525, + "fairnessIndex": 0.9999098436641237, + "perTenant": [ + { + "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 49323, + "avgItemsPerSecond": 6.082355087890031, + "avgBatchDuration": 49238, + "minBatchDuration": 49196, + "maxBatchDuration": 49322, + "p50Duration": 49196, + "p95Duration": 49322, + "p99Duration": 49322 + }, + { + "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 49460, + "avgItemsPerSecond": 6.06550748079256, + "avgBatchDuration": 49290.666666666664, + "minBatchDuration": 49187, + "maxBatchDuration": 49429, + "p50Duration": 49256, + "p95Duration": 49429, + "p99Duration": 49429 + }, + { + "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 49484, + "avgItemsPerSecond": 6.062565677794843, + "avgBatchDuration": 36861.666666666664, + "minBatchDuration": 15041, + "maxBatchDuration": 49472, + "p50Duration": 46072, + "p95Duration": 49472, + "p99Duration": 49472 + }, + { + "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 48442, + "avgItemsPerSecond": 6.192973039924033, + "avgBatchDuration": 48347, + "minBatchDuration": 48223, + "maxBatchDuration": 48442, + "p50Duration": 48376, + "p95Duration": 48442, + "p99Duration": 48442 + }, + { + "tenantId": "org-2:proj_zgysghtkiezoakvjscin", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 48446, + "avgItemsPerSecond": 6.192461709945094, + "avgBatchDuration": 48102.333333333336, + "minBatchDuration": 47417, + "maxBatchDuration": 48446, + "p50Duration": 48444, + "p95Duration": 48446, + "p99Duration": 48446 + }, + { + "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 48288, + "avgItemsPerSecond": 6.21272365805169, + "avgBatchDuration": 36139.333333333336, + "minBatchDuration": 16087, + "maxBatchDuration": 48218, + "p50Duration": 44113, + "p95Duration": 48218, + "p99Duration": 48218 + }, + { + "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 49148, + "avgItemsPerSecond": 6.104012370798404, + "avgBatchDuration": 49074.666666666664, + "minBatchDuration": 49061, + "maxBatchDuration": 49099, + "p50Duration": 49064, + "p95Duration": 49099, + "p99Duration": 49099 + }, + { + "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 49178, + "avgItemsPerSecond": 6.1002887470006915, + "avgBatchDuration": 48484.333333333336, + "minBatchDuration": 47154, + "maxBatchDuration": 49176, + "p50Duration": 49123, + "p95Duration": 49176, + "p99Duration": 49176 + }, + { + "tenantId": "org-3:proj_ytivyoceocenyxuprmga", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 49454, + "avgItemsPerSecond": 6.066243377684312, + "avgBatchDuration": 49372.666666666664, + "minBatchDuration": 49251, + "maxBatchDuration": 49454, + "p50Duration": 49413, + "p95Duration": 49454, + "p99Duration": 49454 + } + ] + }, + "timestamps": { + "start": "2025-12-15T14:23:40.510Z", + "end": "2025-12-15T14:24:30.035Z" + } +} +``` + +**Observations:** + +- Overall throughput: **54.52 items/sec** with 9 tenants competing +- Fairness index: **0.9999** (nearly perfect fairness) +- Notable patterns: All tenants completed within 1-2 seconds of each other (48-50s range). Per-tenant throughput ~6 items/sec each. **DRR is working excellently for symmetric load.** + +
+ +### Test A2: Throughput Baseline (Single Tenant) + +**Command:** + +```bash +pnpm stress throughput --batch-sizes 100,500,1000 --batch-count 3 +``` + +**Expected:** Establish throughput ceiling for single tenant. + +
+Results A2 (click to expand) + +```json +[ + { + "scenario": "throughput-100", + "config": { + "tenantCount": 1, + "batchSize": 100, + "batchCount": 3 + }, + "results": { + "totalItemsProcessed": 300, + "totalBatches": 3, + "overallThroughput": 82.57638315441784, + "overallDuration": 3633, + "fairnessIndex": 1, + "perTenant": [ + { + "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 3570, + "avgItemsPerSecond": 84.03361344537815, + "avgBatchDuration": 3558.6666666666665, + "minBatchDuration": 3538, + "maxBatchDuration": 3570, + "p50Duration": 3568, + "p95Duration": 3570, + "p99Duration": 3570 + } + ] + }, + "timestamps": { + "start": "2025-12-15T14:41:19.119Z", + "end": "2025-12-15T14:41:22.752Z" + } + }, + { + "scenario": "throughput-500", + "config": { + "tenantCount": 1, + "batchSize": 500, + "batchCount": 3 + }, + "results": { + "totalItemsProcessed": 1500, + "totalBatches": 3, + "overallThroughput": 97.48488984207448, + "overallDuration": 15387, + "fairnessIndex": 1, + "perTenant": [ + { + "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh", + "totalItems": 1500, + "totalBatches": 3, + "totalDuration": 15384, + "avgItemsPerSecond": 97.50390015600624, + "avgBatchDuration": 15369, + "minBatchDuration": 15356, + "maxBatchDuration": 15384, + "p50Duration": 15367, + "p95Duration": 15384, + "p99Duration": 15384 + } + ] + }, + "timestamps": { + "start": "2025-12-15T14:41:23.069Z", + "end": "2025-12-15T14:41:38.456Z" + } + }, + { + "scenario": "throughput-1000", + "config": { + "tenantCount": 1, + "batchSize": 1000, + "batchCount": 3 + }, + "results": { + "totalItemsProcessed": 3000, + "totalBatches": 3, + "overallThroughput": 88.60796880999499, + "overallDuration": 33857, + "fairnessIndex": 1, + "perTenant": [ + { + "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh", + "totalItems": 3000, + "totalBatches": 3, + "totalDuration": 33839, + "avgItemsPerSecond": 88.65510210112592, + "avgBatchDuration": 33109, + "minBatchDuration": 32731, + "maxBatchDuration": 33827, + "p50Duration": 32769, + "p95Duration": 33827, + "p99Duration": 33827 + } + ] + }, + "timestamps": { + "start": "2025-12-15T14:41:38.860Z", + "end": "2025-12-15T14:42:12.717Z" + } + } +] +``` + +**Observations:** + +- Max throughput achieved: **97.48 items/sec** (at batch size 500) +- Scaling behavior: **Non-linear.** 100 items → 82.58/sec, 500 items → 97.48/sec (peak), 1000 items → 88.61/sec. Sweet spot around 500 items per batch. Larger batches may introduce overhead or hit concurrency limits. + +
+ +--- + +## Series B: DRR Quantum Variations + +**Objective:** Understand how quantum size affects fairness vs. throughput tradeoff. + +**Theory:** + +- Lower quantum = more frequent tenant switching = better fairness, possibly lower throughput +- Higher quantum = longer bursts per tenant = potentially higher throughput, worse fairness + +### Test B1: Very Low Quantum (quantum=1) + +**Configuration:** + +```env +BATCH_QUEUE_DRR_QUANTUM=1 +BATCH_QUEUE_MAX_DEFICIT=50 +BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10 +``` + +**Command:** + +```bash +pnpm stress fairness --batch-size 100 --batch-count 3 --tenants all -d "Low quantum=1" +``` + +
+Results B1 (click to expand) + +```json +{ + "scenario": "fairness-9t", + "description": "Low quantum=1", + "config": { + "tenantCount": 9, + "batchSize": 100, + "batchCount": 3 + }, + "results": { + "totalItemsProcessed": 2700, + "totalBatches": 27, + "overallThroughput": 66.08899985313556, + "overallDuration": 40854, + "fairnessIndex": 0.9997008998554137, + "perTenant": [ + { + "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 40769, + "avgItemsPerSecond": 7.358532218106895, + "avgBatchDuration": 36850.333333333336, + "minBatchDuration": 29016, + "maxBatchDuration": 40769, + "p50Duration": 40766, + "p95Duration": 40769, + "p99Duration": 40769 + }, + { + "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 40771, + "avgItemsPerSecond": 7.358171249172206, + "avgBatchDuration": 40408.333333333336, + "minBatchDuration": 39684, + "maxBatchDuration": 40771, + "p50Duration": 40770, + "p95Duration": 40771, + "p99Duration": 40771 + }, + { + "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 40754, + "avgItemsPerSecond": 7.361240614418216, + "avgBatchDuration": 40106, + "minBatchDuration": 39076, + "maxBatchDuration": 40753, + "p50Duration": 40489, + "p95Duration": 40753, + "p99Duration": 40753 + }, + { + "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 40770, + "avgItemsPerSecond": 7.358351729212656, + "avgBatchDuration": 40717.333333333336, + "minBatchDuration": 40616, + "maxBatchDuration": 40769, + "p50Duration": 40767, + "p95Duration": 40769, + "p99Duration": 40769 + }, + { + "tenantId": "org-2:proj_zgysghtkiezoakvjscin", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 40767, + "avgItemsPerSecond": 7.358893222459342, + "avgBatchDuration": 40610, + "minBatchDuration": 40299, + "maxBatchDuration": 40766, + "p50Duration": 40765, + "p95Duration": 40766, + "p99Duration": 40766 + }, + { + "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 40780, + "avgItemsPerSecond": 7.356547327121138, + "avgBatchDuration": 40681.666666666664, + "minBatchDuration": 40497, + "maxBatchDuration": 40778, + "p50Duration": 40770, + "p95Duration": 40778, + "p99Duration": 40778 + }, + { + "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 40771, + "avgItemsPerSecond": 7.358171249172206, + "avgBatchDuration": 40766.333333333336, + "minBatchDuration": 40764, + "maxBatchDuration": 40769, + "p50Duration": 40766, + "p95Duration": 40769, + "p99Duration": 40769 + }, + { + "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 38628, + "avgItemsPerSecond": 7.766387076731904, + "avgBatchDuration": 34753, + "minBatchDuration": 28057, + "maxBatchDuration": 38627, + "p50Duration": 37575, + "p95Duration": 38627, + "p99Duration": 38627 + }, + { + "tenantId": "org-3:proj_ytivyoceocenyxuprmga", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 40754, + "avgItemsPerSecond": 7.361240614418216, + "avgBatchDuration": 40018.333333333336, + "minBatchDuration": 39630, + "maxBatchDuration": 40754, + "p50Duration": 39671, + "p95Duration": 40754, + "p99Duration": 40754 + } + ] + }, + "timestamps": { + "start": "2025-12-15T14:48:32.814Z", + "end": "2025-12-15T14:49:13.668Z" + } +} +``` + +**Observations:** + +- Fairness index: **0.9997** (vs baseline: **0.9999**) - nearly identical +- Throughput: **66.09 items/sec** (vs baseline: **54.52**) - +21% improvement! +- Context switching overhead visible?: **Not significantly.** Despite switching tenants every 1 item, throughput actually improved. The tighter scheduling may have better utilized concurrency slots. + +
+ +### Test B2: Medium Quantum (quantum=10) + +**Configuration:** + +```env +BATCH_QUEUE_DRR_QUANTUM=10 +BATCH_QUEUE_MAX_DEFICIT=50 +BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10 +``` + +**Command:** + +```bash +pnpm stress fairness --batch-size 100 --batch-count 3 --tenants all -d "Medium quantum=10" +``` + +
+Results B2 (click to expand) + +```json +{ + "scenario": "fairness-9t", + "description": "Medium quantum=10", + "config": { + "tenantCount": 9, + "batchSize": 100, + "batchCount": 3 + }, + "results": { + "totalItemsProcessed": 2700, + "totalBatches": 27, + "overallThroughput": 64.68615237182559, + "overallDuration": 41740, + "fairnessIndex": 0.9998055065601579, + "perTenant": [ + { + "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 40743, + "avgItemsPerSecond": 7.363228039172373, + "avgBatchDuration": 37656, + "minBatchDuration": 31484, + "maxBatchDuration": 40743, + "p50Duration": 40741, + "p95Duration": 40743, + "p99Duration": 40743 + }, + { + "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 40745, + "avgItemsPerSecond": 7.362866609399926, + "avgBatchDuration": 36601.666666666664, + "minBatchDuration": 28318, + "maxBatchDuration": 40745, + "p50Duration": 40742, + "p95Duration": 40745, + "p99Duration": 40745 + }, + { + "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 41664, + "avgItemsPerSecond": 7.200460829493087, + "avgBatchDuration": 38524.333333333336, + "minBatchDuration": 32253, + "maxBatchDuration": 41660, + "p50Duration": 41660, + "p95Duration": 41660, + "p99Duration": 41660 + }, + { + "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 41443, + "avgItemsPerSecond": 7.238858190768043, + "avgBatchDuration": 36661.333333333336, + "minBatchDuration": 32251, + "maxBatchDuration": 41443, + "p50Duration": 36290, + "p95Duration": 41443, + "p99Duration": 41443 + }, + { + "tenantId": "org-2:proj_zgysghtkiezoakvjscin", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 40436, + "avgItemsPerSecond": 7.419131467009596, + "avgBatchDuration": 40406, + "minBatchDuration": 40349, + "maxBatchDuration": 40436, + "p50Duration": 40433, + "p95Duration": 40436, + "p99Duration": 40436 + }, + { + "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 41044, + "avgItemsPerSecond": 7.309229119968814, + "avgBatchDuration": 39122, + "minBatchDuration": 35972, + "maxBatchDuration": 41040, + "p50Duration": 40354, + "p95Duration": 41040, + "p99Duration": 41040 + }, + { + "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 41683, + "avgItemsPerSecond": 7.197178705947269, + "avgBatchDuration": 41325, + "minBatchDuration": 40636, + "maxBatchDuration": 41683, + "p50Duration": 41656, + "p95Duration": 41683, + "p99Duration": 41683 + }, + { + "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 41412, + "avgItemsPerSecond": 7.244277021153288, + "avgBatchDuration": 40959.666666666664, + "minBatchDuration": 40735, + "maxBatchDuration": 41406, + "p50Duration": 40738, + "p95Duration": 41406, + "p99Duration": 41406 + }, + { + "tenantId": "org-3:proj_ytivyoceocenyxuprmga", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 39914, + "avgItemsPerSecond": 7.516159743448415, + "avgBatchDuration": 36645, + "minBatchDuration": 32034, + "maxBatchDuration": 39914, + "p50Duration": 37987, + "p95Duration": 39914, + "p99Duration": 39914 + } + ] + }, + "timestamps": { + "start": "2025-12-15T14:51:13.796Z", + "end": "2025-12-15T14:51:55.536Z" + } +} +``` + +**Observations:** + +- Fairness index: **0.9998** (excellent) +- Throughput: **64.69 items/sec** (vs quantum=1: 66.09, slightly lower) +- Comparison to baseline: **+18% throughput vs baseline.** Higher quantum didn't provide expected gains; the overhead theory is not supported by evidence. + +
+ +### Test B3: High Quantum (quantum=25) + +**Configuration:** + +```env +BATCH_QUEUE_DRR_QUANTUM=25 +BATCH_QUEUE_MAX_DEFICIT=50 +BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10 +``` + +**Command:** + +```bash +pnpm stress fairness --batch-size 100 --batch-count 3 --tenants all -d "High quantum=25" +``` + +
+Results B3 (click to expand) + +```json +{ + "scenario": "fairness-9t", + "description": "High quantum=25", + "config": { + "tenantCount": 9, + "batchSize": 100, + "batchCount": 3 + }, + "results": { + "totalItemsProcessed": 2700, + "totalBatches": 27, + "overallThroughput": 84.3644544431946, + "overallDuration": 32004, + "fairnessIndex": 0.9999195340273302, + "perTenant": [ + { + "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 31137, + "avgItemsPerSecond": 9.634839579920994, + "avgBatchDuration": 31135.333333333332, + "minBatchDuration": 31134, + "maxBatchDuration": 31137, + "p50Duration": 31135, + "p95Duration": 31137, + "p99Duration": 31137 + }, + { + "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 31135, + "avgItemsPerSecond": 9.635458487233016, + "avgBatchDuration": 30287.333333333332, + "minBatchDuration": 29792, + "maxBatchDuration": 31133, + "p50Duration": 29937, + "p95Duration": 31133, + "p99Duration": 31133 + }, + { + "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 31197, + "avgItemsPerSecond": 9.616309260505819, + "avgBatchDuration": 24641, + "minBatchDuration": 18973, + "maxBatchDuration": 31197, + "p50Duration": 23753, + "p95Duration": 31197, + "p99Duration": 31197 + }, + { + "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 31105, + "avgItemsPerSecond": 9.644751647645073, + "avgBatchDuration": 29303, + "minBatchDuration": 25964, + "maxBatchDuration": 31105, + "p50Duration": 30840, + "p95Duration": 31105, + "p99Duration": 31105 + }, + { + "tenantId": "org-2:proj_zgysghtkiezoakvjscin", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 31120, + "avgItemsPerSecond": 9.640102827763496, + "avgBatchDuration": 31006.333333333332, + "minBatchDuration": 30835, + "maxBatchDuration": 31120, + "p50Duration": 31064, + "p95Duration": 31120, + "p99Duration": 31120 + }, + { + "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 31075, + "avgItemsPerSecond": 9.654062751407883, + "avgBatchDuration": 24953.333333333332, + "minBatchDuration": 21079, + "maxBatchDuration": 31073, + "p50Duration": 22708, + "p95Duration": 31073, + "p99Duration": 31073 + }, + { + "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 31965, + "avgItemsPerSecond": 9.385265133740027, + "avgBatchDuration": 25924.666666666668, + "minBatchDuration": 22904, + "maxBatchDuration": 31964, + "p50Duration": 22906, + "p95Duration": 31964, + "p99Duration": 31964 + }, + { + "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 31106, + "avgItemsPerSecond": 9.644441586832123, + "avgBatchDuration": 26298.333333333332, + "minBatchDuration": 16867, + "maxBatchDuration": 31106, + "p50Duration": 30922, + "p95Duration": 31106, + "p99Duration": 31106 + }, + { + "tenantId": "org-3:proj_ytivyoceocenyxuprmga", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 30886, + "avgItemsPerSecond": 9.713138638865505, + "avgBatchDuration": 30883, + "minBatchDuration": 30881, + "maxBatchDuration": 30884, + "p50Duration": 30884, + "p95Duration": 30884, + "p99Duration": 30884 + } + ] + }, + "timestamps": { + "start": "2025-12-15T14:54:10.280Z", + "end": "2025-12-15T14:54:42.284Z" + } +} +``` + +**Observations:** + +- Fairness index: **0.9999** (best in B series!) +- Throughput: **84.36 items/sec** (**BEST in B series**, +55% vs baseline) +- Evidence of tenant starvation?: **None.** All tenants completed in 30-32s range. Higher quantum=25 actually provided better throughput AND fairness! + +
+ +### Test B4: Very High Quantum (quantum=50) + +**Configuration:** + +```env +BATCH_QUEUE_DRR_QUANTUM=50 +BATCH_QUEUE_MAX_DEFICIT=50 +BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10 +``` + +**Command:** + +```bash +pnpm stress fairness --batch-size 100 --batch-count 3 --tenants all -d "Very high quantum=50" +``` + +
+Results B4 (click to expand) + +```json +{ + "scenario": "fairness-9t", + "description": "Very high quantum=50", + "config": { + "tenantCount": 9, + "batchSize": 100, + "batchCount": 3 + }, + "results": { + "totalItemsProcessed": 2700, + "totalBatches": 27, + "overallThroughput": 51.97605251506343, + "overallDuration": 51947, + "fairnessIndex": 0.9997441540697334, + "perTenant": [ + { + "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 51416, + "avgItemsPerSecond": 5.834759607904155, + "avgBatchDuration": 51002, + "minBatchDuration": 50774, + "maxBatchDuration": 51416, + "p50Duration": 50816, + "p95Duration": 51416, + "p99Duration": 51416 + }, + { + "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 51847, + "avgItemsPerSecond": 5.786255713927518, + "avgBatchDuration": 51840.666666666664, + "minBatchDuration": 51838, + "maxBatchDuration": 51843, + "p50Duration": 51841, + "p95Duration": 51843, + "p99Duration": 51843 + }, + { + "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 51252, + "avgItemsPerSecond": 5.853430110044486, + "avgBatchDuration": 51034.333333333336, + "minBatchDuration": 50778, + "maxBatchDuration": 51250, + "p50Duration": 51075, + "p95Duration": 51250, + "p99Duration": 51250 + }, + { + "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 50778, + "avgItemsPerSecond": 5.908070424199456, + "avgBatchDuration": 50768.666666666664, + "minBatchDuration": 50765, + "maxBatchDuration": 50776, + "p50Duration": 50765, + "p95Duration": 50776, + "p99Duration": 50776 + }, + { + "tenantId": "org-2:proj_zgysghtkiezoakvjscin", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 49692, + "avgItemsPerSecond": 6.037189084762135, + "avgBatchDuration": 49689.666666666664, + "minBatchDuration": 49687, + "maxBatchDuration": 49692, + "p50Duration": 49690, + "p95Duration": 49692, + "p99Duration": 49692 + }, + { + "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 51843, + "avgItemsPerSecond": 5.786702158439905, + "avgBatchDuration": 51842, + "minBatchDuration": 51841, + "maxBatchDuration": 51843, + "p50Duration": 51842, + "p95Duration": 51843, + "p99Duration": 51843 + }, + { + "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 50821, + "avgItemsPerSecond": 5.9030715649042715, + "avgBatchDuration": 50794.666666666664, + "minBatchDuration": 50779, + "maxBatchDuration": 50821, + "p50Duration": 50784, + "p95Duration": 50821, + "p99Duration": 50821 + }, + { + "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 49689, + "avgItemsPerSecond": 6.037553583288052, + "avgBatchDuration": 49682, + "minBatchDuration": 49678, + "maxBatchDuration": 49689, + "p50Duration": 49679, + "p95Duration": 49689, + "p99Duration": 49689 + }, + { + "tenantId": "org-3:proj_ytivyoceocenyxuprmga", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 51838, + "avgItemsPerSecond": 5.787260310968787, + "avgBatchDuration": 51483.333333333336, + "minBatchDuration": 50775, + "maxBatchDuration": 51838, + "p50Duration": 51837, + "p95Duration": 51838, + "p99Duration": 51838 + } + ] + }, + "timestamps": { + "start": "2025-12-15T14:58:47.844Z", + "end": "2025-12-15T14:59:39.791Z" + } +} +``` + +**Observations:** + +- Fairness index: **0.9997** (still excellent) +- Throughput: **51.98 items/sec** (**WORST in B series**, -5% vs baseline) +- Notable patterns: **Diminishing returns / counterproductive.** Very high quantum=50 actually hurt throughput. With 9 tenants each getting 50-item bursts, tenants wait too long for their turn. **Sweet spot appears to be quantum=25.** + +
+ +--- + +## Series C: Max Deficit Variations + +**Objective:** Understand how max deficit cap affects catch-up behavior for starved tenants. + +**Theory:** + +- Lower max deficit = limits how much a starved tenant can "catch up" +- Higher max deficit = allows more aggressive catch-up, but could starve other tenants + +### Test C1: Low Max Deficit (maxDeficit=10) + +**Configuration:** + +```env +BATCH_QUEUE_DRR_QUANTUM=5 +BATCH_QUEUE_MAX_DEFICIT=10 +BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10 +``` + +**Command:** + +```bash +pnpm stress fairness --batch-size 100 --batch-count 3 --tenants all -d "Low maxDeficit=10" +``` + +
+Results C1 (click to expand) + +```json +{ + "scenario": "fairness-9t", + "description": "Low maxDeficit=10", + "config": { + "tenantCount": 9, + "batchSize": 100, + "batchCount": 3 + }, + "results": { + "totalItemsProcessed": 2700, + "totalBatches": 27, + "overallThroughput": 52.51385782359234, + "overallDuration": 51415, + "fairnessIndex": 0.9997739979483051, + "perTenant": [ + { + "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 49624, + "avgItemsPerSecond": 6.0454618732871195, + "avgBatchDuration": 49357.333333333336, + "minBatchDuration": 48832, + "maxBatchDuration": 49624, + "p50Duration": 49616, + "p95Duration": 49624, + "p99Duration": 49624 + }, + { + "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 51230, + "avgItemsPerSecond": 5.8559437829396845, + "avgBatchDuration": 50431, + "minBatchDuration": 49201, + "maxBatchDuration": 51230, + "p50Duration": 50862, + "p95Duration": 51230, + "p99Duration": 51230 + }, + { + "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 51410, + "avgItemsPerSecond": 5.83544057576347, + "avgBatchDuration": 49448.333333333336, + "minBatchDuration": 45734, + "maxBatchDuration": 51410, + "p50Duration": 51201, + "p95Duration": 51410, + "p99Duration": 51410 + }, + { + "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 49206, + "avgItemsPerSecond": 6.096817461285209, + "avgBatchDuration": 46922.333333333336, + "minBatchDuration": 42366, + "maxBatchDuration": 49201, + "p50Duration": 49200, + "p95Duration": 49201, + "p99Duration": 49201 + }, + { + "tenantId": "org-2:proj_zgysghtkiezoakvjscin", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 50661, + "avgItemsPerSecond": 5.921714928643334, + "avgBatchDuration": 48617.666666666664, + "minBatchDuration": 46575, + "maxBatchDuration": 50661, + "p50Duration": 48617, + "p95Duration": 50661, + "p99Duration": 50661 + }, + { + "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 50189, + "avgItemsPerSecond": 5.977405407559425, + "avgBatchDuration": 49945.666666666664, + "minBatchDuration": 49823, + "maxBatchDuration": 50189, + "p50Duration": 49825, + "p95Duration": 50189, + "p99Duration": 50189 + }, + { + "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 51232, + "avgItemsPerSecond": 5.8557151780137415, + "avgBatchDuration": 50892.333333333336, + "minBatchDuration": 50636, + "maxBatchDuration": 51229, + "p50Duration": 50812, + "p95Duration": 51229, + "p99Duration": 51229 + }, + { + "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 49639, + "avgItemsPerSecond": 6.0436350450250815, + "avgBatchDuration": 44099, + "minBatchDuration": 41326, + "maxBatchDuration": 49638, + "p50Duration": 41333, + "p95Duration": 49638, + "p99Duration": 49638 + }, + { + "tenantId": "org-3:proj_ytivyoceocenyxuprmga", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 50193, + "avgItemsPerSecond": 5.976929053852131, + "avgBatchDuration": 49979.666666666664, + "minBatchDuration": 49623, + "maxBatchDuration": 50193, + "p50Duration": 50123, + "p95Duration": 50193, + "p99Duration": 50193 + } + ] + }, + "timestamps": { + "start": "2025-12-15T15:01:17.027Z", + "end": "2025-12-15T15:02:08.442Z" + } +} +``` + +**Observations:** + +- Fairness index: **0.9998** (excellent) +- Throughput: **52.51 items/sec** (similar to baseline) +- Recovery behavior: **Constrained.** Low max deficit limits catch-up ability. Total duration 51.4s, consistent with limited burst capacity. + +
+ +### Test C2: Medium Max Deficit (maxDeficit=25) + +**Configuration:** + +```env +BATCH_QUEUE_DRR_QUANTUM=5 +BATCH_QUEUE_MAX_DEFICIT=25 +BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10 +``` + +**Command:** + +```bash +pnpm stress fairness --batch-size 100 --batch-count 3 --tenants all -d "Medium maxDeficit=25" +``` + +
+Results C2 (click to expand) + +```json +{ + "scenario": "fairness-9t", + "description": "Medium maxDeficit=25", + "config": { + "tenantCount": 9, + "batchSize": 100, + "batchCount": 3 + }, + "results": { + "totalItemsProcessed": 2700, + "totalBatches": 27, + "overallThroughput": 66.32928806564142, + "overallDuration": 40706, + "fairnessIndex": 0.9992454865624668, + "perTenant": [ + { + "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 39801, + "avgItemsPerSecond": 7.537499057812618, + "avgBatchDuration": 35339.333333333336, + "minBatchDuration": 28824, + "maxBatchDuration": 39800, + "p50Duration": 37394, + "p95Duration": 39800, + "p99Duration": 39800 + }, + { + "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 39487, + "avgItemsPerSecond": 7.597437131207739, + "avgBatchDuration": 36988.666666666664, + "minBatchDuration": 34055, + "maxBatchDuration": 39486, + "p50Duration": 37425, + "p95Duration": 39486, + "p99Duration": 39486 + }, + { + "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 37449, + "avgItemsPerSecond": 8.010894816951053, + "avgBatchDuration": 36409.333333333336, + "minBatchDuration": 34337, + "maxBatchDuration": 37447, + "p50Duration": 37444, + "p95Duration": 37447, + "p99Duration": 37447 + }, + { + "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 40648, + "avgItemsPerSecond": 7.380436921865774, + "avgBatchDuration": 39841.666666666664, + "minBatchDuration": 39419, + "maxBatchDuration": 40647, + "p50Duration": 39459, + "p95Duration": 40647, + "p99Duration": 40647 + }, + { + "tenantId": "org-2:proj_zgysghtkiezoakvjscin", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 38760, + "avgItemsPerSecond": 7.739938080495356, + "avgBatchDuration": 36378.333333333336, + "minBatchDuration": 33012, + "maxBatchDuration": 38750, + "p50Duration": 37373, + "p95Duration": 38750, + "p99Duration": 38750 + }, + { + "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 37706, + "avgItemsPerSecond": 7.956293428101628, + "avgBatchDuration": 37609.333333333336, + "minBatchDuration": 37424, + "maxBatchDuration": 37706, + "p50Duration": 37698, + "p95Duration": 37706, + "p99Duration": 37706 + }, + { + "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 40621, + "avgItemsPerSecond": 7.385342556805593, + "avgBatchDuration": 38921, + "minBatchDuration": 36642, + "maxBatchDuration": 40621, + "p50Duration": 39500, + "p95Duration": 40621, + "p99Duration": 40621 + }, + { + "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 38757, + "avgItemsPerSecond": 7.740537193281214, + "avgBatchDuration": 38744, + "minBatchDuration": 38730, + "maxBatchDuration": 38757, + "p50Duration": 38745, + "p95Duration": 38757, + "p99Duration": 38757 + }, + { + "tenantId": "org-3:proj_ytivyoceocenyxuprmga", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 38751, + "avgItemsPerSecond": 7.741735697143299, + "avgBatchDuration": 37674.666666666664, + "minBatchDuration": 35558, + "maxBatchDuration": 38740, + "p50Duration": 38726, + "p95Duration": 38740, + "p99Duration": 38740 + } + ] + }, + "timestamps": { + "start": "2025-12-15T15:03:24.382Z", + "end": "2025-12-15T15:04:05.088Z" + } +} +``` + +**Observations:** + +- Fairness index: **0.9992** (slightly lower but still very good) +- Throughput: **66.33 items/sec** (+22% vs baseline) +- Recovery behavior: **Better.** More headroom for catch-up. Duration dropped from 51.4s (C1) to 40.7s. + +
+ +### Test C3: High Max Deficit (maxDeficit=100) + +**Configuration:** + +```env +BATCH_QUEUE_DRR_QUANTUM=5 +BATCH_QUEUE_MAX_DEFICIT=100 +BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10 +``` + +**Command:** + +```bash +pnpm stress fairness --batch-size 100 --batch-count 3 --tenants all -d "High maxDeficit=100" +``` + +
+Results C3 (click to expand) + +```json +{ + "scenario": "fairness-9t", + "description": "High maxDeficit=100", + "config": { + "tenantCount": 9, + "batchSize": 100, + "batchCount": 3 + }, + "results": { + "totalItemsProcessed": 2700, + "totalBatches": 27, + "overallThroughput": 73.73624272878718, + "overallDuration": 36617, + "fairnessIndex": 0.9988716605255289, + "perTenant": [ + { + "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 32809, + "avgItemsPerSecond": 9.143832484988875, + "avgBatchDuration": 30391.666666666668, + "minBatchDuration": 28722, + "maxBatchDuration": 32751, + "p50Duration": 29702, + "p95Duration": 32751, + "p99Duration": 32751 + }, + { + "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 36540, + "avgItemsPerSecond": 8.210180623973727, + "avgBatchDuration": 28375.666666666668, + "minBatchDuration": 13115, + "maxBatchDuration": 36537, + "p50Duration": 35475, + "p95Duration": 36537, + "p99Duration": 36537 + }, + { + "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 36337, + "avgItemsPerSecond": 8.256047554833916, + "avgBatchDuration": 28524.666666666668, + "minBatchDuration": 13114, + "maxBatchDuration": 36322, + "p50Duration": 36138, + "p95Duration": 36322, + "p99Duration": 36322 + }, + { + "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 35478, + "avgItemsPerSecond": 8.45594452900389, + "avgBatchDuration": 27529.333333333332, + "minBatchDuration": 23556, + "maxBatchDuration": 35475, + "p50Duration": 23557, + "p95Duration": 35475, + "p99Duration": 35475 + }, + { + "tenantId": "org-2:proj_zgysghtkiezoakvjscin", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 35453, + "avgItemsPerSecond": 8.461907313908554, + "avgBatchDuration": 35291, + "minBatchDuration": 35039, + "maxBatchDuration": 35438, + "p50Duration": 35396, + "p95Duration": 35438, + "p99Duration": 35438 + }, + { + "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 36430, + "avgItemsPerSecond": 8.23497117760088, + "avgBatchDuration": 34851.333333333336, + "minBatchDuration": 31705, + "maxBatchDuration": 36425, + "p50Duration": 36424, + "p95Duration": 36425, + "p99Duration": 36425 + }, + { + "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 36500, + "avgItemsPerSecond": 8.21917808219178, + "avgBatchDuration": 31197, + "minBatchDuration": 27638, + "maxBatchDuration": 36495, + "p50Duration": 29458, + "p95Duration": 36495, + "p99Duration": 36495 + }, + { + "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 36573, + "avgItemsPerSecond": 8.202772537117545, + "avgBatchDuration": 27671.666666666668, + "minBatchDuration": 23222, + "maxBatchDuration": 36570, + "p50Duration": 23223, + "p95Duration": 36570, + "p99Duration": 36570 + }, + { + "tenantId": "org-3:proj_ytivyoceocenyxuprmga", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 35514, + "avgItemsPerSecond": 8.44737286703835, + "avgBatchDuration": 35127.666666666664, + "minBatchDuration": 34370, + "maxBatchDuration": 35511, + "p50Duration": 35502, + "p95Duration": 35511, + "p99Duration": 35511 + } + ] + }, + "timestamps": { + "start": "2025-12-15T15:06:48.615Z", + "end": "2025-12-15T15:07:25.232Z" + } +} +``` + +**Observations:** + +- Fairness index: **0.9989** (slightly lower due to more aggressive catch-up) +- Throughput: **73.74 items/sec** (**+35% vs baseline**, best in C series) +- Any evidence of new tenant starvation during catch-up?: **Minor variance.** Some tenants took 13-37s for batches (wider spread) but all finished within reasonable bounds. **Higher max deficit = better throughput but slightly more variance.** + +
+ +--- + +## Series D: Consumer Count Variations + +**Objective:** Test parallelism within a single process. + +**Theory:** + +- More consumers = higher potential throughput (limited by concurrency limits) +- Diminishing returns expected as consumers contend for the same work + +### Test D1: Single Consumer (baseline) + +**Configuration:** + +```env +BATCH_QUEUE_DRR_QUANTUM=5 +BATCH_QUEUE_MAX_DEFICIT=50 +BATCH_QUEUE_CONSUMER_COUNT=1 +BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10 +``` + +**Command:** + +```bash +pnpm stress fairness --batch-size 100 --batch-count 3 --tenants all -d "Single consumer" +``` + +
+Results D1 (click to expand) + +```json +{ + "scenario": "fairness-9t", + "description": "Single consumer", + "config": { + "tenantCount": 9, + "batchSize": 100, + "batchCount": 3 + }, + "results": { + "totalItemsProcessed": 2700, + "totalBatches": 27, + "overallThroughput": 41.326108917255944, + "overallDuration": 65334, + "fairnessIndex": 0.9996736075271871, + "perTenant": [ + { + "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 62086, + "avgItemsPerSecond": 4.832007215797443, + "avgBatchDuration": 60490.333333333336, + "minBatchDuration": 59021, + "maxBatchDuration": 62084, + "p50Duration": 60366, + "p95Duration": 62084, + "p99Duration": 62084 + }, + { + "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 65000, + "avgItemsPerSecond": 4.615384615384616, + "avgBatchDuration": 63631.333333333336, + "minBatchDuration": 61952, + "maxBatchDuration": 64991, + "p50Duration": 63951, + "p95Duration": 64991, + "p99Duration": 64991 + }, + { + "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 62539, + "avgItemsPerSecond": 4.797006667839269, + "avgBatchDuration": 60631, + "minBatchDuration": 58991, + "maxBatchDuration": 62536, + "p50Duration": 60366, + "p95Duration": 62536, + "p99Duration": 62536 + }, + { + "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 64125, + "avgItemsPerSecond": 4.678362573099415, + "avgBatchDuration": 61768.666666666664, + "minBatchDuration": 58220, + "maxBatchDuration": 64118, + "p50Duration": 62968, + "p95Duration": 64118, + "p99Duration": 64118 + }, + { + "tenantId": "org-2:proj_zgysghtkiezoakvjscin", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 62536, + "avgItemsPerSecond": 4.797236791608034, + "avgBatchDuration": 55644, + "minBatchDuration": 48107, + "maxBatchDuration": 62516, + "p50Duration": 56309, + "p95Duration": 62516, + "p99Duration": 62516 + }, + { + "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 65257, + "avgItemsPerSecond": 4.597207962364191, + "avgBatchDuration": 62698.666666666664, + "minBatchDuration": 60933, + "maxBatchDuration": 65257, + "p50Duration": 61906, + "p95Duration": 65257, + "p99Duration": 65257 + }, + { + "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 62654, + "avgItemsPerSecond": 4.788201870590864, + "avgBatchDuration": 60963.666666666664, + "minBatchDuration": 59342, + "maxBatchDuration": 62654, + "p50Duration": 60895, + "p95Duration": 62654, + "p99Duration": 62654 + }, + { + "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 62966, + "avgItemsPerSecond": 4.76447606644856, + "avgBatchDuration": 61417.333333333336, + "minBatchDuration": 60031, + "maxBatchDuration": 62961, + "p50Duration": 61260, + "p95Duration": 62961, + "p99Duration": 62961 + }, + { + "tenantId": "org-3:proj_ytivyoceocenyxuprmga", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 64690, + "avgItemsPerSecond": 4.637501932292472, + "avgBatchDuration": 59325, + "minBatchDuration": 51023, + "maxBatchDuration": 64686, + "p50Duration": 62266, + "p95Duration": 64686, + "p99Duration": 64686 + } + ] + }, + "timestamps": { + "start": "2025-12-15T15:09:10.445Z", + "end": "2025-12-15T15:10:15.779Z" + } +} +``` + +**Observations:** + +- Throughput: **41.33 items/sec** (lowest in D series, -24% vs baseline) +- Fairness: **0.9997** (excellent) +- **Single consumer is a bottleneck.** Total duration 65.3s. Each tenant averaging only ~4.6 items/sec. + +
+ +### Test D2: Multiple Consumers (count=3) + +**Configuration:** + +```env +BATCH_QUEUE_DRR_QUANTUM=5 +BATCH_QUEUE_MAX_DEFICIT=50 +BATCH_QUEUE_CONSUMER_COUNT=3 +BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10 +``` + +**Command:** + +```bash +pnpm stress fairness --batch-size 100 --batch-count 3 --tenants all -d "3 consumers" +``` + +
+Results D2 (click to expand) + +```json +{ + "scenario": "fairness-9t", + "description": "3 consumers", + "config": { + "tenantCount": 9, + "batchSize": 100, + "batchCount": 3 + }, + "results": { + "totalItemsProcessed": 2700, + "totalBatches": 27, + "overallThroughput": 57.55211663895639, + "overallDuration": 46914, + "fairnessIndex": 0.9992021047885379, + "perTenant": [ + { + "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 46746, + "avgItemsPerSecond": 6.417661404184315, + "avgBatchDuration": 46353, + "minBatchDuration": 45585, + "maxBatchDuration": 46746, + "p50Duration": 46728, + "p95Duration": 46746, + "p99Duration": 46746 + }, + { + "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 46812, + "avgItemsPerSecond": 6.40861317610869, + "avgBatchDuration": 46805, + "minBatchDuration": 46794, + "maxBatchDuration": 46812, + "p50Duration": 46809, + "p95Duration": 46812, + "p99Duration": 46812 + }, + { + "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 46606, + "avgItemsPerSecond": 6.436939449856242, + "avgBatchDuration": 38880, + "minBatchDuration": 33105, + "maxBatchDuration": 46604, + "p50Duration": 36931, + "p95Duration": 46604, + "p99Duration": 46604 + }, + { + "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 46821, + "avgItemsPerSecond": 6.407381303261357, + "avgBatchDuration": 46059, + "minBatchDuration": 45679, + "maxBatchDuration": 46810, + "p50Duration": 45688, + "p95Duration": 46810, + "p99Duration": 46810 + }, + { + "tenantId": "org-2:proj_zgysghtkiezoakvjscin", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 43729, + "avgItemsPerSecond": 6.860435866358709, + "avgBatchDuration": 42251, + "minBatchDuration": 39304, + "maxBatchDuration": 43729, + "p50Duration": 43720, + "p95Duration": 43729, + "p99Duration": 43729 + }, + { + "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 46597, + "avgItemsPerSecond": 6.438182715625469, + "avgBatchDuration": 45484.333333333336, + "minBatchDuration": 44415, + "maxBatchDuration": 46567, + "p50Duration": 45471, + "p95Duration": 46567, + "p99Duration": 46567 + }, + { + "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 46808, + "avgItemsPerSecond": 6.409160827209024, + "avgBatchDuration": 45405, + "minBatchDuration": 43679, + "maxBatchDuration": 46761, + "p50Duration": 45775, + "p95Duration": 46761, + "p99Duration": 46761 + }, + { + "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 46811, + "avgItemsPerSecond": 6.408750080109376, + "avgBatchDuration": 45293, + "minBatchDuration": 44526, + "maxBatchDuration": 46784, + "p50Duration": 44569, + "p95Duration": 46784, + "p99Duration": 46784 + }, + { + "tenantId": "org-3:proj_ytivyoceocenyxuprmga", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 43732, + "avgItemsPerSecond": 6.859965242842769, + "avgBatchDuration": 41508, + "minBatchDuration": 39455, + "maxBatchDuration": 43581, + "p50Duration": 41488, + "p95Duration": 43581, + "p99Duration": 43581 + } + ] + }, + "timestamps": { + "start": "2025-12-15T15:11:42.929Z", + "end": "2025-12-15T15:12:29.843Z" + } +} +``` + +**Observations:** + +- Throughput: **57.55 items/sec** (vs single: **1.39x** improvement) +- Fairness impact: **0.9992** (very minor degradation) +- Duration dropped from 65.3s to 46.9s. **3 consumers is a good balance.** + +
+ +### Test D3: Many Consumers (count=5) + +**Configuration:** + +```env +BATCH_QUEUE_DRR_QUANTUM=5 +BATCH_QUEUE_MAX_DEFICIT=50 +BATCH_QUEUE_CONSUMER_COUNT=5 +BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10 +``` + +**Command:** + +```bash +pnpm stress fairness --batch-size 100 --batch-count 3 --tenants all -d "5 consumers" +``` + +
+Results D3 (click to expand) + +```json +{ + "scenario": "fairness-9t", + "description": "5 consumers", + "config": { + "tenantCount": 9, + "batchSize": 100, + "batchCount": 3 + }, + "results": { + "totalItemsProcessed": 2700, + "totalBatches": 27, + "overallThroughput": 63.80263717566993, + "overallDuration": 42318, + "fairnessIndex": 0.9999000645903268, + "perTenant": [ + { + "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 41203, + "avgItemsPerSecond": 7.281023226464092, + "avgBatchDuration": 40931, + "minBatchDuration": 40810, + "maxBatchDuration": 41167, + "p50Duration": 40816, + "p95Duration": 41167, + "p99Duration": 41167 + }, + { + "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 41814, + "avgItemsPerSecond": 7.1746305065289135, + "avgBatchDuration": 37657.333333333336, + "minBatchDuration": 35575, + "maxBatchDuration": 41814, + "p50Duration": 35583, + "p95Duration": 41814, + "p99Duration": 41814 + }, + { + "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 41795, + "avgItemsPerSecond": 7.177892092355545, + "avgBatchDuration": 37201.666666666664, + "minBatchDuration": 34298, + "maxBatchDuration": 41795, + "p50Duration": 35512, + "p95Duration": 41795, + "p99Duration": 41795 + }, + { + "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 41837, + "avgItemsPerSecond": 7.170686234672658, + "avgBatchDuration": 40580.333333333336, + "minBatchDuration": 38694, + "maxBatchDuration": 41833, + "p50Duration": 41214, + "p95Duration": 41833, + "p99Duration": 41833 + }, + { + "tenantId": "org-2:proj_zgysghtkiezoakvjscin", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 41829, + "avgItemsPerSecond": 7.1720576633436135, + "avgBatchDuration": 39891.666666666664, + "minBatchDuration": 37128, + "maxBatchDuration": 41786, + "p50Duration": 40761, + "p95Duration": 41786, + "p99Duration": 41786 + }, + { + "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 42246, + "avgItemsPerSecond": 7.10126402499645, + "avgBatchDuration": 34807.333333333336, + "minBatchDuration": 28353, + "maxBatchDuration": 42154, + "p50Duration": 33915, + "p95Duration": 42154, + "p99Duration": 42154 + }, + { + "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 41654, + "avgItemsPerSecond": 7.202189465597542, + "avgBatchDuration": 40819.333333333336, + "minBatchDuration": 40181, + "maxBatchDuration": 41653, + "p50Duration": 40624, + "p95Duration": 41653, + "p99Duration": 41653 + }, + { + "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 41210, + "avgItemsPerSecond": 7.279786459597185, + "avgBatchDuration": 40886.333333333336, + "minBatchDuration": 40630, + "maxBatchDuration": 41210, + "p50Duration": 40819, + "p95Duration": 41210, + "p99Duration": 41210 + }, + { + "tenantId": "org-3:proj_ytivyoceocenyxuprmga", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 40813, + "avgItemsPerSecond": 7.350599073824517, + "avgBatchDuration": 37843.666666666664, + "minBatchDuration": 31921, + "maxBatchDuration": 40810, + "p50Duration": 40800, + "p95Duration": 40810, + "p99Duration": 40810 + } + ] + }, + "timestamps": { + "start": "2025-12-15T15:13:47.964Z", + "end": "2025-12-15T15:14:30.282Z" + } +} +``` + +**Observations:** + +- Throughput: **63.80 items/sec** (vs 3 consumers: **1.11x** improvement) +- Fairness: **0.9999** (nearly perfect!) +- Diminishing returns?: **Yes, starting to see it.** Going from 3→5 consumers added ~11% throughput. Still beneficial but less dramatic. + +
+ +### Test D4: High Consumer Count (count=10) + +**Configuration:** + +```env +BATCH_QUEUE_DRR_QUANTUM=5 +BATCH_QUEUE_MAX_DEFICIT=50 +BATCH_QUEUE_CONSUMER_COUNT=10 +BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10 +``` + +**Command:** + +```bash +pnpm stress fairness --batch-size 100 --batch-count 3 --tenants all -d "10 consumers" +``` + +
+Results D4 (click to expand) + +```json +{ + "scenario": "fairness-9t", + "description": "10 consumers", + "config": { + "tenantCount": 9, + "batchSize": 100, + "batchCount": 3 + }, + "results": { + "totalItemsProcessed": 2700, + "totalBatches": 27, + "overallThroughput": 114.94252873563218, + "overallDuration": 23490, + "fairnessIndex": 0.9887990604258773, + "perTenant": [ + { + "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 23004, + "avgItemsPerSecond": 13.041210224308816, + "avgBatchDuration": 21971, + "minBatchDuration": 19947, + "maxBatchDuration": 23004, + "p50Duration": 22962, + "p95Duration": 23004, + "p99Duration": 23004 + }, + { + "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 22777, + "avgItemsPerSecond": 13.171181454976512, + "avgBatchDuration": 21004.666666666668, + "minBatchDuration": 18477, + "maxBatchDuration": 22775, + "p50Duration": 21762, + "p95Duration": 22775, + "p99Duration": 22775 + }, + { + "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 21772, + "avgItemsPerSecond": 13.77916590115745, + "avgBatchDuration": 21014.666666666668, + "minBatchDuration": 19505, + "maxBatchDuration": 21772, + "p50Duration": 21767, + "p95Duration": 21772, + "p99Duration": 21772 + }, + { + "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 22339, + "avgItemsPerSecond": 13.42942835399973, + "avgBatchDuration": 21297.333333333332, + "minBatchDuration": 20240, + "maxBatchDuration": 22339, + "p50Duration": 21313, + "p95Duration": 22339, + "p99Duration": 22339 + }, + { + "tenantId": "org-2:proj_zgysghtkiezoakvjscin", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 22597, + "avgItemsPerSecond": 13.276098597158914, + "avgBatchDuration": 20592.333333333332, + "minBatchDuration": 17413, + "maxBatchDuration": 22597, + "p50Duration": 21767, + "p95Duration": 22597, + "p99Duration": 22597 + }, + { + "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 21762, + "avgItemsPerSecond": 13.785497656465399, + "avgBatchDuration": 21700, + "minBatchDuration": 21578, + "maxBatchDuration": 21762, + "p50Duration": 21760, + "p95Duration": 21762, + "p99Duration": 21762 + }, + { + "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 22275, + "avgItemsPerSecond": 13.468013468013467, + "avgBatchDuration": 22239.333333333332, + "minBatchDuration": 22218, + "maxBatchDuration": 22275, + "p50Duration": 22225, + "p95Duration": 22275, + "p99Duration": 22275 + }, + { + "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 16719, + "avgItemsPerSecond": 17.94365691727974, + "avgBatchDuration": 15697, + "minBatchDuration": 13656, + "maxBatchDuration": 16719, + "p50Duration": 16716, + "p95Duration": 16719, + "p99Duration": 16719 + }, + { + "tenantId": "org-3:proj_ytivyoceocenyxuprmga", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 23387, + "avgItemsPerSecond": 12.827639286783254, + "avgBatchDuration": 23160.333333333332, + "minBatchDuration": 22770, + "maxBatchDuration": 23387, + "p50Duration": 23324, + "p95Duration": 23387, + "p99Duration": 23387 + } + ] + }, + "timestamps": { + "start": "2025-12-15T15:16:28.939Z", + "end": "2025-12-15T15:16:52.429Z" + } +} +``` + +**Observations:** + +- Throughput: **114.94 items/sec** (**BEST IN ALL TESTS SO FAR!** 2.78x vs single consumer) +- Fairness: **0.9888** (noticeably lower - some tenants got 17s while others got 23s) +- CPU impact: **Not measured, but likely higher.** +- Contention visible?: **Yes, fairness degradation suggests some contention/scheduling variance.** One tenant (org-3:efaelbvnogkhjnrdfsmi) finished in 16.7s while others took 21-23s. **10 consumers is great for throughput but comes at fairness cost.** + +
+ +--- + +## Series E: Consumer Interval Variations + +**Objective:** Test polling frequency impact on throughput and latency. + +**Theory:** + +- Lower interval = more responsive, higher CPU usage +- Higher interval = lower CPU, higher latency for new work + +### Test E1: Fast Polling (interval=20ms) + +**Configuration:** + +```env +BATCH_QUEUE_DRR_QUANTUM=5 +BATCH_QUEUE_MAX_DEFICIT=50 +BATCH_QUEUE_CONSUMER_COUNT=3 +BATCH_QUEUE_CONSUMER_INTERVAL_MS=20 +BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10 +``` + +**Command:** + +```bash +pnpm stress fairness --batch-size 100 --batch-count 3 --tenants all -d "Fast polling 20ms" +``` + +
+Results E1 (click to expand) + +```json +{ + "scenario": "fairness-9t", + "description": "Fast polling 20ms", + "config": { + "tenantCount": 9, + "batchSize": 100, + "batchCount": 3 + }, + "results": { + "totalItemsProcessed": 2700, + "totalBatches": 27, + "overallThroughput": 70.23019898556379, + "overallDuration": 38445, + "fairnessIndex": 0.9979118686098942, + "perTenant": [ + { + "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 38383, + "avgItemsPerSecond": 7.815960190709429, + "avgBatchDuration": 32665.333333333332, + "minBatchDuration": 28219, + "maxBatchDuration": 38381, + "p50Duration": 31396, + "p95Duration": 38381, + "p99Duration": 38381 + }, + { + "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 37376, + "avgItemsPerSecond": 8.026541095890412, + "avgBatchDuration": 37178.333333333336, + "minBatchDuration": 36801, + "maxBatchDuration": 37376, + "p50Duration": 37358, + "p95Duration": 37376, + "p99Duration": 37376 + }, + { + "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 37361, + "avgItemsPerSecond": 8.029763657289687, + "avgBatchDuration": 29490.666666666668, + "minBatchDuration": 23952, + "maxBatchDuration": 37361, + "p50Duration": 27159, + "p95Duration": 37361, + "p99Duration": 37361 + }, + { + "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 37997, + "avgItemsPerSecond": 7.895360160012633, + "avgBatchDuration": 36532, + "minBatchDuration": 34226, + "maxBatchDuration": 37994, + "p50Duration": 37376, + "p95Duration": 37994, + "p99Duration": 37994 + }, + { + "tenantId": "org-2:proj_zgysghtkiezoakvjscin", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 37889, + "avgItemsPerSecond": 7.917865343503392, + "avgBatchDuration": 36989, + "minBatchDuration": 35364, + "maxBatchDuration": 37867, + "p50Duration": 37736, + "p95Duration": 37867, + "p99Duration": 37867 + }, + { + "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 37359, + "avgItemsPerSecond": 8.030193527664016, + "avgBatchDuration": 36533.666666666664, + "minBatchDuration": 35927, + "maxBatchDuration": 37357, + "p50Duration": 36317, + "p95Duration": 37357, + "p99Duration": 37357 + }, + { + "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 38245, + "avgItemsPerSecond": 7.844162635638646, + "avgBatchDuration": 34851.333333333336, + "minBatchDuration": 28081, + "maxBatchDuration": 38237, + "p50Duration": 38236, + "p95Duration": 38237, + "p99Duration": 38237 + }, + { + "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 38090, + "avgItemsPerSecond": 7.876082961407193, + "avgBatchDuration": 36777.666666666664, + "minBatchDuration": 34984, + "maxBatchDuration": 38019, + "p50Duration": 37330, + "p95Duration": 38019, + "p99Duration": 38019 + }, + { + "tenantId": "org-3:proj_ytivyoceocenyxuprmga", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 33054, + "avgItemsPerSecond": 9.076057360682519, + "avgBatchDuration": 24749.666666666668, + "minBatchDuration": 17350, + "maxBatchDuration": 33048, + "p50Duration": 23851, + "p95Duration": 33048, + "p99Duration": 33048 + } + ] + }, + "timestamps": { + "start": "2025-12-15T15:18:51.094Z", + "end": "2025-12-15T15:19:29.539Z" + } +} +``` + +**Observations:** + +- Throughput: **70.23 items/sec** (good improvement over baseline) +- Fairness: **0.9979** (slightly lower) +- CPU usage (if observed): **Expected higher due to more frequent polling.** Faster polling didn't dramatically improve throughput - suggests bottleneck is elsewhere (processing time, not poll latency). + +
+ +### Test E2: Medium Polling (interval=50ms) + +**Configuration:** + +```env +BATCH_QUEUE_DRR_QUANTUM=5 +BATCH_QUEUE_MAX_DEFICIT=50 +BATCH_QUEUE_CONSUMER_COUNT=3 +BATCH_QUEUE_CONSUMER_INTERVAL_MS=50 +BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10 +``` + +**Command:** + +```bash +pnpm stress fairness --batch-size 100 --batch-count 3 --tenants all -d "Medium polling 50ms" +``` + +
+Results E2 (click to expand) + +```json +{ + "scenario": "fairness-9t", + "description": "Medium polling 50ms", + "config": { + "tenantCount": 9, + "batchSize": 100, + "batchCount": 3 + }, + "results": { + "totalItemsProcessed": 2700, + "totalBatches": 27, + "overallThroughput": 93.56806210146937, + "overallDuration": 28856, + "fairnessIndex": 0.9994094367115308, + "perTenant": [ + { + "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 28584, + "avgItemsPerSecond": 10.495382031905962, + "avgBatchDuration": 26949, + "minBatchDuration": 25606, + "maxBatchDuration": 28582, + "p50Duration": 26659, + "p95Duration": 28582, + "p99Duration": 28582 + }, + { + "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 28733, + "avgItemsPerSecond": 10.440956391605472, + "avgBatchDuration": 28390, + "minBatchDuration": 27709, + "maxBatchDuration": 28732, + "p50Duration": 28729, + "p95Duration": 28732, + "p99Duration": 28732 + }, + { + "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 26660, + "avgItemsPerSecond": 11.252813203300825, + "avgBatchDuration": 24539, + "minBatchDuration": 20303, + "maxBatchDuration": 26660, + "p50Duration": 26654, + "p95Duration": 26660, + "p99Duration": 26660 + }, + { + "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 28729, + "avgItemsPerSecond": 10.442410108252986, + "avgBatchDuration": 28355.333333333332, + "minBatchDuration": 27708, + "maxBatchDuration": 28727, + "p50Duration": 28631, + "p95Duration": 28727, + "p99Duration": 28727 + }, + { + "tenantId": "org-2:proj_zgysghtkiezoakvjscin", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 28732, + "avgItemsPerSecond": 10.44131978282055, + "avgBatchDuration": 23195.666666666668, + "minBatchDuration": 13149, + "maxBatchDuration": 28729, + "p50Duration": 27709, + "p95Duration": 28729, + "p99Duration": 28729 + }, + { + "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 28683, + "avgItemsPerSecond": 10.459156991946449, + "avgBatchDuration": 28325, + "minBatchDuration": 27667, + "maxBatchDuration": 28680, + "p50Duration": 28628, + "p95Duration": 28680, + "p99Duration": 28680 + }, + { + "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 28728, + "avgItemsPerSecond": 10.442773600668337, + "avgBatchDuration": 28049, + "minBatchDuration": 26791, + "maxBatchDuration": 28728, + "p50Duration": 28628, + "p95Duration": 28728, + "p99Duration": 28728 + }, + { + "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 28829, + "avgItemsPerSecond": 10.406188213257483, + "avgBatchDuration": 28452.666666666668, + "minBatchDuration": 27805, + "maxBatchDuration": 28829, + "p50Duration": 28724, + "p95Duration": 28829, + "p99Duration": 28829 + }, + { + "tenantId": "org-3:proj_ytivyoceocenyxuprmga", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 28824, + "avgItemsPerSecond": 10.407993338884264, + "avgBatchDuration": 28786.666666666668, + "minBatchDuration": 28727, + "maxBatchDuration": 28817, + "p50Duration": 28816, + "p95Duration": 28817, + "p99Duration": 28817 + } + ] + }, + "timestamps": { + "start": "2025-12-15T15:20:41.913Z", + "end": "2025-12-15T15:21:10.769Z" + } +} +``` + +**Observations:** + +- Throughput: **93.57 items/sec** (**BEST in E series!** +72% vs baseline) +- Fairness: **0.9994** (excellent) +- **50ms polling with 3 consumers is a sweet spot.** Much better than 20ms (70.23) - suggests 20ms introduced contention overhead. + +
+ +### Test E3: Default Polling (interval=100ms) + +**Configuration:** + +```env +BATCH_QUEUE_DRR_QUANTUM=5 +BATCH_QUEUE_MAX_DEFICIT=50 +BATCH_QUEUE_CONSUMER_COUNT=3 +BATCH_QUEUE_CONSUMER_INTERVAL_MS=100 +BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10 +``` + +**Command:** + +```bash +pnpm stress fairness --batch-size 100 --batch-count 3 --tenants all -d "Default polling 100ms" +``` + +
+Results E3 (click to expand) + +```json +{ + "scenario": "fairness-9t", + "description": "Default polling 100ms", + "config": { + "tenantCount": 9, + "batchSize": 100, + "batchCount": 3 + }, + "results": { + "totalItemsProcessed": 2700, + "totalBatches": 27, + "overallThroughput": 86.52182272639877, + "overallDuration": 31206, + "fairnessIndex": 0.9996560386487722, + "perTenant": [ + { + "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 31119, + "avgItemsPerSecond": 9.640412609659693, + "avgBatchDuration": 24846, + "minBatchDuration": 12442, + "maxBatchDuration": 31116, + "p50Duration": 30980, + "p95Duration": 31116, + "p99Duration": 31116 + }, + { + "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 30083, + "avgItemsPerSecond": 9.972409666589103, + "avgBatchDuration": 28193.666666666668, + "minBatchDuration": 24794, + "maxBatchDuration": 30082, + "p50Duration": 29705, + "p95Duration": 30082, + "p99Duration": 30082 + }, + { + "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 30774, + "avgItemsPerSecond": 9.748488984207448, + "avgBatchDuration": 28711.333333333332, + "minBatchDuration": 26655, + "maxBatchDuration": 30774, + "p50Duration": 28705, + "p95Duration": 30774, + "p99Duration": 30774 + }, + { + "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 29722, + "avgItemsPerSecond": 10.093533409595585, + "avgBatchDuration": 29380.666666666668, + "minBatchDuration": 28699, + "maxBatchDuration": 29722, + "p50Duration": 29721, + "p95Duration": 29722, + "p99Duration": 29722 + }, + { + "tenantId": "org-2:proj_zgysghtkiezoakvjscin", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 30761, + "avgItemsPerSecond": 9.752608822860115, + "avgBatchDuration": 28703, + "minBatchDuration": 27637, + "maxBatchDuration": 30759, + "p50Duration": 27713, + "p95Duration": 30759, + "p99Duration": 30759 + }, + { + "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 29721, + "avgItemsPerSecond": 10.093873019077419, + "avgBatchDuration": 29032.333333333332, + "minBatchDuration": 27674, + "maxBatchDuration": 29719, + "p50Duration": 29704, + "p95Duration": 29719, + "p99Duration": 29719 + }, + { + "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 30089, + "avgItemsPerSecond": 9.97042108411712, + "avgBatchDuration": 23863.666666666668, + "minBatchDuration": 12440, + "maxBatchDuration": 30087, + "p50Duration": 29064, + "p95Duration": 30087, + "p99Duration": 30087 + }, + { + "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 29704, + "avgItemsPerSecond": 10.099649878804202, + "avgBatchDuration": 23475, + "minBatchDuration": 20361, + "maxBatchDuration": 29701, + "p50Duration": 20363, + "p95Duration": 29701, + "p99Duration": 29701 + }, + { + "tenantId": "org-3:proj_ytivyoceocenyxuprmga", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 31114, + "avgItemsPerSecond": 9.641961817831202, + "avgBatchDuration": 30969.333333333332, + "minBatchDuration": 30943, + "maxBatchDuration": 30984, + "p50Duration": 30981, + "p95Duration": 30984, + "p99Duration": 30984 + } + ] + }, + "timestamps": { + "start": "2025-12-15T15:22:27.627Z", + "end": "2025-12-15T15:22:58.833Z" + } +} +``` + +**Observations:** + +- Throughput: **86.52 items/sec** (+59% vs baseline) +- Fairness: **0.9997** (excellent) +- 100ms default is solid but slightly lower than 50ms (93.57). **100ms is a good default for balance.** + +
+ +### Test E4: Slow Polling (interval=250ms) + +**Configuration:** + +```env +BATCH_QUEUE_DRR_QUANTUM=5 +BATCH_QUEUE_MAX_DEFICIT=50 +BATCH_QUEUE_CONSUMER_COUNT=3 +BATCH_QUEUE_CONSUMER_INTERVAL_MS=250 +BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10 +``` + +**Command:** + +```bash +pnpm stress fairness --batch-size 100 --batch-count 3 --tenants all -d "Slow polling 250ms" +``` + +
+Results E4 (click to expand) + +```json +{ + "scenario": "fairness-9t", + "description": "Slow polling 250ms", + "config": { + "tenantCount": 9, + "batchSize": 100, + "batchCount": 3 + }, + "results": { + "totalItemsProcessed": 2700, + "totalBatches": 27, + "overallThroughput": 47.45583970471922, + "overallDuration": 56895, + "fairnessIndex": 0.9993684226112225, + "perTenant": [ + { + "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 56835, + "avgItemsPerSecond": 5.278437582475587, + "avgBatchDuration": 56487.666666666664, + "minBatchDuration": 56107, + "maxBatchDuration": 56835, + "p50Duration": 56521, + "p95Duration": 56835, + "p99Duration": 56835 + }, + { + "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 56711, + "avgItemsPerSecond": 5.289979016416568, + "avgBatchDuration": 48760.666666666664, + "minBatchDuration": 34046, + "maxBatchDuration": 56710, + "p50Duration": 55526, + "p95Duration": 56710, + "p99Duration": 56710 + }, + { + "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 55715, + "avgItemsPerSecond": 5.384546351969847, + "avgBatchDuration": 55026.666666666664, + "minBatchDuration": 54680, + "maxBatchDuration": 55715, + "p50Duration": 54685, + "p95Duration": 55715, + "p99Duration": 55715 + }, + { + "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 56114, + "avgItemsPerSecond": 5.346259400506113, + "avgBatchDuration": 47434, + "minBatchDuration": 42815, + "maxBatchDuration": 56109, + "p50Duration": 43378, + "p95Duration": 56109, + "p99Duration": 56109 + }, + { + "tenantId": "org-2:proj_zgysghtkiezoakvjscin", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 52049, + "avgItemsPerSecond": 5.763799496628177, + "avgBatchDuration": 51173, + "minBatchDuration": 49551, + "maxBatchDuration": 52046, + "p50Duration": 51922, + "p95Duration": 52046, + "p99Duration": 52046 + }, + { + "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 55787, + "avgItemsPerSecond": 5.377596931184685, + "avgBatchDuration": 55069.666666666664, + "minBatchDuration": 53713, + "maxBatchDuration": 55787, + "p50Duration": 55709, + "p95Duration": 55787, + "p99Duration": 55787 + }, + { + "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 55790, + "avgItemsPerSecond": 5.377307761247535, + "avgBatchDuration": 47576, + "minBatchDuration": 31153, + "maxBatchDuration": 55788, + "p50Duration": 55787, + "p95Duration": 55788, + "p99Duration": 55788 + }, + { + "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 56010, + "avgItemsPerSecond": 5.356186395286556, + "avgBatchDuration": 47534.333333333336, + "minBatchDuration": 30885, + "maxBatchDuration": 56010, + "p50Duration": 55708, + "p95Duration": 56010, + "p99Duration": 56010 + }, + { + "tenantId": "org-3:proj_ytivyoceocenyxuprmga", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 55784, + "avgItemsPerSecond": 5.377886132224293, + "avgBatchDuration": 55374.333333333336, + "minBatchDuration": 54732, + "maxBatchDuration": 55784, + "p50Duration": 55607, + "p95Duration": 55784, + "p99Duration": 55784 + } + ] + }, + "timestamps": { + "start": "2025-12-15T15:24:13.441Z", + "end": "2025-12-15T15:25:10.336Z" + } +} +``` + +**Observations:** + +- Throughput: **47.46 items/sec** (-13% vs baseline, worst in E series) +- Fairness: **0.9994** (good) +- Latency impact: **Significant.** Duration 56.9s vs 28.9s for 50ms. **250ms is too slow - introduces unnecessary latency between scheduling rounds.** + +
+ +--- + +## Series F: Concurrency Limit Variations + +**Objective:** Test per-environment processing concurrency limits. + +**Theory:** + +- Lower concurrency = throttled processing per env, better resource control +- Higher concurrency = faster per-env processing, potentially starving smaller tenants + +### Test F1: Low Concurrency (concurrency=2) + +**Configuration:** + +```env +BATCH_QUEUE_DRR_QUANTUM=5 +BATCH_QUEUE_MAX_DEFICIT=50 +BATCH_QUEUE_CONSUMER_COUNT=3 +BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=2 +``` + +**Command:** + +```bash +pnpm stress fairness --batch-size 100 --batch-count 3 --tenants all -d "Low concurrency=2" +``` + +
+Results F1 (click to expand) + +```json +{ + "scenario": "fairness-9t", + "description": "Low concurrency=2", + "config": { + "tenantCount": 9, + "batchSize": 100, + "batchCount": 3 + }, + "results": { + "totalItemsProcessed": 2700, + "totalBatches": 27, + "overallThroughput": 90.7288551362613, + "overallDuration": 29759, + "fairnessIndex": 0.9931064638844554, + "perTenant": [ + { + "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 28794, + "avgItemsPerSecond": 10.418837257762034, + "avgBatchDuration": 27016, + "minBatchDuration": 23890, + "maxBatchDuration": 28794, + "p50Duration": 28364, + "p95Duration": 28794, + "p99Duration": 28794 + }, + { + "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 29532, + "avgItemsPerSecond": 10.158472165786264, + "avgBatchDuration": 27720, + "minBatchDuration": 26284, + "maxBatchDuration": 29529, + "p50Duration": 27347, + "p95Duration": 29529, + "p99Duration": 29529 + }, + { + "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 29439, + "avgItemsPerSecond": 10.19056353816366, + "avgBatchDuration": 22083.333333333332, + "minBatchDuration": 10765, + "maxBatchDuration": 29423, + "p50Duration": 26062, + "p95Duration": 29423, + "p99Duration": 29423 + }, + { + "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 29529, + "avgItemsPerSecond": 10.15950421619425, + "avgBatchDuration": 22252.666666666668, + "minBatchDuration": 17683, + "maxBatchDuration": 29396, + "p50Duration": 19679, + "p95Duration": 29396, + "p99Duration": 29396 + }, + { + "tenantId": "org-2:proj_zgysghtkiezoakvjscin", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 28474, + "avgItemsPerSecond": 10.535927512818711, + "avgBatchDuration": 26113.666666666668, + "minBatchDuration": 24815, + "maxBatchDuration": 28474, + "p50Duration": 25052, + "p95Duration": 28474, + "p99Duration": 28474 + }, + { + "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 22890, + "avgItemsPerSecond": 13.10615989515072, + "avgBatchDuration": 22201.666666666668, + "minBatchDuration": 20834, + "maxBatchDuration": 22887, + "p50Duration": 22884, + "p95Duration": 22887, + "p99Duration": 22887 + }, + { + "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 27952, + "avgItemsPerSecond": 10.732684602175159, + "avgBatchDuration": 21568.333333333332, + "minBatchDuration": 18289, + "maxBatchDuration": 27826, + "p50Duration": 18590, + "p95Duration": 27826, + "p99Duration": 27826 + }, + { + "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 28473, + "avgItemsPerSecond": 10.536297545042672, + "avgBatchDuration": 26193.333333333332, + "minBatchDuration": 25054, + "maxBatchDuration": 28470, + "p50Duration": 25056, + "p95Duration": 28470, + "p99Duration": 28470 + }, + { + "tenantId": "org-3:proj_ytivyoceocenyxuprmga", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 29668, + "avgItemsPerSecond": 10.111905082917621, + "avgBatchDuration": 28195.666666666668, + "minBatchDuration": 25936, + "maxBatchDuration": 29562, + "p50Duration": 29089, + "p95Duration": 29562, + "p99Duration": 29562 + } + ] + }, + "timestamps": { + "start": "2025-12-15T15:27:02.038Z", + "end": "2025-12-15T15:27:31.797Z" + } +} +``` + +**Observations:** + +- Throughput: **90.73 items/sec** (+66% vs baseline) +- Fairness index: **0.9931** (slightly degraded) +- Processing time variance: **Higher.** One tenant (org-2:giomqjmqmqbcngusxqfo) finished in 22.9s while others took 28-30s. **Low concurrency=2 paradoxically gave high throughput but with fairness cost.** The system may be "batching up" work more efficiently but creating uneven distribution. + +
+ +### Test F2: Medium Concurrency (concurrency=5) + +**Configuration:** + +```env +BATCH_QUEUE_DRR_QUANTUM=5 +BATCH_QUEUE_MAX_DEFICIT=50 +BATCH_QUEUE_CONSUMER_COUNT=3 +BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=5 +``` + +**Command:** + +```bash +pnpm stress fairness --batch-size 100 --batch-count 3 --tenants all -d "Medium concurrency=5" +``` + +
+Results F2 (click to expand) + +```json +{ + "scenario": "fairness-9t", + "description": "Medium concurrency=5", + "config": { + "tenantCount": 9, + "batchSize": 100, + "batchCount": 3 + }, + "results": { + "totalItemsProcessed": 2700, + "totalBatches": 27, + "overallThroughput": 58.25368400612742, + "overallDuration": 46349, + "fairnessIndex": 0.9998661349420435, + "perTenant": [ + { + "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 46278, + "avgItemsPerSecond": 6.482561908466226, + "avgBatchDuration": 45090, + "minBatchDuration": 43007, + "maxBatchDuration": 46275, + "p50Duration": 45988, + "p95Duration": 46275, + "p99Duration": 46275 + }, + { + "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 45205, + "avgItemsPerSecond": 6.63643402278509, + "avgBatchDuration": 44818, + "minBatchDuration": 44147, + "maxBatchDuration": 45164, + "p50Duration": 45143, + "p95Duration": 45164, + "p99Duration": 45164 + }, + { + "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 45206, + "avgItemsPerSecond": 6.636287218510818, + "avgBatchDuration": 44442, + "minBatchDuration": 43086, + "maxBatchDuration": 45206, + "p50Duration": 45034, + "p95Duration": 45206, + "p99Duration": 45206 + }, + { + "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 45210, + "avgItemsPerSecond": 6.635700066357001, + "avgBatchDuration": 44481.666666666664, + "minBatchDuration": 43090, + "maxBatchDuration": 45210, + "p50Duration": 45145, + "p95Duration": 45210, + "p99Duration": 45210 + }, + { + "tenantId": "org-2:proj_zgysghtkiezoakvjscin", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 45209, + "avgItemsPerSecond": 6.635846844654825, + "avgBatchDuration": 45130.333333333336, + "minBatchDuration": 44982, + "maxBatchDuration": 45207, + "p50Duration": 45202, + "p95Duration": 45207, + "p99Duration": 45207 + }, + { + "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 46279, + "avgItemsPerSecond": 6.482421832796733, + "avgBatchDuration": 45474, + "minBatchDuration": 44160, + "maxBatchDuration": 46277, + "p50Duration": 45985, + "p95Duration": 46277, + "p99Duration": 46277 + }, + { + "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 45209, + "avgItemsPerSecond": 6.635846844654825, + "avgBatchDuration": 44499.333333333336, + "minBatchDuration": 43086, + "maxBatchDuration": 45206, + "p50Duration": 45206, + "p95Duration": 45206, + "p99Duration": 45206 + }, + { + "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 46278, + "avgItemsPerSecond": 6.482561908466226, + "avgBatchDuration": 45212, + "minBatchDuration": 44153, + "maxBatchDuration": 46276, + "p50Duration": 45207, + "p95Duration": 46276, + "p99Duration": 46276 + }, + { + "tenantId": "org-3:proj_ytivyoceocenyxuprmga", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 46262, + "avgItemsPerSecond": 6.484803942760797, + "avgBatchDuration": 44808, + "minBatchDuration": 42006, + "maxBatchDuration": 46209, + "p50Duration": 46209, + "p95Duration": 46209, + "p99Duration": 46209 + } + ] + }, + "timestamps": { + "start": "2025-12-15T15:28:48.965Z", + "end": "2025-12-15T15:29:35.314Z" + } +} +``` + +**Observations:** + +- Throughput: **58.25 items/sec** (+7% vs baseline) +- Fairness index: **0.9999** (nearly perfect!) +- **Concurrency=5 gives best fairness but modest throughput.** All tenants completed in 45-46s range (very tight clustering). + +
+ +### Test F3: Default Concurrency (concurrency=10) + +**Configuration:** + +```env +BATCH_QUEUE_DRR_QUANTUM=5 +BATCH_QUEUE_MAX_DEFICIT=50 +BATCH_QUEUE_CONSUMER_COUNT=3 +BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10 +``` + +**Command:** + +```bash +pnpm stress fairness --batch-size 100 --batch-count 3 --tenants all -d "Default concurrency=10" +``` + +
+Results F3 (click to expand) + +```json +{ + "scenario": "fairness-9t", + "description": "Default concurrency=10", + "config": { + "tenantCount": 9, + "batchSize": 100, + "batchCount": 3 + }, + "results": { + "totalItemsProcessed": 2700, + "totalBatches": 27, + "overallThroughput": 88.71073728479432, + "overallDuration": 30436, + "fairnessIndex": 0.9986319639223729, + "perTenant": [ + { + "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 29265, + "avgItemsPerSecond": 10.251153254741158, + "avgBatchDuration": 22707.666666666668, + "minBatchDuration": 12771, + "maxBatchDuration": 29256, + "p50Duration": 26096, + "p95Duration": 29256, + "p99Duration": 29256 + }, + { + "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 30322, + "avgItemsPerSecond": 9.893806477145308, + "avgBatchDuration": 23069, + "minBatchDuration": 12719, + "maxBatchDuration": 30322, + "p50Duration": 26166, + "p95Duration": 30322, + "p99Duration": 30322 + }, + { + "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 29049, + "avgItemsPerSecond": 10.327377878756584, + "avgBatchDuration": 26282.333333333332, + "minBatchDuration": 21766, + "maxBatchDuration": 29041, + "p50Duration": 28040, + "p95Duration": 29041, + "p99Duration": 29041 + }, + { + "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 29051, + "avgItemsPerSecond": 10.326666896148154, + "avgBatchDuration": 22590.333333333332, + "minBatchDuration": 11757, + "maxBatchDuration": 29043, + "p50Duration": 26971, + "p95Duration": 29043, + "p99Duration": 29043 + }, + { + "tenantId": "org-2:proj_zgysghtkiezoakvjscin", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 30247, + "avgItemsPerSecond": 9.918339008827322, + "avgBatchDuration": 22704.333333333332, + "minBatchDuration": 12811, + "maxBatchDuration": 30217, + "p50Duration": 25085, + "p95Duration": 30217, + "p99Duration": 30217 + }, + { + "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 30216, + "avgItemsPerSecond": 9.928514694201748, + "avgBatchDuration": 23965.333333333332, + "minBatchDuration": 12747, + "maxBatchDuration": 30216, + "p50Duration": 28933, + "p95Duration": 30216, + "p99Duration": 30216 + }, + { + "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 29163, + "avgItemsPerSecond": 10.287007509515483, + "avgBatchDuration": 29106, + "minBatchDuration": 29005, + "maxBatchDuration": 29163, + "p50Duration": 29150, + "p95Duration": 29163, + "p99Duration": 29163 + }, + { + "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 26728, + "avgItemsPerSecond": 11.224184375935348, + "avgBatchDuration": 24337.666666666668, + "minBatchDuration": 21607, + "maxBatchDuration": 26726, + "p50Duration": 24680, + "p95Duration": 26726, + "p99Duration": 26726 + }, + { + "tenantId": "org-3:proj_ytivyoceocenyxuprmga", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 29268, + "avgItemsPerSecond": 10.25010250102501, + "avgBatchDuration": 28511.333333333332, + "minBatchDuration": 27111, + "maxBatchDuration": 29268, + "p50Duration": 29155, + "p95Duration": 29268, + "p99Duration": 29268 + } + ] + }, + "timestamps": { + "start": "2025-12-15T15:31:05.764Z", + "end": "2025-12-15T15:31:36.200Z" + } +} +``` + +**Observations:** + +- Throughput: **88.71 items/sec** (+63% vs baseline) +- Fairness index: **0.9986** (very good) +- **Default concurrency=10 is solid.** Good balance of throughput and fairness. Duration 30.4s. + +
+ +### Test F4: High Concurrency (concurrency=25) + +**Configuration:** + +```env +BATCH_QUEUE_DRR_QUANTUM=5 +BATCH_QUEUE_MAX_DEFICIT=50 +BATCH_QUEUE_CONSUMER_COUNT=3 +BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=25 +``` + +**Command:** + +```bash +pnpm stress fairness --batch-size 100 --batch-count 3 --tenants all -d "High concurrency=25" +``` + +
+Results F4 (click to expand) + +```json +{ + "scenario": "fairness-9t", + "description": "High concurrency=25", + "config": { + "tenantCount": 9, + "batchSize": 100, + "batchCount": 3 + }, + "results": { + "totalItemsProcessed": 2700, + "totalBatches": 27, + "overallThroughput": 92.11873080859776, + "overallDuration": 29310, + "fairnessIndex": 0.9985800387045317, + "perTenant": [ + { + "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 28196, + "avgItemsPerSecond": 10.63980706483189, + "avgBatchDuration": 21146.333333333332, + "minBatchDuration": 9134, + "maxBatchDuration": 28193, + "p50Duration": 26112, + "p95Duration": 28193, + "p99Duration": 28193 + }, + { + "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 26122, + "avgItemsPerSecond": 11.484572391087971, + "avgBatchDuration": 24917.333333333332, + "minBatchDuration": 22512, + "maxBatchDuration": 26122, + "p50Duration": 26118, + "p95Duration": 26122, + "p99Duration": 26122 + }, + { + "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 27157, + "avgItemsPerSecond": 11.046875575358102, + "avgBatchDuration": 20760.333333333332, + "minBatchDuration": 17529, + "maxBatchDuration": 27154, + "p50Duration": 17598, + "p95Duration": 27154, + "p99Duration": 27154 + }, + { + "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 29234, + "avgItemsPerSecond": 10.262023671067935, + "avgBatchDuration": 27852.666666666668, + "minBatchDuration": 27155, + "maxBatchDuration": 29232, + "p50Duration": 27171, + "p95Duration": 29232, + "p99Duration": 29232 + }, + { + "tenantId": "org-2:proj_zgysghtkiezoakvjscin", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 29230, + "avgItemsPerSecond": 10.263427984946972, + "avgBatchDuration": 28536, + "minBatchDuration": 27154, + "maxBatchDuration": 29230, + "p50Duration": 29224, + "p95Duration": 29230, + "p99Duration": 29230 + }, + { + "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 28199, + "avgItemsPerSecond": 10.63867513032377, + "avgBatchDuration": 28189.666666666668, + "minBatchDuration": 28179, + "maxBatchDuration": 28195, + "p50Duration": 28195, + "p95Duration": 28195, + "p99Duration": 28195 + }, + { + "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 29231, + "avgItemsPerSecond": 10.26307687044576, + "avgBatchDuration": 27848.333333333332, + "minBatchDuration": 27149, + "maxBatchDuration": 29231, + "p50Duration": 27165, + "p95Duration": 29231, + "p99Duration": 29231 + }, + { + "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 28194, + "avgItemsPerSecond": 10.640561821664184, + "avgBatchDuration": 27132.666666666668, + "minBatchDuration": 26095, + "maxBatchDuration": 28190, + "p50Duration": 27113, + "p95Duration": 28190, + "p99Duration": 28190 + }, + { + "tenantId": "org-3:proj_ytivyoceocenyxuprmga", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 29234, + "avgItemsPerSecond": 10.262023671067935, + "avgBatchDuration": 28871, + "minBatchDuration": 28201, + "maxBatchDuration": 29232, + "p50Duration": 29180, + "p95Duration": 29232, + "p99Duration": 29232 + } + ] + }, + "timestamps": { + "start": "2025-12-15T15:32:57.803Z", + "end": "2025-12-15T15:33:27.113Z" + } +} +``` + +**Observations:** + +- Throughput: **92.12 items/sec** (+69% vs baseline, best in F series) +- Fairness index: **0.9986** (very good) +- Resource contention?: **Minimal.** Duration 29.3s, only marginally better than concurrency=10 (30.4s). **Diminishing returns above concurrency=10.** Higher concurrency helps but gains are flattening. + +
+ +--- + +## Series G: Global Rate Limiter + +**Objective:** Test global throughput caps across all consumers. + +**Theory:** + +- Rate limiting provides predictable resource consumption +- Should not affect fairness, only overall throughput + +### Test G1: Moderate Rate Limit (50 items/sec) + +**Configuration:** + +```env +BATCH_QUEUE_DRR_QUANTUM=5 +BATCH_QUEUE_MAX_DEFICIT=50 +BATCH_QUEUE_CONSUMER_COUNT=3 +BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10 +BATCH_QUEUE_GLOBAL_RATE_LIMIT=50 +``` + +**Command:** + +```bash +pnpm stress fairness --batch-size 100 --batch-count 3 --tenants all -d "Rate limit 50/sec" +``` + +
+Results G1 (click to expand) + +```json +{ + "scenario": "fairness-9t", + "description": "Rate limit 50/sec", + "config": { + "tenantCount": 9, + "batchSize": 100, + "batchCount": 3 + }, + "results": { + "totalItemsProcessed": 2700, + "totalBatches": 27, + "overallThroughput": 45.432364670447086, + "overallDuration": 59429, + "fairnessIndex": 0.9985621839167762, + "perTenant": [ + { + "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 54093, + "avgItemsPerSecond": 5.546004104043037, + "avgBatchDuration": 54091, + "minBatchDuration": 54089, + "maxBatchDuration": 54093, + "p50Duration": 54091, + "p95Duration": 54093, + "p99Duration": 54093 + }, + { + "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 54109, + "avgItemsPerSecond": 5.544364153837624, + "avgBatchDuration": 53601, + "minBatchDuration": 52695, + "maxBatchDuration": 54109, + "p50Duration": 53999, + "p95Duration": 54109, + "p99Duration": 54109 + }, + { + "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 59343, + "avgItemsPerSecond": 5.055356149840756, + "avgBatchDuration": 57304.333333333336, + "minBatchDuration": 55866, + "maxBatchDuration": 59029, + "p50Duration": 57018, + "p95Duration": 59029, + "p99Duration": 59029 + }, + { + "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 58371, + "avgItemsPerSecond": 5.139538469445443, + "avgBatchDuration": 57164.333333333336, + "minBatchDuration": 56262, + "maxBatchDuration": 58093, + "p50Duration": 57138, + "p95Duration": 58093, + "p99Duration": 58093 + }, + { + "tenantId": "org-2:proj_zgysghtkiezoakvjscin", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 58285, + "avgItemsPerSecond": 5.147121901003689, + "avgBatchDuration": 55634.333333333336, + "minBatchDuration": 50470, + "maxBatchDuration": 58284, + "p50Duration": 58149, + "p95Duration": 58284, + "p99Duration": 58284 + }, + { + "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 55174, + "avgItemsPerSecond": 5.437343676369304, + "avgBatchDuration": 41686.333333333336, + "minBatchDuration": 28166, + "maxBatchDuration": 55157, + "p50Duration": 41736, + "p95Duration": 55157, + "p99Duration": 55157 + }, + { + "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 58284, + "avgItemsPerSecond": 5.147210212065061, + "avgBatchDuration": 57015.666666666664, + "minBatchDuration": 54788, + "maxBatchDuration": 58246, + "p50Duration": 58013, + "p95Duration": 58246, + "p99Duration": 58246 + }, + { + "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 59296, + "avgItemsPerSecond": 5.059363194819212, + "avgBatchDuration": 58258.666666666664, + "minBatchDuration": 57226, + "maxBatchDuration": 59291, + "p50Duration": 58259, + "p95Duration": 59291, + "p99Duration": 59291 + }, + { + "tenantId": "org-3:proj_ytivyoceocenyxuprmga", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 59408, + "avgItemsPerSecond": 5.049824939402101, + "avgBatchDuration": 58836.333333333336, + "minBatchDuration": 58371, + "maxBatchDuration": 59107, + "p50Duration": 59031, + "p95Duration": 59107, + "p99Duration": 59107 + } + ] + }, + "timestamps": { + "start": "2025-12-15T15:35:48.593Z", + "end": "2025-12-15T15:36:48.022Z" + } +} +``` + +**Observations:** + +- Actual throughput: **45.43 items/sec** (target: 50) - ~91% of limit +- Fairness preserved?: **Yes, 0.9986** (excellent) +- **Rate limiter is working correctly.** Duration 59.4s shows throttling is effective. Some overhead explains why we don't hit exactly 50/sec. + +
+ +### Test G2: Higher Rate Limit (100 items/sec) + +**Configuration:** + +```env +BATCH_QUEUE_DRR_QUANTUM=5 +BATCH_QUEUE_MAX_DEFICIT=50 +BATCH_QUEUE_CONSUMER_COUNT=3 +BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10 +BATCH_QUEUE_GLOBAL_RATE_LIMIT=100 +``` + +**Command:** + +```bash +pnpm stress fairness --batch-size 100 --batch-count 3 --tenants all -d "Rate limit 100/sec" +``` + +
+Results G2 (click to expand) + +```json +{ + "scenario": "fairness-9t", + "description": "Rate limit 100/sec", + "config": { + "tenantCount": 9, + "batchSize": 100, + "batchCount": 3 + }, + "results": { + "totalItemsProcessed": 2700, + "totalBatches": 27, + "overallThroughput": 81.93487694595333, + "overallDuration": 32953, + "fairnessIndex": 0.9994851661042276, + "perTenant": [ + { + "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 31984, + "avgItemsPerSecond": 9.379689844922462, + "avgBatchDuration": 30240.333333333332, + "minBatchDuration": 28799, + "maxBatchDuration": 31980, + "p50Duration": 29942, + "p95Duration": 31980, + "p99Duration": 31980 + }, + { + "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 31947, + "avgItemsPerSecond": 9.3905531035778, + "avgBatchDuration": 26446, + "minBatchDuration": 23696, + "maxBatchDuration": 31945, + "p50Duration": 23697, + "p95Duration": 31945, + "p99Duration": 31945 + }, + { + "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 32914, + "avgItemsPerSecond": 9.114662453667133, + "avgBatchDuration": 31870.666666666668, + "minBatchDuration": 29821, + "maxBatchDuration": 32912, + "p50Duration": 32879, + "p95Duration": 32912, + "p99Duration": 32912 + }, + { + "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 30770, + "avgItemsPerSecond": 9.749756256093598, + "avgBatchDuration": 26862, + "minBatchDuration": 22381, + "maxBatchDuration": 30768, + "p50Duration": 27437, + "p95Duration": 30768, + "p99Duration": 30768 + }, + { + "tenantId": "org-2:proj_zgysghtkiezoakvjscin", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 30770, + "avgItemsPerSecond": 9.749756256093598, + "avgBatchDuration": 29516.666666666668, + "minBatchDuration": 27853, + "maxBatchDuration": 30769, + "p50Duration": 29928, + "p95Duration": 30769, + "p99Duration": 30769 + }, + { + "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 30771, + "avgItemsPerSecond": 9.749439407234084, + "avgBatchDuration": 28237.333333333332, + "minBatchDuration": 25781, + "maxBatchDuration": 30769, + "p50Duration": 28162, + "p95Duration": 30769, + "p99Duration": 30769 + }, + { + "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 31890, + "avgItemsPerSecond": 9.40733772342427, + "avgBatchDuration": 23515.666666666668, + "minBatchDuration": 18184, + "maxBatchDuration": 31888, + "p50Duration": 20475, + "p95Duration": 31888, + "p99Duration": 31888 + }, + { + "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 30956, + "avgItemsPerSecond": 9.691174570357928, + "avgBatchDuration": 28886, + "minBatchDuration": 25781, + "maxBatchDuration": 30947, + "p50Duration": 29930, + "p95Duration": 30947, + "p99Duration": 30947 + }, + { + "tenantId": "org-3:proj_ytivyoceocenyxuprmga", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 31976, + "avgItemsPerSecond": 9.382036527395547, + "avgBatchDuration": 29608.666666666668, + "minBatchDuration": 26943, + "maxBatchDuration": 31974, + "p50Duration": 29909, + "p95Duration": 31974, + "p99Duration": 31974 + } + ] + }, + "timestamps": { + "start": "2025-12-15T15:37:49.150Z", + "end": "2025-12-15T15:38:22.103Z" + } +} +``` + +**Observations:** + +- Actual throughput: **81.93 items/sec** (limit was 100) +- Is this limit binding or above natural throughput?: **Above natural throughput.** Without rate limiting (E2 with same settings), throughput was 93.57/sec. The 100/sec limit is barely constraining. **Use rate limits below ~90/sec to have meaningful impact.** + +
+ +### Test G3: Low Rate Limit (20 items/sec) + +**Configuration:** + +```env +BATCH_QUEUE_DRR_QUANTUM=5 +BATCH_QUEUE_MAX_DEFICIT=50 +BATCH_QUEUE_CONSUMER_COUNT=3 +BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10 +BATCH_QUEUE_GLOBAL_RATE_LIMIT=20 +``` + +**Command:** + +```bash +pnpm stress fairness --batch-size 100 --batch-count 3 --tenants all -d "Rate limit 20/sec" +``` + +
+Results G3 (click to expand) + +```json +{ + "scenario": "fairness-9t", + "description": "Rate limit 20/sec", + "config": { + "tenantCount": 9, + "batchSize": 100, + "batchCount": 3 + }, + "results": { + "totalItemsProcessed": 2700, + "totalBatches": 27, + "overallThroughput": 21.692671090900326, + "overallDuration": 124466, + "fairnessIndex": 0.997793763355806, + "perTenant": [ + { + "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 108861, + "avgItemsPerSecond": 2.755807865075647, + "avgBatchDuration": 99844, + "minBatchDuration": 93281, + "maxBatchDuration": 108850, + "p50Duration": 97401, + "p95Duration": 108850, + "p99Duration": 108850 + }, + { + "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 111633, + "avgItemsPerSecond": 2.687377388406654, + "avgBatchDuration": 85438, + "minBatchDuration": 71861, + "maxBatchDuration": 111516, + "p50Duration": 72937, + "p95Duration": 111516, + "p99Duration": 111516 + }, + { + "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 123298, + "avgItemsPerSecond": 2.433129491151519, + "avgBatchDuration": 96355, + "minBatchDuration": 43494, + "maxBatchDuration": 123297, + "p50Duration": 122274, + "p95Duration": 123297, + "p99Duration": 123297 + }, + { + "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 121568, + "avgItemsPerSecond": 2.4677546722821795, + "avgBatchDuration": 92002, + "minBatchDuration": 53253, + "maxBatchDuration": 121568, + "p50Duration": 101185, + "p95Duration": 121568, + "p99Duration": 121568 + }, + { + "tenantId": "org-2:proj_zgysghtkiezoakvjscin", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 123851, + "avgItemsPerSecond": 2.422265464146434, + "avgBatchDuration": 118623.66666666667, + "minBatchDuration": 111599, + "maxBatchDuration": 123784, + "p50Duration": 120488, + "p95Duration": 123784, + "p99Duration": 123784 + }, + { + "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 120270, + "avgItemsPerSecond": 2.4943876278373662, + "avgBatchDuration": 113171.33333333333, + "minBatchDuration": 109360, + "maxBatchDuration": 120269, + "p50Duration": 109885, + "p95Duration": 120269, + "p99Duration": 120269 + }, + { + "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 123169, + "avgItemsPerSecond": 2.4356778085394866, + "avgBatchDuration": 118514, + "minBatchDuration": 113405, + "maxBatchDuration": 123164, + "p50Duration": 118973, + "p95Duration": 123164, + "p99Duration": 123164 + }, + { + "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 124246, + "avgItemsPerSecond": 2.414564653992885, + "avgBatchDuration": 119631.33333333333, + "minBatchDuration": 114504, + "maxBatchDuration": 124241, + "p50Duration": 120149, + "p95Duration": 124241, + "p99Duration": 124241 + }, + { + "tenantId": "org-3:proj_ytivyoceocenyxuprmga", + "totalItems": 300, + "totalBatches": 3, + "totalDuration": 122197, + "avgItemsPerSecond": 2.4550520880218007, + "avgBatchDuration": 120645.33333333333, + "minBatchDuration": 118022, + "maxBatchDuration": 122197, + "p50Duration": 121717, + "p95Duration": 122197, + "p99Duration": 122197 + } + ] + }, + "timestamps": { + "start": "2025-12-15T15:39:42.414Z", + "end": "2025-12-15T15:41:46.880Z" + } +} +``` + +**Observations:** + +- Actual throughput: **21.69 items/sec** (target: 20) - slightly above limit +- Fairness index: **0.9978** (good but slight variance) +- Duration 124.5s (2+ minutes) - shows throttling working hard. **Strict rate limiting works but creates fairness variance** - some tenants saw 72-124s batch durations (wide spread). + +
+ +--- + +## Series H: Asymmetric Load Testing + +**Objective:** Test fairness when tenants have dramatically different workloads. + +### Test H1: Asymmetric with Default Config + +**Configuration:** + +```env +BATCH_QUEUE_DRR_QUANTUM=5 +BATCH_QUEUE_MAX_DEFICIT=50 +BATCH_QUEUE_CONSUMER_COUNT=3 +BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10 +``` + +**Command:** + +```bash +pnpm stress asymmetric --small-size 50 --large-size 800 --batch-count 3 -d "Asymmetric baseline" +``` + +
+Results H1 (click to expand) + +```json +{ + "scenario": "asymmetric-combined", + "config": { + "tenantCount": 9, + "batchSize": 0, + "batchCount": 3 + }, + "results": { + "totalItemsProcessed": 10350, + "totalBatches": 27, + "overallThroughput": 67.60816590394816, + "overallDuration": 107281, + "fairnessIndex": 0.9978993490004432, + "perTenant": [ + { + "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh", + "totalItems": 150, + "totalBatches": 3, + "totalDuration": 16361, + "avgItemsPerSecond": 9.168143756494102, + "avgBatchDuration": 14312.666666666666, + "minBatchDuration": 12271, + "maxBatchDuration": 16360, + "p50Duration": 14307, + "p95Duration": 16360, + "p99Duration": 16360 + }, + { + "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf", + "totalItems": 150, + "totalBatches": 3, + "totalDuration": 16361, + "avgItemsPerSecond": 9.168143756494102, + "avgBatchDuration": 13968.333333333334, + "minBatchDuration": 11240, + "maxBatchDuration": 16358, + "p50Duration": 14307, + "p95Duration": 16358, + "p99Duration": 16358 + }, + { + "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj", + "totalItems": 150, + "totalBatches": 3, + "totalDuration": 16361, + "avgItemsPerSecond": 9.168143756494102, + "avgBatchDuration": 12117.333333333334, + "minBatchDuration": 8977, + "maxBatchDuration": 16359, + "p50Duration": 11016, + "p95Duration": 16359, + "p99Duration": 16359 + }, + { + "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb", + "totalItems": 150, + "totalBatches": 3, + "totalDuration": 13966, + "avgItemsPerSecond": 10.740369468709723, + "avgBatchDuration": 13624.333333333334, + "minBatchDuration": 12945, + "maxBatchDuration": 13964, + "p50Duration": 13964, + "p95Duration": 13964, + "p99Duration": 13964 + }, + { + "tenantId": "org-2:proj_zgysghtkiezoakvjscin", + "totalItems": 150, + "totalBatches": 3, + "totalDuration": 16116, + "avgItemsPerSecond": 9.307520476545047, + "avgBatchDuration": 14416.666666666666, + "minBatchDuration": 11021, + "maxBatchDuration": 16116, + "p50Duration": 16113, + "p95Duration": 16116, + "p99Duration": 16116 + }, + { + "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo", + "totalItems": 2400, + "totalBatches": 3, + "totalDuration": 107273, + "avgItemsPerSecond": 22.37282447586998, + "avgBatchDuration": 106213, + "minBatchDuration": 105126, + "maxBatchDuration": 107273, + "p50Duration": 106240, + "p95Duration": 107273, + "p99Duration": 107273 + }, + { + "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs", + "totalItems": 2400, + "totalBatches": 3, + "totalDuration": 106584, + "avgItemsPerSecond": 22.51745102454402, + "avgBatchDuration": 105934, + "minBatchDuration": 105097, + "maxBatchDuration": 106574, + "p50Duration": 106131, + "p95Duration": 106574, + "p99Duration": 106574 + }, + { + "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi", + "totalItems": 2400, + "totalBatches": 3, + "totalDuration": 107116, + "avgItemsPerSecond": 22.40561634116285, + "avgBatchDuration": 106374.33333333333, + "minBatchDuration": 105308, + "maxBatchDuration": 107100, + "p50Duration": 106715, + "p95Duration": 107100, + "p99Duration": 107100 + }, + { + "tenantId": "org-3:proj_ytivyoceocenyxuprmga", + "totalItems": 2400, + "totalBatches": 3, + "totalDuration": 107046, + "avgItemsPerSecond": 22.42026792220167, + "avgBatchDuration": 104503.66666666667, + "minBatchDuration": 102401, + "maxBatchDuration": 106989, + "p50Duration": 104121, + "p95Duration": 106989, + "p99Duration": 106989 + } + ] + }, + "timestamps": { + "start": "2025-12-15T15:44:50.480Z", + "end": "2025-12-15T15:46:37.770Z" + } +} +``` + +**Observations:** + +- Small batch completion times: **14-17s** (150 items × 5 tenants) +- Large batch completion times: **105-107s** (2400 items × 4 tenants) +- Were small batches starved?: **NO!** Small batches finished ~6x faster than large batches, exactly as expected. **DRR correctly prioritizes smaller workloads and doesn't let large batches monopolize.** Fairness index 0.9979 is excellent given asymmetric load. + +
+ +### Test H2: Asymmetric with Low Quantum (better fairness?) + +**Configuration:** + +```env +BATCH_QUEUE_DRR_QUANTUM=2 +BATCH_QUEUE_MAX_DEFICIT=50 +BATCH_QUEUE_CONSUMER_COUNT=3 +BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10 +``` + +**Command:** + +```bash +pnpm stress asymmetric --small-size 50 --large-size 800 --batch-count 3 -d "Asymmetric low quantum" +``` + +
+Results H2 (click to expand) + +```json +{ + "scenario": "asymmetric-combined", + "config": { + "tenantCount": 9, + "batchSize": 0, + "batchCount": 3 + }, + "results": { + "totalItemsProcessed": 10350, + "totalBatches": 27, + "overallThroughput": 58.418499446364066, + "overallDuration": 127189, + "fairnessIndex": 0.9991864215649569, + "perTenant": [ + { + "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh", + "totalItems": 150, + "totalBatches": 3, + "totalDuration": 16648, + "avgItemsPerSecond": 9.01009130225853, + "avgBatchDuration": 15360, + "minBatchDuration": 14080, + "maxBatchDuration": 16647, + "p50Duration": 15353, + "p95Duration": 16647, + "p99Duration": 16647 + }, + { + "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf", + "totalItems": 150, + "totalBatches": 3, + "totalDuration": 18118, + "avgItemsPerSecond": 8.279059498840931, + "avgBatchDuration": 16403.333333333332, + "minBatchDuration": 15020, + "maxBatchDuration": 18115, + "p50Duration": 16075, + "p95Duration": 18115, + "p99Duration": 18115 + }, + { + "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj", + "totalItems": 150, + "totalBatches": 3, + "totalDuration": 16307, + "avgItemsPerSecond": 9.198503710063163, + "avgBatchDuration": 15531.333333333334, + "minBatchDuration": 14206, + "maxBatchDuration": 16305, + "p50Duration": 16083, + "p95Duration": 16305, + "p99Duration": 16305 + }, + { + "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb", + "totalItems": 150, + "totalBatches": 3, + "totalDuration": 16646, + "avgItemsPerSecond": 9.011173855580921, + "avgBatchDuration": 14090, + "minBatchDuration": 11256, + "maxBatchDuration": 16645, + "p50Duration": 14369, + "p95Duration": 16645, + "p99Duration": 16645 + }, + { + "tenantId": "org-2:proj_zgysghtkiezoakvjscin", + "totalItems": 150, + "totalBatches": 3, + "totalDuration": 17696, + "avgItemsPerSecond": 8.476491862567812, + "avgBatchDuration": 16322, + "minBatchDuration": 14600, + "maxBatchDuration": 17694, + "p50Duration": 16672, + "p95Duration": 17694, + "p99Duration": 17694 + }, + { + "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo", + "totalItems": 2400, + "totalBatches": 3, + "totalDuration": 125841, + "avgItemsPerSecond": 19.07168569861969, + "avgBatchDuration": 125238.66666666667, + "minBatchDuration": 124354, + "maxBatchDuration": 125800, + "p50Duration": 125562, + "p95Duration": 125800, + "p99Duration": 125800 + }, + { + "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs", + "totalItems": 2400, + "totalBatches": 3, + "totalDuration": 127140, + "avgItemsPerSecond": 18.876828692779615, + "avgBatchDuration": 126407.66666666667, + "minBatchDuration": 125398, + "maxBatchDuration": 127140, + "p50Duration": 126685, + "p95Duration": 127140, + "p99Duration": 127140 + }, + { + "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi", + "totalItems": 2400, + "totalBatches": 3, + "totalDuration": 125854, + "avgItemsPerSecond": 19.069715702321737, + "avgBatchDuration": 125322.66666666667, + "minBatchDuration": 124426, + "maxBatchDuration": 125849, + "p50Duration": 125693, + "p95Duration": 125849, + "p99Duration": 125849 + }, + { + "tenantId": "org-3:proj_ytivyoceocenyxuprmga", + "totalItems": 2400, + "totalBatches": 3, + "totalDuration": 126467, + "avgItemsPerSecond": 18.977282611274088, + "avgBatchDuration": 124552.33333333333, + "minBatchDuration": 123402, + "maxBatchDuration": 126202, + "p50Duration": 124053, + "p95Duration": 126202, + "p99Duration": 126202 + } + ] + }, + "timestamps": { + "start": "2025-12-15T15:50:21.351Z", + "end": "2025-12-15T15:52:28.745Z" + } +} +``` + +**Observations:** + +- Small batch completion times: **14-18s** (slightly slower than H1's 14-17s) +- Large batch completion times: **123-127s** (slower than H1's 105-107s) +- Improvement over H1?: **Mixed.** Fairness improved (0.9992 vs 0.9979) but overall throughput dropped (58.42 vs 67.61 items/sec). **Low quantum=2 improves fairness but at throughput cost.** For asymmetric loads, default quantum=5 is better balance. + +
+ +### Test H3: Burst Test (All Tenants Simultaneous) + +**Configuration:** + +```env +BATCH_QUEUE_DRR_QUANTUM=5 +BATCH_QUEUE_MAX_DEFICIT=50 +BATCH_QUEUE_CONSUMER_COUNT=3 +BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10 +``` + +**Command:** + +```bash +pnpm stress burst --batch-size 500 -d "Burst test baseline" +``` + +
+Results H3 (click to expand) + +```json +{ + "scenario": "burst-9t-500i", + "config": { + "tenantCount": 9, + "batchSize": 500, + "batchCount": 1 + }, + "results": { + "totalItemsProcessed": 4500, + "totalBatches": 9, + "overallThroughput": 68.83786388459714, + "overallDuration": 65371, + "fairnessIndex": 0.9999479889573228, + "perTenant": [ + { + "tenantId": "org-1:proj_czimyjnqtbskjmvimpwh", + "totalItems": 500, + "totalBatches": 1, + "totalDuration": 64508, + "avgItemsPerSecond": 7.750976623054505, + "avgBatchDuration": 64508, + "minBatchDuration": 64508, + "maxBatchDuration": 64508, + "p50Duration": 64508, + "p95Duration": 64508, + "p99Duration": 64508 + }, + { + "tenantId": "org-1:proj_lvfvbfatttkmiocyaojf", + "totalItems": 500, + "totalBatches": 1, + "totalDuration": 64504, + "avgItemsPerSecond": 7.751457273967506, + "avgBatchDuration": 64504, + "minBatchDuration": 64504, + "maxBatchDuration": 64504, + "p50Duration": 64504, + "p95Duration": 64504, + "p99Duration": 64504 + }, + { + "tenantId": "org-1:proj_pogdfmagzpxpjggpwrlj", + "totalItems": 500, + "totalBatches": 1, + "totalDuration": 64404, + "avgItemsPerSecond": 7.7634929507484, + "avgBatchDuration": 64404, + "minBatchDuration": 64404, + "maxBatchDuration": 64404, + "p50Duration": 64404, + "p95Duration": 64404, + "p99Duration": 64404 + }, + { + "tenantId": "org-2:proj_prxnkqpzdapktltqmxhb", + "totalItems": 500, + "totalBatches": 1, + "totalDuration": 64504, + "avgItemsPerSecond": 7.751457273967506, + "avgBatchDuration": 64504, + "minBatchDuration": 64504, + "maxBatchDuration": 64504, + "p50Duration": 64504, + "p95Duration": 64504, + "p99Duration": 64504 + }, + { + "tenantId": "org-2:proj_zgysghtkiezoakvjscin", + "totalItems": 500, + "totalBatches": 1, + "totalDuration": 64507, + "avgItemsPerSecond": 7.7510967801943975, + "avgBatchDuration": 64507, + "minBatchDuration": 64507, + "maxBatchDuration": 64507, + "p50Duration": 64507, + "p95Duration": 64507, + "p99Duration": 64507 + }, + { + "tenantId": "org-2:proj_giomqjmqmqbcngusxqfo", + "totalItems": 500, + "totalBatches": 1, + "totalDuration": 64391, + "avgItemsPerSecond": 7.7650603345187985, + "avgBatchDuration": 64391, + "minBatchDuration": 64391, + "maxBatchDuration": 64391, + "p50Duration": 64391, + "p95Duration": 64391, + "p99Duration": 64391 + }, + { + "tenantId": "org-3:proj_qopvqsgghjbtrrfcwlqs", + "totalItems": 500, + "totalBatches": 1, + "totalDuration": 63375, + "avgItemsPerSecond": 7.889546351084813, + "avgBatchDuration": 63375, + "minBatchDuration": 63375, + "maxBatchDuration": 63375, + "p50Duration": 63375, + "p95Duration": 63375, + "p99Duration": 63375 + }, + { + "tenantId": "org-3:proj_efaelbvnogkhjnrdfsmi", + "totalItems": 500, + "totalBatches": 1, + "totalDuration": 64497, + "avgItemsPerSecond": 7.752298556522009, + "avgBatchDuration": 64497, + "minBatchDuration": 64497, + "maxBatchDuration": 64497, + "p50Duration": 64497, + "p95Duration": 64497, + "p99Duration": 64497 + }, + { + "tenantId": "org-3:proj_ytivyoceocenyxuprmga", + "totalItems": 500, + "totalBatches": 1, + "totalDuration": 65316, + "avgItemsPerSecond": 7.655092167309695, + "avgBatchDuration": 65316, + "minBatchDuration": 65316, + "maxBatchDuration": 65316, + "p50Duration": 65316, + "p95Duration": 65316, + "p99Duration": 65316 + } + ] + }, + "timestamps": { + "start": "2025-12-15T15:53:49.834Z", + "end": "2025-12-15T15:54:55.205Z" + } +} +``` + +**Observations:** + +- Fairness under burst: **0.9999** (nearly perfect!) +- Throughput: **68.84 items/sec** +- Any tenant significantly slower?: **All tenants finished within 2s of each other (63-65s range).** **Excellent burst handling!** Even under simultaneous 4500-item burst from 9 tenants, DRR maintained perfect fairness. This is the ideal scenario for DRR. + +
+ +--- + +## Series I: Combined/Optimized Configurations + +**Objective:** Test promising combinations based on earlier results. + +### Test I1: Throughput-Optimized + +Based on Series B-F results, configure for maximum throughput. + +**Configuration (adjust based on findings):** + +```env +BATCH_QUEUE_DRR_QUANTUM=10 +BATCH_QUEUE_MAX_DEFICIT=50 +BATCH_QUEUE_CONSUMER_COUNT=5 +BATCH_QUEUE_CONSUMER_INTERVAL_MS=50 +BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=25 +``` + +**Command:** + +```bash +pnpm stress fairness --batch-size 100 --batch-count 5 --tenants all -d "Throughput optimized" +``` + +
+Results I1 (click to expand) + +```json +// Paste results here +``` + +**Observations:** + +- Throughput: **\_** items/sec +- Fairness tradeoff: **\_** + +
+ +### Test I2: Fairness-Optimized + +Based on Series B-F results, configure for best fairness. + +**Configuration (adjust based on findings):** + +```env +BATCH_QUEUE_DRR_QUANTUM=2 +BATCH_QUEUE_MAX_DEFICIT=25 +BATCH_QUEUE_CONSUMER_COUNT=3 +BATCH_QUEUE_CONSUMER_INTERVAL_MS=50 +BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=5 +``` + +**Command:** + +```bash +pnpm stress fairness --batch-size 100 --batch-count 5 --tenants all -d "Fairness optimized" +``` + +
+Results I2 (click to expand) + +```json +// Paste results here +``` + +**Observations:** + +- Fairness index: **\_** +- Throughput cost: **\_** + +
+ +### Test I3: Balanced Configuration + +Best balance of throughput and fairness. + +**Configuration (adjust based on findings):** + +```env +BATCH_QUEUE_DRR_QUANTUM=5 +BATCH_QUEUE_MAX_DEFICIT=50 +BATCH_QUEUE_CONSUMER_COUNT=3 +BATCH_QUEUE_CONSUMER_INTERVAL_MS=50 +BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10 +``` + +**Command:** + +```bash +pnpm stress fairness --batch-size 100 --batch-count 5 --tenants all -d "Balanced configuration" +``` + +
+Results I3 (click to expand) + +```json +// Paste results here +``` + +**Observations:** + +- Throughput: **\_** items/sec +- Fairness index: **\_** +- Recommended for production?: **\_** + +
+ +--- + +## Per-Org Concurrency Testing (Optional Series J) + +If time permits, test different concurrency limits per organization by updating the database. + +### Setting Per-Org Concurrency + +```sql +-- Set org-1 to high concurrency (50) +UPDATE "Organization" +SET "batchQueueConcurrencyConfig" = '{"default": 50}' +WHERE slug = 'org-1'; + +-- Set org-2 to medium concurrency (10) +UPDATE "Organization" +SET "batchQueueConcurrencyConfig" = '{"default": 10}' +WHERE slug = 'org-2'; + +-- Set org-3 to low concurrency (2) +UPDATE "Organization" +SET "batchQueueConcurrencyConfig" = '{"default": 2}' +WHERE slug = 'org-3'; +``` + +### Test J1: Mixed Org Concurrency Limits + +**Configuration:** + +```env +BATCH_QUEUE_DRR_QUANTUM=5 +BATCH_QUEUE_MAX_DEFICIT=50 +BATCH_QUEUE_CONSUMER_COUNT=3 +BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10 # fallback +``` + +**Command:** + +```bash +pnpm stress fairness --batch-size 100 --batch-count 3 --tenants all -d "Mixed org concurrency" +``` + +
+Results J1 (click to expand) + +```json +// Paste results here +``` + +**Observations:** + +- Org-1 (high concurrency) throughput: **\_** +- Org-2 (medium concurrency) throughput: **\_** +- Org-3 (low concurrency) throughput: **\_** +- Does DRR still maintain fairness?: **\_** + +
+ +--- + +## Results Summary Table + +| Test | Config Summary | Throughput (items/sec) | Fairness Index | Notes | +| ---- | ---------------------- | ---------------------- | -------------- | ---------------------------------- | +| A1 | Baseline | 54.52 | 0.9999 | Reference point | +| A2 | Throughput baseline | 97.48 (peak @ 500) | 1.0000 | Single tenant max | +| B1 | quantum=1 | 66.09 | 0.9997 | +21% vs baseline | +| B2 | quantum=10 | 64.69 | 0.9998 | +18% | +| B3 | quantum=25 | **84.36** | **0.9999** | **Best B series** | +| B4 | quantum=50 | 51.98 | 0.9997 | Too high, hurts perf | +| C1 | maxDeficit=10 | 52.51 | 0.9998 | Limited catch-up | +| C2 | maxDeficit=25 | 66.33 | 0.9992 | +22% | +| C3 | maxDeficit=100 | **73.74** | 0.9989 | **Best C**, slight fairness cost | +| D1 | consumers=1 | 41.33 | 0.9997 | Bottleneck | +| D2 | consumers=3 | 57.55 | 0.9992 | 1.39x vs D1 | +| D3 | consumers=5 | 63.80 | 0.9999 | Good balance | +| D4 | consumers=10 | **114.94** | 0.9888 | **Best throughput**, fairness cost | +| E1 | interval=20ms | 70.23 | 0.9979 | Contention overhead | +| E2 | interval=50ms | **93.57** | **0.9994** | **Best E series** | +| E3 | interval=100ms | 86.52 | 0.9997 | Good default | +| E4 | interval=250ms | 47.46 | 0.9994 | Too slow | +| F1 | concurrency=2 | 90.73 | 0.9931 | High but unfair | +| F2 | concurrency=5 | 58.25 | **0.9999** | **Best fairness** | +| F3 | concurrency=10 | 88.71 | 0.9986 | Good balance | +| F4 | concurrency=25 | **92.12** | 0.9986 | **Best F series** | +| F5 | concurrency=50 | _not tested_ | _not tested_ | | +| G1 | rateLimit=50 | 45.43 | 0.9986 | Effective throttle | +| G2 | rateLimit=100 | 81.93 | 0.9995 | Not binding | +| G3 | rateLimit=20 | 21.69 | 0.9978 | Strict throttle | +| H1 | asymmetric baseline | 67.61 | 0.9979 | Small batches not starved | +| H2 | asymmetric low quantum | 58.42 | 0.9992 | Better fairness, lower throughput | +| H3 | burst test | 68.84 | **0.9999** | **Perfect burst fairness** | +| I1 | throughput optimized | _pending_ | _pending_ | | +| I2 | fairness optimized | _pending_ | _pending_ | | +| I3 | balanced | _pending_ | _pending_ | | + +--- + +## Recommended Execution Order + +1. **Series A** (Baseline) - Establish reference point +2. **Series B** (DRR Quantum) - Understand core fairness mechanism +3. **Series C** (Max Deficit) - Understand starvation prevention +4. **Series D** (Consumer Count) - Find optimal parallelism +5. **Series E** (Consumer Interval) - Find optimal polling frequency +6. **Series F** (Concurrency) - Find optimal per-env limits +7. **Series G** (Global Rate Limit) - Test throttling mechanism +8. **Series H** (Asymmetric) - Validate fairness under realistic conditions +9. **Series I** (Combined) - Test optimized configurations + +## Between Tests + +After each configuration change: + +1. Restart the webapp to pick up new env vars +2. Wait ~5 seconds for BatchQueue to initialize +3. Consider clearing Redis state between tests if needed: `redis-cli FLUSHDB` + +--- + +## Key Metrics to Watch + +1. **Throughput** (`overallThroughput`) - Higher is better, all else equal +2. **Fairness Index** (`fairnessIndex`) - Closer to 1.0 is better +3. **Per-Tenant Variance** - Look at min/max batch durations across tenants +4. **P95/P99 Latencies** - Watch for tail latencies indicating starvation + +## Notes and Observations + +### Key Findings from Series A-H + +#### 1. DRR Fairness is Excellent + +- **Fairness Index consistently 0.99+** across all tests +- Burst scenario (H3) achieved **0.9999 fairness** with 9 tenants competing simultaneously +- Asymmetric load (H1) showed small batches complete quickly without being starved by large batches + +#### 2. Throughput vs Fairness Tradeoffs + +| Configuration | Effect on Throughput | Effect on Fairness | +| --------------------- | -------------------- | ------------------ | +| Higher quantum (25) | ↑ Significant | → Neutral | +| Higher quantum (50) | ↓ Hurts | → Neutral | +| Higher maxDeficit | ↑ Moderate | ↓ Slight | +| More consumers | ↑↑ Major | ↓ Moderate (at 10) | +| Faster polling (50ms) | ↑ Best | → Neutral | +| Higher concurrency | ↑ Moderate | → Neutral | + +#### 3. Recommended Default Configuration + +Based on test results, the optimal balanced configuration is: + +```env +BATCH_QUEUE_DRR_QUANTUM=25 # Sweet spot from B series +BATCH_QUEUE_MAX_DEFICIT=50 # Default is fine (100 gains throughput but costs fairness) +BATCH_QUEUE_CONSUMER_COUNT=3 # Good balance (10 is faster but less fair) +BATCH_QUEUE_CONSUMER_INTERVAL_MS=50 # Best from E series +BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=10 # Good balance from F series +``` + +**Expected performance:** ~90-100 items/sec with 0.999+ fairness + +#### 4. Configuration Recommendations by Use Case + +**High-throughput priority (fairness acceptable at 0.98+):** + +```env +BATCH_QUEUE_DRR_QUANTUM=25 +BATCH_QUEUE_MAX_DEFICIT=100 +BATCH_QUEUE_CONSUMER_COUNT=10 +BATCH_QUEUE_CONSUMER_INTERVAL_MS=50 +BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=25 +``` + +**Strict fairness priority (throughput can be lower):** + +```env +BATCH_QUEUE_DRR_QUANTUM=5 +BATCH_QUEUE_MAX_DEFICIT=25 +BATCH_QUEUE_CONSUMER_COUNT=3 +BATCH_QUEUE_CONSUMER_INTERVAL_MS=100 +BATCH_CONCURRENCY_DEFAULT_CONCURRENCY=5 +``` + +#### 5. Surprising Results + +1. **quantum=1 outperformed quantum=10**: Lower quantum didn't add expected overhead. May be due to better slot utilization. + +2. **quantum=50 was worst**: Too much time between tenant switches caused overall slowdown. + +3. **concurrency=2 had high throughput but poor fairness**: Some tenants finished much faster than others. + +4. **20ms polling was worse than 50ms**: Too-fast polling created contention. + +#### 6. Rate Limiter Behavior + +- Rate limiter effectively caps throughput +- At 50/sec limit, achieved ~45/sec (91% efficiency) +- At 20/sec limit, achieved ~22/sec (108% - slight overshoot) +- Rate limiting preserved fairness well (0.997-0.999) + +#### 7. Asymmetric Load Handling + +- **Excellent!** Small batches (150 items) completed in 14-17s +- Large batches (2400 items) completed in 105-127s +- DRR prevented large batches from starving small ones +- Lower quantum (2) improved fairness but hurt overall throughput by ~13% + +### Scaling Observations + +- Single consumer: ~41 items/sec +- 3 consumers: ~58 items/sec (1.4x) +- 5 consumers: ~64 items/sec (1.5x) +- 10 consumers: ~115 items/sec (2.8x) + +**Diminishing returns start around 5 consumers, with fairness degradation at 10.** + +### Production Recommendations + +1. **Start with defaults** - They're well-tuned for most cases +2. **Consider quantum=25** for throughput boost without fairness cost +3. **Use 3-5 consumers** for good parallelism without contention +4. **50ms polling** is optimal for most workloads +5. **Monitor fairness index** in production metrics +6. **Use global rate limiter** if you need to cap resource usage + +--- + +**Document created:** $(date) +**Last updated:** diff --git a/docs/docs.json b/docs/docs.json index 13b8b7706d..30d99fd058 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -270,6 +270,13 @@ "management/tasks/batch-trigger" ] }, + { + "group": "Batches API", + "pages": [ + "management/batches/create", + "management/batches/stream-items" + ] + }, { "group": "Runs API", "pages": [ @@ -698,4 +705,4 @@ "destination": "/migrating-from-v3" } ] -} +} \ No newline at end of file diff --git a/docs/limits.mdx b/docs/limits.mdx index db2a18ae6d..c1149c8343 100644 --- a/docs/limits.mdx +++ b/docs/limits.mdx @@ -75,20 +75,44 @@ Additional bundles are available for $10/month per 100 concurrent connections. C ## Task payloads and outputs -| Limit | Details | -| :--------------------- | :-------------------------------------------- | -| Single trigger payload | Must not exceed 3MB | -| Batch trigger payload | The total of all payloads must not exceed 5MB | -| Task outputs | Must not exceed 10MB | +| Limit | Details | +| :--------------------- | :----------------------------------------------------------------- | +| Single trigger payload | Must not exceed 3MB | +| Batch trigger payload | Each item can be up to 3MB (SDK 4.3.1+). Prior: 1MB total combined | +| Task outputs | Must not exceed 10MB | Payloads and outputs that exceed 512KB will be offloaded to object storage and a presigned URL will be provided to download the data when calling `runs.retrieve`. You don't need to do anything to handle this in your tasks however, as we will transparently upload/download these during operation. ## Batch size -A single batch can have a maximum of 500 items. +A single batch can have a maximum of 1,000 items with SDK 4.3.1+. Prior versions are limited to 500 items. +## Batch trigger rate limits + +Batch triggering uses a token bucket algorithm to rate limit the number of runs you can trigger per environment. Each run in a batch consumes one token. + +| Pricing tier | Bucket size | Refill rate | +| :----------- | :---------- | :-------------------- | +| Free | 1,200 runs | 100 runs every 10 sec | +| Hobby | 5,000 runs | 500 runs every 5 sec | +| Pro | 5,000 runs | 500 runs every 5 sec | + +**How it works**: You can burst up to your bucket size, then tokens refill at the specified rate. For example, a Free user can trigger 1,200 runs immediately, then must wait for tokens to refill (100 runs become available every 10 seconds). + +## Batch processing concurrency + +The number of batches that can be processed concurrently per environment. + +| Pricing tier | Limit | +| :----------- | :-------------------- | +| Free | 1 concurrent batch | +| Hobby | 10 concurrent batches | +| Pro | 10 concurrent batches | + +This limits how many batches can have their items actively being processed into runs at the same time. + ## Log retention | Pricing tier | Limit | diff --git a/docs/management/batches/create.mdx b/docs/management/batches/create.mdx new file mode 100644 index 0000000000..f9a11cb2e4 --- /dev/null +++ b/docs/management/batches/create.mdx @@ -0,0 +1,5 @@ +--- +title: "Create batch" +openapi: "openapi POST /api/v3/batches" +--- + diff --git a/docs/management/batches/stream-items.mdx b/docs/management/batches/stream-items.mdx new file mode 100644 index 0000000000..66f074bc42 --- /dev/null +++ b/docs/management/batches/stream-items.mdx @@ -0,0 +1,5 @@ +--- +title: "Stream batch items" +openapi: "openapi POST /api/v3/batches/{batchId}/items" +--- + diff --git a/docs/openapi.yml b/docs/openapi.yml index d5dd12ff46..922e8d7f28 100644 --- a/docs/openapi.yml +++ b/docs/openapi.yml @@ -43,6 +43,159 @@ paths: schema: $ref: "#/components/schemas/Error" + /api/v3/batches: + post: + operationId: createBatch + externalDocs: + description: Find more info here + url: "https://trigger.dev/docs/triggering" + tags: + - Batches + summary: Create a batch (Phase 1) + description: | + Phase 1 of 2-phase batch API. Creates a batch record and optionally blocks the parent run for batchTriggerAndWait. + After creating a batch, stream items via POST /api/v3/batches/{batchId}/items. + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CreateBatchRequest" + responses: + "202": + description: Batch successfully created + content: + application/json: + schema: + $ref: "#/components/schemas/CreateBatchResponse" + headers: + x-trigger-jwt-claims: + description: JWT claims for the batch + schema: + type: string + x-trigger-jwt: + description: JWT token for browser clients + schema: + type: string + "400": + description: Invalid request (e.g., runCount <= 0 or exceeds maximum) + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + "401": + description: Unauthorized - API key is missing or invalid + "422": + description: Validation error + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + "429": + description: Rate limit exceeded + headers: + X-RateLimit-Limit: + description: Maximum number of requests allowed + schema: + type: integer + X-RateLimit-Remaining: + description: Number of requests remaining + schema: + type: integer + X-RateLimit-Reset: + description: Unix timestamp when the rate limit resets + schema: + type: integer + Retry-After: + description: Seconds to wait before retrying + schema: + type: integer + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + "500": + description: Internal server error + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + + /api/v3/batches/{batchId}/items: + post: + operationId: streamBatchItems + externalDocs: + description: Find more info here + url: "https://trigger.dev/docs/triggering" + tags: + - Batches + summary: Stream batch items (Phase 2) + description: | + Phase 2 of 2-phase batch API. Accepts an NDJSON stream of batch items and enqueues them. + Each line in the body should be a valid BatchItemNDJSON object. + The stream is processed with backpressure - items are enqueued as they arrive. + The batch is sealed when the stream completes successfully. + parameters: + - name: batchId + in: path + required: true + description: The batch ID returned from POST /api/v3/batches + schema: + type: string + requestBody: + required: true + content: + application/x-ndjson: + schema: + type: string + description: | + NDJSON (newline-delimited JSON) stream where each line is a BatchItemNDJSON object. + Example: + {"index":0,"task":"my-task","payload":{"key":"value1"}} + {"index":1,"task":"my-task","payload":{"key":"value2"}} + application/ndjson: + schema: + type: string + description: | + NDJSON (newline-delimited JSON) stream where each line is a BatchItemNDJSON object. + responses: + "200": + description: Items successfully processed + content: + application/json: + schema: + $ref: "#/components/schemas/StreamBatchItemsResponse" + "400": + description: Invalid request (e.g., invalid JSON, item exceeds maximum size) + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + "401": + description: Unauthorized - API key is missing or invalid + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + "415": + description: Unsupported Media Type - Content-Type must be application/x-ndjson or application/ndjson + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + "422": + description: Validation error + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + "500": + description: Internal server error + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + components: schemas: Error: @@ -130,6 +283,91 @@ components: type: object additionalProperties: true description: A JSON object that represents the deserialized payload or context. + CreateBatchRequest: + type: object + required: + - runCount + properties: + runCount: + type: integer + minimum: 1 + description: Expected number of items in the batch. Must be a positive integer. + parentRunId: + type: string + description: Parent run ID (friendly ID) for batchTriggerAndWait. + resumeParentOnCompletion: + type: boolean + description: Whether to resume parent on completion. Set to true for batchTriggerAndWait. + idempotencyKey: + type: string + description: Idempotency key for the batch. If provided and a batch with this key already exists, the existing batch will be returned. + CreateBatchResponse: + type: object + required: + - id + - runCount + - isCached + properties: + id: + type: string + description: The batch ID (friendly ID). Use this to stream items via POST /api/v3/batches/{batchId}/items. + runCount: + type: integer + description: The expected run count. + isCached: + type: boolean + description: Whether this response came from a cached/idempotent batch. + idempotencyKey: + type: string + description: The idempotency key if provided. + BatchItemNDJSON: + type: object + required: + - index + - task + properties: + index: + type: integer + minimum: 0 + description: Zero-based index of this item. Used for idempotency and ordering. + task: + type: string + description: The task identifier to trigger. + payload: + description: The payload for this task run. Can be any JSON value. + options: + type: object + additionalProperties: true + description: Options for this specific item. + StreamBatchItemsResponse: + type: object + required: + - id + - itemsAccepted + - itemsDeduplicated + - sealed + properties: + id: + type: string + description: The batch ID. + itemsAccepted: + type: integer + description: Number of items successfully accepted. + itemsDeduplicated: + type: integer + description: Number of items that were deduplicated (already enqueued). + sealed: + type: boolean + description: | + Whether the batch was sealed and is ready for processing. + If false, the batch needs more items before processing can start. + Clients should check this field and retry with missing items if needed. + enqueuedCount: + type: integer + description: Total items currently enqueued. Only present when sealed=false to help with retries. + expectedCount: + type: integer + description: Expected total item count. Only present when sealed=false to help with retries. securitySchemes: BearerAuth: type: http diff --git a/docs/self-hosting/env/webapp.mdx b/docs/self-hosting/env/webapp.mdx index 49e561179b..eb4c1ed7f4 100644 --- a/docs/self-hosting/env/webapp.mdx +++ b/docs/self-hosting/env/webapp.mdx @@ -101,7 +101,9 @@ mode: "wide" | `TASK_PAYLOAD_MAXIMUM_SIZE` | No | 3145728 (3MB) | Max task payload size. | | `BATCH_TASK_PAYLOAD_MAXIMUM_SIZE` | No | 1000000 (1MB) | Max batch payload size. | | `TASK_RUN_METADATA_MAXIMUM_SIZE` | No | 262144 (256KB) | Max metadata size. | -| `MAX_BATCH_V2_TRIGGER_ITEMS` | No | 500 | Max batch size. | +| `MAX_BATCH_V2_TRIGGER_ITEMS` | No | 500 | Max batch size (legacy v2 API). | +| `STREAMING_BATCH_MAX_ITEMS` | No | 1000 | Max items in streaming batch (v3 API, requires SDK 4.3.1+). | +| `STREAMING_BATCH_ITEM_MAXIMUM_SIZE` | No | 3145728 (3MB) | Max size per item in streaming batch. | | `MAXIMUM_DEV_QUEUE_SIZE` | No | — | Max dev queue size. | | `MAXIMUM_DEPLOYED_QUEUE_SIZE` | No | — | Max deployed queue size. | | **OTel limits** | | | | diff --git a/docs/snippets/rate-limit-hit-use-batchtrigger.mdx b/docs/snippets/rate-limit-hit-use-batchtrigger.mdx index cf209d35ad..009ba46c1d 100644 --- a/docs/snippets/rate-limit-hit-use-batchtrigger.mdx +++ b/docs/snippets/rate-limit-hit-use-batchtrigger.mdx @@ -1 +1 @@ -The most common cause of hitting the API rate limit is if you’re calling `trigger()` on a task in a loop, instead of doing this use `batchTrigger()` which will trigger multiple tasks in a single API call. You can have up to 500 tasks in a single batch trigger call. +The most common cause of hitting the API rate limit is if you're calling `trigger()` on a task in a loop, instead of doing this use `batchTrigger()` which will trigger multiple tasks in a single API call. You can have up to 1,000 tasks in a single batch trigger call with SDK 4.3.1+ (500 in prior versions). diff --git a/docs/triggering.mdx b/docs/triggering.mdx index 599fe67e99..464eff79c7 100644 --- a/docs/triggering.mdx +++ b/docs/triggering.mdx @@ -198,8 +198,8 @@ Triggers a single run of a task with the payload you pass in, and any options yo If you need to call `trigger()` on a task in a loop, use - [`batchTrigger()`](#yourTask-batchtrigger) instead which will trigger up to 500 runs in a single - call. + [`batchTrigger()`](#yourTask-batchtrigger) instead which will trigger up to 1,000 runs in a single + call with SDK 4.3.1+ (500 runs in prior versions). ```ts ./trigger/my-task.ts @@ -971,6 +971,44 @@ await yourTask.trigger(payload, { machine: "large-1x" }); If you don't specify a machine it will use the machine preset for your task (or the default for your project). For more information read [the machines guide](/machines). +## Streaming batch triggering + +This feature is only available with SDK 4.3.1+ + +For large batches, you can pass an `AsyncIterable` or `ReadableStream` instead of an array. This allows you to generate items on-demand without loading them all into memory upfront. + +```ts /trigger/my-task.ts +import { task } from "@trigger.dev/sdk"; +import { myOtherTask } from "~/trigger/my-other-task"; + +export const myTask = task({ + id: "my-task", + run: async (payload: { userIds: string[] }) => { + // Use an async generator to stream items + async function* generateItems() { + for (const userId of payload.userIds) { + yield { payload: { userId } }; + } + } + + const batchHandle = await myOtherTask.batchTrigger(generateItems()); + + return { batchId: batchHandle.batchId }; + }, +}); +``` + +This works with all batch trigger methods: + +- `yourTask.batchTrigger()` +- `yourTask.batchTriggerAndWait()` +- `batch.triggerByTask()` +- `batch.triggerByTaskAndWait()` +- `tasks.batchTrigger()` + +Streaming is especially useful when generating batches from database queries, API pagination, or +file processing where you don't want to load all items into memory at once. + ## Large Payloads We recommend keeping your task payloads as small as possible. We currently have a hard limit on task payloads above 10MB. @@ -1049,7 +1087,3 @@ export const myTask = task({ ``` - -### Batch Triggering - -When using triggering a batch, the total size of all payloads cannot exceed 1MB. This means if you are doing a batch of 100 runs, each payload should be less than 100KB. The max batch size is 500 runs. diff --git a/docs/v3-openapi.yaml b/docs/v3-openapi.yaml index 67299f946f..d406ce6c93 100644 --- a/docs/v3-openapi.yaml +++ b/docs/v3-openapi.yaml @@ -1301,7 +1301,7 @@ paths: post: operationId: batch_trigger_task_v1 summary: Batch trigger tasks - description: Batch trigger tasks with up to 500 payloads. + description: Batch trigger tasks with up to 1,000 payloads with SDK 4.3.1+ (500 in prior versions). requestBody: required: true content: