[Main][Task]31233527: Add a Config to Allow Users to Change RequestSizeLimitBytes (microsoft#2511)

Karlie-777 · web-flow · commit 4f7f645fa52b · 2025-04-30T17:55:19.000-07:00
* update

* update

* update

* update
diff --git a/channels/1ds-post-js/src/DataModels.ts b/channels/1ds-post-js/src/DataModels.ts
@@ -60,6 +60,14 @@ export interface IChannelConfiguration {
      */
     disableAutoBatchFlushLimit?: boolean;
 
+    /**
+     * [Optional] Sets the record and request size limit in bytes for serializer.
+     * Default for record size (sync) is 65000, record size (async) is 2000000.
+     * Default for request size (sync) is 65000, request size (async) is 3145728.
+     * @since 3.3.7
+     */
+    requestLimit?: IRequestSizeLimit;
+
     /**
      * [Optional] The HTTP override that should be used to send requests, as an IXHROverride object.
      * By default during the unload of a page or if the event specifies that it wants to use sendBeacon() or sync fetch (with keep-alive),
@@ -326,6 +334,19 @@ export interface ICollectorWebResult {
     authLoginUrl?: string;
 }
 
+export interface IRequestSizeLimit {
+    /**
+     * Request size limit in bytes for serializer.
+     * The value should be two numbers array, with format [async request size limit, sync request size limit]
+     */
+    requestLimit?: number[];
+     /**
+     * Record size limit in bytes for serializer.
+     * The value should be two numbers array, with format [async record size limit, sync record size limit]
+     */
+    recordLimit?: number[];
+}
+
 /**
  * Post channel interface
  */
diff --git a/channels/1ds-post-js/src/HttpManager.ts b/channels/1ds-post-js/src/HttpManager.ts
@@ -246,7 +246,7 @@ export class HttpManager {
                         }
             
                         _useBeacons = !isReactNative(); // Only use beacons if not running in React Native
-                        _serializer = new Serializer(_core, valueSanitizer, stringifyObjects, enableCompoundKey, getCommonSchemaMetaData, _excludeCsMetaData);
+                        _serializer = new Serializer(_core, valueSanitizer, stringifyObjects, enableCompoundKey, getCommonSchemaMetaData, _excludeCsMetaData, channelConfig);
         
                         if (!isNullOrUndefined(channelConfig.useSendBeacon)) {
                             _useBeacons = !!channelConfig.useSendBeacon;
diff --git a/channels/1ds-post-js/src/Index.ts b/channels/1ds-post-js/src/Index.ts
@@ -7,12 +7,13 @@
 
 import { IPayloadData, IXHROverride, OnCompleteCallback, SendPOSTFunction } from "@microsoft/1ds-core-js";
 import {
-    BE_PROFILE, IChannelConfiguration, IPostChannel, NRT_PROFILE, PayloadListenerFunction, PayloadPreprocessorFunction, RT_PROFILE
+    BE_PROFILE, IChannelConfiguration, IPostChannel, IRequestSizeLimit, NRT_PROFILE, PayloadListenerFunction, PayloadPreprocessorFunction,
+    RT_PROFILE
 } from "./DataModels";
 import { PostChannel } from "./PostChannel";
 
 export {
-    PostChannel, IChannelConfiguration,
+    PostChannel, IChannelConfiguration, IRequestSizeLimit,
     BE_PROFILE, NRT_PROFILE, RT_PROFILE, IXHROverride, IPostChannel,
     SendPOSTFunction, IPayloadData, PayloadPreprocessorFunction, PayloadListenerFunction, OnCompleteCallback
 };
diff --git a/channels/1ds-post-js/src/PostChannel.ts b/channels/1ds-post-js/src/PostChannel.ts
@@ -16,7 +16,8 @@ import {
 import { IPromise, createPromise } from "@nevware21/ts-async";
 import { ITimerHandler, isPromiseLike, mathCeil, mathMax, mathMin, objDeepFreeze } from "@nevware21/ts-utils";
 import {
-    BE_PROFILE, EventBatchNotificationReason, IChannelConfiguration, IPostChannel, IPostTransmissionTelemetryItem, NRT_PROFILE, RT_PROFILE
+    BE_PROFILE, EventBatchNotificationReason, IChannelConfiguration, IPostChannel, IPostTransmissionTelemetryItem, IRequestSizeLimit,
+    NRT_PROFILE, RT_PROFILE
 } from "./DataModels";
 import { EventBatch } from "./EventBatch";
 import { HttpManager } from "./HttpManager";
@@ -88,7 +89,8 @@ const defaultPostChannelConfig: IConfigDefaults<IChannelConfiguration> = objDeep
     maxEventRetryAttempts: { isVal: isNumber, v: MaxSendAttempts },
     maxUnloadEventRetryAttempts: { isVal: isNumber, v: MaxSyncUnloadSendAttempts},
     addNoResponse: undefValue,
-    excludeCsMetaData: undefValue
+    excludeCsMetaData: undefValue,
+    requestLimit: {} as IRequestSizeLimit
 });
 
 function isOverrideFn(httpXHROverride: any) {
diff --git a/channels/1ds-post-js/src/Serializer.ts b/channels/1ds-post-js/src/Serializer.ts
@@ -11,7 +11,7 @@ import {
     SendRequestReason, arrIndexOf, doPerf, getCommonSchemaMetaData, getTenantId, isArray, isValueAssigned, objForEachKey, sanitizeProperty,
     strStartsWith
 } from "@microsoft/1ds-core-js";
-import { IPostTransmissionTelemetryItem } from "./DataModels";
+import { IChannelConfiguration, IPostTransmissionTelemetryItem, IRequestSizeLimit } from "./DataModels";
 import { EventBatch } from "./EventBatch";
 import { STR_EMPTY } from "./InternalConstants";
 import { mathMin, strSubstr } from "@nevware21/ts-utils";
@@ -149,8 +149,9 @@ export class Serializer {
      * @param enableCompoundKey - Should compound keys be enabled (defaults to false)
      * @param getEncodedTypeOverride - The callback to get the encoded type for a property defaults to ({@link getCommonSchemaMetaData }(...))
      * @param excludeCsMetaData - (!DANGER!) Should metadata be populated when encoding the event blob (defaults to false) - PII data will NOT be tagged as PII for backend processing
+     * @param cfg channel cfg for setting request and record size limit
      */
-    constructor(perfManager?: IPerfManagerProvider, valueSanitizer?: IValueSanitizer, stringifyObjects?: boolean, enableCompoundKey?: boolean, getEncodedTypeOverride?: SerializerGetEncodedType, excludeCsMetaData?: boolean) {
+    constructor(perfManager?: IPerfManagerProvider, valueSanitizer?: IValueSanitizer, stringifyObjects?: boolean, enableCompoundKey?: boolean, getEncodedTypeOverride?: SerializerGetEncodedType, excludeCsMetaData?: boolean, cfg?: IChannelConfiguration) {
         const strData = "data";
         const strBaseData = "baseData";
         const strExt = "ext";
@@ -161,6 +162,11 @@ export class Serializer {
         let _isReservedCache = {};
         let _excludeCsMetaData: boolean = !!excludeCsMetaData;
         let _getEncodedType: SerializerGetEncodedType = getEncodedTypeOverride || getCommonSchemaMetaData;
+        let _sizeCfg = _getSizeLimtCfg(cfg);
+        let _requestSizeLimitBytes = _validateSizeLimit(_sizeCfg.requestLimit, RequestSizeLimitBytes, 0);
+        let _beaconRequestSizeLimitBytes = _validateSizeLimit(_sizeCfg.requestLimit, BeaconRequestSizeLimitBytes, 1);
+        let _maxRecordSize =  _validateSizeLimit(_sizeCfg.recordLimit, MaxRecordSize, 0);
+        let _maxBeaconRecordSize =  Math.min(_validateSizeLimit(_sizeCfg.recordLimit, MaxBeaconRecordSize, 1), _beaconRequestSizeLimitBytes);
 
         dynamicProto(Serializer, this, (_self) => {
 
@@ -193,8 +199,8 @@ export class Serializer {
                         let sizeExceeded: IPostTransmissionTelemetryItem[] = [];
                         let failedEvts: IPostTransmissionTelemetryItem[] = [];
                         let isBeaconPayload = payload.isBeacon;
-                        let requestMaxSize = isBeaconPayload ? BeaconRequestSizeLimitBytes : RequestSizeLimitBytes;
-                        let recordMaxSize = isBeaconPayload ? MaxBeaconRecordSize : MaxRecordSize;
+                        let requestMaxSize = isBeaconPayload ? _beaconRequestSizeLimitBytes : _requestSizeLimitBytes;
+                        let recordMaxSize = isBeaconPayload ? _maxBeaconRecordSize : _maxRecordSize;
 
                         let lp = 0;
                         let joinCount = 0;
@@ -480,6 +486,25 @@ export class Serializer {
 
 }
 
+function _validateSizeLimit(cfgVal: number[], defaultVal: number, idx: number): number {
+    if (isArray(cfgVal)){
+        let val = cfgVal[idx];
+        if (val > 0 && val <= defaultVal) {
+            return val;
+        }
+    }
+    return defaultVal;
+}
+
+function _getSizeLimtCfg(cfg?: IChannelConfiguration) {
+    let defaultCfg = {} as IRequestSizeLimit;
+    if (cfg && cfg.requestLimit) {
+        return cfg.requestLimit;
+    }
+    return defaultCfg;
+}
+
+
 /**
  * @ignore
  * @param getEncodedType - The function to get the encoded type for the property
diff --git a/channels/1ds-post-js/test/Unit/src/PostChannelTest.ts b/channels/1ds-post-js/test/Unit/src/PostChannelTest.ts
@@ -179,6 +179,7 @@ export class PostChannelTest extends AITestClass {
                     maxUnloadEventRetryAttempts: 2,
                     addNoResponse: undefValue,
                     excludeCsMetaData: undefValue,
+                    requestLimit: {}
                 };
                 let actaulConfig =  postChannel["_getDbgPlgTargets"]()[1];
                 QUnit.assert.deepEqual(expectedConfig, actaulConfig, "default config should be set");
diff --git a/channels/1ds-post-js/test/Unit/src/SerializerTest.ts b/channels/1ds-post-js/test/Unit/src/SerializerTest.ts
@@ -1,9 +1,10 @@
 import { AITestClass } from "@microsoft/ai-test-framework";
 import { HttpManager } from '../../../src/HttpManager';
-import { AppInsightsCore, EventLatency, IEventProperty } from '@microsoft/1ds-core-js';
+import { AppInsightsCore, EventLatency, EventSendType, IEventProperty, SendRequestReason } from '@microsoft/1ds-core-js';
 import { PostChannel, IXHROverride, IPayloadData } from '../../../src/Index';
-import { IPostTransmissionTelemetryItem, EventBatchNotificationReason } from '../../../src/DataModels';
+import { IPostTransmissionTelemetryItem, EventBatchNotificationReason, IChannelConfiguration } from '../../../src/DataModels';
 import { Serializer } from '../../../src/Serializer';
+import { EventBatch } from "../../../src/EventBatch";
 
 
 export class SerializerTest extends AITestClass {
@@ -36,6 +37,138 @@ export class SerializerTest extends AITestClass {
             }
         });
 
+        this.testCase({
+            name: "Append payload with size limit channel config",
+            test: () => {
+                let cfg = {
+                    requestLimit: {
+                        requestLimit: [200, 200],
+                        recordLimit: [200,200]
+                    }
+                } as IChannelConfiguration
+                let serializer = new Serializer(null, null, null, false, null, null, cfg);
+                let ikey = "1234-5678";
+                let event: IPostTransmissionTelemetryItem = {
+                    name: "testEvent",
+                    iKey: ikey,
+                    latency: EventLatency.Normal,       // Should not get serialized
+                    data: {
+                        "testObject.testProperty": 456
+                    },
+                    baseData: {}
+                };
+                let event1: IPostTransmissionTelemetryItem = {
+                    name: "testEvent1",
+                    iKey: ikey,
+                    latency: EventLatency.Normal,       // Should not get serialized
+                    data: {
+                        "testObject.testProperty": 456
+                    },
+                    baseData: {}
+                };
+    
+                let payload = serializer.createPayload(0, false, false, false, SendRequestReason.NormalSchedule, EventSendType.Batched);
+                let batch = EventBatch.create(ikey, [event, event1]);
+                serializer.appendPayload(payload, batch, 100);
+                let evts = payload.payloadBlob;
+                let expectedPayload = "{\"name\":\"testEvent\",\"iKey\":\"o:1234\",\"data\":{\"baseData\":{},\"testObject.testProperty\":456}}" + "\n" +
+                "{\"name\":\"testEvent1\",\"iKey\":\"o:1234\",\"data\":{\"baseData\":{},\"testObject.testProperty\":456}}"
+                QUnit.assert.equal(evts, expectedPayload, "should contain both events");
+                let overflow = payload.overflow;
+                QUnit.assert.equal(overflow, null, "should not have overflow batch");
+                let sizeExceed = payload.sizeExceed;
+                QUnit.assert.equal(sizeExceed.length, 0, "should not have size exceed batch");
+            }
+        });
+
+
+        this.testCase({
+            name: "Append overflow payload with size limit channel config",
+            test: () => {
+                let cfg = {
+                    requestLimit: {
+                        recordLimit: [100,100],
+                        requestLimit: [100, 100]
+                    }
+                } as IChannelConfiguration
+                let serializer = new Serializer(null, null, null, false, null, null, cfg);
+                let ikey = "1234-5678";
+                let event: IPostTransmissionTelemetryItem = {
+                    name: "testEvent",
+                    iKey: ikey,
+                    latency: EventLatency.Normal,       // Should not get serialized
+                    data: {
+                        "testObject.testProperty": 456
+                    },
+                    baseData: {}
+                };
+                let event1: IPostTransmissionTelemetryItem = {
+                    name: "testEvent1",
+                    iKey: ikey,
+                    latency: EventLatency.Normal,       // Should not get serialized
+                    data: {
+                        "testObject.testProperty": 456
+                    },
+                    baseData: {}
+                };
+    
+                let payload = serializer.createPayload(0, false, false, false, SendRequestReason.NormalSchedule, EventSendType.Batched);
+                let batch = EventBatch.create(ikey, [event, event1]);
+                serializer.appendPayload(payload, batch, 100);
+                let evts = payload.payloadBlob;
+                QUnit.assert.equal(evts, "{\"name\":\"testEvent\",\"iKey\":\"o:1234\",\"data\":{\"baseData\":{},\"testObject.testProperty\":456}}", "should contain only one event");
+                let overflow = payload.overflow;
+                QUnit.assert.equal(overflow.count(), 1, "should have only one overflow batch");
+                let overflowEvts = overflow.events();
+                QUnit.assert.equal(overflowEvts.length, 1, "should have only one overflow event");
+                QUnit.assert.equal(overflowEvts[0], event1, "overflow should have event1");
+            }
+        });
+
+
+        this.testCase({
+            name: "Append exceed size payload with size limit channel config",
+            test: () => {
+                let cfg = {
+                    requestLimit: {
+                        recordLimit: [100,100],
+                        requestLimit: [100, 100]
+                    }
+                } as IChannelConfiguration
+                let serializer = new Serializer(null, null, null, false, null, null, cfg);
+                let ikey = "1234-5678";
+                let event: IPostTransmissionTelemetryItem = {
+                    name: "testEvent",
+                    iKey: ikey,
+                    latency: EventLatency.Normal,       // Should not get serialized
+                    data: {
+                        "testObject.testProperty": 456
+                    },
+                    baseData: {}
+                };
+                let largEvent: IPostTransmissionTelemetryItem = {
+                    name: "testEvent",
+                    iKey: ikey,
+                    latency: EventLatency.Normal,       // Should not get serialized
+                    data: {
+                        "testObject.testProperty": new Array(100).join("a")
+                    },
+                    baseData: {}
+                };
+                let payload = serializer.createPayload(0, false, false, false, SendRequestReason.NormalSchedule, EventSendType.Batched);
+                let largeBatch = EventBatch.create(ikey, [event, largEvent]);
+                serializer.appendPayload(payload, largeBatch, 100);
+                let evts = payload.payloadBlob;
+                QUnit.assert.equal(evts, "{\"name\":\"testEvent\",\"iKey\":\"o:1234\",\"data\":{\"baseData\":{},\"testObject.testProperty\":456}}", "should have only one overflow batch");
+                let sizeExceed = payload.sizeExceed;
+                QUnit.assert.equal(sizeExceed.length, 1, "should have only one payload with size exceed");
+                let sizeExceedBatch = sizeExceed[0];
+                QUnit.assert.equal(sizeExceedBatch.count(), 1, "exceed Batch should have only one event");
+                let sizeExceedEvts = sizeExceedBatch.events();
+                QUnit.assert.equal(sizeExceedEvts[0], largEvent, "exceed batch should only contain large event")
+            }
+        });
+
         this.testCase({
             name: 'getEventBlob without compoundKey support',
             test: () => {

Original file line number	Diff line number	Diff line change
`@@ -246,7 +246,7 @@ export class HttpManager {`
`246`	`246`	`}`
`247`	`247`
`248`	`248`	`_useBeacons = !isReactNative(); // Only use beacons if not running in React Native`
`249`		`- _serializer = new Serializer(_core, valueSanitizer, stringifyObjects, enableCompoundKey, getCommonSchemaMetaData, _excludeCsMetaData);`
	`249`	`+ _serializer = new Serializer(_core, valueSanitizer, stringifyObjects, enableCompoundKey, getCommonSchemaMetaData, _excludeCsMetaData, channelConfig);`
`250`	`250`
`251`	`251`	`if (!isNullOrUndefined(channelConfig.useSendBeacon)) {`
`252`	`252`	`_useBeacons = !!channelConfig.useSendBeacon;`