diff --git a/packages/beaconing/changelog.yml b/packages/beaconing/changelog.yml index 3443414bf95..3463d49827c 100644 --- a/packages/beaconing/changelog.yml +++ b/packages/beaconing/changelog.yml @@ -1,3 +1,8 @@ +- version: "1.7.0" + changes: + - description: Update transform source index to scope to the relevant data streams. + type: enhancement + link: https://github.com/elastic/integrations/pull/19998 - version: "1.6.0" changes: - description: Update beaconing transform to add process related fields diff --git a/packages/beaconing/docs/README.md b/packages/beaconing/docs/README.md index 27d7a17ba81..ecc023144bb 100644 --- a/packages/beaconing/docs/README.md +++ b/packages/beaconing/docs/README.md @@ -34,7 +34,9 @@ To inspect the installed assets, you can navigate to **Stack Management > Data > | Transform name | Purpose | Source index | Destination index | Alias | Supported Platform | Event Category | |---------------------------|----------------------------------------------|--------------|------------------------|------------------|-----------------------|----------------| -| beaconing.pivot_transform | Flags beaconing activity in your environment | logs-* | ml_beaconing-[version] | ml_beaconing.all | Linux, macOS, Windows | network | +| beaconing.pivot_transform | Flags beaconing activity in your environment | `logs-endpoint.events.network-*` | ml_beaconing-[version] | ml_beaconing.all | Linux, macOS, Windows | network | + +**Source index patterns**: `logs-endpoint.events.network-*` holds network events collected by the [Elastic Defend](https://www.elastic.co/docs/reference/integrations/endpoint) integration. When querying the destination index to enquire about beaconing activities, we advise using the alias for the destination index (`ml_beaconing.all`). In the event that the underlying package is upgraded, the alias will aid in maintaining the previous findings. diff --git a/packages/beaconing/elasticsearch/transform/pivot_transform/transform.yml b/packages/beaconing/elasticsearch/transform/pivot_transform/transform.yml index 59155f5b728..8ce2052278e 100644 --- a/packages/beaconing/elasticsearch/transform/pivot_transform/transform.yml +++ b/packages/beaconing/elasticsearch/transform/pivot_transform/transform.yml @@ -1,6 +1,6 @@ dest: - index: ml_beaconing-1.6.0 - pipeline: 1.6.0-ml_beaconing_ingest_pipeline + index: ml_beaconing-1.7.0 + pipeline: 1.7.0-ml_beaconing_ingest_pipeline aliases: - alias: ml_beaconing.latest move_on_creation: true @@ -411,7 +411,7 @@ pivot: source: "return doc.containsKey('process.parent.pid') && !doc['process.parent.pid'].isEmpty() ? doc['process.parent.pid'].value.toString() : ''" lang: painless source: - index: logs-* + index: "logs-endpoint.events.network-*" query: bool: filter: @@ -454,5 +454,5 @@ sync: delay: 120s field: "@timestamp" _meta: - fleet_transform_version: 1.6.0 + fleet_transform_version: 1.7.0 run_as_kibana_system: false diff --git a/packages/beaconing/manifest.yml b/packages/beaconing/manifest.yml index f34c596e8bd..c13db975e34 100644 --- a/packages/beaconing/manifest.yml +++ b/packages/beaconing/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.0.0 name: beaconing title: "Network Beaconing Identification" -version: 1.6.0 +version: 1.7.0 source: license: "Elastic-2.0" description: "Package to identify beaconing activity in your network events." diff --git a/packages/ded/changelog.yml b/packages/ded/changelog.yml index 87b6a070dcb..a6f81681c7a 100644 --- a/packages/ded/changelog.yml +++ b/packages/ded/changelog.yml @@ -1,3 +1,8 @@ +- version: "3.3.0" + changes: + - description: Update transform source index to scope to the relevant data streams. + type: enhancement + link: https://github.com/elastic/integrations/pull/19998 - version: "3.2.0" changes: - description: Add process signature fields as influencers to ML jobs and extend the network transform group_by with process.code_signature fields. diff --git a/packages/ded/docs/README.md b/packages/ded/docs/README.md index 9579c100798..644c5515694 100644 --- a/packages/ded/docs/README.md +++ b/packages/ded/docs/README.md @@ -41,7 +41,9 @@ To inspect the installed assets, you can navigate to **Stack Management > Data > | Transform name | Purpose | Source index | Destination index | Alias | Supported Platform | Event Category | |------------------------|---------------------------------------------|--------------|-----------------------------|-----------------------|--------------------|----------------| -| ded.pivot_transform_ea | Collects network logs from your environment | logs-* | ml_network_ded_ea-[version] | ml_network_ded_ea.all | Linux, Windows | network | +| ded.pivot_transform_ea | Collects network logs from your environment | `logs-endpoint.events.network-*,logs-network_traffic.*` | ml_network_ded_ea-[version] | ml_network_ded_ea.all | Linux, Windows | network | + +**Source index patterns**: `logs-endpoint.events.network-*` holds network events collected by the [Elastic Defend](https://www.elastic.co/docs/reference/integrations/endpoint) integration, and `logs-network_traffic.*` holds network packet traffic collected by the [Network Packet Capture](https://www.elastic.co/docs/reference/integrations/network_traffic) integration. **Note**: The transform applies only to network data and does not currently support macOS network logs. diff --git a/packages/ded/elasticsearch/transform/pivot_transform_ea/transform.yml b/packages/ded/elasticsearch/transform/pivot_transform_ea/transform.yml index 55fdba57acf..ed4542d44a0 100644 --- a/packages/ded/elasticsearch/transform/pivot_transform_ea/transform.yml +++ b/packages/ded/elasticsearch/transform/pivot_transform_ea/transform.yml @@ -1,12 +1,12 @@ dest: - index: ml_network_ded_ea-3.2.0 + index: ml_network_ded_ea-3.3.0 aliases: - alias: ml_network_ded_ea.latest move_on_creation: true - alias: ml_network_ded_ea.all move_on_creation: false - pipeline: 3.2.0-ml_ded_ingest_pipeline + pipeline: 3.3.0-ml_ded_ingest_pipeline description: This transform runs every 30 minutes and collects network logs to detect data exfiltration in your environment for the past month up to the runtime. frequency: 30m pivot: @@ -77,7 +77,7 @@ pivot: source: "return doc.containsKey('process.code_signature.trusted') && !doc['process.code_signature.trusted'].isEmpty() ? doc['process.code_signature.trusted'].value.toString() : ''" lang: painless source: - index: logs-* + index: "logs-endpoint.events.network-*,logs-network_traffic.*" query: bool: filter: @@ -113,5 +113,5 @@ sync: delay: 120s field: "@timestamp" _meta: - fleet_transform_version: 3.2.0 + fleet_transform_version: 3.3.0 run_as_kibana_system: false diff --git a/packages/ded/manifest.yml b/packages/ded/manifest.yml index e51bd2b6351..2a107d2ab71 100644 --- a/packages/ded/manifest.yml +++ b/packages/ded/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.0.0 name: ded title: "Data Exfiltration Detection" -version: 3.2.0 +version: 3.3.0 source: license: "Elastic-2.0" description: "ML package to detect data exfiltration in your network and file data." diff --git a/packages/lmd/changelog.yml b/packages/lmd/changelog.yml index ef34d67e01b..9f69c53c1a8 100644 --- a/packages/lmd/changelog.yml +++ b/packages/lmd/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "3.3.0" + changes: + - description: Update transform source index to scope to the relevant data streams. + type: enhancement + link: https://github.com/elastic/integrations/pull/19998 - version: "3.2.0" changes: - description: Add process signature fields as influencers to ML jobs. diff --git a/packages/lmd/docs/README.md b/packages/lmd/docs/README.md index 1a6cab7c912..a2c323db8e9 100644 --- a/packages/lmd/docs/README.md +++ b/packages/lmd/docs/README.md @@ -52,7 +52,9 @@ To inspect the installed assets, you can navigate to **Stack Management > Data > | Transform name | Purpose | Source index | Destination index | Alias | Supported Platform | Event Category | |------------------------|--------------------------------------------------------|--------------|-------------------|-------|--------------------|----------------| -| lmd.pivot_transform_ea | Collects RDP session information from your environment | logs-* | ml-rdp-lmd_ea | | Windows | process | +| lmd.pivot_transform_ea | Collects RDP session information from your environment | `logs-endpoint.events.process-*` | ml-rdp-lmd_ea | | Windows | process | + +**Source index pattern**: `logs-endpoint.events.process-*` holds endpoint process events collected by the [Elastic Defend](https://www.elastic.co/docs/reference/integrations/endpoint) integration. When querying the destination index (`ml-rdp-lmd_ea`) for RDP session logs, we advise using the destination index directly. In the event that the underlying package is upgraded, it will aid in maintaining the previous findings. diff --git a/packages/lmd/elasticsearch/transform/pivot_transform_ea/transform.yml b/packages/lmd/elasticsearch/transform/pivot_transform_ea/transform.yml index 568ae06d986..9cfd54e3a7f 100644 --- a/packages/lmd/elasticsearch/transform/pivot_transform_ea/transform.yml +++ b/packages/lmd/elasticsearch/transform/pivot_transform_ea/transform.yml @@ -1,5 +1,5 @@ source: - index: "logs-*" + index: "logs-endpoint.events.process-*" query: bool: must_not: @@ -86,5 +86,5 @@ sync: delay: 60s field: '@timestamp' _meta: - fleet_transform_version: 3.2.0 + fleet_transform_version: 3.3.0 run_as_kibana_system: false diff --git a/packages/lmd/manifest.yml b/packages/lmd/manifest.yml index 8af19baf205..871622ac881 100644 --- a/packages/lmd/manifest.yml +++ b/packages/lmd/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.0.0 name: lmd title: "Lateral Movement Detection" -version: 3.2.0 +version: 3.3.0 source: license: "Elastic-2.0" description: "ML package to detect lateral movement based on file transfer activity and Windows RDP events." diff --git a/packages/pad/changelog.yml b/packages/pad/changelog.yml index 2406420c714..e54d59dcf1f 100644 --- a/packages/pad/changelog.yml +++ b/packages/pad/changelog.yml @@ -1,3 +1,8 @@ +- version: "2.3.0" + changes: + - description: Update transform source index to scope to the relevant data streams. + type: enhancement + link: https://github.com/elastic/integrations/pull/19998 - version: "2.2.0" changes: - description: Add process signature fields as influencers to ML jobs. diff --git a/packages/pad/docs/README.md b/packages/pad/docs/README.md index 6c57bcf6c4b..64a6a958bc4 100644 --- a/packages/pad/docs/README.md +++ b/packages/pad/docs/README.md @@ -86,8 +86,10 @@ To inspect the installed assets, you can navigate to **Stack Management > Data > | Transform name | Purpose | Source index | Destination index | Alias | Supported Platform | |-------------------------------------------|-------------------------------------------------------------------|--------------|-------------------------------------------------|-------------------------------------------|--------------------| -| pad.pivot_transform_okta_sessions_ea | Collects user session information for Okta events | logs-* | ml_okta_multiple_user_sessions_pad_ea-[version] | ml_okta_multiple_user_sessions_pad_ea.all | Okta | -| pad.pivot_transform_win_privilege_list_ea | Collects special privileges assigned to a user for Windows events | logs-* | ml_windows_privilege_type_pad_ea-[version] | ml_windows_privilege_type_pad_ea.all | Windows | +| pad.pivot_transform_okta_sessions_ea | Collects user session information for Okta events | `logs-okta.system-*` | ml_okta_multiple_user_sessions_pad_ea-[version] | ml_okta_multiple_user_sessions_pad_ea.all | Okta | +| pad.pivot_transform_win_privilege_list_ea | Collects special privileges assigned to a user for Windows events | `logs-system.security-*,logs-windows.*` | ml_windows_privilege_type_pad_ea-[version] | ml_windows_privilege_type_pad_ea.all | Windows | + +**Source index patterns**: `logs-okta.system-*` holds Okta system logs collected by the [Okta](https://www.elastic.co/docs/reference/integrations/okta) integration, `logs-system.security-*` holds Windows security event logs collected by the [System](https://www.elastic.co/docs/reference/integrations/system) integration, and `logs-windows.*` holds Windows event logs collected by the [Windows](https://www.elastic.co/docs/reference/integrations/windows) integration. When querying the destination indices for Okta and Windows logs, we advise using the alias for the destination index (`ml_okta_multiple_user_sessions_pad_ea.all` and `ml_windows_privilege_type_pad_ea.all`). In the event that the underlying package is upgraded, the alias will aid in maintaining the previous findings. diff --git a/packages/pad/elasticsearch/transform/pivot_transform_okta_sessions_ea/transform.yml b/packages/pad/elasticsearch/transform/pivot_transform_okta_sessions_ea/transform.yml index 81551d53dd2..fc4102ec0f6 100644 --- a/packages/pad/elasticsearch/transform/pivot_transform_okta_sessions_ea/transform.yml +++ b/packages/pad/elasticsearch/transform/pivot_transform_okta_sessions_ea/transform.yml @@ -1,5 +1,5 @@ source: - index: "logs-*" + index: "logs-okta.system-*" query: bool: filter: @@ -18,7 +18,7 @@ source: - terms: '_tier': [ "data_cold", "data_frozen" ] dest: - index: ml_okta_multiple_user_sessions_pad_ea-2.2.0 + index: ml_okta_multiple_user_sessions_pad_ea-2.3.0 aliases: - alias: ml_okta_multiple_user_sessions_pad_ea.latest move_on_creation: true @@ -70,5 +70,5 @@ sync: delay: 60s field: '@timestamp' _meta: - fleet_transform_version: 2.2.0 + fleet_transform_version: 2.3.0 run_as_kibana_system: false \ No newline at end of file diff --git a/packages/pad/elasticsearch/transform/pivot_transform_win_privilege_list_ea/transform.yml b/packages/pad/elasticsearch/transform/pivot_transform_win_privilege_list_ea/transform.yml index fd68ca55307..ade7b068aea 100644 --- a/packages/pad/elasticsearch/transform/pivot_transform_win_privilege_list_ea/transform.yml +++ b/packages/pad/elasticsearch/transform/pivot_transform_win_privilege_list_ea/transform.yml @@ -1,5 +1,5 @@ source: - index: "logs-*" + index: "logs-system.security-*,logs-windows.*" query: bool: filter: @@ -20,7 +20,7 @@ source: - terms: '_tier': [ "data_cold", "data_frozen" ] dest: - index: ml_windows_privilege_type_pad_ea-2.2.0 + index: ml_windows_privilege_type_pad_ea-2.3.0 aliases: - alias: ml_windows_privilege_type_pad_ea.latest move_on_creation: true @@ -70,5 +70,5 @@ sync: delay: 60s field: '@timestamp' _meta: - fleet_transform_version: 2.2.0 + fleet_transform_version: 2.3.0 run_as_kibana_system: false \ No newline at end of file diff --git a/packages/pad/manifest.yml b/packages/pad/manifest.yml index c09110c3510..3f86fd821d9 100644 --- a/packages/pad/manifest.yml +++ b/packages/pad/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.0.0 name: pad title: "Privileged Access Detection" -version: 2.2.0 +version: 2.3.0 source: license: "Elastic-2.0" description: "ML package to detect anomalous privileged access activity in Windows, Linux and Okta logs"