Update ML Job influencer fields for Security ML packages

packages/ded/changelog.yml

@@ -1,3 +1,8 @@
+- version: "3.2.0"
+  changes:
+    - description: Add process signature fields as influencers to ML jobs and extend the network transform group_by with process.code_signature fields.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/19558
 - version: "3.1.1"
   changes:
     - description: Add threat_tactics and threat_techniques to ML job custom_settings.

packages/ded/elasticsearch/ingest_pipeline/ml_ded_ingest_pipeline.yml

@@ -8,9 +8,20 @@ processors:
         if (ctx.avg_source_bytes != null) {
             ctx['source']['bytes'] = (long)(ctx.avg_source_bytes);
         }
+        if (ctx.process_code_signature_trusted != null) {
+          String v = ctx.process_code_signature_trusted.toString();
+          if (!v.isEmpty()) {
+            if (ctx.process == null) { ctx.process = new HashMap(); }
+            if (ctx.process.code_signature == null) { ctx.process.code_signature = new HashMap(); }
+            ctx.process.code_signature.trusted = Boolean.parseBoolean(v);
+          }
+        }
   - remove:
       field: avg_source_bytes
       ignore_missing: true
+  - remove:
+      field: process_code_signature_trusted
+      ignore_missing: true
 on_failure:
   - set:
       field: event.kind

packages/ded/elasticsearch/transform/pivot_transform_ea/fields/fields.yml

@@ -22,6 +22,10 @@
   name: source.ip
 - external: ecs
   name: process.name
+- external: ecs
+  name: process.code_signature.subject_name
+- external: ecs
+  name: process.code_signature.trusted
 - external: ecs
   name: '@timestamp'
 - external: ecs

packages/ded/elasticsearch/transform/pivot_transform_ea/transform.yml

@@ -1,12 +1,12 @@
 
 dest:
-  index: ml_network_ded_ea-3.1.1
+  index: ml_network_ded_ea-3.2.0
   aliases:
     - alias: ml_network_ded_ea.latest
       move_on_creation: true
     - alias: ml_network_ded_ea.all
       move_on_creation: false
-  pipeline: 3.1.1-ml_ded_ingest_pipeline
+  pipeline: 3.2.0-ml_ded_ingest_pipeline
 description: This transform runs every 30 minutes and collects network logs to detect data exfiltration in your environment for the past month up to the runtime.
 frequency: 30m
 pivot:
@@ -66,6 +66,16 @@ pivot:
     'process.name':
       terms:
         field: process.name
+    'process.code_signature.subject_name':
+      terms:
+        script:
+          source: "return doc.containsKey('process.code_signature.subject_name') && !doc['process.code_signature.subject_name'].isEmpty() ? doc['process.code_signature.subject_name'].value : ''"
+          lang: painless
+    'process_code_signature_trusted':
+      terms:
+        script:
+          source: "return doc.containsKey('process.code_signature.trusted') && !doc['process.code_signature.trusted'].isEmpty() ? doc['process.code_signature.trusted'].value.toString() : ''"
+          lang: painless
 source:
   index: logs-*
   query:
@@ -103,5 +113,5 @@ sync:
     delay: 120s
     field: "@timestamp"
 _meta:
-  fleet_transform_version: 3.1.1
+  fleet_transform_version: 3.2.0
   run_as_kibana_system: false

packages/ded/kibana/ml_module/ded-ml.json

@@ -79,7 +79,9 @@
                             "destination.geo.continent_name",
                             "destination.geo.country_name",
                             "destination.geo.country_iso_code",
-                            "source.bytes"
+                            "source.bytes",
+                            "process.code_signature.subject_name",
+                            "process.code_signature.trusted"
                         ]
                     },
                     "data_description": {
@@ -125,7 +127,9 @@
                             "process.name",
                             "source.ip",
                             "destination.ip",
-                            "source.bytes"
+                            "source.bytes",
+                            "process.code_signature.subject_name",
+                            "process.code_signature.trusted"
                         ]
                     },
                     "data_description": {
@@ -172,7 +176,9 @@
                             "source.ip",
                             "destination.ip",
                             "destination.port",
-                            "source.bytes"
+                            "source.bytes",
+                            "process.code_signature.subject_name",
+                            "process.code_signature.trusted"
                         ]
                     },
                     "data_description": {
@@ -222,7 +228,9 @@
                             "destination.ip",
                             "destination.geo.city_name",
                             "destination.geo.region_name",
-                            "source.bytes"
+                            "source.bytes",
+                            "process.code_signature.subject_name",
+                            "process.code_signature.trusted"
                         ]
                     },
                     "data_description": {
@@ -269,7 +277,9 @@
                             "file.path",
                             "file.Ext.device.bus_type",
                             "process.name",
-                            "file.size"
+                            "file.size",
+                            "process.code_signature.subject_name",
+                            "process.code_signature.trusted"
                         ]
                     },
                     "data_description": {
@@ -316,7 +326,9 @@
                             "file.name",
                             "file.path",
                             "file.Ext.device.bus_type",
-                            "process.name"
+                            "process.name",
+                            "process.code_signature.subject_name",
+                            "process.code_signature.trusted"
                         ]
                     },
                     "data_description": {
@@ -363,7 +375,9 @@
                             "file.name",
                             "file.path",
                             "process.name",
-                            "file.size"
+                            "file.size",
+                            "process.code_signature.subject_name",
+                            "process.code_signature.trusted"
                         ]
                     },
                     "data_description": {

packages/ded/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 3.0.0
 name: ded
 title: "Data Exfiltration Detection"
-version: 3.1.1
+version: 3.2.0
 source:
   license: "Elastic-2.0"
 description: "ML package to detect data exfiltration in your network and file data."

packages/dga/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "3.1.0"
+  changes:
+    - description: Add process signature fields as influencers to ML jobs.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/19558
 - version: "3.0.1"
   changes:
     - description: Add threat_tactics and threat_techniques to ML job custom_settings.

packages/dga/kibana/ml_module/dga-ml.json

@@ -49,7 +49,10 @@
                         "influencers": [
                             "source.ip",
                             "host.name",
-                            "host.id"
+                            "host.id",
+                            "process.name",
+                            "process.code_signature.subject_name",
+                            "process.code_signature.trusted"
                         ]
                     },
                     "data_description": {

packages/dga/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 3.0.4
 name: dga
 title: "Domain Generation Algorithm Detection"
-version: 3.0.1
+version: 3.1.0
 source:
   license: "Elastic-2.0"
 description: "ML solution package to detect domain generation algorithm (DGA) activity in your network data."

packages/lmd/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "3.2.0"
+  changes:
+    - description: Add process signature fields as influencers to ML jobs.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/19558
 - version: "3.1.2"
   changes:
     - description: Add threat_tactics and threat_techniques to ML job custom_settings.

packages/lmd/elasticsearch/transform/pivot_transform_ea/transform.yml

@@ -86,5 +86,5 @@ sync:
     delay: 60s
     field: '@timestamp'
 _meta:
-  fleet_transform_version: 3.1.2
+  fleet_transform_version: 3.2.0
   run_as_kibana_system: false

packages/lmd/kibana/ml_module/lmd-ml.json

@@ -64,7 +64,9 @@
                             "event.module",
                             "user.id",
                             "event.action",
-                            "process.name"
+                            "process.name",
+                            "process.code_signature.subject_name",
+                            "process.code_signature.trusted"
                         ]
                     },
                     "data_description": {
@@ -109,7 +111,9 @@
                             "event.module",
                             "user.id",
                             "process.name",
-                            "file.size"
+                            "file.size",
+                            "process.code_signature.subject_name",
+                            "process.code_signature.trusted"
                         ]
                     },
                     "data_description": {
@@ -156,7 +160,10 @@
                             "event.module",
                             "user.id",
                             "file.extension",
-                            "file.name"
+                            "file.name",
+                            "process.name",
+                            "process.code_signature.subject_name",
+                            "process.code_signature.trusted"
                         ]
                     },
                     "data_description": {
@@ -201,7 +208,10 @@
                             "event.module",
                             "user.id",
                             "file_directory",
-                            "file.path"
+                            "file.path",
+                            "process.name",
+                            "process.code_signature.subject_name",
+                            "process.code_signature.trusted"
                         ]
                     },
                     "data_description": {

packages/lmd/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 3.0.0
 name: lmd
 title: "Lateral Movement Detection"
-version: 3.1.2
+version: 3.2.0
 source:
   license: "Elastic-2.0"
 description: "ML package to detect lateral movement based on file transfer activity and Windows RDP events."

packages/pad/changelog.yml

@@ -1,3 +1,8 @@
+- version: "2.2.0"
+  changes:
+    - description: Add process signature fields as influencers to ML jobs.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/19558
 - version: "2.1.1"
   changes:
     - description: Add threat_tactics and threat_techniques to ML job custom_settings.

packages/pad/elasticsearch/transform/pivot_transform_okta_sessions_ea/transform.yml

@@ -18,7 +18,7 @@ source:
         - terms:
             '_tier': [ "data_cold", "data_frozen" ]
 dest:
-  index: ml_okta_multiple_user_sessions_pad_ea-2.1.1
+  index: ml_okta_multiple_user_sessions_pad_ea-2.2.0
   aliases:
     - alias: ml_okta_multiple_user_sessions_pad_ea.latest
       move_on_creation: true
@@ -70,5 +70,5 @@ sync:
     delay: 60s
     field: '@timestamp'
 _meta:
-  fleet_transform_version: 2.1.1
+  fleet_transform_version: 2.2.0
   run_as_kibana_system: false

packages/pad/elasticsearch/transform/pivot_transform_win_privilege_list_ea/transform.yml

@@ -20,7 +20,7 @@ source:
         - terms:
             '_tier': [ "data_cold", "data_frozen" ]
 dest:
-  index: ml_windows_privilege_type_pad_ea-2.1.1
+  index: ml_windows_privilege_type_pad_ea-2.2.0
   aliases:
     - alias: ml_windows_privilege_type_pad_ea.latest
       move_on_creation: true
@@ -70,5 +70,5 @@ sync:
     delay: 60s
     field: '@timestamp'
 _meta:
-  fleet_transform_version: 2.1.1
+  fleet_transform_version: 2.2.0
   run_as_kibana_system: false

packages/pad/kibana/ml_module/pad-ml.json

@@ -544,7 +544,9 @@
                 "event.module",
                 "user.id",
                 "process.name",
-                "process.command_line"
+                "process.command_line",
+                "process.code_signature.subject_name",
+                "process.code_signature.trusted"
               ]
             },
             "data_description": {
@@ -589,7 +591,9 @@
                 "user.name",
                 "event.module",
                 "user.id",
-                "process.name"
+                "process.name",
+                "process.code_signature.subject_name",
+                "process.code_signature.trusted"
               ]
             },
             "data_description": {

packages/pad/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 3.0.0
 name: pad
 title: "Privileged Access Detection"
-version: 2.1.1
+version: 2.2.0
 source:
   license: "Elastic-2.0"
 description: "ML package to detect anomalous privileged access activity in Windows, Linux and Okta logs"

packages/problemchild/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "3.1.0"
+  changes:
+    - description: Add process signature fields as influencers to ML jobs.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/19558
 - version: "3.0.2"
   changes:
     - description: Add threat_tactics and threat_techniques to ML job custom_settings.