[8.12] [Fleet] Fix conflicting dynamic template mappings for intermed…

…iate objects (#175970) (#175991)

# Backport

This will backport the following commits from `main` to `8.12`:
- [[Fleet] Fix conflicting dynamic template mappings for intermediate
objects (#175970)](#175970)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Jaime Soriano
Pastor","email":"[email protected]"},"sourceCommit":{"committedDate":"2024-01-31T14:51:39Z","message":"[Fleet]
Fix conflicting dynamic template mappings for intermediate objects
(#175970)\n\nWhen there are multiple dynamic templates with the same
name, we only\r\nuse the first appearance.\r\nAdd an exception for
intermediate objects, so we use the last\r\nappearance. This way we
avoid that less specific dynamic templates\r\ncreated for intermediate
objects match before more specific dynamic\r\ntemplates defined in
packages.\r\n\r\nThis was a problem for example in the Prometheus
package, where there\r\nare definitions for `prometheus.*.value` and
`prometheus.*.histogram`.\r\nFor both cases Fleet tries to create a
dynamic template for the\r\nintermediate objects `prometheus.*`. If this
dynamic template is\r\ninserted before the dynamic template for the
histograms, it matches for\r\nthem, causing mapping errors on ingestion,
like the following one:\r\n```\r\nmapper
[prometheus.queue_duration.histogram.values] cannot be changed from type
[float] to [long]\r\n```\r\nWith this change, when multiple definitions
generate the same dynamic\r\ntemplate for intermediate objects, only the
last one ends up in the\r\ntemplate.\r\n\r\nThese intermediate objects
were introduced in 8.12, after
https://github.com/elastic/kibana/pull/169981.","sha":"7274f6e2e152928418a677fcb6b9c3550b8a6227","branchLabelMapping":{"^v8.13.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:fix","Team:Fleet","backport:prev-minor","v8.12.1","v8.13.0"],"title":"[Fleet]
Fix conflicting dynamic template mappings for intermediate
objects","number":175970,"url":"https://github.com/elastic/kibana/pull/175970","mergeCommit":{"message":"[Fleet]
Fix conflicting dynamic template mappings for intermediate objects
(#175970)\n\nWhen there are multiple dynamic templates with the same
name, we only\r\nuse the first appearance.\r\nAdd an exception for
intermediate objects, so we use the last\r\nappearance. This way we
avoid that less specific dynamic templates\r\ncreated for intermediate
objects match before more specific dynamic\r\ntemplates defined in
packages.\r\n\r\nThis was a problem for example in the Prometheus
package, where there\r\nare definitions for `prometheus.*.value` and
`prometheus.*.histogram`.\r\nFor both cases Fleet tries to create a
dynamic template for the\r\nintermediate objects `prometheus.*`. If this
dynamic template is\r\ninserted before the dynamic template for the
histograms, it matches for\r\nthem, causing mapping errors on ingestion,
like the following one:\r\n```\r\nmapper
[prometheus.queue_duration.histogram.values] cannot be changed from type
[float] to [long]\r\n```\r\nWith this change, when multiple definitions
generate the same dynamic\r\ntemplate for intermediate objects, only the
last one ends up in the\r\ntemplate.\r\n\r\nThese intermediate objects
were introduced in 8.12, after
https://github.com/elastic/kibana/pull/169981.","sha":"7274f6e2e152928418a677fcb6b9c3550b8a6227"}},"sourceBranch":"main","suggestedTargetBranches":["8.12"],"targetPullRequestStates":[{"branch":"8.12","label":"v8.12.1","branchLabelMappingKey":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"main","label":"v8.13.0","branchLabelMappingKey":"^v8.13.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/175970","number":175970,"mergeCommit":{"message":"[Fleet]
Fix conflicting dynamic template mappings for intermediate objects
(#175970)\n\nWhen there are multiple dynamic templates with the same
name, we only\r\nuse the first appearance.\r\nAdd an exception for
intermediate objects, so we use the last\r\nappearance. This way we
avoid that less specific dynamic templates\r\ncreated for intermediate
objects match before more specific dynamic\r\ntemplates defined in
packages.\r\n\r\nThis was a problem for example in the Prometheus
package, where there\r\nare definitions for `prometheus.*.value` and
`prometheus.*.histogram`.\r\nFor both cases Fleet tries to create a
dynamic template for the\r\nintermediate objects `prometheus.*`. If this
dynamic template is\r\ninserted before the dynamic template for the
histograms, it matches for\r\nthem, causing mapping errors on ingestion,
like the following one:\r\n```\r\nmapper
[prometheus.queue_duration.histogram.values] cannot be changed from type
[float] to [long]\r\n```\r\nWith this change, when multiple definitions
generate the same dynamic\r\ntemplate for intermediate objects, only the
last one ends up in the\r\ntemplate.\r\n\r\nThese intermediate objects
were introduced in 8.12, after
https://github.com/elastic/kibana/pull/169981.","sha":"7274f6e2e152928418a677fcb6b9c3550b8a6227"}}]}]
BACKPORT-->

Co-authored-by: Jaime Soriano Pastor <[email protected]>

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts

@@ -1352,6 +1352,63 @@ describe('EPM template', () => {
     expect(mappings).toEqual(runtimeFieldMapping);
   });
 
+  it('tests processing dynamic templates priority of intermediate objects', () => {
+    const textWithRuntimeFieldsLiteralYml = `
+- name: group.*.value
+  type: object
+  object_type: double
+  object_type_mapping_type: "*"
+  metric_type: gauge
+- name: group.*.histogram
+  type: object
+  object_type: histogram
+  object_type_mapping_type: "*"
+`;
+    const runtimeFieldMapping = {
+      properties: {
+        group: {
+          type: 'object',
+          dynamic: true,
+        },
+      },
+      dynamic_templates: [
+        {
+          'group.*.value': {
+            match_mapping_type: '*',
+            path_match: 'group.*.value',
+            mapping: {
+              time_series_metric: 'gauge',
+              type: 'double',
+            },
+          },
+        },
+        {
+          'group.*.histogram': {
+            path_match: 'group.*.histogram',
+            match_mapping_type: '*',
+            mapping: {
+              type: 'histogram',
+            },
+          },
+        },
+        {
+          'group.*': {
+            path_match: 'group.*',
+            match_mapping_type: 'object',
+            mapping: {
+              type: 'object',
+              dynamic: true,
+            },
+          },
+        },
+      ],
+    };
+    const fields: Field[] = safeLoad(textWithRuntimeFieldsLiteralYml);
+    const processedFields = processFields(fields);
+    const mappings = generateMappings(processedFields, true);
+    expect(mappings).toEqual(runtimeFieldMapping);
+  });
+
   it('tests unexpected type for field as dynamic template fails', () => {
     const textWithRuntimeFieldsLiteralYml = `
 - name: labels.*

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts

@@ -149,7 +149,7 @@ export function generateMappings(
   isIndexModeTimeSeries = false
 ): IndexTemplateMappings {
   const dynamicTemplates: Array<Record<string, Properties>> = [];
-  const dynamicTemplateNames = new Set<string>();
+  const dynamicTemplateNames: Record<string, number> = {};
   const runtimeFields: RuntimeFields = {};
 
   const { properties } = _generateMappings(
@@ -163,8 +163,16 @@ export function generateMappings(
         runtimeProperties?: Properties;
       }) => {
         const name = dynamicMapping.path;
-        if (dynamicTemplateNames.has(name)) {
-          return;
+        if (name in dynamicTemplateNames) {
+          if (name.includes('*') && dynamicMapping.properties?.type === 'object') {
+            // This is a conflicting intermediate object, use the last one so
+            // more specific templates are chosen before.
+            const index = dynamicTemplateNames[name];
+            delete dynamicTemplateNames[name];
+            dynamicTemplates.splice(index, 1);
+          } else {
+            return;
+          }
         }
 
         const dynamicTemplate: Properties = {};
@@ -182,8 +190,8 @@ export function generateMappings(
           dynamicTemplate.path_match = dynamicMapping.pathMatch;
         }
 
-        dynamicTemplateNames.add(name);
-        dynamicTemplates.push({ [dynamicMapping.path]: dynamicTemplate });
+        const size = dynamicTemplates.push({ [name]: dynamicTemplate });
+        dynamicTemplateNames[name] = size - 1;
       },
       addRuntimeField: (runtimeField: { path: string; properties: Properties }) => {
         runtimeFields[`${runtimeField.path}`] = runtimeField.properties;