[Fleet] Support user overrides in composable templates

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts

@@ -50,36 +50,46 @@ export const installTemplates = async (
   // build templates per data stream from yml files
   const dataStreams = installablePackage.data_streams;
   if (!dataStreams) return [];
-  // get template refs to save
-  const installedTemplateRefs = dataStreams.map((dataStream) => ({
-    id: generateTemplateName(dataStream),
-    type: ElasticsearchAssetType.indexTemplate,
-  }));
 
-  // add package installation's references to index templates
-  await saveInstalledEsRefs(savedObjectsClient, installablePackage.name, installedTemplateRefs);
-
-  if (dataStreams) {
-    const installTemplatePromises = dataStreams.reduce<Array<Promise<TemplateRef>>>(
-      (acc, dataStream) => {
-        acc.push(
-          installTemplateForDataStream({
-            pkg: installablePackage,
-            esClient,
-            dataStream,
-          })
-        );
-        return acc;
+  const installTemplatePromises = dataStreams.reduce<Array<Promise<TemplateRef>>>(
+    (acc, dataStream) => {
+      acc.push(
+        installTemplateForDataStream({
+          pkg: installablePackage,
+          esClient,
+          dataStream,
+        })
+      );
+      return acc;
+    },
+    []
+  );
+
+  const res = await Promise.all(installTemplatePromises);
+  const installedTemplates = res.flat();
+
+  // get template refs to save
+  const templateRefs = installedTemplates.flatMap((installedTemplate) => {
+    const indexTemplates = [
+      {
+        id: installedTemplate.templateName,
+        type: ElasticsearchAssetType.indexTemplate,
       },
-      []
+    ];
+    const componentTemplates = installedTemplate.indexTemplate.composed_of.map(
+      (componentTemplateId) => ({
+        id: componentTemplateId,
+        type: ElasticsearchAssetType.componentTemplate,
+      })
     );
+    return indexTemplates.concat(componentTemplates);
+  });
 
-    const res = await Promise.all(installTemplatePromises);
-    const installedTemplates = res.flat();
+  // add package installation's references to index templates
+  // await saveInstalledEsRefs(savedObjectsClient, installablePackage.name, installedTemplateRefs);
+  await saveInstalledEsRefs(savedObjectsClient, installablePackage.name, templateRefs);
 
-    return installedTemplates;
-  }
-  return [];
+  return installedTemplates;
 };
 
 const installPreBuiltTemplates = async (paths: string[], esClient: ElasticsearchClient) => {
@@ -172,83 +182,98 @@ export async function installTemplateForDataStream({
 }
 
 function putComponentTemplate(
-  body: object | undefined,
-  name: string,
-  esClient: ElasticsearchClient
-): { clusterPromise: Promise<any>; name: string } | undefined {
-  if (body) {
-    const esClientParams = {
-      name,
-      body,
-    };
-
-    return {
-      // @ts-expect-error body expected to be ClusterPutComponentTemplateRequest
-      clusterPromise: esClient.cluster.putComponentTemplate(esClientParams, { ignore: [404] }),
-      name,
-    };
+  esClient: ElasticsearchClient,
+  params: {
+    body: object;
+    name: string;
+    create?: boolean;
   }
+): { clusterPromise: Promise<any>; name: string } {
+  const { name, body, create = false } = params;
+
+  return {
+    // @ts-expect-error body expected to be ClusterPutComponentTemplateRequest
+    clusterPromise: esClient.cluster.putComponentTemplate(
+      { name, body, create },
+      { ignore: [404] }
+    ),
+    name,
+  };
 }
 
-function buildComponentTemplates(registryElasticsearch: RegistryElasticsearch | undefined) {
-  let mappingsTemplate;
-  let settingsTemplate;
+const mappingsSuffix = '-mappings';
+const settingsSuffix = '-settings';
+const userSettingsSuffix = '-user_settings';
+type TemplateBaseName = string;
+type UserSettingsTemplateName = `${TemplateBaseName}${typeof userSettingsSuffix}`;
+
+const isUserSettingsTemplate = (name: string): name is UserSettingsTemplateName =>
+  name.endsWith(userSettingsSuffix);
+
+function buildComponentTemplates(
+  templateName: string,
+  registryElasticsearch: RegistryElasticsearch | undefined
+) {
+  const mappingsTemplateName = `${templateName}${mappingsSuffix}`;
+  const settingsTemplateName = `${templateName}${settingsSuffix}`;
+  const userSettingsTemplateName = `${templateName}${userSettingsSuffix}`;
+  const templatesMap: Record<string, { template: object }> = {};
 
   if (registryElasticsearch && registryElasticsearch['index_template.mappings']) {
-    mappingsTemplate = {
+    templatesMap[mappingsTemplateName] = {
       template: {
-        mappings: {
-          ...registryElasticsearch['index_template.mappings'],
-        },
+        mappings: registryElasticsearch['index_template.mappings'],
       },
     };
   }
 
   if (registryElasticsearch && registryElasticsearch['index_template.settings']) {
-    settingsTemplate = {
+    templatesMap[settingsTemplateName] = {
       template: {
         settings: registryElasticsearch['index_template.settings'],
       },
     };
+
+    // return empty/stub template
+    templatesMap[userSettingsTemplateName] = {
+      template: {
+        settings: {},
+      },
+    };
   }
-  return { settingsTemplate, mappingsTemplate };
+
+  return templatesMap;
 }
 
 async function installDataStreamComponentTemplates(
   templateName: string,
   registryElasticsearch: RegistryElasticsearch | undefined,
   esClient: ElasticsearchClient
 ) {
-  const templates: string[] = [];
-  const componentPromises: Array<Promise<any>> = [];
-
-  const compTemplates = buildComponentTemplates(registryElasticsearch);
+  const templates = buildComponentTemplates(templateName, registryElasticsearch);
+  const templateNames = Object.keys(templates);
+  const templateEntries = Object.entries(templates);
 
-  const mappings = putComponentTemplate(
-    compTemplates.mappingsTemplate,
-    `${templateName}-mappings`,
-    esClient
-  );
-
-  const settings = putComponentTemplate(
-    compTemplates.settingsTemplate,
-    `${templateName}-settings`,
-    esClient
+  // TODO: Check return values for errors
+  await Promise.all(
+    templateEntries.map(async ([name, body]) => {
+      if (isUserSettingsTemplate(name)) {
+        // look for existing user_settings template
+        const result = await esClient.cluster.getComponentTemplate({ name }, { ignore: [404] });
+        const hasUserSettingsTemplate = result.body.component_templates?.length === 1;
+        if (!hasUserSettingsTemplate) {
+          // only add if one isn't already present
+          const { clusterPromise } = putComponentTemplate(esClient, { body, name, create: true });
+          return clusterPromise;
+        }
+      } else {
+        const { clusterPromise } = putComponentTemplate(esClient, { body, name });
+        return clusterPromise;
+      }
+    })
   );
 
-  if (mappings) {
-    templates.push(mappings.name);
-    componentPromises.push(mappings.clusterPromise);
-  }
-
-  if (settings) {
-    templates.push(settings.name);
-    componentPromises.push(settings.clusterPromise);
-  }
-
-  // TODO: Check return values for errors
-  await Promise.all(componentPromises);
-  return templates;
+  return templateNames;
 }
 
 export async function installTemplate({

x-pack/plugins/fleet/server/services/epm/packages/remove.ts

@@ -89,13 +89,18 @@ function deleteKibanaAssets(
   });
 }
 
-function deleteESAssets(installedObjects: EsAssetReference[], esClient: ElasticsearchClient) {
+function deleteESAssets(
+  installedObjects: EsAssetReference[],
+  esClient: ElasticsearchClient
+): Array<Promise<unknown>> {
   return installedObjects.map(async ({ id, type }) => {
     const assetType = type as AssetType;
     if (assetType === ElasticsearchAssetType.ingestPipeline) {
       return deletePipeline(esClient, id);
     } else if (assetType === ElasticsearchAssetType.indexTemplate) {
-      return deleteTemplate(esClient, id);
+      return deleteIndexTemplate(esClient, id);
+    } else if (assetType === ElasticsearchAssetType.componentTemplate) {
+      return deleteComponentTemplate(esClient, id);
     } else if (assetType === ElasticsearchAssetType.transform) {
       return deleteTransforms(esClient, [id]);
     } else if (assetType === ElasticsearchAssetType.dataStreamIlmPolicy) {
@@ -111,13 +116,30 @@ async function deleteAssets(
 ) {
   const logger = appContextService.getLogger();
 
-  const deletePromises: Array<Promise<unknown>> = [
-    ...deleteESAssets(installedEs, esClient),
-    ...deleteKibanaAssets(installedKibana, savedObjectsClient),
-  ];
+  // must delete index templates first, or component templates which reference them cannot be deleted
+  // separate the assets into Index Templates and other assets
+  type Tuple = [EsAssetReference[], EsAssetReference[]];
+  const [indexTemplates, otherAssets] = installedEs.reduce<Tuple>(
+    ([indexAssetTypes, otherAssetTypes], asset) => {
+      if (asset.type === ElasticsearchAssetType.indexTemplate) {
+        indexAssetTypes.push(asset);
+      } else {
+        otherAssetTypes.push(asset);
+      }
+
+      return [indexAssetTypes, otherAssetTypes];
+    },
+    [[], []]
+  );
 
   try {
-    await Promise.all(deletePromises);
+    // must delete index templates first
+    await Promise.all(deleteESAssets(indexTemplates, esClient));
+    // then the other asset types
+    await Promise.all([
+      ...deleteESAssets(otherAssets, esClient),
+      ...deleteKibanaAssets(installedKibana, savedObjectsClient),
+    ]);
   } catch (err) {
     // in the rollback case, partial installs are likely, so missing assets are not an error
     if (!savedObjectsClient.errors.isNotFoundError(err)) {
@@ -126,13 +148,24 @@ async function deleteAssets(
   }
 }
 
-async function deleteTemplate(esClient: ElasticsearchClient, name: string): Promise<void> {
+async function deleteIndexTemplate(esClient: ElasticsearchClient, name: string): Promise<void> {
   // '*' shouldn't ever appear here, but it still would delete all templates
   if (name && name !== '*') {
     try {
       await esClient.indices.deleteIndexTemplate({ name }, { ignore: [404] });
     } catch {
-      throw new Error(`error deleting template ${name}`);
+      throw new Error(`error deleting index template ${name}`);
+    }
+  }
+}
+
+async function deleteComponentTemplate(esClient: ElasticsearchClient, name: string): Promise<void> {
+  // '*' shouldn't ever appear here, but it still would delete all templates
+  if (name && name !== '*') {
+    try {
+      await esClient.cluster.deleteComponentTemplate({ name }, { ignore: [404] });
+    } catch (error) {
+      throw new Error(`error deleting component template ${name}`);
     }
   }
 }

x-pack/test/fleet_api_integration/apis/epm/install_overrides.ts

@@ -18,9 +18,8 @@ export default function ({ getService }: FtrProviderContext) {
   const mappingsPackage = 'overrides-0.1.0';
   const server = dockerServers.get('registry');
 
-  const deletePackage = async (pkgkey: string) => {
-    await supertest.delete(`/api/fleet/epm/packages/${pkgkey}`).set('kbn-xsrf', 'xxxx');
-  };
+  const deletePackage = async (pkgkey: string) =>
+    supertest.delete(`/api/fleet/epm/packages/${pkgkey}`).set('kbn-xsrf', 'xxxx');
 
   describe('installs packages that include settings and mappings overrides', async () => {
     after(async () => {
@@ -44,14 +43,12 @@ export default function ({ getService }: FtrProviderContext) {
           path: `/_index_template/${templateName}`,
         }));
 
-        // make sure it has the right composed_of array, the contents should be the component templates
-        // that were installed
-        expect(body.index_templates[0].index_template.composed_of).to.contain(
-          `${templateName}-mappings`
-        );
-        expect(body.index_templates[0].index_template.composed_of).to.contain(
-          `${templateName}-settings`
-        );
+        // make sure composed_of array has the correct component templates in the correct order
+        expect(body.index_templates[0].index_template.composed_of).to.eql([
+          `${templateName}-mappings`,
+          `${templateName}-settings`,
+          `${templateName}-user_settings`,
+        ]);
 
         ({ body } = await es.transport.request({
           method: 'GET',
@@ -72,6 +69,46 @@ export default function ({ getService }: FtrProviderContext) {
         expect(
           body.component_templates[0].component_template.template.settings.index.lifecycle.name
         ).to.be('reference');
+
+        ({ body } = await es.transport.request({
+          method: 'GET',
+          path: `/_component_template/${templateName}-user_settings`,
+        }));
+
+        // Make sure that the lifecycle name gets set correct in the settings
+        let storedTemplate = body.component_templates[0].component_template.template.settings;
+        const stubTemplate = {};
+        expect(storedTemplate).to.eql(stubTemplate);
+
+        const userSettingsOverrides = {
+          number_of_shards: 3,
+          index: {
+            lifecycle: { name: 'overridden by user' },
+          },
+        };
+
+        ({ body } = await es.transport.request({
+          method: 'PUT',
+          path: `/_component_template/${templateName}-user_settings`,
+          body: {
+            template: { settings: userSettingsOverrides },
+          },
+        }));
+
+        ({ body } = await es.transport.request({
+          method: 'GET',
+          path: `/_component_template/${templateName}-user_settings`,
+        }));
+        // templateName = 'logs-overrides.test';
+        // console.log({ GET: JSON.stringify(body) });
+        // Make sure that the lifecycle name gets set correct in the settings
+        storedTemplate = body.component_templates[0].component_template.template.settings;
+        expect(storedTemplate).to.eql({
+          index: {
+            number_of_shards: 3,
+            lifecycle: { name: 'overridden by user' },
+          },
+        });
       } else {
         warnAndSkipTest(this, log);
       }