diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts index 83875801300d3..21f0f980dc8b7 100644 --- a/x-pack/plugins/fleet/common/types/models/epm.ts +++ b/x-pack/plugins/fleet/common/types/models/epm.ts @@ -5,6 +5,7 @@ * 2.0. */ +import type { estypes } from '@elastic/elasticsearch'; // Follow pattern from https://github.com/elastic/kibana/pull/52447 // TODO: Update when https://github.com/elastic/kibana/issues/53021 is closed import type { SavedObject, SavedObjectAttributes, SavedObjectReference } from 'src/core/public'; @@ -299,8 +300,8 @@ export interface RegistryDataStream { } export interface RegistryElasticsearch { - 'index_template.settings'?: object; - 'index_template.mappings'?: object; + 'index_template.settings'?: estypes.IndicesIndexSettings; + 'index_template.mappings'?: estypes.MappingTypeMapping; } export interface RegistryDataStreamPermissions { @@ -425,7 +426,7 @@ export interface IndexTemplate { _meta: object; } -export interface TemplateRef { +export interface IndexTemplateEntry { templateName: string; indexTemplate: IndexTemplate; } diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts index d202dab54f5bd..db1fba1eedccd 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts @@ -11,7 +11,7 @@ import type { ElasticsearchClient, SavedObjectsClientContract } from 'src/core/s import { ElasticsearchAssetType } from '../../../../types'; import type { RegistryDataStream, - TemplateRef, + IndexTemplateEntry, RegistryElasticsearch, InstallablePackage, } from '../../../../types'; @@ -19,7 +19,7 @@ import { loadFieldsFromYaml, processFields } from '../../fields/field'; import type { Field } from '../../fields/field'; import { getPipelineNameForInstallation } from '../ingest_pipeline/install'; import { getAsset, getPathParts } from '../../archive'; -import { removeAssetsFromInstalledEsByType, saveInstalledEsRefs } from '../../packages/install'; +import { removeAssetTypesFromInstalledEs, saveInstalledEsRefs } from '../../packages/install'; import { generateMappings, @@ -34,7 +34,7 @@ export const installTemplates = async ( esClient: ElasticsearchClient, paths: string[], savedObjectsClient: SavedObjectsClientContract -): Promise => { +): Promise => { // install any pre-built index template assets, // atm, this is only the base package's global index templates // Install component templates first, as they are used by the index templates @@ -42,44 +42,36 @@ export const installTemplates = async ( await installPreBuiltTemplates(paths, esClient); // remove package installation's references to index templates - await removeAssetsFromInstalledEsByType( - savedObjectsClient, - installablePackage.name, - ElasticsearchAssetType.indexTemplate - ); + await removeAssetTypesFromInstalledEs(savedObjectsClient, installablePackage.name, [ + ElasticsearchAssetType.indexTemplate, + ElasticsearchAssetType.componentTemplate, + ]); // build templates per data stream from yml files const dataStreams = installablePackage.data_streams; if (!dataStreams) return []; + + const installedTemplatesNested = await Promise.all( + dataStreams.map((dataStream) => + installTemplateForDataStream({ + pkg: installablePackage, + esClient, + dataStream, + }) + ) + ); + const installedTemplates = installedTemplatesNested.flat(); + // get template refs to save - const installedTemplateRefs = dataStreams.map((dataStream) => ({ - id: generateTemplateName(dataStream), - type: ElasticsearchAssetType.indexTemplate, - })); + const installedIndexTemplateRefs = getAllTemplateRefs(installedTemplates); // add package installation's references to index templates - await saveInstalledEsRefs(savedObjectsClient, installablePackage.name, installedTemplateRefs); - - if (dataStreams) { - const installTemplatePromises = dataStreams.reduce>>( - (acc, dataStream) => { - acc.push( - installTemplateForDataStream({ - pkg: installablePackage, - esClient, - dataStream, - }) - ); - return acc; - }, - [] - ); - - const res = await Promise.all(installTemplatePromises); - const installedTemplates = res.flat(); + await saveInstalledEsRefs( + savedObjectsClient, + installablePackage.name, + installedIndexTemplateRefs + ); - return installedTemplates; - } - return []; + return installedTemplates; }; const installPreBuiltTemplates = async (paths: string[], esClient: ElasticsearchClient) => { @@ -160,7 +152,7 @@ export async function installTemplateForDataStream({ pkg: InstallablePackage; esClient: ElasticsearchClient; dataStream: RegistryDataStream; -}): Promise { +}): Promise { const fields = await loadFieldsFromYaml(pkg, dataStream.path); return installTemplate({ esClient, @@ -171,84 +163,118 @@ export async function installTemplateForDataStream({ }); } +interface TemplateMapEntry { + _meta: { package: { name: string } }; + template: + | { + mappings: NonNullable; + } + | { + settings: NonNullable | object; + }; +} +type TemplateMap = Record; function putComponentTemplate( - body: object | undefined, - name: string, - esClient: ElasticsearchClient -): { clusterPromise: Promise; name: string } | undefined { - if (body) { - const esClientParams = { - name, - body, - }; - - return { - // @ts-expect-error body expected to be ClusterPutComponentTemplateRequest - clusterPromise: esClient.cluster.putComponentTemplate(esClientParams, { ignore: [404] }), - name, - }; + esClient: ElasticsearchClient, + params: { + body: TemplateMapEntry; + name: string; + create?: boolean; } +): { clusterPromise: Promise; name: string } { + const { name, body, create = false } = params; + return { + clusterPromise: esClient.cluster.putComponentTemplate( + // @ts-expect-error body is missing required key `settings`. TemplateMapEntry has settings *or* mappings + { name, body, create }, + { ignore: [404] } + ), + name, + }; } -function buildComponentTemplates(registryElasticsearch: RegistryElasticsearch | undefined) { - let mappingsTemplate; - let settingsTemplate; +const mappingsSuffix = '@mappings'; +const settingsSuffix = '@settings'; +const userSettingsSuffix = '@custom'; +type TemplateBaseName = string; +type UserSettingsTemplateName = `${TemplateBaseName}${typeof userSettingsSuffix}`; + +const isUserSettingsTemplate = (name: string): name is UserSettingsTemplateName => + name.endsWith(userSettingsSuffix); + +function buildComponentTemplates(params: { + templateName: string; + registryElasticsearch: RegistryElasticsearch | undefined; + packageName: string; +}) { + const { templateName, registryElasticsearch, packageName } = params; + const mappingsTemplateName = `${templateName}${mappingsSuffix}`; + const settingsTemplateName = `${templateName}${settingsSuffix}`; + const userSettingsTemplateName = `${templateName}${userSettingsSuffix}`; + + const templatesMap: TemplateMap = {}; + const _meta = { package: { name: packageName } }; if (registryElasticsearch && registryElasticsearch['index_template.mappings']) { - mappingsTemplate = { + templatesMap[mappingsTemplateName] = { template: { - mappings: { - ...registryElasticsearch['index_template.mappings'], - }, + mappings: registryElasticsearch['index_template.mappings'], }, + _meta, }; } if (registryElasticsearch && registryElasticsearch['index_template.settings']) { - settingsTemplate = { + templatesMap[settingsTemplateName] = { template: { settings: registryElasticsearch['index_template.settings'], }, + _meta, }; } - return { settingsTemplate, mappingsTemplate }; -} -async function installDataStreamComponentTemplates( - templateName: string, - registryElasticsearch: RegistryElasticsearch | undefined, - esClient: ElasticsearchClient -) { - const templates: string[] = []; - const componentPromises: Array> = []; + // return empty/stub template + templatesMap[userSettingsTemplateName] = { + template: { + settings: {}, + }, + _meta, + }; - const compTemplates = buildComponentTemplates(registryElasticsearch); + return templatesMap; +} - const mappings = putComponentTemplate( - compTemplates.mappingsTemplate, - `${templateName}-mappings`, - esClient - ); +async function installDataStreamComponentTemplates(params: { + templateName: string; + registryElasticsearch: RegistryElasticsearch | undefined; + esClient: ElasticsearchClient; + packageName: string; +}) { + const { templateName, registryElasticsearch, esClient, packageName } = params; + const templates = buildComponentTemplates({ templateName, registryElasticsearch, packageName }); + const templateNames = Object.keys(templates); + const templateEntries = Object.entries(templates); - const settings = putComponentTemplate( - compTemplates.settingsTemplate, - `${templateName}-settings`, - esClient + // TODO: Check return values for errors + await Promise.all( + templateEntries.map(async ([name, body]) => { + if (isUserSettingsTemplate(name)) { + // look for existing user_settings template + const result = await esClient.cluster.getComponentTemplate({ name }, { ignore: [404] }); + const hasUserSettingsTemplate = result.body.component_templates?.length === 1; + if (!hasUserSettingsTemplate) { + // only add if one isn't already present + const { clusterPromise } = putComponentTemplate(esClient, { body, name, create: true }); + return clusterPromise; + } + } else { + const { clusterPromise } = putComponentTemplate(esClient, { body, name }); + return clusterPromise; + } + }) ); - if (mappings) { - templates.push(mappings.name); - componentPromises.push(mappings.clusterPromise); - } - - if (settings) { - templates.push(settings.name); - componentPromises.push(settings.clusterPromise); - } - - // TODO: Check return values for errors - await Promise.all(componentPromises); - return templates; + return templateNames; } export async function installTemplate({ @@ -263,7 +289,7 @@ export async function installTemplate({ dataStream: RegistryDataStream; packageVersion: string; packageName: string; -}): Promise { +}): Promise { const validFields = processFields(fields); const mappings = generateMappings(validFields); const templateName = generateTemplateName(dataStream); @@ -310,11 +336,12 @@ export async function installTemplate({ await esClient.indices.putIndexTemplate(updateIndexTemplateParams, { ignore: [404] }); } - const composedOfTemplates = await installDataStreamComponentTemplates( + const composedOfTemplates = await installDataStreamComponentTemplates({ templateName, - dataStream.elasticsearch, - esClient - ); + registryElasticsearch: dataStream.elasticsearch, + esClient, + packageName, + }); const template = getTemplate({ type: dataStream.type, @@ -342,3 +369,21 @@ export async function installTemplate({ indexTemplate: template, }; } + +export function getAllTemplateRefs(installedTemplates: IndexTemplateEntry[]) { + return installedTemplates.flatMap((installedTemplate) => { + const indexTemplates = [ + { + id: installedTemplate.templateName, + type: ElasticsearchAssetType.indexTemplate, + }, + ]; + const componentTemplates = installedTemplate.indexTemplate.composed_of.map( + (componentTemplateId) => ({ + id: componentTemplateId, + type: ElasticsearchAssetType.componentTemplate, + }) + ); + return indexTemplates.concat(componentTemplates); + }); +} diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index 07d0df021c827..158996cc574d7 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -10,7 +10,7 @@ import type { ElasticsearchClient } from 'kibana/server'; import type { Field, Fields } from '../../fields/field'; import type { RegistryDataStream, - TemplateRef, + IndexTemplateEntry, IndexTemplate, IndexTemplateMappings, } from '../../../../types'; @@ -456,7 +456,7 @@ function getBaseTemplate( export const updateCurrentWriteIndices = async ( esClient: ElasticsearchClient, - templates: TemplateRef[] + templates: IndexTemplateEntry[] ): Promise => { if (!templates.length) return; @@ -471,7 +471,7 @@ function isCurrentDataStream(item: CurrentDataStream[] | undefined): item is Cur const queryDataStreamsFromTemplates = async ( esClient: ElasticsearchClient, - templates: TemplateRef[] + templates: IndexTemplateEntry[] ): Promise => { const dataStreamPromises = templates.map((template) => { return getDataStreams(esClient, template); @@ -482,7 +482,7 @@ const queryDataStreamsFromTemplates = async ( const getDataStreams = async ( esClient: ElasticsearchClient, - template: TemplateRef + template: IndexTemplateEntry ): Promise => { const { templateName, indexTemplate } = template; const { body } = await esClient.indices.getDataStream({ name: `${templateName}-*` }); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts index 65d71ac5fdc17..1bbbb1bb9b6a2 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts @@ -10,10 +10,10 @@ import type { ElasticsearchClient, SavedObject, SavedObjectsClientContract } fro import { MAX_TIME_COMPLETE_INSTALL, ASSETS_SAVED_OBJECT_TYPE } from '../../../../common'; import type { InstallablePackage, InstallSource, PackageAssetReference } from '../../../../common'; import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants'; -import { ElasticsearchAssetType } from '../../../types'; import type { AssetReference, Installation, InstallType } from '../../../types'; import { installTemplates } from '../elasticsearch/template/install'; import { installPipelines, deletePreviousPipelines } from '../elasticsearch/ingest_pipeline/'; +import { getAllTemplateRefs } from '../elasticsearch/template/install'; import { installILMPolicy } from '../elasticsearch/ilm/install'; import { installKibanaAssets, getKibanaAssets } from '../kibana/assets/install'; import { updateCurrentWriteIndices } from '../elasticsearch/template/template'; @@ -170,10 +170,7 @@ export async function _installPackage({ installedPkg.attributes.install_version ); } - const installedTemplateRefs = installedTemplates.map((template) => ({ - id: template.templateName, - type: ElasticsearchAssetType.indexTemplate, - })); + const installedTemplateRefs = getAllTemplateRefs(installedTemplates); // make sure the assets are installed (or didn't error) if (installKibanaAssetsError) throw installKibanaAssetsError; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install.ts b/x-pack/plugins/fleet/server/services/epm/packages/install.ts index c6fd9a8f763ab..e00526cbb4ec4 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install.ts @@ -257,8 +257,7 @@ async function installPackageFromRegistry({ const { paths, packageInfo } = await Registry.getRegistryPackage(pkgName, pkgVersion); // try installing the package, if there was an error, call error handler and rethrow - // TODO: without the ts-ignore, TS complains about the type of the value of the returned InstallResult.status - // @ts-ignore + // @ts-expect-error status is string instead of InstallResult.status 'installed' | 'already_installed' return _installPackage({ savedObjectsClient, esClient, @@ -334,8 +333,7 @@ async function installPackageByUpload({ version: packageInfo.version, packageInfo, }); - // TODO: without the ts-ignore, TS complains about the type of the value of the returned InstallResult.status - // @ts-ignore + // @ts-expect-error status is string instead of InstallResult.status 'installed' | 'already_installed' return _installPackage({ savedObjectsClient, esClient, @@ -484,17 +482,17 @@ export const saveInstalledEsRefs = async ( return installedAssets; }; -export const removeAssetsFromInstalledEsByType = async ( +export const removeAssetTypesFromInstalledEs = async ( savedObjectsClient: SavedObjectsClientContract, pkgName: string, - assetType: AssetType + assetTypes: AssetType[] ) => { const installedPkg = await getInstallationObject({ savedObjectsClient, pkgName }); const installedAssets = installedPkg?.attributes.installed_es; if (!installedAssets?.length) return; - const installedAssetsToSave = installedAssets?.filter(({ id, type }) => { - return type !== assetType; - }); + const installedAssetsToSave = installedAssets?.filter( + (asset) => !assetTypes.includes(asset.type) + ); return savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, { installed_es: installedAssetsToSave, diff --git a/x-pack/plugins/fleet/server/services/epm/packages/remove.ts b/x-pack/plugins/fleet/server/services/epm/packages/remove.ts index 706f1bbbaaf35..70167d1156a66 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/remove.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/remove.ts @@ -89,13 +89,18 @@ function deleteKibanaAssets( }); } -function deleteESAssets(installedObjects: EsAssetReference[], esClient: ElasticsearchClient) { +function deleteESAssets( + installedObjects: EsAssetReference[], + esClient: ElasticsearchClient +): Array> { return installedObjects.map(async ({ id, type }) => { const assetType = type as AssetType; if (assetType === ElasticsearchAssetType.ingestPipeline) { return deletePipeline(esClient, id); } else if (assetType === ElasticsearchAssetType.indexTemplate) { - return deleteTemplate(esClient, id); + return deleteIndexTemplate(esClient, id); + } else if (assetType === ElasticsearchAssetType.componentTemplate) { + return deleteComponentTemplate(esClient, id); } else if (assetType === ElasticsearchAssetType.transform) { return deleteTransforms(esClient, [id]); } else if (assetType === ElasticsearchAssetType.dataStreamIlmPolicy) { @@ -111,13 +116,30 @@ async function deleteAssets( ) { const logger = appContextService.getLogger(); - const deletePromises: Array> = [ - ...deleteESAssets(installedEs, esClient), - ...deleteKibanaAssets(installedKibana, savedObjectsClient), - ]; + // must delete index templates first, or component templates which reference them cannot be deleted + // separate the assets into Index Templates and other assets + type Tuple = [EsAssetReference[], EsAssetReference[]]; + const [indexTemplates, otherAssets] = installedEs.reduce( + ([indexAssetTypes, otherAssetTypes], asset) => { + if (asset.type === ElasticsearchAssetType.indexTemplate) { + indexAssetTypes.push(asset); + } else { + otherAssetTypes.push(asset); + } + + return [indexAssetTypes, otherAssetTypes]; + }, + [[], []] + ); try { - await Promise.all(deletePromises); + // must delete index templates first + await Promise.all(deleteESAssets(indexTemplates, esClient)); + // then the other asset types + await Promise.all([ + ...deleteESAssets(otherAssets, esClient), + ...deleteKibanaAssets(installedKibana, savedObjectsClient), + ]); } catch (err) { // in the rollback case, partial installs are likely, so missing assets are not an error if (!savedObjectsClient.errors.isNotFoundError(err)) { @@ -126,13 +148,24 @@ async function deleteAssets( } } -async function deleteTemplate(esClient: ElasticsearchClient, name: string): Promise { +async function deleteIndexTemplate(esClient: ElasticsearchClient, name: string): Promise { // '*' shouldn't ever appear here, but it still would delete all templates if (name && name !== '*') { try { await esClient.indices.deleteIndexTemplate({ name }, { ignore: [404] }); } catch { - throw new Error(`error deleting template ${name}`); + throw new Error(`error deleting index template ${name}`); + } + } +} + +async function deleteComponentTemplate(esClient: ElasticsearchClient, name: string): Promise { + // '*' shouldn't ever appear here, but it still would delete all templates + if (name && name !== '*') { + try { + await esClient.cluster.deleteComponentTemplate({ name }, { ignore: [404] }); + } catch (error) { + throw new Error(`error deleting component template ${name}`); } } } diff --git a/x-pack/plugins/fleet/server/types/index.tsx b/x-pack/plugins/fleet/server/types/index.tsx index 8927676976457..0c08a09e76f4e 100644 --- a/x-pack/plugins/fleet/server/types/index.tsx +++ b/x-pack/plugins/fleet/server/types/index.tsx @@ -63,7 +63,7 @@ export { IndexTemplate, RegistrySearchResults, RegistrySearchResult, - TemplateRef, + IndexTemplateEntry, IndexTemplateMappings, Settings, SettingsSOAttributes, diff --git a/x-pack/test/fleet_api_integration/apis/epm/__snapshots__/install_by_upload.snap b/x-pack/test/fleet_api_integration/apis/epm/__snapshots__/install_by_upload.snap index 7584dfcc8a6c0..13c2dd24f9103 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/__snapshots__/install_by_upload.snap +++ b/x-pack/test/fleet_api_integration/apis/epm/__snapshots__/install_by_upload.snap @@ -341,14 +341,26 @@ Object { "id": "logs-apache.access", "type": "index_template", }, + Object { + "id": "logs-apache.access@custom", + "type": "component_template", + }, Object { "id": "metrics-apache.status", "type": "index_template", }, + Object { + "id": "metrics-apache.status@custom", + "type": "component_template", + }, Object { "id": "logs-apache.error", "type": "index_template", }, + Object { + "id": "logs-apache.error@custom", + "type": "component_template", + }, ], "installed_kibana": Array [ Object { diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_by_upload.ts b/x-pack/test/fleet_api_integration/apis/epm/install_by_upload.ts index 71cf7ed79fa2b..182838f21dbda 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/install_by_upload.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/install_by_upload.ts @@ -70,7 +70,7 @@ export default function (providerContext: FtrProviderContext) { .type('application/gzip') .send(buf) .expect(200); - expect(res.body.response.length).to.be(23); + expect(res.body.response.length).to.be(26); }); it('should install a zip archive correctly and package info should return correctly after validation', async function () { @@ -81,7 +81,7 @@ export default function (providerContext: FtrProviderContext) { .type('application/zip') .send(buf) .expect(200); - expect(res.body.response.length).to.be(23); + expect(res.body.response.length).to.be(26); const packageInfoRes = await supertest .get(`/api/fleet/epm/packages/${testPkgKey}`) diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_overrides.ts b/x-pack/test/fleet_api_integration/apis/epm/install_overrides.ts index 1b916dff573af..204ee8508f468 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/install_overrides.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/install_overrides.ts @@ -7,22 +7,22 @@ import expect from '@kbn/expect'; import { FtrProviderContext } from '../../../api_integration/ftr_provider_context'; -import { warnAndSkipTest } from '../../helpers'; +import { skipIfNoDockerRegistry } from '../../helpers'; -export default function ({ getService }: FtrProviderContext) { +export default function (providerContext: FtrProviderContext) { + const { getService } = providerContext; const supertest = getService('supertest'); const es = getService('es'); const dockerServers = getService('dockerServers'); - const log = getService('log'); const mappingsPackage = 'overrides-0.1.0'; const server = dockerServers.get('registry'); - const deletePackage = async (pkgkey: string) => { - await supertest.delete(`/api/fleet/epm/packages/${pkgkey}`).set('kbn-xsrf', 'xxxx'); - }; + const deletePackage = async (pkgkey: string) => + supertest.delete(`/api/fleet/epm/packages/${pkgkey}`).set('kbn-xsrf', 'xxxx'); describe('installs packages that include settings and mappings overrides', async () => { + skipIfNoDockerRegistry(providerContext); after(async () => { if (server.enabled) { // remove the package just in case it being installed will affect other tests @@ -31,50 +31,107 @@ export default function ({ getService }: FtrProviderContext) { }); it('should install the overrides package correctly', async function () { - if (server.enabled) { - let { body } = await supertest - .post(`/api/fleet/epm/packages/${mappingsPackage}`) - .set('kbn-xsrf', 'xxxx') - .expect(200); - - const templateName = body.response[0].id; - - ({ body } = await es.transport.request({ - method: 'GET', - path: `/_index_template/${templateName}`, - })); - - // make sure it has the right composed_of array, the contents should be the component templates - // that were installed - expect(body.index_templates[0].index_template.composed_of).to.contain( - `${templateName}-mappings` - ); - expect(body.index_templates[0].index_template.composed_of).to.contain( - `${templateName}-settings` - ); - - ({ body } = await es.transport.request({ - method: 'GET', - path: `/_component_template/${templateName}-mappings`, - })); - - // Make sure that the `dynamic` field exists and is set to false (as it is in the package) - expect(body.component_templates[0].component_template.template.mappings.dynamic).to.be( - false - ); - - ({ body } = await es.transport.request({ - method: 'GET', - path: `/_component_template/${templateName}-settings`, - })); - - // Make sure that the lifecycle name gets set correct in the settings - expect( - body.component_templates[0].component_template.template.settings.index.lifecycle.name - ).to.be('reference'); - } else { - warnAndSkipTest(this, log); - } + let { body } = await supertest + .post(`/api/fleet/epm/packages/${mappingsPackage}`) + .set('kbn-xsrf', 'xxxx') + .expect(200); + + const templateName = body.response[0].id; + + const { body: indexTemplateResponse } = await es.transport.request({ + method: 'GET', + path: `/_index_template/${templateName}`, + }); + + // the index template composed_of has the correct component templates in the correct order + const indexTemplate = indexTemplateResponse.index_templates[0].index_template; + expect(indexTemplate.composed_of).to.eql([ + `${templateName}@mappings`, + `${templateName}@settings`, + `${templateName}@custom`, + ]); + + ({ body } = await es.transport.request({ + method: 'GET', + path: `/_component_template/${templateName}@mappings`, + })); + + // The mappings override provided in the package is set in the mappings component template + expect(body.component_templates[0].component_template.template.mappings.dynamic).to.be(false); + + ({ body } = await es.transport.request({ + method: 'GET', + path: `/_component_template/${templateName}@settings`, + })); + + // The settings override provided in the package is set in the settings component template + expect( + body.component_templates[0].component_template.template.settings.index.lifecycle.name + ).to.be('reference'); + + ({ body } = await es.transport.request({ + method: 'GET', + path: `/_component_template/${templateName}@custom`, + })); + + // The user_settings component template is an empty/stub template at first + const storedTemplate = body.component_templates[0].component_template.template.settings; + expect(storedTemplate).to.eql({}); + + // Update the user_settings component template + ({ body } = await es.transport.request({ + method: 'PUT', + path: `/_component_template/${templateName}@custom`, + body: { + template: { + settings: { + number_of_shards: 3, + index: { + lifecycle: { name: 'overridden by user' }, + number_of_shards: 123, + }, + }, + }, + }, + })); + + // simulate the result + ({ body } = await es.transport.request({ + method: 'POST', + path: `/_index_template/_simulate/${templateName}`, + // body: indexTemplate, // I *think* this should work, but it doesn't + body: { + index_patterns: [`${templateName}-*`], + composed_of: [ + `${templateName}@mappings`, + `${templateName}@settings`, + `${templateName}@custom`, + ], + }, + })); + + expect(body).to.eql({ + template: { + settings: { + index: { + lifecycle: { + name: 'overridden by user', + }, + number_of_shards: '3', + }, + }, + mappings: { + dynamic: 'false', + }, + aliases: {}, + }, + overlapping: [ + { + name: 'logs', + index_patterns: ['logs-*-*'], + }, + ], + }); }); }); } diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts b/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts index 8e09e331bf867..85573560177ee 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts @@ -87,6 +87,40 @@ export default function (providerContext: FtrProviderContext) { ); expect(resMetricsTemplate.statusCode).equal(404); }); + it('should have uninstalled the component templates', async function () { + const resMappings = await es.transport.request( + { + method: 'GET', + path: `/_component_template/${logsTemplateName}@mappings`, + }, + { + ignore: [404], + } + ); + expect(resMappings.statusCode).equal(404); + + const resSettings = await es.transport.request( + { + method: 'GET', + path: `/_component_template/${logsTemplateName}@settings`, + }, + { + ignore: [404], + } + ); + expect(resSettings.statusCode).equal(404); + + const resUserSettings = await es.transport.request( + { + method: 'GET', + path: `/_component_template/${logsTemplateName}@custom`, + }, + { + ignore: [404], + } + ); + expect(resUserSettings.statusCode).equal(404); + }); it('should have uninstalled the pipelines', async function () { const res = await es.transport.request( { @@ -328,17 +362,22 @@ const expectAssetsInstalled = ({ }); expect(resPipeline2.statusCode).equal(200); }); - it('should have installed the template components', async function () { - const res = await es.transport.request({ + it('should have installed the component templates', async function () { + const resMappings = await es.transport.request({ method: 'GET', - path: `/_component_template/${logsTemplateName}-mappings`, + path: `/_component_template/${logsTemplateName}@mappings`, }); - expect(res.statusCode).equal(200); + expect(resMappings.statusCode).equal(200); const resSettings = await es.transport.request({ method: 'GET', - path: `/_component_template/${logsTemplateName}-settings`, + path: `/_component_template/${logsTemplateName}@settings`, }); expect(resSettings.statusCode).equal(200); + const resUserSettings = await es.transport.request({ + method: 'GET', + path: `/_component_template/${logsTemplateName}@custom`, + }); + expect(resUserSettings.statusCode).equal(200); }); it('should have installed the transform components', async function () { const res = await es.transport.request({ @@ -487,6 +526,22 @@ const expectAssetsInstalled = ({ }, ], installed_es: [ + { + id: 'logs-all_assets.test_logs@mappings', + type: 'component_template', + }, + { + id: 'logs-all_assets.test_logs@settings', + type: 'component_template', + }, + { + id: 'logs-all_assets.test_logs@custom', + type: 'component_template', + }, + { + id: 'metrics-all_assets.test_metrics@custom', + type: 'component_template', + }, { id: 'logs-all_assets.test_logs-all_assets', type: 'data_stream_ilm_policy', diff --git a/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts b/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts index a6f79414ab8c0..6b4d104423144 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts @@ -199,23 +199,45 @@ export default function (providerContext: FtrProviderContext) { ); expect(resPipeline2.statusCode).equal(404); }); - it('should have updated the template components', async function () { - const res = await es.transport.request({ + it('should have updated the component templates', async function () { + const resMappings = await es.transport.request({ method: 'GET', - path: `/_component_template/${logsTemplateName}-mappings`, + path: `/_component_template/${logsTemplateName}@mappings`, }); - expect(res.statusCode).equal(200); - expect(res.body.component_templates[0].component_template.template.mappings).eql({ + expect(resMappings.statusCode).equal(200); + expect(resMappings.body.component_templates[0].component_template.template.mappings).eql({ dynamic: true, }); const resSettings = await es.transport.request({ method: 'GET', - path: `/_component_template/${logsTemplateName}-settings`, + path: `/_component_template/${logsTemplateName}@settings`, }); - expect(res.statusCode).equal(200); + expect(resSettings.statusCode).equal(200); expect(resSettings.body.component_templates[0].component_template.template.settings).eql({ index: { lifecycle: { name: 'reference2' } }, }); + const resUserSettings = await es.transport.request({ + method: 'GET', + path: `/_component_template/${logsTemplateName}@custom`, + }); + expect(resUserSettings.statusCode).equal(200); + expect(resUserSettings.body).eql({ + component_templates: [ + { + name: 'logs-all_assets.test_logs@custom', + component_template: { + _meta: { + package: { + name: 'all_assets', + }, + }, + template: { + settings: {}, + }, + }, + }, + ], + }); }); it('should have updated the index patterns', async function () { const resIndexPatternLogs = await kibanaServer.savedObjects.get({ @@ -321,14 +343,34 @@ export default function (providerContext: FtrProviderContext) { id: 'logs-all_assets.test_logs', type: 'index_template', }, + { + id: 'logs-all_assets.test_logs@mappings', + type: 'component_template', + }, + { + id: 'logs-all_assets.test_logs@settings', + type: 'component_template', + }, + { + id: 'logs-all_assets.test_logs@custom', + type: 'component_template', + }, { id: 'logs-all_assets.test_logs2', type: 'index_template', }, + { + id: 'logs-all_assets.test_logs2@custom', + type: 'component_template', + }, { id: 'metrics-all_assets.test_metrics', type: 'index_template', }, + { + id: 'metrics-all_assets.test_metrics@custom', + type: 'component_template', + }, ], es_index_patterns: { test_logs: 'logs-all_assets.test_logs-*', diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.1.0/img/logo_overrides_64_color.svg b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.1.0/img/logo_overrides_64_color.svg new file mode 100644 index 0000000000000..b03007a76ffcc --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.1.0/img/logo_overrides_64_color.svg @@ -0,0 +1,7 @@ + + + + + + + diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.1.0/manifest.yml b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.1.0/manifest.yml index bba1a6a4c347d..312cd2874804c 100644 --- a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.1.0/manifest.yml +++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.1.0/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 -name: error_handling +name: error_handling title: Error handling -description: tests error handling and rollback +description: tests error handling and rollback version: 0.1.0 categories: [] release: beta @@ -17,4 +17,4 @@ requirement: icons: - src: '/img/logo_overrides_64_color.svg' size: '16x16' - type: 'image/svg+xml' \ No newline at end of file + type: 'image/svg+xml' diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.2.0/img/logo_overrides_64_color.svg b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.2.0/img/logo_overrides_64_color.svg new file mode 100644 index 0000000000000..b03007a76ffcc --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.2.0/img/logo_overrides_64_color.svg @@ -0,0 +1,7 @@ + + + + + + + diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.2.0/manifest.yml b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.2.0/manifest.yml index 2eb6a41a77ede..c92f0ab5ae7f3 100644 --- a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.2.0/manifest.yml +++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.2.0/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 -name: error_handling +name: error_handling title: Error handling -description: tests error handling and rollback +description: tests error handling and rollback version: 0.2.0 categories: [] release: beta @@ -16,4 +16,4 @@ requirement: icons: - src: '/img/logo_overrides_64_color.svg' - size: '16x16' \ No newline at end of file + size: '16x16' diff --git a/x-pack/test/fleet_api_integration/config.ts b/x-pack/test/fleet_api_integration/config.ts index 52c9760d66c19..d18ba9c55ca96 100644 --- a/x-pack/test/fleet_api_integration/config.ts +++ b/x-pack/test/fleet_api_integration/config.ts @@ -51,17 +51,11 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { waitForLogLine: 'package manifests loaded', }, }), - services: { - ...xPackAPITestsConfig.get('services'), - }, + services: xPackAPITestsConfig.get('services'), junit: { reportName: 'X-Pack EPM API Integration Tests', }, - - esTestCluster: { - ...xPackAPITestsConfig.get('esTestCluster'), - }, - + esTestCluster: xPackAPITestsConfig.get('esTestCluster'), kbnTestServer: { ...xPackAPITestsConfig.get('kbnTestServer'), serverArgs: [